diff --git a/spiders/__pycache__/fdb_spider.cpython-39.pyc b/spiders/__pycache__/fdb_spider.cpython-39.pyc index 3c02d28..ebbced9 100644 Binary files a/spiders/__pycache__/fdb_spider.cpython-39.pyc and b/spiders/__pycache__/fdb_spider.cpython-39.pyc differ diff --git a/spiders/fdb_spider.py b/spiders/fdb_spider.py index 8aa6ae9..ef76a6a 100644 --- a/spiders/fdb_spider.py +++ b/spiders/fdb_spider.py @@ -218,35 +218,62 @@ class fdb_spider(object): print('blabliblub') print('len', len(tree.xpath(fdb_conf_entry_list_parent))) for n in range(len(tree.xpath(fdb_conf_entry_list_parent))): - print('oi inside the loop') - name = tree.xpath( - fdb_conf_entry_list_parent - + fdb_conf_entry_list_child_name - )[n] + + try: + name = tree.xpath( + fdb_conf_entry_list_parent + + "[" + + str(n+1) + + "]" + + fdb_conf_entry_list_child_name + )[0] - info = tree.xpath( - fdb_conf_entry_list_parent - + fdb_conf_entry_list_child_info - )[n] + except Exception as e: + print("name could not be parsed", e) + name = 'NONE' + + try: + info = tree.xpath( + fdb_conf_entry_list_parent + + "[" + + str(n+1) + + "]" + + fdb_conf_entry_list_child_info + )[0] - period = tree.xpath( - fdb_conf_entry_list_parent - + fdb_conf_entry_list_child_period - )[n] - - print('oi ', name) - print('blablidubbiduub') - link = tree.xpath( - fdb_conf_entry_list_parent - # + "[" - # + str(n) - # + "]" - + fdb_conf_entry_list_child_link - )[n] - - print('oi' + name) + except Exception as e: + print("info could not be parsed", e, info) + info = 'NONE' + + try: + period = tree.xpath( + fdb_conf_entry_list_parent + + "[" + + str(n+1) + + "]" + + fdb_conf_entry_list_child_period + )[0] + print('period', period) + except Exception as e: + print("period could not be parsed", e, period) + period = 'NONE' + + try: + link = tree.xpath( + fdb_conf_entry_list_parent + + "[" + + str(n+1) + + "]" + + fdb_conf_entry_list_child_link + )[0] + print('link', link) + + except Exception as e: + print("link could not be parsed", e, link) + link = 'NONE' + - if len(name) > 0: + if len(name) > 0 and name != 'NONE': dictionary_entry_list[n] = {} dictionary_entry_list[n]["name"] = name dictionary_entry_list[n]["info"] = info