diff --git a/main.py b/main.py index 12a01a3..2691b42 100644 --- a/main.py +++ b/main.py @@ -4,7 +4,7 @@ from spiders.fdb_spider import * import sys config = "spiders/config.yaml" -list_of_fdbs = sys.argv[2] +list_of_fdbs = eval(sys.argv[1]) #list_of_fdbs = ["foerderinfo.bund.de-bekanntmachungen"] @@ -14,9 +14,9 @@ spider = fdb_spider(config) spider.download_entry_list_pages_of_funding_databases(list_of_fdbs) -#spider.find_config_parameter(list_of_fdbs) +spider.find_config_parameter(list_of_fdbs) -spider.parse_entry_list_data2dictionary(list_of_fdbs) +#spider.parse_entry_list_data2dictionary(list_of_fdbs) spider.download_entry_data_htmls(list_of_fdbs) diff --git a/spiders/__pycache__/fdb_spider.cpython-311.pyc b/spiders/__pycache__/fdb_spider.cpython-311.pyc index 7e1e819..f3f6ff0 100644 Binary files a/spiders/__pycache__/fdb_spider.cpython-311.pyc and b/spiders/__pycache__/fdb_spider.cpython-311.pyc differ diff --git a/spiders/config.yaml b/spiders/config.yaml index 3627f8a..054bc22 100644 --- a/spiders/config.yaml +++ b/spiders/config.yaml @@ -53,7 +53,7 @@ giz: entry-list: link1: 'https://ausschreibungen.giz.de/Satellite/company/welcome.do?method=showTable&fromSearch=1&tableSortPROJECT_RESULT=2&tableSortAttributePROJECT_RESULT=publicationDate&selectedTablePagePROJECT_RESULT=' link2: '' - iteration-var-list: '[1,2,3,4,5,6,7]' + iteration-var-list: "[1,2,3,4,5,6,7]" #parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']" parent: "//html//body//div//div//table[contains(@class, 'csx-new-table')]//tbody//tr" child-name: "//td[3]//text()" diff --git a/spiders/fdb_spider.py b/spiders/fdb_spider.py index e97ffcd..08dac4c 100644 --- a/spiders/fdb_spider.py +++ b/spiders/fdb_spider.py @@ -434,7 +434,10 @@ class fdb_spider(object): for fdb in list_of_fdbs: try: - iteration_var_list = eval(self.config.get(fdb).get("entry-list").get("iteration-var-list")) + fdb_config = self.config.get(fdb) + print('oi oi',fdb_config) + fdb_config_entrylist = fdb_config.get("entry-list") + iteration_var_list = eval(fdb_config_entrylist.get("iteration-var-list")) except Exception as e: print( "There is a problem with the configuration variable entryList iteration var list in the config.yaml - the original error message is:",