|
@ -5,21 +5,21 @@ import sys |
|
|
|
|
|
|
|
|
config = "spiders/config.yaml" |
|
|
config = "spiders/config.yaml" |
|
|
#list_of_fdbs = eval(sys.argv[1]) |
|
|
#list_of_fdbs = eval(sys.argv[1]) |
|
|
#list_of_fdbs = ["giz","evergabe-online"] |
|
|
|
|
|
list_of_fdbs = ["giz"] |
|
|
|
|
|
|
|
|
list_of_fdbs = ["giz","evergabe-online","foerderinfo.bund.de-bekanntmachungen"] |
|
|
|
|
|
#list_of_fdbs = ["giz"] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# doing the crawling of government websites |
|
|
# doing the crawling of government websites |
|
|
|
|
|
|
|
|
spider = fdb_spider(config) |
|
|
spider = fdb_spider(config) |
|
|
|
|
|
|
|
|
#spider.download_entry_list_pages_of_funding_databases(list_of_fdbs) |
|
|
|
|
|
|
|
|
spider.download_entry_list_pages_of_funding_databases(list_of_fdbs) |
|
|
|
|
|
|
|
|
#spider.find_config_parameter(list_of_fdbs) |
|
|
#spider.find_config_parameter(list_of_fdbs) |
|
|
|
|
|
|
|
|
#spider.parse_entry_list_data2dictionary(list_of_fdbs) |
|
|
|
|
|
|
|
|
spider.parse_entry_list_data2dictionary(list_of_fdbs) |
|
|
|
|
|
|
|
|
#spider.download_entry_data_htmls(list_of_fdbs) |
|
|
|
|
|
|
|
|
spider.download_entry_data_htmls(list_of_fdbs) |
|
|
|
|
|
|
|
|
spider.parse_entry_data2dictionary(list_of_fdbs) |
|
|
spider.parse_entry_data2dictionary(list_of_fdbs) |
|
|
|
|
|
|