2023-11-02 11:54:16 +01:00
|
|
|
from spiders.fdb_spider import *
|
|
|
|
|
|
|
|
|
|
config = "spiders/config.yaml"
|
2023-11-27 16:10:11 +01:00
|
|
|
list_of_fdbs = ["giz"]
|
|
|
|
|
#list_of_fdbs = ["foerderinfo.bund.de-bekanntmachungen"]
|
2023-11-02 11:54:16 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
# doing the crawling of government websites
|
|
|
|
|
|
2023-11-06 20:17:45 +01:00
|
|
|
spider = fdb_spider(config)
|
2023-11-02 11:54:16 +01:00
|
|
|
|
2023-11-27 16:10:11 +01:00
|
|
|
spider.download_entry_list_pages_of_funding_databases(list_of_fdbs)
|
2023-11-02 11:54:16 +01:00
|
|
|
|
2023-11-13 17:44:11 +01:00
|
|
|
#spider.find_config_parameter(list_of_fdbs)
|
|
|
|
|
|
2023-11-14 15:38:45 +01:00
|
|
|
#spider.parse_entry_list_data2dictionary(list_of_fdbs)
|
2023-11-02 11:54:16 +01:00
|
|
|
|
2023-11-22 18:03:15 +01:00
|
|
|
#spider.download_entry_data_htmls(list_of_fdbs)
|
2023-11-02 11:54:16 +01:00
|
|
|
|
2023-11-27 16:10:11 +01:00
|
|
|
#spider.parse_entry_data2dictionary(list_of_fdbs)
|
2023-11-02 11:54:16 +01:00
|
|
|
|
