fdb-spider/main.py

from spiders.fdb_spider import *


import sys

config = "spiders/config.yaml"
#list_of_fdbs = eval(sys.argv[1])
#list_of_fdbs = ["giz","evergabe-online","foerderinfo.bund.de-bekanntmachungen"]
#list_of_fdbs = ["giz","evergabe-online"]
#list_of_fdbs = ["foerderinfo.bund.de-bekanntmachungen"]
list_of_fdbs = ["ted.europa.eu"]
#list_of_fdbs = ["dtvp"]


# doing the crawling of government websites

spider = fdb_spider(config)

spider.download_entry_list_pages_of_funding_databases(list_of_fdbs)

#spider.find_config_parameter(list_of_fdbs)

spider.parse_entry_list_data2dictionary(list_of_fdbs)

#spider.download_entry_data_htmls(list_of_fdbs)

#spider.parse_entry_data2dictionary(list_of_fdbs)