You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

24 lines
514 B

  1. from spiders.fdb_spider import *
  2. import sys
  3. config = "spiders/config.yaml"
  4. list_of_fdbs = eval(sys.argv[1])
  5. #list_of_fdbs = ["foerderinfo.bund.de-bekanntmachungen"]
  6. # doing the crawling of government websites
  7. spider = fdb_spider(config)
  8. #spider.download_entry_list_pages_of_funding_databases(list_of_fdbs)
  9. #spider.find_config_parameter(list_of_fdbs)
  10. #spider.parse_entry_list_data2dictionary(list_of_fdbs)
  11. spider.download_entry_data_htmls(list_of_fdbs)
  12. #spider.parse_entry_data2dictionary(list_of_fdbs)