diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8b8b7d9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/venv + diff --git a/spiders/config.yaml b/spiders/config.yaml new file mode 100644 index 0000000..da83c38 --- /dev/null +++ b/spiders/config.yaml @@ -0,0 +1,20 @@ +# Settings for the PEP crawler per country to crawl +# Follow the syntax and dont use tbody as it gets added by the browser (when researching xpath through inspector) + +# xpath syntax: https://www.w3schools.com/xml/xpath_syntax.asp +# lxml xpath syntax: https://www.geeksforgeeks.org/web-scraping-using-lxml-and-xpath-in-python/ + +foerderinfo.bund.de: + domain: 'http://foerderinfo.bund.de' + entryList: + link: '' + parent: '//html//body//form//table//tr//td//table//tr' + child-name: '//td//a/text()' + child-link: '//td//a/@href' + member: + info-1: + parent: '//html//body//form//table' + #child-name: '//html//body//form//table//tr[1]//td[2]//span' + #child-sum: '//html//body//form//table//tr[2]//td[1]//span//img' + #child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1' + diff --git a/fdb_spider.py b/spiders/fdb_spider.py similarity index 99% rename from fdb_spider.py rename to spiders/fdb_spider.py index 052df02..a4930b8 100644 --- a/fdb_spider.py +++ b/spiders/fdb_spider.py @@ -27,7 +27,7 @@ class fdb_spider(object): for key in self.config: if key in list_of_fdbs: try: - entry_list = self.config.get(key).get("entry_list") + entry_list = self.config.get(key).get("entry-list") except Exception as e: print( "There is a problem with the configuration variable entryList in the config.yaml - the original error message is:",