Browse Source

added first config.yaml template and started creating folder structure

onlinkgen
alpcentaur 1 year ago
parent
commit
5ac07d151a
3 changed files with 23 additions and 1 deletions
  1. +2
    -0
      .gitignore
  2. +20
    -0
      spiders/config.yaml
  3. +1
    -1
      spiders/fdb_spider.py

+ 2
- 0
.gitignore View File

@ -0,0 +1,2 @@
/venv

+ 20
- 0
spiders/config.yaml View File

@ -0,0 +1,20 @@
# Settings for the PEP crawler per country to crawl
# Follow the syntax and dont use tbody as it gets added by the browser (when researching xpath through inspector)
# xpath syntax: https://www.w3schools.com/xml/xpath_syntax.asp
# lxml xpath syntax: https://www.geeksforgeeks.org/web-scraping-using-lxml-and-xpath-in-python/
foerderinfo.bund.de:
domain: 'http://foerderinfo.bund.de'
entryList:
link: ''
parent: '//html//body//form//table//tr//td//table//tr'
child-name: '//td//a/text()'
child-link: '//td//a/@href'
member:
info-1:
parent: '//html//body//form//table'
#child-name: '//html//body//form//table//tr[1]//td[2]//span'
#child-sum: '//html//body//form//table//tr[2]//td[1]//span//img'
#child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1'

fdb_spider.py → spiders/fdb_spider.py View File

@ -27,7 +27,7 @@ class fdb_spider(object):
for key in self.config: for key in self.config:
if key in list_of_fdbs: if key in list_of_fdbs:
try: try:
entry_list = self.config.get(key).get("entry_list")
entry_list = self.config.get(key).get("entry-list")
except Exception as e: except Exception as e:
print( print(
"There is a problem with the configuration variable entryList in the config.yaml - the original error message is:", "There is a problem with the configuration variable entryList in the config.yaml - the original error message is:",

Loading…
Cancel
Save