added first config.yaml template and started creating folder structure
This commit is contained in:
parent
b3011efc73
commit
5ac07d151a
3 changed files with 23 additions and 1 deletions
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
/venv
|
||||||
|
|
20
spiders/config.yaml
Normal file
20
spiders/config.yaml
Normal file
|
@ -0,0 +1,20 @@
|
||||||
|
# Settings for the PEP crawler per country to crawl
|
||||||
|
# Follow the syntax and dont use tbody as it gets added by the browser (when researching xpath through inspector)
|
||||||
|
|
||||||
|
# xpath syntax: https://www.w3schools.com/xml/xpath_syntax.asp
|
||||||
|
# lxml xpath syntax: https://www.geeksforgeeks.org/web-scraping-using-lxml-and-xpath-in-python/
|
||||||
|
|
||||||
|
foerderinfo.bund.de:
|
||||||
|
domain: 'http://foerderinfo.bund.de'
|
||||||
|
entryList:
|
||||||
|
link: ''
|
||||||
|
parent: '//html//body//form//table//tr//td//table//tr'
|
||||||
|
child-name: '//td//a/text()'
|
||||||
|
child-link: '//td//a/@href'
|
||||||
|
member:
|
||||||
|
info-1:
|
||||||
|
parent: '//html//body//form//table'
|
||||||
|
#child-name: '//html//body//form//table//tr[1]//td[2]//span'
|
||||||
|
#child-sum: '//html//body//form//table//tr[2]//td[1]//span//img'
|
||||||
|
#child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1'
|
||||||
|
|
|
@ -27,7 +27,7 @@ class fdb_spider(object):
|
||||||
for key in self.config:
|
for key in self.config:
|
||||||
if key in list_of_fdbs:
|
if key in list_of_fdbs:
|
||||||
try:
|
try:
|
||||||
entry_list = self.config.get(key).get("entry_list")
|
entry_list = self.config.get(key).get("entry-list")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(
|
print(
|
||||||
"There is a problem with the configuration variable entryList in the config.yaml - the original error message is:",
|
"There is a problem with the configuration variable entryList in the config.yaml - the original error message is:",
|
Loading…
Reference in a new issue