alpcentaur/fdb-spider
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

added multi pages configuration and code

Browse source
This commit is contained in:
alpcentaur 2023-11-06 18:17:32 +00:00
parent 7aa903883b
commit 8b20bc178f
2 changed files with 183 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
spiders/config.yaml
View file

@ -7,14 +7,13 @@
foerderinfo.bund.de: foerderinfo.bund.de:
domain: 'http://foerderinfo.bund.de' domain: 'http://foerderinfo.bund.de'
entry-list: entry-list:
link1: 'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/expertensuche/Servicesuche_Formular.html#searchResults' link1: 'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/expertensuche/Servicesuche_Formular.html?gtp=33498_list%253D'
link2: 'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/expertensuche/Servicesuche_Formular.html?gtp=33498_list%253D2#searchResults' link2: '#searchResults'
link3: 'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/expertensuche/Servicesuche_Formular.html?gtp=33498_list%253D3#searchResults' iteration-var-list: [1,2,3,4,5,6,7,8]
iteration-var-list: [1,1
parent: '//html//body//form//table//tr//td//table//tr' parent: '//html//body//form//table//tr//td//table//tr'
child-name: '//td//a/text()' child-name: '//td//a/text()'
child-link: '//td//a/@href' child-link: '//td//a/@href'
member: entry:
info-1: info-1:
parent: '//html//body//form//table' parent: '//html//body//form//table'
#child-name: '//html//body//form//table//tr[1]//td[2]//span' #child-name: '//html//body//form//table//tr[1]//td[2]//span'

75
spiders/fdb_spider.py
View file

@ -34,31 +34,59 @@ class fdb_spider(object):
e, e,
) )
try: try:
entry_list_link = entry_list.get("link") entry_list_link1 = entry_list.get("link1")
except Exception as e: except Exception as e:
print( print(
"No entryListLink defined in config.yaml - the original error message is:", "No link1 defined in config.yaml - the original error message is:",
e, e,
) )
try:
entry_list_link2 = entry_list.get("link2")
except Exception as e:
print(
"No link2 defined in config.yaml - the original error message is:",
e,
)
try:
entry_iteration_var_list = eval(entry_list.get("iteration-var-list"))
except Exception as e:
print(
"No iteration-var-list defined in config.yaml - the original error message is:",
e,
)
for i in entry_iteration_var_list:
# download the html page of the List of entrys # download the html page of the List of entrys
response = urllib.request.urlopen(entry_list_link) response = urllib.request.urlopen(entry_list_link1 + string(i) + entry_list_link2)
web_content = response.read().decode("UTF-8") web_content = response.read().decode("UTF-8")
# save interim results to files # save interim results to files
f = open("spiders/pages/" + key + "entryList.html", "w+") f = open("spiders/pages/" + key + str(i) + "entryList.html", "w+")
f.write(webContent) f.write(webContent)
f.close f.close
def parse_entry_list_data2dictionary(self, list_of_fdbs): def parse_entry_list_data2dictionary(self, list_of_fdbs):
for fdb in list_of_fdbs: for fdb in list_of_fdbs:
try:
iteration_var_list = eval(self.config.get(fdb).get("entry-list").get("iteration-var-list"))
except Exception as e:
print(
"There is a problem with the configuration variable entryList iteration var list in the config.yaml - the original error message is:",
e,
)
for i in iteration_var_list:
try: try:
# use soupparser to handle broken html # use soupparser to handle broken html
tree = lxml.html.soupparser.parse( tree = lxml.html.soupparser.parse(
"spiders/pages/" + fdb + "entryList.html" "spiders/pages/" + fdb + str(i) + "entryList.html"
) )
# for e in tree.iter(): # for e in tree.iter():
@ -73,7 +101,7 @@ class fdb_spider(object):
fdb_conf = self.config.get(fdb) fdb_conf = self.config.get(fdb)
fdb_domain = fdb_conf.get("domain") fdb_domain = fdb_conf.get("domain")
fdb_conf_entry_list = fdb_conf.get("entryList") fdb_conf_entry_list = fdb_conf.get("entry-list")
fdb_conf_entry_list_parent = fdb_conf_entry_list.get("parent") fdb_conf_entry_list_parent = fdb_conf_entry_list.get("parent")
fdb_conf_entry_list_child_name = fdb_conf_entry_list.get("child-name") fdb_conf_entry_list_child_name = fdb_conf_entry_list.get("child-name")
fdb_conf_entry_list_child_link = fdb_conf_entry_list.get("child-link") fdb_conf_entry_list_child_link = fdb_conf_entry_list.get("child-link")
@ -112,13 +140,25 @@ class fdb_spider(object):
# save interim results to files # save interim results to files
f = open("spiders/output/" + fdb + "entryList.txt", "w+") f = open("spiders/output/" + fdb + str(i) + "entryList.txt", "w+")
f.write(str(dictionary_entry_list)) f.write(str(dictionary_entry_list))
f.close f.close
def download_entry_data_htmls(self, list_of_fdbs): def download_entry_data_htmls(self, list_of_fdbs):
for fdb in list_of_fdbs: for fdb in list_of_fdbs:
f = open("spiders/output/" + fdb + "entryList.txt")
try:
iteration_var_list = eval(self.config.get(fdb).get("entry-list").get("iteration-var-list"))
except Exception as e:
print(
"There is a problem with the configuration variable entryList iteration var list in the config.yaml - the original error message is:",
e,
)
for i in iteration_var_list:
f = open("spiders/output/" + fdb + str(i) + "entryList.txt")
text = f.read() text = f.read()
dictionary_entry_list = eval(text) dictionary_entry_list = eval(text)
@ -133,7 +173,7 @@ class fdb_spider(object):
# save interim results to files # save interim results to files
file_name = "spiders/pages/" + fdb + "/" + str(entry_id) + ".html" file_name = "spiders/pages/" + fdb + str(i) + "/" + str(entry_id) + ".html"
os.makedirs(os.path.dirname(file_name), exist_ok=True) os.makedirs(os.path.dirname(file_name), exist_ok=True)
f = open(file_name, "w+") f = open(file_name, "w+")
@ -142,9 +182,20 @@ class fdb_spider(object):
def parse_entry_data2dictionary(self, list_of_fdbs): def parse_entry_data2dictionary(self, list_of_fdbs):
for fdb in list_of_fdbs: for fdb in list_of_fdbs:
try:
iteration_var_list = eval(self.config.get(fdb).get("entry-list").get("iteration-var-list"))
except Exception as e:
print(
"There is a problem with the configuration variable entryList iteration var list in the config.yaml - the original error message is:",
e,
)
for i in iteration_var_list:
print("started to parse data of entry of " + fdb + " ..") print("started to parse data of entry of " + fdb + " ..")
f = open("spiders/output/" + fdb + "entryList.txt") f = open("spiders/output/" + fdb + str(i) + "entryList.txt")
text = f.read() text = f.read()
dictionary_entry_list = eval(text) dictionary_entry_list = eval(text)
@ -165,7 +216,7 @@ class fdb_spider(object):
+ " .." + " .."
) )
file_name = "spiders/pages/" + fdb + "/" + str(entry_id) + ".html" file_name = "spiders/pages/" + fdb + str(i) + "/" + str(entry_id) + ".html"
tree = lxml.html.soupparser.parse(file_name) tree = lxml.html.soupparser.parse(file_name)
@ -181,6 +232,6 @@ class fdb_spider(object):
0 0
] ]
f = open("spiders/output/" + fdb + "entryList.txt", "w+") f = open("spiders/output/" + fdb + str(i) + "entryList.txt", "w+")
f.write(str(dictionary_entry_list)) f.write(str(dictionary_entry_list))
f.close f.close