alpcentaur/fdb-spider
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

added further database in config.yaml, added new exception for downloading js generated html pages

Browse source
This commit is contained in:
alpcentaur 2023-11-27 15:10:11 +00:00
parent df4a8289b8
commit a0075e429d
4 changed files with 58 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
main.py
View file

@ -1,15 +1,15 @@
from spiders.fdb_spider import *
config = "spiders/config.yaml"
#list_of_fdbs = ["foerderinfo.bund.de"]
list_of_fdbs = ["foerderinfo.bund.de-bekanntmachungen"]
list_of_fdbs = ["giz"]
#list_of_fdbs = ["foerderinfo.bund.de-bekanntmachungen"]
# doing the crawling of government websites
spider = fdb_spider(config)
# spider.download_entry_list_pages_of_funding_databases(list_of_fdbs)
spider.download_entry_list_pages_of_funding_databases(list_of_fdbs)
#spider.find_config_parameter(list_of_fdbs)
@ -17,5 +17,5 @@ spider = fdb_spider(config)
#spider.download_entry_data_htmls(list_of_fdbs)
spider.parse_entry_data2dictionary(list_of_fdbs)
#spider.parse_entry_data2dictionary(list_of_fdbs)

BIN
spiders/__pycache__/fdb_spider.cpython-39.pyc
View file

Binary file not shown.

26
spiders/config.yaml
View file

@ -47,3 +47,29 @@ foerderinfo.bund.de-bekanntmachungen:
#child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1'
unifalse:
wordlist: "['Mobilität', 'Energie', 'Off-grid', 'regenerative Energien', 'Solar', 'Energienetze', 'Elektromobilität']"
giz:
domain: 'https://ausschreibungen.giz.de'
entry-list:
link1: 'https://ausschreibungen.giz.de/Satellite/company/welcome.do?method=showTable&fromSearch=1&tableSortPROJECT_RESULT=2&tableSortAttributePROJECT_RESULT=publicationDate&selectedTablePagePROJECT_RESULT='
link2: ''
iteration-var-list: '[1,2,3,4,5,6,7]'
#parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']"
parent: "//html//body//div//div//table[contains(@class, 'csx-new-table')]//tbody"
child-name: "//tr//td[2]/text()"
child-link: "/tr//td[5]/a/@href"
child-info: "/tr//td[3]/text()"
child-period: "/tr/td[1]/text()"
#child-period: "//div[@class='c-teaser__text-wrapper']//small//span/time/text()"
child-sponsor: "//tr/td[4]/text()"
entry:
general:
uniform: 'FALSE'
unitrue:
parent: '//html//body//form//table'
#child-name: '//html//body//form//table//tr[1]//td[2]//span'
#child-sum: '//html//body//form//table//tr[2]//td[1]//span//img'
#child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1'
unifalse:
wordlist: "['Mobilität', 'Energie', 'Off-grid', 'regenerative Energien', 'Solar', 'Energienetze', 'Elektromobilität']"

31
spiders/fdb_spider.py
View file

@ -69,10 +69,35 @@ class fdb_spider(object):
# download the html page of the List of entrys
response = urllib.request.urlopen(entry_list_link1 + str(i) + entry_list_link2)
web_content = response.read().decode("UTF-8")
# web_content = response.read().decode("UTF-8")
try:
web_content = response.read().decode("UTF-8")
except Exception as e:
try:
web_content = response.read().decode("latin-1")
print(
"decoding the respone in utf8 did not work, try to decode latin1 now - the original error message is:",
e,
)
except Exception as ex:
print(ex)
# save interim results to files
if (len(web_content)) < 10:
print('getting the html page through urllib did not work, trying with requests librarys function get')
try:
res = requests.get(entry_list_link1 + str(i) + entry_list_link2)
web_content = res.text
except Exception as e:
print('also requests library did not work, original error is:', e)
print(web_content)
f = open("spiders/pages/" + key + str(i) + "entryList.html", "w+")
f.write(web_content)
f.close