alpcentaur/fdb-spider
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

added other exception for unitrue var text not being found, before saving index 0 to variable produced error to whole execution

Browse source
This commit is contained in:
alpcentaur 2024-01-10 15:28:41 +00:00
parent 20db0028e1
commit af8374f715
6 changed files with 26 additions and 126073 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -2,3 +2,5 @@
/spiders/pages/** /spiders/pages/**
/spiders/output/** /spiders/output/**
/spiders/config.yaml /spiders/config.yaml
/spiders/output_old
/spiders/pages_old

6
main.py
View file

@ -3,7 +3,7 @@ from spiders.fdb_spider import *
import sys import sys
config = "/root/fdb-spider/spiders/config.yaml" config = "spiders/config.yaml"
#list_of_fdbs = eval(sys.argv[1]) #list_of_fdbs = eval(sys.argv[1])
list_of_fdbs = ["giz","evergabe-online"] list_of_fdbs = ["giz","evergabe-online"]
@ -12,11 +12,11 @@ list_of_fdbs = ["giz","evergabe-online"]
spider = fdb_spider(config) spider = fdb_spider(config)
#spider.download_entry_list_pages_of_funding_databases(list_of_fdbs) spider.download_entry_list_pages_of_funding_databases(list_of_fdbs)
#spider.find_config_parameter(list_of_fdbs) #spider.find_config_parameter(list_of_fdbs)
#spider.parse_entry_list_data2dictionary(list_of_fdbs) spider.parse_entry_list_data2dictionary(list_of_fdbs)
spider.download_entry_data_htmls(list_of_fdbs) spider.download_entry_data_htmls(list_of_fdbs)

126054
spider.log
View file

File diff suppressed because it is too large Load diff

BIN
spiders/__pycache__/fdb_spider.cpython-39.pyc
View file

Binary file not shown.

20
spiders/config.yaml
View file

@ -7,8 +7,8 @@ foerderinfo.bund.de:
entry-list: entry-list:
link1: 'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/expertensuche/Servicesuche_Formular.html?gtp=33498_list%253D' link1: 'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/expertensuche/Servicesuche_Formular.html?gtp=33498_list%253D'
link2: '#searchResults' link2: '#searchResults'
#iteration-var-list: '[1,2,3,4,5]' iteration-var-list: '[1,2,3,4,5]'
iteration-var-list: '[1,2,3]' #iteration-var-list: '[1,2]'
parent: "//html//body//div//main//div//div[@class='row']//section[@class='l-search-result-list']//div[@class='l-search-result-list__item']" parent: "//html//body//div//main//div//div[@class='row']//section[@class='l-search-result-list']//div[@class='l-search-result-list__item']"
child-name: "//div[@class='c-search-result__text-wrapper']//span[@class='c-search-result__title']/text()" child-name: "//div[@class='c-search-result__text-wrapper']//span[@class='c-search-result__title']/text()"
jsdomain: 'None' jsdomain: 'None'
@ -30,8 +30,8 @@ foerderinfo.bund.de-bekanntmachungen:
link2: '#searchResults' link2: '#searchResults'
# here jsdomain has to be specified None with this syntax, if html pages of entrylists are not depending on javascript actions # here jsdomain has to be specified None with this syntax, if html pages of entrylists are not depending on javascript actions
jsdomain: 'None' jsdomain: 'None'
#iteration-var-list: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]' iteration-var-list: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]'
iteration-var-list: '[1,2,3]' #iteration-var-list: '[1,2]'
#parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']" #parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']"
parent: "//html//body//div//main//div[@class='row']//a[contains(@class, 'c-teaser--announcement')]" parent: "//html//body//div//main//div[@class='row']//a[contains(@class, 'c-teaser--announcement')]"
child-name: "//div[@class='c-teaser__text-wrapper']//span[@class='c-teaser__title']/text()" child-name: "//div[@class='c-teaser__text-wrapper']//span[@class='c-teaser__title']/text()"
@ -57,8 +57,8 @@ giz:
link1: 'https://ausschreibungen.giz.de/Satellite/company/welcome.do?method=showTable&fromSearch=1&tableSortPROJECT_RESULT=2&tableSortAttributePROJECT_RESULT=publicationDate&selectedTablePagePROJECT_RESULT=' link1: 'https://ausschreibungen.giz.de/Satellite/company/welcome.do?method=showTable&fromSearch=1&tableSortPROJECT_RESULT=2&tableSortAttributePROJECT_RESULT=publicationDate&selectedTablePagePROJECT_RESULT='
link2: '' link2: ''
jsdomain: 'None' jsdomain: 'None'
#iteration-var-list: "[1,2,3,4,5,6,7]" iteration-var-list: "[1,2,3,4,5,6,7]"
iteration-var-list: "[1,2,3]" #iteration-var-list: "[1,2]"
#parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']" #parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']"
parent: "//html//body//div//div//table[contains(@class, 'csx-new-table')]//tbody//tr" parent: "//html//body//div//div//table[contains(@class, 'csx-new-table')]//tbody//tr"
child-name: "//td[3]//text()" child-name: "//td[3]//text()"
@ -86,10 +86,10 @@ evergabe-online:
jsdomain: 'https://www.evergabe-online.de/search.html' jsdomain: 'https://www.evergabe-online.de/search.html'
jslink1: '/html/body/div[8]/main/div[4]/div/div/div[2]/table/thead/tr[1]/td/div[2]/div/span[' jslink1: '/html/body/div[8]/main/div[4]/div/div/div[2]/table/thead/tr[1]/td/div[2]/div/span['
jslink2: ']' jslink2: ']'
#jsiteration-var-list: "[1,2, 3, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,6,7,8,9,10]" jsiteration-var-list: "[1,2, 3, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,6,7,8,9,10]"
jsiteration-var-list: "[1,2, 3, 4, 5, 6]" #jsiteration-var-list: "[1,2, 3]"
#iteration-var-list: "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102]" iteration-var-list: "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102]"
iteration-var-list: "[1, 2, 3, 4, 5, 6]" #iteration-var-list: "[1, 2, 3]"
parent: "/html/body/div[8]/main/div[4]/div/div/div[2]/table/tbody//tr" parent: "/html/body/div[8]/main/div[4]/div/div/div[2]/table/tbody//tr"
child-name: "//td[1]/div/a/text()" child-name: "//td[1]/div/a/text()"
child-link: "//td[1]/div/a/@href" child-link: "//td[1]/div/a/@href"

17
spiders/fdb_spider.py
View file

@ -699,14 +699,19 @@ class fdb_spider(object):
fdb_conf_entry_unitrue_child = fdb_conf_entry_unitrue.get(key) fdb_conf_entry_unitrue_child = fdb_conf_entry_unitrue.get(key)
print('unitrue_child',fdb_conf_entry_unitrue_child)
try:
child = tree.xpath( child = tree.xpath(
fdb_conf_entry_unitrue_child fdb_conf_entry_unitrue_child
)[0] )[0]
print('oi', child)
except:
print('getting unitruechild did not work')
child = 'NONE'
print("oi", child) print("oi", child)
if '.pdf' in child: if '.pdf' in child:
print('child in entry data is pdf, downloading it..') print('child in entry data is pdf, downloading it..')