alpcentaur
/
fdb-spider
1
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 2 Wiki Activity
Browse Source

added first changes to fix js related bug for giz db

master
alpcentaur 10 months ago
parent
dec60f9bf5
commit
20db0028e1
5 changed files with 126086 additions and 21 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +4
    -4
      main.py
  2. +126054
    -0
      spider.log
  3. BIN
      spiders/__pycache__/fdb_spider.cpython-311.pyc
  4. +18
    -7
      spiders/config.yaml
  5. +10
    -10
      spiders/fdb_spider.py

+ 4
- 4
main.py View File

@ -3,9 +3,9 @@ from spiders.fdb_spider import *
import sys
config = "spiders/config.yaml"
list_of_fdbs = eval(sys.argv[1])
#list_of_fdbs = ["foerderinfo.bund.de-bekanntmachungen"]
config = "/root/fdb-spider/spiders/config.yaml"
#list_of_fdbs = eval(sys.argv[1])
list_of_fdbs = ["giz","evergabe-online"]
# doing the crawling of government websites
@ -20,5 +20,5 @@ spider = fdb_spider(config)
spider.download_entry_data_htmls(list_of_fdbs)
#spider.parse_entry_data2dictionary(list_of_fdbs)
spider.parse_entry_data2dictionary(list_of_fdbs)

+ 126054
- 0
spider.log
File diff suppressed because it is too large
View File


BIN
spiders/__pycache__/fdb_spider.cpython-311.pyc View File


+ 18
- 7
spiders/config.yaml View File

@ -7,9 +7,12 @@ foerderinfo.bund.de:
entry-list:
link1: 'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/expertensuche/Servicesuche_Formular.html?gtp=33498_list%253D'
link2: '#searchResults'
iteration-var-list: '[1,2,3,4,5]'
#iteration-var-list: '[1,2,3,4,5]'
iteration-var-list: '[1,2,3]'
parent: "//html//body//div//main//div//div[@class='row']//section[@class='l-search-result-list']//div[@class='l-search-result-list__item']"
child-name: "//div[@class='c-search-result__text-wrapper']//span[@class='c-search-result__title']/text()"
jsdomain: 'None'
javascript-link: ""
child-link: "/a[@class='c-search-result']/@href"
child-info: "//"
child-period: "/"
@ -25,10 +28,14 @@ foerderinfo.bund.de-bekanntmachungen:
entry-list:
link1: 'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/bekanntmachungen/Bekanntmachungen_Formular.html?gtp=407348_list%253D'
link2: '#searchResults'
iteration-var-list: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]'
# here jsdomain has to be specified None with this syntax, if html pages of entrylists are not depending on javascript actions
jsdomain: 'None'
#iteration-var-list: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]'
iteration-var-list: '[1,2,3]'
#parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']"
parent: "//html//body//div//main//div[@class='row']//a[contains(@class, 'c-teaser--announcement')]"
child-name: "//div[@class='c-teaser__text-wrapper']//span[@class='c-teaser__title']/text()"
javascript-link: ""
child-link: "/@href"
child-info: "//div[@class='c-teaser__text-wrapper']//div[@class='c-teaser__text']/p/text()"
#child-period: "//div[@class='c-teaser__text-wrapper']//small[@class='c-topline']/span[@class='c-topline__item']/text()"
@ -49,7 +56,9 @@ giz:
entry-list:
link1: 'https://ausschreibungen.giz.de/Satellite/company/welcome.do?method=showTable&fromSearch=1&tableSortPROJECT_RESULT=2&tableSortAttributePROJECT_RESULT=publicationDate&selectedTablePagePROJECT_RESULT='
link2: ''
iteration-var-list: "[1,2,3,4,5,6,7]"
jsdomain: 'None'
#iteration-var-list: "[1,2,3,4,5,6,7]"
iteration-var-list: "[1,2,3]"
#parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']"
parent: "//html//body//div//div//table[contains(@class, 'csx-new-table')]//tbody//tr"
child-name: "//td[3]//text()"
@ -77,8 +86,10 @@ evergabe-online:
jsdomain: 'https://www.evergabe-online.de/search.html'
jslink1: '/html/body/div[8]/main/div[4]/div/div/div[2]/table/thead/tr[1]/td/div[2]/div/span['
jslink2: ']'
jsiteration-var-list: "[1,2, 3, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,6,7,8,9,10]"
iteration-var-list: "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102]"
#jsiteration-var-list: "[1,2, 3, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,6,7,8,9,10]"
jsiteration-var-list: "[1,2, 3, 4, 5, 6]"
#iteration-var-list: "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102]"
iteration-var-list: "[1, 2, 3, 4, 5, 6]"
parent: "/html/body/div[8]/main/div[4]/div/div/div[2]/table/tbody//tr"
child-name: "//td[1]/div/a/text()"
child-link: "//td[1]/div/a/@href"
@ -88,10 +99,10 @@ evergabe-online:
child-sponsor: "//td[2]/div/text()"
entry:
general:
uniform: 'TRUE'
uniform: 'FALSE'
unitrue:
#parent: '//html//body//form//table'
text: '/html/body/div[2]/div[4]/div/div[5]/div/table/tbody/tr/td[5]/a/@href'
#text: '/html/body/div[2]/div[4]/div/div[5]/div/table/tbody/tr/td[5]/a/@href'
#child-sum: '//html//body//form//table//tr[2]//td[1]//span//img'
#child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1'
unifalse:

+ 10
- 10
spiders/fdb_spider.py View File

@ -407,7 +407,7 @@ class fdb_spider(object):
dictionary_entry_list[n]["name"] = name
dictionary_entry_list[n]["info"] = info
dictionary_entry_list[n]["period"] = period
#print('linklink', link, fdb_domain)
print('linklink', link, fdb_domain)
if fdb_domain in link:
print('oi')
dictionary_entry_list[n]["link"] = link
@ -423,7 +423,7 @@ class fdb_spider(object):
if fdb_domain not in link:
if 'http' not in link:
if 'www' not in link:
#print('oiB')
print('oiB')
if link[0] == '/':
if fdb_domain[-1] != '/':
dictionary_entry_list[n]["link"] = fdb_domain + link
@ -434,12 +434,12 @@ class fdb_spider(object):
if link[0] == '.' and link[1] == '/':
if fdb_domain[-1] != '/':
dictionary_entry_list[n]["link"] = fdb_domain + link[1:]
#print('got into B', dictionary_entry_list[n]["link"])
print('got into B', dictionary_entry_list[n]["link"])
if fdb_domain[-1] == '/':
dictionary_entry_list[n]["link"] = fdb_domain + link[2:]
#print('got into A', dictionary_entry_list[n]["link"])
print('got into A', dictionary_entry_list[n]["link"])
if link[0] != '/':
if link[0] != '/' and link[0] != '.':
dictionary_entry_list[n]["link"] = fdb_domain + '/' + link
#print('got into last else', dictionary_entry_list[n]["link"])
@ -478,7 +478,7 @@ class fdb_spider(object):
driver = webdriver.Chrome(options=options, service=service)
#driver = webdriver.Chrome()
for fdb in list_of_fdbs:
print('spidering ' + fdb + ' ..')
try:
iteration_var_list = eval(self.config.get(fdb).get("entry-list").get("iteration-var-list"))
except Exception as e:
@ -514,10 +514,10 @@ class fdb_spider(object):
entry_link = dictionary_entry_list[entry_id]["link"]
web_content = 'NONE'
# download the html page of the entry
print(entry_link)
if 'javascript' in entry_link:
print('oioioi',fdb_conf_entry_list_parent, entry_id, fdb_conf_entry_list_javascript_link)
element = driver.find_element(
"xpath",
fdb_conf_entry_list_parent
@ -528,8 +528,8 @@ class fdb_spider(object):
)
# to time.sleep was suggested for errors
#import time
#time.sleep(1)
import time
time.sleep(1)
element.click()
window_after = driver.window_handles[1]

Write Preview
Loading…
Cancel
Save