From ece5cf13015a567ca1445fc15d4a9cc43c4c3253 Mon Sep 17 00:00:00 2001 From: alpcentaur Date: Fri, 15 Dec 2023 13:34:23 +0100 Subject: [PATCH] added better logic for getting the right link of entry --- spiders/__pycache__/fdb_spider.cpython-39.pyc | Bin 16864 -> 16897 bytes spiders/config.yaml | 102 +++--------------- spiders/fdb_spider.py | 40 ++++--- .../output/foerderinfo.bund.de1entryList.txt | 1 - 4 files changed, 41 insertions(+), 102 deletions(-) delete mode 100644 spiders/output/foerderinfo.bund.de1entryList.txt diff --git a/spiders/__pycache__/fdb_spider.cpython-39.pyc b/spiders/__pycache__/fdb_spider.cpython-39.pyc index a8d26b3005800c673ae14119ae4d445e31952ed8..396f12f7dfe0db4c408752028bf313fc56fd0c23 100644 GIT binary patch delta 838 zcmZ`%&ubGw6rQ)gY|>;Yk|xz7&I{p4(5K4?5hx&AzA{SN;mfc?bJ{X|qp!6xEIIMWSEv_E z36EF+hy=(UGNd6KJ%xye1S(*Q#vG0@gg9cn(f{%`@IIHf@lTy5j=S|{ZygWU-Skdx z+O-FxfD^0^Hb-N-hy*Y{X0j~Ul93@EyuY&(+0Ms1L#THUx9W0PuZq#Spdpu4@6Owm zJkXKcpw^z?~MjcKOQGS2$>!tTM$Bcf-RkXh$Y~V^F4M}NDB+4mNZwIFW2Oo zrn#6aiVLMPHK|&yrfUk%+{v{h{x6Cw=gh>P0ucHok%tIPj(t8p06qvnH*(|yZ)bVI zE2yBt5Q||U%21diQH7}Nj3=KcN`pN`<34>evBU&V=xJD`ST91+d8^NgV9`YSr)=D1 zB&tp7@X0Am{_^ZLozG4ytGxdK-O8pE&N3d+kJ-#1;c$hAb*khpjNId}!b6>f*1#gZ uewB|qV?1(MbM1G@N#R8wS@n{W%ZY)Sk+~G3VV>41NQ=%G5Ui delta 792 zcmZo{VSLcc$d|~=%f$c$|BP!=rB`m`OJ`zqn_R}Ews|6xg&2xLw z6@tO+%s@6v?25_yl9i0-Ca;vtW{s;AsM4BjD=od*MCt@%y-kfs4SS7%Btwd{4MPoQ zjYt{e1m>cc6d53k0mQ;8Uy~vWm9N1nKPN>FDnAEVK1CENx(Oy)k7O-lkxPpF1m=QM zDGFc`WCh~{plFRq3dn(q*-R5yid1SiQrv*4@*vFUUn60MoCG5)#N}0Q8qBMcypFQKO7^RqHST=ho_pz}x08>{{|IZ22lV$10RtwVJI(eU^8ry6Tvt#lrOC#=T z5VHnEv`kjF@{z6wG0Q+i9gw)iQIL`r5Ask*QQPJ|D^ZX=OKhhx&YNs)ca*VwGo$@d ZM($>i@;M-4+2oB5MvUs4pE$@e0svz4%eepm diff --git a/spiders/config.yaml b/spiders/config.yaml index c0be443..f0e4e2d 100644 --- a/spiders/config.yaml +++ b/spiders/config.yaml @@ -4,93 +4,23 @@ # xpath syntax: https://www.w3schools.com/xml/xpath_syntax.asp # lxml xpath syntax: https://www.geeksforgeeks.org/web-scraping-using-lxml-and-xpath-in-python/ -foerderinfo.bund.de: - domain: 'http://foerderinfo.bund.de' +greenjobs: + domain: 'https://www.greenjobs.de' entry-list: - link1: 'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/expertensuche/Servicesuche_Formular.html?gtp=33498_list%253D' - link2: '#searchResults' - iteration-var-list: '[1,2,3,4,5]' - parent: "//html//body//div//main//div//div[@class='row']//section[@class='l-search-result-list']//div[@class='l-search-result-list__item']" - child-name: "//div[@class='c-search-result__text-wrapper']//span[@class='c-search-result__title']/text()" - child-link: "/a[@class='c-search-result']/@href" - child-info: "//" - child-period: "/" - child-sponsor: "/" - entry: - info-1: - parent: '//html//body//form//table' - #child-name: '//html//body//form//table//tr[1]//td[2]//span' - #child-sum: '//html//body//form//table//tr[2]//td[1]//span//img' - #child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1' - -foerderinfo.bund.de-bekanntmachungen: - domain: 'http://foerderinfo.bund.de' - entry-list: - link1: 'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/bekanntmachungen/Bekanntmachungen_Formular.html?gtp=407348_list%253D' - link2: '#searchResults' - iteration-var-list: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]' - #parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']" - parent: "//html//body//div//main//div[@class='row']//a[contains(@class, 'c-teaser--announcement')]" - child-name: "//div[@class='c-teaser__text-wrapper']//span[@class='c-teaser__title']/text()" - child-link: "/@href" - child-info: "//div[@class='c-teaser__text-wrapper']//div[@class='c-teaser__text']/p/text()" - #child-period: "//div[@class='c-teaser__text-wrapper']//small[@class='c-topline']/span[@class='c-topline__item']/text()" - child-period: "//div[@class='c-teaser__text-wrapper']//small//span/time/text()" - child-sponsor: "//div[@class='c-teaser__text-wrapper']//small[@class='c-topline']//span[@class='c-topline__item']/span[@class='c-topline__category']/text()" - entry: - general: - uniform: 'FALSE' - unitrue: - parent: '//html//body//form//table' - #child-name: '//html//body//form//table//tr[1]//td[2]//span' - #child-sum: '//html//body//form//table//tr[2]//td[1]//span//img' - #child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1' - unifalse: - wordlist: "['Mobilität', 'Energie', 'Off-grid', 'regenerative Energien', 'Solar', 'Energienetze', 'Elektromobilität']" - -giz: - domain: 'https://ausschreibungen.giz.de' - entry-list: - link1: 'https://ausschreibungen.giz.de/Satellite/company/welcome.do?method=showTable&fromSearch=1&tableSortPROJECT_RESULT=2&tableSortAttributePROJECT_RESULT=publicationDate&selectedTablePagePROJECT_RESULT=' - link2: '' - iteration-var-list: "[1,2,3,4,5,6,7]" - #parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']" - parent: "//html//body//div//div//table[contains(@class, 'csx-new-table')]//tbody//tr" - child-name: "//td[3]//text()" - child-link: "//a/@href" - javascript-link: "/td[6]/a" - child-info: "/td[4]/text()[1]" - child-period: "//td[2]/abbr/text()" - #child-period: "//div[@class='c-teaser__text-wrapper']//small//span/time/text()" - child-sponsor: "/tr/td[4]/text()" - entry: - general: - uniform: 'TRUE' - unitrue: - #parent: '//html//body//form//table' - text: '/html/body/div[2]/div[4]/div/div[5]/div/table/tbody/tr/td[5]/a/@href' - #child-sum: '//html//body//form//table//tr[2]//td[1]//span//img' - #child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1' - unifalse: - wordlist: "['Mobilität', 'Energie', 'Off-grid', 'regenerative Energien', 'Solar', 'Energienetze', 'Elektromobilität']" - -evergabe-online: - domain: 'https://www.evergabe-online.de/' - entry-list: - link1: 'https://www.evergabe-online.de/search.html?101-1.-searchPanel-results-searchResults-results-topToolbars-toolbars-1-span-navigator-navigation-' - link2: '-pageLink' - jsdomain: 'https://www.evergabe-online.de/search.html' - jslink1: '/html/body/div[8]/main/div[4]/div/div/div[2]/table/thead/tr[1]/td/div[2]/div/span[' - jslink2: ']' - jsiteration-var-list: "[1,2, 3, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,6,7,8,9,10]" - iteration-var-list: "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102]" - parent: "/html/body/div[8]/main/div[4]/div/div/div[2]/table/tbody//tr" - child-name: "//td[1]/div/a/text()" - child-link: "//td[1]/div/a/@href" - javascript-link: "" - child-info: "//td[3]/div/text()" - child-period: "//td[5]/text()" - child-sponsor: "//td[2]/div/text()" + link1: 'https://www.greenjobs.de/angebote/index.html?s=&loc=&countrycode=de&dist=' + link2: '0&lng=&lat=' + jsdomain: 'NONE' + #jslink1: '/html/body/div[8]/main/div[4]/div/div/div[2]/table/thead/tr[1]/td/div[2]/div/span[' + #jslink2: ']' + #jsiteration-var-list: "[1,2, 3, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,6,7,8,9,10]" + iteration-var-list: "['1']" + parent: "/html/body/div/div/div/div/main/div[2]/table/tbody//tr" + child-name: "//td[1]/a/text()" + child-link: "//td[1]/a/@href" + #javascript-link: "" + child-info: "//td[5]/text()" + child-period: "//td[6]/text()" + child-sponsor: "//td[3]/text()" entry: general: uniform: 'TRUE' diff --git a/spiders/fdb_spider.py b/spiders/fdb_spider.py index 7c197f0..ec648d3 100644 --- a/spiders/fdb_spider.py +++ b/spiders/fdb_spider.py @@ -407,10 +407,12 @@ class fdb_spider(object): dictionary_entry_list[n]["name"] = name dictionary_entry_list[n]["info"] = info dictionary_entry_list[n]["period"] = period - + #print('linklink', link, fdb_domain) if fdb_domain in link: + print('oi') dictionary_entry_list[n]["link"] = link if fdb_domain not in link and 'http:' in link: + print('oiA') dictionary_entry_list[n]["link"] = link if fdb_domain not in link and 'www.' in link: dictionary_entry_list[n]["link"] = link @@ -418,20 +420,28 @@ class fdb_spider(object): dictionary_entry_list[n]["link"] = link if 'javascript:' in link: dictionary_entry_list[n]["link"] = link - if fdb_domain not in link and ('http' or 'https' or 'www.') not in link: - if link[0] == '/': - if fdb_domain[-1] != '/': - dictionary_entry_list[n]["link"] = fdb_domain + link - if fdb_domain[-1] == '/': - dictionary_entry_list[n]["link"] = fdb_domain + link[1:] - if link[0] == '.' and link[1] == '/': - if fdb_domain[-1] != '/': - dictionary_entry_list[n]["link"] = fdb_domain + link[1:] - if fdb_domain[-1] == '/': - dictionary_entry_list[n]["link"] = fdb_domain + link[2:] - - else: - dictionary_entry_list[n]["link"] = fdb_domain + '/' + link + if fdb_domain not in link: + if 'http' not in link: + if 'www' not in link: + #print('oiB') + if link[0] == '/': + if fdb_domain[-1] != '/': + dictionary_entry_list[n]["link"] = fdb_domain + link + #print('got into D', dictionary_entry_list[n]["link"]) + if fdb_domain[-1] == '/': + dictionary_entry_list[n]["link"] = fdb_domain + link[1:] + #print('got into C', dictionary_entry_list[n]["link"]) + if link[0] == '.' and link[1] == '/': + if fdb_domain[-1] != '/': + dictionary_entry_list[n]["link"] = fdb_domain + link[1:] + #print('got into B', dictionary_entry_list[n]["link"]) + if fdb_domain[-1] == '/': + dictionary_entry_list[n]["link"] = fdb_domain + link[2:] + #print('got into A', dictionary_entry_list[n]["link"]) + + if link[0] != '/': + dictionary_entry_list[n]["link"] = fdb_domain + '/' + link + #print('got into last else', dictionary_entry_list[n]["link"]) diff --git a/spiders/output/foerderinfo.bund.de1entryList.txt b/spiders/output/foerderinfo.bund.de1entryList.txt deleted file mode 100644 index b1c823d..0000000 --- a/spiders/output/foerderinfo.bund.de1entryList.txt +++ /dev/null @@ -1 +0,0 @@ -{0: {'name': 'Newsletter', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/news/newsletter/newsletter.html'}, 1: {'name': 'Wettbewerbe, Preise', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/news/wettbewerbe-preise/wettbewerbe-preise.html'}, 2: {'name': 'Veranstaltungen', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/news/veranstaltungen/veranstaltungen.html'}, 3: {'name': 'Projektträger in der Forschungsförderung', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/projekttraeger/projekttraeger-in-der-forschungsfoerderung.html'}, 4: {'name': 'Leichte Sprache', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/services/leichtesprache/leichte-sprache.html'}, 5: {'name': 'Ausführliche Informationen', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/services/leichtesprache/ausfuehrliche-informationen.html'}, 6: {'name': 'Erklärung zur Barrierefreiheit', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/services/leichtesprache/erklaerung-zur-barrierefreiheit.html'}, 7: {'name': 'Darum geht es auf dieser Seite', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/services/leichtesprache/darum-geht-es-auf-dieser-seite.html'}, 8: {'name': 'FAQ', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/faq/faq.html'}, 9: {'name': 'Forschungs- und Innovationsförderung', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/forschungs-und-innovationsfoerderung/forschungs-und-innovationsfoerderung.html'}, 10: {'name': 'Glossar', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/glossar/glossar.html'}, 11: {'name': 'Bei uns sind Sie bestens beraten!', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/erstberatung/bei-uns-sind-sie-bestens-beraten_.html'}, 12: {'name': 'Unser Service', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/unser-service/unser-service.html'}, 13: {'name': 'Was wir tun', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/was-wir-tun/was-wir-tun.html'}, 14: {'name': '„Ich hab‘ da mal eine Idee“ – Die Förderberatung des Bundes im Gespräch', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/_documents/ich-hab-da-mal-eine-idee.html'}} \ No newline at end of file