added better logic for getting the right link of entry

spiders/config.yaml

@@ -4,93 +4,23 @@
 # xpath syntax: https://www.w3schools.com/xml/xpath_syntax.asp
 # lxml xpath syntax: https://www.geeksforgeeks.org/web-scraping-using-lxml-and-xpath-in-python/
 
-foerderinfo.bund.de:
-  domain: 'http://foerderinfo.bund.de'
+greenjobs:
+  domain: 'https://www.greenjobs.de'
   entry-list:
-    link1:  'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/expertensuche/Servicesuche_Formular.html?gtp=33498_list%253D'
-    link2:  '#searchResults'
-    iteration-var-list:  '[1,2,3,4,5]'
-    parent:  "//html//body//div//main//div//div[@class='row']//section[@class='l-search-result-list']//div[@class='l-search-result-list__item']"
-    child-name:  "//div[@class='c-search-result__text-wrapper']//span[@class='c-search-result__title']/text()"
-    child-link:  "/a[@class='c-search-result']/@href"
-    child-info:  "//"
-    child-period:  "/"
-    child-sponsor: "/"
-  entry:
-    info-1:
-      parent:  '//html//body//form//table'
-      #child-name:  '//html//body//form//table//tr[1]//td[2]//span'
-      #child-sum:  '//html//body//form//table//tr[2]//td[1]//span//img'
-      #child-deadline:  '//html/body/form/table/tr[2]/td[3]/span + label.1'
-
-foerderinfo.bund.de-bekanntmachungen:
-  domain: 'http://foerderinfo.bund.de'
-  entry-list:
-    link1:  'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/bekanntmachungen/Bekanntmachungen_Formular.html?gtp=407348_list%253D'
-    link2:  '#searchResults'
-    iteration-var-list:  '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]'
-    #parent:  "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']"
-    parent:  "//html//body//div//main//div[@class='row']//a[contains(@class, 'c-teaser--announcement')]"
-    child-name:  "//div[@class='c-teaser__text-wrapper']//span[@class='c-teaser__title']/text()"
-    child-link:  "/@href"
-    child-info:  "//div[@class='c-teaser__text-wrapper']//div[@class='c-teaser__text']/p/text()"
-    #child-period:  "//div[@class='c-teaser__text-wrapper']//small[@class='c-topline']/span[@class='c-topline__item']/text()"
-    child-period:  "//div[@class='c-teaser__text-wrapper']//small//span/time/text()"
-    child-sponsor: "//div[@class='c-teaser__text-wrapper']//small[@class='c-topline']//span[@class='c-topline__item']/span[@class='c-topline__category']/text()"
-  entry:
-    general:
-      uniform: 'FALSE'
-    unitrue:
-      parent:  '//html//body//form//table'
-      #child-name:  '//html//body//form//table//tr[1]//td[2]//span'
-      #child-sum:  '//html//body//form//table//tr[2]//td[1]//span//img'
-      #child-deadline:  '//html/body/form/table/tr[2]/td[3]/span + label.1'
-    unifalse:
-      wordlist:  "['Mobilität', 'Energie', 'Off-grid', 'regenerative Energien', 'Solar', 'Energienetze', 'Elektromobilität']"
-
-giz:
-  domain: 'https://ausschreibungen.giz.de'
-  entry-list:
-    link1:  'https://ausschreibungen.giz.de/Satellite/company/welcome.do?method=showTable&fromSearch=1&tableSortPROJECT_RESULT=2&tableSortAttributePROJECT_RESULT=publicationDate&selectedTablePagePROJECT_RESULT='
-    link2:  ''
-    iteration-var-list:  "[1,2,3,4,5,6,7]"
-    #parent:  "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']"
-    parent:  "//html//body//div//div//table[contains(@class, 'csx-new-table')]//tbody//tr"
-    child-name:  "//td[3]//text()"
-    child-link:  "//a/@href"
-    javascript-link: "/td[6]/a"
-    child-info:  "/td[4]/text()[1]"
-    child-period:  "//td[2]/abbr/text()"
-    #child-period:  "//div[@class='c-teaser__text-wrapper']//small//span/time/text()"
-    child-sponsor: "/tr/td[4]/text()"
-  entry:
-    general:
-      uniform: 'TRUE'
-    unitrue:
-      #parent:  '//html//body//form//table'
-      text:  '/html/body/div[2]/div[4]/div/div[5]/div/table/tbody/tr/td[5]/a/@href'
-      #child-sum:  '//html//body//form//table//tr[2]//td[1]//span//img'
-      #child-deadline:  '//html/body/form/table/tr[2]/td[3]/span + label.1'
-    unifalse:
-      wordlist:  "['Mobilität', 'Energie', 'Off-grid', 'regenerative Energien', 'Solar', 'Energienetze', 'Elektromobilität']"
-
-evergabe-online:
-  domain: 'https://www.evergabe-online.de/'
-  entry-list:
-    link1: 'https://www.evergabe-online.de/search.html?101-1.-searchPanel-results-searchResults-results-topToolbars-toolbars-1-span-navigator-navigation-'
-    link2: '-pageLink'
-    jsdomain: 'https://www.evergabe-online.de/search.html'
-    jslink1:  '/html/body/div[8]/main/div[4]/div/div/div[2]/table/thead/tr[1]/td/div[2]/div/span['
-    jslink2:  ']'
-    jsiteration-var-list: "[1,2, 3, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,6,7,8,9,10]"
-    iteration-var-list: "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102]"
-    parent:  "/html/body/div[8]/main/div[4]/div/div/div[2]/table/tbody//tr"
-    child-name:  "//td[1]/div/a/text()"
-    child-link:  "//td[1]/div/a/@href"
-    javascript-link: ""
-    child-info:  "//td[3]/div/text()"
-    child-period:  "//td[5]/text()"
-    child-sponsor: "//td[2]/div/text()"
+    link1: 'https://www.greenjobs.de/angebote/index.html?s=&loc=&countrycode=de&dist='
+    link2: '0&lng=&lat='
+    jsdomain: 'NONE'
+    #jslink1:  '/html/body/div[8]/main/div[4]/div/div/div[2]/table/thead/tr[1]/td/div[2]/div/span['
+    #jslink2:  ']'
+    #jsiteration-var-list: "[1,2, 3, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,6,7,8,9,10]"
+    iteration-var-list: "['1']"
+    parent:  "/html/body/div/div/div/div/main/div[2]/table/tbody//tr"
+    child-name:  "//td[1]/a/text()"
+    child-link:  "//td[1]/a/@href"
+    #javascript-link: ""
+    child-info:  "//td[5]/text()"
+    child-period:  "//td[6]/text()"
+    child-sponsor: "//td[3]/text()"
   entry:
     general:
       uniform: 'TRUE'

spiders/fdb_spider.py

@@ -407,10 +407,12 @@ class fdb_spider(object):
                             dictionary_entry_list[n]["name"] = name
                             dictionary_entry_list[n]["info"] = info
                             dictionary_entry_list[n]["period"] = period
-
+                            #print('linklink', link, fdb_domain)
                             if fdb_domain in link:
+                                print('oi')
                                 dictionary_entry_list[n]["link"] = link
                             if fdb_domain not in link and 'http:' in link:
+                                print('oiA')
                                 dictionary_entry_list[n]["link"] = link
                             if fdb_domain not in link and 'www.' in link:
                                 dictionary_entry_list[n]["link"] = link
@@ -418,20 +420,28 @@ class fdb_spider(object):
                                 dictionary_entry_list[n]["link"] = link
                             if 'javascript:' in link:
                                 dictionary_entry_list[n]["link"] = link
-                            if fdb_domain not in link and ('http' or 'https' or 'www.') not in link:
-                                if link[0] == '/':
-                                    if fdb_domain[-1] != '/':
-                                        dictionary_entry_list[n]["link"] = fdb_domain + link
-                                    if fdb_domain[-1] == '/':
-                                        dictionary_entry_list[n]["link"] = fdb_domain + link[1:]
-                                if link[0] == '.' and link[1] == '/':
-                                    if fdb_domain[-1] != '/':
-                                        dictionary_entry_list[n]["link"] = fdb_domain + link[1:]
-                                    if fdb_domain[-1] == '/':
-                                        dictionary_entry_list[n]["link"] = fdb_domain + link[2:]
-                                
-                                else:
-                                    dictionary_entry_list[n]["link"] = fdb_domain + '/' + link
+                            if fdb_domain not in link:
+                                if 'http' not in link:
+                                    if 'www' not in link:
+                                        #print('oiB')
+                                        if link[0] == '/':
+                                            if fdb_domain[-1] != '/':
+                                                dictionary_entry_list[n]["link"] = fdb_domain + link
+                                                #print('got into D', dictionary_entry_list[n]["link"])
+                                            if fdb_domain[-1] == '/':
+                                                dictionary_entry_list[n]["link"] = fdb_domain + link[1:]
+                                                #print('got into C', dictionary_entry_list[n]["link"])
+                                        if link[0] == '.' and link[1] == '/':
+                                            if fdb_domain[-1] != '/':
+                                                dictionary_entry_list[n]["link"] = fdb_domain + link[1:]
+                                                #print('got into B', dictionary_entry_list[n]["link"])
+                                            if fdb_domain[-1] == '/':
+                                                dictionary_entry_list[n]["link"] = fdb_domain + link[2:]
+                                                #print('got into A', dictionary_entry_list[n]["link"])
+                                        
+                                        if link[0] != '/':
+                                            dictionary_entry_list[n]["link"] = fdb_domain + '/' + link
+                                            #print('got into last else', dictionary_entry_list[n]["link"])
                                 
                             
 

spiders/output/foerderinfo.bund.de1entryList.txt

@@ -1 +0,0 @@
-{0: {'name': 'Newsletter', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/news/newsletter/newsletter.html'}, 1: {'name': 'Wettbewerbe, Preise', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/news/wettbewerbe-preise/wettbewerbe-preise.html'}, 2: {'name': 'Veranstaltungen', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/news/veranstaltungen/veranstaltungen.html'}, 3: {'name': 'Projektträger in der Forschungsförderung', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/projekttraeger/projekttraeger-in-der-forschungsfoerderung.html'}, 4: {'name': 'Leichte Sprache', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/services/leichtesprache/leichte-sprache.html'}, 5: {'name': 'Ausführliche Informationen', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/services/leichtesprache/ausfuehrliche-informationen.html'}, 6: {'name': 'Erklärung zur Barrierefreiheit', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/services/leichtesprache/erklaerung-zur-barrierefreiheit.html'}, 7: {'name': 'Darum geht es auf dieser Seite', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/services/leichtesprache/darum-geht-es-auf-dieser-seite.html'}, 8: {'name': 'FAQ', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/faq/faq.html'}, 9: {'name': 'Forschungs- und Innovationsförderung', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/forschungs-und-innovationsfoerderung/forschungs-und-innovationsfoerderung.html'}, 10: {'name': 'Glossar', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/glossar/glossar.html'}, 11: {'name': 'Bei uns sind Sie bestens beraten!', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/erstberatung/bei-uns-sind-sie-bestens-beraten_.html'}, 12: {'name': 'Unser Service', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/unser-service/unser-service.html'}, 13: {'name': 'Was wir tun', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/beratung/was-wir-tun/was-wir-tun.html'}, 14: {'name': '„Ich hab‘ da mal eine Idee“ – Die Förderberatung des Bundes im Gespräch', 'link': 'http://foerderinfo.bund.de/foerderinfo/de/_documents/ich-hab-da-mal-eine-idee.html'}}