From 89dcca20311843bf294f18090a192bd51e084e05 Mon Sep 17 00:00:00 2001
From: alpcentaur <luca@cannabinieri.de>
Date: Tue, 28 Nov 2023 15:27:39 +0000
Subject: [PATCH] added further handling for javascript links not being urls,
 made config for giz work

---
 spiders/__pycache__/fdb_spider.cpython-39.pyc | Bin 11670 -> 12084 bytes
 spiders/config.yaml                           |  12 ++++----
 spiders/fdb_spider.py                         |  29 +++++++++++++++++-
 3 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/spiders/__pycache__/fdb_spider.cpython-39.pyc b/spiders/__pycache__/fdb_spider.cpython-39.pyc
index 8d567f13fe5c379e4eefee13371337d0029daa96..32a32f554ddb2826ce3586cbc49954bb7296513e 100644
GIT binary patch
delta 910
zcmY*WOHUI~6uxJsmUd{NkLh#z7!U<&HJBI{gpJBVgOV7HJPc^%j&|BIQ|=6}<~9b|
zxU9KG7j9r<SV~5JfcgWBTOr1!YZG0#!CM#wW^(SF`+aj>-+6iO-CS2G7?coyBHl;Z
zxUti<9u5>d2vHbAEzfL@y$(;qNA@ec20xe?sfw%35Cb{u{E7GgM%h+WVL7?8fMZl%
zMlFGC+a4U>06>$-81^7rYzb@M0X02<8X3+d*kyVANpcj?2<@Pu6v8R)M_oV7eg8ow
zk&IH=MHLr?mPqP2bI4m5Xa9RSH>Q3(#_dTY&yJ(UB`QzgJTI(P1)gxbT`+-;N*OZs
zW0fN(&=xF?@sWlOMiSS4x+BH;NImRJI1{&nxR(kEWc%r<4M-q-np+=RiuJ%5=T&SM
zMmj5{<&rf|^twILDAshNs_K?{1gct1ThI)<s2ZlNE|HpQn`*s8EX`R=bPMpCt*1Wq
zdVG=)@P!}%X)y^t{^hSA@pFKi1YH7~OHZU^zS9u5tK6)l=x$V;4e*Ulio)z$mRBwU
zb#5tpaF=$s1nV0gCDq<sV+g0QQj0GnnqfdRyn&h#8rc-6v<1y5jaHJIsV&5rXeHh3
z=AvA1_q^wdBJ)g{4Pg7}Bs-r~Aj__2^N@4&>@egSBc@%}h>Fb>qh^+{Zah{?^*S+2
z^JUFqpZnwEL|a_aEZZW$n;(R)o!9Q%ym4EzEZsC3XaBMH%l1M|Jw~Zm#|P4_n50US
zcC5nQ<em=91W&|}hnyl@5;Wr_nt?S!`rALttdK89+T=`!by$t~_>Dc!7rfKF7i*P$
z$Un%6UhoM57uzdHTzE+cK~9Ww=_P3Nv0Z=K8B^c;y=5LsbZ37s1hC_T&Rqd-r489^
VsyGx#`ld(+zYfL-c3u<(_zRch>AC;_

delta 587
zcmYjNO=uHA7@co+yUEsMySv$+?k0`d>Y=GWM3Lg5r&#bHsOX`Tk`i%>v4zNPmO^BL
zN^XL%FwlcgO%7f>h_c=kEO^p`;6Xg-(X%&u@Z=Y}h%<aMk2mwZ$NOGy?5=A!ip6Py
z>)A`T`KI$qdtel(ObCY+l1Qxs(I~G<QtPq!W%OZR+%dPMs{>s;EzPCB%{+)vsUoV@
zk>jhRyCjoXN@)KRkbw;3ZY7qP&1G&4B_PZkSR%Atv12XVbCwA=d5-HIVJ_lQhG!A0
z+}c6sXpvhP>t!q;+h+dZd+!=6|3j6mr~Q9V_Z+v_96D6DDR$nUD7$2$c^*j=?OI%A
zRirZdkWC<5!7z=#VWj@YER2I;j*8Dl#fgio#;HqUl}`=9B`m<27khR9$I^H9H%Py@
z8ik+Yhxgvkt2z|Gmu4Z4H*PwO=K^Sw_~2jm3>>_UbpfmQoMvOEttUA4HE4>5<ujnA
zZ_D3dIiA_lv5b+)^)WDqxsNE2c#7-H8`6Qol~IY$i~$H&wrn6_5^jVtBiBjYPn*>x
z$VFvzsXHQBtczN0jn)k@u60Bus4G_h{h7EFw1KAk!B<teg@2<?x_3ec*iB!YY=Uxo
YLiNPKsYXG|UyO1SH9NYQo;^+B58-8wHUIzs

diff --git a/spiders/config.yaml b/spiders/config.yaml
index 12cdb83..3627f8a 100644
--- a/spiders/config.yaml
+++ b/spiders/config.yaml
@@ -55,13 +55,13 @@ giz:
     link2:  ''
     iteration-var-list:  '[1,2,3,4,5,6,7]'
     #parent:  "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']"
-    parent:  "//html//body//div//div//table[contains(@class, 'csx-new-table')]//tbody"
-    child-name:  "//tr//td[2]/text()"
-    child-link:  "/tr//td[5]/a/@href"
-    child-info:  "/tr//td[3]/text()"
-    child-period:  "/tr/td[1]/text()"
+    parent:  "//html//body//div//div//table[contains(@class, 'csx-new-table')]//tbody//tr"
+    child-name:  "//td[3]//text()"
+    child-link:  "//a/@href"
+    child-info:  "/td[4]/text()[1]"
+    child-period:  "//td[2]/abbr/text()"
     #child-period:  "//div[@class='c-teaser__text-wrapper']//small//span/time/text()"
-    child-sponsor: "//tr/td[4]/text()"
+    child-sponsor: "/tr/td[4]/text()"
   entry:
     general:
       uniform: 'FALSE'
diff --git a/spiders/fdb_spider.py b/spiders/fdb_spider.py
index 3b5978e..e97ffcd 100644
--- a/spiders/fdb_spider.py
+++ b/spiders/fdb_spider.py
@@ -296,6 +296,15 @@ class fdb_spider(object):
                                 + "]"
                                 + fdb_conf_entry_list_child_link
                             )[0]
+                            
+                            if 'javascript:' in link:
+                                #from selenium import webdriver
+                                print('link is javascript element, not url to parse')
+                                #url = 'https://example.com'
+                                #driver = webdriver.Chrome()
+                                #driver.get(url)
+                                #links = [link.get_attribute('href') for link in driver.find_elements_by_tag_name('a')]
+                            
                             #print('link', link)
                         
                         except Exception as e:
@@ -313,7 +322,8 @@ class fdb_spider(object):
                                 dictionary_entry_list[n]["link"] = link
                             if fdb_domain not in link and ('http:' in link or 'www.' in link or 'https:' in link):
                                 dictionary_entry_list[n]["link"] = link
-
+                            if 'javascript:' in link:
+                                dictionary_entry_list[n]["link"] = link
                             else:
                                 if link[-1] == '/':
                                     dictionary_entry_list[n]["link"] = fdb_domain + link
@@ -397,7 +407,24 @@ class fdb_spider(object):
                     
                     else:
                         file_name = "spiders/pages/" + fdb + str(i) + "/" + str(entry_id) + ".html"
+                        
+                        
+                        if not web_content:
+                            print('other downloading approaches did not work, trying requests')
+                            
+                            try:
+                                from requests_html import HTMLSession
+                                session = HTMLSession()
+
+                                r = session.get(entry_link)
 
+                                r.html.render()
+                                web_content = r.text
+                        
+                            except Exception as e:
+                                print('requests_html HTMLSession did not work')
+                        
+                        
                         os.makedirs(os.path.dirname(file_name), exist_ok=True)
                         f = open(file_name, "w+")
                         f.write(web_content)