From b4fd385c5d8600fcaeb59341f3b46a9863dad1d1 Mon Sep 17 00:00:00 2001 From: alpcentaur Date: Tue, 5 Dec 2023 18:23:57 +0100 Subject: [PATCH] did some changes to main.py for using sys.argv --- main.py | 6 +++--- .../__pycache__/fdb_spider.cpython-311.pyc | Bin 12079 -> 28138 bytes spiders/config.yaml | 2 +- spiders/fdb_spider.py | 5 ++++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index 12a01a3..2691b42 100644 --- a/main.py +++ b/main.py @@ -4,7 +4,7 @@ from spiders.fdb_spider import * import sys config = "spiders/config.yaml" -list_of_fdbs = sys.argv[2] +list_of_fdbs = eval(sys.argv[1]) #list_of_fdbs = ["foerderinfo.bund.de-bekanntmachungen"] @@ -14,9 +14,9 @@ spider = fdb_spider(config) spider.download_entry_list_pages_of_funding_databases(list_of_fdbs) -#spider.find_config_parameter(list_of_fdbs) +spider.find_config_parameter(list_of_fdbs) -spider.parse_entry_list_data2dictionary(list_of_fdbs) +#spider.parse_entry_list_data2dictionary(list_of_fdbs) spider.download_entry_data_htmls(list_of_fdbs) diff --git a/spiders/__pycache__/fdb_spider.cpython-311.pyc b/spiders/__pycache__/fdb_spider.cpython-311.pyc index 7e1e81975fd502bf422eae9ba2b04b659b58b4f3..f3f6ff0dbf01bbd5a04eb68e5c9d483e2d81e48d 100644 GIT binary patch literal 28138 zcmeHwdr%z7nP>I;!K;C8XrO^=@lx|>gph;)2?>FO5E6PJjf6*H8mbY^qZ?PZUafA; zMC`_{H$r=d_a@GGZuX27+8Ji74dYvHEZ!U6B6~cN?3vjrTt3=;$Au#96mhZF$QiHLlVo*Wem4iz1R1K=hQxkt`$x}J0o6-;J6|rv&gE|t&NS-PP z%NWdvhh+|CLYR6oYbtv%oA_xabEZs#CgRpk=1%1e<|!y8)k875KcpDFJO3ROMSTbU z#jZg!qc}}D48OwHeupBeaR*qp8yxDfz|b?$2c)-@6UXI9)fH0`?N_Jc>x(SZT* z>+*R6E|1sEj>J}*p?RS#jhp3@sJa1hE6-dT}uhft{xhg5PD69%(B;Fxcl{6g%bspL-th%Z4>SNEV zimTK$)p_bFwm0xG^Tyc2k6!U+a86&3>W79rUQb|XD3~t_NO)Qx zIsZK%Z&0gQxifXMem<*0$f{VQ)P|C|D{D$sR>^AFmbbn0XZf-nLfMWP^_v+p8BbU9 z3*YQmqZC;sDY9y^%#3{9_@;42^!F65H5J6erw$S&DHHMp^Y{!&$}g=8Ki@Q}e;1VE z>x#}Q)xRogP~JtyU*>dLG#})2nl$$^H1K}Uqydk678Say)SV99y?O;uiL@FZ86vIU zhmqkMAfesVjPkKKz?J@GxC-e(niLoQZP*YXzfv*w#&Ogn*PN)G z6Mi5|+CZ-8%^>A62tgqoQmKE~2yJhi6V*a04gCH4PV#k5#7okXVr2P*GrU*U$)+c) z`3-Y1%DB#u$a=>5+VxhiDgG*yDNQpz=4al_B>e-Pss53^=L8BJnsZipcCzp%~V==kX@hY4h%AU(frIvE7&oz`iZc2nJpzqw8n^E3+2>*UTetq-$ zpBdfW0DHY*(&G;Vrw7K|tef`uX%{`s`i3XnQ}k6&V2loox#`&N>}A#!@c6v+6&LGq zK?It7eiXi@Jzh~fC;pPt4aCRCdMF)l7Q46HUSW{XUww zXvRGX^DPDwi)HvS32jd%wEc@F^eoR4aL3Bn0A-BT%Q0YYz4sT+LUU%|P{ZzLx=E-? zZ$r~FhH;Ph7?1Z7sV&y+pZ0m7dc5@I!02w8@i4U47oe~D*ok@?+KLYNAZz%-O_LOw zAo%KYtNK?sN(C=mat9#OSWY+(rg6p#`Le#tm&WMJ?Bt|pczu@m1rY?+{ng8Ef4~o| zHO#u$>wbFlvUh|?1f(3?<(l;SXn7=AdNOeUF?g1pn2?eAcjL3{q*OVP$!E%|b&gyX zWj?Bce4>Vf*GAmam@SJ#dsOYd;+l+VW9&zDaR#C~@F4R_70e5xYM2_rj51brQPm{O zH}wftqv`;v0GXJW<|+$~9o39X`uuK34x5Lbnb<*we4|4!GxSGQK%=S&_w{J3`wU@d zRN;=A2;<}x2#EGNpY2g=EKX9OhCt5Z33!krY<|BxnsL=VJOq6p0MVmL_qC`Bf})C1 zKUM?IYeg4ZfIK=G-&HS8n}}^*NM}@${DRqfF29=3uU=DXEya;4$DOad zJ9K+!;WS^>Dpa+uRP9}^+PgT(SM>>1eY3_{6luh*{U|{BFnl9g8)5-2tKQ0B79__r{q;Y!|<1PMKe7POHr= zi^jjp{@ZM>=PUf~SB2fL^34~7<_ny48{Fv%H}QoFN0sebqbe=Xyg+6ZpX;g0+B+R9 z6|Kt^tz6rYPfl|uPdz;Q@map@GZLd(l&pmsudBXuCzAd-1Xh5_ov?T-Sw?FyO*8aOXqp# zalv_fE@PFh0tw<^$W`7lI>i{_EnpGjiM?JD4$SNX_xXdLcs^zYf5$L zh+?(cd1sPq?)u5d1N*(v4@ddtqulW!zIlkReod%;ZB7+&Hr*YY%a{XAv7#RN8&QZU z@X1wQi*$LpA!4< znhLvktfTw?3i|)QZs=62@2QQQTXpxcO`R3GdledV*IN!K)jv`i4^-=ZlxaF(*Zs(@ zL3gdCi&FoXGImw!erz;#mFs?7u0i+KE!`RF4>OG2jk*slrtVtZhqW4T|D?9&P@eXq zJmX=F;iJ-)9*h3IS#g*$+_xBec4yo#&*^E-xbM`UPqPMncB{bsfYLzx2X)TgV(r6X zV{eDyVWp|J)$p)Y18#}-iob_Md;MFOx5c$r)rJ2^FqPy?6LbSAHSe4gp(gNk{E}lW z8^@CCs%c`$a2x52IIJRb)}+1@*KgHgU5!zN)YAE4eEzP!sV4Jxc&6s>FJHQ_CZtKj zyIRIAXq;(c%Jp_#NS7w2T+`L3jj5I8Zb%zbrnQERX=CbTX=bF2DcANgLn?xN+zn@m zTPL)Ujw8|^Q@ZSUQk$65CnQb$WQ-vt(IN6vzVg(G_K<;8R*r1%NzekNa?~AG2a?(m zpn}O|9YOR&aGdEBVa;ZYV-{vF^Af3$AADS`1u)PtD(^kV~j z{|>A+NPf9yRJS4o!!Y-=*n3zTkl^rdhF~OAe}p$c#oZ9p01eVN1&cya?X-&pMeCVy zWXv^VoFovzup7R=2Iz^)4|oLtD3>Q0ddzjjO%J=>-r&K& z7(hF4Bb?)&N_KlK&SgC0Lw0p!e>RAWHv<3g9;h)MCES8iA8h|GC z49(Ie4Z{XW7lsWa3bTpRj5U}jv`pM!r1r)5jMHIatFduWM`tUMR3V9h6qC@vEG|w& zjj5)D^(@k)wdk*5UEWJ>wt={>O}hY>!6|A~AMgQ!0dm6D!>hwgFbNiyAK7LMPk~lR zFoDHIo~RyGd<4>BOOA2LCCgjrD*_6l#fhjQPgoB{5s#K_6h^9g2*rN0aIE>w!m;+F zbo%t;U8W67qsA57sKOh~#9W8sl8e}kcOvzMDJI{tW8Wv<{boo<1e_$DF$Bc|;3Gro z%X10K4!OXP5G3ryAS&mEF+7l}zZ;tN4JwjfzDAu<7z&;c`n9e#wKgknwaPJHdAIJn zjqf)uR`9hQLT$%N?SbXm1511O+7m+UiIv)c<=O%6jEAos7i!1(stKWLf-`M_8&(i) zfPrM?r3kDaM$FeDW(%mhxp|=O=0f0RB&H-n8mKI0QdZlWH@@-upS?ck=L_qE!n&^= zjuhFyWqZpuH^vt=3Pp`yI}+Ja`)>X1`uPyQWsk6B50_aUaqL-f>|b{5U$XL!BZA`y zmsuTY*uT=yz1+~f^eW$QN@zI6W!Ap1U*mz5#zV`Ehn8;gjc0|%vs`99ph*Rw{&y;} zp#G4~C~&RR^>zK5`WgM2J~OxAb1h|~-)YChw6j?1Pf4hyV-nE#|SMT|uf2rvQS3bDH zS0CgKpW~~~@g?VllJhg&tL7rW7m5xkK`Dj`w?Lvmpu#sHq{20j8CH0Qexri%v z@#bN{Jj|JgKQ~cj6@Sq@7kD>#JIHN6%GVqdYL4+0{X#`QUwT|9J-$XMEyDl>Ae~hp z^hHtSo+`t*30al`45z1a!DZ98_ zc9Ac8O(=Vfx4HzYiz7GTX%}y95zH-|x#dws&AV;4+c@W8uFu0eJ$%KuP%%Dp6v{ta z&zoxma}8&%f%>-;|9QpiBwt)76xZ?Qdcj=Jnd|@Y69Yce@wJ;bw+iM~&fNN_%JJ^t z?Llta5$@O&zio=I@(NYnnSLy!lQ%mAvx74`Vo7=V;s&9(fj2t^vy(GJu=DBTLI6S3 zC`1VYaU<5i&LhWU3~6J^u_Gg_(r<{%F*1EWoRK!> zdh{k;n)3Dbth8y$*1TcWHeH&s^>)A=(jgn#GVVrp$)W2#*{)e!nt0@It|b!(*SeV%OH~%&Lu1E8={xL!@q!jrtPzp1YLS|1%N|FDim4Zm%CZ#Yl zmXP@hC|Z(HtU&sOf_j0n7D8Fg8?=}?eOV>ghYS~DOcSW$QS6(cTqbV<)i;X$7ap)N z3YqqRJCuvARL@|n#5YgoTSR>GWxmD4*FZ|15i-j{OOl}qwve%5EWe&9B~LX|w%J$l z+RWNGO~7ADLT091#5FxZ#$47E0&d6%AT1T)1-@$u85o*`Zx-vw3#@C{*N+aHw2%mo zWwyNF?{Nhg&nl&>Ct*v-GM7Yju{9HyWPFtUe#zx9WEoFd?0_AP`c^0W`ws-asuJN> z&;Hx#=i!g{6yZNYj1LIM1i@=Snl z^#r~HYZWX!$v0??L00P%jj9Rcq9(wJHkNr*jj@B8u|QzDEvUYF^{O+d#TS3uul^rg z&lHWpKNYv-dgO{e;q&`F!;{zP>%PkXBeDQ~66g>(VlR8?q=5(lTZlDGdef4vj^{Pc zvz&y=7i0p|O2P5J6*) zBEbb1b{mo=AdUhVbjJ2$5MpI)8?Co-FsNAji}EEFJKIw%1?vzki^EYwST>3LBh2#O zXp_j)fYM-nLLvQ~Aya^rSO%3}9GpS`=!h~4*hDG?fRz830MHrLiz)d88Il4(0RGlwH~$Gf)Mw)ruz(-7<}Ge^)wrvCl=J3}UTHeK+;sSXm2WyFG@SxGBC_-F%Fd(9JC8nimESoi>>T9G+h5qP<;Y4) z|8h(JgPVNIMWN*)XWsF-@%f{V1h(W&g!YkI1pQ}M*0wgXjZE6YBkfB-02PJfoAt+dZ} z&jMIyqG$(TUuXuDAYf|4ZP~Nv;%)l{+kVcrKT_eq$cevLjYDjN-1J{QUrT1*c1W-t z;%tYW!&yC;zaa?q5;fLEY7D4KTa7hVovg8H`|P3FL#Y}A8zN;Cp1-Fs(G zGW;BG>v&Y|m>=NEo8XRg^>Fmg1(!hYUM%40{gHCVf|)Dd3HM^l60^8(ao>Yc?%b;n zybrwGYh&ExH1`_b>%Xf{RDwJRQU&03UU_Hty#MyTxqXqIlia{LzULfQ*}7;KD)%g& z%_t(K+;8aqO!wz1s7~Ix zU9fKFP0fO-nKLy*(3`p$-Pd(b&Zu!%U$fwE&>HzA^8Xrsq|yPNKgwY z!t0DD>vVjs>14HzuhxKD0?pz)1(n5LUjf6LBs9C@_Yj&@UHBV-W<{G2nFP!!GEb7j zwM2E>5Ksk}D``pv^_#rj0at1hPgp0}Y$;Rc!T?+XaHhI276xE4WDqIB+LSxCWhk5x zFeNjg#*+~9#@mEK8H|q6GlsX7ca1W|Gc!#(qMAwNBj2$Yd~?ceAAY&WTzl#TTHAm!WJq zAVi>s?3>vH42NgRgfcw{0q|tji^yJ@nv)rBg>%RTGMLIIF4)}Y!500Ea*>%{$&B7M zfMm%AGAS2DvKCoqD)&WdkI4%e0pR-!Fbs+rI*R0L!`@E-Cb0=)NCxYZF~Wj0Y!YwB z5^bAa2k!^U6SWN{k;$GDzzq=bYaER`$Tf#MMb=$%lR2i~ZlSd2fGMG<_k~Q1RT>`F z!Wbq3<5O9{hBh>4AlEv`NjsigMgYi3dF!>`O-+V!{2_6FWQydR7$~NgDVZ#oDh%i1 zmVGeL1CB*wZ(b;G<0YJsygl+m`Qz4jfvLIF(3|jrr!;Lafy*B+P6R_A!tIZ8>6IFy z+2s7E@8SO_(oy=t(jgl`pf02`p}I8=W^_LQC~_BpBFjWjBs&C^#!euaM1lZDFyojn z2*zfv#+}Y>!JMhQ?Y$~tOMV)vC1U2Wvml@lsNf>6YV@9|N64iqF7&JZG2rui**5ue5!eb;R zJB8os2_QJ+VN#pyz?4h{C@F4*`zqUx(f1<3b&se4O(%&t8p%JZgDq}Yn=Fp4&$0U| zN}`*wJaLa4se6C0QUtc8jddQdWd9nJIA#9@d!xXlgZ?Ltvh~5fo+lL283mg4q*d{F z^HITk6coHd>o>0a*_E$fd-K|i`tjEK`GNP0ca5B>>Jhzjjp|b5LUKqE(b~0eO|b7@ zDiG|4XO67tvn)3hkIJ^rs#eP?-}bJQ?N~0`u>kwTE^?kAci;xzWjpw?kWdzaI2De> zb}}%RtLP9aI%f5=`q)M`J){Ji!%|eKE4rart!cP(bERhQa?Rew@s*C_%N@tL6R&Wu z`MHkcd`Cd&2&{D6Snhy*iBwq8jb>E)4x$+qP;hTOgl?d-hjB-jlCob^Mrb=&dzAv( z0t7ITXz#YMukbED{;XNG$<{+trZw8)sbrFyHmHPzU{l?TdCf&T)k)U6);&7 zst?X)%)%xz+%>7UREr4IPxbmecSU?{pHK@s?&1aRd47Q*QAJUVU}?8}v-j=ZSHzd}Z$^UEGOt z508C(jITV;y$U`fvxe1*x;uLojC@7AP|-fCU$qv$o$(h3?)1ES{PyvMUcRzJsO)%P z=B)j^wO_FIgJac3&lSyg^R_0z)|A@arWfdKytPTNHgVRbKgU| zo=EHiEct@fw8198Cp}8ixpQ%>ShL{3Mr_q+KhEOPK{r1x?bLKvXg{nl_Rxk8ot=e; zwEB-Uik=d~M_OY~e#S?p)}HK)`vwj8+|Sm4&;5KAx=S?h!Trjb!gPquby`kmTOoyOz2hEI3r z9M3fTeWnK7637J=3Ie%Mur?cqTyz(%BjoY|C)CJx>cP25a0*W1Via(K4J<6CPHTe6 zmnzLkGV+wv;!`owp1mSFe?~fK4Qt4XIzUN@i=2d`mGGzzY2`eUsjl{>mZ-V#OsTmc z4FDk83;!K}5^*^-G3AWWqFN>8L8{g;x((Dg8O%ZyH2wON5Po?PMH ziOwb3xGm&+Il*QuW8;|X&UeX78&j^tz&Rvo(v*QEXfp=2uDS4^K`j^8dM1D#qCI0w zR!S2hb+}<`0KBqZk>02Uhy{^Ek(MbEBoA^n3EGiu3@D8o6OuQF6R>rcTrCC*v35|+ zq!Q~4YsFR;+?SHDG%X7#wyK^DN1nyP|m-Uv{7;TxSakQ}q}s4s1!iNk=@kFk}rCZw%Z>frKFZT7<2)8gK2GQF;I8I!P@JYZ)7GDDI`D z>7@|T_yl&eNk`Z;GQpI|>bWGH^f2YJm^M;tsQ1M$i42uUV@uzl9v$IQY5Y)Wsy(H# zlk%4h^;sg07qk9~*IsQP9WOG_MVdNkTPZI#aURmxNnWfWNvu%G6$Oj@Nqi;z?Fd=t zl7L#G&hW|5(mcXtlA4jV5Mz(^#u){>S0WKxLOD#uW?#MoI^=gyb0!zqT>h@c?Bqxa z7KtXNlBv2|E$wq~x@38%{DppX>5Wf*^k!^)?xWYeuJ@dSabC_*&6mfKq>Z+w@3re^ zMS;3RTR}a_Wkfm}I5ul{rdAUAf>J1zNxAGuQj|+*-2`DAB=k=trLW)YE9uw?^E>fq z*YMA>4fe*rH?j@$KGBJGA*)mtnY#FVWS>MvzVtreEv}$sa_S6OnMUbY2jeXKcAuxb zs@Ms&@EakWUvI;Clcc<|{v?avgz;xIxBiQGRC_rMh}kafZA1q`+rf30WTr0DHKeTq zI;eg;c`hR?3Tbcsj~U&q8wC7s?hk5w9@^(gZKo$|0KqN(02txIM(~7fDvj_cZWkwD zDLR5G+UXQ6790R3;lO1y=#1)MH2_Z3n+!UhlMXpxj+o)bj3;pl$0F{|UP1C25IE7z z>lyX2Q&IK6sWS)3YkEB0R0lP1=wHKbvc!N(9I#eE79@f$ z0@CBs0XY>AP6tdZl*CSGWL`Nqr4dGb^I!JRABuk7Slmnyrl^9$P?N_gAMjo zm%oh;?i{`xAPXvRXtpFPtfz6`FpUc>UiVc}n%IX>UB3blgbEgUeKbjxn5#0ey1hCA zXAuO;i9ei82UW3sqlzjL=XVKAv&sH^uo`X)miXhR8HyPPDMgEIvXBtW!PFbDEDeOE z*pi7=v_C%=1*95D6%t}DoLVwDjD9#D6ImG_h1cj7*^YG(Jru;aC2s5#f(HB!*BYZn z4;=CU;_n@Cvxrbc^>B*e5b{RMm7^N}^dx+bKc^yFWxLE)Iktx%_wB=p*yaQSuv47~ z6%lutHM4X@C1t=I%}#tB`!-r2eIJTfDEkM<(H|orhSBn9@lvxpuuL{t8i^_)l>x^o z;wc%?!u642T~>YsK%y@XVdG)hq8XxAj%J|o`Ou^bjMW2hl6&k73pkR4RBBWYoibLv zXp0@q5q(LLQsY`tG!s|X5=Sihak!C~V{h~lGtn*)TV3N|D7Mb7#u9E|oh^&^JwOWl zHvlAwFBwe}!)&CuWMs(7irz8-Wmd>E==+@T|!M4XR3sA2Drr00$kTCeC;`*_8ebzUZ^_1*yC=ZKN}b z$0iF`(6)G*FWA4dlP@^Tn|cLPFK6m~Y__H|BYgfE6KvLR|JJ^@_Q7`ll1AA7KckP7 zR(teM~+zF=4MF$j*ip#-*xrlzEQsd5lAYf7nB$I-J z^_qEl*AMd-PyV3b127NWw^YH?2R~6gIQ%f{<1Daq<<1T9^w3N%G^nNcG1z@y2XpV0 z^48_@)q62!%q zY|84K_Xvf%7Y{8L9^eWOM4I-kG<7XEbuIPsP5nX>teZF=RkW^AIYn@EB9PgxNc+i^ z_A|@vXSlQH`SwAfeUP*5UNcf)ynN*LkyZKtoUl*@%VR)h4}P9W(G_z$zZnF!JKwrF zdo$8=6wYJ(Or<=m*n&2+V*=EVil@7uJ}JRG>Z#^V#me^XaQW4+WCp7oLN}jzRR#cGA^+lTDY<^`G zdLW(Gt=8>axb(x<`MTpm9h||SaP%u8^(~7nEA4&D?R|XvF`@k!Uw>@A@9CNbzj&g; zgr0&OG9EFhgbc+r;@m;br{t>7af4ss=?emVfs@|gNtSLj_-(AIs%vA`;SEs1XzdMT z_7F(z)_OcGB0@KU|5o(hS~hza1hK9GR`IH8|FU`h%6sj1+xe;%p{fN}EW5CrK<5;X zE2`(N%)9Pf|8~cnj)l(Uik)1=&PZkB!mdU8_gfeJ-`mGm9+>NU`j{kh?Y+Hs_r?xp z#RLwLWDdrYxwfFVbAzvF7Al$-T+0=vBev&UDjM2cN5B54e$S$xuRrjU(+_g*o&E4EU*Gp&gs(sOi-A|TbFc9OE@8mM*Sk1o zg0G*LGpwm~_EXALN5i~lvEqS_bDZEECj`d{XxOSQ1=v|fTH62S`oFvWy&K=ZF`o%* zm*5)iU5*>V)znu83Ei!WD6#;PqTe1Q>R7oKk`dMuNs&@SEi95}aLe>F0aEomHO4)CZMr<^RFQgz|sI9^g5t^dT36%4uW+AMkco`Bi+w2fW!? zd;tp4fj$-qlWV!Wg)eUbn+AI|SKaxO=B3y9u2%&3L8Q+K^f`_`N46sLJg{=7M}IK} zOY=AQv5+tp;_0wJhdDa@&vKUR3BScY@Vk^VI9J-{2^-_F;x2sJH>_T`#(uBLr; zYXj#z`pF1)%Ek8&3;n|&E^fuED1aB0E)%P@vC%BWZ@yA@kF*ued7QpT9g#(P>{ycjVtDKkIc+sQ5$=O;7&N0_0 zr2Yb!g#K}-z0;w*=P(|~(%fq{9nfojq}PD^qtePArS`tk*wbdXpIOY=4cew)HG;voN2epQXGjC8~_$6gIZ_oIpB@^9t_4%rtUskEm zy;Xs3M?Sct3Jt`M7Io%*rC$4=>MgIT4E+uR{(nFSM+^-`jYC6IKISsmmkmWThlaj- z*)h6!!p1KMn3r)#c?Oa|2q6gxNj7O|k`mW2CNUVBj!i?^VO{M* zZ6$Ws>5W$HlYPf%6Te_*w*isq$(7#j!-t%`*Qy;`|nOT4s40_yUc!*qnM*zY|b z84Gy*q4A+WIOd;Vz2j_TBrqED_`So-peH&WWCAStM$|!TZkC8G^!5c9_-%@#_fZLY zAT=%GsFC%NgtjlI#hjR4p-rHqjNB=SE;Ow(mS)1N9kNac)crQvR?XWma0`lL+#CznhY(= zH-JVy*`cj-W-98gk6jzP8hJmmRMEaz(SEBxRneWU=$_4gsGw{v^0jW4VOIX1l~XiW z$cn+FU_9z0m4$=EDKTnp$kz8=JaB|uY`rfQ4J66;+ukDImutvpdX0>xIN5w2xhpFo zpByeQ?xO~b(~_7HX1#iGl97DU;W3M5Ok|8E7?5OGvG$jFym}o4TY;L26n~ zuD0$VXDh6vz@Q~>RO} zox+V{EKX}+NUL0hFF002+ICx1#~MJynhEE5BW}T>6@a%yf=~M$J$UrsRIS&0 zJ~kHedIuxSD_-w#gdOvGWB$QV03Iyd%!Vp2VDtq0hX8Mo_OYX33xHfi(~$CymWHhA3M(iMNzyW7Kz5#VE7!1+sjK?fB0O0 z7l#62!Ts3^Xg$ul0jmmG4LsONgem~O5Ka1#F?=ukSzjm!mTFPD9ipH zAUWRsj%6RMhALu${#I*9_%A9hDH4jTsGVGOHpd^TD1#+wX-esv)B5J5zWJWb_1>u~ zr!JqFJCmhEhH?RB>1DHR);|}xteR8J!Vk4hUpbwud@g0{Puu!YYo`E$>N|nC@Qvds zyEkq3F4>zF?M*3rOWNL&GPkD9tx4g3ueL6!CqvIf#mlx*5_UP_7Q^#&mO`ARGmf(N zrmjq7N}4iu*NT|7l`h*|^RCOobHgiAk*ypk)KR)37u!m+3d*r-MJciES`o{fmIoBT ztR$JDs0h5US~thK1)+CGGbdRX0v9xIXGnIvU`g>FW6l{io)nvvI+ncDCFy zrkow1vY=n^|G|`WRJSA@Et#$($zvyz&pwwv)}QL?-=zB}+lZU<=wS=_viV*8K@q;y ztN~wm?zKnj$%PhM+d)~cS@M-RueV+PRjIbOS^ibC6v{*EArVdt#xOkS_Zh(Ti2R`t z8R&sJ^4iX~L<9GTtIbY+-DV;$v>8dFLo3eb^2zaC<>Yaj0r_7ttzs19Z+qOdl6=0q zye1EcXIdS5!jaUIfEzP|aP=C|kk?IiQfbsmnEa1aTWz{H z2B;C;ngs~oe}~Y$9qx|>+S07i7HF_d2zwAZ5Cm7kNwG+0k7@5?X&th{Nl17Ufl2T@ zUu2XO=%nz+=)^_kJiiRhY(sD&U=U+V5z2{ppDT{UkmCi-7G7X`QM673f@=dC1SVtb z5s<({1Re-7eBK%(gh$zW?6d>r%Hd!r;6n#vC~>y4LseHBcW5eRC1;B* z?M%^XQlsMvnW86-xn)>ACM7dp#4wDmUW~AEt6bQSt;qzsrUqS8OvTjX1(OBSwHm=i z3)T!kUO~@1gxrGNoL7LHZ6ey!o|-2f+5(O6?SURJB@2d!Bc6J;722lcx!jJ&T>t zq1ir!qX4`HzJ`25k??RX*7N$!bssLcp0_@&)tAeW_!1EA*kB{q$0zI#EZK)+`4J8y z3f^ogO{vwLnCGk0LTrp8q61JNY{(MIhJuZF1cG5 z-L1DOQ|_*`yDO>PmZ{sjRJVVzZvUOhRNb+3-La&$Cgbf|^7brxdop_4ideM0@Bsyo z)Yd|5b~R-wgU%uVdGw+$zHD{OwWh7!C2QlNwQ(VsvUa4c9f^JS?4|FGTp7ujpUGI7 zAmSG{uZYDC;{ysH(ep45@|4+a^YV3qSTDO2XXkATPw#mVC88z)l5M7Cc?J`Q?K{WPpJYk*~Eaqa97Y)W61f~ZEz z8*t~?283r31`$R9eox&8q=fxsrC371Tr(OVBct~Cb}4*%tph_JrKMng0}hHxrNCk9 zaDZ+?jUNI#zGfaiE>B|_KgD>-NHh}WH4|(o6dbH$120VkAiH#;NHxKd@=!6|N%n-Q z>UmjYa3nAk%S|(a1_X;3Cmt)>$0E!`D6pHIgen9HD*6#w4!zWsm7NpQFrTMm!nh(x K9{h-07~#JpUP7<{ diff --git a/spiders/config.yaml b/spiders/config.yaml index 3627f8a..054bc22 100644 --- a/spiders/config.yaml +++ b/spiders/config.yaml @@ -53,7 +53,7 @@ giz: entry-list: link1: 'https://ausschreibungen.giz.de/Satellite/company/welcome.do?method=showTable&fromSearch=1&tableSortPROJECT_RESULT=2&tableSortAttributePROJECT_RESULT=publicationDate&selectedTablePagePROJECT_RESULT=' link2: '' - iteration-var-list: '[1,2,3,4,5,6,7]' + iteration-var-list: "[1,2,3,4,5,6,7]" #parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']" parent: "//html//body//div//div//table[contains(@class, 'csx-new-table')]//tbody//tr" child-name: "//td[3]//text()" diff --git a/spiders/fdb_spider.py b/spiders/fdb_spider.py index e97ffcd..08dac4c 100644 --- a/spiders/fdb_spider.py +++ b/spiders/fdb_spider.py @@ -434,7 +434,10 @@ class fdb_spider(object): for fdb in list_of_fdbs: try: - iteration_var_list = eval(self.config.get(fdb).get("entry-list").get("iteration-var-list")) + fdb_config = self.config.get(fdb) + print('oi oi',fdb_config) + fdb_config_entrylist = fdb_config.get("entry-list") + iteration_var_list = eval(fdb_config_entrylist.get("iteration-var-list")) except Exception as e: print( "There is a problem with the configuration variable entryList iteration var list in the config.yaml - the original error message is:",