From a0dd469f256df61efb62eb423edb3e9201116428 Mon Sep 17 00:00:00 2001 From: alpcentaur Date: Fri, 9 Feb 2024 18:38:49 +0000 Subject: [PATCH] added new database ted.europe.eu, created new case of slow downloading, intergrated scrolling into entrylistpagesdownload --- main.py | 4 +- .../__pycache__/fdb_spider.cpython-311.pyc | Bin 41199 -> 42669 bytes spiders/config.yaml | 37 ++++++- spiders/fdb_spider.py | 102 ++++++++++++------ 4 files changed, 108 insertions(+), 35 deletions(-) diff --git a/main.py b/main.py index 17ff0f8..f892b52 100644 --- a/main.py +++ b/main.py @@ -5,8 +5,8 @@ import sys config = "spiders/config.yaml" #list_of_fdbs = eval(sys.argv[1]) -list_of_fdbs = ["giz","evergabe-online","foerderinfo.bund.de-bekanntmachungen"] -#list_of_fdbs = ["giz"] +#list_of_fdbs = ["giz","evergabe-online","foerderinfo.bund.de-bekanntmachungen"] +list_of_fdbs = ["ted.europa.eu"] # doing the crawling of government websites diff --git a/spiders/__pycache__/fdb_spider.cpython-311.pyc b/spiders/__pycache__/fdb_spider.cpython-311.pyc index fd4f9608bb1edb1c8df9e1426da29624f79a406d..549e0a4feaf3db6f704eafeba3198878622d9021 100644 GIT binary patch delta 6555 zcmcIodr({FmA_X)?+8gCp$8yc5C{vz+kkES0P_kMJAMPTF$i5@B!q;ogo*K$v6HNw zI-Umi>n4pSiIq0%q>Xpm41XluO}6%A+dOvNy&`JBWSBVVG`rpIWaUn0)|>5PzjK9* zB$DyW&Qx>tyXSkId(Qd3bI$jD|8-OFx3>h@uPBvr0{{MP^5^cZe|SB6QDEz=9t%vk ztDU|{m)jq5PBGP!Ou*|N4^=v=N9mEu;G~CkGxNU{93g8ye~2KSC0MeZm=h2B!UC4? z0yqB0*A0h-tdJ#7;&!HAMxbMib^f`mFQjBXx@py+gIhEL)`%jaJoLOW7rmXWK&Lk- zQEyXW9v)5(J{eYqnv)C(!fQrfDz}jqe(dh-NqbsbvLEfZMiRtp_@DTZL`Z_lS(BI~ zSYw2O!MZBebWMOe;eSe}x56eh?w!UJ{d;yHS%e2^Q4N#5zWGaKnWiSBPx2w~H@wBD z$eM=?r?dG_dadY+>IHHHZBiGJwP;dgDxM=RJw_8@(HYS+IZZq&93ZC2Iq{_t{P)wu z0m5E4|BU*7v+8~WO3WyNE+*jfdHlz#s%Fei=J?bEo|)j0n!&0>OSeB1IP7sxZ?HF` z%ewPq2{Pox)hf@#q|f8>gnXxlrk$P;lIX43+3rW(uBniF2#@qk&Tr6vNS4n@3F0gf z7ZX}4PB;a!PQl&8>&dnyeg1b&$BAKq5-Nx1OO3S{0cnWI+cyu0#J5HA zfpXbxm3p95cDqzGP(FXMcvP-#!)9Yt0IdK!(SJJ16kR~}0rVhqrJx2lKb8j5)<8VZb!NqX4G>W&loO z_>w%rw9HFeKPP81C4nv)02Uw%APjH@AVoqb?f}3ZMlibp_F%B*Wys5%g(erkIKVeC zEbC~G3p8*tXrfKI^Bgoh2Jkq*699h&a30_Sz>@$M0sb1`8GvU2t^iD;x7rxeI$z!1 zA`=wn&Y#$Qjuc8Ga^4SrDNz+;&w>~r9eg7!OZ5d>#mc}cv>It1(d&(vT>arlp-j@K zc1IqP?Nh^BO>5GtSjLvL+S zplcyDs@yA)NC)%64~6vTNBd0%?BYdhOlVy~#*h^^ngw(o`gB7sitNoIb5^}xCCm;P z(Le9C$TCINp}HM9bYY)Hs9|~WwVC3FbJ!eSg_Be2tZ?pXVP&62q?Hkg(pK7(u5wuC zw_Ax&x}pVP1)U$(r72r*Us+vNlG`S_FoRp3zib`1OtYESI0;r|e4`l)tWOjun#y2m zVDspraXx%9c2){1X0dwSh8EI*XIrcGlUODdp9ZX9Bes3zfr439#TwU^0bRUChfP?& z{8asLj3(0qIc;T4CyAhh)zSImLYTujk{L`LLzl3*tdZ4S!>;ujxEKG!>r)!Z#U#?r zXiRh|n@ihZU52cQc0krhm!a32^yCI~^MJ9kfF^0erwtbh2)59xOZKD7GG$^5B6-R5 zP!1pR-cJT}S&DjQ++s+wi5QX7rrT{^$4u%qCHsUetR=%-`mG?S)SG|5Pd;r=UA|Jf zJfRmhwh8^~K~X~ml*3eLb8-kyqlHO2v?)nJa!t|^3kZk)a7eX@c3iJqmnc)5ho$JM z^!BilRpKOz8Q>)F(xKe?`@J)4L#EiKXuWM_i!yD)Pd+&(VD`|}co_iI%`XIg72?v) zw1rTUIUcXr+SL_%Tt=kzty{kx5F{2gx_T%_oLB;=y5F#gS0csybEN-^b>5Yq5tKG2 zysJWSMuGeP0%v|qtz~}fCZ=$4s6AdtUUntq$vl5_?n;pf(g{Z5A1Tact}QP7hu|*-1JxisI?`i^YPS>C0d(76?nc zc79X*qeCKHN|zSU-?8uxQryQ^Ojx1 z#&|7h5F)&R-N5JMtB2I;u${Gg9m%#xWipg+#GY>VRwWzo%6NX=%6K}%VLHKKD*s>B zo&Cy43)|AwpSLDS(Os;Gewd{oupbwuD<3YW9bucl1dDns*&i>4+9Zv4OgVV_b3a4j ztywqE0o4uGZMM;Es}>&m%dL?Ihh1YeVY&K=W{R+N3xgg1`r z0#@lw{ z;`7)z1DO;(Gt{$vMjiA8rYq^dwBHwS()g@4(-9hTQ<=5Yq?2(@xbYc-@&qY=AVdY- zAqTpdn~&}c$z{u;&eq-o9n2MUcc@T@K!g^6?*P~Ut|IGj_3U!i!TpCiW~dNz3V(_U z1*qe0%nFO(J4NBcA#;iX`KaGOV4P{kjUm1tmS`+A;iE>}V;(m2yUdB7fN7PeBhu@f!gB00RI=0R}NF%YxJRJo~4SeDWAFxr%D; zpN|;GR!j%CC^aug&EI9B(&Cu3 zn3ERY$+g6ZS%J)OmzYK08huwXB~W5|3LQOuoymJPaUR(&nFmcDRW6{yJ;Ap$i6?OE+9KD>{zJ$VK+P2)LH=7o6cXGKq?`TcW z_I$gCvut|1{>>9VX?&|OYH8)#2cwq3sPyJMutqC7V-=kdDKEs18-^&=6Qg=K{<*gT^2+e%Qjza@ob!02)Od^<=cw_x zKvL8uSgCHferB<{W1+g^hIg^EZ=tgf-xzUYr@79)XeS%%WdEBWy2y5H@*4h)C>zd;1_ zc3qD)>jSO4XOrXuv%05V@Dc|fJOEf?XM+bR*RxvdofT_D24ZkLvQ zLn&TX%8wGVWkb^uwe({Zd8AqPv08qlUh%O>b)-u1v0Vf;RU)XV7XsZZ!u>y~Zxs!1 zZW7NfMi+v9PXqi7z#_o^ zpsE+Q>1crn0#Vv)4?6o|S?)oh2zx*Z`o-wRi+PB9QBwIQU?9U_*C!03#Po8OHyH3Q zXHPNsPJN_`S=}bIqpYhG$)eJ$)#NU8=xS#zzw`Jxj1UJ#ylz*BXT|&xnDU(SF!iN$ l>qLN_^0~J&GW0K3KWiPxCb2wUjdOW-`Ig^*O*_x_{{ZMuDM0`L delta 5332 zcmcJTc~Db#9>Cx4h1>)ZAg3XOaO99nJkVC;^uW6yDq4!ji$H_~UjnFoA-2=nb{7@= zbw{V$Bd*7`)gCsxGrBw7>a;s;ciV1)SeVQ*YIkO5_Lv$vwqv{h?C<*^NYEaC>^{in zeShcset*aJ_kHh;PwCrN>6Bk96f%ljyIwx)y#3C_lrdVAU(9+uzK~q76P2t@rl=DX zhia)IQM1WUbCe5e^B13&<`-}R4)u|8Rwkuz=Kh?}<)kkqIHItjGOU`f#b=ggN=RMQ zyveV?Pwdj*^(l*y317`q;}a=8_*;b>AK4`%xmoE2=MY7mC%>r+QJeuEO3~ok{VDhx zMGDlT89O$mnv_0mycR`qdY2hb_GxhHv&tlr$i`chFCja=qs&G{ct~il457g~hVlys zg#D_Gt(F(|H3N14165&_Pd19R&GIV&8>rVrM8uc#>!;djnZ+X+^&_5YlYj5q0e&m z$duK@Vm1{}38=!gMb^cuKx_om0V)6jf~dsj@%q?KcRRa=B<&e&8WcSWXvV=J9omI2 z7k!F4@H53Pq4z_1CHgdr7fOAAUO*q9AF!X`JGAp3D735k?x^#gVRIsk_WqH3nI&DZI1x3aneEl#(OJI7Lu}_dS;*Nbza$hWae|{iB71PC28v636P3Na3ma zJYhPQHk~hPP>M68)J%F&gVL)02?iPN8I&~Bc<^!ZOwCO4ljS6-m}pK5 z!=B8glDQE!pZ4p>0H_8GBH|cQIUV*?Xe53;WAYnFK58_UGFhCSxFWw5FfXVkgKV)uXD&^=F_WW@iiA8Z3GA84 zpcu_^&_Loc#QX6Zn7rB3EY6?O7ls4BmoLn+LfW6@D~q?FDA&wF^DV*I+cM~&2^Q(f ztVA!%*e=-5(PU`IZ|X{1UtAXBNG$i0)y4bey2i9LPT?}&YY4-DZ*5a6%yZ4gXPcGv zh~lRRktvEY^T}j^bUk6R?mGj>lGFRMec*GZFP9}Q1#`e6dp1+Z6fwmw(ickRXlhAp zhnt$rc<)mwY5tsqhc4i9@U13w*35|UTl4)nTeB3mw5SycijWSo1o~hetRwY8X3>Sk zi3$Ysp#&|&x9rBXa~`^clP6Y2lza1i{-MU7J6lyw&;97*R0|Vln9IeRcBODbwkflj%ro7+5;D;cR`GWe>12|Gtj(bF9ow z>=RK<)Rg9N3+|oc_`ps>8o!t*_Vf?$5_No+Y>F=+vP%@W`7xb=%+3F0>pYY=#qq6^ z#O1hbe9ThZDpZ@}>X7~MgWEiB+5qNZe;#LGOndn~%YKzG;>+7Q{W8%~crBB#Yt2`{wKzY!7^Shsi+B z#UCHMz)$fz&G9Fai7AfDOa-TBD!D9haqWV{mEpHDHh-QwmuUBBygq4L5|_ydn@0Q@ zIn((#7qXcuT(PSxCy#k}diYv?g7*JguW3v4nnS|Dv)_ND>(yZR_c>vjoRt@!Hjn^A zX`IumcyRNOcyN*raI5j{*6PfIGH*{Oop27ubU!o&qEW zg9lSxt$nTDHny|JSJ>6*-izPay;j8BM#FZ-{C33pkJY>KihGy5| z9qq>S5Qq>)L`AIAP0HAH__=m_yJJ=m%iG4A#arZtCUB&)eriuZI@ia#OmUEXU_{6^=$l5eJG#i&+Vs+*)*@g>)X;%&5o z$lLJYy%&%GFYL-zB2_1fQTK>lUHM4@lE&Y36_zM9CuGNE;dzUG>%Fw#V&8B2BJ;|_ zl}(X(O%dgeQRR-Xa>q^c;uv|gRGB9U@t*Ewf{1oO=*{jYQD8!sd9pjAD;U)kgmne6 zq*Q$X-Lw>(teLRb&J>MVmX27KUP=!)?v7EE8&%Qdwi;E(Z(+;QNs7v;q3`e{t*nJ- z2F9%ABi8auZ4qnrsI~g!nh9fe*z)kD?GfXuQRAvG|9^KP%NC=g0rEIWX8mx?!ZcA-rX0nCTB!H$^l8LlMMgaQABom0L&# zoIH0nNyDJ)ZO2$?%}8m@mDS-b?cvgzNNLAtX~$TpXQb2MjUmcj=GcU zPOh68If>p8qh>(H$T42D@@PJEn%!KDFk(x5);1-%zEfwb38gWGj<}ACxLriiLj^3yJ)ZUA?9v>7xo+ zU5@x;sj|)_{@5fWa>QDtt}hpjl*_itq}RmCtwQNFU6poAiDa}Gk;G_;tg&1^`p~k* zrSj_wg(TklKOwuR0*4nq6wp{WwCT3U)fS3ohYkHZ4pauh!9EK z5SvghYRE%LYqGW z4&xQWM)VloIc%@}2r_t={3np21W{>Qudk)AwX4_3f24@Y+nsJF+uG%G?)OFIJ+1A| zmaf(Vo?ajC)}vZ{YgmhK41b263cY#Uu2S!Vo(2J@0TI9*{O{A{>SkKFinzaoTP0p` z#+u$s3(2yc#U3FZIHSQw&xkjE3mr%a>{^x|9R-o2NiMI)9Zl(FySh4e7qQc)Asmp= z-*@^_Nu j-5#d5%ejJ0!aqCvMU_{9h^i0fkY