You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

109 lines
6.4 KiB

  1. # Settings for the PEP crawler per country to crawl
  2. # Follow the syntax and dont use tbody as it gets added by the browser (when researching xpath through inspector)
  3. # xpath syntax: https://www.w3schools.com/xml/xpath_syntax.asp
  4. # lxml xpath syntax: https://www.geeksforgeeks.org/web-scraping-using-lxml-and-xpath-in-python/
  5. foerderinfo.bund.de:
  6. domain: 'http://foerderinfo.bund.de'
  7. entry-list:
  8. link1: 'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/expertensuche/Servicesuche_Formular.html?gtp=33498_list%253D'
  9. link2: '#searchResults'
  10. iteration-var-list: '[1,2,3,4,5]'
  11. #iteration-var-list: '[1,2]'
  12. parent: "//html//body//div//main//div//div[@class='row']//section[@class='l-search-result-list']//div[@class='l-search-result-list__item']"
  13. child-name: "//div[@class='c-search-result__text-wrapper']//span[@class='c-search-result__title']/text()"
  14. jsdomain: 'None'
  15. javascript-link: ""
  16. child-link: "/a[@class='c-search-result']/@href"
  17. child-info: "//"
  18. child-period: "/"
  19. child-sponsor: "/"
  20. entry:
  21. info-1:
  22. parent: '//html//body//form//table'
  23. #child-name: '//html//body//form//table//tr[1]//td[2]//span'
  24. #child-sum: '//html//body//form//table//tr[2]//td[1]//span//img'
  25. #child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1'
  26. foerderinfo.bund.de-bekanntmachungen:
  27. domain: 'http://foerderinfo.bund.de'
  28. entry-list:
  29. link1: 'https://www.foerderinfo.bund.de/SiteGlobals/Forms/foerderinfo/bekanntmachungen/Bekanntmachungen_Formular.html?gtp=407348_list%253D'
  30. link2: '#searchResults'
  31. # here jsdomain has to be specified None with this syntax, if html pages of entrylists are not depending on javascript actions
  32. jsdomain: 'None'
  33. iteration-var-list: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]'
  34. #iteration-var-list: '[1,2]'
  35. #parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']"
  36. parent: "//html//body//div//main//div[@class='row']//a[contains(@class, 'c-teaser--announcement')]"
  37. child-name: "//div[@class='c-teaser__text-wrapper']//span[@class='c-teaser__title']/text()"
  38. javascript-link: ""
  39. child-link: "/@href"
  40. child-info: "//div[@class='c-teaser__text-wrapper']//div[@class='c-teaser__text']/p/text()"
  41. #child-period: "//div[@class='c-teaser__text-wrapper']//small[@class='c-topline']/span[@class='c-topline__item']/text()"
  42. child-period: "//div[@class='c-teaser__text-wrapper']//small//span/time/text()"
  43. child-sponsor: "//div[@class='c-teaser__text-wrapper']//small[@class='c-topline']//span[@class='c-topline__item']/span[@class='c-topline__category']/text()"
  44. entry:
  45. general:
  46. uniform: 'FALSE'
  47. unitrue:
  48. parent: '//html//body//form//table'
  49. #child-name: '//html//body//form//table//tr[1]//td[2]//span'
  50. #child-sum: '//html//body//form//table//tr[2]//td[1]//span//img'
  51. #child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1'
  52. unifalse:
  53. wordlist: "['Mobilität', 'Energie', 'Off-grid', 'regenerative Energien', 'Solar', 'Energienetze', 'Elektromobilität']"
  54. giz:
  55. domain: 'https://ausschreibungen.giz.de'
  56. entry-list:
  57. link1: 'https://ausschreibungen.giz.de/Satellite/company/welcome.do?method=showTable&fromSearch=1&tableSortPROJECT_RESULT=2&tableSortAttributePROJECT_RESULT=publicationDate&selectedTablePagePROJECT_RESULT='
  58. link2: ''
  59. jsdomain: 'None'
  60. iteration-var-list: "[1,2,3,4,5,6,7]"
  61. #iteration-var-list: "[1,2]"
  62. #parent: "//html//body//div//main//div//div[@class='row']//div[@class='large-12']//a[@class='c-teaser']"
  63. parent: "//html//body//div//div//table[contains(@class, 'csx-new-table')]//tbody//tr"
  64. child-name: "//td[3]//text()"
  65. child-link: "//a/@href"
  66. javascript-link: "/td[6]/a"
  67. child-info: "/td[4]/text()[1]"
  68. child-period: "//td[2]/abbr/text()"
  69. #child-period: "//div[@class='c-teaser__text-wrapper']//small//span/time/text()"
  70. child-sponsor: "/tr/td[4]/text()"
  71. entry:
  72. general:
  73. uniform: 'TRUE'
  74. unitrue:
  75. #parent: '//html//body//form//table'
  76. text: '/html/body/div[2]/div[4]/div/div[5]/div/table/tbody/tr/td[5]/a/@href'
  77. #child-sum: '//html//body//form//table//tr[2]//td[1]//span//img'
  78. #child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1'
  79. unifalse:
  80. wordlist: "['Mobilität', 'Energie', 'Off-grid', 'regenerative Energien', 'Solar', 'Energienetze', 'Elektromobilität']"
  81. evergabe-online:
  82. domain: 'https://www.evergabe-online.de/'
  83. entry-list:
  84. link1: 'https://www.evergabe-online.de/search.html?101-1.-searchPanel-results-searchResults-results-topToolbars-toolbars-1-span-navigator-navigation-'
  85. link2: '-pageLink'
  86. jsdomain: 'https://www.evergabe-online.de/search.html'
  87. jslink1: '/html/body/div[8]/main/div[4]/div/div/div[2]/table/thead/tr[1]/td/div[2]/div/span['
  88. jslink2: ']'
  89. jsiteration-var-list: "[1,2, 3, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,6,7,8,9,10]"
  90. #jsiteration-var-list: "[1,2, 3]"
  91. iteration-var-list: "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102]"
  92. #iteration-var-list: "[1, 2, 3]"
  93. parent: "/html/body/div[8]/main/div[4]/div/div/div[2]/table/tbody//tr"
  94. child-name: "//td[1]/div/a/text()"
  95. child-link: "//td[1]/div/a/@href"
  96. javascript-link: ""
  97. child-info: "//td[3]/div/text()"
  98. child-period: "//td[5]/text()"
  99. child-sponsor: "//td[2]/div/text()"
  100. entry:
  101. general:
  102. uniform: 'FALSE'
  103. unitrue:
  104. #parent: '//html//body//form//table'
  105. #text: '/html/body/div[2]/div[4]/div/div[5]/div/table/tbody/tr/td[5]/a/@href'
  106. #child-sum: '//html//body//form//table//tr[2]//td[1]//span//img'
  107. #child-deadline: '//html/body/form/table/tr[2]/td[3]/span + label.1'
  108. unifalse:
  109. wordlist: "['Mobilität', 'Energie', 'Off-grid', 'regenerative Energien', 'Solar', 'Energienetze', 'Elektromobilität']"