Browse Source

added dateutil to requirements.txt, added parsing the period from fdb-spider output

master
alpcentaur 10 months ago
parent
commit
1f21166cf7
4 changed files with 26 additions and 6 deletions
  1. +11
    -5
      fdb_spider_interface.py
  2. +1
    -1
      new_message_file.txt
  3. +2
    -0
      requirements.txt
  4. +12
    -0
      roomhistories/657c97e5ebb39dd248d38d51.txt

+ 11
- 5
fdb_spider_interface.py View File

@ -4,7 +4,7 @@ import configparser
import json import json
import requests import requests
import yaml import yaml
import dateutil.parser
import subprocess import subprocess
config = configparser.ConfigParser() config = configparser.ConfigParser()
@ -179,6 +179,7 @@ if __name__ == '__main__':
print(room_fdb, i) print(room_fdb, i)
for key in data[room_fdb][i]: for key in data[room_fdb][i]:
contains_tag = False contains_tag = False
period_data_formatted = dateutil.parser.parse(data[room_fdb][i][key]["period"])
name_data_lower = [word.lower() for word in data[room_fdb][i][key]["name"].split(' ')] name_data_lower = [word.lower() for word in data[room_fdb][i][key]["name"].split(' ')]
info_data_lower = [word.lower() for word in data[room_fdb][i][key]["info"].split(' ')] info_data_lower = [word.lower() for word in data[room_fdb][i][key]["info"].split(' ')]
text_data_lower = [word.lower() for word in data[room_fdb][i][key]["text"].split(' ')] text_data_lower = [word.lower() for word in data[room_fdb][i][key]["text"].split(' ')]
@ -237,7 +238,7 @@ if __name__ == '__main__':
except: except:
url = data[room_fdb][i][key]["link"] url = data[room_fdb][i][key]["link"]
entry_message = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>' + '\n' + data[room_fdb][i][key]["info"] + '\n' + str(tag_list)
entry_message = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>' + '\n' + data[room_fdb][i][key]["info"] + '\n' + 'The period of the entry is:' + str(period_data_formatted) + '\n' + str(tag_list)
entry_message_identifier = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>' entry_message_identifier = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>'
message_was_already_there = False message_was_already_there = False
@ -406,7 +407,7 @@ if __name__ == '__main__':
command : `updaterooms <room-id>` command : `updaterooms <room-id>`
example : `updaterooms 6572012bebb39dd248d08320` example : `updaterooms 6572012bebb39dd248d08320`
----------------------------------------------------------------------------------------- -----------------------------------------------------------------------------------------
To create a room use following command, but be sure to not have spaces in your lists,
To create a room use following command, but be sure to **not have spaces** in your lists,
as a space indicates a new command argument: as a space indicates a new command argument:
command : `createroom <room-name> <list-fdbs> <list-tags>` command : `createroom <room-name> <list-fdbs> <list-tags>`
example : `createroom room-test-1 ['giz','fdb2'] ['tag1','tag2','tag3']` example : `createroom room-test-1 ['giz','fdb2'] ['tag1','tag2','tag3']`
@ -441,8 +442,12 @@ if __name__ == '__main__':
room_file_list = room_file_raw.split('§%§%') room_file_list = room_file_raw.split('§%§%')
room_list.append([room[:-4], room_file_list[0], room_file_list[1], room_file_list[2]]) room_list.append([room[:-4], room_file_list[0], room_file_list[1], room_file_list[2]])
room_list_string = ''
for room in room_list:
room_list_string += str(room) + '\n' + '------------------------------------------------------------------------------' + '\n'
#print(room_list) #print(room_list)
answer = str(room_list)
answer = room_list_string
if user_input_list[0] == 'updaterooms': if user_input_list[0] == 'updaterooms':
try: try:
@ -538,6 +543,7 @@ if __name__ == '__main__':
print(room_fdb, i) print(room_fdb, i)
for key in data[room_fdb][i]: for key in data[room_fdb][i]:
contains_tag = False contains_tag = False
period_data_formatted = dateutil.parser.parse(data[room_fdb][i][key]["period"])
name_data_lower = [word.lower() for word in data[room_fdb][i][key]["name"].split(' ')] name_data_lower = [word.lower() for word in data[room_fdb][i][key]["name"].split(' ')]
info_data_lower = [word.lower() for word in data[room_fdb][i][key]["info"].split(' ')] info_data_lower = [word.lower() for word in data[room_fdb][i][key]["info"].split(' ')]
text_data_lower = [word.lower() for word in data[room_fdb][i][key]["text"].split(' ')] text_data_lower = [word.lower() for word in data[room_fdb][i][key]["text"].split(' ')]
@ -630,7 +636,7 @@ if __name__ == '__main__':
print('101110001101010010010101000111') print('101110001101010010010101000111')
entry_message = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>' + '\n' + data[room_fdb][i][key]["info"] + '\n' + str(tag_list)
entry_message = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>' + '\n' + data[room_fdb][i][key]["info"] + '\n' + 'The period of the entry is:' + str(period_data_formatted) + '\n' + str(tag_list)
entry_message_identifier = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>' entry_message_identifier = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>'

+ 1
- 1
new_message_file.txt View File

@ -1 +1 @@
updaterooms 657c97e5ebb39dd248d38d51§%§%BvLofBxRKbtk9AHkK
printcommands§%§%WM56LadJhkYBuGayZ

+ 2
- 0
requirements.txt View File

@ -1,8 +1,10 @@
certifi==2023.7.22 certifi==2023.7.22
charset-normalizer==3.2.0 charset-normalizer==3.2.0
idna==3.4 idna==3.4
python-dateutil==2.8.2
PyYAML==6.0.1 PyYAML==6.0.1
requests==2.31.0 requests==2.31.0
rocket-python==1.3.4 rocket-python==1.3.4
schedule==1.2.1 schedule==1.2.1
six==1.16.0
urllib3==2.0.4 urllib3==2.0.4

+ 12
- 0
roomhistories/657c97e5ebb39dd248d38d51.txt View File

@ -1 +1,13 @@
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYY6Y1AZS8RDL/de/overview?28|81303963 - Entwicklung eines webbasierten Trainings für Fachthemen im Bereich des nachhaltigem Entrepreneurships> <https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYY6Y1AZS8RDL/de/overview?28|81303963 - Entwicklung eines webbasierten Trainings für Fachthemen im Bereich des nachhaltigem Entrepreneurships>
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYY6Y1AXZRLFW/de/overview?19|81304134-Mission de coaching pour l'accompagnement de 15 entreprises industrielles pour le calcul de l"empreinte carbone selon la norme ISO 14067:2018>
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXR2YYRYYDA/de/overview?38|81298137-Development of tools, instruments and project concepts to leverage climate finance to scale up renewables and energy efficiency in Côte d'Ivoire>
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYY6Y1L7LSVE5/de/overview?53|Pool of energy efficiency experts>
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYY6Y1FMHFNQS/de/overview?63|81303704-Capacity Development on Green Energy Finance for Domestic Financiers>
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYY6Y1FMDZBYH/de/overview?64|81303544-Advisory services to establish Gender Diversity Management in companies supporting the green transformation>
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYY6Y1FK96YAK/de/overview?69|81301307 - Study to identify green FSD opportunities in the MENA region>
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYY6Y1L7Q0R4H/de/overview?71|81203546-Implementation of Gender Mainstreaming and Disability Inclusion activities in Dual TVET in Kenya>
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYYRY1F0U9P72/de/overview?75|Diverse unterschiedliche Güter für den Senegal>
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYY6Y1L1QRSZU/de/overview?85|81303447-Implementing Energy Efficiency during refurbishment of non-residential buildings in Morocco>
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYY6Y1EQWGUKJ/de/overview?90|81301134 - Provide recommendations for Donor Interventions to support Green Industrial Policies and Just Transition Implications>
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYY6Y1CV0XTS6/de/overview?97|81296882-Unterstützungsleistungen für das SV Klima>
<https://ausschreibungen.giz.de/Satellite/public/company/project/CXTRYY6Y1C82BB4U/de/overview?99|81294786-Development of Sustainable Energy and Climate Action Plans for selected Local Self-Government Units in Western Balkans>

Loading…
Cancel
Save