From 77bd441dff3af476eb6d02dcaa3313e9f03a5a47 Mon Sep 17 00:00:00 2001 From: alpcentaur Date: Tue, 12 Dec 2023 12:41:50 +0100 Subject: [PATCH] added command updaterooms --- fdb_spider_interface.py | 52 +++++++++++++++++++++++++++++++++++++++++ new_message_file.txt | 2 +- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/fdb_spider_interface.py b/fdb_spider_interface.py index 46b3ddf..2967750 100644 --- a/fdb_spider_interface.py +++ b/fdb_spider_interface.py @@ -230,6 +230,58 @@ if __name__ == '__main__': #print(room_list) answer = str(room_list) + if user_input_list[0] == 'updaterooms': + answer = "Ich update die Rooms auf Basis der Daten von heute morgen um 6 Uhr.." + data = dict({}) + for room in room_list: + print(room[0]) + room_fdbs = room[2] + room_tags = room[3] + #subprocess.run(["python", spider_directory + 'main.py', fdbs]) + + for room_fdb in eval(room_fdbs): + #print('room_fdb',room_fdb, 'fdb_list',fdb_list) + try: + iteration_var_list = config.get(room_fdb).get("entry-list").get("iteration-var-list") + except Exception as e: + print('there was an error with the entry-list parameter in the config regarding the fdb ', room_fdb) + if room_fdb in fdb_list and room_fdb not in [key for key in data]: + iterdict = {} + for i in eval(iteration_var_list): + f = open(spider_directory + "/spiders/output/" + room_fdb + str(i) + "entryList.txt") + text = f.read() + + dictionary_entry_list = eval(text) + + iterdict[i] = dictionary_entry_list + + data[room_fdb] = iterdict + + for i in eval(iteration_var_list): + try: + print(room_fdb, i) + for key in data[room_fdb][i]: + contains_tag = False + for tag in eval(room_tags): + if tag in (data[room_fdb][i][key]["name"].split(' ') or data[room_fdb][i][key]["info"].split(' ') or data[room_fdb][i][key]["text"].split(' ')): + contains_tag = True + if contains_tag == True: + + try: + url = data[room_fdb][i][key]["domain"] + except: + url = data[room_fdb][i][key]["link"] + + + + entry_message = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>' + '\n' + data[room_fdb][i][key]["info"] + + api.send_message(entry_message, room[0]) + + except Exception as e: + print("probably i was not there in last page, original error is:", e) + + if user_input_list[0] == 'printfdbs': answer = str(fdb_list) diff --git a/new_message_file.txt b/new_message_file.txt index 11a6f40..80d62f2 100644 --- a/new_message_file.txt +++ b/new_message_file.txt @@ -1 +1 @@ -createroom fdbspider-mobi ['giz','foerderinfo.bund.de-bekanntmachungen'] ['Energie','Mobilität','Wasserstoff','Solarenergie','regenerative','regenerativ','erneuerbar','Transport']§%§%nCqXY7riWQkZStD8Y \ No newline at end of file +updaterooms§%§%ramWq2wQnMLem29uR \ No newline at end of file