alpcentaur/fdb-spider-interface
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

added updaterooms for single rooms you specify in second commandline arg

Browse source
This commit is contained in:
alpcentaur 2023-12-12 13:07:05 +01:00
parent 77bd441dff
commit 0bac9ec8a7
1 changed files with 77 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

120
fdb_spider_interface.py
View file

@ -214,6 +214,8 @@ if __name__ == '__main__':
if user_input_list[0] == 'printrooms':
room_was_found = False
# get all the rooms
import os
@ -231,55 +233,87 @@ if __name__ == '__main__':
answer = str(room_list)
if user_input_list[0] == 'updaterooms':
room_list = []
rooms = os.listdir('rooms')
for room in rooms:
room_file = open('rooms/' + room, 'r')
room_file_raw = room_file.read()
room_file.close()
room_file_list = room_file_raw.split('§%§%')
room_list.append([room[:-4], room_file_list[0], room_file_list[1], room_file_list[2]])
with open(spider_directory + '/spiders/config.yaml' , "r") as stream:
try:
config = yaml.safe_load(stream)
except yaml.YAMLError as exc:
print(exc)
fdb_list = []
#print(config)
for key in config:
fdb_list.append(key)
answer = "Ich update die Rooms auf Basis der Daten von heute morgen um 6 Uhr.."
data = dict({})
for room in room_list:
print(room[0])
room_fdbs = room[2]
room_tags = room[3]
#subprocess.run(["python", spider_directory + 'main.py', fdbs])
if room[0] == room_to_update:
room_was_found = True
print(room[0])
room_fdbs = room[2]
room_tags = room[3]
#subprocess.run(["python", spider_directory + 'main.py', fdbs])
for room_fdb in eval(room_fdbs):
#print('room_fdb',room_fdb, 'fdb_list',fdb_list)
try:
iteration_var_list = config.get(room_fdb).get("entry-list").get("iteration-var-list")
except Exception as e:
print('there was an error with the entry-list parameter in the config regarding the fdb ', room_fdb)
if room_fdb in fdb_list and room_fdb not in [key for key in data]:
iterdict = {}
for i in eval(iteration_var_list):
f = open(spider_directory + "/spiders/output/" + room_fdb + str(i) + "entryList.txt")
text = f.read()
dictionary_entry_list = eval(text)
iterdict[i] = dictionary_entry_list
data[room_fdb] = iterdict
for i in eval(iteration_var_list):
for room_fdb in eval(room_fdbs):
#print('room_fdb',room_fdb, 'fdb_list',fdb_list)
try:
print(room_fdb, i)
for key in data[room_fdb][i]:
contains_tag = False
for tag in eval(room_tags):
if tag in (data[room_fdb][i][key]["name"].split(' ') or data[room_fdb][i][key]["info"].split(' ') or data[room_fdb][i][key]["text"].split(' ')):
contains_tag = True
if contains_tag == True:
try:
url = data[room_fdb][i][key]["domain"]
except:
url = data[room_fdb][i][key]["link"]
entry_message = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>' + '\n' + data[room_fdb][i][key]["info"]
api.send_message(entry_message, room[0])
iteration_var_list = config.get(room_fdb).get("entry-list").get("iteration-var-list")
except Exception as e:
print("probably i was not there in last page, original error is:", e)
print('there was an error with the entry-list parameter in the config regarding the fdb ', room_fdb)
if room_fdb in fdb_list and room_fdb not in [key for key in data]:
iterdict = {}
for i in eval(iteration_var_list):
f = open(spider_directory + "/spiders/output/" + room_fdb + str(i) + "entryList.txt")
text = f.read()
dictionary_entry_list = eval(text)
iterdict[i] = dictionary_entry_list
data[room_fdb] = iterdict
for i in eval(iteration_var_list):
try:
print(room_fdb, i)
for key in data[room_fdb][i]:
contains_tag = False
for tag in eval(room_tags):
if tag in (data[room_fdb][i][key]["name"].split(' ') or data[room_fdb][i][key]["info"].split(' ') or data[room_fdb][i][key]["text"].split(' ')):
contains_tag = True
if contains_tag == True:
try:
url = data[room_fdb][i][key]["domain"]
except:
url = data[room_fdb][i][key]["link"]
entry_message = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>' + '\n' + data[room_fdb][i][key]["info"]
api.send_message(entry_message, room[0])
except Exception as e:
print("probably i was not there in last page, original error is:", e)
if room_was_found == False:
answer = 'Den angegebenen Raum gibt es nicht.. Versuche es noch einmal mit -> updaterooms <room-id>'
if room_was_found == True:
answer = "Ich update die Rooms auf Basis der Daten in fdb-spider/spiders/output.."
if user_input_list[0] == 'printfdbs':