|
@ -104,11 +104,11 @@ if __name__ == '__main__': |
|
|
|
|
|
|
|
|
for room_fdb in eval(room_fdbs): |
|
|
for room_fdb in eval(room_fdbs): |
|
|
#print('room_fdb',room_fdb, 'fdb_list',fdb_list) |
|
|
#print('room_fdb',room_fdb, 'fdb_list',fdb_list) |
|
|
|
|
|
try: |
|
|
|
|
|
iteration_var_list = config.get(room_fdb).get("entry-list").get("iteration-var-list") |
|
|
|
|
|
except Exception as e: |
|
|
|
|
|
print('there was an error with the entry-list parameter in the config regarding the fdb ', room_fdb) |
|
|
if room_fdb in fdb_list and room_fdb not in [key for key in data]: |
|
|
if room_fdb in fdb_list and room_fdb not in [key for key in data]: |
|
|
try: |
|
|
|
|
|
iteration_var_list = config.get(room_fdb).get("entry-list").get("iteration-var-list") |
|
|
|
|
|
except Exception as e: |
|
|
|
|
|
print('there was an error with the entry-list parameter in the config regarding the fdb ', room_fdb) |
|
|
|
|
|
iterdict = {} |
|
|
iterdict = {} |
|
|
for i in eval(iteration_var_list): |
|
|
for i in eval(iteration_var_list): |
|
|
f = open(spider_directory + "/spiders/output/" + room_fdb + str(i) + "entryList.txt") |
|
|
f = open(spider_directory + "/spiders/output/" + room_fdb + str(i) + "entryList.txt") |
|
@ -120,29 +120,29 @@ if __name__ == '__main__': |
|
|
|
|
|
|
|
|
data[room_fdb] = iterdict |
|
|
data[room_fdb] = iterdict |
|
|
|
|
|
|
|
|
for i in eval(iteration_var_list): |
|
|
|
|
|
try: |
|
|
|
|
|
print(room_fdb, i) |
|
|
|
|
|
for key in data[room_fdb][i]: |
|
|
|
|
|
contains_tag = False |
|
|
|
|
|
for tag in eval(room_tags): |
|
|
|
|
|
if tag in (data[room_fdb][i][key]["name"].split(' ') or data[room_fdb][i][key]["info"].split(' ') or data[room_fdb][i][key]["text"].split(' ')): |
|
|
|
|
|
contains_tag = True |
|
|
|
|
|
if contains_tag == True: |
|
|
|
|
|
|
|
|
for i in eval(iteration_var_list): |
|
|
|
|
|
try: |
|
|
|
|
|
print(room_fdb, i) |
|
|
|
|
|
for key in data[room_fdb][i]: |
|
|
|
|
|
contains_tag = False |
|
|
|
|
|
for tag in eval(room_tags): |
|
|
|
|
|
if tag in (data[room_fdb][i][key]["name"].split(' ') or data[room_fdb][i][key]["info"].split(' ') or data[room_fdb][i][key]["text"].split(' ')): |
|
|
|
|
|
contains_tag = True |
|
|
|
|
|
if contains_tag == True: |
|
|
|
|
|
|
|
|
try: |
|
|
|
|
|
url = data[room_fdb][i][key]["domain"] |
|
|
|
|
|
except: |
|
|
|
|
|
url = data[room_fdb][i][key]["link"] |
|
|
|
|
|
|
|
|
try: |
|
|
|
|
|
url = data[room_fdb][i][key]["domain"] |
|
|
|
|
|
except: |
|
|
|
|
|
url = data[room_fdb][i][key]["link"] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
entry_message = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>' + '\n' + data[room_fdb][i][key]["info"] |
|
|
|
|
|
|
|
|
entry_message = '<' + url + '|' + data[room_fdb][i][key]["name"]+ '>' + '\n' + data[room_fdb][i][key]["info"] |
|
|
|
|
|
|
|
|
api.send_message(entry_message, room[0]) |
|
|
|
|
|
|
|
|
api.send_message(entry_message, room[0]) |
|
|
|
|
|
|
|
|
except Exception as e: |
|
|
|
|
|
print("probably i was not there in last page, original error is:", e) |
|
|
|
|
|
|
|
|
except Exception as e: |
|
|
|
|
|
print("probably i was not there in last page, original error is:", e) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|