added trafilatura exception
This commit is contained in:
alpcentaur
parent
14ece9bceb
commit
d3335f203b
2 changed files with 69 additions and 13 deletions
Binary file not shown.
|
|
@ -10,6 +10,7 @@ import lxml.html
|
||||||
import lxml.html.soupparser
|
import lxml.html.soupparser
|
||||||
from lxml import html
|
from lxml import html
|
||||||
|
|
||||||
|
from trafilatura import extract
|
||||||
|
|
||||||
|
|
||||||
class fdb_spider(object):
|
class fdb_spider(object):
|
||||||
|
|
@ -215,8 +216,8 @@ class fdb_spider(object):
|
||||||
fdb_conf_entry_list_child_period = fdb_conf_entry_list.get("child-period")
|
fdb_conf_entry_list_child_period = fdb_conf_entry_list.get("child-period")
|
||||||
|
|
||||||
|
|
||||||
print('blabliblub')
|
#print('blabliblub')
|
||||||
print('len', len(tree.xpath(fdb_conf_entry_list_parent)))
|
#print('len', len(tree.xpath(fdb_conf_entry_list_parent)))
|
||||||
for n in range(len(tree.xpath(fdb_conf_entry_list_parent))):
|
for n in range(len(tree.xpath(fdb_conf_entry_list_parent))):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
@ -253,7 +254,7 @@ class fdb_spider(object):
|
||||||
+ "]"
|
+ "]"
|
||||||
+ fdb_conf_entry_list_child_period
|
+ fdb_conf_entry_list_child_period
|
||||||
)[0]
|
)[0]
|
||||||
print('period', period)
|
#print('period', period)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print("period could not be parsed", e, period)
|
print("period could not be parsed", e, period)
|
||||||
period = 'NONE'
|
period = 'NONE'
|
||||||
|
|
@ -266,7 +267,7 @@ class fdb_spider(object):
|
||||||
+ "]"
|
+ "]"
|
||||||
+ fdb_conf_entry_list_child_link
|
+ fdb_conf_entry_list_child_link
|
||||||
)[0]
|
)[0]
|
||||||
print('link', link)
|
#print('link', link)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print("link could not be parsed", e, link)
|
print("link could not be parsed", e, link)
|
||||||
|
|
@ -386,9 +387,9 @@ class fdb_spider(object):
|
||||||
fdb_conf = self.config.get(fdb)
|
fdb_conf = self.config.get(fdb)
|
||||||
fdb_domain = fdb_conf.get("domain")
|
fdb_domain = fdb_conf.get("domain")
|
||||||
fdb_conf_entry = fdb_conf.get("entry")
|
fdb_conf_entry = fdb_conf.get("entry")
|
||||||
print('balubaluba', fdb_conf_entry)
|
#print('balubaluba', fdb_conf_entry)
|
||||||
fdb_conf_entry_general = fdb_conf_entry.get("general")
|
fdb_conf_entry_general = fdb_conf_entry.get("general")
|
||||||
print(fdb_conf_entry_general)
|
#print(fdb_conf_entry_general)
|
||||||
|
|
||||||
|
|
||||||
for entry_id in dictionary_entry_list:
|
for entry_id in dictionary_entry_list:
|
||||||
|
|
@ -424,7 +425,7 @@ class fdb_spider(object):
|
||||||
fdb_conf_entry_unitrue_entry_child
|
fdb_conf_entry_unitrue_entry_child
|
||||||
)
|
)
|
||||||
|
|
||||||
print("oi", child)
|
#print("oi", child)
|
||||||
|
|
||||||
if len(child) > 0:
|
if len(child) > 0:
|
||||||
dictionary_entry_list[entry_id][key] = child[
|
dictionary_entry_list[entry_id][key] = child[
|
||||||
|
|
@ -444,18 +445,73 @@ class fdb_spider(object):
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
text = tree.xpath(
|
p_text = tree.xpath(
|
||||||
"//p//text()"
|
"//p//text()"
|
||||||
)
|
)
|
||||||
|
|
||||||
print("oi", text)
|
div_text = tree.xpath(
|
||||||
|
"//div//text()"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
#print("oi", text)
|
||||||
generaltext = ''
|
generaltext = ''
|
||||||
for n in range(len(text)):
|
for n in range(len(p_text)):
|
||||||
|
|
||||||
if len(text[n]) > 0:
|
if len(p_text[n]) > 0:
|
||||||
generaltext += text[n] + ' '
|
generaltext += p_text[n] + ' '
|
||||||
|
|
||||||
|
for n in range(len(div_text)):
|
||||||
|
|
||||||
|
if len(div_text[n]) > 0 and div_text[n] not in p_text:
|
||||||
|
generaltext += div_text[n] + ' '
|
||||||
|
|
||||||
|
|
||||||
|
generaltextlist = generaltext.split(' ')
|
||||||
|
if len(generaltextlist) > 5000:
|
||||||
|
print('text over 1000 words for entry id', entry_id, ' number of words:', len(generaltextlist))
|
||||||
|
|
||||||
|
file_name = "spiders/pages/" + fdb + str(i) + "/" + str(entry_id) + ".html"
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(file_name , 'r', encoding='utf-8') as file:
|
||||||
|
html_content = file.read()
|
||||||
|
except Exception as e:
|
||||||
|
|
||||||
|
with open(file_name , 'r', encoding='latin-1') as file:
|
||||||
|
html_content = file.read()
|
||||||
|
print('encoding utf8 in opening with trafilatura did not work, trying latin1, original error message is:', e)
|
||||||
|
|
||||||
|
generaltext = extract(html_content)
|
||||||
|
print('generaltext word count was: ', len(generaltextlist), 'but now trafilatura did the job and new wordcount is:', len(generaltext.split(' ')))
|
||||||
|
|
||||||
|
if len(generaltextlist) < 2:
|
||||||
|
print('no text parsed, the wc is', len(generaltextlist))
|
||||||
|
|
||||||
|
print('text under 2 words for entry id', entry_id, ' number of words:', len(generaltextlist))
|
||||||
|
|
||||||
|
file_name = "spiders/pages/" + fdb + str(i) + "/" + str(entry_id) + ".html"
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(file_name , 'r', encoding='utf-8') as file:
|
||||||
|
html_content = file.read()
|
||||||
|
except Exception as e:
|
||||||
|
|
||||||
|
with open(file_name , 'r', encoding='latin-1') as file:
|
||||||
|
html_content = file.read()
|
||||||
|
print('encoding utf8 in opening with trafilatura did not work, trying latin1, original error message is:', e)
|
||||||
|
|
||||||
|
generaltext = extract(html_content)
|
||||||
|
try:
|
||||||
|
if len(generaltext) > 2:
|
||||||
|
print('generaltext word count was: ', len(generaltextlist), 'but now trafilatura did the job and new wordcount is:', len(generaltext.split(' ')))
|
||||||
|
except:
|
||||||
|
|
||||||
|
print('trafilatura got this out:', generaltext , 'setting generaltext to NONE')
|
||||||
|
generaltext = 'NONE'
|
||||||
|
|
||||||
dictionary_entry_list[entry_id]["text"] = generaltext
|
dictionary_entry_list[entry_id]["text"] = generaltext
|
||||||
|
dictionary_entry_list[entry_id]["text-word-count"] = len(generaltextlist)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue