|
@ -1,5 +1,3 @@ |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import os |
|
|
import os |
|
|
|
|
|
|
|
|
import yaml |
|
|
import yaml |
|
@ -12,172 +10,177 @@ import lxml.html |
|
|
import lxml.html.soupparser |
|
|
import lxml.html.soupparser |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class membersParliamentCrawler(object): |
|
|
class membersParliamentCrawler(object): |
|
|
|
|
|
|
|
|
def __init__(self, configFile): |
|
|
def __init__(self, configFile): |
|
|
|
|
|
|
|
|
with open(configFile, "r") as stream: |
|
|
with open(configFile, "r") as stream: |
|
|
try: |
|
|
try: |
|
|
self.config = yaml.safe_load(stream) |
|
|
self.config = yaml.safe_load(stream) |
|
|
except yaml.YAMLError as exc: |
|
|
except yaml.YAMLError as exc: |
|
|
print(exc) |
|
|
print(exc) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# input list of countries in form of ['nicaragua', 'honduras', .. , 'mexico'] |
|
|
# input list of countries in form of ['nicaragua', 'honduras', .. , 'mexico'] |
|
|
|
|
|
|
|
|
def downloadMemberListPagesOfCountries(self, listOfCountries): |
|
|
def downloadMemberListPagesOfCountries(self, listOfCountries): |
|
|
|
|
|
|
|
|
# download only html pages of the countries specified in input |
|
|
# download only html pages of the countries specified in input |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for country in listOfCountries: |
|
|
for country in listOfCountries: |
|
|
for key in self.config: |
|
|
for key in self.config: |
|
|
if key in listOfCountries: |
|
|
if key in listOfCountries: |
|
|
try: |
|
|
try: |
|
|
memberList = self.config.get(key).get('memberList') |
|
|
|
|
|
|
|
|
memberList = self.config.get(key).get("memberList") |
|
|
except Exception as e: |
|
|
except Exception as e: |
|
|
print("There is a problem with the entry memberList in the config.yaml - the original error message is:", e) |
|
|
|
|
|
|
|
|
print( |
|
|
|
|
|
"There is a problem with the entry memberList in the config.yaml - the original error message is:", |
|
|
|
|
|
e, |
|
|
|
|
|
) |
|
|
try: |
|
|
try: |
|
|
memberListLink = memberList.get('link') |
|
|
|
|
|
|
|
|
memberListLink = memberList.get("link") |
|
|
except Exception as e: |
|
|
except Exception as e: |
|
|
print("No memberListLink defined in config.yaml - the original error message is:", e) |
|
|
|
|
|
|
|
|
|
|
|
# download the html page of the List of Members |
|
|
|
|
|
|
|
|
print( |
|
|
|
|
|
"No memberListLink defined in config.yaml - the original error message is:", |
|
|
|
|
|
e, |
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
# download the html page of the List of Members |
|
|
|
|
|
|
|
|
response = urllib.request.urlopen(memberListLink) |
|
|
response = urllib.request.urlopen(memberListLink) |
|
|
webContent = response.read().decode('UTF-8') |
|
|
|
|
|
|
|
|
webContent = response.read().decode("UTF-8") |
|
|
|
|
|
|
|
|
# save interim results to files |
|
|
# save interim results to files |
|
|
|
|
|
|
|
|
f = open('crawlers/pages/' + key +'MemberList.html', 'w+') |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
f = open("crawlers/pages/" + key + "MemberList.html", "w+") |
|
|
f.write(webContent) |
|
|
f.write(webContent) |
|
|
f.close |
|
|
f.close |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def parseMemberListData2dictionary(self, listOfCountries): |
|
|
def parseMemberListData2dictionary(self, listOfCountries): |
|
|
|
|
|
|
|
|
for country in listOfCountries: |
|
|
for country in listOfCountries: |
|
|
|
|
|
|
|
|
try: |
|
|
try: |
|
|
|
|
|
|
|
|
#use soupparser to handle broken html |
|
|
|
|
|
|
|
|
|
|
|
tree = lxml.html.soupparser.parse('crawlers/pages/' + country + 'MemberList.html') |
|
|
|
|
|
|
|
|
|
|
|
# for e in tree.iter(): |
|
|
|
|
|
# |
|
|
|
|
|
# print(e.tag) |
|
|
|
|
|
# |
|
|
|
|
|
# for e in tree.xpath('//html//body//form//table//tr//td//table//tr'): |
|
|
|
|
|
# |
|
|
|
|
|
# #print(etree.tostring(e).decode()) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# use soupparser to handle broken html |
|
|
|
|
|
|
|
|
|
|
|
tree = lxml.html.soupparser.parse( |
|
|
|
|
|
"crawlers/pages/" + country + "MemberList.html" |
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
# for e in tree.iter(): |
|
|
|
|
|
# |
|
|
|
|
|
# print(e.tag) |
|
|
|
|
|
# |
|
|
|
|
|
# for e in tree.xpath('//html//body//form//table//tr//td//table//tr'): |
|
|
|
|
|
# |
|
|
|
|
|
# #print(etree.tostring(e).decode()) |
|
|
|
|
|
|
|
|
dictionaryMemberList = {} |
|
|
dictionaryMemberList = {} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
countryConf = self.config.get(country) |
|
|
countryConf = self.config.get(country) |
|
|
countryDomain = countryConf.get('domain') |
|
|
|
|
|
countryConfMemberList = countryConf.get('memberList') |
|
|
|
|
|
countryConfMemberListParent = countryConfMemberList.get('parent') |
|
|
|
|
|
countryConfMemberListChildName = countryConfMemberList.get('child-name') |
|
|
|
|
|
countryConfMemberListChildLink = countryConfMemberList.get('child-link') |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
countryDomain = countryConf.get("domain") |
|
|
|
|
|
countryConfMemberList = countryConf.get("memberList") |
|
|
|
|
|
countryConfMemberListParent = countryConfMemberList.get("parent") |
|
|
|
|
|
countryConfMemberListChildName = countryConfMemberList.get("child-name") |
|
|
|
|
|
countryConfMemberListChildLink = countryConfMemberList.get("child-link") |
|
|
|
|
|
|
|
|
for n in range(len(tree.xpath(countryConfMemberListParent))): |
|
|
for n in range(len(tree.xpath(countryConfMemberListParent))): |
|
|
|
|
|
|
|
|
name = tree.xpath(countryConfMemberListParent + '[' + str(n) + ']' + countryConfMemberListChildName) |
|
|
|
|
|
link = tree.xpath(countryConfMemberListParent + '[' + str(n) + ']' + countryConfMemberListChildLink) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
name = tree.xpath( |
|
|
|
|
|
countryConfMemberListParent |
|
|
|
|
|
+ "[" |
|
|
|
|
|
+ str(n) |
|
|
|
|
|
+ "]" |
|
|
|
|
|
+ countryConfMemberListChildName |
|
|
|
|
|
) |
|
|
|
|
|
link = tree.xpath( |
|
|
|
|
|
countryConfMemberListParent |
|
|
|
|
|
+ "[" |
|
|
|
|
|
+ str(n) |
|
|
|
|
|
+ "]" |
|
|
|
|
|
+ countryConfMemberListChildLink |
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
if len(name) > 0: |
|
|
if len(name) > 0: |
|
|
|
|
|
|
|
|
dictionaryMemberList[n] = {} |
|
|
dictionaryMemberList[n] = {} |
|
|
dictionaryMemberList[n]['name'] = name[0] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
dictionaryMemberList[n]["name"] = name[0] |
|
|
|
|
|
|
|
|
if countryDomain in link[0]: |
|
|
if countryDomain in link[0]: |
|
|
|
|
|
|
|
|
dictionaryMemberList[n]['link'] = link[0] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
dictionaryMemberList[n]["link"] = link[0] |
|
|
|
|
|
|
|
|
if countryDomain not in link[0]: |
|
|
if countryDomain not in link[0]: |
|
|
|
|
|
|
|
|
dictionaryMemberList[n]['link'] = countryDomain + link[0] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
dictionaryMemberList[n]["link"] = countryDomain + link[0] |
|
|
|
|
|
|
|
|
except Exception as e: |
|
|
except Exception as e: |
|
|
|
|
|
|
|
|
print('parsing the html did not work. Possibly you first have to downloadMemberListPagesOfCountries(). The original error message is:', e) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
print( |
|
|
|
|
|
"parsing the html did not work. Possibly you first have to downloadMemberListPagesOfCountries(). The original error message is:", |
|
|
|
|
|
e, |
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
# save interim results to files |
|
|
# save interim results to files |
|
|
|
|
|
|
|
|
f = open('crawlers/output/' + country +'MemberList.txt', 'w+') |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
f = open("crawlers/output/" + country + "MemberList.txt", "w+") |
|
|
f.write(str(dictionaryMemberList)) |
|
|
f.write(str(dictionaryMemberList)) |
|
|
f.close |
|
|
f.close |
|
|
|
|
|
|
|
|
def downloadMemberDataHtmls(self, listOfCountries): |
|
|
def downloadMemberDataHtmls(self, listOfCountries): |
|
|
|
|
|
|
|
|
for country in listOfCountries: |
|
|
for country in listOfCountries: |
|
|
|
|
|
|
|
|
f = open('crawlers/output/' + country +'MemberList.txt') |
|
|
|
|
|
|
|
|
f = open("crawlers/output/" + country + "MemberList.txt") |
|
|
text = f.read() |
|
|
text = f.read() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
dictionaryMemberList = eval(text) |
|
|
dictionaryMemberList = eval(text) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for memberid in dictionaryMemberList: |
|
|
for memberid in dictionaryMemberList: |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
memberLink = dictionaryMemberList[memberid]['link'] |
|
|
|
|
|
|
|
|
|
|
|
# download the html page of the Member |
|
|
|
|
|
|
|
|
memberLink = dictionaryMemberList[memberid]["link"] |
|
|
|
|
|
|
|
|
|
|
|
# download the html page of the Member |
|
|
|
|
|
|
|
|
response = urllib.request.urlopen(memberLink) |
|
|
response = urllib.request.urlopen(memberLink) |
|
|
webContent = response.read().decode('UTF-8') |
|
|
|
|
|
|
|
|
webContent = response.read().decode("UTF-8") |
|
|
|
|
|
|
|
|
# save interim results to files |
|
|
# save interim results to files |
|
|
|
|
|
|
|
|
filename = 'crawlers/pages/' + country + '/' + str(memberid) +'.html' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
filename = "crawlers/pages/" + country + "/" + str(memberid) + ".html" |
|
|
|
|
|
|
|
|
os.makedirs(os.path.dirname(filename), exist_ok=True) |
|
|
os.makedirs(os.path.dirname(filename), exist_ok=True) |
|
|
f = open( filename, 'w+') |
|
|
|
|
|
|
|
|
f = open(filename, "w+") |
|
|
f.write(webContent) |
|
|
f.write(webContent) |
|
|
f.close |
|
|
f.close |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def parseMemberData2dictionary(self, listOfCountries): |
|
|
def parseMemberData2dictionary(self, listOfCountries): |
|
|
|
|
|
|
|
|
for country in listOfCountries: |
|
|
for country in listOfCountries: |
|
|
|
|
|
|
|
|
print('started to parse data of member of ' + country + ' ..') |
|
|
|
|
|
|
|
|
|
|
|
f = open('crawlers/output/' + country +'MemberList.txt') |
|
|
|
|
|
|
|
|
print("started to parse data of member of " + country + " ..") |
|
|
|
|
|
|
|
|
|
|
|
f = open("crawlers/output/" + country + "MemberList.txt") |
|
|
text = f.read() |
|
|
text = f.read() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
dictionaryMemberList = eval(text) |
|
|
dictionaryMemberList = eval(text) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
countryConf = self.config.get(country) |
|
|
countryConf = self.config.get(country) |
|
|
countryDomain = countryConf.get('domain') |
|
|
|
|
|
countryConfMember = countryConf.get('member') |
|
|
|
|
|
countryConfMemberInfo1 = countryConfMember.get('info-1') |
|
|
|
|
|
countryConfMemberInfo1Parent = countryConfMemberInfo1.get('parent') |
|
|
|
|
|
countryConfMemberInfo1ChildPoliticalParty = countryConfMemberInfo1.get('child-politicalParty') |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
countryDomain = countryConf.get("domain") |
|
|
|
|
|
countryConfMember = countryConf.get("member") |
|
|
|
|
|
countryConfMemberInfo1 = countryConfMember.get("info-1") |
|
|
|
|
|
countryConfMemberInfo1Parent = countryConfMemberInfo1.get("parent") |
|
|
|
|
|
countryConfMemberInfo1ChildPoliticalParty = countryConfMemberInfo1.get( |
|
|
|
|
|
"child-politicalParty" |
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
for memberid in dictionaryMemberList: |
|
|
for memberid in dictionaryMemberList: |
|
|
|
|
|
|
|
|
print('started to parse data of member with name ' + dictionaryMemberList[memberid]['name'] + ' ..') |
|
|
|
|
|
|
|
|
|
|
|
filename = 'crawlers/pages/' + country + '/' + str(memberid) +'.html' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
print( |
|
|
|
|
|
"started to parse data of member with name " |
|
|
|
|
|
+ dictionaryMemberList[memberid]["name"] |
|
|
|
|
|
+ " .." |
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
filename = "crawlers/pages/" + country + "/" + str(memberid) + ".html" |
|
|
|
|
|
|
|
|
tree = lxml.html.soupparser.parse(filename) |
|
|
tree = lxml.html.soupparser.parse(filename) |
|
|
|
|
|
|
|
|
politicalParty = tree.xpath(countryConfMemberInfo1Parent + countryConfMemberInfo1ChildPoliticalParty) |
|
|
|
|
|
|
|
|
|
|
|
print('oi', politicalParty) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
politicalParty = tree.xpath( |
|
|
|
|
|
countryConfMemberInfo1Parent |
|
|
|
|
|
+ countryConfMemberInfo1ChildPoliticalParty |
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
print("oi", politicalParty) |
|
|
|
|
|
|
|
|
if len(politicalParty) > 0: |
|
|
if len(politicalParty) > 0: |
|
|
|
|
|
|
|
|
dictionaryMemberList[memberid]['political party'] = politicalParty[0] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
f = open('crawlers/output/' + country +'MemberList.txt', 'w+') |
|
|
|
|
|
|
|
|
dictionaryMemberList[memberid]["political party"] = politicalParty[ |
|
|
|
|
|
0 |
|
|
|
|
|
] |
|
|
|
|
|
|
|
|
|
|
|
f = open("crawlers/output/" + country + "MemberList.txt", "w+") |
|
|
f.write(str(dictionaryMemberList)) |
|
|
f.write(str(dictionaryMemberList)) |
|
|
f.close |
|
|
f.close |
|
|
|
|
|
|