You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
183 lines
6.9 KiB
183 lines
6.9 KiB
|
|
|
|
import os
|
|
|
|
import yaml
|
|
import json
|
|
|
|
import urllib.request, urllib.error, urllib.parse
|
|
|
|
from lxml import etree
|
|
import lxml.html
|
|
import lxml.html.soupparser
|
|
|
|
|
|
|
|
|
|
class membersParliamentCrawler(object):
|
|
|
|
def __init__(self, configFile):
|
|
|
|
with open(configFile, "r") as stream:
|
|
try:
|
|
self.config = yaml.safe_load(stream)
|
|
except yaml.YAMLError as exc:
|
|
print(exc)
|
|
|
|
|
|
# input list of countries in form of ['nicaragua', 'honduras', .. , 'mexico']
|
|
|
|
def downloadMemberListPagesOfCountries(self, listOfCountries):
|
|
|
|
# download only html pages of the countries specified in input
|
|
|
|
for country in listOfCountries:
|
|
for key in self.config:
|
|
if key in listOfCountries:
|
|
try:
|
|
memberList = self.config.get(key).get('memberList')
|
|
except Exception as e:
|
|
print("There is a problem with the entry memberList in the config.yaml - the original error message is:", e)
|
|
try:
|
|
memberListLink = memberList.get('link')
|
|
except Exception as e:
|
|
print("No memberListLink defined in config.yaml - the original error message is:", e)
|
|
|
|
# download the html page of the List of Members
|
|
|
|
response = urllib.request.urlopen(memberListLink)
|
|
webContent = response.read().decode('UTF-8')
|
|
|
|
# save interim results to files
|
|
|
|
f = open('crawlers/pages/' + key +'MemberList.html', 'w+')
|
|
f.write(webContent)
|
|
f.close
|
|
|
|
|
|
def parseMemberListData2dictionary(self, listOfCountries):
|
|
|
|
for country in listOfCountries:
|
|
|
|
try:
|
|
|
|
#use soupparser to handle broken html
|
|
|
|
tree = lxml.html.soupparser.parse('crawlers/pages/' + country + 'MemberList.html')
|
|
|
|
# for e in tree.iter():
|
|
#
|
|
# print(e.tag)
|
|
#
|
|
# for e in tree.xpath('//html//body//form//table//tr//td//table//tr'):
|
|
#
|
|
# #print(etree.tostring(e).decode())
|
|
|
|
dictionaryMemberList = {}
|
|
|
|
countryConf = self.config.get(country)
|
|
countryDomain = countryConf.get('domain')
|
|
countryConfMemberList = countryConf.get('memberList')
|
|
countryConfMemberListParent = countryConfMemberList.get('parent')
|
|
countryConfMemberListChildName = countryConfMemberList.get('child-name')
|
|
countryConfMemberListChildLink = countryConfMemberList.get('child-link')
|
|
|
|
for n in range(len(tree.xpath(countryConfMemberListParent))):
|
|
|
|
name = tree.xpath(countryConfMemberListParent + '[' + str(n) + ']' + countryConfMemberListChildName)
|
|
link = tree.xpath(countryConfMemberListParent + '[' + str(n) + ']' + countryConfMemberListChildLink)
|
|
|
|
if len(name) > 0:
|
|
|
|
dictionaryMemberList[n] = {}
|
|
dictionaryMemberList[n]['name'] = name[0]
|
|
|
|
if countryDomain in link[0]:
|
|
|
|
dictionaryMemberList[n]['link'] = link[0]
|
|
|
|
if countryDomain not in link[0]:
|
|
|
|
dictionaryMemberList[n]['link'] = countryDomain + link[0]
|
|
|
|
except Exception as e:
|
|
|
|
print('parsing the html did not work. Possibly you first have to downloadMemberListPagesOfCountries(). The original error message is:', e)
|
|
|
|
# save interim results to files
|
|
|
|
f = open('crawlers/output/' + country +'MemberList.txt', 'w+')
|
|
f.write(str(dictionaryMemberList))
|
|
f.close
|
|
|
|
def downloadMemberDataHtmls(self, listOfCountries):
|
|
|
|
for country in listOfCountries:
|
|
|
|
f = open('crawlers/output/' + country +'MemberList.txt')
|
|
text = f.read()
|
|
|
|
dictionaryMemberList = eval(text)
|
|
|
|
|
|
for memberid in dictionaryMemberList:
|
|
|
|
|
|
memberLink = dictionaryMemberList[memberid]['link']
|
|
|
|
# download the html page of the Member
|
|
|
|
response = urllib.request.urlopen(memberLink)
|
|
webContent = response.read().decode('UTF-8')
|
|
|
|
# save interim results to files
|
|
|
|
filename = 'crawlers/pages/' + country + '/' + str(memberid) +'.html'
|
|
|
|
os.makedirs(os.path.dirname(filename), exist_ok=True)
|
|
f = open( filename, 'w+')
|
|
f.write(webContent)
|
|
f.close
|
|
|
|
|
|
def parseMemberData2dictionary(self, listOfCountries):
|
|
|
|
for country in listOfCountries:
|
|
|
|
print('started to parse data of member of ' + country + ' ..')
|
|
|
|
f = open('crawlers/output/' + country +'MemberList.txt')
|
|
text = f.read()
|
|
|
|
dictionaryMemberList = eval(text)
|
|
|
|
|
|
countryConf = self.config.get(country)
|
|
countryDomain = countryConf.get('domain')
|
|
countryConfMember = countryConf.get('member')
|
|
countryConfMemberInfo1 = countryConfMember.get('info-1')
|
|
countryConfMemberInfo1Parent = countryConfMemberInfo1.get('parent')
|
|
countryConfMemberInfo1ChildPoliticalParty = countryConfMemberInfo1.get('child-politicalParty')
|
|
|
|
for memberid in dictionaryMemberList:
|
|
|
|
print('started to parse data of member with name ' + dictionaryMemberList[memberid]['name'] + ' ..')
|
|
|
|
filename = 'crawlers/pages/' + country + '/' + str(memberid) +'.html'
|
|
|
|
tree = lxml.html.soupparser.parse(filename)
|
|
|
|
politicalParty = tree.xpath(countryConfMemberInfo1Parent + countryConfMemberInfo1ChildPoliticalParty)
|
|
|
|
print('oi', politicalParty)
|
|
|
|
if len(politicalParty) > 0:
|
|
|
|
dictionaryMemberList[memberid]['political party'] = politicalParty[0]
|
|
|
|
|
|
|
|
f = open('crawlers/output/' + country +'MemberList.txt', 'w+')
|
|
f.write(str(dictionaryMemberList))
|
|
f.close
|
|
|