|
|
@ -1,5 +1,7 @@ |
|
|
|
|
|
|
|
|
|
|
|
import os |
|
|
|
|
|
|
|
import yaml |
|
|
|
import json |
|
|
|
|
|
|
@ -74,6 +76,7 @@ class membersParliamentCrawler(object): |
|
|
|
dictionaryMemberList = {} |
|
|
|
|
|
|
|
countryConf = self.config.get(country) |
|
|
|
countryDomain = countryConf.get('domain') |
|
|
|
countryConfMemberList = countryConf.get('memberList') |
|
|
|
countryConfMemberListParent = countryConfMemberList.get('parent') |
|
|
|
countryConfMemberListChildName = countryConfMemberList.get('child-name') |
|
|
@ -86,11 +89,17 @@ class membersParliamentCrawler(object): |
|
|
|
|
|
|
|
if len(name) > 0: |
|
|
|
|
|
|
|
dictionaryMemberList[name[0]] = {} |
|
|
|
dictionaryMemberList[name[0]]['name'] = name[0] |
|
|
|
dictionaryMemberList[name[0]]['link'] = link[0] |
|
|
|
dictionaryMemberList[n] = {} |
|
|
|
dictionaryMemberList[n]['name'] = name[0] |
|
|
|
|
|
|
|
|
|
|
|
if countryDomain in link[0]: |
|
|
|
|
|
|
|
dictionaryMemberList[n]['link'] = link[0] |
|
|
|
|
|
|
|
if countryDomain not in link[0]: |
|
|
|
|
|
|
|
dictionaryMemberList[n]['link'] = countryDomain + link[0] |
|
|
|
|
|
|
|
except Exception as e: |
|
|
|
|
|
|
|
print('parsing the html did not work. Possibly you first have to downloadMemberListPagesOfCountries(). The original error message is:', e) |
|
|
@ -101,20 +110,74 @@ class membersParliamentCrawler(object): |
|
|
|
f.write(str(dictionaryMemberList)) |
|
|
|
f.close |
|
|
|
|
|
|
|
def parseMemberData2dictionary(self, listOfCountries): |
|
|
|
def downloadMemberDataHtmls(self, listOfCountries): |
|
|
|
|
|
|
|
for country in listOfCountries: |
|
|
|
|
|
|
|
f = open('output/' + country +'MemberList.txt') |
|
|
|
text = f.read() |
|
|
|
|
|
|
|
# replace quotes with double quotes because of JSON specification - RFC7159 which would result in error for json.loads function |
|
|
|
text = text.replace("\'", "\"") |
|
|
|
dictionaryMemberList = eval(text) |
|
|
|
|
|
|
|
dictionaryMemberList = json.loads(text) |
|
|
|
|
|
|
|
for memberid in dictionaryMemberList: |
|
|
|
|
|
|
|
|
|
|
|
memberLink = dictionaryMemberList[memberid]['link'] |
|
|
|
|
|
|
|
# download the html page of the Member |
|
|
|
|
|
|
|
response = urllib.request.urlopen(memberLink) |
|
|
|
webContent = response.read().decode('UTF-8') |
|
|
|
|
|
|
|
for member in dictionaryMemberList: |
|
|
|
# save interim results to files |
|
|
|
|
|
|
|
filename = 'pages/' + country + '/' + str(memberid) +'.html' |
|
|
|
|
|
|
|
os.makedirs(os.path.dirname(filename), exist_ok=True) |
|
|
|
f = open( filename, 'w+') |
|
|
|
f.write(webContent) |
|
|
|
f.close |
|
|
|
|
|
|
|
print('oi') |
|
|
|
print(dictionaryMemberList[member]['link']) |
|
|
|
|
|
|
|
def parseMemberData2dictionary(self, listOfCountries): |
|
|
|
|
|
|
|
for country in listOfCountries: |
|
|
|
|
|
|
|
print('started to parse data of member of ' + country + ' ..') |
|
|
|
|
|
|
|
f = open('output/' + country +'MemberList.txt') |
|
|
|
text = f.read() |
|
|
|
|
|
|
|
dictionaryMemberList = eval(text) |
|
|
|
|
|
|
|
|
|
|
|
countryConf = self.config.get(country) |
|
|
|
countryDomain = countryConf.get('domain') |
|
|
|
countryConfMember = countryConf.get('member') |
|
|
|
countryConfMemberInfo1 = countryConfMember.get('info-1') |
|
|
|
countryConfMemberInfo1Parent = countryConfMemberInfo1.get('parent') |
|
|
|
countryConfMemberInfo1ChildPoliticalParty = countryConfMemberInfo1.get('child-politicalParty') |
|
|
|
|
|
|
|
for memberid in dictionaryMemberList: |
|
|
|
|
|
|
|
print('started to parse data of member with name ' + dictionaryMemberList[memberid]['name'] + ' ..') |
|
|
|
|
|
|
|
filename = 'pages/' + country + '/' + str(memberid) +'.html' |
|
|
|
|
|
|
|
tree = lxml.html.soupparser.parse(filename) |
|
|
|
|
|
|
|
politicalParty = tree.xpath(countryConfMemberInfo1Parent + countryConfMemberInfo1ChildPoliticalParty) |
|
|
|
|
|
|
|
print('oi', politicalParty) |
|
|
|
|
|
|
|
if len(politicalParty) > 0: |
|
|
|
|
|
|
|
dictionaryMemberList[memberid]['political party'] = politicalParty[0] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
f = open('output/' + country +'MemberList.txt', 'w+') |
|
|
|
f.write(str(dictionaryMemberList)) |
|
|
|
f.close |
|
|
|
|