Browse Source

added function parseMemberData2dictionary() for first property political party

master
corsaronero 1 year ago
parent
commit
a05b5bc33f
6 changed files with 82 additions and 17 deletions
  1. BIN
      crawlers/.MembersParliamentCrawler.py.kate-swp
  2. +74
    -11
      crawlers/MembersParliamentCrawler.py
  3. BIN
      crawlers/__pycache__/MembersParliamentCrawler.cpython-310.pyc
  4. +5
    -5
      crawlers/config.yaml
  5. +1
    -1
      crawlers/output/nicaraguaMemberList.txt
  6. +2
    -0
      crawlers/useMembersParliamentCrawler.py

BIN
crawlers/.MembersParliamentCrawler.py.kate-swp View File


+ 74
- 11
crawlers/MembersParliamentCrawler.py View File

@ -1,5 +1,7 @@
import os
import yaml
import json
@ -74,6 +76,7 @@ class membersParliamentCrawler(object):
dictionaryMemberList = {}
countryConf = self.config.get(country)
countryDomain = countryConf.get('domain')
countryConfMemberList = countryConf.get('memberList')
countryConfMemberListParent = countryConfMemberList.get('parent')
countryConfMemberListChildName = countryConfMemberList.get('child-name')
@ -86,11 +89,17 @@ class membersParliamentCrawler(object):
if len(name) > 0:
dictionaryMemberList[name[0]] = {}
dictionaryMemberList[name[0]]['name'] = name[0]
dictionaryMemberList[name[0]]['link'] = link[0]
dictionaryMemberList[n] = {}
dictionaryMemberList[n]['name'] = name[0]
if countryDomain in link[0]:
dictionaryMemberList[n]['link'] = link[0]
if countryDomain not in link[0]:
dictionaryMemberList[n]['link'] = countryDomain + link[0]
except Exception as e:
print('parsing the html did not work. Possibly you first have to downloadMemberListPagesOfCountries(). The original error message is:', e)
@ -101,20 +110,74 @@ class membersParliamentCrawler(object):
f.write(str(dictionaryMemberList))
f.close
def parseMemberData2dictionary(self, listOfCountries):
def downloadMemberDataHtmls(self, listOfCountries):
for country in listOfCountries:
f = open('output/' + country +'MemberList.txt')
text = f.read()
# replace quotes with double quotes because of JSON specification - RFC7159 which would result in error for json.loads function
text = text.replace("\'", "\"")
dictionaryMemberList = eval(text)
dictionaryMemberList = json.loads(text)
for memberid in dictionaryMemberList:
memberLink = dictionaryMemberList[memberid]['link']
# download the html page of the Member
response = urllib.request.urlopen(memberLink)
webContent = response.read().decode('UTF-8')
for member in dictionaryMemberList:
# save interim results to files
filename = 'pages/' + country + '/' + str(memberid) +'.html'
os.makedirs(os.path.dirname(filename), exist_ok=True)
f = open( filename, 'w+')
f.write(webContent)
f.close
print('oi')
print(dictionaryMemberList[member]['link'])
def parseMemberData2dictionary(self, listOfCountries):
for country in listOfCountries:
print('started to parse data of member of ' + country + ' ..')
f = open('output/' + country +'MemberList.txt')
text = f.read()
dictionaryMemberList = eval(text)
countryConf = self.config.get(country)
countryDomain = countryConf.get('domain')
countryConfMember = countryConf.get('member')
countryConfMemberInfo1 = countryConfMember.get('info-1')
countryConfMemberInfo1Parent = countryConfMemberInfo1.get('parent')
countryConfMemberInfo1ChildPoliticalParty = countryConfMemberInfo1.get('child-politicalParty')
for memberid in dictionaryMemberList:
print('started to parse data of member with name ' + dictionaryMemberList[memberid]['name'] + ' ..')
filename = 'pages/' + country + '/' + str(memberid) +'.html'
tree = lxml.html.soupparser.parse(filename)
politicalParty = tree.xpath(countryConfMemberInfo1Parent + countryConfMemberInfo1ChildPoliticalParty)
print('oi', politicalParty)
if len(politicalParty) > 0:
dictionaryMemberList[memberid]['political party'] = politicalParty[0]
f = open('output/' + country +'MemberList.txt', 'w+')
f.write(str(dictionaryMemberList))
f.close

BIN
crawlers/__pycache__/MembersParliamentCrawler.cpython-310.pyc View File


+ 5
- 5
crawlers/config.yaml View File

@ -13,8 +13,8 @@ nicaragua:
child-link: '//td//a/@href'
member:
info-1:
parent: html/body/form/table
child-name: html/body/form/table/tr.0/td.1/span
child-image: html/body/form/table/tr.1/td.0/span/img
child-role: html/body/form/table/tr.1/td.2/span + label.1
child-politicalParty: html/body/form/table/tr.4/td/span
parent: '//html//body//form//table'
#child-name: '//html//body//form//table//tr[1]//td[2]//span'
#child-image: '//html//body//form//table//tr[2]//td[1]//span//img'
#child-role: '//html/body/form/table/tr[2]/td[3]/span + label.1'
child-politicalParty: '//tr[6]//td//span//text()'

+ 1
- 1
crawlers/output/nicaraguaMemberList.txt
File diff suppressed because it is too large
View File


+ 2
- 0
crawlers/useMembersParliamentCrawler.py View File

@ -13,4 +13,6 @@ Crawler = membersParliamentCrawler(config)
#Crawler.parseMemberListData2dictionary(listOfCountries)
#Crawler.downloadMemberDataHtmls(listOfCountries)
Crawler.parseMemberData2dictionary(listOfCountries)

Loading…
Cancel
Save