added function parseMemberData2dictionary() for first property political party

crawlers/MembersParliamentCrawler.py

@@ -1,5 +1,7 @@
 
 
+import os
+
 import yaml
 import json
 
@@ -74,6 +76,7 @@ class membersParliamentCrawler(object):
                 dictionaryMemberList = {}
                 
                 countryConf = self.config.get(country)
+                countryDomain = countryConf.get('domain')
                 countryConfMemberList = countryConf.get('memberList')
                 countryConfMemberListParent = countryConfMemberList.get('parent')
                 countryConfMemberListChildName = countryConfMemberList.get('child-name')
@@ -86,10 +89,16 @@ class membersParliamentCrawler(object):
                     
                     if len(name) > 0:
                     
-                        dictionaryMemberList[name[0]] = {}
+                        dictionaryMemberList[n] = {}
-                        dictionaryMemberList[name[0]]['name'] = name[0]
+                        dictionaryMemberList[n]['name'] = name[0]
-                        dictionaryMemberList[name[0]]['link'] = link[0]
                         
+                        if countryDomain in link[0]:
+                        
+                            dictionaryMemberList[n]['link'] = link[0]
+                        
+                        if countryDomain not in link[0]:
+                            
+                            dictionaryMemberList[n]['link'] = countryDomain + link[0]
                             
             except Exception as e:
                 
@@ -101,20 +110,74 @@ class membersParliamentCrawler(object):
             f.write(str(dictionaryMemberList))
             f.close
 
-    def parseMemberData2dictionary(self, listOfCountries):
+    def downloadMemberDataHtmls(self, listOfCountries):
         
         for country in listOfCountries:
             
             f = open('output/' + country +'MemberList.txt')
             text = f.read()
             
-            # replace quotes with double quotes because of JSON specification - RFC7159 which would result in error for json.loads function
+            dictionaryMemberList = eval(text)
-            text = text.replace("\'", "\"")
             
-            dictionaryMemberList = json.loads(text)
             
-            for member in dictionaryMemberList:
+            for memberid in dictionaryMemberList:
                 
-                print('oi')
+                
-                print(dictionaryMemberList[member]['link'])
+                memberLink = dictionaryMemberList[memberid]['link']
+                
+                # download the html page of the Member                    
+
+                response = urllib.request.urlopen(memberLink)
+                webContent = response.read().decode('UTF-8')
+
+                # save interim results to files
+                
+                filename = 'pages/' + country + '/' + str(memberid) +'.html'
+                
+                os.makedirs(os.path.dirname(filename), exist_ok=True)
+                f = open( filename, 'w+')
+                f.write(webContent)
+                f.close
+                
+            
+    def parseMemberData2dictionary(self, listOfCountries):
+        
+        for country in listOfCountries:
+            
+            print('started to parse data of member of ' + country + ' ..')
+            
+            f = open('output/' + country +'MemberList.txt')
+            text = f.read()
+            
+            dictionaryMemberList = eval(text)
+            
+            
+            countryConf = self.config.get(country)
+            countryDomain = countryConf.get('domain')
+            countryConfMember = countryConf.get('member')
+            countryConfMemberInfo1 = countryConfMember.get('info-1')
+            countryConfMemberInfo1Parent = countryConfMemberInfo1.get('parent')
+            countryConfMemberInfo1ChildPoliticalParty = countryConfMemberInfo1.get('child-politicalParty')
+        
+            for memberid in dictionaryMemberList:
+                
+                print('started to parse data of member with name ' + dictionaryMemberList[memberid]['name'] + ' ..')
+                
+                filename = 'pages/' + country + '/' + str(memberid) +'.html'
+                
+                tree = lxml.html.soupparser.parse(filename)
+                
+                politicalParty = tree.xpath(countryConfMemberInfo1Parent + countryConfMemberInfo1ChildPoliticalParty)
+                
+                print('oi', politicalParty)
+                
+                if len(politicalParty) > 0:
+                    
+                    dictionaryMemberList[memberid]['political party'] = politicalParty[0]
+                
+                
+                
+            f = open('output/' + country +'MemberList.txt', 'w+')
+            f.write(str(dictionaryMemberList))
+            f.close
         

crawlers/config.yaml

@@ -13,8 +13,8 @@ nicaragua:
     child-link:  '//td//a/@href'
   member:
     info-1:
-      parent:  html/body/form/table
+      parent:  '//html//body//form//table'
-      child-name:  html/body/form/table/tr.0/td.1/span
+      #child-name:  '//html//body//form//table//tr[1]//td[2]//span'
-      child-image:  html/body/form/table/tr.1/td.0/span/img
+      #child-image:  '//html//body//form//table//tr[2]//td[1]//span//img'
-      child-role:  html/body/form/table/tr.1/td.2/span + label.1
+      #child-role:  '//html/body/form/table/tr[2]/td[3]/span + label.1'
-      child-politicalParty:  html/body/form/table/tr.4/td/span
+      child-politicalParty:  '//tr[6]//td//span//text()'

crawlers/useMembersParliamentCrawler.py

@@ -13,4 +13,6 @@ Crawler = membersParliamentCrawler(config)
 
 #Crawler.parseMemberListData2dictionary(listOfCountries)
 
+#Crawler.downloadMemberDataHtmls(listOfCountries)
+
 Crawler.parseMemberData2dictionary(listOfCountries)