|
|
@ -1,13 +1,16 @@ |
|
|
|
|
|
|
|
from crawlers.MembersParliamentCrawler import * |
|
|
|
|
|
|
|
|
|
|
|
from wikidata.wdPEP import * |
|
|
|
|
|
|
|
config = 'crawlers/config.yaml' |
|
|
|
listOfCountries = ['nicaragua'] |
|
|
|
|
|
|
|
|
|
|
|
Crawler = membersParliamentCrawler(config) |
|
|
|
|
|
|
|
# doing the crawling of government websites |
|
|
|
|
|
|
|
#Crawler = membersParliamentCrawler(config) |
|
|
|
|
|
|
|
#Crawler.downloadMemberListPagesOfCountries(listOfCountries) |
|
|
|
|
|
|
@ -15,4 +18,10 @@ Crawler = membersParliamentCrawler(config) |
|
|
|
|
|
|
|
#Crawler.downloadMemberDataHtmls(listOfCountries) |
|
|
|
|
|
|
|
Crawler.parseMemberData2dictionary(listOfCountries) |
|
|
|
#Crawler.parseMemberData2dictionary(listOfCountries) |
|
|
|
|
|
|
|
# processing the resulted dictionary and create wikidata queries |
|
|
|
|
|
|
|
wikidataPEP = WikidataPEP(config) |
|
|
|
|
|
|
|
wikidataPEP.importMembersOfParliamentDict(listOfCountries) |