|
@ -4,57 +4,57 @@ import yaml |
|
|
import json |
|
|
import json |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class WikidataPEP(object): |
|
|
|
|
|
def __init__(self, configFile): |
|
|
|
|
|
with open(configFile, "r") as stream: |
|
|
|
|
|
|
|
|
class wikidata_PEP(object): |
|
|
|
|
|
def __init__(self, config_file): |
|
|
|
|
|
with open(config_file, "r") as stream: |
|
|
try: |
|
|
try: |
|
|
self.config = yaml.safe_load(stream) |
|
|
self.config = yaml.safe_load(stream) |
|
|
except yaml.YAMLError as exc: |
|
|
except yaml.YAMLError as exc: |
|
|
print(exc) |
|
|
print(exc) |
|
|
|
|
|
|
|
|
def importMembersOfParliamentDict(self, listOfCountries): |
|
|
|
|
|
self.fullDictionaryMemberLists = {} |
|
|
|
|
|
|
|
|
def import_members_of_parliament_dict(self, list_of_countries): |
|
|
|
|
|
self.full_dictionary_member_lists = {} |
|
|
|
|
|
|
|
|
for country in listOfCountries: |
|
|
|
|
|
|
|
|
for country in list_of_countries: |
|
|
print("started to parse data of members of " + country + " ..") |
|
|
print("started to parse data of members of " + country + " ..") |
|
|
|
|
|
|
|
|
f = open("crawlers/output/" + country + "MemberList.txt") |
|
|
f = open("crawlers/output/" + country + "MemberList.txt") |
|
|
text = f.read() |
|
|
text = f.read() |
|
|
|
|
|
|
|
|
self.fullDictionaryMemberLists[country] = eval(text) |
|
|
|
|
|
|
|
|
self.full_dictionary_member_lists[country] = eval(text) |
|
|
|
|
|
|
|
|
# print(self.fullDictionaryMemberLists) |
|
|
|
|
|
|
|
|
# print(self.full_dictionary_member_lists) |
|
|
|
|
|
|
|
|
def checkForEntityIds(self, listOfCountries): |
|
|
|
|
|
|
|
|
def check_for_entity_ids(self, list_of_countries): |
|
|
from wikibaseintegrator import WikibaseIntegrator |
|
|
from wikibaseintegrator import WikibaseIntegrator |
|
|
from wikibaseintegrator import wbi_helpers |
|
|
from wikibaseintegrator import wbi_helpers |
|
|
|
|
|
|
|
|
fullDictionaryMemberLists = self.fullDictionaryMemberLists |
|
|
|
|
|
|
|
|
full_dictionary_member_lists = self.full_dictionary_member_lists |
|
|
|
|
|
|
|
|
for country in listOfCountries: |
|
|
|
|
|
for memberId in fullDictionaryMemberLists[country].keys(): |
|
|
|
|
|
name = fullDictionaryMemberLists[country][memberId]["name"] |
|
|
|
|
|
|
|
|
for country in list_of_countries: |
|
|
|
|
|
for member_id in full_dictionary_member_lists[country].keys(): |
|
|
|
|
|
name = full_dictionary_member_lists[country][member_id]["name"] |
|
|
|
|
|
|
|
|
results = wbi_helpers.search_entities(search_string=name) |
|
|
results = wbi_helpers.search_entities(search_string=name) |
|
|
|
|
|
|
|
|
for entityId in results: |
|
|
|
|
|
|
|
|
for entity_id in results: |
|
|
wbi = WikibaseIntegrator() |
|
|
wbi = WikibaseIntegrator() |
|
|
wikidata_item = wbi.item.get(entity_id=entityId) |
|
|
|
|
|
|
|
|
wikidata_item = wbi.item.get(entity_id=entity_id) |
|
|
|
|
|
|
|
|
for claimkey in wikidata_item.get_json()["claims"].keys(): |
|
|
|
|
|
if claimkey == "P31": |
|
|
|
|
|
|
|
|
for claim_key in wikidata_item.get_json()["claims"].keys(): |
|
|
|
|
|
if claim_key == "P31": |
|
|
if ( |
|
|
if ( |
|
|
wikidata_item.get_json()["claims"][claimkey][0][ |
|
|
|
|
|
|
|
|
wikidata_item.get_json()["claims"][claim_key][0][ |
|
|
"mainsnak" |
|
|
"mainsnak" |
|
|
]["datavalue"]["value"]["id"] |
|
|
]["datavalue"]["value"]["id"] |
|
|
== "Q5" |
|
|
== "Q5" |
|
|
): |
|
|
): |
|
|
print(entityId) |
|
|
|
|
|
|
|
|
print(entity_id) |
|
|
print("---------") |
|
|
print("---------") |
|
|
print(name) |
|
|
print(name) |
|
|
print("is a human") |
|
|
print("is a human") |
|
|
|
|
|
|
|
|
def createMemberOnWikidata(self): |
|
|
|
|
|
|
|
|
def create_member_on_wikidata(self): |
|
|
from wikibaseintegrator import wbi_login, WikibaseIntegrator |
|
|
from wikibaseintegrator import wbi_login, WikibaseIntegrator |
|
|
from wikibaseintegrator.datatypes import ExternalID, Item |
|
|
from wikibaseintegrator.datatypes import ExternalID, Item |
|
|
from wikibaseintegrator.wbi_config import config as wbi_config |
|
|
from wikibaseintegrator.wbi_config import config as wbi_config |
|
@ -67,18 +67,18 @@ class WikidataPEP(object): |
|
|
wbi = WikibaseIntegrator(login=login_instance) |
|
|
wbi = WikibaseIntegrator(login=login_instance) |
|
|
|
|
|
|
|
|
# data type object, e.g. for a NCBI gene entrez ID |
|
|
# data type object, e.g. for a NCBI gene entrez ID |
|
|
isHuman = Item(value="Q5", prop_nr="P31") |
|
|
|
|
|
occupationPolitician = ExternalID(value="Q82955", prop_nr="P106") |
|
|
|
|
|
occupationDeputy = ExternalID(value="Q1055894", prop_nr="P106") |
|
|
|
|
|
|
|
|
is_human = Item(value="Q5", prop_nr="P31") |
|
|
|
|
|
occupation_politician = ExternalID(value="Q82955", prop_nr="P106") |
|
|
|
|
|
occupation_deputy = ExternalID(value="Q1055894", prop_nr="P106") |
|
|
# referenceURL = URL(value='http://legislacion.asamblea.gob.ni/Tablas%20Generales.nsf/InfoDiputado.xsp?documentId=3D4CFDC4B3006D70062587C5007C29E1&action=openDocument&SessionID=1868803A06AB73D50B7F89BD0AB', prop_nr='P106') |
|
|
# referenceURL = URL(value='http://legislacion.asamblea.gob.ni/Tablas%20Generales.nsf/InfoDiputado.xsp?documentId=3D4CFDC4B3006D70062587C5007C29E1&action=openDocument&SessionID=1868803A06AB73D50B7F89BD0AB', prop_nr='P106') |
|
|
|
|
|
|
|
|
# print(isHuman) |
|
|
|
|
|
|
|
|
# print(is_human) |
|
|
# print(referenceURL) |
|
|
# print(referenceURL) |
|
|
|
|
|
|
|
|
# data goes into a list, because many data objects can be provided to |
|
|
# data goes into a list, because many data objects can be provided to |
|
|
data1 = [isHuman] |
|
|
|
|
|
data2 = [occupationDeputy] |
|
|
|
|
|
data3 = [occupationPolitician] |
|
|
|
|
|
|
|
|
data1 = [is_human] |
|
|
|
|
|
data2 = [occupation_deputy] |
|
|
|
|
|
data3 = [occupation_politician] |
|
|
|
|
|
|
|
|
# Create a new item |
|
|
# Create a new item |
|
|
item = wbi.item.new() |
|
|
item = wbi.item.new() |
|
@ -115,12 +115,12 @@ class WikidataPEP(object): |
|
|
wbi = WikibaseIntegrator(login=login_instance) |
|
|
wbi = WikibaseIntegrator(login=login_instance) |
|
|
|
|
|
|
|
|
# data type object, e.g. for a NCBI gene entrez ID |
|
|
# data type object, e.g. for a NCBI gene entrez ID |
|
|
# isHuman = Item(value='Q5', prop_nr='P31') |
|
|
|
|
|
# occupationPolitician = Item(value='Q82955', prop_nr='P106') |
|
|
|
|
|
# occupationDeputy = Item(value='Q1055894', prop_nr='P106') |
|
|
|
|
|
|
|
|
# is_human = Item(value='Q5', prop_nr='P31') |
|
|
|
|
|
# occupation_politician = Item(value='Q82955', prop_nr='P106') |
|
|
|
|
|
# occupation_deputy = Item(value='Q1055894', prop_nr='P106') |
|
|
# referenceURL = ExternalID(value='http://legislacion.asamblea.gob.ni/Tablas%20Generales.nsf/InfoDiputado.xsp?documentId=3D4CFDC4B3006D70062587C5007C29E1&action=openDocument&SessionID=1868803A06AB73D50B7F89BD0AB', prop_nr='P854') |
|
|
# referenceURL = ExternalID(value='http://legislacion.asamblea.gob.ni/Tablas%20Generales.nsf/InfoDiputado.xsp?documentId=3D4CFDC4B3006D70062587C5007C29E1&action=openDocument&SessionID=1868803A06AB73D50B7F89BD0AB', prop_nr='P854') |
|
|
|
|
|
|
|
|
# print(isHuman) |
|
|
|
|
|
|
|
|
# print(is_human) |
|
|
# print(referenceURL) |
|
|
# print(referenceURL) |
|
|
|
|
|
|
|
|
references = [ |
|
|
references = [ |
|
@ -137,12 +137,12 @@ class WikidataPEP(object): |
|
|
] |
|
|
] |
|
|
] |
|
|
] |
|
|
|
|
|
|
|
|
occupationDeputy = Item(value="Q1055894", prop_nr="P106", references=references) |
|
|
|
|
|
|
|
|
occupation_deputy = Item(value="Q1055894", prop_nr="P106", references=references) |
|
|
|
|
|
|
|
|
## data goes into a list, because many data objects can be provided to |
|
|
## data goes into a list, because many data objects can be provided to |
|
|
# data1 = [isHuman] |
|
|
|
|
|
data2 = [occupationDeputy] |
|
|
|
|
|
# data3 = [occupationPolitician] |
|
|
|
|
|
|
|
|
# data1 = [is_human] |
|
|
|
|
|
data2 = [occupation_deputy] |
|
|
|
|
|
# data3 = [occupation_politician] |
|
|
# data4 = [referenceURL] |
|
|
# data4 = [referenceURL] |
|
|
## get item for Qid |
|
|
## get item for Qid |
|
|
item = wbi.item.get(entity_id=Qid) |
|
|
item = wbi.item.get(entity_id=Qid) |