You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2223 lines
91 KiB
2223 lines
91 KiB
|
|
# split sentences
|
|
|
|
# in den Listen fehlt noch sondern ( und noch weitere Dinge..)
|
|
|
|
|
|
# Folgende Konjunktionen brauchen keine Satzumformungen:
|
|
# Woraufhin, zudem, zumal, umso - desto,
|
|
|
|
# sondern ist schwierig zu lösen.. am besten mit sondern weg, und anschließend SentGlue
|
|
|
|
|
|
|
|
class SentSeg(object):
|
|
|
|
def __init__(self, language):
|
|
|
|
self.language = language
|
|
|
|
self.punktuation_list = ['.', '?', '!', ';', ':']
|
|
|
|
self.wrappunktuation_list = [',', '-']
|
|
|
|
self.adversativ_list = ['wohingegen', 'Wohingegen', 'aber', 'Aber', 'wobei', 'Wobei', 'hingegen']
|
|
|
|
self.final_list = ['damit','Damit', 'um', 'Um']
|
|
|
|
self.kausal_list = ['weil', 'Weil', 'da', 'Da', 'denn', 'falls', 'Falls' ]
|
|
|
|
self.konditional_list = ['wenn', 'Wenn', 'sobald', 'Sobald', 'als', 'falls']
|
|
|
|
self.konsekutiv_list = ['dass', 'Dass']
|
|
|
|
self.konzessiv_list = ['obwohl', 'Obwohl', 'obgleich', 'Obgleich', 'trotzdem', 'Trotzdem', 'wenngleich', 'doch']
|
|
|
|
self.lokal_list = ['wo', 'Wo']
|
|
|
|
self.temporal_list_vor = ['bevor', 'Bevor']
|
|
|
|
self.temporal_list_nach = ['nachdem', 'Nachdem']
|
|
|
|
self.instrumental_list = ['indem', 'Indem']
|
|
|
|
self.indirectspeech_list = ['ob', 'Ob', 'wann', 'Wann', 'wer', 'Wer', 'wie', 'Wie', 'warum', 'Warum', 'weshalb', 'Weshalb', 'wieso', 'Wieso']
|
|
self.firstwordlist = []
|
|
#self.firstwordlist = ['wann', 'Wann', 'wer', 'Wer', 'wie', 'Wie', 'warum', 'Warum', 'weshalb', 'Weshalb', 'wieso', 'Wieso', 'dies', 'dann', 'jedoch', 'deswegen', 'trotzdem', 'danach', 'davor', 'wenn', 'sobald']
|
|
|
|
self.full_list = self.adversativ_list + self.final_list + self.kausal_list + self.konditional_list + self.konsekutiv_list + self.konzessiv_list + self.lokal_list + self.temporal_list_nach + self.temporal_list_vor + self.instrumental_list + self.indirectspeech_list
|
|
|
|
def ReadDoc2Sent(self, document):
|
|
|
|
splitsentences = []
|
|
splitsentence = []
|
|
|
|
with open(document) as sentences:
|
|
counter = 0
|
|
for sentence in sentences:
|
|
|
|
counter += 1
|
|
if counter % 1000 == 0:
|
|
print(counter)
|
|
|
|
words = sentence.split()
|
|
|
|
|
|
|
|
for word in words:
|
|
|
|
splitsentence.append(word)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if(word[-1] in self.punktuation_list or word in self.punktuation_list) and len(word) > 2:
|
|
|
|
splitsentences.append([splitsentence])
|
|
|
|
splitsentence = []
|
|
|
|
return splitsentences
|
|
|
|
|
|
|
|
|
|
|
|
def AndOrSolver(self, sentences, punctuations):
|
|
|
|
for n in range(len(punctuations)):
|
|
if punctuations[n] == ':' or punctuations[n] == '-':
|
|
punctuations[n] = '.'
|
|
|
|
|
|
#print(sentences, punctuations)
|
|
|
|
splitsentences = []
|
|
|
|
counter = 0
|
|
|
|
newsentences = []
|
|
for sentence in sentences:
|
|
newpunctuationsindexes = []
|
|
utterancenumber = sentence[2]
|
|
commainfo = sentence[1]
|
|
commaornot = commainfo[0]
|
|
sentence = sentence[0]
|
|
|
|
|
|
counter += 1
|
|
doc = self.nlp(' '.join(sentence))
|
|
|
|
subjectcount = 0
|
|
separationwords = []
|
|
subjectcounts = []
|
|
doccounter = 0
|
|
subjectindex = []
|
|
rcornot = 0
|
|
for word in doc:
|
|
doccounter += 1
|
|
if word.dep_ == 'sb' or word.dep_ == 'ep':
|
|
subjectcount += 1
|
|
subjectindex.append(doccounter - 1)
|
|
if word.dep_ == 'rc':
|
|
rcornot = 1
|
|
|
|
|
|
if word.tag_ == '$,':
|
|
|
|
subjectcounts.append([subjectcount, doccounter - 2, subjectindex, rcornot])
|
|
subjectindex = []
|
|
subjectcount = 0
|
|
#print('aleaole',sentence[doccounter - 2])
|
|
if len(sentence[doccounter - 2]) > 1:
|
|
|
|
doccounter -= 1
|
|
|
|
if word.text == 'und' or word.text == 'also' or word.text == 'oder' or word.text == 'schon' or word.text == 'bald' or word.text == 'doch' or word.text == 'jedoch' or word.text == 'sondern':
|
|
separationwords.append(doccounter - 1)
|
|
|
|
#print('separationwords', separationwords)
|
|
#print('subjectcounts', subjectcounts)
|
|
|
|
|
|
|
|
separationwordstocut = []
|
|
listofownsentencessubjectindexes = []
|
|
for n in range(len(subjectcounts) - 1):
|
|
if subjectcounts[n][0] > 0 and subjectcounts[n + 1][0] > 0 and subjectcounts[n + 1][3] == 0:
|
|
listofownsentencessubjectindexes.append(subjectcounts[n])
|
|
for m in range(len(separationwords)):
|
|
if subjectcounts[n][1] < separationwords[m] < subjectcounts[n + 1][1]:
|
|
#print(subjectcounts[n + 1], separationwords[m])
|
|
if subjectcounts[n + 1][0] > 1:
|
|
if subjectcounts[n + 1][2][0] < separationwords[m] <= subjectcounts[n + 1][2][-1]:
|
|
separationwordstocut.append(separationwords[m])
|
|
|
|
processed = 0
|
|
|
|
#print('oioioi')
|
|
#print(listofownsentencessubjectindexes)
|
|
#print(separationwordstocut)
|
|
|
|
if len(listofownsentencessubjectindexes) > 0:
|
|
for n in range(len(listofownsentencessubjectindexes)):
|
|
|
|
sentence[listofownsentencessubjectindexes[n][1]] = sentence[listofownsentencessubjectindexes[n][1]] + 'alohaseparator'
|
|
newpunctuationsindexes.append([punctuations[counter - 1], counter - 1])
|
|
#print('a new punctuation1')
|
|
processed = 1
|
|
if len(separationwordstocut) > 0:
|
|
for n in range(len(separationwordstocut)):
|
|
sentence[separationwordstocut[n] - 1] = sentence[separationwordstocut[n] - 1] + 'alohaseparator'
|
|
#print('a new punctuation2')
|
|
newpunctuationsindexes.append([punctuations[counter - 1], counter - 1])
|
|
processed = 1
|
|
|
|
if processed == 0:
|
|
newsentences.append([sentence])
|
|
|
|
if processed == 1:
|
|
#print(sentence)
|
|
splitsentence = []
|
|
for word in sentence:
|
|
splitsentence.append(word)
|
|
if word[-14:] == 'alohaseparator':
|
|
if splitsentence[-1][-15] == ',':
|
|
splitsentence[-1] = splitsentence[-1][:-15]
|
|
else:
|
|
splitsentence[-1] = splitsentence[-1][:-14]
|
|
newsentences.append([splitsentence])
|
|
splitsentence = []
|
|
newsentences.append([splitsentence])
|
|
|
|
#print(newpunctuationsindexes)
|
|
newpunctuationsindexes = newpunctuationsindexes[::-1]
|
|
for n in range(len(newpunctuationsindexes)):
|
|
punctuations.insert(newpunctuationsindexes[n][1], newpunctuationsindexes[n][0])
|
|
|
|
#print(newsentences, punctuations)
|
|
return newsentences, punctuations
|
|
|
|
|
|
|
|
def LoadBoWModelAndDatabaseOnesZeros(self):
|
|
|
|
|
|
import FASTsearch
|
|
|
|
#print('loading the tag hkl db..')
|
|
self.fsearch1 = FASTsearch.FASTsearch('GS_DB_word.tag_.hkl')
|
|
#print('done')
|
|
|
|
#print('generating BoW Model..')
|
|
self.fsearch1.Gen_BoW_Model(1000, "word")
|
|
#print('done')
|
|
|
|
#print('loading the bow model')
|
|
self.fsearch1.Load_BoW_Model('bagofwordsGS_DB_word.tag_.pkl', 'DataBaseOneZerosGS_DB_word.tag_.hkl')
|
|
#print('done')
|
|
|
|
#print('loading the dep hkl db..')
|
|
self.fsearch2 = FASTsearch.FASTsearch('GS_DB_word.dep_.hkl')
|
|
#print('done')
|
|
|
|
#print('generating BoW Model..')
|
|
self.fsearch2.Gen_BoW_Model(1000, "word")
|
|
#print('done')
|
|
|
|
#print('loading the bow model')
|
|
self.fsearch2.Load_BoW_Model('bagofwordsGS_DB_word.dep_.pkl', 'DataBaseOneZerosGS_DB_word.dep_.hkl')
|
|
#print('done')
|
|
|
|
def LoadSentGlueSGDandGSUtils(self):
|
|
|
|
import GS_Utils
|
|
#print('initializing the gs utils..')
|
|
self.gs = GS_Utils.GS_Utils('de_core_news_sm')
|
|
#print('done')
|
|
|
|
|
|
from SentGlue import SentGlueMach
|
|
#print('loading the Stochastic Gradient models..')
|
|
self.sgm = SentGlueMach('trainedSGD_twolabel.pkl', 'bagofwordstwolabel.pkl')
|
|
#print('done')
|
|
#print('initializing the SGM..')
|
|
self.sgm.initialize()
|
|
#print('done')
|
|
|
|
#print('importing spacy..')
|
|
import spacy
|
|
#print('done')
|
|
|
|
#print('importing german model..')
|
|
self.nlp = spacy.load('de_core_news_sm')
|
|
#print('done')
|
|
|
|
return 'done'
|
|
|
|
def CommaSentenceOrNot(self, sentences):
|
|
|
|
nlp = self.nlp
|
|
|
|
commasentences = []
|
|
counter = 0
|
|
|
|
#print('creating array of comma or not..')
|
|
for sentence in sentences:
|
|
|
|
doc = nlp(' '.join(sentence[0]))
|
|
|
|
#print(doc)
|
|
counter += 1
|
|
#if counter % 100 == 0:
|
|
#print(counter)
|
|
|
|
|
|
n = 0
|
|
firstone = 0
|
|
token = []
|
|
nextword = 0
|
|
for word in doc:
|
|
#print(word.tag_)
|
|
# es eignet sich hierbei word.pos_ fuer noun und verb, word.dep_ fuer sb pd, und evtl tag
|
|
|
|
if firstone == 0:
|
|
token.append(word.text)
|
|
|
|
firstone = 1
|
|
|
|
|
|
if nextword == 1:
|
|
token.append(word.text)
|
|
|
|
nextword = 0
|
|
|
|
if word.tag_ == '$,':
|
|
n += 1
|
|
nextword = 1
|
|
|
|
sentence.append([n, token])
|
|
|
|
commasentences.append(sentence)
|
|
|
|
#print('done')
|
|
return commasentences
|
|
|
|
def EnumerationSolver(self, sentences):
|
|
|
|
|
|
gs = self.gs
|
|
|
|
|
|
nlp = self.nlp
|
|
|
|
sgm = self.sgm
|
|
|
|
|
|
enumerationsentences = []
|
|
counter = 0
|
|
NOTenumerations = []
|
|
#print('processing enumerations..')
|
|
for sentence in sentences:
|
|
|
|
doc = nlp(' '.join(sentence[0]))
|
|
|
|
#print(doc)
|
|
counter += 1
|
|
#if counter % 100 == 0:
|
|
#print(counter)
|
|
|
|
n = 0
|
|
firstone = 0
|
|
token = []
|
|
nextword = 0
|
|
enumeration = False
|
|
|
|
splitsentence = []
|
|
splitsentence_deps = []
|
|
splitsentence_tags = []
|
|
splitsentences = []
|
|
splitsentences_deps = []
|
|
splitsentences_tags = []
|
|
|
|
|
|
|
|
for word in doc:
|
|
#print(word.tag_)
|
|
# es eignet sich hierbei word.pos_ fuer noun und verb, word.dep_ fuer sb pd, und evtl tag
|
|
|
|
|
|
|
|
nextword = 0
|
|
|
|
if word.tag_ == '$,':
|
|
n += 1
|
|
nextword = 1
|
|
|
|
if (word.text == 'und' or word.text == 'oder') and n >= 1:
|
|
enumeration = True
|
|
break
|
|
|
|
|
|
output = []
|
|
if enumeration == True:
|
|
|
|
for word in doc:
|
|
|
|
#print(word.text)
|
|
|
|
if word.text != ',' and word.text != '.' and word.text != 'und':
|
|
|
|
splitsentence.append(word.text)
|
|
splitsentence_deps.append(word.dep_)
|
|
splitsentence_tags.append(word.tag_)
|
|
|
|
if word.text == ',' or word.text == 'und':
|
|
|
|
#print('oi')
|
|
|
|
splitsentences.append(splitsentence)
|
|
splitsentences_deps.append(splitsentence_deps)
|
|
splitsentences_tags.append(splitsentence_tags)
|
|
splitsentence = []
|
|
splitsentence_deps = []
|
|
splitsentence_tags = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
splitsentences_deps.append(splitsentence_deps)
|
|
splitsentences_tags.append(splitsentence_tags)
|
|
|
|
#print( 'splitsentences', splitsentences)
|
|
|
|
token = []
|
|
enumerations = []
|
|
enumerationsSPOs = []
|
|
NOTenumerations = []
|
|
|
|
for sentence in splitsentences:
|
|
token.append(sentence[0])
|
|
|
|
|
|
if sentence[0] not in self.full_list:
|
|
enumerations.append(sentence)
|
|
enumerationsSPOs.append(gs.checkSPO(sentence, 0))
|
|
else:
|
|
NOTenumerations.append(sentence)
|
|
|
|
#print(enumerationsSPOs)
|
|
|
|
|
|
#print('enumerations', enumerations)
|
|
biggest = []
|
|
for i in range(len(enumerationsSPOs)):
|
|
biggest.append([i, sum(enumerationsSPOs[i])])
|
|
|
|
|
|
sortedbiggest = sorted(biggest[::-1], key=lambda tup: tup[1], reverse=True)
|
|
|
|
for i in range(len(sortedbiggest)):
|
|
if sortedbiggest[i][0] == 0:
|
|
mainsentenceIndex = sortedbiggest[i][0]
|
|
lastornot = 0
|
|
break
|
|
|
|
if sortedbiggest[i][0] == len(biggest) - 1:
|
|
mainsentenceIndex = sortedbiggest[i][0]
|
|
lastornot = 1
|
|
break
|
|
|
|
|
|
# Hier muss noch für den Fall Er, sie und der Beamte LACHTEN den Clown aus --> das lachten abgefangen werden mit der Datenbank der Fälle, sprich enumeration im spo 1 0 0 + plural muss dann zu singular werden abhängig von den artikeln.
|
|
#print('enumerations', enumerations)
|
|
mainsentence = enumerations[mainsentenceIndex]
|
|
#print('main', mainsentence)
|
|
probablemainsentences = []
|
|
for i in range(len(enumerations)):
|
|
if i != mainsentenceIndex:
|
|
iprobablemainsentences = []
|
|
probablemainsentence = []
|
|
if lastornot == 0:
|
|
for j in range(1, len(mainsentence)):
|
|
probablemainsentence = mainsentence[0:j] + enumerations[i]
|
|
#print(probablemainsentence)
|
|
iprobablemainsentences.append(' '.join(probablemainsentence))
|
|
if lastornot == 1:
|
|
for j in range(1, len(mainsentence)):
|
|
probablemainsentence = enumerations[i] + mainsentence[-j:]
|
|
iprobablemainsentences.append(' '.join(probablemainsentence))
|
|
probablemainsentences.append(iprobablemainsentences)
|
|
|
|
|
|
# hier wird auf noch da geprüft, aber es ist wichtiger in diesem fall, dass ein tuple nicht zerissen vorkommt AENDERN !!!!
|
|
|
|
#print('probablemainsentences', probablemainsentences)
|
|
tuplesToCheck = []
|
|
tuples = [['ART', 'NN'], ['APPR','NN'], ['ART', 'CARD']]
|
|
for tupl in tuples:
|
|
|
|
|
|
checktupleindex, tupleInWords = gs.checkForAnnotationTuple(mainsentence, tupl , 'word.tag_', 'None')
|
|
if checktupleindex == 2:
|
|
tuplesToCheck.append([tupl, tupleInWords])
|
|
triplesToCheck = []
|
|
triples = [['ART','ADJA','NN'], ['APPR', 'ART', 'NN'], ['KOKOM', 'ART', 'NN']]
|
|
for tripl in triples:
|
|
checktripleindex, tripleInWords = gs.checkForAnnotationTriple(mainsentence, tripl, 'word.tag_', 'None')
|
|
if checktripleindex == 3:
|
|
triplesToCheck.append([tripl, tripleInWords])
|
|
|
|
#print('tuples to check', tuplesToCheck)
|
|
#print('triples to check', triplesToCheck)
|
|
#print('probablemainsentences', probablemainsentences)
|
|
for probsentences in probablemainsentences:
|
|
|
|
checktripleindexes = []
|
|
checktupleindexes = []
|
|
#print(probsentences)
|
|
filteredprobsentences = []
|
|
for sentence in probsentences:
|
|
tuplchecked = 0
|
|
triplchecked = 0
|
|
#print('sentence and tuples to check', sentence, tuplesToCheck)
|
|
for tupl in tuplesToCheck:
|
|
|
|
checkedsecondtime, tupleinWords = gs.checkForAnnotationTuple(sentence.split(), tupl[0], 'word.tag_', tupl[1])
|
|
|
|
#print(sentence, checkedsecondtime)
|
|
if checkedsecondtime == 1:
|
|
|
|
tuplchecked = 0
|
|
if checkedsecondtime == 2:
|
|
|
|
tuplchecked = 1
|
|
|
|
for tripl in triplesToCheck:
|
|
checkedsecondtime, tripleinWords = gs.checkForAnnotationTriple(sentence.split(), tripl[0], 'word.tag_', tripl[1])
|
|
if checkedsecondtime == 1 or checkedsecondtime == 2:
|
|
|
|
triplchecked = 0
|
|
if checkedsecondtime == 3:
|
|
|
|
triplchecked = 1
|
|
|
|
|
|
|
|
|
|
if triplchecked == 1 or tuplchecked == 1:
|
|
filteredprobsentences.append(sentence)
|
|
|
|
#print('filteredprobsentences', filteredprobsentences)
|
|
if len(filteredprobsentences) == 0:
|
|
filteredprobsentences = probsentences
|
|
# here is still the problem, that there are lists of words instead of proper sentences..
|
|
#print('filteredprobsentences', filteredprobsentences)
|
|
probsMatrix = sgm.predictprobsOnSentenceList(filteredprobsentences, filteredprobsentences)
|
|
|
|
#print(probsMatrix)
|
|
|
|
for i in range(len(probsMatrix)):
|
|
probsMatrix[i][0] = i
|
|
|
|
#print(probsMatrix)
|
|
|
|
sortedprobsMatrix = sorted(probsMatrix[::-1], key=lambda tup: tup[1], reverse=True)
|
|
|
|
#print(sortedprobsMatrix)
|
|
|
|
bestindex = sortedprobsMatrix[0][0]
|
|
|
|
#print(bestindex)
|
|
#print('probablemainsentences', filteredprobsentences)
|
|
probablemainsentence = filteredprobsentences[int(bestindex)]
|
|
#print('oi', probablemainsentence)
|
|
|
|
#print('probablemainsentence', probablemainsentence)
|
|
enumerationsentences.append([probablemainsentence])
|
|
|
|
|
|
enumerationsentences.append([' '.join(mainsentence)])
|
|
|
|
for notenum in NOTenumerations:
|
|
#print(enumerationsentences)
|
|
#print(enumerationsentences[-1])
|
|
#print('enum no1', enumerationsentences)
|
|
#print('notenum', notenum)
|
|
enumerationsentences[-1].append(' '.join(notenum))
|
|
#print('enumsentences',enumerationsentences[-1])
|
|
enumerationsentences[-1] = [', '.join(enumerationsentences[-1])]
|
|
|
|
|
|
else:
|
|
enumerationsentences.append([sentence])
|
|
|
|
|
|
|
|
output.append(enumerationsentences)
|
|
|
|
|
|
for n in range(len(output[0])):
|
|
#print('out',output[0][n])
|
|
try:
|
|
output[0][n] = [output[0][n][0].split()]
|
|
except:
|
|
output[0][n] = [output[0][n][0][0]]
|
|
|
|
|
|
#print('done')
|
|
return output[0]
|
|
|
|
|
|
def GetUtteranceNumber(self, sentences):
|
|
|
|
nlp = self.nlp
|
|
|
|
uttersentences = []
|
|
|
|
for sentence in sentences:
|
|
|
|
doc = nlp(' '.join(sentence[0]))
|
|
|
|
subjectcount = 0
|
|
|
|
for word in doc:
|
|
|
|
if word.dep_ == 'sb' or word.dep_ == 'ep':
|
|
subjectcount += 1
|
|
|
|
sentence.append(subjectcount)
|
|
uttersentences.append(sentence)
|
|
|
|
return uttersentences
|
|
|
|
def GetQuestionOrNot(self, sentences):
|
|
|
|
nlp = self.nlp
|
|
|
|
uttersentences = []
|
|
questionmark = 0
|
|
for sentence in sentences:
|
|
|
|
doc = nlp(' '.join(sentence[0]))
|
|
|
|
|
|
count = 0
|
|
for word in doc:
|
|
|
|
|
|
count += 1
|
|
|
|
if word.text == '?':
|
|
questionmark = 1
|
|
|
|
sentence.append(questionmark)
|
|
uttersentences.append(sentence)
|
|
|
|
return uttersentences
|
|
|
|
def SplitSentencesIntoHauptNebenTuple(self, sentences, punctuations):
|
|
|
|
|
|
|
|
oldsplitsentences = []
|
|
#print('hauptneben inputsentences', sentences)
|
|
|
|
gs = self.gs
|
|
|
|
#print('importing spacy..')
|
|
import spacy
|
|
#print('done')
|
|
|
|
nlp = self.nlp
|
|
|
|
outputsentences = []
|
|
sentencesThatAreOutoutput = []
|
|
outsentences = []
|
|
for generalindex in range(len(sentences)):
|
|
presentence = sentences[generalindex]
|
|
|
|
splitsentence = []
|
|
splitsentence_deps = []
|
|
splitsentence_tags = []
|
|
splitsentences = []
|
|
splitsentences_deps = []
|
|
splitsentences_tags = []
|
|
commainfo = presentence[1]
|
|
outputsentence = []
|
|
|
|
|
|
token = commainfo[1]
|
|
|
|
commaornot = commainfo[0]
|
|
|
|
numberutterances = presentence[2]
|
|
|
|
sentence = presentence[0]
|
|
|
|
oldsentence = presentence[0]
|
|
|
|
#print(commaornot)
|
|
if commaornot >= 2:
|
|
#print('nla')
|
|
|
|
sentence[0] = sentence[0].title()
|
|
|
|
doc = nlp(' '.join(sentence))
|
|
|
|
|
|
for word in doc:
|
|
|
|
#print(word.text)
|
|
|
|
if word.text != ',' and word.text != '.':
|
|
|
|
splitsentence.append(word.text)
|
|
splitsentence_deps.append(word.dep_)
|
|
splitsentence_tags.append(word.tag_)
|
|
|
|
if word.text == ',':
|
|
|
|
#print('oi')
|
|
|
|
splitsentences.append(splitsentence)
|
|
splitsentences_deps.append(splitsentence_deps)
|
|
splitsentences_tags.append(splitsentence_tags)
|
|
splitsentence = []
|
|
splitsentence_deps = []
|
|
splitsentence_tags = []
|
|
|
|
|
|
splitsentences.append(splitsentence)
|
|
splitsentences[0][0] = splitsentences[0][0].lower()
|
|
splitsentences_deps.append(splitsentence_deps)
|
|
splitsentences_tags.append(splitsentence_tags)
|
|
oldsplitsentences = splitsentences
|
|
#print(splitsentences)
|
|
#print(splitsentences_tags)
|
|
#print(splitsentences_deps)
|
|
spo = []
|
|
|
|
for n in range(len(splitsentences)):
|
|
prespo = []
|
|
prespo = gs.checkSPO(splitsentences_deps[n], 1)
|
|
prespo.append( gs.checkForAnnotation(splitsentences[n], 'VVINF', 'word.tag_'))
|
|
prespo.append(gs.checkForAnnotation(splitsentences[n], 'VAFIN', 'word.tag_'))
|
|
prespo.append(gs.checkForAnnotation(splitsentences[n], 'VVFIN', 'word.tag_'))
|
|
prespo.append(gs.checkForAnnotation(splitsentences[n], 'VMFIN', 'word.tag_'))
|
|
|
|
|
|
|
|
|
|
|
|
spo.append(prespo)
|
|
#print(splitsentences_deps)
|
|
#print(splitsentences)
|
|
#print(spo)
|
|
|
|
indexSPO = []
|
|
lastm = len(splitsentences)
|
|
for o in range(len(splitsentences)):
|
|
|
|
m = len(splitsentences) - 1 - o
|
|
for n in range(len(splitsentences)):
|
|
|
|
|
|
|
|
if m < n - 1 and n < lastm:
|
|
|
|
#print('spo s',spo[m], spo[n])
|
|
sb = spo[m][0] + spo[n][0]
|
|
Vafin = 1
|
|
if spo[m][3] == 1 or spo[n][3] == 1:
|
|
Vafin = spo[m][3] + spo[n][3]
|
|
Vvinf = 1
|
|
if spo[m][4] == 1 or spo[n][4] == 1:
|
|
Vvinf = spo[m][4] + spo[n][4]
|
|
Vvfin = 1
|
|
if spo[m][5] == 1 or spo[n][5] == 1:
|
|
Vvfin = spo[m][5] + spo[n][5]
|
|
Vmfin = 1
|
|
if spo[m][6] == 1 or spo[n][6] == 1:
|
|
Vmfin == spo[m][6] + spo[n][6]
|
|
#wrapped = 0
|
|
#for n in range(len(indexSPO)):
|
|
#if n == indexSPO[n][0] + 1 and n == indexSPO[n][1] - 1:
|
|
#wrapped = 1
|
|
#print(sb, Vafin, Vvinf, Vvfin, Vmfin, 'm n', m, n)
|
|
if sb == 1 and Vafin == 1 and Vvinf == 1 and (Vvfin == 1 or Vmfin == 1):
|
|
|
|
indexSPO.append([m,n])
|
|
#print([m,n])
|
|
lastm = m
|
|
#print('lastm',lastm)
|
|
|
|
|
|
|
|
#print(splitsentences)
|
|
Hauptsentences = []
|
|
for n in range(len(indexSPO)):
|
|
if indexSPO[n][0] > indexSPO[n][1]:
|
|
i = 1
|
|
j = 0
|
|
else:
|
|
i = 0
|
|
j = 1
|
|
Hauptsentences.append([splitsentences[indexSPO[n][i]] + splitsentences[indexSPO[n][j]] , indexSPO[n][i], indexSPO[n][j] ])
|
|
|
|
HauptSentences = []
|
|
for n in range(len(Hauptsentences)):
|
|
m = len(Hauptsentences) - 1 - n
|
|
HauptSentences.append(Hauptsentences[m])
|
|
|
|
#print('Hauptsentences', Hauptsentences)
|
|
#print('HauptSentences', HauptSentences)
|
|
sentencesThatAreOut =[]
|
|
|
|
for n in range(len(HauptSentences)):
|
|
index = HauptSentences[n][1]
|
|
finish = 0
|
|
#print('Oi',HauptSentences[n])
|
|
if n == len(HauptSentences) - 1:
|
|
|
|
#print('lenHauptsentences', len(HauptSentences))
|
|
|
|
stopindex = len(splitsentences)
|
|
finish = 1
|
|
else:
|
|
stopindex = HauptSentences[n + 1][1]
|
|
#print('stopindex', stopindex)
|
|
vvfinisthere = 0
|
|
if finish == 0:
|
|
if splitsentences_tags[stopindex][0] == 'VVFIN':
|
|
stopindex -= 1
|
|
vvfinisthere = 1
|
|
|
|
if splitsentences_tags[index][0] == 'VVFIN':
|
|
vvfinisthere = 1
|
|
|
|
if vvfinisthere == 1:
|
|
|
|
|
|
HNTuple = HauptSentences[n][0] + [','] + splitsentences[index - 1]
|
|
outputsentence.append(HNTuple)
|
|
sentencesThatAreOut.append(index - 1)
|
|
sentencesThatAreOut.append(Hauptsentences[n][1])
|
|
sentencesThatAreOut.append(Hauptsentences[n][2])
|
|
|
|
for m in range(index + 1, stopindex ):
|
|
if m != HauptSentences[n][2]:
|
|
HNTuple = HauptSentences[n][0] + [','] + splitsentences[m]
|
|
#print('check', HauptSentences[n], n)
|
|
#print('check', splitsentences[m], m)
|
|
#print('double', HNTuple)
|
|
outputsentence.append(HNTuple)
|
|
|
|
|
|
|
|
sentencesThatAreOut.append(m)
|
|
sentencesThatAreOut.append(Hauptsentences[n][1])
|
|
sentencesThatAreOut.append(Hauptsentences[n][2])
|
|
|
|
sentencesThatAreOutoutput.append(sentencesThatAreOut)
|
|
|
|
|
|
cpOrNots = []
|
|
rcOrNots = []
|
|
for splitsentence in splitsentences_deps:
|
|
cpOrNot = gs.checkForAnnotationInTokenizedSentence(splitsentence, 'cp')
|
|
cpOrNots.append(cpOrNot)
|
|
rcOrNot = gs.checkForAnnotationInTokenizedSentence(splitsentence, 'rc')
|
|
rcOrNots.append(rcOrNot)
|
|
|
|
#print('Laenge splitsentences', len(splitsentences))
|
|
#print('laenge cpOrNots', len(cpOrNots))
|
|
#print(cpOrNots)
|
|
#print('rc or nots', rcOrNots)
|
|
pairs = []
|
|
for n in range(len(cpOrNots)):
|
|
index = len(cpOrNots) - 1 - n
|
|
done = 0
|
|
if rcOrNots[index] == 1:
|
|
pairs.append([index, index - 1])
|
|
done = 1
|
|
|
|
|
|
if done == 0 and cpOrNots[index] == 1:
|
|
try:
|
|
if splitsentences_tags[index + 1][0] == 'VVFIN':
|
|
pairs.append([index, index + 1])
|
|
done = 1
|
|
except:
|
|
pass
|
|
try:
|
|
if done == 0 and rcOrNots[index - 1] == 0:
|
|
pairs.append([index, index - 1])
|
|
done = 1
|
|
except:
|
|
pass
|
|
try:
|
|
if done == 0 and rcOrNots[index - 1] == 1:
|
|
if rcOrNots[index - 2] == 0:
|
|
pairs.append([index, index - 2])
|
|
except:
|
|
pass
|
|
|
|
for pair in pairs[::-1]:
|
|
if pair[0] not in set(sentencesThatAreOut) or pair[1] not in set(sentencesThatAreOut):
|
|
outputsentence.append(splitsentences[pair[1]] + [','] + splitsentences[pair[0]])
|
|
#print('hnhn',sentences)
|
|
sentences[generalindex][0] = outputsentence
|
|
|
|
#print('outputsentence hntuple',outputsentence)
|
|
#outputsentences.append([outputsentence , i])
|
|
|
|
#print('Oio', outputsentences)
|
|
#print(sentencesThatAreOutoutput)
|
|
#print(splitsentences)
|
|
#print('oioioioioioioio',sentences)
|
|
|
|
#print(sentences[0][0])
|
|
|
|
|
|
#print('oioi',sentences[n])
|
|
#print('malatesta', sentences[n][0][0])
|
|
#print('generalindex sentences index 0', sentences[generalindex][0])
|
|
try:
|
|
if type(sentences[generalindex][0][0]) == str:
|
|
sentences[generalindex][0] = [sentences[generalindex][0]]
|
|
except:
|
|
pass
|
|
#print('generalindex sentences index 0', sentences[generalindex][0])
|
|
#print('oldsentence', oldsentence)
|
|
newgeneratedsentences = len(sentences[generalindex][0])
|
|
if newgeneratedsentences > 1:
|
|
#print('goti t')
|
|
for sentence in sentences[generalindex][0]:
|
|
punctuations.insert(generalindex, punctuations[generalindex])
|
|
outsentences.append(sentence)
|
|
del punctuations[generalindex]
|
|
if newgeneratedsentences == 1:
|
|
if len(sentences[generalindex][0][0]) > 1:
|
|
outsentences.append(sentences[generalindex][0][0])
|
|
else:
|
|
outsentences.append(oldsentence)
|
|
if newgeneratedsentences == 0:
|
|
#print('case oldsentence', oldsentence)
|
|
outsentences.append(oldsentence)
|
|
#print('oioi', sentences[n])
|
|
# connect alonestanding commatas with the word before
|
|
#print('theoutsentences', outsentences)
|
|
for outsentence in outsentences:
|
|
todelete = []
|
|
for n in range(len(outsentence)):
|
|
if outsentence[n] == ',':
|
|
todelete.append(n)
|
|
outsentence[n-1] = outsentence[n-1] + ','
|
|
for deleteindex in todelete[::-1]:
|
|
del outsentence[deleteindex]
|
|
|
|
for index in range(len(outsentences)):
|
|
outsentences[index] = [outsentences[index]]
|
|
#print('theoutsentences', outsentences)
|
|
|
|
#removing doubles
|
|
doubledsentences = []
|
|
for o in range(len(outsentences)):
|
|
sentence = outsentences[o][0]
|
|
for m in range(len(outsentences)):
|
|
if m != o:
|
|
count = 0
|
|
for n in range(len(sentence)):
|
|
if sentence[n] in outsentences[m][0] or sentence[n][:-1] in outsentences[m][0]:
|
|
count += 1
|
|
if count == len(sentence):
|
|
doubledsentences.append(sentence)
|
|
punctdeleteindex = []
|
|
tmp = set()
|
|
for sentence in doubledsentences:
|
|
tmp.add(tuple(sentence))
|
|
#print(list(tmp))
|
|
doubledsentences = []
|
|
for tup in tmp:
|
|
doubledsentences.append([list(tup)])
|
|
#print('doubledsentences',doubledsentences)
|
|
punctdeleteindexes = []
|
|
for double in doubledsentences:
|
|
if double in outsentences:
|
|
punctdeleteindex = outsentences[::-1].index(double)
|
|
del outsentences[len(outsentences) - 1 - punctdeleteindex]
|
|
punctdeleteindexes.append(punctdeleteindex)
|
|
|
|
for index in punctdeleteindexes[::-1]:
|
|
del punctuations[len(outsentences) - 1 - index]
|
|
|
|
#print('oldsplit',oldsplitsentences)
|
|
#print('outsents',outsentences)
|
|
|
|
for o in range(len(oldsplitsentences)):
|
|
for m in range(len(outsentences)):
|
|
counter = 0
|
|
for n in range(len(oldsplitsentences[o])):
|
|
if oldsplitsentences[o][n] in outsentences[m][0] or oldsplitsentences[o][n] + ',' in outsentences[m][0]:
|
|
counter += 1
|
|
if counter >= len(oldsplitsentences[o]):
|
|
break
|
|
if m == len(outsentences) - 1 and counter < len(oldsplitsentences[o]):
|
|
if o == 0:
|
|
outsentences.insert(0,[oldsplitsentences[o]])
|
|
punctuations.insert(0, punctuations[0])
|
|
else:
|
|
newones = []
|
|
for i in range(len(outsentences)):
|
|
if outsentences[i][0][-1] == oldsplitsentences[o - 1][-1]:
|
|
if len(outsentences[i][0]) > 2 and len(oldsplitsentences[o - 1]) > 2:
|
|
if outsentences[i][0][-2] == oldsplitsentences[o - 1][-2]:
|
|
if outsentences[i][0][-3] == oldsplitsentences[o - 1][-3]:
|
|
newones.append([i + 1, [oldsplitsentences[o]]])
|
|
for newone in newones[::-1]:
|
|
#print(newones)
|
|
outsentences.insert(newone[0], newone[1])
|
|
punctuations.insert(newone[0], punctuations[newone[0] - 1])
|
|
|
|
|
|
|
|
|
|
|
|
#print('outsentences at the very end ', outsentences, punctuations)
|
|
return outsentences, punctuations
|
|
|
|
|
|
# Notiz: Hier muss der Input immer Paare sein, von Hauptsatz/Nebensatz. D.h. eine weitere vorgeschaltete Klasse ist von Nöten.
|
|
|
|
def SplitCommatas(self, Inputsentences, punctuations):
|
|
|
|
gs = self.gs
|
|
|
|
nlp = self.nlp
|
|
|
|
gramcorr_splitsentences = []
|
|
counter = 0
|
|
newpunctuationsindex = []
|
|
for Inputsentence in Inputsentences:
|
|
|
|
counter += 1
|
|
|
|
|
|
commainfo = Inputsentence[1]
|
|
|
|
|
|
token = commainfo[1]
|
|
|
|
commaornot = commainfo[0]
|
|
|
|
numberutterances = Inputsentence[2]
|
|
|
|
|
|
if commaornot == 0:
|
|
gramcorr_splitsentences.append(Inputsentence[0])
|
|
|
|
if commaornot > 1:
|
|
gramcorr_splitsentences.append(Inputsentence[0])
|
|
|
|
if commaornot == 1:
|
|
oldsentence = Inputsentence[0]
|
|
Inputsentence = [[Inputsentence[0]]]
|
|
|
|
|
|
|
|
|
|
for sentence in Inputsentence[0]:
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences = []
|
|
|
|
|
|
|
|
|
|
processed = 0
|
|
wasNotInAnyList = 0
|
|
try:
|
|
for n in range(len(token)):
|
|
|
|
if token[n] in self.final_list:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
if word != token[n]:
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
|
|
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
if n == 1:
|
|
|
|
|
|
if token[n] == 'um' or token[n] == 'Um':
|
|
|
|
splitsentences[n].insert(0,'dies')
|
|
splitsentences[n].insert(0,'um')
|
|
else:
|
|
splitsentences[n].insert(0,'dann')
|
|
|
|
|
|
|
|
if n == 0:
|
|
|
|
if token[n] == 'um' or token[n] == 'Um':
|
|
splitsentences[n].insert(0,'dies')
|
|
splitsentences[n].insert(0,'um')
|
|
splitsentences = splitsentences[::-1]
|
|
else:
|
|
splitsentences[n].insert(0,'dann')
|
|
|
|
splitsentences[0], splitsentences[1] = splitsentences[1] , splitsentences[0]
|
|
|
|
generalrules = [['ADV','VAFIN'], ['ADV', 'VVFIN']]
|
|
processed = 1
|
|
|
|
if token[n] in self.adversativ_list:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
if word != token[n]:
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentences[n].append('jedoch')
|
|
|
|
|
|
generalrules = [['ADV','VAFIN'], ['ADV', 'VVFIN']]
|
|
processed = 1
|
|
|
|
if token[n] in self.kausal_list:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
if word != token[n]:
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
# Da deswegen an den anderen Satz gehaengt wird, muss der input zu commasentences immer ZWEI sentences sein.
|
|
#print('splitsentences in kausal', splitsentences)
|
|
if n == 1:
|
|
splitsentences[n - 1].insert(0,'deswegen')
|
|
splitsentences[0], splitsentences[1] = splitsentences[1] , splitsentences[0]
|
|
|
|
|
|
|
|
if n == 0:
|
|
splitsentences[n + 1].insert(0,'deswegen')
|
|
|
|
|
|
|
|
|
|
#print('splitsentences in kausal', splitsentences)
|
|
|
|
|
|
generalrules = [['PROAV','VAFIN'], ['PROAV', 'VVFIN']]
|
|
processed = 1
|
|
|
|
# from here come konsekutiv sentences, they have to be split according https://www.deutschplus.net/pages/Konsekutivsatz
|
|
if token[n] in self.konsekutiv_list:
|
|
#print('oi konsekutiv')
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
if word != token[n]:
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
generalrules = [['KOUS','PPER']]
|
|
processed = 1
|
|
|
|
|
|
if token[n] in self.konditional_list:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
|
|
if n == 1:
|
|
|
|
spoCount = gs.checkSPO(splitsentences[n], 0)
|
|
|
|
spoCount = sum(spoCount)
|
|
|
|
if spoCount == 2:
|
|
thereisanes = 0
|
|
for word in splitsentences[n]:
|
|
if word == 'es' or word == 'Es':
|
|
thereisanes = 1
|
|
if thereisanes == 0:
|
|
splitsentences[n].append('es')
|
|
|
|
|
|
if n == 0:
|
|
|
|
|
|
spoCount = gs.checkSPO(splitsentences[n], 0)
|
|
|
|
spoCount = sum(spoCount)
|
|
|
|
if spoCount == 2:
|
|
|
|
thereisanes = 0
|
|
for word in splitsentences[n]:
|
|
if word == 'es' or word == 'Es':
|
|
thereisanes = 1
|
|
if thereisanes == 0:
|
|
splitsentences[n].append('es')
|
|
|
|
splitsentences[0], splitsentences[1] = splitsentences[1] , splitsentences[0]
|
|
|
|
generalrules = [['KOUS','PPER']]
|
|
processed = 1
|
|
|
|
if token[n] in self.konzessiv_list:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
if word != token[n]:
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
|
|
if n == 1:
|
|
splitsentences[n - 1].insert(0,'trotzdem')
|
|
splitsentences[0], splitsentences[1] = splitsentences[1] , splitsentences[0]
|
|
|
|
|
|
|
|
if n == 0:
|
|
splitsentences[n + 1].insert(0,'trotzdem')
|
|
|
|
|
|
generalrules = [['PROAV','VAFIN'], ['PROAV', 'VVFIN']]
|
|
processed = 1
|
|
|
|
if token[n] in self.lokal_list:
|
|
#print('lokal ole ole ')
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
if word != token[n]:
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
|
|
if n == 1:
|
|
splitsentences[n - 1].insert(0,'dort')
|
|
splitsentences[0], splitsentences[1] = splitsentences[1] , splitsentences[0]
|
|
|
|
|
|
|
|
if n == 0:
|
|
splitsentences[n + 1].insert(0,'dort')
|
|
|
|
|
|
generalrules = [['PROAV','VAFIN'], ['PROAV', 'VVFIN']]
|
|
processed = 1
|
|
|
|
if token[n] in self.instrumental_list:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
if word != token[n]:
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
|
|
if n == 1:
|
|
splitsentences[n - 1].insert(0,'so')
|
|
splitsentences[0], splitsentences[1] = splitsentences[1] , splitsentences[0]
|
|
|
|
|
|
|
|
if n == 0:
|
|
splitsentences[n + 1].insert(0,'so')
|
|
|
|
|
|
generalrules = [['ADV','VAFIN'], ['ADV', 'VVFIN']]
|
|
processed = 1
|
|
|
|
if token[n] in self.temporal_list_vor:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
if word != token[n]:
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
|
|
if n == 1:
|
|
splitsentences[n].insert(0,'danach')
|
|
|
|
|
|
|
|
|
|
if n == 0:
|
|
splitsentences[n].insert(0,'danach')
|
|
splitsentences[0], splitsentences[1] = splitsentences[1] , splitsentences[0]
|
|
|
|
generalrules = [['PROAV','VAFIN'], ['PROAV', 'VVFIN']]
|
|
processed = 1
|
|
|
|
if token[n] in self.temporal_list_nach:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
if word != token[n]:
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
|
|
if n == 1:
|
|
splitsentences[n].insert(0,'davor')
|
|
|
|
|
|
|
|
|
|
if n == 0:
|
|
splitsentences[n].insert(0,'davor')
|
|
splitsentences[0], splitsentences[1] = splitsentences[1] , splitsentences[0]
|
|
|
|
generalrules = [['PROAV','VAFIN'], ['PROAV', 'VVFIN']]
|
|
processed = 1
|
|
|
|
#print(token[n])
|
|
if token[n] == 'der' or token[n] == 'welcher':
|
|
|
|
tokens = self.nlp(' '.join(sentence))
|
|
for word in tokens:
|
|
if word.dep_ == 'rc':
|
|
wordwithrc = word.text
|
|
|
|
rcORnot = gs.checkForAnnotation(sentence, 'rc', 'word.dep_')
|
|
|
|
oldsplitsentences = splitsentences
|
|
splitsentences = []
|
|
|
|
if rcORnot == 1:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
# das umtauschen wird hier vollzogen, da ansonsten spacy dieser nicht als PDS einliest.. analog in den anderen.
|
|
|
|
if wordwithrc in splitsentences[n]:
|
|
|
|
splitsentences[n][0] = 'dieser'
|
|
|
|
verb = splitsentences[n][-1]
|
|
|
|
splitsentences[n] = splitsentences[n][:-1]
|
|
splitsentences[n].insert(1, verb)
|
|
|
|
|
|
#print('Vorsicht', splitsentences)
|
|
|
|
generalrules = [['PDS','VAFIN'], ['PDS', 'VVFIN']]
|
|
processed = 1
|
|
else:
|
|
splitsentences = oldsplitsentences
|
|
splitsentence = []
|
|
|
|
if token[n] == 'die' or token[n] == 'welche':
|
|
|
|
|
|
tokens = self.nlp(' '.join(sentence))
|
|
for word in tokens:
|
|
if word.dep_ == 'rc':
|
|
wordwithrc = word.text
|
|
|
|
|
|
|
|
rcORnot = gs.checkForAnnotation(sentence, 'rc', 'word.dep_')
|
|
|
|
|
|
oldsplitsentences = splitsentences
|
|
splitsentences = []
|
|
|
|
|
|
if rcORnot == 1:
|
|
#print('it went to rcornot in case die')
|
|
|
|
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
if wordwithrc in splitsentences[n]:
|
|
#print('wordwithrc was in sentence')
|
|
#print(wordwithrc)
|
|
#print(splitsentences[n])
|
|
#print('wordwithrcend')
|
|
splitsentences[n][0] = 'diese'
|
|
|
|
verb = splitsentences[n][-1]
|
|
|
|
splitsentences[n] = splitsentences[n][:-1]
|
|
splitsentences[n].insert(1, verb)
|
|
|
|
|
|
|
|
generalrules = [['PDS','VAFIN'], ['PDS', 'VVFIN']]
|
|
processed = 1
|
|
else:
|
|
|
|
splitsentences = oldsplitsentences
|
|
splitsentence = []
|
|
|
|
if token[n] == 'dem':
|
|
|
|
tokens = self.nlp(' '.join(sentence))
|
|
for word in tokens:
|
|
if word.dep_ == 'rc':
|
|
wordwithrc = word.text
|
|
|
|
rcORnot = gs.checkForAnnotation(sentence, 'rc', 'word.dep_')
|
|
|
|
oldsplitsentences = splitsentences
|
|
splitsentences = []
|
|
|
|
if rcORnot == 1:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',' and word[-1] != '.':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
|
|
if wordwithrc in splitsentences[n]:
|
|
|
|
splitsentences[n][0] = 'diesem'
|
|
|
|
verb = splitsentences[n][-1]
|
|
|
|
splitsentences[n] = splitsentences[n][:-1]
|
|
splitsentences[n].insert(1, verb)
|
|
|
|
|
|
|
|
generalrules = [['PDS','VAFIN'], ['PDS', 'VVFIN']]
|
|
processed = 1
|
|
else:
|
|
splitsentences = oldsplitsentences
|
|
splitsentence = []
|
|
|
|
if token[n] == 'das' or token[n] == 'welches':
|
|
|
|
tokens = self.nlp(' '.join(sentence))
|
|
for word in tokens:
|
|
if word.dep_ == 'rc':
|
|
wordwithrc = word.text
|
|
|
|
|
|
rcORnot = gs.checkForAnnotation(sentence, 'rc', 'word.dep_')
|
|
|
|
#print('Oeeee',rcORnot)
|
|
oldsplitsentences = splitsentences
|
|
splitsentences = []
|
|
if rcORnot == 1:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
#print('splitsentence in das rc', splitsentences)
|
|
if wordwithrc in splitsentences[n]:
|
|
|
|
splitsentences[n][0] = 'dieses'
|
|
|
|
verb = splitsentences[n][-1]
|
|
#print('verb',verb)
|
|
splitsentences[n] = splitsentences[n][:-1]
|
|
splitsentences[n].insert(1, verb)
|
|
|
|
generalrules = [['PDS','VAFIN'], ['PDS', 'VVFIN']]
|
|
processed = 1
|
|
else:
|
|
splitsentences = oldsplitsentences
|
|
splitsentence = []
|
|
|
|
if token[n] == 'dessen' or token[n] == 'wessen':
|
|
|
|
tokens = self.nlp(' '.join(sentence))
|
|
for word in tokens:
|
|
if word.dep_ == 'rc':
|
|
wordwithrc = word.text
|
|
|
|
|
|
rcORnot = gs.checkForAnnotation(sentence, 'rc', 'word.dep_')
|
|
|
|
oldsplitsentences = splitsentences
|
|
splitsentences = []
|
|
|
|
if rcORnot == 1:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
|
|
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
if wordwithrc in splitsentences[n]:
|
|
verb = splitsentences[n][-1]
|
|
|
|
splitsentences[n] = splitsentences[n][:-1]
|
|
splitsentences[n].insert(1, verb)
|
|
|
|
|
|
|
|
generalrules = [['PDS','VAFIN'], ['PDS', 'VVFIN']]
|
|
processed = 1
|
|
else:
|
|
splitsentences = oldsplitsentences
|
|
splitsentence = []
|
|
|
|
if token[n] == 'den' or token[n] == 'welchen':
|
|
|
|
tokens = self.nlp(' '.join(sentence))
|
|
for word in tokens:
|
|
if word.dep_ == 'rc':
|
|
wordwithrc = word.text
|
|
|
|
|
|
rcORnot = gs.checkForAnnotation(sentence, 'rc', 'word.dep_')
|
|
|
|
oldsplitsentences = splitsentences
|
|
splitsentences = []
|
|
|
|
if rcORnot == 1:
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
|
|
if wordwithrc in splitsentences[n]:
|
|
|
|
splitsentences[n][0] = 'diesen'
|
|
|
|
verb = splitsentences[n][-1]
|
|
|
|
splitsentences[n] = splitsentences[n][:-1]
|
|
splitsentences[n].insert(1, verb)
|
|
|
|
|
|
generalrules = [['PDS','VAFIN'], ['PDS', 'VVFIN']]
|
|
processed = 1
|
|
else:
|
|
splitsentences = oldsplitsentences
|
|
splitsentence = []
|
|
|
|
|
|
if token[n] == 'wem' or token[n] == 'Wem' or token[n] == 'welchem':
|
|
|
|
daORnot = gs.checkForAnnotation(sentence, 'da', 'word.dep_')
|
|
|
|
oaORnot = gs.checkForAnnotation(sentence, 'oa', 'word.dep_')
|
|
|
|
reORnot = gs.checkForAnnotation(sentence, 're', 'word.dep_')
|
|
|
|
oldsplitsentences = splitsentences
|
|
splitsentences = []
|
|
|
|
for word in sentence:
|
|
|
|
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
|
|
if n == 0:
|
|
index = 1
|
|
if n == 1:
|
|
index = 0
|
|
|
|
if reORnot == 1:
|
|
pass
|
|
if daORnot == 1 and reORnot == 0:
|
|
splitsentences[index].insert(1, 'das')
|
|
|
|
if oaORnot == 1 and reORnot == 0:
|
|
splitsentences[index].insert(1, 'dem')
|
|
|
|
if n == 1:
|
|
splitsentences[0], splitsentences[1] = splitsentences[1] , splitsentences[0]
|
|
|
|
|
|
generalrules = [['PDS','VAFIN'], ['PDS', 'VVFIN']]
|
|
processed = 1
|
|
|
|
if token[n] in self.indirectspeech_list and token[1] not in self.konsekutiv_list:
|
|
|
|
|
|
reORnot = gs.checkForAnnotation(sentence, 're', 'word.dep_')
|
|
oldsplitsentences = splitsentences
|
|
splitsentences = []
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
|
|
if n == 0:
|
|
index = 1
|
|
if n == 1:
|
|
index = 0
|
|
|
|
if reORnot == 0:
|
|
if splitsentences[index][0] != 'was':
|
|
splitsentences[index].insert(1, 'das')
|
|
|
|
if n == 1:
|
|
splitsentences[0], splitsentences[1] = splitsentences[1] , splitsentences[0]
|
|
|
|
|
|
generalrules = [['PDS','VAFIN'], ['PDS', 'VVFIN']]
|
|
processed = 1
|
|
|
|
if processed == 0 and n == 1:
|
|
|
|
ZUVINFTupelORnot = gs.checkForAnnotationTuple(sentence, ['PTKZU', 'VVINF'], 'word.tag_', 'None')
|
|
if ZUVINFTupelORnot == 0:
|
|
ZUVINFTupelORnot = gs.checkForAnnotationTuple(sentence, ['PTKZU', 'VAINF'], 'word.tag_', 'None')
|
|
|
|
if ZUVINFTupelORnot == 1:
|
|
|
|
reORnot = gs.checkForAnnotation(sentence, 're', 'word.dep_')
|
|
splitsentence = []
|
|
for word in sentence:
|
|
|
|
|
|
if word[-1] == ',':
|
|
splitsentence.append(word[:-1])
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',' :
|
|
splitsentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
splitsentences.append(splitsentence)
|
|
processed = 1
|
|
splitsentence = []
|
|
|
|
splitsentences.append(splitsentence)
|
|
|
|
for m in range(2):
|
|
ZUINForNOT = gs.checkForAnnotationTuple(splitsentences[m], ['PTKZU', 'VVINF'], 'word.tag_','None')
|
|
|
|
if ZUINForNOT == 0:
|
|
ZUINForNOT = gs.checkForAnnotationTuple(splitsentences[m], ['PTKZU', 'VAINF'], 'word.tag_','None')
|
|
|
|
|
|
if ZUINForNOT == 1:
|
|
r = m
|
|
ZUINForNOT = 0
|
|
|
|
|
|
|
|
if r == 0:
|
|
index = 1
|
|
if r == 1:
|
|
index = 0
|
|
|
|
objectORnot = gs.checkForAnnotation(splitsentences[index] , 'oa', 'word.dep_')
|
|
|
|
if reORnot == 0 and objectORnot == 0:
|
|
splitsentences[index].insert(1, 'das')
|
|
|
|
if r == 1:
|
|
splitsentences[0], splitsentences[1] = splitsentences[1] , splitsentences[0]
|
|
else:
|
|
processed == 2
|
|
|
|
|
|
except:
|
|
wasNotInAnyList = 1
|
|
|
|
|
|
#rules = [['ART','ADJA','NN'], ['ART','ADJA','NE'], ['ART', 'NN'], ['ART', 'NE'], ['APPR','NN'], ['APPR','NE'], ['APPR', 'ART', 'NN'], ['APPR', 'ART', 'NE'], ['APPR','ART','NN','ADJA','NN'], ['APPR','ART','NN','ADJA','NE'], ['KOKOM', 'ART', 'NN'], ['KOKOM', 'ART', 'NE'], ['PPOSAT', 'NN'], ['PPOSAT', 'NE'], ['ADV', 'ADJD']]
|
|
|
|
#print('B',splitsentences)
|
|
endsentences = []
|
|
if (processed == 2 or processed == 0) and n == 1:
|
|
wasNotInAnyList = 1
|
|
|
|
|
|
try:
|
|
if wasNotInAnyList == 0:
|
|
newpunctuationsindex.insert(0,[counter-1,punctuations[counter-1]])
|
|
#print('splitsentencee', splitsentences)
|
|
if len(splitsentences) > 2:
|
|
splitsentences = splitsentences[:2]
|
|
|
|
#print('splitsentenceeeees', splitsentences)
|
|
|
|
for splitsentence in splitsentences:
|
|
|
|
#print('splitsentenceeeeeeeeeeee!!',splitsentence)
|
|
wordtoputfirst = 'nada'
|
|
for word in self.firstwordlist:
|
|
if word == splitsentence[0]:
|
|
wordtoputfirst = word
|
|
splitsentence.remove(word)
|
|
|
|
|
|
|
|
#print('get the tuples and triples to check..')
|
|
tuplesTocheck, triplesTocheck, quadruplesTocheck = self.gs.GetTuplesinSentence(splitsentence)
|
|
#print('done')
|
|
#print(tuplesTocheck, 'ole', triplesTocheck ,'aiai', quadruplesTocheck)
|
|
#print('1')
|
|
grammpiecessentence = self.gs.createTupleofGrammarpieces( splitsentence, tuplesTocheck, triplesTocheck, quadruplesTocheck)
|
|
|
|
#print('grammpiece',grammpiecessentence)
|
|
#print('2')
|
|
if len(grammpiecessentence) > 7:
|
|
print('A sentence is too long, too many permutations. \n piping wrong grammar..')
|
|
endsentence = ' '.join(grammpiecessentence)
|
|
|
|
else:
|
|
#print('genrating the permutations')
|
|
permutations = self.sgm.GeneratePermutationsOfSentence(grammpiecessentence)
|
|
#print('done')
|
|
#print(permutations)
|
|
#print('3')
|
|
firstwordwithverblist = ['deswegen', 'danach']
|
|
permutationstodelete = []
|
|
for permutation in permutations:
|
|
#print('4')
|
|
if permutation[0] in firstwordwithverblist:
|
|
#print('4.1')
|
|
count = 1
|
|
for word in self.nlp(permutation[1]):
|
|
#print('4.2')
|
|
if word.tag_[0] != 'V':
|
|
#print('4.3')
|
|
permutationstodelete.append(permutation)
|
|
break
|
|
else:
|
|
break
|
|
#for word in self.nlp(permutation[0]):
|
|
#print('4.2')
|
|
#if word.tag_[0] != 'V':
|
|
#print('4.3')
|
|
#permutationstodelete.append(permutation)
|
|
#break
|
|
#else:
|
|
#break
|
|
for delperm in permutationstodelete:
|
|
try:
|
|
permutations.remove(delperm)
|
|
except:
|
|
|
|
pass
|
|
#print('5')
|
|
|
|
sentencesToCheck = []
|
|
if wordtoputfirst in self.firstwordlist:
|
|
for sentence in permutations:
|
|
sentencesToCheck.append(wordtoputfirst + ' ' + ' '.join(sentence))
|
|
else:
|
|
for sentence in permutations:
|
|
sentencesToCheck.append(' '.join(sentence))
|
|
|
|
endsentence = self.sgm.GetBestSentenceFromSentencesAccordingToGrammar(sentencesToCheck, ' '.join(splitsentence))
|
|
#print('done')
|
|
#print('endsent',endsentence)
|
|
endsentences.append(endsentence)
|
|
except:
|
|
#print('there was an error')
|
|
wasNotInAnyList = 1
|
|
endsentences = []
|
|
todelete = []
|
|
for index in range(len(newpunctuationsindex)):
|
|
if newpunctuationsindex[index][0] == counter - 1:
|
|
todelete.append(index)
|
|
for todel in todelete[::-1]:
|
|
del newpunctuationsindex[todel]
|
|
|
|
|
|
if wasNotInAnyList == 1:
|
|
#print('was not in any list')
|
|
#print(oldsentence)
|
|
endsplisentences = []
|
|
splisentence = []
|
|
for word in oldsentence:
|
|
|
|
|
|
if word[-1] == ',':
|
|
splisentence.append(word[:-1])
|
|
|
|
if word == ',':
|
|
pass
|
|
if word[-1] != ',':
|
|
splisentence.append(word)
|
|
|
|
if word[-1] == ',' or word == ',':
|
|
|
|
endsplisentences.append(splisentence)
|
|
|
|
splisentence = []
|
|
|
|
endsplisentences.append(splisentence)
|
|
|
|
newpunctuationsindex.insert(0,[counter-1,punctuations[counter-1]])
|
|
|
|
#print('endsplisentences',endsplisentences)
|
|
for splsentence in endsplisentences:
|
|
|
|
endsentences.append(' '.join(splsentence))
|
|
|
|
|
|
|
|
|
|
|
|
'''
|
|
|
|
|
|
|
|
fsearch1 = self.fsearch1
|
|
spacyclass1 = 'word.tag_'
|
|
|
|
|
|
gs_sentence1 = gs.Sentence2GrammarSchema(' '.join(splitsentence), spacyclass1)
|
|
|
|
|
|
print('searchPatternMatch for tags')
|
|
bestmatches1 = fsearch1.searchPatternMatch(' '.join(gs_sentence1), 1)
|
|
print('done')
|
|
|
|
#print('oioi', bestmatches1)
|
|
|
|
#print(len(fsearch1.database))
|
|
right_gs_tupel1 = []
|
|
|
|
if len(bestmatches1) < 10:
|
|
bestndocs1 = len(bestmatches1)
|
|
else:
|
|
bestndocs1 = 10
|
|
|
|
for m in range(bestndocs1):
|
|
right_gs_tupel1.append(fsearch1.database[bestmatches1[m][0]])
|
|
|
|
|
|
statistically_correct_sentences1 = gs.Sentence2RightGrammarTupel(' '.join(splitsentence), gs_sentence1, right_gs_tupel1)
|
|
|
|
|
|
fsearch2 = self.fsearch2
|
|
|
|
spacyclass2 = 'word.dep_'
|
|
|
|
gs_sentence2 = gs.Sentence2GrammarSchema(' '.join(splitsentence), spacyclass2)
|
|
|
|
print('searchPatternMatch for deps')
|
|
bestmatches2 = fsearch2.searchPatternMatch(' '.join(gs_sentence2), 1)
|
|
print('done')
|
|
|
|
right_gs_tupel2 = []
|
|
|
|
|
|
if len(bestmatches2) < 10:
|
|
bestndocs2 = len(bestmatches2)
|
|
else:
|
|
bestndocs2 = 10
|
|
|
|
|
|
for m in range(bestndocs2):
|
|
right_gs_tupel2.append(fsearch2.database[bestmatches2[m][0]])
|
|
|
|
#print(' '.join(splitsentence))
|
|
|
|
statistically_correct_sentences2 = gs.Sentence2RightGrammarTupel(' '.join(splitsentence), gs_sentence2, right_gs_tupel2)
|
|
|
|
|
|
print(splitsentence)
|
|
|
|
|
|
Rightsentence = gs.GetBestgsAccordingRules(' '.join(splitsentence) , gs_sentence1, right_gs_tupel1, right_gs_tupel2, statistically_correct_sentences1, statistically_correct_sentences2, rules, generalrules)
|
|
|
|
|
|
'''
|
|
for endsentence in endsentences:
|
|
gramcorr_splitsentences.append(endsentence.split())
|
|
|
|
for index in newpunctuationsindex:
|
|
punctuations.insert(index[0], index[1])
|
|
|
|
return gramcorr_splitsentences, punctuations
|
|
|
|
|
|
|
|
def putAppendixesIntoOwnSentences(self, sentences, punctuations):
|
|
|
|
gs = self.gs
|
|
#triples = [['NN', 'ART', 'NN'], ['NE', 'ART', 'NN'], ['NN', 'ART', 'NN'], ['NE', 'ART', 'NE']]
|
|
quadruples = [['NN', 'APPR', 'NE', 'NN'], ['NN', 'APPR', 'NE', 'NN'], ['NN', 'APPR', 'ART', 'NN'], ['NE', 'APPR', 'ART', 'NN'], ['NN', 'APPR', 'ART', 'NE'], ['NE', 'APPR', 'ART', 'NE']]
|
|
quadruplestochange = []
|
|
triplestochange = []
|
|
newsentences = []
|
|
newpunctuations = []
|
|
Whatisofnouns = []
|
|
oldsentences = sentences
|
|
oldpunctuations = punctuations
|
|
for hauptindex in range(len(sentences)):
|
|
|
|
sentence = sentences[hauptindex]
|
|
try:
|
|
#for triple in triples:
|
|
# AnnoOrNot, tripleInWords = gs.checkForAnnotationTriple(sentence, triple, 'word.tag_', 'None')
|
|
# for tripleinwor in tripleInWords:
|
|
# triplestochange.append([triple, tripleinwor])
|
|
|
|
for quadruple in quadruples:
|
|
AnnoOrNot, quadrupleInWords = gs.checkForAnnotationQuadruple(sentence, quadruple, 'word.tag_', 'None')
|
|
#print('quadinwords', quadrupleInWords)
|
|
#print('ANNOORNOT', AnnoOrNot)
|
|
for quadrupleInWo in quadrupleInWords:
|
|
quadruplestochange.append([quadruple, quadrupleInWo])
|
|
|
|
#print('quadstochange',quadruplestochange)
|
|
for quad in quadruplestochange:
|
|
for n in range(len(sentence) - 4):
|
|
if sentence[n] == quad[1][0]:
|
|
if sentence[n + 1] == quad[1][1]:
|
|
if sentence[n + 2] == quad[1][2]:
|
|
artword = None
|
|
longerWhatisnoun = 0
|
|
for m in range(2):
|
|
for word in self.nlp(sentence[n - m]):
|
|
if word.tag_ == 'ART':
|
|
Nounthatis = sentence[n - m:n + 1]
|
|
import spacy
|
|
nlp = spacy.load('de_core_news_sm')
|
|
token3 = nlp(sentence[n+4])
|
|
counter = 0
|
|
Whatisnoun = sentence[n + 1:n + 4]
|
|
for wor in token3:
|
|
counter += 1
|
|
if wor.tag_ == 'NN' or wor.tag_ == 'NE':
|
|
if counter == 1:
|
|
Whatisnoun = sentence[n + 1:n + 5]
|
|
longerWhatisnoun = 1
|
|
if counter == 2:
|
|
Whatisnoun = sentence[n + 1:n + 4]
|
|
|
|
|
|
|
|
artword = word.text
|
|
#print(sentence[n - 1],'oi')
|
|
if ((artword == 'die' or artword == 'Die') and sentence[n][-1] != 'n') or ((artword == 'der' or artword == 'einer' or artword == 'dieser') and (sentence[n - 2] in ['von', 'in', 'auf', 'ueber', 'unter', 'nach', 'mit'])):
|
|
|
|
if artword == 'der':
|
|
Nounthatis[0] = 'die'
|
|
|
|
donothing = 0
|
|
if sentence[n + 1] == 'mit':
|
|
if sentence[n + 2] == 'den':
|
|
verb = ' hat die '
|
|
Whatisnoun = Whatisnoun[2:]
|
|
if sentence[n + 2] == 'der':
|
|
verb = ' hat eine '
|
|
Whatisnoun = Whatisnoun[2:]
|
|
if sentence[n + 2] != 'der' and sentence[n + 2] != 'den':
|
|
donothing = 1
|
|
else:
|
|
verb = ' ist '
|
|
if donothing == 0:
|
|
newsentence = ' '.join(Nounthatis) + verb + ' '.join(Whatisnoun)
|
|
|
|
|
|
newsentences.append([hauptindex + 1, newsentence.split()])
|
|
newpunctuations.append([hauptindex + 1, punctuations[hauptindex]])
|
|
if longerWhatisnoun == 0:
|
|
Whatisofnouns.append([n + 1, n + 4, hauptindex])
|
|
else:
|
|
Whatisofnouns.append([n + 1, n + 5, hauptindex])
|
|
except:
|
|
print('Konnte nicht ' + str(sentence) + 'in Characterisierung pro Satz prozessieren..')
|
|
try:
|
|
for whatis in Whatisofnouns[::-1]:
|
|
thereisacomma = 0
|
|
#print(sentences[whatis[2]][whatis[1] - 1])
|
|
if sentences[whatis[2]][whatis[1] - 1][-1] == ',':
|
|
|
|
thereisacomma = 1
|
|
if thereisacomma == 1:
|
|
#print(sentences[whatis[2]][whatis[0] - 1])
|
|
sentences[whatis[2]][whatis[0] - 1] = sentences[whatis[2]][whatis[0] - 1] + ','
|
|
del sentences[whatis[2]][whatis[0]:whatis[1]]
|
|
for newsent in newsentences[::-1]:
|
|
sentences.insert(newsent[0], newsent[1])
|
|
for newpunct in newpunctuations[::-1]:
|
|
punctuations.insert(newpunct[0], newpunct[1])
|
|
for sentence in sentences:
|
|
if sentence[-1][-1] == ',':
|
|
sentence[-1] = sentence[-1][:-1]
|
|
except:
|
|
print('konnte nicht die gesammelten Characterisierungen prozessieren')
|
|
sentences = oldsentences
|
|
punctuations = oldpunctuations
|
|
|
|
|
|
|
|
return sentences, punctuations
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|