You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
327 lines
13 KiB
327 lines
13 KiB
# Klasse zum Konvertieren von nicht Aussagen zu postiven Aussagen.
|
|
|
|
|
|
# Notiz: nicht + Adjektiv kann direkt durch Gegenwort ausgetauscht werden.
|
|
# nicht + verb kann zum Gegenwort des Verbes ausgetauscht werden, dabei muss aber nach Hause weggecuttet werden bei bsp Er ging nicht nach Hause. Er blieb
|
|
|
|
|
|
# in wiktionary {{Gegenwörter}} Kategorie
|
|
|
|
import spacy
|
|
import nltk
|
|
from nltk.stem.snowball import SnowballStemmer
|
|
|
|
import hickle as hkl
|
|
import FASTsearch
|
|
|
|
stemmer = SnowballStemmer("german")
|
|
|
|
|
|
class SayYes(object):
|
|
|
|
def __init__(self, hklDatabaseDir_Opposites, hklDatabaseDir_Opposites_All):
|
|
|
|
if hklDatabaseDir_Opposites is not None:
|
|
self.OppositesDB = hkl.load(hklDatabaseDir_Opposites)
|
|
|
|
|
|
#print('loading the german spacy model..')
|
|
self.nlp = spacy.load('de_core_news_sm')
|
|
#print('done')
|
|
|
|
#print('loading the stemmer..')
|
|
self.stemmer = SnowballStemmer("german")
|
|
#print('done')
|
|
|
|
return
|
|
|
|
|
|
def create_hklDB_from_csv(self, csvDbDir):
|
|
|
|
with open(csvDbDir) as lines:
|
|
|
|
self.OppositesDB_All = []
|
|
|
|
for line in lines:
|
|
|
|
#print(line)
|
|
|
|
self.OppositesDB_All.append(list(eval(line)))
|
|
|
|
|
|
|
|
|
|
|
|
self.hkldbOpposites1 = []
|
|
self.hkldbOpposites2 = []
|
|
|
|
|
|
counter = 0
|
|
for n in range(len(self.OppositesDB_All)):
|
|
|
|
counter += 1
|
|
if counter % 1000 == 0:
|
|
print(counter)
|
|
|
|
|
|
self.hkldbOpposites1.append([self.OppositesDB_All[n][0][0]] + [self.stemmer.stem(word) for word in self.OppositesDB_All[n][0]] )
|
|
self.hkldbOpposites2.append([self.OppositesDB_All[n][1][0]] + [stemmer.stem(word) for word in self.OppositesDB_All[n][1]] )
|
|
|
|
|
|
|
|
#print('hkldbOpposites1', self.hkldbOpposites1)
|
|
#print('hkldbOpposites2', self.hkldbOpposites2)
|
|
|
|
print('creating the hkl dump of OppositesDBAll')
|
|
hkl.dump(self.OppositesDB_All, 'hkldbOpposites_All.hkl', mode='w', compression='gzip')
|
|
print('done..')
|
|
|
|
print('Creating the hkl dump of OppositesDB 1')
|
|
hkl.dump(self.hkldbOpposites1, 'hkldbOpposites1.hkl', mode='w', compression='gzip')
|
|
print('done..')
|
|
|
|
print('Creating the hkl dump of OppositesDB 2')
|
|
hkl.dump(self.hkldbOpposites2, 'hkldbOpposites2.hkl', mode='w', compression='gzip')
|
|
print('done..')
|
|
|
|
|
|
|
|
return 'done'
|
|
|
|
def load_DB_into_FASTsearch(self):
|
|
|
|
#print('Loading the hklDB1..')
|
|
self.hkldbOpposites1 = hkl.load('hkldbOpposites1.hkl')
|
|
#print('done')
|
|
|
|
#print('Loading the hklDB2')
|
|
self.hkldbOpposites2 = hkl.load('hkldbOpposites2.hkl')
|
|
#print('done')
|
|
|
|
#print('loading hkldbOpposites 1..')
|
|
self.fsearch1 = FASTsearch.FASTsearch('hkldbOpposites1.hkl')
|
|
#print('done')
|
|
|
|
#print('loading hkldbOpposites 2..')
|
|
self.fsearch2 = FASTsearch.FASTsearch('hkldbOpposites2.hkl')
|
|
#print('done')
|
|
|
|
#print('generating BoW Model 1..')
|
|
#self.fsearch1.Gen_BoW_Model(3000, "word", punctuation = False)
|
|
#print('done')
|
|
|
|
#print('generating BoW Model 2..')
|
|
#self.fsearch2.Gen_BoW_Model(3000, "word", punctuation = False)
|
|
#print('done')
|
|
|
|
|
|
#print('loading the bow model 1')
|
|
self.fsearch1.Load_BoW_Model('bagofwordshkldbOpposites1.pkl', 'DataBaseOneZeroshkldbOpposites1.hkl')
|
|
#print('done')
|
|
|
|
#print('loading the bow model 2')
|
|
self.fsearch2.Load_BoW_Model('bagofwordshkldbOpposites2.pkl', 'DataBaseOneZeroshkldbOpposites2.hkl')
|
|
#print('done')
|
|
|
|
|
|
#print('oi thats the get_feature_names', self.fsearch1.vectorizer.get_feature_names())
|
|
#print('oi thats the get_feature_names', self.fsearch2.vectorizer.get_feature_names())
|
|
|
|
|
|
|
|
|
|
|
|
def replaceOpposites(self, sentences):
|
|
outsentences = []
|
|
#print('wenigstens etwas')
|
|
sentencecount = 0
|
|
for sentence in sentences:
|
|
#print('oloa')
|
|
sentencecount += 1
|
|
#print('processing sentence', sentencecount)
|
|
|
|
listofAdjektives = []
|
|
nichtIndex = None
|
|
KeinIndex = None
|
|
for m in range(len(sentence)):
|
|
if sentence[m] == 'nicht':
|
|
nichtIndex = m
|
|
|
|
|
|
if sentence[m][:4] == 'kein':
|
|
|
|
KeinIndex = m
|
|
|
|
#if KeinIndex not None or nichtIndex not None:
|
|
|
|
|
|
#if len(listofAdjektives) == 0:
|
|
#if word.dep_[0] == 'V':
|
|
#print('ola')
|
|
|
|
if (KeinIndex is not None) or (nichtIndex is not None):
|
|
|
|
doc = self.nlp(' '.join(sentence))
|
|
count = 0
|
|
for word in doc:
|
|
count += 1
|
|
if word.text == ',':
|
|
count -= 1
|
|
#print(word.text, word.tag_, word.tag_[:1])
|
|
if word.tag_[:2] == 'AD':
|
|
|
|
listofAdjektives.append([word.text, count - 1])
|
|
|
|
|
|
|
|
listOfOpposites = []
|
|
if (KeinIndex is not None):
|
|
|
|
#print(sentence[KeinIndex + 1])
|
|
|
|
|
|
if len(listofAdjektives) == 0 or len(listofAdjektives) > 1:
|
|
AdjIndex = 1
|
|
#print('listofadjectives', listofAdjektives)
|
|
for n in range(len(listofAdjektives)):
|
|
ad = listofAdjektives[n]
|
|
#print(ad[1])
|
|
if ad[1] == KeinIndex + 2 and ad[0] == 'zu':
|
|
if listofAdjektives[n + 1][1] == KeinIndex + 3:
|
|
AdjIndex = 2
|
|
|
|
#print('Adj und stemadj 0 2')
|
|
#print(sentence[KeinIndex + AdjIndex])
|
|
#print(self.stemmer.stem(sentence[KeinIndex + AdjIndex]))
|
|
|
|
bestmatches1, matchindex1 = self.fsearch1.search_with_highest_multiplikation_Output(self.stemmer.stem(sentence[KeinIndex + AdjIndex]), 1)
|
|
|
|
bestmatches2, matchindex2 = self.fsearch2.search_with_highest_multiplikation_Output(self.stemmer.stem(sentence[KeinIndex + AdjIndex]), 1)
|
|
|
|
Austauschindex = KeinIndex + AdjIndex
|
|
|
|
else:
|
|
|
|
|
|
|
|
Adjektiv = listofAdjektives[0][0]
|
|
#print('Adj und stemadj')
|
|
#print(Adjektiv)
|
|
#print(self.stemmer.stem(Adjektiv))
|
|
Austauschindex = listofAdjektives[0][1]
|
|
|
|
bestmatches1, matchindex1 = self.fsearch1.search_with_highest_multiplikation_Output(self.stemmer.stem(Adjektiv), 1)
|
|
|
|
bestmatches2, matchindex2 = self.fsearch2.search_with_highest_multiplikation_Output(self.stemmer.stem(Adjektiv), 1)
|
|
|
|
|
|
Opposite = None
|
|
#print('thetheone')
|
|
if (nichtIndex is not None):
|
|
|
|
#print(sentence[nichtIndex + 1])
|
|
#print('theone')
|
|
if len(listofAdjektives) == 0 or len(listofAdjektives) > 1:
|
|
#print('1')
|
|
#print(nichtIndex)
|
|
#print('2')
|
|
if nichtIndex == (len(sentence) - 1):
|
|
Austauschindex = nichtIndex - 1
|
|
else:
|
|
Austauschindex = nichtIndex + 1
|
|
|
|
# TO DO: egal formen auf infinitiv mappen
|
|
# Das muss mit machine learnign gelöst werden..
|
|
# --> ergiebt sich aus den übersetzungen ( welches wort fehl, welches neu
|
|
# da, dann daraus eine maschine die sich die gegenteile merkt =)
|
|
|
|
#itisaVerb = False
|
|
#if doc[Austauschindex].dep_[0] == 'V':
|
|
# itisaVerb = True
|
|
|
|
#someform = sentence[Austauschindex]
|
|
|
|
|
|
|
|
|
|
|
|
bestmatches1, matchindex1 = self.fsearch1.search_with_highest_multiplikation_Output(self.stemmer.stem(sentence[Austauschindex]), 1)
|
|
|
|
bestmatches2, matchindex2 = self.fsearch2.search_with_highest_multiplikation_Output(self.stemmer.stem(sentence[Austauschindex]), 1)
|
|
|
|
|
|
|
|
else:
|
|
|
|
Adjektiv = listofAdjektives[0][0]
|
|
|
|
Austauschindex = listofAdjektives[0][1]
|
|
|
|
bestmatches1, matchindex1 = self.fsearch1.search_with_highest_multiplikation_Output(self.stemmer.stem(Adjektiv), 1)
|
|
|
|
bestmatches2, matchindex2 = self.fsearch2.search_with_highest_multiplikation_Output(self.stemmer.stem(Adjektiv), 1)
|
|
|
|
Opposite = None
|
|
#print(sentence)
|
|
#print(bestmatches1, matchindex1)
|
|
#print(bestmatches1, matchindex1)
|
|
#print(len(listOfOpposites))
|
|
if matchindex1[1] >= 1:
|
|
OppositeIndex = matchindex1[0]
|
|
|
|
Opposite = self.hkldbOpposites2[OppositeIndex][0]
|
|
#print('Opposite in match1', Opposite)
|
|
listOfOpposites.append([Opposite,Austauschindex])
|
|
|
|
if matchindex2[1] >= 1:
|
|
OppositeIndex = matchindex2[0]
|
|
|
|
Opposite = self.hkldbOpposites1[OppositeIndex][0]
|
|
#print('opposite in match2', Opposite)
|
|
listOfOpposites.append([Opposite,Austauschindex])
|
|
|
|
|
|
#print(listOfOpposites)
|
|
for opposite in listOfOpposites:
|
|
if sentence[opposite[1]][-1] == ',':
|
|
|
|
if sentence[opposite[1]][-3:] == 'es,':
|
|
opposite[0] = opposite[0] + 'es'
|
|
if sentence[opposite[1]][-3:] == 'er,':
|
|
opposite[0] = opposite[0] + 'er'
|
|
if sentence[opposite[1]][-3:] == 'em,':
|
|
opposite[0] = opposite[0] + 'em'
|
|
if sentence[opposite[1]][-2:] == 'e,':
|
|
opposite[0] = opposite[0] + 'e'
|
|
sentence[opposite[1]] = opposite[0] + ','
|
|
else:
|
|
|
|
if sentence[opposite[1]][-2:] == 'es':
|
|
opposite[0] = opposite[0] + 'es'
|
|
if sentence[opposite[1]][-2:] == 'er':
|
|
opposite[0] = opposite[0] + 'er'
|
|
if sentence[opposite[1]][-2:] == 'em':
|
|
opposite[0] = opposite[0] + 'em'
|
|
if sentence[opposite[1]][-1:] == 'e':
|
|
opposite[0] = opposite[0] + 'e'
|
|
sentence[opposite[1]] = opposite[0]
|
|
|
|
if KeinIndex is not None and len(listOfOpposites) > 0:
|
|
|
|
#print(KeinIndex)
|
|
sentence[KeinIndex] = sentence[KeinIndex][1:]
|
|
|
|
|
|
|
|
if nichtIndex is not None and len(listOfOpposites) > 0:
|
|
|
|
#print(nichtIndex)
|
|
|
|
printer = sentence.pop(nichtIndex)
|
|
|
|
#print(printer)
|
|
|
|
|
|
|
|
outsentences.append(sentence)
|
|
return outsentences
|