corsaronero
1 year ago
commit
1263460e05
2 changed files with 787 additions and 0 deletions
Unified View
Diff Options
-
+787 -0parse1.py
-
BINparse1.pyc
+ 787
- 0
parse1.py
View File
| @ -0,0 +1,787 @@ | |||||
| # Parse wiktionary.xml with pure python, such that it can be run with pypy (python just in time compiler) | |||||
| # optimization would be possible through cython and assembler loops etc | |||||
| # on a linux system, get the first n lines of a document with: | |||||
| # head -n1000000 dewiktionary-20181201-pages-articles.xml > wiktionaryFirstMio.xml | |||||
| import sys | |||||
| import os | |||||
| import re | |||||
| class Parser(object): | |||||
| def __init__(self, InputDokument, OutputDokument): | |||||
| self.Indok = InputDokument | |||||
| self.Outdok = OutputDokument | |||||
| def GetSeparators(self): | |||||
| with open(self.Indok) as xmldok: | |||||
| with open(self.Outdok , 'w') as getsepdok: | |||||
| seperators = [] | |||||
| counter = 0 | |||||
| for line in xmldok: | |||||
| counter += 1 | |||||
| #print(counter) | |||||
| if (counter % 10) == 0: | |||||
| print(counter) | |||||
| seperator =[] | |||||
| val = 0 | |||||
| #if counter == 10000: | |||||
| #seperatorsSet = [] | |||||
| #getsepdok.write('[' + '\n') | |||||
| #for element in seperators: | |||||
| #seperatorsSet.append(''.join(element)) | |||||
| #for element in set(seperatorsSet): | |||||
| #getsepdok.write(str(''.join(element)) + '\n') | |||||
| #getsepdok.write(']') | |||||
| for letter in line: | |||||
| #print(letter) | |||||
| if letter == '>': | |||||
| val = 0 | |||||
| seperators.append(seperator) | |||||
| seperator = [] | |||||
| if val == 1: | |||||
| seperator.append(letter) | |||||
| else: | |||||
| pass | |||||
| if letter == '<': | |||||
| val = 1 | |||||
| seperatorsSet = [] | |||||
| getsepdok.write('[' + '\n') | |||||
| for element in seperators: | |||||
| seperatorsSet.append(''.join(element)) | |||||
| seperatorsSet = set(seperatorsSet) | |||||
| for element in set(seperatorsSet): | |||||
| getsepdok.write(str(''.join(element)) + '\n') | |||||
| getsepdok.write(']') | |||||
| return seperatorsSet | |||||
| def GetPayloadBetweenTwoSymbols(self, SymbolA, SymbolB , LogLineNumber=False, Doc = True): | |||||
| with open(self.Indok) as xmldok: | |||||
| with open(self.Outdok , 'w') as payloaddok: | |||||
| seperators = [] | |||||
| counter = 0 | |||||
| valA = 0 | |||||
| valB = 0 | |||||
| seperator =[] | |||||
| for line in xmldok: | |||||
| #print(line) | |||||
| counter += 1 | |||||
| if LogLineNumber == True: | |||||
| if (counter % 10000) == 0: | |||||
| print(counter) | |||||
| wait1letterA = False | |||||
| wait1letterB = False | |||||
| #for letter in line.decode('utf-8'): | |||||
| for letter in line: | |||||
| #print(letter) | |||||
| #print(set(range(1, len(SymbolA)))) | |||||
| if valA % len(SymbolA) in set(range(1, len(SymbolA) )): | |||||
| #print('jo') | |||||
| if wait1letterA == True: | |||||
| #print('joo') | |||||
| #print(letter) | |||||
| valA -= valA % len(SymbolA) | |||||
| wait1letterA = False | |||||
| wait1letterA = True | |||||
| if valB in set(range(1, len(SymbolB) )): | |||||
| if wait1letterB == True: | |||||
| valB = 0 | |||||
| wait1letterB = False | |||||
| wait1letterB = True | |||||
| for n in range(len(SymbolB)): | |||||
| if valA >= len(SymbolA) and valB == n and letter == SymbolB[n]: | |||||
| valB = n + 1 | |||||
| wait1letterB = False | |||||
| else: | |||||
| pass | |||||
| if valB == len(SymbolB) and valA >= len(SymbolA): | |||||
| valB = 0 | |||||
| #print(letter) | |||||
| #print(valA) | |||||
| valA -= len(SymbolA) | |||||
| #print(valA) | |||||
| #print(seperators) | |||||
| if valA >= len(SymbolA): | |||||
| seperator.append(letter) | |||||
| else: | |||||
| pass | |||||
| #print(valA) | |||||
| #print(SymbolA[6]) | |||||
| #print(len(SymbolA)) | |||||
| #print(range(len(SymbolA))) | |||||
| if valA == 0: | |||||
| if len(seperator[:-(len(SymbolB)-1)]) >= 1: | |||||
| seperators.append(seperator[:-(len(SymbolB)-1)]) | |||||
| seperator = [] | |||||
| for n in range(len(SymbolA)): | |||||
| #print(n) | |||||
| if valA % len(SymbolA) == n and letter == SymbolA[n]: | |||||
| valA += 1 | |||||
| #print(valA) | |||||
| wait1letterA = False | |||||
| break | |||||
| else: | |||||
| pass | |||||
| seperatorsSet = [] | |||||
| #getsepdok.write('[' + '\n') | |||||
| for element in seperators: | |||||
| seperatorsSet.append(''.join(element)) | |||||
| seperatorsSet = set(seperatorsSet) | |||||
| output = [] | |||||
| ID = 0 | |||||
| ## Set has a probabilistic factor in it!!!! thats why the nmbers change | |||||
| for element in seperatorsSet: | |||||
| output.append([element, ID]) | |||||
| ID += 1 | |||||
| return output | |||||
| def GetPayloadBetweenTwoSymbolsInPayload(self, Payload, SymbolA, SymbolB, LogElementNumber): | |||||
| seperators = [] | |||||
| counter = 0 | |||||
| for element in Payload: | |||||
| counter += 1 | |||||
| if LogElementNumber == True: | |||||
| if (counter % 1000) == 0: | |||||
| print(counter) | |||||
| seperator =[] | |||||
| wait1letterA = False | |||||
| wait1letterB = False | |||||
| valA = 0 | |||||
| valB = 0 | |||||
| for letter in element[0]: | |||||
| #print(letter) | |||||
| #print(set(range(1, len(SymbolA)))) | |||||
| if valA % len(SymbolA) in set(range(1, len(SymbolA) )): | |||||
| #print(valA) | |||||
| #print('jo') | |||||
| if wait1letterA == True: | |||||
| #print('joo') | |||||
| valA -= valA % len(SymbolA) | |||||
| wait1letterA = False | |||||
| wait1letterA = True | |||||
| if valB in set(range(1, len(SymbolB) )) and valA >= len(SymbolA): | |||||
| if wait1letterB == True: | |||||
| valB = 0 | |||||
| wait1letterB = False | |||||
| wait1letterB = True | |||||
| #for n in range(len(SymbolB)): | |||||
| #if valB == n and letter == SymbolB[n]: | |||||
| #valB = n + 1 | |||||
| #wait1letterB = False | |||||
| #else: | |||||
| #pass | |||||
| if letter == SymbolB[valB % len(SymbolB)] and valA >= len(SymbolA): | |||||
| valB += 1 | |||||
| wait1letterB = False | |||||
| else: | |||||
| pass | |||||
| if valB == len(SymbolB) and valA >= len(SymbolA): | |||||
| valB = 0 | |||||
| #print(valA) | |||||
| valA -= len(SymbolA) | |||||
| #print(valA) | |||||
| #print(seperators) | |||||
| if valA >= len(SymbolA): | |||||
| ##print(letter) | |||||
| seperator.append(letter) | |||||
| #print(seperator) | |||||
| else: | |||||
| pass | |||||
| #print(valA) | |||||
| #print(SymbolA[6]) | |||||
| #print(len(SymbolA)) | |||||
| #print(range(len(SymbolA))) | |||||
| if valA == 0: | |||||
| #print('seps') | |||||
| if len(seperator[:-(len(SymbolB)-1)]) >= 1: | |||||
| seperators.append([''.join(seperator[:-(len(SymbolB)-1)]), element[1]]) | |||||
| seperator = [] | |||||
| # Optimierungsmoeglichkeit: Hier kann die for schleife durch viele ifs ersetzt werden, sowas wie start for after zwei ifs. | |||||
| # wuerde einiges an computation wegnehmen, auch da beide symbole | |||||
| #for n in range(len(SymbolA)): | |||||
| ##print(n) | |||||
| #if valA % len(SymbolA) == n and letter == SymbolA[n]: | |||||
| ##print(SymbolA[n]) | |||||
| #valA += 1 | |||||
| #wait1letterA = False | |||||
| #else: | |||||
| #pass | |||||
| for n in range(len(SymbolA)): | |||||
| #print(n) | |||||
| if valA % len(SymbolA) == n and letter == SymbolA[n]: | |||||
| valA += 1 | |||||
| #print(valA) | |||||
| wait1letterA = False | |||||
| break | |||||
| else: | |||||
| pass | |||||
| return seperators | |||||
| def GetPayloadBetweenTwoOneSymbolsInPayload(self, Payload, SymbolA, SymbolB, LogElementNumber, Payloadrow, IDrow): | |||||
| counter = 0 | |||||
| seperator =[] | |||||
| seperators = [] | |||||
| for payload in Payload: | |||||
| val = 0 | |||||
| for letter in payload[Payloadrow]: | |||||
| counter += 1 | |||||
| #print(counter) | |||||
| if LogElementNumber == True: | |||||
| if (counter % 10) == 0: | |||||
| print(counter) | |||||
| #print(letter) | |||||
| if letter == SymbolB: | |||||
| val -= 1 | |||||
| if val >= 1: | |||||
| seperator.append(letter) | |||||
| else: | |||||
| pass | |||||
| if val == 0 and len(seperator) >= 1: | |||||
| seperators.append([''.join(seperator), payload[IDrow]]) | |||||
| seperator = [] | |||||
| if letter == SymbolA: | |||||
| #print(val) | |||||
| val += 1 | |||||
| return seperators | |||||
| def CutTextAtSymbol(self, text, symbol): | |||||
| itisthesymbol = 0 | |||||
| outtext = [] | |||||
| output = [] | |||||
| symbolisthere = 0 | |||||
| for letter in text: | |||||
| outtext.append(letter) | |||||
| #print(letter) | |||||
| if letter != symbol[itisthesymbol]: | |||||
| itisthesymbol = 0 | |||||
| if letter == symbol[itisthesymbol]: | |||||
| itisthesymbol += 1 | |||||
| if itisthesymbol == len(symbol): | |||||
| #print(outtext) | |||||
| output.append(''.join(outtext)) | |||||
| itisthesymbol = 0 | |||||
| symbolisthere = 1 | |||||
| if symbolisthere == 0: | |||||
| output.append(''.join(outtext)) | |||||
| return output[0] | |||||
| def GetPayloadBetweenTwoSymbolsInText(self, text, SymbolA, SymbolB): | |||||
| seperators = [] | |||||
| seperator =[] | |||||
| wait1letterA = False | |||||
| wait1letterB = False | |||||
| valA = 0 | |||||
| valB = 0 | |||||
| for letter in text: | |||||
| #print(letter) | |||||
| #print(SymbolA) | |||||
| if valA % len(SymbolA) in set(range(1, len(SymbolA) )): | |||||
| if wait1letterA == True: | |||||
| valA -= valA % len(SymbolA) | |||||
| wait1letterA = False | |||||
| wait1letterA = True | |||||
| #print('B',valB) | |||||
| #print(valA) | |||||
| if valB in set(range(1, len(SymbolB) )): | |||||
| if wait1letterB == True: | |||||
| valB = 0 | |||||
| wait1letterB = False | |||||
| wait1letterB = True | |||||
| #print('B',valB) | |||||
| #print(valA) | |||||
| if letter == SymbolB[valB % len(SymbolB)]: | |||||
| valB += 1 | |||||
| wait1letterB = False | |||||
| else: | |||||
| pass | |||||
| if valB == len(SymbolB): | |||||
| valB = 0 | |||||
| valA -= len(SymbolA) | |||||
| #print('B',valB) | |||||
| #print(valA) | |||||
| if valA >= len(SymbolA): | |||||
| #print('append') | |||||
| seperator.append(letter) | |||||
| else: | |||||
| pass | |||||
| if valA == 0: | |||||
| if len(seperator[:-(len(SymbolB)-1)]) >= 1: | |||||
| seperators.append([''.join(seperator[:-(len(SymbolB)-1)])]) | |||||
| seperator = [] | |||||
| # Optimierungsmoeglichkeit: Hier kann die for schleife durch viele ifs ersetzt werden, sowas wie start for after zwei ifs. | |||||
| # wuerde einiges an computation wegnehmen, auch da beide symbole | |||||
| #for n in range(len(SymbolA)): | |||||
| #print(SymbolA[valA % len(SymbolA)]) | |||||
| if letter == SymbolA[valA % len(SymbolA)]: | |||||
| #print('oi') | |||||
| valA += 1 | |||||
| wait1letterA = False | |||||
| else: | |||||
| pass | |||||
| return seperators | |||||
| def GetPayloadBetweenTwoSameSymbolsInText(self, text, Symbol): | |||||
| seperators = [] | |||||
| seperator =[] | |||||
| wait1letter = False | |||||
| nowendit = False | |||||
| val = 0 | |||||
| for letter in text: | |||||
| #print(letter) | |||||
| #print(SymbolA) | |||||
| if nowendit == False and letter == Symbol[val % len(Symbol)]: | |||||
| val += 1 | |||||
| if nowendit == True and letter == Symbol[val % len(Symbol)]: | |||||
| val -= 1 | |||||
| if val == len(Symbol): | |||||
| seperator.append(letter) | |||||
| nowendit = True | |||||
| #print('append') | |||||
| if val == 0 and len(seperator) >= 1: | |||||
| seperators.append(' '.join(seperator)) | |||||
| seperator = [] | |||||
| nowendit = False | |||||
| return seperators | |||||
| def ParseWordswithSymbolFromSymbolongoing(self, text, Symbol): | |||||
| seperators = [] | |||||
| #print(text.split()) | |||||
| for word in text.split(): | |||||
| val = 0 | |||||
| waitoneletter = False | |||||
| seperator = [] | |||||
| for letter in word: | |||||
| #print(letter) | |||||
| #print(val) | |||||
| if val < len(Symbol): | |||||
| if letter == Symbol[val]: | |||||
| val += 1 | |||||
| #print(letter) | |||||
| #print(len(Symbol)) | |||||
| #print(val) | |||||
| if val >= len(Symbol): | |||||
| val = len(Symbol) | |||||
| if val < len(Symbol): | |||||
| if letter != Symbol[val]: | |||||
| val = 0 | |||||
| if val == len(Symbol): | |||||
| seperator.append(letter) | |||||
| #print('itsappending') | |||||
| if len(seperator) >= 1: | |||||
| seperators.append(''.join(seperator)) | |||||
| seperator = [] | |||||
| return seperators | |||||
| def ParseWithHighestLetterAccordance(self, inputtext, Letters): | |||||
| # first check if there is a word that has all letters | |||||
| short = False | |||||
| lettervect = [] | |||||
| Lettervector = [] | |||||
| wordscores = [] | |||||
| text = inputtext.lower() | |||||
| if '.' in set(Letters): | |||||
| short = True | |||||
| if short == True: | |||||
| for letter in re.sub("[^a-zA-Züäö.]", " ", Letters): | |||||
| letter = letter.lower() | |||||
| #print(re.sub("[^a-züäö.]", " ", Letters)) | |||||
| if letter != '.' and letter != ' ': | |||||
| lettervect.append(letter) | |||||
| if letter == '.': | |||||
| Lettervector.append(lettervect) | |||||
| lettervect = [] | |||||
| if len(lettervect) >= 1: | |||||
| Lettervector.append(lettervect) | |||||
| else: | |||||
| for letter in re.sub("[^a-zA-Züäö.]", " ", Letters): | |||||
| letter = letter.lower() | |||||
| Lettervector.append([letter]) | |||||
| #print(text) | |||||
| #print(Lettervector) | |||||
| from copy import deepcopy | |||||
| for word in text.split(): | |||||
| lettervector = deepcopy(Lettervector) | |||||
| #print(word) | |||||
| #print(Lettervector) | |||||
| wordscore = [] | |||||
| for n in range(len(lettervector)): | |||||
| wordscore.append([word, 0]) | |||||
| #wordscore = len(lettervector) * [[word, 0 ]] | |||||
| #print(wordscore) | |||||
| firstletter = 0 | |||||
| usedletters = [] | |||||
| for letter in word: | |||||
| firstletter += 1 | |||||
| #print(set(Letters)) | |||||
| #print(wordscore) | |||||
| #print(lettervector[n]) | |||||
| if firstletter == 1: | |||||
| if letter == lettervector[0][0]: | |||||
| #print('oi') | |||||
| #print(lettervector) | |||||
| #print(len(lettervector[2])) | |||||
| wordscore[0][1] += 1 | |||||
| lettervector[0].remove(letter) | |||||
| #print(usedletters) | |||||
| else: | |||||
| lettervector[0].remove(lettervector[0][0]) | |||||
| for n in range(len(lettervector)): | |||||
| #print('1' ,letter) | |||||
| #print(lettervector[n][0]) | |||||
| if letter in set(lettervector[n]): | |||||
| #print('ooioi',usedletters) | |||||
| if letter not in set(usedletters): | |||||
| #print('something was added', letter) | |||||
| wordscore[n][1] += 1 | |||||
| lettervector[n].remove(letter) | |||||
| #print('angesprungen') | |||||
| wordscores.append(wordscore) | |||||
| #print(wordscores) | |||||
| #checkbest_firstlettervector = [] | |||||
| #for n in range(len(wordscores)): | |||||
| #checkbest_firstlettervector.append([ n , wordscores[n][0][1]]) | |||||
| #print('wordscores', wordscores) | |||||
| #best_n_lettervectors = sorted(checkbest_firstlettervector[::-1], key=lambda tup: tup[1], reverse=True) | |||||
| #print(best_n_lettervectors) | |||||
| #for wordscore in wordscores: | |||||
| ntupelscores = [] | |||||
| ntupelscoresm = [] | |||||
| for o in range(len(wordscores)): | |||||
| #print('newlettervectorindex') | |||||
| lastletterexistentindex = 1 | |||||
| lastlettercame = False | |||||
| if wordscores[o][0][1] >= 1: | |||||
| for m in range(1, len(lettervector) + 1): | |||||
| #print(m) | |||||
| if o <= len(text.split()) - (m): | |||||
| triplescore = [] | |||||
| for q in range(len(wordscores[o])): | |||||
| triplescore.append(0) | |||||
| #print(len(lettervector)) | |||||
| for n in range(m): | |||||
| #print(wordscores[lettervectorindex[0] + n][n][1]) | |||||
| #wordscores[lettervectorindex[0] + 1][1][1] + wordscores[lettervectorindex[0] + 2][2][1] | |||||
| for p in range(len(wordscores[o])): | |||||
| #print(wordscore[o + n][p][1]) | |||||
| #print(len(Lettervector[p])) | |||||
| if wordscores[o + n][p][1] == len(Lettervector[p]): | |||||
| triplescore[p] += wordscores[o + n][p][1] | |||||
| letterlength = 0 | |||||
| for r in range(len(lettervector)): | |||||
| letterlength += len(Lettervector[r]) | |||||
| #print(wordscore) | |||||
| #print(sum(triplescore)) | |||||
| if p == len(wordscores[o]) - 1 and wordscores[o + n][p][1] == len(Lettervector[p]) and lastlettercame == False and sum(triplescore) == letterlength: | |||||
| #print('oioioioioioioooioioioiiiiiiiiiiiiiiiiiiiiiiiiiii') | |||||
| lastletterexistentindex = n | |||||
| lastlettercame = True | |||||
| #triplescore += wordscores[o + n][p][1] | |||||
| ntupelscores.append([[o , m, lastletterexistentindex], sum(triplescore)]) | |||||
| #ntupelscoresm.append([m , triplescore]) | |||||
| #print(text.split()) | |||||
| #print('bliblablub', ntupelscores) | |||||
| for tupel in ntupelscores: | |||||
| if text.split()[tupel[0][0]][0] == Lettervector[0][0]: | |||||
| tupel[1] += 3 | |||||
| #print('b',text.split()[tupel[0][0] + tupel[0][1] - 1][0]) | |||||
| #print('a',Lettervector[-1][0]) | |||||
| if text.split()[tupel[0][0] + tupel[0][1] - 1][0] == Lettervector[-1][0]: | |||||
| tupel[1] += 3 | |||||
| # Bestrafe laengere Tupel, sprich wenn durch weitere worte kein score dazukommt | |||||
| tupel[1] -= tupel[0][1] * 0.1 | |||||
| bestntupelscoresorted = sorted(ntupelscores[::-1], key=lambda tup: tup[1], reverse=True) | |||||
| #bestntupelscoresortedm = sorted(ntupelscoresm[::-1], key=lambda tup: tup[1], reverse=True) | |||||
| #print('oioioioioioioioioioi',bestntupelscoresorted) | |||||
| outputntupel = [] | |||||
| #print(bestntupelscoresorted) | |||||
| for s in range(bestntupelscoresorted[0][0][1] ): | |||||
| outputntupel.append(text.split()[bestntupelscoresorted[0][0][0] + s]) | |||||
| #print(outputntupel) | |||||
| return outputntupel | |||||
| #def parseWordsContainingCertainSymbols(self, text, symbols): | |||||
| #print() | |||||
| #fooSeparator = 'title' | |||||
| #cwd = os.getcwd() | |||||
| #with open('dewiktionary-20181201-pages-articles.xml') as xmldok: | |||||
| #with open(cwd + '/' + 'classes.txt', 'w') as Outdok: | |||||
| #n = 0 | |||||
| #done = False | |||||
| #while done == False: | |||||
| #for line in xmldok: | |||||
| #n += 1 | |||||
| ##print(line) | |||||
| ##print(dok_to_token(line)) | |||||
| ##print(n) | |||||
| #for word in line: | |||||
| #print(word) | |||||
| #try: | |||||
| #if dok_to_token(line)[:(len(fooSeparator) + 2)] == '<' + fooSeparator + '>': | |||||
| #Outdok.write(dok_to_token(line)[len(fooSeperator):-len(fooSeperator)] + '\n') | |||||
| #except: | |||||
| #pass | |||||
| #if n >= 100000: | |||||
| #quit() | |||||