You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
341 lines
11 KiB
341 lines
11 KiB
|
|
# The new class FASTsearch. Every DB can be represented in Lists. The Brain actually is constituted from lists. Access to all Documents almost the same moment.
|
|
|
|
# TODO GPU Multithreading has to be implemented.
|
|
|
|
|
|
|
|
# USAGE: Learn scikit-learn count vectorizer on a database of lines or docs.
|
|
|
|
import joblib
|
|
from sklearn.feature_extraction.text import CountVectorizer
|
|
|
|
import numpy as np
|
|
import scipy as sc
|
|
|
|
import tensorflow as tf
|
|
|
|
|
|
import _pickle as cPickle
|
|
|
|
import hickle as hkl
|
|
|
|
import os
|
|
|
|
|
|
# Define function to convert scipy csr matrix to tf tensor for working on gpu
|
|
def convert_sparse_matrix_to_sparse_tensor(X):
|
|
coo = sc.sparse.coo_matrix(X)
|
|
indices = np.mat([coo.row, coo.col]).transpose()
|
|
return tf.SparseTensorValue(indices, coo.data, coo.shape)
|
|
|
|
|
|
|
|
|
|
# The whole class is initialized with input of the database in [['word','word2'],[],[],[]] List format, 2 dimensional, the index of the list in the matrix defines its id
|
|
## in every list element of the input, each document is represented by one string
|
|
# This list must be saved as a hkl dump and then loaded into the database.
|
|
|
|
|
|
def my_tokenizer(s):
|
|
return s.split('\+')
|
|
|
|
class FASTsearch(object):
|
|
|
|
def __init__(self, DatabaseDir):
|
|
|
|
self.DatabaseDir = DatabaseDir[:-4]
|
|
|
|
database = []
|
|
hkl_load = hkl.load(DatabaseDir)
|
|
|
|
for element in hkl_load:
|
|
#print('element',element)
|
|
#print('joined element', ' '.join(element))
|
|
database.append(' '.join(element))
|
|
|
|
|
|
# input has to be hkl format
|
|
self.database = database
|
|
|
|
|
|
|
|
|
|
def Gen_BoW_Model(self, max_features, analyzer, punctuation = False):
|
|
|
|
print("Creating the bag of words...\n")
|
|
from sklearn.feature_extraction.text import CountVectorizer
|
|
|
|
# Initialize the "CountVectorizer" object, which is scikit-learn's
|
|
# bag of words tool.
|
|
if punctuation == False:
|
|
vectorizer = CountVectorizer(analyzer = analyzer, \
|
|
tokenizer = None, \
|
|
preprocessor = None, \
|
|
stop_words = None, \
|
|
max_features = max_features)
|
|
|
|
if punctuation == True:
|
|
vectorizer = CountVectorizer(analyzer = analyzer, \
|
|
tokenizer = my_tokenizer, \
|
|
preprocessor = None, \
|
|
stop_words = None, \
|
|
max_features = max_features)
|
|
|
|
|
|
# token_pattern = r'(?u)\w')
|
|
# fit_transform() does two functions: First, it fits the model
|
|
# and learns the vocabulary; second, it transforms our training data
|
|
# into feature vectors. The input to fit_transform should be a list of
|
|
# strings.
|
|
train_data_features = vectorizer.fit_transform(self.database)
|
|
|
|
joblib.dump(vectorizer, 'bagofwords' + self.DatabaseDir + '.pkl')
|
|
|
|
print('dumping the data to hkl format..')
|
|
hkl.dump(train_data_features, 'DataBaseOneZeros' + self.DatabaseDir + '.hkl', mode='w', compression='gzip')
|
|
print('done')
|
|
|
|
|
|
return vectorizer
|
|
|
|
def Load_BoW_Model(self, BoWModelDir, DatabaseOneZerosDir):
|
|
|
|
# input has to be pkl format
|
|
self.vectorizer = joblib.load(BoWModelDir)
|
|
|
|
self.dbOZ = hkl.load(DatabaseOneZerosDir).astype('float32')
|
|
|
|
|
|
return self.vectorizer
|
|
|
|
|
|
# input: string to search for in the documents, the numberofmatches to get the best n documents
|
|
# output the numberofmatches documents with their indexes on the database which is searched, the highest accordance number plus index [index, number]
|
|
|
|
def search(self, string , numberofmatches):
|
|
|
|
|
|
numberofmatches = numberofmatches
|
|
|
|
|
|
# Convert user input to Zeros and Ones
|
|
user_array = []
|
|
user_array.append(string)
|
|
|
|
user_input_OnesZeros = self.vectorizer.transform(user_array)
|
|
|
|
uOZ = user_input_OnesZeros.toarray()[0].astype(np.float32, copy=False)
|
|
|
|
|
|
uiOZ = uOZ[np.newaxis, :]
|
|
|
|
uiOZ = uiOZ.transpose()
|
|
|
|
sess = tf.Session()
|
|
with tf.device('/gpu:0'):
|
|
with sess.as_default():
|
|
|
|
uiOZ_tensor = tf.constant(uiOZ)
|
|
|
|
dbOZ_tensor_sparse = convert_sparse_matrix_to_sparse_tensor(self.dbOZ)
|
|
|
|
#uiOZ_tensor_sparse =tf.contrib.layers.dense_to_sparse(uiOZ_tensor, eos_token=0, outputs_collections=None, scope=None )
|
|
#dbOZ_tensor_sparse =tf.contrib.layers.dense_to_sparse(dbOZ_tensor, eos_token=0, outputs_collections=None, scope=None )
|
|
|
|
|
|
#wordCountDoku = tf.matmul(uiOZ_tensor, dbOZ_tensor)
|
|
wordCountDoku = tf.sparse_tensor_dense_matmul(dbOZ_tensor_sparse, uiOZ_tensor)
|
|
|
|
wCD = np.array(wordCountDoku.eval())
|
|
|
|
indexedwCD = []
|
|
for n in range(len(wCD)):
|
|
indexedwCD.append([n,wCD[n][0]])
|
|
|
|
|
|
indexedwCD = sorted(indexedwCD[::-1], key=lambda tup: tup[1], reverse=True)
|
|
|
|
best_n_documents = []
|
|
|
|
eq_number = 0
|
|
for number in uiOZ:
|
|
#print(number)
|
|
eq_number += number ** 2
|
|
|
|
#print(eq_number)
|
|
|
|
n = 0
|
|
done = False
|
|
while n < len(indexedwCD) and done == False:
|
|
n += 1
|
|
if indexedwCD[n][1] == eq_number:
|
|
best_n_documents = indexedwCD[n][0]
|
|
done = True
|
|
|
|
if indexedwCD[n][1] < eq_number:
|
|
best_n_documents = indexedwCD[n - 1][0]
|
|
done = True
|
|
|
|
|
|
#for n in range(numberofmatches):
|
|
|
|
#best_n_documents.append([indexedwCD[n][0], indexedwCD[n][1]])
|
|
|
|
|
|
return best_n_documents, indexedwCD[0]
|
|
|
|
def search_with_highest_multiplikation_Output(self, string , numberofmatches):
|
|
|
|
|
|
|
|
|
|
numberofmatches = numberofmatches
|
|
|
|
|
|
# Convert user input to Zeros and Ones
|
|
user_array = []
|
|
user_array.append(string)
|
|
|
|
user_input_OnesZeros = self.vectorizer.transform(user_array)
|
|
|
|
uOZ = user_input_OnesZeros.toarray()[0].astype(np.float32, copy=False)
|
|
|
|
|
|
uiOZ = uOZ[np.newaxis, :]
|
|
|
|
uiOZ = uiOZ.transpose()
|
|
|
|
sess = tf.Session()
|
|
with tf.device('/gpu:0'):
|
|
with sess.as_default():
|
|
|
|
uiOZ_tensor = tf.constant(uiOZ)
|
|
|
|
dbOZ_tensor_sparse = convert_sparse_matrix_to_sparse_tensor(self.dbOZ)
|
|
|
|
#uiOZ_tensor_sparse =tf.contrib.layers.dense_to_sparse(uiOZ_tensor, eos_token=0, outputs_collections=None, scope=None )
|
|
#dbOZ_tensor_sparse =tf.contrib.layers.dense_to_sparse(dbOZ_tensor, eos_token=0, outputs_collections=None, scope=None )
|
|
|
|
|
|
#wordCountDoku = tf.matmul(uiOZ_tensor, dbOZ_tensor)
|
|
wordCountDoku = tf.sparse_tensor_dense_matmul(dbOZ_tensor_sparse, uiOZ_tensor)
|
|
|
|
wCD = np.array(wordCountDoku.eval())
|
|
|
|
indexedwCD = []
|
|
for n in range(len(wCD)):
|
|
indexedwCD.append([n,wCD[n][0]])
|
|
|
|
|
|
indexedwCD = sorted(indexedwCD[::-1], key=lambda tup: tup[1], reverse=True)
|
|
|
|
best_n_documents = []
|
|
|
|
for n in range(numberofmatches):
|
|
best_n_documents.append(indexedwCD[n][0])
|
|
|
|
return best_n_documents, indexedwCD[0]
|
|
|
|
|
|
def searchPatternMatch(self, string , numberofmatches):
|
|
|
|
|
|
numberofmatches = numberofmatches
|
|
|
|
|
|
# Convert user input to Zeros and Ones
|
|
user_array = []
|
|
user_array.append(string)
|
|
|
|
user_input_OnesZeros = self.vectorizer.transform(user_array)
|
|
|
|
uOZ = user_input_OnesZeros.toarray()[0].astype(np.float32, copy=False)
|
|
|
|
|
|
uiOZ = uOZ[np.newaxis, :]
|
|
|
|
uiOZ = uiOZ.transpose()
|
|
|
|
sess = tf.Session()
|
|
with tf.device('/gpu:0'):
|
|
with sess.as_default():
|
|
|
|
uiOZ_tensor = tf.constant(uiOZ)
|
|
|
|
dbOZ_tensor_sparse = convert_sparse_matrix_to_sparse_tensor(self.dbOZ)
|
|
|
|
#uiOZ_tensor_sparse =tf.contrib.layers.dense_to_sparse(uiOZ_tensor, eos_token=0, outputs_collections=None, scope=None )
|
|
#dbOZ_tensor_sparse =tf.contrib.layers.dense_to_sparse(dbOZ_tensor, eos_token=0, outputs_collections=None, scope=None )
|
|
|
|
|
|
#wordCountDoku = tf.matmul(uiOZ_tensor, dbOZ_tensor)
|
|
wordCountDoku = tf.sparse_tensor_dense_matmul(dbOZ_tensor_sparse, uiOZ_tensor)
|
|
|
|
wCD = np.array(wordCountDoku.eval())
|
|
|
|
indexedwCD = []
|
|
for n in range(len(wCD)):
|
|
indexedwCD.append([n,wCD[n][0]])
|
|
|
|
# Sort the biggest matches
|
|
indexedwCD = sorted(indexedwCD[::-1], key=lambda tup: tup[1], reverse=True)
|
|
|
|
best_n_documents = []
|
|
|
|
best_docs_surrounding = []
|
|
|
|
|
|
# Get the number which is result when same words would be in the document as in one grammar scheme
|
|
eq_number = 0
|
|
for number in uiOZ:
|
|
#print(number)
|
|
eq_number += number ** 2
|
|
|
|
print(eq_number)
|
|
|
|
# Create new array of closest grammar schemes, I have chosen around 3 (in the matchnumber, not regarding words or so)
|
|
n = 0
|
|
done = False
|
|
while n < len(indexedwCD) and done == False:
|
|
n += 1
|
|
#print('a',indexedwCD)
|
|
#print('oo', indexedwCD[n])
|
|
if indexedwCD[n][1] == eq_number:
|
|
best_docs_surrounding.append(indexedwCD[n][0])
|
|
|
|
#if indexedwCD[n][1] < eq_number:
|
|
#best_docs_surrounding.append(indexedwCD[n][0])
|
|
|
|
if indexedwCD[n][1] < eq_number :
|
|
done = True
|
|
|
|
|
|
# Count for these docs in surrounding the matches of wordnumbers per word
|
|
# would be much faster when using the sparse class
|
|
|
|
best_docs_surrounding_new = []
|
|
for doc in best_docs_surrounding:
|
|
dok_BoW = self.dbOZ[doc].toarray()[0].astype(np.float32, copy=False)
|
|
Number_equal_words = 0
|
|
for n in range(len(uiOZ)):
|
|
#print(uiOZ[n])
|
|
#print(dok_BoW[n])
|
|
#print('dok_BoW',dok_BoW)
|
|
if uiOZ[n] == dok_BoW[n]:
|
|
Number_equal_words += 1
|
|
best_docs_surrounding_new.append([doc , Number_equal_words])
|
|
|
|
# Sort the result again with the original indexes
|
|
best_n_documents = sorted(best_docs_surrounding_new[::-1], key=lambda tup: tup[1], reverse=True)
|
|
|
|
|
|
|
|
#for n in range(numberofmatches):
|
|
|
|
#best_n_documents.append([indexedwCD[n][0], indexedwCD[n][1]])
|
|
|
|
|
|
return best_n_documents
|
|
|
|
|