You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1578 lines
52 KiB
1578 lines
52 KiB
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright (C) 2011 Radim Rehurek <radimrehurek@seznam.cz>
|
|
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html
|
|
|
|
"""Math helper functions."""
|
|
|
|
from __future__ import with_statement
|
|
|
|
|
|
from itertools import chain
|
|
import logging
|
|
import math
|
|
|
|
from gensim import utils
|
|
|
|
import numpy as np
|
|
import scipy.sparse
|
|
from scipy.stats import entropy
|
|
import scipy.linalg
|
|
from scipy.linalg.lapack import get_lapack_funcs
|
|
from scipy.linalg.special_matrices import triu
|
|
from scipy.special import psi # gamma function utils
|
|
|
|
from six import iteritems, itervalues, string_types
|
|
from six.moves import xrange, zip as izip
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def blas(name, ndarray):
|
|
"""Helper for getting the appropriate BLAS function, using :func:`scipy.linalg.get_blas_funcs`.
|
|
|
|
Parameters
|
|
----------
|
|
name : str
|
|
Name(s) of BLAS functions, without the type prefix.
|
|
ndarray : numpy.ndarray
|
|
Arrays can be given to determine optimal prefix of BLAS routines.
|
|
|
|
Returns
|
|
-------
|
|
object
|
|
BLAS function for the needed operation on the given data type.
|
|
|
|
"""
|
|
return scipy.linalg.get_blas_funcs((name,), (ndarray,))[0]
|
|
|
|
|
|
def argsort(x, topn=None, reverse=False):
|
|
"""Efficiently calculate indices of the `topn` smallest elements in array `x`.
|
|
|
|
Parameters
|
|
----------
|
|
x : array_like
|
|
Array to get the smallest element indices from.
|
|
topn : int, optional
|
|
Number of indices of the smallest (greatest) elements to be returned.
|
|
If not given, indices of all elements will be returned in ascending (descending) order.
|
|
reverse : bool, optional
|
|
Return the `topn` greatest elements in descending order,
|
|
instead of smallest elements in ascending order?
|
|
|
|
Returns
|
|
-------
|
|
numpy.ndarray
|
|
Array of `topn` indices that sort the array in the requested order.
|
|
|
|
"""
|
|
x = np.asarray(x) # unify code path for when `x` is not a np array (list, tuple...)
|
|
if topn is None:
|
|
topn = x.size
|
|
if topn <= 0:
|
|
return []
|
|
if reverse:
|
|
x = -x
|
|
if topn >= x.size or not hasattr(np, 'argpartition'):
|
|
return np.argsort(x)[:topn]
|
|
# np >= 1.8 has a fast partial argsort, use that!
|
|
most_extreme = np.argpartition(x, topn)[:topn]
|
|
return most_extreme.take(np.argsort(x.take(most_extreme))) # resort topn into order
|
|
|
|
|
|
def corpus2csc(corpus, num_terms=None, dtype=np.float64, num_docs=None, num_nnz=None, printprogress=0):
|
|
"""Convert a streamed corpus in bag-of-words format into a sparse matrix `scipy.sparse.csc_matrix`,
|
|
with documents as columns.
|
|
|
|
Notes
|
|
-----
|
|
If the number of terms, documents and non-zero elements is known, you can pass
|
|
them here as parameters and a (much) more memory efficient code path will be taken.
|
|
|
|
Parameters
|
|
----------
|
|
corpus : iterable of iterable of (int, number)
|
|
Input corpus in BoW format
|
|
num_terms : int, optional
|
|
Number of terms in `corpus`. If provided, the `corpus.num_terms` attribute (if any) will be ignored.
|
|
dtype : data-type, optional
|
|
Data type of output CSC matrix.
|
|
num_docs : int, optional
|
|
Number of documents in `corpus`. If provided, the `corpus.num_docs` attribute (in any) will be ignored.
|
|
num_nnz : int, optional
|
|
Number of non-zero elements in `corpus`. If provided, the `corpus.num_nnz` attribute (if any) will be ignored.
|
|
printprogress : int, optional
|
|
Log a progress message at INFO level once every `printprogress` documents. 0 to turn off progress logging.
|
|
|
|
Returns
|
|
-------
|
|
scipy.sparse.csc_matrix
|
|
`corpus` converted into a sparse CSC matrix.
|
|
|
|
See Also
|
|
--------
|
|
:class:`~gensim.matutils.Sparse2Corpus`
|
|
Convert sparse format to Gensim corpus format.
|
|
|
|
"""
|
|
try:
|
|
# if the input corpus has the `num_nnz`, `num_docs` and `num_terms` attributes
|
|
# (as is the case with MmCorpus for example), we can use a more efficient code path
|
|
if num_terms is None:
|
|
num_terms = corpus.num_terms
|
|
if num_docs is None:
|
|
num_docs = corpus.num_docs
|
|
if num_nnz is None:
|
|
num_nnz = corpus.num_nnz
|
|
except AttributeError:
|
|
pass # not a MmCorpus...
|
|
if printprogress:
|
|
logger.info("creating sparse matrix from corpus")
|
|
if num_terms is not None and num_docs is not None and num_nnz is not None:
|
|
# faster and much more memory-friendly version of creating the sparse csc
|
|
posnow, indptr = 0, [0]
|
|
indices = np.empty((num_nnz,), dtype=np.int32) # HACK assume feature ids fit in 32bit integer
|
|
data = np.empty((num_nnz,), dtype=dtype)
|
|
for docno, doc in enumerate(corpus):
|
|
if printprogress and docno % printprogress == 0:
|
|
logger.info("PROGRESS: at document #%i/%i", docno, num_docs)
|
|
posnext = posnow + len(doc)
|
|
indices[posnow: posnext] = [feature_id for feature_id, _ in doc]
|
|
data[posnow: posnext] = [feature_weight for _, feature_weight in doc]
|
|
indptr.append(posnext)
|
|
posnow = posnext
|
|
assert posnow == num_nnz, "mismatch between supplied and computed number of non-zeros"
|
|
result = scipy.sparse.csc_matrix((data, indices, indptr), shape=(num_terms, num_docs), dtype=dtype)
|
|
else:
|
|
# slower version; determine the sparse matrix parameters during iteration
|
|
num_nnz, data, indices, indptr = 0, [], [], [0]
|
|
for docno, doc in enumerate(corpus):
|
|
if printprogress and docno % printprogress == 0:
|
|
logger.info("PROGRESS: at document #%i", docno)
|
|
indices.extend([feature_id for feature_id, _ in doc])
|
|
data.extend([feature_weight for _, feature_weight in doc])
|
|
num_nnz += len(doc)
|
|
indptr.append(num_nnz)
|
|
if num_terms is None:
|
|
num_terms = max(indices) + 1 if indices else 0
|
|
num_docs = len(indptr) - 1
|
|
# now num_docs, num_terms and num_nnz contain the correct values
|
|
data = np.asarray(data, dtype=dtype)
|
|
indices = np.asarray(indices)
|
|
result = scipy.sparse.csc_matrix((data, indices, indptr), shape=(num_terms, num_docs), dtype=dtype)
|
|
return result
|
|
|
|
|
|
def pad(mat, padrow, padcol):
|
|
"""Add additional rows/columns to `mat`. The new rows/columns will be initialized with zeros.
|
|
|
|
Parameters
|
|
----------
|
|
mat : numpy.ndarray
|
|
Input 2D matrix
|
|
padrow : int
|
|
Number of additional rows
|
|
padcol : int
|
|
Number of additional columns
|
|
|
|
Returns
|
|
-------
|
|
numpy.matrixlib.defmatrix.matrix
|
|
Matrix with needed padding.
|
|
|
|
"""
|
|
if padrow < 0:
|
|
padrow = 0
|
|
if padcol < 0:
|
|
padcol = 0
|
|
rows, cols = mat.shape
|
|
return np.bmat([
|
|
[mat, np.matrix(np.zeros((rows, padcol)))],
|
|
[np.matrix(np.zeros((padrow, cols + padcol)))],
|
|
])
|
|
|
|
|
|
def zeros_aligned(shape, dtype, order='C', align=128):
|
|
"""Get array aligned at `align` byte boundary in memory.
|
|
|
|
Parameters
|
|
----------
|
|
shape : int or (int, int)
|
|
Shape of array.
|
|
dtype : data-type
|
|
Data type of array.
|
|
order : {'C', 'F'}, optional
|
|
Whether to store multidimensional data in C- or Fortran-contiguous (row- or column-wise) order in memory.
|
|
align : int, optional
|
|
Boundary for alignment in bytes.
|
|
|
|
Returns
|
|
-------
|
|
numpy.ndarray
|
|
Aligned array.
|
|
|
|
"""
|
|
nbytes = np.prod(shape, dtype=np.int64) * np.dtype(dtype).itemsize
|
|
buffer = np.zeros(nbytes + align, dtype=np.uint8) # problematic on win64 ("maximum allowed dimension exceeded")
|
|
start_index = -buffer.ctypes.data % align
|
|
return buffer[start_index: start_index + nbytes].view(dtype).reshape(shape, order=order)
|
|
|
|
|
|
def ismatrix(m):
|
|
"""Check whether `m` is a 2D `numpy.ndarray` or `scipy.sparse` matrix.
|
|
|
|
Parameters
|
|
----------
|
|
m : object
|
|
Object to check.
|
|
|
|
Returns
|
|
-------
|
|
bool
|
|
Is `m` a 2D `numpy.ndarray` or `scipy.sparse` matrix.
|
|
|
|
"""
|
|
return isinstance(m, np.ndarray) and m.ndim == 2 or scipy.sparse.issparse(m)
|
|
|
|
|
|
def any2sparse(vec, eps=1e-9):
|
|
"""Convert a numpy.ndarray or `scipy.sparse` vector into the Gensim bag-of-words format.
|
|
|
|
Parameters
|
|
----------
|
|
vec : {`numpy.ndarray`, `scipy.sparse`}
|
|
Input vector
|
|
eps : float, optional
|
|
Value used for threshold, all coordinates less than `eps` will not be presented in result.
|
|
|
|
Returns
|
|
-------
|
|
list of (int, float)
|
|
Vector in BoW format.
|
|
|
|
"""
|
|
if isinstance(vec, np.ndarray):
|
|
return dense2vec(vec, eps)
|
|
if scipy.sparse.issparse(vec):
|
|
return scipy2sparse(vec, eps)
|
|
return [(int(fid), float(fw)) for fid, fw in vec if np.abs(fw) > eps]
|
|
|
|
|
|
def scipy2scipy_clipped(matrix, topn, eps=1e-9):
|
|
"""Get the 'topn' elements of the greatest magnitude (absolute value) from a `scipy.sparse` vector or matrix.
|
|
|
|
Parameters
|
|
----------
|
|
matrix : `scipy.sparse`
|
|
Input vector or matrix (1D or 2D sparse array).
|
|
topn : int
|
|
Number of greatest elements, in absolute value, to return.
|
|
eps : float
|
|
Ignored.
|
|
|
|
Returns
|
|
-------
|
|
`scipy.sparse.csr.csr_matrix`
|
|
Clipped matrix.
|
|
|
|
"""
|
|
if not scipy.sparse.issparse(matrix):
|
|
raise ValueError("'%s' is not a scipy sparse vector." % matrix)
|
|
if topn <= 0:
|
|
return scipy.sparse.csr_matrix([])
|
|
# Return clipped sparse vector if input is a sparse vector.
|
|
if matrix.shape[0] == 1:
|
|
# use np.argpartition/argsort and only form tuples that are actually returned.
|
|
biggest = argsort(abs(matrix.data), topn, reverse=True)
|
|
indices, data = matrix.indices.take(biggest), matrix.data.take(biggest)
|
|
return scipy.sparse.csr_matrix((data, indices, [0, len(indices)]))
|
|
# Return clipped sparse matrix if input is a matrix, processing row by row.
|
|
else:
|
|
matrix_indices = []
|
|
matrix_data = []
|
|
matrix_indptr = [0]
|
|
# calling abs() on entire matrix once is faster than calling abs() iteratively for each row
|
|
matrix_abs = abs(matrix)
|
|
for i in range(matrix.shape[0]):
|
|
v = matrix.getrow(i)
|
|
v_abs = matrix_abs.getrow(i)
|
|
# Sort and clip each row vector first.
|
|
biggest = argsort(v_abs.data, topn, reverse=True)
|
|
indices, data = v.indices.take(biggest), v.data.take(biggest)
|
|
# Store the topn indices and values of each row vector.
|
|
matrix_data.append(data)
|
|
matrix_indices.append(indices)
|
|
matrix_indptr.append(matrix_indptr[-1] + min(len(indices), topn))
|
|
matrix_indices = np.concatenate(matrix_indices).ravel()
|
|
matrix_data = np.concatenate(matrix_data).ravel()
|
|
# Instantiate and return a sparse csr_matrix which preserves the order of indices/data.
|
|
return scipy.sparse.csr.csr_matrix(
|
|
(matrix_data, matrix_indices, matrix_indptr),
|
|
shape=(matrix.shape[0], np.max(matrix_indices) + 1)
|
|
)
|
|
|
|
|
|
def scipy2sparse(vec, eps=1e-9):
|
|
"""Convert a scipy.sparse vector into the Gensim bag-of-words format.
|
|
|
|
Parameters
|
|
----------
|
|
vec : `scipy.sparse`
|
|
Sparse vector.
|
|
|
|
eps : float, optional
|
|
Value used for threshold, all coordinates less than `eps` will not be presented in result.
|
|
|
|
Returns
|
|
-------
|
|
list of (int, float)
|
|
Vector in Gensim bag-of-words format.
|
|
|
|
"""
|
|
vec = vec.tocsr()
|
|
assert vec.shape[0] == 1
|
|
return [(int(pos), float(val)) for pos, val in zip(vec.indices, vec.data) if np.abs(val) > eps]
|
|
|
|
|
|
class Scipy2Corpus(object):
|
|
"""Convert a sequence of dense/sparse vectors into a streamed Gensim corpus object.
|
|
|
|
See Also
|
|
--------
|
|
:func:`~gensim.matutils.corpus2csc`
|
|
Convert corpus in Gensim format to `scipy.sparse.csc` matrix.
|
|
|
|
"""
|
|
def __init__(self, vecs):
|
|
"""
|
|
|
|
Parameters
|
|
----------
|
|
vecs : iterable of {`numpy.ndarray`, `scipy.sparse`}
|
|
Input vectors.
|
|
|
|
"""
|
|
self.vecs = vecs
|
|
|
|
def __iter__(self):
|
|
for vec in self.vecs:
|
|
if isinstance(vec, np.ndarray):
|
|
yield full2sparse(vec)
|
|
else:
|
|
yield scipy2sparse(vec)
|
|
|
|
def __len__(self):
|
|
return len(self.vecs)
|
|
|
|
|
|
def sparse2full(doc, length):
|
|
"""Convert a document in Gensim bag-of-words format into a dense numpy array.
|
|
|
|
Parameters
|
|
----------
|
|
doc : list of (int, number)
|
|
Document in BoW format.
|
|
length : int
|
|
Vector dimensionality. This cannot be inferred from the BoW, and you must supply it explicitly.
|
|
This is typically the vocabulary size or number of topics, depending on how you created `doc`.
|
|
|
|
Returns
|
|
-------
|
|
numpy.ndarray
|
|
Dense numpy vector for `doc`.
|
|
|
|
See Also
|
|
--------
|
|
:func:`~gensim.matutils.full2sparse`
|
|
Convert dense array to gensim bag-of-words format.
|
|
|
|
"""
|
|
result = np.zeros(length, dtype=np.float32) # fill with zeroes (default value)
|
|
# convert indices to int as numpy 1.12 no longer indexes by floats
|
|
doc = ((int(id_), float(val_)) for (id_, val_) in doc)
|
|
|
|
doc = dict(doc)
|
|
# overwrite some of the zeroes with explicit values
|
|
result[list(doc)] = list(itervalues(doc))
|
|
return result
|
|
|
|
|
|
def full2sparse(vec, eps=1e-9):
|
|
"""Convert a dense numpy array into the Gensim bag-of-words format.
|
|
|
|
Parameters
|
|
----------
|
|
vec : numpy.ndarray
|
|
Dense input vector.
|
|
eps : float
|
|
Feature weight threshold value. Features with `abs(weight) < eps` are considered sparse and
|
|
won't be included in the BOW result.
|
|
|
|
Returns
|
|
-------
|
|
list of (int, float)
|
|
BoW format of `vec`, with near-zero values omitted (sparse vector).
|
|
|
|
See Also
|
|
--------
|
|
:func:`~gensim.matutils.sparse2full`
|
|
Convert a document in Gensim bag-of-words format into a dense numpy array.
|
|
|
|
"""
|
|
vec = np.asarray(vec, dtype=float)
|
|
nnz = np.nonzero(abs(vec) > eps)[0]
|
|
return list(zip(nnz, vec.take(nnz)))
|
|
|
|
|
|
dense2vec = full2sparse
|
|
|
|
|
|
def full2sparse_clipped(vec, topn, eps=1e-9):
|
|
"""Like :func:`~gensim.matutils.full2sparse`, but only return the `topn` elements of the greatest magnitude (abs).
|
|
|
|
This is more efficient that sorting a vector and then taking the greatest values, especially
|
|
where `len(vec) >> topn`.
|
|
|
|
Parameters
|
|
----------
|
|
vec : numpy.ndarray
|
|
Input dense vector
|
|
topn : int
|
|
Number of greatest (abs) elements that will be presented in result.
|
|
eps : float
|
|
Threshold value, if coordinate in `vec` < eps, this will not be presented in result.
|
|
|
|
Returns
|
|
-------
|
|
list of (int, float)
|
|
Clipped vector in BoW format.
|
|
|
|
See Also
|
|
--------
|
|
:func:`~gensim.matutils.full2sparse`
|
|
Convert dense array to gensim bag-of-words format.
|
|
|
|
"""
|
|
# use np.argpartition/argsort and only form tuples that are actually returned.
|
|
# this is about 40x faster than explicitly forming all 2-tuples to run sort() or heapq.nlargest() on.
|
|
if topn <= 0:
|
|
return []
|
|
vec = np.asarray(vec, dtype=float)
|
|
nnz = np.nonzero(abs(vec) > eps)[0]
|
|
biggest = nnz.take(argsort(abs(vec).take(nnz), topn, reverse=True))
|
|
return list(zip(biggest, vec.take(biggest)))
|
|
|
|
|
|
def corpus2dense(corpus, num_terms, num_docs=None, dtype=np.float32):
|
|
"""Convert corpus into a dense numpy 2D array, with documents as columns.
|
|
|
|
Parameters
|
|
----------
|
|
corpus : iterable of iterable of (int, number)
|
|
Input corpus in the Gensim bag-of-words format.
|
|
num_terms : int
|
|
Number of terms in the dictionary. X-axis of the resulting matrix.
|
|
num_docs : int, optional
|
|
Number of documents in the corpus. If provided, a slightly more memory-efficient code path is taken.
|
|
Y-axis of the resulting matrix.
|
|
dtype : data-type, optional
|
|
Data type of the output matrix.
|
|
|
|
Returns
|
|
-------
|
|
numpy.ndarray
|
|
Dense 2D array that presents `corpus`.
|
|
|
|
See Also
|
|
--------
|
|
:class:`~gensim.matutils.Dense2Corpus`
|
|
Convert dense matrix to Gensim corpus format.
|
|
|
|
"""
|
|
if num_docs is not None:
|
|
# we know the number of documents => don't bother column_stacking
|
|
docno, result = -1, np.empty((num_terms, num_docs), dtype=dtype)
|
|
for docno, doc in enumerate(corpus):
|
|
result[:, docno] = sparse2full(doc, num_terms)
|
|
assert docno + 1 == num_docs
|
|
else:
|
|
result = np.column_stack(sparse2full(doc, num_terms) for doc in corpus)
|
|
return result.astype(dtype)
|
|
|
|
|
|
class Dense2Corpus(object):
|
|
"""Treat dense numpy array as a streamed Gensim corpus in the bag-of-words format.
|
|
|
|
Notes
|
|
-----
|
|
No data copy is made (changes to the underlying matrix imply changes in the streamed corpus).
|
|
|
|
See Also
|
|
--------
|
|
:func:`~gensim.matutils.corpus2dense`
|
|
Convert Gensim corpus to dense matrix.
|
|
:class:`~gensim.matutils.Sparse2Corpus`
|
|
Convert sparse matrix to Gensim corpus format.
|
|
|
|
"""
|
|
def __init__(self, dense, documents_columns=True):
|
|
"""
|
|
|
|
Parameters
|
|
----------
|
|
dense : numpy.ndarray
|
|
Corpus in dense format.
|
|
documents_columns : bool, optional
|
|
Documents in `dense` represented as columns, as opposed to rows?
|
|
|
|
"""
|
|
if documents_columns:
|
|
self.dense = dense.T
|
|
else:
|
|
self.dense = dense
|
|
|
|
def __iter__(self):
|
|
"""Iterate over the corpus.
|
|
|
|
Yields
|
|
------
|
|
list of (int, float)
|
|
Document in BoW format.
|
|
|
|
"""
|
|
for doc in self.dense:
|
|
yield full2sparse(doc.flat)
|
|
|
|
def __len__(self):
|
|
return len(self.dense)
|
|
|
|
|
|
class Sparse2Corpus(object):
|
|
"""Convert a matrix in scipy.sparse format into a streaming Gensim corpus.
|
|
|
|
See Also
|
|
--------
|
|
:func:`~gensim.matutils.corpus2csc`
|
|
Convert gensim corpus format to `scipy.sparse.csc` matrix
|
|
:class:`~gensim.matutils.Dense2Corpus`
|
|
Convert dense matrix to gensim corpus.
|
|
|
|
"""
|
|
def __init__(self, sparse, documents_columns=True):
|
|
"""
|
|
|
|
Parameters
|
|
----------
|
|
sparse : `scipy.sparse`
|
|
Corpus scipy sparse format
|
|
documents_columns : bool, optional
|
|
Documents will be column?
|
|
|
|
"""
|
|
if documents_columns:
|
|
self.sparse = sparse.tocsc()
|
|
else:
|
|
self.sparse = sparse.tocsr().T # make sure shape[1]=number of docs (needed in len())
|
|
|
|
def __iter__(self):
|
|
"""
|
|
|
|
Yields
|
|
------
|
|
list of (int, float)
|
|
Document in BoW format.
|
|
|
|
"""
|
|
for indprev, indnow in izip(self.sparse.indptr, self.sparse.indptr[1:]):
|
|
yield list(zip(self.sparse.indices[indprev:indnow], self.sparse.data[indprev:indnow]))
|
|
|
|
def __len__(self):
|
|
return self.sparse.shape[1]
|
|
|
|
def __getitem__(self, document_index):
|
|
"""Retrieve a document vector from the corpus by its index.
|
|
|
|
Parameters
|
|
----------
|
|
document_index : int
|
|
Index of document
|
|
|
|
Returns
|
|
-------
|
|
list of (int, number)
|
|
Document in BoW format.
|
|
|
|
"""
|
|
indprev = self.sparse.indptr[document_index]
|
|
indnow = self.sparse.indptr[document_index + 1]
|
|
return list(zip(self.sparse.indices[indprev:indnow], self.sparse.data[indprev:indnow]))
|
|
|
|
|
|
def veclen(vec):
|
|
"""Calculate L2 (euclidean) length of a vector.
|
|
|
|
Parameters
|
|
----------
|
|
vec : list of (int, number)
|
|
Input vector in sparse bag-of-words format.
|
|
|
|
Returns
|
|
-------
|
|
float
|
|
Length of `vec`.
|
|
|
|
"""
|
|
if len(vec) == 0:
|
|
return 0.0
|
|
length = 1.0 * math.sqrt(sum(val**2 for _, val in vec))
|
|
assert length > 0.0, "sparse documents must not contain any explicit zero entries"
|
|
return length
|
|
|
|
|
|
def ret_normalized_vec(vec, length):
|
|
"""Normalize a vector in L2 (Euclidean unit norm).
|
|
|
|
Parameters
|
|
----------
|
|
vec : list of (int, number)
|
|
Input vector in BoW format.
|
|
length : float
|
|
Length of vector
|
|
|
|
Returns
|
|
-------
|
|
list of (int, number)
|
|
L2-normalized vector in BoW format.
|
|
|
|
"""
|
|
if length != 1.0:
|
|
return [(termid, val / length) for termid, val in vec]
|
|
else:
|
|
return list(vec)
|
|
|
|
|
|
def ret_log_normalize_vec(vec, axis=1):
|
|
log_max = 100.0
|
|
if len(vec.shape) == 1:
|
|
max_val = np.max(vec)
|
|
log_shift = log_max - np.log(len(vec) + 1.0) - max_val
|
|
tot = np.sum(np.exp(vec + log_shift))
|
|
log_norm = np.log(tot) - log_shift
|
|
vec -= log_norm
|
|
else:
|
|
if axis == 1: # independently normalize each sample
|
|
max_val = np.max(vec, 1)
|
|
log_shift = log_max - np.log(vec.shape[1] + 1.0) - max_val
|
|
tot = np.sum(np.exp(vec + log_shift[:, np.newaxis]), 1)
|
|
log_norm = np.log(tot) - log_shift
|
|
vec = vec - log_norm[:, np.newaxis]
|
|
elif axis == 0: # normalize each feature
|
|
k = ret_log_normalize_vec(vec.T)
|
|
return k[0].T, k[1]
|
|
else:
|
|
raise ValueError("'%s' is not a supported axis" % axis)
|
|
return vec, log_norm
|
|
|
|
|
|
blas_nrm2 = blas('nrm2', np.array([], dtype=float))
|
|
blas_scal = blas('scal', np.array([], dtype=float))
|
|
|
|
|
|
def unitvec(vec, norm='l2', return_norm=False):
|
|
"""Scale a vector to unit length.
|
|
|
|
Parameters
|
|
----------
|
|
vec : {numpy.ndarray, scipy.sparse, list of (int, float)}
|
|
Input vector in any format
|
|
norm : {'l1', 'l2'}, optional
|
|
Metric to normalize in.
|
|
return_norm : bool, optional
|
|
Return the length of vector `vec`, in addition to the normalized vector itself?
|
|
|
|
Returns
|
|
-------
|
|
numpy.ndarray, scipy.sparse, list of (int, float)}
|
|
Normalized vector in same format as `vec`.
|
|
float
|
|
Length of `vec` before normalization, if `return_norm` is set.
|
|
|
|
Notes
|
|
-----
|
|
Zero-vector will be unchanged.
|
|
|
|
"""
|
|
if norm not in ('l1', 'l2'):
|
|
raise ValueError("'%s' is not a supported norm. Currently supported norms are 'l1' and 'l2'." % norm)
|
|
|
|
if scipy.sparse.issparse(vec):
|
|
vec = vec.tocsr()
|
|
if norm == 'l1':
|
|
veclen = np.sum(np.abs(vec.data))
|
|
if norm == 'l2':
|
|
veclen = np.sqrt(np.sum(vec.data ** 2))
|
|
if veclen > 0.0:
|
|
if np.issubdtype(vec.dtype, np.int):
|
|
vec = vec.astype(np.float)
|
|
vec /= veclen
|
|
if return_norm:
|
|
return vec, veclen
|
|
else:
|
|
return vec
|
|
else:
|
|
if return_norm:
|
|
return vec, 1.
|
|
else:
|
|
return vec
|
|
|
|
if isinstance(vec, np.ndarray):
|
|
if norm == 'l1':
|
|
veclen = np.sum(np.abs(vec))
|
|
if norm == 'l2':
|
|
veclen = blas_nrm2(vec)
|
|
if veclen > 0.0:
|
|
if np.issubdtype(vec.dtype, np.int):
|
|
vec = vec.astype(np.float)
|
|
if return_norm:
|
|
return blas_scal(1.0 / veclen, vec).astype(vec.dtype), veclen
|
|
else:
|
|
return blas_scal(1.0 / veclen, vec).astype(vec.dtype)
|
|
else:
|
|
if return_norm:
|
|
return vec, 1
|
|
else:
|
|
return vec
|
|
|
|
try:
|
|
first = next(iter(vec)) # is there at least one element?
|
|
except StopIteration:
|
|
return vec
|
|
|
|
if isinstance(first, (tuple, list)) and len(first) == 2: # gensim sparse format
|
|
if norm == 'l1':
|
|
length = float(sum(abs(val) for _, val in vec))
|
|
if norm == 'l2':
|
|
length = 1.0 * math.sqrt(sum(val ** 2 for _, val in vec))
|
|
assert length > 0.0, "sparse documents must not contain any explicit zero entries"
|
|
if return_norm:
|
|
return ret_normalized_vec(vec, length), length
|
|
else:
|
|
return ret_normalized_vec(vec, length)
|
|
else:
|
|
raise ValueError("unknown input type")
|
|
|
|
|
|
def cossim(vec1, vec2):
|
|
"""Get cosine similarity between two sparse vectors.
|
|
|
|
Cosine similarity is a number between `<-1.0, 1.0>`, higher means more similar.
|
|
|
|
Parameters
|
|
----------
|
|
vec1 : list of (int, float)
|
|
Vector in BoW format.
|
|
vec2 : list of (int, float)
|
|
Vector in BoW format.
|
|
|
|
Returns
|
|
-------
|
|
float
|
|
Cosine similarity between `vec1` and `vec2`.
|
|
|
|
"""
|
|
vec1, vec2 = dict(vec1), dict(vec2)
|
|
if not vec1 or not vec2:
|
|
return 0.0
|
|
vec1len = 1.0 * math.sqrt(sum(val * val for val in itervalues(vec1)))
|
|
vec2len = 1.0 * math.sqrt(sum(val * val for val in itervalues(vec2)))
|
|
assert vec1len > 0.0 and vec2len > 0.0, "sparse documents must not contain any explicit zero entries"
|
|
if len(vec2) < len(vec1):
|
|
vec1, vec2 = vec2, vec1 # swap references so that we iterate over the shorter vector
|
|
result = sum(value * vec2.get(index, 0.0) for index, value in iteritems(vec1))
|
|
result /= vec1len * vec2len # rescale by vector lengths
|
|
return result
|
|
|
|
|
|
def softcossim(vec1, vec2, similarity_matrix):
|
|
"""Get Soft Cosine Measure between two vectors given a term similarity matrix.
|
|
|
|
Return Soft Cosine Measure between two sparse vectors given a sparse term similarity matrix
|
|
in the :class:`scipy.sparse.csc_matrix` format. The similarity is a number between `<-1.0, 1.0>`,
|
|
higher is more similar.
|
|
|
|
Notes
|
|
-----
|
|
Soft Cosine Measure was perhaps first defined by `Grigori Sidorov et al.,
|
|
"Soft Similarity and Soft Cosine Measure: Similarity of Features in Vector Space Model"
|
|
<http://www.cys.cic.ipn.mx/ojs/index.php/CyS/article/view/2043/1921>`_.
|
|
|
|
Parameters
|
|
----------
|
|
vec1 : list of (int, float)
|
|
A query vector in the BoW format.
|
|
vec2 : list of (int, float)
|
|
A document vector in the BoW format.
|
|
similarity_matrix : {:class:`scipy.sparse.csc_matrix`, :class:`scipy.sparse.csr_matrix`}
|
|
A term similarity matrix, typically produced by
|
|
:meth:`~gensim.models.keyedvectors.WordEmbeddingsKeyedVectors.similarity_matrix`.
|
|
|
|
Returns
|
|
-------
|
|
`similarity_matrix.dtype`
|
|
The Soft Cosine Measure between `vec1` and `vec2`.
|
|
|
|
Raises
|
|
------
|
|
ValueError
|
|
When the term similarity matrix is in an unknown format.
|
|
|
|
See Also
|
|
--------
|
|
:meth:`gensim.models.keyedvectors.WordEmbeddingsKeyedVectors.similarity_matrix`
|
|
A term similarity matrix produced from term embeddings.
|
|
:class:`gensim.similarities.docsim.SoftCosineSimilarity`
|
|
A class for performing corpus-based similarity queries with Soft Cosine Measure.
|
|
|
|
"""
|
|
if not isinstance(similarity_matrix, scipy.sparse.csc_matrix):
|
|
if isinstance(similarity_matrix, scipy.sparse.csr_matrix):
|
|
similarity_matrix = similarity_matrix.T
|
|
else:
|
|
raise ValueError('unknown similarity matrix format')
|
|
|
|
if not vec1 or not vec2:
|
|
return 0.0
|
|
|
|
vec1 = dict(vec1)
|
|
vec2 = dict(vec2)
|
|
word_indices = sorted(set(chain(vec1, vec2)))
|
|
dtype = similarity_matrix.dtype
|
|
vec1 = np.array([vec1[i] if i in vec1 else 0 for i in word_indices], dtype=dtype)
|
|
vec2 = np.array([vec2[i] if i in vec2 else 0 for i in word_indices], dtype=dtype)
|
|
dense_matrix = similarity_matrix[[[i] for i in word_indices], word_indices].todense()
|
|
vec1len = vec1.T.dot(dense_matrix).dot(vec1)[0, 0]
|
|
vec2len = vec2.T.dot(dense_matrix).dot(vec2)[0, 0]
|
|
|
|
assert \
|
|
vec1len > 0.0 and vec2len > 0.0, \
|
|
u"sparse documents must not contain any explicit zero entries and the similarity matrix S " \
|
|
u"must satisfy x^T * S * x > 0 for any nonzero bag-of-words vector x."
|
|
|
|
result = vec1.T.dot(dense_matrix).dot(vec2)[0, 0]
|
|
result /= math.sqrt(vec1len) * math.sqrt(vec2len) # rescale by vector lengths
|
|
return np.clip(result, -1.0, 1.0)
|
|
|
|
|
|
def isbow(vec):
|
|
"""Checks if a vector is in the sparse Gensim bag-of-words format.
|
|
|
|
Parameters
|
|
----------
|
|
vec : object
|
|
Object to check.
|
|
|
|
Returns
|
|
-------
|
|
bool
|
|
Is `vec` in BoW format.
|
|
|
|
"""
|
|
if scipy.sparse.issparse(vec):
|
|
vec = vec.todense().tolist()
|
|
try:
|
|
id_, val_ = vec[0] # checking first value to see if it is in bag of words format by unpacking
|
|
int(id_), float(val_)
|
|
except IndexError:
|
|
return True # this is to handle the empty input case
|
|
except (ValueError, TypeError):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _convert_vec(vec1, vec2, num_features=None):
|
|
if scipy.sparse.issparse(vec1):
|
|
vec1 = vec1.toarray()
|
|
if scipy.sparse.issparse(vec2):
|
|
vec2 = vec2.toarray() # converted both the vectors to dense in case they were in sparse matrix
|
|
if isbow(vec1) and isbow(vec2): # if they are in bag of words format we make it dense
|
|
if num_features is not None: # if not None, make as large as the documents drawing from
|
|
dense1 = sparse2full(vec1, num_features)
|
|
dense2 = sparse2full(vec2, num_features)
|
|
return dense1, dense2
|
|
else:
|
|
max_len = max(len(vec1), len(vec2))
|
|
dense1 = sparse2full(vec1, max_len)
|
|
dense2 = sparse2full(vec2, max_len)
|
|
return dense1, dense2
|
|
else:
|
|
# this conversion is made because if it is not in bow format, it might be a list within a list after conversion
|
|
# the scipy implementation of Kullback fails in such a case so we pick up only the nested list.
|
|
if len(vec1) == 1:
|
|
vec1 = vec1[0]
|
|
if len(vec2) == 1:
|
|
vec2 = vec2[0]
|
|
return vec1, vec2
|
|
|
|
|
|
def kullback_leibler(vec1, vec2, num_features=None):
|
|
"""Calculate Kullback-Leibler distance between two probability distributions using `scipy.stats.entropy`.
|
|
|
|
Parameters
|
|
----------
|
|
vec1 : {scipy.sparse, numpy.ndarray, list of (int, float)}
|
|
Distribution vector.
|
|
vec2 : {scipy.sparse, numpy.ndarray, list of (int, float)}
|
|
Distribution vector.
|
|
num_features : int, optional
|
|
Number of features in the vectors.
|
|
|
|
Returns
|
|
-------
|
|
float
|
|
Kullback-Leibler distance between `vec1` and `vec2`.
|
|
Value in range [0, +∞) where values closer to 0 mean less distance (higher similarity).
|
|
|
|
"""
|
|
vec1, vec2 = _convert_vec(vec1, vec2, num_features=num_features)
|
|
return entropy(vec1, vec2)
|
|
|
|
|
|
def jensen_shannon(vec1, vec2, num_features=None):
|
|
"""Calculate Jensen-Shannon distance between two probability distributions using `scipy.stats.entropy`.
|
|
|
|
Parameters
|
|
----------
|
|
vec1 : {scipy.sparse, numpy.ndarray, list of (int, float)}
|
|
Distribution vector.
|
|
vec2 : {scipy.sparse, numpy.ndarray, list of (int, float)}
|
|
Distribution vector.
|
|
num_features : int, optional
|
|
Number of features in the vectors.
|
|
|
|
Returns
|
|
-------
|
|
float
|
|
Jensen-Shannon distance between `vec1` and `vec2`.
|
|
|
|
Notes
|
|
-----
|
|
This is a symmetric and finite "version" of :func:`gensim.matutils.kullback_leibler`.
|
|
|
|
"""
|
|
vec1, vec2 = _convert_vec(vec1, vec2, num_features=num_features)
|
|
avg_vec = 0.5 * (vec1 + vec2)
|
|
return 0.5 * (entropy(vec1, avg_vec) + entropy(vec2, avg_vec))
|
|
|
|
|
|
def hellinger(vec1, vec2):
|
|
"""Calculate Hellinger distance between two probability distributions.
|
|
|
|
Parameters
|
|
----------
|
|
vec1 : {scipy.sparse, numpy.ndarray, list of (int, float)}
|
|
Distribution vector.
|
|
vec2 : {scipy.sparse, numpy.ndarray, list of (int, float)}
|
|
Distribution vector.
|
|
|
|
Returns
|
|
-------
|
|
float
|
|
Hellinger distance between `vec1` and `vec2`.
|
|
Value in range `[0, 1]`, where 0 is min distance (max similarity) and 1 is max distance (min similarity).
|
|
|
|
"""
|
|
if scipy.sparse.issparse(vec1):
|
|
vec1 = vec1.toarray()
|
|
if scipy.sparse.issparse(vec2):
|
|
vec2 = vec2.toarray()
|
|
if isbow(vec1) and isbow(vec2):
|
|
# if it is a BoW format, instead of converting to dense we use dictionaries to calculate appropriate distance
|
|
vec1, vec2 = dict(vec1), dict(vec2)
|
|
indices = set(list(vec1.keys()) + list(vec2.keys()))
|
|
sim = np.sqrt(
|
|
0.5 * sum((np.sqrt(vec1.get(index, 0.0)) - np.sqrt(vec2.get(index, 0.0)))**2 for index in indices)
|
|
)
|
|
return sim
|
|
else:
|
|
sim = np.sqrt(0.5 * ((np.sqrt(vec1) - np.sqrt(vec2))**2).sum())
|
|
return sim
|
|
|
|
|
|
def jaccard(vec1, vec2):
|
|
"""Calculate Jaccard distance between two vectors.
|
|
|
|
Parameters
|
|
----------
|
|
vec1 : {scipy.sparse, numpy.ndarray, list of (int, float)}
|
|
Distribution vector.
|
|
vec2 : {scipy.sparse, numpy.ndarray, list of (int, float)}
|
|
Distribution vector.
|
|
|
|
Returns
|
|
-------
|
|
float
|
|
Jaccard distance between `vec1` and `vec2`.
|
|
Value in range `[0, 1]`, where 0 is min distance (max similarity) and 1 is max distance (min similarity).
|
|
|
|
"""
|
|
|
|
# converting from sparse for easier manipulation
|
|
if scipy.sparse.issparse(vec1):
|
|
vec1 = vec1.toarray()
|
|
if scipy.sparse.issparse(vec2):
|
|
vec2 = vec2.toarray()
|
|
if isbow(vec1) and isbow(vec2):
|
|
# if it's in bow format, we use the following definitions:
|
|
# union = sum of the 'weights' of both the bags
|
|
# intersection = lowest weight for a particular id; basically the number of common words or items
|
|
union = sum(weight for id_, weight in vec1) + sum(weight for id_, weight in vec2)
|
|
vec1, vec2 = dict(vec1), dict(vec2)
|
|
intersection = 0.0
|
|
for feature_id, feature_weight in iteritems(vec1):
|
|
intersection += min(feature_weight, vec2.get(feature_id, 0.0))
|
|
return 1 - float(intersection) / float(union)
|
|
else:
|
|
# if it isn't in bag of words format, we can use sets to calculate intersection and union
|
|
if isinstance(vec1, np.ndarray):
|
|
vec1 = vec1.tolist()
|
|
if isinstance(vec2, np.ndarray):
|
|
vec2 = vec2.tolist()
|
|
vec1 = set(vec1)
|
|
vec2 = set(vec2)
|
|
intersection = vec1 & vec2
|
|
union = vec1 | vec2
|
|
return 1 - float(len(intersection)) / float(len(union))
|
|
|
|
|
|
def jaccard_distance(set1, set2):
|
|
"""Calculate Jaccard distance between two sets.
|
|
|
|
Parameters
|
|
----------
|
|
set1 : set
|
|
Input set.
|
|
set2 : set
|
|
Input set.
|
|
|
|
Returns
|
|
-------
|
|
float
|
|
Jaccard distance between `set1` and `set2`.
|
|
Value in range `[0, 1]`, where 0 is min distance (max similarity) and 1 is max distance (min similarity).
|
|
"""
|
|
|
|
union_cardinality = len(set1 | set2)
|
|
if union_cardinality == 0: # Both sets are empty
|
|
return 1.
|
|
|
|
return 1. - float(len(set1 & set2)) / float(union_cardinality)
|
|
|
|
|
|
try:
|
|
# try to load fast, cythonized code if possible
|
|
from gensim._matutils import logsumexp, mean_absolute_difference, dirichlet_expectation
|
|
|
|
except ImportError:
|
|
def logsumexp(x):
|
|
"""Log of sum of exponentials.
|
|
|
|
Parameters
|
|
----------
|
|
x : numpy.ndarray
|
|
Input 2d matrix.
|
|
|
|
Returns
|
|
-------
|
|
float
|
|
log of sum of exponentials of elements in `x`.
|
|
|
|
Warnings
|
|
--------
|
|
For performance reasons, doesn't support NaNs or 1d, 3d, etc arrays like :func:`scipy.special.logsumexp`.
|
|
|
|
"""
|
|
x_max = np.max(x)
|
|
x = np.log(np.sum(np.exp(x - x_max)))
|
|
x += x_max
|
|
|
|
return x
|
|
|
|
def mean_absolute_difference(a, b):
|
|
"""Mean absolute difference between two arrays.
|
|
|
|
Parameters
|
|
----------
|
|
a : numpy.ndarray
|
|
Input 1d array.
|
|
b : numpy.ndarray
|
|
Input 1d array.
|
|
|
|
Returns
|
|
-------
|
|
float
|
|
mean(abs(a - b)).
|
|
|
|
"""
|
|
return np.mean(np.abs(a - b))
|
|
|
|
def dirichlet_expectation(alpha):
|
|
"""Expected value of log(theta) where theta is drawn from a Dirichlet distribution.
|
|
|
|
Parameters
|
|
----------
|
|
alpha : numpy.ndarray
|
|
Dirichlet parameter 2d matrix or 1d vector, if 2d - each row is treated as a separate parameter vector.
|
|
|
|
Returns
|
|
-------
|
|
numpy.ndarray
|
|
Log of expected values, dimension same as `alpha.ndim`.
|
|
|
|
"""
|
|
if len(alpha.shape) == 1:
|
|
result = psi(alpha) - psi(np.sum(alpha))
|
|
else:
|
|
result = psi(alpha) - psi(np.sum(alpha, 1))[:, np.newaxis]
|
|
return result.astype(alpha.dtype, copy=False) # keep the same precision as input
|
|
|
|
|
|
def qr_destroy(la):
|
|
"""Get QR decomposition of `la[0]`.
|
|
|
|
Parameters
|
|
----------
|
|
la : list of numpy.ndarray
|
|
Run QR decomposition on the first elements of `la`. Must not be empty.
|
|
|
|
Returns
|
|
-------
|
|
(numpy.ndarray, numpy.ndarray)
|
|
Matrices :math:`Q` and :math:`R`.
|
|
|
|
Notes
|
|
-----
|
|
Using this function is less memory intense than calling `scipy.linalg.qr(la[0])`,
|
|
because the memory used in `la[0]` is reclaimed earlier. This makes a difference when
|
|
decomposing very large arrays, where every memory copy counts.
|
|
|
|
Warnings
|
|
--------
|
|
Content of `la` as well as `la[0]` gets destroyed in the process. Again, for memory-effiency reasons.
|
|
|
|
"""
|
|
a = np.asfortranarray(la[0])
|
|
del la[0], la # now `a` is the only reference to the input matrix
|
|
m, n = a.shape
|
|
# perform q, r = QR(a); code hacked out of scipy.linalg.qr
|
|
logger.debug("computing QR of %s dense matrix", str(a.shape))
|
|
geqrf, = get_lapack_funcs(('geqrf',), (a,))
|
|
qr, tau, work, info = geqrf(a, lwork=-1, overwrite_a=True)
|
|
qr, tau, work, info = geqrf(a, lwork=work[0], overwrite_a=True)
|
|
del a # free up mem
|
|
assert info >= 0
|
|
r = triu(qr[:n, :n])
|
|
if m < n: # rare case, #features < #topics
|
|
qr = qr[:, :m] # retains fortran order
|
|
gorgqr, = get_lapack_funcs(('orgqr',), (qr,))
|
|
q, work, info = gorgqr(qr, tau, lwork=-1, overwrite_a=True)
|
|
q, work, info = gorgqr(qr, tau, lwork=work[0], overwrite_a=True)
|
|
assert info >= 0, "qr failed"
|
|
assert q.flags.f_contiguous
|
|
return q, r
|
|
|
|
|
|
class MmWriter(object):
|
|
"""Store a corpus in `Matrix Market format <https://math.nist.gov/MatrixMarket/formats.html>`_,
|
|
using :class:`~gensim.corpora.mmcorpus.MmCorpus`.
|
|
|
|
Notes
|
|
-----
|
|
The output is written one document at a time, not the whole matrix at once (unlike e.g. `scipy.io.mmread`).
|
|
This allows you to write corpora which are larger than the available RAM.
|
|
|
|
The output file is created in a single pass through the input corpus, so that the input can be
|
|
a once-only stream (generator).
|
|
|
|
To achieve this, a fake MM header is written first, corpus statistics are collected
|
|
during the pass (shape of the matrix, number of non-zeroes), followed by a seek back to the beginning of the file,
|
|
rewriting the fake header with the final values.
|
|
|
|
"""
|
|
HEADER_LINE = b'%%MatrixMarket matrix coordinate real general\n' # the only supported MM format
|
|
|
|
def __init__(self, fname):
|
|
"""
|
|
|
|
Parameters
|
|
----------
|
|
fname : str
|
|
Path to output file.
|
|
|
|
"""
|
|
self.fname = fname
|
|
if fname.endswith(".gz") or fname.endswith('.bz2'):
|
|
raise NotImplementedError("compressed output not supported with MmWriter")
|
|
self.fout = utils.smart_open(self.fname, 'wb+') # open for both reading and writing
|
|
self.headers_written = False
|
|
|
|
def write_headers(self, num_docs, num_terms, num_nnz):
|
|
"""Write headers to file.
|
|
|
|
Parameters
|
|
----------
|
|
num_docs : int
|
|
Number of documents in corpus.
|
|
num_terms : int
|
|
Number of term in corpus.
|
|
num_nnz : int
|
|
Number of non-zero elements in corpus.
|
|
|
|
"""
|
|
self.fout.write(MmWriter.HEADER_LINE)
|
|
|
|
if num_nnz < 0:
|
|
# we don't know the matrix shape/density yet, so only log a general line
|
|
logger.info("saving sparse matrix to %s", self.fname)
|
|
self.fout.write(utils.to_utf8(' ' * 50 + '\n')) # 48 digits must be enough for everybody
|
|
else:
|
|
logger.info(
|
|
"saving sparse %sx%s matrix with %i non-zero entries to %s",
|
|
num_docs, num_terms, num_nnz, self.fname
|
|
)
|
|
self.fout.write(utils.to_utf8('%s %s %s\n' % (num_docs, num_terms, num_nnz)))
|
|
self.last_docno = -1
|
|
self.headers_written = True
|
|
|
|
def fake_headers(self, num_docs, num_terms, num_nnz):
|
|
"""Write "fake" headers to file, to be rewritten once we've scanned the entire corpus.
|
|
|
|
Parameters
|
|
----------
|
|
num_docs : int
|
|
Number of documents in corpus.
|
|
num_terms : int
|
|
Number of term in corpus.
|
|
num_nnz : int
|
|
Number of non-zero elements in corpus.
|
|
|
|
"""
|
|
stats = '%i %i %i' % (num_docs, num_terms, num_nnz)
|
|
if len(stats) > 50:
|
|
raise ValueError('Invalid stats: matrix too large!')
|
|
self.fout.seek(len(MmWriter.HEADER_LINE))
|
|
self.fout.write(utils.to_utf8(stats))
|
|
|
|
def write_vector(self, docno, vector):
|
|
"""Write a single sparse vector to the file.
|
|
|
|
Parameters
|
|
----------
|
|
docno : int
|
|
Number of document.
|
|
vector : list of (int, number)
|
|
Document in BoW format.
|
|
|
|
Returns
|
|
-------
|
|
(int, int)
|
|
Max word index in vector and len of vector. If vector is empty, return (-1, 0).
|
|
|
|
"""
|
|
assert self.headers_written, "must write Matrix Market file headers before writing data!"
|
|
assert self.last_docno < docno, "documents %i and %i not in sequential order!" % (self.last_docno, docno)
|
|
vector = sorted((i, w) for i, w in vector if abs(w) > 1e-12) # ignore near-zero entries
|
|
for termid, weight in vector: # write term ids in sorted order
|
|
# +1 because MM format starts counting from 1
|
|
self.fout.write(utils.to_utf8("%i %i %s\n" % (docno + 1, termid + 1, weight)))
|
|
self.last_docno = docno
|
|
return (vector[-1][0], len(vector)) if vector else (-1, 0)
|
|
|
|
@staticmethod
|
|
def write_corpus(fname, corpus, progress_cnt=1000, index=False, num_terms=None, metadata=False):
|
|
"""Save the corpus to disk in `Matrix Market format <https://math.nist.gov/MatrixMarket/formats.html>`_.
|
|
|
|
Parameters
|
|
----------
|
|
fname : str
|
|
Filename of the resulting file.
|
|
corpus : iterable of list of (int, number)
|
|
Corpus in streamed bag-of-words format.
|
|
progress_cnt : int, optional
|
|
Print progress for every `progress_cnt` number of documents.
|
|
index : bool, optional
|
|
Return offsets?
|
|
num_terms : int, optional
|
|
Number of terms in the corpus. If provided, the `corpus.num_terms` attribute (if any) will be ignored.
|
|
metadata : bool, optional
|
|
Generate a metadata file?
|
|
|
|
Returns
|
|
-------
|
|
offsets : {list of int, None}
|
|
List of offsets (if index=True) or nothing.
|
|
|
|
Notes
|
|
-----
|
|
Documents are processed one at a time, so the whole corpus is allowed to be larger than the available RAM.
|
|
|
|
See Also
|
|
--------
|
|
:func:`gensim.corpora.mmcorpus.MmCorpus.save_corpus`
|
|
Save corpus to disk.
|
|
|
|
"""
|
|
mw = MmWriter(fname)
|
|
|
|
# write empty headers to the file (with enough space to be overwritten later)
|
|
mw.write_headers(-1, -1, -1) # will print 50 spaces followed by newline on the stats line
|
|
|
|
# calculate necessary header info (nnz elements, num terms, num docs) while writing out vectors
|
|
_num_terms, num_nnz = 0, 0
|
|
docno, poslast = -1, -1
|
|
offsets = []
|
|
if hasattr(corpus, 'metadata'):
|
|
orig_metadata = corpus.metadata
|
|
corpus.metadata = metadata
|
|
if metadata:
|
|
docno2metadata = {}
|
|
else:
|
|
metadata = False
|
|
for docno, doc in enumerate(corpus):
|
|
if metadata:
|
|
bow, data = doc
|
|
docno2metadata[docno] = data
|
|
else:
|
|
bow = doc
|
|
if docno % progress_cnt == 0:
|
|
logger.info("PROGRESS: saving document #%i", docno)
|
|
if index:
|
|
posnow = mw.fout.tell()
|
|
if posnow == poslast:
|
|
offsets[-1] = -1
|
|
offsets.append(posnow)
|
|
poslast = posnow
|
|
max_id, veclen = mw.write_vector(docno, bow)
|
|
_num_terms = max(_num_terms, 1 + max_id)
|
|
num_nnz += veclen
|
|
if metadata:
|
|
utils.pickle(docno2metadata, fname + '.metadata.cpickle')
|
|
corpus.metadata = orig_metadata
|
|
|
|
num_docs = docno + 1
|
|
num_terms = num_terms or _num_terms
|
|
|
|
if num_docs * num_terms != 0:
|
|
logger.info(
|
|
"saved %ix%i matrix, density=%.3f%% (%i/%i)",
|
|
num_docs, num_terms, 100.0 * num_nnz / (num_docs * num_terms), num_nnz, num_docs * num_terms
|
|
)
|
|
|
|
# now write proper headers, by seeking and overwriting the spaces written earlier
|
|
mw.fake_headers(num_docs, num_terms, num_nnz)
|
|
|
|
mw.close()
|
|
if index:
|
|
return offsets
|
|
|
|
def __del__(self):
|
|
"""Close `self.fout` file. Alias for :meth:`~gensim.matutils.MmWriter.close`.
|
|
|
|
Warnings
|
|
--------
|
|
Closing the file explicitly via the close() method is preferred and safer.
|
|
|
|
"""
|
|
self.close() # does nothing if called twice (on an already closed file), so no worries
|
|
|
|
def close(self):
|
|
"""Close `self.fout` file."""
|
|
logger.debug("closing %s", self.fname)
|
|
if hasattr(self, 'fout'):
|
|
self.fout.close()
|
|
|
|
|
|
try:
|
|
# try to load fast, cythonized code if possible
|
|
from gensim.corpora._mmreader import MmReader
|
|
except ImportError:
|
|
FAST_VERSION = -1
|
|
|
|
class MmReader(object):
|
|
"""Matrix market file reader, used internally in :class:`~gensim.corpora.mmcorpus.MmCorpus`.
|
|
|
|
Wrap a term-document matrix on disk (in matrix-market format), and present it
|
|
as an object which supports iteration over the rows (~documents).
|
|
|
|
Attributes
|
|
----------
|
|
num_docs : int
|
|
Number of documents in market matrix file.
|
|
num_terms : int
|
|
Number of terms.
|
|
num_nnz : int
|
|
Number of non-zero terms.
|
|
|
|
Notes
|
|
-----
|
|
Note that the file is read into memory one document at a time, not the whole matrix at once
|
|
(unlike e.g. `scipy.io.mmread` and other implementations).
|
|
This allows us to process corpora which are larger than the available RAM.
|
|
|
|
"""
|
|
def __init__(self, input, transposed=True):
|
|
"""
|
|
|
|
Parameters
|
|
----------
|
|
input : {str, file-like object}
|
|
Path to the input file in MM format or a file-like object that supports `seek()`
|
|
(e.g. smart_open objects).
|
|
transposed : bool, optional
|
|
Do lines represent `doc_id, term_id, value`, instead of `term_id, doc_id, value`?
|
|
|
|
"""
|
|
logger.info("initializing corpus reader from %s", input)
|
|
self.input, self.transposed = input, transposed
|
|
with utils.open_file(self.input) as lines:
|
|
try:
|
|
header = utils.to_unicode(next(lines)).strip()
|
|
if not header.lower().startswith('%%matrixmarket matrix coordinate real general'):
|
|
raise ValueError(
|
|
"File %s not in Matrix Market format with coordinate real general; instead found: \n%s" %
|
|
(self.input, header)
|
|
)
|
|
except StopIteration:
|
|
pass
|
|
|
|
self.num_docs = self.num_terms = self.num_nnz = 0
|
|
for lineno, line in enumerate(lines):
|
|
line = utils.to_unicode(line)
|
|
if not line.startswith('%'):
|
|
self.num_docs, self.num_terms, self.num_nnz = (int(x) for x in line.split())
|
|
if not self.transposed:
|
|
self.num_docs, self.num_terms = self.num_terms, self.num_docs
|
|
break
|
|
|
|
logger.info(
|
|
"accepted corpus with %i documents, %i features, %i non-zero entries",
|
|
self.num_docs, self.num_terms, self.num_nnz
|
|
)
|
|
|
|
def __len__(self):
|
|
"""Get the corpus size: total number of documents."""
|
|
return self.num_docs
|
|
|
|
def __str__(self):
|
|
return ("MmCorpus(%i documents, %i features, %i non-zero entries)" %
|
|
(self.num_docs, self.num_terms, self.num_nnz))
|
|
|
|
def skip_headers(self, input_file):
|
|
"""Skip file headers that appear before the first document.
|
|
|
|
Parameters
|
|
----------
|
|
input_file : iterable of str
|
|
Iterable taken from file in MM format.
|
|
|
|
"""
|
|
for line in input_file:
|
|
if line.startswith(b'%'):
|
|
continue
|
|
break
|
|
|
|
def __iter__(self):
|
|
"""Iterate through all documents in the corpus.
|
|
|
|
Notes
|
|
------
|
|
Note that the total number of vectors returned is always equal to the number of rows specified
|
|
in the header: empty documents are inserted and yielded where appropriate, even if they are not explicitly
|
|
stored in the Matrix Market file.
|
|
|
|
Yields
|
|
------
|
|
(int, list of (int, number))
|
|
Document id and document in sparse bag-of-words format.
|
|
|
|
"""
|
|
with utils.file_or_filename(self.input) as lines:
|
|
self.skip_headers(lines)
|
|
|
|
previd = -1
|
|
for line in lines:
|
|
docid, termid, val = utils.to_unicode(line).split() # needed for python3
|
|
if not self.transposed:
|
|
termid, docid = docid, termid
|
|
# -1 because matrix market indexes are 1-based => convert to 0-based
|
|
docid, termid, val = int(docid) - 1, int(termid) - 1, float(val)
|
|
assert previd <= docid, "matrix columns must come in ascending order"
|
|
if docid != previd:
|
|
# change of document: return the document read so far (its id is prevId)
|
|
if previd >= 0:
|
|
yield previd, document # noqa:F821
|
|
|
|
# return implicit (empty) documents between previous id and new id
|
|
# too, to keep consistent document numbering and corpus length
|
|
for previd in xrange(previd + 1, docid):
|
|
yield previd, []
|
|
|
|
# from now on start adding fields to a new document, with a new id
|
|
previd = docid
|
|
document = []
|
|
|
|
document.append((termid, val,)) # add another field to the current document
|
|
|
|
# handle the last document, as a special case
|
|
if previd >= 0:
|
|
yield previd, document
|
|
|
|
# return empty documents between the last explicit document and the number
|
|
# of documents as specified in the header
|
|
for previd in xrange(previd + 1, self.num_docs):
|
|
yield previd, []
|
|
|
|
def docbyoffset(self, offset):
|
|
"""Get the document at file offset `offset` (in bytes).
|
|
|
|
Parameters
|
|
----------
|
|
offset : int
|
|
File offset, in bytes, of the desired document.
|
|
|
|
Returns
|
|
------
|
|
list of (int, str)
|
|
Document in sparse bag-of-words format.
|
|
|
|
"""
|
|
# empty documents are not stored explicitly in MM format, so the index marks
|
|
# them with a special offset, -1.
|
|
if offset == -1:
|
|
return []
|
|
if isinstance(self.input, string_types):
|
|
fin, close_fin = utils.smart_open(self.input), True
|
|
else:
|
|
fin, close_fin = self.input, False
|
|
|
|
fin.seek(offset) # works for gzip/bz2 input, too
|
|
previd, document = -1, []
|
|
for line in fin:
|
|
docid, termid, val = line.split()
|
|
if not self.transposed:
|
|
termid, docid = docid, termid
|
|
# -1 because matrix market indexes are 1-based => convert to 0-based
|
|
docid, termid, val = int(docid) - 1, int(termid) - 1, float(val)
|
|
assert previd <= docid, "matrix columns must come in ascending order"
|
|
if docid != previd:
|
|
if previd >= 0:
|
|
break
|
|
previd = docid
|
|
|
|
document.append((termid, val,)) # add another field to the current document
|
|
|
|
if close_fin:
|
|
fin.close()
|
|
return document
|