#!/usr/bin/env python # -*- coding: utf-8 -*- # Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html """Automatically detect common phrases -- multi-word expressions / word n-grams -- from a stream of sentences. Inspired by: * `Mikolov, et. al: "Distributed Representations of Words and Phrases and their Compositionality" `_ * `"Normalized (Pointwise) Mutual Information in Colocation Extraction" by Gerlof Bouma `_ Examples -------- >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus >>> from gensim.models.phrases import Phrases, Phraser >>> >>> sentences = Text8Corpus(datapath('testcorpus.txt')) >>> phrases = Phrases(sentences, min_count=1, threshold=1) # train model >>> phrases[[u'trees', u'graph', u'minors']] # apply model to sentence [u'trees_graph', u'minors'] >>> >>> phrases.add_vocab([["hello", "world"], ["meow"]]) # update model with new sentences >>> >>> bigram = Phraser(phrases) # construct faster model (this is only an wrapper) >>> bigram[[u'trees', u'graph', u'minors']] # apply model to sentence [u'trees_graph', u'minors'] >>> >>> for sent in bigram[sentences]: # apply model to text corpus ... pass """ import sys import os import logging import warnings from collections import defaultdict import functools as ft import itertools as it from math import log import pickle import six from six import iteritems, string_types, PY2, next from gensim import utils, interfaces if PY2: from inspect import getargspec else: from inspect import getfullargspec as getargspec logger = logging.getLogger(__name__) def _is_single(obj): """Check whether `obj` is a single document or an entire corpus. Parameters ---------- obj : object Return ------ (bool, object) (is_single, new) tuple, where `new` yields the same sequence as `obj`. Notes ----- `obj` is a single document if it is an iterable of strings. It is a corpus if it is an iterable of documents. """ obj_iter = iter(obj) temp_iter = obj_iter try: peek = next(obj_iter) obj_iter = it.chain([peek], obj_iter) except StopIteration: # An empty object is a single document return True, obj if isinstance(peek, string_types): # It's a document, return the iterator return True, obj_iter if temp_iter is obj: # Checking for iterator to the object return False, obj_iter else: # If the first item isn't a string, assume obj is a corpus return False, obj class SentenceAnalyzer(object): """Base util class for :class:`~gensim.models.phrases.Phrases` and :class:`~gensim.models.phrases.Phraser`.""" def score_item(self, worda, wordb, components, scorer): """Get bi-gram score statistics. Parameters ---------- worda : str First word of bi-gram. wordb : str Second word of bi-gram. components : generator Contain all phrases. scorer : function Scorer function, as given to :class:`~gensim.models.phrases.Phrases`. See :func:`~gensim.models.phrases.npmi_scorer` and :func:`~gensim.models.phrases.original_scorer`. Returns ------- float Score for given bi-gram, if bi-gram not presented in dictionary - return -1. """ vocab = self.vocab if worda in vocab and wordb in vocab: bigram = self.delimiter.join(components) if bigram in vocab: return scorer( worda_count=float(vocab[worda]), wordb_count=float(vocab[wordb]), bigram_count=float(vocab[bigram])) return -1 def analyze_sentence(self, sentence, threshold, common_terms, scorer): """Analyze a sentence, detecting any bigrams that should be concatenated. Parameters ---------- sentence : iterable of str Token sequence representing the sentence to be analyzed. threshold : float The minimum score for a bigram to be taken into account. common_terms : list of object List of common terms, they have special treatment. scorer : function Scorer function, as given to :class:`~gensim.models.phrases.Phrases`. See :func:`~gensim.models.phrases.npmi_scorer` and :func:`~gensim.models.phrases.original_scorer`. Yields ------ (str, score) If bi-gram detected, a tuple where the first element is a detect bigram, second its score. Otherwise, the first tuple element is a single word and second is None. """ s = [utils.any2utf8(w) for w in sentence] # adding None is a trick that helps getting an automatic happy ending # as it won't be a common_word, nor score s.append(None) last_uncommon = None in_between = [] for word in s: is_common = word in common_terms if not is_common and last_uncommon: chain = [last_uncommon] + in_between + [word] # test between last_uncommon score = self.score_item( worda=last_uncommon, wordb=word, components=chain, scorer=scorer, ) if score > threshold: yield (chain, score) last_uncommon = None in_between = [] else: # release words individually for w in it.chain([last_uncommon], in_between): yield (w, None) in_between = [] last_uncommon = word elif not is_common: last_uncommon = word else: # common term if last_uncommon: # wait for uncommon resolution in_between.append(word) else: yield (word, None) class PhrasesTransformation(interfaces.TransformationABC): """Base util class for :class:`~gensim.models.phrases.Phrases` and :class:`~gensim.models.phrases.Phraser`.""" @classmethod def load(cls, *args, **kwargs): """Load a previously saved :class:`~gensim.models.phrases.Phrases` / :class:`~gensim.models.phrases.Phraser` class. Handles backwards compatibility from older :class:`~gensim.models.phrases.Phrases` / :class:`~gensim.models.phrases.Phraser` versions which did not support pluggable scoring functions. Parameters ---------- args : object Sequence of arguments, see :class:`~gensim.utils.SaveLoad.load` for more information. kwargs : object Sequence of arguments, see :class:`~gensim.utils.SaveLoad.load` for more information. """ model = super(PhrasesTransformation, cls).load(*args, **kwargs) # update older models # if no scoring parameter, use default scoring if not hasattr(model, 'scoring'): logger.info('older version of %s loaded without scoring function', cls.__name__) logger.info('setting pluggable scoring method to original_scorer for compatibility') model.scoring = original_scorer # if there is a scoring parameter, and it's a text value, load the proper scoring function if hasattr(model, 'scoring'): if isinstance(model.scoring, six.string_types): if model.scoring == 'default': logger.info('older version of %s loaded with "default" scoring parameter', cls.__name__) logger.info('setting scoring method to original_scorer pluggable scoring method for compatibility') model.scoring = original_scorer elif model.scoring == 'npmi': logger.info('older version of %s loaded with "npmi" scoring parameter', cls.__name__) logger.info('setting scoring method to npmi_scorer pluggable scoring method for compatibility') model.scoring = npmi_scorer else: raise ValueError( 'failed to load %s model with unknown scoring setting %s' % (cls.__name__, model.scoring)) # if there is non common_terms attribute, initialize if not hasattr(model, "common_terms"): logger.info('older version of %s loaded without common_terms attribute', cls.__name__) logger.info('setting common_terms to empty set') model.common_terms = frozenset() return model class Phrases(SentenceAnalyzer, PhrasesTransformation): """Detect phrases based on collocation counts.""" def __init__(self, sentences=None, min_count=5, threshold=10.0, max_vocab_size=40000000, delimiter=b'_', progress_per=10000, scoring='default', common_terms=frozenset()): """ Parameters ---------- sentences : iterable of list of str, optional The `sentences` iterable can be simply a list, but for larger corpora, consider a generator that streams the sentences directly from disk/network, See :class:`~gensim.models.word2vec.BrownCorpus`, :class:`~gensim.models.word2vec.Text8Corpus` or :class:`~gensim.models.word2vec.LineSentence` for such examples. min_count : float, optional Ignore all words and bigrams with total collected count lower than this value. threshold : float, optional Represent a score threshold for forming the phrases (higher means fewer phrases). A phrase of words `a` followed by `b` is accepted if the score of the phrase is greater than threshold. Hardly depends on concrete socring-function, see the `scoring` parameter. max_vocab_size : int, optional Maximum size (number of tokens) of the vocabulary. Used to control pruning of less common words, to keep memory under control. The default of 40M needs about 3.6GB of RAM. Increase/decrease `max_vocab_size` depending on how much available memory you have. delimiter : str, optional Glue character used to join collocation tokens, should be a byte string (e.g. b'_'). scoring : {'default', 'npmi', function}, optional Specify how potential phrases are scored. `scoring` can be set with either a string that refers to a built-in scoring function, or with a function with the expected parameter names. Two built-in scoring functions are available by setting `scoring` to a string: #. "default" - :func:`~gensim.models.phrases.original_scorer`. #. "npmi" - :func:`~gensim.models.phrases.npmi_scorer`. common_terms : set of str, optional List of "stop words" that won't affect frequency count of expressions containing them. Allow to detect expressions like "bank_of_america" or "eye_of_the_beholder". Notes ----- 'npmi' is more robust when dealing with common words that form part of common bigrams, and ranges from -1 to 1, but is slower to calculate than the default. The default is the PMI-like scoring as described by `Mikolov, et. al: "Distributed Representations of Words and Phrases and their Compositionality" `_. To use a custom scoring function, pass in a function with the following signature: * worda_count - number of corpus occurrences in `sentences` of the first token in the bigram being scored * wordb_count - number of corpus occurrences in `sentences` of the second token in the bigram being scored * bigram_count - number of occurrences in `sentences` of the whole bigram * len_vocab - the number of unique tokens in `sentences` * min_count - the `min_count` setting of the Phrases class * corpus_word_count - the total number of tokens (non-unique) in `sentences` The scoring function **must accept all these parameters**, even if it doesn't use them in its scoring. The scoring function **must be pickleable**. """ if min_count <= 0: raise ValueError("min_count should be at least 1") if threshold <= 0 and scoring == 'default': raise ValueError("threshold should be positive for default scoring") if scoring == 'npmi' and (threshold < -1 or threshold > 1): raise ValueError("threshold should be between -1 and 1 for npmi scoring") # set scoring based on string # intentially override the value of the scoring parameter rather than set self.scoring here, # to still run the check of scoring function parameters in the next code block if isinstance(scoring, six.string_types): if scoring == 'default': scoring = original_scorer elif scoring == 'npmi': scoring = npmi_scorer else: raise ValueError('unknown scoring method string %s specified' % (scoring)) scoring_parameters = [ 'worda_count', 'wordb_count', 'bigram_count', 'len_vocab', 'min_count', 'corpus_word_count' ] if callable(scoring): if all(parameter in getargspec(scoring)[0] for parameter in scoring_parameters): self.scoring = scoring else: raise ValueError('scoring function missing expected parameters') self.min_count = min_count self.threshold = threshold self.max_vocab_size = max_vocab_size self.vocab = defaultdict(int) # mapping between utf8 token => its count self.min_reduce = 1 # ignore any tokens with count smaller than this self.delimiter = delimiter self.progress_per = progress_per self.corpus_word_count = 0 self.common_terms = frozenset(utils.any2utf8(w) for w in common_terms) # ensure picklability of custom scorer try: test_pickle = pickle.dumps(self.scoring) load_pickle = pickle.loads(test_pickle) except pickle.PickleError: raise pickle.PickleError('unable to pickle custom Phrases scoring function') finally: del(test_pickle) del(load_pickle) if sentences is not None: self.add_vocab(sentences) @classmethod def load(cls, *args, **kwargs): """Load a previously saved Phrases class. Handles backwards compatibility from older Phrases versions which did not support pluggable scoring functions. Parameters ---------- args : object Sequence of arguments, see :class:`~gensim.utils.SaveLoad.load` for more information. kwargs : object Sequence of arguments, see :class:`~gensim.utils.SaveLoad.load` for more information. """ model = super(Phrases, cls).load(*args, **kwargs) if not hasattr(model, 'corpus_word_count'): logger.info('older version of %s loaded without corpus_word_count', cls.__name__) logger.info('Setting it to 0, do not use it in your scoring function.') model.corpus_word_count = 0 return model def __str__(self): """Get short string representation of this phrase detector.""" return "%s<%i vocab, min_count=%s, threshold=%s, max_vocab_size=%s>" % ( self.__class__.__name__, len(self.vocab), self.min_count, self.threshold, self.max_vocab_size ) @staticmethod def learn_vocab(sentences, max_vocab_size, delimiter=b'_', progress_per=10000, common_terms=frozenset()): """Collect unigram/bigram counts from the `sentences` iterable. Parameters ---------- sentences : iterable of list of str The `sentences` iterable can be simply a list, but for larger corpora, consider a generator that streams the sentences directly from disk/network, See :class:`~gensim.models.word2vec.BrownCorpus`, :class:`~gensim.models.word2vec.Text8Corpus` or :class:`~gensim.models.word2vec.LineSentence` for such examples. max_vocab_size : int Maximum size (number of tokens) of the vocabulary. Used to control pruning of less common words, to keep memory under control. The default of 40M needs about 3.6GB of RAM. Increase/decrease `max_vocab_size` depending on how much available memory you have. delimiter : str, optional Glue character used to join collocation tokens, should be a byte string (e.g. b'_'). progress_per : int Write logs every `progress_per` sentence. common_terms : set of str, optional List of "stop words" that won't affect frequency count of expressions containing them. Allow to detect expressions like "bank_of_america" or "eye_of_the_beholder". Return ------ (int, dict of (str, int), int) Number of pruned words, counters for each word/bi-gram and total number of words. Example ---------- >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus >>> from gensim.models.phrases import Phrases >>> >>> sentences = Text8Corpus(datapath('testcorpus.txt')) >>> pruned_words, counters, total_words = Phrases.learn_vocab(sentences, 100) >>> (pruned_words, total_words) (1, 29) >>> counters['computer'] 2 >>> counters['response_time'] 1 """ sentence_no = -1 total_words = 0 logger.info("collecting all words and their counts") vocab = defaultdict(int) min_reduce = 1 for sentence_no, sentence in enumerate(sentences): if sentence_no % progress_per == 0: logger.info( "PROGRESS: at sentence #%i, processed %i words and %i word types", sentence_no, total_words, len(vocab), ) s = [utils.any2utf8(w) for w in sentence] last_uncommon = None in_between = [] for word in s: if word not in common_terms: vocab[word] += 1 if last_uncommon is not None: components = it.chain([last_uncommon], in_between, [word]) vocab[delimiter.join(components)] += 1 last_uncommon = word in_between = [] elif last_uncommon is not None: in_between.append(word) total_words += 1 if len(vocab) > max_vocab_size: utils.prune_vocab(vocab, min_reduce) min_reduce += 1 logger.info( "collected %i word types from a corpus of %i words (unigram + bigrams) and %i sentences", len(vocab), total_words, sentence_no + 1 ) return min_reduce, vocab, total_words def add_vocab(self, sentences): """Update model with new `sentences`. Parameters ---------- sentences : iterable of list of str Text corpus. Example ------- >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus >>> from gensim.models.phrases import Phrases >>> #Create corpus and use it for phrase detector >>> sentences = Text8Corpus(datapath('testcorpus.txt')) >>> phrases = Phrases(sentences) # train model >>> assert len(phrases.vocab) == 37 >>> >>> more_sentences = [ ... [u'the', u'mayor', u'of', u'new', u'york', u'was', u'there'], ... [u'machine', u'learning', u'can', u'be', u'new', u'york' , u'sometimes'] ... ] >>> >>> phrases.add_vocab(more_sentences) # add new sentences to model >>> assert len(phrases.vocab) == 60 """ # uses a separate vocab to collect the token counts from `sentences`. # this consumes more RAM than merging new sentences into `self.vocab` # directly, but gives the new sentences a fighting chance to collect # sufficient counts, before being pruned out by the (large) accummulated # counts collected in previous learn_vocab runs. min_reduce, vocab, total_words = self.learn_vocab( sentences, self.max_vocab_size, self.delimiter, self.progress_per, self.common_terms) self.corpus_word_count += total_words if len(self.vocab) > 0: logger.info("merging %i counts into %s", len(vocab), self) self.min_reduce = max(self.min_reduce, min_reduce) for word, count in iteritems(vocab): self.vocab[word] += count if len(self.vocab) > self.max_vocab_size: utils.prune_vocab(self.vocab, self.min_reduce) self.min_reduce += 1 logger.info("merged %s", self) else: # in common case, avoid doubling gigantic dict logger.info("using %i counts as vocab in %s", len(vocab), self) self.vocab = vocab def export_phrases(self, sentences, out_delimiter=b' ', as_tuples=False): """Get all phrases that appear in 'sentences' that pass the bigram threshold. Parameters ---------- sentences : iterable of list of str Text corpus. out_delimiter : str, optional Delimiter used to "glue" together words that form a bigram phrase. as_tuples : bool, optional Yield `(tuple(words), score)` instead of `(out_delimiter.join(words), score)`? Yields ------ ((str, str), float) **or** (str, float) Phrases detected in `sentences`. Return type depends on the `as_tuples` parameter. Example ------- >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus >>> from gensim.models.phrases import Phrases >>> >>> sentences = Text8Corpus(datapath('testcorpus.txt')) >>> phrases = Phrases(sentences, min_count=1, threshold=0.1) >>> >>> for phrase, score in phrases.export_phrases(sentences): ... pass """ analyze_sentence = ft.partial( self.analyze_sentence, threshold=self.threshold, common_terms=self.common_terms, scorer=ft.partial( self.scoring, len_vocab=float(len(self.vocab)), min_count=float(self.min_count), corpus_word_count=float(self.corpus_word_count), ), ) for sentence in sentences: bigrams = analyze_sentence(sentence) # keeps only not None scores filtered = ((words, score) for words, score in bigrams if score is not None) for words, score in filtered: if as_tuples: yield (tuple(words), score) else: yield (out_delimiter.join(words), score) def __getitem__(self, sentence): """Convert the input tokens `sentence` into tokens where detected bigrams are joined by a selected delimiter. If `sentence` is an entire corpus (iterable of sentences rather than a single sentence), return an iterable that converts each of the corpus' sentences into phrases on the fly, one after another. Parameters ---------- sentence : {list of str, iterable of list of str} Sentence or text corpus. Returns ------- {list of str, :class:`gensim.interfaces.TransformedCorpus`} `sentence` with detected phrase bigrams merged together, or a streamed corpus of such sentences if the input was a corpus. Examples ---------- >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus >>> from gensim.models.phrases import Phrases, Phraser >>> >>> #Create corpus >>> sentences = Text8Corpus(datapath('testcorpus.txt')) >>> >>> #Train the detector with: >>> phrases = Phrases(sentences, min_count=1, threshold=1) >>> #Input is a list of unicode strings: >>> sent = [u'trees', u'graph', u'minors'] >>> #Both of these tokens appear in corpus at least twice, and phrase score is higher, than treshold = 1: >>> print(phrases[sent]) [u'trees_graph', u'minors'] >>> >>> sentences = Text8Corpus(datapath('testcorpus.txt')) >>> phrases = Phrases(sentences, min_count=1, threshold=1) >>> phraser = Phraser(phrases) # for speedup >>> >>> sent = [[u'trees', u'graph', u'minors'],[u'graph', u'minors']] >>> for phrase in phraser[sent]: ... pass """ warnings.warn("For a faster implementation, use the gensim.models.phrases.Phraser class") delimiter = self.delimiter # delimiter used for lookup is_single, sentence = _is_single(sentence) if not is_single: # if the input is an entire corpus (rather than a single sentence), # return an iterable stream. return self._apply(sentence) delimiter = self.delimiter bigrams = self.analyze_sentence( sentence, threshold=self.threshold, common_terms=self.common_terms, scorer=ft.partial( self.scoring, len_vocab=float(len(self.vocab)), min_count=float(self.min_count), corpus_word_count=float(self.corpus_word_count), ), ) new_s = [] for words, score in bigrams: if score is not None: words = delimiter.join(words) new_s.append(words) return [utils.to_unicode(w) for w in new_s] def original_scorer(worda_count, wordb_count, bigram_count, len_vocab, min_count, corpus_word_count): """Bigram scoring function, based on the original `Mikolov, et. al: "Distributed Representations of Words and Phrases and their Compositionality" `_. Parameters ---------- worda_count : int Number of occurrences for first word. wordb_count : int Number of occurrences for second word. bigram_count : int Number of co-occurrences for phrase "worda_wordb". len_vocab : int Size of vocabulary. min_count: int Minimum score threshold. corpus_word_count : int Not used in this particular scoring technique. Notes ----- Formula: :math:`\\frac{(bigram\_count - min\_count) * len\_vocab }{ (worda\_count * wordb\_count)}`. """ return (bigram_count - min_count) / worda_count / wordb_count * len_vocab def npmi_scorer(worda_count, wordb_count, bigram_count, len_vocab, min_count, corpus_word_count): """Calculation NPMI score based on `"Normalized (Pointwise) Mutual Information in Colocation Extraction" by Gerlof Bouma `_. Parameters ---------- worda_count : int Number of occurrences for first word. wordb_count : int Number of occurrences for second word. bigram_count : int Number of co-occurrences for phrase "worda_wordb". len_vocab : int Not used. min_count: int Not used. corpus_word_count : int Total number of words in the corpus. Notes ----- Formula: :math:`\\frac{ln(prop(word_a, word_b) / (prop(word_a)*prop(word_b)))}{ -ln(prop(word_a, word_b)}`, where :math:`prob(word) = \\frac{word\_count}{corpus\_word\_count}` """ pa = worda_count / corpus_word_count pb = wordb_count / corpus_word_count pab = bigram_count / corpus_word_count return log(pab / (pa * pb)) / -log(pab) def pseudocorpus(source_vocab, sep, common_terms=frozenset()): """Feeds `source_vocab`'s compound keys back to it, to discover phrases. Parameters ---------- source_vocab : iterable of list of str Tokens vocabulary. sep : str Separator element. common_terms : set, optional Immutable set of stopwords. Yields ------ list of str Phrase. """ for k in source_vocab: if sep not in k: continue unigrams = k.split(sep) for i in range(1, len(unigrams)): if unigrams[i - 1] not in common_terms: # do not join common terms cterms = list(it.takewhile(lambda w: w in common_terms, unigrams[i:])) tail = unigrams[i + len(cterms):] components = [sep.join(unigrams[:i])] + cterms if tail: components.append(sep.join(tail)) yield components class Phraser(SentenceAnalyzer, PhrasesTransformation): """Minimal state & functionality exported from :class:`~gensim.models.phrases.Phrases`. The goal of this class is to cut down memory consumption of `Phrases`, by discarding model state not strictly needed for the bigram detection task. Use this instead of `Phrases` if you do not need to update the bigram statistics with new documents any more. """ def __init__(self, phrases_model): """ Parameters ---------- phrases_model : :class:`~gensim.models.phrases.Phrases` Trained phrases instance. Notes ----- After the one-time initialization, a :class:`~gensim.models.phrases.Phraser` will be much smaller and somewhat faster than using the full :class:`~gensim.models.phrases.Phrases` model. Examples -------- >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus >>> from gensim.models.phrases import Phrases, Phraser >>> >>> sentences = Text8Corpus(datapath('testcorpus.txt')) >>> phrases = Phrases(sentences, min_count=1, threshold=1) >>> >>> bigram = Phraser(phrases) >>> sent = [u'trees', u'graph', u'minors'] >>> print(bigram[sent]) [u'trees_graph', u'minors'] """ self.threshold = phrases_model.threshold self.min_count = phrases_model.min_count self.delimiter = phrases_model.delimiter self.scoring = phrases_model.scoring self.common_terms = phrases_model.common_terms corpus = self.pseudocorpus(phrases_model) self.phrasegrams = {} logger.info('source_vocab length %i', len(phrases_model.vocab)) count = 0 for bigram, score in phrases_model.export_phrases(corpus, self.delimiter, as_tuples=True): if bigram in self.phrasegrams: logger.info('Phraser repeat %s', bigram) self.phrasegrams[bigram] = (phrases_model.vocab[self.delimiter.join(bigram)], score) count += 1 if not count % 50000: logger.info('Phraser added %i phrasegrams', count) logger.info('Phraser built with %i %i phrasegrams', count, len(self.phrasegrams)) def pseudocorpus(self, phrases_model): """Alias for :func:`gensim.models.phrases.pseudocorpus`. Parameters ---------- phrases_model : :class:`~gensim.models.phrases.Phrases` Phrases instance. Return ------ generator Generator with phrases. """ return pseudocorpus(phrases_model.vocab, phrases_model.delimiter, phrases_model.common_terms) def score_item(self, worda, wordb, components, scorer): """Score a bigram. Parameters ---------- worda : str First word for comparison. wordb : str Second word for comparison. components : generator Contain phrases. scorer : {'default', 'npmi'} NOT USED. Returns ------- float Score for given bi-gram, if bi-gram not presented in dictionary - return -1. """ try: return self.phrasegrams[tuple(components)][1] except KeyError: return -1 def __getitem__(self, sentence): """Convert the input sequence of tokens `sentence` into a sequence of tokens where adjacent tokens are replaced by a single token if they form a bigram collocation. Parameters ---------- sentence : {list of str, iterable of list of str} Input sentence or a stream of sentences. Return ------ {list of str, iterable of list of str} Sentence or sentences with phrase tokens joined by `self.delimiter` character. Examples ---------- >>> from gensim.test.utils import datapath >>> from gensim.models.word2vec import Text8Corpus >>> from gensim.models.phrases import Phrases, Phraser >>> >>> sentences = Text8Corpus(datapath('testcorpus.txt')) # Read corpus >>> >>> phrases = Phrases(sentences, min_count=1, threshold=1) # Train model >>> # Create a Phraser object to transform any sentence and turn 2 suitable tokens into 1 phrase >>> phraser_model = Phraser(phrases) >>> >>> sent = [u'trees', u'graph', u'minors'] >>> print(phraser_model[sent]) [u'trees_graph', u'minors'] >>> sent = [[u'trees', u'graph', u'minors'],[u'graph', u'minors']] >>> for phrase in phraser_model[sent]: ... print(phrase) [u'trees_graph', u'minors'] [u'graph_minors'] """ is_single, sentence = _is_single(sentence) if not is_single: # if the input is an entire corpus (rather than a single sentence), # return an iterable stream. return self._apply(sentence) delimiter = self.delimiter bigrams = self.analyze_sentence( sentence, threshold=self.threshold, common_terms=self.common_terms, scorer=None) # we will use our score_item function redefinition new_s = [] for words, score in bigrams: if score is not None: words = delimiter.join(words) new_s.append(words) return [utils.to_unicode(w) for w in new_s] if __name__ == '__main__': logging.basicConfig(format='%(asctime)s : %(threadName)s : %(levelname)s : %(message)s', level=logging.INFO) logging.info("running %s", " ".join(sys.argv)) # check and process cmdline input program = os.path.basename(sys.argv[0]) if len(sys.argv) < 2: print(globals()['__doc__'] % locals()) sys.exit(1) infile = sys.argv[1] from gensim.models import Phrases # noqa:F811 for pickle from gensim.models.word2vec import Text8Corpus sentences = Text8Corpus(infile) # test_doc = LineSentence('test/test_data/testcorpus.txt') bigram = Phrases(sentences, min_count=5, threshold=100) for s in bigram[sentences]: print(utils.to_utf8(u' '.join(s)))