laywerrobot/lib/python3.6/site-packages/nltk/classify/textcat.py

# -*- coding: utf-8 -*-
# Natural Language Toolkit: Language ID module using TextCat algorithm
#
# Copyright (C) 2001-2018 NLTK Project
# Author: Avital Pekker <avital.pekker@utoronto.ca>
#
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

"""
A module for language identification using the TextCat algorithm.
An implementation of the text categorization algorithm
presented in Cavnar, W. B. and J. M. Trenkle, 
"N-Gram-Based Text Categorization".

The algorithm takes advantage of Zipf's law and uses 
n-gram frequencies to profile languages and text-yet to
be identified-then compares using a distance measure.

Language n-grams are provided by the "An Crubadan"
project. A corpus reader was created seperately to read
those files.

For details regarding the algorithm, see:
http://www.let.rug.nl/~vannoord/TextCat/textcat.pdf

For details about An Crubadan, see:
http://borel.slu.edu/crubadan/index.html
"""

# Ensure that literal strings default to unicode rather than str.
from __future__ import print_function, unicode_literals

from nltk.compat import PY3
from nltk.util import trigrams

if PY3:
    from sys import maxsize
else:
    from sys import maxint

# Note: this is NOT "re" you're likely used to. The regex module
# is an alternative to the standard re module that supports
# Unicode codepoint properties with the \p{} syntax.
# You may have to "pip install regx"
try:
    import regex as re
except ImportError:
    re = None
######################################################################
##  Language identification using TextCat
######################################################################

class TextCat(object):

    _corpus = None
    fingerprints = {}
    _START_CHAR = "<"
    _END_CHAR = ">"
    
    last_distances = {}
    
    def __init__(self):
        if not re:
            raise EnvironmentError("classify.textcat requires the regex module that "
                                   "supports unicode. Try '$ pip install regex' and "
                                   "see https://pypi.python.org/pypi/regex for "
                                   "further details.")

        from nltk.corpus import crubadan
        self._corpus = crubadan
        # Load all language ngrams into cache
        for lang in self._corpus.langs():
            self._corpus.lang_freq(lang)
        
    def remove_punctuation(self, text):
        ''' Get rid of punctuation except apostrophes '''
        return re.sub(r"[^\P{P}\']+", "", text)
    
    def profile(self, text):
        ''' Create FreqDist of trigrams within text '''
        from nltk import word_tokenize, FreqDist

        clean_text = self.remove_punctuation(text)
        tokens = word_tokenize(clean_text)
        
        fingerprint = FreqDist()
        for t in tokens:
            token_trigram_tuples = trigrams(self._START_CHAR + t + self._END_CHAR)
            token_trigrams = [''.join(tri) for tri in token_trigram_tuples]

            for cur_trigram in token_trigrams:
                if cur_trigram in fingerprint:
                    fingerprint[cur_trigram] += 1
                else:
                    fingerprint[cur_trigram] = 1

        return fingerprint
        
    def calc_dist(self, lang, trigram, text_profile):
        ''' Calculate the "out-of-place" measure between the
            text and language profile for a single trigram '''

        lang_fd = self._corpus.lang_freq(lang)
        dist = 0

        if trigram in lang_fd:
            idx_lang_profile = list(lang_fd.keys()).index(trigram)
            idx_text = list(text_profile.keys()).index(trigram)

            #print(idx_lang_profile, ", ", idx_text)
            dist = abs(idx_lang_profile - idx_text) 
        else:
            # Arbitrary but should be larger than
            # any possible trigram file length
            # in terms of total lines
            if PY3:
                dist = maxsize
            else:
                dist = maxint

        return dist
        
    def lang_dists(self, text):
        ''' Calculate the "out-of-place" measure between
            the text and all languages '''
        
        distances = {}
        profile = self.profile(text)
        # For all the languages
        for lang in self._corpus._all_lang_freq.keys():
            # Calculate distance metric for every trigram in
            # input text to be identified
            lang_dist = 0
            for trigram in profile:
                lang_dist += self.calc_dist(lang, trigram, profile)
        
            distances[lang] = lang_dist
            
        return distances
    
    def guess_language(self, text):
        ''' Find the language with the min distance
            to the text and return its ISO 639-3 code '''
        self.last_distances = self.lang_dists(text)
        
        return min(self.last_distances, key=self.last_distances.get)
        #################################################')

def demo():
    from nltk.corpus import udhr

    langs = ['Kurdish-UTF8', 'Abkhaz-UTF8', 'Farsi_Persian-UTF8',
             'Hindi-UTF8', 'Hawaiian-UTF8', 'Russian-UTF8', 'Vietnamese-UTF8',
             'Serbian_Srpski-UTF8','Esperanto-UTF8']

    friendly = {'kmr':'Northern Kurdish',
                'abk':'Abkhazian',
                'pes':'Iranian Persian',
                'hin':'Hindi',
                'haw':'Hawaiian',
                'rus':'Russian',
                'vie':'Vietnamese',
                'srp':'Serbian',
                'epo':'Esperanto'}
        
    tc = TextCat()

    for cur_lang in langs:
        # Get raw data from UDHR corpus
        raw_sentences = udhr.sents(cur_lang)
        rows = len(raw_sentences) - 1
        cols = list(map(len, raw_sentences))

        sample = ''
          
        # Generate a sample text of the language
        for i in range(0, rows):
            cur_sent = ''
            for j in range(0, cols[i]):
                cur_sent += ' ' + raw_sentences[i][j]
            
            sample += cur_sent
          
        # Try to detect what it is
        print('Language snippet: ' + sample[0:140] + '...')
        guess = tc.guess_language(sample)
        print('Language detection: %s (%s)' % (guess, friendly[guess]))
        print('#' * 140)


if __name__ == '__main__':
    demo()