#!/usr/bin/python # -*- coding: utf-8 -*- # :Id: $Id: smartquotes.py 8095 2017-05-30 21:04:18Z milde $ # :Copyright: © 2010 Günter Milde, # original `SmartyPants`_: © 2003 John Gruber # smartypants.py: © 2004, 2007 Chad Miller # :Maintainer: docutils-develop@lists.sourceforge.net # :License: Released under the terms of the `2-Clause BSD license`_, in short: # # Copying and distribution of this file, with or without modification, # are permitted in any medium without royalty provided the copyright # notices and this notice are preserved. # This file is offered as-is, without any warranty. # # .. _2-Clause BSD license: http://www.spdx.org/licenses/BSD-2-Clause r""" ========================= Smart Quotes for Docutils ========================= Synopsis ======== "SmartyPants" is a free web publishing plug-in for Movable Type, Blosxom, and BBEdit that easily translates plain ASCII punctuation characters into "smart" typographic punctuation characters. ``smartquotes.py`` is an adaption of "SmartyPants" to Docutils_. * Using Unicode instead of HTML entities for typographic punctuation characters, it works for any output format that supports Unicode. * Supports `language specific quote characters`__. __ http://en.wikipedia.org/wiki/Non-English_usage_of_quotation_marks Authors ======= `John Gruber`_ did all of the hard work of writing this software in Perl for `Movable Type`_ and almost all of this useful documentation. `Chad Miller`_ ported it to Python to use with Pyblosxom_. Adapted to Docutils_ by Günter Milde. Additional Credits ================== Portions of the SmartyPants original work are based on Brad Choate's nifty MTRegex plug-in. `Brad Choate`_ also contributed a few bits of source code to this plug-in. Brad Choate is a fine hacker indeed. `Jeremy Hedley`_ and `Charles Wiltgen`_ deserve mention for exemplary beta testing of the original SmartyPants. `Rael Dornfest`_ ported SmartyPants to Blosxom. .. _Brad Choate: http://bradchoate.com/ .. _Jeremy Hedley: http://antipixel.com/ .. _Charles Wiltgen: http://playbacktime.com/ .. _Rael Dornfest: http://raelity.org/ Copyright and License ===================== SmartyPants_ license (3-Clause BSD license): Copyright (c) 2003 John Gruber (http://daringfireball.net/) All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name "SmartyPants" nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. This software is provided by the copyright holders and contributors "as is" and any express or implied warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. In no event shall the copyright owner or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute goods or services; loss of use, data, or profits; or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this software, even if advised of the possibility of such damage. smartypants.py license (2-Clause BSD license): smartypants.py is a derivative work of SmartyPants. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. This software is provided by the copyright holders and contributors "as is" and any express or implied warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. In no event shall the copyright owner or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute goods or services; loss of use, data, or profits; or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this software, even if advised of the possibility of such damage. .. _John Gruber: http://daringfireball.net/ .. _Chad Miller: http://web.chad.org/ .. _Pyblosxom: http://pyblosxom.bluesock.org/ .. _SmartyPants: http://daringfireball.net/projects/smartypants/ .. _Movable Type: http://www.movabletype.org/ .. _2-Clause BSD license: http://www.spdx.org/licenses/BSD-2-Clause .. _Docutils: http://docutils.sf.net/ Description =========== SmartyPants can perform the following transformations: - Straight quotes ( " and ' ) into "curly" quote characters - Backticks-style quotes (\`\`like this'') into "curly" quote characters - Dashes (``--`` and ``---``) into en- and em-dash entities - Three consecutive dots (``...`` or ``. . .``) into an ellipsis entity This means you can write, edit, and save your posts using plain old ASCII straight quotes, plain dashes, and plain dots, but your published posts (and final HTML output) will appear with smart quotes, em-dashes, and proper ellipses. SmartyPants does not modify characters within ``
``, ````, ````,
```` or ``
He said, "'Quoted' words in a larger quote."

    text = re.sub(r""""'(?=\w)""", smart.opquote+smart.osquote, text)
    text = re.sub(r"""'"(?=\w)""", smart.osquote+smart.opquote, text)

    # Special case for decade abbreviations (the '80s):
    if language.startswith('en'): # TODO similar cases in other languages?
        text = re.sub(r"""'(?=\d{2}s)""", smart.apostrophe, text, re.UNICODE)

    close_class = r"""[^\ \t\r\n\[\{\(\-]"""
    dec_dashes = r"""–|—"""

    # Get most opening single quotes:
    opening_single_quotes_regex = re.compile(r"""
                    (
                            \s          |   # a whitespace char, or
                                   |   # a non-breaking space entity, or
                            --          |   # dashes, or
                            &[mn]dash;  |   # named dash entities
                            %s          |   # or decimal entities
                            &\#x201[34];    # or hex
                    )
                    '                 # the quote
                    (?=\w)            # followed by a word character
                    """ % (dec_dashes,), re.VERBOSE | re.UNICODE)
    text = opening_single_quotes_regex.sub(r'\1'+smart.osquote, text)

    # In many locales, single closing quotes are different from apostrophe:
    if smart.csquote != smart.apostrophe:
        apostrophe_regex = re.compile(r"(?<=(\w|\d))'(?=\w)", re.UNICODE)
        text = apostrophe_regex.sub(smart.apostrophe, text)
    # TODO: keep track of quoting level to recognize apostrophe in, e.g.,
    # "Ich fass' es nicht."

    closing_single_quotes_regex = re.compile(r"""
                    (%s)
                    '
                    (?!\s  |       # whitespace
                       s\b |
                        \d         # digits   ('80s)
                    )
                    """ % (close_class,), re.VERBOSE | re.UNICODE)
    text = closing_single_quotes_regex.sub(r'\1'+smart.csquote, text)

    closing_single_quotes_regex = re.compile(r"""
                    (%s)
                    '
                    (\s | s\b)
                    """ % (close_class,), re.VERBOSE | re.UNICODE)
    text = closing_single_quotes_regex.sub(r'\1%s\2' % smart.csquote, text)

    # Any remaining single quotes should be opening ones:
    text = re.sub(r"""'""", smart.osquote, text)

    # Get most opening double quotes:
    opening_double_quotes_regex = re.compile(r"""
                    (
                            \s          |   # a whitespace char, or
                                   |   # a non-breaking space entity, or
                            --          |   # dashes, or
                            &[mn]dash;  |   # named dash entities
                            %s          |   # or decimal entities
                            &\#x201[34];    # or hex
                    )
                    "                 # the quote
                    (?=\w)            # followed by a word character
                    """ % (dec_dashes,), re.VERBOSE)
    text = opening_double_quotes_regex.sub(r'\1'+smart.opquote, text)

    # Double closing quotes:
    closing_double_quotes_regex = re.compile(r"""
                    #(%s)?   # character that indicates the quote should be closing
                    "
                    (?=\s)
                    """ % (close_class,), re.VERBOSE)
    text = closing_double_quotes_regex.sub(smart.cpquote, text)

    closing_double_quotes_regex = re.compile(r"""
                    (%s)   # character that indicates the quote should be closing
                    "
                    """ % (close_class,), re.VERBOSE)
    text = closing_double_quotes_regex.sub(r'\1'+smart.cpquote, text)

    # Any remaining quotes should be opening ones.
    text = re.sub(r'"', smart.opquote, text)

    return text


def educateBackticks(text, language='en'):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with ``backticks'' -style double quotes
                translated into HTML curly quote entities.
    Example input:  ``Isn't this fun?''
    Example output: “Isn't this fun?“;
    """
    smart = smartchars(language)

    text = re.sub(r"""``""", smart.opquote, text)
    text = re.sub(r"""''""", smart.cpquote, text)
    return text


def educateSingleBackticks(text, language='en'):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with `backticks' -style single quotes
                translated into HTML curly quote entities.

    Example input:  `Isn't this fun?'
    Example output: ‘Isn’t this fun?’
    """
    smart = smartchars(language)

    text = re.sub(r"""`""", smart.osquote, text)
    text = re.sub(r"""'""", smart.csquote, text)
    return text


def educateDashes(text):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each instance of "--" translated to
                an em-dash character.
    """

    text = re.sub(r"""---""", smartchars.endash, text) # en  (yes, backwards)
    text = re.sub(r"""--""", smartchars.emdash, text) # em (yes, backwards)
    return text


def educateDashesOldSchool(text):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each instance of "--" translated to
                an en-dash character, and each "---" translated to
                an em-dash character.
    """

    text = re.sub(r"""---""", smartchars.emdash, text)
    text = re.sub(r"""--""", smartchars.endash, text)
    return text


def educateDashesOldSchoolInverted(text):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each instance of "--" translated to
                an em-dash character, and each "---" translated to
                an en-dash character. Two reasons why: First, unlike the
                en- and em-dash syntax supported by
                EducateDashesOldSchool(), it's compatible with existing
                entries written before SmartyPants 1.1, back when "--" was
                only used for em-dashes.  Second, em-dashes are more
                common than en-dashes, and so it sort of makes sense that
                the shortcut should be shorter to type. (Thanks to Aaron
                Swartz for the idea.)
    """
    text = re.sub(r"""---""", smartchars.endash, text)    # em
    text = re.sub(r"""--""", smartchars.emdash, text)    # en
    return text



def educateEllipses(text):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each instance of "..." translated to
                an ellipsis character.

    Example input:  Huh...?
    Example output: Huh…?
    """

    text = re.sub(r"""\.\.\.""", smartchars.ellipsis, text)
    text = re.sub(r"""\. \. \.""", smartchars.ellipsis, text)
    return text


def stupefyEntities(text, language='en'):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each SmartyPants character translated to
                its ASCII counterpart.

    Example input:  “Hello — world.”
    Example output: "Hello -- world."
    """
    smart = smartchars(language)

    text = re.sub(smart.endash, "-", text)  # en-dash
    text = re.sub(smart.emdash, "--", text) # em-dash

    text = re.sub(smart.osquote, "'", text)  # open single quote
    text = re.sub(smart.csquote, "'", text)  # close single quote

    text = re.sub(smart.opquote, '"', text)  # open double quote
    text = re.sub(smart.cpquote, '"', text)  # close double quote

    text = re.sub(smart.ellipsis, '...', text)# ellipsis

    return text


def processEscapes(text, restore=False):
    r"""
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with after processing the following backslash
                escape sequences. This is useful if you want to force a "dumb"
                quote or other character to appear.

                Escape  Value
                ------  -----
                \\      \
                \"      "
                \'      '
                \.      .
                \-      -
                \`      `
    """
    replacements = ((r'\\', r'\'),
                    (r'\"', r'"'),
                    (r"\'", r'''),
                    (r'\.', r'.'),
                    (r'\-', r'-'),
                    (r'\`', r'`'))
    if restore:
        for (ch, rep) in replacements:
            text = text.replace(rep, ch[1])
    else:
        for (ch, rep) in replacements:
            text = text.replace(ch, rep)

    return text


def tokenize(text):
    """
    Parameter:  String containing HTML markup.
    Returns:    An iterator that yields the tokens comprising the input
                string. Each token is either a tag (possibly with nested,
                tags contained therein, such as , or a
                run of text between tags. Each yielded element is a
                two-element tuple; the first is either 'tag' or 'text';
                the second is the actual value.

    Based on the _tokenize() subroutine from Brad Choate's MTRegex plugin.
        
    """

    pos = 0
    length = len(text)
    # tokens = []

    depth = 6
    nested_tags = "|".join(['(?:<(?:[^<>]',] * depth) + (')*>)' * depth)
    #match = r"""(?:  ) |  # comments
    #               (?: <\? .*? \?> ) |  # directives
    #               %s  # nested tags       """ % (nested_tags,)
    tag_soup = re.compile(r"""([^<]*)(<[^>]*>)""")

    token_match = tag_soup.search(text)

    previous_end = 0
    while token_match is not None:
        if token_match.group(1):
            yield ('text', token_match.group(1))

        yield ('tag', token_match.group(2))

        previous_end = token_match.end()
        token_match = tag_soup.search(text, token_match.end())

    if previous_end < len(text):
        yield ('text', text[previous_end:])



if __name__ == "__main__":

    import itertools
    try:
        import locale # module missing in Jython
        locale.setlocale(locale.LC_ALL, '') # set to user defaults
        defaultlanguage = locale.getdefaultlocale()[0]
    except:
        defaultlanguage = 'en'

    # Normalize and drop unsupported subtags:
    defaultlanguage = defaultlanguage.lower().replace('-','_')
    # split (except singletons, which mark the following tag as non-standard):
    defaultlanguage = re.sub(r'_([a-zA-Z0-9])_', r'_\1-', defaultlanguage)
    _subtags = [subtag for subtag in defaultlanguage.split('_')]
    _basetag = _subtags.pop(0)
    # find all combinations of subtags
    for n in range(len(_subtags), 0, -1):
        for tags in itertools.combinations(_subtags, n):
            _tag = '-'.join((_basetag,)+tags)
            if _tag in smartchars.quotes:
                defaultlanguage = _tag
                break
        else:
            if _basetag in smartchars.quotes:
                defaultlanguage = _basetag
            else:
                defaultlanguage = 'en'


    import argparse
    parser = argparse.ArgumentParser(
                description='Filter stdin making ASCII punctuation "smart".')
    # parser.add_argument("text", help="text to be acted on")
    parser.add_argument("-a", "--action", default="1",
                        help="what to do with the input (see --actionhelp)")
    parser.add_argument("-e", "--encoding", default="utf8",
                        help="text encoding")
    parser.add_argument("-l", "--language", default=defaultlanguage,
                        help="text language (BCP47 tag), Default: %s"%defaultlanguage)
    parser.add_argument("-q", "--alternative-quotes", action="store_true",
                        help="use alternative quote style")
    parser.add_argument("--doc", action="store_true",
                        help="print documentation")
    parser.add_argument("--actionhelp", action="store_true",
                        help="list available actions")
    parser.add_argument("--stylehelp", action="store_true",
                        help="list available quote styles")
    parser.add_argument("--test", action="store_true",
                        help="perform short self-test")
    args = parser.parse_args()

    if args.doc:
        print (__doc__)
    elif args.actionhelp:
        print(options)
    elif args.stylehelp:
        print()
        print("Available styles (primary open/close, secondary open/close)")
        print("language tag   quotes")
        print("============   ======")
        for key in sorted(smartchars.quotes.keys()):
            print("%-14s %s" % (key, smartchars.quotes[key]))
    elif args.test:
        # Unit test output goes to stderr.
        import unittest

        class TestSmartypantsAllAttributes(unittest.TestCase):
            # the default attribute is "1", which means "all".
            def test_dates(self):
                self.assertEqual(smartyPants("1440-80's"), "1440-80’s")
                self.assertEqual(smartyPants("1440-'80s"), "1440-’80s")
                self.assertEqual(smartyPants("1440---'80s"), "1440–’80s")
                self.assertEqual(smartyPants("1960's"), "1960’s")
                self.assertEqual(smartyPants("one two '60s"), "one two ’60s")
                self.assertEqual(smartyPants("'60s"), "’60s")

            def test_educated_quotes(self):
                self.assertEqual(smartyPants('"Isn\'t this fun?"'), '“Isn’t this fun?”')

            def test_html_tags(self):
                text = 'more'
                self.assertEqual(smartyPants(text), text)

        suite = unittest.TestLoader().loadTestsFromTestCase(
                                            TestSmartypantsAllAttributes)
        unittest.TextTestRunner().run(suite)

    else:
        if args.alternative_quotes:
            if '-x-altquot' in args.language:
                args.language = args.language.replace('-x-altquot', '')
            else:
                args.language += '-x-altquot'
        text = sys.stdin.read().decode(args.encoding)
        print(smartyPants(text, attr=args.action,
                          language=args.language).encode(args.encoding))