Your IP : 216.73.216.213

Current Path : /proc/242857/root/opt/alt/python33/lib64/python3.3/test/__pycache__/
Current File : //proc/242857/root/opt/alt/python33/lib64/python3.3/test/__pycache__/test_tokenize.cpython-33.pyc
�
��f�c@s�dZddlmZddlmZmZmZmZmZmZm	Z	m
Z
mZmZm
Z
mZmZddlmZddlmZddlZddlZddlZddlZdd�Zd	d
�Zdd�ZGd
d�de�ZGdd�de�ZGdd�de�ZGdd�de�ZGdd�de�Z ied6ed6Z!dd�Z"e#dkr�e"�ndS(ua
Tests for the tokenize module.

The tests can be really simple. Given a small fragment of source
code, print out a table with tokens. The ENDMARKER is omitted for
brevity.

    >>> dump_tokens("1 + 1")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NUMBER     '1'           (1, 0) (1, 1)
    OP         '+'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)

    >>> dump_tokens("if False:\n"
    ...             "    # NL\n"
    ...             "    True = False # NEWLINE\n")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'if'          (1, 0) (1, 2)
    NAME       'False'       (1, 3) (1, 8)
    OP         ':'           (1, 8) (1, 9)
    NEWLINE    '\n'          (1, 9) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'True'        (3, 4) (3, 8)
    OP         '='           (3, 9) (3, 10)
    NAME       'False'       (3, 11) (3, 16)
    COMMENT    '# NEWLINE'   (3, 17) (3, 26)
    NEWLINE    '\n'          (3, 26) (3, 27)
    DEDENT     ''            (4, 0) (4, 0)

    >>> indent_error_file = """
    ... def k(x):
    ...     x += 2
    ...   x += 5
    ... """
    >>> readline = BytesIO(indent_error_file.encode('utf-8')).readline
    >>> for tok in tokenize(readline): pass
    Traceback (most recent call last):
        ...
    IndentationError: unindent does not match any outer indentation level

There are some standard formatting practices that are easy to get right.

    >>> roundtrip("if x == 1:\n"
    ...           "    print(x)\n")
    True

    >>> roundtrip("# This is a comment\n# This also")
    True

Some people use different formatting conventions, which makes
untokenize a little trickier. Note that this test involves trailing
whitespace after the colon. Note that we use hex escapes to make the
two trailing blanks apparent in the expected output.

    >>> roundtrip("if x == 1 : \n"
    ...           "  print(x)\n")
    True

    >>> f = support.findfile("tokenize_tests.txt")
    >>> roundtrip(open(f, 'rb'))
    True

    >>> roundtrip("if x == 1:\n"
    ...           "    # A comment by itself.\n"
    ...           "    print(x) # Comment here, too.\n"
    ...           "    # Another comment.\n"
    ...           "after_if = True\n")
    True

    >>> roundtrip("if (x # The comments need to go in the right place\n"
    ...           "    == 1):\n"
    ...           "    print('x==1')\n")
    True

    >>> roundtrip("class Test: # A comment here\n"
    ...           "  # A comment with weird indent\n"
    ...           "  after_com = 5\n"
    ...           "  def x(m): return m*5 # a one liner\n"
    ...           "  def y(m): # A whitespace after the colon\n"
    ...           "     return y*4 # 3-space indent\n")
    True

Some error-handling code

    >>> roundtrip("try: import somemodule\n"
    ...           "except ImportError: # comment\n"
    ...           "    print('Can not import' # comment2\n)"
    ...           "else:   print('Loaded')\n")
    True

Balancing continuation

    >>> roundtrip("a = (3,4, \n"
    ...           "5,6)\n"
    ...           "y = [3, 4,\n"
    ...           "5]\n"
    ...           "z = {'a': 5,\n"
    ...           "'b':15, 'c':True}\n"
    ...           "x = len(y) + 5 - a[\n"
    ...           "3] - a[2]\n"
    ...           "+ len(z) - z[\n"
    ...           "'b']\n")
    True

Ordinary integers and binary operators

    >>> dump_tokens("0xff <= 255")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NUMBER     '0xff'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    >>> dump_tokens("0b10 <= 255")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NUMBER     '0b10'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    >>> dump_tokens("0o123 <= 0O123")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NUMBER     '0o123'       (1, 0) (1, 5)
    OP         '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    >>> dump_tokens("1234567 > ~0x15")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NUMBER     '1234567'     (1, 0) (1, 7)
    OP         '>'           (1, 8) (1, 9)
    OP         '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    >>> dump_tokens("2134568 != 1231515")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NUMBER     '2134568'     (1, 0) (1, 7)
    OP         '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    >>> dump_tokens("(-124561-1) & 200000000")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    OP         '('           (1, 0) (1, 1)
    OP         '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    OP         '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         ')'           (1, 10) (1, 11)
    OP         '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    >>> dump_tokens("0xdeadbeef != -1")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    OP         '!='          (1, 11) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    >>> dump_tokens("0xdeadc0de & 12345")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    OP         '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    >>> dump_tokens("0xFF & 0x15 | 1234")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NUMBER     '0xFF'        (1, 0) (1, 4)
    OP         '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    OP         '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)

Long integers

    >>> dump_tokens("x = 0")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0'           (1, 4) (1, 5)
    >>> dump_tokens("x = 0xfffffffffff")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0xffffffffff (1, 4) (1, 17)
    >>> dump_tokens("x = 123141242151251616110")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '123141242151 (1, 4) (1, 25)
    >>> dump_tokens("x = -15921590215012591")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '-'           (1, 4) (1, 5)
    NUMBER     '159215902150 (1, 5) (1, 22)

Floating point numbers

    >>> dump_tokens("x = 3.14159")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    >>> dump_tokens("x = 314159.")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    >>> dump_tokens("x = .314159")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    >>> dump_tokens("x = 3e14159")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    >>> dump_tokens("x = 3E123")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    >>> dump_tokens("x+y = 3e-1230")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    OP         '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    >>> dump_tokens("x = 3.14e159")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)

String literals

    >>> dump_tokens("x = ''; y = \"\"")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    OP         ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    >>> dump_tokens("x = '\"'; y = \"'\"")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    OP         ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    OP         '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    >>> dump_tokens("x = \"doesn't \"shrink\", does it\"")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    >>> dump_tokens("x = 'abc' + 'ABC'")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    >>> dump_tokens('y = "ABC" + "ABC"')
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    >>> dump_tokens("x = r'abc' + r'ABC' + R'ABC' + R'ABC'")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    >>> dump_tokens('y = r"abc" + r"ABC" + R"ABC" + R"ABC"')
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)

    >>> dump_tokens("u'abc' + U'abc'")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    STRING     "u'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    >>> dump_tokens('u"abc" + U"abc"')
    ENCODING   'utf-8'       (0, 0) (0, 0)
    STRING     'u"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)

    >>> dump_tokens("b'abc' + B'abc'")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    STRING     "b'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    >>> dump_tokens('b"abc" + B"abc"')
    ENCODING   'utf-8'       (0, 0) (0, 0)
    STRING     'b"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    >>> dump_tokens("br'abc' + bR'abc' + Br'abc' + BR'abc'")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    STRING     "br'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    >>> dump_tokens('br"abc" + bR"abc" + Br"abc" + BR"abc"')
    ENCODING   'utf-8'       (0, 0) (0, 0)
    STRING     'br"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    >>> dump_tokens("rb'abc' + rB'abc' + Rb'abc' + RB'abc'")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    STRING     "rb'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    >>> dump_tokens('rb"abc" + rB"abc" + Rb"abc" + RB"abc"')
    ENCODING   'utf-8'       (0, 0) (0, 0)
    STRING     'rb"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)

Operators

    >>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    OP         ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    OP         '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    OP         '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    OP         ','           (1, 22) (1, 23)
    OP         '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    OP         ')'           (1, 26) (1, 27)
    OP         ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    >>> dump_tokens("def d01v_(a=1, *k, **w): pass")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    OP         '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         ','           (1, 13) (1, 14)
    OP         '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    OP         '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    OP         ')'           (1, 22) (1, 23)
    OP         ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)

Comparison

    >>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
    ...             "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    OP         '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    OP         '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    OP         '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    OP         '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    OP         ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)

Shift

    >>> dump_tokens("x = 1 << 1 >> 5")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '<<'          (1, 6) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '>>'          (1, 11) (1, 13)
    NUMBER     '5'           (1, 14) (1, 15)

Additive

    >>> dump_tokens("x = 1 - y + 15 - 1 + 0x124 + z + a[5]")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    OP         '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    OP         '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    OP         '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    OP         '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    OP         '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    OP         ']'           (1, 36) (1, 37)

Multiplicative

    >>> dump_tokens("x = 1//1*1/5*12%0x12")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    OP         '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    OP         '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)

Unary

    >>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    OP         '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '^'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    >>> dump_tokens("-1*1/1+1*1//1 - ---1**1")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    OP         '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    OP         '-'           (1, 17) (1, 18)
    OP         '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    OP         '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)

Selector

    >>> dump_tokens("import sys, time\nx = sys.modules['time'].time()")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    OP         ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'x'           (2, 0) (2, 1)
    OP         '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    OP         '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    OP         '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    OP         ']'           (2, 22) (2, 23)
    OP         '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    OP         '('           (2, 28) (2, 29)
    OP         ')'           (2, 29) (2, 30)

Methods

    >>> dump_tokens("@staticmethod\ndef foo(x,y): pass")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    OP         '@'           (1, 0) (1, 1)
    NAME       'staticmethod (1, 1) (1, 13)
    NEWLINE    '\n'          (1, 13) (1, 14)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    OP         '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    OP         ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    OP         ')'           (2, 11) (2, 12)
    OP         ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)

Backslash means line continuation, except for comments

    >>> roundtrip("x=1+\\n"
    ...           "1\n"
    ...           "# This is a comment\\n"
    ...           "# This also\n")
    True
    >>> roundtrip("# Comment \\nx = 0")
    True

Two string literals on the same line

    >>> roundtrip("'' ''")
    True

Test roundtrip on random python modules.
pass the '-ucpu' option to process the full directory.

    >>> import random
    >>> tempdir = os.path.dirname(f) or os.curdir
    >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))

Tokenize is broken on test_pep3131.py because regular expressions are
broken on the obscure unicode identifiers in it. *sigh*
With roundtrip extended to test the 5-tuple mode of  untokenize,
7 more testfiles fail.  Remove them also until the failure is diagnosed.

    >>> testfiles.remove(os.path.join(tempdir, "test_pep3131.py"))
    >>> for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'):
    ...     testfiles.remove(os.path.join(tempdir, "test_%s.py") % f)
    ...
    >>> if not support.is_resource_enabled("cpu"):
    ...     testfiles = random.sample(testfiles, 10)
    ...
    >>> for testfile in testfiles:
    ...     if not roundtrip(open(testfile, 'rb')):
    ...         print("Roundtrip failed for file %s" % testfile)
    ...         break
    ... else: True
    True

Evil tabs

    >>> dump_tokens("def f():\n\tif x\n        \tpass")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    INDENT     '\t'          (2, 0) (2, 1)
    NAME       'if'          (2, 1) (2, 3)
    NAME       'x'           (2, 4) (2, 5)
    NEWLINE    '\n'          (2, 5) (2, 6)
    INDENT     '        \t'  (3, 0) (3, 9)
    NAME       'pass'        (3, 9) (3, 13)
    DEDENT     ''            (4, 0) (4, 0)
    DEDENT     ''            (4, 0) (4, 0)

Non-ascii identifiers

    >>> dump_tokens("Örter = 'places'\ngrün = 'green'")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "'places'"    (1, 8) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "'green'"     (2, 7) (2, 14)

Legacy unicode literals:

    >>> dump_tokens("Örter = u'places'\ngrün = U'green'")
    ENCODING   'utf-8'       (0, 0) (0, 0)
    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    '\n'          (1, 17) (1, 18)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
i(usupport(
utokenizeu	_tokenizeu
untokenizeuNUMBERuNAMEuOPuSTRINGu	ENDMARKERuENCODINGutok_nameudetect_encodinguopenuUntokenizer(uBytesIO(uTestCaseNcCsmt|jd��}xQt|j�D]@\}}}}}|tkrJPnt|}tdt��q%WdS(uPPrint out the tokens in s in a table format.

    The ENDMARKER is omitted.
    uutf-8u0%(type)-10.10s %(token)-13.13r %(start)s %(end)sN(uBytesIOuencodeutokenizeureadlineu	ENDMARKERutok_nameuprintulocals(usufutypeutokenustartuenduline((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyudump_tokens�s%
udump_tokenscCst|t�r!|jd�}n|j�}|j�t|jdd��j}t	t
|��}dd�|D�}t|�}t|jdd��j}dd�t
|�D�}t|�}t|jdd��j}	dd�t
|	�D�}
||ko|
kSS(u
    Test roundtrip for `untokenize`. `f` is an open file or a string.
    The source code in f is tokenized to both 5- and 2-tuples.
    Both sequences are converted back to source code via
    tokenize.untokenize(), and the latter tokenized again to 2-tuples.
    The test fails if the 3 pair tokenizations do not match.

    When untokenize bugs are fixed, untokenize with 5-tuples should
    reproduce code that does not contain a backslash continuation
    following spaces.  A proper test should test this.

    This function would be more useful for correcting bugs if it reported
    the first point of failure, like assertEqual, rather than just
    returning False -- or if it were only used in unittests and not
    doctest and actually used assertEqual.
    uutf-8ukeependscSs g|]}|dd��qS(Ni((u.0utok((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu
<listcomp>�s	uroundtrip.<locals>.<listcomp>cSs g|]}|dd��qS(Ni((u.0utok((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu
<listcomp>�s	cSs g|]}|dd��qS(Ni((u.0utok((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu
<listcomp>�s	T(u
isinstanceustruencodeureaducloseuiteru
splitlinesuTrueu__next__ulistutokenizeu
untokenize(ufucodeureadlineutokens5utokens2ubytes_from2u	readline2u
tokens2_from2ubytes_from5u	readline5u
tokens2_from5((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu	roundtrip�s
u	roundtripcCs�g}tt|jd��j�}x�|D]z\}}}}}|tkr�d|kr�|jtdftdftt	|�ftdfg�q+|j
||f�q+Wt|�jd�S(u�Substitute Decimals for floats in a string of statements.

    >>> from decimal import Decimal
    >>> s = 'print(+21.3e-5*-.1234/81.7)'
    >>> decistmt(s)
    "print (+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7'))"

    The format of the exponent is inherited from the platform C library.
    Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
    we're only showing 11 digits, and the 12th isn't close to 5, the
    rest of the output should be platform-independent.

    >>> exec(s) #doctest: +ELLIPSIS
    -3.2171603427...e-0...7

    Output from calculations with Decimal should be identical across all
    platforms.

    >>> exec(decistmt(s))
    -3.217160342717258261933904529E-7
    uutf-8u.uDecimalu(u)(
utokenizeuBytesIOuencodeureadlineuNUMBERuextenduNAMEuOPuSTRINGurepruappendu
untokenizeudecode(usuresultugutoknumutokvalu_((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyudecistmt�s		udecistmtcBsb|EeZdZdZdd�Zdd�Zdd�Zdd	�Zd
d�Zdd
�Z	dS(uTestTokenizerAdheresToPep0263uU
    Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263.
    cCs4tjjtjjt�|�}tt|d��S(Nurb(uosupathujoinudirnameu__file__u	roundtripuopen(uselfufilenameupath((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu	_testFile�s!u'TestTokenizerAdheresToPep0263._testFilecCs d}|j|j|��dS(Nu9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt(u
assertTrueu	_testFile(uselfuf((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu'test_utf8_coding_cookie_and_no_utf8_bom�suETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bomcCs d}|jt|j|�dS(u�
        As per PEP 0263, if a file starts with a utf-8 BOM signature, the only
        allowed encoding for the comment is 'utf-8'.  The text file used in
        this test starts with a BOM signature, but specifies latin1 as the
        coding, so verify that a SyntaxError is raised, which matches the
        behaviour of the interpreter when it encounters a similar condition.
        u8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtN(uassertRaisesuSyntaxErroru	_testFile(uselfuf((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu&test_latin1_coding_cookie_and_utf8_bom�suDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bomcCs d}|j|j|��dS(Nu9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt(u
assertTrueu	_testFile(uselfuf((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu"test_no_coding_cookie_and_utf8_bom�su@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bomcCs d}|j|j|��dS(Nu6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt(u
assertTrueu	_testFile(uselfuf((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu$test_utf8_coding_cookie_and_utf8_bomsuBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bomcCs0|jt|jd�|jt|jd�dS(Nu
bad_coding.pyubad_coding2.py(uassertRaisesuSyntaxErroru	_testFile(uself((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_bad_coding_cookiesu4TestTokenizerAdheresToPep0263.test_bad_coding_cookieN(
u__name__u
__module__u__qualname__u__doc__u	_testFileu'test_utf8_coding_cookie_and_no_utf8_bomu&test_latin1_coding_cookie_and_utf8_bomu"test_no_coding_cookie_and_utf8_bomu$test_utf8_coding_cookie_and_utf8_bomutest_bad_coding_cookie(u
__locals__((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyuTestTokenizerAdheresToPep0263�suTestTokenizerAdheresToPep0263cBs,|EeZdZdd�Zdd�ZdS(u
Test_Tokenizecsrd}|jd��d���fdd�}tt|dd��dd�}dg}|j||d
�dS(Nu"ЉЊЈЁЂ"uutf-8cs�sd��SdSdS(NsT(uTrue((ufirstuline(u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyureadlinesuNTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readlineuencodingiiiiubytes not decoded with encodingFi����(ii(ii(iu"ЉЊЈЁЂ"(ii(iiu"ЉЊЈЁЂ"(uencodeuFalseulistu	_tokenizeuassertEqual(uselfuliteralureadlineutokensuexpected_tokens((ufirstulineu7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu.test__tokenize_decodes_with_specified_encodings	"	u<Test_Tokenize.test__tokenize_decodes_with_specified_encodingcscd�d
���fdd�}tt|dd��dd�}dg}|j||d	�dS(Nu"ЉЊЈЁЂ"cs�sd��SdSdS(NsT(uTrue((ufirstuliteral(u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyureadline suQTest_Tokenize.test__tokenize_does_not_decode_with_encoding_none.<locals>.readlineuencodingiiiiu*string not tokenized when encoding is NoneFi����(ii(ii(iu"ЉЊЈЁЂ"(ii(iiu"ЉЊЈЁЂ"(uFalseulistu	_tokenizeuNoneuassertEqual(uselfureadlineutokensuexpected_tokens((ufirstuliteralu7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu1test__tokenize_does_not_decode_with_encoding_nones	"	u?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_noneN(u__name__u
__module__u__qualname__u.test__tokenize_decodes_with_specified_encodingu1test__tokenize_does_not_decode_with_encoding_none(u
__locals__((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu
Test_Tokenize	su
Test_TokenizecBs�|EeZdZdd�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dd�Zdd�Zdd�Z
dd�Zdd�Zdd�Zdd�Zdd �Zd!d"�Zd#d$�Zd%d&�Zd'S((uTestDetectEncodingcsd���fdd�}|S(Nics3�t��krt�n��}�d7�|S(Ni(ulenu
StopIteration(uline(uindexulines(u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyureadline3s
	

u1TestDetectEncoding.get_readline.<locals>.readline((uselfulinesureadline((uindexulinesu7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyuget_readline1suTestDetectEncoding.get_readlinecCsUd}t|j|��\}}|j|d�|j|t|dd���dS(Ns# something
sprint(something)
sdo_something(else)
uutf-8i(s# something
sprint(something)
sdo_something(else)
(udetect_encodinguget_readlineuassertEqualulist(uselfulinesuencodinguconsumed_lines((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_no_bom_no_encoding_cookie<su1TestDetectEncoding.test_no_bom_no_encoding_cookiecCsKd}t|j|��\}}|j|d�|j|ddg�dS(Ns# something
sprint(something)
sdo_something(else)
u	utf-8-sigs# something
(s# something
sprint(something)
sdo_something(else)
(udetect_encodinguget_readlineuassertEqual(uselfulinesuencodinguconsumed_lines((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_bom_no_cookieFs	u%TestDetectEncoding.test_bom_no_cookiecCsHd}t|j|��\}}|j|d�|j|dg�dS(Ns# -*- coding: latin-1 -*-
sprint(something)
sdo_something(else)
u
iso-8859-1(s# -*- coding: latin-1 -*-
sprint(something)
sdo_something(else)
(udetect_encodinguget_readlineuassertEqual(uselfulinesuencodinguconsumed_lines((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_cookie_first_line_no_bomQsu0TestDetectEncoding.test_cookie_first_line_no_bomcCsHd}t|j|��\}}|j|d�|j|dg�dS(Ns# coding=utf-8
sprint(something)
sdo_something(else)
u	utf-8-sigs# coding=utf-8
(s# coding=utf-8
sprint(something)
sdo_something(else)
(udetect_encodinguget_readlineuassertEqual(uselfulinesuencodinguconsumed_lines((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu&test_matched_bom_and_cookie_first_line[su9TestDetectEncoding.test_matched_bom_and_cookie_first_linecCs,d}|j|�}|jtt|�dS(Ns## vim: set fileencoding=ascii :
sprint(something)
sdo_something(else)
(s## vim: set fileencoding=ascii :
sprint(something)
sdo_something(else)
(uget_readlineuassertRaisesuSyntaxErrorudetect_encoding(uselfulinesureadline((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu<test_mismatched_bom_and_cookie_first_line_raises_syntaxerrores
uOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorcCsQd}t|j|��\}}|j|d�ddg}|j||�dS(Ns
#! something
s # vim: set fileencoding=ascii :
sprint(something)
sdo_something(else)
uascii(s
#! something
s # vim: set fileencoding=ascii :
sprint(something)
sdo_something(else)
(udetect_encodinguget_readlineuassertEqual(uselfulinesuencodinguconsumed_linesuexpected((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_cookie_second_line_no_bomnsu1TestDetectEncoding.test_cookie_second_line_no_bomcCsKd}t|j|��\}}|j|d�|j|ddg�dS(Ns#! something
sf# coding=utf-8
sprint(something)
sdo_something(else)
u	utf-8-sigs
#! something
(s#! something
sf# coding=utf-8
sprint(something)
sdo_something(else)
(udetect_encodinguget_readlineuassertEqual(uselfulinesuencodinguconsumed_lines((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu'test_matched_bom_and_cookie_second_linezs	u:TestDetectEncoding.test_matched_bom_and_cookie_second_linecCs,d}|j|�}|jtt|�dS(Ns#! something
s # vim: set fileencoding=ascii :
sprint(something)
sdo_something(else)
(s#! something
s # vim: set fileencoding=ascii :
sprint(something)
sdo_something(else)
(uget_readlineuassertRaisesuSyntaxErrorudetect_encoding(uselfulinesureadline((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu=test_mismatched_bom_and_cookie_second_line_raises_syntaxerror�suPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorcCsNd}t|j|��\}}|j|d�dg}|j||�dS(Nsprint('£')
s%# vim: set fileencoding=iso8859-15 :
s
print('€')
uutf-8(sprint('£')
s%# vim: set fileencoding=iso8859-15 :
s
print('€')
(udetect_encodinguget_readlineuassertEqual(uselfulinesuencodinguconsumed_linesuexpected((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu/test_cookie_second_line_noncommented_first_line�s	uBTestDetectEncoding.test_cookie_second_line_noncommented_first_linecCsQd}t|j|��\}}|j|d�ddg}|j||�dS(Ns
#print('£')
s%# vim: set fileencoding=iso8859-15 :
s
print('€')
u
iso8859-15(s
#print('£')
s%# vim: set fileencoding=iso8859-15 :
s
print('€')
(udetect_encodinguget_readlineuassertEqual(uselfulinesuencodinguconsumed_linesuexpected((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu,test_cookie_second_line_commented_first_line�su?TestDetectEncoding.test_cookie_second_line_commented_first_linecCsQd}t|j|��\}}|j|d�ddg}|j||�dS(Ns
s%# vim: set fileencoding=iso8859-15 :
s
print('€')
u
iso8859-15(s
s%# vim: set fileencoding=iso8859-15 :
s
print('€')
(udetect_encodinguget_readlineuassertEqual(uselfulinesuencodinguconsumed_linesuexpected((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu(test_cookie_second_line_empty_first_line�su;TestDetectEncoding.test_cookie_second_line_empty_first_linec	Cs�d}x�|D]}}xtdD]l}|jd|�}d	d
|jd�dd
df}|j|�}t|�\}}|j|d�qWq
WdS(Nulatin-1u
iso-8859-1uiso-latin-1ulatin-1-unixuiso-8859-1-unixuiso-latin-1-macu-u_s#!/usr/bin/python
s
# coding: uasciis
sprint(things)
sdo_something += 4
(ulatin-1u
iso-8859-1uiso-latin-1ulatin-1-unixuiso-8859-1-unixuiso-latin-1-mac(u-u_(ureplaceuencodeuget_readlineudetect_encodinguassertEqual(	uselfu	encodingsuencodingurepuenculinesurlufounduconsumed_lines((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_latin1_normalization�s

	u,TestDetectEncoding.test_latin1_normalizationcCs,d}|j|�}|jtt|�dS(Ns
print("�")(s
print("�")(uget_readlineuassertRaisesuSyntaxErrorudetect_encoding(uselfulinesureadline((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_syntaxerror_latin1�su*TestDetectEncoding.test_syntaxerror_latin1c	Cs�d}x�|D]z}xqdD]i}|jd|�}dd|jd�d	d
f}|j|�}t|�\}}|j|d�qWq
WdS(
Nuutf-8u	utf-8-macu
utf-8-unixu-u_s#!/usr/bin/python
s
# coding: uasciis
s1 + 3
(uutf-8u	utf-8-macu
utf-8-unix(u-u_(ureplaceuencodeuget_readlineudetect_encodinguassertEqual(	uselfu	encodingsuencodingurepuenculinesurlufounduconsumed_lines((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_utf8_normalization�s

	u*TestDetectEncoding.test_utf8_normalizationcCs*|jd�}t|�\}}|j|d�|j|dg�t|jf��\}}|j|d�|j|g�|jd�}t|�\}}|j|d�|j|dg�|jd	�}t|�\}}|j|d�|j|g�|jd
�}|jtt|�dS(Nsprint(something)
uutf-8sprint(something)
u	utf-8-sigss# coding: bad
(sprint(something)
(sprint(something)
(s(s# coding: bad
(uget_readlineudetect_encodinguassertEqualuassertRaisesuSyntaxError(uselfureadlineuencodinguconsumed_lines((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_short_files�s"u#TestDetectEncoding.test_short_filescCsH|jd�}t|�\}}|j|d�|j|dg�dS(Nsprint("#coding=fake")uutf-8(sprint("#coding=fake")(uget_readlineudetect_encodinguassertEqual(uselfureadlineuencodinguconsumed_lines((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_false_encoding�su&TestDetectEncoding.test_false_encodingcCstjd}|jtj|�x�dD]�}t|dd|��+}td|d|�tdd|�WdQXt|��-}|j|j|�|j|j	d	�WdQXq'Wt|ddd
��}tdd|�WdQXt|��-}|j|jd
�|j|j	d	�WdQXdS(Nu.pyuiso-8859-15uutf-8uwuencodingu# coding: %sufileuprint('euro:€')uru	utf-8-sig(uiso-8859-15uutf-8(
usupportuTESTFNu
addCleanupuunlinkuopenuprintu
tokenize_openuassertEqualuencodingumode(uselfufilenameuencodingufp((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu	test_open�s

uTestDetectEncoding.test_opencs�d}d�G�fdd�d�}|jt��'|�|�}|`t|j�WdQX|jtdj|���!|�|�}t|j�WdQXdS(Nusome_file_paths
print("�")cs2|EeZdZdd�Z�fdd�ZdS(u;TestDetectEncoding.test_filename_in_exception.<locals>.BunkcSs||_||_d|_dS(Ni(unameu_linesu_index(uselfulinesupath((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu__init__s		uDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__cs>|jt��krt�n�|j}|jd7_|S(Ni(u_indexulenu
StopIteration(uselfuline(ulines(u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyureadlines
	
uDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readlineN(u__name__u
__module__u__qualname__u__init__ureadline(u
__locals__(ulines(u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyuBunksuBunku.*{}(s
print("�")(uassertRaisesuSyntaxErrorunameudetect_encodingureadlineuassertRaisesRegexuformat(uselfupathuBunkuins((ulinesu7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_filename_in_exception	s
u-TestDetectEncoding.test_filename_in_exceptionN(u__name__u
__module__u__qualname__uget_readlineutest_no_bom_no_encoding_cookieutest_bom_no_cookieutest_cookie_first_line_no_bomu&test_matched_bom_and_cookie_first_lineu<test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorutest_cookie_second_line_no_bomu'test_matched_bom_and_cookie_second_lineu=test_mismatched_bom_and_cookie_second_line_raises_syntaxerroru/test_cookie_second_line_noncommented_first_lineu,test_cookie_second_line_commented_first_lineu(test_cookie_second_line_empty_first_lineutest_latin1_normalizationutest_syntaxerror_latin1utest_utf8_normalizationutest_short_filesutest_false_encodingu	test_openutest_filename_in_exception(u
__locals__((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyuTestDetectEncoding/s&


	


uTestDetectEncodingcBsD|EeZdZdd�Zdd�Zdd�Zdd�Zd	S(
uTestTokenizecs�ddl}t��d��fdd�}�fdd�}d��fdd�}|j}|j}||_||_z8t|�}|jt|�dd	d
ddd
g�Wd||_||_X|j���dS(Nics�ddgfS(Nufirstusecond((ureadline(uencoding(u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyumock_detect_encoding,su8TestTokenize.test_tokenize.<locals>.mock_detect_encodingcs9|�g}x&|�}|r1|j|�qn|SdS(N(uappend(ureadlineuencodinguoutu	next_line(u
encoding_used(u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyumock__tokenize/s	
u2TestTokenize.test_tokenize.<locals>.mock__tokenizecs�d7��dkrdS�S(Niis(((ucounter(u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu
mock_readline;s
u1TestTokenize.test_tokenize.<locals>.mock_readlineufirstusecondiiii(utokenizeuobjectuNoneudetect_encodingu	_tokenizeuassertEqualulistu
assertTrue(uselfutokenize_moduleumock_detect_encodingumock__tokenizeu
mock_readlineuorig_detect_encodinguorig__tokenizeuresults((ucounteruencodingu
encoding_usedu7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu
test_tokenize(s"					,	
uTestTokenize.test_tokenizecGs�ttt|jd��j��}t|�}|jt|�d|�|jtj|dj	tjt
�xAt|�D]3}|jtj||dj	tj||�q|W|jtj|d|j	tjtj�dS(Nuutf-8iii(
ulistutokenizeuBytesIOuencodeureadlineulenuassertEqualutokenutok_nameu
exact_typeuENCODINGurangeu	ENDMARKER(uselfuopstruoptypesutokensunum_optypesui((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyuassertExactTypeEqualOs$u!TestTokenize.assertExactTypeEqualc
Cs�|jdtjtj�|jdtjtj�|jdtj�|jdtj�|jdtj�|jdtj	�|jdtj
�|jdtj�|jd	tj�|jd
tj
�|jdtj�|jdtj�|jd
tj�|jdtj�|jdtj�|jdtj�|jdtjtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj �|jdtj!�|jdtj"�|jdtj#�|jd tj$�|jd!tj%�|jd"tj&�|jd"tj&�|jd#tj'�|jd$tj(�|jd%tj)�|jd&tj*�|jd'tj+�|jd(tj,�|jd)t-tjt.tj	t-tjt.tjt-tjt.�|jd*tjtj.tjtj.tjtj.tj�|jd+tjtjtj-tjtj.tj�dS(,Nu()u[]u:u,u;u+u-u*u/u|u&u<u>u=u.u%u{}u==u!=u<=u>=u~u^u<<u>>u**u+=u-=u*=u/=u%=u&=u|=u^=u<<=u>>=u**=u//u//=u@ua**2+b**2==c**2u	{1, 2, 3}u
^(x & 0x1)(/uassertExactTypeEqualutokenuLPARuRPARuLSQBuRSQBuCOLONuCOMMAuSEMIuPLUSuMINUSuSTARuSLASHuVBARuAMPERuLESSuGREATERuEQUALuDOTuPERCENTuLBRACEuRBRACEuEQEQUALuNOTEQUALu	LESSEQUALuGREATEREQUALuTILDEu
CIRCUMFLEXu	LEFTSHIFTu
RIGHTSHIFTu
DOUBLESTARu	PLUSEQUALuMINEQUALu	STAREQUALu
SLASHEQUALuPERCENTEQUALu
AMPEREQUALu	VBAREQUALuCIRCUMFLEXEQUALuLEFTSHIFTEQUALuRIGHTSHIFTEQUALuDOUBLESTAREQUALuDOUBLESLASHuDOUBLESLASHEQUALuATuNAMEuNUMBER(uself((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_exact_type[st		
	uTestTokenize.test_exact_typecCs|jdtj�dS(Nu@          (uassertExactTypeEqualutokenuAT(uself((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu%test_pathological_trailing_whitespace�su2TestTokenize.test_pathological_trailing_whitespaceN(u__name__u
__module__u__qualname__u
test_tokenizeuassertExactTypeEqualutest_exact_typeu%test_pathological_trailing_whitespace(u
__locals__((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyuTestTokenize&s'=uTestTokenizecBs8|EeZdZdd�Zdd�Zdd�ZdS(uUntokenizeTestcCstt�}d|_d|_|jt��}|jd�WdQX|j|jjdd�|jt|jd�dS(Niiiiu'start (1,3) precedes previous end (2,2)(ii(ii(	uUntokenizeruprev_rowuprev_coluassertRaisesu
ValueErroruadd_whitespaceuassertEqualu	exceptionuargs(uselfuuucm((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_bad_input_order�s			u#UntokenizeTest.test_bad_input_ordercCs�t�}d|_d|_g|_|jd	�|j|jdg�d|_|jd
�|j|jdddg�|jtd��dS(Niiiu\
iu\
\
u    ua
  b
    c
  \
  c
(ii(ii(uUntokenizeruprev_rowuprev_colutokensuadd_whitespaceuassertEqualu
assertTrueu	roundtrip(uselfuu((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_backslash_continuation�s				
	
u*UntokenizeTest.test_backslash_continuationcCs�t�}tdf}tdf|g}|j|tg��|j|jdg�t�}|j|jt|g��d�t�}|j|jt|��d�|j|jd�|jtt|��d�dS(NuHellouutf-8uHello sHello (	uUntokenizeruNAMEuENCODINGucompatuiteruassertEqualutokensu
untokenizeuencoding(uselfuuutokenutokens((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyutest_iter_compat�s		"	uUntokenizeTest.test_iter_compatN(u__name__u
__module__u__qualname__utest_bad_input_orderutest_backslash_continuationutest_iter_compat(u
__locals__((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyuUntokenizeTest�s
uUntokenizeTestudoctestscCseddlm}tj|d�tjt�tjt�tjt�tjt	�tjt
�dS(Ni(u
test_tokenizeT(utestu
test_tokenizeusupporturun_doctestuTrueurun_unittestuTestTokenizerAdheresToPep0263u
Test_TokenizeuTestDetectEncodinguTestTokenizeuUntokenizeTest(u
test_tokenize((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu	test_main�s



u	test_mainu__main__($udoctestsutestusupportutokenizeu	_tokenizeu
untokenizeuNUMBERuNAMEuOPuSTRINGu	ENDMARKERuENCODINGutok_nameudetect_encodinguopenu
tokenize_openuUntokenizeruiouBytesIOuunittestuTestCaseuosusysuglobutokenudump_tokensu	roundtripudecistmtuTestTokenizerAdheresToPep0263u
Test_TokenizeuTestDetectEncodinguTestTokenizeuUntokenizeTestu__test__u	test_mainu__name__(((u7/opt/alt/python33/lib64/python3.3/test/test_tokenize.pyu<module>�s$X$&%%&�v)