Your IP : 216.73.216.85

Current Path : /opt/alt/python35/lib64/python3.5/test/__pycache__/
Current File : //opt/alt/python35/lib64/python3.5/test/__pycache__/test_tokenize.cpython-35.opt-2.pyc


���]w��@s|ddlmZddlmZmZmZmZmZmZmZm	Z	m
Z
mZmZm
ZmZddlmZddlmZmZddlZddlZGdd�de�Zdd	�ZGd
d�de�ZGdd
�d
e�ZGdd�de�ZGdd�de�ZGdd�de�ZGdd�de�ZGdd�de�Ze dkrxej!�dS)�)�support)
�tokenize�	_tokenize�
untokenize�NUMBER�NAME�OP�STRING�	ENDMARKER�ENCODING�tok_name�detect_encoding�open�Untokenizer)�BytesIO)�TestCase�mockNc@s�eZdZdd�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dd�Zdd�Zdd�Z
dd�Zdd�Zdd�Zdd�Zdd �Zd!d"�Zd#d$�Zd%S)&�TokenizeTestc
Cs�g}t|jd��}xQt|j�D]@\}}}}}	|tkrMPt|}|jdt��q+W|j|dg|j	�j
��dS)Nzutf-8z4    %(type)-10.10s %(token)-13.13r %(start)s %(end)sz*    ENCODING   'utf-8'       (0, 0) (0, 0))r�encoder�readliner
r�append�locals�assertEqual�rstrip�
splitlines)
�self�s�expected�result�f�type�token�start�end�line�r%�2/opt/alt/python35/lib64/python3.5/test_tokenize.py�check_tokenizes%
		zTokenizeTest.check_tokenizec	Csj|jdd�|jdd�d}t|�j}|jtd��xt|�D]}qUWWdQRXdS)Nz1 + 1z�    NUMBER     '1'           (1, 0) (1, 1)
    OP         '+'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    z.if False:
    # NL
    True = False # NEWLINE
a;    NAME       'if'          (1, 0) (1, 2)
    NAME       'False'       (1, 3) (1, 8)
    OP         ':'           (1, 8) (1, 9)
    NEWLINE    '\n'          (1, 9) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'True'        (3, 4) (3, 8)
    OP         '='           (3, 9) (3, 10)
    NAME       'False'       (3, 11) (3, 16)
    COMMENT    '# NEWLINE'   (3, 17) (3, 26)
    NEWLINE    '\n'          (3, 26) (3, 27)
    DEDENT     ''            (4, 0) (4, 0)
    sdef k(x):
    x += 2
  x += 5
z3unindent does not match any outer indentation level)r'rr�assertRaisesRegex�IndentationErrorr)rZindent_error_filer�tokr%r%r&�
test_basic!s			
zTokenizeTest.test_basiccCs�|jdd�|jdd�|jdd�|jdd�|jd	d
�|jdd�|jd
d�|jdd�|jdd�dS)Nz0xff <= 255z�    NUMBER     '0xff'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    z0b10 <= 255z�    NUMBER     '0b10'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    z0o123 <= 0O123z�    NUMBER     '0o123'       (1, 0) (1, 5)
    OP         '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    z1234567 > ~0x15z�    NUMBER     '1234567'     (1, 0) (1, 7)
    OP         '>'           (1, 8) (1, 9)
    OP         '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    z2134568 != 1231515z�    NUMBER     '2134568'     (1, 0) (1, 7)
    OP         '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    z(-124561-1) & 200000000ac    OP         '('           (1, 0) (1, 1)
    OP         '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    OP         '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         ')'           (1, 10) (1, 11)
    OP         '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    z0xdeadbeef != -1z�    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    OP         '!='          (1, 11) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    z0xdeadc0de & 12345z�    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    OP         '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    z0xFF & 0x15 | 1234z�    NUMBER     '0xFF'        (1, 0) (1, 4)
    OP         '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    OP         '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    )r')rr%r%r&�test_intDs$										zTokenizeTest.test_intcCsD|jdd�|jdd�|jdd�|jdd�dS)	Nzx = 0z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0'           (1, 4) (1, 5)
    zx = 0xfffffffffffz�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0xffffffffff (1, 4) (1, 17)
    zx = 123141242151251616110z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '123141242151 (1, 4) (1, 25)
    zx = -15921590215012591z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '-'           (1, 4) (1, 5)
    NUMBER     '159215902150 (1, 5) (1, 22)
    )r')rr%r%r&�	test_long}s				zTokenizeTest.test_longcCst|jdd�|jdd�|jdd�|jdd�|jd	d
�|jdd�|jd
d�dS)Nzx = 3.14159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    zx = 314159.z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    zx = .314159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    zx = 3e14159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    z	x = 3E123z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    z
x+y = 3e-1230z�    NAME       'x'           (1, 0) (1, 1)
    OP         '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    OP         '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    zx = 3.14e159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    )r')rr%r%r&�
test_float�s							zTokenizeTest.test_floatcCs�|jdd�|jdd�|jdd�|jdd�|jd	d
�|jdd�|jd
d�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�dS)Nzx = ''; y = ""a5    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    OP         ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    zx = '"'; y = "'"a6    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    OP         ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    OP         '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    zx = "doesn't "shrink", does it"z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    zx = 'abc' + 'ABC'z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    zy = "ABC" + "ABC"z�    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    z%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    z%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a�    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    zu'abc' + U'abc'z�    STRING     "u'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    zu"abc" + U"abc"z�    STRING     'u"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    zb'abc' + B'abc'z�    STRING     "b'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    zb"abc" + B"abc"z�    STRING     'b"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    z%br'abc' + bR'abc' + Br'abc' + BR'abc'a;    STRING     "br'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    z%br"abc" + bR"abc" + Br"abc" + BR"abc"a;    STRING     'br"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    z%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a;    STRING     "rb'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    z%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a;    STRING     'rb"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    )r')rr%r%r&�test_string�s<						
	
								zTokenizeTest.test_stringcCs$|jdd�|jdd�dS)Nz!def d22(a, b, c=2, d=2, *k): passa    NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    OP         ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    OP         '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    OP         '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    OP         ','           (1, 22) (1, 23)
    OP         '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    OP         ')'           (1, 26) (1, 27)
    OP         ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    zdef d01v_(a=1, *k, **w): passa�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    OP         '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         ','           (1, 13) (1, 14)
    OP         '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    OP         '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    OP         ')'           (1, 22) (1, 23)
    OP         ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    )r')rr%r%r&�
test_function7s		zTokenizeTest.test_functioncCs|jdd�dS)NzXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa�    NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    OP         '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    OP         '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    OP         '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    OP         '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    OP         ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    )r')rr%r%r&�test_comparison`s	"zTokenizeTest.test_comparisoncCs|jdd�dS)Nzx = 1 << 1 >> 5a6    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '<<'          (1, 6) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '>>'          (1, 11) (1, 13)
    NUMBER     '5'           (1, 14) (1, 15)
    )r')rr%r%r&�
test_shift�s	zTokenizeTest.test_shiftcCs|jdd�dS)Nz%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    OP         '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    OP         '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    OP         '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    OP         '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    OP         '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    OP         ']'           (1, 36) (1, 37)
    )r')rr%r%r&�
test_additive�s	zTokenizeTest.test_additivecCs|jdd�dS)Nzx = 1//1*1/5*12%0x12@42a�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    OP         '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    OP         '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    OP         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    )r')rr%r%r&�test_multiplicative�s	z TokenizeTest.test_multiplicativecCs$|jdd�|jdd�dS)Nz~1 ^ 1 & 1 |1 ^ -1a�    OP         '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '^'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    z-1*1/1+1*1//1 - ---1**1aH    OP         '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    OP         '-'           (1, 17) (1, 18)
    OP         '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    OP         '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    )r')rr%r%r&�
test_unary�s		zTokenizeTest.test_unarycCs|jdd�dS)Nz/import sys, time
x = sys.modules['time'].time()a�    NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    OP         ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'x'           (2, 0) (2, 1)
    OP         '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    OP         '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    OP         '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    OP         ']'           (2, 22) (2, 23)
    OP         '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    OP         '('           (2, 28) (2, 29)
    OP         ')'           (2, 29) (2, 30)
    )r')rr%r%r&�
test_selector�s	zTokenizeTest.test_selectorcCs|jdd�dS)Nz @staticmethod
def foo(x,y): passa    OP         '@'           (1, 0) (1, 1)
    NAME       'staticmethod (1, 1) (1, 13)
    NEWLINE    '\n'          (1, 13) (1, 14)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    OP         '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    OP         ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    OP         ')'           (2, 11) (2, 12)
    OP         ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    )r')rr%r%r&�test_method�s	
zTokenizeTest.test_methodcCs|jdd�dS)Nzdef f():
	if x
        	passa_    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    INDENT     '\t'          (2, 0) (2, 1)
    NAME       'if'          (2, 1) (2, 3)
    NAME       'x'           (2, 4) (2, 5)
    NEWLINE    '\n'          (2, 5) (2, 6)
    INDENT     '        \t'  (3, 0) (3, 9)
    NAME       'pass'        (3, 9) (3, 13)
    DEDENT     ''            (4, 0) (4, 0)
    DEDENT     ''            (4, 0) (4, 0)
    )r')rr%r%r&�	test_tabs	s	zTokenizeTest.test_tabscCs|jdd�dS)Nu!Örter = 'places'
grün = 'green'u7    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "'places'"    (1, 8) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "'green'"     (2, 7) (2, 14)
    )r')rr%r%r&�test_non_ascii_identifierss	z'TokenizeTest.test_non_ascii_identifierscCs|jdd�dS)Nu#Örter = u'places'
grün = U'green'u7    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    '\n'          (1, 17) (1, 18)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    )r')rr%r%r&�test_unicode*s	zTokenizeTest.test_unicodecCsT|jdd�|jdd�|jdd�|jdd�|jd	d
�|jdd�|jd
d�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd �|jd!d"�|jd#d$�|jd%d&�|jd'd(�|jd)d*�dS)+Nz	async = 1z�    NAME       'async'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    za = (async = 1)a8    NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '('           (1, 4) (1, 5)
    NAME       'async'       (1, 5) (1, 10)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    zasync()z�    NAME       'async'       (1, 0) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    zclass async(Bar):passa<    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    OP         ')'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    zclass async:passz�    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    z	await = 1z�    NAME       'await'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    z	foo.asyncz�    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    zasync for a in b: passa;    NAME       'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    zasync with a as b: passa<    NAME       'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    z	async.fooz�    NAME       'async'       (1, 0) (1, 5)
    OP         '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    �asyncz/    NAME       'async'       (1, 0) (1, 5)
    zasync
#comment
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    COMMENT    '#comment'    (2, 0) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NAME       'await'       (3, 0) (3, 5)
    zasync
...
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    OP         '...'         (2, 0) (2, 3)
    NEWLINE    '\n'          (2, 3) (2, 4)
    NAME       'await'       (3, 0) (3, 5)
    zasync
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    NAME       'await'       (2, 0) (2, 5)
    z
foo.async + 1z�    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    zasync def foo(): passa;    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    zNasync def foo():
  def foo(await):
    await = 1
  if 1:
    await
async += 1
a�    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    OP         '('           (2, 9) (2, 10)
    AWAIT      'await'       (2, 10) (2, 15)
    OP         ')'           (2, 15) (2, 16)
    OP         ':'           (2, 16) (2, 17)
    NEWLINE    '\n'          (2, 17) (2, 18)
    INDENT     '    '        (3, 0) (3, 4)
    AWAIT      'await'       (3, 4) (3, 9)
    OP         '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    '\n'          (3, 13) (3, 14)
    DEDENT     ''            (4, 2) (4, 2)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    OP         ':'           (4, 6) (4, 7)
    NEWLINE    '\n'          (4, 7) (4, 8)
    INDENT     '    '        (5, 0) (5, 4)
    AWAIT      'await'       (5, 4) (5, 9)
    NEWLINE    '\n'          (5, 9) (5, 10)
    DEDENT     ''            (6, 0) (6, 0)
    DEDENT     ''            (6, 0) (6, 0)
    NAME       'async'       (6, 0) (6, 5)
    OP         '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    '\n'          (6, 10) (6, 11)
    z)async def foo():
  async for i in 1: passa�    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    ASYNC      'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    OP         ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (3, 0) (3, 0)
    zasync def foo(async): awaitah    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    ASYNC      'async'       (1, 14) (1, 19)
    OP         ')'           (1, 19) (1, 20)
    OP         ':'           (1, 20) (1, 21)
    AWAIT      'await'       (1, 22) (1, 27)
    z@def f():

  def baz(): pass
  async def bar(): pass

  await = 2a�    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    ASYNC      'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    zFasync def f():

  def baz(): pass
  async def bar(): pass

  await = 2a    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    OP         '('           (1, 11) (1, 12)
    OP         ')'           (1, 12) (1, 13)
    OP         ':'           (1, 13) (1, 14)
    NEWLINE    '\n'          (1, 14) (1, 15)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    ASYNC      'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    AWAIT      'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    )r')rr%r%r&�
test_async6s\																#		zTokenizeTest.test_asyncN)�__name__�
__module__�__qualname__r'r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r<r%r%r%r&rs$#9(z)&%rcCs�g}tt|jd��j�}x�|D]z\}}}}}|tkr�d|kr�|jtdftdftt	|�ftdfg�q+|j
||f�q+Wt|�jd�S)Nzutf-8�.�Decimal�(�))
rrrrr�extendrrr	�reprrr�decode)rr�g�toknum�tokval�_r%r%r&�decistmtIs		rKc@seZdZdd�ZdS)�TestMisccCsnddlm}d}|jt|�d�|jtt|��d�|jtt|��|d��dS)Nr)rAz+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)�decimalrArrKZassertRegexrE�eval)rrArr%r%r&�
test_decistmtZszTestMisc.test_decistmtN)r=r>r?rOr%r%r%r&rLXsrLc@sXeZdZdd�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
S)�TestTokenizerAdheresToPep0263cCs>tjjtjjt�|�}tj|t|d��dS)N�rb)�os�path�join�dirname�__file__�
TestRoundtrip�check_roundtripr)r�filenamerSr%r%r&�	_testFilets!z'TestTokenizerAdheresToPep0263._testFilecCsd}|j|�dS)Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt)rZ)rrr%r%r&�'test_utf8_coding_cookie_and_no_utf8_bomxszETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bomcCs d}|jt|j|�dS)Nz8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt)�assertRaises�SyntaxErrorrZ)rrr%r%r&�&test_latin1_coding_cookie_and_utf8_bom|szDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bomcCsd}|j|�dS)Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt)rZ)rrr%r%r&�"test_no_coding_cookie_and_utf8_bom�sz@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bomcCsd}|j|�dS)Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt)rZ)rrr%r%r&�$test_utf8_coding_cookie_and_utf8_bom�szBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bomcCs0|jt|jd�|jt|jd�dS)Nz
bad_coding.pyzbad_coding2.py)r\r]rZ)rr%r%r&�test_bad_coding_cookie�sz4TestTokenizerAdheresToPep0263.test_bad_coding_cookieN)	r=r>r?rZr[r^r_r`rar%r%r%r&rPosrPc@s(eZdZdd�Zdd�ZdS)�
Test_Tokenizecsrd}|jd��d���fdd�}tt|dd��dd�}dg}|j||d�dS)N�"ЉЊЈЁЂ"zutf-8Fcs�sd��SdSdS)NT�r%r%)�firstr$r%r&r�szNTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline�encoding��r�zbytes not decoded with encoding����rgr�rgri)rhrcrkrlrc)r�listrr)r�literalr�tokens�expected_tokensr%)rer$r&�.test__tokenize_decodes_with_specified_encoding�s	"	z<Test_Tokenize.test__tokenize_decodes_with_specified_encodingcscd�d���fdd�}tt|dd��dd�}dg}|j||d
�dS)N�"ЉЊЈЁЂ"Fcs�sd��SdSdS)NTrdr%r%)rernr%r&r�szQTest_Tokenize.test__tokenize_does_not_decode_with_encoding_none.<locals>.readlinerfrgrhrriz*string not tokenized when encoding is Nonerj�rgr�rgri)rhrrrsrtrr)rmrr)rrrorpr%)rernr&�1test__tokenize_does_not_decode_with_encoding_none�s	"	z?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_noneN)r=r>r?rqrur%r%r%r&rb�srbc@seZdZdd�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dd�Zdd�Zdd�Z
dd�Zdd�Zdd�Zdd�Zdd �Zd!d"�Zd#d$�Zd%d&�Zd'd(�Zd)S)*�TestDetectEncodingcsd���fdd�}|S)Nrcs0�t��krt���}�d7�|S)Nrg)�len�
StopIteration)r$)�index�linesr%r&r�s


z1TestDetectEncoding.get_readline.<locals>.readliner%)rrzrr%)ryrzr&�get_readline�szTestDetectEncoding.get_readlinecCsUd}t|j|��\}}|j|d�|j|t|dd���dS)N�# something
�print(something)
�do_something(else)
zutf-8�)r|r}r~)r
r{rrm)rrzrf�consumed_linesr%r%r&�test_no_bom_no_encoding_cookie�sz1TestDetectEncoding.test_no_bom_no_encoding_cookiecCsKd}t|j|��\}}|j|d�|j|ddg�dS)N�# something
�print(something)
�do_something(else)
z	utf-8-sigs# something
)r�r�r�)r
r{r)rrzrfr�r%r%r&�test_bom_no_cookie�s	z%TestDetectEncoding.test_bom_no_cookiecCsHd}t|j|��\}}|j|d�|j|dg�dS)N�# -*- coding: latin-1 -*-
�print(something)
�do_something(else)
z
iso-8859-1)r�r�r�)r
r{r)rrzrfr�r%r%r&�test_cookie_first_line_no_bom�sz0TestDetectEncoding.test_cookie_first_line_no_bomcCsHd}t|j|��\}}|j|d�|j|dg�dS)N�# coding=utf-8
�print(something)
�do_something(else)
z	utf-8-sigs# coding=utf-8
)r�r�r�)r
r{r)rrzrfr�r%r%r&�&test_matched_bom_and_cookie_first_line�sz9TestDetectEncoding.test_matched_bom_and_cookie_first_linecCs,d}|j|�}|jtt|�dS)N�## vim: set fileencoding=ascii :
�print(something)
�do_something(else)
)r�r�r�)r{r\r]r
)rrzrr%r%r&�<test_mismatched_bom_and_cookie_first_line_raises_syntaxerror�s
zOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorcCsQd}t|j|��\}}|j|d�ddg}|j||�dS)N�
#! something
� # vim: set fileencoding=ascii :
�print(something)
�do_something(else)
�ascii)r�r�r�r�)r
r{r)rrzrfr�rr%r%r&�test_cookie_second_line_no_bom�sz1TestDetectEncoding.test_cookie_second_line_no_bomcCsKd}t|j|��\}}|j|d�|j|ddg�dS)N�#! something
�f# coding=utf-8
�print(something)
�do_something(else)
z	utf-8-sigs
#! something
)r�r�r�r�)r
r{r)rrzrfr�r%r%r&�'test_matched_bom_and_cookie_second_lines	z:TestDetectEncoding.test_matched_bom_and_cookie_second_linecCs,d}|j|�}|jtt|�dS)N�#! something
� # vim: set fileencoding=ascii :
�print(something)
�do_something(else)
)r�r�r�r�)r{r\r]r
)rrzrr%r%r&�=test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorszPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorcCsNd}t|j|��\}}|j|d�dg}|j||�dS)N�print('£')
�%# vim: set fileencoding=iso8859-15 :
�
print('€')
zutf-8)r�r�r�)r
r{r)rrzrfr�rr%r%r&�/test_cookie_second_line_noncommented_first_lines	zBTestDetectEncoding.test_cookie_second_line_noncommented_first_linecCsQd}t|j|��\}}|j|d�ddg}|j||�dS)N�
#print('£')
�%# vim: set fileencoding=iso8859-15 :
�
print('€')
z
iso8859-15)r�r�r�)r
r{r)rrzrfr�rr%r%r&�,test_cookie_second_line_commented_first_line&sz?TestDetectEncoding.test_cookie_second_line_commented_first_linecCsQd}t|j|��\}}|j|d�ddg}|j||�dS)N�
�%# vim: set fileencoding=iso8859-15 :
�
print('€')
z
iso8859-15)r�r�r�)r
r{r)rrzrfr�rr%r%r&�(test_cookie_second_line_empty_first_line1sz;TestDetectEncoding.test_cookie_second_line_empty_first_linec	Cs�d}x�|D]}}xtdD]l}|jd|�}d	d
|jd�dd
df}|j|�}t|�\}}|j|d�qWq
WdS)N�latin-1�
iso-8859-1�iso-latin-1�latin-1-unix�iso-8859-1-unix�iso-latin-1-mac�-rJs#!/usr/bin/python
s
# coding: r�s
sprint(things)
sdo_something += 4
)r�r�r�r�r�r�)r�rJ)�replacerr{r
r)	r�	encodingsrf�rep�encrz�rl�foundr�r%r%r&�test_latin1_normalization<s

	z,TestDetectEncoding.test_latin1_normalizationcCs,d}|j|�}|jtt|�dS)N�
print("�"))r�)r{r\r]r
)rrzrr%r%r&�test_syntaxerror_latin1Ksz*TestDetectEncoding.test_syntaxerror_latin1c	Cs�d}x�|D]z}xqdD]i}|jd|�}dd|jd�d	d
f}|j|�}t|�\}}|j|d�qWq
WdS)
N�utf-8�	utf-8-mac�
utf-8-unixr�rJs#!/usr/bin/python
s
# coding: r�s
s1 + 3
)r�r�r�)r�rJ)r�rr{r
r)	rr�rfr�r�rzr�r�r�r%r%r&�test_utf8_normalizationUs

	z*TestDetectEncoding.test_utf8_normalizationcCs*|jd�}t|�\}}|j|d�|j|dg�t|jf��\}}|j|d�|j|g�|jd�}t|�\}}|j|d�|j|dg�|jd	�}t|�\}}|j|d�|j|g�|jd
�}|jtt|�dS)N�print(something)
zutf-8�print(something)
z	utf-8-sig��# coding: bad
)r�)r�)r�)r�)r{r
rr\r])rrrfr�r%r%r&�test_short_filesbs"z#TestDetectEncoding.test_short_filescCsH|jd�}t|�\}}|j|d�|j|dg�dS)N�print("#coding=fake")zutf-8)r�)r{r
r)rrrfr�r%r%r&�test_false_encodingysz&TestDetectEncoding.test_false_encodingcCstjd}|jtj|�x�dD]�}t|dd|��+}td|d|�tdd|�WdQRXt|��-}|j|j|�|j|j	d	�WdQRXq'Wt|ddd
��}tdd|�WdQRXt|��-}|j|jd
�|j|j	d	�WdQRXdS)Nz.py�iso-8859-15�utf-8�wrfz# coding: %s�fileuprint('euro:€')�rz	utf-8-sig)r�r�)
rZTESTFNZ
addCleanup�unlinkr�print�
tokenize_openrrf�mode)rrYrf�fpr%r%r&�	test_open�s

zTestDetectEncoding.test_opencs�d}d�G�fdd�d�}|jt��'|�|�}|`t|j�WdQRX|jtdj|���!|�|�}t|j�WdQRXdS)NZsome_file_path�
print("�")cs.eZdZdd�Z�fdd�ZdS)z;TestDetectEncoding.test_filename_in_exception.<locals>.BunkcSs||_||_d|_dS)Nr)�nameZ_lines�_index)rrzrSr%r%r&�__init__�s		zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__cs;|jt��krt��|j}|jd7_|S)Nrg)r�rwrx)rr$)rzr%r&r�s

zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readlineN)r=r>r?r�rr%)rzr%r&�Bunk�sr�z.*{})r�)r\r]r�r
rr(�format)rrSr�Zinsr%)rzr&�test_filename_in_exception�s
z-TestDetectEncoding.test_filename_in_exceptioncCsPtd�}tjdd|��|jttd�WdQRX|j|j�dS)Ns#coding:xxxztokenize._builtin_openZreturn_valueZfoobar)rrZpatchr\r]r��
assertTrue�closed)r�mr%r%r&�test_open_error�sz"TestDetectEncoding.test_open_errorN)r=r>r?r{r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r%r%r%r&rv�s(


	


rvc@sLeZdZdd�Zdd�Zdd�Zdd�Zd	d
�ZdS)�TestTokenizecs�ddl}t��d��fdd�}�fdd�}d��fdd�}|j}|j}||_||_z8t|�}|jt|�dd	d
ddd
g�Wd||_||_X|j���dS)Nrcs�ddgfS)Nsfirstssecondr%)r)rfr%r&�mock_detect_encoding�sz8TestTokenize.test_tokenize.<locals>.mock_detect_encodingcs7|�g}x$|�}|r.|j|�q|SWdS)N)r)rrf�outZ	next_line)�
encoding_usedr%r&�mock__tokenize�s	
z2TestTokenize.test_tokenize.<locals>.mock__tokenizecs*�d7��dkrdSt��j�S)Nrg�rd)�strrr%)�counterr%r&�
mock_readline�s
z1TestTokenize.test_tokenize.<locals>.mock_readlinesfirstssecond�1�2�3�4)r�objectr
rrrmr�)rZtokenize_moduler�r�r�Zorig_detect_encodingZorig__tokenize�resultsr%)r�rfr�r&�
test_tokenize�s$						
zTestTokenize.test_tokenizecCs�g}x-td�D]}|jdjd|��qW|jd�dj|�}ttt|jd��j��}|j	|dj
d�dS)	Ni�zdef i{i}(): return {i}�iZOK�
zutf-8r���)�rangerr�rTrmrrrrr�string)r�bufr�Ztoksr%r%r&�test_oneline_defs�s
$zTestTokenize.test_oneline_defscGs�ttt|jd��j��}t|�}|jt|�d|�|jtj|dj	tjt
�xAt|�D]3}|jtj||dj	tj||�q|W|jtj|d|j	tjtj�dS)Nzutf-8rrrg)
rmrrrrrwrr!r�
exact_typerr�r
)rZopstrZoptypesroZnum_optypesr�r%r%r&�assertExactTypeEqual�s$z!TestTokenize.assertExactTypeEqualc
Cs�|jdtjtj�|jdtjtj�|jdtj�|jdtj�|jdtj�|jdtj	�|jdtj
�|jdtj�|jd	tj�|jd
tj
�|jdtj�|jdtj�|jd
tj�|jdtj�|jdtj�|jdtj�|jdtjtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj �|jdtj!�|jdtj"�|jdtj#�|jd tj$�|jd!tj%�|jd"tj&�|jd"tj&�|jd#tj'�|jd$tj(�|jd%tj)�|jd&tj*�|jd'tj+�|jd(tj,�|jd)tj-�|jd*t.tjt/tj	t.tjt/tjt.tjt/�|jd+tjtj/tjtj/tjtj/tj�|jd,tjtjtj.tjtj/tj�dS)-Nz()z[]�:�,�;�+r��*�/�|�&�<�>�=r@�%z{}z==z!=z<=z>=�~�^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=�@z@=za**2+b**2==c**2z	{1, 2, 3}z
^(x & 0x1))0r�r!�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENT�LBRACE�RBRACE�EQEQUAL�NOTEQUAL�	LESSEQUAL�GREATEREQUAL�TILDE�
CIRCUMFLEX�	LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR�	PLUSEQUAL�MINEQUAL�	STAREQUAL�
SLASHEQUAL�PERCENTEQUAL�
AMPEREQUAL�	VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL�DOUBLESLASH�DOUBLESLASHEQUAL�AT�ATEQUALrr)rr%r%r&�test_exact_type�sv		
	zTestTokenize.test_exact_typecCs|jdtj�dS)Nz@          )r�r!r1)rr%r%r&�%test_pathological_trailing_whitespace7sz2TestTokenize.test_pathological_trailing_whitespaceN)r=r>r?r�r�r�r3r4r%r%r%r&r��s
(>r�c@s4eZdZdd�Zdd�Zdd�ZdS)�UntokenizeTestcCsut�}d|_d|_|jt��}|jd�WdQRX|j|jjdd�|jt|jd�dS)Nrrgrhrz'start (1,3) precedes previous end (2,2))rgrh)rrg)	r�prev_row�prev_colr\�
ValueError�add_whitespacer�	exception�args)r�u�cmr%r%r&�test_bad_input_order>s			z#UntokenizeTest.test_bad_input_ordercCs�t�}d|_d|_g|_|jd	�|j|jdg�d|_|jd
�|j|jdddg�tj|d�dS)Nrgrrz\
�z\
\
z    za
  b
    c
  \
  c
)rr)r?r?)rr6r7ror9rrWrX)rr<r%r%r&�test_backslash_continuationJs				
	
z*UntokenizeTest.test_backslash_continuationcCs�t�}tdf}tdf|g}|j|tg��|j|jdg�t�}|j|jt|g��d�t�}|j|jt|��d�|j|jd�|jtt|��d�dS)NZHellozutf-8zHello sHello )	rrr�compat�iterrrorrf)rr<r!ror%r%r&�test_iter_compatWs		"	zUntokenizeTest.test_iter_compatN)r=r>r?r>r@rCr%r%r%r&r5<s
r5c@speZdZdd�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dS)rWcCst|t�r!|jd�}n|j�}|j�t|jdd��j}tt	|��}dd�|D�}t
|�}t|jdd��j}dd�t	|�D�}|j||�t
|�}	t|	jdd��j}
dd�t	|
�D�}|j||�dS)Nzutf-8�keependsTcSs g|]}|dd��qS)Nrr%)�.0r*r%r%r&�
<listcomp>{s	z1TestRoundtrip.check_roundtrip.<locals>.<listcomp>cSs g|]}|dd��qS)Nrr%)rEr*r%r%r&rFs	cSs g|]}|dd��qS)Nrr%)rEr*r%r%r&rF�s	)�
isinstancer�r�read�closerBr�__next__rmrrr)rr�coderZtokens5Ztokens2Zbytes_from2Z	readline2Z
tokens2_from2Zbytes_from5Z	readline5Z
tokens2_from5r%r%r&rXgs

zTestRoundtrip.check_roundtripc	Cs�|jd�|jd�|jd�tjd�}t|d��}|j|�WdQRX|jd�|jd�|jd�|jd	�dS)
Nzif x == 1:
    print(x)
z# This is a comment
# This alsozif x == 1 : 
  print(x)
ztokenize_tests.txtrQzoif x == 1:
    # A comment by itself.
    print(x) # Comment here, too.
    # Another comment.
after_if = True
zPif (x # The comments need to go in the right place
    == 1):
    print('x==1')
z�class Test: # A comment here
  # A comment with weird indent
  after_com = 5
  def x(m): return m*5 # a one liner
  def y(m): # A whitespace after the colon
     return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
    print('Can not import' # comment2
)else:   print('Loaded')
)rXr�findfiler)r�fnrr%r%r&�test_roundtrip�s





zTestRoundtrip.test_roundtripcCs|jd�dS)Nzna = (3,4, 
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
)rX)rr%r%r&�test_continuation�szTestRoundtrip.test_continuationcCs|jd�|jd�dS)Nz)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0)rX)rr%r%r&r@�s
z)TestRoundtrip.test_backslash_continuationcCs|jd�dS)Nz'' '')rX)rr%r%r&�test_string_concatenation�sz'TestRoundtrip.test_string_concatenationcCs#ddl}ddl}tjd�}tjj|�p?tj}|jtjj|d��}|j	tjj|d��x.dD]&}|j	tjj|d�|�q�Wtj
d
�s�|j|d�}xN|D]F}t|d��.}|j
d|��|j|�WdQRXWdQRXq�WdS)Nrztokenize_tests.txtztest*.pyztest_pep3131.py�buffer�builtin�fileio�inspectrR�platform�sysz
test_%s.pyZcpu�
rQr�)rQrRrSrTrRrUrV)�glob�randomrrLrRrSrU�curdirrT�removeZis_resource_enabled�samplerZsubTestrX)rrXrYrM�tempdirZ	testfilesrZtestfiler%r%r&�test_random_files�s
$
zTestRoundtrip.test_random_filescCs@t|t�r|jd�}ttt|�j��jd�S)Nzutf-8)rGr�rrrrrrF)rrKr%r%r&�	roundtrip�szTestRoundtrip.roundtripcCsGd}|j|�jd�}|j|d|d�|j|�dS)Nzif False:
	x=3
	x=3
r�rgr)r_�splitrrX)rrKZ	codelinesr%r%r&�#test_indentation_semantics_retained�sz1TestRoundtrip.test_indentation_semantics_retainedN)r=r>r?rXrNrOr@rPr^r_rar%r%r%r&rWes '
	rW�__main__)"Ztestrrrrrrrr	r
rrr
rr�r�iorZunittestrrrRr!rrKrLrPrbrvr�r5rWr=�mainr%r%r%r&�<module>s&X���A%&��)�