Your IP : 216.73.216.196

Current Path : /opt/alt/python312/lib64/python3.12/test/__pycache__/
Upload File :
Current File : //opt/alt/python312/lib64/python3.12/test/__pycache__/test_tokenize.cpython-312.opt-2.pyc

�

�#�j���8�ddlZddlZddlZddlZddlmZmZmZmZmZm	Z	m
Z
mZmZm
Z
mZmZmZmZmZmZmZmZddlmZmZddlmZddlmZmZddlmZddl m!Z!m"Z"ddl#m$Z$dd	l%m&Z&m'Z'm(Z(d
�Z)Gd�de�Z*Gd
�de*�Z+d�Z,Gd�de�Z-Gd�de�Z.Gd�de�Z/Gd�de�Z0Gd�de�Z1Gd�de�Z2d�Z3Gd�de�Z4Gd�d e�Z5Gd!�d"e�Z6Gd#�d$ej8�Z7e8d%k(rejr�yy)&�N)�tokenize�
untokenize�NUMBER�NAME�OP�STRING�	ENDMARKER�ENCODING�tok_name�detect_encoding�open�Untokenizer�generate_tokens�NEWLINE�!_generate_tokens_from_c_tokenizer�DEDENT�	TokenInfo�
TokenError)�BytesIO�StringIO)�dedent)�TestCase�mock)�support)�VALID_UNDERSCORE_LITERALS�INVALID_UNDERSCORE_LITERALS)�	os_helper)�run_test_script�make_script�run_python_until_endc���g}t|j��}|ddv}|D]P\}}}}}	|tk(r|S|r|tk(r	|d|k(r�)t|}|jd|d�d|d�d|�d|����R|S)N���z
r�    �10� �13)�len�
splitlinesr	rr�append)
�token_generator�
source_string�result�	num_lines�missing_trailing_nl�type�token�start�end�lines
          �9/opt/alt/python312/lib64/python3.12/test/test_tokenize.py�stringify_tokens_from_sourcer5s���
�F��M�,�,�.�/�I�'��+�6�9��)8�%��e�U�C���9����M��4�7�?�s�1�v��7J����~���
�
��T�"�I�Q�u�r�l�!�E�7�!�C�5�A�B�*9��M�c��eZdZd�Zd�Zd�Zd�Zd�Zd�Zd�Z	d�Z
d	�Zd
�Zd�Z
d�Zd
�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zy)�TokenizeTestc���t|jd��}tt|j�|�}|j|dg|j
�j�z�y)N�utf-8z*    ENCODING   'utf-8'       (0, 0) (0, 0))r�encoder5r�readline�assertEqual�rstripr(��self�s�expected�fr,s     r4�check_tokenizezTokenizeTest.check_tokenize.sZ��
�A�H�H�W�%�&��-�h�q�z�z�.B�A�F������F�G�!���*�5�5�7�8�	9r6c���d�}|jt�5tt|�j��ddd�d�}|jt�5tt|�j��ddd�d�}|jt�5tt|�j��ddd�y#1swY��xYw#1swY�ZxYw#1swYyxYw)Nc3�K�d��d��y�w)N�sdfosdg�rHr6r4�genz/TokenizeTest.test_invalid_readline.<locals>.gen8s�����O��O���
c3�K�d��d��y�w)NssdfosdgrHrHr6r4rIz/TokenizeTest.test_invalid_readline.<locals>.gen>s��������rJc3�K�d��ddzy�w)NrG�rrHrHr6r4rIz/TokenizeTest.test_invalid_readline.<locals>.genDs�����O�
�a�C�s�
)�assertRaises�	TypeError�listr�__next__r�ZeroDivisionError)r@rIs  r4�test_invalid_readlinez"TokenizeTest.test_invalid_readline7s���	��
�
�y�
)���#�%�.�.�)�*�*�	��
�
�y�
)�������0�1�*�	��
�
�0�
1�������0�1�2�
1�*�
)��*�
)��2�
1�s#�$C�$C�!$C&�C�C#�&C/c��tdjd��}tt|j��}|j|djt�|j|djt�y)N�xr:���r")	rr;rPrr<r=r/rr	)r@rC�tokenss   r4�test_implicit_newlinez"TokenizeTest.test_implicit_newlineJsZ��
�C�J�J�w�'�(���h�q�z�z�*�+����������'�2���������)�4r6c�P�|jdd�|jdd�|jdd�|jdd�d	}t|�j}|jtd
�5}t|�D]}�	ddd�|j
jjd�|j
|jjd�|j
|jjd�|j
|jjd�|j
|jjd
�|j
|jjd
�|j
|jjd�y#1swY��xYw)Nz1 + 1z�    NUMBER     '1'           (1, 0) (1, 1)
    OP         '+'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    z3if False:
    # NL
    
    True = False # NEWLINE
af    NAME       'if'          (1, 0) (1, 2)
    NAME       'False'       (1, 3) (1, 8)
    OP         ':'           (1, 8) (1, 9)
    NEWLINE    '\n'          (1, 9) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NL         '\n'          (3, 4) (3, 5)
    INDENT     '    '        (4, 0) (4, 4)
    NAME       'True'        (4, 4) (4, 8)
    OP         '='           (4, 9) (4, 10)
    NAME       'False'       (4, 11) (4, 16)
    COMMENT    '# NEWLINE'   (4, 17) (4, 26)
    NEWLINE    '\n'          (4, 26) (4, 27)
    DEDENT     ''            (5, 0) (5, 0)
    z%if True:
    # NL
    foo='bar'

a@    NAME       'if'          (1, 0) (1, 2)
    NAME       'True'        (1, 3) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\r\n'        (1, 8) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\r\n'        (2, 8) (2, 10)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'foo'         (3, 4) (3, 7)
    OP         '='           (3, 7) (3, 8)
    STRING     "'bar'"       (3, 8) (3, 13)
    NEWLINE    '\r\n'        (3, 13) (3, 15)
    NL         '\r\n'        (4, 0) (4, 2)
    DEDENT     ''            (5, 0) (5, 0)
            zx = 1 + \
1
a    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (2, 0) (2, 1)
    NEWLINE    '\r\n'        (2, 1) (2, 3)
            sdef k(x):
    x += 2
  x += 5
z3unindent does not match any outer indentation level��<string>�	z  x += 5)rDrr<�assertRaisesRegex�IndentationErrorrr=�	exception�lineno�filename�
end_lineno�
end_offset�msg�offset�text)r@�indent_error_filer<�e�toks     r4�
test_basiczTokenizeTest.test_basicRsu�����G�&�		�
	
���;�=�		�(	
���K�N�	� 	
���1�4�	���
�,�-�6�6��
�
#�
#�$4�%>�?�BC���)���*�?�
	
������+�+�Q�/�������-�-�z�:�������/�/��6�������/�/��6����
�K�K�O�O�A�	C�	
������+�+�Q�/�������)�)�:�6�?�?�s�6F�F%c�H�|jdd�|jdd�|jdd�|jdd�|jd	d
�|jdd�|jd
d�|jdd�|jdd�y)N�0xff <= 255z�    NUMBER     '0xff'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    �0b10 <= 255z�    NUMBER     '0b10'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    �0o123 <= 0O123z�    NUMBER     '0o123'       (1, 0) (1, 5)
    OP         '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    �1234567 > ~0x15z�    NUMBER     '1234567'     (1, 0) (1, 7)
    OP         '>'           (1, 8) (1, 9)
    OP         '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    �2134568 != 1231515z�    NUMBER     '2134568'     (1, 0) (1, 7)
    OP         '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    �(-124561-1) & 200000000ac    OP         '('           (1, 0) (1, 1)
    OP         '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    OP         '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         ')'           (1, 10) (1, 11)
    OP         '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    �0xdeadbeef != -1z�    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    OP         '!='          (1, 11) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    �0xdeadc0de & 12345z�    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    OP         '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    �0xFF & 0x15 | 1234z�    NUMBER     '0xFF'        (1, 0) (1, 4)
    OP         '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    OP         '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    �rD�r@s r4�test_intzTokenizeTest.test_int�s������M�,�		�
	
���M�,�		�
	
���,�/�		�
	
���-�0�		�	
���0�3�		�
	
���5�	8�			�	
���.�1�		�	
���0�3�		�
	
���0�3�		r6c��|jdd�|jdd�|jdd�|jdd�y)	Nzx = 0z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0'           (1, 4) (1, 5)
    zx = 0xfffffffffffz�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0xfffffffffff' (1, 4) (1, 17)
    zx = 123141242151251616110z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '123141242151251616110' (1, 4) (1, 25)
    zx = -15921590215012591z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '-'           (1, 4) (1, 5)
    NUMBER     '15921590215012591' (1, 5) (1, 22)
    rurvs r4�	test_longzTokenizeTest.test_long�s_�����G�&�		�
	
���/�2�		�
	
���7�:�		�
	
���4�7�		r6c��|jdd�|jdd�|jdd�|jdd�|jd	d
�|jdd�|jd
d�y)N�x = 3.14159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    �x = 314159.z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    �x = .314159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    �x = 3e14159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    �	x = 3E123z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    �
x+y = 3e-1230z�    NAME       'x'           (1, 0) (1, 1)
    OP         '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    OP         '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    �x = 3.14e159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    rurvs r4�
test_floatzTokenizeTest.test_float�s������M�,�		�
	
���M�,�		�
	
���M�,�		�
	
���M�,�		�
	
���K�*�		�
	
���O�.�		�	
���N�-�		r6c���d�}tD]}d|vr�|j||�|��!ddh}tD](}||vr�	||�|j	||�|��*y#t$rY�7wxYw)Nc��t|jd��}t|j�D]\}}}}}|tk(s�|cSy)Nr:z
invalid token)rr;rr<r)rArC�toktyper0r1r2r3s       r4�number_tokenz;TokenizeTest.test_underscore_literals.<locals>.number_tokensF�������)�*�A�4<�Q�Z�Z�4H�0����s�D��f�$� �L�5I�#r6�(�0_7�09_99)rr=rr�assertNotEqual)r@r��lit�extra_valid_casess    r4�test_underscore_literalsz%TokenizeTest.test_underscore_literalss���	#�-�C��c�z�����\�#�.��4�	-�#�G�,��.�C��'�'��
��S�!�
����S� 1�3�7�/��
�
��
�s�A"�"	A.�-A.c�8�|jdd�|jdd�|jdd�|jdd�|jd	d
�|jdd�|jd
d�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd �|jd!d"�|jd#d$�|jd%d&�|jd'd(�|jd)d*�|jd+d,�|jd-d.�|jd/d0�|jd1d2�|jd3d4�|jd5d6�|jd7d8�|jd9d:�|jd;d<�|jd=d>�|jd?d@�|jdAdB�|jdCdD�|jdEdF�|jdGdH�|jdIdJ�y)KN�x = ''; y = ""a5    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    OP         ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    �x = '"'; y = "'"a6    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    OP         ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    OP         '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    �x = "doesn't "shrink", does it"z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    �x = 'abc' + 'ABC'z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    �y = "ABC" + "ABC"z�    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    �%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    �%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a�    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    �u'abc' + U'abc'z�    STRING     "u'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    �u"abc" + U"abc"z�    STRING     'u"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    �b'abc' + B'abc'z�    STRING     "b'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    �b"abc" + B"abc"z�    STRING     'b"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    �%br'abc' + bR'abc' + Br'abc' + BR'abc'a;    STRING     "br'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    �%br"abc" + bR"abc" + Br"abc" + BR"abc"a;    STRING     'br"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    �%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a;    STRING     "rb'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    �%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a;    STRING     'rb"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    �"a\
de\
fg"�3    STRING     '"a\\\nde\\\nfg"' (1, 0) (3, 3)
    �u"a\
de"�/    STRING     'u"a\\\nde"'  (1, 0) (2, 3)
    �rb"a\
d"�/    STRING     'rb"a\\\nd"'  (1, 0) (2, 2)
    �
"""a\
b"""�0    STRING     '"""a\\\nb"""' (1, 0) (2, 4)
    �u"""a\
b"""�1    STRING     'u"""a\\\nb"""' (1, 0) (2, 4)
    �rb"""a\
b\
c"""�7    STRING     'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4)
    �f"abc"��    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE 'abc'         (1, 2) (1, 5)
    FSTRING_END '"'           (1, 5) (1, 6)
    �	fR"a{b}c"a=    FSTRING_START 'fR"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'a'           (1, 3) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    NAME       'b'           (1, 5) (1, 6)
    OP         '}'           (1, 6) (1, 7)
    FSTRING_MIDDLE 'c'           (1, 7) (1, 8)
    FSTRING_END '"'           (1, 8) (1, 9)
    zfR"a{{{b!r}}}c"a�    FSTRING_START 'fR"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'a{'          (1, 3) (1, 5)
    OP         '{'           (1, 6) (1, 7)
    NAME       'b'           (1, 7) (1, 8)
    OP         '!'           (1, 8) (1, 9)
    NAME       'r'           (1, 9) (1, 10)
    OP         '}'           (1, 10) (1, 11)
    FSTRING_MIDDLE '}'           (1, 11) (1, 12)
    FSTRING_MIDDLE 'c'           (1, 13) (1, 14)
    FSTRING_END '"'           (1, 14) (1, 15)
    zf"{{{1+1}}}"a�    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE '{'           (1, 2) (1, 3)
    OP         '{'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '}'           (1, 8) (1, 9)
    FSTRING_MIDDLE '}'           (1, 9) (1, 10)
    FSTRING_END '"'           (1, 11) (1, 12)
    zf"""{f'''{f'{f"{1+1}"}'}'''}"""ad    FSTRING_START 'f"""'        (1, 0) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    FSTRING_START "f'''"        (1, 5) (1, 9)
    OP         '{'           (1, 9) (1, 10)
    FSTRING_START "f'"          (1, 10) (1, 12)
    OP         '{'           (1, 12) (1, 13)
    FSTRING_START 'f"'          (1, 13) (1, 15)
    OP         '{'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '+'           (1, 17) (1, 18)
    NUMBER     '1'           (1, 18) (1, 19)
    OP         '}'           (1, 19) (1, 20)
    FSTRING_END '"'           (1, 20) (1, 21)
    OP         '}'           (1, 21) (1, 22)
    FSTRING_END "'"           (1, 22) (1, 23)
    OP         '}'           (1, 23) (1, 24)
    FSTRING_END "'''"         (1, 24) (1, 27)
    OP         '}'           (1, 27) (1, 28)
    FSTRING_END '"""'         (1, 28) (1, 31)
    z.f"""     x
str(data, encoding={invalid!r})
"""a�    FSTRING_START 'f"""'        (1, 0) (1, 4)
    FSTRING_MIDDLE '     x\nstr(data, encoding=' (1, 4) (2, 19)
    OP         '{'           (2, 19) (2, 20)
    NAME       'invalid'     (2, 20) (2, 27)
    OP         '!'           (2, 27) (2, 28)
    NAME       'r'           (2, 28) (2, 29)
    OP         '}'           (2, 29) (2, 30)
    FSTRING_MIDDLE ')\n'         (2, 30) (3, 0)
    FSTRING_END '"""'         (3, 0) (3, 3)
    z#f"""123456789
something{None}bad"""aO    FSTRING_START 'f"""'        (1, 0) (1, 4)
    FSTRING_MIDDLE '123456789\nsomething' (1, 4) (2, 9)
    OP         '{'           (2, 9) (2, 10)
    NAME       'None'        (2, 10) (2, 14)
    OP         '}'           (2, 14) (2, 15)
    FSTRING_MIDDLE 'bad'         (2, 15) (2, 18)
    FSTRING_END '"""'         (2, 18) (2, 21)
    �
f"""abc"""��    FSTRING_START 'f"""'        (1, 0) (1, 4)
    FSTRING_MIDDLE 'abc'         (1, 4) (1, 7)
    FSTRING_END '"""'         (1, 7) (1, 10)
    �f"abc\
def"��    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE 'abc\\\ndef'  (1, 2) (2, 3)
    FSTRING_END '"'           (2, 3) (2, 4)
    �Rf"abc\
def"��    FSTRING_START 'Rf"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'abc\\\ndef'  (1, 3) (2, 3)
    FSTRING_END '"'           (2, 3) (2, 4)
    z5f'some words {a+b:.3f} more words {c+d=} final words'aA    FSTRING_START "f'"          (1, 0) (1, 2)
    FSTRING_MIDDLE 'some words ' (1, 2) (1, 13)
    OP         '{'           (1, 13) (1, 14)
    NAME       'a'           (1, 14) (1, 15)
    OP         '+'           (1, 15) (1, 16)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    FSTRING_MIDDLE '.3f'         (1, 18) (1, 21)
    OP         '}'           (1, 21) (1, 22)
    FSTRING_MIDDLE ' more words ' (1, 22) (1, 34)
    OP         '{'           (1, 34) (1, 35)
    NAME       'c'           (1, 35) (1, 36)
    OP         '+'           (1, 36) (1, 37)
    NAME       'd'           (1, 37) (1, 38)
    OP         '='           (1, 38) (1, 39)
    OP         '}'           (1, 39) (1, 40)
    FSTRING_MIDDLE ' final words' (1, 40) (1, 52)
    FSTRING_END "'"           (1, 52) (1, 53)
    z
f'''{
3
=}'''a`    FSTRING_START "f'''"        (1, 0) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    NL         '\n'          (1, 5) (1, 6)
    NUMBER     '3'           (2, 0) (2, 1)
    NL         '\n'          (2, 1) (2, 2)
    OP         '='           (3, 0) (3, 1)
    OP         '}'           (3, 1) (3, 2)
    FSTRING_END "'''"         (3, 2) (3, 5)
    �f'''__{
    x:a
}__'''a�    FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    OP         '{'           (1, 6) (1, 7)
    NL         '\n'          (1, 7) (1, 8)
    NAME       'x'           (2, 4) (2, 5)
    OP         ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n'         (2, 6) (3, 0)
    OP         '}'           (3, 0) (3, 1)
    FSTRING_MIDDLE '__'          (3, 1) (3, 3)
    FSTRING_END "'''"         (3, 3) (3, 6)
    �+f'''__{
    x:a
    b
     c
      d
}__'''a�    FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    OP         '{'           (1, 6) (1, 7)
    NL         '\n'          (1, 7) (1, 8)
    NAME       'x'           (2, 4) (2, 5)
    OP         ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n    b\n     c\n      d\n' (2, 6) (6, 0)
    OP         '}'           (6, 0) (6, 1)
    FSTRING_MIDDLE '__'          (6, 1) (6, 3)
    FSTRING_END "'''"         (6, 3) (6, 6)
    �f'__{
    x:d
}__'a�    FSTRING_START "f'"          (1, 0) (1, 2)
    FSTRING_MIDDLE '__'          (1, 2) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    NL         '\n'          (1, 5) (1, 6)
    NAME       'x'           (2, 4) (2, 5)
    OP         ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'd'           (2, 6) (2, 7)
    NL         '\n'          (2, 7) (2, 8)
    OP         '}'           (3, 0) (3, 1)
    FSTRING_MIDDLE '__'          (3, 1) (3, 3)
    FSTRING_END "'"           (3, 3) (3, 4)
    uz    '''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli
    aktualni pracownicy, obecni pracownicy'''
u    INDENT     '    '        (1, 0) (1, 4)
    STRING     "'''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli\n    aktualni pracownicy, obecni pracownicy'''" (1, 4) (2, 45)
    NEWLINE    '\n'          (2, 45) (2, 46)
    DEDENT     ''            (3, 0) (3, 0)
    rurvs r4�test_stringzTokenizeTest.test_string+s������.�1�		�	
���1�4�		�	
���A�D�		�	
���/�2�		�	
���/�2�		�	
���C�
F�
		�	
���C�
F�
		�	
���-�0�		�
	
���-�0�		�	
���-�0�		�
	
���-�0�		�
	
���C�F�		�	
���C�F�		�	
���C�F�		�	
���C�F�		�	
�����		�
	
�����		�	
�����		�	
�����		�	
�����		�	
�����		�
	
���H�'�		�
	
���K�*�		�	
���-�0�		�	
���N�
-�
		�	
���E�H�		�*	
���N�
Q�
		�	
���B�E�		�	
���L�+�		�
	
�����		�	
�����		�	
���S�V�		�(	
���	�	�		�	
���
��		�	
���
��
		�$	
����
�		�"	
�����		r6c�p�|jdd�|jdd�|jdd�y)N�!def d22(a, b, c=2, d=2, *k): passa    NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    OP         ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    OP         '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    OP         '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    OP         ','           (1, 22) (1, 23)
    OP         '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    OP         ')'           (1, 26) (1, 27)
    OP         ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    �def d01v_(a=1, *k, **w): passa�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    OP         '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         ','           (1, 13) (1, 14)
    OP         '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    OP         '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    OP         ')'           (1, 22) (1, 23)
    OP         ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    �&def d23(a: str, b: int=3) -> int: passa�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    OP         ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    OP         '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    OP         ')'           (1, 24) (1, 25)
    OP         '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    OP         ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    rurvs r4�
test_functionzTokenizeTest.test_function{sJ�����?�B�		�,	
���;�>�		�"	
���D�G�		r6c�(�|jdd�y)N�Xif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa�    NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    OP         '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    OP         '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    OP         '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    OP         '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    OP         ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    rurvs r4�test_comparisonzTokenizeTest.test_comparison�������M�!O�"		r6c�(�|jdd�y)Nzx = 1 << 1 >> 5a6    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '<<'          (1, 6) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '>>'          (1, 11) (1, 13)
    NUMBER     '5'           (1, 14) (1, 15)
    rurvs r4�
test_shiftzTokenizeTest.test_shift�s�����-�0�		r6c�(�|jdd�y)N�%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    OP         '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    OP         '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    OP         '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    OP         '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    OP         '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    OP         ']'           (1, 36) (1, 37)
    rurvs r4�
test_additivezTokenizeTest.test_additive�������C�F�		r6c�(�|jdd�y)N�x = 1//1*1/5*12%0x12@42a�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    OP         '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    OP         '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    OP         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    rurvs r4�test_multiplicativez TokenizeTest.test_multiplicative������5�8�		r6c�L�|jdd�|jdd�y)N�~1 ^ 1 & 1 |1 ^ -1a�    OP         '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '^'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    �-1*1/1+1*1//1 - ---1**1aH    OP         '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    OP         '-'           (1, 17) (1, 18)
    OP         '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    OP         '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    rurvs r4�
test_unaryzTokenizeTest.test_unarys0�����0�3�		�	
���5�8�		r6c�(�|jdd�y)N�/import sys, time
x = sys.modules['time'].time()a�    NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    OP         ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'x'           (2, 0) (2, 1)
    OP         '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    OP         '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    OP         '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    OP         ']'           (2, 22) (2, 23)
    OP         '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    OP         '('           (2, 28) (2, 29)
    OP         ')'           (2, 29) (2, 30)
    rurvs r4�
test_selectorzTokenizeTest.test_selector9������N�Q�		r6c�(�|jdd�y)N� @staticmethod
def foo(x,y): passa    OP         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    '\n'          (1, 13) (1, 14)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    OP         '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    OP         ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    OP         ')'           (2, 11) (2, 12)
    OP         ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    rurvs r4�test_methodzTokenizeTest.test_methodO������?�
B�
		r6c�(�|jdd�y)Nzdef f():
	if x
        	passa_    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    INDENT     '\t'          (2, 0) (2, 1)
    NAME       'if'          (2, 1) (2, 3)
    NAME       'x'           (2, 4) (2, 5)
    NEWLINE    '\n'          (2, 5) (2, 6)
    INDENT     '        \t'  (3, 0) (3, 9)
    NAME       'pass'        (3, 9) (3, 13)
    DEDENT     ''            (4, 0) (4, 0)
    DEDENT     ''            (4, 0) (4, 0)
    rurvs r4�	test_tabszTokenizeTest.test_tabs`s�����-�/�		r6c�(�|jdd�y)Nu!Örter = 'places'
grün = 'green'u7    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "'places'"    (1, 8) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "'green'"     (2, 7) (2, 14)
    rurvs r4�test_non_ascii_identifiersz'TokenizeTest.test_non_ascii_identifiersus�����@�C�		r6c�(�|jdd�y)N�#Örter = u'places'
grün = U'green'u7    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    '\n'          (1, 17) (1, 18)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    rurvs r4�test_unicodezTokenizeTest.test_unicode�������B�E�		r6c��|jdd�|jdd�|jdd�|jdd�|jd	d
�|jdd�|jd
d�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd �|jd!d"�|jd#d$�|jd%d&�|jd'd(�|jd)d*�y)+N�	async = 1z�    NAME       'async'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    �a = (async = 1)a8    NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '('           (1, 4) (1, 5)
    NAME       'async'       (1, 5) (1, 10)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    �async()z�    NAME       'async'       (1, 0) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    �class async(Bar):passa<    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    OP         ')'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    �class async:passz�    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    �	await = 1z�    NAME       'await'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    �	foo.asyncz�    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    �async for a in b: passa;    NAME       'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    �async with a as b: passa<    NAME       'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    �	async.fooz�    NAME       'async'       (1, 0) (1, 5)
    OP         '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    �asyncz/    NAME       'async'       (1, 0) (1, 5)
    �async
#comment
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    COMMENT    '#comment'    (2, 0) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NAME       'await'       (3, 0) (3, 5)
    �async
...
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    OP         '...'         (2, 0) (2, 3)
    NEWLINE    '\n'          (2, 3) (2, 4)
    NAME       'await'       (3, 0) (3, 5)
    �async
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    NAME       'await'       (2, 0) (2, 5)
    �
foo.async + 1z�    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    �async def foo(): passa;    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    �Nasync def foo():
  def foo(await):
    await = 1
  if 1:
    await
async += 1
a�    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    OP         '('           (2, 9) (2, 10)
    NAME       'await'       (2, 10) (2, 15)
    OP         ')'           (2, 15) (2, 16)
    OP         ':'           (2, 16) (2, 17)
    NEWLINE    '\n'          (2, 17) (2, 18)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'await'       (3, 4) (3, 9)
    OP         '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    '\n'          (3, 13) (3, 14)
    DEDENT     ''            (4, 2) (4, 2)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    OP         ':'           (4, 6) (4, 7)
    NEWLINE    '\n'          (4, 7) (4, 8)
    INDENT     '    '        (5, 0) (5, 4)
    NAME       'await'       (5, 4) (5, 9)
    NEWLINE    '\n'          (5, 9) (5, 10)
    DEDENT     ''            (6, 0) (6, 0)
    DEDENT     ''            (6, 0) (6, 0)
    NAME       'async'       (6, 0) (6, 5)
    OP         '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    '\n'          (6, 10) (6, 11)
    �)async def foo():
  async for i in 1: passa�    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    OP         ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (3, 0) (3, 0)
    �async def foo(async): awaitah    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    NAME       'async'       (1, 14) (1, 19)
    OP         ')'           (1, 19) (1, 20)
    OP         ':'           (1, 20) (1, 21)
    NAME       'await'       (1, 22) (1, 27)
    �@def f():

  def baz(): pass
  async def bar(): pass

  await = 2a�    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    �Fasync def f():

  def baz(): pass
  async def bar(): pass

  await = 2a    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    OP         '('           (1, 11) (1, 12)
    OP         ')'           (1, 12) (1, 13)
    OP         ':'           (1, 13) (1, 14)
    NEWLINE    '\n'          (1, 14) (1, 15)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    rurvs r4�
test_asynczTokenizeTest.test_async�s������K�*�		�	
���-�0�		�	
���I�(�		�	
���3�6�		�	
���.�1�		�	
���K�*�		�	
���K�*�		�	
���4�7�		�	
���5�8�		�	
���K�*�		�	
���G�&�		�	
���4�7�		�	
���/�2�		�	
���N�-�		�	
���O�.�		�	
���3�6�		�	
����#�*		�X	
�����		�*	
���=�	@�			�	
�����
#		�J	
�����
$		r6c�(�|jdd�y)Nz[
    # A comment here
    1
]
ad    OP         '['           (1, 0) (1, 1)
    NL         '\n'          (1, 1) (1, 2)
    COMMENT    '# A comment here' (2, 4) (2, 20)
    NL         '\n'          (2, 20) (2, 21)
    NUMBER     '1'           (3, 4) (3, 5)
    NL         '\n'          (3, 5) (3, 6)
    OP         ']'           (4, 0) (4, 1)
    NEWLINE    '\n'          (4, 1) (4, 2)
    rurvs r4�3test_newline_after_parenthesized_block_with_commentz@TokenizeTest.test_newline_after_parenthesized_block_with_comment�s������
	�		r6c�(�|jdd�y)Nz); xz�    OP         ')'           (1, 0) (1, 1)
    OP         ';'           (1, 1) (1, 2)
    NAME       'x'           (1, 3) (1, 4)
    rurvs r4�,test_closing_parenthesis_from_different_linez9TokenizeTest.test_closing_parenthesis_from_different_line�s�����F�%�		r6c�(�|jdd�y)NuTa = f'''
    Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli'''u%    NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    FSTRING_START "f'''"        (1, 4) (1, 8)
    FSTRING_MIDDLE '\n    Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli' (1, 8) (2, 68)
    FSTRING_END "'''"         (2, 68) (2, 71)
    rurvs r4� test_multiline_non_ascii_fstringz-TokenizeTest.test_multiline_non_ascii_fstring�s�����O�Q�		r6c�(�|jdd�y)Nu4f'''
    🔗 This is a test {test_arg1}🔗
🔗'''uZ    FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '\n    🔗 This is a test ' (1, 4) (2, 21)
    OP         '{'           (2, 21) (2, 22)
    NAME       'test_arg1'   (2, 22) (2, 31)
    OP         '}'           (2, 31) (2, 32)
    FSTRING_MIDDLE '🔗\n🔗'        (2, 32) (3, 1)
    FSTRING_END "'''"         (3, 1) (3, 4)
    rurvs r4�*test_multiline_non_ascii_fstring_with_exprz7TokenizeTest.test_multiline_non_ascii_fstring_with_expr�s������
�		r6N)�__name__�
__module__�__qualname__rDrSrXrjrwryr�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�rHr6r4r8r8's���9�2�&5�F7�P7	�r	�0&	�P8�0N	�`
:	�x$	�L
	�	�.	�(#	�J	�,	�"	�*
	�
	�P	�d	�"	�		�	r6r8c��eZdZd�Zy)�GenerateTokensTestc��t|�}tt|j�|�}|j	||j�j
��y�N)rr5rr<r=r>r(r?s     r4rDz!GenerateTokensTest.check_tokenize�sB��
�Q�K��-�o�a�j�j�.I�1�M���������!2�!=�!=�!?�@r6N)r�r�r�rDrHr6r4rr�s��Ar6rc	�x�g}tt|jd��j�}|D]_\}}}}}|tk(r;d|vr7|jtdftdftt|�ftdfg��M|j||f��at|�jd�j�S)Nr:�.�Decimalr��))rrr;r<r�extendrrr�reprr)r�decode�strip)rAr,�g�toknum�tokval�_s      r4�decistmtr�s���
�F������'�*�+�4�4�5�A�$%�����1�a��V���v�
��M�M��y�!��S�	���f��&��S�	�	�
�
�M�M�6�6�*�+�%&��f��$�$�W�-�3�3�5�5r6c��eZdZd�Zy)�TestMiscc���ddlm}d}|jt|�d�|j	tt
|��d�|jt
t|��|d��y)Nr)rz+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)�decimalrr=r�assertRegexr	�eval)r@rrAs   r4�
test_decistmtzTestMisc.test_decistmt�sd��	$�"������!��S�	U�	
����d�1�g��(B�C�	
����h�q�k�*� �!D�E�	Gr6N)r�r�r�rrHr6r4rr�s��Gr6rc�2�eZdZ	d�Zd�Zd�Zd�Zd�Zd�Zy)�TestTokenizerAdheresToPep0263c��tjjtjjt�d|�}t|d�5}tj||�ddd�y#1swYyxYw)N�tokenizedata�rb)�os�path�join�dirname�__file__r
�
TestRoundtrip�check_roundtrip)r@rarrCs    r4�	_testFilez'TestTokenizerAdheresToPep0263._testFilesM���w�w�|�|�B�G�G�O�O�H�5�~�x�P��
�$��
���)�)�$��2��
�
�s�A/�/A8c�*�d}|j|�y)Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt�r$�r@rCs  r4�'test_utf8_coding_cookie_and_no_utf8_bomzETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bom	���G�����q�r6c�L�	d}|jt|j|�y)Nz8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt�rN�SyntaxErrorr$r's  r4�&test_latin1_coding_cookie_and_utf8_bomzDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bom
s%��	�
G�����+�t�~�~�q�9r6c�*�d}|j|�y)Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtr&r's  r4�"test_no_coding_cookie_and_utf8_bomz@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bomr)r6c�*�d}|j|�y)Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtr&r's  r4�$test_utf8_coding_cookie_and_utf8_bomzBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_boms��D�����q�r6c��|jt|jd�|jt|jd�y)Nz
bad_coding.pyzbad_coding2.pyr+rvs r4�test_bad_coding_cookiez4TestTokenizerAdheresToPep0263.test_bad_coding_cookie s/�����+�t�~�~��G����+�t�~�~�7G�Hr6N)	r�r�r�r$r(r-r/r1r3rHr6r4rr�s&���3�
�	:���Ir6rc��eZdZd�Zy)�
Test_Tokenizec�����d}|jd��d���fd�}tt|�jdd���dd}t	ddd	d
d�g}|j||d�y)Nu"ЉЊЈЁЂ"r:Fc3�(�K��sd����yd��y�w)NTr6rH)�firstr3s��r4r<zNTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline+s���������
��	�s�T)�encoding�extra_tokensrVrZ�rMr�rM�zbytes not decoded with encoding)r;rPrrQrr=)r@�literalr<rW�expected_tokensr8r3s     @@r4�.test__tokenize_decodes_with_specified_encodingz<Test_Tokenize.test__tokenize_decodes_with_specified_encoding'sx��� ���~�~�g�&����	��7��
�8K�8K�V]�#'�)�*�*-�2�/��$�Q�����W�X�������:�	<r6N)r�r�r�r@rHr6r4r5r5%s��<r6r5c��eZdZd�Zd�Zd�Zd�Zd�Zd�Zd�Z	d�Z
d	�Zd
�Zd�Z
d�Zd
�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zy)�TestDetectEncodingc����d���fd�}|S)Nrc�D���t��k(rt���}�dz
�|S�NrM)r'�
StopIteration)r3�index�liness ��r4r<z1TestDetectEncoding.get_readline.<locals>.readline?s,�����E�
�"�#�#���<�D��Q�J�E��Kr6rH)r@rHr<rGs ` @r4�get_readlinezTestDetectEncoding.get_readline=s�����	��r6c��d}t|j|��\}}|j|d�|j|t|dd��y)N)�# something
�print(something)
�do_something(else)
r:�)rrIr=rP�r@rHr9�consumed_liness    r4�test_no_bom_no_encoding_cookiez1TestDetectEncoding.test_no_bom_no_encoding_cookieHsO��
��
$3�4�3D�3D�U�3K�#L� ��.�����7�+������e�B�Q�i��9r6c��d}t|j|��\}}|j|d�|j|ddg�y)N)s# something
rLrM�	utf-8-sigrKrL�rrIr=rOs    r4�test_bom_no_cookiez%TestDetectEncoding.test_bom_no_cookieRsO��
��
$3�4�3D�3D�U�3K�#L� ��.�����;�/�����*�,A�B�	Dr6c��d}t|j|��\}}|j|d�|j|dg�y)N)�# -*- coding: latin-1 -*-
rLrM�
iso-8859-1rWrTrOs    r4�test_cookie_first_line_no_bomz0TestDetectEncoding.test_cookie_first_line_no_bom]sI��
��
$3�4�3D�3D�U�3K�#L� ��.�����<�0�����*H�)I�Jr6c��d}t|j|��\}}|j|d�|j|dg�y)N)s# coding=utf-8
rLrMrSs# coding=utf-8
rTrOs    r4�&test_matched_bom_and_cookie_first_linez9TestDetectEncoding.test_matched_bom_and_cookie_first_linegsI��
��
$3�4�3D�3D�U�3K�#L� ��.�����;�/�����*=�)>�?r6c�`�d}|j|�}|jtt|�y)N)s## vim: set fileencoding=ascii :
rLrM�rIrNr,r�r@rHr<s   r4�<test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorzOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorqs-��
��
�$�$�U�+�����+���Ar6c��d}t|j|��\}}|j|d�ddg}|j||�y)N)�
#! something
� # vim: set fileencoding=ascii :
rLrM�asciirarbrT�r@rHr9rPrBs     r4�test_cookie_second_line_no_bomz1TestDetectEncoding.test_cookie_second_line_no_bomzsP��
��$3�4�3D�3D�U�3K�#L� ��.�����7�+�%�'K�L�������2r6c��d}t|j|��\}}|j|d�|j|ddg�y)N)�#! something
�f# coding=utf-8
rLrMrSrarhrTrOs    r4�'test_matched_bom_and_cookie_second_linez:TestDetectEncoding.test_matched_bom_and_cookie_second_line�sO��
��$3�4�3D�3D�U�3K�#L� ��.�����;�/�����+�-A�B�	Dr6c�`�d}|j|�}|jtt|�y)N)rgrbrLrMr]r^s   r4�=test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorzPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerror�s-��
���$�$�U�+�����+���Ar6c��d}t|j|��\}}|j|d�dg}|j||�y)N)�print('£')
�%# vim: set fileencoding=iso8859-15 :
�
print('€')
r:rmrTrds     r4�/test_cookie_second_line_noncommented_first_linezBTestDetectEncoding.test_cookie_second_line_noncommented_first_line�sM��
��
$3�4�3D�3D�U�3K�#L� ��.�����7�+�*�+�������2r6c��d}t|j|��\}}|j|d�ddg}|j||�y)N)�
#print('£')
rnro�
iso8859-15rrrnrTrds     r4�,test_cookie_second_line_commented_first_linez?TestDetectEncoding.test_cookie_second_line_commented_first_line�sP��
��
$3�4�3D�3D�U�3K�#L� ��.�����<�0�+�-V�W�������2r6c��d}t|j|��\}}|j|d�ddg}|j||�y)N)�
rnrorsrvrnrTrds     r4�(test_cookie_second_line_empty_first_linez;TestDetectEncoding.test_cookie_second_line_empty_first_line�sP��
��
$3�4�3D�3D�U�3K�#L� ��.�����<�0��D�E�������2r6c���d}|D]g}dD]`}|jd|�}dd|jd�zdzdd	f}|j|�}t|�\}}|j	|d
��b�iy)N)�latin-1rXziso-latin-1zlatin-1-unixziso-8859-1-unixziso-latin-1-mac��-rr{�#!/usr/bin/python
�
# coding: rcrvsprint(things)
sdo_something += 4
rX��replacer;rIrr=�	r@�	encodingsr9�rep�encrH�rl�foundrPs	         r4�test_latin1_normalizationz,TestDetectEncoding.test_latin1_normalization�s���;�	�!�H�!���&�&�s�C�0��/�&����G�)<�<�u�D�+�/�1���&�&�u�-��(7��(;�%��~�� � ���5�"�"r6c�`�d}|j|�}|jtt|�y)N�s
print("�")r]r^s   r4�test_syntaxerror_latin1z*TestDetectEncoding.test_syntaxerror_latin1�s-�����$�$�U�+�����+���Ar6c���d}|D]f}dD]_}|jd|�}dd|jd�zdzdf}|j|�}t|�\}}|j	|d	��a�hy)
N)r:z	utf-8-macz
utf-8-unixrzr{r|r}rcrvs1 + 3
r:r~r�s	         r4�test_utf8_normalizationz*TestDetectEncoding.test_utf8_normalization�s���8�	�!�H�!���&�&�s�C�0��/�&����G�)<�<�u�D�#�%���&�&�u�-��(7��(;�%��~�� � ���0�"�"r6c�t�|jd�}t|�\}}|j|d�|j|dg�t|jd��\}}|j|d�|j|g�|jd�}t|�\}}|j|d�|j|dg�|jd�}t|�\}}|j|d�|j|g�|jd�}|jtt|�y)	N)rLr:rLrH)sprint(something)
rS)�)s# coding: bad
)rIrr=rNr,�r@r<r9rPs    r4�test_short_filesz#TestDetectEncoding.test_short_files�s'���$�$�%=�>��#2�8�#<� ��.�����7�+�����*?�)@�A�#2�4�3D�3D�R�3H�#I� ��.�����7�+������,��$�$�%I�J��#2�8�#<� ��.�����;�/�����*?�)@�A��$�$�%7�8��#2�8�#<� ��.�����;�/������,��$�$�%:�;�����+���Ar6c��|jd�}t|�\}}|j|d�|j|dg�y)N)�print("#coding=fake")r:r�)rIrr=r�s    r4�test_false_encodingz&TestDetectEncoding.test_false_encoding�sF���$�$�%@�A��#2�8�#<� ��.�����7�+�����*B�)C�Dr6c��tjdz}|jtj|�dD]�}t	|d|��5}td|z|��td|��ddd�t
|�5}|j|j|�|j|jd�ddd���t	|dd	��5}td|��ddd�t
|�5}|j|jd	�|j|jd�ddd�y#1swY��xYw#1swY��xYw#1swY�oxYw#1swYyxYw)
Nz.py)ziso-8859-15r:�w�r9z# coding: %s��fileuprint('euro:€')�rrS)
r�TESTFN�
addCleanup�unlinkr
�print�
tokenize_openr=r9�mode)r@rar9�fps    r4�	test_openzTestDetectEncoding.test_opens���#�#�e�+�����	�(�(�(�3�1�H��h��h�7�2��n�x�/�b�9��,�2�6�8��x�(�B�� � ����h�7�� � ����#�.�)�(�	1��(�C�+�
6�"��(�r�2�7�
�8�
$�����R�[�[�+�6����R�W�W�c�*�%�
$�8�7��)�(��
7�
6��
$�
$�s0�D,�89D8�	E�*9E�,D5	�8E	�E�Ec�j��d}d�G�fd�d�}|jt�5|�|�}|`t|j�ddd�|jtdj
|��5|�|�}t|j�ddd�y#1swY�VxYw#1swYyxYw)N�some_file_pathr�c���eZdZd�Z�fd�Zy)�;TestDetectEncoding.test_filename_in_exception.<locals>.Bunkc�.�||_||_d|_y)Nr)�name�_lines�_index)r@rHrs   r4�__init__zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__s�� ��	�#�����r6c���|jt��k(rt��|j}|xjdz
c_|SrE)r�r'rF)r@r3rHs  �r4r<zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readline!s:����;�;�#�e�*�,�'�'��T�[�[�)�����q� ���r6N)r�r�r�r�r<)rHs�r4�Bunkr�s
���
 �

r6r�z.*{})rNr,r�rr<r]�format)r@rr��insrHs    @r4�test_filename_in_exceptionz-TestDetectEncoding.test_filename_in_exceptions��������	�	��
�
�{�
+��u�d�#�C����C�L�L�)�	,�
�
#�
#�K����t�1D�
E��u�d�#�C��C�L�L�)�F�
E�,�
+��
F�
E�s�!B�5B)�B&�)B2c���td�}tjd|��5|jtt
d�ddd�|j
|j�y#1swY�%xYw)Ns#coding:xxxztokenize._builtin_open)�return_value�foobar)rr�patchrNr,r��
assertTrue�closed)r@�ms  r4�test_open_errorz"TestDetectEncoding.test_open_error1sM���N�#��
�Z�Z�0�q�
A����k�=�(�C�B�������!�B�
A�s�A#�#A,N)r�r�r�rIrQrUrYr[r_rerirkrprtrwr�r�r�r�r�r�r�r�rHr6r4rBrB;so��	�:�	D�K�@�B�
3�
D�B�	3�	3�	3�
6�B�1�B�.E�+�(*�8"r6rBc�<�eZdZd�Zd�Zd�Zd�Zd�Zd�Zd�Z	d�Z
y	)
�TestTokenizec�F��	�
�ddl}d�	d�
�	fd�}�
fd�}d��fd�}|j}|j}||_||_	t|�}|jt	|�ddgd��||_||_|j�
�	�y#||_||_wxYw)Nrr:c����ddgfS)N�first�secondrH)r<r9s �r4�mock_detect_encodingz8TestTokenize.test_tokenize.<locals>.mock_detect_encoding?s����h�	�2�2�2r6c�l��|�g}		|�}|r|j|��|S#t$r|cYSwxYwr)rFr))r<r9�kwargs�out�	next_line�
encoding_useds     �r4�mock__tokenizez2TestTokenize.test_tokenize.<locals>.mock__tokenizeBsK���$�M��C��� (�
�I���J�J�y�)���
��%���J��s�%�3�3c�L���dz
��dk(ryt��j�S)NrM�r6)�strr;)�counters�r4�
mock_readlinez1TestTokenize.test_tokenize.<locals>.mock_readlineQs)����q�L�G��!�|���w�<�&�&�(�(r6rM)r�r��1�2�3�4)rrrr=rP)r@�tokenize_moduler�r�r��orig_detect_encoding�orig_c_token�resultsr�r9r�s        @@@r4�
test_tokenizezTestTokenize.test_tokenize;s����*����
�	3�	���	)� /�>�>��&�H�H��*>��'�<J��9�	M��}�-�G����T�'�]�1�2�.�J�
L�/C�O�+�@L�O�=������1��/C�O�+�@L�O�=�s�+B�B c	�X�g}td�D]#}|jdj|����%|jd�dj|�}t	tt
|jd��j��}|j|djd�y)Ni�zdef i{i}(): return {i})�i�OK�
r:���)�ranger)r�rrPrrr;r<r=�string)r@�bufr��tokss    r4�test_oneline_defszTestTokenize.test_oneline_defsfs������s��A��J�J�/�6�6��6�;�<���
�
�4���i�i��n���H�W�S�Z�Z��%8�9�B�B�C�D������b����$�/r6c	��ttt|jd��j��}t|�}|j
t|�d|z�|j
t|djtt�t|�D]5}|j
t||dzjt||��7|j
t|d|zjttj�|j
t|d|zjttj�y)Nr:rZrrMrN)rPrrr;r<r'r=r�
exact_typer
r�r0rr	)r@�opstr�optypesrW�num_optypesr�s      r4�assertExactTypeEqualz!TestTokenize.assertExactTypeEqualrs
���h�w�u�|�|�G�'<�=�F�F�G�H���'�l������V��a�+�o�6�����&��)�"6�"6�7�!�(�+�	-��{�#�A����X�f�Q��U�m�&>�&>�?�%�g�a�j�1�
3�$�	
����&��[��"9�"D�"D�E�!�%�-�-�0�	2�����&��[��"9�"D�"D�E�!�%�/�/�2�	4r6c�\�|jdtjtj�|jdtjtj
�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jdtj�|jd	tj�|jd
tj�|jdtj�|jdtj�|jd
tj �|jdtj"�|jdtj$�|jdtj&�|jdtj(tj*�|jdtj,�|jdtj.�|jdtj0�|jdtj2�|jdtj4�|jdtj6�|jdtj8�|jdtj:�|jdtj<�|jdtj>�|jdtj@�|jdtjB�|jdtjD�|jdtjF�|jd tjH�|jd!tjJ�|jd"tjL�|jd"tjL�|jd#tjN�|jd$tjP�|jd%tjR�|jd&tjT�|jd'tjV�|jd(tjX�|jd)tjZ�|jd*tj\�|jd+tj^�|jd,tj`�|jd-tbtj<tdtjtbtj<tdtj,tbtj<td�|jd.tj(tjdtjtjdtjtjdtj*�|jd/tj6tjtjbtjtjdtj�y)0Nz()z[]�:�,�;�+r{�*�/�|�&�<�>�=r�%z{}z==z!=z<=z>=�~�^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->�@z@=za**2+b**2==c**2z	{1, 2, 3}z
^(x & 0x1))3r�r0�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENT�LBRACE�RBRACE�EQEQUAL�NOTEQUAL�	LESSEQUAL�GREATEREQUAL�TILDE�
CIRCUMFLEX�	LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR�	PLUSEQUAL�MINEQUAL�	STAREQUAL�
SLASHEQUAL�PERCENTEQUAL�
AMPEREQUAL�	VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL�DOUBLESLASH�DOUBLESLASHEQUAL�
COLONEQUAL�ELLIPSIS�RARROW�AT�ATEQUALrrrvs r4�test_exact_typezTestTokenize.test_exact_type�sz���!�!�$��
�
�E�J�J�?��!�!�$��
�
�E�J�J�?��!�!�#�u�{�{�3��!�!�#�u�{�{�3��!�!�#�u�z�z�2��!�!�#�u�z�z�2��!�!�#�u�{�{�3��!�!�#�u�z�z�2��!�!�#�u�{�{�3��!�!�#�u�z�z�2��!�!�#�u�{�{�3��!�!�#�u�z�z�2��!�!�#�u�}�}�5��!�!�#�u�{�{�3��!�!�#�u�y�y�1��!�!�#�u�}�}�5��!�!�$����e�l�l�C��!�!�$��
�
�6��!�!�$����7��!�!�$����8��!�!�$��(:�(:�;��!�!�#�u�{�{�3��!�!�#�u�'7�'7�8��!�!�$����8��!�!�$��(8�(8�9��!�!�$��(8�(8�9��!�!�$����8��!�!�$����7��!�!�$����8��!�!�$��(8�(8�9��!�!�$��(:�(:�;��!�!�$��(8�(8�9��!�!�$����8��!�!�$��(=�(=�>��!�!�$��(=�(=�>��!�!�%��)=�)=�>��!�!�%��)>�)>�?��!�!�%��)>�)>�?��!�!�$��(9�(9�:��!�!�%��)?�)?�@��!�!�$��(8�(8�9��!�!�%����8��!�!�$����5��!�!�#�u�x�x�0��!�!�$��
�
�6��!�!�"3�"&��(8�(8�&�"'�*�*�"&��(8�(8�&�"'�-�-�"&��(8�(8�&�	B�	
�!�!�+�"'�,�,�"'�,�,����"'�,�,����"'�,�,�"'�,�,�	0�	
�!�!�,�"'�"2�"2�"'�*�*�"'�*�*�e�k�k�5�<�<�"'�*�*�		.r6c�D�|jdtj�y)Nz@          )r�r0rrvs r4�%test_pathological_trailing_whitespacez2TestTokenize.test_pathological_trailing_whitespace�s���!�!�-����:r6c��d}ttjdddd��ttjdddd	��ttjd
ddd	��ttj
d
ddd	��ttjdddd	��ttjdddd��ttjdddd��ttjdddd��ttjdddd��g	}ttt|jd��j��}|j||�y)Nzb = 1

#testr:�rr��r/r�r1r2r3�br;�rMrMzb = 1
r��rMrN�rMrZ�1�rM��rMr�r��rM��rNr�rNrMz#test�rZr)rZr�)rZr1)r.r)rr0r
rrrr�NL�COMMENTr	rPrrr;r<r=�r@�sourcer?rWs    r4�5test_comment_at_the_end_of_the_source_without_newlinezBTestTokenize.test_comment_at_the_end_of_the_source_without_newline�s���!���5�>�>�'��V�Z\�]��5�:�:�c��V�R[�\��5�8�8�C�v�6�PY�Z��5�<�<��6�v�T]�^��5�=�=��V��V_�`��5�8�8�D��F�QU�V��5�=�=���F�Y`�a��5�8�8�B�f�&�w�W��5�?�?�2�V��VX�Y�

���h�w�v�}�}�W�'=�>�G�G�H�I�������1r6c��d}ttjdddd��ttjdddd	��ttjd
ddd	��ttj
ddd
d��ttjdddd��g}ttt|jd��j��}|j||�y)Nza
 r:r%r&)r�r1r2r3�ar;r)za
r�r*r3)rNrNr%r4)
rr0r
rrr5r	rPrrr;r<r=r7s    r4�?test_newline_and_space_at_the_end_of_the_source_without_newlinezLTestTokenize.test_newline_and_space_at_the_end_of_the_source_without_newline�s������e�n�n�W�F��UW�X��e�j�j��F��U�S��e�m�m�D��F�QV�W��e�h�h�r��V�#�N��e�o�o�b��F�QS�T�
���h�w�v�}�}�W�'=�>�G�G�H�I�������1r6c�R�d}tj�5}tjj	|d�}t|d�5}|j
|�ddd�t|�\}}|jd|j�ddd�y#1swY�<xYw#1swYyxYw)NsF"""
        �"""z	script.py�wbsSyntaxError)
r�temp_dirrrrr
�writer �assertIn�err)r@�scriptr?rar��rsrs       r4�(test_invalid_character_in_fstring_middlez5TestTokenize.test_invalid_character_in_fstring_middle�s������
�
�
!�X��w�w�|�|�H�k�:�H��h��%���
�
�6�"�&�(��2�E�B���M�M�.�"�&�&�1�"�
!�%�%��"�
!�s#�-B�B�2B�B	�B�B&N)r�r�r�r�r�r�r!r#r9r<rErHr6r4r�r�9s,��)2�V	0�4�?.�B;�2�$2�
2r6r�c��eZdZd�Zd�Zd�Zy)�UntokenizeTestc�>�t�}d|_d|_|jt�5}|jd�ddd�|j
jjdd�|jt|j
d�y#1swY�TxYw)NrNr+rz'start (1,3) precedes previous end (2,2)r3)	r�prev_row�prev_colrN�
ValueError�add_whitespacer=r_�args)r@�u�cms   r4�test_bad_input_orderz#UntokenizeTest.test_bad_input_order�s���M����
���
�
�
�
�z�
*�b�
���U�#�+�������*�*�1�-�9�	;�	
���*�a�&6�&6��>�+�
*�s�B�Bc�6�t�}d|_d|_g|_|j	d�|j|jdg�d|_|j	d�|j|jgd��tj|d�y)NrMr2�\
rN)r.r.)rRz\
\
r#za
  b
    c
  \
  c
)rrIrJrWrLr=r"r#)r@rNs  r4�test_backslash_continuationz*UntokenizeTest.test_backslash_continuations}���M����
���
����	���� �������F�8�,���
�	���� �������#?�@��%�%�d�,H�Ir6c��t�}tdf}tdf|g}|j|t	g��|j|jdg�t�}|j|jt	|g��d�t�}|j|jt	|��d�|j|jd�|jtt	|��d�y)N�Hellor:zHello sHello )	rrr
�compat�iterr=rWrr9)r@rNr0rWs    r4�test_iter_compatzUntokenizeTest.test_iter_compat
s����M���w����W�%�u�-��	�����R��!�������H�:�.��M��������d�E�7�m�4�h�?��M��������d�6�l�3�X�>�������W�-�����D��L�1�9�=r6N)r�r�r�rPrSrXrHr6r4rGrG�s��
?�J�>r6rGc�T�	tjd�}|j|�duS)Ns\n\s*\\\r?\n)�re�compile�search)r8�patterns  r4�contains_ambiguous_backslashr^s,��
��j�j�)�*�G��>�>�&�!��-�-r6c�B�eZdZd�Zd�Zd�Zd�Zd�Zd�Zd�Z	d�Z
d	�Zy
)r"c�b�	t|t�r|jd�}n|j�}t	|jd���j}tt|��}|D�cgc]}|dd��	}}t|�}t	|jd���j}t|�D�cgc]}|dd��	}	}|j|	|�t|�}
t	|
jd���j}t|�D�cgc]}|dd��	}}|j||�t|�s]|jd�}
t	|
jd���j}tt|��}|j|
|�yycc}wcc}wcc}w)Nr:T��keependsrNr�)
�
isinstancer�r;�readrWr(rQrPrrr=r^�removeprefix)r@rC�coder<�tokens5ri�tokens2�bytes_from2�	readline2�
tokens2_from2�bytes_from5�	readline5�
tokens2_from5�code_without_bom�untokenized_codes               r4r#zTestRoundtrip.check_roundtrip-s���
	��a����8�8�G�$�D��6�6�8�D�������6�7�@�@���x��)�*��&-�.�g�s�3�r��7�g��.� ��)����/�/��/�>�?�H�H�	�,4�Y�,?�@�,?�S��R�a��,?�
�@������0� ��)����/�/��/�>�?�H�H�	�,4�Y�,?�@�,?�S��R�a��,?�
�@������0�+�D�1�#�0�0��A���,�7�7��7�F�G�P�P�H�)�(�8�*<�=�����-�/?�@�2��/��A��
As�1F"�<F'�F,c��t|t�r|jd�}n|j�}t	|jd���j}t|�D]^}|jtthvr�|j|j|j|jd|jd��`y)Nr:TrarM)rcr�r;rdrWr(rQrr/r
r	r=r�r3r1r2)r@rCrfr<ris     r4�check_line_extractionz#TestRoundtrip.check_line_extractionVs����a����8�8�G�$�D��6�6�8�D�������6�7�@�@���H�%�C��x�x�X�y�1�1�����S�Z�Z����#�)�)�A�,�����
�)K�L�&r6c���|jd�|jd�|jd�tjdd��}t|d�5}|j|�ddd�|jd�|jd	�|jd
�|jd�|jd�|jd
�|jd�|jd�|jd�|jd�|jd�|jd�|jd�|jd�|jd�|jd�|jd�|jd�|jd�|jd�|jd�|jd�|jd�|jd�|jd �|jd!�|jd"�|jd#�|jd$�|jd%�|jd&�|jd'�|jd(�|jd)�|jd*�gd+�}|D]}|j|��y#1swY��zxYw),Nzif x == 1:
    print(x)
z # This is a comment
# This also
zif x == 1 : 
  print(x)
ztokenize_tests.txtr)�subdirrzoif x == 1:
    # A comment by itself.
    print(x) # Comment here, too.
    # Another comment.
after_if = True
zPif (x # The comments need to go in the right place
    == 1):
    print('x==1')
z�class Test: # A comment here
  # A comment with weird indent
  after_com = 5
  def x(m): return m*5 # a one liner
  def y(m): # A whitespace after the colon
     return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
    print('Can not import' # comment2
)else:   print('Loaded')
zf'\N{EXCLAMATION MARK}'z
f'\\N{SNAKE}'zf'\\N{{SNAKE}}'zf'\N{SNAKE}'zf'\\\N{SNAKE}'zf'\\\\\N{SNAKE}'zf'\\\\\\\N{SNAKE}'z	f'\\N{1}'zf'\\\\N{2}'z
f'\\\\\\N{3}'zf'\\\\\\\\N{4}'zf'\\N{{'z
f'\\\\N{{'zf'\\\\\\N{{'zf'\\\\\\\\N{{'zf'\n{{foo}}'z
f'\\n{{foo}}'zf'\\\n{{foo}}'zf'\\\\n{{foo}}'zf'\t{{foo}}'z
f'\\t{{foo}}'zf'\\\t{{foo}}'zf'\\\\t{{foo}}'z
rf'\t{{foo}}'zrf'\\t{{foo}}'zrf'\\\t{{foo}}'zrf'\\\\t{{foo}}'zrf'\{{foo}}'zf'\\{{foo}}'zrf'\\\{{foo}}'zf'\\\\{{foo}}')z
if 1:
    "foo"
"bar"
z
if 1:
    ("foo"
     "bar")
z
if 1:
    "foo"
    "bar"
)r#r�findfiler
)r@�fnrC�cases�cases     r4�test_roundtripzTestRoundtrip.test_roundtripas���	
���.�	/����-�	.�	
���,�	-�
�
�
�2�>�
J��
�"�d�^�q�� � ��#�����1�	2�
	
���3�	4�	
���B�	C�	
���9�	:�
	
���7�8����-�.����/�0����_�-����.�/����0�1����2�3����\�*����^�,����-�.����/�0����[�)����]�+����_�-����.�/����_�-����-�.����.�/����/�0����_�-����-�.����.�/����/�0����-�.����.�/����/�0����0�1����_�-����_�-����.�/����.�/��� �D�� � ��&��Y�^�s�K!�!K+c�&�|jd�y)Nzna = (3,4, 
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
�r#rvs r4�test_continuationzTestRoundtrip.test_continuation�s�����	&�		'r6c�H�|jd�|jd�y)Nz)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0r{rvs r4rSz)TestRoundtrip.test_backslash_continuation�s*�����-�	.�	
���%�	&r6c�&�|jd�y)Nz'' ''r{rvs r4�test_string_concatenationz'TestRoundtrip.test_string_concatenation�s�����W�%r6c�h�ddl}ddl}tjj	t
�xstj}|jtjj|j|�d��}tjd�s|j|d�}|D]s}tjdk\rtd|�t|d�5}|j|��5|j!|�|j#|�ddd�ddd��uy#1swY�xYw#1swY��xYw)	Nrztest*.py�cpu�
rNrrr�)�glob�randomrrr r!�curdirr�escaper�is_resource_enabled�sample�verboser�r
�subTestr#rr)r@r�r��tempdir�	testfiles�testfilerCs       r4�test_random_fileszTestRoundtrip.test_random_files�s���	��'�'�/�/�(�+�8�r�y�y���I�I�b�g�g�l�l�4�;�;�w�+?��L�M�	��*�*�5�1��
�
�i��4�I�!�H����!�#��j�(�+��h��%���\�\�x�\�0��(�(��+��.�.�q�1�1�&�%�"�1�0��&�%�s$�D(�&#D�	D(�D%�!D(�(D1	c��t|t�r|jd�}tt	t|�j��jd�S)Nr:)rcr�r;rrrr<r
)r@rfs  r4�	roundtripzTestRoundtrip.roundtrip�s?���d�C� ��;�;�w�'�D��(�7�4�=�#9�#9�:�;�B�B�7�K�Kr6c��	d}|j|�jd�}|j|d|d�|j|�y)Nzif False:
	x=3
	x=3
r�rMrN)r��splitr=r#)r@rf�	codeliness   r4�#test_indentation_semantics_retainedz1TestRoundtrip.test_indentation_semantics_retained�sM��	�+���N�N�4�(�.�.�t�4�	�����1��y��|�4����T�"r6N)r�r�r�r#rrryr|rSrr�r�r�rHr6r4r"r"+s5��'A�R	M�]'�@'�&�&�2�(L�
#r6r"c��eZdZd�Zd�Zy)�InvalidPythonTestsc
��d}ttjdddd��ttjdddd��ttjddd	d��ttjd
d	dd��ttjddd
d��ttj
dd
dd��ttjdddd��g}ttt|�j��}|j||�y)Nz2sin(x)�2r;r)r'�sinr-r�r/rUr0rr<r&)rM�r2)rr0rrrrr	rPrrr<r=r7s    r4�test_number_followed_by_namez/InvalidPythonTests.test_number_followed_by_names������5�<�<��6�v�T]�^��5�:�:�e�6�v�T]�^��5�8�8�C�v�6�PY�Z��5�:�:�c��V�R[�\��5�8�8�C�v�6�PY�Z��5�=�=��6�v�T]�^��5�?�?�2�V��VX�Y�
���o�h�v�&6�&?�&?�@�A�������1r6c	�,�d}ttjdddd��ttjdddd��ttjdddd��g}tt
t|�j��}|j||�y)N�01234r;r/r'r&r0r2)
rr0rrr	rPrrr<r=r7s    r4�test_number_starting_with_zeroz1InvalidPythonTests.test_number_starting_with_zerosy�����5�<�<��v�6�X_�`��5�=�=��6�v�T[�\��5�?�?�2�V��VX�Y�
���o�h�v�&6�&?�&?�@�A�������1r6N)r�r�r�r�r�rHr6r4r�r��s��2� 	2r6r�c�x�eZdZd�Zd�Zd�Zd�Zd�Zd�Zd�Z	d�Z
d	�Zd
�Zd�Z
d�Zd
�Zd�Zd�Zd�Zd�Zd�Zy)�
CTokenizeTestc��t|�}|j|��5tt|j�|�}|j||j
�j��ddd�y#1swYyxYw)N)r8)rr�r5rr<r=r>r(r?s     r4rDzCTokenizeTest.check_tokenizesa��
�Q�K��
�\�\��\�
#�1�1�!�*�*�=�q��F�
���V�X�_�_�%6�%A�%A�%C�D�	$�
#�
#�s�AA5�5A>c��d�}ttdddd��ttdddd��ttddd	d��ttd
d	dd��ttd
ddd
��g}d
D]V}|j|��5t
t||�jd|���}|j||�ddd��Xy#1swY�cxYw)Nc3�2K�dj|���y�w)N�1+1)r;r�s r4r<z-CTokenizeTest.test_encoding.<locals>.readline's�����,�,�x�(�(�s�r,r;r)r�r'r�r*r+r&r-r2)r:ryzutf-16r�T)r:r9)
rrrrr	r�rPrrQr=)r@r<rBr9rWs     r4�
test_encodingzCTokenizeTest.test_encoding&s���	)�
�6�#�V��e�T��2�c��V�%�P��6�#�V��e�T��7�2�V��e�T��9�R�v�6�PR�S�
��7�H����x��0��?��X�&�/�/�!%�%����
� � ���2�
1�0�7�0�0�s�<:C�C
	c�H�|jdd�|jdd�|jdd�|jdd�|jd	d
�|jdd�|jd
d�|jdd�|jdd�y)Nrlz�    NUMBER     '0xff'        (1, 0) (1, 4)
    LESSEQUAL  '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    rmz�    NUMBER     '0b10'        (1, 0) (1, 4)
    LESSEQUAL  '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    rnz�    NUMBER     '0o123'       (1, 0) (1, 5)
    LESSEQUAL  '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    roz�    NUMBER     '1234567'     (1, 0) (1, 7)
    GREATER    '>'           (1, 8) (1, 9)
    TILDE      '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    rpz�    NUMBER     '2134568'     (1, 0) (1, 7)
    NOTEQUAL   '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    rqac    LPAR       '('           (1, 0) (1, 1)
    MINUS      '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    MINUS      '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    RPAR       ')'           (1, 10) (1, 11)
    AMPER      '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    rrz�    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    NOTEQUAL   '!='          (1, 11) (1, 13)
    MINUS      '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    rsz�    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    AMPER      '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    rtz�    NUMBER     '0xFF'        (1, 0) (1, 4)
    AMPER      '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    VBAR       '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    rurvs r4rwzCTokenizeTest.test_int:s������M�,�		�	
���M�,�		�	
���,�/�		�	
���-�0�		�	
���0�3�		�	
���5�	8�			�	
���.�1�		�	
���0�3�		�	
���0�3�		r6c��|jdd�|jdd�|jdd�|jdd�|jd	d
�|jdd�|jd
d�y)Nr{z�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    r|z�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    r}z�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    r~z�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    rz�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    r�z�    NAME       'x'           (1, 0) (1, 1)
    PLUS       '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    EQUAL      '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    r�z�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    rurvs r4r�zCTokenizeTest.test_float{s������M�,�		�	
���M�,�		�	
���M�,�		�	
���M�,�		�	
���K�*�		�	
���O�.�		�	
���N�-�		r6c��|jdd�|jdd�|jdd�|jdd�|jd	d
�|jdd�|jd
d�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd �|jd!d"�|jd#d$�|jd%d&�|jd'd(�|jd)d*�|jd+d,�|jd-d.�|jd/d0�|jd1d2�|jd3d4�|jd5d6�|jd7d8�|jd9d:�|jd;d<�|jd=d>�|jd?d@�y)ANr�a5    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    SEMI       ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    EQUAL      '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    r�a6    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    SEMI       ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    EQUAL      '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    r�z�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    r�z�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    r�z�    NAME       'y'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    r�a�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    PLUS       '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    PLUS       '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    PLUS       '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    r�a�    NAME       'y'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    PLUS       '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    PLUS       '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    PLUS       '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    r�z�    STRING     "u'abc'"      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    r�z�    STRING     'u"abc"'      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    r�z�    STRING     "b'abc'"      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    r�z�    STRING     'b"abc"'      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    r�a;    STRING     "br'abc'"     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    r�a;    STRING     'br"abc"'     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    r�a;    STRING     "rb'abc'"     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    r�a;    STRING     'rb"abc"'     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    r�r�r�r�r�r�r�r�r�r�r�r�z"hola\\\r\ndfgf"z9    STRING     '"hola\\\\\\r\\ndfgf"' (1, 0) (1, 16)
    r�r�r�a=    FSTRING_START 'fR"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'a'           (1, 3) (1, 4)
    LBRACE     '{'           (1, 4) (1, 5)
    NAME       'b'           (1, 5) (1, 6)
    RBRACE     '}'           (1, 6) (1, 7)
    FSTRING_MIDDLE 'c'           (1, 7) (1, 8)
    FSTRING_END '"'           (1, 8) (1, 9)
    r�r�r�r�zf"{
a}"z�    FSTRING_START 'f"'          (1, 0) (1, 2)
    LBRACE     '{'           (1, 2) (1, 3)
    NAME       'a'           (2, 0) (2, 1)
    RBRACE     '}'           (2, 1) (2, 2)
    FSTRING_END '"'           (2, 2) (2, 3)
    r�r�zf"hola\\\r\ndfgf"z�    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE 'hola\\\\\\r\\ndfgf' (1, 2) (1, 16)
    FSTRING_END '"'           (1, 16) (1, 17)
    r�a�    FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    LBRACE     '{'           (1, 6) (1, 7)
    NAME       'x'           (2, 4) (2, 5)
    COLON      ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n'         (2, 6) (3, 0)
    RBRACE     '}'           (3, 0) (3, 1)
    FSTRING_MIDDLE '__'          (3, 1) (3, 3)
    FSTRING_END "'''"         (3, 3) (3, 6)
    r�a�    FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    LBRACE     '{'           (1, 6) (1, 7)
    NAME       'x'           (2, 4) (2, 5)
    COLON      ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n    b\n     c\n      d\n' (2, 6) (6, 0)
    RBRACE     '}'           (6, 0) (6, 1)
    FSTRING_MIDDLE '__'          (6, 1) (6, 3)
    FSTRING_END "'''"         (6, 3) (6, 6)
    r�a�    FSTRING_START "f'"          (1, 0) (1, 2)
    FSTRING_MIDDLE '__'          (1, 2) (1, 4)
    LBRACE     '{'           (1, 4) (1, 5)
    NAME       'x'           (2, 4) (2, 5)
    COLON      ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'd'           (2, 6) (2, 7)
    RBRACE     '}'           (3, 0) (3, 1)
    FSTRING_MIDDLE '__'          (3, 1) (3, 3)
    FSTRING_END "'"           (3, 3) (3, 4)
    rurvs r4r�zCTokenizeTest.test_string�s�����.�1�		�	
���1�4�		�	
���>�A�		�	
���/�2�		�	
���/�2�		�	
���C�
F�
		�	
���C�
F�
		�	
���-�0�		�	
���-�0�		�	
���-�0�		�	
���-�0�		�	
���C�F�		�	
���C�F�		�	
���C�F�		�	
���C�F�		�	
���-�0�		�	
���L�+�		�	
���L�+�		�	
�����		�	
�����		�	
�����		�	
���/�2�		�	
���H�'�		�	
���K�*�		�	
���L�+�		�	
�����		�	
����	�		�	
�����		�	
���0�3�		�	
���
�
�
		�	
���
�
�
		�$	
����

�
		r6c�p�|jdd�|jdd�|jdd�y)Nr�a    NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    LPAR       '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    COMMA      ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    COMMA      ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    EQUAL      '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    COMMA      ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    EQUAL      '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    COMMA      ','           (1, 22) (1, 23)
    STAR       '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    RPAR       ')'           (1, 26) (1, 27)
    COLON      ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    r�a�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    LPAR       '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    EQUAL      '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    COMMA      ','           (1, 13) (1, 14)
    STAR       '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    COMMA      ','           (1, 17) (1, 18)
    DOUBLESTAR '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    RPAR       ')'           (1, 22) (1, 23)
    COLON      ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    r�a�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    LPAR       '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    COLON      ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    COMMA      ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    COLON      ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    EQUAL      '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    RPAR       ')'           (1, 24) (1, 25)
    RARROW     '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    COLON      ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    rurvs r4r�zCTokenizeTest.test_function�	sJ�����?�B�		�.	
���;�>�		�$	
���D�G�		r6c�(�|jdd�y)Nr�a�    NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    LESS       '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    GREATER    '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    EQEQUAL    '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    GREATEREQUAL '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    LESSEQUAL  '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    LESSEQUAL  '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    NOTEQUAL   '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    COLON      ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    rurvs r4r�zCTokenizeTest.test_comparison�	r�r6c�(�|jdd�y)Nr�a$    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    MINUS      '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    MINUS      '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    PLUS       '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    PLUS       '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    PLUS       '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    LSQB       '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    RSQB       ']'           (1, 36) (1, 37)
    rurvs r4r�zCTokenizeTest.test_additive
r�r6c�(�|jdd�y)Nr�a�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    DOUBLESLASH '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    STAR       '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    SLASH      '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    STAR       '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    PERCENT    '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    AT         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    rurvs r4r�z!CTokenizeTest.test_multiplicative-
r�r6c�L�|jdd�|jdd�y)Nr�a�    TILDE      '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    CIRCUMFLEX '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    AMPER      '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    VBAR       '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    CIRCUMFLEX '^'           (1, 14) (1, 15)
    MINUS      '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    r�aI    MINUS      '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    STAR       '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    SLASH      '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    PLUS       '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    STAR       '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    DOUBLESLASH '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    MINUS      '-'           (1, 14) (1, 15)
    MINUS      '-'           (1, 16) (1, 17)
    MINUS      '-'           (1, 17) (1, 18)
    MINUS      '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    DOUBLESTAR '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    rurvs r4r�zCTokenizeTest.test_unaryA
s0�����0�3�		�	
���5�8�		r6c�(�|jdd�y)Nr�a�    NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    COMMA      ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    NAME       'x'           (2, 0) (2, 1)
    EQUAL      '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    DOT        '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    LSQB       '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    RSQB       ']'           (2, 22) (2, 23)
    DOT        '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    LPAR       '('           (2, 28) (2, 29)
    RPAR       ')'           (2, 29) (2, 30)
    rurvs r4r�zCTokenizeTest.test_selectorg
r�r6c�(�|jdd�y�Nr�a    AT         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    ''            (1, 13) (1, 13)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    LPAR       '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    COMMA      ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    RPAR       ')'           (2, 11) (2, 12)
    COLON      ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    rurvs r4r�zCTokenizeTest.test_method}
r�r6c�(�|jdd�yr�rurvs r4r�zCTokenizeTest.test_tabs�
r�r6c��|jdd�|jdd�|jdd�|jdd�|jd	d
�|jdd�|jd
d�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd�|jdd �|jd!d"�|jd#d$�|jd%d&�|jd'd(�|jd)d*�y)+Nr�z�    ASYNC      'async'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    r�a8    NAME       'a'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    LPAR       '('           (1, 4) (1, 5)
    ASYNC      'async'       (1, 5) (1, 10)
    EQUAL      '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    r�z�    ASYNC      'async'       (1, 0) (1, 5)
    LPAR       '('           (1, 5) (1, 6)
    RPAR       ')'           (1, 6) (1, 7)
    r�a<    NAME       'class'       (1, 0) (1, 5)
    ASYNC      'async'       (1, 6) (1, 11)
    LPAR       '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    RPAR       ')'           (1, 15) (1, 16)
    COLON      ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    r�z�    NAME       'class'       (1, 0) (1, 5)
    ASYNC      'async'       (1, 6) (1, 11)
    COLON      ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    r�z�    AWAIT      'await'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    r�z�    NAME       'foo'         (1, 0) (1, 3)
    DOT        '.'           (1, 3) (1, 4)
    ASYNC      'async'       (1, 4) (1, 9)
    r�a;    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    COLON      ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    r�a<    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    COLON      ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    r�z�    ASYNC      'async'       (1, 0) (1, 5)
    DOT        '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    r�z/    ASYNC      'async'       (1, 0) (1, 5)
    r�z�    ASYNC      'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    AWAIT      'await'       (3, 0) (3, 5)
    r�z�    ASYNC      'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    ELLIPSIS   '...'         (2, 0) (2, 3)
    NEWLINE    ''            (2, 3) (2, 3)
    AWAIT      'await'       (3, 0) (3, 5)
    r�z�    ASYNC      'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    AWAIT      'await'       (2, 0) (2, 5)
    r�z�    NAME       'foo'         (1, 0) (1, 3)
    DOT        '.'           (1, 3) (1, 4)
    ASYNC      'async'       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    r�a;    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    r�a�    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    INDENT     ''            (2, -1) (2, -1)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    LPAR       '('           (2, 9) (2, 10)
    AWAIT      'await'       (2, 10) (2, 15)
    RPAR       ')'           (2, 15) (2, 16)
    COLON      ':'           (2, 16) (2, 17)
    NEWLINE    ''            (2, 17) (2, 17)
    INDENT     ''            (3, -1) (3, -1)
    AWAIT      'await'       (3, 4) (3, 9)
    EQUAL      '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    ''            (3, 13) (3, 13)
    DEDENT     ''            (4, -1) (4, -1)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    COLON      ':'           (4, 6) (4, 7)
    NEWLINE    ''            (4, 7) (4, 7)
    INDENT     ''            (5, -1) (5, -1)
    AWAIT      'await'       (5, 4) (5, 9)
    NEWLINE    ''            (5, 9) (5, 9)
    DEDENT     ''            (6, -1) (6, -1)
    DEDENT     ''            (6, -1) (6, -1)
    ASYNC      'async'       (6, 0) (6, 5)
    PLUSEQUAL  '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    ''            (6, 10) (6, 10)
    r�a�    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    INDENT     ''            (2, -1) (2, -1)
    ASYNC      'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    COLON      ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (2, -1) (2, -1)
    r�ah    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    ASYNC      'async'       (1, 14) (1, 19)
    RPAR       ')'           (1, 19) (1, 20)
    COLON      ':'           (1, 20) (1, 21)
    AWAIT      'await'       (1, 22) (1, 27)
    r�a~    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    LPAR       '('           (1, 5) (1, 6)
    RPAR       ')'           (1, 6) (1, 7)
    COLON      ':'           (1, 7) (1, 8)
    NEWLINE    ''            (1, 8) (1, 8)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    LPAR       '('           (3, 9) (3, 10)
    RPAR       ')'           (3, 10) (3, 11)
    COLON      ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    ''            (3, 17) (3, 17)
    ASYNC      'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    LPAR       '('           (4, 15) (4, 16)
    RPAR       ')'           (4, 16) (4, 17)
    COLON      ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    ''            (4, 23) (4, 23)
    AWAIT      'await'       (6, 2) (6, 7)
    EQUAL      '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
    r�a�    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    LPAR       '('           (1, 11) (1, 12)
    RPAR       ')'           (1, 12) (1, 13)
    COLON      ':'           (1, 13) (1, 14)
    NEWLINE    ''            (1, 14) (1, 14)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    LPAR       '('           (3, 9) (3, 10)
    RPAR       ')'           (3, 10) (3, 11)
    COLON      ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    ''            (3, 17) (3, 17)
    ASYNC      'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    LPAR       '('           (4, 15) (4, 16)
    RPAR       ')'           (4, 16) (4, 17)
    COLON      ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    ''            (4, 23) (4, 23)
    AWAIT      'await'       (6, 2) (6, 7)
    EQUAL      '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
    rurvs r4r�zCTokenizeTest.test_async�
s������K�*�		�	
���-�0�		�	
���I�(�		�	
���3�6�		�	
���.�1�		�	
���K�*�		�	
���K�*�		�	
���4�7�		�	
���5�8�		�	
���K�*�		�	
���G�&�		�	
���4�7�		�	
���/�2�		�	
���N�-�		�	
���O�.�		�	
���3�6�		�	
����#�*		�X	
���H�K�		�&	
���9�	<�			�	
�����
!		�F	
�����
"		r6c�(�|jdd�y)Nr�u7    NAME       'Örter'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    ''            (1, 17) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    EQUAL      '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    rurvs r4r�zCTokenizeTest.test_unicode�r�r6c��d�}dD]4}|j|��5|jt||�ddd��6y#1swY�AxYw)Nc�T�t|�}tt|j��Sr)rrPrr<)r��
the_strings  r4�
get_tokensz5CTokenizeTest.test_invalid_syntax.<locals>.get_tokens�s#��!�&�)�J��9�*�:M�:M�N�O�Or6)z(1+2]z(1+2}z{1+2]�1_�1.2_�1e2_�1e+� u€�0b12�0b1_2�0b2�0b1_�0b�0o18�0o1_8�0o8�0o1_�0o�0x1_�0xr��012r�r�r�z'sdfsdfz'''sdfsdf''a�((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((a))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))�])rx)r�rNr)r@r�rxs   r4�test_invalid_syntaxz!CTokenizeTest.test_invalid_syntax�sJ��	P� 
�D�B���4��(��!�!�*�j�$�?�)�(�C 
�B)�(�s	�>�A	c���d}d�}||dz
�}t|��tt�j��}|j	|dj
t�|j	|dj
t�t|dd�||�}t|��|jt�fd��|jtt|dd�y)	N�dc�^�djd�t|�D��}|d|zdzz
}|S)Nr&c3�,K�|]}d|zdz���y�w)�  z	if True:
NrH)�.0rUs  r4�	<genexpr>zICTokenizeTest.test_max_indent.<locals>.generate_source.<locals>.<genexpr>�s����O��1�d�Q�h�,�6��s�r�zpass
)rr�)�indentsr8s  r4�generate_sourcez6CTokenizeTest.test_max_indent.<locals>.generate_source�s2���W�W�O��g��O�O�F��d�W�n�x�/�/�F��Mr6rMrVr"r[�execc�@��tt�j��Sr)rPrr<)�	the_inputs�r4�<lambda>z/CTokenizeTest.test_max_indent.<locals>.<lambda>�s���D�9Z�[d�[m�[m�9n�4or6)rrPrr<r=r/rr	r[rNr^)r@�	MAXINDENTr��validrW�invalidr�s      @r4�test_max_indentzCTokenizeTest.test_max_indent�s�����	�	�
 �	�A�
�.���U�O�	��7�	�8J�8J�K�L����������&�1���������)�4���z�6�*�!�)�,���W�%�	����*�,o�p�����g�w�
�F�	
r6c��d�}td�}|j|d�td�}|j||�||��td�}|j|d�td�}|j||�||��td�}|j|d	�td
�}|j||�||��y)Nc�~�t|�}t|j�D���cgc]	^}}}||f��c}}}Scc}}}wr)rrr<)r�r��kindrs    r4r�zECTokenizeTest.test_continuation_lines_indentation.<locals>.get_tokens�sJ��!�&�)�J�8��9L�9L�M�O�M�(:��f�q�T�6�N�M�O�
O��Os�8z�
            def fib(n):
                \
            '''Print a Fibonacci series up to n.'''
                \
            a, b = 0, 1
        ae    NAME       'def'         (2, 0) (2, 3)
    NAME       'fib'         (2, 4) (2, 7)
    LPAR       '('           (2, 7) (2, 8)
    NAME       'n'           (2, 8) (2, 9)
    RPAR       ')'           (2, 9) (2, 10)
    COLON      ':'           (2, 10) (2, 11)
    NEWLINE    ''            (2, 11) (2, 11)
    INDENT     ''            (4, -1) (4, -1)
    STRING     "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39)
    NEWLINE    ''            (4, 39) (4, 39)
    NAME       'a'           (6, 0) (6, 1)
    COMMA      ','           (6, 1) (6, 2)
    NAME       'b'           (6, 3) (6, 4)
    EQUAL      '='           (6, 5) (6, 6)
    NUMBER     '0'           (6, 7) (6, 8)
    COMMA      ','           (6, 8) (6, 9)
    NUMBER     '1'           (6, 10) (6, 11)
    NEWLINE    ''            (6, 11) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
        zu
            def fib(n):
                '''Print a Fibonacci series up to n.'''
                a, b = 0, 1
        z>
            pass
                \

            pass
        z�    NAME       'pass'        (2, 0) (2, 4)
    NEWLINE    ''            (2, 4) (2, 4)
    NAME       'pass'        (5, 0) (5, 4)
    NEWLINE    ''            (5, 4) (5, 4)
        z+
            pass
            pass
        z�
            if x:
                y = 1
                \
                        \
                    \
                \
                foo = 1
        aj    NAME       'if'          (2, 0) (2, 2)
    NAME       'x'           (2, 3) (2, 4)
    COLON      ':'           (2, 4) (2, 5)
    NEWLINE    ''            (2, 5) (2, 5)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'y'           (3, 4) (3, 5)
    EQUAL      '='           (3, 6) (3, 7)
    NUMBER     '1'           (3, 8) (3, 9)
    NEWLINE    ''            (3, 9) (3, 9)
    NAME       'foo'         (8, 4) (8, 7)
    EQUAL      '='           (8, 8) (8, 9)
    NUMBER     '1'           (8, 10) (8, 11)
    NEWLINE    ''            (8, 11) (8, 11)
    DEDENT     ''            (8, -1) (8, -1)
        zI
            if x:
                y = 1
                foo = 1
        )rrDr=)r@r�rf�code_no_conts    r4�#test_continuation_lines_indentationz1CTokenizeTest.test_continuation_lines_indentation�s��	O�
��
��	
���D�#�	
�,��
��	
����D�)�:�l�+C�D���
��	
���D�#�	
���
��
	
����D�)�:�l�+C�D���
��	
���D�#�	
�"��
��	
����D�)�:�l�+C�Dr6N)r�r�r�rDr�rwr�r�r�r�r�r�r�r�r�r�r�r�r�r�r�rHr6r4r�r�sj��E�3�(?	�B,	�\F	�P=	�~$	�L	�.	�($	�L	�,	�"	�"H	�T
	�'@�R
�,bEr6r�c��eZdZd�Zy)�CTokenizerBufferTestsc��dddz�dddz��}tj�5}t|d|�}t|�ddd�y#1swYyxYw)Nz"        #coding: latin-1
        #r;i'z

        #i'�foo)rr?rr)r@�test_scriptr?�	file_names    r4�!test_newline_at_the_end_of_bufferz7CTokenizerBufferTests.test_newline_at_the_end_of_bufferZsW��
�

�e�)��
�

�e�)�����
�
�
!�X�#�H�e�[�A�I��I�&�"�
!�
!�s�A�AN)r�r�r�r�rHr6r4r�r�Ys��
'r6r��__main__):rrZr0�unittestrrrrrrr	r
rrr
r�rrrrrrr�iorr�textwraprrr�testr�test.test_grammarrr�test.supportr�test.support.script_helperrrr r5r8rrrrr5rBr�rGr^r"r�r�r�r��mainrHr6r4�<module>r�s��	�	���!�!�!�!�!�
!��#��<�"�Y�Y�� g	�8�g	�R%A��A�
6�G�x�G�.#I�H�#I�L<�H�<�,{"��{"�|v2�8�v2�r&>�X�&>�R
.� Q#�H�Q#�h2��2�8{E�H�{E�|!'�H�-�-�'��z���H�M�M�O�r6