
    dafW                    6   d dl mZ d dlmZ d dlmZmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZ d dlmZmZ d dlZd dlmZ d dlmZmZ d dlm Z m!Z! d dlmZ d d	l"m#Z#m$Z$m%Z% d dl&Z&d dl'Z'd
 Z( G d de      Z) G d de)      Z*d Z+ G d de      Z, G d de      Z- G d de      Z. G d de      Z/ G d de      Z0 G d de      Z1 G d de      Z2 G d de      Z3 G d  d!e      Z4 G d" d#ej:                        Z5e6d$k(  r ejn                          yy)%    )support)	os_helper)tokenize
untokenizeNUMBERNAMEOPSTRING	ENDMARKERENCODINGtok_namedetect_encodingopenUntokenizergenerate_tokensNEWLINE!_generate_tokens_from_c_tokenizerDEDENT	TokenInfo
TokenError)BytesIOStringION)dedent)TestCasemock)VALID_UNDERSCORE_LITERALSINVALID_UNDERSCORE_LITERALS)run_test_scriptmake_scriptrun_python_until_endc                     g }t        |j                               }|d   dv}| D ]P  \  }}}}}	|t        k(  r |S |r|t        k(  r	|d   |k(  r)t        |   }|j                  d|dd|dd| d|        R |S )Nz
r       10 13)len
splitlinesr   r   r   append)
token_generatorsource_stringresult	num_linesmissing_trailing_nltypetokenstartendlines
             -/root/Python-3.12.4/Lib/test/test_tokenize.pystringify_tokens_from_sourcer5      s    FM,,./I'+69)8 C%eUC9 M 47?s1v7J~T"IQurl!E7!C5ABC M    c                       e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zy)TokenizeTestc                     t        |j                  d            }t        t        |j                        |      }| j                  |dg|j                         j                         z          y )Nutf-8z*    ENCODING   'utf-8'       (0, 0) (0, 0))r   encoder5   r   readlineassertEqualrstripr(   selfsexpectedfr,   s        r4   check_tokenizezTokenizeTest.check_tokenize.   sZ     AHHW%&-hqzz.BAFFG!*5578	9r6   c                    d }| j                  t              5  t        t         |       j                               d d d        d }| j                  t              5  t        t         |       j                               d d d        d }| j                  t              5  t        t         |       j                               d d d        y # 1 sw Y   xY w# 1 sw Y   ZxY w# 1 sw Y   y xY w)Nc               3      K   d d y w)Nsdfosdg rH   r6   r4   genz/TokenizeTest.test_invalid_readline.<locals>.gen8   s     OO   
c               3      K   d d y w)Ns   sdfosdgrH   rH   r6   r4   rI   z/TokenizeTest.test_invalid_readline.<locals>.gen>   s     rJ   c               3      K   d ddz   y w)NrG      r   rH   rH   r6   r4   rI   z/TokenizeTest.test_invalid_readline.<locals>.genD   s     OaCs   )assertRaises	TypeErrorlistr   __next__r   ZeroDivisionError)r@   rI   s     r4   test_invalid_readlinez"TokenizeTest.test_invalid_readline7   s    	 y) 	+#%..)*	+	 y) 	201	2	 01 	201	2 	2	+ 	+	2 	2	2 	2s#   $C$C!$C&CC#&C/c                     t        dj                  d            }t        t        |j                              }| j                  |d   j                  t               | j                  |d   j                  t               y )Nxr:   r"   )	r   r;   rP   r   r<   r=   r/   r   r   )r@   rC   tokenss      r4   test_implicit_newlinez"TokenizeTest.test_implicit_newlineJ   sZ     CJJw'(hqzz*+'2)4r6   c                 P   | j                  dd       | j                  dd       | j                  dd       | j                  dd       d	}t        |      j                  }| j                  t        d
      5 }t        |      D ]  } 	 d d d        | j                  j                  j                  d       | j                  |j                  j                  d       | j                  |j                  j                  d        | j                  |j                  j                  d        | j                  |j                  j                  d
       | j                  |j                  j                  d       | j                  |j                  j                  d       y # 1 sw Y   xY w)Nz1 + 1z    NUMBER     '1'           (1, 0) (1, 1)
    OP         '+'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    z3if False:
    # NL
    
    True = False # NEWLINE
af      NAME       'if'          (1, 0) (1, 2)
    NAME       'False'       (1, 3) (1, 8)
    OP         ':'           (1, 8) (1, 9)
    NEWLINE    '\n'          (1, 9) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NL         '\n'          (3, 4) (3, 5)
    INDENT     '    '        (4, 0) (4, 4)
    NAME       'True'        (4, 4) (4, 8)
    OP         '='           (4, 9) (4, 10)
    NAME       'False'       (4, 11) (4, 16)
    COMMENT    '# NEWLINE'   (4, 17) (4, 26)
    NEWLINE    '\n'          (4, 26) (4, 27)
    DEDENT     ''            (5, 0) (5, 0)
    z%if True:
    # NL
    foo='bar'

a@      NAME       'if'          (1, 0) (1, 2)
    NAME       'True'        (1, 3) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\r\n'        (1, 8) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\r\n'        (2, 8) (2, 10)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'foo'         (3, 4) (3, 7)
    OP         '='           (3, 7) (3, 8)
    STRING     "'bar'"       (3, 8) (3, 13)
    NEWLINE    '\r\n'        (3, 13) (3, 15)
    NL         '\r\n'        (4, 0) (4, 2)
    DEDENT     ''            (5, 0) (5, 0)
            zx = 1 + \
1
a      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (2, 0) (2, 1)
    NEWLINE    '\r\n'        (2, 1) (2, 3)
            s   def k(x):
    x += 2
  x += 5
z3unindent does not match any outer indentation level   <string>	   z  x += 5)rD   r   r<   assertRaisesRegexIndentationErrorr   r=   	exceptionlinenofilename
end_lineno
end_offsetmsgoffsettext)r@   indent_error_filer<   etoks        r4   
test_basiczTokenizeTest.test_basicR   s   G & 		
 	 ;=		( 	K N 	  	1 4 	
 ,-66##$4%>? 	BC) 	
 	++Q/--z://6//6KKOOA	C 	++Q/)):6	 	s   6FF%c                 H   | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  d	d
       | j                  dd       | j                  dd       | j                  dd       | j                  dd       y )N0xff <= 255z    NUMBER     '0xff'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    0b10 <= 255z    NUMBER     '0b10'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    0o123 <= 0O123z    NUMBER     '0o123'       (1, 0) (1, 5)
    OP         '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    1234567 > ~0x15z    NUMBER     '1234567'     (1, 0) (1, 7)
    OP         '>'           (1, 8) (1, 9)
    OP         '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    2134568 != 1231515z    NUMBER     '2134568'     (1, 0) (1, 7)
    OP         '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    (-124561-1) & 200000000ac      OP         '('           (1, 0) (1, 1)
    OP         '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    OP         '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         ')'           (1, 10) (1, 11)
    OP         '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    0xdeadbeef != -1z    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    OP         '!='          (1, 11) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    0xdeadc0de & 12345z    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    OP         '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    0xFF & 0x15 | 1234z    NUMBER     '0xFF'        (1, 0) (1, 4)
    OP         '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    OP         '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    rD   r@   s    r4   test_intzTokenizeTest.test_int   s    M , 		
 	M , 		
 	, / 		
 	- 0 		 	0 3 		
 	5 	8 			 	. 1 		 	0 3 		
 	0 3 		r6   c                     | j                  dd       | j                  dd       | j                  dd       | j                  dd       y )	Nzx = 0z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0'           (1, 4) (1, 5)
    zx = 0xfffffffffffz    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0xfffffffffff' (1, 4) (1, 17)
    zx = 123141242151251616110z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '123141242151251616110' (1, 4) (1, 25)
    zx = -15921590215012591z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '-'           (1, 4) (1, 5)
    NUMBER     '15921590215012591' (1, 5) (1, 22)
    ru   rv   s    r4   	test_longzTokenizeTest.test_long   s_    G & 		
 	/ 2 		
 	7 : 		
 	4 7 		r6   c                     | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  d	d
       | j                  dd       | j                  dd       y )Nx = 3.14159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    x = 314159.z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    x = .314159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    x = 3e14159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    	x = 3E123z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    x+y = 3e-1230z    NAME       'x'           (1, 0) (1, 1)
    OP         '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    OP         '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    x = 3.14e159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    ru   rv   s    r4   
test_floatzTokenizeTest.test_float   s    M , 		
 	M , 		
 	M , 		
 	M , 		
 	K * 		
 	O . 		 	N - 		r6   c                     d }t         D ]  }d|v r| j                   ||      |       ! ddh}t        D ](  }||v r	  ||       | j	                   ||      |       * y # t        $ r Y 7w xY w)Nc                     t        | j                  d            }t        |j                        D ]  \  }}}}}|t        k(  s|c S  y)Nr:   zinvalid token)r   r;   r   r<   r   )rA   rC   toktyper0   r1   r2   r3   s          r4   number_tokenz;TokenizeTest.test_underscore_literals.<locals>.number_token  sJ    )*A4<QZZ4H !0sDf$ L! #r6   (0_709_99)r   r=   r   r   assertNotEqual)r@   r   litextra_valid_casess       r4   test_underscore_literalsz%TokenizeTest.test_underscore_literals  s    	# - 	5Ccz\#.4		5 #G,. 	8C''S! S 137	8
  s   A""	A.-A.c                 8   | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  d	d
       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd        | j                  d!d"       | j                  d#d$       | j                  d%d&       | j                  d'd(       | j                  d)d*       | j                  d+d,       | j                  d-d.       | j                  d/d0       | j                  d1d2       | j                  d3d4       | j                  d5d6       | j                  d7d8       | j                  d9d:       | j                  d;d<       | j                  d=d>       | j                  d?d@       | j                  dAdB       | j                  dCdD       | j                  dEdF       | j                  dGdH       | j                  dIdJ       y )KNx = ''; y = ""a5      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    OP         ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    x = '"'; y = "'"a6      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    OP         ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    OP         '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    x = "doesn't "shrink", does it"z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    x = 'abc' + 'ABC'z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    y = "ABC" + "ABC"z    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    %x = r'abc' + r'ABC' + R'ABC' + R'ABC'a      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    %y = r"abc" + r"ABC" + R"ABC" + R"ABC"a      NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    u'abc' + U'abc'z    STRING     "u'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    u"abc" + U"abc"z    STRING     'u"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    b'abc' + B'abc'z    STRING     "b'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    b"abc" + B"abc"z    STRING     'b"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    %br'abc' + bR'abc' + Br'abc' + BR'abc'a;      STRING     "br'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    %br"abc" + bR"abc" + Br"abc" + BR"abc"a;      STRING     'br"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    %rb'abc' + rB'abc' + Rb'abc' + RB'abc'a;      STRING     "rb'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    %rb"abc" + rB"abc" + Rb"abc" + RB"abc"a;      STRING     'rb"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    "a\
de\
fg"3    STRING     '"a\\\nde\\\nfg"' (1, 0) (3, 3)
    u"a\
de"/    STRING     'u"a\\\nde"'  (1, 0) (2, 3)
    rb"a\
d"/    STRING     'rb"a\\\nd"'  (1, 0) (2, 2)
    
"""a\
b"""0    STRING     '"""a\\\nb"""' (1, 0) (2, 4)
    u"""a\
b"""1    STRING     'u"""a\\\nb"""' (1, 0) (2, 4)
    rb"""a\
b\
c"""7    STRING     'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4)
    f"abc"    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE 'abc'         (1, 2) (1, 5)
    FSTRING_END '"'           (1, 5) (1, 6)
    	fR"a{b}c"a=      FSTRING_START 'fR"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'a'           (1, 3) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    NAME       'b'           (1, 5) (1, 6)
    OP         '}'           (1, 6) (1, 7)
    FSTRING_MIDDLE 'c'           (1, 7) (1, 8)
    FSTRING_END '"'           (1, 8) (1, 9)
    zfR"a{{{b!r}}}c"a      FSTRING_START 'fR"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'a{'          (1, 3) (1, 5)
    OP         '{'           (1, 6) (1, 7)
    NAME       'b'           (1, 7) (1, 8)
    OP         '!'           (1, 8) (1, 9)
    NAME       'r'           (1, 9) (1, 10)
    OP         '}'           (1, 10) (1, 11)
    FSTRING_MIDDLE '}'           (1, 11) (1, 12)
    FSTRING_MIDDLE 'c'           (1, 13) (1, 14)
    FSTRING_END '"'           (1, 14) (1, 15)
    zf"{{{1+1}}}"a      FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE '{'           (1, 2) (1, 3)
    OP         '{'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '}'           (1, 8) (1, 9)
    FSTRING_MIDDLE '}'           (1, 9) (1, 10)
    FSTRING_END '"'           (1, 11) (1, 12)
    zf"""{f'''{f'{f"{1+1}"}'}'''}"""ad      FSTRING_START 'f"""'        (1, 0) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    FSTRING_START "f'''"        (1, 5) (1, 9)
    OP         '{'           (1, 9) (1, 10)
    FSTRING_START "f'"          (1, 10) (1, 12)
    OP         '{'           (1, 12) (1, 13)
    FSTRING_START 'f"'          (1, 13) (1, 15)
    OP         '{'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '+'           (1, 17) (1, 18)
    NUMBER     '1'           (1, 18) (1, 19)
    OP         '}'           (1, 19) (1, 20)
    FSTRING_END '"'           (1, 20) (1, 21)
    OP         '}'           (1, 21) (1, 22)
    FSTRING_END "'"           (1, 22) (1, 23)
    OP         '}'           (1, 23) (1, 24)
    FSTRING_END "'''"         (1, 24) (1, 27)
    OP         '}'           (1, 27) (1, 28)
    FSTRING_END '"""'         (1, 28) (1, 31)
    z.f"""     x
str(data, encoding={invalid!r})
"""a      FSTRING_START 'f"""'        (1, 0) (1, 4)
    FSTRING_MIDDLE '     x\nstr(data, encoding=' (1, 4) (2, 19)
    OP         '{'           (2, 19) (2, 20)
    NAME       'invalid'     (2, 20) (2, 27)
    OP         '!'           (2, 27) (2, 28)
    NAME       'r'           (2, 28) (2, 29)
    OP         '}'           (2, 29) (2, 30)
    FSTRING_MIDDLE ')\n'         (2, 30) (3, 0)
    FSTRING_END '"""'         (3, 0) (3, 3)
    z#f"""123456789
something{None}bad"""aO      FSTRING_START 'f"""'        (1, 0) (1, 4)
    FSTRING_MIDDLE '123456789\nsomething' (1, 4) (2, 9)
    OP         '{'           (2, 9) (2, 10)
    NAME       'None'        (2, 10) (2, 14)
    OP         '}'           (2, 14) (2, 15)
    FSTRING_MIDDLE 'bad'         (2, 15) (2, 18)
    FSTRING_END '"""'         (2, 18) (2, 21)
    
f"""abc"""    FSTRING_START 'f"""'        (1, 0) (1, 4)
    FSTRING_MIDDLE 'abc'         (1, 4) (1, 7)
    FSTRING_END '"""'         (1, 7) (1, 10)
    f"abc\
def"    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE 'abc\\\ndef'  (1, 2) (2, 3)
    FSTRING_END '"'           (2, 3) (2, 4)
    Rf"abc\
def"    FSTRING_START 'Rf"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'abc\\\ndef'  (1, 3) (2, 3)
    FSTRING_END '"'           (2, 3) (2, 4)
    z5f'some words {a+b:.3f} more words {c+d=} final words'aA      FSTRING_START "f'"          (1, 0) (1, 2)
    FSTRING_MIDDLE 'some words ' (1, 2) (1, 13)
    OP         '{'           (1, 13) (1, 14)
    NAME       'a'           (1, 14) (1, 15)
    OP         '+'           (1, 15) (1, 16)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    FSTRING_MIDDLE '.3f'         (1, 18) (1, 21)
    OP         '}'           (1, 21) (1, 22)
    FSTRING_MIDDLE ' more words ' (1, 22) (1, 34)
    OP         '{'           (1, 34) (1, 35)
    NAME       'c'           (1, 35) (1, 36)
    OP         '+'           (1, 36) (1, 37)
    NAME       'd'           (1, 37) (1, 38)
    OP         '='           (1, 38) (1, 39)
    OP         '}'           (1, 39) (1, 40)
    FSTRING_MIDDLE ' final words' (1, 40) (1, 52)
    FSTRING_END "'"           (1, 52) (1, 53)
    zf'''{
3
=}'''a`      FSTRING_START "f'''"        (1, 0) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    NL         '\n'          (1, 5) (1, 6)
    NUMBER     '3'           (2, 0) (2, 1)
    NL         '\n'          (2, 1) (2, 2)
    OP         '='           (3, 0) (3, 1)
    OP         '}'           (3, 1) (3, 2)
    FSTRING_END "'''"         (3, 2) (3, 5)
    f'''__{
    x:a
}__'''a      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    OP         '{'           (1, 6) (1, 7)
    NL         '\n'          (1, 7) (1, 8)
    NAME       'x'           (2, 4) (2, 5)
    OP         ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n'         (2, 6) (3, 0)
    OP         '}'           (3, 0) (3, 1)
    FSTRING_MIDDLE '__'          (3, 1) (3, 3)
    FSTRING_END "'''"         (3, 3) (3, 6)
    +f'''__{
    x:a
    b
     c
      d
}__'''a      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    OP         '{'           (1, 6) (1, 7)
    NL         '\n'          (1, 7) (1, 8)
    NAME       'x'           (2, 4) (2, 5)
    OP         ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n    b\n     c\n      d\n' (2, 6) (6, 0)
    OP         '}'           (6, 0) (6, 1)
    FSTRING_MIDDLE '__'          (6, 1) (6, 3)
    FSTRING_END "'''"         (6, 3) (6, 6)
    f'__{
    x:d
}__'a      FSTRING_START "f'"          (1, 0) (1, 2)
    FSTRING_MIDDLE '__'          (1, 2) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    NL         '\n'          (1, 5) (1, 6)
    NAME       'x'           (2, 4) (2, 5)
    OP         ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'd'           (2, 6) (2, 7)
    NL         '\n'          (2, 7) (2, 8)
    OP         '}'           (3, 0) (3, 1)
    FSTRING_MIDDLE '__'          (3, 1) (3, 3)
    FSTRING_END "'"           (3, 3) (3, 4)
    uz       '''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli
    aktualni pracownicy, obecni pracownicy'''
u      INDENT     '    '        (1, 0) (1, 4)
    STRING     "'''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli\n    aktualni pracownicy, obecni pracownicy'''" (1, 4) (2, 45)
    NEWLINE    '\n'          (2, 45) (2, 46)
    DEDENT     ''            (3, 0) (3, 0)
    ru   rv   s    r4   test_stringzTokenizeTest.test_string+  s   . 1 		 	1 4 		 	A D 		 	/ 2 		 	/ 2 		 	C 
F 
		 	C 
F 
		 	- 0 		
 	- 0 		 	- 0 		
 	- 0 		
 	C F 		 	C F 		 	C F 		 	C F 		 	 		
 	 		 	 		 	 		 	 		 	 		
 	H ' 		
 	K * 		 	- 0 		 	N 
- 
		 	E H 		* 	N 
Q 
		 	B E 		 	L + 		
 	 		 	 		 	S V 		( 	 				 	 
		 	 
		$ 	 
		" 	 		r6   c                 p    | j                  dd       | j                  dd       | j                  dd       y )N!def d22(a, b, c=2, d=2, *k): passa      NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    OP         ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    OP         '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    OP         '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    OP         ','           (1, 22) (1, 23)
    OP         '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    OP         ')'           (1, 26) (1, 27)
    OP         ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    def d01v_(a=1, *k, **w): passa      NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    OP         '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         ','           (1, 13) (1, 14)
    OP         '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    OP         '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    OP         ')'           (1, 22) (1, 23)
    OP         ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    &def d23(a: str, b: int=3) -> int: passa      NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    OP         ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    OP         '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    OP         ')'           (1, 24) (1, 25)
    OP         '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    OP         ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    ru   rv   s    r4   test_functionzTokenizeTest.test_function{  sJ    ? B 		, 	; > 		" 	D G 		r6   c                 (    | j                  dd       y )NXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa      NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    OP         '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    OP         '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    OP         '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    OP         '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    OP         ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    ru   rv   s    r4   test_comparisonzTokenizeTest.test_comparison       M!O"		r6   c                 (    | j                  dd       y )Nzx = 1 << 1 >> 5a6      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '<<'          (1, 6) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '>>'          (1, 11) (1, 13)
    NUMBER     '5'           (1, 14) (1, 15)
    ru   rv   s    r4   
test_shiftzTokenizeTest.test_shift  s    - 0 		r6   c                 (    | j                  dd       y )N%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    OP         '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    OP         '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    OP         '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    OP         '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    OP         '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    OP         ']'           (1, 36) (1, 37)
    ru   rv   s    r4   test_additivezTokenizeTest.test_additive      C F 		r6   c                 (    | j                  dd       y )Nx = 1//1*1/5*12%0x12@42a      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    OP         '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    OP         '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    OP         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    ru   rv   s    r4   test_multiplicativez TokenizeTest.test_multiplicative       5 8 		r6   c                 L    | j                  dd       | j                  dd       y )N~1 ^ 1 & 1 |1 ^ -1a      OP         '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '^'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    -1*1/1+1*1//1 - ---1**1aH      OP         '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    OP         '-'           (1, 17) (1, 18)
    OP         '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    OP         '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    ru   rv   s    r4   
test_unaryzTokenizeTest.test_unary  s0    0 3 		 	5 8 		r6   c                 (    | j                  dd       y )N/import sys, time
x = sys.modules['time'].time()a      NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    OP         ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'x'           (2, 0) (2, 1)
    OP         '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    OP         '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    OP         '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    OP         ']'           (2, 22) (2, 23)
    OP         '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    OP         '('           (2, 28) (2, 29)
    OP         ')'           (2, 29) (2, 30)
    ru   rv   s    r4   test_selectorzTokenizeTest.test_selector9      N Q 		r6   c                 (    | j                  dd       y )N @staticmethod
def foo(x,y): passa      OP         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    '\n'          (1, 13) (1, 14)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    OP         '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    OP         ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    OP         ')'           (2, 11) (2, 12)
    OP         ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    ru   rv   s    r4   test_methodzTokenizeTest.test_methodO      ? B 		r6   c                 (    | j                  dd       y )Nzdef f():
	if x
        	passa_      NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    INDENT     '\t'          (2, 0) (2, 1)
    NAME       'if'          (2, 1) (2, 3)
    NAME       'x'           (2, 4) (2, 5)
    NEWLINE    '\n'          (2, 5) (2, 6)
    INDENT     '        \t'  (3, 0) (3, 9)
    NAME       'pass'        (3, 9) (3, 13)
    DEDENT     ''            (4, 0) (4, 0)
    DEDENT     ''            (4, 0) (4, 0)
    ru   rv   s    r4   	test_tabszTokenizeTest.test_tabs`  s     -/		r6   c                 (    | j                  dd       y )Nu!   Örter = 'places'
grün = 'green'u7      NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "'places'"    (1, 8) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "'green'"     (2, 7) (2, 14)
    ru   rv   s    r4   test_non_ascii_identifiersz'TokenizeTest.test_non_ascii_identifiersu  s    @ C 		r6   c                 (    | j                  dd       y )N#   Örter = u'places'
grün = U'green'u7      NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    '\n'          (1, 17) (1, 18)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    ru   rv   s    r4   test_unicodezTokenizeTest.test_unicode      B E 		r6   c                    | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  d	d
       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd        | j                  d!d"       | j                  d#d$       | j                  d%d&       | j                  d'd(       | j                  d)d*       y )+N	async = 1z    NAME       'async'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    a = (async = 1)a8      NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '('           (1, 4) (1, 5)
    NAME       'async'       (1, 5) (1, 10)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    async()z    NAME       'async'       (1, 0) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    class async(Bar):passa<      NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    OP         ')'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    class async:passz    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    	await = 1z    NAME       'await'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    	foo.asyncz    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    async for a in b: passa;      NAME       'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    async with a as b: passa<      NAME       'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    	async.fooz    NAME       'async'       (1, 0) (1, 5)
    OP         '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    asyncz/    NAME       'async'       (1, 0) (1, 5)
    async
#comment
awaitz    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    COMMENT    '#comment'    (2, 0) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NAME       'await'       (3, 0) (3, 5)
    async
...
awaitz    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    OP         '...'         (2, 0) (2, 3)
    NEWLINE    '\n'          (2, 3) (2, 4)
    NAME       'await'       (3, 0) (3, 5)
    async
awaitz    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    NAME       'await'       (2, 0) (2, 5)
    foo.async + 1z    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    async def foo(): passa;      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    Nasync def foo():
  def foo(await):
    await = 1
  if 1:
    await
async += 1
a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    OP         '('           (2, 9) (2, 10)
    NAME       'await'       (2, 10) (2, 15)
    OP         ')'           (2, 15) (2, 16)
    OP         ':'           (2, 16) (2, 17)
    NEWLINE    '\n'          (2, 17) (2, 18)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'await'       (3, 4) (3, 9)
    OP         '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    '\n'          (3, 13) (3, 14)
    DEDENT     ''            (4, 2) (4, 2)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    OP         ':'           (4, 6) (4, 7)
    NEWLINE    '\n'          (4, 7) (4, 8)
    INDENT     '    '        (5, 0) (5, 4)
    NAME       'await'       (5, 4) (5, 9)
    NEWLINE    '\n'          (5, 9) (5, 10)
    DEDENT     ''            (6, 0) (6, 0)
    DEDENT     ''            (6, 0) (6, 0)
    NAME       'async'       (6, 0) (6, 5)
    OP         '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    '\n'          (6, 10) (6, 11)
    )async def foo():
  async for i in 1: passa      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    OP         ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (3, 0) (3, 0)
    async def foo(async): awaitah      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    NAME       'async'       (1, 14) (1, 19)
    OP         ')'           (1, 19) (1, 20)
    OP         ':'           (1, 20) (1, 21)
    NAME       'await'       (1, 22) (1, 27)
    @def f():

  def baz(): pass
  async def bar(): pass

  await = 2a      NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    Fasync def f():

  def baz(): pass
  async def bar(): pass

  await = 2a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    OP         '('           (1, 11) (1, 12)
    OP         ')'           (1, 12) (1, 13)
    OP         ':'           (1, 13) (1, 14)
    NEWLINE    '\n'          (1, 14) (1, 15)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    ru   rv   s    r4   
test_asynczTokenizeTest.test_async  s   K * 		 	- 0 		 	I ( 		 	3 6 		 	. 1 		 	K * 		 	K * 		 	4 7 		 	5 8 		 	K * 		 	G & 		 	4 7 		 	/ 2 		 	N - 		 	O . 		 	3 6 		 	 #*		X 	 		* 	= 	@ 			 	 #		J 	 $		r6   c                 (    | j                  dd       y )Nz[
    # A comment here
    1
]
ad      OP         '['           (1, 0) (1, 1)
    NL         '\n'          (1, 1) (1, 2)
    COMMENT    '# A comment here' (2, 4) (2, 20)
    NL         '\n'          (2, 20) (2, 21)
    NUMBER     '1'           (3, 4) (3, 5)
    NL         '\n'          (3, 5) (3, 6)
    OP         ']'           (4, 0) (4, 1)
    NEWLINE    '\n'          (4, 1) (4, 2)
    ru   rv   s    r4   3test_newline_after_parenthesized_block_with_commentz@TokenizeTest.test_newline_after_parenthesized_block_with_comment  s     
			r6   c                 (    | j                  dd       y )Nz); xz    OP         ')'           (1, 0) (1, 1)
    OP         ';'           (1, 1) (1, 2)
    NAME       'x'           (1, 3) (1, 4)
    ru   rv   s    r4   ,test_closing_parenthesis_from_different_linez9TokenizeTest.test_closing_parenthesis_from_different_line  s    F % 		r6   N)__name__
__module____qualname__rD   rS   rX   rj   rw   ry   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rH   r6   r4   r8   r8   '   s    92&5F7P7	r	0&	P80N	`
:	x$	L
		.	(#	J	,	"	*
	
	P	d	"	r6   r8   c                       e Zd Zd Zy)GenerateTokensTestc                     t        |      }t        t        |j                        |      }| j	                  ||j                         j                                y N)r   r5   r   r<   r=   r>   r(   r?   s        r4   rD   z!GenerateTokensTest.check_tokenize  sB     QK-oajj.I1M!2!=!=!?@r6   N)r   r   r   rD   rH   r6   r4   r   r     s    Ar6   r   c           	      x   g }t        t        | j                  d            j                        }|D ]_  \  }}}}}|t        k(  r;d|v r7|j                  t        dft        dft        t        |      ft        dfg       M|j                  ||f       a t        |      j                  d      j                         S )Nr:   .Decimalr   ))r   r   r;   r<   r   extendr   r	   r
   reprr)   r   decodestrip)rA   r,   gtoknumtokval_s         r4   decistmtr    s    F'*+445A$% 	,1aVvMMy!S	f&S		  MM66*+	, f$$W-3355r6   c                       e Zd Zd Zy)TestMiscc                     ddl m} d}| j                  t        |      d       | j	                  t        t        |            d       | j                  t        t        |             |d             y )Nr   )r  z+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)decimalr  r=   r  assertRegexr  eval)r@   r  rA   s      r4   test_decistmtzTestMisc.test_decistmt  sd     	$"!S	U 	d1g(BC 	hqk* !DE	Gr6   N)r   r   r   r  rH   r6   r4   r  r    s    Gr6   r  c                   4    e Zd ZdZd Zd Zd Zd Zd Zd Z	y)	TestTokenizerAdheresToPep0263zU
    Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263.
    c                     t         j                  j                  t         j                  j                  t              d|      }t        |d      5 }t        j                  | |       d d d        y # 1 sw Y   y xY w)Ntokenizedatarb)ospathjoindirname__file__r   TestRoundtripcheck_roundtrip)r@   ra   r  rC   s       r4   	_testFilez'TestTokenizerAdheresToPep0263._testFile  sV    ww||BGGOOH5~xP$ 	3))$2	3 	3 	3s   A//A8c                 *    d}| j                  |       y )Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txtr   r@   rC   s     r4   'test_utf8_coding_cookie_and_no_utf8_bomzETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bom      Gqr6   c                 J    d}| j                  t        | j                  |       y)a  
        As per PEP 0263, if a file starts with a utf-8 BOM signature, the only
        allowed encoding for the comment is 'utf-8'.  The text file used in
        this test starts with a BOM signature, but specifies latin1 as the
        coding, so verify that a SyntaxError is raised, which matches the
        behaviour of the interpreter when it encounters a similar condition.
        z8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtNrN   SyntaxErrorr   r#  s     r4   &test_latin1_coding_cookie_and_utf8_bomzDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bom  s      G+t~~q9r6   c                 *    d}| j                  |       y )Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtr"  r#  s     r4   "test_no_coding_cookie_and_utf8_bomz@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bom  r%  r6   c                 *    d}| j                  |       y )Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtr"  r#  s     r4   $test_utf8_coding_cookie_and_utf8_bomzBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bom  s    Dqr6   c                     | j                  t        | j                  d       | j                  t        | j                  d       y )Nzbad_coding.pyzbad_coding2.pyr'  rv   s    r4   test_bad_coding_cookiez4TestTokenizerAdheresToPep0263.test_bad_coding_cookie  s/    +t~~G+t~~7GHr6   N)
r   r   r   __doc__r   r$  r)  r+  r-  r/  rH   r6   r4   r  r    s&    3
	:Ir6   r  c                       e Zd Zd Zy)Test_Tokenizec                     d}|j                  d      dfd}t        t         |       j                  dd            d d }t	        ddd	d
d      g}| j                  ||d       y )Nu   "ЉЊЈЁЂ"r:   Fc               3   (   K    sd  y d y w)NTr6   rH   )firstr3   s   r4   r<   zNTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline  s     
	s   T)encodingextra_tokensrV   rZ   rM   r   rM      zbytes not decoded with encoding)r;   rP   r   rQ   r   r=   )r@   literalr<   rW   expected_tokensr5  r3   s        @@r4   .test__tokenize_decodes_with_specified_encodingz<Test_Tokenize.test__tokenize_decodes_with_specified_encoding  sx     ~~g&	 7
8K8KV]#') **-2/$QWX:	<r6   N)r   r   r   r=  rH   r6   r4   r2  r2    s    <r6   r2  c                       e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zy)TestDetectEncodingc                     dfd}|S )Nr   c                  D    t              k(  rt           } dz  | S NrM   )r'   StopIteration)r3   indexliness    r4   r<   z1TestDetectEncoding.get_readline.<locals>.readline&  s,    E
"##<DQJEKr6   rH   )r@   rE  r<   rD  s    ` @r4   get_readlinezTestDetectEncoding.get_readline$  s    	 r6   c                     d}t        | j                  |            \  }}| j                  |d       | j                  |t        |d d              y )N)   # something
   print(something)
   do_something(else)
r:      )r   rF  r=   rP   r@   rE  r6  consumed_liness       r4   test_no_bom_no_encoding_cookiez1TestDetectEncoding.test_no_bom_no_encoding_cookie/  sO    

 $343D3DU3K#L .7+eBQi9r6   c                     d}t        | j                  |            \  }}| j                  |d       | j                  |ddg       y )N)s   ﻿# something
rI  rJ  	utf-8-sigrH  rI  r   rF  r=   rL  s       r4   test_bom_no_cookiez%TestDetectEncoding.test_bom_no_cookie9  sO    

 $343D3DU3K#L .;/*,AB	Dr6   c                     d}t        | j                  |            \  }}| j                  |d       | j                  |dg       y )N)   # -*- coding: latin-1 -*-
rI  rJ  
iso-8859-1rT  rQ  rL  s       r4   test_cookie_first_line_no_bomz0TestDetectEncoding.test_cookie_first_line_no_bomD  sI    

 $343D3DU3K#L .<0*H)IJr6   c                     d}t        | j                  |            \  }}| j                  |d       | j                  |dg       y )N)s   ﻿# coding=utf-8
rI  rJ  rP  s   # coding=utf-8
rQ  rL  s       r4   &test_matched_bom_and_cookie_first_linez9TestDetectEncoding.test_matched_bom_and_cookie_first_lineN  sI    

 $343D3DU3K#L .;/*=)>?r6   c                 `    d}| j                  |      }| j                  t        t        |       y )N)s#   ﻿# vim: set fileencoding=ascii :
rI  rJ  rF  rN   r(  r   r@   rE  r<   s      r4   <test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorzOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorX  s-    

 $$U++Ar6   c                     d}t        | j                  |            \  }}| j                  |d       ddg}| j                  ||       y )N)   #! something
    # vim: set fileencoding=ascii :
rI  rJ  asciir^  r_  rQ  r@   rE  r6  rM  rB   s        r4   test_cookie_second_line_no_bomz1TestDetectEncoding.test_cookie_second_line_no_boma  sP    
 $343D3DU3K#L .7+%'KL2r6   c                     d}t        | j                  |            \  }}| j                  |d       | j                  |ddg       y )N)   ﻿#! something
   f# coding=utf-8
rI  rJ  rP  r^  re  rQ  rL  s       r4   'test_matched_bom_and_cookie_second_linez:TestDetectEncoding.test_matched_bom_and_cookie_second_linem  sO    
 $343D3DU3K#L .;/+-AB	Dr6   c                 `    d}| j                  |      }| j                  t        t        |       y )N)rd  r_  rI  rJ  rZ  r[  s      r4   =test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorzPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerrory  s-    
 $$U++Ar6   c                     d}t        | j                  |            \  }}| j                  |d       dg}| j                  ||       y )N)   print('£')
%   # vim: set fileencoding=iso8859-15 :
   print('€')
r:   rj  rQ  ra  s        r4   /test_cookie_second_line_noncommented_first_linezBTestDetectEncoding.test_cookie_second_line_noncommented_first_line  sM    

 $343D3DU3K#L .7+*+2r6   c                     d}t        | j                  |            \  }}| j                  |d       ddg}| j                  ||       y )N)   #print('£')
rk  rl  
iso8859-15ro  rk  rQ  ra  s        r4   ,test_cookie_second_line_commented_first_linez?TestDetectEncoding.test_cookie_second_line_commented_first_line  sP    

 $343D3DU3K#L .<0+-VW2r6   c                     d}t        | j                  |            \  }}| j                  |d       ddg}| j                  ||       y )N)   
rk  rl  rp  rs  rk  rQ  ra  s        r4   (test_cookie_second_line_empty_first_linez;TestDetectEncoding.test_cookie_second_line_empty_first_line  sP    

 $343D3DU3K#L .<0DE2r6   c                     d}|D ]g  }dD ]`  }|j                  d|      }dd|j                  d      z   dz   dd	f}| j                  |      }t        |      \  }}| j	                  |d
       b i y )N)latin-1rU  ziso-latin-1zlatin-1-unixziso-8859-1-unixziso-latin-1-mac-r  rx     #!/usr/bin/python

   # coding: r`  rs  s   print(things)
s   do_something += 4
rU  replacer;   rF  r   r=   	r@   	encodingsr6  repencrE  rlfoundrM  s	            r4   test_latin1_normalizationz,TestDetectEncoding.test_latin1_normalization  s    ;	! 		6H! 6&&sC0/&G)<<uD+/1 &&u-(7(;%~  56		6r6   c                 `    d}| j                  |      }| j                  t        t        |       y )Ns
   print("")rZ  r[  s      r4   test_syntaxerror_latin1z*TestDetectEncoding.test_syntaxerror_latin1  s-     $$U++Ar6   c                     d}|D ]f  }dD ]_  }|j                  d|      }dd|j                  d      z   dz   df}| j                  |      }t        |      \  }}| j	                  |d	       a h y )
N)r:   z	utf-8-macz
utf-8-unixrw  rx  ry  rz  r`  rs  s   1 + 3
r:   r{  r}  s	            r4   test_utf8_normalizationz*TestDetectEncoding.test_utf8_normalization  s    8	! 	1H! 1&&sC0/&G)<<uD#% &&u-(7(;%~  01	1r6   c                 t   | j                  d      }t        |      \  }}| j                  |d       | j                  |dg       t        | j                  d            \  }}| j                  |d       | j                  |g        | j                  d      }t        |      \  }}| j                  |d       | j                  |dg       | j                  d      }t        |      \  }}| j                  |d       | j                  |g        | j                  d      }| j                  t        t        |       y )	N)rI  r:   rI  rH   )s   ﻿print(something)
rP  )s   ﻿)s   # coding: bad
)rF  r   r=   rN   r(  r@   r<   r6  rM  s       r4   test_short_filesz#TestDetectEncoding.test_short_files  s'   $$%=>#28#< .7+*?)@A#243D3DR3H#I .7+,$$%IJ#28#< .;/*?)@A$$%78#28#< .;/,$$%:;+Ar6   c                     | j                  d      }t        |      \  }}| j                  |d       | j                  |dg       y )N)   print("#coding=fake")r:   r  )rF  r   r=   r  s       r4   test_false_encodingz&TestDetectEncoding.test_false_encoding  sF    $$%@A#28#< .7+*B)CDr6   c                    t         j                  dz   }| j                  t         j                  |       dD ]  }t	        |d|      5 }t        d|z  |       t        d|       d d d        t        |      5 }| j                  |j                  |       | j                  |j                  d       d d d         t	        |dd	      5 }t        d|       d d d        t        |      5 }| j                  |j                  d	       | j                  |j                  d       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   oxY w# 1 sw Y   y xY w)
Nz.py)ziso-8859-15r:   wr6  z# coding: %sfileu   print('euro:€')rrP  )
r   TESTFN
addCleanupunlinkr   printtokenize_openr=   r6  mode)r@   ra   r6  fps       r4   	test_openzTestDetectEncoding.test_open  s>   ##e+	(((3 1 	/Hhh7 72nx/b9,267 x( /B  h7  #./ /		/ (C+6 	3"(r2	38$ 	+R[[+6RWWc*	+ 	+7 7/ /
	3 	3	+ 	+s0   D,89D8	E*9E,D5	8E	EEc                 j   d}d G fdd      }| j                  t              5   ||      }|`t        |j                         d d d        | j                  t        dj                  |            5   ||      }t        |j                         d d d        y # 1 sw Y   VxY w# 1 sw Y   y xY w)Nsome_file_pathr  c                       e Zd Zd Z fdZy);TestDetectEncoding.test_filename_in_exception.<locals>.Bunkc                 .    || _         || _        d| _        y )Nr   )name_lines_index)r@   rE  r  s      r4   __init__zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__  s     	#r6   c                     | j                   t              k(  rt        | j                      }| xj                   dz  c_         |S rB  )r  r'   rC  )r@   r3   rE  s     r4   r<   zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readline  s:    ;;#e*,''T[[)q r6   N)r   r   r   r  r<   )rE  s   r4   Bunkr    s     
r6   r  z.*{})rN   r(  r  r   r<   r]   format)r@   r  r  insrE  s       @r4   test_filename_in_exceptionz-TestDetectEncoding.test_filename_in_exception  s    	 	 {+ 	*ud#CCLL)		*
 ##Kt1DE 	*ud#CCLL)	* 	*	* 	*
	* 	*s   !B5B)B&)B2c                     t        d      }t        j                  d|      5  | j                  t        t
        d       d d d        | j                  |j                         y # 1 sw Y   %xY w)Ns   #coding:xxxztokenize._builtin_open)return_valuefoobar)r   r   patchrN   r(  r  
assertTrueclosed)r@   ms     r4   test_open_errorz"TestDetectEncoding.test_open_error  sV    N#ZZ0qA 	Dk=(C	D!	D 	Ds   A##A,N)r   r   r   rF  rN  rR  rV  rX  r\  rb  rf  rh  rm  rq  rt  r  r  r  r  r  r  r  r  rH   r6   r4   r?  r?  "  so    	:	DK@B
3
DB	3	3	36B1B.E+(*8"r6   r?  c                   <    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
y	)
TestTokenizec                 F  	
 dd l }d	d 
	fd}
fd}dfd}|j                  }|j                  }||_        ||_        	 t        |      }| j                  t	        |      dd  g d       ||_        ||_        | j                  
	       y # ||_        ||_        w xY w)Nr   r:   c                     ddgfS )N   first   secondrH   )r<   r6  s    r4   mock_detect_encodingz8TestTokenize.test_tokenize.<locals>.mock_detect_encoding&  s    h	222r6   c                 l    |g }	 	  |        }|r|j                  |       |S # t         $ r |cY S w xY wr   )rC  r)   )r<   r6  kwargsout	next_lineencoding_useds        r4   mock__tokenizez2TestTokenize.test_tokenize.<locals>.mock__tokenize)  sK    $MC (
I JJy)
 % Js   % 33c                  L     dz    dk(  ryt               j                         S )NrM      r6   )strr;   )counters   r4   mock_readlinez1TestTokenize.test_tokenize.<locals>.mock_readline8  s)    qLG!|w<&&((r6   rM   )r  r     1   2   3   4)r   r   r   r=   rP   )r@   tokenize_moduler  r  r  orig_detect_encodingorig_c_tokenresultsr  r6  r  s           @@@r4   test_tokenizezTestTokenize.test_tokenize"  s    *	3	 	)  />>&HH*>'<J9	M}-GT']12.JL /CO+@LO=1 /CO+@LO=s   +B B c           	      X   g }t        d      D ]#  }|j                  dj                  |             % |j                  d       dj                  |      }t	        t        t        |j                  d            j                              }| j                  |d   j                  d       y )Ni  zdef i{i}(): return {i})iOK
r:   )ranger)   r  r  rP   r   r   r;   r<   r=   string)r@   bufr  tokss       r4   test_oneline_defszTestTokenize.test_oneline_defsM  s    s 	=AJJ/666;<	=

4iin HWSZZ%89BBCDb$/r6   c           	         t        t        t        |j                  d            j                              }t        |      }| j                  t        |      d|z          | j                  t        |d   j                     t        t                  t        |      D ]5  }| j                  t        ||dz      j                     t        ||             7 | j                  t        |d|z      j                     t        t        j                            | j                  t        |d|z      j                     t        t        j                            y )Nr:   rZ   r   rM   rK  )rP   r   r   r;   r<   r'   r=   r   
exact_typer   r  r0   r   r   )r@   opstroptypesrW   num_optypesr  s         r4   assertExactTypeEqualz!TestTokenize.assertExactTypeEqualY  s   hwu||G'<=FFGH'lVa+o6&)"6"67!(+	-{# 	3AXfQUm&>&>?%gaj13	3 	&["9"D"DE!%--0	2&["9"D"DE!%//2	4r6   c                 \   | j                  dt        j                  t        j                         | j                  dt        j                  t        j
                         | j                  dt        j                         | j                  dt        j                         | j                  dt        j                         | j                  dt        j                         | j                  dt        j                         | j                  dt        j                         | j                  d	t        j                         | j                  d
t        j                         | j                  dt        j                         | j                  dt        j                         | j                  dt        j                          | j                  dt        j"                         | j                  dt        j$                         | j                  dt        j&                         | j                  dt        j(                  t        j*                         | j                  dt        j,                         | j                  dt        j.                         | j                  dt        j0                         | j                  dt        j2                         | j                  dt        j4                         | j                  dt        j6                         | j                  dt        j8                         | j                  dt        j:                         | j                  dt        j<                         | j                  dt        j>                         | j                  dt        j@                         | j                  dt        jB                         | j                  dt        jD                         | j                  dt        jF                         | j                  d t        jH                         | j                  d!t        jJ                         | j                  d"t        jL                         | j                  d"t        jL                         | j                  d#t        jN                         | j                  d$t        jP                         | j                  d%t        jR                         | j                  d&t        jT                         | j                  d't        jV                         | j                  d(t        jX                         | j                  d)t        jZ                         | j                  d*t        j\                         | j                  d+t        j^                         | j                  d,t        j`                         | j                  d-tb        t        j<                  td        t        j                  tb        t        j<                  td        t        j,                  tb        t        j<                  td               | j                  d.t        j(                  t        jd                  t        j                  t        jd                  t        j                  t        jd                  t        j*                         | j                  d/t        j6                  t        j                  t        jb                  t        j                  t        jd                  t        j                         y )0Nz()z[]:,;+rx  */|&<>=r  %z{}z==z!=z<=z>=~^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->@z@=za**2+b**2==c**2z	{1, 2, 3}z
^(x & 0x1))3r  r0   LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL	LESSEQUALGREATEREQUALTILDE
CIRCUMFLEX	LEFTSHIFT
RIGHTSHIFT
DOUBLESTAR	PLUSEQUALMINEQUAL	STAREQUAL
SLASHEQUALPERCENTEQUAL
AMPEREQUAL	VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUALDOUBLESLASHDOUBLESLASHEQUAL
COLONEQUALELLIPSISRARROWATATEQUALr   r   rv   s    r4   test_exact_typezTestTokenize.test_exact_typeg  sz   !!$

EJJ?!!$

EJJ?!!#u{{3!!#u{{3!!#uzz2!!#uzz2!!#u{{3!!#uzz2!!#u{{3!!#uzz2!!#u{{3!!#uzz2!!#u}}5!!#u{{3!!#uyy1!!#u}}5!!$ellC!!$6!!$7!!$8!!$(:(:;!!#u{{3!!#u'7'78!!$8!!$(8(89!!$(8(89!!$8!!$7!!$8!!$(8(89!!$(:(:;!!$(8(89!!$8!!$(=(=>!!$(=(=>!!%)=)=>!!%)>)>?!!%)>)>?!!$(9(9:!!%)?)?@!!$(8(89!!%8!!$5!!#uxx0!!$6!!"3"&(8(8&"'**"&(8(8&"'--"&(8(8&	B 	!!+"',,"',,"',,"',,"',,	0 	!!,"'"2"2"'**"'**ekk5<<"'**		.r6   c                 D    | j                  dt        j                         y )Nz@          )r  r0   r  rv   s    r4   %test_pathological_trailing_whitespacez2TestTokenize.test_pathological_trailing_whitespace  s    !!-:r6   c                    d}t        t        j                  dddd      t        t        j                  dddd	      t        t        j                  d
ddd	      t        t        j
                  dddd	      t        t        j                  dddd	      t        t        j                  dddd      t        t        j                  dddd      t        t        j                  dddd      t        t        j                  dddd      g	}t        t        t        |j                  d            j                              }| j                  ||       y )Nzb = 1

#testr:   r   r    r/   r  r1   r2   r3   br8  rM   rM   zb = 1
r  rM   rK  rM   rZ   1rM      rM   r  r  rM      rK  r   rK  rM   z#testrZ   r   )rZ   r  )rZ   r-  )r*  r   )r   r0   r   r   r	   r   r   NLCOMMENTr   rP   r   r   r;   r<   r=   r@   sourcer<  rW   s       r4   5test_comment_at_the_end_of_the_source_without_newlinezBTestTokenize.test_comment_at_the_end_of_the_source_without_newline  s    !5>>'VZ\]5::cVR[\588Cv6PYZ5<<6vT]^5==VV_`588DFQUV5==FY`a588Bf&wW5??2VVXY

 hwv}}W'=>GGHI1r6   c                    d}t        t        j                  dddd      t        t        j                  dddd	      t        t        j                  d
ddd	      t        t        j
                  dddd      t        t        j                  dddd      g}t        t        t        |j                  d            j                              }| j                  ||       y )Nza
 r:   r!  r"  )r  r1   r2   r3   ar8  r%  za
r  r&  r/  )rK  rK  r%   r0  )r   r0   r   r   r   r1  r   rP   r   r   r;   r<   r=   r3  s       r4   ?test_newline_and_space_at_the_end_of_the_source_without_newlinezLTestTokenize.test_newline_and_space_at_the_end_of_the_source_without_newline  s    ennWFUWXejjFUSemmDFQVWehhrV#NeoobFQST
 hwv}}W'=>GGHI1r6   c                 R   d}t        j                         5 }t        j                  j	                  |d      }t        |d      5 }|j                  |       d d d        t        |      \  }}| j                  d|j                         d d d        y # 1 sw Y   <xY w# 1 sw Y   y xY w)Ns   F"""
        """z	script.pywbs   SyntaxError)
r   temp_dirr  r  r  r   writer    assertInerr)r@   scriptr;  ra   r  rsr  s          r4   (test_invalid_character_in_fstring_middlez5TestTokenize.test_invalid_character_in_fstring_middle  s     ! 	2Xww||Hk:Hh% #

6"#(2EBMM."&&1	2 	2# #	2 	2s#   -BB2BB	BB&N)r   r   r   r  r  r  r  r  r5  r8  rA  rH   r6   r4   r  r     s,    )2V	04?.B;2$2
2r6   r  c                       e Zd Zd Zd Zd Zy)UntokenizeTestc                 >   t               }d|_        d|_        | j                  t              5 }|j                  d       d d d        | j                  j                  j                  d   d       | j                  t        |j
                  d       y # 1 sw Y   TxY w)NrK  r'  r   z'start (1,3) precedes previous end (2,2)r/  )	r   prev_rowprev_colrN   
ValueErroradd_whitespacer=   r_   args)r@   ucms      r4   test_bad_input_orderz#UntokenizeTest.test_bad_input_order  s    M

z* 	$bU#	$**1-9	; 	*a&6&6>	$ 	$s   BBc                 6   t               }d|_        d|_        g |_        |j	                  d       | j                  |j                  dg       d|_        |j	                  d       | j                  |j                  g d       t        j                  | d       y )NrM   r.  \
rK  )r*  r*  )rN  z\
\
r#   za
  b
    c
  \
  c
)r   rE  rF  rW   rH  r=   r  r  )r@   rJ  s     r4   test_backslash_continuationz*UntokenizeTest.test_backslash_continuation  s}    M

	 F8,
	 #?@%%d,HIr6   c                    t               }t        df}t        df|g}|j                  |t	        g              | j                  |j                  dg       t               }| j                  |j                  t	        |g            d       t               }| j                  |j                  t	        |            d       | j                  |j                  d       | j                  t        t	        |            d       y )NHellor:   zHello s   Hello )	r   r   r   compatiterr=   rW   r   r6  )r@   rJ  r0   rW   s       r4   test_iter_compatzUntokenizeTest.test_iter_compat  s    MwW%u-	R!H:.MdE7m4h?Md6l3X>W-DL19=r6   N)r   r   r   rL  rO  rT  rH   r6   r4   rC  rC    s    
?J>r6   rC  c                   B    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zy
)r  c                    t        |t              r|j                  d      }n|j                         }t	        |j                  d            j                  }t        t        |            }|D cg c]  }|dd 	 }}t        |      }t	        |j                  d            j                  }t        |      D cg c]  }|dd 	 }	}| j                  |	|       t        |      }
t	        |
j                  d            j                  }t        |      D cg c]  }|dd 	 }}| j                  ||       yc c}w c c}w c c}w )a)  
        Test roundtrip for `untokenize`. `f` is an open file or a string.
        The source code in f is tokenized to both 5- and 2-tuples.
        Both sequences are converted back to source code via
        tokenize.untokenize(), and the latter tokenized again to 2-tuples.
        The test fails if the 3 pair tokenizations do not match.

        When untokenize bugs are fixed, untokenize with 5-tuples should
        reproduce code that does not contain a backslash continuation
        following spaces.  A proper test should test this.
        r:   TkeependsNrK  )
isinstancer  r;   readrS  r(   rQ   rP   r   r   r=   )r@   rC   coder<   tokens5ri   tokens2bytes_from2	readline2tokens2_from2bytes_from5	readline5tokens2_from5s                r4   r  zTestRoundtrip.check_roundtrip  s'    a88G$D668D67@@x)*&-.s3r7.. )///>?HH	,4Y,?@SRa@@0 )///>?HH	,4Y,?@SRa@@0 / A
 As   0D9;D>Ec                    t        |t              r|j                  d      }n|j                         }t	        |j                  d            j                  }t        |      D ]^  }|j                  t        t        hv r| j                  |j                  |j                  |j                  d   |j                  d           ` y )Nr:   TrW  rM   )rY  r  r;   rZ  rS  r(   rQ   r   r/   r   r   r=   r  r3   r1   r2   )r@   rC   r[  r<   ri   s        r4   check_line_extractionz#TestRoundtrip.check_line_extraction#  s    a88G$D668D67@@H% 	MCxxXy11SZZ#))A,
)KL	Mr6   c                    | j                  d       | j                  d       | j                  d       t        j                  dd      }t        |d      5 }| j                  |       d d d        | j                  d       | j                  d	       | j                  d
       | j                  d       | j                  d       | j                  d       | j                  d       | j                  d       | j                  d       | j                  d       | j                  d       | j                  d       | j                  d       | j                  d       | j                  d       | j                  d       | j                  d       | j                  d       | j                  d       g d}|D ]  }| j                  |        y # 1 sw Y   jxY w)Nzif x == 1:
    print(x)
z # This is a comment
# This also
zif x == 1 : 
  print(x)
ztokenize_tests.txtr  )subdirr  zoif x == 1:
    # A comment by itself.
    print(x) # Comment here, too.
    # Another comment.
after_if = True
zPif (x # The comments need to go in the right place
    == 1):
    print('x==1')
zclass Test: # A comment here
  # A comment with weird indent
  after_com = 5
  def x(m): return m*5 # a one liner
  def y(m): # A whitespace after the colon
     return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
    print('Can not import' # comment2
)else:   print('Loaded')
zf'\N{EXCLAMATION MARK}'zf'\\N{SNAKE}'zf'\\N{{SNAKE}}'zf'\N{SNAKE}'zf'\\\N{SNAKE}'zf'\\\\\N{SNAKE}'zf'\\\\\\\N{SNAKE}'z	f'\\N{1}'zf'\\\\N{2}'zf'\\\\\\N{3}'zf'\\\\\\\\N{4}'zf'\\N{{'z
f'\\\\N{{'zf'\\\\\\N{{'zf'\\\\\\\\N{{')z
if 1:
    "foo"
"bar"
z
if 1:
    ("foo"
     "bar")
z
if 1:
    "foo"
    "bar"
)r  r   findfiler   )r@   fnrC   casescases        r4   test_roundtripzTestRoundtrip.test_roundtrip.  s    	 . 	/ - 	. 	 , 	-2>J"d^ 	$q  #	$ 1 	2
 	 3 	4 	 B 	C 	 9 	:
 	78-./0_-./0123\*^,-./0[)]+_-./   	'D  &	'q	$ 	$s   GGc                 &    | j                  d       y )Nzna = (3,4, 
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
r  rv   s    r4   test_continuationzTestRoundtrip.test_continuationz  s     	& 		'r6   c                 H    | j                  d       | j                  d       y )Nz)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0rn  rv   s    r4   rO  z)TestRoundtrip.test_backslash_continuation  s*     - 	. 	 % 	&r6   c                 &    | j                  d       y )Nz'' ''rn  rv   s    r4   test_string_concatenationz'TestRoundtrip.test_string_concatenation  s    W%r6   c                 h   dd l }dd l}t        j                  j	                  t
              xs t        j                  }|j                  t        j                  j                  |j                  |      d            }t        j                  d      s|j                  |d      }|D ]s  }t        j                  dk\  rt        d|       t        |d      5 }| j                  |      5  | j!                  |       | j#                  |       d d d        d d d        u y # 1 sw Y   xY w# 1 sw Y   xY w)	Nr   ztest*.pycpu
   rK  r   r  r  )globrandomr  r  r  r  curdirr  escaper   is_resource_enabledsampleverboser  r   subTestr  re  )r@   rv  rw  tempdir	testfilestestfilerC   s          r4   test_random_fileszTestRoundtrip.test_random_files  s     	''//(+8ryyIIbggll4;;w+?LM	**51i4I! 	2H!#j(+h% 2\\x\0 2((+..q122 2	22 22 2s$   D(&#D	D(D%!D((D1	c                     t        |t              r|j                  d      }t        t	        t        |      j                              j                  d      S )Nr:   )rY  r  r;   r   r   r   r<   r  )r@   r[  s     r4   	roundtripzTestRoundtrip.roundtrip  s?    dC ;;w'D(74=#9#9:;BB7KKr6   c                     d}| j                  |      j                  d      }| j                  |d   |d          | j                  |       y)z
        Ensure that although whitespace might be mutated in a roundtrip,
        the semantic meaning of the indentation remains consistent.
        zif False:
	x=3
	x=3
r  rM   rK  N)r  splitr=   r  )r@   r[  	codeliness      r4   #test_indentation_semantics_retainedz1TestRoundtrip.test_indentation_semantics_retained  sH    
 +NN4(..t4	1y|4T"r6   N)r   r   r   r  re  rl  ro  rO  rr  r  r  r  rH   r6   r4   r  r    s3    1>	MI'X'&&2(L
#r6   r  c                       e Zd Zd Zd Zy)InvalidPythonTestsc                    d}t        t        j                  dddd      t        t        j                  dddd      t        t        j                  ddd	d      t        t        j                  d
d	dd      t        t        j                  dddd      t        t        j
                  dddd      t        t        j                  dddd      g}t        t        t        |      j                              }| j                  ||       y )Nz2sin(x)2r8  r%  r#  sinr)  r   r+  rU   r,  r  r9  r"  )rM      r.  )r   r0   r   r   r	   r   r   rP   r   r   r<   r=   r3  s       r4   test_number_followed_by_namez/InvalidPythonTests.test_number_followed_by_name  s    5<<6vT]^5::e6vT]^588Cv6PYZ5::cVR[\588Cv6PYZ5==6vT]^5??2VVXY
 ohv&6&?&?@A1r6   c           	      ,   d}t        t        j                  dddd      t        t        j                  dddd      t        t        j                  dddd      g}t        t        t        |      j                              }| j                  ||       y )N01234r8  r+  r#  r"  r,  r.  )
r   r0   r   r   r   rP   r   r   r<   r=   r3  s       r4   test_number_starting_with_zeroz1InvalidPythonTests.test_number_starting_with_zero  sy    5<<v6X_`5==6vT[\5??2VVXY
 ohv&6&?&?@A1r6   N)r   r   r   r  r  rH   r6   r4   r  r    s    2 	2r6   r  c                   x    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zy)CTokenizeTestc                    t        |      }| j                  |      5  t        t        |j                        |      }| j                  ||j                         j                                d d d        y # 1 sw Y   y xY w)N)r4  )r   r}  r5   r   r<   r=   r>   r(   r?   s        r4   rD   zCTokenizeTest.check_tokenize  sn     QK\\\# 	E11!**=qF VX__%6%A%A%CD		E 	E 	Es   AA55A>c                    d }t        t        dddd      t        t        dddd      t        t        ddd	d      t        t        d
d	dd      t        t        d
ddd
      g}dD ]V  }| j                  |      5  t        t         ||      j                  d|            }| j                  ||       d d d        X y # 1 sw Y   cxY w)Nc              3   2   K   dj                  |        y w)N1+1)r;   r  s    r4   r<   z-CTokenizeTest.test_encoding.<locals>.readline  s     ,,x((s   r(  r8  r%  r  r#  r  r&  r'  r"  r)  r.  )r:   rv  zutf-16r  T)r7  r6  )
r   r   r	   r   r   r}  rP   r   rQ   r=   )r@   r<   rB   r6  rW   s        r4   test_encodingzCTokenizeTest.test_encoding  s    	) 6#VeT2cV%P6#VeT72VeT9Rv6PRS
 7 	3Hx0 3?X&//!%% 
   23 3	33 3s   <:CC
	c                 H   | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  d	d
       | j                  dd       | j                  dd       | j                  dd       | j                  dd       y )Nrl   z    NUMBER     '0xff'        (1, 0) (1, 4)
    LESSEQUAL  '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    rm   z    NUMBER     '0b10'        (1, 0) (1, 4)
    LESSEQUAL  '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    rn   z    NUMBER     '0o123'       (1, 0) (1, 5)
    LESSEQUAL  '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    ro   z    NUMBER     '1234567'     (1, 0) (1, 7)
    GREATER    '>'           (1, 8) (1, 9)
    TILDE      '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    rp   z    NUMBER     '2134568'     (1, 0) (1, 7)
    NOTEQUAL   '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    rq   ac      LPAR       '('           (1, 0) (1, 1)
    MINUS      '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    MINUS      '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    RPAR       ')'           (1, 10) (1, 11)
    AMPER      '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    rr   z    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    NOTEQUAL   '!='          (1, 11) (1, 13)
    MINUS      '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    rs   z    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    AMPER      '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    rt   z    NUMBER     '0xFF'        (1, 0) (1, 4)
    AMPER      '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    VBAR       '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    ru   rv   s    r4   rw   zCTokenizeTest.test_int  s    M , 		 	M , 		 	, / 		 	- 0 		 	0 3 		 	5 	8 			 	. 1 		 	0 3 		 	0 3 		r6   c                     | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  d	d
       | j                  dd       | j                  dd       y )Nr{   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    r|   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    r}   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    r~   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    r   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    r   z    NAME       'x'           (1, 0) (1, 1)
    PLUS       '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    EQUAL      '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    r   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    ru   rv   s    r4   r   zCTokenizeTest.test_float4  s    M , 		 	M , 		 	M , 		 	M , 		 	K * 		 	O . 		 	N - 		r6   c                    | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  d	d
       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd        | j                  d!d"       | j                  d#d$       | j                  d%d&       | j                  d'd(       | j                  d)d*       | j                  d+d,       | j                  d-d.       | j                  d/d0       | j                  d1d2       | j                  d3d4       | j                  d5d6       | j                  d7d8       | j                  d9d:       | j                  d;d<       | j                  d=d>       | j                  d?d@       y )ANr   a5      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    SEMI       ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    EQUAL      '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    r   a6      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    SEMI       ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    EQUAL      '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    r   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    r   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    r   z    NAME       'y'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    r   a      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    PLUS       '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    PLUS       '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    PLUS       '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    r   a      NAME       'y'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    PLUS       '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    PLUS       '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    PLUS       '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    r   z    STRING     "u'abc'"      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    r   z    STRING     'u"abc"'      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    r   z    STRING     "b'abc'"      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    r   z    STRING     'b"abc"'      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    r   a;      STRING     "br'abc'"     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    r   a;      STRING     'br"abc"'     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    r   a;      STRING     "rb'abc'"     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    r   a;      STRING     'rb"abc"'     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    r   r   r   r   r   r   r   r   r   r   r   r   z"hola\\\r\ndfgf"z9    STRING     '"hola\\\\\\r\\ndfgf"' (1, 0) (1, 16)
    r   r   r   a=      FSTRING_START 'fR"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'a'           (1, 3) (1, 4)
    LBRACE     '{'           (1, 4) (1, 5)
    NAME       'b'           (1, 5) (1, 6)
    RBRACE     '}'           (1, 6) (1, 7)
    FSTRING_MIDDLE 'c'           (1, 7) (1, 8)
    FSTRING_END '"'           (1, 8) (1, 9)
    r   r   r   r   zf"{
a}"z    FSTRING_START 'f"'          (1, 0) (1, 2)
    LBRACE     '{'           (1, 2) (1, 3)
    NAME       'a'           (2, 0) (2, 1)
    RBRACE     '}'           (2, 1) (2, 2)
    FSTRING_END '"'           (2, 2) (2, 3)
    r   r   zf"hola\\\r\ndfgf"z    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE 'hola\\\\\\r\\ndfgf' (1, 2) (1, 16)
    FSTRING_END '"'           (1, 16) (1, 17)
    r   a      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    LBRACE     '{'           (1, 6) (1, 7)
    NAME       'x'           (2, 4) (2, 5)
    COLON      ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n'         (2, 6) (3, 0)
    RBRACE     '}'           (3, 0) (3, 1)
    FSTRING_MIDDLE '__'          (3, 1) (3, 3)
    FSTRING_END "'''"         (3, 3) (3, 6)
    r   a      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    LBRACE     '{'           (1, 6) (1, 7)
    NAME       'x'           (2, 4) (2, 5)
    COLON      ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n    b\n     c\n      d\n' (2, 6) (6, 0)
    RBRACE     '}'           (6, 0) (6, 1)
    FSTRING_MIDDLE '__'          (6, 1) (6, 3)
    FSTRING_END "'''"         (6, 3) (6, 6)
    r   a      FSTRING_START "f'"          (1, 0) (1, 2)
    FSTRING_MIDDLE '__'          (1, 2) (1, 4)
    LBRACE     '{'           (1, 4) (1, 5)
    NAME       'x'           (2, 4) (2, 5)
    COLON      ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'd'           (2, 6) (2, 7)
    RBRACE     '}'           (3, 0) (3, 1)
    FSTRING_MIDDLE '__'          (3, 1) (3, 3)
    FSTRING_END "'"           (3, 3) (3, 4)
    ru   rv   s    r4   r   zCTokenizeTest.test_stringb  s   . 1 		 	1 4 		 	> A 		 	/ 2 		 	/ 2 		 	C 
F 
		 	C 
F 
		 	- 0 		 	- 0 		 	- 0 		 	- 0 		 	C F 		 	C F 		 	C F 		 	C F 		 	- 0 		 	L + 		 	L + 		 	 		 	 		 	 		 	/ 2 		 	H ' 		 	K * 		 	L + 		 	 		 	 			 	 		 	0 3 		 	 

		 	 

		$ 	 

		r6   c                 p    | j                  dd       | j                  dd       | j                  dd       y )Nr   a      NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    LPAR       '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    COMMA      ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    COMMA      ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    EQUAL      '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    COMMA      ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    EQUAL      '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    COMMA      ','           (1, 22) (1, 23)
    STAR       '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    RPAR       ')'           (1, 26) (1, 27)
    COLON      ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    r   a      NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    LPAR       '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    EQUAL      '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    COMMA      ','           (1, 13) (1, 14)
    STAR       '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    COMMA      ','           (1, 17) (1, 18)
    DOUBLESTAR '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    RPAR       ')'           (1, 22) (1, 23)
    COLON      ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    r   a      NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    LPAR       '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    COLON      ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    COMMA      ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    COLON      ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    EQUAL      '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    RPAR       ')'           (1, 24) (1, 25)
    RARROW     '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    COLON      ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    ru   rv   s    r4   r   zCTokenizeTest.test_functionj	  sJ    ? B 		. 	; > 		$ 	D G 		r6   c                 (    | j                  dd       y )Nr   a      NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    LESS       '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    GREATER    '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    EQEQUAL    '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    GREATEREQUAL '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    LESSEQUAL  '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    LESSEQUAL  '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    NOTEQUAL   '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    COLON      ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    ru   rv   s    r4   r   zCTokenizeTest.test_comparison	  r   r6   c                 (    | j                  dd       y )Nr   a$      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    MINUS      '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    MINUS      '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    PLUS       '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    PLUS       '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    PLUS       '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    LSQB       '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    RSQB       ']'           (1, 36) (1, 37)
    ru   rv   s    r4   r   zCTokenizeTest.test_additive	  r   r6   c                 (    | j                  dd       y )Nr   a      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    DOUBLESLASH '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    STAR       '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    SLASH      '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    STAR       '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    PERCENT    '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    AT         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    ru   rv   s    r4   r   z!CTokenizeTest.test_multiplicative	  r   r6   c                 L    | j                  dd       | j                  dd       y )Nr   a      TILDE      '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    CIRCUMFLEX '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    AMPER      '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    VBAR       '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    CIRCUMFLEX '^'           (1, 14) (1, 15)
    MINUS      '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    r   aI      MINUS      '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    STAR       '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    SLASH      '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    PLUS       '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    STAR       '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    DOUBLESLASH '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    MINUS      '-'           (1, 14) (1, 15)
    MINUS      '-'           (1, 16) (1, 17)
    MINUS      '-'           (1, 17) (1, 18)
    MINUS      '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    DOUBLESTAR '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    ru   rv   s    r4   r   zCTokenizeTest.test_unary	  s0    0 3 		 	5 8 		r6   c                 (    | j                  dd       y )Nr   a      NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    COMMA      ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    NAME       'x'           (2, 0) (2, 1)
    EQUAL      '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    DOT        '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    LSQB       '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    RSQB       ']'           (2, 22) (2, 23)
    DOT        '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    LPAR       '('           (2, 28) (2, 29)
    RPAR       ')'           (2, 29) (2, 30)
    ru   rv   s    r4   r   zCTokenizeTest.test_selector 
  r   r6   c                 (    | j                  dd       y Nr   a      AT         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    ''            (1, 13) (1, 13)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    LPAR       '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    COMMA      ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    RPAR       ')'           (2, 11) (2, 12)
    COLON      ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    ru   rv   s    r4   r   zCTokenizeTest.test_method6
  r   r6   c                 (    | j                  dd       y r  ru   rv   s    r4   r   zCTokenizeTest.test_tabsG
  r   r6   c                    | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  d	d
       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  dd        | j                  d!d"       | j                  d#d$       | j                  d%d&       | j                  d'd(       | j                  d)d*       y )+Nr   z    ASYNC      'async'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    r   a8      NAME       'a'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    LPAR       '('           (1, 4) (1, 5)
    ASYNC      'async'       (1, 5) (1, 10)
    EQUAL      '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    r   z    ASYNC      'async'       (1, 0) (1, 5)
    LPAR       '('           (1, 5) (1, 6)
    RPAR       ')'           (1, 6) (1, 7)
    r   a<      NAME       'class'       (1, 0) (1, 5)
    ASYNC      'async'       (1, 6) (1, 11)
    LPAR       '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    RPAR       ')'           (1, 15) (1, 16)
    COLON      ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    r   z    NAME       'class'       (1, 0) (1, 5)
    ASYNC      'async'       (1, 6) (1, 11)
    COLON      ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    r   z    AWAIT      'await'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    r   z    NAME       'foo'         (1, 0) (1, 3)
    DOT        '.'           (1, 3) (1, 4)
    ASYNC      'async'       (1, 4) (1, 9)
    r   a;      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    COLON      ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    r   a<      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    COLON      ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    r   z    ASYNC      'async'       (1, 0) (1, 5)
    DOT        '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    r   z/    ASYNC      'async'       (1, 0) (1, 5)
    r   z    ASYNC      'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    AWAIT      'await'       (3, 0) (3, 5)
    r   z    ASYNC      'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    ELLIPSIS   '...'         (2, 0) (2, 3)
    NEWLINE    ''            (2, 3) (2, 3)
    AWAIT      'await'       (3, 0) (3, 5)
    r   z    ASYNC      'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    AWAIT      'await'       (2, 0) (2, 5)
    r   z    NAME       'foo'         (1, 0) (1, 3)
    DOT        '.'           (1, 3) (1, 4)
    ASYNC      'async'       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    r   a;      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    r   a      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    INDENT     ''            (2, -1) (2, -1)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    LPAR       '('           (2, 9) (2, 10)
    AWAIT      'await'       (2, 10) (2, 15)
    RPAR       ')'           (2, 15) (2, 16)
    COLON      ':'           (2, 16) (2, 17)
    NEWLINE    ''            (2, 17) (2, 17)
    INDENT     ''            (3, -1) (3, -1)
    AWAIT      'await'       (3, 4) (3, 9)
    EQUAL      '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    ''            (3, 13) (3, 13)
    DEDENT     ''            (4, -1) (4, -1)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    COLON      ':'           (4, 6) (4, 7)
    NEWLINE    ''            (4, 7) (4, 7)
    INDENT     ''            (5, -1) (5, -1)
    AWAIT      'await'       (5, 4) (5, 9)
    NEWLINE    ''            (5, 9) (5, 9)
    DEDENT     ''            (6, -1) (6, -1)
    DEDENT     ''            (6, -1) (6, -1)
    ASYNC      'async'       (6, 0) (6, 5)
    PLUSEQUAL  '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    ''            (6, 10) (6, 10)
    r   a      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    INDENT     ''            (2, -1) (2, -1)
    ASYNC      'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    COLON      ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (2, -1) (2, -1)
    r   ah      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    ASYNC      'async'       (1, 14) (1, 19)
    RPAR       ')'           (1, 19) (1, 20)
    COLON      ':'           (1, 20) (1, 21)
    AWAIT      'await'       (1, 22) (1, 27)
    r   a~      NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    LPAR       '('           (1, 5) (1, 6)
    RPAR       ')'           (1, 6) (1, 7)
    COLON      ':'           (1, 7) (1, 8)
    NEWLINE    ''            (1, 8) (1, 8)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    LPAR       '('           (3, 9) (3, 10)
    RPAR       ')'           (3, 10) (3, 11)
    COLON      ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    ''            (3, 17) (3, 17)
    ASYNC      'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    LPAR       '('           (4, 15) (4, 16)
    RPAR       ')'           (4, 16) (4, 17)
    COLON      ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    ''            (4, 23) (4, 23)
    AWAIT      'await'       (6, 2) (6, 7)
    EQUAL      '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
    r   a      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    LPAR       '('           (1, 11) (1, 12)
    RPAR       ')'           (1, 12) (1, 13)
    COLON      ':'           (1, 13) (1, 14)
    NEWLINE    ''            (1, 14) (1, 14)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    LPAR       '('           (3, 9) (3, 10)
    RPAR       ')'           (3, 10) (3, 11)
    COLON      ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    ''            (3, 17) (3, 17)
    ASYNC      'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    LPAR       '('           (4, 15) (4, 16)
    RPAR       ')'           (4, 16) (4, 17)
    COLON      ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    ''            (4, 23) (4, 23)
    AWAIT      'await'       (6, 2) (6, 7)
    EQUAL      '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
    ru   rv   s    r4   r   zCTokenizeTest.test_asyncX
  s   K * 		 	- 0 		 	I ( 		 	3 6 		 	. 1 		 	K * 		 	K * 		 	4 7 		 	5 8 		 	K * 		 	G & 		 	4 7 		 	/ 2 		 	N - 		 	O . 		 	3 6 		 	 #*		X 	H K 		& 	9 	< 			 	 !		F 	 "		r6   c                 (    | j                  dd       y )Nr   u7      NAME       'Örter'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    ''            (1, 17) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    EQUAL      '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    ru   rv   s    r4   r   zCTokenizeTest.test_unicodeb  r   r6   c                     d }dD ]4  }| j                  |      5  | j                  t        ||       d d d        6 y # 1 sw Y   AxY w)Nc                 T    t        |       }t        t        |j                              S r   )r   rP   r   r<   )r  
the_strings     r4   
get_tokensz5CTokenizeTest.test_invalid_syntax.<locals>.get_tokenso  s#    !&)J9*:M:MNOOr6   )z(1+2]z(1+2}z{1+2]1_1.2_1e2_1e+    u   €0b120b1_20b20b1_0b0o180o1_80o80o1_0o0x1_0xr  012r  r  r  z'sdfsdfz'''sdfsdf''a  ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((a))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))])rk  )r}  rN   r   )r@   r  rk  s      r4   test_invalid_syntaxz!CTokenizeTest.test_invalid_syntaxn  s_    	P 
 "	@DB 4( @!!*j$?@ @C"	@B@ @s	   >A	c                    d}d } ||dz
        }t        |      t        t        j                              }| j	                  |d   j
                  t               | j	                  |d   j
                  t               t        |dd        ||      }t        |      | j                  t        fd       | j                  t        t        |dd       y )	Nd   c                 ^    dj                  d t        |       D              }|d| z  dz   z  }|S )Nr"  c              3   ,   K   | ]  }d |z  dz     yw)  z	if True:
NrH   ).0rU   s     r4   	<genexpr>zICTokenizeTest.test_max_indent.<locals>.generate_source.<locals>.<genexpr>  s     O1dQh,6Os   r  zpass
)r  r  )indentsr4  s     r4   generate_sourcez6CTokenizeTest.test_max_indent.<locals>.generate_source  s2    WWOgOOFdWnx//FMr6   rM   rV   r"   r[   execc                  @    t        t         j                              S r   )rP   r   r<   )	the_inputs   r4   <lambda>z/CTokenizeTest.test_max_indent.<locals>.<lambda>  s    D9Z[d[m[m9n4o r6   )r   rP   r   r<   r=   r/   r   r   compilerN   r^   )r@   	MAXINDENTr  validrW   invalidr  s         @r4   test_max_indentzCTokenizeTest.test_max_indent  s    		
  	A.UO	7	8J8JKL&1)4z6*!),W%	*,opgw
F	
r6   c                    d }t        d      }| j                  |d       t        d      }| j                   ||       ||             t        d      }| j                  |d       t        d      }| j                   ||       ||             t        d      }| j                  |d	       t        d
      }| j                   ||       ||             y )Nc                 ~    t        |       }t        |j                        D  cg c]	  ^}} }|| f c}} }S c c}} }w r   )r   r   r<   )r  r  kindr  s       r4   r  zECTokenizeTest.test_continuation_lines_indentation.<locals>.get_tokens  sH    !&)J89L9LMO O'9fqT6N O O Os   8z
            def fib(n):
                \
            '''Print a Fibonacci series up to n.'''
                \
            a, b = 0, 1
        ae      NAME       'def'         (2, 0) (2, 3)
    NAME       'fib'         (2, 4) (2, 7)
    LPAR       '('           (2, 7) (2, 8)
    NAME       'n'           (2, 8) (2, 9)
    RPAR       ')'           (2, 9) (2, 10)
    COLON      ':'           (2, 10) (2, 11)
    NEWLINE    ''            (2, 11) (2, 11)
    INDENT     ''            (4, -1) (4, -1)
    STRING     "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39)
    NEWLINE    ''            (4, 39) (4, 39)
    NAME       'a'           (6, 0) (6, 1)
    COMMA      ','           (6, 1) (6, 2)
    NAME       'b'           (6, 3) (6, 4)
    EQUAL      '='           (6, 5) (6, 6)
    NUMBER     '0'           (6, 7) (6, 8)
    COMMA      ','           (6, 8) (6, 9)
    NUMBER     '1'           (6, 10) (6, 11)
    NEWLINE    ''            (6, 11) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
        zu
            def fib(n):
                '''Print a Fibonacci series up to n.'''
                a, b = 0, 1
        z>
            pass
                \

            pass
        z    NAME       'pass'        (2, 0) (2, 4)
    NEWLINE    ''            (2, 4) (2, 4)
    NAME       'pass'        (5, 0) (5, 4)
    NEWLINE    ''            (5, 4) (5, 4)
        z+
            pass
            pass
        z
            if x:
                y = 1
                \
                        \
                    \
                \
                foo = 1
        aj      NAME       'if'          (2, 0) (2, 2)
    NAME       'x'           (2, 3) (2, 4)
    COLON      ':'           (2, 4) (2, 5)
    NEWLINE    ''            (2, 5) (2, 5)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'y'           (3, 4) (3, 5)
    EQUAL      '='           (3, 6) (3, 7)
    NUMBER     '1'           (3, 8) (3, 9)
    NEWLINE    ''            (3, 9) (3, 9)
    NAME       'foo'         (8, 4) (8, 7)
    EQUAL      '='           (8, 8) (8, 9)
    NUMBER     '1'           (8, 10) (8, 11)
    NEWLINE    ''            (8, 11) (8, 11)
    DEDENT     ''            (8, -1) (8, -1)
        zI
            if x:
                y = 1
                foo = 1
        )r   rD   r=   )r@   r  r[  code_no_conts       r4   #test_continuation_lines_indentationz1CTokenizeTest.test_continuation_lines_indentation  s   	O
    	D # 	,    	D):l+CD   	D # 	   
 	D):l+CD   	D # 	"    	D):l+CDr6   N)r   r   r   rD   r  rw   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  rH   r6   r4   r  r    sj    E3(?	B,	\F	P=	~$	L	.	($	L	,	"	"H	T
	'@R
,bEr6   r  c                       e Zd Zd Zy)CTokenizerBufferTestsc                     dddz   dddz   }t        j                         5 }t        |d|      }t        |       d d d        y # 1 sw Y   y xY w)Nz"        #coding: latin-1
        #r7  i'  z

        #i'  foo)r   r;  r   r   )r@   test_scriptr;  	file_names       r4   !test_newline_at_the_end_of_bufferz7CTokenizerBufferTests.test_newline_at_the_end_of_buffer  s`    

e) 

e) ! 	'X#He[AII&	' 	' 	's   AAN)r   r   r   r  rH   r6   r4   r  r    s    
'r6   r  __main__)8testr   test.supportr   r   r   r   r   r	   r
   r   r   r   r   r   r  r   r   r   r   r   r   r   ior   r   unittesttextwrapr   r   r   test.test_grammarr   r   test.support.script_helperr   r   r    r  r0   r5   r8   r   r  r  r  r2  r?  r  rC  r  r  r  r  r   mainrH   r6   r4   <module>r     s    "! ! ! ! !
 !   #< " Y Y 	  N	8 N	`$A A6Gx G.#IH #IL<H <,{" {"|v28 v2r&>X &>Rs#H s#l2 28{EH {E|!'H-- ' zHMMO r6   