Skip to content

C
cpython
Commit b8d37359 authored by Brett Cannon's avatar Brett Cannon
Browse files
Options

Move test_tokenize to doctest. 

Done as GHOP 238 by Josip Dzolonga.
parent 66865d2e
Hide whitespace changes
Inline Side-by-side
Showing with 501 additions and 845 deletions
Lib/test/output/test_tokenize deleted 100644 → 0
View file @ 66865d2e
test_tokenize
1,0-1,34: COMMENT "# Tests for the 'tokenize' module."
1,34-1,35: NL '\n'
2,0-2,42: COMMENT '# Large bits stolen from test_grammar.py. '
2,42-2,43: NL '\n'
3,0-3,1: NL '\n'
4,0-4,10: COMMENT '# Comments'
4,10-4,11: NL '\n'
5,0-5,3: STRING '"#"'
5,3-5,4: NEWLINE '\n'
6,0-6,2: COMMENT "#'"
6,2-6,3: NL '\n'
7,0-7,2: COMMENT '#"'
7,2-7,3: NL '\n'
8,0-8,2: COMMENT '#\\'
8,2-8,3: NL '\n'
9,7-9,8: COMMENT '#'
9,8-9,9: NL '\n'
10,4-10,9: COMMENT '# abc'
10,9-10,10: NL '\n'
11,0-12,4: STRING "'''#\n#'''"
12,4-12,5: NEWLINE '\n'
13,0-13,1: NL '\n'
14,0-14,1: NAME 'x'
14,2-14,3: OP '='
14,4-14,5: NUMBER '1'
14,7-14,8: COMMENT '#'
14,8-14,9: NEWLINE '\n'
15,0-15,1: NL '\n'
16,0-16,24: COMMENT '# Balancing continuation'
16,24-16,25: NL '\n'
17,0-17,1: NL '\n'
18,0-18,1: NAME 'a'
18,2-18,3: OP '='
18,4-18,5: OP '('
18,5-18,6: NUMBER '3'
18,6-18,7: OP ','
18,8-18,9: NUMBER '4'
18,9-18,10: OP ','
18,10-18,11: NL '\n'
19,2-19,3: NUMBER '5'
19,3-19,4: OP ','
19,5-19,6: NUMBER '6'
19,6-19,7: OP ')'
19,7-19,8: NEWLINE '\n'
20,0-20,1: NAME 'y'
20,2-20,3: OP '='
20,4-20,5: OP '['
20,5-20,6: NUMBER '3'
20,6-20,7: OP ','
20,8-20,9: NUMBER '4'
20,9-20,10: OP ','
20,10-20,11: NL '\n'
21,2-21,3: NUMBER '5'
21,3-21,4: OP ']'
21,4-21,5: NEWLINE '\n'
22,0-22,1: NAME 'z'
22,2-22,3: OP '='
22,4-22,5: OP '{'
22,5-22,8: STRING "'a'"
22,8-22,9: OP ':'
22,9-22,10: NUMBER '5'
22,10-22,11: OP ','
22,11-22,12: NL '\n'
23,2-23,5: STRING "'b'"
23,5-23,6: OP ':'
23,6-23,7: NUMBER '6'
23,7-23,8: OP '}'
23,8-23,9: NEWLINE '\n'
24,0-24,1: NAME 'x'
24,2-24,3: OP '='
24,4-24,5: OP '('
24,5-24,8: NAME 'len'
24,8-24,9: OP '('
24,9-24,10: OP '`'
24,10-24,11: NAME 'y'
24,11-24,12: OP '`'
24,12-24,13: OP ')'
24,14-24,15: OP '+'
24,16-24,17: NUMBER '5'
24,17-24,18: OP '*'
24,18-24,19: NAME 'x'
24,20-24,21: OP '-'
24,22-24,23: NAME 'a'
24,23-24,24: OP '['
24,24-24,25: NL '\n'
25,3-25,4: NUMBER '3'
25,5-25,6: OP ']'
25,6-25,7: NL '\n'
26,3-26,4: OP '-'
26,5-26,6: NAME 'x'
26,7-26,8: OP '+'
26,9-26,12: NAME 'len'
26,12-26,13: OP '('
26,13-26,14: OP '{'
26,14-26,15: NL '\n'
27,3-27,4: OP '}'
27,4-27,5: NL '\n'
28,4-28,5: OP ')'
28,5-28,6: NL '\n'
29,2-29,3: OP ')'
29,3-29,4: NEWLINE '\n'
30,0-30,1: NL '\n'
31,0-31,36: COMMENT '# Backslash means line continuation:'
31,36-31,37: NL '\n'
32,0-32,1: NAME 'x'
32,2-32,3: OP '='
32,4-32,5: NUMBER '1'
33,0-33,1: OP '+'
33,2-33,3: NUMBER '1'
33,3-33,4: NEWLINE '\n'
34,0-34,1: NL '\n'
35,0-35,54: COMMENT '# Backslash does not means continuation in comments :\\'
35,54-35,55: NL '\n'
36,0-36,1: NAME 'x'
36,2-36,3: OP '='
36,4-36,5: NUMBER '0'
36,5-36,6: NEWLINE '\n'
37,0-37,1: NL '\n'
38,0-38,19: COMMENT '# Ordinary integers'
38,19-38,20: NL '\n'
39,0-39,4: NUMBER '0xff'
39,5-39,7: OP '<>'
39,8-39,11: NUMBER '255'
39,11-39,12: NEWLINE '\n'
40,0-40,4: NUMBER '0377'
40,5-40,7: OP '<>'
40,8-40,11: NUMBER '255'
40,11-40,12: NEWLINE '\n'
41,0-41,10: NUMBER '2147483647'
41,13-41,15: OP '!='
41,16-41,28: NUMBER '017777777777'
41,28-41,29: NEWLINE '\n'
42,0-42,1: OP '-'
42,1-42,11: NUMBER '2147483647'
42,11-42,12: OP '-'
42,12-42,13: NUMBER '1'
42,14-42,16: OP '!='
42,17-42,29: NUMBER '020000000000'
42,29-42,30: NEWLINE '\n'
43,0-43,12: NUMBER '037777777777'
43,13-43,15: OP '!='
43,16-43,17: OP '-'
43,17-43,18: NUMBER '1'
43,18-43,19: NEWLINE '\n'
44,0-44,10: NUMBER '0xffffffff'
44,11-44,13: OP '!='
44,14-44,15: OP '-'
44,15-44,16: NUMBER '1'
44,16-44,17: NEWLINE '\n'
45,0-45,1: NL '\n'
46,0-46,15: COMMENT '# Long integers'
46,15-46,16: NL '\n'
47,0-47,1: NAME 'x'
47,2-47,3: OP '='
47,4-47,6: NUMBER '0L'
47,6-47,7: NEWLINE '\n'
48,0-48,1: NAME 'x'
48,2-48,3: OP '='
48,4-48,6: NUMBER '0l'
48,6-48,7: NEWLINE '\n'
49,0-49,1: NAME 'x'
49,2-49,3: OP '='
49,4-49,23: NUMBER '0xffffffffffffffffL'
49,23-49,24: NEWLINE '\n'
50,0-50,1: NAME 'x'
50,2-50,3: OP '='
50,4-50,23: NUMBER '0xffffffffffffffffl'
50,23-50,24: NEWLINE '\n'
51,0-51,1: NAME 'x'
51,2-51,3: OP '='
51,4-51,23: NUMBER '077777777777777777L'
51,23-51,24: NEWLINE '\n'
52,0-52,1: NAME 'x'
52,2-52,3: OP '='
52,4-52,23: NUMBER '077777777777777777l'
52,23-52,24: NEWLINE '\n'
53,0-53,1: NAME 'x'
53,2-53,3: OP '='
53,4-53,35: NUMBER '123456789012345678901234567890L'
53,35-53,36: NEWLINE '\n'
54,0-54,1: NAME 'x'
54,2-54,3: OP '='
54,4-54,35: NUMBER '123456789012345678901234567890l'
54,35-54,36: NEWLINE '\n'
55,0-55,1: NL '\n'
56,0-56,24: COMMENT '# Floating-point numbers'
56,24-56,25: NL '\n'
57,0-57,1: NAME 'x'
57,2-57,3: OP '='
57,4-57,8: NUMBER '3.14'
57,8-57,9: NEWLINE '\n'
58,0-58,1: NAME 'x'
58,2-58,3: OP '='
58,4-58,8: NUMBER '314.'
58,8-58,9: NEWLINE '\n'
59,0-59,1: NAME 'x'
59,2-59,3: OP '='
59,4-59,9: NUMBER '0.314'
59,9-59,10: NEWLINE '\n'
60,0-60,17: COMMENT '# XXX x = 000.314'
60,17-60,18: NL '\n'
61,0-61,1: NAME 'x'
61,2-61,3: OP '='
61,4-61,8: NUMBER '.314'
61,8-61,9: NEWLINE '\n'
62,0-62,1: NAME 'x'
62,2-62,3: OP '='
62,4-62,8: NUMBER '3e14'
62,8-62,9: NEWLINE '\n'
63,0-63,1: NAME 'x'
63,2-63,3: OP '='
63,4-63,8: NUMBER '3E14'
63,8-63,9: NEWLINE '\n'
64,0-64,1: NAME 'x'
64,2-64,3: OP '='
64,4-64,9: NUMBER '3e-14'
64,9-64,10: NEWLINE '\n'
65,0-65,1: NAME 'x'
65,2-65,3: OP '='
65,4-65,9: NUMBER '3e+14'
65,9-65,10: NEWLINE '\n'
66,0-66,1: NAME 'x'
66,2-66,3: OP '='
66,4-66,9: NUMBER '3.e14'
66,9-66,10: NEWLINE '\n'
67,0-67,1: NAME 'x'
67,2-67,3: OP '='
67,4-67,9: NUMBER '.3e14'
67,9-67,10: NEWLINE '\n'
68,0-68,1: NAME 'x'
68,2-68,3: OP '='
68,4-68,9: NUMBER '3.1e4'
68,9-68,10: NEWLINE '\n'
69,0-69,1: NL '\n'
70,0-70,17: COMMENT '# String literals'
70,17-70,18: NL '\n'
71,0-71,1: NAME 'x'
71,2-71,3: OP '='
71,4-71,6: STRING "''"
71,6-71,7: OP ';'
71,8-71,9: NAME 'y'
71,10-71,11: OP '='
71,12-71,14: STRING '""'
71,14-71,15: OP ';'
71,15-71,16: NEWLINE '\n'
72,0-72,1: NAME 'x'
72,2-72,3: OP '='
72,4-72,8: STRING "'\\''"
72,8-72,9: OP ';'
72,10-72,11: NAME 'y'
72,12-72,13: OP '='
72,14-72,17: STRING '"\'"'
72,17-72,18: OP ';'
72,18-72,19: NEWLINE '\n'
73,0-73,1: NAME 'x'
73,2-73,3: OP '='
73,4-73,7: STRING '\'"\''
73,7-73,8: OP ';'
73,9-73,10: NAME 'y'
73,11-73,12: OP '='
73,13-73,17: STRING '"\\""'
73,17-73,18: OP ';'
73,18-73,19: NEWLINE '\n'
74,0-74,1: NAME 'x'
74,2-74,3: OP '='
74,4-74,32: STRING '"doesn\'t \\"shrink\\" does it"'
74,32-74,33: NEWLINE '\n'
75,0-75,1: NAME 'y'
75,2-75,3: OP '='
75,4-75,31: STRING '\'doesn\\\'t "shrink" does it\''
75,31-75,32: NEWLINE '\n'
76,0-76,1: NAME 'x'
76,2-76,3: OP '='
76,4-76,32: STRING '"does \\"shrink\\" doesn\'t it"'
76,32-76,33: NEWLINE '\n'
77,0-77,1: NAME 'y'
77,2-77,3: OP '='
77,4-77,31: STRING '\'does "shrink" doesn\\\'t it\''
77,31-77,32: NEWLINE '\n'
78,0-78,1: NAME 'x'
78,2-78,3: OP '='
78,4-83,3: STRING '"""\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n"""'
83,3-83,4: NEWLINE '\n'
84,0-84,1: NAME 'y'
84,2-84,3: OP '='
84,4-84,63: STRING '\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\''
84,63-84,64: NEWLINE '\n'
85,0-85,1: NAME 'y'
85,2-85,3: OP '='
85,4-90,3: STRING '\'\'\'\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n\'\'\''
90,3-90,4: OP ';'
90,4-90,5: NEWLINE '\n'
91,0-91,1: NAME 'y'
91,2-91,3: OP '='
91,4-96,1: STRING '"\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \'lazy\' dog.\\n\\\n"'
96,1-96,2: OP ';'
96,2-96,3: NEWLINE '\n'
97,0-97,1: NAME 'y'
97,2-97,3: OP '='
97,4-102,1: STRING '\'\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \\\'lazy\\\' dog.\\n\\\n\''
102,1-102,2: OP ';'
102,2-102,3: NEWLINE '\n'
103,0-103,1: NAME 'x'
103,2-103,3: OP '='
103,4-103,9: STRING "r'\\\\'"
103,10-103,11: OP '+'
103,12-103,17: STRING "R'\\\\'"
103,17-103,18: NEWLINE '\n'
104,0-104,1: NAME 'x'
104,2-104,3: OP '='
104,4-104,9: STRING "r'\\''"
104,10-104,11: OP '+'
104,12-104,14: STRING "''"
104,14-104,15: NEWLINE '\n'
105,0-105,1: NAME 'y'
105,2-105,3: OP '='
105,4-107,6: STRING "r'''\nfoo bar \\\\\nbaz'''"
107,7-107,8: OP '+'
107,9-108,6: STRING "R'''\nfoo'''"
108,6-108,7: NEWLINE '\n'
109,0-109,1: NAME 'y'
109,2-109,3: OP '='
109,4-111,3: STRING 'r"""foo\nbar \\\\ baz\n"""'
111,4-111,5: OP '+'
111,6-112,3: STRING "R'''spam\n'''"
112,3-112,4: NEWLINE '\n'
113,0-113,1: NAME 'x'
113,2-113,3: OP '='
113,4-113,10: STRING "u'abc'"
113,11-113,12: OP '+'
113,13-113,19: STRING "U'ABC'"
113,19-113,20: NEWLINE '\n'
114,0-114,1: NAME 'y'
114,2-114,3: OP '='
114,4-114,10: STRING 'u"abc"'
114,11-114,12: OP '+'
114,13-114,19: STRING 'U"ABC"'
114,19-114,20: NEWLINE '\n'
115,0-115,1: NAME 'x'
115,2-115,3: OP '='
115,4-115,11: STRING "ur'abc'"
115,12-115,13: OP '+'
115,14-115,21: STRING "Ur'ABC'"
115,22-115,23: OP '+'
115,24-115,31: STRING "uR'ABC'"
115,32-115,33: OP '+'
115,34-115,41: STRING "UR'ABC'"
115,41-115,42: NEWLINE '\n'
116,0-116,1: NAME 'y'
116,2-116,3: OP '='
116,4-116,11: STRING 'ur"abc"'
116,12-116,13: OP '+'
116,14-116,21: STRING 'Ur"ABC"'
116,22-116,23: OP '+'
116,24-116,31: STRING 'uR"ABC"'
116,32-116,33: OP '+'
116,34-116,41: STRING 'UR"ABC"'
116,41-116,42: NEWLINE '\n'
117,0-117,1: NAME 'x'
117,2-117,3: OP '='
117,4-117,10: STRING "ur'\\\\'"
117,11-117,12: OP '+'
117,13-117,19: STRING "UR'\\\\'"
117,19-117,20: NEWLINE '\n'
118,0-118,1: NAME 'x'
118,2-118,3: OP '='
118,4-118,10: STRING "ur'\\''"
118,11-118,12: OP '+'
118,13-118,15: STRING "''"
118,15-118,16: NEWLINE '\n'
119,0-119,1: NAME 'y'
119,2-119,3: OP '='
119,4-121,6: STRING "ur'''\nfoo bar \\\\\nbaz'''"
121,7-121,8: OP '+'
121,9-122,6: STRING "UR'''\nfoo'''"
122,6-122,7: NEWLINE '\n'
123,0-123,1: NAME 'y'
123,2-123,3: OP '='
123,4-125,3: STRING 'Ur"""foo\nbar \\\\ baz\n"""'
125,4-125,5: OP '+'
125,6-126,3: STRING "uR'''spam\n'''"
126,3-126,4: NEWLINE '\n'
127,0-127,1: NL '\n'
128,0-128,13: COMMENT '# Indentation'
128,13-128,14: NL '\n'
129,0-129,2: NAME 'if'
129,3-129,4: NUMBER '1'
129,4-129,5: OP ':'
129,5-129,6: NEWLINE '\n'
130,0-130,4: INDENT ' '
130,4-130,5: NAME 'x'
130,6-130,7: OP '='
130,8-130,9: NUMBER '2'
130,9-130,10: NEWLINE '\n'
131,0-131,0: DEDENT ''
131,0-131,2: NAME 'if'
131,3-131,4: NUMBER '1'
131,4-131,5: OP ':'
131,5-131,6: NEWLINE '\n'
132,0-132,8: INDENT ' '
132,8-132,9: NAME 'x'
132,10-132,11: OP '='
132,12-132,13: NUMBER '2'
132,13-132,14: NEWLINE '\n'
133,0-133,0: DEDENT ''
133,0-133,2: NAME 'if'
133,3-133,4: NUMBER '1'
133,4-133,5: OP ':'
133,5-133,6: NEWLINE '\n'
134,0-134,4: INDENT ' '
134,4-134,9: NAME 'while'
134,10-134,11: NUMBER '0'
134,11-134,12: OP ':'
134,12-134,13: NEWLINE '\n'
135,0-135,5: INDENT ' '
135,5-135,7: NAME 'if'
135,8-135,9: NUMBER '0'
135,9-135,10: OP ':'
135,10-135,11: NEWLINE '\n'
136,0-136,11: INDENT ' '
136,11-136,12: NAME 'x'
136,13-136,14: OP '='
136,15-136,16: NUMBER '2'
136,16-136,17: NEWLINE '\n'
137,5-137,5: DEDENT ''
137,5-137,6: NAME 'x'
137,7-137,8: OP '='
137,9-137,10: NUMBER '2'
137,10-137,11: NEWLINE '\n'
138,0-138,0: DEDENT ''
138,0-138,0: DEDENT ''
138,0-138,2: NAME 'if'
138,3-138,4: NUMBER '0'
138,4-138,5: OP ':'
138,5-138,6: NEWLINE '\n'
139,0-139,2: INDENT ' '
139,2-139,4: NAME 'if'
139,5-139,6: NUMBER '2'
139,6-139,7: OP ':'
139,7-139,8: NEWLINE '\n'
140,0-140,3: INDENT ' '
140,3-140,8: NAME 'while'
140,9-140,10: NUMBER '0'
140,10-140,11: OP ':'
140,11-140,12: NEWLINE '\n'
141,0-141,8: INDENT ' '
141,8-141,10: NAME 'if'
141,11-141,12: NUMBER '1'
141,12-141,13: OP ':'
141,13-141,14: NEWLINE '\n'
142,0-142,10: INDENT ' '
142,10-142,11: NAME 'x'
142,12-142,13: OP '='
142,14-142,15: NUMBER '2'
142,15-142,16: NEWLINE '\n'
143,0-143,1: NL '\n'
144,0-144,11: COMMENT '# Operators'
144,11-144,12: NL '\n'
145,0-145,1: NL '\n'
146,0-146,0: DEDENT ''
146,0-146,0: DEDENT ''
146,0-146,0: DEDENT ''
146,0-146,0: DEDENT ''
146,0-146,3: NAME 'def'
146,4-146,7: NAME 'd22'
146,7-146,8: OP '('
146,8-146,9: NAME 'a'
146,9-146,10: OP ','
146,11-146,12: NAME 'b'
146,12-146,13: OP ','
146,14-146,15: NAME 'c'
146,15-146,16: OP '='
146,16-146,17: NUMBER '1'
146,17-146,18: OP ','
146,19-146,20: NAME 'd'
146,20-146,21: OP '='
146,21-146,22: NUMBER '2'
146,22-146,23: OP ')'
146,23-146,24: OP ':'
146,25-146,29: NAME 'pass'
146,29-146,30: NEWLINE '\n'
147,0-147,3: NAME 'def'
147,4-147,8: NAME 'd01v'
147,8-147,9: OP '('
147,9-147,10: NAME 'a'
147,10-147,11: OP '='
147,11-147,12: NUMBER '1'
147,12-147,13: OP ','
147,14-147,15: OP '*'
147,15-147,20: NAME 'restt'
147,20-147,21: OP ','
147,22-147,24: OP '**'
147,24-147,29: NAME 'restd'
147,29-147,30: OP ')'
147,30-147,31: OP ':'
147,32-147,36: NAME 'pass'
147,36-147,37: NEWLINE '\n'
148,0-148,1: NL '\n'
149,0-149,1: OP '('
149,1-149,2: NAME 'x'
149,2-149,3: OP ','
149,4-149,5: NAME 'y'
149,5-149,6: OP ')'
149,7-149,9: OP '<>'
149,10-149,11: OP '('
149,11-149,12: OP '{'
149,12-149,15: STRING "'a'"
149,15-149,16: OP ':'
149,16-149,17: NUMBER '1'
149,17-149,18: OP '}'
149,18-149,19: OP ','
149,20-149,21: OP '{'
149,21-149,24: STRING "'b'"
149,24-149,25: OP ':'
149,25-149,26: NUMBER '2'
149,26-149,27: OP '}'
149,27-149,28: OP ')'
149,28-149,29: NEWLINE '\n'
150,0-150,1: NL '\n'
151,0-151,12: COMMENT '# comparison'
151,12-151,13: NL '\n'
152,0-152,2: NAME 'if'
152,3-152,4: NUMBER '1'
152,5-152,6: OP '<'
152,7-152,8: NUMBER '1'
152,9-152,10: OP '>'
152,11-152,12: NUMBER '1'
152,13-152,15: OP '=='
152,16-152,17: NUMBER '1'
152,18-152,20: OP '>='
152,21-152,22: NUMBER '1'
152,23-152,25: OP '<='
152,26-152,27: NUMBER '1'
152,28-152,30: OP '<>'
152,31-152,32: NUMBER '1'
152,33-152,35: OP '!='
152,36-152,37: NUMBER '1'
152,38-152,40: NAME 'in'
152,41-152,42: NUMBER '1'
152,43-152,46: NAME 'not'
152,47-152,49: NAME 'in'
152,50-152,51: NUMBER '1'
152,52-152,54: NAME 'is'
152,55-152,56: NUMBER '1'
152,57-152,59: NAME 'is'
152,60-152,63: NAME 'not'
152,64-152,65: NUMBER '1'
152,65-152,66: OP ':'
152,67-152,71: NAME 'pass'
152,71-152,72: NEWLINE '\n'
153,0-153,1: NL '\n'
154,0-154,8: COMMENT '# binary'
154,8-154,9: NL '\n'
155,0-155,1: NAME 'x'
155,2-155,3: OP '='
155,4-155,5: NUMBER '1'
155,6-155,7: OP '&'
155,8-155,9: NUMBER '1'
155,9-155,10: NEWLINE '\n'
156,0-156,1: NAME 'x'
156,2-156,3: OP '='
156,4-156,5: NUMBER '1'
156,6-156,7: OP '^'
156,8-156,9: NUMBER '1'
156,9-156,10: NEWLINE '\n'
157,0-157,1: NAME 'x'
157,2-157,3: OP '='
157,4-157,5: NUMBER '1'
157,6-157,7: OP '|'
157,8-157,9: NUMBER '1'
157,9-157,10: NEWLINE '\n'
158,0-158,1: NL '\n'
159,0-159,7: COMMENT '# shift'
159,7-159,8: NL '\n'
160,0-160,1: NAME 'x'
160,2-160,3: OP '='
160,4-160,5: NUMBER '1'
160,6-160,8: OP '<<'
160,9-160,10: NUMBER '1'
160,11-160,13: OP '>>'
160,14-160,15: NUMBER '1'
160,15-160,16: NEWLINE '\n'
161,0-161,1: NL '\n'
162,0-162,10: COMMENT '# additive'
162,10-162,11: NL '\n'
163,0-163,1: NAME 'x'
163,2-163,3: OP '='
163,4-163,5: NUMBER '1'
163,6-163,7: OP '-'
163,8-163,9: NUMBER '1'
163,10-163,11: OP '+'
163,12-163,13: NUMBER '1'
163,14-163,15: OP '-'
163,16-163,17: NUMBER '1'
163,18-163,19: OP '+'
163,20-163,21: NUMBER '1'
163,21-163,22: NEWLINE '\n'
164,0-164,1: NL '\n'
165,0-165,16: COMMENT '# multiplicative'
165,16-165,17: NL '\n'
166,0-166,1: NAME 'x'
166,2-166,3: OP '='
166,4-166,5: NUMBER '1'
166,6-166,7: OP '/'
166,8-166,9: NUMBER '1'
166,10-166,11: OP '*'
166,12-166,13: NUMBER '1'
166,14-166,15: OP '%'
166,16-166,17: NUMBER '1'
166,17-166,18: NEWLINE '\n'
167,0-167,1: NL '\n'
168,0-168,7: COMMENT '# unary'
168,7-168,8: NL '\n'
169,0-169,1: NAME 'x'
169,2-169,3: OP '='
169,4-169,5: OP '~'
169,5-169,6: NUMBER '1'
169,7-169,8: OP '^'
169,9-169,10: NUMBER '1'
169,11-169,12: OP '&'
169,13-169,14: NUMBER '1'
169,15-169,16: OP '|'
169,17-169,18: NUMBER '1'
169,19-169,20: OP '&'
169,21-169,22: NUMBER '1'
169,23-169,24: OP '^'
169,25-169,26: OP '-'
169,26-169,27: NUMBER '1'
169,27-169,28: NEWLINE '\n'
170,0-170,1: NAME 'x'
170,2-170,3: OP '='
170,4-170,5: OP '-'
170,5-170,6: NUMBER '1'
170,6-170,7: OP '*'
170,7-170,8: NUMBER '1'
170,8-170,9: OP '/'
170,9-170,10: NUMBER '1'
170,11-170,12: OP '+'
170,13-170,14: NUMBER '1'
170,14-170,15: OP '*'
170,15-170,16: NUMBER '1'
170,17-170,18: OP '-'
170,19-170,20: OP '-'
170,20-170,21: OP '-'
170,21-170,22: OP '-'
170,22-170,23: NUMBER '1'
170,23-170,24: OP '*'
170,24-170,25: NUMBER '1'
170,25-170,26: NEWLINE '\n'
171,0-171,1: NL '\n'
172,0-172,10: COMMENT '# selector'
172,10-172,11: NL '\n'
173,0-173,6: NAME 'import'
173,7-173,10: NAME 'sys'
173,10-173,11: OP ','
173,12-173,16: NAME 'time'
173,16-173,17: NEWLINE '\n'
174,0-174,1: NAME 'x'
174,2-174,3: OP '='
174,4-174,7: NAME 'sys'
174,7-174,8: OP '.'
174,8-174,15: NAME 'modules'
174,15-174,16: OP '['
174,16-174,22: STRING "'time'"
174,22-174,23: OP ']'
174,23-174,24: OP '.'
174,24-174,28: NAME 'time'
174,28-174,29: OP '('
174,29-174,30: OP ')'
174,30-174,31: NEWLINE '\n'
175,0-175,1: NL '\n'
176,0-176,1: OP '@'
176,1-176,13: NAME 'staticmethod'
176,13-176,14: NEWLINE '\n'
177,0-177,3: NAME 'def'
177,4-177,7: NAME 'foo'
177,7-177,8: OP '('
177,8-177,9: OP ')'
177,9-177,10: OP ':'
177,11-177,15: NAME 'pass'
177,15-177,16: NEWLINE '\n'
178,0-178,1: NL '\n'
179,0-179,0: ENDMARKER ''
Lib/test/test_tokenize.py
View file @ b8d37359
"""Tests for the tokenize module.
doctests = """
Tests for the tokenize module.
The tests were originally written in the old Python style, where the
test output was compared to a golden file. This docstring represents
the first steps towards rewriting the entire test as a doctest.
>>> import glob, random, sys
The tests can be really simple. Given a small fragment of source
code, print out a table with the tokens. The ENDMARK is omitted for
The tests can be really simple. Given a small fragment of source
code, print out a table with thokens. The ENDMARK is omitted for
brevity.
>>> dump_tokens("1 + 1")
NUMBER '1' (1, 0) (1, 1)
OP '+' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
A comment generates a token here, unlike in the parser module. The
comment token is followed by an NL or a NEWLINE token, depending on
whether the line contains the completion of a statement.
>>> dump_tokens("if False:\\n"
... " # NL\\n"
... " True = False # NEWLINE\\n")
NAME 'if' (1, 0) (1, 2)
NAME 'False' (1, 3) (1, 8)
OP ':' (1, 8) (1, 9)
NEWLINE '\\n' (1, 9) (1, 10)
COMMENT '# NL' (2, 4) (2, 8)
NL '\\n' (2, 8) (2, 9)
INDENT ' ' (3, 0) (3, 4)
NAME 'True' (3, 4) (3, 8)
OP '=' (3, 9) (3, 10)
NAME 'False' (3, 11) (3, 16)
COMMENT '# NEWLINE' (3, 17) (3, 26)
NEWLINE '\\n' (3, 26) (3, 27)
DEDENT '' (4, 0) (4, 0)
There will be a bunch more tests of specific source patterns.
The tokenize module also defines an untokenize function that should
regenerate the original program text from the tokens.
There are some standard formatting practices that are easy to get right.
>>> roundtrip("if x == 1:\\n"
... " print x\\n")
if x == 1:
print x
>>> dump_tokens("1 + 1")
NUMBER '1' (1, 0) (1, 1)
OP '+' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
>>> dump_tokens("if False:\\n"
... " # NL\\n"
... " True = False # NEWLINE\\n")
NAME 'if' (1, 0) (1, 2)
NAME 'False' (1, 3) (1, 8)
OP ':' (1, 8) (1, 9)
NEWLINE '\\n' (1, 9) (1, 10)
COMMENT '# NL' (2, 4) (2, 8)
NL '\\n' (2, 8) (2, 9)
INDENT ' ' (3, 0) (3, 4)
NAME 'True' (3, 4) (3, 8)
OP '=' (3, 9) (3, 10)
NAME 'False' (3, 11) (3, 16)
COMMENT '# NEWLINE' (3, 17) (3, 26)
NEWLINE '\\n' (3, 26) (3, 27)
DEDENT '' (4, 0) (4, 0)
>>> indent_error_file = \"""
... def k(x):
... x += 2
... x += 5
... \"""
>>> for tok in generate_tokens(StringIO(indent_error_file).readline): pass
Traceback (most recent call last):
...
IndentationError: unindent does not match any outer indentation level
Test roundtrip for `untokenize`. `f` is an open file or a string. The source
code in f is tokenized, converted back to source code via tokenize.untokenize(),
and tokenized again from the latter. The test fails if the second tokenization
doesn't match the first.
>>> def roundtrip(f):
... if isinstance(f, str): f = StringIO(f)
... token_list = list(generate_tokens(f.readline))
... f.close()
... tokens1 = [tok[:2] for tok in token_list]
... new_text = untokenize(tokens1)
... readline = iter(new_text.splitlines(1)).next
... tokens2 = [tok[:2] for tok in generate_tokens(readline)]
... return tokens1 == tokens2
...
There are some standard formattig practises that are easy to get right.
>>> roundtrip("if x == 1:\\n"
... " print x\\n")
True
>>> roundtrip("# This is a comment\\n# This also")
True
Some people use different formatting conventions, which makes
untokenize a little trickier. Note that this test involves trailing
whitespace after the colon. Note that we use hex escapes to make the
two trailing blanks apparent in the expected output.
>>> roundtrip("if x == 1 : \\n"
... " print x\\n")
if x == 1 :\x20\x20
print x
Comments need to go in the right place.
>>> roundtrip("if x == 1:\\n"
... " # A comment by itself.\\n"
... " print x # Comment here, too.\\n"
... " # Another comment.\\n"
... "after_if = True\\n")
if x == 1:
# A comment by itself.
print x # Comment here, too.
# Another comment.
after_if = True
>>> roundtrip("if (x # The comments need to go in the right place\\n"
... " == 1):\\n"
... " print 'x == 1'\\n")
if (x # The comments need to go in the right place
== 1):
print 'x == 1'
untokenize a little trickier. Note that this test involves trailing
whitespace after the colon. Note that we use hex escapes to make the
two trailing blanks apperant in the expected output.
>>> roundtrip("if x == 1 : \\n"
... " print x\\n")
True
>>> f = test_support.findfile("tokenize_tests" + os.extsep + "txt")
>>> roundtrip(open(f))
True
>>> roundtrip("if x == 1:\\n"
... " # A comment by itself.\\n"
... " print x # Comment here, too.\\n"
... " # Another comment.\\n"
... "after_if = True\\n")
True
>>> roundtrip("if (x # The comments need to go in the right place\\n"
... " == 1):\\n"
... " print 'x==1'\\n")
True
>>> roundtrip("class Test: # A comment here\\n"
... " # A comment with weird indent\\n"
... " after_com = 5\\n"
... " def x(m): return m*5 # a one liner\\n"
... " def y(m): # A whitespace after the colon\\n"
... " return y*4 # 3-space indent\\n")
True
Some error-handling code
>>> roundtrip("try: import somemodule\\n"
... "except ImportError: # comment\\n"
... " print 'Can not import' # comment2\\n"
... "else: print 'Loaded'\\n")
True
Balancing contunuation
>>> roundtrip("a = (3,4, \\n"
... "5,6)\\n"
... "y = [3, 4,\\n"
... "5]\\n"
... "z = {'a': 5,\\n"
... "'b':15, 'c':True}\\n"
... "x = len(y) + 5 - a[\\n"
... "3] - a[2]\\n"
... "+ len(z) - z[\\n"
... "'b']\\n")
True
Ordinary integers and binary operators
>>> dump_tokens("0xff <= 255")
NUMBER '0xff' (1, 0) (1, 4)
OP '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
>>> dump_tokens("01234567 > ~0x15")
NUMBER '01234567' (1, 0) (1, 8)
OP '>' (1, 9) (1, 10)
OP '~' (1, 11) (1, 12)
NUMBER '0x15' (1, 12) (1, 16)
>>> dump_tokens("2134568 != 01231515")
NUMBER '2134568' (1, 0) (1, 7)
OP '!=' (1, 8) (1, 10)
NUMBER '01231515' (1, 11) (1, 19)
>>> dump_tokens("(-124561-1) & 0200000000")
OP '(' (1, 0) (1, 1)
OP '-' (1, 1) (1, 2)
NUMBER '124561' (1, 2) (1, 8)
OP '-' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP ')' (1, 10) (1, 11)
OP '&' (1, 12) (1, 13)
NUMBER '0200000000' (1, 14) (1, 24)
>>> dump_tokens("0xdeadbeef != -1")
NUMBER '0xdeadbeef' (1, 0) (1, 10)
OP '!=' (1, 11) (1, 13)
OP '-' (1, 14) (1, 15)
NUMBER '1' (1, 15) (1, 16)
>>> dump_tokens("0xdeadc0de & 012345")
NUMBER '0xdeadc0de' (1, 0) (1, 10)
OP '&' (1, 11) (1, 12)
NUMBER '012345' (1, 13) (1, 19)
>>> dump_tokens("0xFF & 0x15 | 1234")
NUMBER '0xFF' (1, 0) (1, 4)
OP '&' (1, 5) (1, 6)
NUMBER '0x15' (1, 7) (1, 11)
OP '|' (1, 12) (1, 13)
NUMBER '1234' (1, 14) (1, 18)
Long integers
>>> dump_tokens("x = 0L")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '0L' (1, 4) (1, 6)
>>> dump_tokens("x = 0xfffffffffff")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '0xffffffffff (1, 4) (1, 17)
>>> dump_tokens("x = 123141242151251616110l")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '123141242151 (1, 4) (1, 26)
>>> dump_tokens("x = -15921590215012591L")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
OP '-' (1, 4) (1, 5)
NUMBER '159215902150 (1, 5) (1, 23)
Floating point numbers
>>> dump_tokens("x = 3.14159")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3.14159' (1, 4) (1, 11)
>>> dump_tokens("x = 314159.")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '314159.' (1, 4) (1, 11)
>>> dump_tokens("x = .314159")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '.314159' (1, 4) (1, 11)
>>> dump_tokens("x = 3e14159")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3e14159' (1, 4) (1, 11)
>>> dump_tokens("x = 3E123")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3E123' (1, 4) (1, 9)
>>> dump_tokens("x+y = 3e-1230")
NAME 'x' (1, 0) (1, 1)
OP '+' (1, 1) (1, 2)
NAME 'y' (1, 2) (1, 3)
OP '=' (1, 4) (1, 5)
NUMBER '3e-1230' (1, 6) (1, 13)
>>> dump_tokens("x = 3.14e159")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3.14e159' (1, 4) (1, 12)
String literals
>>> dump_tokens("x = ''; y = \\\"\\\"")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "''" (1, 4) (1, 6)
OP ';' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
OP '=' (1, 10) (1, 11)
STRING '""' (1, 12) (1, 14)
>>> dump_tokens("x = '\\\"'; y = \\\"'\\\"")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '\\'"\\'' (1, 4) (1, 7)
OP ';' (1, 7) (1, 8)
NAME 'y' (1, 9) (1, 10)
OP '=' (1, 11) (1, 12)
STRING '"\\'"' (1, 13) (1, 16)
>>> dump_tokens("x = \\\"doesn't \\\"shrink\\\", does it\\\"")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '"doesn\\'t "' (1, 4) (1, 14)
NAME 'shrink' (1, 14) (1, 20)
STRING '", does it"' (1, 20) (1, 31)
>>> dump_tokens("x = u'abc' + U'ABC'")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "u'abc'" (1, 4) (1, 10)
OP '+' (1, 11) (1, 12)
STRING "U'ABC'" (1, 13) (1, 19)
>>> dump_tokens('y = u"ABC" + U"ABC"')
NAME 'y' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING 'u"ABC"' (1, 4) (1, 10)
OP '+' (1, 11) (1, 12)
STRING 'U"ABC"' (1, 13) (1, 19)
>>> dump_tokens("x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC'")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "ur'abc'" (1, 4) (1, 11)
OP '+' (1, 12) (1, 13)
STRING "Ur'ABC'" (1, 14) (1, 21)
OP '+' (1, 22) (1, 23)
STRING "uR'ABC'" (1, 24) (1, 31)
OP '+' (1, 32) (1, 33)
STRING "UR'ABC'" (1, 34) (1, 41)
>>> dump_tokens('y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC"')
NAME 'y' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING 'ur"abc"' (1, 4) (1, 11)
OP '+' (1, 12) (1, 13)
STRING 'Ur"ABC"' (1, 14) (1, 21)
OP '+' (1, 22) (1, 23)
STRING 'uR"ABC"' (1, 24) (1, 31)
OP '+' (1, 32) (1, 33)
STRING 'UR"ABC"' (1, 34) (1, 41)
Operators
>>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")
NAME 'def' (1, 0) (1, 3)
NAME 'd22' (1, 4) (1, 7)
OP '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
OP ',' (1, 9) (1, 10)
NAME 'b' (1, 11) (1, 12)
OP ',' (1, 12) (1, 13)
NAME 'c' (1, 14) (1, 15)
OP '=' (1, 15) (1, 16)
NUMBER '2' (1, 16) (1, 17)
OP ',' (1, 17) (1, 18)
NAME 'd' (1, 19) (1, 20)
OP '=' (1, 20) (1, 21)
NUMBER '2' (1, 21) (1, 22)
OP ',' (1, 22) (1, 23)
OP '*' (1, 24) (1, 25)
NAME 'k' (1, 25) (1, 26)
OP ')' (1, 26) (1, 27)
OP ':' (1, 27) (1, 28)
NAME 'pass' (1, 29) (1, 33)
>>> dump_tokens("def d01v_(a=1, *k, **w): pass")
NAME 'def' (1, 0) (1, 3)
NAME 'd01v_' (1, 4) (1, 9)
OP '(' (1, 9) (1, 10)
NAME 'a' (1, 10) (1, 11)
OP '=' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP ',' (1, 13) (1, 14)
OP '*' (1, 15) (1, 16)
NAME 'k' (1, 16) (1, 17)
OP ',' (1, 17) (1, 18)
OP '**' (1, 19) (1, 21)
NAME 'w' (1, 21) (1, 22)
OP ')' (1, 22) (1, 23)
OP ':' (1, 23) (1, 24)
NAME 'pass' (1, 25) (1, 29)
Comparison
>>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
... "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass")
NAME 'if' (1, 0) (1, 2)
NUMBER '1' (1, 3) (1, 4)
OP '<' (1, 5) (1, 6)
NUMBER '1' (1, 7) (1, 8)
OP '>' (1, 9) (1, 10)
NUMBER '1' (1, 11) (1, 12)
OP '==' (1, 13) (1, 15)
NUMBER '1' (1, 16) (1, 17)
OP '>=' (1, 18) (1, 20)
NUMBER '5' (1, 21) (1, 22)
OP '<=' (1, 23) (1, 25)
NUMBER '0x15' (1, 26) (1, 30)
OP '<=' (1, 31) (1, 33)
NUMBER '0x12' (1, 34) (1, 38)
OP '!=' (1, 39) (1, 41)
NUMBER '1' (1, 42) (1, 43)
NAME 'and' (1, 44) (1, 47)
NUMBER '5' (1, 48) (1, 49)
NAME 'in' (1, 50) (1, 52)
NUMBER '1' (1, 53) (1, 54)
NAME 'not' (1, 55) (1, 58)
NAME 'in' (1, 59) (1, 61)
NUMBER '1' (1, 62) (1, 63)
NAME 'is' (1, 64) (1, 66)
NUMBER '1' (1, 67) (1, 68)
NAME 'or' (1, 69) (1, 71)
NUMBER '5' (1, 72) (1, 73)
NAME 'is' (1, 74) (1, 76)
NAME 'not' (1, 77) (1, 80)
NUMBER '1' (1, 81) (1, 82)
OP ':' (1, 82) (1, 83)
NAME 'pass' (1, 84) (1, 88)
Shift
>>> dump_tokens("x = 1 << 1 >> 5")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '<<' (1, 6) (1, 8)
NUMBER '1' (1, 9) (1, 10)
OP '>>' (1, 11) (1, 13)
NUMBER '5' (1, 14) (1, 15)
Additive
>>> dump_tokens("x = 1 - y + 15 - 01 + 0x124 + z + a[5]")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '-' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
OP '+' (1, 10) (1, 11)
NUMBER '15' (1, 12) (1, 14)
OP '-' (1, 15) (1, 16)
NUMBER '01' (1, 17) (1, 19)
OP '+' (1, 20) (1, 21)
NUMBER '0x124' (1, 22) (1, 27)
OP '+' (1, 28) (1, 29)
NAME 'z' (1, 30) (1, 31)
OP '+' (1, 32) (1, 33)
NAME 'a' (1, 34) (1, 35)
OP '[' (1, 35) (1, 36)
NUMBER '5' (1, 36) (1, 37)
OP ']' (1, 37) (1, 38)
Multiplicative
>>> dump_tokens("x = 1//1*1/5*12%0x12")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '//' (1, 5) (1, 7)
NUMBER '1' (1, 7) (1, 8)
OP '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP '/' (1, 10) (1, 11)
NUMBER '5' (1, 11) (1, 12)
OP '*' (1, 12) (1, 13)
NUMBER '12' (1, 13) (1, 15)
OP '%' (1, 15) (1, 16)
NUMBER '0x12' (1, 16) (1, 20)
Unary
>>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1")
OP '~' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
OP '^' (1, 3) (1, 4)
NUMBER '1' (1, 5) (1, 6)
OP '&' (1, 7) (1, 8)
NUMBER '1' (1, 9) (1, 10)
OP '|' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP '^' (1, 14) (1, 15)
OP '-' (1, 16) (1, 17)
NUMBER '1' (1, 17) (1, 18)
>>> dump_tokens("-1*1/1+1*1//1 - ---1**1")
OP '-' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
OP '*' (1, 2) (1, 3)
NUMBER '1' (1, 3) (1, 4)
OP '/' (1, 4) (1, 5)
NUMBER '1' (1, 5) (1, 6)
OP '+' (1, 6) (1, 7)
NUMBER '1' (1, 7) (1, 8)
OP '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP '//' (1, 10) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP '-' (1, 14) (1, 15)
OP '-' (1, 16) (1, 17)
OP '-' (1, 17) (1, 18)
OP '-' (1, 18) (1, 19)
NUMBER '1' (1, 19) (1, 20)
OP '**' (1, 20) (1, 22)
NUMBER '1' (1, 22) (1, 23)
Selector
>>> dump_tokens("import sys, time\\nx = sys.modules['time'].time()")
NAME 'import' (1, 0) (1, 6)
NAME 'sys' (1, 7) (1, 10)
OP ',' (1, 10) (1, 11)
NAME 'time' (1, 12) (1, 16)
NEWLINE '\\n' (1, 16) (1, 17)
NAME 'x' (2, 0) (2, 1)
OP '=' (2, 2) (2, 3)
NAME 'sys' (2, 4) (2, 7)
OP '.' (2, 7) (2, 8)
NAME 'modules' (2, 8) (2, 15)
OP '[' (2, 15) (2, 16)
STRING "'time'" (2, 16) (2, 22)
OP ']' (2, 22) (2, 23)
OP '.' (2, 23) (2, 24)
NAME 'time' (2, 24) (2, 28)
OP '(' (2, 28) (2, 29)
OP ')' (2, 29) (2, 30)
Methods
>>> dump_tokens("@staticmethod\\ndef foo(x,y): pass")
OP '@' (1, 0) (1, 1)
NAME 'staticmethod (1, 1) (1, 13)
NEWLINE '\\n' (1, 13) (1, 14)
NAME 'def' (2, 0) (2, 3)
NAME 'foo' (2, 4) (2, 7)
OP '(' (2, 7) (2, 8)
NAME 'x' (2, 8) (2, 9)
OP ',' (2, 9) (2, 10)
NAME 'y' (2, 10) (2, 11)
OP ')' (2, 11) (2, 12)
OP ':' (2, 12) (2, 13)
NAME 'pass' (2, 14) (2, 18)
Backslash means line continuation, except for comments
>>> roundtrip("x=1+\\\\n"
... "1\\n"
... "# This is a comment\\\\n"
... "# This also\\n")
True
>>> roundtrip("# Comment \\\\nx = 0")
True
>>>
>>> tempdir = os.path.dirname(f) or os.curdir
>>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
>>> if not test_support.is_resource_enabled("compiler"):
... testfiles = random.sample(testfiles, 10)
...
>>> for testfile in testfiles:
... if not roundtrip(open(testfile)): break
... else: True
True
"""
import os, glob, random, time, sys
from cStringIO import StringIO
from test.test_support import (verbose, findfile, is_resource_enabled,
TestFailed)
from tokenize import (tokenize, generate_tokens, untokenize, tok_name,
ENDMARKER, NUMBER, NAME, OP, STRING, COMMENT)
# How much time in seconds can pass before we print a 'Still working' message.
_PRINT_WORKING_MSG_INTERVAL = 5 * 60
# Test roundtrip for `untokenize`. `f` is a file path. The source code in f
# is tokenized, converted back to source code via tokenize.untokenize(),
# and tokenized again from the latter. The test fails if the second
# tokenization doesn't match the first.
def test_roundtrip(f):
## print 'Testing:', f
fobj = open(f)
try:
fulltok = list(generate_tokens(fobj.readline))
finally:
fobj.close()
t1 = [tok[:2] for tok in fulltok]
newtext = untokenize(t1)
readline = iter(newtext.splitlines(1)).next
t2 = [tok[:2] for tok in generate_tokens(readline)]
if t1 != t2:
raise TestFailed("untokenize() roundtrip failed for %r" % f)
from test import test_support
from tokenize import (tokenize, untokenize, generate_tokens, NUMBER, NAME, OP,
STRING, ENDMARKER, tok_name)
from StringIO import StringIO
import os
def dump_tokens(s):
"""Print out the tokens in s in a table format.
......@@ -118,12 +508,7 @@ def dump_tokens(s):
if type == ENDMARKER:
break
type = tok_name[type]
print "%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals()
def roundtrip(s):
f = StringIO(s)
source = untokenize(generate_tokens(f.readline))
print source,
print("%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals())
# This is an example from the docs, set up as a doctest.
def decistmt(s):
......@@ -163,61 +548,13 @@ def decistmt(s):
result.append((toknum, tokval))
return untokenize(result)
def test_main():
if verbose:
print 'starting...'
next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
# This displays the tokenization of tokenize_tests.py to stdout, and
# regrtest.py checks that this equals the expected output (in the
# test/output/ directory).
f = open(findfile('tokenize_tests' + os.extsep + 'txt'))
tokenize(f.readline)
f.close()
# Now run test_roundtrip() over tokenize_test.py too, and over all
# (if the "compiler" resource is enabled) or a small random sample (if
# "compiler" is not enabled) of the test*.py files.
f = findfile('tokenize_tests' + os.extsep + 'txt')
test_roundtrip(f)
testdir = os.path.dirname(f) or os.curdir
testfiles = glob.glob(testdir + os.sep + 'test*.py')
if not is_resource_enabled('compiler'):
testfiles = random.sample(testfiles, 10)
for f in testfiles:
# Print still working message since this test can be really slow
if next_time <= time.time():
next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
print >>sys.__stdout__, ' test_main still working, be patient...'
sys.__stdout__.flush()
test_roundtrip(f)
# Test detecton of IndentationError.
sampleBadText = """\
def foo():
bar
baz
"""
try:
for tok in generate_tokens(StringIO(sampleBadText).readline):
pass
except IndentationError:
pass
else:
raise TestFailed("Did not detect IndentationError:")
# Run the doctests in this module.
from test import test_tokenize # i.e., this module
from test.test_support import run_doctest
run_doctest(test_tokenize, verbose)
if verbose:
print 'finished'
__test__ = {"doctests" : doctests, 'decistmt': decistmt}
def test_main():
from test import test_tokenize
test_support.run_doctest(test_tokenize, True)
if __name__ == "__main__":
test_main()
Misc/ACKS
View file @ b8d37359
......@@ -178,6 +178,7 @@ Luke Dunstan
Andy Dustman
Gary Duzan
Eugene Dvurechenski
Josip Dzolonga
Maxim Dzumanenko
Hans Eckardt
Grant Edwards
......
Misc/NEWS
View file @ b8d37359
......@@ -50,6 +50,8 @@ Library
Tests
-----
- GHOP 238: Convert test_tokenize to use doctest.
- GHOP 237: Rewrite test_thread using unittest.
- Patch #2232: os.tmpfile might fail on Windows if the user has no
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7