Skip to content

C
cpython
Commit b8d37359 authored by Brett Cannon's avatar Brett Cannon
Browse files
Options

Move test_tokenize to doctest. 

Done as GHOP 238 by Josip Dzolonga.
parent 66865d2e
Show whitespace changes
Inline Side-by-side
Showing with 501 additions and 845 deletions
Lib/test/output/test_tokenize deleted 100644 → 0
View file @ 66865d2e
test_tokenize
1,0-1,34: COMMENT "# Tests for the 'tokenize' module."
1,34-1,35: NL '\n'
2,0-2,42: COMMENT '# Large bits stolen from test_grammar.py. '
2,42-2,43: NL '\n'
3,0-3,1: NL '\n'
4,0-4,10: COMMENT '# Comments'
4,10-4,11: NL '\n'
5,0-5,3: STRING '"#"'
5,3-5,4: NEWLINE '\n'
6,0-6,2: COMMENT "#'"
6,2-6,3: NL '\n'
7,0-7,2: COMMENT '#"'
7,2-7,3: NL '\n'
8,0-8,2: COMMENT '#\\'
8,2-8,3: NL '\n'
9,7-9,8: COMMENT '#'
9,8-9,9: NL '\n'
10,4-10,9: COMMENT '# abc'
10,9-10,10: NL '\n'
11,0-12,4: STRING "'''#\n#'''"
12,4-12,5: NEWLINE '\n'
13,0-13,1: NL '\n'
14,0-14,1: NAME 'x'
14,2-14,3: OP '='
14,4-14,5: NUMBER '1'
14,7-14,8: COMMENT '#'
14,8-14,9: NEWLINE '\n'
15,0-15,1: NL '\n'
16,0-16,24: COMMENT '# Balancing continuation'
16,24-16,25: NL '\n'
17,0-17,1: NL '\n'
18,0-18,1: NAME 'a'
18,2-18,3: OP '='
18,4-18,5: OP '('
18,5-18,6: NUMBER '3'
18,6-18,7: OP ','
18,8-18,9: NUMBER '4'
18,9-18,10: OP ','
18,10-18,11: NL '\n'
19,2-19,3: NUMBER '5'
19,3-19,4: OP ','
19,5-19,6: NUMBER '6'
19,6-19,7: OP ')'
19,7-19,8: NEWLINE '\n'
20,0-20,1: NAME 'y'
20,2-20,3: OP '='
20,4-20,5: OP '['
20,5-20,6: NUMBER '3'
20,6-20,7: OP ','
20,8-20,9: NUMBER '4'
20,9-20,10: OP ','
20,10-20,11: NL '\n'
21,2-21,3: NUMBER '5'
21,3-21,4: OP ']'
21,4-21,5: NEWLINE '\n'
22,0-22,1: NAME 'z'
22,2-22,3: OP '='
22,4-22,5: OP '{'
22,5-22,8: STRING "'a'"
22,8-22,9: OP ':'
22,9-22,10: NUMBER '5'
22,10-22,11: OP ','
22,11-22,12: NL '\n'
23,2-23,5: STRING "'b'"
23,5-23,6: OP ':'
23,6-23,7: NUMBER '6'
23,7-23,8: OP '}'
23,8-23,9: NEWLINE '\n'
24,0-24,1: NAME 'x'
24,2-24,3: OP '='
24,4-24,5: OP '('
24,5-24,8: NAME 'len'
24,8-24,9: OP '('
24,9-24,10: OP '`'
24,10-24,11: NAME 'y'
24,11-24,12: OP '`'
24,12-24,13: OP ')'
24,14-24,15: OP '+'
24,16-24,17: NUMBER '5'
24,17-24,18: OP '*'
24,18-24,19: NAME 'x'
24,20-24,21: OP '-'
24,22-24,23: NAME 'a'
24,23-24,24: OP '['
24,24-24,25: NL '\n'
25,3-25,4: NUMBER '3'
25,5-25,6: OP ']'
25,6-25,7: NL '\n'
26,3-26,4: OP '-'
26,5-26,6: NAME 'x'
26,7-26,8: OP '+'
26,9-26,12: NAME 'len'
26,12-26,13: OP '('
26,13-26,14: OP '{'
26,14-26,15: NL '\n'
27,3-27,4: OP '}'
27,4-27,5: NL '\n'
28,4-28,5: OP ')'
28,5-28,6: NL '\n'
29,2-29,3: OP ')'
29,3-29,4: NEWLINE '\n'
30,0-30,1: NL '\n'
31,0-31,36: COMMENT '# Backslash means line continuation:'
31,36-31,37: NL '\n'
32,0-32,1: NAME 'x'
32,2-32,3: OP '='
32,4-32,5: NUMBER '1'
33,0-33,1: OP '+'
33,2-33,3: NUMBER '1'
33,3-33,4: NEWLINE '\n'
34,0-34,1: NL '\n'
35,0-35,54: COMMENT '# Backslash does not means continuation in comments :\\'
35,54-35,55: NL '\n'
36,0-36,1: NAME 'x'
36,2-36,3: OP '='
36,4-36,5: NUMBER '0'
36,5-36,6: NEWLINE '\n'
37,0-37,1: NL '\n'
38,0-38,19: COMMENT '# Ordinary integers'
38,19-38,20: NL '\n'
39,0-39,4: NUMBER '0xff'
39,5-39,7: OP '<>'
39,8-39,11: NUMBER '255'
39,11-39,12: NEWLINE '\n'
40,0-40,4: NUMBER '0377'
40,5-40,7: OP '<>'
40,8-40,11: NUMBER '255'
40,11-40,12: NEWLINE '\n'
41,0-41,10: NUMBER '2147483647'
41,13-41,15: OP '!='
41,16-41,28: NUMBER '017777777777'
41,28-41,29: NEWLINE '\n'
42,0-42,1: OP '-'
42,1-42,11: NUMBER '2147483647'
42,11-42,12: OP '-'
42,12-42,13: NUMBER '1'
42,14-42,16: OP '!='
42,17-42,29: NUMBER '020000000000'
42,29-42,30: NEWLINE '\n'
43,0-43,12: NUMBER '037777777777'
43,13-43,15: OP '!='
43,16-43,17: OP '-'
43,17-43,18: NUMBER '1'
43,18-43,19: NEWLINE '\n'
44,0-44,10: NUMBER '0xffffffff'
44,11-44,13: OP '!='
44,14-44,15: OP '-'
44,15-44,16: NUMBER '1'
44,16-44,17: NEWLINE '\n'
45,0-45,1: NL '\n'
46,0-46,15: COMMENT '# Long integers'
46,15-46,16: NL '\n'
47,0-47,1: NAME 'x'
47,2-47,3: OP '='
47,4-47,6: NUMBER '0L'
47,6-47,7: NEWLINE '\n'
48,0-48,1: NAME 'x'
48,2-48,3: OP '='
48,4-48,6: NUMBER '0l'
48,6-48,7: NEWLINE '\n'
49,0-49,1: NAME 'x'
49,2-49,3: OP '='
49,4-49,23: NUMBER '0xffffffffffffffffL'
49,23-49,24: NEWLINE '\n'
50,0-50,1: NAME 'x'
50,2-50,3: OP '='
50,4-50,23: NUMBER '0xffffffffffffffffl'
50,23-50,24: NEWLINE '\n'
51,0-51,1: NAME 'x'
51,2-51,3: OP '='
51,4-51,23: NUMBER '077777777777777777L'
51,23-51,24: NEWLINE '\n'
52,0-52,1: NAME 'x'
52,2-52,3: OP '='
52,4-52,23: NUMBER '077777777777777777l'
52,23-52,24: NEWLINE '\n'
53,0-53,1: NAME 'x'
53,2-53,3: OP '='
53,4-53,35: NUMBER '123456789012345678901234567890L'
53,35-53,36: NEWLINE '\n'
54,0-54,1: NAME 'x'
54,2-54,3: OP '='
54,4-54,35: NUMBER '123456789012345678901234567890l'
54,35-54,36: NEWLINE '\n'
55,0-55,1: NL '\n'
56,0-56,24: COMMENT '# Floating-point numbers'
56,24-56,25: NL '\n'
57,0-57,1: NAME 'x'
57,2-57,3: OP '='
57,4-57,8: NUMBER '3.14'
57,8-57,9: NEWLINE '\n'
58,0-58,1: NAME 'x'
58,2-58,3: OP '='
58,4-58,8: NUMBER '314.'
58,8-58,9: NEWLINE '\n'
59,0-59,1: NAME 'x'
59,2-59,3: OP '='
59,4-59,9: NUMBER '0.314'
59,9-59,10: NEWLINE '\n'
60,0-60,17: COMMENT '# XXX x = 000.314'
60,17-60,18: NL '\n'
61,0-61,1: NAME 'x'
61,2-61,3: OP '='
61,4-61,8: NUMBER '.314'
61,8-61,9: NEWLINE '\n'
62,0-62,1: NAME 'x'
62,2-62,3: OP '='
62,4-62,8: NUMBER '3e14'
62,8-62,9: NEWLINE '\n'
63,0-63,1: NAME 'x'
63,2-63,3: OP '='
63,4-63,8: NUMBER '3E14'
63,8-63,9: NEWLINE '\n'
64,0-64,1: NAME 'x'
64,2-64,3: OP '='
64,4-64,9: NUMBER '3e-14'
64,9-64,10: NEWLINE '\n'
65,0-65,1: NAME 'x'
65,2-65,3: OP '='
65,4-65,9: NUMBER '3e+14'
65,9-65,10: NEWLINE '\n'
66,0-66,1: NAME 'x'
66,2-66,3: OP '='
66,4-66,9: NUMBER '3.e14'
66,9-66,10: NEWLINE '\n'
67,0-67,1: NAME 'x'
67,2-67,3: OP '='
67,4-67,9: NUMBER '.3e14'
67,9-67,10: NEWLINE '\n'
68,0-68,1: NAME 'x'
68,2-68,3: OP '='
68,4-68,9: NUMBER '3.1e4'
68,9-68,10: NEWLINE '\n'
69,0-69,1: NL '\n'
70,0-70,17: COMMENT '# String literals'
70,17-70,18: NL '\n'
71,0-71,1: NAME 'x'
71,2-71,3: OP '='
71,4-71,6: STRING "''"
71,6-71,7: OP ';'
71,8-71,9: NAME 'y'
71,10-71,11: OP '='
71,12-71,14: STRING '""'
71,14-71,15: OP ';'
71,15-71,16: NEWLINE '\n'
72,0-72,1: NAME 'x'
72,2-72,3: OP '='
72,4-72,8: STRING "'\\''"
72,8-72,9: OP ';'
72,10-72,11: NAME 'y'
72,12-72,13: OP '='
72,14-72,17: STRING '"\'"'
72,17-72,18: OP ';'
72,18-72,19: NEWLINE '\n'
73,0-73,1: NAME 'x'
73,2-73,3: OP '='
73,4-73,7: STRING '\'"\''
73,7-73,8: OP ';'
73,9-73,10: NAME 'y'
73,11-73,12: OP '='
73,13-73,17: STRING '"\\""'
73,17-73,18: OP ';'
73,18-73,19: NEWLINE '\n'
74,0-74,1: NAME 'x'
74,2-74,3: OP '='
74,4-74,32: STRING '"doesn\'t \\"shrink\\" does it"'
74,32-74,33: NEWLINE '\n'
75,0-75,1: NAME 'y'
75,2-75,3: OP '='
75,4-75,31: STRING '\'doesn\\\'t "shrink" does it\''
75,31-75,32: NEWLINE '\n'
76,0-76,1: NAME 'x'
76,2-76,3: OP '='
76,4-76,32: STRING '"does \\"shrink\\" doesn\'t it"'
76,32-76,33: NEWLINE '\n'
77,0-77,1: NAME 'y'
77,2-77,3: OP '='
77,4-77,31: STRING '\'does "shrink" doesn\\\'t it\''
77,31-77,32: NEWLINE '\n'
78,0-78,1: NAME 'x'
78,2-78,3: OP '='
78,4-83,3: STRING '"""\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n"""'
83,3-83,4: NEWLINE '\n'
84,0-84,1: NAME 'y'
84,2-84,3: OP '='
84,4-84,63: STRING '\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\''
84,63-84,64: NEWLINE '\n'
85,0-85,1: NAME 'y'
85,2-85,3: OP '='
85,4-90,3: STRING '\'\'\'\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n\'\'\''
90,3-90,4: OP ';'
90,4-90,5: NEWLINE '\n'
91,0-91,1: NAME 'y'
91,2-91,3: OP '='
91,4-96,1: STRING '"\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \'lazy\' dog.\\n\\\n"'
96,1-96,2: OP ';'
96,2-96,3: NEWLINE '\n'
97,0-97,1: NAME 'y'
97,2-97,3: OP '='
97,4-102,1: STRING '\'\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \\\'lazy\\\' dog.\\n\\\n\''
102,1-102,2: OP ';'
102,2-102,3: NEWLINE '\n'
103,0-103,1: NAME 'x'
103,2-103,3: OP '='
103,4-103,9: STRING "r'\\\\'"
103,10-103,11: OP '+'
103,12-103,17: STRING "R'\\\\'"
103,17-103,18: NEWLINE '\n'
104,0-104,1: NAME 'x'
104,2-104,3: OP '='
104,4-104,9: STRING "r'\\''"
104,10-104,11: OP '+'
104,12-104,14: STRING "''"
104,14-104,15: NEWLINE '\n'
105,0-105,1: NAME 'y'
105,2-105,3: OP '='
105,4-107,6: STRING "r'''\nfoo bar \\\\\nbaz'''"
107,7-107,8: OP '+'
107,9-108,6: STRING "R'''\nfoo'''"
108,6-108,7: NEWLINE '\n'
109,0-109,1: NAME 'y'
109,2-109,3: OP '='
109,4-111,3: STRING 'r"""foo\nbar \\\\ baz\n"""'
111,4-111,5: OP '+'
111,6-112,3: STRING "R'''spam\n'''"
112,3-112,4: NEWLINE '\n'
113,0-113,1: NAME 'x'
113,2-113,3: OP '='
113,4-113,10: STRING "u'abc'"
113,11-113,12: OP '+'
113,13-113,19: STRING "U'ABC'"
113,19-113,20: NEWLINE '\n'
114,0-114,1: NAME 'y'
114,2-114,3: OP '='
114,4-114,10: STRING 'u"abc"'
114,11-114,12: OP '+'
114,13-114,19: STRING 'U"ABC"'
114,19-114,20: NEWLINE '\n'
115,0-115,1: NAME 'x'
115,2-115,3: OP '='
115,4-115,11: STRING "ur'abc'"
115,12-115,13: OP '+'
115,14-115,21: STRING "Ur'ABC'"
115,22-115,23: OP '+'
115,24-115,31: STRING "uR'ABC'"
115,32-115,33: OP '+'
115,34-115,41: STRING "UR'ABC'"
115,41-115,42: NEWLINE '\n'
116,0-116,1: NAME 'y'
116,2-116,3: OP '='
116,4-116,11: STRING 'ur"abc"'
116,12-116,13: OP '+'
116,14-116,21: STRING 'Ur"ABC"'
116,22-116,23: OP '+'
116,24-116,31: STRING 'uR"ABC"'
116,32-116,33: OP '+'
116,34-116,41: STRING 'UR"ABC"'
116,41-116,42: NEWLINE '\n'
117,0-117,1: NAME 'x'
117,2-117,3: OP '='
117,4-117,10: STRING "ur'\\\\'"
117,11-117,12: OP '+'
117,13-117,19: STRING "UR'\\\\'"
117,19-117,20: NEWLINE '\n'
118,0-118,1: NAME 'x'
118,2-118,3: OP '='
118,4-118,10: STRING "ur'\\''"
118,11-118,12: OP '+'
118,13-118,15: STRING "''"
118,15-118,16: NEWLINE '\n'
119,0-119,1: NAME 'y'
119,2-119,3: OP '='
119,4-121,6: STRING "ur'''\nfoo bar \\\\\nbaz'''"
121,7-121,8: OP '+'
121,9-122,6: STRING "UR'''\nfoo'''"
122,6-122,7: NEWLINE '\n'
123,0-123,1: NAME 'y'
123,2-123,3: OP '='
123,4-125,3: STRING 'Ur"""foo\nbar \\\\ baz\n"""'
125,4-125,5: OP '+'
125,6-126,3: STRING "uR'''spam\n'''"
126,3-126,4: NEWLINE '\n'
127,0-127,1: NL '\n'
128,0-128,13: COMMENT '# Indentation'
128,13-128,14: NL '\n'
129,0-129,2: NAME 'if'
129,3-129,4: NUMBER '1'
129,4-129,5: OP ':'
129,5-129,6: NEWLINE '\n'
130,0-130,4: INDENT ' '
130,4-130,5: NAME 'x'
130,6-130,7: OP '='
130,8-130,9: NUMBER '2'
130,9-130,10: NEWLINE '\n'
131,0-131,0: DEDENT ''
131,0-131,2: NAME 'if'
131,3-131,4: NUMBER '1'
131,4-131,5: OP ':'
131,5-131,6: NEWLINE '\n'
132,0-132,8: INDENT ' '
132,8-132,9: NAME 'x'
132,10-132,11: OP '='
132,12-132,13: NUMBER '2'
132,13-132,14: NEWLINE '\n'
133,0-133,0: DEDENT ''
133,0-133,2: NAME 'if'
133,3-133,4: NUMBER '1'
133,4-133,5: OP ':'
133,5-133,6: NEWLINE '\n'
134,0-134,4: INDENT ' '
134,4-134,9: NAME 'while'
134,10-134,11: NUMBER '0'
134,11-134,12: OP ':'
134,12-134,13: NEWLINE '\n'
135,0-135,5: INDENT ' '
135,5-135,7: NAME 'if'
135,8-135,9: NUMBER '0'
135,9-135,10: OP ':'
135,10-135,11: NEWLINE '\n'
136,0-136,11: INDENT ' '
136,11-136,12: NAME 'x'
136,13-136,14: OP '='
136,15-136,16: NUMBER '2'
136,16-136,17: NEWLINE '\n'
137,5-137,5: DEDENT ''
137,5-137,6: NAME 'x'
137,7-137,8: OP '='
137,9-137,10: NUMBER '2'
137,10-137,11: NEWLINE '\n'
138,0-138,0: DEDENT ''
138,0-138,0: DEDENT ''
138,0-138,2: NAME 'if'
138,3-138,4: NUMBER '0'
138,4-138,5: OP ':'
138,5-138,6: NEWLINE '\n'
139,0-139,2: INDENT ' '
139,2-139,4: NAME 'if'
139,5-139,6: NUMBER '2'
139,6-139,7: OP ':'
139,7-139,8: NEWLINE '\n'
140,0-140,3: INDENT ' '
140,3-140,8: NAME 'while'
140,9-140,10: NUMBER '0'
140,10-140,11: OP ':'
140,11-140,12: NEWLINE '\n'
141,0-141,8: INDENT ' '
141,8-141,10: NAME 'if'
141,11-141,12: NUMBER '1'
141,12-141,13: OP ':'
141,13-141,14: NEWLINE '\n'
142,0-142,10: INDENT ' '
142,10-142,11: NAME 'x'
142,12-142,13: OP '='
142,14-142,15: NUMBER '2'
142,15-142,16: NEWLINE '\n'
143,0-143,1: NL '\n'
144,0-144,11: COMMENT '# Operators'
144,11-144,12: NL '\n'
145,0-145,1: NL '\n'
146,0-146,0: DEDENT ''
146,0-146,0: DEDENT ''
146,0-146,0: DEDENT ''
146,0-146,0: DEDENT ''
146,0-146,3: NAME 'def'
146,4-146,7: NAME 'd22'
146,7-146,8: OP '('
146,8-146,9: NAME 'a'
146,9-146,10: OP ','
146,11-146,12: NAME 'b'
146,12-146,13: OP ','
146,14-146,15: NAME 'c'
146,15-146,16: OP '='
146,16-146,17: NUMBER '1'
146,17-146,18: OP ','
146,19-146,20: NAME 'd'
146,20-146,21: OP '='
146,21-146,22: NUMBER '2'
146,22-146,23: OP ')'
146,23-146,24: OP ':'
146,25-146,29: NAME 'pass'
146,29-146,30: NEWLINE '\n'
147,0-147,3: NAME 'def'
147,4-147,8: NAME 'd01v'
147,8-147,9: OP '('
147,9-147,10: NAME 'a'
147,10-147,11: OP '='
147,11-147,12: NUMBER '1'
147,12-147,13: OP ','
147,14-147,15: OP '*'
147,15-147,20: NAME 'restt'
147,20-147,21: OP ','
147,22-147,24: OP '**'
147,24-147,29: NAME 'restd'
147,29-147,30: OP ')'
147,30-147,31: OP ':'
147,32-147,36: NAME 'pass'
147,36-147,37: NEWLINE '\n'
148,0-148,1: NL '\n'
149,0-149,1: OP '('
149,1-149,2: NAME 'x'
149,2-149,3: OP ','
149,4-149,5: NAME 'y'
149,5-149,6: OP ')'
149,7-149,9: OP '<>'
149,10-149,11: OP '('
149,11-149,12: OP '{'
149,12-149,15: STRING "'a'"
149,15-149,16: OP ':'
149,16-149,17: NUMBER '1'
149,17-149,18: OP '}'
149,18-149,19: OP ','
149,20-149,21: OP '{'
149,21-149,24: STRING "'b'"
149,24-149,25: OP ':'
149,25-149,26: NUMBER '2'
149,26-149,27: OP '}'
149,27-149,28: OP ')'
149,28-149,29: NEWLINE '\n'
150,0-150,1: NL '\n'
151,0-151,12: COMMENT '# comparison'
151,12-151,13: NL '\n'
152,0-152,2: NAME 'if'
152,3-152,4: NUMBER '1'
152,5-152,6: OP '<'
152,7-152,8: NUMBER '1'
152,9-152,10: OP '>'
152,11-152,12: NUMBER '1'
152,13-152,15: OP '=='
152,16-152,17: NUMBER '1'
152,18-152,20: OP '>='
152,21-152,22: NUMBER '1'
152,23-152,25: OP '<='
152,26-152,27: NUMBER '1'
152,28-152,30: OP '<>'
152,31-152,32: NUMBER '1'
152,33-152,35: OP '!='
152,36-152,37: NUMBER '1'
152,38-152,40: NAME 'in'
152,41-152,42: NUMBER '1'
152,43-152,46: NAME 'not'
152,47-152,49: NAME 'in'
152,50-152,51: NUMBER '1'
152,52-152,54: NAME 'is'
152,55-152,56: NUMBER '1'
152,57-152,59: NAME 'is'
152,60-152,63: NAME 'not'
152,64-152,65: NUMBER '1'
152,65-152,66: OP ':'
152,67-152,71: NAME 'pass'
152,71-152,72: NEWLINE '\n'
153,0-153,1: NL '\n'
154,0-154,8: COMMENT '# binary'
154,8-154,9: NL '\n'
155,0-155,1: NAME 'x'
155,2-155,3: OP '='
155,4-155,5: NUMBER '1'
155,6-155,7: OP '&'
155,8-155,9: NUMBER '1'
155,9-155,10: NEWLINE '\n'
156,0-156,1: NAME 'x'
156,2-156,3: OP '='
156,4-156,5: NUMBER '1'
156,6-156,7: OP '^'
156,8-156,9: NUMBER '1'
156,9-156,10: NEWLINE '\n'
157,0-157,1: NAME 'x'
157,2-157,3: OP '='
157,4-157,5: NUMBER '1'
157,6-157,7: OP '|'
157,8-157,9: NUMBER '1'
157,9-157,10: NEWLINE '\n'
158,0-158,1: NL '\n'
159,0-159,7: COMMENT '# shift'
159,7-159,8: NL '\n'
160,0-160,1: NAME 'x'
160,2-160,3: OP '='
160,4-160,5: NUMBER '1'
160,6-160,8: OP '<<'
160,9-160,10: NUMBER '1'
160,11-160,13: OP '>>'
160,14-160,15: NUMBER '1'
160,15-160,16: NEWLINE '\n'
161,0-161,1: NL '\n'
162,0-162,10: COMMENT '# additive'
162,10-162,11: NL '\n'
163,0-163,1: NAME 'x'
163,2-163,3: OP '='
163,4-163,5: NUMBER '1'
163,6-163,7: OP '-'
163,8-163,9: NUMBER '1'
163,10-163,11: OP '+'
163,12-163,13: NUMBER '1'
163,14-163,15: OP '-'
163,16-163,17: NUMBER '1'
163,18-163,19: OP '+'
163,20-163,21: NUMBER '1'
163,21-163,22: NEWLINE '\n'
164,0-164,1: NL '\n'
165,0-165,16: COMMENT '# multiplicative'
165,16-165,17: NL '\n'
166,0-166,1: NAME 'x'
166,2-166,3: OP '='
166,4-166,5: NUMBER '1'
166,6-166,7: OP '/'
166,8-166,9: NUMBER '1'
166,10-166,11: OP '*'
166,12-166,13: NUMBER '1'
166,14-166,15: OP '%'
166,16-166,17: NUMBER '1'
166,17-166,18: NEWLINE '\n'
167,0-167,1: NL '\n'
168,0-168,7: COMMENT '# unary'
168,7-168,8: NL '\n'
169,0-169,1: NAME 'x'
169,2-169,3: OP '='
169,4-169,5: OP '~'
169,5-169,6: NUMBER '1'
169,7-169,8: OP '^'
169,9-169,10: NUMBER '1'
169,11-169,12: OP '&'
169,13-169,14: NUMBER '1'
169,15-169,16: OP '|'
169,17-169,18: NUMBER '1'
169,19-169,20: OP '&'
169,21-169,22: NUMBER '1'
169,23-169,24: OP '^'
169,25-169,26: OP '-'
169,26-169,27: NUMBER '1'
169,27-169,28: NEWLINE '\n'
170,0-170,1: NAME 'x'
170,2-170,3: OP '='
170,4-170,5: OP '-'
170,5-170,6: NUMBER '1'
170,6-170,7: OP '*'
170,7-170,8: NUMBER '1'
170,8-170,9: OP '/'
170,9-170,10: NUMBER '1'
170,11-170,12: OP '+'
170,13-170,14: NUMBER '1'
170,14-170,15: OP '*'
170,15-170,16: NUMBER '1'
170,17-170,18: OP '-'
170,19-170,20: OP '-'
170,20-170,21: OP '-'
170,21-170,22: OP '-'
170,22-170,23: NUMBER '1'
170,23-170,24: OP '*'
170,24-170,25: NUMBER '1'
170,25-170,26: NEWLINE '\n'
171,0-171,1: NL '\n'
172,0-172,10: COMMENT '# selector'
172,10-172,11: NL '\n'
173,0-173,6: NAME 'import'
173,7-173,10: NAME 'sys'
173,10-173,11: OP ','
173,12-173,16: NAME 'time'
173,16-173,17: NEWLINE '\n'
174,0-174,1: NAME 'x'
174,2-174,3: OP '='
174,4-174,7: NAME 'sys'
174,7-174,8: OP '.'
174,8-174,15: NAME 'modules'
174,15-174,16: OP '['
174,16-174,22: STRING "'time'"
174,22-174,23: OP ']'
174,23-174,24: OP '.'
174,24-174,28: NAME 'time'
174,28-174,29: OP '('
174,29-174,30: OP ')'
174,30-174,31: NEWLINE '\n'
175,0-175,1: NL '\n'
176,0-176,1: OP '@'
176,1-176,13: NAME 'staticmethod'
176,13-176,14: NEWLINE '\n'
177,0-177,3: NAME 'def'
177,4-177,7: NAME 'foo'
177,7-177,8: OP '('
177,8-177,9: OP ')'
177,9-177,10: OP ':'
177,11-177,15: NAME 'pass'
177,15-177,16: NEWLINE '\n'
178,0-178,1: NL '\n'
179,0-179,0: ENDMARKER ''
Lib/test/test_tokenize.py
View file @ b8d37359
"""Tests for the tokenize module. doctests = """
Tests for the tokenize module.
The tests were originally written in the old Python style, where the >>> import glob, random, sys
test output was compared to a golden file. This docstring represents
the first steps towards rewriting the entire test as a doctest.
The tests can be really simple. Given a small fragment of source The tests can be really simple. Given a small fragment of source
code, print out a table with the tokens. The ENDMARK is omitted for code, print out a table with thokens. The ENDMARK is omitted for
brevity. brevity.
>>> dump_tokens("1 + 1") >>> dump_tokens("1 + 1")
NUMBER '1' (1, 0) (1, 1) NUMBER '1' (1, 0) (1, 1)
OP '+' (1, 2) (1, 3) OP '+' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5) NUMBER '1' (1, 4) (1, 5)
A comment generates a token here, unlike in the parser module. The >>> dump_tokens("if False:\\n"
comment token is followed by an NL or a NEWLINE token, depending on ... " # NL\\n"
whether the line contains the completion of a statement. ... " True = False # NEWLINE\\n")
NAME 'if' (1, 0) (1, 2)
>>> dump_tokens("if False:\\n" NAME 'False' (1, 3) (1, 8)
... " # NL\\n" OP ':' (1, 8) (1, 9)
... " True = False # NEWLINE\\n") NEWLINE '\\n' (1, 9) (1, 10)
NAME 'if' (1, 0) (1, 2) COMMENT '# NL' (2, 4) (2, 8)
NAME 'False' (1, 3) (1, 8) NL '\\n' (2, 8) (2, 9)
OP ':' (1, 8) (1, 9) INDENT ' ' (3, 0) (3, 4)
NEWLINE '\\n' (1, 9) (1, 10) NAME 'True' (3, 4) (3, 8)
COMMENT '# NL' (2, 4) (2, 8) OP '=' (3, 9) (3, 10)
NL '\\n' (2, 8) (2, 9) NAME 'False' (3, 11) (3, 16)
INDENT ' ' (3, 0) (3, 4) COMMENT '# NEWLINE' (3, 17) (3, 26)
NAME 'True' (3, 4) (3, 8) NEWLINE '\\n' (3, 26) (3, 27)
OP '=' (3, 9) (3, 10) DEDENT '' (4, 0) (4, 0)
NAME 'False' (3, 11) (3, 16)
COMMENT '# NEWLINE' (3, 17) (3, 26) >>> indent_error_file = \"""
NEWLINE '\\n' (3, 26) (3, 27) ... def k(x):
DEDENT '' (4, 0) (4, 0) ... x += 2
... x += 5
... \"""
There will be a bunch more tests of specific source patterns.
>>> for tok in generate_tokens(StringIO(indent_error_file).readline): pass
The tokenize module also defines an untokenize function that should Traceback (most recent call last):
regenerate the original program text from the tokens. ...
IndentationError: unindent does not match any outer indentation level
There are some standard formatting practices that are easy to get right.
Test roundtrip for `untokenize`. `f` is an open file or a string. The source
>>> roundtrip("if x == 1:\\n" code in f is tokenized, converted back to source code via tokenize.untokenize(),
... " print x\\n") and tokenized again from the latter. The test fails if the second tokenization
if x == 1: doesn't match the first.
print x
>>> def roundtrip(f):
... if isinstance(f, str): f = StringIO(f)
... token_list = list(generate_tokens(f.readline))
... f.close()
... tokens1 = [tok[:2] for tok in token_list]
... new_text = untokenize(tokens1)
... readline = iter(new_text.splitlines(1)).next
... tokens2 = [tok[:2] for tok in generate_tokens(readline)]
... return tokens1 == tokens2
...
There are some standard formattig practises that are easy to get right.
>>> roundtrip("if x == 1:\\n"
... " print x\\n")
True
>>> roundtrip("# This is a comment\\n# This also")
True
Some people use different formatting conventions, which makes Some people use different formatting conventions, which makes
untokenize a little trickier. Note that this test involves trailing untokenize a little trickier. Note that this test involves trailing
whitespace after the colon. Note that we use hex escapes to make the whitespace after the colon. Note that we use hex escapes to make the
two trailing blanks apparent in the expected output. two trailing blanks apperant in the expected output.
>>> roundtrip("if x == 1 : \\n" >>> roundtrip("if x == 1 : \\n"
... " print x\\n") ... " print x\\n")
if x == 1 :\x20\x20 True
print x
>>> f = test_support.findfile("tokenize_tests" + os.extsep + "txt")
Comments need to go in the right place. >>> roundtrip(open(f))
True
>>> roundtrip("if x == 1:\\n"
... " # A comment by itself.\\n" >>> roundtrip("if x == 1:\\n"
... " print x # Comment here, too.\\n" ... " # A comment by itself.\\n"
... " # Another comment.\\n" ... " print x # Comment here, too.\\n"
... "after_if = True\\n") ... " # Another comment.\\n"
if x == 1: ... "after_if = True\\n")
# A comment by itself. True
print x # Comment here, too.
# Another comment. >>> roundtrip("if (x # The comments need to go in the right place\\n"
after_if = True ... " == 1):\\n"
... " print 'x==1'\\n")
>>> roundtrip("if (x # The comments need to go in the right place\\n" True
... " == 1):\\n"
... " print 'x == 1'\\n") >>> roundtrip("class Test: # A comment here\\n"
if (x # The comments need to go in the right place ... " # A comment with weird indent\\n"
== 1): ... " after_com = 5\\n"
print 'x == 1' ... " def x(m): return m*5 # a one liner\\n"
... " def y(m): # A whitespace after the colon\\n"
... " return y*4 # 3-space indent\\n")
True
Some error-handling code
>>> roundtrip("try: import somemodule\\n"
... "except ImportError: # comment\\n"
... " print 'Can not import' # comment2\\n"
... "else: print 'Loaded'\\n")
True
Balancing contunuation
>>> roundtrip("a = (3,4, \\n"
... "5,6)\\n"
... "y = [3, 4,\\n"
... "5]\\n"
... "z = {'a': 5,\\n"
... "'b':15, 'c':True}\\n"
... "x = len(y) + 5 - a[\\n"
... "3] - a[2]\\n"
... "+ len(z) - z[\\n"
... "'b']\\n")
True
Ordinary integers and binary operators
>>> dump_tokens("0xff <= 255")
NUMBER '0xff' (1, 0) (1, 4)
OP '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
>>> dump_tokens("01234567 > ~0x15")
NUMBER '01234567' (1, 0) (1, 8)
OP '>' (1, 9) (1, 10)
OP '~' (1, 11) (1, 12)
NUMBER '0x15' (1, 12) (1, 16)
>>> dump_tokens("2134568 != 01231515")
NUMBER '2134568' (1, 0) (1, 7)
OP '!=' (1, 8) (1, 10)
NUMBER '01231515' (1, 11) (1, 19)
>>> dump_tokens("(-124561-1) & 0200000000")
OP '(' (1, 0) (1, 1)
OP '-' (1, 1) (1, 2)
NUMBER '124561' (1, 2) (1, 8)
OP '-' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP ')' (1, 10) (1, 11)
OP '&' (1, 12) (1, 13)
NUMBER '0200000000' (1, 14) (1, 24)
>>> dump_tokens("0xdeadbeef != -1")
NUMBER '0xdeadbeef' (1, 0) (1, 10)
OP '!=' (1, 11) (1, 13)
OP '-' (1, 14) (1, 15)
NUMBER '1' (1, 15) (1, 16)
>>> dump_tokens("0xdeadc0de & 012345")
NUMBER '0xdeadc0de' (1, 0) (1, 10)
OP '&' (1, 11) (1, 12)
NUMBER '012345' (1, 13) (1, 19)
>>> dump_tokens("0xFF & 0x15 | 1234")
NUMBER '0xFF' (1, 0) (1, 4)
OP '&' (1, 5) (1, 6)
NUMBER '0x15' (1, 7) (1, 11)
OP '|' (1, 12) (1, 13)
NUMBER '1234' (1, 14) (1, 18)
Long integers
>>> dump_tokens("x = 0L")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '0L' (1, 4) (1, 6)
>>> dump_tokens("x = 0xfffffffffff")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '0xffffffffff (1, 4) (1, 17)
>>> dump_tokens("x = 123141242151251616110l")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '123141242151 (1, 4) (1, 26)
>>> dump_tokens("x = -15921590215012591L")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
OP '-' (1, 4) (1, 5)
NUMBER '159215902150 (1, 5) (1, 23)
Floating point numbers
>>> dump_tokens("x = 3.14159")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3.14159' (1, 4) (1, 11)
>>> dump_tokens("x = 314159.")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '314159.' (1, 4) (1, 11)
>>> dump_tokens("x = .314159")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '.314159' (1, 4) (1, 11)
>>> dump_tokens("x = 3e14159")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3e14159' (1, 4) (1, 11)
>>> dump_tokens("x = 3E123")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3E123' (1, 4) (1, 9)
>>> dump_tokens("x+y = 3e-1230")
NAME 'x' (1, 0) (1, 1)
OP '+' (1, 1) (1, 2)
NAME 'y' (1, 2) (1, 3)
OP '=' (1, 4) (1, 5)
NUMBER '3e-1230' (1, 6) (1, 13)
>>> dump_tokens("x = 3.14e159")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3.14e159' (1, 4) (1, 12)
String literals
>>> dump_tokens("x = ''; y = \\\"\\\"")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "''" (1, 4) (1, 6)
OP ';' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
OP '=' (1, 10) (1, 11)
STRING '""' (1, 12) (1, 14)
>>> dump_tokens("x = '\\\"'; y = \\\"'\\\"")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '\\'"\\'' (1, 4) (1, 7)
OP ';' (1, 7) (1, 8)
NAME 'y' (1, 9) (1, 10)
OP '=' (1, 11) (1, 12)
STRING '"\\'"' (1, 13) (1, 16)
>>> dump_tokens("x = \\\"doesn't \\\"shrink\\\", does it\\\"")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '"doesn\\'t "' (1, 4) (1, 14)
NAME 'shrink' (1, 14) (1, 20)
STRING '", does it"' (1, 20) (1, 31)
>>> dump_tokens("x = u'abc' + U'ABC'")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "u'abc'" (1, 4) (1, 10)
OP '+' (1, 11) (1, 12)
STRING "U'ABC'" (1, 13) (1, 19)
>>> dump_tokens('y = u"ABC" + U"ABC"')
NAME 'y' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING 'u"ABC"' (1, 4) (1, 10)
OP '+' (1, 11) (1, 12)
STRING 'U"ABC"' (1, 13) (1, 19)
>>> dump_tokens("x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC'")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "ur'abc'" (1, 4) (1, 11)
OP '+' (1, 12) (1, 13)
STRING "Ur'ABC'" (1, 14) (1, 21)
OP '+' (1, 22) (1, 23)
STRING "uR'ABC'" (1, 24) (1, 31)
OP '+' (1, 32) (1, 33)
STRING "UR'ABC'" (1, 34) (1, 41)
>>> dump_tokens('y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC"')
NAME 'y' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING 'ur"abc"' (1, 4) (1, 11)
OP '+' (1, 12) (1, 13)
STRING 'Ur"ABC"' (1, 14) (1, 21)
OP '+' (1, 22) (1, 23)
STRING 'uR"ABC"' (1, 24) (1, 31)
OP '+' (1, 32) (1, 33)
STRING 'UR"ABC"' (1, 34) (1, 41)
Operators
>>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")
NAME 'def' (1, 0) (1, 3)
NAME 'd22' (1, 4) (1, 7)
OP '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
OP ',' (1, 9) (1, 10)
NAME 'b' (1, 11) (1, 12)
OP ',' (1, 12) (1, 13)
NAME 'c' (1, 14) (1, 15)
OP '=' (1, 15) (1, 16)
NUMBER '2' (1, 16) (1, 17)
OP ',' (1, 17) (1, 18)
NAME 'd' (1, 19) (1, 20)
OP '=' (1, 20) (1, 21)
NUMBER '2' (1, 21) (1, 22)
OP ',' (1, 22) (1, 23)
OP '*' (1, 24) (1, 25)
NAME 'k' (1, 25) (1, 26)
OP ')' (1, 26) (1, 27)
OP ':' (1, 27) (1, 28)
NAME 'pass' (1, 29) (1, 33)
>>> dump_tokens("def d01v_(a=1, *k, **w): pass")
NAME 'def' (1, 0) (1, 3)
NAME 'd01v_' (1, 4) (1, 9)
OP '(' (1, 9) (1, 10)
NAME 'a' (1, 10) (1, 11)
OP '=' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP ',' (1, 13) (1, 14)
OP '*' (1, 15) (1, 16)
NAME 'k' (1, 16) (1, 17)
OP ',' (1, 17) (1, 18)
OP '**' (1, 19) (1, 21)
NAME 'w' (1, 21) (1, 22)
OP ')' (1, 22) (1, 23)
OP ':' (1, 23) (1, 24)
NAME 'pass' (1, 25) (1, 29)
Comparison
>>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
... "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass")
NAME 'if' (1, 0) (1, 2)
NUMBER '1' (1, 3) (1, 4)
OP '<' (1, 5) (1, 6)
NUMBER '1' (1, 7) (1, 8)
OP '>' (1, 9) (1, 10)
NUMBER '1' (1, 11) (1, 12)
OP '==' (1, 13) (1, 15)
NUMBER '1' (1, 16) (1, 17)
OP '>=' (1, 18) (1, 20)
NUMBER '5' (1, 21) (1, 22)
OP '<=' (1, 23) (1, 25)
NUMBER '0x15' (1, 26) (1, 30)
OP '<=' (1, 31) (1, 33)
NUMBER '0x12' (1, 34) (1, 38)
OP '!=' (1, 39) (1, 41)
NUMBER '1' (1, 42) (1, 43)
NAME 'and' (1, 44) (1, 47)
NUMBER '5' (1, 48) (1, 49)
NAME 'in' (1, 50) (1, 52)
NUMBER '1' (1, 53) (1, 54)
NAME 'not' (1, 55) (1, 58)
NAME 'in' (1, 59) (1, 61)
NUMBER '1' (1, 62) (1, 63)
NAME 'is' (1, 64) (1, 66)
NUMBER '1' (1, 67) (1, 68)
NAME 'or' (1, 69) (1, 71)
NUMBER '5' (1, 72) (1, 73)
NAME 'is' (1, 74) (1, 76)
NAME 'not' (1, 77) (1, 80)
NUMBER '1' (1, 81) (1, 82)
OP ':' (1, 82) (1, 83)
NAME 'pass' (1, 84) (1, 88)
Shift
>>> dump_tokens("x = 1 << 1 >> 5")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '<<' (1, 6) (1, 8)
NUMBER '1' (1, 9) (1, 10)
OP '>>' (1, 11) (1, 13)
NUMBER '5' (1, 14) (1, 15)
Additive
>>> dump_tokens("x = 1 - y + 15 - 01 + 0x124 + z + a[5]")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '-' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
OP '+' (1, 10) (1, 11)
NUMBER '15' (1, 12) (1, 14)
OP '-' (1, 15) (1, 16)
NUMBER '01' (1, 17) (1, 19)
OP '+' (1, 20) (1, 21)
NUMBER '0x124' (1, 22) (1, 27)
OP '+' (1, 28) (1, 29)
NAME 'z' (1, 30) (1, 31)
OP '+' (1, 32) (1, 33)
NAME 'a' (1, 34) (1, 35)
OP '[' (1, 35) (1, 36)
NUMBER '5' (1, 36) (1, 37)
OP ']' (1, 37) (1, 38)
Multiplicative
>>> dump_tokens("x = 1//1*1/5*12%0x12")
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '//' (1, 5) (1, 7)
NUMBER '1' (1, 7) (1, 8)
OP '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP '/' (1, 10) (1, 11)
NUMBER '5' (1, 11) (1, 12)
OP '*' (1, 12) (1, 13)
NUMBER '12' (1, 13) (1, 15)
OP '%' (1, 15) (1, 16)
NUMBER '0x12' (1, 16) (1, 20)
Unary
>>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1")
OP '~' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
OP '^' (1, 3) (1, 4)
NUMBER '1' (1, 5) (1, 6)
OP '&' (1, 7) (1, 8)
NUMBER '1' (1, 9) (1, 10)
OP '|' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP '^' (1, 14) (1, 15)
OP '-' (1, 16) (1, 17)
NUMBER '1' (1, 17) (1, 18)
>>> dump_tokens("-1*1/1+1*1//1 - ---1**1")
OP '-' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
OP '*' (1, 2) (1, 3)
NUMBER '1' (1, 3) (1, 4)
OP '/' (1, 4) (1, 5)
NUMBER '1' (1, 5) (1, 6)
OP '+' (1, 6) (1, 7)
NUMBER '1' (1, 7) (1, 8)
OP '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP '//' (1, 10) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP '-' (1, 14) (1, 15)
OP '-' (1, 16) (1, 17)
OP '-' (1, 17) (1, 18)
OP '-' (1, 18) (1, 19)
NUMBER '1' (1, 19) (1, 20)
OP '**' (1, 20) (1, 22)
NUMBER '1' (1, 22) (1, 23)
Selector
>>> dump_tokens("import sys, time\\nx = sys.modules['time'].time()")
NAME 'import' (1, 0) (1, 6)
NAME 'sys' (1, 7) (1, 10)
OP ',' (1, 10) (1, 11)
NAME 'time' (1, 12) (1, 16)
NEWLINE '\\n' (1, 16) (1, 17)
NAME 'x' (2, 0) (2, 1)
OP '=' (2, 2) (2, 3)
NAME 'sys' (2, 4) (2, 7)
OP '.' (2, 7) (2, 8)
NAME 'modules' (2, 8) (2, 15)
OP '[' (2, 15) (2, 16)
STRING "'time'" (2, 16) (2, 22)
OP ']' (2, 22) (2, 23)
OP '.' (2, 23) (2, 24)
NAME 'time' (2, 24) (2, 28)
OP '(' (2, 28) (2, 29)
OP ')' (2, 29) (2, 30)
Methods
>>> dump_tokens("@staticmethod\\ndef foo(x,y): pass")
OP '@' (1, 0) (1, 1)
NAME 'staticmethod (1, 1) (1, 13)
NEWLINE '\\n' (1, 13) (1, 14)
NAME 'def' (2, 0) (2, 3)
NAME 'foo' (2, 4) (2, 7)
OP '(' (2, 7) (2, 8)
NAME 'x' (2, 8) (2, 9)
OP ',' (2, 9) (2, 10)
NAME 'y' (2, 10) (2, 11)
OP ')' (2, 11) (2, 12)
OP ':' (2, 12) (2, 13)
NAME 'pass' (2, 14) (2, 18)
Backslash means line continuation, except for comments
>>> roundtrip("x=1+\\\\n"
... "1\\n"
... "# This is a comment\\\\n"
... "# This also\\n")
True
>>> roundtrip("# Comment \\\\nx = 0")
True
>>>
>>> tempdir = os.path.dirname(f) or os.curdir
>>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
>>> if not test_support.is_resource_enabled("compiler"):
... testfiles = random.sample(testfiles, 10)
...
>>> for testfile in testfiles:
... if not roundtrip(open(testfile)): break
... else: True
True
""" """
import os, glob, random, time, sys
from cStringIO import StringIO from test import test_support
from test.test_support import (verbose, findfile, is_resource_enabled, from tokenize import (tokenize, untokenize, generate_tokens, NUMBER, NAME, OP,
TestFailed) STRING, ENDMARKER, tok_name)
from tokenize import (tokenize, generate_tokens, untokenize, tok_name, from StringIO import StringIO
ENDMARKER, NUMBER, NAME, OP, STRING, COMMENT) import os
# How much time in seconds can pass before we print a 'Still working' message.
_PRINT_WORKING_MSG_INTERVAL = 5 * 60
# Test roundtrip for `untokenize`. `f` is a file path. The source code in f
# is tokenized, converted back to source code via tokenize.untokenize(),
# and tokenized again from the latter. The test fails if the second
# tokenization doesn't match the first.
def test_roundtrip(f):
## print 'Testing:', f
fobj = open(f)
try:
fulltok = list(generate_tokens(fobj.readline))
finally:
fobj.close()
t1 = [tok[:2] for tok in fulltok]
newtext = untokenize(t1)
readline = iter(newtext.splitlines(1)).next
t2 = [tok[:2] for tok in generate_tokens(readline)]
if t1 != t2:
raise TestFailed("untokenize() roundtrip failed for %r" % f)
def dump_tokens(s): def dump_tokens(s):
"""Print out the tokens in s in a table format. """Print out the tokens in s in a table format.
...@@ -118,12 +508,7 @@ def dump_tokens(s): ...@@ -118,12 +508,7 @@ def dump_tokens(s):
if type == ENDMARKER: if type == ENDMARKER:
break break
type = tok_name[type] type = tok_name[type]
print "%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals() print("%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals())
def roundtrip(s):
f = StringIO(s)
source = untokenize(generate_tokens(f.readline))
print source,
# This is an example from the docs, set up as a doctest. # This is an example from the docs, set up as a doctest.
def decistmt(s): def decistmt(s):
...@@ -163,61 +548,13 @@ def decistmt(s): ...@@ -163,61 +548,13 @@ def decistmt(s):
result.append((toknum, tokval)) result.append((toknum, tokval))
return untokenize(result) return untokenize(result)
def test_main():
if verbose:
print 'starting...'
next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
# This displays the tokenization of tokenize_tests.py to stdout, and
# regrtest.py checks that this equals the expected output (in the
# test/output/ directory).
f = open(findfile('tokenize_tests' + os.extsep + 'txt'))
tokenize(f.readline)
f.close()
# Now run test_roundtrip() over tokenize_test.py too, and over all
# (if the "compiler" resource is enabled) or a small random sample (if
# "compiler" is not enabled) of the test*.py files.
f = findfile('tokenize_tests' + os.extsep + 'txt')
test_roundtrip(f)
testdir = os.path.dirname(f) or os.curdir
testfiles = glob.glob(testdir + os.sep + 'test*.py')
if not is_resource_enabled('compiler'):
testfiles = random.sample(testfiles, 10)
for f in testfiles:
# Print still working message since this test can be really slow
if next_time <= time.time():
next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
print >>sys.__stdout__, ' test_main still working, be patient...'
sys.__stdout__.flush()
test_roundtrip(f)
# Test detecton of IndentationError.
sampleBadText = """\
def foo():
bar
baz
"""
try: __test__ = {"doctests" : doctests, 'decistmt': decistmt}
for tok in generate_tokens(StringIO(sampleBadText).readline):
pass
except IndentationError:
pass
else:
raise TestFailed("Did not detect IndentationError:")
# Run the doctests in this module.
from test import test_tokenize # i.e., this module
from test.test_support import run_doctest
run_doctest(test_tokenize, verbose)
if verbose: def test_main():
print 'finished' from test import test_tokenize
test_support.run_doctest(test_tokenize, True)
if __name__ == "__main__": if __name__ == "__main__":
test_main() test_main()
Misc/ACKS
View file @ b8d37359
...@@ -178,6 +178,7 @@ Luke Dunstan ...@@ -178,6 +178,7 @@ Luke Dunstan
Andy Dustman Andy Dustman
Gary Duzan Gary Duzan
Eugene Dvurechenski Eugene Dvurechenski
Josip Dzolonga
Maxim Dzumanenko Maxim Dzumanenko
Hans Eckardt Hans Eckardt
Grant Edwards Grant Edwards
......
Misc/NEWS
View file @ b8d37359
...@@ -50,6 +50,8 @@ Library ...@@ -50,6 +50,8 @@ Library
Tests Tests
----- -----
- GHOP 238: Convert test_tokenize to use doctest.
- GHOP 237: Rewrite test_thread using unittest. - GHOP 237: Rewrite test_thread using unittest.
- Patch #2232: os.tmpfile might fail on Windows if the user has no - Patch #2232: os.tmpfile might fail on Windows if the user has no
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7