Skip to content

C
cpython
Commit 5150a339 authored by Serhiy Storchaka's avatar Serhiy Storchaka
Browse files
Options

Merge heads

parents a525fc16 40ce22ed
Hide whitespace changes
Inline Side-by-side
Showing with 392 additions and 105 deletions
Lib/test/test_tools.py
View file @ 5150a339
......@@ -9,10 +9,13 @@ import sys
import importlib.machinery
import unittest
from unittest import mock
import shutil
import subprocess
import sysconfig
import tempfile
import textwrap
from test import support
from test.script_helper import assert_python_ok
from test.script_helper import assert_python_ok, temp_dir
if not sysconfig.is_python_build():
# XXX some installers do contain the tools, should we detect that
......@@ -36,6 +39,327 @@ class ReindentTests(unittest.TestCase):
self.assertGreater(err, b'')
class PindentTests(unittest.TestCase):
script = os.path.join(scriptsdir, 'pindent.py')
def assertFileEqual(self, fn1, fn2):
with open(fn1) as f1, open(fn2) as f2:
self.assertEqual(f1.readlines(), f2.readlines())
def pindent(self, source, *args):
with subprocess.Popen(
(sys.executable, self.script) + args,
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
universal_newlines=True) as proc:
out, err = proc.communicate(source)
self.assertIsNone(err)
return out
def lstriplines(self, data):
return '\n'.join(line.lstrip() for line in data.splitlines()) + '\n'
def test_selftest(self):
with temp_dir() as directory:
data_path = os.path.join(directory, '_test.py')
with open(self.script) as f:
closed = f.read()
with open(data_path, 'w') as f:
f.write(closed)
rc, out, err = assert_python_ok(self.script, '-d', data_path)
self.assertEqual(out, b'')
self.assertEqual(err, b'')
backup = data_path + '~'
self.assertTrue(os.path.exists(backup))
with open(backup) as f:
self.assertEqual(f.read(), closed)
with open(data_path) as f:
clean = f.read()
compile(clean, '_test.py', 'exec')
self.assertEqual(self.pindent(clean, '-c'), closed)
self.assertEqual(self.pindent(closed, '-d'), clean)
rc, out, err = assert_python_ok(self.script, '-c', data_path)
self.assertEqual(out, b'')
self.assertEqual(err, b'')
with open(backup) as f:
self.assertEqual(f.read(), clean)
with open(data_path) as f:
self.assertEqual(f.read(), closed)
broken = self.lstriplines(closed)
with open(data_path, 'w') as f:
f.write(broken)
rc, out, err = assert_python_ok(self.script, '-r', data_path)
self.assertEqual(out, b'')
self.assertEqual(err, b'')
with open(backup) as f:
self.assertEqual(f.read(), broken)
with open(data_path) as f:
indented = f.read()
compile(indented, '_test.py', 'exec')
self.assertEqual(self.pindent(broken, '-r'), indented)
def pindent_test(self, clean, closed):
self.assertEqual(self.pindent(clean, '-c'), closed)
self.assertEqual(self.pindent(closed, '-d'), clean)
broken = self.lstriplines(closed)
self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '4'), closed)
def test_statements(self):
clean = textwrap.dedent("""\
if a:
pass
if a:
pass
else:
pass
if a:
pass
elif:
pass
else:
pass
while a:
break
while a:
break
else:
pass
for i in a:
break
for i in a:
break
else:
pass
try:
pass
finally:
pass
try:
pass
except TypeError:
pass
except ValueError:
pass
else:
pass
try:
pass
except TypeError:
pass
except ValueError:
pass
finally:
pass
with a:
pass
class A:
pass
def f():
pass
""")
closed = textwrap.dedent("""\
if a:
pass
# end if
if a:
pass
else:
pass
# end if
if a:
pass
elif:
pass
else:
pass
# end if
while a:
break
# end while
while a:
break
else:
pass
# end while
for i in a:
break
# end for
for i in a:
break
else:
pass
# end for
try:
pass
finally:
pass
# end try
try:
pass
except TypeError:
pass
except ValueError:
pass
else:
pass
# end try
try:
pass
except TypeError:
pass
except ValueError:
pass
finally:
pass
# end try
with a:
pass
# end with
class A:
pass
# end class A
def f():
pass
# end def f
""")
self.pindent_test(clean, closed)
def test_multilevel(self):
clean = textwrap.dedent("""\
def foobar(a, b):
if a == b:
a = a+1
elif a < b:
b = b-1
if b > a: a = a-1
else:
print 'oops!'
""")
closed = textwrap.dedent("""\
def foobar(a, b):
if a == b:
a = a+1
elif a < b:
b = b-1
if b > a: a = a-1
# end if
else:
print 'oops!'
# end if
# end def foobar
""")
self.pindent_test(clean, closed)
def test_preserve_indents(self):
clean = textwrap.dedent("""\
if a:
if b:
pass
""")
closed = textwrap.dedent("""\
if a:
if b:
pass
# end if
# end if
""")
self.assertEqual(self.pindent(clean, '-c'), closed)
self.assertEqual(self.pindent(closed, '-d'), clean)
broken = self.lstriplines(closed)
self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '9'), closed)
clean = textwrap.dedent("""\
if a:
\tif b:
\t\tpass
""")
closed = textwrap.dedent("""\
if a:
\tif b:
\t\tpass
\t# end if
# end if
""")
self.assertEqual(self.pindent(clean, '-c'), closed)
self.assertEqual(self.pindent(closed, '-d'), clean)
broken = self.lstriplines(closed)
self.assertEqual(self.pindent(broken, '-r'), closed)
def test_escaped_newline(self):
clean = textwrap.dedent("""\
class\\
\\
A:
def\
\\
f:
pass
""")
closed = textwrap.dedent("""\
class\\
\\
A:
def\
\\
f:
pass
# end def f
# end class A
""")
self.assertEqual(self.pindent(clean, '-c'), closed)
self.assertEqual(self.pindent(closed, '-d'), clean)
def test_empty_line(self):
clean = textwrap.dedent("""\
if a:
pass
""")
closed = textwrap.dedent("""\
if a:
pass
# end if
""")
self.pindent_test(clean, closed)
def test_oneline(self):
clean = textwrap.dedent("""\
if a: pass
""")
closed = textwrap.dedent("""\
if a: pass
# end if
""")
self.pindent_test(clean, closed)
class TestSundryScripts(unittest.TestCase):
# At least make sure the rest don't have syntax errors. When tests are
# added for a script it should be added to the whitelist below.
......
Misc/NEWS
View file @ 5150a339
......@@ -424,6 +424,8 @@ Library
Tests
-----
- Issue #15539: Added regression tests for Tools/scripts/pindent.py.
- Issue #16925: test_configparser now works with unittest test discovery.
Patch by Zachary Ware.
......@@ -571,6 +573,11 @@ Documentation
Tools/Demos
-----------
- Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py. Now
pindent.py works with a "with" statement. pindent.py no longer produces
improper indentation. pindent.py now works with continued lines broken after
"class" or "def" keywords and with continuations at the start of line.
- Issue #15378: Fix Tools/unicode/comparecodecs.py. Patch by Serhiy Storchaka.
......
Tools/scripts/pindent.py
View file @ 5150a339
......@@ -79,8 +79,9 @@
# Defaults
STEPSIZE = 8
TABSIZE = 8
EXPANDTABS = 0
EXPANDTABS = False
import io
import re
import sys
......@@ -89,7 +90,8 @@ next['if'] = next['elif'] = 'elif', 'else', 'end'
next['while'] = next['for'] = 'else', 'end'
next['try'] = 'except', 'finally'
next['except'] = 'except', 'else', 'finally', 'end'
next['else'] = next['finally'] = next['def'] = next['class'] = 'end'
next['else'] = next['finally'] = next['with'] = \
next['def'] = next['class'] = 'end'
next['end'] = ()
start = 'if', 'while', 'for', 'try', 'with', 'def', 'class'
......@@ -105,11 +107,11 @@ class PythonIndenter:
self.expandtabs = expandtabs
self._write = fpo.write
self.kwprog = re.compile(
r'^\s*(?P<kw>[a-z]+)'
r'(\s+(?P<id>[a-zA-Z_]\w*))?'
r'^(?:\s|\\\n)*(?P<kw>[a-z]+)'
r'((?:\s|\\\n)+(?P<id>[a-zA-Z_]\w*))?'
r'[^\w]')
self.endprog = re.compile(
r'^\s*#?\s*end\s+(?P<kw>[a-z]+)'
r'^(?:\s|\\\n)*#?\s*end\s+(?P<kw>[a-z]+)'
r'(\s+(?P<id>[a-zA-Z_]\w*))?'
r'[^\w]')
self.wsprog = re.compile(r'^[ \t]*')
......@@ -125,7 +127,7 @@ class PythonIndenter:
def readline(self):
line = self.fpi.readline()
if line: self.lineno = self.lineno + 1
if line: self.lineno += 1
# end if
return line
# end def readline
......@@ -143,27 +145,24 @@ class PythonIndenter:
line2 = self.readline()
if not line2: break
# end if
line = line + line2
line += line2
# end while
return line
# end def getline
def putline(self, line, indent = None):
if indent is None:
self.write(line)
return
# end if
def putline(self, line, indent):
tabs, spaces = divmod(indent*self.indentsize, self.tabsize)
i = 0
m = self.wsprog.match(line)
if m: i = m.end()
i = self.wsprog.match(line).end()
line = line[i:]
if line[:1] not in ('\n', '\r', ''):
line = '\t'*tabs + ' '*spaces + line
# end if
self.write('\t'*tabs + ' '*spaces + line[i:])
self.write(line)
# end def putline
def reformat(self):
stack = []
while 1:
while True:
line = self.getline()
if not line: break # EOF
# end if
......@@ -173,10 +172,9 @@ class PythonIndenter:
kw2 = m.group('kw')
if not stack:
self.error('unexpected end')
elif stack[-1][0] != kw2:
elif stack.pop()[0] != kw2:
self.error('unmatched end')
# end if
del stack[-1:]
self.putline(line, len(stack))
continue
# end if
......@@ -208,23 +206,23 @@ class PythonIndenter:
def delete(self):
begin_counter = 0
end_counter = 0
while 1:
while True:
line = self.getline()
if not line: break # EOF
# end if
m = self.endprog.match(line)
if m:
end_counter = end_counter + 1
end_counter += 1
continue
# end if
m = self.kwprog.match(line)
if m:
kw = m.group('kw')
if kw in start:
begin_counter = begin_counter + 1
begin_counter += 1
# end if
# end if
self.putline(line)
self.write(line)
# end while
if begin_counter - end_counter < 0:
sys.stderr.write('Warning: input contained more end tags than expected\n')
......@@ -234,17 +232,12 @@ class PythonIndenter:
# end def delete
def complete(self):
self.indentsize = 1
stack = []
todo = []
thisid = ''
current, firstkw, lastkw, topid = 0, '', '', ''
while 1:
currentws = thisid = firstkw = lastkw = topid = ''
while True:
line = self.getline()
i = 0
m = self.wsprog.match(line)
if m: i = m.end()
# end if
i = self.wsprog.match(line).end()
m = self.endprog.match(line)
if m:
thiskw = 'end'
......@@ -269,7 +262,9 @@ class PythonIndenter:
thiskw = ''
# end if
# end if
indent = len(line[:i].expandtabs(self.tabsize))
indentws = line[:i]
indent = len(indentws.expandtabs(self.tabsize))
current = len(currentws.expandtabs(self.tabsize))
while indent < current:
if firstkw:
if topid:
......@@ -278,11 +273,11 @@ class PythonIndenter:
else:
s = '# end %s\n' % firstkw
# end if
self.putline(s, current)
self.write(currentws + s)
firstkw = lastkw = ''
# end if
current, firstkw, lastkw, topid = stack[-1]
del stack[-1]
currentws, firstkw, lastkw, topid = stack.pop()
current = len(currentws.expandtabs(self.tabsize))
# end while
if indent == current and firstkw:
if thiskw == 'end':
......@@ -297,18 +292,18 @@ class PythonIndenter:
else:
s = '# end %s\n' % firstkw
# end if
self.putline(s, current)
self.write(currentws + s)
firstkw = lastkw = topid = ''
# end if
# end if
if indent > current:
stack.append((current, firstkw, lastkw, topid))
stack.append((currentws, firstkw, lastkw, topid))
if thiskw and thiskw not in start:
# error
thiskw = ''
# end if
current, firstkw, lastkw, topid = \
indent, thiskw, thiskw, thisid
currentws, firstkw, lastkw, topid = \
indentws, thiskw, thiskw, thisid
# end if
if thiskw:
if thiskw in start:
......@@ -326,7 +321,6 @@ class PythonIndenter:
self.write(line)
# end while
# end def complete
# end class PythonIndenter
# Simplified user interface
......@@ -352,76 +346,34 @@ def reformat_filter(input = sys.stdin, output = sys.stdout,
pi.reformat()
# end def reformat_filter
class StringReader:
def __init__(self, buf):
self.buf = buf
self.pos = 0
self.len = len(self.buf)
# end def __init__
def read(self, n = 0):
if n <= 0:
n = self.len - self.pos
else:
n = min(n, self.len - self.pos)
# end if
r = self.buf[self.pos : self.pos + n]
self.pos = self.pos + n
return r
# end def read
def readline(self):
i = self.buf.find('\n', self.pos)
return self.read(i + 1 - self.pos)
# end def readline
def readlines(self):
lines = []
line = self.readline()
while line:
lines.append(line)
line = self.readline()
# end while
return lines
# end def readlines
# seek/tell etc. are left as an exercise for the reader
# end class StringReader
class StringWriter:
def __init__(self):
self.buf = ''
# end def __init__
def write(self, s):
self.buf = self.buf + s
# end def write
def getvalue(self):
return self.buf
# end def getvalue
# end class StringWriter
def complete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
input = StringReader(source)
output = StringWriter()
input = io.StringIO(source)
output = io.StringIO()
pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
pi.complete()
return output.getvalue()
# end def complete_string
def delete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
input = StringReader(source)
output = StringWriter()
input = io.StringIO(source)
output = io.StringIO()
pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
pi.delete()
return output.getvalue()
# end def delete_string
def reformat_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
input = StringReader(source)
output = StringWriter()
input = io.StringIO(source)
output = io.StringIO()
pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
pi.reformat()
return output.getvalue()
# end def reformat_string
def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
source = open(filename, 'r').read()
with open(filename, 'r') as f:
source = f.read()
# end with
result = complete_string(source, stepsize, tabsize, expandtabs)
if source == result: return 0
# end if
......@@ -429,14 +381,16 @@ def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs =
try: os.rename(filename, filename + '~')
except os.error: pass
# end try
f = open(filename, 'w')
f.write(result)
f.close()
with open(filename, 'w') as f:
f.write(result)
# end with
return 1
# end def complete_file
def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
source = open(filename, 'r').read()
with open(filename, 'r') as f:
source = f.read()
# end with
result = delete_string(source, stepsize, tabsize, expandtabs)
if source == result: return 0
# end if
......@@ -444,14 +398,16 @@ def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = E
try: os.rename(filename, filename + '~')
except os.error: pass
# end try
f = open(filename, 'w')
f.write(result)
f.close()
with open(filename, 'w') as f:
f.write(result)
# end with
return 1
# end def delete_file
def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
source = open(filename, 'r').read()
with open(filename, 'r') as f:
source = f.read()
# end with
result = reformat_string(source, stepsize, tabsize, expandtabs)
if source == result: return 0
# end if
......@@ -459,9 +415,9 @@ def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs =
try: os.rename(filename, filename + '~')
except os.error: pass
# end try
f = open(filename, 'w')
f.write(result)
f.close()
with open(filename, 'w') as f:
f.write(result)
# end with
return 1
# end def reformat_file
......@@ -474,7 +430,7 @@ usage: pindent (-c|-d|-r) [-s stepsize] [-t tabsize] [-e] [file] ...
-r : reformat a completed program (use #end directives)
-s stepsize: indentation step (default %(STEPSIZE)d)
-t tabsize : the worth in spaces of a tab (default %(TABSIZE)d)
-e : expand TABs into spaces (defailt OFF)
-e : expand TABs into spaces (default OFF)
[file] ... : files are changed in place, with backups in file~
If no files are specified or a single - is given,
the program acts as a filter (reads stdin, writes stdout).
......@@ -517,7 +473,7 @@ def test():
elif o == '-t':
tabsize = int(a)
elif o == '-e':
expandtabs = 1
expandtabs = True
# end if
# end for
if not action:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7