[Python-checkins] cpython (merge 3.2 -> 3.3): Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py.
serhiy.storchaka
python-checkins at python.org
Fri Jan 11 11:40:11 CET 2013
http://hg.python.org/cpython/rev/01df1f7841b2
changeset: 81406:01df1f7841b2
branch: 3.3
parent: 81398:3e884b3804b3
parent: 81405:9df6b707aef9
user: Serhiy Storchaka <storchaka at gmail.com>
date: Fri Jan 11 12:10:57 2013 +0200
summary:
Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py.
Now pindent.py works with a "with" statement. pindent.py no longer produces
improper indentation. pindent.py now works with continued lines broken after
"class" or "def" keywords and with continuations at the start of line. Added
regression tests for pindent.py. Modernized pindent.py.
files:
Lib/test/test_tools.py | 326 ++++++++++++++++++++++++++-
Misc/NEWS | 7 +
Tools/scripts/pindent.py | 166 +++++--------
3 files changed, 393 insertions(+), 106 deletions(-)
diff --git a/Lib/test/test_tools.py b/Lib/test/test_tools.py
--- a/Lib/test/test_tools.py
+++ b/Lib/test/test_tools.py
@@ -9,10 +9,13 @@
import importlib.machinery
import unittest
from unittest import mock
+import shutil
+import subprocess
import sysconfig
import tempfile
+import textwrap
from test import support
-from test.script_helper import assert_python_ok
+from test.script_helper import assert_python_ok, temp_dir
if not sysconfig.is_python_build():
# XXX some installers do contain the tools, should we detect that
@@ -36,6 +39,327 @@
self.assertGreater(err, b'')
+class PindentTests(unittest.TestCase):
+ script = os.path.join(scriptsdir, 'pindent.py')
+
+ def assertFileEqual(self, fn1, fn2):
+ with open(fn1) as f1, open(fn2) as f2:
+ self.assertEqual(f1.readlines(), f2.readlines())
+
+ def pindent(self, source, *args):
+ with subprocess.Popen(
+ (sys.executable, self.script) + args,
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ universal_newlines=True) as proc:
+ out, err = proc.communicate(source)
+ self.assertIsNone(err)
+ return out
+
+ def lstriplines(self, data):
+ return '\n'.join(line.lstrip() for line in data.splitlines()) + '\n'
+
+ def test_selftest(self):
+ with temp_dir() as directory:
+ data_path = os.path.join(directory, '_test.py')
+ with open(self.script) as f:
+ closed = f.read()
+ with open(data_path, 'w') as f:
+ f.write(closed)
+
+ rc, out, err = assert_python_ok(self.script, '-d', data_path)
+ self.assertEqual(out, b'')
+ self.assertEqual(err, b'')
+ backup = data_path + '~'
+ self.assertTrue(os.path.exists(backup))
+ with open(backup) as f:
+ self.assertEqual(f.read(), closed)
+ with open(data_path) as f:
+ clean = f.read()
+ compile(clean, '_test.py', 'exec')
+ self.assertEqual(self.pindent(clean, '-c'), closed)
+ self.assertEqual(self.pindent(closed, '-d'), clean)
+
+ rc, out, err = assert_python_ok(self.script, '-c', data_path)
+ self.assertEqual(out, b'')
+ self.assertEqual(err, b'')
+ with open(backup) as f:
+ self.assertEqual(f.read(), clean)
+ with open(data_path) as f:
+ self.assertEqual(f.read(), closed)
+
+ broken = self.lstriplines(closed)
+ with open(data_path, 'w') as f:
+ f.write(broken)
+ rc, out, err = assert_python_ok(self.script, '-r', data_path)
+ self.assertEqual(out, b'')
+ self.assertEqual(err, b'')
+ with open(backup) as f:
+ self.assertEqual(f.read(), broken)
+ with open(data_path) as f:
+ indented = f.read()
+ compile(indented, '_test.py', 'exec')
+ self.assertEqual(self.pindent(broken, '-r'), indented)
+
+ def pindent_test(self, clean, closed):
+ self.assertEqual(self.pindent(clean, '-c'), closed)
+ self.assertEqual(self.pindent(closed, '-d'), clean)
+ broken = self.lstriplines(closed)
+ self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '4'), closed)
+
+ def test_statements(self):
+ clean = textwrap.dedent("""\
+ if a:
+ pass
+
+ if a:
+ pass
+ else:
+ pass
+
+ if a:
+ pass
+ elif:
+ pass
+ else:
+ pass
+
+ while a:
+ break
+
+ while a:
+ break
+ else:
+ pass
+
+ for i in a:
+ break
+
+ for i in a:
+ break
+ else:
+ pass
+
+ try:
+ pass
+ finally:
+ pass
+
+ try:
+ pass
+ except TypeError:
+ pass
+ except ValueError:
+ pass
+ else:
+ pass
+
+ try:
+ pass
+ except TypeError:
+ pass
+ except ValueError:
+ pass
+ finally:
+ pass
+
+ with a:
+ pass
+
+ class A:
+ pass
+
+ def f():
+ pass
+ """)
+
+ closed = textwrap.dedent("""\
+ if a:
+ pass
+ # end if
+
+ if a:
+ pass
+ else:
+ pass
+ # end if
+
+ if a:
+ pass
+ elif:
+ pass
+ else:
+ pass
+ # end if
+
+ while a:
+ break
+ # end while
+
+ while a:
+ break
+ else:
+ pass
+ # end while
+
+ for i in a:
+ break
+ # end for
+
+ for i in a:
+ break
+ else:
+ pass
+ # end for
+
+ try:
+ pass
+ finally:
+ pass
+ # end try
+
+ try:
+ pass
+ except TypeError:
+ pass
+ except ValueError:
+ pass
+ else:
+ pass
+ # end try
+
+ try:
+ pass
+ except TypeError:
+ pass
+ except ValueError:
+ pass
+ finally:
+ pass
+ # end try
+
+ with a:
+ pass
+ # end with
+
+ class A:
+ pass
+ # end class A
+
+ def f():
+ pass
+ # end def f
+ """)
+ self.pindent_test(clean, closed)
+
+ def test_multilevel(self):
+ clean = textwrap.dedent("""\
+ def foobar(a, b):
+ if a == b:
+ a = a+1
+ elif a < b:
+ b = b-1
+ if b > a: a = a-1
+ else:
+ print 'oops!'
+ """)
+ closed = textwrap.dedent("""\
+ def foobar(a, b):
+ if a == b:
+ a = a+1
+ elif a < b:
+ b = b-1
+ if b > a: a = a-1
+ # end if
+ else:
+ print 'oops!'
+ # end if
+ # end def foobar
+ """)
+ self.pindent_test(clean, closed)
+
+ def test_preserve_indents(self):
+ clean = textwrap.dedent("""\
+ if a:
+ if b:
+ pass
+ """)
+ closed = textwrap.dedent("""\
+ if a:
+ if b:
+ pass
+ # end if
+ # end if
+ """)
+ self.assertEqual(self.pindent(clean, '-c'), closed)
+ self.assertEqual(self.pindent(closed, '-d'), clean)
+ broken = self.lstriplines(closed)
+ self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '9'), closed)
+ clean = textwrap.dedent("""\
+ if a:
+ \tif b:
+ \t\tpass
+ """)
+ closed = textwrap.dedent("""\
+ if a:
+ \tif b:
+ \t\tpass
+ \t# end if
+ # end if
+ """)
+ self.assertEqual(self.pindent(clean, '-c'), closed)
+ self.assertEqual(self.pindent(closed, '-d'), clean)
+ broken = self.lstriplines(closed)
+ self.assertEqual(self.pindent(broken, '-r'), closed)
+
+ def test_escaped_newline(self):
+ clean = textwrap.dedent("""\
+ class\\
+ \\
+ A:
+ def\
+ \\
+ f:
+ pass
+ """)
+ closed = textwrap.dedent("""\
+ class\\
+ \\
+ A:
+ def\
+ \\
+ f:
+ pass
+ # end def f
+ # end class A
+ """)
+ self.assertEqual(self.pindent(clean, '-c'), closed)
+ self.assertEqual(self.pindent(closed, '-d'), clean)
+
+ def test_empty_line(self):
+ clean = textwrap.dedent("""\
+ if a:
+
+ pass
+ """)
+ closed = textwrap.dedent("""\
+ if a:
+
+ pass
+ # end if
+ """)
+ self.pindent_test(clean, closed)
+
+ def test_oneline(self):
+ clean = textwrap.dedent("""\
+ if a: pass
+ """)
+ closed = textwrap.dedent("""\
+ if a: pass
+ # end if
+ """)
+ self.pindent_test(clean, closed)
+
+
class TestSundryScripts(unittest.TestCase):
# At least make sure the rest don't have syntax errors. When tests are
# added for a script it should be added to the whitelist below.
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -424,6 +424,8 @@
Tests
-----
+- Issue #15539: Added regression tests for Tools/scripts/pindent.py.
+
- Issue #16925: test_configparser now works with unittest test discovery.
Patch by Zachary Ware.
@@ -571,6 +573,11 @@
Tools/Demos
-----------
+- Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py. Now
+ pindent.py works with a "with" statement. pindent.py no longer produces
+ improper indentation. pindent.py now works with continued lines broken after
+ "class" or "def" keywords and with continuations at the start of line.
+
- Issue #15378: Fix Tools/unicode/comparecodecs.py. Patch by Serhiy Storchaka.
diff --git a/Tools/scripts/pindent.py b/Tools/scripts/pindent.py
--- a/Tools/scripts/pindent.py
+++ b/Tools/scripts/pindent.py
@@ -79,8 +79,9 @@
# Defaults
STEPSIZE = 8
TABSIZE = 8
-EXPANDTABS = 0
+EXPANDTABS = False
+import io
import re
import sys
@@ -89,7 +90,8 @@
next['while'] = next['for'] = 'else', 'end'
next['try'] = 'except', 'finally'
next['except'] = 'except', 'else', 'finally', 'end'
-next['else'] = next['finally'] = next['def'] = next['class'] = 'end'
+next['else'] = next['finally'] = next['with'] = \
+ next['def'] = next['class'] = 'end'
next['end'] = ()
start = 'if', 'while', 'for', 'try', 'with', 'def', 'class'
@@ -105,11 +107,11 @@
self.expandtabs = expandtabs
self._write = fpo.write
self.kwprog = re.compile(
- r'^\s*(?P<kw>[a-z]+)'
- r'(\s+(?P<id>[a-zA-Z_]\w*))?'
+ r'^(?:\s|\\\n)*(?P<kw>[a-z]+)'
+ r'((?:\s|\\\n)+(?P<id>[a-zA-Z_]\w*))?'
r'[^\w]')
self.endprog = re.compile(
- r'^\s*#?\s*end\s+(?P<kw>[a-z]+)'
+ r'^(?:\s|\\\n)*#?\s*end\s+(?P<kw>[a-z]+)'
r'(\s+(?P<id>[a-zA-Z_]\w*))?'
r'[^\w]')
self.wsprog = re.compile(r'^[ \t]*')
@@ -125,7 +127,7 @@
def readline(self):
line = self.fpi.readline()
- if line: self.lineno = self.lineno + 1
+ if line: self.lineno += 1
# end if
return line
# end def readline
@@ -143,27 +145,24 @@
line2 = self.readline()
if not line2: break
# end if
- line = line + line2
+ line += line2
# end while
return line
# end def getline
- def putline(self, line, indent = None):
- if indent is None:
- self.write(line)
- return
+ def putline(self, line, indent):
+ tabs, spaces = divmod(indent*self.indentsize, self.tabsize)
+ i = self.wsprog.match(line).end()
+ line = line[i:]
+ if line[:1] not in ('\n', '\r', ''):
+ line = '\t'*tabs + ' '*spaces + line
# end if
- tabs, spaces = divmod(indent*self.indentsize, self.tabsize)
- i = 0
- m = self.wsprog.match(line)
- if m: i = m.end()
- # end if
- self.write('\t'*tabs + ' '*spaces + line[i:])
+ self.write(line)
# end def putline
def reformat(self):
stack = []
- while 1:
+ while True:
line = self.getline()
if not line: break # EOF
# end if
@@ -173,10 +172,9 @@
kw2 = m.group('kw')
if not stack:
self.error('unexpected end')
- elif stack[-1][0] != kw2:
+ elif stack.pop()[0] != kw2:
self.error('unmatched end')
# end if
- del stack[-1:]
self.putline(line, len(stack))
continue
# end if
@@ -208,23 +206,23 @@
def delete(self):
begin_counter = 0
end_counter = 0
- while 1:
+ while True:
line = self.getline()
if not line: break # EOF
# end if
m = self.endprog.match(line)
if m:
- end_counter = end_counter + 1
+ end_counter += 1
continue
# end if
m = self.kwprog.match(line)
if m:
kw = m.group('kw')
if kw in start:
- begin_counter = begin_counter + 1
+ begin_counter += 1
# end if
# end if
- self.putline(line)
+ self.write(line)
# end while
if begin_counter - end_counter < 0:
sys.stderr.write('Warning: input contained more end tags than expected\n')
@@ -234,17 +232,12 @@
# end def delete
def complete(self):
- self.indentsize = 1
stack = []
todo = []
- thisid = ''
- current, firstkw, lastkw, topid = 0, '', '', ''
- while 1:
+ currentws = thisid = firstkw = lastkw = topid = ''
+ while True:
line = self.getline()
- i = 0
- m = self.wsprog.match(line)
- if m: i = m.end()
- # end if
+ i = self.wsprog.match(line).end()
m = self.endprog.match(line)
if m:
thiskw = 'end'
@@ -269,7 +262,9 @@
thiskw = ''
# end if
# end if
- indent = len(line[:i].expandtabs(self.tabsize))
+ indentws = line[:i]
+ indent = len(indentws.expandtabs(self.tabsize))
+ current = len(currentws.expandtabs(self.tabsize))
while indent < current:
if firstkw:
if topid:
@@ -278,11 +273,11 @@
else:
s = '# end %s\n' % firstkw
# end if
- self.putline(s, current)
+ self.write(currentws + s)
firstkw = lastkw = ''
# end if
- current, firstkw, lastkw, topid = stack[-1]
- del stack[-1]
+ currentws, firstkw, lastkw, topid = stack.pop()
+ current = len(currentws.expandtabs(self.tabsize))
# end while
if indent == current and firstkw:
if thiskw == 'end':
@@ -297,18 +292,18 @@
else:
s = '# end %s\n' % firstkw
# end if
- self.putline(s, current)
+ self.write(currentws + s)
firstkw = lastkw = topid = ''
# end if
# end if
if indent > current:
- stack.append((current, firstkw, lastkw, topid))
+ stack.append((currentws, firstkw, lastkw, topid))
if thiskw and thiskw not in start:
# error
thiskw = ''
# end if
- current, firstkw, lastkw, topid = \
- indent, thiskw, thiskw, thisid
+ currentws, firstkw, lastkw, topid = \
+ indentws, thiskw, thiskw, thisid
# end if
if thiskw:
if thiskw in start:
@@ -326,7 +321,6 @@
self.write(line)
# end while
# end def complete
-
# end class PythonIndenter
# Simplified user interface
@@ -352,76 +346,34 @@
pi.reformat()
# end def reformat_filter
-class StringReader:
- def __init__(self, buf):
- self.buf = buf
- self.pos = 0
- self.len = len(self.buf)
- # end def __init__
- def read(self, n = 0):
- if n <= 0:
- n = self.len - self.pos
- else:
- n = min(n, self.len - self.pos)
- # end if
- r = self.buf[self.pos : self.pos + n]
- self.pos = self.pos + n
- return r
- # end def read
- def readline(self):
- i = self.buf.find('\n', self.pos)
- return self.read(i + 1 - self.pos)
- # end def readline
- def readlines(self):
- lines = []
- line = self.readline()
- while line:
- lines.append(line)
- line = self.readline()
- # end while
- return lines
- # end def readlines
- # seek/tell etc. are left as an exercise for the reader
-# end class StringReader
-
-class StringWriter:
- def __init__(self):
- self.buf = ''
- # end def __init__
- def write(self, s):
- self.buf = self.buf + s
- # end def write
- def getvalue(self):
- return self.buf
- # end def getvalue
-# end class StringWriter
-
def complete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
- input = StringReader(source)
- output = StringWriter()
+ input = io.StringIO(source)
+ output = io.StringIO()
pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
pi.complete()
return output.getvalue()
# end def complete_string
def delete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
- input = StringReader(source)
- output = StringWriter()
+ input = io.StringIO(source)
+ output = io.StringIO()
pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
pi.delete()
return output.getvalue()
# end def delete_string
def reformat_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
- input = StringReader(source)
- output = StringWriter()
+ input = io.StringIO(source)
+ output = io.StringIO()
pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
pi.reformat()
return output.getvalue()
# end def reformat_string
def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
- source = open(filename, 'r').read()
+ with open(filename, 'r') as f:
+ source = f.read()
+ # end with
result = complete_string(source, stepsize, tabsize, expandtabs)
if source == result: return 0
# end if
@@ -429,14 +381,16 @@
try: os.rename(filename, filename + '~')
except os.error: pass
# end try
- f = open(filename, 'w')
- f.write(result)
- f.close()
+ with open(filename, 'w') as f:
+ f.write(result)
+ # end with
return 1
# end def complete_file
def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
- source = open(filename, 'r').read()
+ with open(filename, 'r') as f:
+ source = f.read()
+ # end with
result = delete_string(source, stepsize, tabsize, expandtabs)
if source == result: return 0
# end if
@@ -444,14 +398,16 @@
try: os.rename(filename, filename + '~')
except os.error: pass
# end try
- f = open(filename, 'w')
- f.write(result)
- f.close()
+ with open(filename, 'w') as f:
+ f.write(result)
+ # end with
return 1
# end def delete_file
def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
- source = open(filename, 'r').read()
+ with open(filename, 'r') as f:
+ source = f.read()
+ # end with
result = reformat_string(source, stepsize, tabsize, expandtabs)
if source == result: return 0
# end if
@@ -459,9 +415,9 @@
try: os.rename(filename, filename + '~')
except os.error: pass
# end try
- f = open(filename, 'w')
- f.write(result)
- f.close()
+ with open(filename, 'w') as f:
+ f.write(result)
+ # end with
return 1
# end def reformat_file
@@ -474,7 +430,7 @@
-r : reformat a completed program (use #end directives)
-s stepsize: indentation step (default %(STEPSIZE)d)
-t tabsize : the worth in spaces of a tab (default %(TABSIZE)d)
--e : expand TABs into spaces (defailt OFF)
+-e : expand TABs into spaces (default OFF)
[file] ... : files are changed in place, with backups in file~
If no files are specified or a single - is given,
the program acts as a filter (reads stdin, writes stdout).
@@ -517,7 +473,7 @@
elif o == '-t':
tabsize = int(a)
elif o == '-e':
- expandtabs = 1
+ expandtabs = True
# end if
# end for
if not action:
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list