[Python-Dev] [Python-checkins] cpython (merge 3.3 -> default): Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py.
Brett Cannon
brett at python.org
Fri Jan 11 18:08:18 CET 2013
This seems to have caused the Windows buildbots to fail.
On Fri, Jan 11, 2013 at 5:40 AM, serhiy.storchaka
<python-checkins at python.org> wrote:
> http://hg.python.org/cpython/rev/8452c23139c6
> changeset: 81407:8452c23139c6
> parent: 81399:5ec8daab477a
> parent: 81406:01df1f7841b2
> user: Serhiy Storchaka <storchaka at gmail.com>
> date: Fri Jan 11 12:12:32 2013 +0200
> summary:
> Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py.
> Now pindent.py works with a "with" statement. pindent.py no longer produces
> improper indentation. pindent.py now works with continued lines broken after
> "class" or "def" keywords and with continuations at the start of line. Added
> regression tests for pindent.py. Modernized pindent.py.
>
> files:
> Lib/test/test_tools.py | 326 ++++++++++++++++++++++++++-
> Misc/NEWS | 7 +
> Tools/scripts/pindent.py | 166 +++++--------
> 3 files changed, 393 insertions(+), 106 deletions(-)
>
>
> diff --git a/Lib/test/test_tools.py b/Lib/test/test_tools.py
> --- a/Lib/test/test_tools.py
> +++ b/Lib/test/test_tools.py
> @@ -9,10 +9,13 @@
> import importlib.machinery
> import unittest
> from unittest import mock
> +import shutil
> +import subprocess
> import sysconfig
> import tempfile
> +import textwrap
> from test import support
> -from test.script_helper import assert_python_ok
> +from test.script_helper import assert_python_ok, temp_dir
>
> if not sysconfig.is_python_build():
> # XXX some installers do contain the tools, should we detect that
> @@ -36,6 +39,327 @@
> self.assertGreater(err, b'')
>
>
> +class PindentTests(unittest.TestCase):
> + script = os.path.join(scriptsdir, 'pindent.py')
> +
> + def assertFileEqual(self, fn1, fn2):
> + with open(fn1) as f1, open(fn2) as f2:
> + self.assertEqual(f1.readlines(), f2.readlines())
> +
> + def pindent(self, source, *args):
> + with subprocess.Popen(
> + (sys.executable, self.script) + args,
> + stdin=subprocess.PIPE, stdout=subprocess.PIPE,
> + universal_newlines=True) as proc:
> + out, err = proc.communicate(source)
> + self.assertIsNone(err)
> + return out
> +
> + def lstriplines(self, data):
> + return '\n'.join(line.lstrip() for line in data.splitlines()) + '\n'
> +
> + def test_selftest(self):
> + with temp_dir() as directory:
> + data_path = os.path.join(directory, '_test.py')
> + with open(self.script) as f:
> + closed = f.read()
> + with open(data_path, 'w') as f:
> + f.write(closed)
> +
> + rc, out, err = assert_python_ok(self.script, '-d', data_path)
> + self.assertEqual(out, b'')
> + self.assertEqual(err, b'')
> + backup = data_path + '~'
> + self.assertTrue(os.path.exists(backup))
> + with open(backup) as f:
> + self.assertEqual(f.read(), closed)
> + with open(data_path) as f:
> + clean = f.read()
> + compile(clean, '_test.py', 'exec')
> + self.assertEqual(self.pindent(clean, '-c'), closed)
> + self.assertEqual(self.pindent(closed, '-d'), clean)
> +
> + rc, out, err = assert_python_ok(self.script, '-c', data_path)
> + self.assertEqual(out, b'')
> + self.assertEqual(err, b'')
> + with open(backup) as f:
> + self.assertEqual(f.read(), clean)
> + with open(data_path) as f:
> + self.assertEqual(f.read(), closed)
> +
> + broken = self.lstriplines(closed)
> + with open(data_path, 'w') as f:
> + f.write(broken)
> + rc, out, err = assert_python_ok(self.script, '-r', data_path)
> + self.assertEqual(out, b'')
> + self.assertEqual(err, b'')
> + with open(backup) as f:
> + self.assertEqual(f.read(), broken)
> + with open(data_path) as f:
> + indented = f.read()
> + compile(indented, '_test.py', 'exec')
> + self.assertEqual(self.pindent(broken, '-r'), indented)
> +
> + def pindent_test(self, clean, closed):
> + self.assertEqual(self.pindent(clean, '-c'), closed)
> + self.assertEqual(self.pindent(closed, '-d'), clean)
> + broken = self.lstriplines(closed)
> + self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '4'), closed)
> +
> + def test_statements(self):
> + clean = textwrap.dedent("""\
> + if a:
> + pass
> +
> + if a:
> + pass
> + else:
> + pass
> +
> + if a:
> + pass
> + elif:
> + pass
> + else:
> + pass
> +
> + while a:
> + break
> +
> + while a:
> + break
> + else:
> + pass
> +
> + for i in a:
> + break
> +
> + for i in a:
> + break
> + else:
> + pass
> +
> + try:
> + pass
> + finally:
> + pass
> +
> + try:
> + pass
> + except TypeError:
> + pass
> + except ValueError:
> + pass
> + else:
> + pass
> +
> + try:
> + pass
> + except TypeError:
> + pass
> + except ValueError:
> + pass
> + finally:
> + pass
> +
> + with a:
> + pass
> +
> + class A:
> + pass
> +
> + def f():
> + pass
> + """)
> +
> + closed = textwrap.dedent("""\
> + if a:
> + pass
> + # end if
> +
> + if a:
> + pass
> + else:
> + pass
> + # end if
> +
> + if a:
> + pass
> + elif:
> + pass
> + else:
> + pass
> + # end if
> +
> + while a:
> + break
> + # end while
> +
> + while a:
> + break
> + else:
> + pass
> + # end while
> +
> + for i in a:
> + break
> + # end for
> +
> + for i in a:
> + break
> + else:
> + pass
> + # end for
> +
> + try:
> + pass
> + finally:
> + pass
> + # end try
> +
> + try:
> + pass
> + except TypeError:
> + pass
> + except ValueError:
> + pass
> + else:
> + pass
> + # end try
> +
> + try:
> + pass
> + except TypeError:
> + pass
> + except ValueError:
> + pass
> + finally:
> + pass
> + # end try
> +
> + with a:
> + pass
> + # end with
> +
> + class A:
> + pass
> + # end class A
> +
> + def f():
> + pass
> + # end def f
> + """)
> + self.pindent_test(clean, closed)
> +
> + def test_multilevel(self):
> + clean = textwrap.dedent("""\
> + def foobar(a, b):
> + if a == b:
> + a = a+1
> + elif a < b:
> + b = b-1
> + if b > a: a = a-1
> + else:
> + print 'oops!'
> + """)
> + closed = textwrap.dedent("""\
> + def foobar(a, b):
> + if a == b:
> + a = a+1
> + elif a < b:
> + b = b-1
> + if b > a: a = a-1
> + # end if
> + else:
> + print 'oops!'
> + # end if
> + # end def foobar
> + """)
> + self.pindent_test(clean, closed)
> +
> + def test_preserve_indents(self):
> + clean = textwrap.dedent("""\
> + if a:
> + if b:
> + pass
> + """)
> + closed = textwrap.dedent("""\
> + if a:
> + if b:
> + pass
> + # end if
> + # end if
> + """)
> + self.assertEqual(self.pindent(clean, '-c'), closed)
> + self.assertEqual(self.pindent(closed, '-d'), clean)
> + broken = self.lstriplines(closed)
> + self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '9'), closed)
> + clean = textwrap.dedent("""\
> + if a:
> + \tif b:
> + \t\tpass
> + """)
> + closed = textwrap.dedent("""\
> + if a:
> + \tif b:
> + \t\tpass
> + \t# end if
> + # end if
> + """)
> + self.assertEqual(self.pindent(clean, '-c'), closed)
> + self.assertEqual(self.pindent(closed, '-d'), clean)
> + broken = self.lstriplines(closed)
> + self.assertEqual(self.pindent(broken, '-r'), closed)
> +
> + def test_escaped_newline(self):
> + clean = textwrap.dedent("""\
> + class\\
> + \\
> + A:
> + def\
> + \\
> + f:
> + pass
> + """)
> + closed = textwrap.dedent("""\
> + class\\
> + \\
> + A:
> + def\
> + \\
> + f:
> + pass
> + # end def f
> + # end class A
> + """)
> + self.assertEqual(self.pindent(clean, '-c'), closed)
> + self.assertEqual(self.pindent(closed, '-d'), clean)
> +
> + def test_empty_line(self):
> + clean = textwrap.dedent("""\
> + if a:
> +
> + pass
> + """)
> + closed = textwrap.dedent("""\
> + if a:
> +
> + pass
> + # end if
> + """)
> + self.pindent_test(clean, closed)
> +
> + def test_oneline(self):
> + clean = textwrap.dedent("""\
> + if a: pass
> + """)
> + closed = textwrap.dedent("""\
> + if a: pass
> + # end if
> + """)
> + self.pindent_test(clean, closed)
> +
> +
> class TestSundryScripts(unittest.TestCase):
> # At least make sure the rest don't have syntax errors. When tests are
> # added for a script it should be added to the whitelist below.
> diff --git a/Misc/NEWS b/Misc/NEWS
> --- a/Misc/NEWS
> +++ b/Misc/NEWS
> @@ -621,6 +621,8 @@
> Tests
> -----
>
> +- Issue #15539: Added regression tests for Tools/scripts/pindent.py.
> +
> - Issue #16836: Enable IPv6 support even if IPv6 is disabled on the build host.
>
> - Issue #16925: test_configparser now works with unittest test discovery.
> @@ -777,6 +779,11 @@
> Tools/Demos
> -----------
>
> +- Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py. Now
> + pindent.py works with a "with" statement. pindent.py no longer produces
> + improper indentation. pindent.py now works with continued lines broken after
> + "class" or "def" keywords and with continuations at the start of line.
> +
> - Issue #11797: Add a 2to3 fixer that maps reload() to imp.reload().
>
> - Issue #10966: Remove the concept of unexpected skipped tests.
> diff --git a/Tools/scripts/pindent.py b/Tools/scripts/pindent.py
> --- a/Tools/scripts/pindent.py
> +++ b/Tools/scripts/pindent.py
> @@ -79,8 +79,9 @@
> # Defaults
> STEPSIZE = 8
> TABSIZE = 8
> -EXPANDTABS = 0
> +EXPANDTABS = False
>
> +import io
> import re
> import sys
>
> @@ -89,7 +90,8 @@
> next['while'] = next['for'] = 'else', 'end'
> next['try'] = 'except', 'finally'
> next['except'] = 'except', 'else', 'finally', 'end'
> -next['else'] = next['finally'] = next['def'] = next['class'] = 'end'
> +next['else'] = next['finally'] = next['with'] = \
> + next['def'] = next['class'] = 'end'
> next['end'] = ()
> start = 'if', 'while', 'for', 'try', 'with', 'def', 'class'
>
> @@ -105,11 +107,11 @@
> self.expandtabs = expandtabs
> self._write = fpo.write
> self.kwprog = re.compile(
> - r'^\s*(?P<kw>[a-z]+)'
> - r'(\s+(?P<id>[a-zA-Z_]\w*))?'
> + r'^(?:\s|\\\n)*(?P<kw>[a-z]+)'
> + r'((?:\s|\\\n)+(?P<id>[a-zA-Z_]\w*))?'
> r'[^\w]')
> self.endprog = re.compile(
> - r'^\s*#?\s*end\s+(?P<kw>[a-z]+)'
> + r'^(?:\s|\\\n)*#?\s*end\s+(?P<kw>[a-z]+)'
> r'(\s+(?P<id>[a-zA-Z_]\w*))?'
> r'[^\w]')
> self.wsprog = re.compile(r'^[ \t]*')
> @@ -125,7 +127,7 @@
>
> def readline(self):
> line = self.fpi.readline()
> - if line: self.lineno = self.lineno + 1
> + if line: self.lineno += 1
> # end if
> return line
> # end def readline
> @@ -143,27 +145,24 @@
> line2 = self.readline()
> if not line2: break
> # end if
> - line = line + line2
> + line += line2
> # end while
> return line
> # end def getline
>
> - def putline(self, line, indent = None):
> - if indent is None:
> - self.write(line)
> - return
> + def putline(self, line, indent):
> + tabs, spaces = divmod(indent*self.indentsize, self.tabsize)
> + i = self.wsprog.match(line).end()
> + line = line[i:]
> + if line[:1] not in ('\n', '\r', ''):
> + line = '\t'*tabs + ' '*spaces + line
> # end if
> - tabs, spaces = divmod(indent*self.indentsize, self.tabsize)
> - i = 0
> - m = self.wsprog.match(line)
> - if m: i = m.end()
> - # end if
> - self.write('\t'*tabs + ' '*spaces + line[i:])
> + self.write(line)
> # end def putline
>
> def reformat(self):
> stack = []
> - while 1:
> + while True:
> line = self.getline()
> if not line: break # EOF
> # end if
> @@ -173,10 +172,9 @@
> kw2 = m.group('kw')
> if not stack:
> self.error('unexpected end')
> - elif stack[-1][0] != kw2:
> + elif stack.pop()[0] != kw2:
> self.error('unmatched end')
> # end if
> - del stack[-1:]
> self.putline(line, len(stack))
> continue
> # end if
> @@ -208,23 +206,23 @@
> def delete(self):
> begin_counter = 0
> end_counter = 0
> - while 1:
> + while True:
> line = self.getline()
> if not line: break # EOF
> # end if
> m = self.endprog.match(line)
> if m:
> - end_counter = end_counter + 1
> + end_counter += 1
> continue
> # end if
> m = self.kwprog.match(line)
> if m:
> kw = m.group('kw')
> if kw in start:
> - begin_counter = begin_counter + 1
> + begin_counter += 1
> # end if
> # end if
> - self.putline(line)
> + self.write(line)
> # end while
> if begin_counter - end_counter < 0:
> sys.stderr.write('Warning: input contained more end tags than expected\n')
> @@ -234,17 +232,12 @@
> # end def delete
>
> def complete(self):
> - self.indentsize = 1
> stack = []
> todo = []
> - thisid = ''
> - current, firstkw, lastkw, topid = 0, '', '', ''
> - while 1:
> + currentws = thisid = firstkw = lastkw = topid = ''
> + while True:
> line = self.getline()
> - i = 0
> - m = self.wsprog.match(line)
> - if m: i = m.end()
> - # end if
> + i = self.wsprog.match(line).end()
> m = self.endprog.match(line)
> if m:
> thiskw = 'end'
> @@ -269,7 +262,9 @@
> thiskw = ''
> # end if
> # end if
> - indent = len(line[:i].expandtabs(self.tabsize))
> + indentws = line[:i]
> + indent = len(indentws.expandtabs(self.tabsize))
> + current = len(currentws.expandtabs(self.tabsize))
> while indent < current:
> if firstkw:
> if topid:
> @@ -278,11 +273,11 @@
> else:
> s = '# end %s\n' % firstkw
> # end if
> - self.putline(s, current)
> + self.write(currentws + s)
> firstkw = lastkw = ''
> # end if
> - current, firstkw, lastkw, topid = stack[-1]
> - del stack[-1]
> + currentws, firstkw, lastkw, topid = stack.pop()
> + current = len(currentws.expandtabs(self.tabsize))
> # end while
> if indent == current and firstkw:
> if thiskw == 'end':
> @@ -297,18 +292,18 @@
> else:
> s = '# end %s\n' % firstkw
> # end if
> - self.putline(s, current)
> + self.write(currentws + s)
> firstkw = lastkw = topid = ''
> # end if
> # end if
> if indent > current:
> - stack.append((current, firstkw, lastkw, topid))
> + stack.append((currentws, firstkw, lastkw, topid))
> if thiskw and thiskw not in start:
> # error
> thiskw = ''
> # end if
> - current, firstkw, lastkw, topid = \
> - indent, thiskw, thiskw, thisid
> + currentws, firstkw, lastkw, topid = \
> + indentws, thiskw, thiskw, thisid
> # end if
> if thiskw:
> if thiskw in start:
> @@ -326,7 +321,6 @@
> self.write(line)
> # end while
> # end def complete
> -
> # end class PythonIndenter
>
> # Simplified user interface
> @@ -352,76 +346,34 @@
> pi.reformat()
> # end def reformat_filter
>
> -class StringReader:
> - def __init__(self, buf):
> - self.buf = buf
> - self.pos = 0
> - self.len = len(self.buf)
> - # end def __init__
> - def read(self, n = 0):
> - if n <= 0:
> - n = self.len - self.pos
> - else:
> - n = min(n, self.len - self.pos)
> - # end if
> - r = self.buf[self.pos : self.pos + n]
> - self.pos = self.pos + n
> - return r
> - # end def read
> - def readline(self):
> - i = self.buf.find('\n', self.pos)
> - return self.read(i + 1 - self.pos)
> - # end def readline
> - def readlines(self):
> - lines = []
> - line = self.readline()
> - while line:
> - lines.append(line)
> - line = self.readline()
> - # end while
> - return lines
> - # end def readlines
> - # seek/tell etc. are left as an exercise for the reader
> -# end class StringReader
> -
> -class StringWriter:
> - def __init__(self):
> - self.buf = ''
> - # end def __init__
> - def write(self, s):
> - self.buf = self.buf + s
> - # end def write
> - def getvalue(self):
> - return self.buf
> - # end def getvalue
> -# end class StringWriter
> -
> def complete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
> - input = StringReader(source)
> - output = StringWriter()
> + input = io.StringIO(source)
> + output = io.StringIO()
> pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
> pi.complete()
> return output.getvalue()
> # end def complete_string
>
> def delete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
> - input = StringReader(source)
> - output = StringWriter()
> + input = io.StringIO(source)
> + output = io.StringIO()
> pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
> pi.delete()
> return output.getvalue()
> # end def delete_string
>
> def reformat_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
> - input = StringReader(source)
> - output = StringWriter()
> + input = io.StringIO(source)
> + output = io.StringIO()
> pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
> pi.reformat()
> return output.getvalue()
> # end def reformat_string
>
> def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
> - source = open(filename, 'r').read()
> + with open(filename, 'r') as f:
> + source = f.read()
> + # end with
> result = complete_string(source, stepsize, tabsize, expandtabs)
> if source == result: return 0
> # end if
> @@ -429,14 +381,16 @@
> try: os.rename(filename, filename + '~')
> except OSError: pass
> # end try
> - f = open(filename, 'w')
> - f.write(result)
> - f.close()
> + with open(filename, 'w') as f:
> + f.write(result)
> + # end with
> return 1
> # end def complete_file
>
> def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
> - source = open(filename, 'r').read()
> + with open(filename, 'r') as f:
> + source = f.read()
> + # end with
> result = delete_string(source, stepsize, tabsize, expandtabs)
> if source == result: return 0
> # end if
> @@ -444,14 +398,16 @@
> try: os.rename(filename, filename + '~')
> except OSError: pass
> # end try
> - f = open(filename, 'w')
> - f.write(result)
> - f.close()
> + with open(filename, 'w') as f:
> + f.write(result)
> + # end with
> return 1
> # end def delete_file
>
> def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
> - source = open(filename, 'r').read()
> + with open(filename, 'r') as f:
> + source = f.read()
> + # end with
> result = reformat_string(source, stepsize, tabsize, expandtabs)
> if source == result: return 0
> # end if
> @@ -459,9 +415,9 @@
> try: os.rename(filename, filename + '~')
> except OSError: pass
> # end try
> - f = open(filename, 'w')
> - f.write(result)
> - f.close()
> + with open(filename, 'w') as f:
> + f.write(result)
> + # end with
> return 1
> # end def reformat_file
>
> @@ -474,7 +430,7 @@
> -r : reformat a completed program (use #end directives)
> -s stepsize: indentation step (default %(STEPSIZE)d)
> -t tabsize : the worth in spaces of a tab (default %(TABSIZE)d)
> --e : expand TABs into spaces (defailt OFF)
> +-e : expand TABs into spaces (default OFF)
> [file] ... : files are changed in place, with backups in file~
> If no files are specified or a single - is given,
> the program acts as a filter (reads stdin, writes stdout).
> @@ -517,7 +473,7 @@
> elif o == '-t':
> tabsize = int(a)
> elif o == '-e':
> - expandtabs = 1
> + expandtabs = True
> # end if
> # end for
> if not action:
>
> --
> Repository URL: http://hg.python.org/cpython
>
> _______________________________________________
> Python-checkins mailing list
> Python-checkins at python.org
> http://mail.python.org/mailman/listinfo/python-checkins
>
More information about the Python-Dev
mailing list