Mailman 3 June 2015 - Python-checkins

cpython (2.7): Issue #20387: Backport fix from Python 3.4
by jason.coombs June 28, 2015

June 28, 2015

https://hg.python.org/cpython/rev/cb9df1ae287b changeset: 96716:cb9df1ae287b branch: 2.7 user: Jason R. Coombs <jaraco(a)jaraco.com> date: Sun Jun 28 13:05:19 2015 -0400 summary: Issue #20387: Backport fix from Python 3.4 files: Lib/tokenize.py | 17 +++++++++++++++++ Misc/NEWS | 3 +++ 2 files changed, 20 insertions(+), 0 deletions(-) diff --git a/Lib/tokenize.py b/Lib/tokenize.py --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -198,6 +198,8 @@ … [View More]

1 0

cpython (merge 3.4 -> 3.5): Issue #20387: Merge test and patch from 3.4.4
by jason.coombs June 28, 2015

June 28, 2015

https://hg.python.org/cpython/rev/98380a6e037c changeset: 96713:98380a6e037c branch: 3.5 parent: 96704:3039cb5b673c parent: 96712:9ce5c1f371f7 user: Jason R. Coombs <jaraco(a)jaraco.com> date: Sun Jun 28 11:13:30 2015 -0400 summary: Issue #20387: Merge test and patch from 3.4.4 files: Lib/test/test_tokenize.py | 21 ++++++++++++++++++++- Lib/tokenize.py | 17 +++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 40 … [View More]insertions(+), 1 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -5,6 +5,8 @@ code, print out a table with tokens. The ENDMARKER is omitted for brevity. + >>> import glob + >>> dump_tokens("1 + 1") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '1' (1, 0) (1, 1) @@ -835,7 +837,7 @@ open as tokenize_open, Untokenizer) from io import BytesIO from unittest import TestCase, mock -import os, sys, glob +import os import token def dump_tokens(s): @@ -1427,6 +1429,22 @@ self.assertEqual(untokenize(iter(tokens)), b'Hello ') +class TestRoundtrip(TestCase): + def roundtrip(self, code): + if isinstance(code, str): + code = code.encode('utf-8') + return untokenize(tokenize(BytesIO(code).readline)).decode('utf-8') + + def test_indentation_semantics_retained(self): + """ + Ensure that although whitespace might be mutated in a roundtrip, + the semantic meaning of the indentation remains consistent. + """ + code = "if False:\n\tx=3\n\tx=3\n" + codelines = self.roundtrip(code).split('\n') + self.assertEqual(codelines[1], codelines[2]) + + __test__ = {"doctests" : doctests, 'decistmt': decistmt} def test_main(): @@ -1437,6 +1455,7 @@ support.run_unittest(TestDetectEncoding) support.run_unittest(TestTokenize) support.run_unittest(UntokenizeTest) + support.run_unittest(TestRoundtrip) if __name__ == "__main__": test_main() diff --git a/Lib/tokenize.py b/Lib/tokenize.py --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -244,6 +244,8 @@ def untokenize(self, iterable): it = iter(iterable) + indents = [] + startline = False for t in it: if len(t) == 2: self.compat(t, it) @@ -254,6 +256,21 @@ continue if tok_type == ENDMARKER: break + if tok_type == INDENT: + indents.append(token) + continue + elif tok_type == DEDENT: + indents.pop() + self.prev_row, self.prev_col = end + continue + elif tok_type in (NEWLINE, NL): + startline = True + elif startline and indents: + indent = indents[-1] + if start[1] >= len(indent): + self.tokens.append(indent) + self.prev_col = len(indent) + startline = False self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -25,6 +25,9 @@ Library ------- +- Issue #20387: Restore semantic round-trip correctness in tokenize/untokenize + for tab-indented blocks. + - Issue #24456: Fixed possible buffer over-read in adpcm2lin() and lin2adpcm() functions of the audioop module. -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython (merge 3.5 -> default): Issue #20387: Merge with 3.5
by jason.coombs June 28, 2015

June 28, 2015

https://hg.python.org/cpython/rev/f2f5d1c928eb changeset: 96714:f2f5d1c928eb parent: 96705:0e1d9018e74b parent: 96713:98380a6e037c user: Jason R. Coombs <jaraco(a)jaraco.com> date: Sun Jun 28 11:15:13 2015 -0400 summary: Issue #20387: Merge with 3.5 files: Lib/test/test_tokenize.py | 21 ++++++++++++++++++++- Lib/tokenize.py | 17 +++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 40 insertions(+), 1 deletions(-) diff --… [View More]git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -5,6 +5,8 @@ code, print out a table with tokens. The ENDMARKER is omitted for brevity. + >>> import glob + >>> dump_tokens("1 + 1") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '1' (1, 0) (1, 1) @@ -835,7 +837,7 @@ open as tokenize_open, Untokenizer) from io import BytesIO from unittest import TestCase, mock -import os, sys, glob +import os import token def dump_tokens(s): @@ -1427,6 +1429,22 @@ self.assertEqual(untokenize(iter(tokens)), b'Hello ') +class TestRoundtrip(TestCase): + def roundtrip(self, code): + if isinstance(code, str): + code = code.encode('utf-8') + return untokenize(tokenize(BytesIO(code).readline)).decode('utf-8') + + def test_indentation_semantics_retained(self): + """ + Ensure that although whitespace might be mutated in a roundtrip, + the semantic meaning of the indentation remains consistent. + """ + code = "if False:\n\tx=3\n\tx=3\n" + codelines = self.roundtrip(code).split('\n') + self.assertEqual(codelines[1], codelines[2]) + + __test__ = {"doctests" : doctests, 'decistmt': decistmt} def test_main(): @@ -1437,6 +1455,7 @@ support.run_unittest(TestDetectEncoding) support.run_unittest(TestTokenize) support.run_unittest(UntokenizeTest) + support.run_unittest(TestRoundtrip) if __name__ == "__main__": test_main() diff --git a/Lib/tokenize.py b/Lib/tokenize.py --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -244,6 +244,8 @@ def untokenize(self, iterable): it = iter(iterable) + indents = [] + startline = False for t in it: if len(t) == 2: self.compat(t, it) @@ -254,6 +256,21 @@ continue if tok_type == ENDMARKER: break + if tok_type == INDENT: + indents.append(token) + continue + elif tok_type == DEDENT: + indents.pop() + self.prev_row, self.prev_col = end + continue + elif tok_type in (NEWLINE, NL): + startline = True + elif startline and indents: + indent = indents[-1] + if start[1] >= len(indent): + self.tokens.append(indent) + self.prev_col = len(indent) + startline = False self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -47,6 +47,9 @@ Library ------- +- Issue #20387: Restore semantic round-trip correctness in tokenize/untokenize + for tab-indented blocks. + - Issue #24456: Fixed possible buffer over-read in adpcm2lin() and lin2adpcm() functions of the audioop module. -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython (3.4): Issue #20387: Restore retention of indentation during untokenize.
by jason.coombs June 28, 2015

June 28, 2015

https://hg.python.org/cpython/rev/ff47efeeed48 changeset: 96709:ff47efeeed48 branch: 3.4 user: Dingyuan Wang <abcdoyle888(a)gmail.com> date: Mon Jun 22 10:01:12 2015 +0800 summary: Issue #20387: Restore retention of indentation during untokenize. files: Lib/tokenize.py | 17 +++++++++++++++++ 1 files changed, 17 insertions(+), 0 deletions(-) diff --git a/Lib/tokenize.py b/Lib/tokenize.py --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -244,6 +244,8 @@ … [View More]

1 0

cpython (3.4): Issue #20387: Add test capturing failure to roundtrip indented code in tokenize
by jason.coombs June 28, 2015

June 28, 2015

https://hg.python.org/cpython/rev/b784c842a63c changeset: 96707:b784c842a63c branch: 3.4 user: Jason R. Coombs <jaraco(a)jaraco.com> date: Sat Jun 20 19:52:22 2015 -0400 summary: Issue #20387: Add test capturing failure to roundtrip indented code in tokenize module. files: Lib/test/test_tokenize.py | 17 +++++++++++++++++ 1 files changed, 17 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py --- a/Lib/test/… [View More]

1 0

cpython (3.4): Issue #20387: Correct test to properly capture expectation.
by jason.coombs June 28, 2015

June 28, 2015

https://hg.python.org/cpython/rev/49323e5f6391 changeset: 96708:49323e5f6391 branch: 3.4 user: Jason R. Coombs <jaraco(a)jaraco.com> date: Thu Jun 25 22:42:24 2015 -0400 summary: Issue #20387: Correct test to properly capture expectation. files: Lib/test/test_tokenize.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@… [View More]

1 0

cpython (3.4): Remove unused import and remove doctest-only import into doctests.
by jason.coombs June 28, 2015

June 28, 2015

https://hg.python.org/cpython/rev/8bf1c158332f changeset: 96706:8bf1c158332f branch: 3.4 parent: 96615:6d574b426eac user: Jason R. Coombs <jaraco(a)jaraco.com> date: Sat Jun 20 19:13:50 2015 -0400 summary: Remove unused import and remove doctest-only import into doctests. files: Lib/test/test_tokenize.py | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py --- a/Lib/test/… [View More]

1 0

cpython (3.4): Issue #20387: Update Misc/NEWS
by jason.coombs June 28, 2015

June 28, 2015

https://hg.python.org/cpython/rev/4856ae883041 changeset: 96710:4856ae883041 branch: 3.4 user: Jason R. Coombs <jaraco(a)jaraco.com> date: Fri Jun 26 17:45:09 2015 -0400 summary: Issue #20387: Update Misc/NEWS files: Misc/NEWS | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -60,6 +60,9 @@ Library ------- +- Issue #20387: Restore semantic round-trip correctness in tokenize/… [View More]

1 0

cpython (merge 3.4 -> 3.4): Issue #20387: Merge patch and test
by jason.coombs June 28, 2015

June 28, 2015

https://hg.python.org/cpython/rev/330e28b28334 changeset: 96711:330e28b28334 branch: 3.4 parent: 96698:f0053d05ed6d parent: 96710:4856ae883041 user: Jason R. Coombs <jaraco(a)jaraco.com> date: Sun Jun 28 10:23:11 2015 -0400 summary: Issue #20387: Merge patch and test files: Lib/test/test_tokenize.py | 21 ++++++++++++++++++++- Lib/tokenize.py | 17 +++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 40 insertions(+), 1 … [View More]deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -5,6 +5,8 @@ code, print out a table with tokens. The ENDMARKER is omitted for brevity. + >>> import glob + >>> dump_tokens("1 + 1") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '1' (1, 0) (1, 1) @@ -647,7 +649,7 @@ open as tokenize_open, Untokenizer) from io import BytesIO from unittest import TestCase, mock -import os, sys, glob +import os import token def dump_tokens(s): @@ -1227,6 +1229,22 @@ self.assertEqual(untokenize(iter(tokens)), b'Hello ') +class TestRoundtrip(TestCase): + def roundtrip(self, code): + if isinstance(code, str): + code = code.encode('utf-8') + return untokenize(tokenize(BytesIO(code).readline)).decode('utf-8') + + def test_indentation_semantics_retained(self): + """ + Ensure that although whitespace might be mutated in a roundtrip, + the semantic meaning of the indentation remains consistent. + """ + code = "if False:\n\tx=3\n\tx=3\n" + codelines = self.roundtrip(code).split('\n') + self.assertEqual(codelines[1], codelines[2]) + + __test__ = {"doctests" : doctests, 'decistmt': decistmt} def test_main(): @@ -1237,6 +1255,7 @@ support.run_unittest(TestDetectEncoding) support.run_unittest(TestTokenize) support.run_unittest(UntokenizeTest) + support.run_unittest(TestRoundtrip) if __name__ == "__main__": test_main() diff --git a/Lib/tokenize.py b/Lib/tokenize.py --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -244,6 +244,8 @@ def untokenize(self, iterable): it = iter(iterable) + indents = [] + startline = False for t in it: if len(t) == 2: self.compat(t, it) @@ -254,6 +256,21 @@ continue if tok_type == ENDMARKER: break + if tok_type == INDENT: + indents.append(token) + continue + elif tok_type == DEDENT: + indents.pop() + self.prev_row, self.prev_col = end + continue + elif tok_type in (NEWLINE, NL): + startline = True + elif startline and indents: + indent = indents[-1] + if start[1] >= len(indent): + self.tokens.append(indent) + self.prev_col = len(indent) + startline = False self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -60,6 +60,9 @@ Library ------- +- Issue #20387: Restore semantic round-trip correctness in tokenize/untokenize + for tab-indented blocks. + - Issue #24336: The contextmanager decorator now works with functions with keyword arguments called "func" and "self". Patch by Martin Panter. -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython (merge 3.4 -> 3.4): Issue #20387: Merge
by jason.coombs June 28, 2015

June 28, 2015

https://hg.python.org/cpython/rev/9ce5c1f371f7 changeset: 96712:9ce5c1f371f7 branch: 3.4 parent: 96703:fd17e168b59f parent: 96711:330e28b28334 user: Jason R. Coombs <jaraco(a)jaraco.com> date: Sun Jun 28 11:10:29 2015 -0400 summary: Issue #20387: Merge files: Lib/test/test_tokenize.py | 21 ++++++++++++++++++++- Lib/tokenize.py | 17 +++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 40 insertions(+), 1 deletions(-) … [View More]diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -5,6 +5,8 @@ code, print out a table with tokens. The ENDMARKER is omitted for brevity. + >>> import glob + >>> dump_tokens("1 + 1") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '1' (1, 0) (1, 1) @@ -647,7 +649,7 @@ open as tokenize_open, Untokenizer) from io import BytesIO from unittest import TestCase, mock -import os, sys, glob +import os import token def dump_tokens(s): @@ -1227,6 +1229,22 @@ self.assertEqual(untokenize(iter(tokens)), b'Hello ') +class TestRoundtrip(TestCase): + def roundtrip(self, code): + if isinstance(code, str): + code = code.encode('utf-8') + return untokenize(tokenize(BytesIO(code).readline)).decode('utf-8') + + def test_indentation_semantics_retained(self): + """ + Ensure that although whitespace might be mutated in a roundtrip, + the semantic meaning of the indentation remains consistent. + """ + code = "if False:\n\tx=3\n\tx=3\n" + codelines = self.roundtrip(code).split('\n') + self.assertEqual(codelines[1], codelines[2]) + + __test__ = {"doctests" : doctests, 'decistmt': decistmt} def test_main(): @@ -1237,6 +1255,7 @@ support.run_unittest(TestDetectEncoding) support.run_unittest(TestTokenize) support.run_unittest(UntokenizeTest) + support.run_unittest(TestRoundtrip) if __name__ == "__main__": test_main() diff --git a/Lib/tokenize.py b/Lib/tokenize.py --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -244,6 +244,8 @@ def untokenize(self, iterable): it = iter(iterable) + indents = [] + startline = False for t in it: if len(t) == 2: self.compat(t, it) @@ -254,6 +256,21 @@ continue if tok_type == ENDMARKER: break + if tok_type == INDENT: + indents.append(token) + continue + elif tok_type == DEDENT: + indents.pop() + self.prev_row, self.prev_col = end + continue + elif tok_type in (NEWLINE, NL): + startline = True + elif startline and indents: + indent = indents[-1] + if start[1] >= len(indent): + self.tokens.append(indent) + self.prev_col = len(indent) + startline = False self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -60,6 +60,9 @@ Library ------- +- Issue #20387: Restore semantic round-trip correctness in tokenize/untokenize + for tab-indented blocks. + - Issue #24456: Fixed possible buffer over-read in adpcm2lin() and lin2adpcm() functions of the audioop module. -- Repository URL: https://hg.python.org/cpython [View Less]

1 0