[Python-checkins] bpo-32874: IDLE: add tests for pyparse (GH-5755)
Terry Jan Reedy
webhook-mailer at python.org
Wed Feb 21 22:48:47 EST 2018
https://github.com/python/cpython/commit/c84cf6c03fce1fb73bfaf91d7909f1c2708f14a2
commit: c84cf6c03fce1fb73bfaf91d7909f1c2708f14a2
branch: master
author: Cheryl Sabella <cheryl.sabella at gmail.com>
committer: Terry Jan Reedy <tjreedy at udel.edu>
date: 2018-02-21T22:48:36-05:00
summary:
bpo-32874: IDLE: add tests for pyparse (GH-5755)
There are no code changes other than comments and docstrings.
files:
A Lib/idlelib/idle_test/test_pyparse.py
A Misc/NEWS.d/next/IDLE/2018-02-19-10-56-41.bpo-32874.6pZ9Gv.rst
M Lib/idlelib/pyparse.py
diff --git a/Lib/idlelib/idle_test/test_pyparse.py b/Lib/idlelib/idle_test/test_pyparse.py
new file mode 100644
index 000000000000..b84e9f843081
--- /dev/null
+++ b/Lib/idlelib/idle_test/test_pyparse.py
@@ -0,0 +1,523 @@
+"""Unittest for idlelib.pyparse.py."""
+
+from collections import namedtuple
+import unittest
+from idlelib import pyparse
+
+
+class StringTranslatePseudoMappingTest(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ whitespace_chars = ' \t\n\r'
+ cls.preserve_dict = {ord(c): ord(c) for c in whitespace_chars}
+ cls.default = ord('x')
+ cls.mapping = pyparse.StringTranslatePseudoMapping(
+ cls.preserve_dict, default_value=ord('x'))
+
+ @classmethod
+ def tearDownClass(cls):
+ del cls.preserve_dict, cls.default, cls.mapping
+
+ def test__init__(self):
+ m = self.mapping
+ self.assertEqual(m._non_defaults, self.preserve_dict)
+ self.assertEqual(m._default_value, self.default)
+
+ def test__get_item__(self):
+ self.assertEqual(self.mapping[ord('\t')], ord('\t'))
+ self.assertEqual(self.mapping[ord('a')], self.default)
+
+ def test__len__(self):
+ self.assertEqual(len(self.mapping), len(self.preserve_dict))
+
+ def test__iter__(self):
+ count = 0
+ for key, value in self.mapping.items():
+ self.assertIn(key, self.preserve_dict)
+ count += 1
+ self.assertEqual(count, len(self.mapping))
+
+ def test_get(self):
+ self.assertEqual(self.mapping.get(ord('\t')), ord('\t'))
+ self.assertEqual(self.mapping.get('a'), self.default)
+ # Default is a parameter, but it isn't used.
+ self.assertEqual(self.mapping.get('a', default=500), self.default)
+
+
+class PyParseTest(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = pyparse.Parser(indentwidth=4, tabwidth=4)
+
+ @classmethod
+ def tearDownClass(cls):
+ del cls.parser
+
+ def test_init(self):
+ self.assertEqual(self.parser.indentwidth, 4)
+ self.assertEqual(self.parser.tabwidth, 4)
+
+ def test_set_str(self):
+ eq = self.assertEqual
+ p = self.parser
+ setstr = p.set_str
+
+ # Not empty and doesn't end with newline.
+ with self.assertRaises(AssertionError):
+ setstr('a')
+
+ tests = ('',
+ 'a\n')
+
+ for string in tests:
+ with self.subTest(string=string):
+ setstr(string)
+ eq(p.str, string)
+ eq(p.study_level, 0)
+
+ def test_find_good_parse_start(self):
+ eq = self.assertEqual
+ p = self.parser
+ setstr = p.set_str
+ start = p.find_good_parse_start
+
+ # Split def across lines.
+ setstr('"""This is a module docstring"""\n'
+ 'class C():\n'
+ ' def __init__(self, a,\n'
+ ' b=True):\n'
+ ' pass\n'
+ )
+
+ # No value sent for is_char_in_string().
+ self.assertIsNone(start())
+
+ # Make text look like a string. This returns pos as the start
+ # position, but it's set to None.
+ self.assertIsNone(start(is_char_in_string=lambda index: True))
+
+ # Make all text look like it's not in a string. This means that it
+ # found a good start position.
+ eq(start(is_char_in_string=lambda index: False), 44)
+
+ # If the beginning of the def line is not in a string, then it
+ # returns that as the index.
+ eq(start(is_char_in_string=lambda index: index > 44), 44)
+ # If the beginning of the def line is in a string, then it
+ # looks for a previous index.
+ eq(start(is_char_in_string=lambda index: index >= 44), 33)
+ # If everything before the 'def' is in a string, then returns None.
+ # The non-continuation def line returns 44 (see below).
+ eq(start(is_char_in_string=lambda index: index < 44), None)
+
+ # Code without extra line break in def line - mostly returns the same
+ # values.
+ setstr('"""This is a module docstring"""\n'
+ 'class C():\n'
+ ' def __init__(self, a, b=True):\n'
+ ' pass\n'
+ )
+ eq(start(is_char_in_string=lambda index: False), 44)
+ eq(start(is_char_in_string=lambda index: index > 44), 44)
+ eq(start(is_char_in_string=lambda index: index >= 44), 33)
+ # When the def line isn't split, this returns which doesn't match the
+ # split line test.
+ eq(start(is_char_in_string=lambda index: index < 44), 44)
+
+ def test_set_lo(self):
+ code = (
+ '"""This is a module docstring"""\n'
+ 'class C():\n'
+ ' def __init__(self, a,\n'
+ ' b=True):\n'
+ ' pass\n'
+ )
+ p = self.parser
+ p.set_str(code)
+
+ # Previous character is not a newline.
+ with self.assertRaises(AssertionError):
+ p.set_lo(5)
+
+ # A value of 0 doesn't change self.str.
+ p.set_lo(0)
+ self.assertEqual(p.str, code)
+
+ # An index that is preceded by a newline.
+ p.set_lo(44)
+ self.assertEqual(p.str, code[44:])
+
+ def test_tran(self):
+ self.assertEqual('\t a([{b}])b"c\'d\n'.translate(self.parser._tran),
+ 'xxx(((x)))x"x\'x\n')
+
+ def test_study1(self):
+ eq = self.assertEqual
+ p = self.parser
+ setstr = p.set_str
+ study = p._study1
+
+ (NONE, BACKSLASH, FIRST, NEXT, BRACKET) = range(5)
+ TestInfo = namedtuple('TestInfo', ['string', 'goodlines',
+ 'continuation'])
+ tests = (
+ TestInfo('', [0], NONE),
+ # Docstrings.
+ TestInfo('"""This is a complete docstring."""\n', [0, 1], NONE),
+ TestInfo("'''This is a complete docstring.'''\n", [0, 1], NONE),
+ TestInfo('"""This is a continued docstring.\n', [0, 1], FIRST),
+ TestInfo("'''This is a continued docstring.\n", [0, 1], FIRST),
+ TestInfo('"""Closing quote does not match."\n', [0, 1], FIRST),
+ TestInfo('"""Bracket in docstring [\n', [0, 1], FIRST),
+ TestInfo("'''Incomplete two line docstring.\n\n", [0, 2], NEXT),
+ # Single-quoted strings.
+ TestInfo('"This is a complete string."\n', [0, 1], NONE),
+ TestInfo('"This is an incomplete string.\n', [0, 1], NONE),
+ TestInfo("'This is more incomplete.\n\n", [0, 1, 2], NONE),
+ # Comment (backslash does not continue comments).
+ TestInfo('# Comment\\\n', [0, 1], NONE),
+ # Brackets.
+ TestInfo('("""Complete string in bracket"""\n', [0, 1], BRACKET),
+ TestInfo('("""Open string in bracket\n', [0, 1], FIRST),
+ TestInfo('a = (1 + 2) - 5 *\\\n', [0, 1], BACKSLASH), # No bracket.
+ TestInfo('\n def function1(self, a,\n b):\n',
+ [0, 1, 3], NONE),
+ TestInfo('\n def function1(self, a,\\\n', [0, 1, 2], BRACKET),
+ TestInfo('\n def function1(self, a,\n', [0, 1, 2], BRACKET),
+ TestInfo('())\n', [0, 1], NONE), # Extra closer.
+ TestInfo(')(\n', [0, 1], BRACKET), # Extra closer.
+ # For the mismatched example, it doesn't look like contination.
+ TestInfo('{)(]\n', [0, 1], NONE), # Mismatched.
+ )
+
+ for test in tests:
+ with self.subTest(string=test.string):
+ setstr(test.string) # resets study_level
+ study()
+ eq(p.study_level, 1)
+ eq(p.goodlines, test.goodlines)
+ eq(p.continuation, test.continuation)
+
+ # Called again, just returns without reprocessing.
+ self.assertIsNone(study())
+
+ def test_get_continuation_type(self):
+ eq = self.assertEqual
+ p = self.parser
+ setstr = p.set_str
+ gettype = p.get_continuation_type
+
+ (NONE, BACKSLASH, FIRST, NEXT, BRACKET) = range(5)
+ TestInfo = namedtuple('TestInfo', ['string', 'continuation'])
+ tests = (
+ TestInfo('', NONE),
+ TestInfo('"""This is a continuation docstring.\n', FIRST),
+ TestInfo("'''This is a multiline-continued docstring.\n\n", NEXT),
+ TestInfo('a = (1 + 2) - 5 *\\\n', BACKSLASH),
+ TestInfo('\n def function1(self, a,\\\n', BRACKET)
+ )
+
+ for test in tests:
+ with self.subTest(string=test.string):
+ setstr(test.string)
+ eq(gettype(), test.continuation)
+
+ def test_study2(self):
+ eq = self.assertEqual
+ p = self.parser
+ setstr = p.set_str
+ study = p._study2
+
+ TestInfo = namedtuple('TestInfo', ['string', 'start', 'end', 'lastch',
+ 'openbracket', 'bracketing'])
+ tests = (
+ TestInfo('', 0, 0, '', None, ((0, 0),)),
+ TestInfo("'''This is a multiline continutation docstring.\n\n",
+ 0, 49, "'", None, ((0, 0), (0, 1), (49, 0))),
+ TestInfo(' # Comment\\\n',
+ 0, 12, '', None, ((0, 0), (1, 1), (12, 0))),
+ # A comment without a space is a special case
+ TestInfo(' #Comment\\\n',
+ 0, 0, '', None, ((0, 0),)),
+ # Backslash continuation.
+ TestInfo('a = (1 + 2) - 5 *\\\n',
+ 0, 19, '*', None, ((0, 0), (4, 1), (11, 0))),
+ # Bracket continuation with close.
+ TestInfo('\n def function1(self, a,\n b):\n',
+ 1, 48, ':', None, ((1, 0), (17, 1), (46, 0))),
+ # Bracket continuation with unneeded backslash.
+ TestInfo('\n def function1(self, a,\\\n',
+ 1, 28, ',', 17, ((1, 0), (17, 1))),
+ # Bracket continuation.
+ TestInfo('\n def function1(self, a,\n',
+ 1, 27, ',', 17, ((1, 0), (17, 1))),
+ # Bracket continuation with comment at end of line with text.
+ TestInfo('\n def function1(self, a, # End of line comment.\n',
+ 1, 51, ',', 17, ((1, 0), (17, 1), (28, 2), (51, 1))),
+ # Multi-line statement with comment line in between code lines.
+ TestInfo(' a = ["first item",\n # Comment line\n "next item",\n',
+ 0, 55, ',', 6, ((0, 0), (6, 1), (7, 2), (19, 1),
+ (23, 2), (38, 1), (42, 2), (53, 1))),
+ TestInfo('())\n',
+ 0, 4, ')', None, ((0, 0), (0, 1), (2, 0), (3, 0))),
+ TestInfo(')(\n', 0, 3, '(', 1, ((0, 0), (1, 0), (1, 1))),
+ # Wrong closers still decrement stack level.
+ TestInfo('{)(]\n',
+ 0, 5, ']', None, ((0, 0), (0, 1), (2, 0), (2, 1), (4, 0))),
+ # Character after backslash.
+ TestInfo(':\\a\n', 0, 4, '\\a', None, ((0, 0),)),
+ TestInfo('\n', 0, 0, '', None, ((0, 0),)),
+ )
+
+ for test in tests:
+ # There is a bug where this is carried forward from last item.
+ p.lastopenbracketpos = None
+ with self.subTest(string=test.string):
+ setstr(test.string)
+ study()
+ eq(p.study_level, 2)
+ eq(p.stmt_start, test.start)
+ eq(p.stmt_end, test.end)
+ eq(p.lastch, test.lastch)
+ eq(p.lastopenbracketpos, test.openbracket)
+ eq(p.stmt_bracketing, test.bracketing)
+
+ # Called again, just returns without reprocessing.
+ self.assertIsNone(study())
+
+ def test_get_num_lines_in_stmt(self):
+ eq = self.assertEqual
+ p = self.parser
+ setstr = p.set_str
+ getlines = p.get_num_lines_in_stmt
+
+ TestInfo = namedtuple('TestInfo', ['string', 'lines'])
+ tests = (
+ TestInfo('[x for x in a]\n', 1), # Closed on one line.
+ TestInfo('[x\nfor x in a\n', 2), # Not closed.
+ TestInfo('[x\\\nfor x in a\\\n', 2), # "", uneeded backslashes.
+ TestInfo('[x\nfor x in a\n]\n', 3), # Closed on multi-line.
+ TestInfo('\n"""Docstring comment L1"""\nL2\nL3\nL4\n', 1),
+ TestInfo('\n"""Docstring comment L1\nL2"""\nL3\nL4\n', 1),
+ TestInfo('\n"""Docstring comment L1\\\nL2\\\nL3\\\nL4\\\n', 4),
+ TestInfo('\n\n"""Docstring comment L1\\\nL2\\\nL3\\\nL4\\\n"""\n', 5)
+ )
+
+ # Blank string doesn't have enough elements in goodlines.
+ setstr('')
+ with self.assertRaises(IndexError):
+ getlines()
+
+ for test in tests:
+ with self.subTest(string=test.string):
+ setstr(test.string)
+ eq(getlines(), test.lines)
+
+ def test_compute_bracket_indent(self):
+ eq = self.assertEqual
+ p = self.parser
+ setstr = p.set_str
+ indent = p.compute_bracket_indent
+
+ TestInfo = namedtuple('TestInfo', ['string', 'spaces'])
+ tests = (
+ TestInfo('def function1(self, a,\n', 14),
+ # Characters after bracket.
+ TestInfo('\n def function1(self, a,\n', 18),
+ TestInfo('\n\tdef function1(self, a,\n', 18),
+ # No characters after bracket.
+ TestInfo('\n def function1(\n', 8),
+ TestInfo('\n\tdef function1(\n', 8),
+ TestInfo('\n def function1( \n', 8), # Ignore extra spaces.
+ TestInfo('[\n"first item",\n # Comment line\n "next item",\n', 0),
+ TestInfo('[\n "first item",\n # Comment line\n "next item",\n', 2),
+ TestInfo('["first item",\n # Comment line\n "next item",\n', 1),
+ TestInfo('(\n', 4),
+ TestInfo('(a\n', 1),
+ )
+
+ # Must be C_BRACKET continuation type.
+ setstr('def function1(self, a, b):\n')
+ with self.assertRaises(AssertionError):
+ indent()
+
+ for test in tests:
+ setstr(test.string)
+ eq(indent(), test.spaces)
+
+ def test_compute_backslash_indent(self):
+ eq = self.assertEqual
+ p = self.parser
+ setstr = p.set_str
+ indent = p.compute_backslash_indent
+
+ # Must be C_BACKSLASH continuation type.
+ errors = (('def function1(self, a, b\\\n'), # Bracket.
+ (' """ (\\\n'), # Docstring.
+ ('a = #\\\n'), # Inline comment.
+ )
+ for string in errors:
+ with self.subTest(string=string):
+ setstr(string)
+ with self.assertRaises(AssertionError):
+ indent()
+
+ TestInfo = namedtuple('TestInfo', ('string', 'spaces'))
+ tests = (TestInfo('a = (1 + 2) - 5 *\\\n', 4),
+ TestInfo('a = 1 + 2 - 5 *\\\n', 4),
+ TestInfo(' a = 1 + 2 - 5 *\\\n', 8),
+ TestInfo(' a = "spam"\\\n', 6),
+ TestInfo(' a = \\\n"a"\\\n', 4),
+ TestInfo(' a = #\\\n"a"\\\n', 5),
+ TestInfo('a == \\\n', 2),
+ TestInfo('a != \\\n', 2),
+ # Difference between containing = and those not.
+ TestInfo('\\\n', 2),
+ TestInfo(' \\\n', 6),
+ TestInfo('\t\\\n', 6),
+ TestInfo('a\\\n', 3),
+ TestInfo('{}\\\n', 4),
+ TestInfo('(1 + 2) - 5 *\\\n', 3),
+ )
+ for test in tests:
+ with self.subTest(string=test.string):
+ setstr(test.string)
+ eq(indent(), test.spaces)
+
+ def test_get_base_indent_string(self):
+ eq = self.assertEqual
+ p = self.parser
+ setstr = p.set_str
+ baseindent = p.get_base_indent_string
+
+ TestInfo = namedtuple('TestInfo', ['string', 'indent'])
+ tests = (TestInfo('', ''),
+ TestInfo('def a():\n', ''),
+ TestInfo('\tdef a():\n', '\t'),
+ TestInfo(' def a():\n', ' '),
+ TestInfo(' def a(\n', ' '),
+ TestInfo('\t\n def a(\n', ' '),
+ TestInfo('\t\n # Comment.\n', ' '),
+ )
+
+ for test in tests:
+ with self.subTest(string=test.string):
+ setstr(test.string)
+ eq(baseindent(), test.indent)
+
+ def test_is_block_opener(self):
+ yes = self.assertTrue
+ no = self.assertFalse
+ p = self.parser
+ setstr = p.set_str
+ opener = p.is_block_opener
+
+ TestInfo = namedtuple('TestInfo', ['string', 'assert_'])
+ tests = (
+ TestInfo('def a():\n', yes),
+ TestInfo('\n def function1(self, a,\n b):\n', yes),
+ TestInfo(':\n', yes),
+ TestInfo('a:\n', yes),
+ TestInfo('):\n', yes),
+ TestInfo('(:\n', yes),
+ TestInfo('":\n', no),
+ TestInfo('\n def function1(self, a,\n', no),
+ TestInfo('def function1(self, a):\n pass\n', no),
+ TestInfo('# A comment:\n', no),
+ TestInfo('"""A docstring:\n', no),
+ TestInfo('"""A docstring:\n', no),
+ )
+
+ for test in tests:
+ with self.subTest(string=test.string):
+ setstr(test.string)
+ test.assert_(opener())
+
+ def test_is_block_closer(self):
+ yes = self.assertTrue
+ no = self.assertFalse
+ p = self.parser
+ setstr = p.set_str
+ closer = p.is_block_closer
+
+ TestInfo = namedtuple('TestInfo', ['string', 'assert_'])
+ tests = (
+ TestInfo('return\n', yes),
+ TestInfo('\tbreak\n', yes),
+ TestInfo(' continue\n', yes),
+ TestInfo(' raise\n', yes),
+ TestInfo('pass \n', yes),
+ TestInfo('pass\t\n', yes),
+ TestInfo('return #\n', yes),
+ TestInfo('raised\n', no),
+ TestInfo('returning\n', no),
+ TestInfo('# return\n', no),
+ TestInfo('"""break\n', no),
+ TestInfo('"continue\n', no),
+ TestInfo('def function1(self, a):\n pass\n', yes),
+ )
+
+ for test in tests:
+ with self.subTest(string=test.string):
+ setstr(test.string)
+ test.assert_(closer())
+
+ def test_get_last_open_bracket_pos(self):
+ eq = self.assertEqual
+ p = self.parser
+ setstr = p.set_str
+ openbracket = p.get_last_open_bracket_pos
+
+ TestInfo = namedtuple('TestInfo', ['string', 'position'])
+ tests = (
+ TestInfo('', None),
+ TestInfo('a\n', None),
+ TestInfo('# (\n', None),
+ TestInfo('""" (\n', None),
+ TestInfo('a = (1 + 2) - 5 *\\\n', None),
+ TestInfo('\n def function1(self, a,\n', 17),
+ TestInfo('\n def function1(self, a, # End of line comment.\n', 17),
+ TestInfo('{)(]\n', None),
+ TestInfo('(((((((((()))))))\n', 2),
+ TestInfo('(((((((((())\n)))\n))\n', 2),
+ )
+
+ for test in tests:
+ # There is a bug where the value is carried forward from last item.
+ p.lastopenbracketpos = None
+ with self.subTest(string=test.string):
+ setstr(test.string)
+ eq(openbracket(), test.position)
+
+ def test_get_last_stmt_bracketing(self):
+ eq = self.assertEqual
+ p = self.parser
+ setstr = p.set_str
+ bracketing = p.get_last_stmt_bracketing
+
+ TestInfo = namedtuple('TestInfo', ['string', 'bracket'])
+ tests = (
+ TestInfo('', ((0, 0),)),
+ TestInfo('a\n', ((0, 0),)),
+ TestInfo('()()\n', ((0, 0), (0, 1), (2, 0), (2, 1), (4, 0))),
+ TestInfo('(\n)()\n', ((0, 0), (0, 1), (3, 0), (3, 1), (5, 0))),
+ TestInfo('()\n()\n', ((3, 0), (3, 1), (5, 0))),
+ TestInfo('()(\n)\n', ((0, 0), (0, 1), (2, 0), (2, 1), (5, 0))),
+ TestInfo('(())\n', ((0, 0), (0, 1), (1, 2), (3, 1), (4, 0))),
+ TestInfo('(\n())\n', ((0, 0), (0, 1), (2, 2), (4, 1), (5, 0))),
+ # Same as matched test.
+ TestInfo('{)(]\n', ((0, 0), (0, 1), (2, 0), (2, 1), (4, 0))),
+ TestInfo('(((())\n',
+ ((0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (5, 3), (6, 2))),
+ )
+
+ for test in tests:
+ with self.subTest(string=test.string):
+ setstr(test.string)
+ eq(bracketing(), test.bracket)
+
+
+if __name__ == '__main__':
+ unittest.main(verbosity=2)
diff --git a/Lib/idlelib/pyparse.py b/Lib/idlelib/pyparse.py
index 536b2d7f5fef..72bd9e00c1e9 100644
--- a/Lib/idlelib/pyparse.py
+++ b/Lib/idlelib/pyparse.py
@@ -1,8 +1,20 @@
+"""Define partial Python code Parser used by editor and hyperparser.
+
+Instances of StringTranslatePseudoMapping are used with str.translate.
+
+The following bound search and match functions are defined:
+_synchre - start of popular statement;
+_junkre - whitespace or comment line;
+_match_stringre: string, possibly without closer;
+_itemre - line that may have bracket structure start;
+_closere - line that must be followed by dedent.
+_chew_ordinaryre - non-special characters.
+"""
from collections.abc import Mapping
import re
import sys
-# Reason last stmt is continued (or C_NONE if it's not).
+# Reason last statement is continued (or C_NONE if it's not).
(C_NONE, C_BACKSLASH, C_STRING_FIRST_LINE,
C_STRING_NEXT_LINES, C_BRACKET) = range(5)
@@ -10,7 +22,7 @@
def dump(*stuff):
sys.__stdout__.write(" ".join(map(str, stuff)) + "\n")
-# Find what looks like the start of a popular stmt.
+# Find what looks like the start of a popular statement.
_synchre = re.compile(r"""
^
@@ -70,7 +82,7 @@ def dump(*stuff):
[^\s#\\] # if we match, m.end()-1 is the interesting char
""", re.VERBOSE).match
-# Match start of stmts that should be followed by a dedent.
+# Match start of statements that should be followed by a dedent.
_closere = re.compile(r"""
\s*
@@ -146,19 +158,20 @@ def set_str(self, s):
self.str = s
self.study_level = 0
- # Return index of a good place to begin parsing, as close to the
- # end of the string as possible. This will be the start of some
- # popular stmt like "if" or "def". Return None if none found:
- # the caller should pass more prior context then, if possible, or
- # if not (the entire program text up until the point of interest
- # has already been tried) pass 0 to set_lo.
- #
- # This will be reliable iff given a reliable is_char_in_string
- # function, meaning that when it says "no", it's absolutely
- # guaranteed that the char is not in a string.
-
def find_good_parse_start(self, is_char_in_string=None,
_synchre=_synchre):
+ """
+ Return index of a good place to begin parsing, as close to the
+ end of the string as possible. This will be the start of some
+ popular stmt like "if" or "def". Return None if none found:
+ the caller should pass more prior context then, if possible, or
+ if not (the entire program text up until the point of interest
+ has already been tried) pass 0 to set_lo().
+
+ This will be reliable iff given a reliable is_char_in_string()
+ function, meaning that when it says "no", it's absolutely
+ guaranteed that the char is not in a string.
+ """
str, pos = self.str, None
if not is_char_in_string:
@@ -173,7 +186,7 @@ def find_good_parse_start(self, is_char_in_string=None,
i = str.rfind(":\n", 0, limit)
if i < 0:
break
- i = str.rfind('\n', 0, i) + 1 # start of colon line
+ i = str.rfind('\n', 0, i) + 1 # start of colon line (-1+1=0)
m = _synchre(str, i, limit)
if m and not is_char_in_string(m.start()):
pos = m.start()
@@ -206,10 +219,11 @@ def find_good_parse_start(self, is_char_in_string=None,
break
return pos
- # Throw away the start of the string. Intended to be called with
- # find_good_parse_start's result.
-
def set_lo(self, lo):
+ """ Throw away the start of the string.
+
+ Intended to be called with the result of find_good_parse_start().
+ """
assert lo == 0 or self.str[lo-1] == '\n'
if lo > 0:
self.str = self.str[lo:]
@@ -224,11 +238,13 @@ def set_lo(self, lo):
_tran.update((ord(c), ord(c)) for c in "\"'\\\n#")
_tran = StringTranslatePseudoMapping(_tran, default_value=ord('x'))
- # As quickly as humanly possible <wink>, find the line numbers (0-
- # based) of the non-continuation lines.
- # Creates self.{goodlines, continuation}.
-
def _study1(self):
+ """Find the line numbers of non-continuation lines.
+
+ As quickly as humanly possible <wink>, find the line numbers (0-
+ based) of the non-continuation lines.
+ Creates self.{goodlines, continuation}.
+ """
if self.study_level >= 1:
return
self.study_level = 1
@@ -244,8 +260,8 @@ def _study1(self):
str = str.replace('xx', 'x')
str = str.replace('xx', 'x')
str = str.replace('\nx', '\n')
- # note that replacing x\n with \n would be incorrect, because
- # x may be preceded by a backslash
+ # Replacing x\n with \n would be incorrect because
+ # x may be preceded by a backslash.
# March over the squashed version of the program, accumulating
# the line numbers of non-continued stmts, and determining
@@ -360,24 +376,25 @@ def get_continuation_type(self):
self._study1()
return self.continuation
- # study1 was sufficient to determine the continuation status,
- # but doing more requires looking at every character. study2
- # does this for the last interesting statement in the block.
- # Creates:
- # self.stmt_start, stmt_end
- # slice indices of last interesting stmt
- # self.stmt_bracketing
- # the bracketing structure of the last interesting stmt;
- # for example, for the statement "say(boo) or die", stmt_bracketing
- # will be [(0, 0), (3, 1), (8, 0)]. Strings and comments are
- # treated as brackets, for the matter.
- # self.lastch
- # last non-whitespace character before optional trailing
- # comment
- # self.lastopenbracketpos
- # if continuation is C_BRACKET, index of last open bracket
-
def _study2(self):
+ """
+ study1 was sufficient to determine the continuation status,
+ but doing more requires looking at every character. study2
+ does this for the last interesting statement in the block.
+ Creates:
+ self.stmt_start, stmt_end
+ slice indices of last interesting stmt
+ self.stmt_bracketing
+ the bracketing structure of the last interesting stmt; for
+ example, for the statement "say(boo) or die",
+ stmt_bracketing will be ((0, 0), (0, 1), (2, 0), (2, 1),
+ (4, 0)). Strings and comments are treated as brackets, for
+ the matter.
+ self.lastch
+ last interesting character before optional trailing comment
+ self.lastopenbracketpos
+ if continuation is C_BRACKET, index of last open bracket
+ """
if self.study_level >= 2:
return
self._study1()
@@ -385,11 +402,11 @@ def _study2(self):
# Set p and q to slice indices of last interesting stmt.
str, goodlines = self.str, self.goodlines
- i = len(goodlines) - 1
- p = len(str) # index of newest line
+ i = len(goodlines) - 1 # Index of newest line.
+ p = len(str) # End of goodlines[i]
while i:
assert p
- # p is the index of the stmt at line number goodlines[i].
+ # Make p be the index of the stmt at line number goodlines[i].
# Move p back to the stmt at line number goodlines[i-1].
q = p
for nothing in range(goodlines[i-1], goodlines[i]):
@@ -483,10 +500,11 @@ def _study2(self):
self.lastopenbracketpos = stack[-1]
self.stmt_bracketing = tuple(bracketing)
- # Assuming continuation is C_BRACKET, return the number
- # of spaces the next line should be indented.
-
def compute_bracket_indent(self):
+ """Return number of spaces the next line should be indented.
+
+ Line continuation must be C_BRACKET.
+ """
self._study2()
assert self.continuation == C_BRACKET
j = self.lastopenbracketpos
@@ -513,20 +531,22 @@ def compute_bracket_indent(self):
extra = self.indentwidth
return len(str[i:j].expandtabs(self.tabwidth)) + extra
- # Return number of physical lines in last stmt (whether or not
- # it's an interesting stmt! this is intended to be called when
- # continuation is C_BACKSLASH).
-
def get_num_lines_in_stmt(self):
+ """Return number of physical lines in last stmt.
+
+ The statement doesn't have to be an interesting statement. This is
+ intended to be called when continuation is C_BACKSLASH.
+ """
self._study1()
goodlines = self.goodlines
return goodlines[-1] - goodlines[-2]
- # Assuming continuation is C_BACKSLASH, return the number of spaces
- # the next line should be indented. Also assuming the new line is
- # the first one following the initial line of the stmt.
-
def compute_backslash_indent(self):
+ """Return number of spaces the next line should be indented.
+
+ Line continuation must be C_BACKSLASH. Also assume that the new
+ line is the first one following the initial line of the stmt.
+ """
self._study2()
assert self.continuation == C_BACKSLASH
str = self.str
@@ -551,6 +571,8 @@ def compute_backslash_indent(self):
elif ch == '"' or ch == "'":
i = _match_stringre(str, i, endpos).end()
elif ch == '#':
+ # This line is unreachable because the # makes a comment of
+ # everything after it.
break
elif level == 0 and ch == '=' and \
(i == 0 or str[i-1] not in "=<>!") and \
@@ -576,10 +598,10 @@ def compute_backslash_indent(self):
return len(str[self.stmt_start:i].expandtabs(\
self.tabwidth)) + 1
- # Return the leading whitespace on the initial line of the last
- # interesting stmt.
-
def get_base_indent_string(self):
+ """Return the leading whitespace on the initial line of the last
+ interesting stmt.
+ """
self._study2()
i, n = self.stmt_start, self.stmt_end
j = i
@@ -588,30 +610,37 @@ def get_base_indent_string(self):
j = j + 1
return str[i:j]
- # Did the last interesting stmt open a block?
-
def is_block_opener(self):
+ "Return True if the last interesting statemtent opens a block."
self._study2()
return self.lastch == ':'
- # Did the last interesting stmt close a block?
-
def is_block_closer(self):
+ "Return True if the last interesting statement closes a block."
self._study2()
return _closere(self.str, self.stmt_start) is not None
- # index of last open bracket ({[, or None if none
+ # XXX - is this used?
lastopenbracketpos = None
def get_last_open_bracket_pos(self):
+ "Return index of last open bracket or None."
self._study2()
return self.lastopenbracketpos
- # the structure of the bracketing of the last interesting statement,
- # in the format defined in _study2, or None if the text didn't contain
- # anything
+ # XXX - is this used?
stmt_bracketing = None
def get_last_stmt_bracketing(self):
+ """Return a tuple of the structure of the bracketing of the last
+ interesting statement.
+
+ Tuple is in the format defined in _study2().
+ """
self._study2()
return self.stmt_bracketing
+
+
+if __name__ == '__main__': #pragma: nocover
+ import unittest
+ unittest.main('idlelib.idle_test.test_pyparse', verbosity=2)
diff --git a/Misc/NEWS.d/next/IDLE/2018-02-19-10-56-41.bpo-32874.6pZ9Gv.rst b/Misc/NEWS.d/next/IDLE/2018-02-19-10-56-41.bpo-32874.6pZ9Gv.rst
new file mode 100644
index 000000000000..79655315fff4
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2018-02-19-10-56-41.bpo-32874.6pZ9Gv.rst
@@ -0,0 +1 @@
+Add tests for pyparse.
More information about the Python-checkins
mailing list