https://github.com/python/cpython/commit/7e5763469e2fc9d08a3f6b6205f87f20a1b... commit: 7e5763469e2fc9d08a3f6b6205f87f20a1bdd465 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> committer: GitHub <noreply@github.com> date: 2018-02-28T15:08:21-08:00 summary: bpo-32940: IDLE: Simplify StringTranslatePseudoMapping in pyparse (GH-5862) The new code also runs faster. (cherry picked from commit f0daa880a405c8de6743e44fa46006754aa145c9) Co-authored-by: Cheryl Sabella <cheryl.sabella@gmail.com> files: A Misc/NEWS.d/next/IDLE/2018-02-24-18-20-50.bpo-32940.ZaJ1Rf.rst M Lib/idlelib/idle_test/test_pyparse.py M Lib/idlelib/pyparse.py diff --git a/Lib/idlelib/idle_test/test_pyparse.py b/Lib/idlelib/idle_test/test_pyparse.py index c7b645caa6a8..574b19d9d6f1 100644 --- a/Lib/idlelib/idle_test/test_pyparse.py +++ b/Lib/idlelib/idle_test/test_pyparse.py @@ -8,44 +8,20 @@ from idlelib import pyparse -class StringTranslatePseudoMappingTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - whitespace_chars = ' \t\n\r' - cls.preserve_dict = {ord(c): ord(c) for c in whitespace_chars} - cls.default = ord('x') - cls.mapping = pyparse.StringTranslatePseudoMapping( - cls.preserve_dict, default_value=ord('x')) - - @classmethod - def tearDownClass(cls): - del cls.preserve_dict, cls.default, cls.mapping - - def test__init__(self): - m = self.mapping - self.assertEqual(m._non_defaults, self.preserve_dict) - self.assertEqual(m._default_value, self.default) - - def test__get_item__(self): - self.assertEqual(self.mapping[ord('\t')], ord('\t')) - self.assertEqual(self.mapping[ord('a')], self.default) - - def test__len__(self): - self.assertEqual(len(self.mapping), len(self.preserve_dict)) - - def test__iter__(self): - count = 0 - for key, value in self.mapping.items(): - self.assertIn(key, self.preserve_dict) - count += 1 - self.assertEqual(count, len(self.mapping)) - - def test_get(self): - self.assertEqual(self.mapping.get(ord('\t')), ord('\t')) - self.assertEqual(self.mapping.get('a'), self.default) - # Default is a parameter, but it isn't used. - self.assertEqual(self.mapping.get('a', default=500), self.default) +class ParseMapTest(unittest.TestCase): + + def test_parsemap(self): + keepwhite = {ord(c): ord(c) for c in ' \t\n\r'} + mapping = pyparse.ParseMap(keepwhite) + self.assertEqual(mapping[ord('\t')], ord('\t')) + self.assertEqual(mapping[ord('a')], ord('x')) + self.assertEqual(mapping[1000], ord('x')) + + def test_trans(self): + # trans is the production instance of ParseMap, used in _study1 + parser = pyparse.Parser(4, 4) + self.assertEqual('\t a([{b}])b"c\'d\n'.translate(pyparse.trans), + 'xxx(((x)))x"x\'x\n') class PyParseTest(unittest.TestCase): @@ -152,10 +128,6 @@ def test_set_lo(self): p.set_lo(44) self.assertEqual(p.code, code[44:]) - def test_tran(self): - self.assertEqual('\t a([{b}])b"c\'d\n'.translate(self.parser._tran), - 'xxx(((x)))x"x\'x\n') - def test_study1(self): eq = self.assertEqual p = self.parser diff --git a/Lib/idlelib/pyparse.py b/Lib/idlelib/pyparse.py index 443d198d1efe..6196c2b7edc9 100644 --- a/Lib/idlelib/pyparse.py +++ b/Lib/idlelib/pyparse.py @@ -1,6 +1,6 @@ """Define partial Python code Parser used by editor and hyperparser. -Instances of StringTranslatePseudoMapping are used with str.translate. +Instances of ParseMap are used with str.translate. The following bound search and match functions are defined: _synchre - start of popular statement; @@ -10,7 +10,6 @@ _closere - line that must be followed by dedent. _chew_ordinaryre - non-special characters. """ -from collections.abc import Mapping import re import sys @@ -101,46 +100,27 @@ """, re.VERBOSE).match -class StringTranslatePseudoMapping(Mapping): - r"""Utility class to be used with str.translate() +class ParseMap(dict): + r"""Dict subclass that maps anything not in dict to 'x'. - This Mapping class wraps a given dict. When a value for a key is - requested via __getitem__() or get(), the key is looked up in the - given dict. If found there, the value from the dict is returned. - Otherwise, the default value given upon initialization is returned. + This is designed to be used with str.translate in study1. + Anything not specifically mapped otherwise becomes 'x'. + Example: replace everything except whitespace with 'x'. - This allows using str.translate() to make some replacements, and to - replace all characters for which no replacement was specified with - a given character instead of leaving them as-is. - - For example, to replace everything except whitespace with 'x': - - >>> whitespace_chars = ' \t\n\r' - >>> preserve_dict = {ord(c): ord(c) for c in whitespace_chars} - >>> mapping = StringTranslatePseudoMapping(preserve_dict, ord('x')) - >>> text = "a + b\tc\nd" - >>> text.translate(mapping) + >>> keepwhite = ParseMap((ord(c), ord(c)) for c in ' \t\n\r') + >>> "a + b\tc\nd".translate(keepwhite) 'x x x\tx\nx' """ - def __init__(self, non_defaults, default_value): - self._non_defaults = non_defaults - self._default_value = default_value - - def _get(key, _get=non_defaults.get, _default=default_value): - return _get(key, _default) - self._get = _get + # Calling this triples access time; see bpo-32940 + def __missing__(self, key): + return 120 # ord('x') - def __getitem__(self, item): - return self._get(item) - def __len__(self): - return len(self._non_defaults) - - def __iter__(self): - return iter(self._non_defaults) - - def get(self, key, default=None): - return self._get(key) +# Map all ascii to 120 to avoid __missing__ call, then replace some. +trans = ParseMap.fromkeys(range(128), 120) +trans.update((ord(c), ord('(')) for c in "({[") # open brackets => '('; +trans.update((ord(c), ord(')')) for c in ")}]") # close brackets => ')'. +trans.update((ord(c), ord(c)) for c in "\"'\\\n#") # Keep these. class Parser: @@ -224,16 +204,6 @@ def set_lo(self, lo): if lo > 0: self.code = self.code[lo:] - # Build a translation table to map uninteresting chars to 'x', open - # brackets to '(', close brackets to ')' while preserving quotes, - # backslashes, newlines and hashes. This is to be passed to - # str.translate() in _study1(). - _tran = {} - _tran.update((ord(c), ord('(')) for c in "({[") - _tran.update((ord(c), ord(')')) for c in ")}]") - _tran.update((ord(c), ord(c)) for c in "\"'\\\n#") - _tran = StringTranslatePseudoMapping(_tran, default_value=ord('x')) - def _study1(self): """Find the line numbers of non-continuation lines. @@ -250,7 +220,7 @@ def _study1(self): # uninteresting characters. This can cut the number of chars # by a factor of 10-40, and so greatly speed the following loop. code = self.code - code = code.translate(self._tran) + code = code.translate(trans) code = code.replace('xxxxxxxx', 'x') code = code.replace('xxxx', 'x') code = code.replace('xx', 'x') diff --git a/Misc/NEWS.d/next/IDLE/2018-02-24-18-20-50.bpo-32940.ZaJ1Rf.rst b/Misc/NEWS.d/next/IDLE/2018-02-24-18-20-50.bpo-32940.ZaJ1Rf.rst new file mode 100644 index 000000000000..958f9522d4f8 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2018-02-24-18-20-50.bpo-32940.ZaJ1Rf.rst @@ -0,0 +1 @@ +Simplify and rename StringTranslatePseudoMapping in pyparse.