Mailman 3 October 2014 - Python-checkins

peps: 2.7.9 concrete dates
by benjamin.peterson 31 Oct '14

31 Oct '14

https://hg.python.org/peps/rev/8df0058f32cc changeset: 5591:8df0058f32cc user: Benjamin Peterson <benjamin(a)python.org> date: Fri Oct 31 19:54:32 2014 -0400 summary: 2.7.9 concrete dates files: pep-0373.txt | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/pep-0373.txt b/pep-0373.txt --- a/pep-0373.txt +++ b/pep-0373.txt @@ -71,7 +71,8 @@ Planned future release dates: -- 2.7.9 December 2014 +- 2.7.9rc1 2014-11-22 +- 2.7.9 2014-12-05 - 2.7.10 June 2015 - beyond this date, releases as needed -- Repository URL: https://hg.python.org/peps

1 0

cpython: issue22780: update NotImplemented description
by ethan.furman 31 Oct '14

31 Oct '14

https://hg.python.org/cpython/rev/26d0a17affb5 changeset: 93327:26d0a17affb5 parent: 93325:c95ce9b0a085 user: Ethan Furman <ethan(a)stoneleaf.us> date: Fri Oct 31 16:48:41 2014 -0700 summary: issue22780: update NotImplemented description files: Doc/library/constants.rst | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -26,9 +26,9 @@ .. data:: NotImplemented - Special value which can be returned by the "rich comparison" special methods - (:meth:`__eq__`, :meth:`__lt__`, and friends), to indicate that the comparison - is not implemented with respect to the other type. + Special value which should be returned by the special methods + (:meth:`__eq__`, :meth:`__lt__`, :meth:`__add__`, etc.) to indicate + that the operation is not implemented with respect to the other type. .. data:: Ellipsis -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Closes #22772: fix __ifloordiv__ and __itruediv__ docstring.
by georg.brandl 31 Oct '14

31 Oct '14

https://hg.python.org/cpython/rev/3f13f1620d70 changeset: 93326:3f13f1620d70 branch: 2.7 parent: 93323:6cd4b9827755 user: Georg Brandl <georg(a)python.org> date: Fri Oct 31 13:12:57 2014 +0100 summary: Closes #22772: fix __ifloordiv__ and __itruediv__ docstring. files: Objects/typeobject.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5993,9 +5993,9 @@ BINSLOT("__truediv__", nb_true_divide, slot_nb_true_divide, "/"), RBINSLOT("__rtruediv__", nb_true_divide, slot_nb_true_divide, "/"), IBSLOT("__ifloordiv__", nb_inplace_floor_divide, - slot_nb_inplace_floor_divide, wrap_binaryfunc, "//"), + slot_nb_inplace_floor_divide, wrap_binaryfunc, "//="), IBSLOT("__itruediv__", nb_inplace_true_divide, - slot_nb_inplace_true_divide, wrap_binaryfunc, "/"), + slot_nb_inplace_true_divide, wrap_binaryfunc, "/="), NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc, "x[y:z] <==> x[y.__index__():z.__index__()]"), MPSLOT("__len__", mp_length, slot_mp_length, wrap_lenfunc, -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.4): Closes #22772: fix __ifloordiv__ and __itruediv__ docstring.
by georg.brandl 31 Oct '14

31 Oct '14

https://hg.python.org/cpython/rev/147518a95b60 changeset: 93324:147518a95b60 branch: 3.4 parent: 93320:7981cb1556cf user: Georg Brandl <georg(a)python.org> date: Fri Oct 31 13:12:57 2014 +0100 summary: Closes #22772: fix __ifloordiv__ and __itruediv__ docstring. files: Objects/typeobject.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6271,9 +6271,9 @@ BINSLOT("__truediv__", nb_true_divide, slot_nb_true_divide, "/"), RBINSLOT("__rtruediv__", nb_true_divide, slot_nb_true_divide, "/"), IBSLOT("__ifloordiv__", nb_inplace_floor_divide, - slot_nb_inplace_floor_divide, wrap_binaryfunc, "//"), + slot_nb_inplace_floor_divide, wrap_binaryfunc, "//="), IBSLOT("__itruediv__", nb_inplace_true_divide, - slot_nb_inplace_true_divide, wrap_binaryfunc, "/"), + slot_nb_inplace_true_divide, wrap_binaryfunc, "/="), NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc, "__index__($self, /)\n--\n\n" "Return self converted to an integer, if self is suitable " -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.4 -> default): merge with 3.4
by georg.brandl 31 Oct '14

31 Oct '14

https://hg.python.org/cpython/rev/c95ce9b0a085 changeset: 93325:c95ce9b0a085 parent: 93321:79eb4ab497b1 parent: 93324:147518a95b60 user: Georg Brandl <georg(a)python.org> date: Fri Oct 31 13:13:09 2014 +0100 summary: merge with 3.4 files: Objects/typeobject.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6280,9 +6280,9 @@ BINSLOT("__truediv__", nb_true_divide, slot_nb_true_divide, "/"), RBINSLOT("__rtruediv__", nb_true_divide, slot_nb_true_divide, "/"), IBSLOT("__ifloordiv__", nb_inplace_floor_divide, - slot_nb_inplace_floor_divide, wrap_binaryfunc, "//"), + slot_nb_inplace_floor_divide, wrap_binaryfunc, "//="), IBSLOT("__itruediv__", nb_inplace_true_divide, - slot_nb_inplace_true_divide, wrap_binaryfunc, "/"), + slot_nb_inplace_true_divide, wrap_binaryfunc, "/="), NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc, "__index__($self, /)\n--\n\n" "Return self converted to an integer, if self is suitable " -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Backported the optimization of compiling charsets in regular expressions
by serhiy.storchaka 31 Oct '14

31 Oct '14

https://hg.python.org/cpython/rev/ebd48b4f650d changeset: 93322:ebd48b4f650d branch: 2.7 parent: 93316:9e57654dfdcc user: Serhiy Storchaka <storchaka(a)gmail.com> date: Fri Oct 31 13:34:06 2014 +0200 summary: Backported the optimization of compiling charsets in regular expressions (issue #19329). This is needed to apply the patch from issue #17381. files: Lib/sre_compile.py | 225 ++++++++++++++------------------ Misc/NEWS | 2 + 2 files changed, 102 insertions(+), 125 deletions(-) diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py --- a/Lib/sre_compile.py +++ b/Lib/sre_compile.py @@ -207,149 +207,124 @@ def _optimize_charset(charset, fixup): # internal: optimize character set out = [] - outappend = out.append - charmap = [0]*256 - try: - for op, av in charset: - if op is NEGATE: - outappend((op, av)) - elif op is LITERAL: - charmap[fixup(av)] = 1 - elif op is RANGE: - for i in range(fixup(av[0]), fixup(av[1])+1): - charmap[i] = 1 - elif op is CATEGORY: - # XXX: could append to charmap tail - return charset # cannot compress - except IndexError: - # character set contains unicode characters - return _optimize_unicode(charset, fixup) + tail = [] + charmap = bytearray(256) + for op, av in charset: + while True: + try: + if op is LITERAL: + charmap[fixup(av)] = 1 + elif op is RANGE: + for i in range(fixup(av[0]), fixup(av[1])+1): + charmap[i] = 1 + elif op is NEGATE: + out.append((op, av)) + else: + tail.append((op, av)) + except IndexError: + if len(charmap) == 256: + # character set contains non-UCS1 character codes + charmap += b'\0' * 0xff00 + continue + # character set contains non-BMP character codes + tail.append((op, av)) + break + # compress character map - i = p = n = 0 runs = [] - runsappend = runs.append - for c in charmap: - if c: - if n == 0: - p = i - n = n + 1 - elif n: - runsappend((p, n)) - n = 0 - i = i + 1 - if n: - runsappend((p, n)) - if len(runs) <= 2: + q = 0 + while True: + p = charmap.find(b'\1', q) + if p < 0: + break + if len(runs) >= 2: + runs = None + break + q = charmap.find(b'\0', p) + if q < 0: + runs.append((p, len(charmap))) + break + runs.append((p, q)) + if runs is not None: # use literal/range - for p, n in runs: - if n == 1: - outappend((LITERAL, p)) + for p, q in runs: + if q - p == 1: + out.append((LITERAL, p)) else: - outappend((RANGE, (p, p+n-1))) + out.append((RANGE, (p, q - 1))) + out += tail if len(out) < len(charset): return out - else: - # use bitmap + return charset + + # use bitmap + if len(charmap) == 256: data = _mk_bitmap(charmap) - outappend((CHARSET, data)) + out.append((CHARSET, data)) + out += tail return out - return charset -def _mk_bitmap(bits): - data = [] - dataappend = data.append - if _sre.CODESIZE == 2: - start = (1, 0) - else: - start = (1L, 0L) - m, v = start - for c in bits: - if c: - v = v + m - m = m + m - if m > MAXCODE: - dataappend(v) - m, v = start - return data + # To represent a big charset, first a bitmap of all characters in the + # set is constructed. Then, this bitmap is sliced into chunks of 256 + # characters, duplicate chunks are eliminated, and each chunk is + # given a number. In the compiled expression, the charset is + # represented by a 32-bit word sequence, consisting of one word for + # the number of different chunks, a sequence of 256 bytes (64 words) + # of chunk numbers indexed by their original chunk position, and a + # sequence of 256-bit chunks (8 words each). -# To represent a big charset, first a bitmap of all characters in the -# set is constructed. Then, this bitmap is sliced into chunks of 256 -# characters, duplicate chunks are eliminated, and each chunk is -# given a number. In the compiled expression, the charset is -# represented by a 32-bit word sequence, consisting of one word for -# the number of different chunks, a sequence of 256 bytes (64 words) -# of chunk numbers indexed by their original chunk position, and a -# sequence of 256-bit chunks (8 words each). + # Compression is normally good: in a typical charset, large ranges of + # Unicode will be either completely excluded (e.g. if only cyrillic + # letters are to be matched), or completely included (e.g. if large + # subranges of Kanji match). These ranges will be represented by + # chunks of all one-bits or all zero-bits. -# Compression is normally good: in a typical charset, large ranges of -# Unicode will be either completely excluded (e.g. if only cyrillic -# letters are to be matched), or completely included (e.g. if large -# subranges of Kanji match). These ranges will be represented by -# chunks of all one-bits or all zero-bits. + # Matching can be also done efficiently: the more significant byte of + # the Unicode character is an index into the chunk number, and the + # less significant byte is a bit index in the chunk (just like the + # CHARSET matching). -# Matching can be also done efficiently: the more significant byte of -# the Unicode character is an index into the chunk number, and the -# less significant byte is a bit index in the chunk (just like the -# CHARSET matching). + # In UCS-4 mode, the BIGCHARSET opcode still supports only subsets + # of the basic multilingual plane; an efficient representation + # for all of Unicode has not yet been developed. -# In UCS-4 mode, the BIGCHARSET opcode still supports only subsets -# of the basic multilingual plane; an efficient representation -# for all of Unicode has not yet been developed. This means, -# in particular, that negated charsets cannot be represented as -# bigcharsets. + charmap = bytes(charmap) # should be hashable + comps = {} + mapping = bytearray(256) + block = 0 + data = bytearray() + for i in range(0, 65536, 256): + chunk = charmap[i: i + 256] + if chunk in comps: + mapping[i // 256] = comps[chunk] + else: + mapping[i // 256] = comps[chunk] = block + block += 1 + data += chunk + data = _mk_bitmap(data) + data[0:0] = [block] + _bytes_to_codes(mapping) + out.append((BIGCHARSET, data)) + out += tail + return out -def _optimize_unicode(charset, fixup): - try: - import array - except ImportError: - return charset - charmap = [0]*65536 - negate = 0 - try: - for op, av in charset: - if op is NEGATE: - negate = 1 - elif op is LITERAL: - charmap[fixup(av)] = 1 - elif op is RANGE: - for i in xrange(fixup(av[0]), fixup(av[1])+1): - charmap[i] = 1 - elif op is CATEGORY: - # XXX: could expand category - return charset # cannot compress - except IndexError: - # non-BMP characters - return charset - if negate: - if sys.maxunicode != 65535: - # XXX: negation does not work with big charsets - return charset - for i in xrange(65536): - charmap[i] = not charmap[i] - comps = {} - mapping = [0]*256 - block = 0 - data = [] - for i in xrange(256): - chunk = tuple(charmap[i*256:(i+1)*256]) - new = comps.setdefault(chunk, block) - mapping[i] = new - if new == block: - block = block + 1 - data = data + _mk_bitmap(chunk) - header = [block] +_CODEBITS = _sre.CODESIZE * 8 +_BITS_TRANS = b'0' + b'1' * 255 +def _mk_bitmap(bits, _CODEBITS=_CODEBITS, _int=int): + s = bytes(bits).translate(_BITS_TRANS)[::-1] + return [_int(s[i - _CODEBITS: i], 2) + for i in range(len(s), 0, -_CODEBITS)] + +def _bytes_to_codes(b): + # Convert block indices to word array + import array if _sre.CODESIZE == 2: code = 'H' else: code = 'I' - # Convert block indices to byte array of 256 bytes - mapping = array.array('B', mapping).tostring() - # Convert byte array to word array - mapping = array.array(code, mapping) - assert mapping.itemsize == _sre.CODESIZE - header = header + mapping.tolist() - data[0:0] = header - return [(BIGCHARSET, data)] + a = array.array(code, bytes(b)) + assert a.itemsize == _sre.CODESIZE + assert len(a) * a.itemsize == len(b) + return a.tolist() def _simple(av): # check if av is a "simple" operator diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -37,6 +37,8 @@ Library ------- +- Issue #19329: Optimized compiling charsets in regular expressions. + - Issue #22410: Module level functions in the re module now cache compiled locale-dependent regular expressions taking into account the locale. -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Issue #17381: Fixed ranges handling in case-insensitive regular expressions.
by serhiy.storchaka 31 Oct '14

31 Oct '14

https://hg.python.org/cpython/rev/6cd4b9827755 changeset: 93323:6cd4b9827755 branch: 2.7 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Fri Oct 31 13:53:21 2014 +0200 summary: Issue #17381: Fixed ranges handling in case-insensitive regular expressions. files: Lib/sre_compile.py | 64 +++++++++++++++++++++++++------- Lib/test/test_re.py | 37 +++++++++++++++++++ Misc/NEWS | 2 + 3 files changed, 88 insertions(+), 15 deletions(-) diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py --- a/Lib/sre_compile.py +++ b/Lib/sre_compile.py @@ -21,9 +21,6 @@ else: MAXCODE = 0xFFFFFFFFL -def _identityfunction(x): - return x - _LITERAL_CODES = set([LITERAL, NOT_LITERAL]) _REPEATING_CODES = set([REPEAT, MIN_REPEAT, MAX_REPEAT]) _SUCCESS_CODES = set([SUCCESS, FAILURE]) @@ -52,7 +49,7 @@ return _sre.getlower(literal, flags) else: emit(OPCODES[op]) - fixup = _identityfunction + fixup = None skip = _len(code); emit(0) _compile_charset(av, flags, code, fixup) code[skip] = _len(code) - skip @@ -178,17 +175,15 @@ def _compile_charset(charset, flags, code, fixup=None): # compile charset subprogram emit = code.append - if fixup is None: - fixup = _identityfunction - for op, av in _optimize_charset(charset, fixup): + for op, av in _optimize_charset(charset, fixup, flags & SRE_FLAG_UNICODE): emit(OPCODES[op]) if op is NEGATE: pass elif op is LITERAL: - emit(fixup(av)) + emit(av) elif op is RANGE: - emit(fixup(av[0])) - emit(fixup(av[1])) + emit(av[0]) + emit(av[1]) elif op is CHARSET: code.extend(av) elif op is BIGCHARSET: @@ -204,7 +199,7 @@ raise error, "internal: unsupported set operator" emit(OPCODES[FAILURE]) -def _optimize_charset(charset, fixup): +def _optimize_charset(charset, fixup, isunicode): # internal: optimize character set out = [] tail = [] @@ -213,9 +208,15 @@ while True: try: if op is LITERAL: - charmap[fixup(av)] = 1 + i = av + if fixup: + i = fixup(i) + charmap[i] = 1 elif op is RANGE: - for i in range(fixup(av[0]), fixup(av[1])+1): + r = range(av[0], av[1]+1) + if fixup: + r = map(fixup, r) + for i in r: charmap[i] = 1 elif op is NEGATE: out.append((op, av)) @@ -227,7 +228,20 @@ charmap += b'\0' * 0xff00 continue # character set contains non-BMP character codes - tail.append((op, av)) + if fixup and isunicode and op is RANGE: + lo, hi = av + ranges = [av] + # There are only two ranges of cased astral characters: + # 10400-1044F (Deseret) and 118A0-118DF (Warang Citi). + _fixup_range(max(0x10000, lo), min(0x11fff, hi), + ranges, fixup) + for lo, hi in ranges: + if lo == hi: + tail.append((LITERAL, hi)) + else: + tail.append((RANGE, (lo, hi))) + else: + tail.append((op, av)) break # compress character map @@ -253,8 +267,10 @@ else: out.append((RANGE, (p, q - 1))) out += tail - if len(out) < len(charset): + # if the case was changed or new representation is more compact + if fixup or len(out) < len(charset): return out + # else original character set is good enough return charset # use bitmap @@ -307,6 +323,24 @@ out += tail return out +def _fixup_range(lo, hi, ranges, fixup): + for i in map(fixup, range(lo, hi+1)): + for k, (lo, hi) in enumerate(ranges): + if i < lo: + if l == lo - 1: + ranges[k] = (i, hi) + else: + ranges.insert(k, (i, i)) + break + elif i > hi: + if i == hi + 1: + ranges[k] = (lo, i) + break + else: + break + else: + ranges.append((i, i)) + _CODEBITS = _sre.CODESIZE * 8 _BITS_TRANS = b'0' + b'1' * 255 def _mk_bitmap(bits, _CODEBITS=_CODEBITS, _int=int): diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -474,6 +474,43 @@ self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a") self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa") + def test_ignore_case_range(self): + # Issues #3511, #17381. + self.assertTrue(re.match(r'[9-a]', '_', re.I)) + self.assertIsNone(re.match(r'[9-A]', '_', re.I)) + self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I)) + self.assertIsNone(re.match(r'[\xc0-\xde]', '\xf7', re.I)) + self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7',re.I)) + self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xd7', re.I)) + if have_unicode: + self.assertTrue(re.match(u(r'[9-a]'), u(r'_'), re.U | re.I)) + self.assertIsNone(re.match(u(r'[9-A]'), u(r'_'), re.U | re.I)) + self.assertTrue(re.match(u(r'[\xc0-\xde]'), + u(r'\xd7'), re.U | re.I)) + self.assertIsNone(re.match(u(r'[\xc0-\xde]'), + u(r'\xf7'), re.U | re.I)) + self.assertTrue(re.match(u(r'[\xe0-\xfe]'), + u(r'\xf7'), re.U | re.I)) + self.assertIsNone(re.match(u(r'[\xe0-\xfe]'), + u(r'\xd7'), re.U | re.I)) + self.assertTrue(re.match(u(r'[\u0430-\u045f]'), + u(r'\u0450'), re.U | re.I)) + self.assertTrue(re.match(u(r'[\u0430-\u045f]'), + u(r'\u0400'), re.U | re.I)) + self.assertTrue(re.match(u(r'[\u0400-\u042f]'), + u(r'\u0450'), re.U | re.I)) + self.assertTrue(re.match(u(r'[\u0400-\u042f]'), + u(r'\u0400'), re.U | re.I)) + if sys.maxunicode > 0xffff: + self.assertTrue(re.match(u(r'[\U00010428-\U0001044f]'), + u(r'\U00010428'), re.U | re.I)) + self.assertTrue(re.match(u(r'[\U00010428-\U0001044f]'), + u(r'\U00010400'), re.U | re.I)) + self.assertTrue(re.match(u(r'[\U00010400-\U00010427]'), + u(r'\U00010428'), re.U | re.I)) + self.assertTrue(re.match(u(r'[\U00010400-\U00010427]'), + u(r'\U00010400'), re.U | re.I)) + def test_category(self): self.assertEqual(re.match(r"(\s)", " ").group(1), " ") diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -37,6 +37,8 @@ Library ------- +- Issue #17381: Fixed ranges handling in case-insensitive regular expressions. + - Issue #19329: Optimized compiling charsets in regular expressions. - Issue #22410: Module level functions in the re module now cache compiled -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.4 -> default): Null merge
by serhiy.storchaka 31 Oct '14

31 Oct '14

https://hg.python.org/cpython/rev/79eb4ab497b1 changeset: 93321:79eb4ab497b1 parent: 93319:6f52a3d0f548 parent: 93320:7981cb1556cf user: Serhiy Storchaka <storchaka(a)gmail.com> date: Fri Oct 31 12:40:11 2014 +0200 summary: Null merge files: -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.4): Issue #17381: Fixed handling of case-insensitive ranges in regular expressions.
by serhiy.storchaka 31 Oct '14

31 Oct '14

https://hg.python.org/cpython/rev/7981cb1556cf changeset: 93320:7981cb1556cf branch: 3.4 parent: 93317:335ff0c6d879 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Fri Oct 31 12:37:50 2014 +0200 summary: Issue #17381: Fixed handling of case-insensitive ranges in regular expressions. files: Lib/sre_compile.py | 64 +++++++++++++++++++++++++------- Lib/test/test_re.py | 19 +++++++++ Misc/NEWS | 3 + 3 files changed, 71 insertions(+), 15 deletions(-) diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py --- a/Lib/sre_compile.py +++ b/Lib/sre_compile.py @@ -22,9 +22,6 @@ else: MAXCODE = 0xFFFFFFFF -def _identityfunction(x): - return x - _LITERAL_CODES = set([LITERAL, NOT_LITERAL]) _REPEATING_CODES = set([REPEAT, MIN_REPEAT, MAX_REPEAT]) _SUCCESS_CODES = set([SUCCESS, FAILURE]) @@ -53,7 +50,7 @@ return _sre.getlower(literal, flags) else: emit(OPCODES[op]) - fixup = _identityfunction + fixup = None skip = _len(code); emit(0) _compile_charset(av, flags, code, fixup) code[skip] = _len(code) - skip @@ -172,17 +169,15 @@ def _compile_charset(charset, flags, code, fixup=None): # compile charset subprogram emit = code.append - if fixup is None: - fixup = _identityfunction - for op, av in _optimize_charset(charset, fixup): + for op, av in _optimize_charset(charset, fixup, flags & SRE_FLAG_UNICODE): emit(OPCODES[op]) if op is NEGATE: pass elif op is LITERAL: - emit(fixup(av)) + emit(av) elif op is RANGE: - emit(fixup(av[0])) - emit(fixup(av[1])) + emit(av[0]) + emit(av[1]) elif op is CHARSET: code.extend(av) elif op is BIGCHARSET: @@ -198,7 +193,7 @@ raise error("internal: unsupported set operator") emit(OPCODES[FAILURE]) -def _optimize_charset(charset, fixup): +def _optimize_charset(charset, fixup, isunicode): # internal: optimize character set out = [] tail = [] @@ -207,9 +202,15 @@ while True: try: if op is LITERAL: - charmap[fixup(av)] = 1 + i = av + if fixup: + i = fixup(i) + charmap[i] = 1 elif op is RANGE: - for i in range(fixup(av[0]), fixup(av[1])+1): + r = range(av[0], av[1]+1) + if fixup: + r = map(fixup, r) + for i in r: charmap[i] = 1 elif op is NEGATE: out.append((op, av)) @@ -221,7 +222,20 @@ charmap += b'\0' * 0xff00 continue # character set contains non-BMP character codes - tail.append((op, av)) + if fixup and isunicode and op is RANGE: + lo, hi = av + ranges = [av] + # There are only two ranges of cased astral characters: + # 10400-1044F (Deseret) and 118A0-118DF (Warang Citi). + _fixup_range(max(0x10000, lo), min(0x11fff, hi), + ranges, fixup) + for lo, hi in ranges: + if lo == hi: + tail.append((LITERAL, hi)) + else: + tail.append((RANGE, (lo, hi))) + else: + tail.append((op, av)) break # compress character map @@ -247,8 +261,10 @@ else: out.append((RANGE, (p, q - 1))) out += tail - if len(out) < len(charset): + # if the case was changed or new representation is more compact + if fixup or len(out) < len(charset): return out + # else original character set is good enough return charset # use bitmap @@ -297,6 +313,24 @@ out += tail return out +def _fixup_range(lo, hi, ranges, fixup): + for i in map(fixup, range(lo, hi+1)): + for k, (lo, hi) in enumerate(ranges): + if i < lo: + if l == lo - 1: + ranges[k] = (i, hi) + else: + ranges.insert(k, (i, i)) + break + elif i > hi: + if i == hi + 1: + ranges[k] = (lo, i) + break + else: + break + else: + ranges.append((i, i)) + _CODEBITS = _sre.CODESIZE * 8 _BITS_TRANS = b'0' + b'1' * 255 def _mk_bitmap(bits, _CODEBITS=_CODEBITS, _int=int): diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -583,6 +583,25 @@ self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a") self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa") + def test_ignore_case_range(self): + # Issues #3511, #17381. + self.assertTrue(re.match(r'[9-a]', '_', re.I)) + self.assertIsNone(re.match(r'[9-A]', '_', re.I)) + self.assertTrue(re.match(br'[9-a]', b'_', re.I)) + self.assertIsNone(re.match(br'[9-A]', b'_', re.I)) + self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I)) + self.assertIsNone(re.match(r'[\xc0-\xde]', '\xf7', re.I)) + self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I)) + self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xd7', re.I)) + self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I)) + self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0400', re.I)) + self.assertTrue(re.match(r'[\u0400-\u042f]', '\u0450', re.I)) + self.assertTrue(re.match(r'[\u0400-\u042f]', '\u0400', re.I)) + self.assertTrue(re.match(r'[\U00010428-\U0001044f]', '\U00010428', re.I)) + self.assertTrue(re.match(r'[\U00010428-\U0001044f]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I)) + self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I)) + def test_category(self): self.assertEqual(re.match(r"(\s)", " ").group(1), " ") diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -33,6 +33,9 @@ Library ------- +- Issue #17381: Fixed handling of case-insensitive ranges in regular + expressions. + - Issue #22410: Module level functions in the re module now cache compiled locale-dependent regular expressions taking into account the locale. -- Repository URL: https://hg.python.org/cpython

1 0

cpython: Issue #17381: Fixed handling of case-insensitive ranges in regular expressions.
by serhiy.storchaka 31 Oct '14

31 Oct '14

https://hg.python.org/cpython/rev/6f52a3d0f548 changeset: 93319:6f52a3d0f548 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Fri Oct 31 12:36:56 2014 +0200 summary: Issue #17381: Fixed handling of case-insensitive ranges in regular expressions. Added new opcode RANGE_IGNORE. files: Lib/sre_compile.py | 35 +++++++++++++++++----------- Lib/sre_constants.py | 9 ++++-- Lib/test/test_re.py | 19 +++++++++++++++ Misc/NEWS | 3 ++ Modules/_sre.c | 28 ++++++++++++++++++++-- Modules/sre.h | 2 +- Modules/sre_constants.h | 3 +- Modules/sre_lib.h | 28 ++++++++++++++++++---- 8 files changed, 99 insertions(+), 28 deletions(-) diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py --- a/Lib/sre_compile.py +++ b/Lib/sre_compile.py @@ -22,9 +22,6 @@ else: MAXCODE = 0xFFFFFFFF -def _identityfunction(x): - return x - _LITERAL_CODES = set([LITERAL, NOT_LITERAL]) _REPEATING_CODES = set([REPEAT, MIN_REPEAT, MAX_REPEAT]) _SUCCESS_CODES = set([SUCCESS, FAILURE]) @@ -53,7 +50,7 @@ return _sre.getlower(literal, flags) else: emit(OPCODES[op]) - fixup = _identityfunction + fixup = None skip = _len(code); emit(0) _compile_charset(av, flags, code, fixup) code[skip] = _len(code) - skip @@ -172,17 +169,15 @@ def _compile_charset(charset, flags, code, fixup=None): # compile charset subprogram emit = code.append - if fixup is None: - fixup = _identityfunction for op, av in _optimize_charset(charset, fixup): emit(OPCODES[op]) if op is NEGATE: pass elif op is LITERAL: - emit(fixup(av)) - elif op is RANGE: - emit(fixup(av[0])) - emit(fixup(av[1])) + emit(av) + elif op is RANGE or op is RANGE_IGNORE: + emit(av[0]) + emit(av[1]) elif op is CHARSET: code.extend(av) elif op is BIGCHARSET: @@ -207,9 +202,14 @@ while True: try: if op is LITERAL: - charmap[fixup(av)] = 1 + if fixup: + av = fixup(av) + charmap[av] = 1 elif op is RANGE: - for i in range(fixup(av[0]), fixup(av[1])+1): + r = range(av[0], av[1]+1) + if fixup: + r = map(fixup, r) + for i in r: charmap[i] = 1 elif op is NEGATE: out.append((op, av)) @@ -220,7 +220,12 @@ # character set contains non-UCS1 character codes charmap += b'\0' * 0xff00 continue - # character set contains non-BMP character codes + # Character set contains non-BMP character codes. + # There are only two ranges of cased non-BMP characters: + # 10400-1044F (Deseret) and 118A0-118DF (Warang Citi), + # and for both ranges RANGE_IGNORE works. + if fixup and op is RANGE: + op = RANGE_IGNORE tail.append((op, av)) break @@ -247,8 +252,10 @@ else: out.append((RANGE, (p, q - 1))) out += tail - if len(out) < len(charset): + # if the case was changed or new representation is more compact + if fixup or len(out) < len(charset): return out + # else original character set is good enough return charset # use bitmap diff --git a/Lib/sre_constants.py b/Lib/sre_constants.py --- a/Lib/sre_constants.py +++ b/Lib/sre_constants.py @@ -13,7 +13,7 @@ # update when constants are added or removed -MAGIC = 20031017 +MAGIC = 20140917 from _sre import MAXREPEAT, MAXGROUPS @@ -56,6 +56,7 @@ NOT_LITERAL = "not_literal" NOT_LITERAL_IGNORE = "not_literal_ignore" RANGE = "range" +RANGE_IGNORE = "range_ignore" REPEAT = "repeat" REPEAT_ONE = "repeat_one" SUBPATTERN = "subpattern" @@ -121,7 +122,8 @@ REPEAT, REPEAT_ONE, SUBPATTERN, - MIN_REPEAT_ONE + MIN_REPEAT_ONE, + RANGE_IGNORE, ] @@ -159,7 +161,8 @@ GROUPREF: GROUPREF_IGNORE, IN: IN_IGNORE, LITERAL: LITERAL_IGNORE, - NOT_LITERAL: NOT_LITERAL_IGNORE + NOT_LITERAL: NOT_LITERAL_IGNORE, + RANGE: RANGE_IGNORE, } AT_MULTILINE = { diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -601,6 +601,25 @@ self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a") self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa") + def test_ignore_case_range(self): + # Issues #3511, #17381. + self.assertTrue(re.match(r'[9-a]', '_', re.I)) + self.assertIsNone(re.match(r'[9-A]', '_', re.I)) + self.assertTrue(re.match(br'[9-a]', b'_', re.I)) + self.assertIsNone(re.match(br'[9-A]', b'_', re.I)) + self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I)) + self.assertIsNone(re.match(r'[\xc0-\xde]', '\xf7', re.I)) + self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I)) + self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xd7', re.I)) + self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I)) + self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0400', re.I)) + self.assertTrue(re.match(r'[\u0400-\u042f]', '\u0450', re.I)) + self.assertTrue(re.match(r'[\u0400-\u042f]', '\u0400', re.I)) + self.assertTrue(re.match(r'[\U00010428-\U0001044f]', '\U00010428', re.I)) + self.assertTrue(re.match(r'[\U00010428-\U0001044f]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I)) + self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I)) + def test_category(self): self.assertEqual(re.match(r"(\s)", " ").group(1), " ") diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -180,6 +180,9 @@ Library ------- +- Issue #17381: Fixed handling of case-insensitive ranges in regular + expressions. + - Issue #22410: Module level functions in the re module now cache compiled locale-dependent regular expressions taking into account the locale. diff --git a/Modules/_sre.c b/Modules/_sre.c --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -113,6 +113,11 @@ return ((ch) < 128 ? Py_TOLOWER(ch) : ch); } +static unsigned int sre_upper(unsigned int ch) +{ + return ((ch) < 128 ? Py_TOUPPER(ch) : ch); +} + /* locale-specific character predicates */ /* !(c & ~N) == (c < N+1) for any unsigned c, this avoids * warnings when c's type supports only numbers < N+1 */ @@ -124,6 +129,11 @@ return ((ch) < 256 ? (unsigned int)tolower((ch)) : ch); } +static unsigned int sre_upper_locale(unsigned int ch) +{ + return ((ch) < 256 ? (unsigned int)toupper((ch)) : ch); +} + /* unicode-specific character predicates */ #define SRE_UNI_IS_DIGIT(ch) Py_UNICODE_ISDECIMAL(ch) @@ -137,6 +147,11 @@ return (unsigned int) Py_UNICODE_TOLOWER(ch); } +static unsigned int sre_upper_unicode(unsigned int ch) +{ + return (unsigned int) Py_UNICODE_TOUPPER(ch); +} + LOCAL(int) sre_category(SRE_CODE category, unsigned int ch) { @@ -377,12 +392,18 @@ state->pos = start; state->endpos = end; - if (pattern->flags & SRE_FLAG_LOCALE) + if (pattern->flags & SRE_FLAG_LOCALE) { state->lower = sre_lower_locale; - else if (pattern->flags & SRE_FLAG_UNICODE) + state->upper = sre_upper_locale; + } + else if (pattern->flags & SRE_FLAG_UNICODE) { state->lower = sre_lower_unicode; - else + state->upper = sre_upper_unicode; + } + else { state->lower = sre_lower; + state->upper = sre_upper; + } return string; err: @@ -1567,6 +1588,7 @@ break; case SRE_OP_RANGE: + case SRE_OP_RANGE_IGNORE: GET_ARG; GET_ARG; break; diff --git a/Modules/sre.h b/Modules/sre.h --- a/Modules/sre.h +++ b/Modules/sre.h @@ -84,7 +84,7 @@ /* current repeat context */ SRE_REPEAT *repeat; /* hooks */ - SRE_TOLOWER_HOOK lower; + SRE_TOLOWER_HOOK lower, upper; } SRE_STATE; typedef struct { diff --git a/Modules/sre_constants.h b/Modules/sre_constants.h --- a/Modules/sre_constants.h +++ b/Modules/sre_constants.h @@ -11,7 +11,7 @@ * See the _sre.c file for information on usage and redistribution. */ -#define SRE_MAGIC 20031017 +#define SRE_MAGIC 20140917 #define SRE_OP_FAILURE 0 #define SRE_OP_SUCCESS 1 #define SRE_OP_ANY 2 @@ -44,6 +44,7 @@ #define SRE_OP_REPEAT_ONE 29 #define SRE_OP_SUBPATTERN 30 #define SRE_OP_MIN_REPEAT_ONE 31 +#define SRE_OP_RANGE_IGNORE 32 #define SRE_AT_BEGINNING 0 #define SRE_AT_BEGINNING_LINE 1 #define SRE_AT_BEGINNING_STRING 2 diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -101,7 +101,7 @@ } LOCAL(int) -SRE(charset)(SRE_CODE* set, SRE_CODE ch) +SRE(charset)(SRE_STATE* state, SRE_CODE* set, SRE_CODE ch) { /* check if character is a member of the given set */ @@ -142,6 +142,20 @@ set += 2; break; + case SRE_OP_RANGE_IGNORE: + /* <RANGE_IGNORE> <lower> <upper> */ + { + SRE_CODE uch; + /* ch is already lower cased */ + if (set[0] <= ch && ch <= set[1]) + return ok; + uch = state->upper(ch); + if (set[0] <= uch && uch <= set[1]) + return ok; + set += 2; + break; + } + case SRE_OP_NEGATE: ok = !ok; break; @@ -193,7 +207,7 @@ case SRE_OP_IN: /* repeated set */ TRACE(("|%p|%p|COUNT IN\n", pattern, ptr)); - while (ptr < end && SRE(charset)(pattern + 2, *ptr)) + while (ptr < end && SRE(charset)(state, pattern + 2, *ptr)) ptr++; break; @@ -628,7 +642,8 @@ /* match set member (or non_member) */ /* <IN> <skip> <set> */ TRACE(("|%p|%p|IN\n", ctx->pattern, ctx->ptr)); - if (ctx->ptr >= end || !SRE(charset)(ctx->pattern + 1, *ctx->ptr)) + if (ctx->ptr >= end || + !SRE(charset)(state, ctx->pattern + 1, *ctx->ptr)) RETURN_FAILURE; ctx->pattern += ctx->pattern[0]; ctx->ptr++; @@ -657,7 +672,7 @@ case SRE_OP_IN_IGNORE: TRACE(("|%p|%p|IN_IGNORE\n", ctx->pattern, ctx->ptr)); if (ctx->ptr >= end - || !SRE(charset)(ctx->pattern+1, + || !SRE(charset)(state, ctx->pattern+1, (SRE_CODE)state->lower(*ctx->ptr))) RETURN_FAILURE; ctx->pattern += ctx->pattern[0]; @@ -688,7 +703,8 @@ continue; if (ctx->pattern[1] == SRE_OP_IN && (ctx->ptr >= end || - !SRE(charset)(ctx->pattern + 3, (SRE_CODE) *ctx->ptr))) + !SRE(charset)(state, ctx->pattern + 3, + (SRE_CODE) *ctx->ptr))) continue; state->ptr = ctx->ptr; DO_JUMP(JUMP_BRANCH, jump_branch, ctx->pattern+1); @@ -1310,7 +1326,7 @@ /* pattern starts with a character from a known set */ end = (SRE_CHAR *)state->end; for (;;) { - while (ptr < end && !SRE(charset)(charset, *ptr)) + while (ptr < end && !SRE(charset)(state, charset, *ptr)) ptr++; if (ptr >= end) return 0; -- Repository URL: https://hg.python.org/cpython

1 0