[Python-checkins] cpython (2.7): Issue #12728: Different Unicode characters having the same uppercase but
serhiy.storchaka
python-checkins at python.org
Mon Nov 10 11:47:06 CET 2014
https://hg.python.org/cpython/rev/4caa695af94c
changeset: 93456:4caa695af94c
branch: 2.7
parent: 93453:61e99438c237
user: Serhiy Storchaka <storchaka at gmail.com>
date: Mon Nov 10 12:37:02 2014 +0200
summary:
Issue #12728: Different Unicode characters having the same uppercase but
different lowercase are now matched in case-insensitive regular expressions.
files:
Lib/sre_compile.py | 87 ++++++++++++++++++++++++++++----
Lib/test/test_re.py | 45 +++++++++++++++++
Misc/NEWS | 3 +
3 files changed, 124 insertions(+), 11 deletions(-)
diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py
--- a/Lib/sre_compile.py
+++ b/Lib/sre_compile.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
#
# Secret Labs' Regular Expression Engine
#
@@ -26,6 +27,40 @@
_SUCCESS_CODES = set([SUCCESS, FAILURE])
_ASSERT_CODES = set([ASSERT, ASSERT_NOT])
+# Sets of lowercase characters which have the same uppercase.
+_equivalences = (
+ # LATIN SMALL LETTER I, LATIN SMALL LETTER DOTLESS I
+ (0x69, 0x131), # iı
+ # LATIN SMALL LETTER S, LATIN SMALL LETTER LONG S
+ (0x73, 0x17f), # sſ
+ # MICRO SIGN, GREEK SMALL LETTER MU
+ (0xb5, 0x3bc), # µμ
+ # COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI
+ (0x345, 0x3b9, 0x1fbe), # \u0345ιι
+ # GREEK SMALL LETTER BETA, GREEK BETA SYMBOL
+ (0x3b2, 0x3d0), # βϐ
+ # GREEK SMALL LETTER EPSILON, GREEK LUNATE EPSILON SYMBOL
+ (0x3b5, 0x3f5), # εϵ
+ # GREEK SMALL LETTER THETA, GREEK THETA SYMBOL
+ (0x3b8, 0x3d1), # θϑ
+ # GREEK SMALL LETTER KAPPA, GREEK KAPPA SYMBOL
+ (0x3ba, 0x3f0), # κϰ
+ # GREEK SMALL LETTER PI, GREEK PI SYMBOL
+ (0x3c0, 0x3d6), # πϖ
+ # GREEK SMALL LETTER RHO, GREEK RHO SYMBOL
+ (0x3c1, 0x3f1), # ρϱ
+ # GREEK SMALL LETTER FINAL SIGMA, GREEK SMALL LETTER SIGMA
+ (0x3c2, 0x3c3), # ςσ
+ # GREEK SMALL LETTER PHI, GREEK PHI SYMBOL
+ (0x3c6, 0x3d5), # φϕ
+ # LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT ABOVE
+ (0x1e61, 0x1e9b), # ṡẛ
+)
+
+# Maps the lowercase code to lowercase codes which have the same uppercase.
+_ignorecase_fixes = {i: tuple(j for j in t if i != j)
+ for t in _equivalences for i in t}
+
def _compile(code, pattern, flags):
# internal: compile a (sub)pattern
emit = code.append
@@ -34,11 +69,29 @@
REPEATING_CODES = _REPEATING_CODES
SUCCESS_CODES = _SUCCESS_CODES
ASSERT_CODES = _ASSERT_CODES
+ if (flags & SRE_FLAG_IGNORECASE and
+ not (flags & SRE_FLAG_LOCALE) and
+ flags & SRE_FLAG_UNICODE):
+ fixes = _ignorecase_fixes
+ else:
+ fixes = None
for op, av in pattern:
if op in LITERAL_CODES:
if flags & SRE_FLAG_IGNORECASE:
- emit(OPCODES[OP_IGNORE[op]])
- emit(_sre.getlower(av, flags))
+ lo = _sre.getlower(av, flags)
+ if fixes and lo in fixes:
+ emit(OPCODES[IN_IGNORE])
+ skip = _len(code); emit(0)
+ if op is NOT_LITERAL:
+ emit(OPCODES[NEGATE])
+ for k in (lo,) + fixes[lo]:
+ emit(OPCODES[LITERAL])
+ emit(k)
+ emit(OPCODES[FAILURE])
+ code[skip] = _len(code) - skip
+ else:
+ emit(OPCODES[OP_IGNORE[op]])
+ emit(lo)
else:
emit(OPCODES[op])
emit(av)
@@ -51,7 +104,7 @@
emit(OPCODES[op])
fixup = None
skip = _len(code); emit(0)
- _compile_charset(av, flags, code, fixup)
+ _compile_charset(av, flags, code, fixup, fixes)
code[skip] = _len(code) - skip
elif op is ANY:
if flags & SRE_FLAG_DOTALL:
@@ -172,10 +225,11 @@
else:
raise ValueError, ("unsupported operand type", op)
-def _compile_charset(charset, flags, code, fixup=None):
+def _compile_charset(charset, flags, code, fixup=None, fixes=None):
# compile charset subprogram
emit = code.append
- for op, av in _optimize_charset(charset, fixup, flags & SRE_FLAG_UNICODE):
+ for op, av in _optimize_charset(charset, fixup, fixes,
+ flags & SRE_FLAG_UNICODE):
emit(OPCODES[op])
if op is NEGATE:
pass
@@ -199,7 +253,7 @@
raise error, "internal: unsupported set operator"
emit(OPCODES[FAILURE])
-def _optimize_charset(charset, fixup, isunicode):
+def _optimize_charset(charset, fixup, fixes, isunicode):
# internal: optimize character set
out = []
tail = []
@@ -208,16 +262,27 @@
while True:
try:
if op is LITERAL:
- i = av
if fixup:
- i = fixup(i)
- charmap[i] = 1
+ i = fixup(av)
+ charmap[i] = 1
+ if fixes and i in fixes:
+ for k in fixes[i]:
+ charmap[k] = 1
+ else:
+ charmap[av] = 1
elif op is RANGE:
r = range(av[0], av[1]+1)
if fixup:
r = map(fixup, r)
- for i in r:
- charmap[i] = 1
+ if fixup and fixes:
+ for i in r:
+ charmap[i] = 1
+ if i in fixes:
+ for k in fixes[i]:
+ charmap[k] = 1
+ else:
+ for i in r:
+ charmap[i] = 1
elif op is NEGATE:
out.append((op, av))
else:
diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py
--- a/Lib/test/test_re.py
+++ b/Lib/test/test_re.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
from test.test_support import verbose, run_unittest, import_module
from test.test_support import precisionbigmemtest, _2G, cpython_only
from test.test_support import captured_stdout, have_unicode, requires_unicode, u
@@ -510,6 +511,39 @@
self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a")
self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa")
+ if have_unicode:
+ assert u(r'\u212a').lower() == u'k' # 'K'
+ self.assertTrue(re.match(ur'K', u(r'\u212a'), re.U | re.I))
+ self.assertTrue(re.match(ur'k', u(r'\u212a'), re.U | re.I))
+ self.assertTrue(re.match(u(r'\u212a'), u'K', re.U | re.I))
+ self.assertTrue(re.match(u(r'\u212a'), u'k', re.U | re.I))
+ assert u(r'\u017f').upper() == u'S' # 'ſ'
+ self.assertTrue(re.match(ur'S', u(r'\u017f'), re.U | re.I))
+ self.assertTrue(re.match(ur's', u(r'\u017f'), re.U | re.I))
+ self.assertTrue(re.match(u(r'\u017f'), u'S', re.U | re.I))
+ self.assertTrue(re.match(u(r'\u017f'), u's', re.U | re.I))
+
+ def test_ignore_case_set(self):
+ self.assertTrue(re.match(r'[19A]', 'A', re.I))
+ self.assertTrue(re.match(r'[19a]', 'a', re.I))
+ self.assertTrue(re.match(r'[19a]', 'A', re.I))
+ self.assertTrue(re.match(r'[19A]', 'a', re.I))
+ if have_unicode:
+ self.assertTrue(re.match(ur'[19A]', u'A', re.U | re.I))
+ self.assertTrue(re.match(ur'[19a]', u'a', re.U | re.I))
+ self.assertTrue(re.match(ur'[19a]', u'A', re.U | re.I))
+ self.assertTrue(re.match(ur'[19A]', u'a', re.U | re.I))
+ assert u(r'\u212a').lower() == u'k' # 'K'
+ self.assertTrue(re.match(u(r'[19K]'), u(r'\u212a'), re.U | re.I))
+ self.assertTrue(re.match(u(r'[19k]'), u(r'\u212a'), re.U | re.I))
+ self.assertTrue(re.match(u(r'[19\u212a]'), u'K', re.U | re.I))
+ self.assertTrue(re.match(u(r'[19\u212a]'), u'k', re.U | re.I))
+ assert u(r'\u017f').upper() == u'S' # 'ſ'
+ self.assertTrue(re.match(ur'[19S]', u(r'\u017f'), re.U | re.I))
+ self.assertTrue(re.match(ur'[19s]', u(r'\u017f'), re.U | re.I))
+ self.assertTrue(re.match(u(r'[19\u017f]'), u'S', re.U | re.I))
+ self.assertTrue(re.match(u(r'[19\u017f]'), u's', re.U | re.I))
+
def test_ignore_case_range(self):
# Issues #3511, #17381.
self.assertTrue(re.match(r'[9-a]', '_', re.I))
@@ -547,6 +581,17 @@
self.assertTrue(re.match(u(r'[\U00010400-\U00010427]'),
u(r'\U00010400'), re.U | re.I))
+ assert u(r'\u212a').lower() == u'k' # 'K'
+ self.assertTrue(re.match(ur'[J-M]', u(r'\u212a'), re.U | re.I))
+ self.assertTrue(re.match(ur'[j-m]', u(r'\u212a'), re.U | re.I))
+ self.assertTrue(re.match(u(r'[\u2129-\u212b]'), u'K', re.U | re.I))
+ self.assertTrue(re.match(u(r'[\u2129-\u212b]'), u'k', re.U | re.I))
+ assert u(r'\u017f').upper() == u'S' # 'ſ'
+ self.assertTrue(re.match(ur'[R-T]', u(r'\u017f'), re.U | re.I))
+ self.assertTrue(re.match(ur'[r-t]', u(r'\u017f'), re.U | re.I))
+ self.assertTrue(re.match(u(r'[\u017e-\u0180]'), u'S', re.U | re.I))
+ self.assertTrue(re.match(u(r'[\u017e-\u0180]'), u's', re.U | re.I))
+
def test_category(self):
self.assertEqual(re.match(r"(\s)", " ").group(1), " ")
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -37,6 +37,9 @@
Library
-------
+- Issue #12728: Different Unicode characters having the same uppercase but
+ different lowercase are now matched in case-insensitive regular expressions.
+
- Issue #22821: Fixed fcntl() with integer argument on 64-bit big-endian
platforms.
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list