Skip to content

C
cpython
Commit e927757d authored by Serhiy Storchaka's avatar Serhiy Storchaka
Browse files
Options

Issue #12728: Different Unicode characters having the same uppercase but 

different lowercase are now matched in case-insensitive regular expressions.
parent 069bdcbb
Hide whitespace changes
Inline Side-by-side
Showing with 124 additions and 11 deletions
Lib/sre_compile.py
View file @ e927757d
# -*- coding: utf-8 -*-
#
# Secret Labs' Regular Expression Engine
#
......@@ -26,6 +27,40 @@ _REPEATING_CODES = set([REPEAT, MIN_REPEAT, MAX_REPEAT])
_SUCCESS_CODES = set([SUCCESS, FAILURE])
_ASSERT_CODES = set([ASSERT, ASSERT_NOT])
# Sets of lowercase characters which have the same uppercase.
_equivalences = (
# LATIN SMALL LETTER I, LATIN SMALL LETTER DOTLESS I
(0x69, 0x131), # iı
# LATIN SMALL LETTER S, LATIN SMALL LETTER LONG S
(0x73, 0x17f), # sſ
# MICRO SIGN, GREEK SMALL LETTER MU
(0xb5, 0x3bc), # µμ
# COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI
(0x345, 0x3b9, 0x1fbe), # \u0345ιι
# GREEK SMALL LETTER BETA, GREEK BETA SYMBOL
(0x3b2, 0x3d0), # βϐ
# GREEK SMALL LETTER EPSILON, GREEK LUNATE EPSILON SYMBOL
(0x3b5, 0x3f5), # εϵ
# GREEK SMALL LETTER THETA, GREEK THETA SYMBOL
(0x3b8, 0x3d1), # θϑ
# GREEK SMALL LETTER KAPPA, GREEK KAPPA SYMBOL
(0x3ba, 0x3f0), # κϰ
# GREEK SMALL LETTER PI, GREEK PI SYMBOL
(0x3c0, 0x3d6), # πϖ
# GREEK SMALL LETTER RHO, GREEK RHO SYMBOL
(0x3c1, 0x3f1), # ρϱ
# GREEK SMALL LETTER FINAL SIGMA, GREEK SMALL LETTER SIGMA
(0x3c2, 0x3c3), # ςσ
# GREEK SMALL LETTER PHI, GREEK PHI SYMBOL
(0x3c6, 0x3d5), # φϕ
# LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT ABOVE
(0x1e61, 0x1e9b), # ṡẛ
)
# Maps the lowercase code to lowercase codes which have the same uppercase.
_ignorecase_fixes = {i: tuple(j for j in t if i != j)
for t in _equivalences for i in t}
def _compile(code, pattern, flags):
# internal: compile a (sub)pattern
emit = code.append
......@@ -34,11 +69,29 @@ def _compile(code, pattern, flags):
REPEATING_CODES = _REPEATING_CODES
SUCCESS_CODES = _SUCCESS_CODES
ASSERT_CODES = _ASSERT_CODES
if (flags & SRE_FLAG_IGNORECASE and
not (flags & SRE_FLAG_LOCALE) and
flags & SRE_FLAG_UNICODE):
fixes = _ignorecase_fixes
else:
fixes = None
for op, av in pattern:
if op in LITERAL_CODES:
if flags & SRE_FLAG_IGNORECASE:
emit(OPCODES[OP_IGNORE[op]])
emit(_sre.getlower(av, flags))
lo = _sre.getlower(av, flags)
if fixes and lo in fixes:
emit(OPCODES[IN_IGNORE])
skip = _len(code); emit(0)
if op is NOT_LITERAL:
emit(OPCODES[NEGATE])
for k in (lo,) + fixes[lo]:
emit(OPCODES[LITERAL])
emit(k)
emit(OPCODES[FAILURE])
code[skip] = _len(code) - skip
else:
emit(OPCODES[OP_IGNORE[op]])
emit(lo)
else:
emit(OPCODES[op])
emit(av)
......@@ -51,7 +104,7 @@ def _compile(code, pattern, flags):
emit(OPCODES[op])
fixup = None
skip = _len(code); emit(0)
_compile_charset(av, flags, code, fixup)
_compile_charset(av, flags, code, fixup, fixes)
code[skip] = _len(code) - skip
elif op is ANY:
if flags & SRE_FLAG_DOTALL:
......@@ -172,10 +225,11 @@ def _compile(code, pattern, flags):
else:
raise ValueError, ("unsupported operand type", op)
def _compile_charset(charset, flags, code, fixup=None):
def _compile_charset(charset, flags, code, fixup=None, fixes=None):
# compile charset subprogram
emit = code.append
for op, av in _optimize_charset(charset, fixup, flags & SRE_FLAG_UNICODE):
for op, av in _optimize_charset(charset, fixup, fixes,
flags & SRE_FLAG_UNICODE):
emit(OPCODES[op])
if op is NEGATE:
pass
......@@ -199,7 +253,7 @@ def _compile_charset(charset, flags, code, fixup=None):
raise error, "internal: unsupported set operator"
emit(OPCODES[FAILURE])
def _optimize_charset(charset, fixup, isunicode):
def _optimize_charset(charset, fixup, fixes, isunicode):
# internal: optimize character set
out = []
tail = []
......@@ -208,16 +262,27 @@ def _optimize_charset(charset, fixup, isunicode):
while True:
try:
if op is LITERAL:
i = av
if fixup:
i = fixup(i)
charmap[i] = 1
i = fixup(av)
charmap[i] = 1
if fixes and i in fixes:
for k in fixes[i]:
charmap[k] = 1
else:
charmap[av] = 1
elif op is RANGE:
r = range(av[0], av[1]+1)
if fixup:
r = map(fixup, r)
for i in r:
charmap[i] = 1
if fixup and fixes:
for i in r:
charmap[i] = 1
if i in fixes:
for k in fixes[i]:
charmap[k] = 1
else:
for i in r:
charmap[i] = 1
elif op is NEGATE:
out.append((op, av))
else:
......
Lib/test/test_re.py
View file @ e927757d
# -*- coding: utf-8 -*-
from test.test_support import verbose, run_unittest, import_module
from test.test_support import precisionbigmemtest, _2G, cpython_only
from test.test_support import captured_stdout, have_unicode, requires_unicode, u
......@@ -510,6 +511,39 @@ class ReTests(unittest.TestCase):
self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a")
self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa")
if have_unicode:
assert u(r'\u212a').lower() == u'k' # 'K'
self.assertTrue(re.match(ur'K', u(r'\u212a'), re.U | re.I))
self.assertTrue(re.match(ur'k', u(r'\u212a'), re.U | re.I))
self.assertTrue(re.match(u(r'\u212a'), u'K', re.U | re.I))
self.assertTrue(re.match(u(r'\u212a'), u'k', re.U | re.I))
assert u(r'\u017f').upper() == u'S' # 'ſ'
self.assertTrue(re.match(ur'S', u(r'\u017f'), re.U | re.I))
self.assertTrue(re.match(ur's', u(r'\u017f'), re.U | re.I))
self.assertTrue(re.match(u(r'\u017f'), u'S', re.U | re.I))
self.assertTrue(re.match(u(r'\u017f'), u's', re.U | re.I))
def test_ignore_case_set(self):
self.assertTrue(re.match(r'[19A]', 'A', re.I))
self.assertTrue(re.match(r'[19a]', 'a', re.I))
self.assertTrue(re.match(r'[19a]', 'A', re.I))
self.assertTrue(re.match(r'[19A]', 'a', re.I))
if have_unicode:
self.assertTrue(re.match(ur'[19A]', u'A', re.U | re.I))
self.assertTrue(re.match(ur'[19a]', u'a', re.U | re.I))
self.assertTrue(re.match(ur'[19a]', u'A', re.U | re.I))
self.assertTrue(re.match(ur'[19A]', u'a', re.U | re.I))
assert u(r'\u212a').lower() == u'k' # 'K'
self.assertTrue(re.match(u(r'[19K]'), u(r'\u212a'), re.U | re.I))
self.assertTrue(re.match(u(r'[19k]'), u(r'\u212a'), re.U | re.I))
self.assertTrue(re.match(u(r'[19\u212a]'), u'K', re.U | re.I))
self.assertTrue(re.match(u(r'[19\u212a]'), u'k', re.U | re.I))
assert u(r'\u017f').upper() == u'S' # 'ſ'
self.assertTrue(re.match(ur'[19S]', u(r'\u017f'), re.U | re.I))
self.assertTrue(re.match(ur'[19s]', u(r'\u017f'), re.U | re.I))
self.assertTrue(re.match(u(r'[19\u017f]'), u'S', re.U | re.I))
self.assertTrue(re.match(u(r'[19\u017f]'), u's', re.U | re.I))
def test_ignore_case_range(self):
# Issues #3511, #17381.
self.assertTrue(re.match(r'[9-a]', '_', re.I))
......@@ -547,6 +581,17 @@ class ReTests(unittest.TestCase):
self.assertTrue(re.match(u(r'[\U00010400-\U00010427]'),
u(r'\U00010400'), re.U | re.I))
assert u(r'\u212a').lower() == u'k' # 'K'
self.assertTrue(re.match(ur'[J-M]', u(r'\u212a'), re.U | re.I))
self.assertTrue(re.match(ur'[j-m]', u(r'\u212a'), re.U | re.I))
self.assertTrue(re.match(u(r'[\u2129-\u212b]'), u'K', re.U | re.I))
self.assertTrue(re.match(u(r'[\u2129-\u212b]'), u'k', re.U | re.I))
assert u(r'\u017f').upper() == u'S' # 'ſ'
self.assertTrue(re.match(ur'[R-T]', u(r'\u017f'), re.U | re.I))
self.assertTrue(re.match(ur'[r-t]', u(r'\u017f'), re.U | re.I))
self.assertTrue(re.match(u(r'[\u017e-\u0180]'), u'S', re.U | re.I))
self.assertTrue(re.match(u(r'[\u017e-\u0180]'), u's', re.U | re.I))
def test_category(self):
self.assertEqual(re.match(r"(\s)", " ").group(1), " ")
......
Misc/NEWS
View file @ e927757d
......@@ -37,6 +37,9 @@ Core and Builtins
Library
-------
- Issue #12728: Different Unicode characters having the same uppercase but
different lowercase are now matched in case-insensitive regular expressions.
- Issue #22821: Fixed fcntl() with integer argument on 64-bit big-endian
platforms.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7