New re version from AMK

Lib/re.py

 #!/usr/bin/env python
 # -*- mode: python -*-
-# $Id$
-
 
 import sys
 import string
@@ -12,11 +10,12 @@ from pcre import *
 #
 
 # pcre.error and re.error should be the same, since exceptions can be
-# raised from  either module.
+# raised from either module.
 
 # compilation flags
 
 I = IGNORECASE
+L = LOCALE
 M = MULTILINE
 S = DOTALL 
 X = VERBOSE 
@@ -61,8 +60,25 @@ def split(pattern, string, maxsplit=0):
 	pattern = _cachecompile(pattern)
     return pattern.split(string, maxsplit)
 
+def escape(pattern):
+    "Escape all non-alphanumeric characters in pattern."
+    result = []
+    alphanum=string.letters+'_'+string.digits
+    for char in pattern:
+	if char not in alphanum:
+	    result.append('\\')
+	result.append(char)
+    return string.join(result, '')
+
+def compile(pattern, flags=0):
+    "Compile a regular expression pattern, returning a RegexObject."
+    groupindex={}
+    code=pcre_compile(pattern, flags, groupindex)
+    return RegexObject(pattern, flags, code, groupindex)
+    
+
 #
-#
+#   Class definitions
 #
 
 class RegexObject:
@@ -71,31 +87,54 @@ class RegexObject:
 	self.flags = flags
 	self.pattern = pattern
 	self.groupindex = groupindex
-    def search(self, string, pos=0):
-	regs = self.code.match(string, pos, 0)
+
+    def search(self, string, pos=0, endpos=None):
+	"""Scan through string looking for a match to the pattern, returning
+	a MatchObject instance, or None if no match was found."""
+
+	if endpos is None or endpos>len(string): 
+	    endpos=len(string)
+	if endpos<pos: endpos=pos
+	regs = self.code.match(string, pos, endpos, 0)
 	if regs is None:
 	    return None
-	self.num_regs=len(regs)
+	self._num_regs=len(regs)
 	
 	return MatchObject(self,
 			   string,
-			   pos,
+			   pos, endpos,
 			   regs)
     
-    def match(self, string, pos=0):
-	regs = self.code.match(string, pos, ANCHORED)
+    def match(self, string, pos=0, endpos=None):
+	"""Try to apply the pattern at the start of the string, returning
+	a MatchObject instance, or None if no match was found."""
+
+	if endpos is None or endpos>len(string): 
+	    endpos=len(string)
+	if endpos<pos: endpos=pos
+	regs = self.code.match(string, pos, endpos, ANCHORED)
 	if regs is None:
 	    return None
-	self.num_regs=len(regs)
+	self._num_regs=len(regs)
 	return MatchObject(self,
 			   string,
-			   pos,
+			   pos, endpos,
 			   regs)
     
     def sub(self, repl, string, count=0):
+	"""Return the string obtained by replacing the leftmost
+	non-overlapping occurrences of the pattern in string by the
+	replacement repl""" 
+
         return self.subn(repl, string, count)[0]
     
-    def subn(self, repl, source, count=0):
+    def subn(self, repl, source, count=0): 
+	"""Return a 2-tuple containing (new_string, number).
+	new_string is the string obtained by replacing the leftmost
+	non-overlapping occurrences of the pattern in string by the
+	replacement repl.  number is the number of substitutions that
+	were made."""
+
 	if count < 0:
 	    raise error, "negative substitution count"
 	if count == 0:
@@ -134,6 +173,9 @@ class RegexObject:
 	return (string.join(results, ''), n)
 									    
     def split(self, source, maxsplit=0):
+	"""Split \var{string} by the occurrences of the pattern,
+	returning a list containing the resulting substrings."""
+
 	if maxsplit < 0:
 	    raise error, "negative split count"
 	if maxsplit == 0:
@@ -156,21 +198,39 @@ class RegexObject:
 		pos = pos+1
 		continue
 	    results.append(source[lastmatch:i])
-	    g = m.group()
+	    g = m.groups()
 	    if g:
+		if type(g)==type( "" ): g = [g]
 		results[len(results):] = list(g)
 	    pos = lastmatch = j
 	results.append(source[lastmatch:])
 	return results
 
+    # The following 3 functions were contributed by Mike Fletcher, and
+    # allow pickling and unpickling of RegexObject instances.
+    def __getinitargs__(self):
+        return (None,None,None,None) # any 4 elements, to work around
+                                     # problems with the
+				     # pickle/cPickle modules not yet 
+				     # ignoring the __init__ function
+    def __getstate__(self):
+        return self.pattern, self.flags, self.groupindex
+    def __setstate__(self, statetuple):
+        self.pattern = statetuple[0]
+        self.flags = statetuple[1]
+        self.groupindex = statetuple[2]
+        self.code = apply(pcre_compile, statetuple)
+
 class MatchObject:
-    def __init__(self, re, string, pos, regs):
+    def __init__(self, re, string, pos, endpos, regs):
 	self.re = re
 	self.string = string
-	self.pos = pos
+	self.pos = pos 
+	self.endpos = endpos
 	self.regs = regs
 	
-    def start(self, g):
+    def start(self, g = 0):
+	"Return the start of the substring matched by group g"
 	if type(g) == type(''):
 	    try:
 		g = self.re.groupindex[g]
@@ -178,7 +238,8 @@ class MatchObject:
 		raise IndexError, ('group "' + g + '" is undefined')
 	return self.regs[g][0]
     
-    def end(self, g):
+    def end(self, g = 0):
+	"Return the end of the substring matched by group g"
 	if type(g) == type(''):
 	    try:
 		g = self.re.groupindex[g]
@@ -186,7 +247,9 @@ class MatchObject:
 		raise IndexError, ('group "' + g + '" is undefined')
 	return self.regs[g][1]
     
-    def span(self, g):
+    def span(self, g = 0):
+	"""Return a tuple containing the start,end of the substring 
+	matched by group g"""
 	if type(g) == type(''):
 	    try:
 		g = self.re.groupindex[g]
@@ -194,12 +257,18 @@ class MatchObject:
 		raise IndexError, ('group "' + g + '" is undefined')
 	return self.regs[g]
     
+    def groups(self):
+	"Return a tuple containing all subgroups of the match object"
+
+	# If _num_regs==1, we don't want to call self.group with an
+	# empty tuple.
+	if self.re._num_regs == 1: return ()
+	return apply(self.group, tuple(range(1, self.re._num_regs) ) )
+
     def group(self, *groups):
+	"Return one or more groups of the match."
 	if len(groups) == 0:
-	    groups = range(1, self.re.num_regs)
-	    use_all = 1
-	else:
-	    use_all = 0
+	    groups = (0,)
 	result = []
 	for g in groups:
 	    if type(g) == type(''):
@@ -212,25 +281,10 @@ class MatchObject:
 		result.append(None)
 	    else:
 		result.append(self.string[self.regs[g][0]:self.regs[g][1]])
-	if use_all or len(result) > 1:
+	if len(result) > 1:
 	    return tuple(result)
 	elif len(result) == 1:
 	    return result[0]
 	else:
 	    return ()
 
-def escape(pattern):
-    result = []
-    alphanum=string.letters+'_'+string.digits
-    for char in pattern:
-	if char not in alphanum:
-	    result.append('\\')
-	result.append(char)
-    return string.join(result, '')
-
-def compile(pattern, flags=0):
-    groupindex={}
-    code=pcre_compile(pattern, flags, groupindex)
-    return RegexObject(pattern, flags, code, groupindex)
-    
-

Lib/test/re_tests.py

@@ -2,7 +2,7 @@
 # -*- mode: python -*-
 # $Id$
 
-# Re test suite and benchmark suite v1.5a2
+# Re test suite and benchmark suite v1.5b2
 
 # The 3 possible outcomes for each pattern
 [SUCCEED, FAIL, SYNTAX_ERROR] = range(3)
@@ -47,7 +47,62 @@ benchmarks = [
 #
 # If the regex isn't expected to work, the latter two elements can be omitted.
 
-tests = [
+tests = [ 
+    # Test ?P< and ?P= extensions
+    ('(?P<foo_123', '', SYNTAX_ERROR),      # Unterminated group identifier
+    ('(?P<1>a)', '', SYNTAX_ERROR),         # Begins with a digit
+    ('(?P<!>a)', '', SYNTAX_ERROR),         # Begins with an illegal char
+    ('(?P<foo!>a)', '', SYNTAX_ERROR),      # Begins with an illegal char
+
+    # Same tests, for the ?P= form
+    ('(?P<foo_123>a)(?P=foo_123', 'aa', SYNTAX_ERROR),
+    ('(?P<foo_123>a)(?P=1)', 'aa', SYNTAX_ERROR),
+    ('(?P<foo_123>a)(?P=!)', 'aa', SYNTAX_ERROR),
+    ('(?P<foo_123>a)(?P=foo_124', 'aa', SYNTAX_ERROR),  # Backref to undefined group
+
+    ('(?P<foo_123>a)', 'a', SUCCEED, 'g1', 'a'),
+    ('(?P<foo_123>a)(?P=foo_123)', 'aa', SUCCEED, 'g1', 'a'),
+    
+    # Test octal escapes
+    ('\\1', 'a', SYNTAX_ERROR),
+    ('\\09', chr(0) + '9', SUCCEED, 'found', chr(0) + '9'),
+    ('\\141', 'a', SUCCEED, 'found', 'a'),
+    ('(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)\\119', 'abcdefghijklk9', SUCCEED, 'found+"-"+g11', 'abcdefghijklk9-k'),
+
+    # Test that a literal \0 is handled everywhere
+    ('\0', '\0', SUCCEED, 'found', '\0'),
+    (r'\0', '\0', SUCCEED, 'found', '\0'),
+    ('[\0a]', '\0', SUCCEED, 'found', '\0'),
+    ('[a\0]', '\0', SUCCEED, 'found', '\0'),
+    ('[^a\0]', '\0', FAIL),
+    (r'[\0a]', '\0', SUCCEED, 'found', '\0'),
+    (r'[a\0]', '\0', SUCCEED, 'found', '\0'),
+    (r'[^a\0]', '\0', FAIL),
+
+    # Test various letter escapes
+    (r'\a[\b]\f\n\r\t\v', '\a\b\f\n\r\t\v', SUCCEED, 'found', '\a\b\f\n\r\t\v'),
+    (r'[\a][\b][\f][\n][\r][\t][\v]', '\a\b\f\n\r\t\v', SUCCEED, 'found', '\a\b\f\n\r\t\v'),
+    (r'\u', '', SYNTAX_ERROR),    # A Perl escape
+    (r'\c\e\g\h\i\j\k\m\o\p\q\y\z', 'ceghijkmopqyz', SUCCEED, 'found', 'ceghijkmopqyz'),
+    (r'\xff', '\377', SUCCEED, 'found', chr(255)),
+    (r'\x00ffffffffffffff', '\377', SUCCEED, 'found', chr(255)),
+    (r'\x00f', '\017', SUCCEED, 'found', chr(15)),
+    (r'\x00fe', '\376', SUCCEED, 'found', chr(254)),
+
+    (r"^\w+=(\\[\000-\277]|[^\n\\])*", "SRC=eval.c g.c blah blah blah \\\\\n\tapes.c", 
+     SUCCEED, 'found', "SRC=eval.c g.c blah blah blah \\\\"),
+     
+    # Test that . only matches \n in DOTALL mode
+    ('a.b', 'acb', SUCCEED, 'found', 'acb'),
+    ('a.b', 'a\nb', FAIL),
+    ('a.*b', 'acc\nccb', FAIL),
+    ('a.{4,5}b', 'acc\nccb', FAIL),
+    ('a.b', 'a\rb', SUCCEED, 'found', 'a\rb'),
+    ('a.b(?s)', 'a\nb', SUCCEED, 'found', 'a\nb'),
+    ('a.*(?s)b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'),
+    ('(?s)a.{4,5}b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'),
+    ('(?s)a.b', 'a\nb', SUCCEED, 'found', 'a\nb'),
+
     ('abc', 'abc', SUCCEED, 'found', 'abc'),
     ('abc', 'xbc', FAIL),
     ('abc', 'axc', FAIL),
@@ -338,8 +393,9 @@ tests = [
     ('(.*)c(.*)', 'abcde', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcde-ab-de'),
     ('\\((.*), (.*)\\)', '(a, b)', SUCCEED, 'g2+"-"+g1', 'b-a'),
     ('[k]', 'ab', FAIL),
-    ##('abcd', 'abcd', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'abcd-$&-\\abcd'),
-    ##('a(bc)d', 'abcd', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'bc-$1-\\bc'),
+# XXX
+#    ('abcd', 'abcd', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'abcd-$&-\\abcd'),
+#    ('a(bc)d', 'abcd', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'bc-$1-\\bc'),
     ('a[-]?c', 'ac', SUCCEED, 'found', 'ac'),
     ('(abc)\\1', 'abcabc', SUCCEED, 'g1', 'abc'),
     ('([a-c]*)\\1', 'abcabc', SUCCEED, 'g1', 'abc'),
@@ -470,15 +526,14 @@ tests = [
     ('(?i)(.*)c(.*)', 'ABCDE', SUCCEED, 'found+"-"+g1+"-"+g2', 'ABCDE-AB-DE'),
     ('(?i)\\((.*), (.*)\\)', '(A, B)', SUCCEED, 'g2+"-"+g1', 'B-A'),
     ('(?i)[k]', 'AB', FAIL),
-    ##('(?i)abcd', 'ABCD', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'ABCD-$&-\\ABCD'),
-    ##('(?i)a(bc)d', 'ABCD', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'BC-$1-\\BC'),
+#    ('(?i)abcd', 'ABCD', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'ABCD-$&-\\ABCD'),
+#    ('(?i)a(bc)d', 'ABCD', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'BC-$1-\\BC'),
     ('(?i)a[-]?c', 'AC', SUCCEED, 'found', 'AC'),
     ('(?i)(abc)\\1', 'ABCABC', SUCCEED, 'g1', 'ABC'),
     ('(?i)([a-c]*)\\1', 'ABCABC', SUCCEED, 'g1', 'ABC'),
-    # these zero-width assertions are not supported
-    #('a(?!b).', 'abad', SUCCEED, 'found', 'ad'),
-    #('a(?=d).', 'abad', SUCCEED, 'found', 'ad'),
-    #('a(?=c|d).', 'abad', SUCCEED, 'found', 'ad'),
+    ('a(?!b).', 'abad', SUCCEED, 'found', 'ad'),
+    ('a(?=d).', 'abad', SUCCEED, 'found', 'ad'),
+    ('a(?=c|d).', 'abad', SUCCEED, 'found', 'ad'),
     ('a(?:b|c|d)(.)', 'ace', SUCCEED, 'g1', 'e'),
     ('a(?:b|c|d)*(.)', 'ace', SUCCEED, 'g1', 'e'),
     ('a(?:b|c|d)+?(.)', 'ace', SUCCEED, 'g1', 'e'),
@@ -535,5 +590,5 @@ xyzabc
     (r'\t\n\v\r\f\a\g', '\t\n\v\r\f\ag', SUCCEED, 'found', '\t\n\v\r\f\ag'),
     ('\t\n\v\r\f\a\g', '\t\n\v\r\f\ag', SUCCEED, 'found', '\t\n\v\r\f\ag'),
     (r'\t\n\v\r\f\a', '\t\n\v\r\f\a', SUCCEED, 'found', chr(9)+chr(10)+chr(11)+chr(13)+chr(12)+chr(7)),
-    (r'[\t][\n][\v][\r][\f][\a][\A][\b][\B][\Z][\g]', '\t\n\v\r\f\aA\bBZg', SUCCEED, 'found', '\t\n\v\r\f\aA\bBZg'),
+    (r'[\t][\n][\v][\r][\f][\b]', '\t\n\v\r\f\b', SUCCEED, 'found', '\t\n\v\r\f\b'),
 ]

Lib/test/test_re.py

@@ -4,7 +4,6 @@
 
 from test_support import verbose, TestFailed
 import re
-import reop
 import sys, os, string, traceback
 
 # Misc tests from Tim Peters' re.doc
@@ -23,12 +22,13 @@ try:
     
     assert re.sub('.', lambda m: r"\n", 'x') == '\\n'
     assert re.sub('.', r"\n", 'x') == '\n'
-    
+
     s = r"\1\1"
     assert re.sub('(.)', s, 'x') == 'xx'
     assert re.sub('(.)', re.escape(s), 'x') == s
     assert re.sub('(.)', lambda m: s, 'x') == s
 
+    assert re.sub('(?P<a>x)', '\g<a>\g<a>', 'xx') == 'xxxx'
     assert re.sub('(?P<unk>x)', '\g<unk>\g<unk>', 'xx') == 'xxxx'
 
     assert re.sub('a', r'\t\n\v\r\f\a\b\B\Z\a\A\w\W\s\S\d\D', 'a') == '\t\n\v\r\f\a\bBZ\aAwWsSdD'
@@ -98,13 +98,9 @@ try:
     assert re.subn("b+", "x", "bbbb BBBB") == ('x BBBB', 1)
     assert re.subn("b+", "x", "xyz") == ('xyz', 0)
     assert re.subn("b*", "x", "xyz") == ('xxxyxzx', 4)
-    
 except AssertionError:
     raise TestFailed, "re.subn"
 
-if verbose:
-    print 'Running tests on re.split'
-
 try:
     assert re.split(":", ":a:b::c") == ['', 'a', 'b', '', 'c']
     assert re.split(":*", ":a:b::c") == ['', 'a', 'b', 'c']
@@ -119,12 +115,37 @@ try:
 except AssertionError:
     raise TestFailed, "re.split"
 
+if verbose:
+    print 'Pickling a RegexObject instance'
+    import pickle
+    pat = re.compile('a(?:b|(c|e){1,2}?|d)+?(.)')
+    s = pickle.dumps(pat)
+    pat = pickle.loads(s)
+
+if verbose:
+    print 'Running tests on re.split'
+    
+try:
+    assert re.I == re.IGNORECASE
+    assert re.L == re.LOCALE
+    assert re.M == re.MULTILINE
+    assert re.S == re.DOTALL 
+    assert re.X == re.VERBOSE 
+except AssertionError:
+    raise TestFailed, 're module constants'
+
+for flags in [re.I, re.M, re.X, re.S]:
+    try:
+	r = re.compile('^pattern$', flags)
+    except:
+	print 'Exception raised on flag', flags
+
 from re_tests import *
 if verbose:
     print 'Running re_tests test suite'
 else:
     # To save time, only run the first and last 10 tests
-    tests = tests[:10] + tests[-10:]
+    pass #tests = tests[:10] + tests[-10:]
 
 for t in tests:
     sys.stdout.flush()
@@ -150,7 +171,7 @@ for t in tests:
     else:
 	try:
 	    result=obj.search(s)
-	except (re.error, reop.error), msg:
+	except (re.error), msg:
 	    print '=== Unexpected exception', t, repr(msg)
 	if outcome==SYNTAX_ERROR:
 	    # This should have been a syntax error; forget it.
@@ -190,9 +211,22 @@ for t in tests:
 	    else:
 		print '=== Failed incorrectly', t
 
+	    # Try the match with the search area limited to the extent
+	    # of the match and see if it still succeeds.  \B will
+	    # break (because it won't match at the end or start of a
+	    # string), so we'll ignore patterns that feature it.
+	    
+	    if pattern[:2]!='\\B' and pattern[-2:]!='\\B':
+		obj=re.compile(pattern)
+		result=obj.search(s, pos=result.start(0), endpos=result.end(0)+1)
+		if result==None:
+		    print '=== Failed on range-limited match', t
+
             # Try the match with IGNORECASE enabled, and check that it
 	    # still succeeds.
             obj=re.compile(pattern, re.IGNORECASE)
             result=obj.search(s)
             if result==None:
                 print '=== Fails on case-insensitive match', t
+
+