Skip to content

G
go
Commit 7e1a3e9f authored by Russ Cox's avatar Russ Cox
Browse files
Options

exp/regexp/syntax: incremental concat, alternate 

Also reuse of *Regexp nodes.

I believe this is the end of the parser.
The only non-execution code that remains is
the code to expand x{3,5} into simpler operations.

R=sam.thorogood, r
CC=golang-dev
https://golang.org/cl/4629078
parent 3379414b
Expand all Show whitespace changes
Inline Side-by-side
Showing with 365 additions and 175 deletions
src/pkg/exp/regexp/syntax/parse.go
View file @ 7e1a3e9f
This diff is collapsed.
src/pkg/exp/regexp/syntax/parse_test.go
View file @ 7e1a3e9f
...@@ -16,148 +16,146 @@ var parseTests = []struct { ...@@ -16,148 +16,146 @@ var parseTests = []struct {
Dump string Dump string
}{ }{
// Base cases // Base cases
{"a", "lit{a}"}, {`a`, `lit{a}`},
{"a.", "cat{lit{a}dot{}}"}, {`a.`, `cat{lit{a}dot{}}`},
{"a.b", "cat{lit{a}dot{}lit{b}}"}, {`a.b`, `cat{lit{a}dot{}lit{b}}`},
// { "ab", "str{ab}" }, {`ab`, `str{ab}`},
{"ab", "cat{lit{a}lit{b}}"}, {`a.b.c`, `cat{lit{a}dot{}lit{b}dot{}lit{c}}`},
{"a.b.c", "cat{lit{a}dot{}lit{b}dot{}lit{c}}"}, {`abc`, `str{abc}`},
// { "abc", "str{abc}" }, {`a|^`, `alt{lit{a}bol{}}`},
{"abc", "cat{lit{a}lit{b}lit{c}}"}, {`a|b`, `cc{0x61-0x62}`},
{"a|^", "alt{lit{a}bol{}}"}, {`(a)`, `cap{lit{a}}`},
// { "a|b", "cc{0x61-0x62}" }, {`(a)|b`, `alt{cap{lit{a}}lit{b}}`},
{"a|b", "alt{lit{a}lit{b}}"}, {`a*`, `star{lit{a}}`},
{"(a)", "cap{lit{a}}"}, {`a+`, `plus{lit{a}}`},
{"(a)|b", "alt{cap{lit{a}}lit{b}}"}, {`a?`, `que{lit{a}}`},
{"a*", "star{lit{a}}"}, {`a{2}`, `rep{2,2 lit{a}}`},
{"a+", "plus{lit{a}}"}, {`a{2,3}`, `rep{2,3 lit{a}}`},
{"a?", "que{lit{a}}"}, {`a{2,}`, `rep{2,-1 lit{a}}`},
{"a{2}", "rep{2,2 lit{a}}"}, {`a*?`, `nstar{lit{a}}`},
{"a{2,3}", "rep{2,3 lit{a}}"}, {`a+?`, `nplus{lit{a}}`},
{"a{2,}", "rep{2,-1 lit{a}}"}, {`a??`, `nque{lit{a}}`},
{"a*?", "nstar{lit{a}}"}, {`a{2}?`, `nrep{2,2 lit{a}}`},
{"a+?", "nplus{lit{a}}"}, {`a{2,3}?`, `nrep{2,3 lit{a}}`},
{"a??", "nque{lit{a}}"}, {`a{2,}?`, `nrep{2,-1 lit{a}}`},
{"a{2}?", "nrep{2,2 lit{a}}"}, {``, `emp{}`},
{"a{2,3}?", "nrep{2,3 lit{a}}"}, // { `|`, `emp{}` }, // alt{emp{}emp{}} but got factored
{"a{2,}?", "nrep{2,-1 lit{a}}"}, {`|`, `alt{emp{}emp{}}`},
{"", "emp{}"}, {`|x|`, `alt{emp{}lit{x}emp{}}`},
// { "|", "emp{}" }, // alt{emp{}emp{}} but got factored {`.`, `dot{}`},
{"|", "alt{emp{}emp{}}"}, {`^`, `bol{}`},
{"|x|", "alt{emp{}lit{x}emp{}}"}, {`$`, `eol{}`},
{".", "dot{}"}, {`\|`, `lit{|}`},
{"^", "bol{}"}, {`\(`, `lit{(}`},
{"$", "eol{}"}, {`\)`, `lit{)}`},
{"\\|", "lit{|}"}, {`\*`, `lit{*}`},
{"\\(", "lit{(}"}, {`\+`, `lit{+}`},
{"\\)", "lit{)}"}, {`\?`, `lit{?}`},
{"\\*", "lit{*}"}, {`{`, `lit{{}`},
{"\\+", "lit{+}"}, {`}`, `lit{}}`},
{"\\?", "lit{?}"}, {`\.`, `lit{.}`},
{"{", "lit{{}"}, {`\^`, `lit{^}`},
{"}", "lit{}}"}, {`\$`, `lit{$}`},
{"\\.", "lit{.}"}, {`\\`, `lit{\}`},
{"\\^", "lit{^}"}, {`[ace]`, `cc{0x61 0x63 0x65}`},
{"\\$", "lit{$}"}, {`[abc]`, `cc{0x61-0x63}`},
{"\\\\", "lit{\\}"}, {`[a-z]`, `cc{0x61-0x7a}`},
{"[ace]", "cc{0x61 0x63 0x65}"}, {`[a]`, `lit{a}`},
{"[abc]", "cc{0x61-0x63}"}, {`\-`, `lit{-}`},
{"[a-z]", "cc{0x61-0x7a}"}, {`-`, `lit{-}`},
// { "[a]", "lit{a}" }, {`\_`, `lit{_}`},
{"[a]", "cc{0x61}"},
{"\\-", "lit{-}"},
{"-", "lit{-}"},
{"\\_", "lit{_}"},
// Posix and Perl extensions // Posix and Perl extensions
{"[[:lower:]]", "cc{0x61-0x7a}"}, {`[[:lower:]]`, `cc{0x61-0x7a}`},
{"[a-z]", "cc{0x61-0x7a}"}, {`[a-z]`, `cc{0x61-0x7a}`},
{"[^[:lower:]]", "cc{0x0-0x60 0x7b-0x10ffff}"}, {`[^[:lower:]]`, `cc{0x0-0x60 0x7b-0x10ffff}`},
{"[[:^lower:]]", "cc{0x0-0x60 0x7b-0x10ffff}"}, {`[[:^lower:]]`, `cc{0x0-0x60 0x7b-0x10ffff}`},
{"(?i)[[:lower:]]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}"}, {`(?i)[[:lower:]]`, `cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}`},
{"(?i)[a-z]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}"}, {`(?i)[a-z]`, `cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}`},
{"(?i)[^[:lower:]]", "cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}"}, {`(?i)[^[:lower:]]`, `cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`},
{"(?i)[[:^lower:]]", "cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}"}, {`(?i)[[:^lower:]]`, `cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`},
{"\\d", "cc{0x30-0x39}"}, {`\d`, `cc{0x30-0x39}`},
{"\\D", "cc{0x0-0x2f 0x3a-0x10ffff}"}, {`\D`, `cc{0x0-0x2f 0x3a-0x10ffff}`},
{"\\s", "cc{0x9-0xa 0xc-0xd 0x20}"}, {`\s`, `cc{0x9-0xa 0xc-0xd 0x20}`},
{"\\S", "cc{0x0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}"}, {`\S`, `cc{0x0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}`},
{"\\w", "cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a}"}, {`\w`, `cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a}`},
{"\\W", "cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x10ffff}"}, {`\W`, `cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x10ffff}`},
{"(?i)\\w", "cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a 0x17f 0x212a}"}, {`(?i)\w`, `cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a 0x17f 0x212a}`},
{"(?i)\\W", "cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}"}, {`(?i)\W`, `cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`},
{"[^\\\\]", "cc{0x0-0x5b 0x5d-0x10ffff}"}, {`[^\\]`, `cc{0x0-0x5b 0x5d-0x10ffff}`},
// { "\\C", "byte{}" }, // { `\C`, `byte{}` }, // probably never
// Unicode, negatives, and a double negative. // Unicode, negatives, and a double negative.
{"\\p{Braille}", "cc{0x2800-0x28ff}"}, {`\p{Braille}`, `cc{0x2800-0x28ff}`},
{"\\P{Braille}", "cc{0x0-0x27ff 0x2900-0x10ffff}"}, {`\P{Braille}`, `cc{0x0-0x27ff 0x2900-0x10ffff}`},
{"\\p{^Braille}", "cc{0x0-0x27ff 0x2900-0x10ffff}"}, {`\p{^Braille}`, `cc{0x0-0x27ff 0x2900-0x10ffff}`},
{"\\P{^Braille}", "cc{0x2800-0x28ff}"}, {`\P{^Braille}`, `cc{0x2800-0x28ff}`},
{"\\pZ", "cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}"}, {`\pZ`, `cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`},
{"[\\p{Braille}]", "cc{0x2800-0x28ff}"}, {`[\p{Braille}]`, `cc{0x2800-0x28ff}`},
{"[\\P{Braille}]", "cc{0x0-0x27ff 0x2900-0x10ffff}"}, {`[\P{Braille}]`, `cc{0x0-0x27ff 0x2900-0x10ffff}`},
{"[\\p{^Braille}]", "cc{0x0-0x27ff 0x2900-0x10ffff}"}, {`[\p{^Braille}]`, `cc{0x0-0x27ff 0x2900-0x10ffff}`},
{"[\\P{^Braille}]", "cc{0x2800-0x28ff}"}, {`[\P{^Braille}]`, `cc{0x2800-0x28ff}`},
{"[\\pZ]", "cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}"}, {`[\pZ]`, `cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`},
{"\\p{Lu}", mkCharClass(unicode.IsUpper)}, {`\p{Lu}`, mkCharClass(unicode.IsUpper)},
{"[\\p{Lu}]", mkCharClass(unicode.IsUpper)}, {`[\p{Lu}]`, mkCharClass(unicode.IsUpper)},
{"(?i)[\\p{Lu}]", mkCharClass(isUpperFold)}, {`(?i)[\p{Lu}]`, mkCharClass(isUpperFold)},
// Hex, octal. // Hex, octal.
{"[\\012-\\234]\\141", "cat{cc{0xa-0x9c}lit{a}}"}, {`[\012-\234]\141`, `cat{cc{0xa-0x9c}lit{a}}`},
{"[\\x{41}-\\x7a]\\x61", "cat{cc{0x41-0x7a}lit{a}}"}, {`[\x{41}-\x7a]\x61`, `cat{cc{0x41-0x7a}lit{a}}`},
// More interesting regular expressions. // More interesting regular expressions.
// { "a{,2}", "str{a{,2}}" }, {`a{,2}`, `str{a{,2}}`},
// { "\\.\\^\\$\\\\", "str{.^$\\}" }, {`\.\^\$\\`, `str{.^$\}`},
{"[a-zABC]", "cc{0x41-0x43 0x61-0x7a}"}, {`[a-zABC]`, `cc{0x41-0x43 0x61-0x7a}`},
{"[^a]", "cc{0x0-0x60 0x62-0x10ffff}"}, {`[^a]`, `cc{0x0-0x60 0x62-0x10ffff}`},
{"[\xce\xb1-\xce\xb5\xe2\x98\xba]", "cc{0x3b1-0x3b5 0x263a}"}, // utf-8 {`[α-ε☺]`, `cc{0x3b1-0x3b5 0x263a}`}, // utf-8
{"a*{", "cat{star{lit{a}}lit{{}}"}, {`a*{`, `cat{star{lit{a}}lit{{}}`},
// Test precedences // Test precedences
// { "(?:ab)*", "star{str{ab}}" }, {`(?:ab)*`, `star{str{ab}}`},
// { "(ab)*", "star{cap{str{ab}}}" }, {`(ab)*`, `star{cap{str{ab}}}`},
// { "ab|cd", "alt{str{ab}str{cd}}" }, {`ab|cd`, `alt{str{ab}str{cd}}`},
// { "a(b|c)d", "cat{lit{a}cap{cc{0x62-0x63}}lit{d}}" }, {`a(b|c)d`, `cat{lit{a}cap{cc{0x62-0x63}}lit{d}}`},
{"(?:ab)*", "star{cat{lit{a}lit{b}}}"},
{"(ab)*", "star{cap{cat{lit{a}lit{b}}}}"},
{"ab|cd", "alt{cat{lit{a}lit{b}}cat{lit{c}lit{d}}}"},
{"a(b|c)d", "cat{lit{a}cap{alt{lit{b}lit{c}}}lit{d}}"},
// Test flattening. // Test flattening.
{"(?:a)", "lit{a}"}, {`(?:a)`, `lit{a}`},
// { "(?:ab)(?:cd)", "str{abcd}" }, {`(?:ab)(?:cd)`, `str{abcd}`},
// { "(?:a|b)|(?:c|d)", "cc{0x61-0x64}" }, {`(?:a+b+)(?:c+d+)`, `cat{plus{lit{a}}plus{lit{b}}plus{lit{c}}plus{lit{d}}}`},
// { "a|.", "dot{}" }, {`(?:a+|b+)|(?:c+|d+)`, `alt{plus{lit{a}}plus{lit{b}}plus{lit{c}}plus{lit{d}}}`},
// { ".|a", "dot{}" }, {`(?:a|b)|(?:c|d)`, `cc{0x61-0x64}`},
{`a|.`, `dot{}`},
{`.|a`, `dot{}`},
{`(?:[abc]|A|Z|hello|world)`, `alt{cc{0x41 0x5a 0x61-0x63}str{hello}str{world}}`},
{`(?:[abc]|A|Z)`, `cc{0x41 0x5a 0x61-0x63}`},
// Test Perl quoted literals // Test Perl quoted literals
{"\\Q+|*?{[\\E", "str{+|*?{[}"}, {`\Q+|*?{[\E`, `str{+|*?{[}`},
{"\\Q+\\E+", "plus{lit{+}}"}, {`\Q+\E+`, `plus{lit{+}}`},
{"\\Q\\\\E", "lit{\\}"}, {`\Q\\E`, `lit{\}`},
{"\\Q\\\\\\E", "str{\\\\}"}, {`\Q\\\E`, `str{\\}`},
// Test Perl \A and \z // Test Perl \A and \z
{"(?m)^", "bol{}"}, {`(?m)^`, `bol{}`},
{"(?m)$", "eol{}"}, {`(?m)$`, `eol{}`},
{"(?-m)^", "bot{}"}, {`(?-m)^`, `bot{}`},
{"(?-m)$", "eot{}"}, {`(?-m)$`, `eot{}`},
{"(?m)\\A", "bot{}"}, {`(?m)\A`, `bot{}`},
{"(?m)\\z", "eot{\\z}"}, {`(?m)\z`, `eot{\z}`},
{"(?-m)\\A", "bot{}"}, {`(?-m)\A`, `bot{}`},
{"(?-m)\\z", "eot{\\z}"}, {`(?-m)\z`, `eot{\z}`},
// Test named captures // Test named captures
{"(?P<name>a)", "cap{name:lit{a}}"}, {`(?P<name>a)`, `cap{name:lit{a}}`},
// Case-folded literals // Case-folded literals
// { "[Aa]", "litfold{a}" }, {`[Aa]`, `litfold{A}`},
{`[\x{100}\x{101}]`, `litfold{Ā}`},
{`[Δδ]`, `litfold{Δ}`},
// Strings // Strings
// { "abcde", "str{abcde}" }, {`abcde`, `str{abcde}`},
// { "[Aa][Bb]cd", "cat{strfold{ab}str{cd}}" }, {`[Aa][Bb]cd`, `cat{strfold{AB}str{cd}}`},
} }
const testFlags = MatchNL | PerlX | UnicodeGroups const testFlags = MatchNL | PerlX | UnicodeGroups
...@@ -230,8 +228,9 @@ func dumpRegexp(b *bytes.Buffer, re *Regexp) { ...@@ -230,8 +228,9 @@ func dumpRegexp(b *bytes.Buffer, re *Regexp) {
} }
if re.Flags&FoldCase != 0 { if re.Flags&FoldCase != 0 {
for _, r := range re.Rune { for _, r := range re.Rune {
if unicode.ToUpper(r) != r { if unicode.SimpleFold(r) != r {
b.WriteString("fold") b.WriteString("fold")
break
} }
} }
} }
......
src/pkg/exp/regexp/syntax/regexp.go
View file @ 7e1a3e9f
...@@ -33,6 +33,8 @@ type Regexp struct { ...@@ -33,6 +33,8 @@ type Regexp struct {
type Op uint8 type Op uint8
// Operators are listed in precedence order, tightest binding to weakest. // Operators are listed in precedence order, tightest binding to weakest.
// Character class operators are listed simplest to most complex
// (OpLiteral, OpCharClass, OpAnyCharNotNL, OpAnyChar).
const ( const (
OpNoMatch Op = 1 + iota // matches no strings OpNoMatch Op = 1 + iota // matches no strings
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7