Commit 10a428b6 authored by Monson Shao's avatar Monson Shao Committed by Benjamin Peterson

closes bpo-34515: Support non-ASCII identifiers in lib2to3. (GH-8950)

parent d2067318
...@@ -56,7 +56,7 @@ def _combinations(*l): ...@@ -56,7 +56,7 @@ def _combinations(*l):
Whitespace = r'[ \f\t]*' Whitespace = r'[ \f\t]*'
Comment = r'#[^\r\n]*' Comment = r'#[^\r\n]*'
Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
Name = r'[a-zA-Z_]\w*' Name = r'\w+'
Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'
...@@ -107,8 +107,8 @@ ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + ...@@ -107,8 +107,8 @@ ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
PseudoExtras = group(r'\\\r?\n', Comment, Triple) PseudoExtras = group(r'\\\r?\n', Comment, Triple)
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
tokenprog, pseudoprog, single3prog, double3prog = list(map( tokenprog, pseudoprog, single3prog, double3prog = map(
re.compile, (Token, PseudoToken, Single3, Double3))) re.compile, (Token, PseudoToken, Single3, Double3))
_strprefixes = ( _strprefixes = (
_combinations('r', 'R', 'f', 'F') | _combinations('r', 'R', 'f', 'F') |
...@@ -349,7 +349,6 @@ def generate_tokens(readline): ...@@ -349,7 +349,6 @@ def generate_tokens(readline):
logical line; continuation lines are included. logical line; continuation lines are included.
""" """
lnum = parenlev = continued = 0 lnum = parenlev = continued = 0
namechars, numchars = string.ascii_letters + '_', '0123456789'
contstr, needcont = '', 0 contstr, needcont = '', 0
contline = None contline = None
indents = [0] indents = [0]
...@@ -451,7 +450,7 @@ def generate_tokens(readline): ...@@ -451,7 +450,7 @@ def generate_tokens(readline):
spos, epos, pos = (lnum, start), (lnum, end), end spos, epos, pos = (lnum, start), (lnum, end), end
token, initial = line[start:end], line[start] token, initial = line[start:end], line[start]
if initial in numchars or \ if initial in string.digits or \
(initial == '.' and token != '.'): # ordinary number (initial == '.' and token != '.'): # ordinary number
yield (NUMBER, token, spos, epos, line) yield (NUMBER, token, spos, epos, line)
elif initial in '\r\n': elif initial in '\r\n':
...@@ -501,7 +500,7 @@ def generate_tokens(readline): ...@@ -501,7 +500,7 @@ def generate_tokens(readline):
yield stashed yield stashed
stashed = None stashed = None
yield (STRING, token, spos, epos, line) yield (STRING, token, spos, epos, line)
elif initial in namechars: # ordinary name elif initial.isidentifier(): # ordinary name
if token in ('async', 'await'): if token in ('async', 'await'):
if async_def: if async_def:
yield (ASYNC if token == 'async' else AWAIT, yield (ASYNC if token == 'async' else AWAIT,
......
...@@ -529,6 +529,16 @@ class TestSetLiteral(GrammarTest): ...@@ -529,6 +529,16 @@ class TestSetLiteral(GrammarTest):
self.validate("""x = {2, 3, 4,}""") self.validate("""x = {2, 3, 4,}""")
# Adapted from Python 3's Lib/test/test_unicode_identifiers.py and
# Lib/test/test_tokenize.py:TokenizeTest.test_non_ascii_identifiers
class TestIdentfier(GrammarTest):
def test_non_ascii_identifiers(self):
self.validate("Örter = 'places'\ngrün = 'green'")
self.validate("蟒 = a蟒 = 锦蛇 = 1")
self.validate("µ = aµ = µµ = 1")
self.validate("𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = a_𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1")
class TestNumericLiterals(GrammarTest): class TestNumericLiterals(GrammarTest):
def test_new_octal_notation(self): def test_new_octal_notation(self):
self.validate("""0o7777777777777""") self.validate("""0o7777777777777""")
......
Fix parsing non-ASCII identifiers in :mod:`lib2to3.pgen2.tokenize` (PEP 3131).
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment