Commit 70bb2a3e authored by Stefan Behnel's avatar Stefan Behnel

identifiers are encoded strings

parent 73bc8c3f
...@@ -11,7 +11,7 @@ import stat ...@@ -11,7 +11,7 @@ import stat
import sys import sys
from time import time from time import time
from Cython import Plex from Cython import Plex, Utils
from Cython.Plex import Scanner from Cython.Plex import Scanner
from Cython.Plex.Errors import UnrecognizedInput from Cython.Plex.Errors import UnrecognizedInput
from Errors import CompileError, error from Errors import CompileError, error
...@@ -320,8 +320,12 @@ class PyrexScanner(Scanner): ...@@ -320,8 +320,12 @@ class PyrexScanner(Scanner):
sy, systring = self.read() sy, systring = self.read()
except UnrecognizedInput: except UnrecognizedInput:
self.error("Unrecognized character") self.error("Unrecognized character")
if sy == 'IDENT' and systring in self.resword_dict: if sy == 'IDENT':
sy = systring if systring in self.resword_dict:
sy = systring
else:
systring = Utils.EncodedString(systring)
systring.encoding = self.source_encoding
self.sy = sy self.sy = sy
self.systring = systring self.systring = systring
if debug_scanner: if debug_scanner:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment