Commit b754b352 authored by Robert Bradshaw's avatar Robert Bradshaw

Further bootstrapping of Scanner

parent 75b241ff
...@@ -2316,10 +2316,11 @@ class PyClassDefNode(ClassDefNode): ...@@ -2316,10 +2316,11 @@ class PyClassDefNode(ClassDefNode):
elif len(bases) == 1: elif len(bases) == 1:
base = bases[0] base = bases[0]
path = [] path = []
while isinstance(base, ExprNodes.AttributeNode): from ExprNodes import AttributeNode, NameNode
while isinstance(base, AttributeNode):
path.insert(0, base.attribute) path.insert(0, base.attribute)
base = base.obj base = base.obj
if isinstance(base, ExprNodes.NameNode): if isinstance(base, NameNode):
path.insert(0, base.name) path.insert(0, base.name)
base_class_name = path[-1] base_class_name = path[-1]
if len(path) > 1: if len(path) > 1:
...@@ -4399,6 +4400,9 @@ class FromImportStatNode(StatNode): ...@@ -4399,6 +4400,9 @@ class FromImportStatNode(StatNode):
env.use_utility_code(ExprNodes.type_test_utility_code) env.use_utility_code(ExprNodes.type_test_utility_code)
break break
else: else:
entry = env.lookup(target.name)
if entry.is_type and entry.type.name == name and entry.type.module_name == self.module.module_name.value:
continue # already cimported
self.interned_items.append( self.interned_items.append(
(env.intern_identifier(name), target)) (env.intern_identifier(name), target))
target.analyse_target_expression(env, None) target.analyse_target_expression(env, None)
......
from Cython.Plex.Scanners cimport Scanner
cdef class PyrexScanner(Scanner):
cdef public context
cdef public list included_files
cdef public compile_time_env
cdef public bint compile_time_eval
cdef public bint compile_time_expr
cdef public bint parse_comments
cdef public source_encoding
cdef public list indentation_stack
cdef public indentation_char
cdef public int bracket_nesting_level
cdef public sy
cdef public systring
...@@ -12,14 +12,17 @@ import sys ...@@ -12,14 +12,17 @@ import sys
from time import time from time import time
from Cython import Plex, Utils from Cython import Plex, Utils
from Cython.Plex import Scanner from Cython.Plex.Scanners import Scanner
from Cython.Plex.Errors import UnrecognizedInput from Cython.Plex.Errors import UnrecognizedInput
from Errors import CompileError, error from Errors import CompileError, error
from Lexicon import string_prefixes, raw_prefixes, make_lexicon from Lexicon import string_prefixes, raw_prefixes, make_lexicon
from StringEncoding import EncodedString from StringEncoding import EncodedString
plex_version = getattr(Plex, '_version', None) try:
plex_version = Plex._version
except AttributeError:
plex_version = None
#print "Plex version:", plex_version ### #print "Plex version:", plex_version ###
debug_scanner = 0 debug_scanner = 0
...@@ -91,7 +94,7 @@ def try_to_unpickle_lexicon(): ...@@ -91,7 +94,7 @@ def try_to_unpickle_lexicon():
source_file = os.path.join(dir, "Lexicon.py") source_file = os.path.join(dir, "Lexicon.py")
lexicon_hash = hash_source_file(source_file) lexicon_hash = hash_source_file(source_file)
lexicon_pickle = os.path.join(dir, "Lexicon.pickle") lexicon_pickle = os.path.join(dir, "Lexicon.pickle")
f = open_pickled_lexicon(expected_hash = lexicon_hash) f = open_pickled_lexicon(lexicon_hash)
if f: if f:
if notify_lexicon_unpickling: if notify_lexicon_unpickling:
t0 = time() t0 = time()
...@@ -165,6 +168,8 @@ def build_resword_dict(): ...@@ -165,6 +168,8 @@ def build_resword_dict():
d[word] = 1 d[word] = 1
return d return d
resword_dict = build_resword_dict()
#------------------------------------------------------------------ #------------------------------------------------------------------
class CompileTimeScope(object): class CompileTimeScope(object):
...@@ -286,7 +291,6 @@ class PyrexScanner(Scanner): ...@@ -286,7 +291,6 @@ class PyrexScanner(Scanner):
# compile_time_env dict Environment for conditional compilation # compile_time_env dict Environment for conditional compilation
# compile_time_eval boolean In a true conditional compilation context # compile_time_eval boolean In a true conditional compilation context
# compile_time_expr boolean In a compile-time expression context # compile_time_expr boolean In a compile-time expression context
resword_dict = build_resword_dict()
def __init__(self, file, filename, parent_scanner = None, def __init__(self, file, filename, parent_scanner = None,
scope = None, context = None, source_encoding=None, parse_comments=True, initial_pos=None): scope = None, context = None, source_encoding=None, parse_comments=True, initial_pos=None):
...@@ -404,7 +408,7 @@ class PyrexScanner(Scanner): ...@@ -404,7 +408,7 @@ class PyrexScanner(Scanner):
except UnrecognizedInput: except UnrecognizedInput:
self.error("Unrecognized character") self.error("Unrecognized character")
if sy == 'IDENT': if sy == 'IDENT':
if systring in self.resword_dict: if systring in resword_dict:
sy = systring sy = systring
else: else:
systring = EncodedString(systring) systring = EncodedString(systring)
......
cdef class Scanner:
cdef public lexicon
cdef public stream
cdef public name
cdef public buffer
cdef public long buf_start_pos
cdef public long next_pos
cdef public long cur_pos
cdef public long cur_line
cdef public long cur_line_start
cdef public long start_pos
cdef public long start_line
cdef public long start_col
cdef public text
cdef public initial_state
cdef public state_name
cdef public list queue
cdef public bint trace
cdef public cur_char
cdef public input_state
cdef public level
\ No newline at end of file
...@@ -10,6 +10,8 @@ ...@@ -10,6 +10,8 @@
import Errors import Errors
from Regexps import BOL, EOL, EOF from Regexps import BOL, EOL, EOF
import cython
class Scanner: class Scanner:
""" """
A Scanner is used to read tokens from a stream of characters A Scanner is used to read tokens from a stream of characters
...@@ -42,23 +44,23 @@ class Scanner: ...@@ -42,23 +44,23 @@ class Scanner:
""" """
lexicon = None # Lexicon # lexicon = None # Lexicon
stream = None # file-like object # stream = None # file-like object
name = '' # name = ''
buffer = '' # buffer = ''
buf_start_pos = 0 # position in input of start of buffer # buf_start_pos = 0 # position in input of start of buffer
next_pos = 0 # position in input of next char to read # next_pos = 0 # position in input of next char to read
cur_pos = 0 # position in input of current char # cur_pos = 0 # position in input of current char
cur_line = 1 # line number of current char # cur_line = 1 # line number of current char
cur_line_start = 0 # position in input of start of current line # cur_line_start = 0 # position in input of start of current line
start_pos = 0 # position in input of start of token # start_pos = 0 # position in input of start of token
start_line = 0 # line number of start of token # start_line = 0 # line number of start of token
start_col = 0 # position in line of start of token # start_col = 0 # position in line of start of token
text = None # text of last token read # text = None # text of last token read
initial_state = None # Node # initial_state = None # Node
state_name = '' # Name of initial state # state_name = '' # Name of initial state
queue = None # list of tokens to be returned # queue = None # list of tokens to be returned
trace = 0 # trace = 0
def __init__(self, lexicon, stream, name = '', initial_pos = None): def __init__(self, lexicon, stream, name = '', initial_pos = None):
""" """
...@@ -73,6 +75,17 @@ class Scanner: ...@@ -73,6 +75,17 @@ class Scanner:
|name| is optional, and may be the name of the file being |name| is optional, and may be the name of the file being
scanned or any other identifying string. scanned or any other identifying string.
""" """
self.buffer = ''
self.buf_start_pos = 0
self.next_pos = 0
self.cur_pos = 0
self.cur_line = 1
self.start_pos = 0
self.start_line = 0
self.start_col = 0
self.text = None
self.state_name = None
self.lexicon = lexicon self.lexicon = lexicon
self.stream = stream self.stream = stream
self.name = name self.name = name
...@@ -374,6 +387,3 @@ class Scanner: ...@@ -374,6 +387,3 @@ class Scanner:
Override this method if you want something to be done at Override this method if you want something to be done at
end of file. end of file.
""" """
# For backward compatibility:
setattr(Scanner, "yield", Scanner.produce)
...@@ -31,7 +31,7 @@ except ValueError: ...@@ -31,7 +31,7 @@ except ValueError:
try: try:
from Cython.Compiler.Main import compile from Cython.Compiler.Main import compile
source_root = os.path.dirname(__file__) source_root = os.path.dirname(__file__)
compiled_modules = ["Cython.Plex.Scanners"] compiled_modules = ["Cython.Plex.Scanners", "Cython.Compiler.Scanning"]
extensions = [] extensions = []
for module in compiled_modules: for module in compiled_modules:
source_file = os.path.join(source_root, *module.split('.')) source_file = os.path.join(source_root, *module.split('.'))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment