Further bootstrapping of Scanner

Cython/Compiler/Nodes.py

@@ -2316,10 +2316,11 @@ class PyClassDefNode(ClassDefNode):
         elif len(bases) == 1:
             base = bases[0]
             path = []
-            while isinstance(base, ExprNodes.AttributeNode):
+            from ExprNodes import AttributeNode, NameNode
+            while isinstance(base, AttributeNode):
                 path.insert(0, base.attribute)
                 base = base.obj
-            if isinstance(base, ExprNodes.NameNode):
+            if isinstance(base, NameNode):
                 path.insert(0, base.name)
                 base_class_name = path[-1]
                 if len(path) > 1:
@@ -4399,6 +4400,9 @@ class FromImportStatNode(StatNode):
                         env.use_utility_code(ExprNodes.type_test_utility_code)
                         break
             else:
+                entry =  env.lookup(target.name)
+                if entry.is_type and entry.type.name == name and entry.type.module_name == self.module.module_name.value:
+                    continue # already cimported
                 self.interned_items.append(
                     (env.intern_identifier(name), target))
                 target.analyse_target_expression(env, None)

Cython/Compiler/Scanning.pxd

+from Cython.Plex.Scanners cimport Scanner
+
+cdef class PyrexScanner(Scanner):
+    cdef public context
+    cdef public list included_files
+    cdef public compile_time_env
+    cdef public bint compile_time_eval
+    cdef public bint compile_time_expr
+    cdef public bint parse_comments
+    cdef public source_encoding
+    cdef public list indentation_stack
+    cdef public indentation_char
+    cdef public int bracket_nesting_level
+    cdef public sy
+    cdef public systring

Cython/Compiler/Scanning.py

@@ -12,14 +12,17 @@ import sys
 from time import time
 
 from Cython import Plex, Utils
-from Cython.Plex import Scanner
+from Cython.Plex.Scanners import Scanner
 from Cython.Plex.Errors import UnrecognizedInput
 from Errors import CompileError, error
 from Lexicon import string_prefixes, raw_prefixes, make_lexicon
 
 from StringEncoding import EncodedString
 
-plex_version = getattr(Plex, '_version', None)
+try:
+    plex_version = Plex._version
+except AttributeError:
+    plex_version = None
 #print "Plex version:", plex_version ###
 
 debug_scanner = 0
@@ -91,7 +94,7 @@ def try_to_unpickle_lexicon():
     source_file = os.path.join(dir, "Lexicon.py")
     lexicon_hash = hash_source_file(source_file)
     lexicon_pickle = os.path.join(dir, "Lexicon.pickle")
-    f = open_pickled_lexicon(expected_hash = lexicon_hash)
+    f = open_pickled_lexicon(lexicon_hash)
     if f:
         if notify_lexicon_unpickling:
             t0 = time()
@@ -165,6 +168,8 @@ def build_resword_dict():
         d[word] = 1
     return d
 
+resword_dict = build_resword_dict()
+
 #------------------------------------------------------------------
 
 class CompileTimeScope(object):
@@ -286,7 +291,6 @@ class PyrexScanner(Scanner):
     #  compile_time_env   dict     Environment for conditional compilation
     #  compile_time_eval  boolean  In a true conditional compilation context
     #  compile_time_expr  boolean  In a compile-time expression context
-    resword_dict = build_resword_dict()
 
     def __init__(self, file, filename, parent_scanner = None, 
                  scope = None, context = None, source_encoding=None, parse_comments=True, initial_pos=None):
@@ -404,7 +408,7 @@ class PyrexScanner(Scanner):
         except UnrecognizedInput:
             self.error("Unrecognized character")
         if sy == 'IDENT':
-            if systring in self.resword_dict:
+            if systring in resword_dict:
                 sy = systring
             else:
                 systring = EncodedString(systring)

Cython/Plex/Scanners.pxd

+cdef class Scanner:
+    cdef public lexicon
+    cdef public stream
+    cdef public name
+    cdef public buffer
+    cdef public long buf_start_pos
+    cdef public long next_pos
+    cdef public long cur_pos
+    cdef public long cur_line
+    cdef public long cur_line_start
+    cdef public long start_pos
+    cdef public long start_line
+    cdef public long start_col
+    cdef public text
+    cdef public initial_state
+    cdef public state_name
+    cdef public list queue
+    cdef public bint trace
+    cdef public cur_char
+    cdef public input_state
+    
+    cdef public level
+    
\ No newline at end of file

Cython/Plex/Scanners.py

@@ -10,6 +10,8 @@
 import Errors
 from Regexps import BOL, EOL, EOF
 
+import cython
+
 class Scanner:
   """
   A Scanner is used to read tokens from a stream of characters
@@ -42,23 +44,23 @@ class Scanner:
 
   """
 
-  lexicon = None        # Lexicon
-  stream = None         # file-like object
-  name = ''
-  buffer = ''
-  buf_start_pos = 0     # position in input of start of buffer
-  next_pos = 0          # position in input of next char to read
-  cur_pos = 0           # position in input of current char
-  cur_line = 1          # line number of current char
-  cur_line_start = 0    # position in input of start of current line
-  start_pos = 0         # position in input of start of token
-  start_line = 0        # line number of start of token
-  start_col = 0         # position in line of start of token
-  text = None           # text of last token read
-  initial_state = None  # Node
-  state_name = ''       # Name of initial state
-  queue = None          # list of tokens to be returned
-  trace = 0
+#  lexicon = None        # Lexicon
+#  stream = None         # file-like object
+#  name = ''
+#  buffer = ''
+#  buf_start_pos = 0     # position in input of start of buffer
+#  next_pos = 0          # position in input of next char to read
+#  cur_pos = 0           # position in input of current char
+#  cur_line = 1          # line number of current char
+#  cur_line_start = 0    # position in input of start of current line
+#  start_pos = 0         # position in input of start of token
+#  start_line = 0        # line number of start of token
+#  start_col = 0         # position in line of start of token
+#  text = None           # text of last token read
+#  initial_state = None  # Node
+#  state_name = ''       # Name of initial state
+#  queue = None          # list of tokens to be returned
+#  trace = 0
 
   def __init__(self, lexicon, stream, name = '', initial_pos = None):
     """
@@ -73,6 +75,17 @@ class Scanner:
       |name| is optional, and may be the name of the file being
       scanned or any other identifying string.
     """
+    self.buffer = ''
+    self.buf_start_pos = 0
+    self.next_pos = 0
+    self.cur_pos = 0
+    self.cur_line = 1
+    self.start_pos = 0
+    self.start_line = 0
+    self.start_col = 0
+    self.text = None
+    self.state_name = None
+    
     self.lexicon = lexicon
     self.stream = stream
     self.name = name
@@ -374,6 +387,3 @@ class Scanner:
     Override this method if you want something to be done at
     end of file.
     """
-
-# For backward compatibility:
-setattr(Scanner, "yield", Scanner.produce)

setup.py

@@ -31,7 +31,7 @@ except ValueError:
     try:
         from Cython.Compiler.Main import compile
         source_root = os.path.dirname(__file__)
-        compiled_modules = ["Cython.Plex.Scanners"]
+        compiled_modules = ["Cython.Plex.Scanners", "Cython.Compiler.Scanning"]
         extensions = []
         for module in compiled_modules:
             source_file = os.path.join(source_root, *module.split('.'))