merge

Cython/Compiler/Nodes.py

@@ -717,9 +717,11 @@ class CVarDefNode(StatNode):
     #  in_pxd        boolean
     #  api           boolean
     #  need_properties [entry]
+    #  pxd_locals    [CVarDefNode]  (used for functions declared in pxd)
 
     child_attrs = ["base_type", "declarators"]
     need_properties = ()
+    pxd_locals = []
     
     def analyse_declarations(self, env, dest_scope = None):
         if not dest_scope:
@@ -755,6 +757,7 @@ class CVarDefNode(StatNode):
                 entry = dest_scope.declare_cfunction(name, type, declarator.pos,
                     cname = cname, visibility = self.visibility, in_pxd = self.in_pxd,
                     api = self.api)
+                entry.pxd_locals = self.pxd_locals
             else:
                 if self.in_pxd and self.visibility != 'extern':
                     error(self.pos, 
@@ -891,10 +894,12 @@ class FuncDefNode(StatNode, BlockNode):
     #  #filename        string        C name of filename string const
     #  entry           Symtab.Entry
     #  needs_closure   boolean        Whether or not this function has inner functions/classes/yield
+    #  pxd_locals      [CVarDefNode]   locals defined in the pxd
     
     py_func = None
     assmt = None
     needs_closure = False
+    pxd_locals = []
     
     def analyse_default_values(self, env):
         genv = env.global_scope()
@@ -1498,7 +1503,8 @@ class DefNode(FuncDefNode):
                             with_gil = cfunc.type.with_gil,
                             nogil = cfunc.type.nogil,
                             visibility = 'private',
-                            api = False)
+                            api = False,
+                            pxd_locals = cfunc.pxd_locals)
     
     def analyse_declarations(self, env):
         if 'locals' in env.directives:
@@ -2317,10 +2323,11 @@ class PyClassDefNode(ClassDefNode):
         elif len(bases) == 1:
             base = bases[0]
             path = []
-            while isinstance(base, ExprNodes.AttributeNode):
+            from ExprNodes import AttributeNode, NameNode
+            while isinstance(base, AttributeNode):
                 path.insert(0, base.attribute)
                 base = base.obj
-            if isinstance(base, ExprNodes.NameNode):
+            if isinstance(base, NameNode):
                 path.insert(0, base.name)
                 base_class_name = path[-1]
                 if len(path) > 1:
@@ -4439,6 +4446,9 @@ class FromImportStatNode(StatNode):
                         env.use_utility_code(ExprNodes.type_test_utility_code)
                         break
             else:
+                entry =  env.lookup(target.name)
+                if entry.is_type and entry.type.name == name and entry.type.module_name == self.module.module_name.value:
+                    continue # already cimported
                 self.interned_items.append(
                     (env.intern_identifier(name), target))
                 target.analyse_target_expression(env, None)

Cython/Compiler/ParseTreeTransforms.py

@@ -235,6 +235,22 @@ class PxdPostParse(CythonTransform):
             and node.name in ('__getbuffer__', '__releasebuffer__')):
             ok = True
             
+        if isinstance(node, CFuncDefNode):
+            ok = True
+            for stat in node.body.stats:
+                if not isinstance(stat, CVarDefNode):
+                    self.context.error("C function definition not allowed here")
+                    ok = False
+                    break
+            node = CVarDefNode(node.pos, 
+                visibility = node.visibility,
+                base_type = node.base_type,
+                declarators = [node.declarator],
+                in_pxd = True,
+                api = node.api,
+                overridable = node.overridable, 
+                pxd_locals = node.body.stats)
+
         if not ok:
             self.context.nonfatal_error(PostParseError(node.pos,
                 self.ERR_FUNCDEF_NOT_ALLOWED))
@@ -547,6 +563,8 @@ property NAME:
                     lenv.declare_var(var, type, type_node.pos)
                 else:
                     error(type_node.pos, "Not a type")
+        for stat in node.pxd_locals:
+            stat.analyse_declarations(lenv)
         node.body.analyse_declarations(lenv)
         self.env_stack.append(lenv)
         self.visitchildren(node)

Cython/Compiler/Parsing.py

@@ -2145,7 +2145,7 @@ def p_c_func_or_var_declaration(s, pos, ctx):
                                 assignable = 1, nonempty = 1)
     declarator.overridable = ctx.overridable
     if s.sy == ':':
-        if ctx.level not in ('module', 'c_class'):
+        if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd'):
             s.error("C function definition not allowed here")
         doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)
         result = Nodes.CFuncDefNode(pos,

Cython/Compiler/Scanning.pxd

+from Cython.Plex.Scanners cimport Scanner
+
+cdef class PyrexScanner(Scanner):
+    cdef public context
+    cdef public list included_files
+    cdef public compile_time_env
+    cdef public bint compile_time_eval
+    cdef public bint compile_time_expr
+    cdef public bint parse_comments
+    cdef public source_encoding
+    cdef public list indentation_stack
+    cdef public indentation_char
+    cdef public int bracket_nesting_level
+    cdef public sy
+    cdef public systring
+
+    cdef long current_level(self)
+    cpdef begin(self, state)
+    cpdef next(self)
+    cpdef bint expect(self, what, message = *) except -2
+    
+    cpdef indentation_action(self, text):
+        cdef:
+            long current_level
+            long new_level
+    

Cython/Compiler/Scanning.py

@@ -12,14 +12,17 @@ import sys
 from time import time
 
 from Cython import Plex, Utils
-from Cython.Plex import Scanner
+from Cython.Plex.Scanners import Scanner
 from Cython.Plex.Errors import UnrecognizedInput
 from Errors import CompileError, error
 from Lexicon import string_prefixes, raw_prefixes, make_lexicon
 
 from StringEncoding import EncodedString
 
-plex_version = getattr(Plex, '_version', None)
+try:
+    plex_version = Plex._version
+except AttributeError:
+    plex_version = None
 #print "Plex version:", plex_version ###
 
 debug_scanner = 0
@@ -91,7 +94,7 @@ def try_to_unpickle_lexicon():
     source_file = os.path.join(dir, "Lexicon.py")
     lexicon_hash = hash_source_file(source_file)
     lexicon_pickle = os.path.join(dir, "Lexicon.pickle")
-    f = open_pickled_lexicon(expected_hash = lexicon_hash)
+    f = open_pickled_lexicon(lexicon_hash)
     if f:
         if notify_lexicon_unpickling:
             t0 = time()
@@ -165,6 +168,8 @@ def build_resword_dict():
         d[word] = 1
     return d
 
+resword_dict = build_resword_dict()
+
 #------------------------------------------------------------------
 
 class CompileTimeScope(object):
@@ -286,7 +291,6 @@ class PyrexScanner(Scanner):
     #  compile_time_env   dict     Environment for conditional compilation
     #  compile_time_eval  boolean  In a true conditional compilation context
     #  compile_time_expr  boolean  In a compile-time expression context
-    resword_dict = build_resword_dict()
 
     def __init__(self, file, filename, parent_scanner = None, 
                  scope = None, context = None, source_encoding=None, parse_comments=True, initial_pos=None):
@@ -404,14 +408,14 @@ class PyrexScanner(Scanner):
         except UnrecognizedInput:
             self.error("Unrecognized character")
         if sy == 'IDENT':
-            if systring in self.resword_dict:
+            if systring in resword_dict:
                 sy = systring
             else:
                 systring = EncodedString(systring)
                 systring.encoding = self.source_encoding
         self.sy = sy
         self.systring = systring
-        if debug_scanner:
+        if False: # debug_scanner:
             _, line, col = self.position()
             if not self.systring or self.sy == self.systring:
                 t = self.sy

Cython/Compiler/Symtab.py

@@ -821,6 +821,8 @@ class ModuleScope(Scope):
         self.cached_builtins = []
         self.undeclared_cached_builtins = []
         self.namespace_cname = self.module_cname
+        for name in ['__builtins__', '__name__', '__file__', '__doc__']:
+            self.declare_var(EncodedString(name), py_object_type, None)
     
     def qualifying_scope(self):
         return self.parent_module

Cython/Plex/Scanners.pxd

+import cython
+
+cdef class Scanner:
+    cdef public lexicon
+    cdef public stream
+    cdef public name
+    cdef public buffer
+    cdef public long buf_start_pos
+    cdef public long next_pos
+    cdef public long cur_pos
+    cdef public long cur_line
+    cdef public long cur_line_start
+    cdef public long start_pos
+    cdef public long start_line
+    cdef public long start_col
+    cdef public text
+    cdef public initial_state # int?
+    cdef public state_name
+    cdef public list queue
+    cdef public bint trace
+    cdef public cur_char
+    cdef public input_state
+    
+    cdef public level # int?
+    
+    cpdef next_char(self):
+        cdef:
+            long input_state
+    
+    cpdef run_machine_inlined(self):
+        cdef:
+            long cur_pos
+            long cur_line
+            long cur_line_start
+            long input_state
+            long next_pos
+            long buf_start_pos
+            long buf_len
+            long buf_index
+            bint trace
+            long discard
+    
+    cpdef begin(self, state)
+    cpdef produce(self, value, text = *)

Cython/Plex/Scanners.py

@@ -10,6 +10,8 @@
 import Errors
 from Regexps import BOL, EOL, EOF
 
+import cython
+
 class Scanner:
   """
   A Scanner is used to read tokens from a stream of characters
@@ -42,23 +44,23 @@ class Scanner:
 
   """
 
-  lexicon = None        # Lexicon
-  stream = None         # file-like object
-  name = ''
-  buffer = ''
-  buf_start_pos = 0     # position in input of start of buffer
-  next_pos = 0          # position in input of next char to read
-  cur_pos = 0           # position in input of current char
-  cur_line = 1          # line number of current char
-  cur_line_start = 0    # position in input of start of current line
-  start_pos = 0         # position in input of start of token
-  start_line = 0        # line number of start of token
-  start_col = 0         # position in line of start of token
-  text = None           # text of last token read
-  initial_state = None  # Node
-  state_name = ''       # Name of initial state
-  queue = None          # list of tokens to be returned
-  trace = 0
+#  lexicon = None        # Lexicon
+#  stream = None         # file-like object
+#  name = ''
+#  buffer = ''
+#  buf_start_pos = 0     # position in input of start of buffer
+#  next_pos = 0          # position in input of next char to read
+#  cur_pos = 0           # position in input of current char
+#  cur_line = 1          # line number of current char
+#  cur_line_start = 0    # position in input of start of current line
+#  start_pos = 0         # position in input of start of token
+#  start_line = 0        # line number of start of token
+#  start_col = 0         # position in line of start of token
+#  text = None           # text of last token read
+#  initial_state = None  # Node
+#  state_name = ''       # Name of initial state
+#  queue = None          # list of tokens to be returned
+#  trace = 0
 
   def __init__(self, lexicon, stream, name = '', initial_pos = None):
     """
@@ -73,6 +75,17 @@ class Scanner:
       |name| is optional, and may be the name of the file being
       scanned or any other identifying string.
     """
+    self.buffer = ''
+    self.buf_start_pos = 0
+    self.next_pos = 0
+    self.cur_pos = 0
+    self.cur_line = 1
+    self.start_pos = 0
+    self.start_line = 0
+    self.start_col = 0
+    self.text = None
+    self.state_name = None
+    
     self.lexicon = lexicon
     self.stream = stream
     self.name = name
@@ -374,6 +387,3 @@ class Scanner:
     Override this method if you want something to be done at
     end of file.
     """
-
-# For backward compatibility:
-setattr(Scanner, "yield", Scanner.produce)

setup.py

@@ -31,7 +31,7 @@ except ValueError:
     try:
         from Cython.Compiler.Main import compile
         source_root = os.path.dirname(__file__)
-        compiled_modules = ["Cython.Plex.Scanners"]
+        compiled_modules = ["Cython.Plex.Scanners", "Cython.Compiler.Scanning"]
         extensions = []
         for module in compiled_modules:
             source_file = os.path.join(source_root, *module.split('.'))