more compiler cythonization

more balancing needs to be done.

Cython/Compiler/Nodes.py

@@ -716,9 +716,11 @@ class CVarDefNode(StatNode):
     #  in_pxd        boolean
     #  api           boolean
     #  need_properties [entry]
+    #  pxd_locals    [CVarDefNode]  (used for functions declared in pxd)
 
     child_attrs = ["base_type", "declarators"]
     need_properties = ()
+    pxd_locals = []
     
     def analyse_declarations(self, env, dest_scope = None):
         if not dest_scope:
@@ -754,6 +756,7 @@ class CVarDefNode(StatNode):
                 entry = dest_scope.declare_cfunction(name, type, declarator.pos,
                     cname = cname, visibility = self.visibility, in_pxd = self.in_pxd,
                     api = self.api)
+                entry.pxd_locals = self.pxd_locals
             else:
                 if self.in_pxd and self.visibility != 'extern':
                     error(self.pos, 
@@ -890,10 +893,12 @@ class FuncDefNode(StatNode, BlockNode):
     #  #filename        string        C name of filename string const
     #  entry           Symtab.Entry
     #  needs_closure   boolean        Whether or not this function has inner functions/classes/yield
+    #  pxd_locals      [CVarDefNode]   locals defined in the pxd
     
     py_func = None
     assmt = None
     needs_closure = False
+    pxd_locals = []
     
     def analyse_default_values(self, env):
         genv = env.global_scope()
@@ -1497,7 +1502,8 @@ class DefNode(FuncDefNode):
                             with_gil = cfunc.type.with_gil,
                             nogil = cfunc.type.nogil,
                             visibility = 'private',
-                            api = False)
+                            api = False,
+                            pxd_locals = cfunc.pxd_locals)
     
     def analyse_declarations(self, env):
         if 'locals' in env.directives:

Cython/Compiler/ParseTreeTransforms.py

@@ -235,6 +235,22 @@ class PxdPostParse(CythonTransform):
             and node.name in ('__getbuffer__', '__releasebuffer__')):
             ok = True
             
+        if isinstance(node, CFuncDefNode):
+            ok = True
+            for stat in node.body.stats:
+                if not isinstance(stat, CVarDefNode):
+                    self.context.error("C function definition not allowed here")
+                    ok = False
+                    break
+            node = CVarDefNode(node.pos, 
+                visibility = node.visibility,
+                base_type = node.base_type,
+                declarators = [node.declarator],
+                in_pxd = True,
+                api = node.api,
+                overridable = node.overridable, 
+                pxd_locals = node.body.stats)
+
         if not ok:
             self.context.nonfatal_error(PostParseError(node.pos,
                 self.ERR_FUNCDEF_NOT_ALLOWED))
@@ -547,6 +563,8 @@ property NAME:
                     lenv.declare_var(var, type, type_node.pos)
                 else:
                     error(type_node.pos, "Not a type")
+        for stat in node.pxd_locals:
+            stat.analyse_declarations(lenv)
         node.body.analyse_declarations(lenv)
         self.env_stack.append(lenv)
         self.visitchildren(node)

Cython/Compiler/Parsing.py

@@ -2128,7 +2128,7 @@ def p_c_func_or_var_declaration(s, pos, ctx):
                                 assignable = 1, nonempty = 1)
     declarator.overridable = ctx.overridable
     if s.sy == ':':
-        if ctx.level not in ('module', 'c_class'):
+        if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd'):
             s.error("C function definition not allowed here")
         doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)
         result = Nodes.CFuncDefNode(pos,

Cython/Compiler/Scanning.pxd

@@ -13,3 +13,14 @@ cdef class PyrexScanner(Scanner):
     cdef public int bracket_nesting_level
     cdef public sy
     cdef public systring
+
+    cdef long current_level(self)
+    cpdef begin(self, state)
+    cpdef next(self)
+    cpdef bint expect(self, what, message = *) except -2
+    
+    cpdef indentation_action(self, text):
+        cdef:
+            long current_level
+            long new_level
+    

Cython/Compiler/Scanning.py

@@ -415,7 +415,7 @@ class PyrexScanner(Scanner):
                 systring.encoding = self.source_encoding
         self.sy = sy
         self.systring = systring
-        if debug_scanner:
+        if False: # debug_scanner:
             _, line, col = self.position()
             if not self.systring or self.sy == self.systring:
                 t = self.sy

Cython/Plex/Scanners.pxd

+import cython
+
 cdef class Scanner:
     cdef public lexicon
     cdef public stream
@@ -12,12 +14,31 @@ cdef class Scanner:
     cdef public long start_line
     cdef public long start_col
     cdef public text
-    cdef public initial_state
+    cdef public initial_state # int?
     cdef public state_name
     cdef public list queue
     cdef public bint trace
     cdef public cur_char
     cdef public input_state
     
-    cdef public level
+    cdef public level # int?
+    
+    cpdef next_char(self):
+        cdef:
+            long input_state
+    
+    cpdef run_machine_inlined(self):
+        cdef:
+            long cur_pos
+            long cur_line
+            long cur_line_start
+            long input_state
+            long next_pos
+            long buf_start_pos
+            long buf_len
+            long buf_index
+            bint trace
+            long discard
     
+    cpdef begin(self, state)
+    cpdef produce(self, value, text = *)