Support utility code written in Cython

Cython/Compiler/Code.py

@@ -57,6 +57,9 @@ class UtilityCode(object):
         self.specialize_list = []
         self.proto_block = proto_block
 
+    def get_tree(self):
+        pass
+    
     def specialize(self, pyrex_type=None, **data):
         # Dicts aren't hashable...
         if pyrex_type is not None:

Cython/Compiler/ModuleNode.py

@@ -69,16 +69,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         selfscope = self.scope
         selfscope.utility_code_list.extend(scope.utility_code_list)
         if merge_scope:
-            selfscope.entries.update(scope.entries)
-            for x in ('const_entries',
-                      'type_entries',
-                      'sue_entries',
-                      'arg_entries',
-                      'var_entries',
-                      'pyfunc_entries',
-                      'cfunc_entries',
-                      'c_class_entries'):
-                getattr(selfscope, x).extend(getattr(scope, x))
+            selfscope.merge_in(scope)
 
     def analyse_declarations(self, env):
         if Options.embed_pos_in_docstring:

Cython/Compiler/Pipeline.py

@@ -60,11 +60,22 @@ def inject_pxd_code_stage_factory(context):
         return module_node
     return inject_pxd_code_stage
 
+def inject_utility_code_stage(module_node):
+    added = []
+    # need to copy list as the list will be altered!
+    for utilcode in module_node.scope.utility_code_list[:]:
+        if utilcode in added: continue
+        added.append(utilcode)
+        tree = utilcode.get_tree()
+        if tree:
+            module_node.merge_in(tree.body, tree.scope, merge_scope=True)
+    return module_node
+
 #
 # Pipeline factories
 #
 
-def create_pipeline(context, mode):
+def create_pipeline(context, mode, exclude_classes=()):
     assert mode in ('pyx', 'py', 'pxd')
     from Visitor import PrintTree
     from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, PxdPostParse
@@ -98,7 +109,7 @@ def create_pipeline(context, mode):
     else:
         _align_function_definitions = None
 
-    return [
+    stages = [
         NormalizeTree(context),
         PostParse(context),
         _specific_post_parse,
@@ -134,9 +145,13 @@ def create_pipeline(context, mode):
         FinalOptimizePhase(context),
         GilCheck(),
         ]
+    filtered_stages = []
+    for s in stages:
+        if s.__class__ not in exclude_classes:
+            filtered_stages.append(s)
+    return filtered_stages
 
-
-def create_pyx_pipeline(context, options, result, py=False):
+def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()):
     if py:
         mode = 'py'
     else:
@@ -157,9 +172,10 @@ def create_pyx_pipeline(context, options, result, py=False):
 
     return list(itertools.chain(
         [parse_stage_factory(context)],
-        create_pipeline(context, mode),
+        create_pipeline(context, mode, exclude_classes=exclude_classes),
         test_support,
         [inject_pxd_code_stage_factory(context),
+         inject_utility_code_stage,
          abort_on_errors],
         debug_transform,
         [generate_pyx_code_stage_factory(options, result)]))
@@ -184,17 +200,15 @@ def create_pyx_as_pxd_pipeline(context, result):
     from Optimize import ConstantFolding, FlattenInListTransform
     from Nodes import StatListNode
     pipeline = []
-    pyx_pipeline = create_pyx_pipeline(context, context.options, result)
+    pyx_pipeline = create_pyx_pipeline(context, context.options, result,
+                                       exclude_classes=[
+                                           AlignFunctionDefinitions,
+                                           MarkClosureVisitor,
+                                           ConstantFolding,
+                                           FlattenInListTransform,
+                                           WithTransform
+                                           ])
     for stage in pyx_pipeline:
-        if stage.__class__ in [
-                AlignFunctionDefinitions,
-                MarkClosureVisitor,
-                ConstantFolding,
-                FlattenInListTransform,
-                WithTransform,
-                ]:
-            # Skip these unnecessary stages.
-            continue
         pipeline.append(stage)
         if isinstance(stage, AnalyseDeclarationsTransform):
             # This is the last stage we need.

Cython/Compiler/PyrexTypes.py

@@ -1968,6 +1968,9 @@ class StructUtilityCode(object):
     def __hash__(self):
         return hash(self.header)
 
+    def get_tree(self):
+        pass
+    
     def put_code(self, output):
         code = output['utility_code_def']
         proto = output['utility_code_proto']
@@ -1995,6 +1998,9 @@ class StructUtilityCode(object):
             proto.putln(self.type.declaration_code('') + ';')
         proto.putln(self.header + ";")
 
+    def inject_tree_and_scope_into(self, module_node):
+        pass
+        
 
 class CStructOrUnionType(CType):
     #  name          string

Cython/Compiler/Symtab.py

@@ -122,6 +122,9 @@ class Entry(object):
     # assignments      [ExprNode] List of expressions that get assigned to this entry.
     # might_overflow   boolean    In an arithmetic expression that could cause
     #                             overflow (used for type inference).
+    # utility_code_definition     For some Cython builtins, the utility code
+    #                             which contains the definition of the entry.
+    #                             Currently only supported for CythonScope entries.
 
     inline_func_in_pxd = False
     borrowed = 0
@@ -173,6 +176,7 @@ class Entry(object):
     buffer_aux = None
     prev_entry = None
     might_overflow = 0
+    utility_code_definition = None
     in_with_gil_block = 0
 
     def __init__(self, name, cname, type, pos = None, init = None):
@@ -274,6 +278,20 @@ class Scope(object):
         self.return_type = None
         self.id_counters = {}
 
+    def merge_in(self, other):
+        # Use with care...
+        self.entries.update(other.entries)
+        for x in ('const_entries',
+                  'type_entries',
+                  'sue_entries',
+                  'arg_entries',
+                  'var_entries',
+                  'pyfunc_entries',
+                  'cfunc_entries',
+                  'c_class_entries'):
+            getattr(self, x).extend(getattr(other, x))
+
+
     def __str__(self):
         return "<%s %s>" % (self.__class__.__name__, self.qualified_name)
 
@@ -879,8 +897,9 @@ class ModuleScope(Scope):
     has_import_star = 0
 
     def __init__(self, name, parent_module, context):
+        import Builtin
         self.parent_module = parent_module
-        outer_scope = context.find_submodule("__builtin__")
+        outer_scope = Builtin.builtin_scope
         Scope.__init__(self, name, outer_scope, parent_module)
         if name != "__init__":
             self.module_name = name
@@ -921,7 +940,8 @@ class ModuleScope(Scope):
         entry = self.lookup_here(name)
         if entry is not None:
             return entry
-        return self.outer_scope.lookup(name, language_level = self.context.language_level)
+        language_level = self.context.language_level if self.context is not None else 3
+        return self.outer_scope.lookup(name, language_level=language_level)
 
     def declare_builtin(self, name, pos):
         if not hasattr(builtins, name) \

Cython/Compiler/TreeFragment.py

@@ -20,7 +20,8 @@ Support for parsing strings into code trees.
 """
 
 class StringParseContext(Main.Context):
-    def __init__(self, include_directories, name):
+    def __init__(self, name, include_directories=None):
+        if include_directories is None: include_directories = []
         Main.Context.__init__(self, include_directories, {})
         self.module_name = name
 
@@ -29,7 +30,8 @@ class StringParseContext(Main.Context):
             raise AssertionError("Not yet supporting any cimports/includes from string code snippets")
         return ModuleScope(module_name, parent_module = None, context = self)
 
-def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None):
+def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None,
+                       context=None):
     """
     Utility method to parse a (unicode) string of code. This is mostly
     used for internal Cython compiler purposes (creating code snippets
@@ -37,8 +39,14 @@ def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None):
 
     code - a unicode string containing Cython (module-level) code
     name - a descriptive name for the code source (to use in error messages etc.)
-    """
 
+    RETURNS
+
+    The tree, i.e. a ModuleNode. The ModuleNode's scope attribute is
+    set to the scope used when parsing.
+    """
+    if context is None:
+        context = StringParseContext(name)
     # Since source files carry an encoding, it makes sense in this context
     # to use a unicode string so that code fragments don't have to bother
     # with encoding. This means that test code passed in should not have an
@@ -51,7 +59,6 @@ def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None):
         initial_pos = (name, 1, 0)
     code_source = StringSourceDescriptor(name, code)
 
-    context = StringParseContext([], name)
     scope = context.find_module(module_name, pos = initial_pos, need_pxd = 0)
 
     buf = StringIO(code)
@@ -61,8 +68,10 @@ def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None):
     if level is None:
         tree = Parsing.p_module(scanner, 0, module_name)
         tree.scope = scope
+        tree.is_pxd = False
     else:
         tree = Parsing.p_code(scanner, level=level)
+    tree.scope = scope
     return tree
 
 class TreeCopier(VisitorTransform):

Cython/Compiler/UtilityCode.py

+from TreeFragment import parse_from_strings, StringParseContext
+from Scanning import StringSourceDescriptor
+import Symtab
+import Naming
+
+class NonManglingModuleScope(Symtab.ModuleScope):
+    def mangle(self, prefix, name=None):
+        if name:
+            if prefix in (Naming.typeobj_prefix, Naming.func_prefix):
+                # Functions, classes etc. gets a manually defined prefix easily
+                # manually callable instead (the one passed to CythonUtilityCode)
+                prefix = self.prefix
+            return "%s%s" % (prefix, name)
+        else:
+            return self.base.name
+
+class CythonUtilityCodeContext(StringParseContext):
+    scope = None
+    
+    def find_module(self, module_name, relative_to = None, pos = None, need_pxd = 1):
+        if module_name != self.module_name:
+            raise AssertionError("Not yet supporting any cimports/includes from string code snippets")
+        if self.scope is None:
+            self.scope = NonManglingModuleScope(module_name, parent_module = None, context = self)
+            self.scope.prefix = self.prefix
+        return self.scope
+
+class CythonUtilityCode:
+    """
+    Utility code written in the Cython language itself.
+    """
+
+    def __init__(self, pyx, name="<utility code>", prefix=""):
+        # 1) We need to delay the parsing/processing, so that all modules can be
+        #    imported without import loops
+        # 2) The same utility code object can be used for multiple source files;
+        #    while the generated node trees can be altered in the compilation of a
+        #    single file.
+        # Hence, delay any processing until later.
+        self.pyx = pyx
+        self.name = name
+        self.prefix = prefix
+
+    def get_tree(self):
+        from AnalysedTreeTransforms import AutoTestDictTransform
+        excludes = [AutoTestDictTransform]
+        
+        import Pipeline
+        context = CythonUtilityCodeContext(self.name)
+        context.prefix = self.prefix
+        tree = parse_from_strings(self.name, self.pyx, context=context)
+        pipeline = Pipeline.create_pipeline(context, 'pyx', exclude_classes=excludes)
+        (err, tree) = Pipeline.run_pipeline(pipeline, tree)
+        assert not err
+        return tree
+
+    def put_code(self, output):
+        pass
+
+
+