Refactored pipeline construction to Pipeline.py

Cython/Build/Inline.py

@@ -16,6 +16,7 @@ from Cython.Compiler.Main import Context, CompilationOptions, default_options
 from Cython.Compiler.ParseTreeTransforms import CythonTransform, SkipDeclarations, AnalyseDeclarationsTransform
 from Cython.Compiler.TreeFragment import parse_from_strings
 from Cython.Build.Dependencies import strip_string_literals, cythonize
+from Cython.Compiler import Pipeline
 
 # A utility function to convert user-supplied ASCII strings to unicode.
 if sys.version_info[0] < 3:
@@ -43,7 +44,7 @@ def unbound_symbols(code, context=None):
         context = Context([], default_options)
     from Cython.Compiler.ParseTreeTransforms import AnalyseDeclarationsTransform
     tree = parse_from_strings('(tree fragment)', code)
-    for phase in context.create_pipeline(pxd=False):
+    for phase in Pipeline.create_pipeline(context, 'pyx'):
         if phase is None:
             continue
         tree = phase(tree)

Cython/Compiler/ModuleNode.py

@@ -52,6 +52,34 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     child_attrs = ["body"]
     directives = None
 
+    def merge_in(self, tree, scope, merge_scope=False):
+        # Merges in the contents of another tree, and possibly scope. With the
+        # current implementation below, this must be done right prior
+        # to code generation.
+        #
+        # Note: This way of doing it seems strange -- I believe the
+        # right concept is to split ModuleNode into a ModuleNode and a
+        # CodeGenerator, and tell that CodeGenerator to generate code
+        # from multiple sources.
+        assert isinstance(self.body, Nodes.StatListNode)
+        if isinstance(tree, Nodes.StatListNode):
+            self.body.stats.extend(tree.stats)
+        else:
+            self.body.stats.append(tree)
+        selfscope = self.scope
+        selfscope.utility_code_list.extend(scope.utility_code_list)
+        if merge_scope:
+            selfscope.entries.update(scope.entries)
+            for x in ('const_entries',
+                      'type_entries',
+                      'sue_entries',
+                      'arg_entries',
+                      'var_entries',
+                      'pyfunc_entries',
+                      'cfunc_entries',
+                      'c_class_entries'):
+                getattr(selfscope, x).extend(getattr(scope, x))
+
     def analyse_declarations(self, env):
         if Options.embed_pos_in_docstring:
             env.doc = EncodedString(u'File: %s (starting at line %s)' % Nodes.relative_position(self.pos))

Cython/Compiler/Pipeline.py

+import itertools
+from time import time
+
+import Errors
+import DebugFlags
+import Options
+from Errors import PyrexError, CompileError, InternalError, AbortError, error
+
+#
+# Really small pipeline stages
+#
+def dumptree(t):
+    # For quick debugging in pipelines
+    print t.dump()
+    return t
+
+def abort_on_errors(node):
+    # Stop the pipeline if there are any errors.
+    if Errors.num_errors != 0:
+        raise AbortError, "pipeline break"
+    return node
+
+def parse_stage_factory(context):
+    def parse(compsrc):
+        source_desc = compsrc.source_desc
+        full_module_name = compsrc.full_module_name
+        initial_pos = (source_desc, 1, 0)
+        saved_cimport_from_pyx, Options.cimport_from_pyx = Options.cimport_from_pyx, False
+        scope = context.find_module(full_module_name, pos = initial_pos, need_pxd = 0)
+        Options.cimport_from_pyx = saved_cimport_from_pyx
+        tree = context.parse(source_desc, scope, pxd = 0, full_module_name = full_module_name)
+        tree.compilation_source = compsrc
+        tree.scope = scope
+        tree.is_pxd = False
+        return tree
+    return parse
+
+def parse_pxd_stage_factory(context, scope, module_name):
+    def parse(source_desc):
+        tree = context.parse(source_desc, scope, pxd=True,
+                             full_module_name=module_name)
+        tree.scope = scope
+        tree.is_pxd = True
+        return tree
+    return parse
+
+def generate_pyx_code_stage_factory(options, result):
+    def generate_pyx_code_stage(module_node):
+        module_node.process_implementation(options, result)
+        result.compilation_source = module_node.compilation_source
+        return result
+    return generate_pyx_code_stage
+
+def inject_pxd_code_stage_factory(context):
+    def inject_pxd_code_stage(module_node):
+        from textwrap import dedent
+        stats = module_node.body.stats
+        for name, (statlistnode, scope) in context.pxds.iteritems():
+            module_node.merge_in(statlistnode, scope)
+        return module_node
+    return inject_pxd_code_stage
+
+#
+# Pipeline factories
+#
+
+def create_pipeline(context, mode):
+    assert mode in ('pyx', 'py', 'pxd')
+    from Visitor import PrintTree
+    from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, PxdPostParse
+    from ParseTreeTransforms import ForwardDeclareTypes, AnalyseDeclarationsTransform
+    from ParseTreeTransforms import AnalyseExpressionsTransform
+    from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
+    from ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods
+    from ParseTreeTransforms import ExpandInplaceOperators, ParallelRangeTransform
+    from TypeInference import MarkAssignments, MarkOverflowingArithmetic
+    from ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions
+    from ParseTreeTransforms import RemoveUnreachableCode, GilCheck
+    from FlowControl import CreateControlFlowGraph
+    from AnalysedTreeTransforms import AutoTestDictTransform
+    from AutoDocTransforms import EmbedSignature
+    from Optimize import FlattenInListTransform, SwitchTransform, IterationTransform
+    from Optimize import EarlyReplaceBuiltinCalls, OptimizeBuiltinCalls
+    from Optimize import ConstantFolding, FinalOptimizePhase
+    from Optimize import DropRefcountingTransform
+    from Buffer import IntroduceBufferAuxiliaryVars
+    from ModuleNode import check_c_declarations, check_c_declarations_pxd
+
+    if mode == 'pxd':
+        _check_c_declarations = check_c_declarations_pxd
+        _specific_post_parse = PxdPostParse(context)
+    else:
+        _check_c_declarations = check_c_declarations
+        _specific_post_parse = None
+
+    if mode == 'py':
+        _align_function_definitions = AlignFunctionDefinitions(context)
+    else:
+        _align_function_definitions = None
+
+    return [
+        NormalizeTree(context),
+        PostParse(context),
+        _specific_post_parse,
+        InterpretCompilerDirectives(context, context.compiler_directives),
+        ParallelRangeTransform(context),
+        AdjustDefByDirectives(context),
+        MarkClosureVisitor(context),
+        _align_function_definitions,
+        RemoveUnreachableCode(context),
+        ConstantFolding(),
+        FlattenInListTransform(),
+        WithTransform(context),
+        DecoratorTransform(context),
+        ForwardDeclareTypes(context),
+        AnalyseDeclarationsTransform(context),
+        AutoTestDictTransform(context),
+        EmbedSignature(context),
+        EarlyReplaceBuiltinCalls(context),  ## Necessary?
+        TransformBuiltinMethods(context),  ## Necessary?
+        CreateControlFlowGraph(context),
+        RemoveUnreachableCode(context),
+        MarkAssignments(context),
+        MarkOverflowingArithmetic(context),
+        IntroduceBufferAuxiliaryVars(context),
+        _check_c_declarations,
+        AnalyseExpressionsTransform(context),
+        CreateClosureClasses(context),  ## After all lookups and type inference
+        ExpandInplaceOperators(context),
+        OptimizeBuiltinCalls(context),  ## Necessary?
+        IterationTransform(),
+        SwitchTransform(),
+        DropRefcountingTransform(),
+        FinalOptimizePhase(context),
+        GilCheck(),
+        ]
+
+
+def create_pyx_pipeline(context, options, result, py=False):
+    if py:
+        mode = 'py'
+    else:
+        mode = 'pyx'
+    test_support = []
+    if options.evaluate_tree_assertions:
+        from Cython.TestUtils import TreeAssertVisitor
+        test_support.append(TreeAssertVisitor())
+
+    if options.gdb_debug:
+        from Cython.Debugger import DebugWriter
+        from ParseTreeTransforms import DebugTransform
+        context.gdb_debug_outputwriter = DebugWriter.CythonDebugWriter(
+            options.output_dir)
+        debug_transform = [DebugTransform(context, options, result)]
+    else:
+        debug_transform = []
+
+    return list(itertools.chain(
+        [parse_stage_factory(context)],
+        create_pipeline(context, mode),
+        test_support,
+        [inject_pxd_code_stage_factory(context),
+         abort_on_errors],
+        debug_transform,
+        [generate_pyx_code_stage_factory(options, result)]))
+
+def create_pxd_pipeline(context, scope, module_name):
+    from CodeGeneration import ExtractPxdCode
+
+    # The pxd pipeline ends up with a CCodeWriter containing the
+    # code of the pxd, as well as a pxd scope.
+    return [
+        parse_pxd_stage_factory(context, scope, module_name)
+        ] + create_pipeline(context, 'pxd') + [
+        ExtractPxdCode(context)
+        ]
+
+def create_py_pipeline(context, options, result):
+    return create_pyx_pipeline(context, options, result, py=True)
+
+def create_pyx_as_pxd_pipeline(context, result):
+    from ParseTreeTransforms import (AlignFunctionDefinitions,
+        MarkClosureVisitor, WithTransform, AnalyseDeclarationsTransform)
+    from Optimize import ConstantFolding, FlattenInListTransform
+    from Nodes import StatListNode
+    pipeline = []
+    pyx_pipeline = create_pyx_pipeline(context, context.options, result)
+    for stage in pyx_pipeline:
+        if stage.__class__ in [
+                AlignFunctionDefinitions,
+                MarkClosureVisitor,
+                ConstantFolding,
+                FlattenInListTransform,
+                WithTransform,
+                ]:
+            # Skip these unnecessary stages.
+            continue
+        pipeline.append(stage)
+        if isinstance(stage, AnalyseDeclarationsTransform):
+            # This is the last stage we need.
+            break
+    def fake_pxd(root):
+        for entry in root.scope.entries.values():
+            entry.defined_in_pxd = 1
+        return StatListNode(root.pos, stats=[]), root.scope
+    pipeline.append(fake_pxd)
+    return pipeline
+
+#
+# Running a pipeline
+#
+
+def run_pipeline(pipeline, source):
+    error = None
+    data = source
+    try:
+        try:
+            for phase in pipeline:
+                if phase is not None:
+                    if DebugFlags.debug_verbose_pipeline:
+                        t = time()
+                        print "Entering pipeline phase %r" % phase
+                    data = phase(data)
+                    if DebugFlags.debug_verbose_pipeline:
+                        print "    %.3f seconds" % (time() - t)
+        except CompileError, err:
+            # err is set
+            Errors.report_error(err)
+            error = err
+    except InternalError, err:
+        # Only raise if there was not an earlier error
+        if Errors.num_errors == 0:
+            raise
+        error = err
+    except AbortError, err:
+        error = err
+    return (error, data)