Make it possible to construct sub-magic-modules of cython

Cython/Compiler/CythonScope.py

 from Symtab import ModuleScope
 from PyrexTypes import *
 from UtilityCode import CythonUtilityCode
+from Errors import error
+from Scanning import StringSourceDescriptor
 
 class CythonScope(ModuleScope):
+    is_cython_builtin = 1
+
     def __init__(self):
         ModuleScope.__init__(self, u'cython', None, None)
         self.pxd_file_loaded = True
@@ -15,7 +19,31 @@ class CythonScope(ModuleScope):
             return type
 
     def find_module(self, module_name, pos):
-        error("cython.%s is not available" % module_name)
+        error("cython.%s is not available" % module_name, pos)
+
+    def find_submodule(self, module_name):
+        entry = self.entries.get(module_name, None)
+        if entry and entry.as_module:
+            return entry.as_module
+        else:
+            # TODO: fix find_submodule control flow so that we're not
+            # expected to create a submodule here (to protect CythonScope's
+            # possible immutability). Hack ourselves out of the situation
+            # for now.
+            raise error((StringSourceDescriptor(u"cython", u""), 0, 0),
+                  "cython.%s is not available" % module_name)
+
+    def lookup_qualified_name(self, qname):
+        # ExprNode.as_cython_attribute generates qnames and we untangle it here...
+        name_path = qname.split(u'.')
+        scope = self
+        while len(name_path) > 1:
+            scope = scope.lookup_here(name_path[0]).as_module
+            del name_path[0]
+            if scope is None:
+                return None
+        else:
+            return scope.lookup_here(name_path[0])
 
     def populate_cython_scope(self):
         # These are used to optimize isinstance in FinalOptimizePhase
@@ -25,28 +53,43 @@ class CythonScope(ModuleScope):
             pos = None,
             cname = 'PyTypeObject')
         type_object.is_void = True
+        type_object_type = type_object.type
         
         self.declare_cfunction(
             'PyObject_TypeCheck',
             CFuncType(c_bint_type, [CFuncTypeArg("o", py_object_type, None),
-                                    CFuncTypeArg("t", c_ptr_type(type_object), None)]),
+                                    CFuncTypeArg("t", c_ptr_type(type_object_type), None)]),
             pos = None,
             defining = 1,
             cname = 'PyObject_TypeCheck')
 
-        #
         # A special function just to make it easy to test the scope and
         # utility code functionality in isolation. It is available to
         # "end-users" but nobody will know it is there anyway...
+        entry = self.declare_cfunction(
+            '_testscope',
+            CFuncType(py_object_type, [CFuncTypeArg("value", c_int_type, None)]),
+            pos=None,
+            defining=1,
+            cname='__pyx_cython__testscope'
+        )
+        entry.utility_code_definition = cython_testscope_utility_code
+
         #
-        testcythonscope = self.declare_cfunction(
-            '_testcythonscope',
+        # The view sub-scope
+        #
+        self.viewscope = viewscope = ModuleScope(u'cython.view', self, None)
+        self.declare_module('view', viewscope, None)
+        viewscope.is_cython_builtin = True
+        viewscope.pxd_file_loaded = True
+        entry = viewscope.declare_cfunction(
+            '_testscope',
             CFuncType(py_object_type, [CFuncTypeArg("value", c_int_type, None)]),
             pos=None,
             defining=1,
-            cname='__pyx_cython__testcythonscope'
+            cname='__pyx_cython_view__testscope'
         )
-        testcythonscope.utility_code_definition = cython_testscope_utility_code
+        entry.utility_code_definition = cythonview_testscope_utility_code
 
 def create_cython_scope(context):
     # One could in fact probably make it a singleton,
@@ -55,6 +98,11 @@ def create_cython_scope(context):
     return CythonScope()
 
 cython_testscope_utility_code = CythonUtilityCode(u"""
-cdef object _testcythonscope(int value):
-    return "hello value=%d" % value
+cdef object _testscope(int value):
+    return "hello from cython scope, value=%d" % value
 """, name="cython utility code", prefix="__pyx_cython_")
+
+cythonview_testscope_utility_code = CythonUtilityCode(u"""
+cdef object _testscope(int value):
+    return "hello from cython.view scope, value=%d" % value
+""", name="cython utility code", prefix="__pyx_cython_view_")

Cython/Compiler/ModuleNode.py

@@ -1029,6 +1029,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     entry.type.typeptr_cname)
 
     def generate_cvariable_declarations(self, env, code, definition):
+        if env.is_cython_builtin:
+            return
         for entry in env.var_entries:
             if (entry.in_cinclude or entry.in_closure or
                 (entry.visibility == 'private' and

Cython/Compiler/ParseTreeTransforms.py

@@ -2193,7 +2193,7 @@ class TransformBuiltinMethods(EnvTransform):
                                           entry=self.current_env().builtin_scope().lookup_here(attribute))
             elif PyrexTypes.parse_basic_type(attribute):
                 pass
-            elif self.context.cython_scope.lookup(attribute):
+            elif self.context.cython_scope.lookup_qualified_name(attribute):
                 pass
             else:
                 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
@@ -2284,11 +2284,9 @@ class TransformBuiltinMethods(EnvTransform):
                     node.cdivision = True
             elif function == u'set':
                 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
+            elif self.context.cython_scope.lookup_qualified_name(function):
+                pass
             else:
-                entry = self.context.cython_scope.lookup(function)
-                if entry and entry.utility_code_definition:
-                    self.env_stack[0].use_utility_code(entry.utility_code_definition)
-                if not entry:
                 error(node.function.pos,
                       u"'%s' not a valid cython language construct" % function)
 

Cython/Compiler/Pipeline.py

@@ -60,16 +60,30 @@ def inject_pxd_code_stage_factory(context):
         return module_node
     return inject_pxd_code_stage
 
-def inject_utility_code_stage(module_node):
+def use_utility_code_definitions(scope, target):
+    for entry in scope.entries.itervalues():
+        if entry.used and entry.utility_code_definition:
+            target.use_utility_code(entry.utility_code_definition)
+        elif entry.as_module:
+            use_utility_code_definitions(entry.as_module, target)
+
+def inject_utility_code_stage_factory(context):
+    def inject_utility_code_stage(module_node):
+        # First, make sure any utility code pulled in by using symbols in the cython
+        # scope is included
+        use_utility_code_definitions(context.cython_scope, module_node.scope)
+        
         added = []
-    # need to copy list as the list will be altered!
-    for utilcode in module_node.scope.utility_code_list[:]:
+        # Note: the list might be extended inside the loop (if some utility code
+        # pulls in other utility code)
+        for utilcode in module_node.scope.utility_code_list:
             if utilcode in added: continue
             added.append(utilcode)
             tree = utilcode.get_tree()
             if tree:
                 module_node.merge_in(tree.body, tree.scope, merge_scope=True)
         return module_node
+    return inject_utility_code_stage
 
 #
 # Pipeline factories
@@ -109,6 +123,9 @@ def create_pipeline(context, mode, exclude_classes=()):
     else:
         _align_function_definitions = None
 
+    # NOTE: This is the "common" parts of the pipeline, which is also
+    # code in pxd files. So it will be run multiple times in a
+    # compilation stage.
     stages = [
         NormalizeTree(context),
         PostParse(context),
@@ -175,7 +192,7 @@ def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()):
         create_pipeline(context, mode, exclude_classes=exclude_classes),
         test_support,
         [inject_pxd_code_stage_factory(context),
-         inject_utility_code_stage,
+         inject_utility_code_stage_factory(context),
          abort_on_errors],
         debug_transform,
         [generate_pyx_code_stage_factory(options, result)]))

Cython/Compiler/Symtab.py

@@ -892,9 +892,11 @@ class ModuleScope(Scope):
     # types_imported       {PyrexType : 1}    Set of types for which import code generated
     # has_import_star      boolean            Module contains import *
     # cpp                  boolean            Compiling a C++ file
+    # is_cython_builtin    boolean            Is this the Cython builtin scope (or a child scope)
 
     is_module_scope = 1
     has_import_star = 0
+    is_cython_builtin = 0
 
     def __init__(self, name, parent_module, context):
         import Builtin

tests/run/cythonscope.pyx

+"""
+>>> f()
+hello from cython scope, value=4
+hello from cython.view scope, value=4
+hello from cython scope, value=3
+hello from cython.view scope, value=3
+"""
+
+cimport cython
+
+from cython cimport _testscope as tester
+from cython.view cimport _testscope as viewtester
+
+
+def f():
+    print cython._testscope(4)
+    print cython.view._testscope(4)
+    print tester(3)
+    print viewtester(3)