major cleanup refactoring of the utility code loading support,

simple support for loading requirements (at least from the same utility code type and file),
caching support

Cython/Compiler/Buffer.py

@@ -748,8 +748,11 @@ def get_type_information_cname(code, dtype, maxdepth=None):
                         ), safe=True)
     return name
 
-def load_buffer_utility(util_code_name, **kwargs):
-    return UtilityCode.load(util_code_name, "Buffer.c", **kwargs)
+def load_buffer_utility(util_code_name, context=None, **kwargs):
+    if context is None:
+        return UtilityCode.load(util_code_name, "Buffer.c", **kwargs)
+    else:
+        return TempitaUtilityCode.load(util_code_name, "Buffer.c", context=context, **kwargs)
 
 context = dict(max_dims=str(Options.buffer_max_dims))
 buffer_struct_declare_code = load_buffer_utility("BufferStructDeclare",

Cython/Compiler/ExprNodes.py

@@ -4,7 +4,8 @@
 
 import cython
 cython.declare(error=object, warning=object, warn_once=object, InternalError=object,
-               CompileError=object, UtilityCode=object, StringEncoding=object, operator=object,
+               CompileError=object, UtilityCode=object, TempitaUtilityCode=object,
+               StringEncoding=object, operator=object,
                Naming=object, Nodes=object, PyrexTypes=object, py_object_type=object,
                list_type=object, tuple_type=object, set_type=object, dict_type=object, \
                unicode_type=object, str_type=object, bytes_type=object, type_type=object,
@@ -15,7 +16,7 @@ import operator
 
 from Errors import error, warning, warn_once, InternalError, CompileError
 from Errors import hold_errors, release_errors, held_errors, report_error
-from Code import UtilityCode
+from Code import UtilityCode, TempitaUtilityCode
 import StringEncoding
 import Naming
 import Nodes
@@ -9967,9 +9968,9 @@ proto="""
         (((x) < 0) & ((unsigned long)(x) == 0-(unsigned long)(x)))
 """)
 
-binding_cfunc_utility_code = UtilityCode.load("CythonFunction",
-                                              context=vars(Naming))
-fused_function_utility_code = UtilityCode.load(
+binding_cfunc_utility_code = TempitaUtilityCode.load(
+    "CythonFunction", context=vars(Naming))
+fused_function_utility_code = TempitaUtilityCode.load(
         "FusedFunction",
         "CythonFunction.c",
         context=vars(Naming),

Cython/Compiler/MemoryView.py

@@ -2,7 +2,7 @@ from Errors import CompileError, error
 import ExprNodes
 from ExprNodes import IntNode, NameNode, AttributeNode
 import Options
-from Code import UtilityCode
+from Code import UtilityCode, TempitaUtilityCode
 from UtilityCode import CythonUtilityCode
 import Buffer
 import PyrexTypes
@@ -931,8 +931,11 @@ def load_memview_cy_utility(util_code_name, context=None, **kwargs):
                                   context=context, **kwargs)
 
 def load_memview_c_utility(util_code_name, context=None, **kwargs):
-    return UtilityCode.load(util_code_name, "MemoryView_C.c",
-                            context=context, **kwargs)
+    if context is None:
+        return UtilityCode.load(util_code_name, "MemoryView_C.c", **kwargs)
+    else:
+        return TempitaUtilityCode.load(util_code_name, "MemoryView_C.c",
+                                       context=context, **kwargs)
 
 def use_cython_array_utility_code(env):
     env.global_scope().context.cython_scope.lookup('array_cwrapper').used = True

Cython/Compiler/Optimize.py

@@ -15,7 +15,7 @@ import Symtab
 import Options
 import Naming
 
-from Code import UtilityCode, ContentHashingUtilityCode
+from Code import UtilityCode
 from StringEncoding import EncodedString, BytesLiteral
 from Errors import error
 from ParseTreeTransforms import SkipDeclarations
@@ -32,11 +32,8 @@ try:
 except ImportError:
     basestring = str # Python 3
 
-_utility_cache = {}
 def load_c_utility(name):
-    if name not in _utility_cache:
-      _utility_cache[name] = ContentHashingUtilityCode.load(name, "Optimize.c")
-    return _utility_cache[name]
+    return UtilityCode.load_cached(name, "Optimize.c")
 
 class FakePythonEnv(object):
     "A fake environment for creating type test nodes etc."

Cython/Compiler/PyrexTypes.py

@@ -2,7 +2,7 @@
 #   Cython/Python language types
 #
 
-from Code import UtilityCode, LazyUtilityCode, ContentHashingUtilityCode
+from Code import UtilityCode, LazyUtilityCode, TempitaUtilityCode
 import StringEncoding
 import Naming
 import copy
@@ -610,14 +610,14 @@ class MemoryViewSliceType(PyrexType):
         return cname + '.memview'
 
     def create_from_py_utility_code(self, env):
-        import MemoryView, Buffer, Code
+        import MemoryView, Buffer
 
         # We don't have 'code', so use a LazyUtilityCode with a callback.
         def lazy_utility_callback(code):
             context['dtype_typeinfo'] = Buffer.get_type_information_cname(
                                                           code, self.dtype)
-            return ContentHashingUtilityCode.load(
-                        "ObjectToMemviewSlice", "MemoryView_C.c", context)
+            return TempitaUtilityCode.load(
+                        "ObjectToMemviewSlice", "MemoryView_C.c", context=context)
 
         env.use_utility_code(Buffer.acquire_utility_code)
         env.use_utility_code(MemoryView.memviewslice_init_code)
@@ -693,7 +693,7 @@ class MemoryViewSliceType(PyrexType):
                 error_condition = error_condition,
             )
 
-        utility = ContentHashingUtilityCode.load(
+        utility = TempitaUtilityCode.load(
                         utility_name, "MemoryView_C.c", context=context)
         env.use_utility_code(utility)
         return get_function, set_function
@@ -2796,8 +2796,8 @@ class CStructOrUnionType(CType):
                 funcname = self.from_py_function,
                 init = '%s 0 %s' % ('{' * nesting_depth, '}' * nesting_depth)
             )
-            self._convert_from_py_code = ContentHashingUtilityCode.load(
-                      "FromPyStructUtility", "TypeConversion.c", context)
+            self._convert_from_py_code = TempitaUtilityCode.load(
+                      "FromPyStructUtility", "TypeConversion.c", context=context)
 
         env.use_utility_code(self._convert_from_py_code)
         return True

Cython/Compiler/Tests/TestUtilityLoad.py

@@ -12,10 +12,8 @@ class TestUtilityLoader(unittest.TestCase):
     """
 
     expected = "test {{loader}} prototype", "test {{loader}} impl"
-    expected_tempita = (expected[0].replace('{{loader}}', 'Loader'),
-                        expected[1].replace('{{loader}}', 'Loader'))
 
-    required = "I am a dependency proto", "I am a dependency impl"
+    required = "req {{loader}} proto", "req {{loader}} impl"
 
     context = dict(loader='Loader')
 
@@ -30,24 +28,55 @@ class TestUtilityLoader(unittest.TestCase):
         got = strip_2tup(self.cls.load_as_string(self.name, self.filename))
         self.assertEquals(got, self.expected)
 
-        got = strip_2tup(self.cls.load_as_string(self.name, context=self.context))
-        self.assertEquals(got, self.expected_tempita)
-
     def test_load(self):
         utility = self.cls.load(self.name)
         got = strip_2tup((utility.proto, utility.impl))
         self.assertEquals(got, self.expected)
 
-        # Not implemented yet
-        #required, = utility.requires
-        #self.assertEquals((required.proto, required.impl), self.required)
+        required, = utility.requires
+        got = strip_2tup((required.proto, required.impl))
+        self.assertEquals(got, self.required)
 
         utility = self.cls.load(self.name, from_file=self.filename)
         got = strip_2tup((utility.proto, utility.impl))
         self.assertEquals(got, self.expected)
 
+        utility = self.cls.load_cached(self.name, from_file=self.filename)
+        got = strip_2tup((utility.proto, utility.impl))
+        self.assertEquals(got, self.expected)
+
 
-class TestCythonUtilityLoader(TestUtilityLoader):
+class TestTempitaUtilityLoader(TestUtilityLoader):
+    """
+    Test loading UtilityCodes with Tempita substitution
+    """
+    expected_tempita = (TestUtilityLoader.expected[0].replace('{{loader}}', 'Loader'),
+                        TestUtilityLoader.expected[1].replace('{{loader}}', 'Loader'))
+
+    required_tempita = (TestUtilityLoader.required[0].replace('{{loader}}', 'Loader'),
+                        TestUtilityLoader.required[1].replace('{{loader}}', 'Loader'))
+
+    cls = Code.TempitaUtilityCode
+
+    def test_load_as_string(self):
+        got = strip_2tup(self.cls.load_as_string(self.name, context=self.context))
+        self.assertEquals(got, self.expected_tempita)
+
+    def test_load(self):
+        utility = self.cls.load(self.name, context=self.context)
+        got = strip_2tup((utility.proto, utility.impl))
+        self.assertEquals(got, self.expected_tempita)
+
+        required, = utility.requires
+        got = strip_2tup((required.proto, required.impl))
+        self.assertEquals(got, self.required_tempita)
+
+        utility = self.cls.load(self.name, from_file=self.filename, context=self.context)
+        got = strip_2tup((utility.proto, utility.impl))
+        self.assertEquals(got, self.expected_tempita)
+
+
+class TestCythonUtilityLoader(TestTempitaUtilityLoader):
     """
     Test loading CythonUtilityCodes
     """
@@ -56,7 +85,8 @@ class TestCythonUtilityLoader(TestUtilityLoader):
     expected = None, "test {{cy_loader}} impl"
     expected_tempita = None, "test CyLoader impl"
 
-    required = None, "I am a Cython dependency impl"
+    required = None, "req {{cy_loader}} impl"
+    required_tempita = None, "req CyLoader impl"
 
     context = dict(cy_loader='CyLoader')
 
@@ -65,4 +95,7 @@ class TestCythonUtilityLoader(TestUtilityLoader):
     cls = UtilityCode.CythonUtilityCode
 
     # Small hack to pass our tests above
-    cls.proto = None
\ No newline at end of file
+    cls.proto = None
+
+    test_load = TestUtilityLoader.test_load
+    test_load_tempita = TestTempitaUtilityLoader.test_load

Cython/Compiler/UtilityCode.py

@@ -66,13 +66,15 @@ class CythonUtilityCode(Code.UtilityCodeBase):
     is_cython_utility = True
 
     def __init__(self, impl, name="__pyxutil", prefix="", requires=None,
-                 file=None, from_scope=None):
+                 file=None, from_scope=None, context=None):
         # 1) We need to delay the parsing/processing, so that all modules can be
         #    imported without import loops
         # 2) The same utility code object can be used for multiple source files;
         #    while the generated node trees can be altered in the compilation of a
         #    single file.
         # Hence, delay any processing until later.
+        if context is not None:
+            impl = Code.sub_tempita(impl, context, file, name)
         self.impl = impl
         self.name = name
         self.file = file
@@ -128,6 +130,14 @@ class CythonUtilityCode(Code.UtilityCodeBase):
     def put_code(self, output):
         pass
 
+    @classmethod
+    def load_as_string(cls, util_code_name, from_file=None, **kwargs):
+        """
+        Load a utility code as a string. Returns (proto, implementation)
+        """
+        util = cls.load(util_code_name, from_file, **kwargs)
+        return util.proto, util.impl # keep line numbers => no lstrip()
+
     def declare_in_scope(self, dest_scope, used=False, cython_scope=None):
         """
         Declare all entries from the utility code in dest_scope. Code will only

Cython/Utility/TestCyUtilityLoader.pyx

 ########## TestCyUtilityLoader ##########
+#@requires: OtherUtility
+
 test {{cy_loader}} impl
+
+
+########## OtherUtility ##########
+req {{cy_loader}} impl

Cython/Utility/TestUtilityLoader.c

@@ -2,4 +2,11 @@
 test {{loader}} prototype
 
 ////////// TestUtilityLoader //////////
+//@requires: OtherUtility
 test {{loader}} impl
+
+////////// OtherUtility.proto //////////
+req {{loader}} proto
+
+////////// OtherUtility //////////
+req {{loader}} impl