reuse existing types/names in module for cfunc wrapping instead of redeclaring...

reuse existing types/names in module for cfunc wrapping instead of redeclaring them in the utility code

Cython/Compiler/PyrexTypes.py

@@ -2659,94 +2659,56 @@ class CFuncType(CType):
         for arg in self.args:
             if not arg.type.is_pyobject and not arg.type.create_from_py_utility_code(env):
                 return False
-            if arg.type.is_extension_type:
-                env.use_utility_code(UtilityCode.load_cached("ExtTypeTest", "ObjectHandling.c"))
+        if not (self.return_type.is_pyobject or self.return_type.is_void or
+                self.return_type.create_to_py_utility_code(env)):
+            return False
 
-        def declared_type(ctype, type_name):
-            py_arg_type = ''
-            type_convert = ''
-            type_cname = ctype.declaration_code("")
+        def declared_type(ctype):
             type_displayname = str(ctype.declaration_code("", for_display=True))
+            if ctype.is_pyobject:
+                arg_ctype = type_name = type_displayname
                 if ctype.is_builtin_type:
-                py_arg_type = ctype.name
-            elif ctype.is_extension_type:
-                type_convert = '<%s>' % type_name
-            elif ctype.is_pyobject:
-                type_convert = ''
+                    arg_ctype = ctype.name
+                elif not ctype.is_extension_type:
+                    type_name = 'object'
+                    type_displayname = None
+                else:
+                    type_displayname = repr(type_displayname)
             elif ctype is c_bint_type:
-                type_name = py_arg_type = 'bint'
-            elif ctype.is_typedef or ctype.is_struct_or_union:
-                type_convert = '%s_to_py' % type_name
+                type_name = arg_ctype = 'bint'
+            else:
+                type_name = arg_ctype = type_displayname
+                if ctype is c_double_type:
+                    type_displayname = 'float'
                 else:
-                type_name = py_arg_type = type_displayname
-            return type_name, type_cname, py_arg_type, type_displayname, type_convert
+                    type_displayname = repr(type_displayname)
+            return type_name, arg_ctype, type_displayname
 
         class Arg(object):
-            def __init__(self, ix, arg):
-                self.ix = ix
-                self.name = arg.name or 'ARG%s' % ix
-                self.type = arg.type
-                self.type_name, self.type_cname, self.py_arg_type, self.type_displayname, self.type_convert = (
-                    declared_type(self.type, 'TYPE%s' % ix))
-
-            def declare_type_def(self):
-                if self.type.is_extension_type or (not self.type.is_pyobject and not self.py_arg_type):
-                    return 'ctypedef void* %s "%s"' % (self.type_name, self.type_cname)
-
-            def declare_type_convert(self):
-                if self.type.is_extension_type:
-                    return 'cdef PyTypeObject* %s_TYPE "%s"' % (self.type_name, self.type.typeptr_cname)
-                elif self.type.is_pyobject:
-                    return ''
-                elif not self.py_arg_type:
-                    return 'cdef %s %s "%s"(object) except *' % (
-                        self.type_name, self.type_convert, self.type.from_py_function)
-
-            def check_type(self):
-                if self.type.is_extension_type:
-                    return '__Pyx_TypeTest(<PyObject*>%s, %s_TYPE)' % (self.name, self.type_name)
+            def __init__(self, arg_name, arg_type):
+                self.name = arg_name
+                self.type = arg_type
+                self.type_cname, self.ctype, self.type_displayname = declared_type(arg_type)
 
         if self.return_type.is_void:
-            return_type = 'void'
-            declare_return_type = ''
-            declare_return_type_convert = ''
-            return_type_displayname = ''
-            maybe_return = ''
             except_clause = 'except *'
-        else:
-            return_type, return_type_cname, return_arg_type, return_type_displayname, _ = (
-                declared_type(self.return_type, 'RETURN_TYPE'))
-            if self.return_type.is_pyobject:
-                to_py = '(PyObject*)'
+        elif self.return_type.is_pyobject:
             except_clause = ''
+        elif self.exception_value:
+            except_clause = ('except? %s' if self.exception_check else 'except %s') % self.exception_value
         else:
-                if not self.return_type.create_to_py_utility_code(env):
-                    return False
-                to_py = self.return_type.to_py_function
             except_clause = 'except *'
-            if return_arg_type:
-                return_type = return_arg_type
-                declare_return_type = ''
-                declare_return_type_convert = ''
-                maybe_return = 'return '
-            else:
-                declare_return_type = 'ctypedef void* RETURN_TYPE "%s"' % return_type_cname
-                declare_return_type_convert = 'cdef object RETURN_TYPE_from_py "%s" (RETURN_TYPE)' % to_py
-                maybe_return = 'return RETURN_TYPE_from_py'
 
         context = {
             'cname': self.to_py_function,
-            'args': [Arg(ix, arg) for ix, arg in enumerate(self.args)],
-            'return_type': return_type,
-            'return_type_displayname': return_type_displayname,
-            'declare_return_type': declare_return_type,
-            'declare_return_type_convert': declare_return_type_convert,
-            'maybe_return': maybe_return,
+            'args': [Arg(arg.name, arg.type) for arg in self.args],
+            'return_type': Arg('return', self.return_type),
             'except_clause': except_clause,
         }
         # FIXME: directives come from first defining environment and do not adapt for reuse
         env.use_utility_code(CythonUtilityCode.load(
             "cfunc.to_py", "CFuncConvert.pyx",
+            outer_module_scope=env.global_scope(),  # need access to types declared in module
             context=context, compiler_directives=dict(env.directives)))
         return True
 

Cython/Compiler/Symtab.py

@@ -1041,15 +1041,13 @@ class ModuleScope(Scope):
     def global_scope(self):
         return self
 
-    def lookup(self, name):
+    def lookup(self, name, language_level=None):
         entry = self.lookup_here(name)
         if entry is not None:
             return entry
 
-        if self.context is not None:
-            language_level = self.context.language_level
-        else:
-            language_level = 3
+        if language_level is None:
+            language_level = self.context.language_level if self.context is not None else 3
 
         return self.outer_scope.lookup(name, language_level=language_level)
 

Cython/Compiler/UtilityCode.py

@@ -68,7 +68,8 @@ class CythonUtilityCode(Code.UtilityCodeBase):
     is_cython_utility = True
 
     def __init__(self, impl, name="__pyxutil", prefix="", requires=None,
-                 file=None, from_scope=None, context=None, compiler_directives=None):
+                 file=None, from_scope=None, context=None, compiler_directives=None,
+                 outer_module_scope=None):
         # 1) We need to delay the parsing/processing, so that all modules can be
         #    imported without import loops
         # 2) The same utility code object can be used for multiple source files;
@@ -83,6 +84,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
         self.prefix = prefix
         self.requires = requires or []
         self.from_scope = from_scope
+        self.outer_module_scope = outer_module_scope
         self.compiler_directives = compiler_directives
 
     def get_tree(self, entries_only=False, cython_scope=None):
@@ -127,6 +129,16 @@ class CythonUtilityCode(Code.UtilityCodeBase):
             pipeline = Pipeline.insert_into_pipeline(pipeline, scope_transform,
                                                      before=transform)
 
+        if self.outer_module_scope:
+            # inject outer module between utility code module and builtin module
+            def scope_transform(module_node):
+                module_node.scope.outer_scope = self.outer_module_scope
+                return module_node
+
+            transform = ParseTreeTransforms.AnalyseDeclarationsTransform
+            pipeline = Pipeline.insert_into_pipeline(pipeline, scope_transform,
+                                                     before=transform)
+
         (err, tree) = Pipeline.run_pipeline(pipeline, tree, printtree=False)
         assert not err, err
         return tree

Cython/Utility/CFuncConvert.pyx

 #################### cfunc.to_py ####################
 
-cdef extern from *:
-    ctypedef struct PyObject
-    ctypedef struct PyTypeObject
-    cdef bint __Pyx_TypeTest(PyObject*, PyTypeObject* type) except 0
-  {{for arg in args}}
-    {{arg.declare_type_def()}}
-    {{arg.declare_type_convert()}}
-  {{endfor}}
-    {{declare_return_type}}
-    {{declare_return_type_convert}}
-
 @cname("{{cname}}")
-cdef object {{cname}}({{return_type}} (*f)({{ ', '.join(arg.type_name for arg in args) }}) {{except_clause}}):
-    def wrap({{ ', '.join('{arg.py_arg_type} {arg.name}'.format(arg=arg) for arg in args) }}):
-        """wrap({{', '.join('{arg.name}: {arg.type_displayname!r}'.format(arg=arg) for arg in args)}}){{if return_type_displayname}} -> {{repr(return_type_displayname)}}{{endif}}"""
-        {{for arg in args}}
-        {{arg.check_type()}}
-        {{endfor}}
-        {{maybe_return}}(f({{ ', '.join('%s(%s)' % (arg.type_convert, arg.name) for arg in args) }}))
+cdef object {{cname}}({{return_type.ctype}} (*f)({{ ', '.join(arg.type_cname for arg in args) }}) {{except_clause}}):
+    def wrap({{ ', '.join('{arg.ctype} {arg.name}'.format(arg=arg) for arg in args) }}):
+        """wrap({{', '.join(('{arg.name}: {arg.type_displayname}'.format(arg=arg) if arg.type_displayname else arg.name) for arg in args)}}){{if return_type.type_displayname}} -> {{return_type.type_displayname}}{{endif}}"""
+        {{'' if return_type.type.is_void else 'return '}}f({{ ', '.join(arg.name for arg in args) }})
     return wrap

tests/run/cfunc_convert.pyx

@@ -38,7 +38,7 @@ def return_square_c():
     >>> square_c(x=4)
     16.0
     >>> square_c.__doc__   # FIXME: try to make original C function name available
-    "wrap(x: 'double') -> 'double'"
+    'wrap(x: float) -> float'
     """
     return square_c
 
@@ -51,7 +51,7 @@ def return_libc_sqrt():
     >>> sqrt(x=9)
     3.0
     >>> sqrt.__doc__
-    "wrap(x: 'double') -> 'double'"
+    'wrap(x: float) -> float'
     """
     return sqrt
 
@@ -96,7 +96,7 @@ def return_abc():
     >>> abc(2, 3, 5)
     False
     >>> abc.__doc__
-    "wrap(a: 'long long', b: 'long long', c: 'long long') -> 'bool'"
+    "wrap(a: 'long long', b: 'long long', c: 'long long') -> bool"
     """
     return abc
 
@@ -144,7 +144,7 @@ def return_struct_builder():
     >>> d['y']['a']
     1
     >>> make.__doc__
-    "wrap(which: 'int', a: 'int', b: 'double') -> 'my_struct'"
+    "wrap(which: 'int', a: 'int', b: float) -> 'my_struct'"
     """
     return c_struct_builder
 
@@ -174,6 +174,16 @@ def test_builtin_params(a, b):
     """
     return (<object>test_builtin_params_cfunc)(a, b)
 
+def return_builtin_params_cfunc():
+    """
+    >>> cfunc = return_builtin_params_cfunc()
+    >>> cfunc([1, 2], {'a': 3})
+    ([1, 2], {'a': 3})
+    >>> cfunc.__doc__
+    'wrap(a: list, b: dict) -> tuple'
+    """
+    return test_builtin_params_cfunc
+
 
 cdef class A:
     def __repr__(self):
@@ -192,6 +202,6 @@ def test_cdef_class_params(a, b):
     >>> test_cdef_class_params(B(), A())
     Traceback (most recent call last):
     ...
-    TypeError: Cannot convert cfunc_convert.A to cfunc_convert.B
+    TypeError: Argument 'b' has incorrect type (expected cfunc_convert.B, got cfunc_convert.A)
     """
     return (<object>test_cdef_class_params_cfunc)(a, b)