Merge pull request #70 from markflorisson88/fusedsuper

Fused classmethods & staticmethods & super

Cython/Compiler/ExprNodes.py

@@ -5766,6 +5766,59 @@ class PyClassNamespaceNode(ExprNode, ModuleNameMixin):
                 code.error_goto_if_null(self.result(), self.pos)))
         code.put_gotref(self.py_result())
 
+
+class ClassCellInjectorNode(ExprNode):
+    # Initialize CyFunction.func_classobj
+    is_temp = True
+    type = py_object_type
+    subexprs = []
+    is_active = False
+
+    def analyse_expressions(self, env):
+        if self.is_active:
+            env.use_utility_code(cyfunction_class_cell_utility_code)
+
+    def generate_evaluation_code(self, code):
+        if self.is_active:
+            self.allocate_temp_result(code)
+            code.putln(
+                '%s = PyList_New(0); %s' % (
+                    self.result(),
+                    code.error_goto_if_null(self.result(), self.pos)))
+            code.put_gotref(self.result())
+
+    def generate_injection_code(self, code, classobj_cname):
+        if self.is_active:
+            code.putln('__Pyx_CyFunction_InitClassCell(%s, %s);' % (
+                self.result(), classobj_cname))
+
+
+class ClassCellNode(ExprNode):
+    # Class Cell for noargs super()
+    subexprs = []
+    is_temp = True
+    is_generator = False
+    type = py_object_type
+
+    def analyse_types(self, env):
+        pass
+
+    def generate_result_code(self, code):
+        if not self.is_generator:
+            code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
+                self.result(),
+                Naming.self_cname))
+        else:
+            code.putln('%s =  %s->classobj;' % (
+                self.result(), Naming.cur_scope_cname))
+        code.putln(
+            'if (!%s) { PyErr_SetString(PyExc_SystemError, '
+            '"super(): empty __class__ cell"); %s }' % (
+                self.result(),
+                code.error_goto(self.pos)));
+        code.put_incref(self.result(), py_object_type)
+
+
 class BoundMethodNode(ExprNode):
     #  Helper class used in the implementation of Python
     #  class definitions. Constructs an bound method
@@ -5832,6 +5885,7 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin):
     #  pymethdef_cname   string             PyMethodDef structure
     #  self_object       ExprNode or None
     #  binding           bool
+    #  def_node          DefNode            the Python function node
     #  module_name       EncodedString      Name of defining module
     #  code_object       CodeObjectNode     the PyCodeObject creator node
 
@@ -5840,6 +5894,7 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin):
     self_object = None
     code_object = None
     binding = False
+    def_node = None
 
     type = py_object_type
     is_temp = 1
@@ -5873,25 +5928,52 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin):
 
     def generate_result_code(self, code):
         if self.binding:
+            self.generate_cyfunction_code(code)
+        else:
+            self.generate_pycfunction_code(code)
+
+    def generate_pycfunction_code(self, code):
+        py_mod_name = self.get_py_mod_name(code)
+        code.putln(
+            '%s = PyCFunction_NewEx(&%s, %s, %s); %s' % (
+                self.result(),
+                self.pymethdef_cname,
+                self.self_result_code(),
+                py_mod_name,
+                code.error_goto_if_null(self.result(), self.pos)))
+
+        code.put_gotref(self.py_result())
+
+    def generate_cyfunction_code(self, code):
         if self.specialized_cpdefs:
             constructor = "__pyx_FusedFunction_NewEx"
+            def_node = self.specialized_cpdefs[0]
         else:
             constructor = "__Pyx_CyFunction_NewEx"
+            def_node = self.def_node
 
         if self.code_object:
-                code_object_result = ', ' + self.code_object.py_result()
+            code_object_result = self.code_object.py_result()
         else:
-                code_object_result = ', NULL'
+            code_object_result = 'NULL'
+
+        flags = []
+        if def_node.is_staticmethod:
+            flags.append('__Pyx_CYFUNCTION_STATICMETHOD')
+        elif def_node.is_classmethod:
+            flags.append('__Pyx_CYFUNCTION_CLASSMETHOD')
+        if flags:
+            flags = ' | '.join(flags)
         else:
-            constructor = "PyCFunction_NewEx"
-            code_object_result = ''
+            flags = '0'
 
         py_mod_name = self.get_py_mod_name(code)
         code.putln(
-            '%s = %s(&%s, %s, %s%s); %s' % (
+            '%s = %s(&%s, %s, %s, %s, %s); %s' % (
                 self.result(),
                 constructor,
                 self.pymethdef_cname,
+                flags,
                 self.self_result_code(),
                 py_mod_name,
                 code_object_result,
@@ -5899,10 +5981,20 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin):
 
         code.put_gotref(self.py_result())
 
+        if def_node.requires_classobj:
+            assert code.pyclass_stack, "pyclass_stack is empty"
+            class_node = code.pyclass_stack[-1]
+            code.put_incref(self.py_result(), py_object_type)
+            code.putln(
+                'PyList_Append(%s, %s);' % (
+                    class_node.class_cell.result(),
+                    self.result()))
+            code.put_giveref(self.py_result())
+
         if self.specialized_cpdefs:
-            self.generate_fused_cpdef(code, code_object_result)
+            self.generate_fused_cpdef(code, code_object_result, flags)
 
-    def generate_fused_cpdef(self, code, code_object_result):
+    def generate_fused_cpdef(self, code, code_object_result, flags):
         """
         Generate binding function objects for all specialized cpdefs, and the
         original fused one. The fused function gets a dict __signatures__
@@ -5924,6 +6016,7 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin):
             py_mod_name=self.get_py_mod_name(code),
             self=self.self_result_code(),
             code=code_object_result,
+            flags=flags,
             func=code.funcstate.allocate_temp(py_object_type,
                                               manage_ref=True),
             signature=code.funcstate.allocate_temp(py_object_type,
@@ -5945,7 +6038,7 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin):
                                     '"%(signature_string)s")')
 
             goto_err("%(func)s = __pyx_FusedFunction_NewEx("
-                            "&%(pymethdef_cname)s, %(self)s, %(py_mod_name)s %(code)s)")
+                            "&%(pymethdef_cname)s, %(flags)s, %(self)s, %(py_mod_name)s, %(code)s)")
 
             s = "PyDict_SetItem(%(sigdict)s, %(signature)s, %(func)s)"
             code.put_error_if_neg(self.pos, s % fmt_dict)
@@ -6040,13 +6133,13 @@ class LambdaNode(InnerFunctionNode):
 
     child_attrs = ['def_node']
 
-    def_node = None
     name = StringEncoding.EncodedString('<lambda>')
 
     def analyse_declarations(self, env):
         self.def_node.no_assignment_synthesis = True
         self.def_node.pymethdef_required = True
         self.def_node.analyse_declarations(env)
+        self.def_node.is_cyfunction = True
         self.pymethdef_cname = self.def_node.entry.pymethdef_cname
         env.add_lambda_def(self.def_node)
 
@@ -6074,6 +6167,7 @@ class GeneratorExpressionNode(LambdaNode):
         super(GeneratorExpressionNode, self).analyse_declarations(env)
         # No pymethdef required
         self.def_node.pymethdef_required = False
+        self.def_node.is_cyfunction = False
         # Force genexpr signature
         self.def_node.entry.signature = TypeSlots.pyfunction_noargs
 
@@ -9858,7 +9952,10 @@ fused_function_utility_code = UtilityCode.load(
         "CythonFunction.c",
         context=vars(Naming),
         requires=[binding_cfunc_utility_code])
-
+cyfunction_class_cell_utility_code = UtilityCode.load(
+    "CyFunctionClassCell",
+    "CythonFunction.c",
+    requires=[binding_cfunc_utility_code])
 
 generator_utility_code = UtilityCode(
 proto="""

Cython/Compiler/Nodes.py

@@ -1302,6 +1302,7 @@ class FuncDefNode(StatNode, BlockNode):
     has_fused_arguments = False
     star_arg = None
     starstar_arg = None
+    is_cyfunction = False
 
     def analyse_default_values(self, env):
         genv = env.global_scope()
@@ -1510,7 +1511,13 @@ class FuncDefNode(StatNode, BlockNode):
             code.put_gotref(Naming.cur_scope_cname)
             # Note that it is unsafe to decref the scope at this point.
         if self.needs_outer_scope:
-            code.putln("%s = (%s)%s;" % (
+            if self.is_cyfunction:
+                code.putln("%s = (%s) __Pyx_CyFunction_GetClosure(%s);" % (
+                    outer_scope_cname,
+                    cenv.scope_class.type.declaration_code(''),
+                    Naming.self_cname))
+            else:
+                code.putln("%s = (%s) %s;" % (
                     outer_scope_cname,
                     cenv.scope_class.type.declaration_code(''),
                     Naming.self_cname))
@@ -2636,6 +2643,7 @@ class DefNode(FuncDefNode):
     acquire_gil = 0
     self_in_stararg = 0
     py_cfunc_node = None
+    requires_classobj = False
     doc = None
 
     fused_py_func = False
@@ -2841,6 +2849,15 @@ class DefNode(FuncDefNode):
             self.self_in_stararg = 1
             nfixed = 0
 
+        if self.is_staticmethod and env.is_c_class_scope:
+            nfixed = 0
+            self.self_in_stararg = True
+
+            self.entry.signature = sig = copy.copy(sig)
+            sig.fixed_arg_format = "*"
+            sig.is_staticmethod = True
+            sig.has_generic_args = True
+
         for i in range(min(nfixed, len(self.args))):
             arg = self.args[i]
             arg.is_generic = 0
@@ -2977,7 +2994,7 @@ class DefNode(FuncDefNode):
                 decorator.decorator.analyse_expressions(env)
 
     def needs_assignment_synthesis(self, env, code=None):
-        if self.specialized_cpdefs:
+        if self.specialized_cpdefs or self.is_staticmethod:
             return True
         if self.no_assignment_synthesis:
             return False
@@ -3003,21 +3020,22 @@ class DefNode(FuncDefNode):
 
         if genv.is_closure_scope:
             rhs = self.py_cfunc_node = ExprNodes.InnerFunctionNode(
-                self.pos, pymethdef_cname = self.entry.pymethdef_cname,
-                code_object = ExprNodes.CodeObjectNode(self))
+                self.pos, def_node=self,
+                pymethdef_cname=self.entry.pymethdef_cname,
+                code_object=ExprNodes.CodeObjectNode(self))
         else:
             rhs = ExprNodes.PyCFunctionNode(
                 self.pos,
+                def_node=self,
                 pymethdef_cname=self.entry.pymethdef_cname,
                 binding=env.directives['binding'],
                 specialized_cpdefs=self.specialized_cpdefs,
                 code_object=ExprNodes.CodeObjectNode(self))
 
         if env.is_py_class_scope:
-            if not self.is_staticmethod and not self.is_classmethod:
             rhs.binding = True
-            else:
-                rhs.binding = False
+
+        self.is_cyfunction = rhs.binding
 
         if self.decorators:
             for decorator in self.decorators[::-1]:
@@ -3754,6 +3772,12 @@ class GeneratorDefNode(DefNode):
         code.putln('%s.resume_label = 0;' % generator_cname)
         code.putln('%s.body = (__pyx_generator_body_t) %s;' % (generator_cname, body_cname))
         code.put_giveref(Naming.cur_scope_cname)
+        if self.requires_classobj:
+            classobj_cname = '%s->classobj' % Naming.cur_scope_cname
+            code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
+                classobj_cname, Naming.self_cname))
+            code.put_incref(classobj_cname, py_object_type)
+            code.put_giveref(classobj_cname)
         code.put_finish_refcount_context()
         code.putln("return (PyObject *) %s;" % Naming.cur_scope_cname);
 
@@ -3949,7 +3973,8 @@ class PyClassDefNode(ClassDefNode):
     #  classobj ClassNode  Class object
     #  target   NameNode   Variable to assign class object to
 
-    child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "class_result", "target"]
+    child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "class_result",
+                   "target", "class_cell"]
     decorators = None
     class_result = None
     py3_style_class = False # Python3 style class (bases+kwargs)
@@ -4006,6 +4031,7 @@ class PyClassDefNode(ClassDefNode):
             self.classobj = ExprNodes.ClassNode(pos, name = name,
                     bases = bases, dict = self.dict, doc = doc_node)
         self.target = ExprNodes.NameNode(pos, name = name)
+        self.class_cell = ExprNodes.ClassCellInjectorNode(self.pos)
 
     def as_cclass(self):
         """
@@ -4084,6 +4110,7 @@ class PyClassDefNode(ClassDefNode):
         cenv = self.scope
         self.body.analyse_expressions(cenv)
         self.target.analyse_target_expression(env, self.classobj)
+        self.class_cell.analyse_expressions(cenv)
 
     def generate_function_definitions(self, env, code):
         self.generate_lambda_definitions(self.scope, code)
@@ -4098,8 +4125,12 @@ class PyClassDefNode(ClassDefNode):
             self.metaclass.generate_evaluation_code(code)
         self.dict.generate_evaluation_code(code)
         cenv.namespace_cname = cenv.class_obj_cname = self.dict.result()
+        self.class_cell.generate_evaluation_code(code)
         self.body.generate_execution_code(code)
         self.class_result.generate_evaluation_code(code)
+        self.class_cell.generate_injection_code(
+            code, self.class_result.result())
+        self.class_cell.generate_disposal_code(code)
         cenv.namespace_cname = cenv.class_obj_cname = self.classobj.result()
         self.target.generate_assignment_code(self.class_result, code)
         self.dict.generate_disposal_code(code)

Cython/Compiler/ParseTreeTransforms.py

@@ -2137,6 +2137,12 @@ class CreateClosureClasses(CythonTransform):
         class_scope = entry.type.scope
         class_scope.is_internal = True
 
+        if node.is_generator and node.requires_classobj:
+            class_scope.declare_var(pos=node.pos, name='classobj',
+                                    cname='classobj',
+                                    type=PyrexTypes.py_object_type,
+                                    is_cdef=True)
+
         if from_closure:
             assert cscope.is_closure_scope
             class_scope.declare_var(pos=node.pos,
@@ -2411,6 +2417,34 @@ class TransformBuiltinMethods(EnvTransform):
                     node.pos, self.current_scope_node(), lenv))
         return node
 
+    def _inject_super(self, node, func_name):
+        lenv = self.current_env()
+        entry = lenv.lookup_here(func_name)
+        if entry or node.args:
+            return node
+        # Inject no-args super
+        def_node = self.current_scope_node()
+        if (not isinstance(def_node, Nodes.DefNode) or not def_node.args or
+            len(self.env_stack) < 2):
+            return node
+        class_node, class_scope = self.env_stack[-2]
+        if class_scope.is_py_class_scope:
+            def_node.requires_classobj = True
+            class_node.class_cell.is_active = True
+            node.args = [
+                ExprNodes.ClassCellNode(
+                    node.pos, is_generator=def_node.is_generator),
+                ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
+                ]
+        elif class_scope.is_c_class_scope:
+            node.args = [
+                ExprNodes.NameNode(
+                    node.pos, name=class_node.scope.name,
+                    entry=class_node.entry),
+                ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
+                ]
+        return node
+
     def visit_SimpleCallNode(self, node):
         # cython.foo
         function = node.function.as_cython_attribute()
@@ -2471,6 +2505,8 @@ class TransformBuiltinMethods(EnvTransform):
                 return self._inject_locals(node, func_name)
             if func_name == 'eval':
                 return self._inject_eval(node, func_name)
+            if func_name == 'super':
+                return self._inject_super(node, func_name)
         return node
 
 

Cython/Compiler/TypeSlots.py

@@ -68,6 +68,9 @@ class Signature(object):
         # and are not looked up in here
     }
 
+    type_to_format_map = dict([(type_, format_)
+                                   for format_, type_ in format_map.iteritems()])
+
     error_value_map = {
         'O': "NULL",
         'T': "NULL",
@@ -91,6 +94,7 @@ class Signature(object):
         self.fixed_arg_format = arg_format
         self.ret_format = ret_format
         self.error_value = self.error_value_map.get(ret_format, None)
+        self.is_staticmethod = False
 
     def num_fixed_args(self):
         return len(self.fixed_arg_format)
@@ -109,6 +113,11 @@ class Signature(object):
     def return_type(self):
         return self.format_map[self.ret_format]
 
+    def format_from_type(self, arg_type):
+        if arg_type.is_pyobject:
+            arg_type = PyrexTypes.py_object_type
+        return self.type_to_format_map[arg_type]
+
     def exception_value(self):
         return self.error_value_map.get(self.ret_format)
 
@@ -141,6 +150,9 @@ class Signature(object):
                     return [method_noargs]
             elif full_args in ["OO", "TO"] and not self.has_generic_args:
                 return [method_onearg]
+
+            if self.is_staticmethod:
+                return [method_varargs, method_keywords]
         return None
 
 

Cython/Compiler/Visitor.py

@@ -334,6 +334,10 @@ class EnvTransform(CythonTransform):
         self.env_stack.pop()
         return node
 
+    def visit_GeneratorBodyDefNode(self, node):
+        self.visitchildren(node)
+        return node
+
     def visit_ClassDefNode(self, node):
         self.env_stack.append((node, node.scope))
         self.visitchildren(node)

tests/run/fused_def.pyx

@@ -128,3 +128,106 @@ def args_kwargs(fused_t obj, cython.floating myf = 1.2, *args, **kwargs):
     """
     print cython.typeof(obj), cython.typeof(myf)
     print obj, "%.2f" % myf, "%.2f" % f, args, kwargs
+
+
+class BaseClass(object):
+    """
+    Test fused class/static/normal methods and super() without args
+    """
+
+    @staticmethod
+    def mystaticmethod(cython.integral arg1):
+        print cython.typeof(arg1), arg1
+
+    @classmethod
+    def myclassmethod(cls, cython.integral arg1):
+        print cls, cython.typeof(arg1), arg1
+
+    def normalmethod(self, cython.integral arg1):
+        print self, cython.typeof(arg1), arg1
+
+    def __repr__(self):
+        return "<%s.%s object>" % (__name__, type(self).__name__)
+
+class SubClass(BaseClass):
+
+    @staticmethod
+    def mystaticmethod(self, cython.integral arg1):
+        print cython.typeof(arg1), arg1
+        super().mystaticmethod(arg1 + 1)
+
+    @classmethod
+    def myclassmethod(cls, cython.integral arg1):
+        print cls, cython.typeof(arg1), arg1
+        super().myclassmethod(arg1 + 1)
+
+    def normalmethod(self, cython.integral arg1):
+        print self, cython.typeof(arg1), arg1
+        super().normalmethod(arg1 + 1)
+
+class SubSubClass(SubClass):
+    pass
+
+def test_fused_def_super():
+    """
+    >>> test_fused_def_super()
+    long 10
+    long 11
+    long 11
+    long 12
+    short 12
+    long 13
+    short 13
+    long 14
+    <class 'fused_def.SubClass'> long 14
+    <class 'fused_def.SubClass'> long 15
+    <class 'fused_def.SubClass'> long 15
+    <class 'fused_def.SubClass'> long 16
+    <class 'fused_def.SubClass'> short 16
+    <class 'fused_def.SubClass'> long 17
+    <class 'fused_def.SubClass'> short 17
+    <class 'fused_def.SubClass'> long 18
+    <fused_def.SubClass object> long 18
+    <fused_def.SubClass object> long 19
+    <fused_def.SubClass object> long 19
+    <fused_def.SubClass object> long 20
+    <fused_def.SubClass object> short 20
+    <fused_def.SubClass object> long 21
+    <fused_def.SubClass object> short 21
+    <fused_def.SubClass object> long 22
+    """
+    obj = SubClass()
+    cls = SubClass
+
+    obj.mystaticmethod(obj, 10)
+    cls.mystaticmethod(obj, 11)
+    obj.mystaticmethod[cy.short](obj, 12)
+    cls.mystaticmethod[cy.short](obj, 13)
+
+    obj.myclassmethod(14)
+    cls.myclassmethod(15)
+    obj.myclassmethod[cy.short](16)
+    cls.myclassmethod[cy.short](17)
+
+    obj.normalmethod(18)
+    cls.normalmethod(obj, 19)
+    obj.normalmethod[cy.short](20)
+    cls.normalmethod[cy.short](obj, 21)
+
+def test_fused_def_classmethod():
+    """
+    >>> test_fused_def_classmethod()
+    <class 'fused_def.SubSubClass'> long 10
+    <class 'fused_def.SubSubClass'> long 11
+    <class 'fused_def.SubSubClass'> long 11
+    <class 'fused_def.SubSubClass'> long 12
+    <class 'fused_def.SubSubClass'> short 12
+    <class 'fused_def.SubSubClass'> long 13
+    <class 'fused_def.SubSubClass'> short 13
+    <class 'fused_def.SubSubClass'> long 14
+    """
+    SubSubClass().myclassmethod(10)
+    SubSubClass.myclassmethod(11)
+
+    SubSubClass().myclassmethod[cy.short](12)
+    SubSubClass.myclassmethod[cy.short](13)

tests/run/py3k_super.pyx

+# mode: run
+# tags: py3k_super
+
+class A(object):
+    def method(self):
+        return 1
+
+    @classmethod
+    def class_method(cls):
+        return 2
+
+    @staticmethod
+    def static_method():
+        return 3
+
+    def generator_test(self):
+        return [1, 2, 3]
+
+
+class B(A):
+    """
+    >>> obj = B()
+    >>> obj.method()
+    1
+    >>> B.class_method()
+    2
+    >>> B.static_method(obj)
+    3
+    >>> list(obj.generator_test())
+    [1, 2, 3]
+    """
+    def method(self):
+        return super().method()
+
+    @classmethod
+    def class_method(cls):
+        return super().class_method()
+
+    @staticmethod
+    def static_method(instance):
+        return super().static_method()
+
+    def generator_test(self):
+        for i in super().generator_test():
+            yield i
+
+
+def test_class_cell_empty():
+    """
+    >>> test_class_cell_empty()
+    Traceback (most recent call last):
+    ...
+    SystemError: super(): empty __class__ cell
+    """
+    class Base(type):
+        def __new__(cls, name, bases, attrs):
+            attrs['foo'](None)
+
+    class EmptyClassCell(metaclass=Base):
+        def foo(self):
+            super()
+
+
+cdef class CClassBase(object):
+    def method(self):
+        return 1
+
+cdef class CClassSuper(CClassBase):
+    """
+    >>> CClassSuper().method()
+    1
+    """
+
+    def method(self):
+        return super().method()

tests/run/staticmethod.pyx

@@ -43,3 +43,61 @@ def nested_class():
         def plus1(a):
             return a + 1
     return class5
+
+cdef class BaseClass(object):
+    """
+    Test cdef static methods with super() and Python subclasses
+
+    >>> obj = BaseClass()
+    >>> obj.mystaticmethod(obj, 1)
+    1
+    >>> BaseClass.mystaticmethod(obj, 1)
+    1
+    >>> obj.mystaticmethod2(1, 2, 3)
+    1 2 3
+    >>> BaseClass.mystaticmethod2(1, 2, 3)
+    1 2 3
+    """
+
+    @staticmethod
+    def mystaticmethod(self, arg1):
+        print arg1
+
+    @staticmethod
+    def mystaticmethod2(a, b, c):
+        print a, b, c
+
+cdef class SubClass(BaseClass):
+    """
+    >>> obj = SubClass()
+    >>> obj.mystaticmethod(obj, 1)
+    1
+    2
+    >>> SubClass.mystaticmethod(obj, 1)
+    1
+    2
+    """
+
+    @staticmethod
+    def mystaticmethod(self, arg1):
+        print arg1
+        super().mystaticmethod(self, arg1 + 1)
+
+class SubSubClass(SubClass):
+    """
+    >>> obj = SubSubClass()
+    >>> obj.mystaticmethod(obj, 1)
+    1
+    2
+    3
+    >>> SubSubClass.mystaticmethod(obj, 1)
+    1
+    2
+    3
+    """
+
+    @staticmethod
+    def mystaticmethod(self, arg1):
+        print arg1
+        super().mystaticmethod(self, arg1 + 1)
+