Make self argument for binops typed (GH-4436)

The type for the self argument for binops depends on the compile
directives. Therefore it needs a set of type slots that depends
on the compiler directives.

I've therefore got rid of the big static list of typeslots in
TypeSlots.py in favour of a class that defines them all (and
can be initialized with suitable compiler directives as needed).
This involves moving a static dictionary and list out of the
global scope too, so that they too can be part of the class.
The passing of the dictionary and list to all the constructors
is a bit awkward

Fixes https://github.com/cython/cython/issues/4434

Cython/Compiler/ModuleNode.py

@@ -1437,7 +1437,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                         warning(scope.parent_type.pos,
                                 "total_ordering directive used, but no comparison and equality methods defined")
 
-                    for slot in TypeSlots.PyNumberMethods:
+                    for slot in TypeSlots.get_slot_table(code.globalstate.directives).PyNumberMethods:
                         if slot.is_binop and scope.defines_any_special(slot.user_methods):
                             self.generate_binop_function(scope, slot, code, entry.pos)
 
@@ -1845,7 +1845,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("}")
 
     def generate_clear_function(self, scope, code, cclass_entry):
-        tp_slot = TypeSlots.get_slot_by_name("tp_clear")
+        tp_slot = TypeSlots.get_slot_by_name("tp_clear", scope.directives)
         slot_func = scope.mangle_internal("tp_clear")
         base_type = scope.parent_type.base_type
         if tp_slot.slot_code(scope) != slot_func:
@@ -2225,10 +2225,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if preprocessor_guard:
             code.putln(preprocessor_guard)
 
-        if slot.left_slot.signature == TypeSlots.binaryfunc:
+        if slot.left_slot.signature in (TypeSlots.binaryfunc, TypeSlots.ibinaryfunc):
             slot_type = 'binaryfunc'
             extra_arg = extra_arg_decl = ''
-        elif slot.left_slot.signature == TypeSlots.ternaryfunc:
+        elif slot.left_slot.signature in (TypeSlots.ternaryfunc, TypeSlots.iternaryfunc):
             slot_type = 'ternaryfunc'
             extra_arg = ', extra_arg'
             extra_arg_decl = ', PyObject* extra_arg'
@@ -2519,17 +2519,17 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         ext_type = entry.type
         scope = ext_type.scope
 
-        members_slot = TypeSlots.get_slot_by_name("tp_members")
+        members_slot = TypeSlots.get_slot_by_name("tp_members", code.globalstate.directives)
         members_slot.generate_substructure_spec(scope, code)
 
-        buffer_slot = TypeSlots.get_slot_by_name("tp_as_buffer")
+        buffer_slot = TypeSlots.get_slot_by_name("tp_as_buffer", code.globalstate.directives)
         if not buffer_slot.is_empty(scope):
             code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
             buffer_slot.generate_substructure(scope, code)
             code.putln("#endif")
 
         code.putln("static PyType_Slot %s_slots[] = {" % ext_type.typeobj_cname)
-        for slot in TypeSlots.slot_table:
+        for slot in TypeSlots.get_slot_table(code.globalstate.directives):
             slot.generate_spec(scope, code)
         code.putln("{0, 0},")
         code.putln("};")
@@ -2543,14 +2543,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln('"%s.%s",' % (self.full_module_name, classname.replace('"', '')))
         code.putln("sizeof(%s)," % objstruct)
         code.putln("0,")
-        code.putln("%s," % TypeSlots.get_slot_by_name("tp_flags").slot_code(scope))
+        code.putln("%s," % TypeSlots.get_slot_by_name("tp_flags", scope.directives).slot_code(scope))
         code.putln("%s_slots," % ext_type.typeobj_cname)
         code.putln("};")
 
     def generate_typeobj_definition(self, modname, entry, code):
         type = entry.type
         scope = type.scope
-        for suite in TypeSlots.substructures:
+        for suite in TypeSlots.get_slot_table(code.globalstate.directives).substructures:
             suite.generate_substructure(scope, code)
         code.putln("")
         if entry.visibility == 'public':
@@ -2574,7 +2574,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             "sizeof(%s), /*tp_basicsize*/" % objstruct)
         code.putln(
             "0, /*tp_itemsize*/")
-        for slot in TypeSlots.slot_table:
+        for slot in TypeSlots.get_slot_table(code.globalstate.directives):
             slot.generate(scope, code)
         code.putln(
             "};")

Cython/Compiler/Nodes.py

@@ -2417,7 +2417,7 @@ class FuncDefNode(StatNode, BlockNode):
         if not self.entry.is_special:
             return None
         name = self.entry.name
-        slot = TypeSlots.method_name_to_slot.get(name)
+        slot = TypeSlots.get_slot_table(self.local_scope.directives).get_slot_by_method_name(name)
         if not slot:
             return None
         if name == '__long__' and not self.entry.scope.lookup_here('__int__'):
@@ -5335,7 +5335,7 @@ class CClassDefNode(ClassDefNode):
                         UtilityCode.load_cached('ValidateBasesTuple', 'ExtensionTypes.c'))
                     code.put_error_if_neg(entry.pos, "__Pyx_validate_bases_tuple(%s.name, %s, %s)" % (
                         typespec_cname,
-                        TypeSlots.get_slot_by_name("tp_dictoffset").slot_code(scope),
+                        TypeSlots.get_slot_by_name("tp_dictoffset", scope.directives).slot_code(scope),
                         bases_tuple_cname or tuple_temp,
                     ))
 
@@ -5359,7 +5359,7 @@ class CClassDefNode(ClassDefNode):
                     ))
 
             # The buffer interface is not currently supported by PyType_FromSpec().
-            buffer_slot = TypeSlots.get_slot_by_name("tp_as_buffer")
+            buffer_slot = TypeSlots.get_slot_by_name("tp_as_buffer", code.globalstate.directives)
             if not buffer_slot.is_empty(scope):
                 code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
                 code.putln("%s->%s = %s;" % (
@@ -5369,7 +5369,8 @@ class CClassDefNode(ClassDefNode):
                 ))
                 # Still need to inherit buffer methods since PyType_Ready() didn't do it for us.
                 for buffer_method_name in ("__getbuffer__", "__releasebuffer__"):
-                    buffer_slot = TypeSlots.get_slot_by_method_name(buffer_method_name)
+                    buffer_slot = TypeSlots.get_slot_table(
+                        code.globalstate.directives).get_slot_by_method_name(buffer_method_name)
                     if buffer_slot.slot_code(scope) == "0" and not TypeSlots.get_base_slot_function(scope, buffer_slot):
                         code.putln("if (!%s->tp_as_buffer->%s &&"
                                    " %s->tp_base->tp_as_buffer &&"
@@ -5405,7 +5406,7 @@ class CClassDefNode(ClassDefNode):
 
             code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
             # FIXME: these still need to get initialised even with the limited-API
-            for slot in TypeSlots.slot_table:
+            for slot in TypeSlots.get_slot_table(code.globalstate.directives):
                 slot.generate_dynamic_init_code(scope, code)
             code.putln("#endif")
 
@@ -5450,7 +5451,8 @@ class CClassDefNode(ClassDefNode):
                 is_buffer = func.name in ('__getbuffer__', '__releasebuffer__')
                 if (func.is_special and Options.docstrings and
                         func.wrapperbase_cname and not is_buffer):
-                    slot = TypeSlots.method_name_to_slot.get(func.name)
+                    slot = TypeSlots.get_slot_table(
+                        entry.type.scope.directives).get_slot_by_method_name(func.name)
                     preprocessor_guard = slot.preprocessor_guard_code() if slot else None
                     if preprocessor_guard:
                         code.putln(preprocessor_guard)

Cython/Compiler/Symtab.py

@@ -20,7 +20,7 @@ from . import PyrexTypes
 from .PyrexTypes import py_object_type, unspecified_type
 from .TypeSlots import (
     pyfunction_signature, pymethod_signature, richcmp_special_methods,
-    get_special_method_signature, get_property_accessor_signature)
+    get_slot_table, get_property_accessor_signature)
 from . import Future
 
 from . import Code
@@ -2240,7 +2240,8 @@ class CClassScope(ClassScope):
                 error(pos,
                     "C attributes cannot be added in implementation part of"
                     " extension type defined in a pxd")
-            if not self.is_closure_class_scope and get_special_method_signature(name):
+            if (not self.is_closure_class_scope and
+                    get_slot_table(self.directives).get_special_method_signature(name)):
                 error(pos,
                     "The name '%s' is reserved for a special method."
                         % name)
@@ -2312,7 +2313,7 @@ class CClassScope(ClassScope):
                 "in a future version of Pyrex and Cython. Use __cinit__ instead.")
         entry = self.declare_var(name, py_object_type, pos,
                                  visibility='extern')
-        special_sig = get_special_method_signature(name)
+        special_sig = get_slot_table(self.directives).get_special_method_signature(name)
         if special_sig:
             # Special methods get put in the method table with a particular
             # signature declared in advance.
@@ -2344,7 +2345,8 @@ class CClassScope(ClassScope):
                           cname=None, visibility='private', api=0, in_pxd=0,
                           defining=0, modifiers=(), utility_code=None, overridable=False):
         name = self.mangle_class_private_name(name)
-        if get_special_method_signature(name) and not self.parent_type.is_builtin_type:
+        if (get_slot_table(self.directives).get_special_method_signature(name)
+                and not self.parent_type.is_builtin_type):
             error(pos, "Special methods must be declared with 'def', not 'cdef'")
         args = type.args
         if not type.is_static_method:

Cython/Compiler/Visitor.py

@@ -675,7 +675,7 @@ class MethodDispatcherTransform(EnvTransform):
         method_handler = self._find_handler(
             "method_%s_%s" % (type_name, attr_name), kwargs)
         if method_handler is None:
-            if (attr_name in TypeSlots.method_name_to_slot
+            if (attr_name in TypeSlots.special_method_names
                     or attr_name in ['__new__', '__class__']):
                 method_handler = self._find_handler(
                     "slot%s" % attr_name, kwargs)

docs/src/userguide/migrating_to_cy30.rst

@@ -157,3 +157,18 @@ Here ``Base.__bar`` is mangled to ``_Base__bar`` and ``Derived.__bar``
 to ``_Derived__bar``. Therefore ``call_bar`` will always call 
 ``_Base__bar``. This matches established Python behaviour and applies
 for ``def``, ``cdef`` and ``cpdef`` methods and attributes.
+
+Arithmetic special methods
+==========================
+
+The behaviour of arithmetic special methods (for example ``__add__``
+and ``__pow__``) of cdef classes has changed in Cython 3.0. They now 
+support separate "reversed" versions of these methods (e.g. 
+``__radd__``, ``__rpow__``) that behave like in pure Python.
+The main incompatible change is that the type of the first operand
+(usually ``__self__``) is now assumed to be that of the defining class,
+rather than relying on the user to test and cast the type of each operand.
+
+The old behaviour can be restored with the 
+:ref:`directive <compiler-directives>` ``c_api_binop_methods=True``.
+More details are given in :ref:`arithmetic_methods`.

tests/run/binop_reverse_methods_GH2056.pyx

@@ -37,25 +37,29 @@ class Base(object):
         self.implemented = implemented
 
     def __add__(self, other):
-        if (<Base>self).implemented:
+        assert cython.typeof(self) == "Base"
+        if self.implemented:
             return "Base.__add__(%s, %s)" % (self, other)
         else:
             return NotImplemented
 
     def __radd__(self, other):
-        if (<Base>self).implemented:
+        assert cython.typeof(self) == "Base"
+        if self.implemented:
             return "Base.__radd__(%s, %s)" % (self, other)
         else:
             return NotImplemented
 
     def __pow__(self, other, mod):
-        if (<Base>self).implemented:
+        assert cython.typeof(self) == "Base"
+        if self.implemented:
             return "Base.__pow__(%s, %s, %s)" % (self, other, mod)
         else:
             return NotImplemented
 
     def __rpow__(self, other, mod):
-        if (<Base>self).implemented:
+        assert cython.typeof(self) == "Base"
+        if self.implemented:
             return "Base.__rpow__(%s, %s, %s)" % (self, other, mod)
         else:
             return NotImplemented
@@ -94,7 +98,8 @@ class OverloadLeft(Base):
         self.derived_implemented = implemented
 
     def __add__(self, other):
-        if (<OverloadLeft>self).derived_implemented:
+        assert cython.typeof(self) == "OverloadLeft"
+        if self.derived_implemented:
             return "OverloadLeft.__add__(%s, %s)" % (self, other)
         else:
             return NotImplemented
@@ -130,7 +135,8 @@ class OverloadRight(Base):
         self.derived_implemented = implemented
 
     def __radd__(self, other):
-        if (<OverloadRight>self).derived_implemented:
+        assert cython.typeof(self) == "OverloadRight"
+        if self.derived_implemented:
             return "OverloadRight.__radd__(%s, %s)" % (self, other)
         else:
             return NotImplemented
@@ -166,6 +172,7 @@ class OverloadCApi(Base):
         self.derived_implemented = derived_implemented
 
     def __add__(self, other):
+        assert cython.typeof(self) != "OverloadCApi"  # should be untyped
         if isinstance(self, OverloadCApi):
             derived_implemented = (<OverloadCApi>self).derived_implemented
         else: