Fuse the cypclass wrappers and the underlying cyobjects into a single object

Cython/Compiler/CypclassWrapper.py

@@ -22,11 +22,6 @@ from .Errors import error, warning
 from .StringEncoding import EncodedString
 from .ParseTreeTransforms import NormalizeTree, InterpretCompilerDirectives, AnalyseDeclarationsTransform
 
-
-# cython name for underlying cypclass attribute in cypclass wrappers
-underlying_name = EncodedString("nogil_cyobject")
-
-
 #
 #   Visitor for wrapper cclass injection
 #
@@ -45,22 +40,22 @@ class CypclassWrapperInjection(Visitor.CythonTransform):
     unlocked_property = TreeFragment.TreeFragment(u"""
 property NAME:
     def __get__(self):
-        OBJ = <TYPE> self.UNDERLYING
+        OBJ = <TYPE> <CyObject> self
         return OBJ.ATTR
     def __set__(self, value):
-        OBJ = <TYPE> self.UNDERLYING
+        OBJ = <TYPE> <CyObject> self
         OBJ.ATTR = value
     """, level='c_class', pipeline=[NormalizeTree(None)])
 
     locked_property = TreeFragment.TreeFragment(u"""
 property NAME:
     def __get__(self):
-        OBJ = <TYPE> self.UNDERLYING
+        OBJ = <TYPE> <CyObject> self
         with rlocked OBJ:
             value = OBJ.ATTR
         return value
     def __set__(self, value):
-        OBJ = <TYPE> self.UNDERLYING
+        OBJ = <TYPE> <CyObject> self
         with wlocked OBJ:
             OBJ.ATTR = value
     """, level='c_class', pipeline=[NormalizeTree(None)])
@@ -68,40 +63,40 @@ property NAME:
     # method wrapper templates
     unlocked_method = TreeFragment.TreeFragment(u"""
 def NAME(self, ARGDECLS):
-    OBJ = <TYPE> self.UNDERLYING
+    OBJ = <TYPE> <CyObject> self
     return OBJ.NAME(ARGS)
     """, level='c_class', pipeline=[NormalizeTree(None)])
 
     unlocked_method_no_return = TreeFragment.TreeFragment(u"""
 def NAME(self, ARGDECLS):
-    OBJ = <TYPE> self.UNDERLYING
+    OBJ = <TYPE> <CyObject> self
     OBJ.NAME(ARGS)
     """, level='c_class', pipeline=[NormalizeTree(None)])
 
     rlocked_method = TreeFragment.TreeFragment(u"""
 def NAME(self, ARGDECLS):
-    OBJ = <TYPE> self.UNDERLYING
+    OBJ = <TYPE> <CyObject> self
     with rlocked OBJ:
         return OBJ.NAME(ARGS)
     """, level='c_class', pipeline=[NormalizeTree(None)])
 
     rlocked_method_no_return = TreeFragment.TreeFragment(u"""
 def NAME(self, ARGDECLS):
-    OBJ = <TYPE> self.UNDERLYING
+    OBJ = <TYPE> <CyObject> self
     with rlocked OBJ:
         OBJ.NAME(ARGS)
     """, level='c_class', pipeline=[NormalizeTree(None)])
 
     wlocked_method = TreeFragment.TreeFragment(u"""
 def NAME(self, ARGDECLS):
-    OBJ = <TYPE> self.UNDERLYING
+    OBJ = <TYPE> <CyObject> self
     with wlocked OBJ:
         return OBJ.NAME(ARGS)
     """, level='c_class', pipeline=[NormalizeTree(None)])
 
     wlocked_method_no_return = TreeFragment.TreeFragment(u"""
 def NAME(self, ARGDECLS):
-    OBJ = <TYPE> self.UNDERLYING
+    OBJ = <TYPE> <CyObject> self
     with wlocked OBJ:
         OBJ.NAME(ARGS)
     """, level='c_class', pipeline=[NormalizeTree(None)])
@@ -164,15 +159,17 @@ def NAME(self, ARGDECLS):
         self.derive_names(node)
         self.collected_cypclasses.append(node)
 
-    def create_unique_name(self, name):
+    def create_unique_name(self, name, entries=None):
         # output: name(_u_*)?
         # guarantees:
         # - different inputs always result in different outputs
-        # - the output is not in the module scope dictionary
+        # - the output is not among the given entries
+        # if entries is None, the module scope entries are used
         unique_name = name
-        if unique_name in self.module_scope.entries:
+        entries = self.module_scope.entries if entries is None else entries
+        if unique_name in entries:
             unique_name = "%s_u" % unique_name
-        while unique_name in self.module_scope.entries:
+        while unique_name in entries:
             unique_name = "%s_" % unique_name
         return EncodedString(unique_name)
 
@@ -292,11 +289,6 @@ def NAME(self, ARGDECLS):
         cclass_bases = self.synthesize_base_tuple(node)
 
         stats = []
-        if not cclass_bases.args:
-            # the memory layout for the underlying cyobject should always be the same
-            # and match the memory layout of CyPyObject.
-            underlying_cyobject = self.synthesize_underlying_cyobject_attribute(node)
-            stats.append(underlying_cyobject)
 
         # insert method wrappers in the statement list
         self.insert_cypclass_method_wrappers(node, cclass_name, stats)
@@ -314,7 +306,7 @@ def NAME(self, ARGDECLS):
             class_name = cclass_name,
             as_name = cclass_name,
             bases = cclass_bases,
-            objstruct_name = None,
+            objstruct_name = Naming.cypclass_wrapper_layout_type,
             typeobj_name = None,
             check_size = None,
             in_pxd = node.in_pxd,
@@ -326,41 +318,6 @@ def NAME(self, ARGDECLS):
 
         return wrapper
 
-    def synthesize_underlying_cyobject_attribute(self, node):
-        base_type = PyrexTypes.cy_object_type
-
-        base_type_node = Nodes.CSimpleBaseTypeNode(
-            node.pos,
-            name = base_type.name,
-            module_path = [],
-            is_basic_c_type = 0,
-            signed = 1,
-            complex = 0,
-            longness = 0,
-            is_self_arg = 0,
-            templates = None
-        )
-
-        underlying_name_declarator = Nodes.CNameDeclaratorNode(
-            node.pos,
-            name=underlying_name,
-            cname=Naming.cypclass_wrapper_underlying_attr
-        )
-
-        underlying_cyobject = Nodes.CVarDefNode(
-            pos = node.pos,
-            visibility = 'private',
-            base_type = base_type_node,
-            declarators = [underlying_name_declarator],
-            in_pxd = node.in_pxd,
-            doc = None,
-            api = 0,
-            modifiers = [],
-            overridable = 0
-        )
-
-        return underlying_cyobject
-
     def insert_cypclass_method_wrappers(self, node, cclass_name, stats):
         for attr in node.attributes:
             if isinstance(attr, Nodes.CFuncDefNode):
@@ -382,11 +339,11 @@ def NAME(self, ARGDECLS):
             template = self.locked_property
         else:
             template = self.unlocked_property
+        underlying_name = EncodedString("o")
         property = template.substitute({
             "ATTR": attr_entry.name,
             "TYPE": node_entry.type,
             "OBJ": ExprNodes.NameNode(attr_entry.pos, name=underlying_name),
-            "UNDERLYING": underlying_name
         }, pos=attr_entry.pos).stats[0]
         property.name = attr_entry.name
         property.doc = attr_entry.doc
@@ -455,13 +412,15 @@ def NAME(self, ARGDECLS):
         else:
             template = self.unlocked_method if need_return else self.unlocked_method_no_return
 
+        # > derive a unique name that doesn't collide with the arguments
+        underlying_name = self.create_unique_name("o", entries=[arg.name for arg in arg_objs])
+
         # > instanciate the wrapper from the template
         method_wrapper = template.substitute({
             "NAME": cfunc_name,
             "ARGDECLS": py_args_decls,
             "TYPE": underlying_type,
             "OBJ": ExprNodes.NameNode(cfunc_method.pos, name=underlying_name),
-            "UNDERLYING": underlying_name,
             "ARGS": arg_objs
         }).stats[0]
         method_wrapper.doc = py_doc
@@ -1124,9 +1083,7 @@ def generate_cypclass_wrapper_allocation(code, wrapper_type):
 
     objstruct_cname = wrapper_type.objstruct_cname
     code.putln("if (self) {")
-    code.putln("%s * wrapper = (%s *) ::operator new(sizeof *wrapper);" % (objstruct_cname, objstruct_cname))
+    code.putln("%s * wrapper = (%s *) self;" % (objstruct_cname, objstruct_cname))
     code.putln("((PyObject *)wrapper)->ob_refcnt = 0;")
     code.putln("((PyObject *)wrapper)->ob_type = %s;" % wrapper_type.typeptr_cname)
-    code.putln("((%s *)wrapper)->%s = self;" % (Naming.cypclass_wrapper_layout_type, Naming.cypclass_wrapper_underlying_attr))
-    code.putln("self->cy_pyobject = (PyObject *) wrapper;")
     code.putln("}")

Cython/Compiler/ModuleNode.py

@@ -582,11 +582,20 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         vtab_dict, vtab_dict_order = {}, []
         vtabslot_dict, vtabslot_dict_order = {}, []
 
+        def vtab_key_func(entry_type):
+            return entry_type.vtabstruct_cname
+
+        def vtabslot_key_func(entry_type):
+            if entry_type.is_cyp_wrapper:
+                # cyp_wrappers all have the same objstruct_cname
+                return entry_type.wrapped_cname
+            return entry_type.objstruct_cname
+
         for module in module_list:
             for entry in module.c_class_entries:
                 if entry.used and not entry.in_cinclude:
                     type = entry.type
-                    key = type.vtabstruct_cname
+                    key = vtab_key_func(type)
                     if not key:
                         continue
                     if key in vtab_dict:
@@ -604,25 +613,26 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     type = entry.type
                     if type.is_extension_type and not entry.in_cinclude:
                         type = entry.type
-                        key = type.objstruct_cname
+                        key = vtabslot_key_func(type)
                         assert key not in vtabslot_dict, key
                         vtabslot_dict[key] = entry
                         vtabslot_dict_order.append(key)
 
-        def vtabstruct_cname(entry_type):
-            return entry_type.vtabstruct_cname
         vtab_list = self.sort_types_by_inheritance(
-            vtab_dict, vtab_dict_order, vtabstruct_cname)
+            vtab_dict, vtab_dict_order, vtab_key_func)
 
-        def objstruct_cname(entry_type):
-            return entry_type.objstruct_cname
         vtabslot_list = self.sort_types_by_inheritance(
-            vtabslot_dict, vtabslot_dict_order, objstruct_cname)
+            vtabslot_dict, vtabslot_dict_order, vtabslot_key_func)
 
         return (vtab_list, vtabslot_list)
 
     def sort_cdef_classes(self, env):
-        key_func = operator.attrgetter('objstruct_cname')
+        def key_func(entry_type):
+            if entry_type.is_cyp_wrapper:
+                # cyp_wrappers all have the same objstruct_cname
+                return entry_type.wrapped_cname
+            return entry_type.objstruct_cname
+
         entry_dict, entry_order = {}, []
         for entry in env.c_class_entries:
             key = key_func(entry.type)
@@ -1296,6 +1306,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln(self.sue_predeclaration(type, "struct", type.objstruct_cname))
 
     def generate_objstruct_definition(self, type, code):
+        if type.is_cyp_wrapper:
+            # cclass wrappers for cypclass already have an objstruct
+            return
+
         code.mark_pos(type.pos)
         # Generate object struct definition for an
         # extension type.
@@ -1686,11 +1700,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         # for cyp wrappers, just decrement the atomic counter of the underlying type
         parent_type = scope.parent_type
         if parent_type.is_cyp_wrapper:
-            underlying_attribute_name = Naming.cypclass_wrapper_underlying_attr
             self.generate_self_cast(scope, code)
             code.putln(
-                "CyObject * p_nogil_cyobject = p->%s;"
-                % underlying_attribute_name
+                "CyObject * p_nogil_cyobject = static_cast<CyObject *>(p);"
             )
             code.putln("Cy_DECREF(p_nogil_cyobject);")
             code.putln("}")

Cython/Compiler/Nodes.py

@@ -5586,6 +5586,8 @@ class CypclassWrapperDefNode(CClassDefNode):
         self.entry.type.is_cyp_wrapper = 1
         # > associate the wrapper type to the wrapped type
         self.wrapped_cypclass.entry.type.wrapper_type = self.entry.type
+        # > remember the declaration of the wrapped type
+        self.entry.type.wrapped_cname = self.wrapped_cypclass.entry.type.empty_declaration_code()
 
 
 class PropertyNode(StatNode):

Cython/Compiler/PyrexTypes.py

@@ -1499,12 +1499,15 @@ class PyExtensionType(PyObjectType):
     #  defered_declarations [thunk]      Used to declare class hierarchies in order
     #  check_size       'warn', 'error', 'ignore'    What to do if tp_basicsize does not match
     #  is_cyp_wrapper   boolean          Whether this extension type wraps a cypclass
+    #  wrapped_cname    string or None   The full namespace declaration of the wrapped type when this is a cyp_wrapper
 
     is_extension_type = 1
     has_attributes = 1
     early_init = 1
     is_cyp_wrapper = 0
 
+    wrapper_cname = None
+
     objtypedef_cname = None
 
     def __init__(self, name, typedef_flag, base_type, is_external=0, check_size=None):

Cython/Utility/CyObjects.cpp

@@ -56,15 +56,18 @@
             int trywlock();
     };
 
-    class CyObject {
+    struct CyPyObject {
+        PyObject_HEAD
+    };
+
+    class CyObject : public CyPyObject {
         private:
           CyObject_ATOMIC_REFCOUNT_TYPE nogil_ob_refcnt;
           //pthread_rwlock_t ob_lock;
           RecursiveUpgradeableRWLock ob_lock;
         public:
-          PyObject * cy_pyobject;
-          CyObject(): nogil_ob_refcnt(1), cy_pyobject(NULL) {}
-          virtual ~CyObject();
+          CyObject(): nogil_ob_refcnt(1) {}
+          virtual ~CyObject() {}
           void CyObject_INCREF();
           int CyObject_DECREF();
           int CyObject_GETREF();
@@ -75,19 +78,6 @@
           int CyObject_TRYWLOCK();
     };
 
-    /*
-     * A POD type that has a compatible memory layout with any wrapper for a cypclass.
-     *
-     * Serves as a:
-     *  - convenience type to cast a wrapper and access its underlying cyobject pointer.
-     *  - reference for the memory layout that all cypclass wrappers must respect.
-     */
-    struct CyPyObject {
-        PyObject_HEAD
-        CyObject * nogil_cyobject;
-    };
-
-
     /* All this is made available by member injection inside the module scope */
 
     struct ActhonResultInterface : public CyObject {
@@ -194,7 +184,7 @@
             Py_INCREF(Py_None);
             return Py_None;
         }
-        PyObject * ob = cy->cy_pyobject;
+        PyObject * ob = reinterpret_cast<PyObject *>(static_cast<CyPyObject *>(cy));
         // artificial atomic increment the first time Python gets a reference
         if (ob->ob_refcnt == 0)
             cy->CyObject_INCREF();
@@ -224,7 +214,7 @@
         }
 
         CyPyObject * wrapper = (CyPyObject *)ob;
-        U * underlying = dynamic_cast<U *>(wrapper->nogil_cyobject);
+        U * underlying = dynamic_cast<U *>(static_cast<CyObject *>(wrapper));
 
         // no underlying cyobject: shouldn't happen, playing it safe for now
         if (underlying == NULL) {
@@ -463,12 +453,6 @@ int RecursiveUpgradeableRWLock::trywlock() {
     return 0;
 }
 
-CyObject::~CyObject() {
-  if (cy_pyobject) {
-    ::operator delete(cy_pyobject);
-  }
-}
-
 void CyObject::CyObject_INCREF()
 {
   atomic_fetch_add(&(this->nogil_ob_refcnt), 1);