Support multiple inheritance

The core functionality is to calculate and store tp_mro and tp_bases
instead of just tp_base.  This gives the runtime a bit harder of a
time to bootstrap itself since now a fully-built class depends on
a few more classes, so the bootstrapping section got larger:
- object_cls (base of the tp_base hierarchy)
- type_cls (base of the metaclass hierarchy)
- str_cls (for ht_name)
- tuple_cls (for tp_mro)
- list_cls (for calculating the mro)

There were a few places that needed to be updated now that we have
multiple inheritance:
- typeLookup()
- isSubclass()
- typeNew()
- super()

This change doesn't even attempt to add multiple inheritance rules
around old-style classes.

src/capi/typeobject.h

@@ -24,6 +24,13 @@ bool update_slot(BoxedClass* self, const std::string& attr) noexcept;
 
 void fixup_slot_dispatchers(BoxedClass* self) noexcept;
 void commonClassSetup(BoxedClass* cls);
+
+// We need to expose these due to our different file organization (they
+// are defined as part of the CPython copied into typeobject.c, but used
+// from Pyston code).
+// We could probably unify things more but that's for later.
+PyTypeObject* best_base(PyObject* bases) noexcept;
+PyObject* mro_external(PyObject* self) noexcept;
 }
 
 #endif

src/runtime/objmodel.cpp

@@ -195,10 +195,11 @@ bool isInstance(Box* obj, BoxedClass* cls) {
 
 extern "C" bool isSubclass(BoxedClass* child, BoxedClass* parent) {
     // TODO the class is allowed to override this using __subclasscheck__
-    while (child) {
-        if (child == parent)
+    assert(child->tp_mro);
+    assert(child->tp_mro->cls == tuple_cls);
+    for (auto b : static_cast<BoxedTuple*>(child->tp_mro)->elts) {
+        if (b == parent)
             return true;
-        child = child->tp_base;
     }
     return false;
 }
@@ -314,7 +315,10 @@ BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset
     if (cls == NULL) {
         assert(type_cls == NULL);
     } else {
-        assert(isSubclass(cls, type_cls));
+        // The (cls == type_cls) part of the check is important because during bootstrapping
+        // we might not have set up enough stuff in order to do proper subclass checking,
+        // but those clases will either have cls == NULL or cls == type_cls
+        assert(cls == type_cls || isSubclass(cls, type_cls));
     }
 
     assert(tp_dealloc == NULL);
@@ -375,15 +379,22 @@ BoxedHeapClass::BoxedHeapClass(BoxedClass* base, gcvisit_func gc_visit, int attr
 
 BoxedHeapClass* BoxedHeapClass::create(BoxedClass* metaclass, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
                                        int instance_size, bool is_user_defined, const std::string& name) {
-    return create(metaclass, base, gc_visit, attrs_offset, instance_size, is_user_defined, new BoxedString(name));
+    return create(metaclass, base, gc_visit, attrs_offset, instance_size, is_user_defined, new BoxedString(name), NULL);
 }
 
 BoxedHeapClass* BoxedHeapClass::create(BoxedClass* metaclass, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
-                                       int instance_size, bool is_user_defined, BoxedString* name) {
+                                       int instance_size, bool is_user_defined, BoxedString* name, BoxedTuple* bases) {
     BoxedHeapClass* made = new (metaclass)
         BoxedHeapClass(base, gc_visit, attrs_offset, instance_size, is_user_defined, name);
 
+    // While it might be ok if these were set, it'd indicate a difference in
+    // expectations as to who was going to calculate them:
+    assert(!made->tp_mro);
+    assert(!made->tp_bases);
+    made->tp_bases = bases;
+
     made->finishInitialization();
+    assert(made->tp_mro);
 
     return made;
 }
@@ -651,19 +662,45 @@ Box* typeLookup(BoxedClass* cls, const std::string& attr, GetattrRewriteArgs* re
 
         RewriterVar* obj_saved = rewrite_args->obj;
 
-        val = cls->getattr(attr, rewrite_args);
-        assert(rewrite_args->out_success);
-        if (!val and cls->tp_base) {
+        auto _mro = cls->tp_mro;
+        assert(_mro->cls == tuple_cls);
+        BoxedTuple* mro = static_cast<BoxedTuple*>(_mro);
+
+        // Guarding approach:
+        // Guard on the value of the tp_mro slot, which should be a tuple and thus be
+        // immutable.  Then we don't have to figure out the guards to emit that check
+        // the individual mro entries.
+        // We can probably move this guard to after we call getattr() on the given cls.
+        //
+        // TODO this can fail if we replace the mro with another mro that lives in the same
+        // address.
+        obj_saved->addAttrGuard(offsetof(BoxedClass, tp_mro), (intptr_t)mro);
+
+        for (auto base : mro->elts) {
             rewrite_args->out_success = false;
-            rewrite_args->obj = obj_saved->getAttr(offsetof(BoxedClass, tp_base));
-            val = typeLookup(cls->tp_base, attr, rewrite_args);
+            if (base == cls) {
+                // Small optimization: don't have to load the class again since it was given to us in
+                // a register.
+                assert(rewrite_args->obj == obj_saved);
+            } else {
+                rewrite_args->obj = rewrite_args->rewriter->loadConst((intptr_t)base, Location::any());
+            }
+            val = base->getattr(attr, rewrite_args);
+            assert(rewrite_args->out_success);
+            if (val)
+                return val;
         }
-        return val;
+
+        return NULL;
     } else {
-        val = cls->getattr(attr, NULL);
-        if (!val and cls->tp_base)
-            return typeLookup(cls->tp_base, attr, NULL);
-        return val;
+        assert(cls->tp_mro);
+        assert(cls->tp_mro->cls == tuple_cls);
+        for (auto b : static_cast<BoxedTuple*>(cls->tp_mro)->elts) {
+            val = b->getattr(attr, NULL);
+            if (val)
+                return val;
+        }
+        return NULL;
     }
 }
 
@@ -1516,9 +1553,8 @@ extern "C" Box* getattr(Box* obj, const char* attr) {
         // This doesn't belong here either:
         if (strcmp(attr, "__bases__") == 0 && isSubclass(obj->cls, type_cls)) {
             BoxedClass* cls = static_cast<BoxedClass*>(obj);
-            if (cls->tp_base)
-                return new BoxedTuple({ static_cast<BoxedClass*>(obj)->tp_base });
-            return EmptyTuple;
+            assert(cls->tp_bases);
+            return cls->tp_bases;
         }
     }
 
@@ -2026,10 +2062,16 @@ extern "C" void dump(void* p) {
             auto cls = static_cast<BoxedClass*>(b);
             printf("Type name: %s\n", getFullNameOfClass(cls).c_str());
 
-            printf("MRO: %s", getFullNameOfClass(cls).c_str());
-            while (cls->tp_base) {
-                printf(" -> %s", getFullNameOfClass(cls->tp_base).c_str());
-                cls = cls->tp_base;
+            printf("MRO:");
+
+            if (cls->tp_mro && cls->tp_mro->cls == tuple_cls) {
+                bool first = true;
+                for (auto b : static_cast<BoxedTuple*>(cls->tp_mro)->elts) {
+                    if (!first)
+                        printf(" ->");
+                    first = false;
+                    printf(" %s", getFullNameOfClass(static_cast<BoxedClass*>(b)).c_str());
+                }
             }
             printf("\n");
         }
@@ -3567,10 +3609,10 @@ Box* typeNew(Box* _cls, Box* arg1, Box* arg2, Box** _args) {
     if (!isSubclass(_cls->cls, type_cls))
         raiseExcHelper(TypeError, "type.__new__(X): X is not a type object (%s)", getTypeName(_cls));
 
-    BoxedClass* cls = static_cast<BoxedClass*>(_cls);
-    if (!isSubclass(cls, type_cls))
-        raiseExcHelper(TypeError, "type.__new__(%s): %s is not a subtype of type", getNameOfClass(cls),
-                       getNameOfClass(cls));
+    BoxedClass* metatype = static_cast<BoxedClass*>(_cls);
+    if (!isSubclass(metatype, type_cls))
+        raiseExcHelper(TypeError, "type.__new__(%s): %s is not a subtype of type", getNameOfClass(metatype),
+                       getNameOfClass(metatype));
 
     if (arg2 == NULL) {
         assert(arg3 == NULL);
@@ -3587,28 +3629,62 @@ Box* typeNew(Box* _cls, Box* arg1, Box* arg2, Box** _args) {
     RELEASE_ASSERT(arg1->cls == str_cls, "");
     BoxedString* name = static_cast<BoxedString*>(arg1);
 
-    BoxedClass* base;
     if (bases->elts.size() == 0) {
         bases = new BoxedTuple({ object_cls });
     }
 
-    RELEASE_ASSERT(bases->elts.size() == 1, "");
-    Box* _base = bases->elts[0];
-    RELEASE_ASSERT(isSubclass(_base->cls, type_cls), "");
-    base = static_cast<BoxedClass*>(_base);
+    // Ported from CPython:
+    int nbases = bases->elts.size();
+    BoxedClass* winner = metatype;
+    for (auto tmp : bases->elts) {
+        auto tmptype = tmp->cls;
+        if (tmptype == classobj_cls)
+            continue;
+        if (isSubclass(winner, tmptype))
+            continue;
+        if (isSubclass(tmptype, winner)) {
+            winner = tmptype;
+            continue;
+        }
+        raiseExcHelper(TypeError, "metaclass conflict: "
+                                  "the metaclass of a derived class "
+                                  "must be a (non-strict) subclass "
+                                  "of the metaclasses of all its bases");
+    }
 
-    if ((base->tp_flags & Py_TPFLAGS_BASETYPE) == 0)
+    if (winner != metatype) {
+        if (getattr(winner, "__new__") != getattr(type_cls, "__new__")) {
+            RELEASE_ASSERT(0, "untested");
+            return callattr(winner, &new_str, CallattrFlags({.cls_only = true, .null_on_nonexistent = false }),
+                            ArgPassSpec(3), arg1, arg2, arg3, _args + 1, NULL);
+        }
+        metatype = winner;
+    }
+
+    BoxedClass* base = best_base(bases);
+    checkAndThrowCAPIException();
+    assert(base);
+    if (!PyType_HasFeature(base, Py_TPFLAGS_BASETYPE))
         raiseExcHelper(TypeError, "type '%.100s' is not an acceptable base type", base->tp_name);
+    assert(isSubclass(base->cls, type_cls));
+
+
+    // TODO I don't think we have to implement the __slots__ memory savings behavior
+    // (we get almost all of that automatically with hidden classes), but adding a __slots__
+    // adds other restrictions (ex for multiple inheritance) that we won't end up enforcing.
+    // I guess it should be ok if we're more permissive?
+    // auto slots = PyDict_GetItemString(attr_dict, "__slots__");
+    // RELEASE_ASSERT(!slots, "__slots__ unsupported");
 
 
     BoxedClass* made;
 
     if (base->instancesHaveDictAttrs() || base->instancesHaveHCAttrs()) {
-        made = BoxedHeapClass::create(cls, base, NULL, base->attrs_offset, base->tp_basicsize, true, name);
+        made = BoxedHeapClass::create(metatype, base, NULL, base->attrs_offset, base->tp_basicsize, true, name, bases);
     } else {
         assert(base->tp_basicsize % sizeof(void*) == 0);
-        made = BoxedHeapClass::create(cls, base, NULL, base->tp_basicsize, base->tp_basicsize + sizeof(HCAttrs), true,
-                                      name);
+        made = BoxedHeapClass::create(metatype, base, NULL, base->tp_basicsize, base->tp_basicsize + sizeof(HCAttrs),
+                                      true, name, bases);
     }
 
     // TODO: how much of these should be in BoxedClass::finishInitialization()?

src/runtime/super.cpp

@@ -16,6 +16,7 @@
 
 #include <sstream>
 
+#include "capi/types.h"
 #include "core/types.h"
 #include "gc/collector.h"
 #include "runtime/objmodel.h"
@@ -66,15 +67,68 @@ Box* superGetattribute(Box* _s, Box* _attr) {
     }
 
     if (!skip) {
-        // We don't support multiple inheritance yet, so the lookup order is simple:
-        Box* r = typeLookup(s->type->tp_base, attr->s, NULL);
-
-        if (r) {
-            return processDescriptor(r, (s->obj == s->obj_type ? None : s->obj), s->obj_type);
+        PyObject* mro, *res, *tmp, *dict;
+        PyTypeObject* starttype;
+        descrgetfunc f;
+        Py_ssize_t i, n;
+
+        starttype = s->obj_type;
+        mro = starttype->tp_mro;
+
+        if (mro == NULL)
+            n = 0;
+        else {
+            assert(PyTuple_Check(mro));
+            n = PyTuple_GET_SIZE(mro);
+        }
+        for (i = 0; i < n; i++) {
+            if ((PyObject*)(s->type) == PyTuple_GET_ITEM(mro, i))
+                break;
+        }
+        i++;
+        res = NULL;
+        for (; i < n; i++) {
+            tmp = PyTuple_GET_ITEM(mro, i);
+
+// Pyston change:
+#if 0
+            if (PyType_Check(tmp))
+                dict = ((PyTypeObject *)tmp)->tp_dict;
+            else if (PyClass_Check(tmp))
+                dict = ((PyClassObject *)tmp)->cl_dict;
+            else
+                continue;
+            res = PyDict_GetItem(dict, name);
+#endif
+            res = tmp->getattr(attr->s);
+
+            if (res != NULL) {
+// Pyston change:
+#if 0
+                Py_INCREF(res);
+                f = Py_TYPE(res)->tp_descr_get;
+                if (f != NULL) {
+                    tmp = f(res,
+                        /* Only pass 'obj' param if
+                           this is instance-mode sper
+                           (See SF ID #743627)
+                        */
+                        (s->obj == (PyObject *)
+                                    s->obj_type
+                            ? (PyObject *)NULL
+                            : s->obj),
+                        (PyObject *)starttype);
+                    Py_DECREF(res);
+                    res = tmp;
+                }
+#endif
+                return processDescriptor(res, (s->obj == s->obj_type ? None : s->obj), s->obj_type);
+            }
         }
     }
 
     Box* r = typeLookup(s->cls, attr->s, NULL);
+    // TODO implement this
     RELEASE_ASSERT(r, "should call the equivalent of objectGetattr here");
     return processDescriptor(r, s, s->cls);
 }

src/runtime/types.cpp

@@ -97,10 +97,16 @@ extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems)
         assert(static_cast<BoxedClass*>(e)->is_pyston_class);
     }
 #endif
-    BoxedClass* b = cls;
-    while (b) {
-        ASSERT(b->is_pyston_class, "%s (%s)", cls->tp_name, b->tp_name);
-        b = b->tp_base;
+    if (!cls->tp_mro) {
+        assert(!list_cls);
+    } else {
+        assert(cls->tp_mro && "maybe we should just skip these checks if !mro");
+        assert(cls->tp_mro->cls == tuple_cls);
+        for (auto b : static_cast<BoxedTuple*>(cls->tp_mro)->elts) {
+            assert(isSubclass(b->cls, type_cls));
+            ASSERT(static_cast<BoxedClass*>(b)->is_pyston_class, "%s (%s)", cls->tp_name,
+                   static_cast<BoxedClass*>(b)->tp_name);
+        }
     }
 #endif
 
@@ -903,6 +909,15 @@ Box* typeHash(BoxedClass* self) {
     return boxInt(reinterpret_cast<intptr_t>(self) >> 4);
 }
 
+Box* typeMro(BoxedClass* self) {
+    assert(isSubclass(self->cls, type_cls));
+
+    Box* r = mro_external(self);
+    if (!r)
+        throwCAPIException();
+    return r;
+}
+
 Box* moduleRepr(BoxedModule* m) {
     assert(m->cls == module_cls);
 
@@ -1229,19 +1244,15 @@ void setupRuntime() {
     PyObject_Init(object_cls, type_cls);
     PyObject_Init(type_cls, type_cls);
 
-    object_cls->finishInitialization();
-    type_cls->finishInitialization();
-
-    none_cls = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, sizeof(Box), false, NULL);
+    none_cls = new BoxedHeapClass(object_cls, NULL, 0, sizeof(Box), false, NULL);
     None = new (none_cls) Box();
     assert(None->cls);
     gc::registerPermanentRoot(None);
 
     // You can't actually have an instance of basestring
-    basestring_cls = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, sizeof(Box), false, NULL);
+    basestring_cls = new BoxedHeapClass(object_cls, NULL, 0, sizeof(Box), false, NULL);
 
-    // TODO we leak all the string data!
-    str_cls = BoxedHeapClass::create(type_cls, basestring_cls, NULL, 0, sizeof(BoxedString), false, NULL);
+    str_cls = new BoxedHeapClass(basestring_cls, NULL, 0, sizeof(BoxedString), false, NULL);
 
     // Hold off on assigning names until str_cls is ready
     object_cls->tp_name = "object";
@@ -1268,9 +1279,26 @@ void setupRuntime() {
     object_cls->giveAttr("__base__", None);
 
 
-    tuple_cls = BoxedHeapClass::create(type_cls, object_cls, &tupleGCHandler, 0, sizeof(BoxedTuple), false, "tuple");
+    tuple_cls = new BoxedHeapClass(object_cls, &tupleGCHandler, 0, sizeof(BoxedTuple), false, boxStrConstant("tuple"));
     EmptyTuple = new BoxedTuple({});
     gc::registerPermanentRoot(EmptyTuple);
+    list_cls = new BoxedHeapClass(object_cls, &listGCHandler, 0, sizeof(BoxedList), false, boxStrConstant("list"));
+
+    // Kind of hacky, but it's easier to manually construct the mro for a couple key classes
+    // than try to make the MRO construction code be safe against say, tuple_cls not having
+    // an mro (since the mro is stored as a tuple).
+    tuple_cls->tp_mro = new BoxedTuple({ tuple_cls, object_cls });
+    list_cls->tp_mro = new BoxedTuple({ list_cls, object_cls });
+    type_cls->tp_mro = new BoxedTuple({ type_cls, object_cls });
+
+    object_cls->finishInitialization();
+    type_cls->finishInitialization();
+    basestring_cls->finishInitialization();
+    str_cls->finishInitialization();
+    none_cls->finishInitialization();
+    tuple_cls->finishInitialization();
+    list_cls->finishInitialization();
+
 
 
     module_cls = BoxedHeapClass::create(type_cls, object_cls, NULL, offsetof(BoxedModule, attrs), sizeof(BoxedModule),
@@ -1357,6 +1385,9 @@ void setupRuntime() {
     type_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)typeRepr, STR, 1)));
     type_cls->giveAttr("__hash__", new BoxedFunction(boxRTFunction((void*)typeHash, BOXED_INT, 1)));
     type_cls->giveAttr("__module__", new (pyston_getset_cls) BoxedGetsetDescriptor(typeModule, typeSetModule, NULL));
+    type_cls->giveAttr("__mro__",
+                       new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedClass, tp_mro)));
+    type_cls->giveAttr("mro", new BoxedFunction(boxRTFunction((void*)typeMro, UNKNOWN, 1)));
     type_cls->freeze();
 
     none_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)noneRepr, STR, 1)));

src/runtime/types.h

@@ -213,9 +213,9 @@ public:
     BoxedString* ht_name;
     PyObject** ht_slots;
 
-    // This is the preferred way to construct new types:
+    // These functions are the preferred way to construct new types:
     static BoxedHeapClass* create(BoxedClass* metatype, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
-                                  int instance_size, bool is_user_defined, BoxedString* name);
+                                  int instance_size, bool is_user_defined, BoxedString* name, BoxedTuple* bases);
     static BoxedHeapClass* create(BoxedClass* metatype, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
                                   int instance_size, bool is_user_defined, const std::string& name);
 
@@ -227,6 +227,8 @@ private:
                    BoxedString* name);
 
     friend void setupRuntime();
+
+    DEFAULT_CLASS(type_cls);
 };
 
 static_assert(sizeof(pyston::Box) == sizeof(struct _object), "");

test/tests/multiple_inheritance.py

-# expected: fail
-# - wip
-
 # Testing the basic multiple-inheritance rules and functionality:
 
 class C(object):
@@ -92,3 +89,6 @@ s = S()
 s.c()
 s.d()
 s.f()
+
+for cls in [object, tuple, list, type, int, bool]:
+    print cls.__mro__