Keep tp_new in sync with cls.__new__

include/object.h

@@ -322,6 +322,17 @@ typedef struct {
 } PyBufferProcs;
 
 
+// Pyston change: hacks to allow C++ features
+#ifndef __cplusplus
+typedef struct _typeobject PyTypeObject;
+#else
+namespace pyston {
+class BoxedClass;
+}
+typedef pyston::BoxedClass PyTypeObject;
+#endif
+
+
 typedef void (*freefunc)(void *);
 typedef void (*destructor)(PyObject *);
 typedef int (*printfunc)(PyObject *, FILE *, int);
@@ -338,8 +349,9 @@ typedef PyObject *(*iternextfunc) (PyObject *);
 typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *);
 typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *);
 typedef int (*initproc)(PyObject *, PyObject *, PyObject *);
-typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *);
-typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t);
+// Pyston change: renamed from struct _typeobject to PyTypeObject
+typedef PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *);
+typedef PyObject *(*allocfunc)(PyTypeObject *, Py_ssize_t);
 
 // Pyston change: moved the field definitions of a PyTypeObject to this macro
 #define PyTypeObject_BODY                                                       \
@@ -449,17 +461,6 @@ struct _typeobject {
     bool _flags[2];
 };
 
-// Pyston change: hacks to allow C++ features
-#ifndef __cplusplus
-typedef struct _typeobject PyTypeObject;
-#else
-namespace pyston {
-class BoxedClass;
-}
-typedef pyston::BoxedClass PyTypeObject;
-#endif
-
-
 /* The *real* layout of a type object when allocated on the heap */
 typedef struct _heaptypeobject {
     /* Note: there's a dependency on the order of these members

src/Makefile

@@ -708,7 +708,7 @@ pyston_profile: $(PROFILE_OBJS) $(LLVM_PROFILE_DEPS)
 
 .PHONY: clean
 clean:
-	@ find . $(TOOLS_DIR) ../test \( -name '*.o' -o -name '*.d' -o -name '*.py_cache' -o -name '*.bc' -o -name '*.o.ll' -o -name '*.pub.ll' -o -name '*.cache' -o -name 'stdlib*.ll' -o -name '*.pyc' -o -name '*.so' -o -name '*.a' -o -name '*.expected_cache' -o -name '*.pch' \) -print -delete
+	@ find . $(TOOLS_DIR) ../test ../lib_python/2.7_Modules \( -name '*.o' -o -name '*.d' -o -name '*.py_cache' -o -name '*.bc' -o -name '*.o.ll' -o -name '*.pub.ll' -o -name '*.cache' -o -name 'stdlib*.ll' -o -name '*.pyc' -o -name '*.so' -o -name '*.a' -o -name '*.expected_cache' -o -name '*.pch' \) -print -delete
 	@ find \( -name 'pyston*' -executable -type f \) -print -delete
 	@ find $(TOOLS_DIR) -maxdepth 0 -executable -type f -print -delete
 	@ rm -rf oprofile_data
@@ -889,11 +889,11 @@ ext: ../test/test_extension/test.so
 
 $(FROM_CPYTHON_SRCS:.c=.o): %.o: %.c $(BUILD_SYSTEM_DEPS)
 	$(ECHO) Compiling C file to $@
-	$(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $<.d -O0
+	$(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $(patsubst %.o,%.d,$@) -O0
 
 $(FROM_CPYTHON_SRCS:.c=.release.o): %.release.o: %.c $(BUILD_SYSTEM_DEPS)
 	$(ECHO) Compiling C file to $@
-	$(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $<.d
+	$(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $(patsubst %.o,%.d,$@)
 
 # These are necessary until we support unicode:
 ../lib_python/2.7_Modules/_sre.o: EXT_CFLAGS += -Wno-sometimes-uninitialized

src/core/types.h

@@ -399,6 +399,7 @@ public:
 static_assert(offsetof(BoxVar, ob_size) == offsetof(struct _varobject, ob_size), "");
 
 extern "C" const std::string* getTypeName(Box* o);
+std::string getFullTypeName(Box* o);
 
 
 
@@ -442,11 +443,8 @@ public:
 
     BoxedClass(BoxedClass* metaclass, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset, int instance_size,
                bool is_user_defined);
-    void freeze() {
-        assert(!is_constant);
-        assert(getattr("__name__")); // otherwise debugging will be very hard
-        is_constant = true;
-    }
+
+    void freeze();
 };
 
 static_assert(sizeof(pyston::Box) == sizeof(struct _object), "");

src/runtime/capi.cpp

@@ -32,8 +32,9 @@ BoxedClass* method_cls;
 class BoxedMethodDescriptor : public Box {
 public:
     PyMethodDef* method;
+    BoxedClass* type;
 
-    BoxedMethodDescriptor(PyMethodDef* method) : Box(method_cls), method(method) {}
+    BoxedMethodDescriptor(PyMethodDef* method, BoxedClass* type) : Box(method_cls), method(method), type(type) {}
 
     static Box* __get__(BoxedMethodDescriptor* self, Box* inst, Box* owner) {
         RELEASE_ASSERT(self->cls == method_cls, "");
@@ -51,6 +52,10 @@ public:
         assert(varargs->cls == tuple_cls);
         assert(kwargs->cls == dict_cls);
 
+        if (!isSubclass(obj->cls, self->type))
+            raiseExcHelper(TypeError, "descriptor '%s' requires a '%s' object but received a '%s'",
+                           self->method->ml_name, getFullNameOfClass(self->type).c_str(), getFullTypeName(obj).c_str());
+
         threading::GLPromoteRegion _gil_lock;
 
         int ml_flags = self->method->ml_flags;
@@ -138,6 +143,62 @@ extern "C" PyObject* PyType_GenericAlloc(PyTypeObject* cls, Py_ssize_t nitems) {
     return rtn;
 }
 
+/*
+// These are for supporting things like tp_call.
+// tp_new is handled differently, which is the only one supported right now,
+// which is why these are (temporarily) commented out.
+BoxedClass* wrapperdescr_cls, *wrapperobject_cls;
+class BoxedWrapperDescriptor : public Box {
+public:
+    const int offset;
+    BoxedWrapperDescriptor(int offset) : Box(wrapperdescr_cls), offset(offset) {}
+
+    static Box* __get__(BoxedWrapperDescriptor* self, Box* inst, Box* owner);
+};
+
+class BoxedWrapperObject : public Box {
+public:
+    BoxedWrapperDescriptor* descr;
+    Box* obj;
+
+    BoxedWrapperObject(BoxedWrapperDescriptor* descr, Box* obj) : Box(wrapperobject_cls), descr(descr), obj(obj) {}
+
+    static Box* __call__(BoxedWrapperObject* self, Box* args, Box* kwds) {
+        assert(args->cls == tuple_cls);
+        assert(kwds->cls == dict_cls);
+
+        abort();
+    }
+};
+
+Box* BoxedWrapperDescriptor::__get__(BoxedWrapperDescriptor* self, Box* inst, Box* owner) {
+    RELEASE_ASSERT(self->cls == wrapperdescr_cls, "");
+
+    if (inst == None)
+        return self;
+
+    return new BoxedWrapperObject(self, inst);
+}
+*/
+
+PyObject* tp_new_wrapper(PyTypeObject* self, BoxedTuple* args, Box* kwds) {
+    RELEASE_ASSERT(isSubclass(self->cls, type_cls), "");
+
+    // ASSERT(self->tp_new != Py_CallPythonNew, "going to get in an infinite loop");
+
+    RELEASE_ASSERT(args->cls == tuple_cls, "");
+    RELEASE_ASSERT(kwds->cls == dict_cls, "");
+    RELEASE_ASSERT(args->elts.size() >= 1, "");
+
+    BoxedClass* subtype = static_cast<BoxedClass*>(args->elts[0]);
+    RELEASE_ASSERT(isSubclass(subtype->cls, type_cls), "");
+    RELEASE_ASSERT(isSubclass(subtype, self), "");
+
+    BoxedTuple* new_args = new BoxedTuple(BoxedTuple::GCVector(args->elts.begin() + 1, args->elts.end()));
+
+    return self->tp_new(subtype, new_args, kwds);
+}
+
 extern "C" int PyType_Ready(PyTypeObject* cls) {
     gc::registerNonheapRootObject(cls);
 
@@ -186,7 +247,7 @@ extern "C" int PyType_Ready(PyTypeObject* cls) {
     INITIALIZE(cls->dependent_icgetattrs);
 #undef INITIALIZE
 
-    cls->base = object_cls;
+    BoxedClass* base = cls->base = object_cls;
     if (!cls->cls)
         cls->cls = cls->base->cls;
 
@@ -196,23 +257,12 @@ extern "C" int PyType_Ready(PyTypeObject* cls) {
     // tp_basicsize, tp_itemsize
     // tp_doc
 
-    if (cls->tp_new) {
-        // Wrap the tp_new in a MethodDescriptor, which will take care of
-        // converting between C API mode and Pyston mode:
-        static bool initialized = false;
-        static PyMethodDef new_method_def;
-        if (!initialized) {
-            new_method_def.ml_name = "__new__";
-            new_method_def.ml_meth = (PyCFunction)cls->tp_new;
-            new_method_def.ml_flags = METH_VARARGS | METH_KEYWORDS;
-            new_method_def.ml_doc = NULL;
-            initialized = true;
-        }
-        cls->giveAttr("__new__", new BoxedMethodDescriptor(&new_method_def));
+    if (!cls->tp_new && base != object_cls)
+        cls->tp_new = base->tp_new;
 
-        // Clear tp_new for now since we are not keeping it updated, and would rather that
-        // clients crash instead of use unguaranteed behavior:
-        cls->tp_new = NULL;
+    if (cls->tp_new) {
+        cls->giveAttr("__new__",
+                      new BoxedCApiFunction(METH_VARARGS | METH_KEYWORDS, cls, "__new__", (PyCFunction)tp_new_wrapper));
     }
 
     if (!cls->tp_alloc) {
@@ -220,7 +270,7 @@ extern "C" int PyType_Ready(PyTypeObject* cls) {
     }
 
     for (PyMethodDef* method = cls->tp_methods; method && method->ml_name; ++method) {
-        cls->giveAttr(method->ml_name, new BoxedMethodDescriptor(method));
+        cls->giveAttr(method->ml_name, new BoxedMethodDescriptor(method, cls));
     }
 
     for (PyMemberDef* member = cls->tp_members; member && member->name; ++member) {
@@ -894,6 +944,20 @@ void setupCAPI() {
     method_cls->giveAttr("__call__", new BoxedFunction(boxRTFunction((void*)BoxedMethodDescriptor::__call__, UNKNOWN, 2,
                                                                      0, true, true)));
     method_cls->freeze();
+
+    /*
+    wrapperdescr_cls = new BoxedClass(type_cls, object_cls, NULL, 0, sizeof(BoxedWrapperDescriptor), false);
+    wrapperdescr_cls->giveAttr("__name__", boxStrConstant("wrapper_descriptor"));
+    wrapperdescr_cls->giveAttr("__get__",
+                               new BoxedFunction(boxRTFunction((void*)BoxedWrapperDescriptor::__get__, UNKNOWN, 3)));
+    wrapperdescr_cls->freeze();
+
+    wrapperobject_cls = new BoxedClass(type_cls, object_cls, NULL, 0, sizeof(BoxedWrapperObject), false);
+    wrapperobject_cls->giveAttr("__name__", boxStrConstant("method-wrapper"));
+    wrapperobject_cls->giveAttr(
+        "__call__", new BoxedFunction(boxRTFunction((void*)BoxedWrapperObject::__call__, UNKNOWN, 1, 0, true, true)));
+    wrapperobject_cls->freeze();
+    */
 }
 
 void teardownCAPI() {

src/runtime/objmodel.cpp

@@ -54,6 +54,11 @@
 
 namespace pyston {
 
+// TODO should centralize all of these:
+static const std::string _call_str("__call__"), _new_str("__new__"), _init_str("__init__"), _get_str("__get__");
+static const std::string _getattr_str("__getattr__");
+static const std::string _getattribute_str("__getattribute__");
+
 struct GetattrRewriteArgs {
     Rewriter* rewriter;
     RewriterVarUsage obj;
@@ -405,10 +410,41 @@ extern "C" Box** unpackIntoArray(Box* obj, int64_t expected_size) {
     return &elts[0];
 }
 
+PyObject* Py_CallPythonNew(PyTypeObject* self, PyObject* args, PyObject* kwds) {
+    try {
+        Py_FatalError("this function is unverified");
+
+        Box* new_attr = typeLookup(self, _new_str, NULL);
+        assert(new_attr);
+        new_attr = processDescriptor(new_attr, None, self);
+        return runtimeCallInternal(new_attr, NULL, ArgPassSpec(1, 0, true, true), self, args, kwds, NULL, NULL);
+    } catch (Box* e) {
+        abort();
+    }
+}
+
+void BoxedClass::freeze() {
+    assert(!is_constant);
+    assert(getattr("__name__")); // otherwise debugging will be very hard
+
+    if (!tp_new) {
+        this->tp_new = &Py_CallPythonNew;
+    } else if (tp_new != Py_CallPythonNew) {
+        ASSERT(0, "need to set __new__?");
+    }
+
+    is_constant = true;
+}
+
 BoxedClass::BoxedClass(BoxedClass* metaclass, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
                        int instance_size, bool is_user_defined)
-    : BoxVar(metaclass, 0), tp_basicsize(instance_size), tp_dealloc(NULL), base(base), gc_visit(gc_visit),
-      attrs_offset(attrs_offset), is_constant(false), is_user_defined(is_user_defined) {
+    : BoxVar(metaclass, 0), base(base), gc_visit(gc_visit), attrs_offset(attrs_offset), is_constant(false),
+      is_user_defined(is_user_defined) {
+
+    // Zero out the CPython tp_* slots:
+    memset(&tp_name, 0, (char*)(&tp_version_tag + 1) - (char*)(&tp_name));
+    tp_basicsize = instance_size;
+
     if (metaclass == NULL) {
         assert(type_cls == NULL);
     } else {
@@ -600,9 +636,6 @@ Box* Box::getattr(const std::string& attr, GetattrRewriteArgs* rewrite_args) {
     return rtn;
 }
 
-// TODO should centralize all of these:
-static const std::string _call_str("__call__"), _new_str("__new__"), _init_str("__init__"), _get_str("__get__");
-
 void Box::setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite_args) {
     assert(cls->instancesHaveAttrs());
     assert(gc::isValidGCObject(val));
@@ -621,28 +654,9 @@ void Box::setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite
 
 
     static const std::string none_str("None");
-    static const std::string getattr_str("__getattr__");
-    static const std::string getattribute_str("__getattribute__");
 
     RELEASE_ASSERT(attr != none_str || this == builtins_module, "can't assign to None");
 
-    if (isSubclass(this->cls, type_cls)) {
-        BoxedClass* self = static_cast<BoxedClass*>(this);
-
-        if (attr == getattr_str || attr == getattribute_str) {
-            // Will have to embed the clear in the IC, so just disable the patching for now:
-            rewrite_args = NULL;
-
-            // TODO should put this clearing behavior somewhere else, since there are probably more
-            // cases in which we want to do it.
-            self->dependent_icgetattrs.invalidateAll();
-        }
-
-        if (attr == "__base__" && getattr("__base__")) {
-            raiseExcHelper(TypeError, "readonly attribute");
-        }
-    }
-
     HCAttrs* attrs = getAttrsPtr();
     HiddenClass* hcls = attrs->hcls;
     int numattrs = hcls->attr_offsets.size();
@@ -799,7 +813,7 @@ Box* descriptorClsSpecialCases(GetattrRewriteArgs* rewrite_args, BoxedClass* cls
         if (rewrite_args)
             r_descr.addAttrGuard(BOX_CLS_OFFSET, (uint64_t)descr->cls);
 
-        if (!for_call) {
+        if (!for_call && descr->cls == function_cls) {
             if (rewrite_args) {
                 assert(rewrite_args->call_done_guarding);
                 rewrite_args->rewriter->setDoneGuarding();
@@ -811,17 +825,17 @@ Box* descriptorClsSpecialCases(GetattrRewriteArgs* rewrite_args, BoxedClass* cls
                 rewrite_args->out_success = true;
             }
             return boxUnboundInstanceMethod(descr);
-        } else {
-            if (rewrite_args) {
-                if (rewrite_args->call_done_guarding)
-                    rewrite_args->rewriter->setDoneGuarding();
-                rewrite_args->obj.setDoneUsing();
-                rewrite_args->out_success = true;
-                rewrite_args->out_rtn = std::move(r_descr);
-            }
-            // leave should_bind_out set to false
-            return descr;
         }
+
+        if (rewrite_args) {
+            if (rewrite_args->call_done_guarding)
+                rewrite_args->rewriter->setDoneGuarding();
+            rewrite_args->obj.setDoneUsing();
+            rewrite_args->out_success = true;
+            rewrite_args->out_rtn = std::move(r_descr);
+        }
+        // leave should_bind_out set to false
+        return descr;
     }
 
     // Special case: member descriptor
@@ -1447,6 +1461,29 @@ void setattrInternal(Box* obj, const std::string& attr, Box* val, SetattrRewrite
         descr = typeLookup(obj->cls, attr, NULL);
     }
 
+    if (isSubclass(obj->cls, type_cls)) {
+        BoxedClass* self = static_cast<BoxedClass*>(obj);
+
+        if (attr == _getattr_str || attr == _getattribute_str) {
+            // Will have to embed the clear in the IC, so just disable the patching for now:
+            rewrite_args = NULL;
+
+            // TODO should put this clearing behavior somewhere else, since there are probably more
+            // cases in which we want to do it.
+            self->dependent_icgetattrs.invalidateAll();
+        }
+
+        if (attr == "__base__" && self->getattr("__base__"))
+            raiseExcHelper(TypeError, "readonly attribute");
+
+        if (attr == "__new__") {
+            self->tp_new = &Py_CallPythonNew;
+            // TODO update subclasses
+
+            rewrite_args = NULL;
+        }
+    }
+
     Box* _set_ = NULL;
     RewriterVarUsage r_set(RewriterVarUsage::empty());
     if (descr) {
@@ -3351,6 +3388,7 @@ Box* typeNew(Box* _cls, Box* arg1, Box* arg2, Box** _args) {
 
     // TODO this function (typeNew) should probably call PyType_Ready
 
+    made->tp_new = base->tp_new;
     made->tp_alloc = reinterpret_cast<decltype(cls->tp_alloc)>(PyType_GenericAlloc);
 
     return made;
@@ -3413,15 +3451,15 @@ Box* typeCallInternal(BoxedFunction* f, CallRewriteArgs* rewrite_args, ArgPassSp
 
     if (rewrite_args) {
         GetattrRewriteArgs grewrite_args(rewrite_args->rewriter, r_ccls.addUse(), rewrite_args->destination, false);
+        // TODO: if tp_new != Py_CallPythonNew, call that instead?
         new_attr = typeLookup(cls, _new_str, &grewrite_args);
 
         if (!grewrite_args.out_success)
             rewrite_args = NULL;
         else {
-            if (new_attr) {
-                r_new = std::move(grewrite_args.out_rtn);
-                r_new.addGuard((intptr_t)new_attr);
-            }
+            assert(new_attr);
+            r_new = std::move(grewrite_args.out_rtn);
+            r_new.addGuard((intptr_t)new_attr);
         }
 
         // Special-case functions to allow them to still rewrite: