Support setting __dict__

It's trickier than just setting all of the attributes, since any updates
to either the original dict or to the object's attributes will get
mirrored in the other object.  I don't know if anyone uses this,
but I don't think there's any good way for us to tell if this is
going to happen so we just have to be conservative.

So, add a new distinction between "types" of hidden classes, and add
a new "dict backed" type.  Instead of having an array of attributes,
has a single attribute which is a dict.

There are some pretty tricky corner cases that we don't support yet,
such as if you access and save __dict__, then set __dict__, and then
try to access the saved version.  At least we can detect that and fail.

src/codegen/unwinding.cpp

@@ -658,7 +658,7 @@ BoxedDict* getLocals(bool only_user_visible, bool includeClosure) {
             // Add the locals from the closure
             for (; closure != NULL; closure = closure->parent) {
                 assert(closure->cls == closure_cls);
-                for (auto& attr_offset : closure->attrs.hcls->attr_offsets) {
+                for (auto& attr_offset : closure->attrs.hcls->getAttrOffsets()) {
                     const std::string& name = attr_offset.first();
                     int offset = attr_offset.second;
                     Box* val = closure->attrs.attr_list->attrs[offset];

src/core/types.h

@@ -414,7 +414,7 @@ struct DelattrRewriteArgs;
 
 struct HCAttrs {
 public:
-    struct AttrList : public GCAllocated<gc::GCKind::PRECISE> {
+    struct AttrList {
         Box* attrs[0];
     };
 
@@ -428,6 +428,9 @@ class BoxedDict;
 class BoxedString;
 
 class Box {
+private:
+    BoxedDict** getDictPtr();
+
 public:
     // Add a no-op constructor to make sure that we don't zero-initialize cls
     Box() {}
@@ -441,6 +444,7 @@ public:
     llvm::iterator_range<BoxIterator> pyElements();
 
     HCAttrs* getHCAttrsPtr();
+    void setDict(BoxedDict* d);
     BoxedDict* getDict();
 
     void setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite_args);

src/runtime/builtin_modules/builtins.cpp

@@ -170,12 +170,12 @@ extern "C" Box* dir(Box* obj) {
         result = new BoxedList();
     }
 
-    for (auto const& kv : obj->cls->attrs.hcls->attr_offsets) {
+    for (auto const& kv : obj->cls->attrs.hcls->getAttrOffsets()) {
         listAppend(result, boxString(kv.first()));
     }
     if (obj->cls->instancesHaveHCAttrs()) {
         HCAttrs* attrs = obj->getHCAttrsPtr();
-        for (auto const& kv : attrs->hcls->attr_offsets) {
+        for (auto const& kv : attrs->hcls->getAttrOffsets()) {
             listAppend(result, boxString(kv.first()));
         }
     }

src/runtime/dict.cpp

@@ -87,6 +87,8 @@ Box* dictValues(BoxedDict* self) {
 }
 
 Box* dictKeys(BoxedDict* self) {
+    RELEASE_ASSERT(isSubclass(self->cls, dict_cls), "");
+
     BoxedList* rtn = new BoxedList();
     for (const auto& p : self->d) {
         listAppendInternal(rtn, p.first);
@@ -321,14 +323,14 @@ Box* dictDelitem(BoxedDict* self, Box* k) {
 }
 
 extern "C" int PyDict_DelItem(PyObject* op, PyObject* key) noexcept {
+    ASSERT(isSubclass(op->cls, dict_cls) || op->cls == attrwrapper_cls, "%s", getTypeName(op));
     try {
-        dictDelitem((BoxedDict*)op, key);
+        delitem(op, key);
+        return 0;
     } catch (ExcInfo e) {
         setCAPIException(e);
         return -1;
     }
-
-    return 0;
 }
 
 extern "C" int PyDict_DelItemString(PyObject* v, const char* key) noexcept {

src/runtime/objmodel.cpp

@@ -481,6 +481,8 @@ const char* getNameOfClass(BoxedClass* cls) {
 }
 
 HiddenClass* HiddenClass::getOrMakeChild(const std::string& attr) {
+    assert(type == NORMAL);
+
     auto it = children.find(attr);
     if (it != children.end())
         return it->second;
@@ -498,6 +500,7 @@ HiddenClass* HiddenClass::getOrMakeChild(const std::string& attr) {
  * del attr from current HiddenClass, pertain the orders of remaining attrs
  */
 HiddenClass* HiddenClass::delAttrToMakeHC(const std::string& attr) {
+    assert(type == NORMAL);
     int idx = getOffset(attr);
     assert(idx >= 0);
 
@@ -527,13 +530,26 @@ HCAttrs* Box::getHCAttrsPtr() {
     return reinterpret_cast<HCAttrs*>(p);
 }
 
-BoxedDict* Box::getDict() {
+BoxedDict** Box::getDictPtr() {
     assert(cls->instancesHaveDictAttrs());
 
     char* p = reinterpret_cast<char*>(this);
     p += cls->tp_dictoffset;
 
     BoxedDict** d_ptr = reinterpret_cast<BoxedDict**>(p);
+    return d_ptr;
+}
+
+void Box::setDict(BoxedDict* d) {
+    assert(cls->instancesHaveDictAttrs());
+
+    *getDictPtr() = d;
+}
+
+BoxedDict* Box::getDict() {
+    assert(cls->instancesHaveDictAttrs());
+
+    BoxedDict** d_ptr = getDictPtr();
     BoxedDict* d = *d_ptr;
     if (!d) {
         d = *d_ptr = new BoxedDict();
@@ -574,12 +590,25 @@ Box* Box::getattr(const std::string& attr, GetattrRewriteArgs* rewrite_args) {
     // structure (ex user class) and the same hidden classes, because
     // otherwise the guard will fail anyway.;
     if (cls->instancesHaveHCAttrs()) {
-        if (rewrite_args)
-            rewrite_args->out_success = true;
-
         HCAttrs* attrs = getHCAttrsPtr();
         HiddenClass* hcls = attrs->hcls;
 
+        if (hcls->type == HiddenClass::DICT_BACKED) {
+            if (rewrite_args)
+                assert(!rewrite_args->out_success);
+            rewrite_args = NULL;
+            Box* d = attrs->attr_list->attrs[0];
+            assert(d);
+            Box* r = PyDict_GetItemString(d, attr.c_str());
+            // r can be NULL if the item didn't exist
+            return r;
+        }
+
+        assert(hcls->type == HiddenClass::NORMAL);
+
+        if (rewrite_args)
+            rewrite_args->out_success = true;
+
         if (rewrite_args) {
             if (!rewrite_args->obj_hcls_guarded)
                 rewrite_args->obj->addAttrGuard(cls->attrs_offset + HCATTRS_HCLS_OFFSET, (intptr_t)hcls);
@@ -641,7 +670,20 @@ void Box::setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite
     if (cls->instancesHaveHCAttrs()) {
         HCAttrs* attrs = getHCAttrsPtr();
         HiddenClass* hcls = attrs->hcls;
-        int numattrs = hcls->attr_offsets.size();
+
+        if (hcls->type == HiddenClass::DICT_BACKED) {
+            if (rewrite_args)
+                assert(!rewrite_args->out_success);
+            rewrite_args = NULL;
+            Box* d = attrs->attr_list->attrs[0];
+            assert(d);
+            PyDict_SetItemString(d, attr.c_str(), val);
+            checkAndThrowCAPIException();
+            return;
+        }
+
+        assert(hcls->type == HiddenClass::NORMAL);
+        int numattrs = hcls->getAttrOffsets().size();
 
         int offset = hcls->getOffset(attr);
 
@@ -672,10 +714,10 @@ void Box::setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite
         HiddenClass* new_hcls = hcls->getOrMakeChild(attr);
 
         // TODO need to make sure we don't need to rearrange the attributes
-        assert(new_hcls->attr_offsets[attr] == numattrs);
+        assert(new_hcls->getAttrOffsets().lookup(attr) == numattrs);
 #ifndef NDEBUG
-        for (const auto& p : hcls->attr_offsets) {
-            assert(new_hcls->attr_offsets[p.first()] == p.second);
+        for (const auto& p : hcls->getAttrOffsets()) {
+            assert(new_hcls->getAttrOffsets().lookup(p.first()) == p.second);
         }
 #endif
 
@@ -3629,12 +3671,25 @@ void Box::delattr(const std::string& attr, DelattrRewriteArgs* rewrite_args) {
         // as soon as the hcls changes, the guard on hidden class won't pass.
         HCAttrs* attrs = getHCAttrsPtr();
         HiddenClass* hcls = attrs->hcls;
+
+        if (hcls->type == HiddenClass::DICT_BACKED) {
+            if (rewrite_args)
+                assert(!rewrite_args->out_success);
+            rewrite_args = NULL;
+            Box* d = attrs->attr_list->attrs[0];
+            assert(d);
+            PyDict_DelItemString(d, attr.c_str());
+            checkAndThrowCAPIException();
+            return;
+        }
+
+        assert(hcls->type == HiddenClass::NORMAL);
         HiddenClass* new_hcls = hcls->delAttrToMakeHC(attr);
 
         // The order of attributes is pertained as delAttrToMakeHC constructs
         // the new HiddenClass by invoking getOrMakeChild in the prevous order
         // of remaining attributes
-        int num_attrs = hcls->attr_offsets.size();
+        int num_attrs = hcls->getAttrOffsets().size();
         int offset = hcls->getOffset(attr);
         assert(offset >= 0);
         Box** start = attrs->attr_list->attrs;
@@ -4372,7 +4427,7 @@ extern "C" Box* importStar(Box* _from_module, BoxedModule* to_module) {
     }
 
     HCAttrs* module_attrs = from_module->getHCAttrsPtr();
-    for (auto& p : module_attrs->hcls->attr_offsets) {
+    for (auto& p : module_attrs->hcls->getAttrOffsets()) {
         if (p.first()[0] == '_')
             continue;
 

src/runtime/types.h

@@ -264,14 +264,29 @@ static_assert(sizeof(pyston::BoxedHeapClass) == sizeof(PyHeapTypeObject), "");
 
 
 class HiddenClass : public GCAllocated<gc::GCKind::HIDDEN_CLASS> {
+public:
+    // We have a couple different storage strategies for attributes, which
+    // are distinguished by having a different hidden class type.
+    enum HCType {
+        NORMAL,      // attributes stored in attributes array, name->offset map stored in hidden class
+        DICT_BACKED, // first attribute in array is a dict-like object which stores the attributes
+    } const type;
+
+    static HiddenClass* dict_backed;
+
 private:
-    HiddenClass() {}
-    HiddenClass(HiddenClass* parent) : attr_offsets() {
+    HiddenClass(HCType type) : type(type) {}
+    HiddenClass(HiddenClass* parent) : type(NORMAL), attr_offsets() {
+        assert(parent->type == NORMAL);
         for (auto& p : parent->attr_offsets) {
             this->attr_offsets.insert(&p);
         }
     }
 
+    // Only makes sense for NORMAL hidden classes.  Clients should access through getAttrOffsets():
+    llvm::StringMap<int> attr_offsets;
+    llvm::StringMap<HiddenClass*> children;
+
 public:
     static HiddenClass* makeRoot() {
 #ifndef NDEBUG
@@ -279,27 +294,45 @@ public:
         assert(!made);
         made = true;
 #endif
-        return new HiddenClass();
+        return new HiddenClass(NORMAL);
+    }
+    static HiddenClass* makeDictBacked() {
+#ifndef NDEBUG
+        static bool made = false;
+        assert(!made);
+        made = true;
+#endif
+        return new HiddenClass(DICT_BACKED);
     }
 
-    llvm::StringMap<int> attr_offsets;
-    llvm::StringMap<HiddenClass*> children;
+    void gc_visit(GCVisitor* visitor) {
+        // Visit children even for the dict-backed case, since children will just be empty
+        for (const auto& p : children) {
+            visitor->visit(p.second);
+        }
+    }
+
+
+    // Only makes sense for NORMAL hidden classes:
+    const llvm::StringMap<int>& getAttrOffsets() {
+        assert(type == NORMAL);
+        return attr_offsets;
+    }
 
+    // Only makes sense for NORMAL hidden classes:
     HiddenClass* getOrMakeChild(const std::string& attr);
 
+    // Only makes sense for NORMAL hidden classes:
     int getOffset(const std::string& attr) {
+        assert(type == NORMAL);
         auto it = attr_offsets.find(attr);
         if (it == attr_offsets.end())
             return -1;
         return it->second;
     }
-    HiddenClass* delAttrToMakeHC(const std::string& attr);
 
-    void gc_visit(GCVisitor* visitor) {
-        for (const auto& p : children) {
-            visitor->visit(p.second);
-        }
-    }
+    // Only makes sense for NORMAL hidden classes:
+    HiddenClass* delAttrToMakeHC(const std::string& attr);
 };
 
 class BoxedInt : public Box {

test/tests/dict_setting.py

-# expected: fail
-# - we don't support setting __dict__ yet
+try:
+    object().__dict__ = 1
+except AttributeError as e:
+    print e
 
 class C(object):
     pass
@@ -23,3 +25,16 @@ p()
 c1.a = 6
 c2.b = 7
 p()
+
+print c1.a, c1.b, c2.a, c2.b
+del c1.a
+try:
+    del c1.a
+except AttributeError as e:
+    # the error message CPython gives here is just "a" which I don't think we should copy.
+    print "caught AttributeError"
+p()
+
+c1.__dict__ = d = {}
+d['i'] = 5
+p()

test/tests/dict_setting_old_dict.py

+# expected: fail
+# - haven't bothered to implement this yet
+
+# Funny case: if we get the attrwrapper of an object, then change it to be dict-backed,
+# the original attrwrapper should remain valid but no longer connected to that object.
+
+class C(object):
+    pass
+
+c1 = C()
+aw = c1.__dict__
+c1.a = 1
+print aw.items()
+c1.__dict__ = d = {}
+print aw.items()