Commit 8b94bbc4 authored by Kevin Modzelewski's avatar Kevin Modzelewski

Support setting __dict__

It's trickier than just setting all of the attributes, since any updates
to either the original dict or to the object's attributes will get
mirrored in the other object.  I don't know if anyone uses this,
but I don't think there's any good way for us to tell if this is
going to happen so we just have to be conservative.

So, add a new distinction between "types" of hidden classes, and add
a new "dict backed" type.  Instead of having an array of attributes,
has a single attribute which is a dict.

There are some pretty tricky corner cases that we don't support yet,
such as if you access and save __dict__, then set __dict__, and then
try to access the saved version.  At least we can detect that and fail.
parent 415673dc
......@@ -658,7 +658,7 @@ BoxedDict* getLocals(bool only_user_visible, bool includeClosure) {
// Add the locals from the closure
for (; closure != NULL; closure = closure->parent) {
assert(closure->cls == closure_cls);
for (auto& attr_offset : closure->attrs.hcls->attr_offsets) {
for (auto& attr_offset : closure->attrs.hcls->getAttrOffsets()) {
const std::string& name = attr_offset.first();
int offset = attr_offset.second;
Box* val = closure->attrs.attr_list->attrs[offset];
......
......@@ -414,7 +414,7 @@ struct DelattrRewriteArgs;
struct HCAttrs {
public:
struct AttrList : public GCAllocated<gc::GCKind::PRECISE> {
struct AttrList {
Box* attrs[0];
};
......@@ -428,6 +428,9 @@ class BoxedDict;
class BoxedString;
class Box {
private:
BoxedDict** getDictPtr();
public:
// Add a no-op constructor to make sure that we don't zero-initialize cls
Box() {}
......@@ -441,6 +444,7 @@ public:
llvm::iterator_range<BoxIterator> pyElements();
HCAttrs* getHCAttrsPtr();
void setDict(BoxedDict* d);
BoxedDict* getDict();
void setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite_args);
......
......@@ -170,12 +170,12 @@ extern "C" Box* dir(Box* obj) {
result = new BoxedList();
}
for (auto const& kv : obj->cls->attrs.hcls->attr_offsets) {
for (auto const& kv : obj->cls->attrs.hcls->getAttrOffsets()) {
listAppend(result, boxString(kv.first()));
}
if (obj->cls->instancesHaveHCAttrs()) {
HCAttrs* attrs = obj->getHCAttrsPtr();
for (auto const& kv : attrs->hcls->attr_offsets) {
for (auto const& kv : attrs->hcls->getAttrOffsets()) {
listAppend(result, boxString(kv.first()));
}
}
......
......@@ -87,6 +87,8 @@ Box* dictValues(BoxedDict* self) {
}
Box* dictKeys(BoxedDict* self) {
RELEASE_ASSERT(isSubclass(self->cls, dict_cls), "");
BoxedList* rtn = new BoxedList();
for (const auto& p : self->d) {
listAppendInternal(rtn, p.first);
......@@ -321,14 +323,14 @@ Box* dictDelitem(BoxedDict* self, Box* k) {
}
extern "C" int PyDict_DelItem(PyObject* op, PyObject* key) noexcept {
ASSERT(isSubclass(op->cls, dict_cls) || op->cls == attrwrapper_cls, "%s", getTypeName(op));
try {
dictDelitem((BoxedDict*)op, key);
delitem(op, key);
return 0;
} catch (ExcInfo e) {
setCAPIException(e);
return -1;
}
return 0;
}
extern "C" int PyDict_DelItemString(PyObject* v, const char* key) noexcept {
......
......@@ -481,6 +481,8 @@ const char* getNameOfClass(BoxedClass* cls) {
}
HiddenClass* HiddenClass::getOrMakeChild(const std::string& attr) {
assert(type == NORMAL);
auto it = children.find(attr);
if (it != children.end())
return it->second;
......@@ -498,6 +500,7 @@ HiddenClass* HiddenClass::getOrMakeChild(const std::string& attr) {
* del attr from current HiddenClass, pertain the orders of remaining attrs
*/
HiddenClass* HiddenClass::delAttrToMakeHC(const std::string& attr) {
assert(type == NORMAL);
int idx = getOffset(attr);
assert(idx >= 0);
......@@ -527,13 +530,26 @@ HCAttrs* Box::getHCAttrsPtr() {
return reinterpret_cast<HCAttrs*>(p);
}
BoxedDict* Box::getDict() {
BoxedDict** Box::getDictPtr() {
assert(cls->instancesHaveDictAttrs());
char* p = reinterpret_cast<char*>(this);
p += cls->tp_dictoffset;
BoxedDict** d_ptr = reinterpret_cast<BoxedDict**>(p);
return d_ptr;
}
void Box::setDict(BoxedDict* d) {
assert(cls->instancesHaveDictAttrs());
*getDictPtr() = d;
}
BoxedDict* Box::getDict() {
assert(cls->instancesHaveDictAttrs());
BoxedDict** d_ptr = getDictPtr();
BoxedDict* d = *d_ptr;
if (!d) {
d = *d_ptr = new BoxedDict();
......@@ -574,12 +590,25 @@ Box* Box::getattr(const std::string& attr, GetattrRewriteArgs* rewrite_args) {
// structure (ex user class) and the same hidden classes, because
// otherwise the guard will fail anyway.;
if (cls->instancesHaveHCAttrs()) {
if (rewrite_args)
rewrite_args->out_success = true;
HCAttrs* attrs = getHCAttrsPtr();
HiddenClass* hcls = attrs->hcls;
if (hcls->type == HiddenClass::DICT_BACKED) {
if (rewrite_args)
assert(!rewrite_args->out_success);
rewrite_args = NULL;
Box* d = attrs->attr_list->attrs[0];
assert(d);
Box* r = PyDict_GetItemString(d, attr.c_str());
// r can be NULL if the item didn't exist
return r;
}
assert(hcls->type == HiddenClass::NORMAL);
if (rewrite_args)
rewrite_args->out_success = true;
if (rewrite_args) {
if (!rewrite_args->obj_hcls_guarded)
rewrite_args->obj->addAttrGuard(cls->attrs_offset + HCATTRS_HCLS_OFFSET, (intptr_t)hcls);
......@@ -641,7 +670,20 @@ void Box::setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite
if (cls->instancesHaveHCAttrs()) {
HCAttrs* attrs = getHCAttrsPtr();
HiddenClass* hcls = attrs->hcls;
int numattrs = hcls->attr_offsets.size();
if (hcls->type == HiddenClass::DICT_BACKED) {
if (rewrite_args)
assert(!rewrite_args->out_success);
rewrite_args = NULL;
Box* d = attrs->attr_list->attrs[0];
assert(d);
PyDict_SetItemString(d, attr.c_str(), val);
checkAndThrowCAPIException();
return;
}
assert(hcls->type == HiddenClass::NORMAL);
int numattrs = hcls->getAttrOffsets().size();
int offset = hcls->getOffset(attr);
......@@ -672,10 +714,10 @@ void Box::setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite
HiddenClass* new_hcls = hcls->getOrMakeChild(attr);
// TODO need to make sure we don't need to rearrange the attributes
assert(new_hcls->attr_offsets[attr] == numattrs);
assert(new_hcls->getAttrOffsets().lookup(attr) == numattrs);
#ifndef NDEBUG
for (const auto& p : hcls->attr_offsets) {
assert(new_hcls->attr_offsets[p.first()] == p.second);
for (const auto& p : hcls->getAttrOffsets()) {
assert(new_hcls->getAttrOffsets().lookup(p.first()) == p.second);
}
#endif
......@@ -3629,12 +3671,25 @@ void Box::delattr(const std::string& attr, DelattrRewriteArgs* rewrite_args) {
// as soon as the hcls changes, the guard on hidden class won't pass.
HCAttrs* attrs = getHCAttrsPtr();
HiddenClass* hcls = attrs->hcls;
if (hcls->type == HiddenClass::DICT_BACKED) {
if (rewrite_args)
assert(!rewrite_args->out_success);
rewrite_args = NULL;
Box* d = attrs->attr_list->attrs[0];
assert(d);
PyDict_DelItemString(d, attr.c_str());
checkAndThrowCAPIException();
return;
}
assert(hcls->type == HiddenClass::NORMAL);
HiddenClass* new_hcls = hcls->delAttrToMakeHC(attr);
// The order of attributes is pertained as delAttrToMakeHC constructs
// the new HiddenClass by invoking getOrMakeChild in the prevous order
// of remaining attributes
int num_attrs = hcls->attr_offsets.size();
int num_attrs = hcls->getAttrOffsets().size();
int offset = hcls->getOffset(attr);
assert(offset >= 0);
Box** start = attrs->attr_list->attrs;
......@@ -4372,7 +4427,7 @@ extern "C" Box* importStar(Box* _from_module, BoxedModule* to_module) {
}
HCAttrs* module_attrs = from_module->getHCAttrsPtr();
for (auto& p : module_attrs->hcls->attr_offsets) {
for (auto& p : module_attrs->hcls->getAttrOffsets()) {
if (p.first()[0] == '_')
continue;
......
......@@ -410,13 +410,22 @@ extern "C" void boxGCHandler(GCVisitor* v, Box* b) {
HCAttrs* attrs = b->getHCAttrsPtr();
v->visit(attrs->hcls);
int nattrs = attrs->hcls->attr_offsets.size();
switch (attrs->hcls->type) {
case HiddenClass::NORMAL: {
int nattrs = attrs->hcls->getAttrOffsets().size();
if (nattrs) {
HCAttrs::AttrList* attr_list = attrs->attr_list;
assert(attr_list);
v->visit(attr_list);
v->visitRange((void**)&attr_list->attrs[0], (void**)&attr_list->attrs[nattrs]);
}
break;
}
case HiddenClass::DICT_BACKED: {
v->visit(attrs->attr_list->attrs[0]);
break;
}
}
}
if (b->cls->instancesHaveDictAttrs()) {
......@@ -459,7 +468,28 @@ static Box* typeDict(Box* obj, void* context) {
}
static void typeSetDict(Box* obj, Box* val, void* context) {
Py_FatalError("unimplemented");
if (obj->cls->instancesHaveDictAttrs()) {
RELEASE_ASSERT(val->cls == dict_cls, "");
obj->setDict(static_cast<BoxedDict*>(val));
return;
}
if (obj->cls->instancesHaveHCAttrs()) {
RELEASE_ASSERT(val->cls == dict_cls || val->cls == attrwrapper_cls, "");
auto new_attr_list
= (HCAttrs::AttrList*)gc_alloc(sizeof(HCAttrs::AttrList) + sizeof(Box*), gc::GCKind::UNTRACKED);
new_attr_list->attrs[0] = val;
HCAttrs* hcattrs = obj->getHCAttrsPtr();
hcattrs->hcls = HiddenClass::dict_backed;
hcattrs->attr_list = new_attr_list;
return;
}
// This should have thrown an exception rather than get here:
abort();
}
Box* dict_descr = NULL;
......@@ -781,6 +811,7 @@ Box* range_obj = NULL;
}
HiddenClass* root_hcls;
HiddenClass* HiddenClass::dict_backed;
extern "C" Box* createSlice(Box* start, Box* stop, Box* step) {
BoxedSlice* rtn = new BoxedSlice(start, stop, step);
......@@ -999,7 +1030,7 @@ private:
// Iterating over the an attrwrapper (~=dict) just gives the keys, which
// just depends on the hidden class of the object. Let's store only that:
HiddenClass* hcls;
decltype(HiddenClass::attr_offsets)::iterator it;
llvm::StringMap<int>::const_iterator it;
public:
AttrWrapperIter(AttrWrapper* aw);
......@@ -1025,7 +1056,16 @@ private:
Box* b;
public:
AttrWrapper(Box* b) : b(b) { assert(b->cls->instancesHaveHCAttrs()); }
AttrWrapper(Box* b) : b(b) {
assert(b->cls->instancesHaveHCAttrs());
// We currently don't support creating an attrwrapper around a dict-backed object,
// so try asserting that here.
// This check doesn't cover all cases, since an attrwrapper could be created around
// a normal object which then becomes dict-backed, so we RELEASE_ASSERT later
// that that doesn't happen.
assert(b->getHCAttrsPtr()->hcls->type == HiddenClass::NORMAL);
}
DEFAULT_CLASS(attrwrapper_cls);
......@@ -1091,6 +1131,21 @@ public:
return r;
}
static Box* delitem(Box* _self, Box* _key) {
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
_key = coerceUnicodeToStr(_key);
RELEASE_ASSERT(_key->cls == str_cls, "%s", _key->cls->tp_name);
BoxedString* key = static_cast<BoxedString*>(_key);
if (self->b->getattr(key->s))
self->b->delattr(key->s, NULL);
else
raiseExcHelper(KeyError, "'%s'", key->s.c_str());
return None;
}
static Box* str(Box* _self) {
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
......@@ -1099,8 +1154,9 @@ public:
os << "attrwrapper({";
HCAttrs* attrs = self->b->getHCAttrsPtr();
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL, "");
bool first = true;
for (const auto& p : attrs->hcls->attr_offsets) {
for (const auto& p : attrs->hcls->getAttrOffsets()) {
if (!first)
os << ", ";
first = false;
......@@ -1131,7 +1187,8 @@ public:
BoxedList* rtn = new BoxedList();
HCAttrs* attrs = self->b->getHCAttrsPtr();
for (const auto& p : attrs->hcls->attr_offsets) {
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL, "");
for (const auto& p : attrs->hcls->getAttrOffsets()) {
listAppend(rtn, boxString(p.first()));
}
return rtn;
......@@ -1144,7 +1201,8 @@ public:
BoxedList* rtn = new BoxedList();
HCAttrs* attrs = self->b->getHCAttrsPtr();
for (const auto& p : attrs->hcls->attr_offsets) {
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL, "");
for (const auto& p : attrs->hcls->getAttrOffsets()) {
listAppend(rtn, attrs->attr_list->attrs[p.second]);
}
return rtn;
......@@ -1157,7 +1215,8 @@ public:
BoxedList* rtn = new BoxedList();
HCAttrs* attrs = self->b->getHCAttrsPtr();
for (const auto& p : attrs->hcls->attr_offsets) {
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL, "");
for (const auto& p : attrs->hcls->getAttrOffsets()) {
BoxedTuple* t = new BoxedTuple({ boxString(p.first()), attrs->attr_list->attrs[p.second] });
listAppend(rtn, t);
}
......@@ -1171,7 +1230,8 @@ public:
BoxedDict* rtn = new BoxedDict();
HCAttrs* attrs = self->b->getHCAttrsPtr();
for (const auto& p : attrs->hcls->attr_offsets) {
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL, "");
for (const auto& p : attrs->hcls->getAttrOffsets()) {
rtn->d[boxString(p.first())] = attrs->attr_list->attrs[p.second];
}
return rtn;
......@@ -1182,7 +1242,8 @@ public:
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
HCAttrs* attrs = self->b->getHCAttrsPtr();
return boxInt(attrs->hcls->attr_offsets.size());
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL, "");
return boxInt(attrs->hcls->getAttrOffsets().size());
}
static Box* update(Box* _self, Box* _container) {
......@@ -1193,7 +1254,8 @@ public:
AttrWrapper* container = static_cast<AttrWrapper*>(_container);
HCAttrs* attrs = container->b->getHCAttrsPtr();
for (const auto& p : attrs->hcls->attr_offsets) {
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL, "");
for (const auto& p : attrs->hcls->getAttrOffsets()) {
self->b->setattr(p.first(), attrs->attr_list->attrs[p.second], NULL);
}
} else if (_container->cls == dict_cls) {
......@@ -1221,21 +1283,24 @@ public:
AttrWrapperIter::AttrWrapperIter(AttrWrapper* aw) {
hcls = aw->b->getHCAttrsPtr()->hcls;
assert(hcls);
it = hcls->attr_offsets.begin();
RELEASE_ASSERT(hcls->type == HiddenClass::NORMAL, "");
it = hcls->getAttrOffsets().begin();
}
Box* AttrWrapperIter::hasnext(Box* _self) {
RELEASE_ASSERT(_self->cls == attrwrapperiter_cls, "");
AttrWrapperIter* self = static_cast<AttrWrapperIter*>(_self);
RELEASE_ASSERT(self->hcls->type == HiddenClass::NORMAL, "");
return boxBool(self->it != self->hcls->attr_offsets.end());
return boxBool(self->it != self->hcls->getAttrOffsets().end());
}
Box* AttrWrapperIter::next(Box* _self) {
RELEASE_ASSERT(_self->cls == attrwrapperiter_cls, "");
AttrWrapperIter* self = static_cast<AttrWrapperIter*>(_self);
RELEASE_ASSERT(self->hcls->type == HiddenClass::NORMAL, "");
assert(self->it != self->hcls->attr_offsets.end());
assert(self->it != self->hcls->getAttrOffsets().end());
Box* r = boxString(self->it->first());
++self->it;
return r;
......@@ -1243,6 +1308,10 @@ Box* AttrWrapperIter::next(Box* _self) {
Box* makeAttrWrapper(Box* b) {
assert(b->cls->instancesHaveHCAttrs());
if (b->getHCAttrsPtr()->hcls->type == HiddenClass::DICT_BACKED) {
return b->getHCAttrsPtr()->attr_list->attrs[0];
}
return new AttrWrapper(b);
}
......@@ -1687,6 +1756,8 @@ bool TRACK_ALLOCATIONS = false;
void setupRuntime() {
root_hcls = HiddenClass::makeRoot();
gc::registerPermanentRoot(root_hcls);
HiddenClass::dict_backed = HiddenClass::makeDictBacked();
gc::registerPermanentRoot(HiddenClass::dict_backed);
// Disable the GC while we do some manual initialization of the object hierarchy:
gc::disableGC();
......@@ -2009,6 +2080,7 @@ void setupRuntime() {
attrwrapper_cls->giveAttr("__setitem__", new BoxedFunction(boxRTFunction((void*)AttrWrapper::setitem, UNKNOWN, 3)));
attrwrapper_cls->giveAttr("__getitem__", new BoxedFunction(boxRTFunction((void*)AttrWrapper::getitem, UNKNOWN, 2)));
attrwrapper_cls->giveAttr("__delitem__", new BoxedFunction(boxRTFunction((void*)AttrWrapper::delitem, UNKNOWN, 2)));
attrwrapper_cls->giveAttr("setdefault",
new BoxedFunction(boxRTFunction((void*)AttrWrapper::setdefault, UNKNOWN, 3)));
attrwrapper_cls->giveAttr(
......@@ -2123,13 +2195,6 @@ BoxedModule* createModule(const std::string& name, const std::string& fn, const
return module;
}
void freeHiddenClasses(HiddenClass* hcls) {
for (const auto& p : hcls->children) {
freeHiddenClasses(p.second);
}
gc::gc_free(hcls);
}
void teardownRuntime() {
// Things start to become very precarious after this point, as the basic classes stop to work.
// TODO it's probably a waste of time to tear these down in non-debugging mode
......@@ -2190,7 +2255,5 @@ void teardownRuntime() {
decref(none_cls);
decref(type_cls);
*/
freeHiddenClasses(root_hcls);
}
}
......@@ -264,14 +264,29 @@ static_assert(sizeof(pyston::BoxedHeapClass) == sizeof(PyHeapTypeObject), "");
class HiddenClass : public GCAllocated<gc::GCKind::HIDDEN_CLASS> {
public:
// We have a couple different storage strategies for attributes, which
// are distinguished by having a different hidden class type.
enum HCType {
NORMAL, // attributes stored in attributes array, name->offset map stored in hidden class
DICT_BACKED, // first attribute in array is a dict-like object which stores the attributes
} const type;
static HiddenClass* dict_backed;
private:
HiddenClass() {}
HiddenClass(HiddenClass* parent) : attr_offsets() {
HiddenClass(HCType type) : type(type) {}
HiddenClass(HiddenClass* parent) : type(NORMAL), attr_offsets() {
assert(parent->type == NORMAL);
for (auto& p : parent->attr_offsets) {
this->attr_offsets.insert(&p);
}
}
// Only makes sense for NORMAL hidden classes. Clients should access through getAttrOffsets():
llvm::StringMap<int> attr_offsets;
llvm::StringMap<HiddenClass*> children;
public:
static HiddenClass* makeRoot() {
#ifndef NDEBUG
......@@ -279,27 +294,45 @@ public:
assert(!made);
made = true;
#endif
return new HiddenClass();
return new HiddenClass(NORMAL);
}
static HiddenClass* makeDictBacked() {
#ifndef NDEBUG
static bool made = false;
assert(!made);
made = true;
#endif
return new HiddenClass(DICT_BACKED);
}
llvm::StringMap<int> attr_offsets;
llvm::StringMap<HiddenClass*> children;
void gc_visit(GCVisitor* visitor) {
// Visit children even for the dict-backed case, since children will just be empty
for (const auto& p : children) {
visitor->visit(p.second);
}
}
// Only makes sense for NORMAL hidden classes:
const llvm::StringMap<int>& getAttrOffsets() {
assert(type == NORMAL);
return attr_offsets;
}
// Only makes sense for NORMAL hidden classes:
HiddenClass* getOrMakeChild(const std::string& attr);
// Only makes sense for NORMAL hidden classes:
int getOffset(const std::string& attr) {
assert(type == NORMAL);
auto it = attr_offsets.find(attr);
if (it == attr_offsets.end())
return -1;
return it->second;
}
HiddenClass* delAttrToMakeHC(const std::string& attr);
void gc_visit(GCVisitor* visitor) {
for (const auto& p : children) {
visitor->visit(p.second);
}
}
// Only makes sense for NORMAL hidden classes:
HiddenClass* delAttrToMakeHC(const std::string& attr);
};
class BoxedInt : public Box {
......
# expected: fail
# - we don't support setting __dict__ yet
try:
object().__dict__ = 1
except AttributeError as e:
print e
class C(object):
pass
......@@ -23,3 +25,16 @@ p()
c1.a = 6
c2.b = 7
p()
print c1.a, c1.b, c2.a, c2.b
del c1.a
try:
del c1.a
except AttributeError as e:
# the error message CPython gives here is just "a" which I don't think we should copy.
print "caught AttributeError"
p()
c1.__dict__ = d = {}
d['i'] = 5
p()
# expected: fail
# - haven't bothered to implement this yet
# Funny case: if we get the attrwrapper of an object, then change it to be dict-backed,
# the original attrwrapper should remain valid but no longer connected to that object.
class C(object):
pass
c1 = C()
aw = c1.__dict__
c1.a = 1
print aw.items()
c1.__dict__ = d = {}
print aw.items()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment