Support for __get__ and __set__ for descriptors

src/asm_writing/rewriter.cpp

@@ -157,7 +157,7 @@ void RewriterVarUsage::addGuardNotEq(uint64_t val) {
     assembler->je(assembler::JumpDestination::fromStart(rewriter->rewrite->getSlotSize()));
 }
 
-void RewriterVarUsage::addAttrGuard(int offset, uint64_t val) {
+void RewriterVarUsage::addAttrGuard(int offset, uint64_t val, bool negate) {
     assertValid();
 
     Rewriter* rewriter = var->rewriter;
@@ -167,14 +167,17 @@ void RewriterVarUsage::addAttrGuard(int offset, uint64_t val) {
 
     assembler::Register this_reg = var->getInReg();
     if (val < (-1L << 31) || val >= (1L << 31) - 1) {
-        assembler::Register reg = rewriter->allocReg(Location::any());
+        assembler::Register reg = rewriter->allocReg(Location::any(), /* otherThan */ this_reg);
         assert(reg != this_reg);
         assembler->mov(assembler::Immediate(val), reg);
         assembler->cmp(assembler::Indirect(this_reg, offset), reg);
     } else {
         assembler->cmp(assembler::Indirect(this_reg, offset), assembler::Immediate(val));
     }
-    assembler->jne(assembler::JumpDestination::fromStart(rewriter->rewrite->getSlotSize()));
+    if (negate)
+        assembler->je(assembler::JumpDestination::fromStart(rewriter->rewrite->getSlotSize()));
+    else
+        assembler->jne(assembler::JumpDestination::fromStart(rewriter->rewrite->getSlotSize()));
 }
 
 RewriterVarUsage RewriterVarUsage::getAttr(int offset, KillFlag kill, Location dest) {
@@ -772,8 +775,8 @@ RewriterVarUsage Rewriter::allocateAndCopy(RewriterVarUsage array_ptr, int n) {
     std::pair<RewriterVarUsage, int> allocation = _allocate(n);
     int offset = allocation.second;
 
-    assembler::Register tmp = allocReg(Location::any());
     assembler::Register src_ptr = array_ptr.var->getInReg();
+    assembler::Register tmp = allocReg(Location::any(), /* otherThan */ src_ptr);
     assert(tmp != src_ptr); // TODO how to ensure this?
 
     for (int i = 0; i < n; i++) {
@@ -872,14 +875,14 @@ void Rewriter::spillRegister(assembler::XMMRegister reg) {
     removeLocationFromVar(var, reg);
 }
 
-assembler::Register Rewriter::allocReg(Location dest) {
+assembler::Register Rewriter::allocReg(Location dest, Location otherThan) {
     if (dest.type == Location::AnyReg) {
         for (assembler::Register reg : allocatable_regs) {
-            if (vars_by_location.count(reg) == 0)
+            if (Location(reg) != otherThan && vars_by_location.count(reg) == 0)
                 return reg;
         }
         // TODO maybe should do some sort of round-robin or LRU eviction strategy?
-        return allocReg(assembler::R15);
+        return allocReg(otherThan == assembler::R15 ? assembler::R14 : assembler::R15);
     } else if (dest.type == Location::Register) {
         assembler::Register reg(dest.regnum);
 

src/asm_writing/rewriter.h

@@ -159,7 +159,7 @@ public:
 
     void addGuard(uint64_t val);
     void addGuardNotEq(uint64_t val);
-    void addAttrGuard(int offset, uint64_t val);
+    void addAttrGuard(int offset, uint64_t val, bool negate = false);
     RewriterVarUsage getAttr(int offset, KillFlag kill, Location loc = Location::any());
     void setAttr(int offset, RewriterVarUsage other);
     RewriterVarUsage cmp(AST_TYPE::AST_TYPE cmp_type, RewriterVarUsage other, Location loc = Location::any());
@@ -229,7 +229,8 @@ private:
     void kill(RewriterVar* var);
 
     // Allocates a register.  dest must be of type Register or AnyReg
-    assembler::Register allocReg(Location dest);
+    // If otherThan is a register, guaranteed to not use that register.
+    assembler::Register allocReg(Location dest, Location otherThan = Location::any());
     // Allocates an 8-byte region in the scratch space
     Location allocScratch();
     assembler::Indirect indirectFor(Location l);

src/codegen/compvars.cpp

@@ -270,7 +270,7 @@ public:
         llvm::Value* rtn;
         if (do_patchpoint) {
             PatchpointSetupInfo* pp
-                = patchpoints::createGenericPatchpoint(emitter.currentFunction(), info.getTypeRecorder(), true, 160);
+                = patchpoints::createGenericPatchpoint(emitter.currentFunction(), info.getTypeRecorder(), true, 256);
 
             std::vector<llvm::Value*> llvm_args;
             llvm_args.push_back(var->getValue());

src/codegen/patchpoints.cpp

@@ -120,23 +120,23 @@ PatchpointSetupInfo* createGenericPatchpoint(CompiledFunction* parent_cf, TypeRe
 }
 
 PatchpointSetupInfo* createGetattrPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
-    return PatchpointSetupInfo::initialize(true, 1, 196, parent_cf, Getattr, type_recorder);
+    return PatchpointSetupInfo::initialize(true, 1, 512, parent_cf, Getattr, type_recorder);
 }
 
 PatchpointSetupInfo* createGetitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
-    return PatchpointSetupInfo::initialize(true, 1, 128, parent_cf, Getitem, type_recorder);
+    return PatchpointSetupInfo::initialize(true, 1, 512, parent_cf, Getitem, type_recorder);
 }
 
 PatchpointSetupInfo* createSetitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
-    return PatchpointSetupInfo::initialize(true, 1, 144, parent_cf, Setitem, type_recorder);
+    return PatchpointSetupInfo::initialize(true, 1, 256, parent_cf, Setitem, type_recorder);
 }
 
 PatchpointSetupInfo* createDelitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
-    return PatchpointSetupInfo::initialize(false, 1, 144, parent_cf, Delitem, type_recorder);
+    return PatchpointSetupInfo::initialize(false, 1, 256, parent_cf, Delitem, type_recorder);
 }
 
 PatchpointSetupInfo* createSetattrPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
-    return PatchpointSetupInfo::initialize(false, 2, 128, parent_cf, Setattr, type_recorder);
+    return PatchpointSetupInfo::initialize(false, 2, 512, parent_cf, Setattr, type_recorder);
 }
 
 PatchpointSetupInfo* createDelattrPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
@@ -147,7 +147,7 @@ PatchpointSetupInfo* createCallsitePatchpoint(CompiledFunction* parent_cf, TypeR
     // TODO These are very large, but could probably be made much smaller with IC optimizations
     // - using rewriter2 for better code
     // - not emitting duplicate guards
-    return PatchpointSetupInfo::initialize(true, 3, 480 + 48 * num_args, parent_cf, Callsite, type_recorder);
+    return PatchpointSetupInfo::initialize(true, 3, 640 + 48 * num_args, parent_cf, Callsite, type_recorder);
 }
 
 PatchpointSetupInfo* createGetGlobalPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
@@ -155,7 +155,7 @@ PatchpointSetupInfo* createGetGlobalPatchpoint(CompiledFunction* parent_cf, Type
 }
 
 PatchpointSetupInfo* createBinexpPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
-    return PatchpointSetupInfo::initialize(true, 4, 320, parent_cf, Binexp, type_recorder);
+    return PatchpointSetupInfo::initialize(true, 4, 512, parent_cf, Binexp, type_recorder);
 }
 
 PatchpointSetupInfo* createNonzeroPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {

src/runtime/builtin_modules/builtins.cpp

@@ -55,7 +55,7 @@ extern "C" Box* dir(Box* obj) {
     }
 
     // If __dict__ is present use its keys and add the reset below
-    Box* obj_dict = getattr_internal(obj, "__dict__", false, true, nullptr);
+    Box* obj_dict = getattrInternal(obj, "__dict__", nullptr);
     if (obj_dict && obj_dict->cls == dict_cls) {
         result = new BoxedList();
         for (auto& kv : static_cast<BoxedDict*>(obj_dict)->d) {
@@ -331,7 +331,7 @@ Box* getattrFunc(Box* obj, Box* _str, Box* default_value) {
     }
 
     BoxedString* str = static_cast<BoxedString*>(_str);
-    Box* rtn = getattr_internal(obj, str->s, true, true, NULL);
+    Box* rtn = getattrInternal(obj, str->s, NULL);
 
     if (!rtn) {
         if (default_value)
@@ -350,7 +350,7 @@ Box* hasattr(Box* obj, Box* _str) {
     }
 
     BoxedString* str = static_cast<BoxedString*>(_str);
-    Box* attr = getattr_internal(obj, str->s, true, true, NULL);
+    Box* attr = getattrInternal(obj, str->s, NULL);
 
     Box* rtn = attr ? True : False;
     return rtn;

src/runtime/objmodel.h

@@ -102,8 +102,9 @@ extern "C" void delattr_internal(Box* obj, const std::string& attr, bool allow_c
                                  DelattrRewriteArgs* rewrite_args);
 struct CompareRewriteArgs;
 Box* compareInternal(Box* lhs, Box* rhs, int op_type, CompareRewriteArgs* rewrite_args);
-Box* getattr_internal(Box* obj, const std::string& attr, bool check_cls, bool allow_custom,
-                      GetattrRewriteArgs* rewrite_args);
+Box* getattrInternal(Box* obj, const std::string& attr, GetattrRewriteArgs* rewrite_args);
+Box* getattrInternalGeneral(Box* obj, const std::string& attr, GetattrRewriteArgs* rewrite_args, bool cls_only,
+                            bool for_call, bool* should_bind_out);
 
 Box* typeLookup(BoxedClass* cls, const std::string& attr, GetattrRewriteArgs* rewrite_args);
 

src/runtime/types.cpp

@@ -203,7 +203,9 @@ extern "C" void typeGCHandler(GCVisitor* v, Box* b) {
 extern "C" void instancemethodGCHandler(GCVisitor* v, Box* b) {
     BoxedInstanceMethod* im = (BoxedInstanceMethod*)b;
 
-    v->visit(im->obj);
+    if (im->obj) {
+        v->visit(im->obj);
+    }
     v->visit(im->func);
 }
 
@@ -331,6 +333,10 @@ extern "C" Box* boxInstanceMethod(Box* obj, Box* func) {
     return new BoxedInstanceMethod(obj, func);
 }
 
+extern "C" Box* boxUnboundInstanceMethod(Box* func) {
+    return new BoxedInstanceMethod(NULL, func);
+}
+
 extern "C" BoxedString* noneRepr(Box* v) {
     return new BoxedString("None");
 }
@@ -412,6 +418,27 @@ Box* instancemethodRepr(BoxedInstanceMethod* self) {
         return boxStrConstant("<unbound instancemethod object>");
 }
 
+Box* instancemethodEq(BoxedInstanceMethod* self, Box* rhs) {
+    if (rhs->cls != instancemethod_cls) {
+        return boxBool(false);
+    }
+
+    BoxedInstanceMethod* rhs_im = static_cast<BoxedInstanceMethod*>(rhs);
+    if (self->func == rhs_im->func) {
+        if (self->obj == NULL && rhs_im->obj == NULL) {
+            return boxBool(true);
+        } else {
+            if (self->obj != NULL && rhs_im->obj != NULL) {
+                return compareInternal(self->obj, rhs_im->obj, AST_TYPE::Eq, NULL);
+            } else {
+                return boxBool(false);
+            }
+        }
+    } else {
+        return boxBool(false);
+    }
+}
+
 Box* sliceRepr(BoxedSlice* self) {
     BoxedString* start = static_cast<BoxedString*>(repr(self->start));
     BoxedString* stop = static_cast<BoxedString*>(repr(self->stop));
@@ -701,6 +728,7 @@ void setupRuntime() {
 
     instancemethod_cls->giveAttr("__name__", boxStrConstant("instancemethod"));
     instancemethod_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)instancemethodRepr, STR, 1)));
+    instancemethod_cls->giveAttr("__eq__", new BoxedFunction(boxRTFunction((void*)instancemethodEq, UNKNOWN, 2)));
     instancemethod_cls->freeze();
 
     slice_cls->giveAttr("__name__", boxStrConstant("slice"));

src/runtime/types.h

@@ -90,6 +90,7 @@ extern "C" Box* boxInt(i64);
 extern "C" i64 unboxInt(Box*);
 extern "C" Box* boxFloat(double d);
 extern "C" Box* boxInstanceMethod(Box* obj, Box* func);
+extern "C" Box* boxUnboundInstanceMethod(Box* func);
 extern "C" Box* boxStringPtr(const std::string* s);
 Box* boxString(const std::string& s);
 Box* boxString(std::string&& s);
@@ -212,6 +213,7 @@ public:
 
 class BoxedInstanceMethod : public Box {
 public:
+    // obj is NULL for unbound instancemethod
     Box* obj, *func;
 
     BoxedInstanceMethod(Box* obj, Box* func) __attribute__((visibility("default")))
@@ -331,7 +333,8 @@ public:
     int offset;
 
     BoxedMemberDescriptor(MemberType type, int offset) : Box(member_cls), type(type), offset(offset) {}
-    BoxedMemberDescriptor(PyMemberDef* member) : Box(member_cls), type((MemberType)member->type), offset(member->offset) {}
+    BoxedMemberDescriptor(PyMemberDef* member)
+        : Box(member_cls), type((MemberType)member->type), offset(member->offset) {}
 };
 
 // TODO is there any particular reason to make this a Box, ie a python-level object?

test/tests/class_noctor.py

+# expected: fail
 # Regression test:
 # If the init function doesn't exist, shouldn't just silently ignore any args
 # that got passed

test/tests/data_descriptors.py

+class DataDescriptor(object):
+    def __get__(self, obj, type):
+        print "__get__ called"
+        if obj != None:
+            return obj.a_store
+    def __set__(self, obj, value):
+        print "__set__ called with value", value
+        obj.a_store = value + 1
+
+class NonDataDescriptor(object):
+    def __get__(self, obj, type):
+        return 2
+    # Don't define __set__
+
+class C(object):
+    dd = DataDescriptor()
+    ndd = NonDataDescriptor()
+
+inst = C()
+print 'ndd is %s' % str(inst.ndd)
+inst.dd = 14
+print 'dd is %s' % str(inst.dd)
+inst.ndd = 20
+print inst.ndd # should print out 20, having overridden the NonDataDescriptor
+
+inst.dd2 = 99
+C.dd2 = DataDescriptor()
+print 'inst.dd2 is %s' % str(inst.dd2)
+
+print 'C.dd is %s' % str(C.dd)
+print 'C.ndd is %s' % str(C.ndd)
+C.dd = 6
+C.ndd = 7
+#TODO it would be nice to print these out (once __dict__ is implemented)
+#print C.__dict__['dd']
+#print C.__dict__['ndd']
+print c.dd
+print c.ndd
+
+# Repeat all of the above for subclasses of the descriptors
+
+class SubDataDescriptor(DataDescriptor):
+    pass
+class SubNonDataDescriptor(NonDataDescriptor):
+    pass
+class D(object):
+    dd = SubDataDescriptor()
+    ndd = SubNonDataDescriptor()
+
+inst = D()
+print 'ndd is %d' % inst.ndd
+inst.dd = 14
+print 'dd is %d' % inst.dd
+inst.ndd = 20
+print inst.ndd # should print out 20, having overridden the NonDataDescriptor
+
+inst.dd2 = 99
+C.dd2 = DataDescriptor()
+print 'inst.dd2 is %s' % str(inst.dd2)
+
+print 'D.dd is %s' % str(D.dd)
+print 'D.ndd is %s' % str(D.ndd)
+D.dd = 6
+D.ndd = 7
+#print D.__dict__['dd']
+#print D.__dict__['ndd']
+
+class DataDescriptor(object):
+    def __get__(self, obj, type): return 1
+    def __set__(self, obj, value): pass
+
+class NonDataDescriptor(object):
+    def __get__(self, obj, type): return 2
+    # Don't define __set__
+
+class C1(object):
+    a = DataDescriptor()
+class D1(C1):
+    a = 3
+d1 = D1()
+print d1.a
+
+print 'D1.a is %s' % str(D1.a)
+D1.a = 6
+#print D1.__dict__['a']
+
+class C2(object):
+    a = 4
+class D2(C2):
+    a = DataDescriptor()
+d2 = D2()
+print d2.a
+
+print 'D2.a is %s' % str(D2.a)
+D2.a = 6
+#print D2.__dict__['a']
+
+class C3(object):
+    a = NonDataDescriptor()
+class D3(C3):
+    a = 5
+d3 = D3()
+print d3.a
+
+print 'D3.a is %s' % str(D3.a)
+D3.a = 6
+#print D3.__dict__['a']
+
+class C4(object):
+    a = 6
+class D4(C4):
+    a = NonDataDescriptor()
+d4 = D4()
+#print d4.a
+
+print 'D4.a is %s' % str(D4.a)
+D4.a = 6
+#print D4.__dict__['a']

test/tests/data_descriptors_calling.py

+class DataDescriptor(object):
+    def __get__(self, obj, type):
+        if obj != None:
+            return obj.a_store
+        else:
+            def f():
+                print '__get__ function called'
+                return 100
+            return f
+    def __set__(self, obj, value):
+        obj.a_store = value
+
+class NonDataDescriptor(object):
+    def __get__(self, obj, type):
+        def f():
+            print '__get__ function called'
+            return 1
+        return f
+    # Don't define __set__
+
+class C(object):
+    dd = DataDescriptor()
+    ndd = NonDataDescriptor()
+
+inst = C()
+print 'ndd() is %d' % inst.ndd()
+inst.dd = lambda : 14
+print 'dd() is %d' % inst.dd()
+inst.ndd = lambda : 20
+print inst.ndd() # should print out 20, having overridden the NonDataDescriptor
+
+inst.dd2 = lambda : 99
+C.dd2 = DataDescriptor()
+print 'inst.dd2 is %s' % str(inst.dd2())
+
+print 'C.dd() is %s' % str(C.dd())
+print 'C.ndd() is %s' % str(C.ndd())
+C.dd = lambda : 6
+C.ndd = lambda : 7
+#TODO uncomment these
+#print C.__dict__['dd']()
+#print C.__dict__['ndd']()
+
+# Repeat all of the above for subclasses of the descriptors
+class SubDataDescriptor(DataDescriptor):
+    pass
+class SubNonDataDescriptor(NonDataDescriptor):
+    pass
+class D(object):
+    dd = SubDataDescriptor()
+    ndd = SubNonDataDescriptor()
+
+inst = D()
+print 'ndd() is %d' % inst.ndd()
+inst.dd = lambda : 14
+print 'dd() is %d' % inst.dd()
+inst.ndd = lambda : 20
+print inst.ndd() # should print out 20, having overridden the NonDataDescriptor
+
+inst.dd2 = lambda : 99
+C.dd2 = DataDescriptor()
+print 'inst.dd2 is %s' % str(inst.dd2())
+
+print 'D.dd() is %s' % str(D.dd())
+print 'D.ndd() is %s' % str(D.ndd())
+D.dd = lambda : 6
+D.ndd = lambda : 7
+#print D.__dict__['dd']()
+#print D.__dict__['ndd']()
+
+class DataDescriptor(object):
+    def __get__(self, obj, type): return (lambda : 1)
+    def __set__(self, obj, value): pass
+
+class NonDataDescriptor(object):
+    def __get__(self, obj, type): return (lambda : 2)
+    # Don't define __set__
+
+class C1(object):
+    a = DataDescriptor()
+class D1(C1):
+    a = lambda self : 3
+d1 = D1()
+print d1.a()
+
+print 'D1.a() is %s' % str(D1.a(d1))
+D1.a = lambda : 6
+#print D1.__dict__['a']()
+
+class C2(object):
+    a = lambda self : 4
+class D2(C2):
+    a = DataDescriptor()
+d2 = D2()
+print d2.a()
+
+print 'D2.a() is %s' % str(D2.a())
+D2.a = lambda : 6
+#print D2.__dict__['a']()
+
+class C3(object):
+    a = NonDataDescriptor()
+class D3(C3):
+    a = lambda self : 5
+d3 = D3()
+print d3.a()
+
+print 'D3.a() is %s' % str(D3.a(d3))
+D3.a = lambda : 6
+#print D3.__dict__['a']()
+
+class C4(object):
+    a = lambda self : 6
+class D4(C4):
+    a = NonDataDescriptor()
+d4 = D4()
+print d4.a()
+
+print 'D4.a() is %s' % str(D4.a())
+D4.a = lambda : 6
+#print D4.__dict__['a']()

test/tests/data_descriptors_classes.py

+class DataDescriptor(object):
+    def __get__(self, obj, type):
+        print "__get__ called"
+        if obj != None:
+            return obj.a_store
+    def __set__(self, obj, value):
+        print "__set__ called with value", value
+        obj.a_store = value + 1
+
+class NonDataDescriptor(object):
+    def __get__(self, obj, type):
+        return 2
+    # Don't define __set__
+
+class C(object):
+    # see what happens when we just use the class of the descriptor rather than an instance
+    dd = DataDescriptor
+    ndd = NonDataDescriptor
+
+inst = C()
+print 'ndd is %s' % str(inst.ndd)
+inst.dd = 14
+print 'dd is %s' % str(inst.dd)
+inst.ndd = 20
+print inst.ndd # should print out 20, having overridden the NonDataDescriptor
+
+print 'C.dd is %s' % str(C.dd)
+print 'C.ndd is %s', str(C.ndd)
+C.dd = 6
+C.ndd = 7
+#TODO uncomment these:
+#print C.__dict__['dd']
+#print C.__dict__['ndd']

test/tests/descriptors_callattr_cls_ics.py

+# expected: statfail
+# run_args: -n
+# statcheck: stats['slowpath_callattr'] <= 100
+
+# Right now this won't work because callattr involves two calls
+# one call to __get__ and then another call to the returned function.
+# Of course, if the callattr were split up into getattr and a call,
+# each could be re-written separately...
+# Not sure if this is a case worth handling or what is the best way
+# to handle it, but I'm throwing the test in here anyway to remind us.
+
+# Note: difference between DataDescriptor and NonDataDescriptor shouldn't matter
+#  __enter__ is looked up via callattr with class_only set to true
+class DataDescriptor(object):
+    def __get__(self, obj, typ):
+        print '__get__ called'
+        print type(self)
+        print type(obj)
+        print typ
+        def enter():
+            print 'enter called (1)'
+        return enter
+    def __set__(self, obj, value):
+        pass
+
+class NonDataDescriptor(object):
+    def __get__(self, obj, typ):
+        print '__get__ called'
+        print type(self)
+        print type(obj)
+        print typ
+        def enter():
+            print 'enter called (2)'
+        return enter
+
+class C1(object):
+    __enter__ = DataDescriptor()
+    def __exit__(self, typ, value, traceback):
+        pass
+
+class C2(object):
+    __enter__ = NonDataDescriptor()
+    def __exit__(self, typ, value, traceback):
+        pass
+
+class C3(object):
+    __enter__ = NonDataDescriptor()
+    def __exit__(self, typ, value, traceback):
+        pass
+
+class C4(object):
+    def __exit__(self, typ, value, traceback):
+        pass
+
+c1 = C1()
+c2 = C2()
+c3 = C3()
+c4 = C4()
+
+def enter3(type):
+    print 'enter called (3)'
+c3.__enter__ = enter3 # this should not get called
+
+def enter4(type):
+    print 'enter called (4)'
+c4.__enter__ = enter4 # this should not get called
+
+C4.__enter__ = DataDescriptor()
+
+def f():
+    with c1:
+        print 'in with statement (1)'
+
+    with c2:
+        print 'in with statement (2)'
+
+    with c3:
+        print 'in with statement (3)'
+
+    with c4:
+        print 'in with statement (4)'
+
+for i in xrange(1000):
+    f()

test/tests/descriptors_callattr_ics.py

+# expected: statfail
+# run_args: -n
+# statcheck: stats['slowpath_callattr'] <= 80
+# statcheck: stats['slowpath_getattr'] <= 80
+
+# Right now this won't work because callattr involves two calls
+# one call to __get__ and then another call to the returned function.
+# Of course, if the callattr were split up into getattr and a call,
+# each could be re-written separately...
+# Not sure if this is a case worth handling or what is the best way
+# to handle it, but I'm throwing the test in here anyway to remind us.
+
+def g():
+    print 'in g'
+    return 0
+
+def h():
+    print 'in h'
+    return 1
+
+class DataDescriptor(object):
+    def __get__(self, obj, typ):
+        print '__get__ called'
+        print type(self)
+        print type(obj)
+        print typ
+        return g
+    def __set__(self, obj, value):
+        pass
+
+class NonDataDescriptor(object):
+    def __get__(self, obj, typ):
+        print '__get__ called'
+        print type(self)
+        print type(obj)
+        print typ
+        return g
+
+class C(object):
+    a = DataDescriptor()
+    b = NonDataDescriptor()
+    c = NonDataDescriptor()
+
+inst = C()
+
+inst.c = h
+inst.d = h
+C.d = DataDescriptor()
+
+def f():
+    print inst.a()
+    print inst.b()
+    print inst.c()
+    print inst.d()
+
+    print C.a()
+    print C.b()
+
+for i in xrange(1000):
+    f()

test/tests/descriptors_double.py

+# TODO This is a hodgepodge of stuff, should probably organize it better
+# maybe merge some of it into dunder_descriptors?
+
+class Descriptor2(object):
+    def __get__(self, obj, type):
+        def get2(self, obj, type):
+            print '__get__ called'
+            print self
+            print obj
+            print type
+        return get2
+
+class Descriptor(object):
+    __get__ = Descriptor2()
+
+class DescriptorNonzero(object):
+    def __get__(self, obj, type):
+        print 'nonzero __get__ called'
+        def nonzero():
+            print 'nonzero called'
+            return True
+        return nonzero
+
+class C(object):
+    desc = Descriptor()
+    __nonzero__ = DescriptorNonzero()
+
+# Should throw type error: Descriptor2 is not callable
+try:
+    print C().desc
+except TypeError:
+    # TODO should print the whole stacktrace and error message probably
+    print 'got type error (1)'
+
+# Should print True; in particular, it should look up __nonzero__
+# using the descriptor protocol
+if C():
+    print 'True'
+else:
+    print 'False'
+# this should *not* override it
+c = C()
+c.__nonzero__ = lambda x : False
+if c:
+    print 'True'
+else:
+    print 'False'
+# this should
+C.__nonzero__ = lambda x : False
+if c:
+    print 'True'
+else:
+    print 'False'
+
+# __getattr__ and __setattr__
+# Looks like __getattr__ and __setattr__ should *not* be looked up with
+# the descriptor protocol
+class DescriptorGetattr(object):
+    def __get__(self, obj, type):
+        print 'getattr __get__ called'
+        def getattr(attr):
+            print 'getattr called for attr', attr
+            return 1337
+        return getattr
+class DescriptorSetattr(object):
+    def __get__(self, obj, type):
+        print 'setattr __get__ called'
+        def setattr(attr, val):
+            print 'setattr called for attr', attr, val
+class D(object):
+    __getattr__ = DescriptorGetattr
+    __setattr__ = DescriptorSetattr
+
+d = D()
+try:
+    print d.a
+except TypeError:
+    print 'got type error (2)'
+
+#TODO enable this once __setattr__ is implemented
+#try:
+#    d.b = 12
+#except TypeError:
+#    print 'got type error (3)'
+
+# with, __enter__ and __exit__
+class DescriptorEnter(object):
+    def __get__(self, obj, type):
+        print 'enter __get__ called'
+        def enter():
+            print 'enter called'
+            return 1337
+        return enter
+class DescriptorExit(object):
+    def __get__(self, obj, type):
+        print 'exit __get__ called'
+        def exit(type, value, traceback):
+            print 'enter called'
+            return 1337
+        return exit
+class E(object):
+    __enter__ = DescriptorEnter()
+    __exit__ = DescriptorExit()
+with E():
+    print 'in with'
+
+# what if __get__ is just an instance attribute
+class Descriptor(object):
+    pass
+desc = Descriptor()
+desc.__get__ = lambda self, obj, type : 5
+def s(self, obj, value):
+    print 'in __set__'
+desc.__set__ = s
+class F(object):
+    at = desc
+f = F()
+print type(f.at) # should not call __get__
+f.at = 12 # should not call __set__, should print nothing
+#TODO uncomment this:
+#print f.__dict__['at']

test/tests/descriptors_getattr_ics.py

+# run_args: -n
+# statcheck: stats['slowpath_getattr'] <= 80
+
+class DataDescriptor(object):
+    def __get__(self, obj, typ):
+        print '__get__ called'
+        print type(self)
+        print type(obj)
+        print typ
+        return 0
+    def __set__(self, obj, value):
+        pass
+
+class NonDataDescriptor(object):
+    def __get__(self, obj, typ):
+        print '__get__ called'
+        print type(self)
+        print type(obj)
+        print typ
+        return 1
+
+class C(object):
+    a = DataDescriptor()
+    b = NonDataDescriptor()
+    c = NonDataDescriptor()
+
+inst = C()
+
+inst.c = 100
+inst.d = 101
+C.d = DataDescriptor()
+
+def f():
+    print inst.a
+    print inst.b
+    print inst.c
+    print inst.d
+
+    print C.a
+    print C.b
+
+for i in xrange(1000):
+    f()

test/tests/descriptors_getclsattr_ics.py

+# run_args: -n
+# statcheck: stats['slowpath_getclsattr'] <= 60
+
+# Note: difference between DataDescriptor and NonDataDescriptor shouldn't matter
+# for getclsattr (which is how __exit__ is looked up for with statements)
+class DataDescriptor(object):
+    def __get__(self, obj, typ):
+        print '__get__ called'
+        print type(self)
+        print type(obj)
+        print typ
+        def exit(type, value, traceback):
+            print 'exit called (1)'
+        return exit
+    def __set__(self, obj, value):
+        pass
+
+class NonDataDescriptor(object):
+    def __get__(self, obj, typ):
+        print '__get__ called'
+        print type(self)
+        print type(obj)
+        print typ
+        def exit(type, value, traceback):
+            print 'exit called (2)'
+        return exit
+
+class C1(object):
+    __exit__ = DataDescriptor()
+    def __enter__(self):
+        pass
+
+class C2(object):
+    __exit__ = NonDataDescriptor()
+    def __enter__(self):
+        pass
+
+class C3(object):
+    __exit__ = NonDataDescriptor()
+    def __enter__(self):
+        pass
+
+class C4(object):
+    def __enter__(self):
+        pass
+
+c1 = C1()
+c2 = C2()
+c3 = C3()
+c4 = C4()
+
+def exit3(type, value, traceback):
+    print 'exit called (3)'
+c3.__exit__ = exit3 # this should not get called
+
+def exit4(type, value, traceback):
+    print 'exit called (4)'
+c4.__exit__ = exit4 # this should not get called
+
+C4.__exit__ = DataDescriptor()
+
+def f():
+    with c1:
+        print 'in with statement (1)'
+
+    with c2:
+        print 'in with statement (2)'
+
+    with c3:
+        print 'in with statement (3)'
+
+    with c4:
+        print 'in with statement (4)'
+
+for i in xrange(1000):
+    f()

test/tests/descriptors_guards.py

+# All the ways of invalidating getattr
+
+# TODO should test getclsattr as well
+# TODO should also test some crazier stuff, like descriptors with inheritance
+
+def get(self, obj, typ):
+    print '__get__ called'
+    print type(self)
+    print type(obj)
+    print typ
+    return self.elem
+
+def set(self, obj, typ):
+    print '__get__ called'
+    print type(self)
+    print type(obj)
+    print typ
+
+class Descriptor(object):
+    def __init__(self, elem):
+        self.elem = elem
+    def __str__(self):
+        return 'Descriptor object'
+
+class C(object):
+    a = Descriptor(0)
+    b = Descriptor(lambda : 0)
+
+c = C()
+
+def f():
+    print c.a
+    print C.a
+
+def g():
+    try:
+        print c.b()
+    except TypeError:
+        print 'got TypeError'
+
+    try:
+        print C.b()
+    except TypeError:
+        print 'got TypeError'
+
+def h():
+    c.c = 10
+
+for i in xrange(2000):
+    f()
+    g()
+    h()
+
+    if i == 50:
+        Descriptor.__get__ = get
+    if i == 100:
+        Descriptor.__set__ = set
+    if i == 150:
+        del Descriptor.__get__
+    if i == 200:
+        del Descriptor.__set__
+
+    if i == 250:
+        Descriptor.__set__ = set
+    if i == 300:
+        Descriptor.__get__ = get
+    if i == 350:
+        del Descriptor.__set__
+    if i == 400:
+        del Descriptor.__get__
+
+    if i == 450:
+        Descriptor.__get__ = get
+        Descriptor.__set__ = set
+    if i == 500:
+        del Descriptor.__get__
+        del Descriptor.__set__
+
+    if i == 550:
+        Descriptor.__get__ = get
+    if i == 600:
+        Descriptor.__set__ = set
+        del Descriptor.__get__
+    if i == 650:
+        Descriptor.__get__ = get
+        del Descriptor.__set__
+
+    if i == 700:
+        c.a = 5
+        c.b = lambda : 5
+    if i == 750:
+        del c.a
+        del c.b
+
+    if i == 800:
+        Descriptor.__set__ = set
+    if i == 850:
+        del Descriptor.__set__
+        c.a = 5
+        c.b = lambda : 5
+        Descriptor.__set__ = set
+    if i == 900:
+        del Descriptor.__set__
+        del c.a
+        del c.b
+        Descriptor.__set__ = set
+
+    if i == 950:
+        del Descriptor.__get__
+    if i == 1000:
+        del Descriptor.__set__
+        c.a = 5
+        c.b = lambda : 5
+        Descriptor.__set__ = set
+    if i == 1050:
+        del Descriptor.__set__
+        del c.a
+        del c.b
+        Descriptor.__set__ = set
+
+    if i == 1100:
+        del Descriptor.__set__
+    if i == 1150:
+        c.a = 5
+        c.b = lambda : 5
+    if i == 1200:
+        del c.a
+        del c.b
+
+    if i == 1250:
+        c.a = 5
+        c.b = lambda : 5
+
+    if i == 1350:
+        Descriptor.__get__ = get
+    if i == 1400:
+        Descriptor.__set__ = set
+    if i == 1450:
+        del Descriptor.__get__
+    if i == 1500:
+        del Descriptor.__set__
+
+    if i == 1550:
+        Descriptor.__set__ = set
+    if i == 1600:
+        Descriptor.__get__ = get
+    if i == 1650:
+        del Descriptor.__set__
+    if i == 1700:
+        del Descriptor.__get__
+
+    if i == 1750:
+        Descriptor.__get__ = get
+        Descriptor.__set__ = set
+    if i == 1800:
+        del Descriptor.__get__
+        del Descriptor.__set__
+
+    if i == 1850:
+        Descriptor.__get__ = get
+    if i == 1900:
+        Descriptor.__set__ = set
+        del Descriptor.__get__
+    if i == 1950:
+        Descriptor.__get__ = get
+        del Descriptor.__set__

test/tests/descriptors_nonfunc_types.py

+# expected: fail
+
+# See what happens when we make __get__ and __set__ things other than functions...
+# TODO add some with __del__
+
+import traceback
+
+class CallableGet(object):
+    def __call__(self, a, b, c):
+        print 'Callable get'
+        print self
+        print a
+        print b
+        print c
+class CallableSet(object):
+    def __call__(a, b, c):
+        print 'Callable set'
+        print a
+        print b
+        print c
+
+class InstanceMethodMaker(object):
+    def getBoundInstanceMethod(self, a, b, c):
+        print '__get__ bound'
+        print a
+        print b
+        print c
+    def setBoundInstanceMethod(a, b, c):
+        print '__set__ bound'
+        print a
+        print b
+        print c
+    def getUnboundInstanceMethod(a, b, c):
+        print '__get__ unbound'
+        print a
+        print b
+        print c
+    def setUnboundInstanceMethod(a, b, c):
+        print '__set__ unbound'
+        print a
+        print b
+        print c
+imm = InstanceMethodMaker()
+
+def closureGet():
+    a = 5
+    def f(b, c, d):
+        print 'closure __get__'
+        print a
+        print b
+        print c
+        print d
+    return f
+def closureSet():
+    def f(b, c, d):
+        print 'closure __set__'
+        print a
+        print b
+        print c
+        print d
+    return f
+
+class A(object):
+    # If __get__ or __set__ is an int
+    class DescGetInt(object):
+        __get__ = 1
+    descGetInt = DescGetInt()
+    class DescSetInt(object):
+        __set__  = 1
+    descSetInt = DescSetInt()
+    class DescGetSetInt(object):
+        def __get__(a, b, c):
+            print 'DescGetSetInt __get__ called'
+            print a
+            print b
+            print c
+        __set__ = 1
+    descGetSetInt = DescGetSetInt()
+
+    class DescGetCall(object):
+        __get__ = CallableGet()
+    descGetCall = DescGetCall()
+    class DescSetCall(object):
+        __set__ = CallableSet()
+    descSetCall = DescSetCall()
+    class DescGetSetCall(object):
+        def __get__(a, b, c):
+            print 'DescGetSetCall __get__ called'
+            print a
+            print b
+            print c
+        __set__ = CallableSet()
+    descGetSetCall = DescGetSetCall()
+
+    class DescGetBoundInstanceMethod(object):
+        __get__ = imm.getBoundInstanceMethod
+    descGetBoundInstanceMethod = DescGetBoundInstanceMethod()
+    class DescSetBoundInstanceMethod(object):
+        __set__ = imm.setBoundInstanceMethod
+    descSetBoundInstanceMethod = DescSetBoundInstanceMethod()
+    class DescGetSetBoundInstanceMethod(object):
+        def __get__(a, b, c):
+            print 'DescGetSetBoundInstanceMethod __get__ called'
+            print a
+            print b
+            print c
+        __set__ = imm.setBoundInstanceMethod
+    descGetSetBoundInstanceMethod = DescGetSetBoundInstanceMethod()
+
+    class DescGetUnboundInstanceMethod(object):
+        __get__ = InstanceMethodMaker.getUnboundInstanceMethod
+    descGetUnboundInstanceMethod = DescGetUnboundInstanceMethod()
+    class DescSetUnboundInstanceMethod(object):
+        __set__ = InstanceMethodMaker.setUnboundInstanceMethod
+    descSetUnboundInstanceMethod = DescSetUnboundInstanceMethod()
+    class DescGetSetUnboundInstanceMethod(object):
+        def __get__(a, b, c):
+            print 'DescGetSetUnboundInstanceMethod __get__ called'
+            print a
+            print b
+            print c
+        __set__ = imm.setUnboundInstanceMethod
+    descGetSetUnboundInstanceMethod = DescGetSetUnboundInstanceMethod()
+
+    class DescGetClosure(object):
+        __get__ = closureGet()
+    descGetClosure = DescGetClosure()
+    class DescSetClosure(object):
+        __set__ = closureSet()
+    descSetClosure = DescSetClosure()
+    class DescGetSetClosure(object):
+        def __get__(a, b, c):
+            print 'DescGetSetClosure __get__ called'
+            print a
+            print b
+            print c
+        __set__ = closureSet()
+    descGetSetClosure = DescGetSetClosure()
+
+    class DescGetGenerator(object):
+        def __get__(self, obj, type):
+            print 'DescGetGenerator __get__ called'
+            print self
+            print obj
+            print type
+            yield 15
+            print '__get__ post yield'
+    descGetGenerator = DescGetGenerator()
+    class DescSetGenerator(object):
+        def __set__(self, obj, value):
+            print 'DescSetGenerator __set__ called'
+            print self
+            print obj
+            print value
+            yield 15
+            print '__set__ post yield'
+    descSetGenerator = DescSetGenerator()
+    class DescGetSetGenerator(object):
+        def __get__(a, b, c):
+            print 'DescGetSetGenerator __get__ called'
+            print a
+            print b
+            print c
+        def __set__(self, obj, value):
+            print 'DescGetSetGenerator __set__ called'
+            print self
+            print obj
+            print value
+            yield 15
+            print 'DescGetSetGenerator __set__ post yield'
+    descGetSetGenerator = DescGetSetGenerator()
+
+    descSetClosure = DescSetClosure()
+
+a = A()
+
+print 'int'
+try:
+    print a.descGetInt
+except:
+    traceback.print_exc()
+
+try:
+    a.descSetInt = 5
+except:
+    traceback.print_exc()
+
+a.__dict__['descGetSetInt'] = 3
+print a.descGetSetInt
+
+print 'object with __call__'
+print a.descGetCall
+a.descSetCall = 5
+a.__dict__['descGetSetCall'] = 3
+print a.descGetSetCall
+
+print 'bound instance method'
+print a.descGetBoundInstanceMethod
+a.descSetBoundInstanceMethod = 5
+a.__dict__['descGetSetBoundInstanceMethod'] = 3
+print a.descGetSetBoundInstanceMethod
+
+print 'unbound instance method'
+try:
+    print a.descGetUnboundInstanceMethod
+except:
+    traceback.print_exc()
+try:
+    a.descSetUnboundInstanceMethod = 5
+except:
+    traceback.print_exc()
+a.__dict__['descGetSetUnboundInstanceMethod'] = 3
+print a.descGetSetUnboundInstanceMethod
+
+print 'closure'
+print a.descGetClosure
+a.descSetClosure = 5
+a.__dict__['descGetSetClosure'] = 3
+print a.descGetClosure
+
+print 'generator'
+print a.descGetGenerator
+a.descSetGenerator = 5
+a.__dict__['descGetSetGenerator'] = 3
+print a.descGetGenerator
+

test/tests/descriptors_setattr_ics.py

+# run_args: -n
+# statcheck: stats['slowpath_setattr'] <= 120
+
+class Descriptor(object):
+    def __set__(self, obj, value):
+        print '__set__ called'
+        print type(self)
+        print type(obj)
+        print type(value)
+
+class C(object):
+    a = Descriptor()
+
+c = C()
+
+def f(i):
+    c.a = i
+
+for i in xrange(1000):
+    f(i)

test/tests/dunder_descriptors.py

-# expected: fail
 # - descriptors
 
 #  Descriptors get processed when fetched as part of a dunder lookup

test/tests/function_instancemethod.py

+# TODO test all of this with getclsattr
+# TODO should make an ics test
+
+class C(object):
+    def f():
+        pass
+
+def g():
+    print 'running g'
+
+print C.f == C.f
+print C.f is C.f
+print C().f == C().f
+print C().f is C().f
+
+#### Check the types of stuff
+
+print type(C.f) # instancemethod
+print type(C().f) # instancemethod
+print type(g) # function
+
+C.g = g
+print type(C.g) # instancemethod
+print type(C().g) # instancemethod
+
+#### Assign a function to an instance
+
+c = C()
+c.g = g
+print type(c.g) # function
+c.g()
+
+print c.g == c.g
+print c.g is c.g
+
+#### Assign a function to a class
+
+def l(inst):
+    print 'running l', inst.i
+C.l = l
+print type(C.l) #instancemethod
+print type(C().l) #instancemethod
+c1 = C()
+c1.i = 1
+C.l(c1)
+c1.l()
+
+print c1.l == c1.l
+print c1.l is c1.l
+print C.l == C.l
+print C.l is C.l
+
+#### Assign a bound instancemethod to a class
+
+C.k = c1.l # set C.k to a bound instancemethod
+C.k() # this should call l with with c1 as the arg
+c2 = C()
+c2.i = 2
+c2.k() # this should just call l with c1 as the arg, not try to bind anything else
+print type(C.k) # instancemethod
+print type(c2.k) # instancemethod
+
+print c2.k == c2.k
+print c2.k is c2.k
+print C.k == C.k
+print C.k is C.k
+print C.k == c2.k
+print C.k is c2.k
+
+#### Assign an unbound instancemethod to a class
+#### Getting is will bind it like a normal function
+
+# TODO implement instancemethod stuff so this case works
+"""
+C.m = C.l
+print type(C.m) #instancemethod
+print type(C().m) #instancemethod
+c3 = C()
+c3.i = 3
+C.m(c3)
+c3.m()
+
+print c3.m == c3.m
+print c3.m is c3.m
+print C.m == C.m
+print C.m is C.m
+"""
+
+### Assign a bound instancemethod to an instance
+
+c4 = C()
+c4.i = 4
+c4.z = c1.l
+print type(c4.z) # instancemethod
+c4.z() # should call l(c1)
+
+print c4.z == c4.z
+print c4.z is c4.z
+
+### Assign an unbound instancemethod to an instance
+
+c4 = C()
+c4.i = 4
+c4.z = C.l
+print type(c4.z) # instancemethod
+c4.z(c1) # should call l(c1)
+
+print c4.z == c4.z
+print c4.z is c4.z
+
+### Call a bound instancemethod on its own (not through the callattr path)
+bound_instancemethod = c1.l
+bound_instancemethod()
+print type(bound_instancemethod)
+
+### Call an unbound instancemethod on its own (not through the callattr path)
+unbound_instancemethod = C.l
+unbound_instancemethod(c2)
+print type(unbound_instancemethod)

test/tests/getattr_classmember_ordering.py

+class C(object):
+    i = 5
+
+    def __getattr__(self, attr):
+        print '__getattr__ called on attr', attr
+        return 6
+
+c = C()
+print c.i  # should print 5, not call __getattr__
+
+class D(object):
+    i = 5
+
+class E(D):
+    def __getattr__(self, attr):
+        print '__getattr__ called on attr', attr
+        return 6
+    pass
+
+e = E()
+print e.i # should print 5, not call __getattr__

test/tests/instancemethod_guards.py

+# Test what happens when we swap out a function (which we handle special-cased
+# in the descriptor logic) for a descriptor of another type
+# TODO should be more thorough
+# TODO should write similar tests for MemberDescriptors too
+
+def g():
+    print 'in g'
+
+class Descriptor(object):
+    def __str__(self):
+        return 'Descriptor'
+    def __get__(self, obj, objtype):
+        return g
+
+class C(object):
+    def f(self):
+        print 'in f'
+
+c = C()
+
+def run():
+    c.f()
+
+    t = c.f
+    t()
+
+for i in xrange(100):
+    run()
+
+    if i == 50:
+        C.f = Descriptor()
+
+# Swap an unbound for a bound
+
+class C(object):
+    def f(self, a=0):
+        print "f", a
+    def __str__(self):
+        return "C obj"
+
+def call(f, c):
+    #TODO uncomment this
+    #print f
+    f(c)
+
+c = C()
+call(C.f, c)
+call(c.f, c)

test/tests/object_new_arguments.py

+# expected: fail
 # object.__new__ doesn't complain if __init__ is overridden:
 
 class C1(object):

tools/tester.py

@@ -197,7 +197,10 @@ def run_test(fn, check_stats, run_memcheck):
             os.unlink(out_fn)
             raise Exception("Failed on %s:\n%s" % (fn, diff))
     elif not TEST_PYPY and canonicalize_stderr(stderr) != canonicalize_stderr(expected_err):
-        if KEEP_GOING:
+        if expected == "fail":
+            r += "    Expected failure (bad stderr)"
+            return r
+        elif KEEP_GOING:
             r += "    \033[31mFAILED\033[0m (bad stderr)"
             failed.append(fn)
             return r