Merge pull request #295 from toshok/tp_getattr

add tp_getattr support, PyCFunction_NewEx, and most of PyNumber_Int's behavior

from_cpython/Modules/selectmodule.c

@@ -1830,6 +1830,9 @@ initselect(void)
     {
 #endif
         Py_TYPE(&poll_Type) = &PyType_Type;
+        if (PyType_Ready(&poll_Type) < 0)
+          return;
+
         PyModule_AddIntConstant(m, "POLLIN", POLLIN);
         PyModule_AddIntConstant(m, "POLLPRI", POLLPRI);
         PyModule_AddIntConstant(m, "POLLOUT", POLLOUT);

src/capi/typeobject.cpp

@@ -22,6 +22,8 @@ namespace pyston {
 
 // FIXME duplicated with objmodel.cpp
 static const std::string _new_str("__new__");
+static const std::string _getattr_str("__getattr__");
+static const std::string _getattribute_str("__getattribute__");
 
 extern "C" void conservativeGCHandler(GCVisitor* v, Box* b) noexcept {
     v->visitPotentialRange((void* const*)b, (void* const*)((char*)b + b->cls->tp_basicsize));
@@ -623,6 +625,73 @@ static PyObject* slot_tp_iternext(PyObject* self) noexcept {
     return call_method(self, "next", &next_str, "()");
 }
 
+static PyObject* slot_tp_getattro(PyObject* self, PyObject* name) noexcept {
+    static PyObject* getattribute_str = NULL;
+    return call_method(self, "__getattribute__", &getattribute_str, "(O)", name);
+}
+
+static PyObject* call_attribute(PyObject* self, PyObject* attr, PyObject* name) noexcept {
+    PyObject* res, * descr = NULL;
+    descrgetfunc f = Py_TYPE(attr)->tp_descr_get;
+
+    if (f != NULL) {
+        descr = f(attr, self, (PyObject*)(Py_TYPE(self)));
+        if (descr == NULL)
+            return NULL;
+        else
+            attr = descr;
+    }
+    try {
+        res = runtimeCall(attr, ArgPassSpec(1, 0, true, true), name, NULL, NULL, NULL, NULL);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        Py_XDECREF(descr);
+        return NULL;
+    }
+    Py_XDECREF(descr);
+    return res;
+}
+
+static PyObject* slot_tp_getattr_hook(PyObject* self, PyObject* name) noexcept {
+    PyObject* getattr, *getattribute, * res = NULL;
+    static PyObject* getattribute_str = NULL;
+    static PyObject* getattr_str = NULL;
+
+    /* speed hack: we could use lookup_maybe, but that would resolve the
+         method fully for each attribute lookup for classes with
+         __getattr__, even when the attribute is present. So we use
+         _PyType_Lookup and create the method only when needed, with
+         call_attribute. */
+    getattr = typeLookup(self->cls, _getattr_str, NULL);
+    if (getattr == NULL) {
+        /* No __getattr__ hook: use a simpler dispatcher */
+        self->cls->tp_getattro = slot_tp_getattro;
+        return slot_tp_getattro(self, name);
+    }
+    /* speed hack: we could use lookup_maybe, but that would resolve the
+         method fully for each attribute lookup for classes with
+         __getattr__, even when self has the default __getattribute__
+         method. So we use _PyType_Lookup and create the method only when
+         needed, with call_attribute. */
+    getattribute = typeLookup(self->cls, _getattribute_str, NULL);
+    if (getattribute == NULL
+        || (Py_TYPE(getattribute) == wrapperdescr_cls
+            && ((BoxedWrapperDescriptor*)getattribute)->wrapped == (void*)PyObject_GenericGetAttr)) {
+        res = PyObject_GenericGetAttr(self, name);
+    } else {
+        res = call_attribute(self, getattribute, name);
+    }
+    if (res == NULL) {
+        try {
+            res = runtimeCall(getattr, ArgPassSpec(2, 0, true, true), self, name, NULL, NULL, NULL);
+        } catch (ExcInfo e) {
+            setCAPIException(e);
+            return NULL;
+        }
+    }
+    return res;
+}
+
 static PyObject* slot_tp_new(PyTypeObject* self, PyObject* args, PyObject* kwds) noexcept {
     try {
         // TODO: runtime ICs?
@@ -1063,11 +1132,16 @@ static void** slotptr(BoxedClass* type, int offset) noexcept {
     ETSLOT(NAME, as_number.SLOT, FUNCTION, wrap_binaryfunc_r, "x." NAME "(y) <==> " DOC)
 
 static slotdef slotdefs[]
-    = { TPSLOT("__repr__", tp_repr, slot_tp_repr, wrap_unaryfunc, "x.__repr__() <==> repr(x)"),
+    = { TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""),
+
+        TPSLOT("__repr__", tp_repr, slot_tp_repr, wrap_unaryfunc, "x.__repr__() <==> repr(x)"),
         TPSLOT("__hash__", tp_hash, slot_tp_hash, wrap_hashfunc, "x.__hash__() <==> hash(x)"),
         FLSLOT("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call, "x.__call__(...) <==> x(...)",
                PyWrapperFlag_KEYWORDS),
         TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc, "x.__str__() <==> str(x)"),
+
+        TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""),
+
         TPSLOT("__lt__", tp_richcompare, slot_tp_richcompare, richcmp_lt, "x.__lt__(y) <==> x<y"),
         TPSLOT("__le__", tp_richcompare, slot_tp_richcompare, richcmp_le, "x.__le__(y) <==> x<=y"),
         TPSLOT("__eq__", tp_richcompare, slot_tp_richcompare, richcmp_eq, "x.__eq__(y) <==> x==y"),
@@ -1714,7 +1788,6 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
     gc::registerNonheapRootObject(cls);
 
     // unhandled fields:
-    RELEASE_ASSERT(cls->tp_getattr == NULL, "");
     RELEASE_ASSERT(cls->tp_setattr == NULL, "");
     RELEASE_ASSERT(cls->tp_compare == NULL, "");
 

src/runtime/capi.cpp

@@ -319,7 +319,12 @@ extern "C" PyObject* PyObject_GetAttr(PyObject* o, PyObject* attr_name) noexcept
 }
 
 extern "C" PyObject* PyObject_GenericGetAttr(PyObject* o, PyObject* name) noexcept {
-    Py_FatalError("unimplemented");
+    try {
+        return getattr(o, static_cast<BoxedString*>(name)->s.c_str());
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
 }
 
 extern "C" PyObject* PyObject_GetItem(PyObject* o, PyObject* key) noexcept {
@@ -646,7 +651,7 @@ void setCAPIException(const ExcInfo& e) {
 
 void throwCAPIException() {
     checkAndThrowCAPIException();
-    llvm_unreachable("No exception was thrown?");
+    raiseExcHelper(SystemError, "error return without exception set");
 }
 
 void checkAndThrowCAPIException() {
@@ -1045,7 +1050,64 @@ extern "C" int PyNumber_CoerceEx(PyObject**, PyObject**) noexcept {
 }
 
 extern "C" PyObject* PyNumber_Int(PyObject* o) noexcept {
-    Py_FatalError("unimplemented");
+    PyNumberMethods* m;
+    const char* buffer;
+    Py_ssize_t buffer_len;
+
+    if (o == NULL) {
+        PyErr_SetString(PyExc_SystemError, "null argument to internal routing");
+        return NULL;
+    }
+    if (PyInt_CheckExact(o)) {
+        Py_INCREF(o);
+        return o;
+    }
+    m = o->cls->tp_as_number;
+    if (m && m->nb_int) { /* This should include subclasses of int */
+        /* Classic classes always take this branch. */
+        PyObject* res = m->nb_int(o);
+        if (res && (!PyInt_Check(res) && !PyLong_Check(res))) {
+            PyErr_Format(PyExc_TypeError, "__int__ returned non-int (type %.200s)", res->cls->tp_name);
+            Py_DECREF(res);
+            return NULL;
+        }
+        return res;
+    }
+    if (PyInt_Check(o)) { /* A int subclass without nb_int */
+        BoxedInt* io = (BoxedInt*)o;
+        return PyInt_FromLong(io->n);
+    }
+
+    Py_FatalError("unimplemented __trunc__ and string -> int conversion");
+// the remainder of PyNumber_Int deals with __trunc__ usage, and converting from unicode/string to int
+#if 0
+    PyObject* trunc_func = getattr(o, "__trunc__");
+    if (trunc_func) {
+      PyObject *truncated = PyEval_CallObject(trunc_func, NULL);
+      Py_DECREF(trunc_func);
+      /* __trunc__ is specified to return an Integral type, but
+	 int() needs to return an int. */
+      return _PyNumber_ConvertIntegralToInt(
+					  truncated,
+					  "__trunc__ returned non-Integral (type %.200s)");
+    }
+    PyErr_Clear();  /* It's not an error if  o.__trunc__ doesn't exist. */
+
+    if (PyString_Check(o))
+      return int_from_string(PyString_AS_STRING(o),
+                 PyString_GET_SIZE(o));
+#ifdef Py_USING_UNICODE
+    if (PyUnicode_Check(o))
+      return PyInt_FromUnicode(PyUnicode_AS_UNICODE(o),
+                   PyUnicode_GET_SIZE(o),
+                   10);
+#endif
+    if (!PyObject_AsCharBuffer(o, &buffer, &buffer_len))
+      return int_from_string((char*)buffer, buffer_len);
+
+    return type_error("int() argument must be a string or a "
+              "number, not '%.200s'", o);
+#endif
 }
 
 extern "C" PyObject* PyNumber_Long(PyObject* o) noexcept {
@@ -1371,7 +1433,8 @@ extern "C" PyObject* Py_FindMethod(PyMethodDef* methods, PyObject* self, const c
 }
 
 extern "C" PyObject* PyCFunction_NewEx(PyMethodDef* ml, PyObject* self, PyObject* module) noexcept {
-    Py_FatalError("unimplemented");
+
+    return new BoxedCApiFunction(ml->ml_flags, self, ml->ml_name, ml->ml_meth);
 }
 
 extern "C" int _PyEval_SliceIndex(PyObject* v, Py_ssize_t* pi) noexcept {

src/runtime/objmodel.cpp

@@ -483,6 +483,7 @@ BoxedDict* Box::getDict() {
 }
 
 Box* Box::getattr(const std::string& attr, GetattrRewriteArgs* rewrite_args) {
+
     if (rewrite_args)
         rewrite_args->obj->addAttrGuard(BOX_CLS_OFFSET, (intptr_t)cls);
 
@@ -1414,6 +1415,13 @@ Box* getattrInternalGeneral(Box* obj, const std::string& attr, GetattrRewriteArg
         // invalidation rather than guards
         rewrite_args = NULL;
         REWRITE_ABORTED("");
+
+        if (obj->cls->tp_getattr) {
+            Box* rtn = obj->cls->tp_getattr(obj, const_cast<char*>(attr.c_str()));
+            if (rtn == NULL)
+                throwCAPIException();
+            return rtn;
+        }
         Box* getattr = typeLookup(obj->cls, getattr_str, NULL);
         if (getattr) {
             Box* boxstr = boxString(attr);

src/runtime/types.h

@@ -226,8 +226,7 @@ public:
     // that we can't rely on for extension classes.
     bool is_pyston_class;
 
-    // will need to update this once we support tp_getattr-style overriding:
-    bool hasGenericGetattr() { return true; }
+    bool hasGenericGetattr() { return tp_getattr != NULL; }
 
     void freeze();
 

test/test_extension/slots_test.c

@@ -524,6 +524,54 @@ static PyTypeObject slots_tester_sub = {
     0,                                  /* tp_free */
 };
 
+static PyObject*
+getattr_returnnull(PyObject* self, const char* attr)
+{
+  return NULL;
+}
+
+static PyTypeObject slots_tester_nullreturngetattr = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "slots_test.slots_tester_nullreturngetattr",            /* tp_name */
+    sizeof(slots_tester_object_sub),          /* tp_basicsize */
+    0,                                  /* tp_itemsize */
+    /* methods */
+    0,                                  /* tp_dealloc */
+    0,                                  /* tp_print */
+    (getattrfunc)getattr_returnnull,    /* tp_getattr */
+    0,                                  /* tp_setattr */
+    0,                                  /* tp_compare */
+    0,        /* tp_repr */
+    0,                                  /* tp_as_number */
+    0,          /* tp_as_sequence */
+    0,                                  /* tp_as_mapping */
+    0,                                  /* tp_hash */
+    0,     /* tp_call */
+    0,     /* tp_str */
+    0,                                  /* tp_getattro */
+    0,                                  /* tp_setattro */
+    0,                                  /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES,                 /* tp_flags */
+    0,                   /* tp_doc */
+    0,                                  /* tp_traverse */
+    0,                                  /* tp_clear */
+    0,                                  /* tp_richcompare */
+    0,                                  /* tp_weaklistoffset */
+    0,                                  /* tp_iter */
+    0,                                  /* tp_iternext */
+    0,                                  /* tp_methods */
+    0,                                  /* tp_members */
+    0,                                  /* tp_getset */
+    &slots_tester_seq,                                  /* tp_base */
+    0,                                  /* tp_dict */
+    0,                                  /* tp_descr_get */
+    0,                                  /* tp_descr_set */
+    0,                                  /* tp_dictoffset */
+    0,                                  /* tp_init */
+    0,                                  /* tp_alloc */
+    0,                   /* tp_new */
+    0,                                  /* tp_free */
+};
 
 // Tests the correctness of the CAPI slots when the attributes get set in Python code:
 static PyObject *
@@ -572,6 +620,12 @@ call_funcs(PyObject* _module, PyObject* args) {
         printf("tp_call doesnt exist\n");
     }
 
+    if (cls->tp_getattr) {
+        printf("tp_getattr exists\n");
+    } else {
+        printf("tp_getattr doesnt exist\n");
+    }
+
     if (cls->tp_as_mapping) {
         printf("tp_as_mapping exists\n");
         PyMappingMethods* map = cls->tp_as_mapping;
@@ -714,6 +768,10 @@ initslots_test(void)
     if (res < 0)
         return;
 
+    res = PyType_Ready(&slots_tester_nullreturngetattr);
+    if (res < 0)
+        return;
+
     // Not sure if the result of PyInt_FromLong needs to be decref'd
     PyDict_SetItemString(slots_tester_seq.tp_dict, "set_through_tpdict", PyInt_FromLong(123));
 
@@ -722,4 +780,5 @@ initslots_test(void)
     PyModule_AddObject(m, "SlotsTesterNum", (PyObject *)&slots_tester_num);
     PyModule_AddObject(m, "SlotsTesterSub", (PyObject *)&slots_tester_sub);
     PyModule_AddObject(m, "SlotsTesterNonsubclassable", (PyObject *)&slots_tester_nonsubclassable);
+    PyModule_AddObject(m, "SlotsTesterNullReturnGetAttr", (PyObject *)&slots_tester_nullreturngetattr);
 }

test/tests/capi_slots.py

@@ -115,3 +115,46 @@ try:
     print s.__dict__
 except AttributeError, e:
     print e
+
+
+class C5(C):
+    def __getattr__(self, attr):
+        print "getattr", attr
+
+c = C5()
+slots_test.call_funcs(c)
+c.foo
+c.bar
+c.baz
+
+
+def _getattr_(self, attr):
+    print "_getattr_", attr
+
+class C6(C):
+    pass
+
+c = C6()
+c.__getattr__ = _getattr_
+slots_test.call_funcs(c)
+
+try:
+    c.foo
+except AttributeError, e:
+    print e
+
+c.__getattro__ = _getattr_
+slots_test.call_funcs(c)
+
+try:
+    c.foo
+except AttributeError, e:
+    print e
+
+c = slots_test.SlotsTesterNullReturnGetAttr(5)
+try:
+    print c.foo
+except SystemError, e:
+    print e
+
+print c.foo()

test/tests/select_test.py

@@ -12,3 +12,11 @@ for k in sorted(dir(select)):
     else:
         print k, getattr(select, k)
 
+
+p = select.poll()
+f = open('/dev/null')
+try:
+    p.register(f)
+    print p.poll(10)
+finally:
+    f.close()