Add a number of C API fixes

from_cpython/Include/methodobject.h

@@ -14,6 +14,8 @@ extern "C" {
 
 // Pyston change: this is no longer a static object
 //PyAPI_DATA(PyTypeObject) PyCFunction_Type;
+PyAPI_DATA(PyTypeObject*) builtin_function_or_method_cls;
+#define PyCFunction_Type (*builtin_function_or_method_cls)
 
 #define PyCFunction_Check(op) (Py_TYPE(op) == &PyCFunction_Type)
 

from_cpython/Include/pyport.h

@@ -385,6 +385,11 @@ typedef PY_LONG_LONG            Py_intptr_t;
 #endif /* !HAVE_SYS_TIME_H */
 #endif /* !TIME_WITH_SYS_TIME */
 
+#ifdef SIZE_MAX
+#define PY_SIZE_MAX SIZE_MAX
+#else
+#define PY_SIZE_MAX ((size_t)-1)
+#endif
 
 /* Py_ARITHMETIC_RIGHT_SHIFT
  * C doesn't define whether a right-shift of a signed integer sign-extends

from_cpython/Include/sliceobject.h

@@ -23,15 +23,12 @@ names are from range).  After much talk with Guido, it was decided to
 let these be any arbitrary python type.  Py_None stands for omitted values.
 */
 
-// Pyston change: comment this out since this is not the format we're using
-#if 0
+// Pyston note: this happens to be the same format we use (not a lot going on here),
+// and we assert so in runtime/types.h
 typedef struct {
     PyObject_HEAD
     PyObject *start, *stop, *step;	/* not NULL */
 } PySliceObject;
-#endif
-struct _PySliceObject;
-typedef struct _PySliceObject PySliceObject;
 
 // Pyston change: these are no longer static objects
 PyAPI_DATA(PyTypeObject*) slice_cls;

from_cpython/Include/traceback.h

@@ -29,7 +29,7 @@ PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, const char *, int, int) PYSTON
 /* Reveal traceback type so we can typecheck traceback objects */
 // Pyston change: not a static type any more
 PyAPI_DATA(PyTypeObject*) traceback_cls;
-#define PyTraceback_Type (*traceback_cls)
+#define PyTraceBack_Type (*traceback_cls)
 // PyAPI_DATA(PyTypeObject) PyTraceBack_Type;
 #define PyTraceBack_Check(v) (Py_TYPE(v) == &PyTraceBack_Type)
 

src/capi/abstract.cpp

@@ -28,8 +28,41 @@
 namespace pyston {
 
 extern "C" Py_ssize_t _PyObject_LengthHint(PyObject* o, Py_ssize_t defaultvalue) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return -1;
+    static PyObject* hintstrobj = NULL;
+    PyObject* ro, *hintmeth;
+    Py_ssize_t rv;
+
+    /* try o.__len__() */
+    rv = PyObject_Size(o);
+    if (rv >= 0)
+        return rv;
+    if (PyErr_Occurred()) {
+        if (!PyErr_ExceptionMatches(PyExc_TypeError) && !PyErr_ExceptionMatches(PyExc_AttributeError))
+            return -1;
+        PyErr_Clear();
+    }
+
+    if (PyInstance_Check(o))
+        return defaultvalue;
+    /* try o.__length_hint__() */
+    hintmeth = _PyObject_LookupSpecial(o, "__length_hint__", &hintstrobj);
+    if (hintmeth == NULL) {
+        if (PyErr_Occurred())
+            return -1;
+        else
+            return defaultvalue;
+    }
+    ro = PyObject_CallFunctionObjArgs(hintmeth, NULL);
+    Py_DECREF(hintmeth);
+    if (ro == NULL) {
+        if (!PyErr_ExceptionMatches(PyExc_TypeError) && !PyErr_ExceptionMatches(PyExc_AttributeError))
+            return -1;
+        PyErr_Clear();
+        return defaultvalue;
+    }
+    rv = PyNumber_Check(ro) ? PyInt_AsSsize_t(ro) : defaultvalue;
+    Py_DECREF(ro);
+    return rv;
 }
 
 static int _IsFortranContiguous(Py_buffer* view) {
@@ -1126,6 +1159,85 @@ extern "C" int PySequence_Contains(PyObject* seq, PyObject* ob) noexcept {
     return Py_SAFE_DOWNCAST(result, Py_ssize_t, int);
 }
 
+extern "C" PyObject* PySequence_Tuple(PyObject* v) noexcept {
+    PyObject* it; /* iter(v) */
+    Py_ssize_t n; /* guess for result tuple size */
+    PyObject* result = NULL;
+    Py_ssize_t j;
+
+    if (v == NULL)
+        return null_error();
+
+    /* Special-case the common tuple and list cases, for efficiency. */
+    if (PyTuple_CheckExact(v)) {
+        /* Note that we can't know whether it's safe to return
+           a tuple *subclass* instance as-is, hence the restriction
+           to exact tuples here.  In contrast, lists always make
+           a copy, so there's no need for exactness below. */
+        Py_INCREF(v);
+        return v;
+    }
+    if (PyList_Check(v))
+        return PyList_AsTuple(v);
+
+    /* Get iterator. */
+    it = PyObject_GetIter(v);
+    if (it == NULL)
+        return NULL;
+
+    /* Guess result size and allocate space. */
+    n = _PyObject_LengthHint(v, 10);
+    if (n == -1)
+        goto Fail;
+    result = PyTuple_New(n);
+    if (result == NULL)
+        goto Fail;
+
+    /* Fill the tuple. */
+    for (j = 0;; ++j) {
+        PyObject* item = PyIter_Next(it);
+        if (item == NULL) {
+            if (PyErr_Occurred())
+                goto Fail;
+            break;
+        }
+        if (j >= n) {
+            Py_ssize_t oldn = n;
+            /* The over-allocation strategy can grow a bit faster
+               than for lists because unlike lists the
+               over-allocation isn't permanent -- we reclaim
+               the excess before the end of this routine.
+               So, grow by ten and then add 25%.
+            */
+            n += 10;
+            n += n >> 2;
+            if (n < oldn) {
+                /* Check for overflow */
+                PyErr_NoMemory();
+                Py_DECREF(item);
+                goto Fail;
+            }
+            if (_PyTuple_Resize(&result, n) != 0) {
+                Py_DECREF(item);
+                goto Fail;
+            }
+        }
+        PyTuple_SET_ITEM(result, j, item);
+    }
+
+    /* Cut tuple back if guess was too large. */
+    if (j < n && _PyTuple_Resize(&result, j) != 0)
+        goto Fail;
+
+    Py_DECREF(it);
+    return result;
+
+Fail:
+    Py_XDECREF(result);
+    Py_DECREF(it);
+    return NULL;
+}
+
 extern "C" PyObject* PyObject_CallFunction(PyObject* callable, const char* format, ...) noexcept {
     va_list va;
     PyObject* args;
@@ -1318,14 +1430,20 @@ extern "C" PyObject* PyNumber_And(PyObject* lhs, PyObject* rhs) noexcept {
     }
 }
 
-extern "C" PyObject* PyNumber_Xor(PyObject*, PyObject*) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return nullptr;
+extern "C" PyObject* PyNumber_Xor(PyObject* lhs, PyObject* rhs) noexcept {
+    try {
+        return binop(lhs, rhs, AST_TYPE::BitXor);
+    } catch (ExcInfo e) {
+        Py_FatalError("unimplemented");
+    }
 }
 
-extern "C" PyObject* PyNumber_Or(PyObject*, PyObject*) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return nullptr;
+extern "C" PyObject* PyNumber_Or(PyObject* lhs, PyObject* rhs) noexcept {
+    try {
+        return binop(lhs, rhs, AST_TYPE::BitOr);
+    } catch (ExcInfo e) {
+        Py_FatalError("unimplemented");
+    }
 }
 
 extern "C" PyObject* PyNumber_InPlaceAdd(PyObject*, PyObject*) noexcept {

src/capi/modsupport.cpp

@@ -381,8 +381,29 @@ extern "C" PyObject* Py_BuildValue(const char* fmt, ...) noexcept {
     return r;
 }
 
+extern "C" {
+char* _Py_PackageContext = NULL;
+}
 extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, const char* doc, PyObject* self,
                                     int apiver) noexcept {
+    // Comment from CPython:
+    /* Make sure name is fully qualified.
+
+       This is a bit of a hack: when the shared library is loaded,
+       the module name is "package.module", but the module calls
+       Py_InitModule*() with just "module" for the name.  The shared
+       library loader squirrels away the true name of the module in
+       _Py_PackageContext, and Py_InitModule*() will substitute this
+       (if the name actually matches).
+    */
+    if (_Py_PackageContext != NULL) {
+        const char* p = strrchr(_Py_PackageContext, '.');
+        if (p != NULL && strcmp(name, p + 1) == 0) {
+            name = _Py_PackageContext;
+            _Py_PackageContext = NULL;
+        }
+    }
+
     BoxedModule* module = createModule(name, "__builtin__", doc);
 
     // Pass self as is, even if NULL we are not allowed to change it to None

src/runtime/capi.cpp

@@ -258,11 +258,16 @@ extern "C" PyObject* PyObject_GetAttr(PyObject* o, PyObject* attr_name) noexcept
 }
 
 extern "C" PyObject* PyObject_GenericGetAttr(PyObject* o, PyObject* name) noexcept {
-    Box* r = getattrInternalGeneric(o, static_cast<BoxedString*>(name)->data(), NULL, false, false, NULL, NULL);
-    if (!r)
-        PyErr_Format(PyExc_AttributeError, "'%.50s' object has no attribute '%.400s'", o->cls->tp_name,
-                     PyString_AS_STRING(name));
-    return r;
+    try {
+        Box* r = getattrInternalGeneric(o, static_cast<BoxedString*>(name)->s.data(), NULL, false, false, NULL, NULL);
+        if (!r)
+            PyErr_Format(PyExc_AttributeError, "'%.50s' object has no attribute '%.400s'", o->cls->tp_name,
+                         PyString_AS_STRING(name));
+        return r;
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
 }
 
 extern "C" PyObject* _PyObject_GenericGetAttrWithDict(PyObject* obj, PyObject* name, PyObject* dict) noexcept {
@@ -399,7 +404,6 @@ extern "C" PyObject* PyObject_Call(PyObject* callable_object, PyObject* args, Py
 
 extern "C" int PyObject_GetBuffer(PyObject* obj, Py_buffer* view, int flags) noexcept {
     if (!PyObject_CheckBuffer(obj)) {
-        printf("%s\n", obj->cls->tp_name);
         PyErr_Format(PyExc_TypeError, "'%100s' does not have the buffer interface", Py_TYPE(obj)->tp_name);
         return -1;
     }
@@ -520,16 +524,6 @@ extern "C" Py_ssize_t PySequence_Index(PyObject* o, PyObject* value) noexcept {
     return -1;
 }
 
-extern "C" PyObject* PySequence_Tuple(PyObject* o) noexcept {
-    if (o->cls == tuple_cls)
-        return o;
-    if (PyList_Check(o))
-        return PyList_AsTuple(o);
-
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return nullptr;
-}
-
 extern "C" PyObject* PyIter_Next(PyObject* iter) noexcept {
     try {
         Box* hasnext = iter->hasnextOrNullIC();
@@ -1239,6 +1233,14 @@ extern "C" PyObject* PyCFunction_NewEx(PyMethodDef* ml, PyObject* self, PyObject
     return new BoxedCApiFunction(ml->ml_flags, self, ml->ml_name, ml->ml_meth);
 }
 
+extern "C" PyCFunction PyCFunction_GetFunction(PyObject* op) noexcept {
+    if (!PyCFunction_Check(op)) {
+        PyErr_BadInternalCall();
+        return NULL;
+    }
+    return ((PyCFunctionObject*)op)->m_ml->ml_meth;
+}
+
 extern "C" int _PyEval_SliceIndex(PyObject* v, Py_ssize_t* pi) noexcept {
     if (v != NULL) {
         Py_ssize_t x;
@@ -1278,6 +1280,7 @@ extern "C" char* PyModule_GetName(PyObject* m) noexcept {
 BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path) {
     void* handle = dlopen(path.c_str(), RTLD_NOW);
     if (!handle) {
+        // raiseExcHelper(ImportError, "%s", dlerror());
         fprintf(stderr, "%s\n", dlerror());
         exit(1);
     }
@@ -1288,17 +1291,26 @@ BoxedModule* importCExtension(const std::string& full_name, const std::string& l
 
     char* error;
     if ((error = dlerror()) != NULL) {
+        // raiseExcHelper(ImportError, "%s", error);
         fprintf(stderr, "%s\n", error);
         exit(1);
     }
 
     assert(init);
+
+    char* packagecontext = strdup(full_name.c_str());
+    char* oldcontext = _Py_PackageContext;
+    _Py_PackageContext = packagecontext;
     (*init)();
+    _Py_PackageContext = oldcontext;
+    free(packagecontext);
+
+    checkAndThrowCAPIException();
 
     BoxedDict* sys_modules = getSysModulesDict();
     Box* s = boxStrConstant(full_name.c_str());
     Box* _m = sys_modules->d[s];
-    RELEASE_ASSERT(_m, "module failed to initialize properly?");
+    RELEASE_ASSERT(_m, "dynamic module not initialized properly");
     assert(_m->cls == module_cls);
 
     BoxedModule* m = static_cast<BoxedModule*>(_m);

src/runtime/float.cpp

@@ -27,6 +27,10 @@ extern "C" PyObject* PyFloat_FromDouble(double d) noexcept {
     return boxFloat(d);
 }
 
+extern "C" PyObject* PyFloat_FromString(PyObject* v, char** pend) noexcept {
+    Py_FatalError("unimplemented");
+}
+
 extern "C" double PyFloat_AsDouble(PyObject* o) noexcept {
     if (o->cls == float_cls)
         return static_cast<BoxedFloat*>(o)->d;

src/runtime/types.cpp

@@ -2336,13 +2336,21 @@ void setupRuntime() {
 
 BoxedModule* createModule(const std::string& name, const std::string& fn, const char* doc) {
     assert(fn.size() && "probably wanted to set the fn to <stdin>?");
-    BoxedModule* module = new BoxedModule(name, fn, doc);
 
     BoxedDict* d = getSysModulesDict();
     Box* b_name = boxStringPtr(&name);
-    ASSERT(d->d.count(b_name) == 0, "%s", name.c_str());
-    d->d[b_name] = module;
 
+    // Surprisingly, there are times that we need to return the existing module if
+    // one exists:
+    Box*& ptr = d->d[b_name];
+    if (ptr && isSubclass(ptr->cls, module_cls)) {
+        return static_cast<BoxedModule*>(ptr);
+    } else {
+        ptr = NULL;
+    }
+
+    BoxedModule* module = new BoxedModule(name, fn, doc);
+    ptr = module;
     return module;
 }
 

src/runtime/types.h

@@ -655,6 +655,10 @@ public:
 
     DEFAULT_CLASS_SIMPLE(slice_cls);
 };
+static_assert(sizeof(BoxedSlice) == sizeof(PySliceObject), "");
+static_assert(offsetof(BoxedSlice, start) == offsetof(PySliceObject, start), "");
+static_assert(offsetof(BoxedSlice, stop) == offsetof(PySliceObject, stop), "");
+static_assert(offsetof(BoxedSlice, step) == offsetof(PySliceObject, step), "");
 
 class BoxedMemberDescriptor : public Box {
 public: