Add the io/_io Module and make import gzip work

This adds quite a few C API functions
and I had to introduce atleast two temporary hacks:
- do not depend on the ABC module which uses weak references
- use a dummy PyErr_CheckSignals() function

Makefile

@@ -292,8 +292,8 @@ STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
 STDLIB_RELEASE_OBJS := stdlib.release.bc.o
 ASM_SRCS := $(wildcard src/runtime/*.S)
 
-STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c $(EXTRA_STDMODULE_SRCS)
-STDOBJECT_SRCS := structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c $(EXTRA_STDOBJECT_SRCS)
+STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c _io/bufferedio.c _io/bytesio.c _io/fileio.c _io/iobase.c _io/_iomodule.c _io/stringio.c _io/textio.c $(EXTRA_STDMODULE_SRCS)
+STDOBJECT_SRCS := structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c memoryobject.c $(EXTRA_STDOBJECT_SRCS)
 STDPYTHON_SRCS := pyctype.c getargs.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c structmember.c $(EXTRA_STDPYTHON_SRCS)
 FROM_CPYTHON_SRCS := $(addprefix from_cpython/Modules/,$(STDMODULE_SRCS)) $(addprefix from_cpython/Objects/,$(STDOBJECT_SRCS)) $(addprefix from_cpython/Python/,$(STDPYTHON_SRCS))
 

from_cpython/CMakeLists.txt

@@ -15,10 +15,10 @@ endforeach(STDLIB_FILE)
 add_custom_target(copy_stdlib ALL DEPENDS ${STDLIB_TARGETS})
 
 # compile specified files in from_cpython/Modules
-file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c)
+file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c bufferedio.c bytesio.c fileio.c iobase.c _iomodule.c stringio.c textio.c)
 
 # compile specified files in from_cpython/Objects
-file(GLOB_RECURSE STDOBJECT_SRCS Objects structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c)
+file(GLOB_RECURSE STDOBJECT_SRCS Objects structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c memoryobject.c)
 
 # compile specified files in from_cpython/Python
 file(GLOB_RECURSE STDPYTHON_SRCS Python getargs.c pyctype.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c structmember.c)

from_cpython/Include/Python.h

@@ -57,6 +57,7 @@
 #ifndef WITHOUT_COMPLEX
 #include "complexobject.h"
 #endif
+#include "memoryobject.h"
 #include "stringobject.h"
 #include "bufferobject.h"
 #include "bytesobject.h"

from_cpython/Include/memoryobject.h

@@ -20,7 +20,7 @@ PyAPI_DATA(PyTypeObject) PyMemoryView_Type;
 
 PyAPI_FUNC(PyObject *) PyMemoryView_GetContiguous(PyObject *base, 
 						  int buffertype, 
-						  char fort);
+                          char fort) PYSTON_NOEXCEPT;
 
     /* Return a contiguous chunk of memory representing the buffer
        from an object in a memory view object.  If a copy is made then the
@@ -53,9 +53,9 @@ PyAPI_FUNC(PyObject *) PyMemoryView_GetContiguous(PyObject *base,
        A new reference is returned that must be DECREF'd when finished.
     */
 
-PyAPI_FUNC(PyObject *) PyMemoryView_FromObject(PyObject *base);
+PyAPI_FUNC(PyObject *) PyMemoryView_FromObject(PyObject *base) PYSTON_NOEXCEPT;
 
-PyAPI_FUNC(PyObject *) PyMemoryView_FromBuffer(Py_buffer *info);
+PyAPI_FUNC(PyObject *) PyMemoryView_FromBuffer(Py_buffer *info) PYSTON_NOEXCEPT;
     /* create new if bufptr is NULL 
         will be a new bytesobject in base */
 

from_cpython/Include/pyconfig.h

@@ -27,6 +27,7 @@
 #define SIZEOF_INT 4
 #define SIZEOF_LONG 8
 #define SIZEOF_LONG_LONG 8
+#define SIZEOF_OFF_T 8
 #define SIZEOF_PTHREAD_T 8
 #define HAVE_COPYSIGN 1
 #define HAVE_ROUND 1

from_cpython/Include/sliceobject.h

@@ -8,9 +8,11 @@ extern "C" {
 
 /* The unique ellipsis object "..." */
 
-PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */
-
-#define Py_Ellipsis (&_Py_EllipsisObject)
+// Pyston change
+// PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */
+// #define Py_Ellipsis (&_Py_EllipsisObject)
+PyAPI_DATA(PyObject) *Ellipsis; /* Don't use this directly */
+#define Py_Ellipsis Ellipsis
 
 /* Slice object interface */
 

from_cpython/Modules/_io/_iomodule.c

@@ -628,10 +628,13 @@ init_io(void)
     if (m == NULL)
         return;
 
+    // Pyston change: we don't support importing os during init, but it looks like it works without it...
+#if 0
     /* put os in the module state */
     _PyIO_os_module = PyImport_ImportModule("os");
     if (_PyIO_os_module == NULL)
         goto fail;
+#endif
 
 #define ADD_TYPE(type, name) \
     if (PyType_Ready(type) < 0) \
@@ -647,9 +650,13 @@ init_io(void)
         goto fail;
 
     /* UnsupportedOperation inherits from ValueError and IOError */
-    _PyIO_unsupported_operation = PyObject_CallFunction(
-        (PyObject *)&PyType_Type, "s(OO){}",
-        "UnsupportedOperation", PyExc_ValueError, PyExc_IOError);
+    // Pyston change: during init we have to supply the module name by ourself.
+    _PyIO_unsupported_operation = PyObject_CallFunction((PyObject *)&PyType_Type, "s(OO){s:s}",
+          "UnsupportedOperation", PyExc_ValueError, PyExc_IOError, "__module__", "io");
+    // _PyIO_unsupported_operation = PyObject_CallFunction(
+    //    (PyObject *)&PyType_Type, "s(OO){}",
+    //    "UnsupportedOperation", PyExc_ValueError, PyExc_IOError);
+
     if (_PyIO_unsupported_operation == NULL)
         goto fail;
     Py_INCREF(_PyIO_unsupported_operation);
@@ -760,6 +767,35 @@ init_io(void)
     if (!(_PyIO_zero = PyLong_FromLong(0L)))
         goto fail;
 
+    // Pyston change: register with gc
+    // TODO: automatically register static variables as roots
+    PyGC_AddRoot(_PyIO_str_close);
+    PyGC_AddRoot(_PyIO_str_closed);
+    PyGC_AddRoot(_PyIO_str_decode);
+    PyGC_AddRoot(_PyIO_str_encode);
+    PyGC_AddRoot(_PyIO_str_fileno);
+    PyGC_AddRoot(_PyIO_str_flush);
+    PyGC_AddRoot(_PyIO_str_getstate);
+    PyGC_AddRoot(_PyIO_str_isatty);
+    PyGC_AddRoot(_PyIO_str_newlines);
+    PyGC_AddRoot(_PyIO_str_nl);
+    PyGC_AddRoot(_PyIO_str_read);
+    PyGC_AddRoot(_PyIO_str_read1);
+    PyGC_AddRoot(_PyIO_str_readable);
+    PyGC_AddRoot(_PyIO_str_readinto);
+    PyGC_AddRoot(_PyIO_str_readline);
+    PyGC_AddRoot(_PyIO_str_reset);
+    PyGC_AddRoot(_PyIO_str_seek);
+    PyGC_AddRoot(_PyIO_str_seekable);
+    PyGC_AddRoot(_PyIO_str_setstate);
+    PyGC_AddRoot(_PyIO_str_tell);
+    PyGC_AddRoot(_PyIO_str_truncate);
+    PyGC_AddRoot(_PyIO_str_writable);
+    PyGC_AddRoot(_PyIO_str_write);
+    // PyGC_AddRoot(_PyIO_empty_str); // this is already registered as root
+    PyGC_AddRoot(_PyIO_empty_bytes);
+    PyGC_AddRoot(_PyIO_zero);
+
     return;
 
   fail:

from_cpython/Modules/_io/iobase.c

@@ -201,10 +201,12 @@ _PyIOBase_finalize(PyObject *self)
 
     /* If _PyIOBase_finalize() is called from a destructor, we need to
        resurrect the object as calling close() can invoke arbitrary code. */
-    is_zombie = (Py_REFCNT(self) == 0);
-    if (is_zombie) {
-        ++Py_REFCNT(self);
-    }
+    // Pyston change:
+    is_zombie = 0;
+    // is_zombie = (Py_REFCNT(self) == 0);
+    //if (is_zombie) {
+    //    ++Py_REFCNT(self);
+    //}
     PyErr_Fetch(&tp, &v, &tb);
     /* If `closed` doesn't exist or can't be evaluated as bool, then the
        object is probably in an unusable state, so ignore. */
@@ -230,6 +232,8 @@ _PyIOBase_finalize(PyObject *self)
     }
     PyErr_Restore(tp, v, tb);
     if (is_zombie) {
+// Pyston change:
+#if 0
         if (--Py_REFCNT(self) != 0) {
             /* The object lives again. The following code is taken from
                slot_tp_del in typeobject.c. */
@@ -245,12 +249,14 @@ _PyIOBase_finalize(PyObject *self)
              * _Py_NewReference bumped tp_allocs:  both of those need to be
              * undone.
              */
+
 #ifdef COUNT_ALLOCS
             --Py_TYPE(self)->tp_frees;
             --Py_TYPE(self)->tp_allocs;
 #endif
             return -1;
         }
+#endif
     }
     return 0;
 }

from_cpython/Modules/_io/textio.c

@@ -305,8 +305,9 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
     if (!final) {
         if (output_len > 0
             && PyUnicode_AS_UNICODE(output)[output_len - 1] == '\r') {
-
-            if (Py_REFCNT(output) == 1) {
+            // Pyston change:
+            // if (Py_REFCNT(output) == 1) {
+            if (0) {
                 if (PyUnicode_Resize(&output, output_len - 1) < 0)
                     goto error;
             }
@@ -405,7 +406,9 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
             PyObject *translated = NULL;
             Py_UNICODE *out_str;
             Py_UNICODE *in, *out, *end;
-            if (Py_REFCNT(output) != 1) {
+            // Pyston change
+            // if (Py_REFCNT(output) != 1) {
+            if (1) {
                 /* We could try to optimize this so that we only do a copy
                    when there is something to translate. On the other hand,
                    most decoders should only output non-shared strings, i.e.
@@ -413,7 +416,7 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
                 translated = PyUnicode_FromUnicode(NULL, len);
                 if (translated == NULL)
                     goto error;
-                assert(Py_REFCNT(translated) == 1);
+                // assert(Py_REFCNT(translated) == 1); Pyston change
                 memcpy(PyUnicode_AS_UNICODE(translated),
                        PyUnicode_AS_UNICODE(output),
                        len * sizeof(Py_UNICODE));
@@ -1798,7 +1801,9 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit)
         /* Our line ends in the current buffer */
         self->decoded_chars_used = endpos - offset_to_buffer;
         if (start > 0 || endpos < PyUnicode_GET_SIZE(line)) {
-            if (start == 0 && Py_REFCNT(line) == 1) {
+            // Pyston change:
+            // if (start == 0 && Py_REFCNT(line) == 1) {
+            if (0) {
                 if (PyUnicode_Resize(&line, endpos) < 0)
                     goto error;
             }

from_cpython/Modules/signalmodule.c

@@ -892,6 +892,9 @@ finisignal(void)
 int
 PyErr_CheckSignals(void)
 {
+    if (!is_tripped)
+        return 0;
+
     // Pyston change:
     Py_FatalError("TODO");
 
@@ -899,9 +902,6 @@ PyErr_CheckSignals(void)
     int i;
     PyObject *f;
 
-    if (!is_tripped)
-        return 0;
-
 #ifdef WITH_THREAD
     if (PyThread_get_thread_ident() != main_thread)
         return 0;

from_cpython/Modules/zlibmodule.c

@@ -1055,6 +1055,11 @@ PyInit_zlib(void)
     PyObject *m, *ver;
     Py_TYPE(&Comptype) = &PyType_Type;
     Py_TYPE(&Decomptype) = &PyType_Type;
+
+    // Pyston change
+    PyType_Ready(&Comptype);
+    PyType_Ready(&Decomptype);
+
     m = Py_InitModule4("zlib", zlib_methods,
                        zlib_module_documentation,
                        (PyObject*)NULL,PYTHON_API_VERSION);

from_cpython/Objects/memoryobject.c

@@ -801,7 +801,9 @@ static PyBufferProcs memory_as_buffer = {
 
 
 PyTypeObject PyMemoryView_Type = {
-    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    // Pyston change
+    //PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    PyVarObject_HEAD_INIT(NULL, 0)
     "memoryview",
     sizeof(PyMemoryViewObject),
     0,

src/capi/abstract.cpp

@@ -341,6 +341,33 @@ extern "C" PyObject* PyObject_CallMethod(PyObject* o, const char* name, const ch
     Py_FatalError("unimplemented");
 }
 
+extern "C" PyObject* PyObject_CallMethodObjArgs(PyObject* callable, PyObject* name, ...) noexcept {
+    PyObject* args, *tmp;
+    va_list vargs;
+
+    if (callable == NULL || name == NULL)
+        return null_error();
+
+    callable = PyObject_GetAttr(callable, name);
+    if (callable == NULL)
+        return NULL;
+
+    /* count the args */
+    va_start(vargs, name);
+    args = objargs_mktuple(vargs);
+    va_end(vargs);
+    if (args == NULL) {
+        Py_DECREF(callable);
+        return NULL;
+    }
+    tmp = PyObject_Call(callable, args, NULL);
+    Py_DECREF(args);
+    Py_DECREF(callable);
+
+    return tmp;
+}
+
+
 extern "C" PyObject* _PyObject_CallMethod_SizeT(PyObject* o, const char* name, const char* format, ...) noexcept {
     // TODO it looks like this could be made much more efficient by calling our callattr(), but
     // I haven't taken the time to verify that that has the same behavior
@@ -550,6 +577,110 @@ extern "C" PyObject* PySequence_Fast(PyObject* v, const char* m) noexcept {
     return v;
 }
 
+extern "C" void* PyBuffer_GetPointer(Py_buffer* view, Py_ssize_t* indices) noexcept {
+    char* pointer;
+    int i;
+    pointer = (char*)view->buf;
+    for (i = 0; i < view->ndim; i++) {
+        pointer += view->strides[i] * indices[i];
+        if ((view->suboffsets != NULL) && (view->suboffsets[i] >= 0)) {
+            pointer = *((char**)pointer) + view->suboffsets[i];
+        }
+    }
+    return (void*)pointer;
+}
+
+extern "C" void _Py_add_one_to_index_F(int nd, Py_ssize_t* index, const Py_ssize_t* shape) noexcept {
+    int k;
+
+    for (k = 0; k < nd; k++) {
+        if (index[k] < shape[k] - 1) {
+            index[k]++;
+            break;
+        } else {
+            index[k] = 0;
+        }
+    }
+}
+
+extern "C" void _Py_add_one_to_index_C(int nd, Py_ssize_t* index, const Py_ssize_t* shape) noexcept {
+    int k;
+
+    for (k = nd - 1; k >= 0; k--) {
+        if (index[k] < shape[k] - 1) {
+            index[k]++;
+            break;
+        } else {
+            index[k] = 0;
+        }
+    }
+}
+
+extern "C" int PyObject_CopyData(PyObject* dest, PyObject* src) noexcept {
+    Py_buffer view_dest, view_src;
+    int k;
+    Py_ssize_t* indices, elements;
+    char* dptr, *sptr;
+
+    if (!PyObject_CheckBuffer(dest) || !PyObject_CheckBuffer(src)) {
+        PyErr_SetString(PyExc_TypeError, "both destination and source must have the "
+                                         "buffer interface");
+        return -1;
+    }
+
+    if (PyObject_GetBuffer(dest, &view_dest, PyBUF_FULL) != 0)
+        return -1;
+    if (PyObject_GetBuffer(src, &view_src, PyBUF_FULL_RO) != 0) {
+        PyBuffer_Release(&view_dest);
+        return -1;
+    }
+
+    if (view_dest.len < view_src.len) {
+        PyErr_SetString(PyExc_BufferError, "destination is too small to receive data from source");
+        PyBuffer_Release(&view_dest);
+        PyBuffer_Release(&view_src);
+        return -1;
+    }
+
+    if ((PyBuffer_IsContiguous(&view_dest, 'C') && PyBuffer_IsContiguous(&view_src, 'C'))
+        || (PyBuffer_IsContiguous(&view_dest, 'F') && PyBuffer_IsContiguous(&view_src, 'F'))) {
+        /* simplest copy is all that is needed */
+        memcpy(view_dest.buf, view_src.buf, view_src.len);
+        PyBuffer_Release(&view_dest);
+        PyBuffer_Release(&view_src);
+        return 0;
+    }
+
+    /* Otherwise a more elaborate copy scheme is needed */
+
+    /* XXX(nnorwitz): need to check for overflow! */
+    indices = (Py_ssize_t*)PyMem_Malloc(sizeof(Py_ssize_t) * view_src.ndim);
+    if (indices == NULL) {
+        PyErr_NoMemory();
+        PyBuffer_Release(&view_dest);
+        PyBuffer_Release(&view_src);
+        return -1;
+    }
+    for (k = 0; k < view_src.ndim; k++) {
+        indices[k] = 0;
+    }
+    elements = 1;
+    for (k = 0; k < view_src.ndim; k++) {
+        /* XXX(nnorwitz): can this overflow? */
+        elements *= view_src.shape[k];
+    }
+    while (elements--) {
+        _Py_add_one_to_index_C(view_src.ndim, indices, view_src.shape);
+        dptr = (char*)PyBuffer_GetPointer(&view_dest, indices);
+        sptr = (char*)PyBuffer_GetPointer(&view_src, indices);
+        memcpy(dptr, sptr, view_src.itemsize);
+    }
+    PyMem_Free(indices);
+    PyBuffer_Release(&view_dest);
+    PyBuffer_Release(&view_src);
+    return 0;
+}
+
 static PyObject* binary_op1(PyObject* v, PyObject* w, const int op_slot) {
     PyObject* x;
     binaryfunc slotv = NULL;
@@ -666,7 +797,20 @@ extern "C" PyObject* PySequence_List(PyObject* v) noexcept {
 }
 
 extern "C" PyObject* PyObject_CallFunction(PyObject* callable, const char* format, ...) noexcept {
-    Py_FatalError("unimplemented");
+    va_list va;
+    PyObject* args;
+
+    if (callable == NULL)
+        return null_error();
+
+    if (format && *format) {
+        va_start(va, format);
+        args = Py_VaBuildValue(format, va);
+        va_end(va);
+    } else
+        args = PyTuple_New(0);
+
+    return call_function_tail(callable, args);
 }
 
 extern "C" int PyMapping_Check(PyObject* o) noexcept {

src/capi/modsupport.cpp

@@ -83,7 +83,7 @@ static int _ustrlen(Py_UNICODE* u) {
 
 static PyObject* do_mktuple(const char**, va_list*, int, int, int) noexcept;
 static PyObject* do_mklist(const char**, va_list*, int, int, int) noexcept;
-// static PyObject *do_mkdict(const char**, va_list *, int, int, int) noexcept;
+static PyObject* do_mkdict(const char**, va_list*, int, int, int) noexcept;
 static PyObject* do_mkvalue(const char**, va_list*, int) noexcept;
 
 static PyObject* do_mkvalue(const char** p_format, va_list* p_va, int flags) noexcept {
@@ -95,10 +95,8 @@ static PyObject* do_mkvalue(const char** p_format, va_list* p_va, int flags) noe
             case '[':
                 return do_mklist(p_format, p_va, ']', countformat(*p_format, ']'), flags);
 
-#if 0
             case '{':
                 return do_mkdict(p_format, p_va, '}', countformat(*p_format, '}'), flags);
-#endif
 
             case 'b':
             case 'B':
@@ -198,6 +196,13 @@ static PyObject* do_mkvalue(const char** p_format, va_list* p_va, int flags) noe
                 return v;
             }
 #endif
+
+            case ':':
+            case ',':
+            case ' ':
+            case '\t':
+                break;
+
             default:
                 RELEASE_ASSERT(0, "%c", *((*p_format) - 1));
         }
@@ -239,6 +244,48 @@ static PyObject* do_mktuple(const char** p_format, va_list* p_va, int endchar, i
     return v;
 }
 
+static PyObject* do_mkdict(const char** p_format, va_list* p_va, int endchar, int n, int flags) noexcept {
+    PyObject* d;
+    int i;
+    int itemfailed = 0;
+    if (n < 0)
+        return NULL;
+    if ((d = PyDict_New()) == NULL)
+        return NULL;
+    /* Note that we can't bail immediately on error as this will leak
+       refcounts on any 'N' arguments. */
+    for (i = 0; i < n; i += 2) {
+        PyObject* k, *v;
+        int err;
+        k = do_mkvalue(p_format, p_va, flags);
+        if (k == NULL) {
+            itemfailed = 1;
+            Py_INCREF(Py_None);
+            k = Py_None;
+        }
+        v = do_mkvalue(p_format, p_va, flags);
+        if (v == NULL) {
+            itemfailed = 1;
+            Py_INCREF(Py_None);
+            v = Py_None;
+        }
+        err = PyDict_SetItem(d, k, v);
+        Py_DECREF(k);
+        Py_DECREF(v);
+        if (err < 0 || itemfailed) {
+            Py_DECREF(d);
+            return NULL;
+        }
+    }
+    if (d != NULL && **p_format != endchar) {
+        Py_DECREF(d);
+        d = NULL;
+        PyErr_SetString(PyExc_SystemError, "Unmatched paren in format");
+    } else if (endchar)
+        ++*p_format;
+    return d;
+}
+
 static PyObject* do_mklist(const char** p_format, va_list* p_va, int endchar, int n, int flags) noexcept {
     PyObject* v;
     int i;

src/capi/object.cpp

@@ -151,6 +151,16 @@ extern "C" int PyObject_SetAttr(PyObject* v, PyObject* name, PyObject* value) no
     Py_FatalError("unimplemented");
 }
 
+extern "C" int PyObject_SetAttrString(PyObject* v, const char* name, PyObject* w) noexcept {
+    try {
+        setattr(v, name, w);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return -1;
+    }
+    return 0;
+}
+
 extern "C" PyObject* PyObject_GetAttrString(PyObject* o, const char* attr) noexcept {
     // TODO do something like this?  not sure if this is safe; will people expect that calling into a known function
     // won't end up doing a GIL check?

src/capi/typeobject.cpp

@@ -2315,7 +2315,8 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
     RELEASE_ASSERT(cls->tp_getattro == NULL || cls->tp_getattro == PyObject_GenericGetAttr, "");
     RELEASE_ASSERT(cls->tp_setattro == NULL || cls->tp_setattro == PyObject_GenericSetAttr, "");
 
-    int ALLOWABLE_FLAGS = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_CHECKTYPES;
+    int ALLOWABLE_FLAGS = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_CHECKTYPES
+                          | Py_TPFLAGS_HAVE_NEWBUFFER;
     RELEASE_ASSERT((cls->tp_flags & ~ALLOWABLE_FLAGS) == 0, "");
     if (cls->tp_as_number) {
         RELEASE_ASSERT(cls->tp_flags & Py_TPFLAGS_CHECKTYPES, "Pyston doesn't yet support non-checktypes behavior");

src/runtime/builtin_modules/builtins.cpp

@@ -44,6 +44,8 @@
 namespace pyston {
 
 extern "C" {
+Box* Ellipsis = 0;
+
 // Copied from CPython:
 #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
 const char* Py_FileSystemDefaultEncoding = "mbcs";
@@ -957,7 +959,7 @@ void setupBuiltins() {
 
     BoxedHeapClass* ellipsis_cls
         = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "ellipsis");
-    Box* Ellipsis = new (ellipsis_cls) Box();
+    Ellipsis = new (ellipsis_cls) Box();
     assert(Ellipsis->cls);
     gc::registerPermanentRoot(Ellipsis);
 
@@ -1136,6 +1138,12 @@ void setupBuiltins() {
     builtins_module->giveAttr("property", property_cls);
     builtins_module->giveAttr("staticmethod", staticmethod_cls);
     builtins_module->giveAttr("classmethod", classmethod_cls);
+
+    assert(memoryview_cls);
+    Py_TYPE(&PyMemoryView_Type) = &PyType_Type;
+    PyType_Ready(&PyMemoryView_Type);
+    builtins_module->giveAttr("memoryview", memoryview_cls);
+
     builtins_module->giveAttr(
         "eval", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)eval, UNKNOWN, 1, 0, false, false), "eval"));
 

src/runtime/capi.cpp

@@ -86,8 +86,36 @@ Box* BoxedWrapperDescriptor::__get__(BoxedWrapperDescriptor* self, Box* inst, Bo
     return new BoxedWrapperObject(self, inst);
 }
 
+static PyObject* null_error(void) {
+    if (!PyErr_Occurred())
+        PyErr_SetString(PyExc_SystemError, "null argument to internal routine");
+    return NULL;
+}
+
 extern "C" int PyObject_AsCharBuffer(PyObject* obj, const char** buffer, Py_ssize_t* buffer_len) noexcept {
-    Py_FatalError("unimplemented");
+    PyBufferProcs* pb;
+    char* pp;
+    Py_ssize_t len;
+
+    if (obj == NULL || buffer == NULL || buffer_len == NULL) {
+        null_error();
+        return -1;
+    }
+    pb = obj->cls->tp_as_buffer;
+    if (pb == NULL || pb->bf_getcharbuffer == NULL || pb->bf_getsegcount == NULL) {
+        PyErr_SetString(PyExc_TypeError, "expected a character buffer object");
+        return -1;
+    }
+    if ((*pb->bf_getsegcount)(obj, NULL) != 1) {
+        PyErr_SetString(PyExc_TypeError, "expected a single-segment buffer object");
+        return -1;
+    }
+    len = (*pb->bf_getcharbuffer)(obj, 0, &pp);
+    if (len < 0)
+        return -1;
+    *buffer = pp;
+    *buffer_len = len;
+    return 0;
 }
 
 // copied from CPython's getargs.c:
@@ -131,11 +159,14 @@ extern "C" void PyBuffer_Release(Py_buffer* view) noexcept {
     }
 
     PyObject* obj = view->obj;
-    assert(obj);
-    assert(obj->cls == str_cls);
-    if (obj && Py_TYPE(obj)->tp_as_buffer && Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer)
-        Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer(obj, view);
-    Py_XDECREF(obj);
+    if (obj) {
+        assert(obj->cls == str_cls);
+        if (obj && Py_TYPE(obj)->tp_as_buffer && Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer)
+            Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer(obj, view);
+        Py_XDECREF(obj);
+    }
+
+
     view->obj = NULL;
 }
 
@@ -309,7 +340,8 @@ extern "C" PyObject* PyObject_GetAttr(PyObject* o, PyObject* attr_name) noexcept
     try {
         return getattr(o, static_cast<BoxedString*>(attr_name)->s.c_str());
     } catch (ExcInfo e) {
-        Py_FatalError("unimplemented");
+        setCAPIException(e);
+        return NULL;
     }
 }
 
@@ -340,7 +372,36 @@ extern "C" int PyObject_DelItem(PyObject* o, PyObject* key) noexcept {
 }
 
 extern "C" PyObject* PyObject_RichCompare(PyObject* o1, PyObject* o2, int opid) noexcept {
-    Py_FatalError("unimplemented");
+    int translated_op;
+    switch (opid) {
+        case Py_LT:
+            translated_op = AST_TYPE::Lt;
+            break;
+        case Py_LE:
+            translated_op = AST_TYPE::LtE;
+            break;
+        case Py_EQ:
+            translated_op = AST_TYPE::Eq;
+            break;
+        case Py_NE:
+            translated_op = AST_TYPE::NotEq;
+            break;
+        case Py_GT:
+            translated_op = AST_TYPE::Gt;
+            break;
+        case Py_GE:
+            translated_op = AST_TYPE::GtE;
+            break;
+        default:
+            Py_FatalError("unimplemented");
+    };
+
+    try {
+        return compare(o1, o2, translated_op);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
 }
 
 extern "C" {
@@ -1150,6 +1211,13 @@ extern "C" PyObject* PyNumber_Float(PyObject* o) noexcept {
 }
 
 extern "C" PyObject* PyNumber_Index(PyObject* o) noexcept {
+    PyObject* result = NULL;
+    if (o == NULL)
+        return null_error();
+    if (PyInt_Check(o) || PyLong_Check(o)) {
+        return o;
+    }
+
     Py_FatalError("unimplemented");
 }
 
@@ -1158,16 +1226,75 @@ extern "C" PyObject* PyNumber_ToBase(PyObject* n, int base) noexcept {
 }
 
 extern "C" Py_ssize_t PyNumber_AsSsize_t(PyObject* o, PyObject* exc) noexcept {
-    RELEASE_ASSERT(o->cls != long_cls, "unhandled");
 
-    RELEASE_ASSERT(isSubclass(o->cls, int_cls), "??");
-    int64_t n = static_cast<BoxedInt*>(o)->n;
-    static_assert(sizeof(n) == sizeof(Py_ssize_t), "");
-    return n;
+    if (isSubclass(o->cls, int_cls)) {
+        int64_t n = static_cast<BoxedInt*>(o)->n;
+        static_assert(sizeof(n) == sizeof(Py_ssize_t), "");
+        return n;
+    } else if (isSubclass(o->cls, long_cls)) {
+        return PyLong_AsSsize_t(o);
+    }
+
+    Py_FatalError("unimplemented");
+}
+
+static int _IsFortranContiguous(Py_buffer* view) {
+    Py_ssize_t sd, dim;
+    int i;
+
+    if (view->ndim == 0)
+        return 1;
+    if (view->strides == NULL)
+        return (view->ndim == 1);
+
+    sd = view->itemsize;
+    if (view->ndim == 1)
+        return (view->shape[0] == 1 || sd == view->strides[0]);
+    for (i = 0; i < view->ndim; i++) {
+        dim = view->shape[i];
+        if (dim == 0)
+            return 1;
+        if (view->strides[i] != sd)
+            return 0;
+        sd *= dim;
+    }
+    return 1;
+}
+
+static int _IsCContiguous(Py_buffer* view) {
+    Py_ssize_t sd, dim;
+    int i;
+
+    if (view->ndim == 0)
+        return 1;
+    if (view->strides == NULL)
+        return 1;
+
+    sd = view->itemsize;
+    if (view->ndim == 1)
+        return (view->shape[0] == 1 || sd == view->strides[0]);
+    for (i = view->ndim - 1; i >= 0; i--) {
+        dim = view->shape[i];
+        if (dim == 0)
+            return 1;
+        if (view->strides[i] != sd)
+            return 0;
+        sd *= dim;
+    }
+    return 1;
 }
 
 extern "C" int PyBuffer_IsContiguous(Py_buffer* view, char fort) noexcept {
-    Py_FatalError("unimplemented");
+    if (view->suboffsets != NULL)
+        return 0;
+
+    if (fort == 'C')
+        return _IsCContiguous(view);
+    else if (fort == 'F')
+        return _IsFortranContiguous(view);
+    else if (fort == 'A')
+        return (_IsCContiguous(view) || _IsFortranContiguous(view));
+    return 0;
 }
 
 extern "C" int PyOS_snprintf(char* str, size_t size, const char* format, ...) noexcept {

src/runtime/import.cpp

@@ -295,7 +295,14 @@ extern "C" PyObject* PyImport_ImportModule(const char* name) noexcept {
     if (strcmp("__builtin__", name) == 0)
         return builtins_module;
 
-    Py_FatalError("unimplemented");
+    try {
+        // TODO: check if this has the same behaviour as the cpython implementation
+        std::string str = name;
+        return import(0, None, &str);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
 }
 
 extern "C" Box* import(int level, Box* from_imports, const std::string* module_name) {

src/runtime/int.cpp

@@ -33,6 +33,10 @@
 namespace pyston {
 
 extern "C" unsigned long PyInt_AsUnsignedLongMask(PyObject* op) noexcept {
+    if (op && PyInt_Check(op))
+        return PyInt_AS_LONG((PyIntObject*)op);
+    if (op && PyLong_Check(op))
+        return PyLong_AsUnsignedLongMask(op);
     Py_FatalError("unimplemented");
 }
 

src/runtime/long.cpp

@@ -78,7 +78,12 @@ extern "C" int _PyLong_AsInt(PyObject* obj) noexcept {
     return (int)result;
 }
 
-extern "C" unsigned long PyLong_AsUnsignedLongMask(PyObject* op) noexcept {
+extern "C" unsigned long PyLong_AsUnsignedLongMask(PyObject* vv) noexcept {
+    if (PyLong_Check(vv)) {
+        BoxedLong* l = static_cast<BoxedLong*>(vv);
+        return mpz_get_ui(l->n);
+    }
+
     Py_FatalError("unimplemented");
 }
 
@@ -136,10 +141,29 @@ extern "C" unsigned long PyLong_AsUnsignedLong(PyObject* vv) noexcept {
 }
 
 extern "C" long PyLong_AsLong(PyObject* vv) noexcept {
+    int overflow;
+    long result = PyLong_AsLongAndOverflow(vv, &overflow);
+    if (overflow) {
+        /* XXX: could be cute and give a different
+           message for overflow == -1 */
+        PyErr_SetString(PyExc_OverflowError, "Python int too large to convert to C long");
+    }
+    return result;
+}
+
+extern "C" Py_ssize_t PyLong_AsSsize_t(PyObject* vv) noexcept {
     RELEASE_ASSERT(PyLong_Check(vv), "");
-    BoxedLong* l = static_cast<BoxedLong*>(vv);
-    RELEASE_ASSERT(mpz_fits_slong_p(l->n), "");
-    return mpz_get_si(l->n);
+
+    if (PyLong_Check(vv)) {
+        BoxedLong* l = static_cast<BoxedLong*>(vv);
+        if (mpz_fits_slong_p(l->n)) {
+            return mpz_get_si(l->n);
+        } else {
+            PyErr_SetString(PyExc_OverflowError, "long int too large to convert to int");
+            return -1;
+        }
+    }
+    Py_FatalError("unimplemented");
 }
 
 extern "C" long PyLong_AsLongAndOverflow(Box* vv, int* overflow) noexcept {
@@ -155,18 +179,35 @@ extern "C" long PyLong_AsLongAndOverflow(Box* vv, int* overflow) noexcept {
     if (PyInt_Check(vv))
         return PyInt_AsLong(vv);
 
-    if (PyLong_Check(vv)) {
-        BoxedLong* l = static_cast<BoxedLong*>(vv);
-        if (mpz_fits_slong_p(l->n)) {
-            return mpz_get_si(l->n);
-        } else {
-            *overflow = mpz_sgn(l->n);
+    if (!PyLong_Check(vv)) {
+        PyNumberMethods* nb;
+        nb = vv->cls->tp_as_number;
+        if (nb == NULL || nb->nb_int == NULL) {
+            PyErr_SetString(PyExc_TypeError, "an integer is required");
             return -1;
         }
+
+        vv = (*nb->nb_int)(vv);
+        if (vv == NULL)
+            return -1;
+
+        if (PyInt_Check(vv))
+            return PyInt_AsLong(vv);
+
+        if (!PyLong_Check(vv)) {
+            PyErr_SetString(PyExc_TypeError, "nb_int should return int object");
+            return -1;
+        }
+        // fallthrough: this has to be a long
     }
 
-    // TODO CPython tries to go through tp_as_number
-    Py_FatalError("unsupported case");
+    BoxedLong* l = static_cast<BoxedLong*>(vv);
+    if (mpz_fits_slong_p(l->n)) {
+        return mpz_get_si(l->n);
+    } else {
+        *overflow = mpz_sgn(l->n);
+        return -1;
+    }
 }
 
 extern "C" double PyLong_AsDouble(PyObject* vv) noexcept {
@@ -288,11 +329,6 @@ extern "C" PyObject* _PyLong_FromByteArray(const unsigned char* bytes, size_t n,
     if (n == 0)
         return PyLong_FromLong(0);
 
-    if (is_signed) {
-        Py_FatalError("unimplemented");
-        return 0;
-    }
-
     if (!little_endian) {
         // TODO: check if the behaviour of mpz_import is right when big endian is specified.
         Py_FatalError("unimplemented");
@@ -302,6 +338,17 @@ extern "C" PyObject* _PyLong_FromByteArray(const unsigned char* bytes, size_t n,
     BoxedLong* rtn = new BoxedLong();
     mpz_init(rtn->n);
     mpz_import(rtn->n, 1, 1, n, little_endian ? -1 : 1, 0, &bytes[0]);
+
+
+    RELEASE_ASSERT(little_endian, "");
+    if (is_signed && bytes[n - 1] >= 0x80) { // todo add big endian support
+        mpz_t t;
+        mpz_init(t);
+        mpz_setbit(t, n * 8);
+        mpz_sub(rtn->n, rtn->n, t);
+        mpz_clear(t);
+    }
+
     return rtn;
 }
 
@@ -367,6 +414,8 @@ BoxedLong* _longNew(Box* val, Box* _base) {
             const std::string& s = static_cast<BoxedString*>(val)->s;
             int r = mpz_init_set_str(rtn->n, s.c_str(), 10);
             RELEASE_ASSERT(r == 0, "");
+        } else if (val->cls == float_cls) {
+            mpz_init_set_si(rtn->n, static_cast<BoxedFloat*>(val)->d);
         } else {
             static const std::string long_str("__long__");
             Box* r = callattr(val, &long_str, CallattrFlags({.cls_only = true, .null_on_nonexistent = true }),

src/runtime/str.cpp

@@ -307,6 +307,10 @@ extern "C" Box* strAdd(BoxedString* lhs, Box* _rhs) {
     return new BoxedString(lhs->s + rhs->s);
 }
 
+extern "C" PyObject* PyString_InternFromString(const char* s) noexcept {
+    return new BoxedString(s);
+}
+
 /* Format codes
  * F_LJUST      '-'
  * F_SIGN       '+'
@@ -1523,6 +1527,16 @@ Box* strJoin(BoxedString* self, Box* rhs) {
     return boxString(std::move(output_str));
 }
 
+extern "C" PyObject* _PyString_Join(PyObject* sep, PyObject* x) noexcept {
+    try {
+        RELEASE_ASSERT(sep->cls == str_cls, "");
+        return strJoin((BoxedString*)sep, x);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+}
+
 Box* strReplace(Box* _self, Box* _old, Box* _new, Box** _args) {
     if (_self->cls != str_cls)
         raiseExcHelper(TypeError, "descriptor 'replace' requires a 'str' object but received a '%s'",
@@ -2193,6 +2207,14 @@ static Py_ssize_t string_buffer_getsegcount(PyObject* o, Py_ssize_t* lenp) noexc
     return 1;
 }
 
+static Py_ssize_t string_buffer_getcharbuf(PyStringObject* self, Py_ssize_t index, const char** ptr) {
+    if (index != 0) {
+        PyErr_SetString(PyExc_SystemError, "accessing non-existent string segment");
+        return -1;
+    }
+    return string_buffer_getreadbuf((PyObject*)self, index, (const void**)ptr);
+}
+
 static int string_buffer_getbuffer(BoxedString* self, Py_buffer* view, int flags) noexcept {
     assert(self->cls == str_cls);
     return PyBuffer_FillInfo(view, (PyObject*)self, &self->s[0], self->s.size(), 1, flags);
@@ -2202,7 +2224,7 @@ static PyBufferProcs string_as_buffer = {
     (readbufferproc)string_buffer_getreadbuf, // comments are the only way I've found of
     (writebufferproc)NULL,                    // forcing clang-format to break these onto multiple lines
     (segcountproc)string_buffer_getsegcount,  //
-    (charbufferproc)NULL,                     //
+    (charbufferproc)string_buffer_getcharbuf, //
     (getbufferproc)string_buffer_getbuffer,   //
     (releasebufferproc)NULL,
 };

src/runtime/types.cpp

@@ -69,6 +69,7 @@ extern "C" void _PyUnicode_Init();
 extern "C" void initunicodedata();
 extern "C" void init_weakref();
 extern "C" void initcStringIO();
+extern "C" void init_io();
 
 namespace pyston {
 
@@ -1540,6 +1541,7 @@ void setupRuntime() {
     initunicodedata();
     init_weakref();
     initcStringIO();
+    init_io();
 
     // some additional setup to ensure weakrefs participate in our GC
     BoxedClass* weakref_ref_cls = &_PyWeakref_RefType;

src/runtime/types.h

@@ -84,6 +84,7 @@ extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *long_cls, *float
     *builtin_function_or_method_cls;
 }
 #define unicode_cls (&PyUnicode_Type)
+#define memoryview_cls (&PyMemoryView_Type)
 
 extern "C" {
 extern Box* None, *NotImplemented, *True, *False;

test/tests/gzip_test.py

+# allow-warning: import level 0 will be treated as -1
+import gzip
+import io
+
+text = "Hello World!"
+
+# generate zip in memory
+outdata = io.BytesIO()
+timestamp = 1
+f = gzip.GzipFile(None, "wb", 9, outdata, timestamp)
+print f.write(text)
+f.close()
+
+output = outdata.getvalue()
+print output.encode("base64")
+
+# decode zip from memory
+f = gzip.GzipFile(None, "rb", 9, io.BytesIO(output))
+print f.read()
+f.close()

test/tests/io_test.py

+# allow-warning: import level 0 will be treated as -1
+import io
+filename = "io_test_.txt"
+
+f = io.open(filename, "w")
+print type(f)
+print f.isatty()
+print f.write(unicode("Hello World"))
+print f.tell()
+f.close()
+
+f = io.StringIO()
+print type(f)
+f.write(unicode('First line.\n'))
+f.write(unicode('Second line.\n'))
+print f.getvalue()
+print f.isatty()
+print f.tell()
+f.close()
+
+f = io.open(filename)
+print type(f)
+print f.read()
+f.close()

test/tests/long.py

@@ -45,6 +45,9 @@ except ValueError, e:
 print long("100", 16)
 print long("100", 10)
 print long("100", 26)
+print long(-1.1)
+print long(1.9)
+print long(-1.9)
 
 print type(hash(1L))
 print hash(1L) == hash(2L)