Basic _collections support

Fixes #192

Makefile

@@ -290,7 +290,7 @@ SRCS := $(MAIN_SRCS) $(STDLIB_SRCS)
 STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
 STDLIB_RELEASE_OBJS := stdlib.release.bc.o
 
-STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c $(EXTRA_STDMODULE_SRCS)
+STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c $(EXTRA_STDMODULE_SRCS)
 STDOBJECT_SRCS := structseq.c capsule.c stringobject.c $(EXTRA_STDOBJECT_SRCS)
 STDPYTHON_SRCS := pyctype.c getargs.c formatter_string.c pystrtod.c dtoa.c $(EXTRA_STDPYTHON_SRCS)
 FROM_CPYTHON_SRCS := $(addprefix from_cpython/Modules/,$(STDMODULE_SRCS)) $(addprefix from_cpython/Objects/,$(STDOBJECT_SRCS)) $(addprefix from_cpython/Python/,$(STDPYTHON_SRCS))

from_cpython/CMakeLists.txt

@@ -15,7 +15,7 @@ endforeach(STDLIB_FILE)
 add_custom_target(copy_stdlib ALL DEPENDS ${STDLIB_TARGETS})
 
 # compile specified files in from_cpython/Modules
-file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c)
+file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c)
 
 # compile specified files in from_cpython/Objects
 file(GLOB_RECURSE STDOBJECT_SRCS Objects structseq.c capsule.c stringobject.c)

from_cpython/Include/dictobject.h

@@ -92,8 +92,10 @@ struct _dictobject {
     PyDictEntry ma_smalltable[PyDict_MINSIZE];
 };
 #endif
-PyAPI_DATA(PyTypeObject*) list_cls;
-#define PyList_Type (*list_cls)
+typedef struct {
+    PyObject_HEAD;
+    char _filler[48];
+} PyDictObject;
 
 // Pyston change: these are no longer static objects:
 #if 0

from_cpython/Include/pystate.h

@@ -116,6 +116,8 @@ typedef struct _ts {
     PyObject *curexc_type;
     PyObject *curexc_value;
     PyObject *curexc_traceback;
+
+    PyObject *dict;  /* Stores per-thread state */
 } PyThreadState;
 
 

src/capi/abstract.cpp

@@ -468,4 +468,28 @@ extern "C" PyObject* PySequence_Concat(PyObject* s, PyObject* o) noexcept {
     }
     return type_error("'%.200s' object can't be concatenated", s);
 }
+
+extern "C" PyObject* PySequence_List(PyObject* v) noexcept {
+    PyObject* result; /* result list */
+    PyObject* rv;     /* return value from PyList_Extend */
+
+    if (v == NULL)
+        return null_error();
+
+    result = PyList_New(0);
+    if (result == NULL)
+        return NULL;
+
+    rv = _PyList_Extend((PyListObject*)result, v);
+    if (rv == NULL) {
+        Py_DECREF(result);
+        return NULL;
+    }
+    Py_DECREF(rv);
+    return result;
+}
+
+extern "C" PyObject* PyObject_CallFunction(PyObject* callable, char* format, ...) noexcept {
+    Py_FatalError("unimplemented");
+}
 }

src/capi/object.cpp

@@ -128,4 +128,65 @@ extern "C" int PyObject_RichCompareBool(PyObject* v, PyObject* w, int op) noexce
 extern "C" PyObject** _PyObject_GetDictPtr(PyObject* obj) noexcept {
     Py_FatalError("unimplemented");
 }
+
+/* These methods are used to control infinite recursion in repr, str, print,
+   etc.  Container objects that may recursively contain themselves,
+   e.g. builtin dictionaries and lists, should used Py_ReprEnter() and
+   Py_ReprLeave() to avoid infinite recursion.
+
+   Py_ReprEnter() returns 0 the first time it is called for a particular
+   object and 1 every time thereafter.  It returns -1 if an exception
+   occurred.  Py_ReprLeave() has no return value.
+
+   See dictobject.c and listobject.c for examples of use.
+*/
+
+#define KEY "Py_Repr"
+
+extern "C" int Py_ReprEnter(PyObject* obj) noexcept {
+    PyObject* dict;
+    PyObject* list;
+    Py_ssize_t i;
+
+    dict = PyThreadState_GetDict();
+    if (dict == NULL)
+        return 0;
+    list = PyDict_GetItemString(dict, KEY);
+    if (list == NULL) {
+        list = PyList_New(0);
+        if (list == NULL)
+            return -1;
+        if (PyDict_SetItemString(dict, KEY, list) < 0)
+            return -1;
+        Py_DECREF(list);
+    }
+    i = PyList_GET_SIZE(list);
+    while (--i >= 0) {
+        if (PyList_GET_ITEM(list, i) == obj)
+            return 1;
+    }
+    PyList_Append(list, obj);
+    return 0;
+}
+
+extern "C" void Py_ReprLeave(PyObject* obj) noexcept {
+    PyObject* dict;
+    PyObject* list;
+    Py_ssize_t i;
+
+    dict = PyThreadState_GetDict();
+    if (dict == NULL)
+        return;
+    list = PyDict_GetItemString(dict, KEY);
+    if (list == NULL || !PyList_Check(list))
+        return;
+    i = PyList_GET_SIZE(list);
+    /* Count backwards because we always expect obj to be list[-1] */
+    while (--i >= 0) {
+        if (PyList_GET_ITEM(list, i) == obj) {
+            PyList_SetSlice(list, i, i + 1, NULL);
+            break;
+        }
+    }
+}
 }

src/capi/typeobject.cpp

@@ -1723,7 +1723,6 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
     gc::registerNonheapRootObject(cls);
 
     // unhandled fields:
-    RELEASE_ASSERT(cls->tp_print == NULL, "");
     RELEASE_ASSERT(cls->tp_getattr == NULL, "");
     RELEASE_ASSERT(cls->tp_setattr == NULL, "");
     RELEASE_ASSERT(cls->tp_compare == NULL, "");
@@ -1749,6 +1748,8 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
     RELEASE_ASSERT(cls->tp_del == NULL, "");
     RELEASE_ASSERT(cls->tp_version_tag == 0, "");
 
+// Pyston doesn't handle tp_print, but it looks like it's just for optimization so it should be ok to skip for now?
+
 // I think it is safe to ignore these for for now:
 // RELEASE_ASSERT(cls->tp_weaklistoffset == 0, "");
 // RELEASE_ASSERT(cls->tp_traverse == NULL, "");

src/core/threading.cpp

@@ -36,7 +36,7 @@ namespace threading {
 
 extern "C" {
 __thread PyThreadState cur_thread_state
-    = { 0, NULL, NULL, NULL }; // not sure if we need to explicitly request zero-initialization
+    = { 0, NULL, NULL, NULL, NULL }; // not sure if we need to explicitly request zero-initialization
 }
 
 PthreadFastMutex threading_lock;

src/runtime/capi.cpp

@@ -531,10 +531,6 @@ extern "C" Py_ssize_t PySequence_Index(PyObject* o, PyObject* value) noexcept {
     Py_FatalError("unimplemented");
 }
 
-extern "C" PyObject* PySequence_List(PyObject* o) noexcept {
-    Py_FatalError("unimplemented");
-}
-
 extern "C" PyObject* PySequence_Tuple(PyObject* o) noexcept {
     Py_FatalError("unimplemented");
 }
@@ -1221,6 +1217,14 @@ static int dev_urandom_python(char* buffer, Py_ssize_t size) noexcept {
 }
 }
 
+extern "C" PyObject* PyThreadState_GetDict(void) noexcept {
+    Box* dict = cur_thread_state.dict;
+    if (!dict) {
+        dict = cur_thread_state.dict = new BoxedDict();
+    }
+    return dict;
+}
+
 extern "C" int _PyOS_URandom(void* buffer, Py_ssize_t size) noexcept {
     if (size < 0) {
         PyErr_Format(PyExc_ValueError, "negative argument not allowed");

src/runtime/list.cpp

@@ -341,20 +341,26 @@ Box* listMul(BoxedList* self, Box* rhs) {
 }
 
 Box* listIAdd(BoxedList* self, Box* _rhs) {
-    if (_rhs->cls != list_cls) {
-        raiseExcHelper(TypeError, "can only concatenate list (not \"%s\") to list", getTypeName(_rhs)->c_str());
-    }
-
     LOCK_REGION(self->lock.asWrite());
 
-    BoxedList* rhs = static_cast<BoxedList*>(_rhs);
+    if (_rhs->cls == list_cls) {
+        // This branch is safe if self==rhs:
+        BoxedList* rhs = static_cast<BoxedList*>(_rhs);
 
-    int s1 = self->size;
-    int s2 = rhs->size;
-    self->ensure(s1 + s2);
+        int s1 = self->size;
+        int s2 = rhs->size;
+        self->ensure(s1 + s2);
+
+        memcpy(self->elts->elts + s1, rhs->elts->elts, sizeof(rhs->elts->elts[0]) * s2);
+        self->size = s1 + s2;
+        return self;
+    }
+
+    RELEASE_ASSERT(_rhs != self, "unsupported");
+
+    for (auto* b : _rhs->pyElements())
+        listAppendInternal(self, b);
 
-    memcpy(self->elts->elts + s1, rhs->elts->elts, sizeof(rhs->elts->elts[0]) * s2);
-    self->size = s1 + s2;
     return self;
 }
 
@@ -578,6 +584,39 @@ Box* listNe(BoxedList* self, Box* rhs) {
     return _listCmp(self, static_cast<BoxedList*>(rhs), AST_TYPE::NotEq);
 }
 
+extern "C" PyObject* _PyList_Extend(PyListObject* self, PyObject* b) noexcept {
+    BoxedList* l = (BoxedList*)self;
+    assert(l->cls == list_cls);
+
+    try {
+        return listIAdd(l, b);
+    } catch (Box* b) {
+        PyErr_SetObject(b->cls, b);
+        return NULL;
+    }
+}
+
+extern "C" int PyList_SetSlice(PyObject* a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject* v) noexcept {
+    if (!PyList_Check(a)) {
+        PyErr_BadInternalCall();
+        return -1;
+    }
+
+    BoxedList* l = (BoxedList*)a;
+    ASSERT(l->cls == list_cls, "%s", l->cls->tp_name);
+
+    try {
+        if (v)
+            listSetitemSlice(l, new BoxedSlice(boxInt(ilow), boxInt(ihigh), None), v);
+        else
+            listDelitemSlice(l, new BoxedSlice(boxInt(ilow), boxInt(ihigh), None));
+        return 0;
+    } catch (Box* b) {
+        PyErr_SetObject(b->cls, b);
+        return -1;
+    }
+}
+
 void setupList() {
     list_iterator_cls = new BoxedHeapClass(object_cls, &listIteratorGCHandler, 0, sizeof(BoxedList), false);
 

src/runtime/types.cpp

@@ -50,6 +50,7 @@ extern "C" void initposix();
 extern "C" void init_struct();
 extern "C" void initdatetime();
 extern "C" void init_functools();
+extern "C" void init_collections();
 
 namespace pyston {
 
@@ -1145,6 +1146,7 @@ void setupRuntime() {
     init_struct();
     initdatetime();
     init_functools();
+    init_collections();
 
     setupSysEnd();
 

src/runtime/types.h

@@ -431,6 +431,7 @@ public:
         return NULL;
     }
 };
+static_assert(sizeof(BoxedDict) == sizeof(PyDictObject), "");
 
 class BoxedFunction : public Box {
 public:

test/tests/_collections_test.py

+import _collections
+
+d = _collections.deque()
+print d
+d.append(1)
+d.appendleft(2)
+d.append(3)
+print d
+
+print
+print type(iter(d))
+for i in d:
+    print i
+
+while d:
+    print d.popleft()