Merge branch 'pickle'

Makefile

@@ -290,7 +290,7 @@ SRCS := $(MAIN_SRCS) $(STDLIB_SRCS)
 STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
 STDLIB_RELEASE_OBJS := stdlib.release.bc.o
 
-STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c $(EXTRA_STDMODULE_SRCS)
+STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c $(EXTRA_STDMODULE_SRCS)
 STDOBJECT_SRCS := structseq.c capsule.c stringobject.c $(EXTRA_STDOBJECT_SRCS)
 STDPYTHON_SRCS := pyctype.c getargs.c formatter_string.c pystrtod.c dtoa.c $(EXTRA_STDPYTHON_SRCS)
 FROM_CPYTHON_SRCS := $(addprefix from_cpython/Modules/,$(STDMODULE_SRCS)) $(addprefix from_cpython/Objects/,$(STDOBJECT_SRCS)) $(addprefix from_cpython/Python/,$(STDPYTHON_SRCS))

from_cpython/CMakeLists.txt

@@ -15,7 +15,7 @@ endforeach(STDLIB_FILE)
 add_custom_target(copy_stdlib ALL DEPENDS ${STDLIB_TARGETS})
 
 # compile specified files in from_cpython/Modules
-file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c)
+file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c)
 
 # compile specified files in from_cpython/Objects
 file(GLOB_RECURSE STDOBJECT_SRCS Objects structseq.c capsule.c stringobject.c)

from_cpython/Include/pydebug.h

@@ -36,11 +36,11 @@ PyAPI_DATA(int) Py_HashRandomizationFlag;
 
 // Pyston change: make Py_FatalError a macro so that it can access linenumber info, similar to assert:
 //PyAPI_FUNC(void) Py_FatalError(const char *message) __attribute__((__noreturn__));
-#define _STRINGIFY(N) #N
-#define STRINGIFY(N) _STRINGIFY(N)
+#define _PYSTON_STRINGIFY(N) #N
+#define PYSTON_STRINGIFY(N) _PYSTON_STRINGIFY(N)
 #define Py_FatalError(message)                                                                                          \
 do {                                                                                                                    \
-    fprintf(stderr, __FILE__ ":" STRINGIFY(__LINE__) ": %s: Fatal Python error: %s\n", __PRETTY_FUNCTION__, message);   \
+    fprintf(stderr, __FILE__ ":" PYSTON_STRINGIFY(__LINE__) ": %s: Fatal Python error: %s\n", __PRETTY_FUNCTION__, message);   \
     abort();                                                                                                            \
 } while (0)
 

lib_pyston/cPickle.py

+from pickle import *
+

lib_pyston/marshal.py

+# Very simple stub for the marshal module.
+# - pickle imports marshal
+
+def loads(*args, **kw):
+    raise NotImplementedError()

src/capi/abstract.cpp

@@ -26,7 +26,61 @@
 
 namespace pyston {
 
+static PyObject* type_error(const char* msg, PyObject* obj) noexcept {
+    PyErr_Format(PyExc_TypeError, msg, Py_TYPE(obj)->tp_name);
+    return NULL;
+}
+
+static PyObject* null_error(void) noexcept {
+    if (!PyErr_Occurred())
+        PyErr_SetString(PyExc_SystemError, "null argument to internal routine");
+    return NULL;
+}
+
+static PyObject* objargs_mktuple(va_list va) noexcept {
+    int i, n = 0;
+    va_list countva;
+    PyObject* result, *tmp;
+
+#ifdef VA_LIST_IS_ARRAY
+    memcpy(countva, va, sizeof(va_list));
+#else
+#ifdef __va_copy
+    __va_copy(countva, va);
+#else
+    countva = va;
+#endif
+#endif
+
+    while (((PyObject*)va_arg(countva, PyObject*)) != NULL)
+        ++n;
+    result = PyTuple_New(n);
+    if (result != NULL && n > 0) {
+        for (i = 0; i < n; ++i) {
+            tmp = (PyObject*)va_arg(va, PyObject*);
+            PyTuple_SET_ITEM(result, i, tmp);
+            Py_INCREF(tmp);
+        }
+    }
+    return result;
+}
+
 extern "C" PyObject* PyObject_CallFunctionObjArgs(PyObject* callable, ...) {
-    Py_FatalError("unimplemented");
+    PyObject* args, *tmp;
+    va_list vargs;
+
+    if (callable == NULL)
+        return null_error();
+
+    /* count the args */
+    va_start(vargs, callable);
+    args = objargs_mktuple(vargs);
+    va_end(vargs);
+    if (args == NULL)
+        return NULL;
+    tmp = PyObject_Call(callable, args, NULL);
+    Py_DECREF(args);
+
+    return tmp;
 }
 }

src/capi/object.cpp

@@ -67,4 +67,12 @@ extern "C" PyObject* PyObject_Str(PyObject* v) {
 extern "C" PyObject* PyObject_SelfIter(PyObject* obj) {
     Py_FatalError("unimplemented");
 }
+
+extern "C" int PyObject_GenericSetAttr(PyObject* obj, PyObject* name, PyObject* value) {
+    Py_FatalError("unimplemented");
+}
+
+extern "C" int PyObject_AsWriteBuffer(PyObject* obj, void** buffer, Py_ssize_t* buffer_len) {
+    Py_FatalError("unimplemented");
+}
 }

src/runtime/dict.cpp

@@ -127,6 +127,11 @@ extern "C" Py_ssize_t PyDict_Size(PyObject* op) {
     return static_cast<BoxedDict*>(op)->d.size();
 }
 
+extern "C" void PyDict_Clear(PyObject* op) {
+    RELEASE_ASSERT(PyDict_Check(op), "");
+    static_cast<BoxedDict*>(op)->d.clear();
+}
+
 Box* dictGetitem(BoxedDict* self, Box* k) {
     assert(self->cls == dict_cls);
 

src/runtime/long.cpp

@@ -32,6 +32,32 @@ namespace pyston {
 
 BoxedClass* long_cls;
 
+#define IS_LITTLE_ENDIAN (int)*(unsigned char*)&one
+#define PY_ABS_LLONG_MIN (0 - (unsigned PY_LONG_LONG)PY_LLONG_MIN)
+
+extern "C" int _PyLong_Sign(PyObject* l) {
+    return mpz_sgn(static_cast<BoxedLong*>(l)->n);
+}
+
+extern "C" unsigned PY_LONG_LONG PyLong_AsUnsignedLongLong(PyObject* vv) {
+    unsigned PY_LONG_LONG bytes;
+    int one = 1;
+    int res;
+
+    if (vv == NULL || !PyLong_Check(vv)) {
+        PyErr_BadInternalCall();
+        return (unsigned PY_LONG_LONG) - 1;
+    }
+
+    res = _PyLong_AsByteArray((PyLongObject*)vv, (unsigned char*)&bytes, SIZEOF_LONG_LONG, IS_LITTLE_ENDIAN, 0);
+
+    /* Plan 9 can't handle PY_LONG_LONG in ? : expressions */
+    if (res < 0)
+        return (unsigned PY_LONG_LONG)res;
+    else
+        return bytes;
+}
+
 extern "C" unsigned long PyLong_AsUnsignedLongMask(PyObject* op) {
     Py_FatalError("unimplemented");
 }
@@ -133,9 +159,6 @@ extern "C" PyObject* PyLong_FromUnsignedLong(unsigned long ival) {
     return rtn;
 }
 
-#define IS_LITTLE_ENDIAN (int)*(unsigned char*)&one
-#define PY_ABS_LLONG_MIN (0 - (unsigned PY_LONG_LONG)PY_LLONG_MIN)
-
 extern "C" PyObject* PyLong_FromSsize_t(Py_ssize_t ival) {
     Py_ssize_t bytes = ival;
     int one = 1;
@@ -154,6 +177,73 @@ extern "C" double _PyLong_Frexp(PyLongObject* a, Py_ssize_t* e) {
     Py_FatalError("unimplemented");
 }
 
+/* Create a new long (or int) object from a C pointer */
+
+extern "C" PyObject* PyLong_FromVoidPtr(void* p) {
+#if SIZEOF_VOID_P <= SIZEOF_LONG
+    if ((long)p < 0)
+        return PyLong_FromUnsignedLong((unsigned long)p);
+    return PyInt_FromLong((long)p);
+#else
+
+#ifndef HAVE_LONG_LONG
+#error "PyLong_FromVoidPtr: sizeof(void*) > sizeof(long), but no long long"
+#endif
+#if SIZEOF_LONG_LONG < SIZEOF_VOID_P
+#error "PyLong_FromVoidPtr: sizeof(PY_LONG_LONG) < sizeof(void*)"
+#endif
+    /* optimize null pointers */
+    if (p == NULL)
+        return PyInt_FromLong(0);
+    return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)p);
+
+#endif /* SIZEOF_VOID_P <= SIZEOF_LONG */
+}
+
+/* Get a C pointer from a long object (or an int object in some cases) */
+
+extern "C" void* PyLong_AsVoidPtr(PyObject* vv) {
+/* This function will allow int or long objects. If vv is neither,
+   then the PyLong_AsLong*() functions will raise the exception:
+   PyExc_SystemError, "bad argument to internal function"
+*/
+#if SIZEOF_VOID_P <= SIZEOF_LONG
+    long x;
+
+    if (PyInt_Check(vv))
+        x = PyInt_AS_LONG(vv);
+    else if (PyLong_Check(vv) && _PyLong_Sign(vv) < 0)
+        x = PyLong_AsLong(vv);
+    else
+        x = PyLong_AsUnsignedLong(vv);
+#else
+
+#ifndef HAVE_LONG_LONG
+#error "PyLong_AsVoidPtr: sizeof(void*) > sizeof(long), but no long long"
+#endif
+#if SIZEOF_LONG_LONG < SIZEOF_VOID_P
+#error "PyLong_AsVoidPtr: sizeof(PY_LONG_LONG) < sizeof(void*)"
+#endif
+    PY_LONG_LONG x;
+
+    if (PyInt_Check(vv))
+        x = PyInt_AS_LONG(vv);
+    else if (PyLong_Check(vv) && _PyLong_Sign(vv) < 0)
+        x = PyLong_AsLongLong(vv);
+    else
+        x = PyLong_AsUnsignedLongLong(vv);
+
+#endif /* SIZEOF_VOID_P <= SIZEOF_LONG */
+
+    if (x == -1 && PyErr_Occurred())
+        return NULL;
+    return (void*)x;
+}
+
+extern "C" int _PyLong_AsByteArray(PyLongObject* v, unsigned char* bytes, size_t n, int little_endian, int is_signed) {
+    Py_FatalError("unimplemented");
+}
+
 extern "C" PyObject* _PyLong_FromByteArray(const unsigned char* bytes, size_t n, int little_endian, int is_signed) {
     if (n == 0)
         return PyLong_FromLong(0);

src/runtime/types.cpp

@@ -46,6 +46,7 @@ extern "C" void initoperator();
 extern "C" void initbinascii();
 extern "C" void initpwd();
 extern "C" void initposix();
+extern "C" void init_struct();
 
 namespace pyston {
 
@@ -995,6 +996,7 @@ void setupRuntime() {
     initbinascii();
     initpwd();
     initposix();
+    init_struct();
 
     setupSysEnd();
 

test/test_extension/slots_test.c

@@ -18,16 +18,26 @@ slots_tester_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     if (!PyArg_ParseTuple(args, "i", &n))
         return NULL;
 
+    printf("slots_tester_seq.__new__, %d\n", n);
+
     /* create attrgetterobject structure */
     obj = PyObject_New(slots_tester_object, type);
     if (obj == NULL)
         return NULL;
 
-    obj->n = n;
+    obj->n = n - 1;
 
     return (PyObject *)obj;
 }
 
+static int
+slots_tester_init(PyObject *self, PyObject *args, PyObject *kwds)
+{
+    printf("slots_tester_seq.__init__, %d\n", ((slots_tester_object*)self)->n);
+
+    return 0;
+}
+
 static long slots_tester_seq_hash(slots_tester_object* obj) {
     printf("slots_tester_seq.__hash__\n");
     return obj->n ^ 1;
@@ -121,7 +131,7 @@ static PyTypeObject slots_tester_seq = {
     0,                                  /* tp_descr_get */
     0,                                  /* tp_descr_set */
     0,                                  /* tp_dictoffset */
-    0,                                  /* tp_init */
+    slots_tester_init,                  /* tp_init */
     0,                                  /* tp_alloc */
     slots_tester_new,                   /* tp_new */
     0,                                  /* tp_free */
@@ -216,6 +226,18 @@ call_funcs(PyObject* _module, PyObject* args) {
         Py_DECREF(rtn);
     }
 
+    if (cls->tp_new) {
+        printf("tp_new exists\n");
+    } else {
+        printf("tp_new doesnt exist\n");
+    }
+
+    if (cls->tp_init) {
+        printf("tp_init exists\n");
+    } else {
+        printf("tp_init doesnt exist\n");
+    }
+
     if (cls->tp_call) {
         printf("tp_call exists\n");
     } else {

test/tests/pickle_test.py

+import pickle
+
+l = [[], (123,)]
+l.append(l)
+s = pickle.dumps(l)
+print repr(s)
+
+l2 = pickle.loads(s)
+l3 = l2.pop()
+print l2, l3, l2 is l3
+
+# This doesn't work yet; we need str.decode("string-escape")
+# print pickle.loads(pickle.dumps("hello world"))

test/tests/struct_test.py

+import struct
+s = struct.pack("II", 1, 1234)
+print repr(s)
+print struct.unpack("II", s)