Merge pull request #341 from undingen/cStringIO

Add cStringIO module + basic support for setting member descriptors

Makefile

@@ -291,9 +291,9 @@ STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
 STDLIB_RELEASE_OBJS := stdlib.release.bc.o
 ASM_SRCS := $(wildcard src/runtime/*.S)
 
-STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c $(EXTRA_STDMODULE_SRCS)
+STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c $(EXTRA_STDMODULE_SRCS)
 STDOBJECT_SRCS := structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c $(EXTRA_STDOBJECT_SRCS)
-STDPYTHON_SRCS := pyctype.c getargs.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c $(EXTRA_STDPYTHON_SRCS)
+STDPYTHON_SRCS := pyctype.c getargs.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c structmember.c $(EXTRA_STDPYTHON_SRCS)
 FROM_CPYTHON_SRCS := $(addprefix from_cpython/Modules/,$(STDMODULE_SRCS)) $(addprefix from_cpython/Objects/,$(STDOBJECT_SRCS)) $(addprefix from_cpython/Python/,$(STDPYTHON_SRCS))
 
 # The stdlib objects have slightly longer dependency chains,

from_cpython/CMakeLists.txt

@@ -15,13 +15,13 @@ endforeach(STDLIB_FILE)
 add_custom_target(copy_stdlib ALL DEPENDS ${STDLIB_TARGETS})
 
 # compile specified files in from_cpython/Modules
-file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c)
+file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c)
 
 # compile specified files in from_cpython/Objects
 file(GLOB_RECURSE STDOBJECT_SRCS Objects structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c)
 
 # compile specified files in from_cpython/Python
-file(GLOB_RECURSE STDPYTHON_SRCS Python getargs.c pyctype.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c)
+file(GLOB_RECURSE STDPYTHON_SRCS Python getargs.c pyctype.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c structmember.c)
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-tautological-compare -Wno-type-limits")
 add_library(FROM_CPYTHON OBJECT ${STDMODULE_SRCS} ${STDOBJECT_SRCS} ${STDPYTHON_SRCS})

from_cpython/Include/cStringIO.h

+// PYSTON_NOEXCEPT This file is originally from CPython 2.7, with modifications for Pyston
+
+#ifndef Py_CSTRINGIO_H
+#define Py_CSTRINGIO_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+
+  This header provides access to cStringIO objects from C.
+  Functions are provided for calling cStringIO objects and
+  macros are provided for testing whether you have cStringIO
+  objects.
+
+  Before calling any of the functions or macros, you must initialize
+  the routines with:
+
+    PycString_IMPORT
+
+  This would typically be done in your init function.
+
+*/
+
+#define PycStringIO_CAPSULE_NAME "cStringIO.cStringIO_CAPI"
+
+#define PycString_IMPORT \
+  PycStringIO = ((struct PycStringIO_CAPI*)PyCapsule_Import(\
+    PycStringIO_CAPSULE_NAME, 0))
+
+/* Basic functions to manipulate cStringIO objects from C */
+
+static struct PycStringIO_CAPI {
+
+ /* Read a string from an input object.  If the last argument
+    is -1, the remainder will be read.
+    */
+  int(*cread)(PyObject *, char **, Py_ssize_t);
+
+ /* Read a line from an input object.  Returns the length of the read
+    line as an int and a pointer inside the object buffer as char** (so
+    the caller doesn't have to provide its own buffer as destination).
+    */
+  int(*creadline)(PyObject *, char **);
+
+  /* Write a string to an output object*/
+  int(*cwrite)(PyObject *, const char *, Py_ssize_t);
+
+  /* Get the output object as a Python string (returns new reference). */
+  PyObject *(*cgetvalue)(PyObject *);
+
+  /* Create a new output object */
+  PyObject *(*NewOutput)(int);
+
+  /* Create an input object from a Python string
+     (copies the Python string reference).
+     */
+  PyObject *(*NewInput)(PyObject *);
+
+  /* The Python types for cStringIO input and output objects.
+     Note that you can do input on an output object.
+     */
+  PyTypeObject *InputType, *OutputType;
+
+} *PycStringIO;
+
+/* These can be used to test if you have one */
+#define PycStringIO_InputCheck(O) \
+  (Py_TYPE(O)==PycStringIO->InputType)
+#define PycStringIO_OutputCheck(O) \
+  (Py_TYPE(O)==PycStringIO->OutputType)
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_CSTRINGIO_H */

src/runtime/capi.cpp

@@ -1471,6 +1471,10 @@ extern "C" PyObject* PyBuffer_FromMemory(void* ptr, Py_ssize_t size) noexcept {
     Py_FatalError("unimplemented");
 }
 
+extern "C" int PyEval_GetRestricted(void) noexcept {
+    return 0; // We don't support restricted mode
+}
+
 BoxedModule* importTestExtension(const std::string& name) {
     std::string pathname_name = "test/test_extension/" + name + ".pyston.so";
     const char* pathname = pathname_name.c_str();

src/runtime/file.cpp

@@ -1098,7 +1098,7 @@ void setupFile() {
     file_cls->giveAttr("next", new BoxedFunction(boxRTFunction((void*)fileIterNext, STR, 1)));
 
     file_cls->giveAttr("softspace",
-                       new BoxedMemberDescriptor(BoxedMemberDescriptor::INT, offsetof(BoxedFile, f_softspace)));
+                       new BoxedMemberDescriptor(BoxedMemberDescriptor::INT, offsetof(BoxedFile, f_softspace), false));
 
     file_cls->giveAttr("__new__", new BoxedFunction(boxRTFunction((void*)fileNew, UNKNOWN, 3, 1, false, false),
                                                     { boxStrConstant("r") }));

src/runtime/objmodel.cpp

@@ -175,13 +175,13 @@ extern "C" bool softspace(Box* b, bool newval) {
     }
 
     bool r;
-    Box* gotten = b->getattr("softspace");
+    Box* gotten = getattrInternal(b, "softspace", NULL);
     if (!gotten) {
         r = 0;
     } else {
         r = nonzero(gotten);
     }
-    b->setattr("softspace", boxInt(newval), NULL);
+    setattrInternal(b, "softspace", boxInt(newval), NULL);
     return r;
 }
 
@@ -853,9 +853,12 @@ Box* dataDescriptorInstanceSpecialCases(GetattrRewriteArgs* rewrite_args, const
         if (rewrite_args) {
             // TODO we could use offset as the index in the assembly lookup rather than hardcoding
             // the value in the assembly and guarding on it be the same.
-            r_descr->addAttrGuard(offsetof(BoxedMemberDescriptor, offset), member_desc->offset);
 
             // This could be optimized if addAttrGuard supported things < 64 bits
+            static_assert(sizeof(member_desc->offset) == 4, "assumed by assembly instruction below");
+            r_descr->getAttr(offsetof(BoxedMemberDescriptor, offset), Location::any(), assembler::MovType::ZLQ)
+                ->addGuard(member_desc->offset);
+
             static_assert(sizeof(member_desc->type) == 4, "assumed by assembly instruction below");
             r_descr->getAttr(offsetof(BoxedMemberDescriptor, type), Location::any(), assembler::MovType::ZLQ)
                 ->addGuard(member_desc->type);
@@ -940,7 +943,6 @@ Box* dataDescriptorInstanceSpecialCases(GetattrRewriteArgs* rewrite_args, const
                 CASE_INTEGER_TYPE(LONGLONG, long long, PyLong_FromLongLong, long long)
                 CASE_INTEGER_TYPE(ULONGLONG, unsigned long long, PyLong_FromUnsignedLongLong, unsigned long long)
                 CASE_INTEGER_TYPE(PYSSIZET, Py_ssize_t, boxInt, Py_ssize_t)
-
             case BoxedMemberDescriptor::STRING: {
                 if (rewrite_args) {
                     RewriterVar* r_interm = rewrite_args->obj->getAttr(member_desc->offset, rewrite_args->destination);
@@ -1565,6 +1567,17 @@ bool dataDescriptorSetSpecialCases(Box* obj, Box* val, Box* descr, SetattrRewrit
 
         getset_descr->set(obj, val, getset_descr->closure);
 
+        return true;
+    } else if (descr->cls == member_cls) {
+        BoxedMemberDescriptor* member_desc = static_cast<BoxedMemberDescriptor*>(descr);
+        PyMemberDef member_def;
+        memset(&member_def, 0, sizeof(member_def));
+        member_def.offset = member_desc->offset;
+        member_def.type = member_desc->type;
+        if (member_desc->readonly)
+            member_def.flags |= READONLY;
+        PyMember_SetOne((char*)obj, &member_def, val);
+        checkAndThrowCAPIException();
         return true;
     }
 
@@ -1636,6 +1649,8 @@ void setattrInternal(Box* obj, const std::string& attr, Box* val, SetattrRewrite
         // We don't need to to the invalidation stuff in this case.
         return;
     } else {
+        if (!obj->cls->instancesHaveHCAttrs() && !obj->cls->instancesHaveDictAttrs())
+            raiseAttributeError(obj, attr.c_str());
         obj->setattr(attr, val, rewrite_args);
     }
 
@@ -1670,10 +1685,6 @@ extern "C" void setattr(Box* obj, const char* attr, Box* attr_val) {
     static StatCounter slowpath_setattr("slowpath_setattr");
     slowpath_setattr.log();
 
-    if (!obj->cls->instancesHaveHCAttrs() && !obj->cls->instancesHaveDictAttrs()) {
-        raiseAttributeError(obj, attr);
-    }
-
     if (obj->cls == type_cls) {
         BoxedClass* cobj = static_cast<BoxedClass*>(obj);
         if (!isUserDefined(cobj)) {

src/runtime/types.cpp

@@ -67,6 +67,7 @@ extern "C" void init_socket();
 extern "C" void _PyUnicode_Init();
 extern "C" void initunicodedata();
 extern "C" void init_weakref();
+extern "C" void initcStringIO();
 
 namespace pyston {
 
@@ -1379,8 +1380,8 @@ void setupRuntime() {
 
     function_cls->giveAttr("__name__", new (pyston_getset_cls) BoxedGetsetDescriptor(funcName, funcSetName, NULL));
     function_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)functionRepr, STR, 1)));
-    function_cls->giveAttr("__module__",
-                           new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedFunction, modname)));
+    function_cls->giveAttr("__module__", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT,
+                                                                   offsetof(BoxedFunction, modname), false));
     function_cls->giveAttr("__get__", new BoxedFunction(boxRTFunction((void*)functionGet, UNKNOWN, 3)));
     function_cls->giveAttr("__call__",
                            new BoxedFunction(boxRTFunction((void*)functionCall, UNKNOWN, 1, 0, true, true)));
@@ -1473,6 +1474,7 @@ void setupRuntime() {
     init_socket();
     initunicodedata();
     init_weakref();
+    initcStringIO();
 
     // some additional setup to ensure weakrefs participate in our GC
     BoxedClass* weakref_ref_cls = &_PyWeakref_RefType;

src/runtime/types.h

@@ -495,9 +495,12 @@ public:
     } type;
 
     int offset;
+    bool readonly;
 
-    BoxedMemberDescriptor(MemberType type, int offset) : type(type), offset(offset) {}
-    BoxedMemberDescriptor(PyMemberDef* member) : type((MemberType)member->type), offset(member->offset) {}
+    BoxedMemberDescriptor(MemberType type, int offset, bool readonly = true)
+        : type(type), offset(offset), readonly(readonly) {}
+    BoxedMemberDescriptor(PyMemberDef* member)
+        : type((MemberType)member->type), offset(member->offset), readonly(member->flags & READONLY) {}
 
     DEFAULT_CLASS_SIMPLE(member_cls);
 };

test/tests/complex.py

@@ -17,3 +17,8 @@ print (0.5j + 1.5).real
 print (0.5j + 1.5).imag
 
 print complex(1, 1.0) / 2.0
+
+try:
+    complex(1, 1.0).real = 1
+except TypeError, e:
+    print e

test/tests/cstringio_test.py

+import cStringIO
+output = cStringIO.StringIO()
+output.write("Hello\n")
+print >>output, "World"
+print output.getvalue()
+output.close()

test/tests/file.py

@@ -50,5 +50,6 @@ except IOError as e:
 
 f = open("/dev/null", "w")
 print f.write("hello world")
+f.softspace = 0
 print f.flush()
 print f.close()