Merge #381 from toshok:iobench

from_cpython/Include/pyconfig.h

@@ -55,6 +55,7 @@
 #define HAVE_SOCKETPAIR 1
 #define HAVE_GETPEERNAME 1
 #define HAVE_STRFTIME 1
+#define HAVE_TIMES 1
 
 #define PY_FORMAT_LONG_LONG "ll"
 #define PY_FORMAT_SIZE_T "z"

src/capi/abstract.cpp

@@ -1341,6 +1341,47 @@ extern "C" int PyNumber_CoerceEx(PyObject**, PyObject**) noexcept {
     Py_FatalError("unimplemented");
 }
 
+extern "C" PyObject* _PyNumber_ConvertIntegralToInt(PyObject* integral, const char* error_format) noexcept {
+    const char* type_name;
+    static PyObject* int_name = NULL;
+    if (int_name == NULL) {
+        int_name = PyString_InternFromString("__int__");
+        if (int_name == NULL)
+            return NULL;
+    }
+
+    if (integral && (!PyInt_Check(integral) && !PyLong_Check(integral))) {
+        /* Don't go through tp_as_number->nb_int to avoid
+           hitting the classic class fallback to __trunc__. */
+        PyObject* int_func = PyObject_GetAttr(integral, int_name);
+        if (int_func == NULL) {
+            PyErr_Clear(); /* Raise a different error. */
+            goto non_integral_error;
+        }
+        Py_DECREF(integral);
+        integral = PyEval_CallObject(int_func, NULL);
+        Py_DECREF(int_func);
+        if (integral && (!PyInt_Check(integral) && !PyLong_Check(integral))) {
+            goto non_integral_error;
+        }
+    }
+    return integral;
+
+non_integral_error:
+    if (PyInstance_Check(integral)) {
+        Py_FatalError("unimplemented");
+        /* cpython has this:
+        type_name = PyString_AS_STRING(((PyInstanceObject *)integral)
+                                       ->in_class->cl_name);
+        */
+    } else {
+        type_name = integral->cls->tp_name;
+    }
+    PyErr_Format(PyExc_TypeError, error_format, type_name);
+    Py_DECREF(integral);
+    return NULL;
+}
+
 extern "C" PyObject* PyNumber_Int(PyObject* o) noexcept {
     PyNumberMethods* m;
     const char* buffer;
@@ -1370,21 +1411,20 @@ extern "C" PyObject* PyNumber_Int(PyObject* o) noexcept {
         return PyInt_FromLong(io->n);
     }
 
-    Py_FatalError("unimplemented __trunc__ and string -> int conversion");
-// the remainder of PyNumber_Int deals with __trunc__ usage, and converting from unicode/string to int
-#if 0
     PyObject* trunc_func = getattr(o, "__trunc__");
     if (trunc_func) {
-      PyObject *truncated = PyEval_CallObject(trunc_func, NULL);
-      Py_DECREF(trunc_func);
-      /* __trunc__ is specified to return an Integral type, but
-     int() needs to return an int. */
-      return _PyNumber_ConvertIntegralToInt(
-                      truncated,
-                      "__trunc__ returned non-Integral (type %.200s)");
+        PyObject* truncated = PyEval_CallObject(trunc_func, NULL);
+        Py_DECREF(trunc_func);
+        /* __trunc__ is specified to return an Integral type, but
+       int() needs to return an int. */
+
+        return _PyNumber_ConvertIntegralToInt(truncated, "__trunc__ returned non-Integral (type %.200s)");
     }
-    PyErr_Clear();  /* It's not an error if  o.__trunc__ doesn't exist. */
+    PyErr_Clear(); /* It's not an error if  o.__trunc__ doesn't exist. */
 
+    // the remainder of PyNumber_Int deals with converting from unicode/string to int
+    Py_FatalError("unimplemented string -> int conversion");
+#if 0
     if (PyString_Check(o))
       return int_from_string(PyString_AS_STRING(o),
                  PyString_GET_SIZE(o));

src/capi/modsupport.cpp

@@ -124,6 +124,9 @@ static PyObject* do_mkvalue(const char** p_format, va_list* p_va, int flags) noe
             case 'l':
                 return PyInt_FromLong(va_arg(*p_va, long));
 
+            case 'd':
+                return PyFloat_FromDouble(va_arg(*p_va, double));
+
             case 'N':
             case 'S':
             case 'O':
@@ -380,7 +383,7 @@ extern "C" PyObject* Py_BuildValue(const char* fmt, ...) noexcept {
 
 extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, const char* doc, PyObject* self,
                                     int apiver) noexcept {
-    BoxedModule* module = createModule(name, "__builtin__");
+    BoxedModule* module = createModule(name, "__builtin__", doc);
 
     // Pass self as is, even if NULL we are not allowed to change it to None
     Box* passthrough = static_cast<Box*>(self);
@@ -394,10 +397,6 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
         methods++;
     }
 
-    if (doc) {
-        module->setattr("__doc__", boxStrConstant(doc), NULL);
-    }
-
     return module;
 }
 

src/codegen/ast_interpreter.cpp

@@ -675,7 +675,6 @@ Box* ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::v
         }
         assert(closure);
     }
-
     return boxCLFunction(cl, closure, is_generator, u.il);
 }
 

src/codegen/irgen.cpp

@@ -413,6 +413,8 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
                     ptr = entry_emitter->getBuilder()->CreateBitCast(ptr, g.double_->getPointerTo());
                 } else if (p.second == GENERATOR) {
                     ptr = entry_emitter->getBuilder()->CreateBitCast(ptr, g.llvm_generator_type_ptr->getPointerTo());
+                } else if (p.second == CLOSURE) {
+                    ptr = entry_emitter->getBuilder()->CreateBitCast(ptr, g.llvm_closure_type_ptr->getPointerTo());
                 } else {
                     assert(p.second->llvmType() == g.llvm_value_type_ptr);
                 }

src/codegen/irgen/hooks.cpp

@@ -101,6 +101,17 @@ const std::string SourceInfo::getName() {
     }
 }
 
+Box* SourceInfo::getDocString() {
+    AST_Str* first_str = NULL;
+
+    if (body.size() > 0 && body[0]->type == AST_TYPE::Expr
+        && static_cast<AST_Expr*>(body[0])->value->type == AST_TYPE::Str) {
+        return boxString(static_cast<AST_Str*>(static_cast<AST_Expr*>(body[0])->value)->str_data);
+    }
+
+    return None;
+}
+
 ScopeInfo* SourceInfo::getScopeInfo() {
     return scoping->getScopeInfoForNode(ast);
 }
@@ -298,6 +309,8 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
         SourceInfo* si = new SourceInfo(bm, scoping, m, m->body);
         CLFunction* cl_f = new CLFunction(0, 0, false, false, si);
 
+        bm->setattr("__doc__", si->getDocString(), NULL);
+
         EffortLevel effort = initialEffort();
 
         cf = compileFunction(cl_f, new FunctionSpecialization(VOID), effort, NULL);

src/core/types.h

@@ -259,6 +259,8 @@ public:
     const std::string getName();
     InternedString mangleName(InternedString id);
 
+    Box* getDocString();
+
     SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body);
 };
 
@@ -514,7 +516,7 @@ class BoxedClass;
 void setupRuntime();
 void teardownRuntime();
 BoxedModule* createAndRunModule(const std::string& name, const std::string& fn);
-BoxedModule* createModule(const std::string& name, const std::string& fn);
+BoxedModule* createModule(const std::string& name, const std::string& fn, const char* doc = NULL);
 
 // TODO where to put this
 void appendToSysPath(const std::string& path);

src/runtime/builtin_modules/builtins.cpp

@@ -1019,7 +1019,9 @@ Box* builtinCmp(Box* lhs, Box* rhs) {
 }
 
 void setupBuiltins() {
-    builtins_module = createModule("__builtin__", "__builtin__");
+    builtins_module = createModule("__builtin__", "__builtin__",
+                                   "Built-in functions, exceptions, and other objects.\n\nNoteworthy: None is "
+                                   "the `nil' object; Ellipsis represents `...' in slices.");
 
     BoxedHeapClass* ellipsis_cls
         = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "ellipsis");

src/runtime/float.cpp

@@ -652,6 +652,36 @@ Box* floatRepr(BoxedFloat* self) {
     return boxString(floatFmt(self->d, 16, 'g'));
 }
 
+Box* floatTrunc(BoxedFloat* self) {
+    if (!isSubclass(self->cls, float_cls))
+        raiseExcHelper(TypeError, "descriptor '__trunc__' requires a 'float' object but received a '%s'",
+                       getTypeName(self));
+
+    double wholepart; /* integral portion of x, rounded toward 0 */
+
+    (void)modf(self->d, &wholepart);
+    /* Try to get out cheap if this fits in a Python int.  The attempt
+     * to cast to long must be protected, as C doesn't define what
+     * happens if the double is too big to fit in a long.  Some rare
+     * systems raise an exception then (RISCOS was mentioned as one,
+     * and someone using a non-default option on Sun also bumped into
+     * that).  Note that checking for <= LONG_MAX is unsafe: if a long
+     * has more bits of precision than a double, casting LONG_MAX to
+     * double may yield an approximation, and if that's rounded up,
+     * then, e.g., wholepart=LONG_MAX+1 would yield true from the C
+     * expression wholepart<=LONG_MAX, despite that wholepart is
+     * actually greater than LONG_MAX.  However, assuming a two's complement
+     * machine with no trap representation, LONG_MIN will be a power of 2 (and
+     * hence exactly representable as a double), and LONG_MAX = -1-LONG_MIN, so
+     * the comparisons with (double)LONG_MIN below should be safe.
+     */
+    if ((double)LONG_MIN <= wholepart && wholepart < -(double)LONG_MIN) {
+        const long aslong = (long)wholepart;
+        return PyInt_FromLong(aslong);
+    }
+    return PyLong_FromDouble(wholepart);
+}
+
 extern "C" void printFloat(double d) {
     std::string s = floatFmt(d, 12, 'g');
     printf("%s", s.c_str());
@@ -711,6 +741,9 @@ void setupFloat() {
     // float_cls->giveAttr("__nonzero__", new BoxedFunction(boxRTFunction((void*)floatNonzero, NULL, 1)));
     float_cls->giveAttr("__str__", new BoxedFunction(boxRTFunction((void*)floatStr, STR, 1)));
     float_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)floatRepr, STR, 1)));
+
+    float_cls->giveAttr("__trunc__", new BoxedFunction(boxRTFunction((void*)floatTrunc, BOXED_INT, 1)));
+
     float_cls->freeze();
 }
 

src/runtime/import.cpp

@@ -685,7 +685,9 @@ Box* impLoadModule(Box* _name, Box* _file, Box* _pathname, Box** args) {
 }
 
 void setupImport() {
-    BoxedModule* imp_module = createModule("imp", "__builtin__");
+    BoxedModule* imp_module
+        = createModule("imp", "__builtin__", "'This module provides the components needed to build your own\n"
+                                             "__import__ function.  Undocumented functions are obsolete.'");
 
     imp_module->giveAttr("PY_SOURCE", boxInt(SearchResult::PY_SOURCE));
     imp_module->giveAttr("PY_COMPILED", boxInt(SearchResult::PY_COMPILED));

src/runtime/int.cpp

@@ -916,6 +916,14 @@ extern "C" Box* intOct(BoxedInt* self) {
     return new BoxedString(std::string(buf, len));
 }
 
+extern "C" Box* intTrunc(BoxedInt* self) {
+    if (!isSubclass(self->cls, int_cls))
+        raiseExcHelper(TypeError, "descriptor '__trunc__' requires a 'int' object but received a '%s'",
+                       getTypeName(self));
+
+    return self;
+}
+
 static Box* _intNew(Box* val, Box* base) {
     if (isSubclass(val->cls, int_cls)) {
         RELEASE_ASSERT(!base, "");
@@ -1072,6 +1080,8 @@ void setupInt() {
     int_cls->giveAttr("__hex__", new BoxedFunction(boxRTFunction((void*)intHex, STR, 1)));
     int_cls->giveAttr("__oct__", new BoxedFunction(boxRTFunction((void*)intOct, STR, 1)));
 
+    int_cls->giveAttr("__trunc__", new BoxedFunction(boxRTFunction((void*)intTrunc, BOXED_INT, 1)));
+
     int_cls->giveAttr(
         "__new__", new BoxedFunction(boxRTFunction((void*)intNew, UNKNOWN, 3, 2, false, false), { boxInt(0), NULL }));
 

src/runtime/long.cpp

@@ -1035,6 +1035,14 @@ Box* longHash(BoxedLong* self) {
     return boxInt(n);
 }
 
+extern "C" Box* longTrunc(BoxedLong* self) {
+    if (!isSubclass(self->cls, long_cls))
+        raiseExcHelper(TypeError, "descriptor '__trunc__' requires a 'long' object but received a '%s'",
+                       getTypeName(self));
+
+    return self;
+}
+
 void* customised_allocation(size_t alloc_size) {
     return gc::gc_alloc(alloc_size, gc::GCKind::CONSERVATIVE);
 }
@@ -1097,6 +1105,8 @@ void setupLong() {
     long_cls->giveAttr("__nonzero__", new BoxedFunction(boxRTFunction((void*)longNonzero, BOXED_BOOL, 1)));
     long_cls->giveAttr("__hash__", new BoxedFunction(boxRTFunction((void*)longHash, BOXED_INT, 1)));
 
+    long_cls->giveAttr("__trunc__", new BoxedFunction(boxRTFunction((void*)longTrunc, UNKNOWN, 1)));
+
     long_cls->freeze();
 }
 }

src/runtime/types.cpp

@@ -263,22 +263,22 @@ std::string builtinStr("__builtin__");
 
 extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f)
     : in_weakreflist(NULL), f(f), closure(NULL), isGenerator(false), ndefaults(0), defaults(NULL), modname(NULL),
-      name(NULL) {
+      name(NULL), doc(NULL) {
     if (f->source) {
         this->modname = f->source->parent_module->getattr("__name__", NULL);
+        this->doc = f->source->getDocString();
     } else {
         this->modname = boxStringPtr(&builtinStr);
+        this->doc = None;
     }
 
-    this->giveAttr("__doc__", None);
-
     assert(f->num_defaults == ndefaults);
 }
 
 extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f, std::initializer_list<Box*> defaults,
                                                 BoxedClosure* closure, bool isGenerator)
     : in_weakreflist(NULL), f(f), closure(closure), isGenerator(isGenerator), ndefaults(0), defaults(NULL),
-      modname(NULL), name(NULL) {
+      modname(NULL), name(NULL), doc(NULL) {
     if (defaults.size()) {
         // make sure to initialize defaults first, since the GC behavior is triggered by ndefaults,
         // and a GC can happen within this constructor:
@@ -289,8 +289,10 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f, std::initializer_
 
     if (f->source) {
         this->modname = f->source->parent_module->getattr("__name__", NULL);
+        this->doc = f->source->getDocString();
     } else {
         this->modname = boxStringPtr(&builtinStr);
+        this->doc = None;
     }
 
     assert(f->num_defaults == ndefaults);
@@ -309,21 +311,22 @@ BoxedFunction::BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults
     if (f->source) {
         this->name = static_cast<BoxedString*>(boxString(f->source->getName()));
     }
-
-    this->giveAttr("__doc__", None);
 }
 
-BoxedBuiltinFunctionOrMethod::BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name)
+BoxedBuiltinFunctionOrMethod::BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name, const char* doc)
     : BoxedBuiltinFunctionOrMethod(f, name, {}) {
+
+    this->doc = doc ? boxStrConstant(doc) : None;
 }
 
 BoxedBuiltinFunctionOrMethod::BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name,
                                                            std::initializer_list<Box*> defaults, BoxedClosure* closure,
-                                                           bool isGenerator)
+                                                           bool isGenerator, const char* doc)
     : BoxedFunctionBase(f, defaults, closure, isGenerator) {
 
     assert(name);
     this->name = static_cast<BoxedString*>(boxString(name));
+    this->doc = doc ? boxStrConstant(doc) : None;
 }
 
 extern "C" void functionGCHandler(GCVisitor* v, Box* b) {
@@ -338,6 +341,9 @@ extern "C" void functionGCHandler(GCVisitor* v, Box* b) {
     if (f->modname)
         v->visit(f->modname);
 
+    if (f->doc)
+        v->visit(f->doc);
+
     if (f->closure)
         v->visit(f->closure);
 
@@ -360,9 +366,10 @@ static void functionDtor(Box* b) {
     self->dependent_ics.~ICInvalidator();
 }
 
-BoxedModule::BoxedModule(const std::string& name, const std::string& fn) : fn(fn) {
+BoxedModule::BoxedModule(const std::string& name, const std::string& fn, const char* doc) : fn(fn) {
     this->giveAttr("__name__", boxString(name));
     this->giveAttr("__file__", boxString(fn));
+    this->giveAttr("__doc__", doc ? boxStrConstant(doc) : None);
 }
 
 std::string BoxedModule::name() {
@@ -929,6 +936,7 @@ static void typeSetModule(Box* _type, PyObject* value, void* context) {
     type->setattr("__module__", value, NULL);
 }
 
+
 Box* typeHash(BoxedClass* self) {
     assert(isSubclass(self->cls, type_cls));
 
@@ -1893,6 +1901,8 @@ void setupRuntime() {
     function_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)functionRepr, STR, 1)));
     function_cls->giveAttr("__module__", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT,
                                                                    offsetof(BoxedFunction, modname), false));
+    function_cls->giveAttr(
+        "__doc__", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedFunction, doc), false));
     function_cls->giveAttr("__get__", new BoxedFunction(boxRTFunction((void*)functionGet, UNKNOWN, 3)));
     function_cls->giveAttr("__call__",
                            new BoxedFunction(boxRTFunction((void*)functionCall, UNKNOWN, 1, 0, true, true)));
@@ -1906,6 +1916,9 @@ void setupRuntime() {
         "__repr__", new BoxedFunction(boxRTFunction((void*)builtinFunctionOrMethodRepr, STR, 1)));
     builtin_function_or_method_cls->giveAttr(
         "__name__", new (pyston_getset_cls) BoxedGetsetDescriptor(builtinFunctionOrMethodName, NULL, NULL));
+    builtin_function_or_method_cls->giveAttr(
+        "__doc__",
+        new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedBuiltinFunctionOrMethod, doc), false));
     builtin_function_or_method_cls->freeze();
 
     instancemethod_cls->giveAttr(
@@ -2036,16 +2049,15 @@ void setupRuntime() {
     TRACK_ALLOCATIONS = true;
 }
 
-BoxedModule* createModule(const std::string& name, const std::string& fn) {
+BoxedModule* createModule(const std::string& name, const std::string& fn, const char* doc) {
     assert(fn.size() && "probably wanted to set the fn to <stdin>?");
-    BoxedModule* module = new BoxedModule(name, fn);
+    BoxedModule* module = new BoxedModule(name, fn, doc);
 
     BoxedDict* d = getSysModulesDict();
     Box* b_name = boxStringPtr(&name);
     ASSERT(d->d.count(b_name) == 0, "%s", name.c_str());
     d->d[b_name] = module;
 
-    module->giveAttr("__doc__", None);
     return module;
 }
 

src/runtime/types.h

@@ -447,6 +447,7 @@ public:
     // Accessed via member descriptor
     Box* modname;      // __module__
     BoxedString* name; // __name__ (should be here or in one of the derived classes?)
+    Box* doc;          // __doc__
 
     BoxedFunctionBase(CLFunction* f);
     BoxedFunctionBase(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure = NULL,
@@ -466,9 +467,9 @@ public:
 
 class BoxedBuiltinFunctionOrMethod : public BoxedFunctionBase {
 public:
-    BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name);
+    BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name, const char* doc = NULL);
     BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name, std::initializer_list<Box*> defaults,
-                                 BoxedClosure* closure = NULL, bool isGenerator = false);
+                                 BoxedClosure* closure = NULL, bool isGenerator = false, const char* doc = NULL);
 
     DEFAULT_CLASS(builtin_function_or_method_cls);
 };
@@ -481,7 +482,7 @@ public:
     std::string fn;
     FutureFlags future_flags;
 
-    BoxedModule(const std::string& name, const std::string& fn);
+    BoxedModule(const std::string& name, const std::string& fn, const char* doc = NULL);
     std::string name();
 
     DEFAULT_CLASS(module_cls);

test/tests/docstrings.py

@@ -2,6 +2,19 @@ print __doc__
 __doc__ = "module_doc"
 print __doc__
 
+import test_package
+print test_package.__doc__
+test_package.__doc__ = "changeable module docs"
+print test_package.__doc__
+
+def foo():
+    """ foo docs go here """
+    pass
+
+print foo.__doc__
+foo.__doc__ = "no they don't"
+print foo.__doc__
+
 class C1(object):
     print 1, __doc__
     "hello world"

test/tests/test_package/__init__.py

+"""
+  module docs go here
+"""

test/tests/trunc_test.py

+import math
+
+def type_trunc(type, arg):
+    try:
+        print type.__trunc__(arg)
+    except TypeError as e:
+        print e
+
+type_trunc(float, 5.25)
+type_trunc(float, 5)
+type_trunc(float, 5L)
+type_trunc(float, "5")
+
+type_trunc(int, 5.25)
+type_trunc(int, 5)
+type_trunc(int, 5L)
+type_trunc(int, "5")
+
+type_trunc(long, 5.25)
+type_trunc(long, 5)
+type_trunc(long, 5L)
+type_trunc(long, "5")
+
+class Test:
+    def __trunc__(self):
+        print "made it"
+        return 5
+
+t = Test()
+print math.trunc(t)
+
+class TruncReturnsNonInt:
+    def __trunc__(self):
+        print "made it"
+        return "hi"
+
+t2 = TruncReturnsNonInt()
+print math.trunc(t2)