Add round(x,d), handle 'int << int' overflow, fix int(9223372036854775808, 0)

add support for assigning a generator to a list slice

from_cpython/Include/pyconfig.h

@@ -27,6 +27,8 @@
 #define SIZEOF_INT 4
 #define SIZEOF_LONG 8
 #define SIZEOF_LONG_LONG 8
+#define SIZEOF_DOUBLE 8
+#define SIZEOF_FLOAT 4
 #define SIZEOF_OFF_T 8
 #define SIZEOF_PTHREAD_T 8
 #define HAVE_COPYSIGN 1

src/capi/abstract.cpp

@@ -557,8 +557,29 @@ extern "C" int PyObject_CheckReadBuffer(PyObject* obj) noexcept {
 }
 
 extern "C" int PyObject_AsReadBuffer(PyObject* obj, const void** buffer, Py_ssize_t* buffer_len) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return -1;
+    PyBufferProcs* pb;
+    void* pp;
+    Py_ssize_t len;
+
+    if (obj == NULL || buffer == NULL || buffer_len == NULL) {
+        null_error();
+        return -1;
+    }
+    pb = obj->cls->tp_as_buffer;
+    if (pb == NULL || pb->bf_getreadbuffer == NULL || pb->bf_getsegcount == NULL) {
+        PyErr_SetString(PyExc_TypeError, "expected a readable buffer object");
+        return -1;
+    }
+    if ((*pb->bf_getsegcount)(obj, NULL) != 1) {
+        PyErr_SetString(PyExc_TypeError, "expected a single-segment buffer object");
+        return -1;
+    }
+    len = (*pb->bf_getreadbuffer)(obj, 0, &pp);
+    if (len < 0)
+        return -1;
+    *buffer = pp;
+    *buffer_len = len;
+    return 0;
 }
 
 static PyObject* call_function_tail(PyObject* callable, PyObject* args) {
@@ -1818,6 +1839,21 @@ extern "C" PyObject* PyNumber_Long(PyObject* o) noexcept {
 }
 
 extern "C" PyObject* PyNumber_Float(PyObject* o) noexcept {
+    if (o == NULL)
+        return null_error();
+
+    if (o->cls == float_cls)
+        return o;
+
+    if (PyInt_Check(o))
+        return boxFloat(((BoxedInt*)o)->n);
+    else if (PyLong_Check(o)) {
+        double result = PyLong_AsDouble(o);
+        if (result == -1.0 && PyErr_Occurred())
+            return NULL;
+        return boxFloat(result);
+    }
+
     fatalOrError(PyExc_NotImplementedError, "unimplemented");
     return nullptr;
 }

src/capi/float.cpp

@@ -14,6 +14,8 @@
 
 // This file is mostly copied from CPython
 
+#include <cfloat>
+
 #include "Python.h"
 
 #include "core/types.h"
@@ -25,8 +27,45 @@ extern "C" {
 
 typedef enum { unknown_format, ieee_big_endian_format, ieee_little_endian_format } float_format_type;
 
+static float_format_type _detect_double_format() {
+    float_format_type format;
+#if SIZEOF_DOUBLE == 8
+    {
+        double x = 9006104071832581.0;
+        if (memcmp(&x, "\x43\x3f\xff\x01\x02\x03\x04\x05", 8) == 0)
+            format = ieee_big_endian_format;
+        else if (memcmp(&x, "\x05\x04\x03\x02\x01\xff\x3f\x43", 8) == 0)
+            format = ieee_little_endian_format;
+        else
+            format = unknown_format;
+    }
+#else
+    format = unknown_format;
+#endif
+    return format;
+}
+
+static float_format_type _detect_float_format() {
+    float_format_type format;
+#if SIZEOF_FLOAT == 4
+    {
+        float y = 16711938.0;
+        if (memcmp(&y, "\x4b\x7f\x01\x02", 4) == 0)
+            format = ieee_big_endian_format;
+        else if (memcmp(&y, "\x02\x01\x7f\x4b", 4) == 0)
+            format = ieee_little_endian_format;
+        else
+            format = unknown_format;
+    }
+#else
+    format = unknown_format;
+#endif
+    return format;
+}
+
 static float_format_type double_format, float_format;
-static float_format_type detected_double_format, detected_float_format;
+static float_format_type detected_double_format = _detect_double_format(),
+                         detected_float_format = _detect_float_format();
 
 static PyObject* float_getformat(PyTypeObject* v, PyObject* arg) noexcept {
     float_format_type r;
@@ -492,5 +531,143 @@ double _PyFloat_Unpack8(const unsigned char* p, int le) noexcept {
         return x;
     }
 }
+
+#if DBL_MANT_DIG == 53
+#define FIVE_POW_LIMIT 22
+#else
+#error "C doubles do not appear to be IEEE 754 binary64 format"
+#endif
+
+PyObject* _Py_double_round(double x, int ndigits) noexcept {
+
+    double rounded, m;
+    Py_ssize_t buflen, mybuflen = 100;
+    char* buf, *buf_end, shortbuf[100], * mybuf = shortbuf;
+    int decpt, sign, val, halfway_case;
+    PyObject* result = NULL;
+    _Py_SET_53BIT_PRECISION_HEADER;
+
+    /* Easy path for the common case ndigits == 0. */
+    if (ndigits == 0) {
+        rounded = round(x);
+        if (fabs(rounded - x) == 0.5)
+            /* halfway between two integers; use round-away-from-zero */
+            rounded = x + (x > 0.0 ? 0.5 : -0.5);
+        return PyFloat_FromDouble(rounded);
+    }
+
+    /* The basic idea is very simple: convert and round the double to a
+       decimal string using _Py_dg_dtoa, then convert that decimal string
+       back to a double with _Py_dg_strtod.  There's one minor difficulty:
+       Python 2.x expects round to do round-half-away-from-zero, while
+       _Py_dg_dtoa does round-half-to-even.  So we need some way to detect
+       and correct the halfway cases.
+
+       Detection: a halfway value has the form k * 0.5 * 10**-ndigits for
+       some odd integer k.  Or in other words, a rational number x is
+       exactly halfway between two multiples of 10**-ndigits if its
+       2-valuation is exactly -ndigits-1 and its 5-valuation is at least
+       -ndigits.  For ndigits >= 0 the latter condition is automatically
+       satisfied for a binary float x, since any such float has
+       nonnegative 5-valuation.  For 0 > ndigits >= -22, x needs to be an
+       integral multiple of 5**-ndigits; we can check this using fmod.
+       For -22 > ndigits, there are no halfway cases: 5**23 takes 54 bits
+       to represent exactly, so any odd multiple of 0.5 * 10**n for n >=
+       23 takes at least 54 bits of precision to represent exactly.
+
+       Correction: a simple strategy for dealing with halfway cases is to
+       (for the halfway cases only) call _Py_dg_dtoa with an argument of
+       ndigits+1 instead of ndigits (thus doing an exact conversion to
+       decimal), round the resulting string manually, and then convert
+       back using _Py_dg_strtod.
+    */
+
+    /* nans, infinities and zeros should have already been dealt
+       with by the caller (in this case, builtin_round) */
+    assert(std::isfinite(x) && x != 0.0);
+
+    /* find 2-valuation val of x */
+    m = frexp(x, &val);
+    while (m != floor(m)) {
+        m *= 2.0;
+        val--;
+    }
+
+    /* determine whether this is a halfway case */
+    if (val == -ndigits - 1) {
+        if (ndigits >= 0)
+            halfway_case = 1;
+        else if (ndigits >= -FIVE_POW_LIMIT) {
+            double five_pow = 1.0;
+            int i;
+            for (i = 0; i < -ndigits; i++)
+                five_pow *= 5.0;
+            halfway_case = fmod(x, five_pow) == 0.0;
+        } else
+            halfway_case = 0;
+    } else
+        halfway_case = 0;
+
+    /* round to a decimal string; use an extra place for halfway case */
+    _Py_SET_53BIT_PRECISION_START;
+    buf = _Py_dg_dtoa(x, 3, ndigits + halfway_case, &decpt, &sign, &buf_end);
+    _Py_SET_53BIT_PRECISION_END;
+    if (buf == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+    buflen = buf_end - buf;
+
+    /* in halfway case, do the round-half-away-from-zero manually */
+    if (halfway_case) {
+        int i, carry;
+        /* sanity check: _Py_dg_dtoa should not have stripped
+           any zeros from the result: there should be exactly
+           ndigits+1 places following the decimal point, and
+           the last digit in the buffer should be a '5'.*/
+        assert(buflen - decpt == ndigits + 1);
+        assert(buf[buflen - 1] == '5');
+
+        /* increment and shift right at the same time. */
+        decpt += 1;
+        carry = 1;
+        for (i = buflen - 1; i-- > 0;) {
+            carry += buf[i] - '0';
+            buf[i + 1] = carry % 10 + '0';
+            carry /= 10;
+        }
+        buf[0] = carry + '0';
+    }
+
+    /* Get new buffer if shortbuf is too small.  Space needed <= buf_end -
+       buf + 8: (1 extra for '0', 1 for sign, 5 for exp, 1 for '\0'). */
+    if (buflen + 8 > mybuflen) {
+        mybuflen = buflen + 8;
+        mybuf = (char*)PyMem_Malloc(mybuflen);
+        if (mybuf == NULL) {
+            PyErr_NoMemory();
+            goto exit;
+        }
+    }
+    /* copy buf to mybuf, adding exponent, sign and leading 0 */
+    PyOS_snprintf(mybuf, mybuflen, "%s0%se%d", (sign ? "-" : ""), buf, decpt - (int)buflen);
+
+    /* and convert the resulting string back to a double */
+    errno = 0;
+    _Py_SET_53BIT_PRECISION_START;
+    rounded = _Py_dg_strtod(mybuf, NULL);
+    _Py_SET_53BIT_PRECISION_END;
+    if (errno == ERANGE && fabs(rounded) >= 1.)
+        PyErr_SetString(PyExc_OverflowError, "rounded value too large to represent");
+    else
+        result = PyFloat_FromDouble(rounded);
+
+    /* done computing value;  now clean up */
+    if (mybuf != shortbuf)
+        PyMem_Free(mybuf);
+exit:
+    _Py_dg_freedtoa(buf);
+    return result;
+}
 }
 }

src/runtime/builtin_modules/builtins.cpp

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include <algorithm>
+#include <cfloat>
 #include <cstddef>
 #include <err.h>
 
@@ -371,6 +372,14 @@ Box* issubclass_func(Box* child, Box* parent) {
     return boxBool(rtn);
 }
 
+Box* intern_func(Box* str) {
+    if (!PyString_CheckExact(str)) // have to use exact check!
+        raiseExcHelper(TypeError, "can't intern subclass of string");
+    PyString_InternInPlace(&str);
+    checkAndThrowCAPIException();
+    return str;
+}
+
 Box* bltinImport(Box* name, Box* globals, Box* locals, Box** args) {
     Box* fromlist = args[0];
     Box* level = args[1];
@@ -586,7 +595,6 @@ public:
     static Box* __reduce__(Box* self) {
         RELEASE_ASSERT(isSubclass(self->cls, BaseException), "");
         BoxedException* exc = static_cast<BoxedException*>(self);
-
         return BoxedTuple::create({ self->cls, EmptyTuple, self->getAttrWrapper() });
     }
 };
@@ -963,16 +971,38 @@ Box* builtinRound(Box* _number, Box* _ndigits) {
         raiseExcHelper(TypeError, "a float is required");
 
     BoxedFloat* number = (BoxedFloat*)_number;
+    double x = number->d;
 
-    if (isSubclass(_ndigits->cls, int_cls)) {
-        BoxedInt* ndigits = (BoxedInt*)_ndigits;
+    /* interpret 2nd argument as a Py_ssize_t; clip on overflow */
+    Py_ssize_t ndigits = PyNumber_AsSsize_t(_ndigits, NULL);
+    if (ndigits == -1 && PyErr_Occurred())
+        throwCAPIException();
 
-        if (ndigits->n == 0)
-            return boxFloat(round(number->d));
+    /* nans, infinities and zeros round to themselves */
+    if (!std::isfinite(x) || x == 0.0)
+        return boxFloat(x);
+
+/* Deal with extreme values for ndigits. For ndigits > NDIGITS_MAX, x
+   always rounds to itself.  For ndigits < NDIGITS_MIN, x always
+   rounds to +-0.0.  Here 0.30103 is an upper bound for log10(2). */
+#define NDIGITS_MAX ((int)((DBL_MANT_DIG - DBL_MIN_EXP) * 0.30103))
+#define NDIGITS_MIN (-(int)((DBL_MAX_EXP + 1) * 0.30103))
+    if (ndigits > NDIGITS_MAX)
+        /* return x */
+        return boxFloat(x);
+    else if (ndigits < NDIGITS_MIN)
+        /* return 0.0, but with sign of x */
+        return boxFloat(0.0 * x);
+    else {
+        /* finite x, and ndigits is not unreasonably large */
+        /* _Py_double_round is defined in floatobject.c */
+        Box* rtn = _Py_double_round(x, (int)ndigits);
+        if (!rtn)
+            throwCAPIException();
+        return rtn;
     }
-
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    throwCAPIException();
+#undef NDIGITS_MAX
+#undef NDIGITS_MIN
 }
 
 Box* builtinCmp(Box* a, Box* b) {
@@ -1099,6 +1129,9 @@ void setupBuiltins() {
         = new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)issubclass_func, BOXED_BOOL, 2), "issubclass");
     builtins_module->giveAttr("issubclass", issubclass_obj);
 
+    Box* intern_obj = new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)intern_func, UNKNOWN, 1), "intern");
+    builtins_module->giveAttr("intern", intern_obj);
+
     CLFunction* import_func = boxRTFunction((void*)bltinImport, UNKNOWN, 5, 4, false, false,
                                             ParamNames({ "name", "globals", "locals", "fromlist", "level" }, "", ""));
     builtins_module->giveAttr("__import__", new BoxedBuiltinFunctionOrMethod(import_func, "__import__",

src/runtime/int.cpp

@@ -200,7 +200,7 @@ extern "C" PyObject* PyInt_FromString(const char* s, char** pend, int base) noex
         if (x < 0)
             return PyLong_FromString(s, pend, base);
     } else
-        x = strtoul(s, &end, base);
+        x = strtol(s, &end, base);
     if (end == s || !isalnum(Py_CHARMASK(end[-1])))
         goto bad;
     while (*end && isspace(Py_CHARMASK(*end)))
@@ -672,8 +672,13 @@ extern "C" Box* intLShiftInt(BoxedInt* lhs, BoxedInt* rhs) {
     if (rhs->n < 0)
         raiseExcHelper(ValueError, "negative shift count");
 
-    // TODO overflow?
-    return boxInt(lhs->n << rhs->n);
+    bool undefined = rhs->n >= sizeof(rhs->n) * 8;
+    if (!undefined) {
+        int64_t res = lhs->n << rhs->n;
+        if ((res >> rhs->n) == lhs->n)
+            return boxInt(lhs->n << rhs->n);
+    }
+    return longLshift(boxLong(lhs->n), rhs);
 }
 
 extern "C" Box* intLShift(BoxedInt* lhs, Box* rhs) {

src/runtime/list.cpp

@@ -25,6 +25,7 @@
 #include "core/stats.h"
 #include "core/types.h"
 #include "gc/collector.h"
+#include "gc/roots.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
 #include "runtime/util.h"
@@ -423,23 +424,21 @@ extern "C" Box* listSetitemSlice(BoxedList* self, BoxedSlice* slice, Box* v) {
     size_t v_size;
     Box** v_elts;
 
+    RootedBox v_as_seq((Box*)nullptr);
     if (!v) {
         v_size = 0;
         v_elts = NULL;
-    } else if (v->cls == list_cls || isSubclass(v->cls, list_cls)) {
-        BoxedList* lv = static_cast<BoxedList*>(v);
-        v_size = lv->size;
+    } else {
+        v_as_seq = RootedBox(PySequence_Fast(v, "can only assign an iterable"));
+        if (v_as_seq == NULL)
+            throwCAPIException();
+
+        v_size = PySequence_Fast_GET_SIZE(v_as_seq);
         // If lv->size is 0, lv->elts->elts is garbage
         if (v_size)
-            v_elts = lv->elts->elts;
+            v_elts = PySequence_Fast_ITEMS(v_as_seq);
         else
             v_elts = NULL;
-    } else if (v->cls == tuple_cls || isSubclass(v->cls, tuple_cls)) {
-        BoxedTuple* tv = static_cast<BoxedTuple*>(v);
-        v_size = tv->size();
-        v_elts = &tv->elts[0];
-    } else {
-        RELEASE_ASSERT(0, "unsupported type for list slice assignment: '%s'", getTypeName(v));
     }
 
     // If self->size is 0, self->elts->elts is garbage

src/runtime/long.cpp

@@ -116,7 +116,48 @@ extern "C" unsigned PY_LONG_LONG PyLong_AsUnsignedLongLongMask(PyObject* vv) noe
 }
 
 extern "C" PY_LONG_LONG PyLong_AsLongLong(PyObject* vv) noexcept {
-    Py_FatalError("unimplemented");
+    PY_LONG_LONG bytes;
+    int one = 1;
+    int res;
+
+    if (vv == NULL) {
+        PyErr_BadInternalCall();
+        return -1;
+    }
+    if (!PyLong_Check(vv)) {
+        PyNumberMethods* nb;
+        PyObject* io;
+        if (PyInt_Check(vv))
+            return (PY_LONG_LONG)PyInt_AsLong(vv);
+        if ((nb = vv->cls->tp_as_number) == NULL || nb->nb_int == NULL) {
+            PyErr_SetString(PyExc_TypeError, "an integer is required");
+            return -1;
+        }
+        io = (*nb->nb_int)(vv);
+        if (io == NULL)
+            return -1;
+        if (PyInt_Check(io)) {
+            bytes = PyInt_AsLong(io);
+            Py_DECREF(io);
+            return bytes;
+        }
+        if (PyLong_Check(io)) {
+            bytes = PyLong_AsLongLong(io);
+            Py_DECREF(io);
+            return bytes;
+        }
+        Py_DECREF(io);
+        PyErr_SetString(PyExc_TypeError, "integer conversion failed");
+        return -1;
+    }
+
+    res = _PyLong_AsByteArray((PyLongObject*)vv, (unsigned char*)&bytes, SIZEOF_LONG_LONG, IS_LITTLE_ENDIAN, 1);
+
+    /* Plan 9 can't handle PY_LONG_LONG in ? : expressions */
+    if (res < 0)
+        return (PY_LONG_LONG)-1;
+    else
+        return bytes;
 }
 
 extern "C" PY_LONG_LONG PyLong_AsLongLongAndOverflow(PyObject* obj, int* overflow) noexcept {
@@ -442,7 +483,12 @@ extern "C" void* PyLong_AsVoidPtr(PyObject* vv) noexcept {
 
 extern "C" int _PyLong_AsByteArray(PyLongObject* v, unsigned char* bytes, size_t n, int little_endian,
                                    int is_signed) noexcept {
-    Py_FatalError("unimplemented");
+    RELEASE_ASSERT(little_endian == 1, "not implemented");
+    RELEASE_ASSERT(n == 8, "did not yet check if the behaviour is correct for sizes other than 8");
+    size_t count = 0;
+    mpz_export(bytes, &count, -1, n, 0, 0, ((BoxedLong*)v)->n);
+    RELEASE_ASSERT(count <= n, "overflow handling is not yet implemented");
+    return 0;
 }
 
 extern "C" PyObject* _PyLong_FromByteArray(const unsigned char* bytes, size_t n, int little_endian,

src/runtime/str.cpp

@@ -2302,8 +2302,17 @@ extern "C" PyObject* PyString_FromStringAndSize(const char* s, ssize_t n) noexce
     return boxStrConstantSize(s, n);
 }
 
+static /*const*/ char* string_getbuffer(register PyObject* op) noexcept {
+    char* s;
+    Py_ssize_t len;
+    if (PyString_AsStringAndSize(op, &s, &len))
+        return NULL;
+    return s;
+}
+
 extern "C" char* PyString_AsString(PyObject* o) noexcept {
-    RELEASE_ASSERT(isSubclass(o->cls, str_cls), "");
+    if (!PyString_Check(o))
+        return string_getbuffer(o);
 
     BoxedString* s = static_cast<BoxedString*>(o);
     return getWriteableStringContents(s);

src/runtime/types.cpp

@@ -978,7 +978,39 @@ Box* sliceRepr(BoxedSlice* self) {
 
 extern "C" int PySlice_GetIndices(PySliceObject* r, Py_ssize_t length, Py_ssize_t* start, Py_ssize_t* stop,
                                   Py_ssize_t* step) noexcept {
-    Py_FatalError("unimplemented");
+    /* XXX support long ints */
+    if (r->step == Py_None) {
+        *step = 1;
+    } else {
+        if (!PyInt_Check(r->step) && !PyLong_Check(r->step))
+            return -1;
+        *step = PyInt_AsSsize_t(r->step);
+    }
+    if (r->start == Py_None) {
+        *start = *step < 0 ? length - 1 : 0;
+    } else {
+        if (!PyInt_Check(r->start) && !PyLong_Check(r->step))
+            return -1;
+        *start = PyInt_AsSsize_t(r->start);
+        if (*start < 0)
+            *start += length;
+    }
+    if (r->stop == Py_None) {
+        *stop = *step < 0 ? -1 : length;
+    } else {
+        if (!PyInt_Check(r->stop) && !PyLong_Check(r->step))
+            return -1;
+        *stop = PyInt_AsSsize_t(r->stop);
+        if (*stop < 0)
+            *stop += length;
+    }
+    if (*stop > length)
+        return -1;
+    if (*start >= length)
+        return -1;
+    if (*step == 0)
+        return -1;
+    return 0;
 }
 
 extern "C" int PySlice_GetIndicesEx(PySliceObject* _r, Py_ssize_t length, Py_ssize_t* start, Py_ssize_t* stop,

test/tests/builtins.py

@@ -105,8 +105,8 @@ print callable(lambda: 1)
 
 print range(5L, 7L)
 
-print round(-1.1), round(-1.9)
-print round(0.5), round(-0.5)
+for n in [0, 1, 2, 3, 4, 5]:
+    print round(-1.1, n), round(-1.9, n), round(0.5, n), round(-0.5, n), round(-0.123456789, n)
 
 print list(iter(xrange(100).__iter__().next, 20))
 

test/tests/intmethods.py

@@ -66,6 +66,9 @@ class L(object):
 print type(int(L()))
 
 print int(u'123')
+print int("9223372036854775808", 0)
+print 1 << 63, 1 << 64, -1 << 63, -1 << 64, 2 << 63
+print type(1 << 63), type(1 << 64), type(-1 << 63), type(-1 << 64), type(2 << 63)
 
 for b in range(26):
     try:

test/tests/list.py

@@ -124,7 +124,13 @@ for i in xrange(3):
             l[j:k] = ["added"]
             print i, j, k, l
 
-
+def G():
+    yield "a"
+    yield "b"
+    yield "c"
+l = [0, 1, 2, 3, 4, 5]
+l[1:] = G()
+print l
 
 l = [1, 3, 5, 7, 2, 4]
 print l.sort(key=lambda x:x%3)