Merge pull request #479 from undingen/babel

Fix misc smaller issues

from_cpython/Include/pyconfig.h

@@ -94,6 +94,7 @@
 #define HAVE_POLL_H 1
 #define HAVE_PTHREAD_H 1
 #define HAVE_PTY_H 1
+#define HAVE_READLINK 1
 #define HAVE_SETVBUF 1
 #define HAVE_SHADOW_H 1
 #define HAVE_SIGNAL_H 1

from_cpython/Objects/stringobject.c

@@ -268,3 +268,99 @@ PyObject *PyString_AsEncodedObject(PyObject *str,
  onError:
     return NULL;
 }
+
+PyObject *string_join(PyStringObject *self, PyObject *orig)
+{
+    char *sep = PyString_AS_STRING(self);
+    const Py_ssize_t seplen = PyString_GET_SIZE(self);
+    PyObject *res = NULL;
+    char *p;
+    Py_ssize_t seqlen = 0;
+    size_t sz = 0;
+    Py_ssize_t i;
+    PyObject *seq, *item;
+
+    seq = PySequence_Fast(orig, "");
+    if (seq == NULL) {
+        return NULL;
+    }
+
+    seqlen = PySequence_Size(seq);
+    if (seqlen == 0) {
+        Py_DECREF(seq);
+        return PyString_FromString("");
+    }
+    if (seqlen == 1) {
+        item = PySequence_Fast_GET_ITEM(seq, 0);
+        if (PyString_CheckExact(item) || PyUnicode_CheckExact(item)) {
+            Py_INCREF(item);
+            Py_DECREF(seq);
+            return item;
+        }
+    }
+
+    /* There are at least two things to join, or else we have a subclass
+     * of the builtin types in the sequence.
+     * Do a pre-pass to figure out the total amount of space we'll
+     * need (sz), see whether any argument is absurd, and defer to
+     * the Unicode join if appropriate.
+     */
+    for (i = 0; i < seqlen; i++) {
+        const size_t old_sz = sz;
+        item = PySequence_Fast_GET_ITEM(seq, i);
+        if (!PyString_Check(item)){
+#ifdef Py_USING_UNICODE
+            if (PyUnicode_Check(item)) {
+                /* Defer to Unicode join.
+                 * CAUTION:  There's no gurantee that the
+                 * original sequence can be iterated over
+                 * again, so we must pass seq here.
+                 */
+                PyObject *result;
+                result = PyUnicode_Join((PyObject *)self, seq);
+                Py_DECREF(seq);
+                return result;
+            }
+#endif
+            PyErr_Format(PyExc_TypeError,
+                         "sequence item %zd: expected string,"
+                         " %.80s found",
+                         i, Py_TYPE(item)->tp_name);
+            Py_DECREF(seq);
+            return NULL;
+        }
+        sz += PyString_GET_SIZE(item);
+        if (i != 0)
+            sz += seplen;
+        if (sz < old_sz || sz > PY_SSIZE_T_MAX) {
+            PyErr_SetString(PyExc_OverflowError,
+                "join() result is too long for a Python string");
+            Py_DECREF(seq);
+            return NULL;
+        }
+    }
+
+    /* Allocate result space. */
+    res = PyString_FromStringAndSize((char*)NULL, sz);
+    if (res == NULL) {
+        Py_DECREF(seq);
+        return NULL;
+    }
+
+    /* Catenate everything. */
+    p = PyString_AS_STRING(res);
+    for (i = 0; i < seqlen; ++i) {
+        size_t n;
+        item = PySequence_Fast_GET_ITEM(seq, i);
+        n = PyString_GET_SIZE(item);
+        Py_MEMCPY(p, PyString_AS_STRING(item), n);
+        p += n;
+        if (i < seqlen - 1) {
+            Py_MEMCPY(p, sep, seplen);
+            p += seplen;
+        }
+    }
+
+    Py_DECREF(seq);
+    return res;
+}

src/runtime/list.cpp

@@ -1067,7 +1067,7 @@ void setupList() {
     list_cls->giveAttr("pop", new BoxedFunction(boxRTFunction((void*)listPop, UNKNOWN, 2, 1, false, false), { None }));
 
     list_cls->giveAttr("append", new BoxedFunction(boxRTFunction((void*)listAppend, NONE, 2)));
-    list_cls->giveAttr("extend", new BoxedFunction(boxRTFunction((void*)listIAdd, LIST, 2)));
+    list_cls->giveAttr("extend", new BoxedFunction(boxRTFunction((void*)listIAdd, UNKNOWN, 2)));
 
     CLFunction* setitem = createRTFunction(3, 0, false, false);
     addRTFunction(setitem, (void*)listSetitemInt, NONE, std::vector<ConcreteCompilerType*>{ LIST, BOXED_INT, UNKNOWN });

src/runtime/long.cpp

@@ -957,6 +957,13 @@ Box* longDiv(BoxedLong* v1, Box* _v2) {
     }
 }
 
+Box* longFloorDiv(BoxedLong* v1, Box* _v2) {
+    if (!isSubclass(v1->cls, long_cls))
+        raiseExcHelper(TypeError, "descriptor '__floordiv__' requires a 'long' object but received a '%s'",
+                       getTypeName(v1));
+    return longDiv(v1, _v2);
+}
+
 Box* longMod(BoxedLong* v1, Box* _v2) {
     if (!isSubclass(v1->cls, long_cls))
         raiseExcHelper(TypeError, "descriptor '__mod__' requires a 'long' object but received a '%s'", getTypeName(v1));
@@ -1038,7 +1045,8 @@ extern "C" Box* longDivmod(BoxedLong* lhs, Box* _rhs) {
 
 Box* longRdiv(BoxedLong* v1, Box* _v2) {
     if (!isSubclass(v1->cls, long_cls))
-        raiseExcHelper(TypeError, "descriptor '__div__' requires a 'long' object but received a '%s'", getTypeName(v1));
+        raiseExcHelper(TypeError, "descriptor '__rdiv__' requires a 'long' object but received a '%s'",
+                       getTypeName(v1));
 
     if (mpz_cmp_si(v1->n, 0) == 0)
         raiseExcHelper(ZeroDivisionError, "long division or modulo by zero");
@@ -1062,6 +1070,14 @@ Box* longRdiv(BoxedLong* v1, Box* _v2) {
     }
 }
 
+Box* longRfloorDiv(BoxedLong* v1, Box* _v2) {
+    if (!isSubclass(v1->cls, long_cls))
+        raiseExcHelper(TypeError, "descriptor '__rfloordiv__' requires a 'long' object but received a '%s'",
+                       getTypeName(v1));
+
+    return longRdiv(v1, _v2);
+}
+
 Box* longTrueDiv(BoxedLong* v1, Box* _v2) {
     if (!isSubclass(v1->cls, long_cls))
         raiseExcHelper(TypeError, "descriptor '__truediv__' requires a 'long' object but received a '%s'",
@@ -1259,6 +1275,8 @@ void setupLong() {
 
     long_cls->giveAttr("__div__", new BoxedFunction(boxRTFunction((void*)longDiv, UNKNOWN, 2)));
     long_cls->giveAttr("__rdiv__", new BoxedFunction(boxRTFunction((void*)longRdiv, UNKNOWN, 2)));
+    long_cls->giveAttr("__floordiv__", new BoxedFunction(boxRTFunction((void*)longFloorDiv, UNKNOWN, 2)));
+    long_cls->giveAttr("__rfloordiv__", new BoxedFunction(boxRTFunction((void*)longRfloorDiv, UNKNOWN, 2)));
     long_cls->giveAttr("__truediv__", new BoxedFunction(boxRTFunction((void*)longTrueDiv, UNKNOWN, 2)));
     long_cls->giveAttr("__rtruediv__", new BoxedFunction(boxRTFunction((void*)longRTrueDiv, UNKNOWN, 2)));
     long_cls->giveAttr("__mod__", new BoxedFunction(boxRTFunction((void*)longMod, UNKNOWN, 2)));

src/runtime/set.cpp

@@ -72,7 +72,7 @@ Box* setiteratorIter(BoxedSetIterator* self) {
 }
 
 Box* setAdd2(Box* _self, Box* b) {
-    assert(_self->cls == set_cls || _self->cls == frozenset_cls);
+    assert(PyAnySet_Check(_self));
     BoxedSet* self = static_cast<BoxedSet*>(_self);
 
     self->s.insert(b);
@@ -82,7 +82,7 @@ Box* setAdd2(Box* _self, Box* b) {
 Box* setNew(Box* _cls, Box* container) {
     assert(_cls->cls == type_cls);
     BoxedClass* cls = static_cast<BoxedClass*>(_cls);
-    assert(cls == set_cls || cls == frozenset_cls);
+    assert(isSubclass(cls, set_cls) || isSubclass(cls, frozenset_cls));
 
     Box* rtn = new (cls) BoxedSet();
 
@@ -124,8 +124,8 @@ Box* frozensetRepr(BoxedSet* self) {
 }
 
 Box* setOrSet(BoxedSet* lhs, BoxedSet* rhs) {
-    assert(lhs->cls == set_cls || lhs->cls == frozenset_cls);
-    assert(rhs->cls == set_cls || rhs->cls == frozenset_cls);
+    assert(PyAnySet_Check(lhs));
+    assert(PyAnySet_Check(rhs));
 
     BoxedSet* rtn = new (lhs->cls) BoxedSet();
 
@@ -139,8 +139,8 @@ Box* setOrSet(BoxedSet* lhs, BoxedSet* rhs) {
 }
 
 Box* setAndSet(BoxedSet* lhs, BoxedSet* rhs) {
-    assert(lhs->cls == set_cls || lhs->cls == frozenset_cls);
-    assert(rhs->cls == set_cls || rhs->cls == frozenset_cls);
+    assert(PyAnySet_Check(lhs));
+    assert(PyAnySet_Check(rhs));
 
     BoxedSet* rtn = new (lhs->cls) BoxedSet();
 
@@ -152,8 +152,8 @@ Box* setAndSet(BoxedSet* lhs, BoxedSet* rhs) {
 }
 
 Box* setSubSet(BoxedSet* lhs, BoxedSet* rhs) {
-    assert(lhs->cls == set_cls || lhs->cls == frozenset_cls);
-    assert(rhs->cls == set_cls || rhs->cls == frozenset_cls);
+    assert(PyAnySet_Check(lhs));
+    assert(PyAnySet_Check(rhs));
 
     BoxedSet* rtn = new (lhs->cls) BoxedSet();
 
@@ -167,8 +167,8 @@ Box* setSubSet(BoxedSet* lhs, BoxedSet* rhs) {
 }
 
 Box* setXorSet(BoxedSet* lhs, BoxedSet* rhs) {
-    assert(lhs->cls == set_cls || lhs->cls == frozenset_cls);
-    assert(rhs->cls == set_cls || rhs->cls == frozenset_cls);
+    assert(PyAnySet_Check(lhs));
+    assert(PyAnySet_Check(rhs));
 
     BoxedSet* rtn = new (lhs->cls) BoxedSet();
 
@@ -186,17 +186,17 @@ Box* setXorSet(BoxedSet* lhs, BoxedSet* rhs) {
 }
 
 Box* setIter(BoxedSet* self) {
-    assert(self->cls == set_cls || self->cls == frozenset_cls);
+    assert(PyAnySet_Check(self));
     return new BoxedSetIterator(self);
 }
 
 Box* setLen(BoxedSet* self) {
-    assert(self->cls == set_cls || self->cls == frozenset_cls);
+    assert(PyAnySet_Check(self));
     return boxInt(self->s.size());
 }
 
 Box* setAdd(BoxedSet* self, Box* v) {
-    assert(self->cls == set_cls || self->cls == frozenset_cls);
+    assert(PyAnySet_Check(self));
     self->s.insert(v);
     return None;
 }
@@ -264,7 +264,7 @@ Box* setUnion(BoxedSet* self, BoxedTuple* args) {
 }
 
 Box* setDifference(BoxedSet* self, BoxedTuple* args) {
-    if (!isSubclass(self->cls, set_cls) && !isSubclass(self->cls, frozenset_cls))
+    if (!PyAnySet_Check(self))
         raiseExcHelper(TypeError, "descriptor 'difference' requires a 'set' object but received a '%s'",
                        getTypeName(self));
 
@@ -315,7 +315,7 @@ static Box* setIssuperset(BoxedSet* self, Box* container) {
 }
 
 Box* setIntersection(BoxedSet* self, BoxedTuple* args) {
-    if (!isSubclass(self->cls, set_cls))
+    if (!PyAnySet_Check(self))
         raiseExcHelper(TypeError, "descriptor 'intersection' requires a 'set' object but received a '%s'",
                        getTypeName(self));
 
@@ -347,12 +347,12 @@ Box* setPop(BoxedSet* self) {
 }
 
 Box* setContains(BoxedSet* self, Box* v) {
-    assert(self->cls == set_cls || self->cls == frozenset_cls);
+    assert(PyAnySet_Check(self));
     return boxBool(self->s.count(v) != 0);
 }
 
 Box* setEq(BoxedSet* self, BoxedSet* rhs) {
-    assert(self->cls == set_cls || self->cls == frozenset_cls);
+    assert(PyAnySet_Check(self));
     if (rhs->cls != set_cls && rhs->cls != frozenset_cls)
         return NotImplemented;
 
@@ -413,7 +413,7 @@ extern "C" PyObject* PyFrozenSet_New(PyObject* iterable) noexcept {
 }
 
 extern "C" int PySet_Add(PyObject* set, PyObject* key) noexcept {
-    if (!PySet_Check(set) && !PyFrozenSet_Check(set)) {
+    if (!PyAnySet_Check(set)) {
         PyErr_BadInternalCall();
         return -1;
     }

src/runtime/str.cpp

@@ -36,6 +36,7 @@
 #include "runtime/util.h"
 
 extern "C" PyObject* string_count(PyStringObject* self, PyObject* args) noexcept;
+extern "C" PyObject* string_join(PyStringObject* self, PyObject* orig) noexcept;
 extern "C" PyObject* string_split(PyStringObject* self, PyObject* args) noexcept;
 extern "C" PyObject* string_rsplit(PyStringObject* self, PyObject* args) noexcept;
 extern "C" PyObject* string_find(PyStringObject* self, PyObject* args) noexcept;
@@ -1711,76 +1712,9 @@ Box* strIsTitle(BoxedString* self) {
     return boxBool(cased);
 }
 
-Box* strJoin(BoxedString* self, Box* rhs) {
-    assert(isSubclass(self->cls, str_cls));
-
-    int i = 0;
-    if (rhs->cls == str_cls) {
-        BoxedString* srhs = static_cast<BoxedString*>(rhs);
-        if (srhs->size() == 0)
-            return EmptyString;
-
-        size_t rtn_len = self->size() * (srhs->size() - 1) + srhs->size();
-        BoxedString* rtn = new (rtn_len) BoxedString(nullptr, rtn_len);
-        char* p = rtn->data();
-        for (auto c : srhs->s) {
-            if (i > 0) {
-                memmove(p, self->data(), self->size());
-                p += self->size();
-            }
-            *p++ = c;
-            ++i;
-        }
-        return rtn;
-    } else if (rhs->cls == list_cls) {
-        BoxedList* lrhs = static_cast<BoxedList*>(rhs);
-        if (lrhs->size == 0)
-            return EmptyString;
-
-        size_t rtn_len = self->size() * (lrhs->size - 1);
-
-        for (int l = 0; l < lrhs->size; l++) {
-            // if we hit anything but a string (unicode objects can be here), bail out since we're
-            // going to pay the cost of converting the unicode to string objects anyway.
-            if (lrhs->elts->elts[l]->cls != str_cls)
-                goto fallback;
-            rtn_len += static_cast<BoxedString*>(lrhs->elts->elts[l])->size();
-        }
-        BoxedString* rtn = new (rtn_len) BoxedString(nullptr, rtn_len);
-        char* p = rtn->data();
-        for (i = 0; i < lrhs->size; i++) {
-            if (i > 0) {
-                memmove(p, self->data(), self->size());
-                p += self->size();
-            }
-            auto lrhs_el = static_cast<BoxedString*>(lrhs->elts->elts[i]);
-            memmove(p, lrhs_el->data(), lrhs_el->size());
-            p += lrhs_el->size();
-        }
-        return rtn;
-    }
-
-fallback:
-    std::string output_str;
-    llvm::raw_string_ostream os(output_str);
-    for (Box* e : rhs->pyElements()) {
-        if (i > 0)
-            os << self->s;
-        os << str(e)->s;
-        ++i;
-    }
-    os.flush();
-    return boxString(std::move(output_str));
-}
-
 extern "C" PyObject* _PyString_Join(PyObject* sep, PyObject* x) noexcept {
-    try {
-        RELEASE_ASSERT(isSubclass(sep->cls, str_cls), "");
-        return strJoin((BoxedString*)sep, x);
-    } catch (ExcInfo e) {
-        setCAPIException(e);
-        return NULL;
-    }
+    RELEASE_ASSERT(isSubclass(sep->cls, str_cls), "");
+    return string_join((PyStringObject*)sep, x);
 }
 
 Box* strReplace(Box* _self, Box* _old, Box* _new, Box** _args) {
@@ -2670,6 +2604,7 @@ static PyBufferProcs string_as_buffer = {
 
 static PyMethodDef string_methods[] = {
     { "count", (PyCFunction)string_count, METH_VARARGS, NULL },
+    { "join", (PyCFunction)string_join, METH_O, NULL },
     { "split", (PyCFunction)string_split, METH_VARARGS, NULL },
     { "rsplit", (PyCFunction)string_rsplit, METH_VARARGS, NULL },
     { "find", (PyCFunction)string_find, METH_VARARGS, NULL },
@@ -2770,8 +2705,6 @@ void setupStr() {
 
     str_cls->giveAttr("__iter__", new BoxedFunction(boxRTFunction((void*)strIter, typeFromClass(str_iterator_cls), 1)));
 
-    str_cls->giveAttr("join", new BoxedFunction(boxRTFunction((void*)strJoin, STR, 2)));
-
     str_cls->giveAttr("replace",
                       new BoxedFunction(boxRTFunction((void*)strReplace, UNKNOWN, 4, 1, false, false), { boxInt(-1) }));
 

src/runtime/types.cpp

@@ -1637,6 +1637,13 @@ static PyObject* reduce_2(PyObject* obj) noexcept {
             PyErr_Clear();
             state = Py_None;
             Py_INCREF(state);
+        } else {
+            // Pyston change: convert attrwrapper to a real dict
+            if (state->cls == attrwrapper_cls) {
+                PyObject* real_dict = PyDict_New();
+                PyDict_Update(real_dict, state);
+                state = real_dict;
+            }
         }
         names = slotnames(cls);
         if (names == NULL)

test/tests/copy_test.py

+import copy
+class C(object):
+    pass
+
+c = C()
+c.a = 1
+c.b = (["str", 1], 2L)
+print sorted(c.__dict__.items())
+cc = copy.deepcopy(c)
+del c.a
+print sorted(cc.__dict__.items())
+

test/tests/long.py

@@ -18,6 +18,7 @@ def test(a, b):
     if not isinstance(a, float) and not isinstance(b, float):
         print a ^ b, a | b, a & b
         print a.__hex__(), b.__hex__(), a.__oct__(), b.__oct__()
+        print a // b, b // a, a.__floordiv__(b), b.__floordiv__(a)
 
 
 print 1L / 5L

test/tests/unicode_test.py

@@ -150,3 +150,5 @@ s = " \thello world\t "
 for m in str.strip, str.lstrip, str.rstrip:
     for args in [], [" "], [u" "]:
         print repr(m(s, *args))
+
+print "".join([u"\xB2", u"\xB3"])