Merge pull request #920 from kmod/heaptype

Match cpython's HEAPTYPE flag

CMakeLists.txt

@@ -336,4 +336,4 @@ else()
 endif()
 
 # last file added (need to change this if we add a file that is added via a glob):
-# from_cpython/Lib/test/test_re.py
+# from_cpython/Lib/test/pickletester.py

from_cpython/Lib/test/test_pickle.py

+# expected: fail
+import pickle
+from cStringIO import StringIO
+
+from test import test_support
+
+from test.pickletester import (AbstractPickleTests,
+                               AbstractPickleModuleTests,
+                               AbstractPersistentPicklerTests,
+                               AbstractPicklerUnpicklerObjectTests,
+                               BigmemPickleTests)
+
+class PickleTests(AbstractPickleTests, AbstractPickleModuleTests):
+
+    def dumps(self, arg, proto=0, fast=0):
+        # Ignore fast
+        return pickle.dumps(arg, proto)
+
+    def loads(self, buf):
+        # Ignore fast
+        return pickle.loads(buf)
+
+    module = pickle
+    error = KeyError
+
+class PicklerTests(AbstractPickleTests):
+
+    error = KeyError
+
+    def dumps(self, arg, proto=0, fast=0):
+        f = StringIO()
+        p = pickle.Pickler(f, proto)
+        if fast:
+            p.fast = fast
+        p.dump(arg)
+        f.seek(0)
+        return f.read()
+
+    def loads(self, buf):
+        f = StringIO(buf)
+        u = pickle.Unpickler(f)
+        return u.load()
+
+class PersPicklerTests(AbstractPersistentPicklerTests):
+
+    def dumps(self, arg, proto=0, fast=0):
+        class PersPickler(pickle.Pickler):
+            def persistent_id(subself, obj):
+                return self.persistent_id(obj)
+        f = StringIO()
+        p = PersPickler(f, proto)
+        if fast:
+            p.fast = fast
+        p.dump(arg)
+        f.seek(0)
+        return f.read()
+
+    def loads(self, buf):
+        class PersUnpickler(pickle.Unpickler):
+            def persistent_load(subself, obj):
+                return self.persistent_load(obj)
+        f = StringIO(buf)
+        u = PersUnpickler(f)
+        return u.load()
+
+class PicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests):
+
+    pickler_class = pickle.Pickler
+    unpickler_class = pickle.Unpickler
+
+class PickleBigmemPickleTests(BigmemPickleTests):
+
+    def dumps(self, arg, proto=0, fast=0):
+        # Ignore fast
+        return pickle.dumps(arg, proto)
+
+    def loads(self, buf):
+        # Ignore fast
+        return pickle.loads(buf)
+
+
+def test_main():
+    test_support.run_unittest(
+        PickleTests,
+        PicklerTests,
+        PersPicklerTests,
+        PicklerUnpicklerObjectTests,
+        PickleBigmemPickleTests,
+    )
+    test_support.run_doctest(pickle)
+
+if __name__ == "__main__":
+    test_main()

from_cpython/Lib/types.py

@@ -58,7 +58,9 @@ InstanceType = type(_x)
 MethodType = type(_x._m)
 
 BuiltinFunctionType = type(len)
-BuiltinMethodType = type([].append)     # Same as BuiltinFunctionType
+# Pyston change:
+# BuiltinMethodType = type([].append)     # Same as BuiltinFunctionType
+BuiltinMethodType = type((1.0).hex)     # Same as BuiltinFunctionType
 
 ModuleType = type(sys)
 FileType = file
@@ -85,8 +87,9 @@ NotImplementedType = type(NotImplemented)
 AttrwrapperType = type(_C().__dict__)
 
 # For Jython, the following two types are identical
-# Pyston change: don't support these yet
-# GetSetDescriptorType = type(FunctionType.func_code)
+GetSetDescriptorType = type(FunctionType.func_code)
+# Pyston change:
 # MemberDescriptorType = type(FunctionType.func_globals)
+MemberDescriptorType = type(file.softspace)
 
 del sys, _f, _g, _C, _x                           # Not for export

src/capi/typeobject.cpp

@@ -2615,7 +2615,9 @@ static void inherit_special(PyTypeObject* type, PyTypeObject* base) noexcept {
            inherit tp_new; static extension types that specify some
            other built-in type as the default are considered
            new-style-aware so they also inherit object.__new__. */
-        if (base != object_cls || (type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
+        // Pyston change:
+        // if (base != object_cls || (type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
+        if (base != object_cls || !type->is_user_defined) {
             if (type->tp_new == NULL)
                 type->tp_new = base->tp_new;
         }

src/codegen/unwinding.cpp

@@ -1252,8 +1252,8 @@ llvm::JITEventListener* makeTracebacksListener() {
 }
 
 void setupUnwinding() {
-    unwind_session_cls = BoxedHeapClass::create(type_cls, object_cls, PythonUnwindSession::gcHandler, 0, 0,
-                                                sizeof(PythonUnwindSession), false, "unwind_session");
+    unwind_session_cls = BoxedClass::create(type_cls, object_cls, PythonUnwindSession::gcHandler, 0, 0,
+                                            sizeof(PythonUnwindSession), false, "unwind_session");
     unwind_session_cls->freeze();
 }
 }

src/runtime/bool.cpp

@@ -87,6 +87,9 @@ extern "C" Box* boolXor(BoxedBool* lhs, BoxedBool* rhs) {
 
 
 void setupBool() {
+    static PyNumberMethods bool_as_number;
+    bool_cls->tp_as_number = &bool_as_number;
+
     bool_cls->giveAttr("__nonzero__", new BoxedFunction(boxRTFunction((void*)boolNonzero, BOXED_BOOL, 1)));
     bool_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)boolRepr, STR, 1)));
     bool_cls->giveAttr("__hash__", new BoxedFunction(boxRTFunction((void*)boolHash, BOXED_INT, 1)));

src/runtime/builtin_modules/ast.cpp

@@ -71,15 +71,14 @@ void setupAST() {
 
 // ::create takes care of registering the class as a GC root.
 #define MAKE_CLS(name, base_cls)                                                                                       \
-    BoxedClass* name##_cls = BoxedHeapClass::create(type_cls, base_cls, /* gchandler = */ NULL, 0, 0,                  \
-                                                    sizeof(BoxedAST), false, STRINGIFY(name));                         \
+    BoxedClass* name##_cls = BoxedClass::create(type_cls, base_cls, /* gchandler = */ NULL, 0, 0, sizeof(BoxedAST),    \
+                                                false, STRINGIFY(name));                                               \
     ast_module->giveAttr(STRINGIFY(name), name##_cls);                                                                 \
     type_to_cls[AST_TYPE::name] = name##_cls;                                                                          \
     name##_cls->giveAttr("__module__", boxString("_ast"));                                                             \
     name##_cls->freeze()
 
-    AST_cls
-        = BoxedHeapClass::create(type_cls, object_cls, /* gchandler = */ NULL, 0, 0, sizeof(BoxedAST), false, "AST");
+    AST_cls = BoxedClass::create(type_cls, object_cls, /* gchandler = */ NULL, 0, 0, sizeof(BoxedAST), false, "AST");
     // ::create takes care of registering the class as a GC root.
     AST_cls->giveAttr("__module__", boxString("_ast"));
     AST_cls->freeze();

src/runtime/builtin_modules/builtins.cpp

@@ -1105,8 +1105,7 @@ static BoxedClass* makeBuiltinException(BoxedClass* base, const char* name, int
     if (size == 0)
         size = base->tp_basicsize;
 
-    BoxedClass* cls
-        = BoxedHeapClass::create(type_cls, base, NULL, offsetof(BoxedException, attrs), 0, size, false, name);
+    BoxedClass* cls = BoxedClass::create(type_cls, base, NULL, offsetof(BoxedException, attrs), 0, size, false, name);
     cls->giveAttr("__module__", boxString("exceptions"));
 
     if (base == object_cls) {
@@ -1516,8 +1515,7 @@ void setupBuiltins() {
                                    "Built-in functions, exceptions, and other objects.\n\nNoteworthy: None is "
                                    "the `nil' object; Ellipsis represents `...' in slices.");
 
-    BoxedHeapClass* ellipsis_cls
-        = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "ellipsis");
+    BoxedClass* ellipsis_cls = BoxedClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "ellipsis");
     Ellipsis = new (ellipsis_cls) Box();
     assert(Ellipsis->cls);
     gc::registerPermanentRoot(Ellipsis);
@@ -1530,15 +1528,13 @@ void setupBuiltins() {
     builtins_module->giveAttr(
         "print", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)print, NONE, 0, true, true), "print"));
 
-    notimplemented_cls
-        = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "NotImplementedType");
+    notimplemented_cls = BoxedClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "NotImplementedType");
     notimplemented_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)notimplementedRepr, STR, 1)));
     notimplemented_cls->freeze();
     NotImplemented = new (notimplemented_cls) Box();
     gc::registerPermanentRoot(NotImplemented);
 
     builtins_module->giveAttr("NotImplemented", NotImplemented);
-    builtins_module->giveAttr("NotImplementedType", notimplemented_cls);
 
     builtins_module->giveAttr("all", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)all, BOXED_BOOL, 1), "all"));
     builtins_module->giveAttr("any", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)any, BOXED_BOOL, 1), "any"));
@@ -1629,8 +1625,8 @@ void setupBuiltins() {
     builtins_module->giveAttr("__import__", new BoxedBuiltinFunctionOrMethod(import_func, "__import__",
                                                                              { None, None, None, new BoxedInt(-1) }));
 
-    enumerate_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedEnumerate::gcHandler, 0, 0,
-                                           sizeof(BoxedEnumerate), false, "enumerate");
+    enumerate_cls = BoxedClass::create(type_cls, object_cls, &BoxedEnumerate::gcHandler, 0, 0, sizeof(BoxedEnumerate),
+                                       false, "enumerate");
     enumerate_cls->giveAttr(
         "__new__",
         new BoxedFunction(boxRTFunction((void*)BoxedEnumerate::new_, UNKNOWN, 3, false, false), { boxInt(0) }));
@@ -1724,7 +1720,6 @@ void setupBuiltins() {
     builtins_module->giveAttr("set", set_cls);
     builtins_module->giveAttr("frozenset", frozenset_cls);
     builtins_module->giveAttr("tuple", tuple_cls);
-    builtins_module->giveAttr("instancemethod", instancemethod_cls);
     builtins_module->giveAttr("complex", complex_cls);
     builtins_module->giveAttr("super", super_cls);
     builtins_module->giveAttr("property", property_cls);

src/runtime/builtin_modules/sys.cpp

@@ -652,8 +652,8 @@ void setupSys() {
     sys_module->giveAttr("maxint", boxInt(PYSTON_INT_MAX));
     sys_module->giveAttr("maxsize", boxInt(PY_SSIZE_T_MAX));
 
-    sys_flags_cls = new (0) BoxedHeapClass(object_cls, BoxedSysFlags::gcHandler, 0, 0, sizeof(BoxedSysFlags), false,
-                                           static_cast<BoxedString*>(boxString("flags")));
+    sys_flags_cls = new (0)
+        BoxedClass(object_cls, BoxedSysFlags::gcHandler, 0, 0, sizeof(BoxedSysFlags), false, "flags");
     sys_flags_cls->giveAttr("__new__",
                             new BoxedFunction(boxRTFunction((void*)BoxedSysFlags::__new__, UNKNOWN, 1, true, true)));
 #define ADD(name)                                                                                                      \

src/runtime/builtin_modules/thread.cpp

@@ -210,7 +210,7 @@ void setupThread() {
     thread_module->giveAttr(
         "stack_size", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)stackSize, BOXED_INT, 0), "stack_size"));
 
-    thread_lock_cls = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(BoxedThreadLock), false, "lock");
+    thread_lock_cls = BoxedClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(BoxedThreadLock), false, "lock");
     thread_lock_cls->tp_dealloc = BoxedThreadLock::threadLockDestructor;
     thread_lock_cls->has_safe_tp_dealloc = true;
 
@@ -228,8 +228,8 @@ void setupThread() {
     thread_lock_cls->giveAttr("locked_lock", thread_lock_cls->getattr(internStringMortal("locked")));
     thread_lock_cls->freeze();
 
-    ThreadError = BoxedHeapClass::create(type_cls, Exception, NULL, Exception->attrs_offset,
-                                         Exception->tp_weaklistoffset, Exception->tp_basicsize, false, "error");
+    ThreadError = BoxedClass::create(type_cls, Exception, NULL, Exception->attrs_offset, Exception->tp_weaklistoffset,
+                                     Exception->tp_basicsize, false, "error");
     ThreadError->giveAttr("__module__", boxString("thread"));
     ThreadError->freeze();
 

src/runtime/classobj.cpp

@@ -16,6 +16,7 @@
 
 #include <sstream>
 
+#include "capi/typeobject.h"
 #include "capi/types.h"
 #include "core/types.h"
 #include "runtime/objmodel.h"
@@ -213,7 +214,7 @@ static const char* set_bases(PyClassObject* c, PyObject* v) {
     return "";
 }
 
-static Box* classobjSetattr(Box* _cls, Box* _attr, Box* _value) {
+static void _classobjSetattr(Box* _cls, Box* _attr, Box* _value) {
     RELEASE_ASSERT(_cls->cls == classobj_cls, "");
     BoxedClassobj* cls = static_cast<BoxedClassobj*>(_cls);
 
@@ -221,26 +222,30 @@ static Box* classobjSetattr(Box* _cls, Box* _attr, Box* _value) {
     BoxedString* attr = static_cast<BoxedString*>(_attr);
 
     if (attr->s() == "__bases__") {
+        RELEASE_ASSERT(_value, "can't delete __bases__");
+
         const char* error_str = set_bases((PyClassObject*)cls, _value);
         if (error_str && error_str[0] != '\0')
             raiseExcHelper(TypeError, "%s", error_str);
         static BoxedString* bases_str = internStringImmortal("__bases__");
         cls->setattr(bases_str, _value, NULL);
-        return None;
+        return;
     }
     PyObject_GenericSetAttr(cls, _attr, _value);
     checkAndThrowCAPIException();
+}
+
+static Box* classobjSetattr(Box* _cls, Box* _attr, Box* _value) {
+    assert(_value);
+
+    _classobjSetattr(_cls, _attr, _value);
     return None;
 }
 
 static int classobj_setattro(Box* cls, Box* attr, Box* value) noexcept {
     try {
-        if (value) {
-            classobjSetattr(cls, attr, value);
-            return 0;
-        } else {
-            RELEASE_ASSERT(0, "");
-        }
+        _classobjSetattr(cls, attr, value);
+        return 0;
     } catch (ExcInfo e) {
         setCAPIException(e);
         return -1;
@@ -1494,11 +1499,10 @@ extern "C" PyObject* PyMethod_Class(PyObject* im) noexcept {
 }
 
 void setupClassobj() {
-    classobj_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedClassobj::gcHandler,
-                                          offsetof(BoxedClassobj, attrs), 0, sizeof(BoxedClassobj), false, "classobj");
-    instance_cls
-        = BoxedHeapClass::create(type_cls, object_cls, &BoxedInstance::gcHandler, offsetof(BoxedInstance, attrs),
-                                 offsetof(BoxedInstance, weakreflist), sizeof(BoxedInstance), false, "instance");
+    classobj_cls = BoxedClass::create(type_cls, object_cls, &BoxedClassobj::gcHandler, offsetof(BoxedClassobj, attrs),
+                                      0, sizeof(BoxedClassobj), false, "classobj");
+    instance_cls = BoxedClass::create(type_cls, object_cls, &BoxedInstance::gcHandler, offsetof(BoxedInstance, attrs),
+                                      offsetof(BoxedInstance, weakreflist), sizeof(BoxedInstance), false, "instance");
 
     classobj_cls->giveAttr("__new__", new BoxedFunction(boxRTFunction((void*)classobjNew, UNKNOWN, 4, false, false)));
 
@@ -1513,7 +1517,14 @@ void setupClassobj() {
     classobj_cls->freeze();
     classobj_cls->tp_getattro = classobj_getattro;
     classobj_cls->tp_setattro = classobj_setattro;
-
+    add_operators(classobj_cls);
+
+    static PyNumberMethods instance_as_number;
+    instance_cls->tp_as_number = &instance_as_number;
+    static PySequenceMethods instance_as_sequence;
+    instance_cls->tp_as_sequence = &instance_as_sequence;
+    static PyMappingMethods instance_as_mapping;
+    instance_cls->tp_as_mapping = &instance_as_mapping;
 
     instance_cls->giveAttr("__getattribute__",
                            new BoxedFunction(boxRTFunction((void*)instanceGetattroInternal<CXX>, UNKNOWN, 2)));

src/runtime/code.cpp

@@ -111,8 +111,7 @@ extern "C" PyCodeObject* PyCode_New(int, int, int, int, PyObject*, PyObject*, Py
 }
 
 void setupCode() {
-    code_cls
-        = BoxedHeapClass::create(type_cls, object_cls, &BoxedCode::gcHandler, 0, 0, sizeof(BoxedCode), false, "code");
+    code_cls = BoxedClass::create(type_cls, object_cls, &BoxedCode::gcHandler, 0, 0, sizeof(BoxedCode), false, "code");
 
     code_cls->giveAttr("__new__", None); // Hacky way of preventing users from instantiating this
 

src/runtime/complex.cpp

@@ -14,6 +14,7 @@
 
 #include "runtime/complex.h"
 
+#include "capi/typeobject.h"
 #include "core/types.h"
 #include "runtime/float.h"
 #include "runtime/inline/boxing.h"
@@ -1224,6 +1225,9 @@ static PyMethodDef complex_methods[] = {
 };
 
 void setupComplex() {
+    static PyNumberMethods complex_as_number;
+    complex_cls->tp_as_number = &complex_as_number;
+
     auto complex_new = boxRTFunction((void*)complexNew<CXX>, UNKNOWN, 3, false, false,
                                      ParamNames({ "", "real", "imag" }, "", ""), CXX);
     addRTFunction(complex_new, (void*)complexNew<CAPI>, UNKNOWN, CAPI);
@@ -1278,6 +1282,8 @@ void setupComplex() {
         complex_cls->giveAttr(md.ml_name, new BoxedMethodDescriptor(&md, complex_cls));
     }
 
+    add_operators(complex_cls);
+
     complex_cls->freeze();
     complex_cls->tp_as_number->nb_negative = (unaryfunc)complex_neg;
     complex_cls->tp_richcompare = complex_richcompare;

src/runtime/dict.cpp

@@ -775,15 +775,20 @@ void BoxedDict::dealloc(Box* b) noexcept {
 }
 
 void setupDict() {
-    dict_iterator_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedDictIterator::gcHandler, 0, 0,
-                                               sizeof(BoxedDictIterator), false, "dictionary-itemiterator");
-
-    dict_keys_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedDictView::gcHandler, 0, 0, sizeof(BoxedDictView),
-                                           false, "dict_keys");
-    dict_values_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedDictView::gcHandler, 0, 0,
-                                             sizeof(BoxedDictView), false, "dict_values");
-    dict_items_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedDictView::gcHandler, 0, 0,
-                                            sizeof(BoxedDictView), false, "dict_items");
+    static PyMappingMethods dict_as_mapping;
+    dict_cls->tp_as_mapping = &dict_as_mapping;
+    static PySequenceMethods dict_as_sequence;
+    dict_cls->tp_as_sequence = &dict_as_sequence;
+
+    dict_iterator_cls = BoxedClass::create(type_cls, object_cls, &BoxedDictIterator::gcHandler, 0, 0,
+                                           sizeof(BoxedDictIterator), false, "dictionary-itemiterator");
+
+    dict_keys_cls = BoxedClass::create(type_cls, object_cls, &BoxedDictView::gcHandler, 0, 0, sizeof(BoxedDictView),
+                                       false, "dict_keys");
+    dict_values_cls = BoxedClass::create(type_cls, object_cls, &BoxedDictView::gcHandler, 0, 0, sizeof(BoxedDictView),
+                                         false, "dict_values");
+    dict_items_cls = BoxedClass::create(type_cls, object_cls, &BoxedDictView::gcHandler, 0, 0, sizeof(BoxedDictView),
+                                        false, "dict_items");
 
     dict_cls->tp_dealloc = &BoxedDict::dealloc;
     dict_cls->has_safe_tp_dealloc = true;

src/runtime/exceptions.cpp

@@ -192,11 +192,13 @@ extern "C" void raise3_capi(Box* arg0, Box* arg1, Box* arg2) noexcept {
 }
 
 void raiseExcHelper(BoxedClass* cls, Box* arg) {
+    assert(!PyErr_Occurred());
     Box* exc_obj = runtimeCall(cls, ArgPassSpec(1), arg, NULL, NULL, NULL, NULL);
     raiseExc(exc_obj);
 }
 
 void raiseExcHelper(BoxedClass* cls, const char* msg, ...) {
+    assert(!PyErr_Occurred());
     if (msg != NULL) {
         va_list ap;
         va_start(ap, msg);

src/runtime/float.cpp

@@ -18,6 +18,7 @@
 #include <cstring>
 #include <gmp.h>
 
+#include "capi/typeobject.h"
 #include "capi/types.h"
 #include "core/types.h"
 #include "runtime/inline/boxing.h"
@@ -1625,6 +1626,10 @@ exit:
     return result;
 }
 
+static PyObject* float_getnewargs(PyFloatObject* v) noexcept {
+    return Py_BuildValue("(d)", v->ob_fval);
+}
+
 static PyMethodDef float_methods[] = { { "hex", (PyCFunction)float_hex, METH_NOARGS, NULL },
                                        { "fromhex", (PyCFunction)float_fromhex, METH_O | METH_CLASS, NULL },
                                        { "as_integer_ratio", (PyCFunction)float_as_integer_ratio, METH_NOARGS, NULL },
@@ -1633,6 +1638,12 @@ static PyMethodDef float_methods[] = { { "hex", (PyCFunction)float_hex, METH_NOA
                                        { "__format__", (PyCFunction)float__format__, METH_VARARGS, NULL } };
 
 void setupFloat() {
+    static PyNumberMethods float_as_number;
+    float_cls->tp_as_number = &float_as_number;
+
+    float_cls->giveAttr("__getnewargs__", new BoxedFunction(boxRTFunction((void*)float_getnewargs, UNKNOWN, 1,
+                                                                          ParamNames::empty(), CAPI)));
+
     _addFunc("__add__", BOXED_FLOAT, (void*)floatAddFloat, (void*)floatAddInt, (void*)floatAdd);
     float_cls->giveAttr("__radd__", float_cls->getattr(internStringMortal("__add__")));
 
@@ -1687,6 +1698,7 @@ void setupFloat() {
         float_cls->giveAttr(md.ml_name, new BoxedMethodDescriptor(&md, float_cls));
     }
 
+    add_operators(float_cls);
     float_cls->freeze();
 
     floatFormatInit();

src/runtime/frame.cpp

@@ -150,8 +150,8 @@ Box* getFrame(int depth) {
 }
 
 void setupFrame() {
-    frame_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedFrame::gchandler, 0, 0, sizeof(BoxedFrame), false,
-                                       "frame");
+    frame_cls
+        = BoxedClass::create(type_cls, object_cls, &BoxedFrame::gchandler, 0, 0, sizeof(BoxedFrame), false, "frame");
     frame_cls->tp_dealloc = BoxedFrame::simpleDestructor;
     frame_cls->has_safe_tp_dealloc = true;
 

src/runtime/generator.cpp

@@ -478,8 +478,8 @@ void generatorDestructor(Box* b) {
 
 void setupGenerator() {
     generator_cls
-        = BoxedHeapClass::create(type_cls, object_cls, &BoxedGenerator::gcHandler, 0,
-                                 offsetof(BoxedGenerator, weakreflist), sizeof(BoxedGenerator), false, "generator");
+        = BoxedClass::create(type_cls, object_cls, &BoxedGenerator::gcHandler, 0, offsetof(BoxedGenerator, weakreflist),
+                             sizeof(BoxedGenerator), false, "generator");
     generator_cls->tp_dealloc = generatorDestructor;
     generator_cls->has_safe_tp_dealloc = true;
     generator_cls->giveAttr("__iter__",

src/runtime/import.cpp

@@ -234,7 +234,7 @@ SearchResult findModule(const std::string& name, BoxedString* full_name, BoxedLi
 
         PyObject* importer = get_path_importer(path_importer_cache, path_hooks, _p);
         if (importer == NULL)
-            return SearchResult("", SearchResult::SEARCH_ERROR);
+            throwCAPIException();
 
         if (importer != None) {
             CallattrFlags callattr_flags{.cls_only = false, .null_on_nonexistent = false, .argspec = ArgPassSpec(1) };
@@ -469,10 +469,8 @@ static bool loadNext(Box* mod, Box* altmod, std::string& name, std::string& buf,
             buf = subname;
         }
     }
-    if (result == NULL) {
-        *rtn = NULL;
-        return false;
-    }
+    if (result == NULL)
+        throwCAPIException();
 
     if (result == None)
         raiseExcHelper(ImportError, "No module named %.200s", local_name.c_str());
@@ -892,7 +890,7 @@ void setupImport() {
     imp_module->giveAttr("C_BUILTIN", boxInt(SearchResult::C_BUILTIN));
     imp_module->giveAttr("PY_FROZEN", boxInt(SearchResult::PY_FROZEN));
 
-    null_importer_cls = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "NullImporter");
+    null_importer_cls = BoxedClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "NullImporter");
     null_importer_cls->giveAttr(
         "__init__", new BoxedFunction(boxRTFunction((void*)nullImporterInit, NONE, 2, false, false), { None }));
     null_importer_cls->giveAttr("find_module", new BoxedBuiltinFunctionOrMethod(

src/runtime/inline/xrange.cpp

@@ -206,9 +206,12 @@ Box* xrangeLen(Box* self) {
 }
 
 void setupXrange() {
-    xrange_cls = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(BoxedXrange), false, "xrange");
-    xrange_iterator_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedXrangeIterator::gcHandler, 0, 0,
-                                                 sizeof(BoxedXrangeIterator), false, "rangeiterator");
+    xrange_cls = BoxedClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(BoxedXrange), false, "xrange");
+    xrange_iterator_cls = BoxedClass::create(type_cls, object_cls, &BoxedXrangeIterator::gcHandler, 0, 0,
+                                             sizeof(BoxedXrangeIterator), false, "rangeiterator");
+
+    static PySequenceMethods xrange_as_sequence;
+    xrange_cls->tp_as_sequence = &xrange_as_sequence;
 
     xrange_cls->giveAttr(
         "__new__",

src/runtime/int.cpp

@@ -1206,12 +1206,22 @@ static PyObject* int_richcompare(PyObject* v, PyObject* w, int op) noexcept {
     }
 }
 
+static PyObject* int_getnewargs(BoxedInt* v) noexcept {
+    return Py_BuildValue("(l)", v->n);
+}
+
 void setupInt() {
+    static PyNumberMethods int_as_number;
+    int_cls->tp_as_number = &int_as_number;
+
     for (int i = 0; i < NUM_INTERNED_INTS; i++) {
         interned_ints[i] = new BoxedInt(i);
         gc::registerPermanentRoot(interned_ints[i]);
     }
 
+    int_cls->giveAttr("__getnewargs__",
+                      new BoxedFunction(boxRTFunction((void*)int_getnewargs, UNKNOWN, 1, ParamNames::empty(), CAPI)));
+
     _addFuncIntFloatUnknown("__add__", (void*)intAddInt, (void*)intAddFloat, (void*)intAdd);
     _addFuncIntUnknown("__and__", BOXED_INT, (void*)intAndInt, (void*)intAnd);
     _addFuncIntUnknown("__or__", BOXED_INT, (void*)intOrInt, (void*)intOr);

src/runtime/iterobject.cpp

@@ -181,8 +181,8 @@ bool calliter_hasnext(Box* b) {
 
 
 void setupIter() {
-    seqiter_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedSeqIter::gcHandler, 0, 0, sizeof(BoxedSeqIter),
-                                         false, "iterator");
+    seqiter_cls = BoxedClass::create(type_cls, object_cls, &BoxedSeqIter::gcHandler, 0, 0, sizeof(BoxedSeqIter), false,
+                                     "iterator");
 
     seqiter_cls->giveAttr("next", new BoxedFunction(boxRTFunction((void*)seqiterNext, UNKNOWN, 1)));
     seqiter_cls->giveAttr("__hasnext__", new BoxedFunction(boxRTFunction((void*)seqiterHasnext, BOXED_BOOL, 1)));
@@ -191,8 +191,8 @@ void setupIter() {
     seqiter_cls->freeze();
     seqiter_cls->tpp_hasnext = seqiterHasnextUnboxed;
 
-    seqreviter_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedSeqIter::gcHandler, 0, 0, sizeof(BoxedSeqIter),
-                                            false, "reversed");
+    seqreviter_cls = BoxedClass::create(type_cls, object_cls, &BoxedSeqIter::gcHandler, 0, 0, sizeof(BoxedSeqIter),
+                                        false, "reversed");
 
     seqreviter_cls->giveAttr("next", new BoxedFunction(boxRTFunction((void*)seqiterNext, UNKNOWN, 1)));
     seqreviter_cls->giveAttr("__hasnext__", new BoxedFunction(boxRTFunction((void*)seqreviterHasnext, BOXED_BOOL, 1)));
@@ -200,8 +200,8 @@ void setupIter() {
 
     seqreviter_cls->freeze();
 
-    iterwrapper_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedIterWrapper::gcHandler, 0, 0,
-                                             sizeof(BoxedIterWrapper), false, "iterwrapper");
+    iterwrapper_cls = BoxedClass::create(type_cls, object_cls, &BoxedIterWrapper::gcHandler, 0, 0,
+                                         sizeof(BoxedIterWrapper), false, "iterwrapper");
 
     iterwrapper_cls->giveAttr("next", new BoxedFunction(boxRTFunction((void*)iterwrapperNext, UNKNOWN, 1)));
     iterwrapper_cls->giveAttr("__hasnext__",

src/runtime/list.cpp

@@ -148,6 +148,42 @@ static Py_ssize_t list_length(Box* self) noexcept {
     return static_cast<BoxedList*>(self)->size;
 }
 
+static PyObject* list_concat(PyListObject* a, PyObject* bb) noexcept {
+    Py_ssize_t size;
+    Py_ssize_t i;
+    PyObject** src, **dest;
+    PyListObject* np;
+    if (!PyList_Check(bb)) {
+        PyErr_Format(PyExc_TypeError, "can only concatenate list (not \"%.200s\") to list", bb->cls->tp_name);
+        return NULL;
+    }
+#define b ((PyListObject*)bb)
+    size = Py_SIZE(a) + Py_SIZE(b);
+    if (size < 0)
+        return PyErr_NoMemory();
+    np = (PyListObject*)PyList_New(size);
+    if (np == NULL) {
+        return NULL;
+    }
+    src = a->ob_item;
+    dest = np->ob_item;
+    for (i = 0; i < Py_SIZE(a); i++) {
+        PyObject* v = src[i];
+        Py_INCREF(v);
+        dest[i] = v;
+    }
+    src = b->ob_item;
+    dest = np->ob_item + Py_SIZE(a);
+    for (i = 0; i < Py_SIZE(b); i++) {
+        PyObject* v = src[i];
+        Py_INCREF(v);
+        dest[i] = v;
+    }
+    return (PyObject*)np;
+#undef b
+}
+
+
 Box* _listSlice(BoxedList* self, i64 start, i64 stop, i64 step, i64 length) {
     assert(step != 0);
     if (step > 0) {
@@ -921,10 +957,48 @@ static inline int list_contains_shared(BoxedList* self, Box* elt) {
     return false;
 }
 
-static int list_contains(PyListObject* a, PyObject* el) {
+static int list_contains(PyListObject* a, PyObject* el) noexcept {
     return list_contains_shared((BoxedList*)a, el);
 }
 
+static PyObject* list_repeat(PyListObject* a, Py_ssize_t n) noexcept {
+    Py_ssize_t i, j;
+    Py_ssize_t size;
+    PyListObject* np;
+    PyObject** p, **items;
+    PyObject* elem;
+    if (n < 0)
+        n = 0;
+    if (n > 0 && Py_SIZE(a) > PY_SSIZE_T_MAX / n)
+        return PyErr_NoMemory();
+    size = Py_SIZE(a) * n;
+    if (size == 0)
+        return PyList_New(0);
+    np = (PyListObject*)PyList_New(size);
+    if (np == NULL)
+        return NULL;
+
+    items = np->ob_item;
+    if (Py_SIZE(a) == 1) {
+        elem = a->ob_item[0];
+        for (i = 0; i < n; i++) {
+            items[i] = elem;
+            Py_INCREF(elem);
+        }
+        return (PyObject*)np;
+    }
+    p = np->ob_item;
+    items = a->ob_item;
+    for (i = 0; i < n; i++) {
+        for (j = 0; j < Py_SIZE(a); j++) {
+            *p = items[j];
+            Py_INCREF(*p);
+            p++;
+        }
+    }
+    return (PyObject*)np;
+}
+
 Box* listContains(BoxedList* self, Box* elt) {
     return boxBool(list_contains_shared(self, elt));
 }
@@ -1198,10 +1272,16 @@ void BoxedList::gcHandler(GCVisitor* v, Box* b) {
 }
 
 void setupList() {
-    list_iterator_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedListIterator::gcHandler, 0, 0,
-                                               sizeof(BoxedListIterator), false, "listiterator");
-    list_reverse_iterator_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedListIterator::gcHandler, 0, 0,
-                                                       sizeof(BoxedListIterator), false, "listreverseiterator");
+    static PySequenceMethods list_as_sequence;
+    list_cls->tp_as_sequence = &list_as_sequence;
+    static PyMappingMethods list_as_mapping;
+    list_cls->tp_as_mapping = &list_as_mapping;
+
+    list_cls->tp_iter = listIter;
+    list_iterator_cls = BoxedClass::create(type_cls, object_cls, &BoxedListIterator::gcHandler, 0, 0,
+                                           sizeof(BoxedListIterator), false, "listiterator");
+    list_reverse_iterator_cls = BoxedClass::create(type_cls, object_cls, &BoxedListIterator::gcHandler, 0, 0,
+                                                   sizeof(BoxedListIterator), false, "listreverseiterator");
 
     list_cls->giveAttr("__len__", new BoxedFunction(boxRTFunction((void*)listLen, BOXED_INT, 1)));
 
@@ -1281,13 +1361,14 @@ void setupList() {
     list_cls->giveAttr("__hash__", None);
     list_cls->freeze();
 
-    list_cls->tp_iter = listIter;
     list_cls->tp_as_sequence->sq_length = list_length;
+    list_cls->tp_as_sequence->sq_concat = (binaryfunc)list_concat;
     list_cls->tp_as_sequence->sq_item = (ssizeargfunc)list_item;
     list_cls->tp_as_sequence->sq_slice = list_slice;
     list_cls->tp_as_sequence->sq_ass_item = (ssizeobjargproc)list_ass_item;
     list_cls->tp_as_sequence->sq_ass_slice = (ssizessizeobjargproc)list_ass_slice;
     list_cls->tp_as_sequence->sq_contains = (objobjproc)list_contains;
+    list_cls->tp_as_sequence->sq_repeat = (ssizeargfunc)list_repeat;
 
     CLFunction* hasnext = boxRTFunction((void*)listiterHasnextUnboxed, BOOL, 1);
     addRTFunction(hasnext, (void*)listiterHasnext, BOXED_BOOL);

src/runtime/long.cpp

@@ -74,7 +74,7 @@ extern "C" int _PyLong_Sign(PyObject* l) noexcept {
 
 extern "C" PyObject* _PyLong_Copy(PyLongObject* src) noexcept {
     BoxedLong* rtn = new BoxedLong();
-    mpz_init_set(((BoxedLong*)src)->n, rtn->n);
+    mpz_init_set(rtn->n, ((BoxedLong*)src)->n);
     return rtn;
 }
 
@@ -1449,7 +1449,14 @@ static Box* long1(Box* b, void*) {
     return boxLong(1);
 }
 
+static PyObject* long_getnewargs(PyLongObject* v) noexcept {
+    return Py_BuildValue("(N)", _PyLong_Copy(v));
+}
+
 void setupLong() {
+    static PyNumberMethods long_as_number;
+    long_cls->tp_as_number = &long_as_number;
+
     mp_set_memory_functions(customised_allocation, customised_realloc, customised_free);
 
     _addFuncPow("__pow__", UNKNOWN, (void*)longPowFloat, (void*)longPow);
@@ -1512,6 +1519,9 @@ void setupLong() {
     long_cls->giveAttr("numerator", new (pyston_getset_cls) BoxedGetsetDescriptor(longLong, NULL, NULL));
     long_cls->giveAttr("denominator", new (pyston_getset_cls) BoxedGetsetDescriptor(long1, NULL, NULL));
 
+    long_cls->giveAttr("__getnewargs__",
+                       new BoxedFunction(boxRTFunction((void*)long_getnewargs, UNKNOWN, 1, ParamNames::empty(), CAPI)));
+
     add_operators(long_cls);
     long_cls->freeze();
 

src/runtime/long.h

@@ -41,6 +41,8 @@ public:
 extern "C" Box* createLong(llvm::StringRef s);
 extern "C" BoxedLong* boxLong(int64_t n);
 
+Box* longRepr(BoxedLong* v);
+
 Box* longNeg(BoxedLong* lhs);
 Box* longAbs(BoxedLong* v1);
 

src/runtime/objmodel.cpp

@@ -420,7 +420,7 @@ void BoxedClass::freeze() {
 }
 
 BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset, int weaklist_offset,
-                       int instance_size, bool is_user_defined)
+                       int instance_size, bool is_user_defined, const char* name)
     : attrs(HiddenClass::makeSingleton()),
       gc_visit(gc_visit),
       attrs_offset(attrs_offset),
@@ -435,6 +435,7 @@ BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset
     memset(&tp_name, 0, (char*)(&tp_version_tag + 1) - (char*)(&tp_name));
     tp_basicsize = instance_size;
     tp_weaklistoffset = weaklist_offset;
+    tp_name = name;
 
     tp_flags |= Py_TPFLAGS_DEFAULT_EXTERNAL;
     tp_flags |= Py_TPFLAGS_CHECKTYPES;
@@ -515,6 +516,24 @@ BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset
         gc::registerPermanentRoot(this);
 }
 
+BoxedClass* BoxedClass::create(BoxedClass* metaclass, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
+                               int weaklist_offset, int instance_size, bool is_user_defined, const char* name) {
+    assert(!is_user_defined);
+    BoxedClass* made = new (metaclass, 0)
+        BoxedClass(base, gc_visit, attrs_offset, weaklist_offset, instance_size, is_user_defined, name);
+
+    // While it might be ok if these were set, it'd indicate a difference in
+    // expectations as to who was going to calculate them:
+    assert(!made->tp_mro);
+    assert(!made->tp_bases);
+    made->tp_bases = NULL;
+
+    made->finishInitialization();
+    assert(made->tp_mro);
+
+    return made;
+}
+
 void BoxedClass::finishInitialization() {
     assert(!tp_traverse);
     assert(!tp_clear);
@@ -531,19 +550,19 @@ void BoxedClass::finishInitialization() {
 
 BoxedHeapClass::BoxedHeapClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset, int weaklist_offset,
                                int instance_size, bool is_user_defined, BoxedString* name)
-    : BoxedClass(base, gc_visit, attrs_offset, weaklist_offset, instance_size, is_user_defined),
+    : BoxedClass(base, gc_visit, attrs_offset, weaklist_offset, instance_size, is_user_defined, name->data()),
       ht_name(name),
       ht_slots(NULL) {
+    assert(is_user_defined);
 
     tp_as_number = &as_number;
     tp_as_mapping = &as_mapping;
     tp_as_sequence = &as_sequence;
     tp_as_buffer = &as_buffer;
     tp_flags |= Py_TPFLAGS_HEAPTYPE;
+
     if (!ht_name)
         assert(str_cls == NULL);
-    else
-        tp_name = ht_name->data();
 
     memset(&as_number, 0, sizeof(as_number));
     memset(&as_mapping, 0, sizeof(as_mapping));
@@ -551,13 +570,6 @@ BoxedHeapClass::BoxedHeapClass(BoxedClass* base, gcvisit_func gc_visit, int attr
     memset(&as_buffer, 0, sizeof(as_buffer));
 }
 
-BoxedHeapClass* BoxedHeapClass::create(BoxedClass* metaclass, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
-                                       int weaklist_offset, int instance_size, bool is_user_defined,
-                                       llvm::StringRef name) {
-    return create(metaclass, base, gc_visit, attrs_offset, weaklist_offset, instance_size, is_user_defined,
-                  boxString(name), NULL, 0);
-}
-
 BoxedHeapClass* BoxedHeapClass::create(BoxedClass* metaclass, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
                                        int weaklist_offset, int instance_size, bool is_user_defined, BoxedString* name,
                                        BoxedTuple* bases, size_t nslots) {

src/runtime/set.cpp

@@ -637,6 +637,30 @@ extern "C" PyObject* PyFrozenSet_New(PyObject* iterable) noexcept {
     }
 }
 
+static PyObject* set_reduce(BoxedSet* so) noexcept {
+    PyObject* keys = NULL, * args = NULL, * result = NULL, * dict = NULL;
+
+    keys = PySequence_List((PyObject*)so);
+    if (keys == NULL)
+        goto done;
+    args = PyTuple_Pack(1, keys);
+    if (args == NULL)
+        goto done;
+    dict = PyObject_GetAttrString((PyObject*)so, "__dict__");
+    if (dict == NULL) {
+        PyErr_Clear();
+        dict = Py_None;
+        Py_INCREF(dict);
+    }
+    result = PyTuple_Pack(3, Py_TYPE(so), args, dict);
+done:
+    Py_XDECREF(args);
+    Py_XDECREF(keys);
+    Py_XDECREF(dict);
+    return result;
+}
+
+
 } // namespace set
 
 void BoxedSet::dealloc(Box* b) noexcept {
@@ -646,12 +670,28 @@ void BoxedSet::dealloc(Box* b) noexcept {
 
 using namespace pyston::set;
 
+static PyMethodDef set_methods[] = {
+    { "__reduce__", (PyCFunction)set_reduce, METH_NOARGS, NULL },
+};
+static PyMethodDef frozenset_methods[] = {
+    { "__reduce__", (PyCFunction)set_reduce, METH_NOARGS, NULL },
+};
+
 void setupSet() {
+    static PySequenceMethods set_as_sequence;
+    set_cls->tp_as_sequence = &set_as_sequence;
+    static PyNumberMethods set_as_number;
+    set_cls->tp_as_number = &set_as_number;
+    static PySequenceMethods frozenset_as_sequence;
+    frozenset_cls->tp_as_sequence = &frozenset_as_sequence;
+    static PyNumberMethods frozenset_as_number;
+    frozenset_cls->tp_as_number = &frozenset_as_number;
+
     set_cls->tp_dealloc = frozenset_cls->tp_dealloc = BoxedSet::dealloc;
     set_cls->has_safe_tp_dealloc = frozenset_cls->has_safe_tp_dealloc = true;
 
-    set_iterator_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedSetIterator::gcHandler, 0, 0,
-                                              sizeof(BoxedSetIterator), false, "setiterator");
+    set_iterator_cls = BoxedClass::create(type_cls, object_cls, &BoxedSetIterator::gcHandler, 0, 0,
+                                          sizeof(BoxedSetIterator), false, "setiterator");
     set_iterator_cls->giveAttr(
         "__iter__", new BoxedFunction(boxRTFunction((void*)setiteratorIter, typeFromClass(set_iterator_cls), 1)));
     set_iterator_cls->giveAttr("__hasnext__",
@@ -771,6 +811,14 @@ void setupSet() {
     set_cls->giveAttr("copy", new BoxedFunction(boxRTFunction((void*)setCopy, UNKNOWN, 1)));
     set_cls->giveAttr("pop", new BoxedFunction(boxRTFunction((void*)setPop, UNKNOWN, 1)));
 
+    for (auto& md : set_methods) {
+        set_cls->giveAttr(md.ml_name, new BoxedMethodDescriptor(&md, set_cls));
+    }
+
+    for (auto& md : frozenset_methods) {
+        frozenset_cls->giveAttr(md.ml_name, new BoxedMethodDescriptor(&md, frozenset_cls));
+    }
+
     set_cls->freeze();
     frozenset_cls->freeze();
 

src/runtime/str.cpp

@@ -2786,6 +2786,10 @@ static int string_buffer_getbuffer(BoxedString* self, Py_buffer* view, int flags
     return PyBuffer_FillInfo(view, (PyObject*)self, self->data(), self->size(), 1, flags);
 }
 
+static PyObject* string_getnewargs(BoxedString* v) noexcept {
+    return Py_BuildValue("(s#)", v->data(), v->size());
+}
+
 static PyBufferProcs string_as_buffer = {
     (readbufferproc)string_buffer_getreadbuf, // comments are the only way I've found of
     (writebufferproc)NULL,                    // forcing clang-format to break these onto multiple lines
@@ -2811,10 +2815,17 @@ static PyMethodDef string_methods[] = {
 };
 
 void setupStr() {
+    static PySequenceMethods string_as_sequence;
+    str_cls->tp_as_sequence = &string_as_sequence;
+    static PyNumberMethods str_as_number;
+    str_cls->tp_as_number = &str_as_number;
+    static PyMappingMethods str_as_mapping;
+    str_cls->tp_as_mapping = &str_as_mapping;
+
     str_cls->tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER;
 
-    str_iterator_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedStringIterator::gcHandler, 0, 0,
-                                              sizeof(BoxedStringIterator), false, "striterator");
+    str_iterator_cls = BoxedClass::create(type_cls, object_cls, &BoxedStringIterator::gcHandler, 0, 0,
+                                          sizeof(BoxedStringIterator), false, "striterator");
     str_iterator_cls->giveAttr("__hasnext__",
                                new BoxedFunction(boxRTFunction((void*)BoxedStringIterator::hasnext, BOXED_BOOL, 1)));
     str_iterator_cls->giveAttr("__iter__",
@@ -2827,6 +2838,9 @@ void setupStr() {
     str_cls->tp_as_buffer = &string_as_buffer;
     str_cls->tp_print = string_print;
 
+    str_cls->giveAttr("__getnewargs__", new BoxedFunction(boxRTFunction((void*)string_getnewargs, UNKNOWN, 1,
+                                                                        ParamNames::empty(), CAPI)));
+
     str_cls->giveAttr("__len__", new BoxedFunction(boxRTFunction((void*)strLen, BOXED_INT, 1)));
     str_cls->giveAttr("__str__", new BoxedFunction(boxRTFunction((void*)strStr, STR, 1)));
     str_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)strRepr, STR, 1)));

src/runtime/super.cpp

@@ -197,8 +197,8 @@ Box* superInit(Box* _self, Box* _type, Box* obj) {
 }
 
 void setupSuper() {
-    super_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedSuper::gcHandler, 0, 0, sizeof(BoxedSuper), false,
-                                       "super");
+    super_cls
+        = BoxedClass::create(type_cls, object_cls, &BoxedSuper::gcHandler, 0, 0, sizeof(BoxedSuper), false, "super");
 
     // super_cls->giveAttr("__getattribute__", new BoxedFunction(boxRTFunction((void*)superGetattribute, UNKNOWN, 2)));
     super_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)superRepr, STR, 1)));

src/runtime/traceback.cpp

@@ -120,8 +120,8 @@ void BoxedTraceback::here(LineInfo lineInfo, Box** tb) {
 }
 
 void setupTraceback() {
-    traceback_cls = BoxedHeapClass::create(type_cls, object_cls, BoxedTraceback::gcHandler, 0, 0,
-                                           sizeof(BoxedTraceback), false, "traceback");
+    traceback_cls = BoxedClass::create(type_cls, object_cls, BoxedTraceback::gcHandler, 0, 0, sizeof(BoxedTraceback),
+                                       false, "traceback");
 
     traceback_cls->giveAttr("getLines", new BoxedFunction(boxRTFunction((void*)BoxedTraceback::getLines, UNKNOWN, 1)));
 

src/runtime/tuple.cpp

@@ -518,7 +518,7 @@ static PyObject* tuplerichcompare(PyObject* v, PyObject* w, int op) noexcept {
     return PyObject_RichCompare(vt->elts[i], wt->elts[i], op);
 }
 
-static PyObject* tupleslice(PyTupleObject* a, Py_ssize_t ilow, Py_ssize_t ihigh) {
+static PyObject* tupleslice(PyTupleObject* a, Py_ssize_t ilow, Py_ssize_t ihigh) noexcept {
     PyTupleObject* np;
     PyObject** src, **dest;
     Py_ssize_t i;
@@ -556,10 +556,80 @@ static PyObject* tupleitem(register PyTupleObject* a, register Py_ssize_t i) {
     return a->ob_item[i];
 }
 
-static Py_ssize_t tuplelength(PyTupleObject* a) {
+static Py_ssize_t tuplelength(PyTupleObject* a) noexcept {
     return Py_SIZE(a);
 }
 
+static Box* tupleconcat(PyTupleObject* a, Box* bb) noexcept {
+    Py_ssize_t size;
+    Py_ssize_t i;
+    PyObject** src, **dest;
+    PyTupleObject* np;
+    if (!PyTuple_Check(bb)) {
+        PyErr_Format(PyExc_TypeError, "can only concatenate tuple (not \"%.200s\") to tuple", Py_TYPE(bb)->tp_name);
+        return NULL;
+    }
+#define b ((PyTupleObject*)bb)
+    size = Py_SIZE(a) + Py_SIZE(b);
+    if (size < 0)
+        return PyErr_NoMemory();
+    np = (PyTupleObject*)PyTuple_New(size);
+    if (np == NULL) {
+        return NULL;
+    }
+    src = a->ob_item;
+    dest = np->ob_item;
+    for (i = 0; i < Py_SIZE(a); i++) {
+        PyObject* v = src[i];
+        Py_INCREF(v);
+        dest[i] = v;
+    }
+    src = b->ob_item;
+    dest = np->ob_item + Py_SIZE(a);
+    for (i = 0; i < Py_SIZE(b); i++) {
+        PyObject* v = src[i];
+        Py_INCREF(v);
+        dest[i] = v;
+    }
+    return (PyObject*)np;
+#undef b
+}
+
+static PyObject* tuplerepeat(PyTupleObject* a, Py_ssize_t n) noexcept {
+    Py_ssize_t i, j;
+    Py_ssize_t size;
+    PyTupleObject* np;
+    PyObject** p, **items;
+    if (n < 0)
+        n = 0;
+    if (Py_SIZE(a) == 0 || n == 1) {
+        if (PyTuple_CheckExact((BoxedTuple*)a)) {
+            /* Since tuples are immutable, we can return a shared
+             *                copy in this case */
+            Py_INCREF(a);
+            return (PyObject*)a;
+        }
+        if (Py_SIZE(a) == 0)
+            return PyTuple_New(0);
+    }
+    size = Py_SIZE(a) * n;
+    if (size / Py_SIZE(a) != n)
+        return PyErr_NoMemory();
+    np = (PyTupleObject*)PyTuple_New(size);
+    if (np == NULL)
+        return NULL;
+    p = np->ob_item;
+    items = a->ob_item;
+    for (i = 0; i < n; i++) {
+        for (j = 0; j < Py_SIZE(a); j++) {
+            *p = items[j];
+            Py_INCREF(*p);
+            p++;
+        }
+    }
+    return (PyObject*)np;
+}
+
 void BoxedTuple::gcHandler(GCVisitor* v, Box* b) {
     Box::gcHandler(v, b);
 
@@ -573,9 +643,20 @@ extern "C" void BoxedTupleIterator::gcHandler(GCVisitor* v, Box* b) {
     v->visit(&it->t);
 }
 
+static Box* tuple_getnewargs(Box* _self) noexcept {
+    RELEASE_ASSERT(PyTuple_Check(_self), "");
+    PyTupleObject* v = reinterpret_cast<PyTupleObject*>(_self);
+    return Py_BuildValue("(N)", tupleslice(v, 0, Py_SIZE(v)));
+}
+
 void setupTuple() {
-    tuple_iterator_cls = BoxedHeapClass::create(type_cls, object_cls, &BoxedTupleIterator::gcHandler, 0, 0,
-                                                sizeof(BoxedTupleIterator), false, "tuple");
+    static PySequenceMethods tuple_as_sequence;
+    tuple_cls->tp_as_sequence = &tuple_as_sequence;
+    static PyMappingMethods tuple_as_mapping;
+    tuple_cls->tp_as_mapping = &tuple_as_mapping;
+
+    tuple_iterator_cls = BoxedClass::create(type_cls, object_cls, &BoxedTupleIterator::gcHandler, 0, 0,
+                                            sizeof(BoxedTupleIterator), false, "tuple");
 
     tuple_cls->giveAttr("__new__", new BoxedFunction(boxRTFunction((void*)tupleNew, UNKNOWN, 1, true, true)));
     CLFunction* getitem = createRTFunction(2, 0, 0);
@@ -606,6 +687,9 @@ void setupTuple() {
     tuple_cls->giveAttr("__mul__", new BoxedFunction(boxRTFunction((void*)tupleMul, BOXED_TUPLE, 2)));
     tuple_cls->giveAttr("__rmul__", new BoxedFunction(boxRTFunction((void*)tupleMul, BOXED_TUPLE, 2)));
 
+    tuple_cls->giveAttr("__getnewargs__", new BoxedFunction(boxRTFunction((void*)tuple_getnewargs, UNKNOWN, 1,
+                                                                          ParamNames::empty(), CAPI)));
+
     tuple_cls->tp_hash = (hashfunc)tuple_hash;
     tuple_cls->tp_as_sequence->sq_slice = (ssizessizeargfunc)&tupleslice;
     add_operators(tuple_cls);
@@ -614,6 +698,8 @@ void setupTuple() {
 
     tuple_cls->tp_as_sequence->sq_item = (ssizeargfunc)tupleitem;
     tuple_cls->tp_as_sequence->sq_length = (lenfunc)tuplelength;
+    tuple_cls->tp_as_sequence->sq_concat = (binaryfunc)tupleconcat;
+    tuple_cls->tp_as_sequence->sq_repeat = (ssizeargfunc)tuplerepeat;
     tuple_cls->tp_iter = tupleIter;
 
     CLFunction* hasnext = boxRTFunction((void*)tupleiterHasnextUnboxed, BOOL, 1);

src/runtime/types.h

@@ -268,15 +268,27 @@ public:
 
     void freeze();
 
+    static void gcHandler(GCVisitor* v, Box* b);
+
+    typedef size_t SlotOffset;
+    SlotOffset* slotOffsets() { return (BoxedClass::SlotOffset*)((char*)this + this->cls->tp_basicsize); }
+
+    // These should only be used for builtin types:
+    static BoxedClass* create(BoxedClass* metatype, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
+                              int weaklist_offset, int instance_size, bool is_user_defined, const char* name);
+
+    BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset, int weaklist_offset, int instance_size,
+               bool is_user_defined, const char* name);
+
+
+    DEFAULT_CLASS_VAR(type_cls, sizeof(SlotOffset));
+
 protected:
     // These functions are not meant for external callers and will mostly just be called
     // by BoxedHeapClass::create(), but setupRuntime() also needs to do some manual class
     // creation due to bootstrapping issues.
     void finishInitialization();
 
-    BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset, int weaklist_offset, int instance_size,
-               bool is_user_defined);
-
     friend void setupRuntime();
     friend void setupSysEnd();
     friend void setupThread();
@@ -292,18 +304,12 @@ public:
     BoxedString* ht_name;
     PyObject* ht_slots;
 
-    typedef size_t SlotOffset;
-    SlotOffset* slotOffsets() { return (BoxedHeapClass::SlotOffset*)((char*)this + this->cls->tp_basicsize); }
     size_t nslots() { return this->ob_size; }
 
     // These functions are the preferred way to construct new types:
     static BoxedHeapClass* create(BoxedClass* metatype, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
                                   int weaklist_offset, int instance_size, bool is_user_defined, BoxedString* name,
                                   BoxedTuple* bases, size_t nslots);
-    static BoxedHeapClass* create(BoxedClass* metatype, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
-                                  int weaklist_offset, int instance_size, bool is_user_defined, llvm::StringRef name);
-
-    static void gcHandler(GCVisitor* v, Box* b);
 
 private:
     // These functions are not meant for external callers and will mostly just be called
@@ -315,8 +321,6 @@ private:
     friend void setupRuntime();
     friend void setupSys();
     friend void setupThread();
-
-    DEFAULT_CLASS_VAR(type_cls, sizeof(SlotOffset));
 };
 
 static_assert(sizeof(pyston::Box) == sizeof(struct _object), "");

src/runtime/util.cpp

@@ -17,6 +17,7 @@
 #include "codegen/codegen.h"
 #include "core/options.h"
 #include "core/types.h"
+#include "runtime/long.h"
 #include "runtime/objmodel.h"
 
 // Temp hack to get CType sort of importing
@@ -238,6 +239,11 @@ extern "C" void dumpEx(void* p, int levels) {
             printf("Int value: %ld\n", static_cast<BoxedInt*>(b)->n);
         }
 
+        if (PyLong_Check(b)) {
+            PyObject* str = longRepr(static_cast<BoxedLong*>(b));
+            printf("Long value: %s\n", static_cast<BoxedString*>(str)->c_str());
+        }
+
         if (PyList_Check(b)) {
             auto l = static_cast<BoxedList*>(b);
             printf("%ld elements\n", l->size);

test/CPYTHON_TEST_NOTES.md

@@ -57,6 +57,7 @@ test_mutants            unknown failure
 test_optparse           assertion instead of exceptions for long("invalid number")
 test_pep277             segfaults
 test_pep352             various unique bugs
+test_pickle             unknown
 test_pkg                unknown bug
 test_random             long("invalid number")
 test_repr               complex.__hash__; some unknown issues

test/cpython/test_pickle.py

+../../from_cpython/Lib/test/test_pickle.py
\ No newline at end of file

test/test_extension/slots_test.c

@@ -806,8 +806,25 @@ call_funcs(PyObject* _module, PyObject* args) {
     Py_RETURN_NONE;
 }
 
+static PyObject *
+view_tp_as(PyObject* _module, PyObject* _type) {
+    assert(PyType_Check(_type));
+    PyTypeObject* type = (PyTypeObject*)_type;
+
+    printf("%s:", type->tp_name);
+    if (type->tp_as_number)
+        printf(" tp_as_number");
+    if (type->tp_as_sequence)
+        printf(" tp_as_sequence");
+    if (type->tp_as_mapping)
+        printf(" tp_as_mapping");
+    printf("\n");
+    Py_RETURN_NONE;
+}
+
 static PyMethodDef SlotsMethods[] = {
     {"call_funcs", call_funcs, METH_VARARGS, "Call slotted functions."},
+    {"view_tp_as", view_tp_as, METH_O, "Check which tp_as_ slots are defined."},
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 

test/tests/pickle_test.py

@@ -22,3 +22,23 @@ c = C()
 c.a = 1
 print repr(pickle.dumps(c))
 print pickle.loads(pickle.dumps(c)).a
+
+
+for obj in [(1, 2), "hello world", u"hola world", 1.0, 1j, 1L, 2, {1:2}, set([3]), frozenset([4])]:
+    print
+    print "Testing pickling subclasses of %s..." % type(obj)
+
+    class MySubclass(type(obj)):
+        pass
+
+    o = MySubclass(obj)
+    print repr(o), type(o)
+
+    for protocol in (0, 1, 2):
+        print "Protocol", protocol
+
+        s = pickle.dumps(o, protocol=protocol)
+        print repr(s)
+
+        o2 = pickle.loads(s)
+        print repr(o2), type(o2)

test/tests/type_flags.py

+import slots_test
+
+HEAPTYPE = 1<<9
+
+def test(cls):
+    print cls
+    slots_test.view_tp_as(cls)
+    if HEAPTYPE & cls.__flags__:
+        print "heaptype"
+
+def testall(module):
+    for n in sorted(dir((module))):
+        if n in ("reversed", "AttrwrapperType", "BuiltinMethodType", "BufferType", "DictProxyType"):
+            continue
+
+        cls = getattr(module, n)
+        if not isinstance(cls, type):
+            continue
+        print n
+        test(cls)
+
+testall(__builtins__)
+import types
+testall(types)
+
+
+class C(object):
+    pass
+test(C)