Merge pull request #458 from kmod/metaserver_misc

Metaserver misc

.gitignore

@@ -47,6 +47,8 @@ gmon.out
 find_problem.status
 *.expected_cache
 
+plugins/clang_capi
+
 *.so
 *.pch
 

CMakeLists.txt

@@ -206,7 +206,7 @@ add_test(NAME gc_unittest COMMAND gc_unittest)
 add_test(NAME analysis_unittest COMMAND analysis_unittest)
 add_test(NAME pyston_defaults COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
 # we pass -I to cpython tests and skip failing ones b/c they are slooow otherwise
-add_test(NAME pyston_defaults_cpython_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -a=-I -k --exit-code-only --skip-failing ${CMAKE_SOURCE_DIR}/test/cpython)
+add_test(NAME pyston_defaults_cpython_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing ${CMAKE_SOURCE_DIR}/test/cpython)
 add_test(NAME pyston_defaults_integration_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t60 ${CMAKE_SOURCE_DIR}/test/integration)
 add_test(NAME pyston_max_compilation_tier COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-O -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
 add_test(NAME pyston_old_parser COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -a=-x -R ./pyston -j1 -a=-n -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)

Makefile

@@ -493,7 +493,7 @@ check:
 
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests & skip failing ones because they are sloooow otherwise
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S -a=-I --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t60 $(TEST_DIR)/integration $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-n -a=-x -a=-S $(TESTS_DIR) $(ARGS)
 	@# skip -O for dbg
@@ -510,7 +510,7 @@ check:
 	@# since we can make different decisions about which internal functions to inline or not.
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-S $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-S -a=-I --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t60 $(TEST_DIR)/integration $(ARGS)
 	@# skip -n for dbg
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-O -a=-x -a=-S $(TESTS_DIR) $(ARGS)
@@ -955,7 +955,7 @@ $(eval \
 check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1 ext_pyston
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -a=-I -k --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t=60 $(TEST_DIR)/integration $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -a=-S -k $(TESTS_DIR) $(ARGS)
@@ -1182,6 +1182,27 @@ $(FROM_CPYTHON_SRCS:.c=.prof.o): %.prof.o: %.c $(BUILD_SYSTEM_DEPS)
 	$(ECHO) Compiling C file to $@
 	$(VERB) $(CC_PROFILE) $(EXT_CFLAGS_PROFILE) -c $< -o $@ -g -MMD -MP -MF $(patsubst %.o,%.d,$@)
 
-# These are necessary until we support unicode:
-../from_cpython/Modules/_sre.o: EXT_CFLAGS += -Wno-sometimes-uninitialized
-../from_cpython/Modules/_sre.release.o: EXT_CFLAGS += -Wno-sometimes-uninitialized
+
+
+
+# TESTING:
+
+_plugins/clang_capi.so: plugins/clang_capi.cpp $(BUILD_SYSTEM_DEPS)
+	@# $(CXX) $< -o $@ -c -I/usr/lib/llvm-3.5/include -std=c++11 -D__STDC_CONSTANT_MACROS -D__STDC_LIMIT_MACROS
+	$(CXX) $< -o $@ -std=c++11 $(LLVM_CXXFLAGS) -I$(LLVM_SRC)/tools/clang/include -I$(LLVM_BUILD)/tools/clang/include -shared
+
+plugins/clang_capi.o: plugins/clang_capi.cpp $(BUILD_SYSTEM_DEPS)
+	@# $(CXX) $< -o $@ -c -I/usr/lib/llvm-3.5/include -std=c++11 -D__STDC_CONSTANT_MACROS -D__STDC_LIMIT_MACROS
+	$(CXX) $< -o $@ -std=c++11 $(LLVM_CXXFLAGS) -O0 -I$(LLVM_SRC)/tools/clang/include -I$(LLVM_BUILD)/tools/clang/include -c
+plugins/clang_capi: plugins/clang_capi.o $(BUILD_SYSTEM_DEPS)
+	@# $(CXX) $< -o $@ -c -I/usr/lib/llvm-3.5/include -std=c++11 -D__STDC_CONSTANT_MACROS -D__STDC_LIMIT_MACROS
+	$(CXX) $< -o $@ -L$(LLVM_BUILD)/Release/lib -lclangASTMatchers -lclangRewrite -lclangFrontend -lclangDriver -lclangTooling -lclangParse -lclangSema -lclangAnalysis -lclangAST -lclangEdit -lclangLex -lclangBasic -lclangSerialization $(shell $(LLVM_BUILD)/Release+Asserts/bin/llvm-config --ldflags --system-libs --libs all)
+plugins/clang_capi.so: plugins/clang_capi.o $(BUILD_SYSTEM_DEPS)
+	@# $(CXX) $< -o $@ -c -I/usr/lib/llvm-3.5/include -std=c++11 -D__STDC_CONSTANT_MACROS -D__STDC_LIMIT_MACROS
+	$(CXX) $< -o $@ -shared
+.PHONY: plugin_test
+plugin_test: plugins/clang_capi.so
+	$(CLANG_CXX) -Xclang -load -Xclang plugins/clang_capi.so -Xclang -add-plugin -Xclang print-fns test/test.cpp -c -S -o -
+.PHONY: tool_test
+tool_test: plugins/clang_capi
+	plugins/clang_capi test/test.cpp --

from_cpython/Include/methodobject.h

@@ -14,6 +14,8 @@ extern "C" {
 
 // Pyston change: this is no longer a static object
 //PyAPI_DATA(PyTypeObject) PyCFunction_Type;
+PyAPI_DATA(PyTypeObject*) builtin_function_or_method_cls;
+#define PyCFunction_Type (*builtin_function_or_method_cls)
 
 #define PyCFunction_Check(op) (Py_TYPE(op) == &PyCFunction_Type)
 

from_cpython/Include/object.h

@@ -117,7 +117,7 @@ struct _varobject {
 };
 
 // Pyston change: hacks to allow C++ features
-#ifndef __cplusplus
+#ifndef _PYSTON_API
 typedef struct _object PyObject;
 typedef struct _varobject PyVarObject;
 #define Py_TYPE(ob)             (((PyObject*)(ob))->ob_type)
@@ -323,7 +323,7 @@ typedef struct {
 
 
 // Pyston change: hacks to allow C++ features
-#ifndef __cplusplus
+#ifndef _PYSTON_API
 typedef struct _typeobject PyTypeObject;
 #else
 namespace pyston {
@@ -826,7 +826,7 @@ PyAPI_FUNC(void) _Py_AddToAllObjects(PyObject *, int force) PYSTON_NOEXCEPT;
 
 
 #define Py_INCREF(op) ((void)(op))
-#define Py_DECREF(op) asm volatile("" : : "X"(op))
+#define Py_DECREF(op) __asm volatile("" : : "X"(op))
 
 /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear
  * and tp_dealloc implementatons.
@@ -874,7 +874,7 @@ PyAPI_FUNC(void) _Py_AddToAllObjects(PyObject *, int force) PYSTON_NOEXCEPT;
 /* Macros to use in case the object pointer may be NULL: */
 // Pyston change: made these noops as well
 #define Py_XINCREF(op) ((void)(op))
-#define Py_XDECREF(op) asm volatile("" : : "X"(op))
+#define Py_XDECREF(op) __asm volatile("" : : "X"(op))
 
 /*
 These are provided as conveniences to Python runtime embedders, so that

from_cpython/Include/pyport.h

@@ -7,10 +7,21 @@
 #include <stdint.h>
 
 #ifdef __cplusplus
+#if __cplusplus > 199711L
 #define PYSTON_NOEXCEPT noexcept
 #else
 #define PYSTON_NOEXCEPT
 #endif
+#else
+#define PYSTON_NOEXCEPT
+#endif
+
+// HACK: we should set this manually rather than cluing off of the C++ version
+#ifdef __cplusplus
+#if __cplusplus > 199711L
+#define _PYSTON_API
+#endif
+#endif
 
 #define Py_PROTO(x) x
 
@@ -374,6 +385,11 @@ typedef PY_LONG_LONG            Py_intptr_t;
 #endif /* !HAVE_SYS_TIME_H */
 #endif /* !TIME_WITH_SYS_TIME */
 
+#ifdef SIZE_MAX
+#define PY_SIZE_MAX SIZE_MAX
+#else
+#define PY_SIZE_MAX ((size_t)-1)
+#endif
 
 /* Py_ARITHMETIC_RIGHT_SHIFT
  * C doesn't define whether a right-shift of a signed integer sign-extends

from_cpython/Include/sliceobject.h

@@ -23,15 +23,12 @@ names are from range).  After much talk with Guido, it was decided to
 let these be any arbitrary python type.  Py_None stands for omitted values.
 */
 
-// Pyston change: comment this out since this is not the format we're using
-#if 0
+// Pyston note: this happens to be the same format we use (not a lot going on here),
+// and we assert so in runtime/types.h
 typedef struct {
     PyObject_HEAD
     PyObject *start, *stop, *step;	/* not NULL */
 } PySliceObject;
-#endif
-struct _PySliceObject;
-typedef struct _PySliceObject PySliceObject;
 
 // Pyston change: these are no longer static objects
 PyAPI_DATA(PyTypeObject*) slice_cls;

from_cpython/Include/traceback.h

@@ -29,7 +29,7 @@ PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, const char *, int, int) PYSTON
 /* Reveal traceback type so we can typecheck traceback objects */
 // Pyston change: not a static type any more
 PyAPI_DATA(PyTypeObject*) traceback_cls;
-#define PyTraceback_Type (*traceback_cls)
+#define PyTraceBack_Type (*traceback_cls)
 // PyAPI_DATA(PyTypeObject) PyTraceBack_Type;
 #define PyTraceBack_Check(v) (Py_TYPE(v) == &PyTraceBack_Type)
 

from_cpython/Lib/modulefinder.py

@@ -2,6 +2,12 @@
 # This module should be kept compatible with Python 2.2, see PEP 291.
 
 from __future__ import generators
+
+# Pyston change:
+import sys
+del sys.modules['modulefinder']
+raise ImportError("This isn't really supported in Pyston yet")
+
 import dis
 import imp
 import marshal

plugins/clang_capi.cpp

+//------------------------------------------------------------------------------
+// Tooling sample. Demonstrates:
+//
+// * How to write a simple source tool using libTooling.
+// * How to use RecursiveASTVisitor to find interesting AST nodes.
+// * How to use the Rewriter API to rewrite the source code.
+//
+// Eli Bendersky (eliben@gmail.com)
+// This code is in the public domain
+//------------------------------------------------------------------------------
+#include <sstream>
+#include <string>
+
+#include "clang/AST/AST.h"
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Frontend/ASTConsumers.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Tooling.h"
+#include "clang/Rewrite/Core/Rewriter.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+using namespace clang;
+using namespace clang::driver;
+using namespace clang::tooling;
+using namespace clang::ast_matchers;
+
+static llvm::cl::OptionCategory ToolingSampleCategory("Tooling Sample");
+
+//auto matcher = memberExpr().bind("member_expr");
+auto matcher = memberExpr(member(fieldDecl(hasName("ob_refcnt")).bind("field"))).bind("member_expr");
+
+MatchFinder Finder;
+Rewriter TheRewriter;
+
+class Replacer : public MatchFinder::MatchCallback {
+public:
+    virtual void run (const MatchFinder::MatchResult &Result) {
+        errs() << "matched!\n";
+        if (const MemberExpr* ME = Result.Nodes.getNodeAs<clang::MemberExpr>("member_expr")) {
+            ME->dump();
+            //cast<FieldDecl>(ME->getMemberDecl())->dump();
+            TheRewriter.InsertTextBefore(ME->getSourceRange().getBegin(), "/* Pyston change, was:  ");
+            TheRewriter.InsertTextAfter(ME->getSourceRange().getEnd(), "*/ 2");
+            for (auto c : ME->Stmt::children()) {
+                c->dump();
+            }
+        }
+        if (const FieldDecl* ME = Result.Nodes.getNodeAs<clang::FieldDecl>("field")) {
+            ME->dump();
+            TheRewriter.InsertTextAfter(ME->getLocStart(), "/**/");
+        }
+        //Result.dump();
+    }
+};
+
+class MyFrontendAction : public ASTFrontendAction {
+public:
+    MyFrontendAction() {}
+
+    void EndSourceFileAction() override {
+        SourceManager &SM = TheRewriter.getSourceMgr();
+        // Now emit the rewritten buffer.
+        TheRewriter.getEditBuffer(SM.getMainFileID()).write(llvm::outs());
+    }
+
+    std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance& CI, StringRef file) override {
+        TheRewriter.setSourceMgr(CI.getSourceManager(), CI.getLangOpts());
+        return Finder.newASTConsumer();
+    }
+};
+
+int main(int argc, const char **argv) {
+  CommonOptionsParser op(argc, argv, ToolingSampleCategory);
+  ClangTool Tool(op.getCompilations(), op.getSourcePathList());
+
+  Replacer replacer;
+  Finder.addMatcher(matcher, &replacer);
+
+  return Tool.run(newFrontendActionFactory<MyFrontendAction>().get());
+}

src/capi/abstract.cpp

@@ -28,8 +28,41 @@
 namespace pyston {
 
 extern "C" Py_ssize_t _PyObject_LengthHint(PyObject* o, Py_ssize_t defaultvalue) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return -1;
+    static PyObject* hintstrobj = NULL;
+    PyObject* ro, *hintmeth;
+    Py_ssize_t rv;
+
+    /* try o.__len__() */
+    rv = PyObject_Size(o);
+    if (rv >= 0)
+        return rv;
+    if (PyErr_Occurred()) {
+        if (!PyErr_ExceptionMatches(PyExc_TypeError) && !PyErr_ExceptionMatches(PyExc_AttributeError))
+            return -1;
+        PyErr_Clear();
+    }
+
+    if (PyInstance_Check(o))
+        return defaultvalue;
+    /* try o.__length_hint__() */
+    hintmeth = _PyObject_LookupSpecial(o, "__length_hint__", &hintstrobj);
+    if (hintmeth == NULL) {
+        if (PyErr_Occurred())
+            return -1;
+        else
+            return defaultvalue;
+    }
+    ro = PyObject_CallFunctionObjArgs(hintmeth, NULL);
+    Py_DECREF(hintmeth);
+    if (ro == NULL) {
+        if (!PyErr_ExceptionMatches(PyExc_TypeError) && !PyErr_ExceptionMatches(PyExc_AttributeError))
+            return -1;
+        PyErr_Clear();
+        return defaultvalue;
+    }
+    rv = PyNumber_Check(ro) ? PyInt_AsSsize_t(ro) : defaultvalue;
+    Py_DECREF(ro);
+    return rv;
 }
 
 static int _IsFortranContiguous(Py_buffer* view) {
@@ -1126,6 +1159,85 @@ extern "C" int PySequence_Contains(PyObject* seq, PyObject* ob) noexcept {
     return Py_SAFE_DOWNCAST(result, Py_ssize_t, int);
 }
 
+extern "C" PyObject* PySequence_Tuple(PyObject* v) noexcept {
+    PyObject* it; /* iter(v) */
+    Py_ssize_t n; /* guess for result tuple size */
+    PyObject* result = NULL;
+    Py_ssize_t j;
+
+    if (v == NULL)
+        return null_error();
+
+    /* Special-case the common tuple and list cases, for efficiency. */
+    if (PyTuple_CheckExact(v)) {
+        /* Note that we can't know whether it's safe to return
+           a tuple *subclass* instance as-is, hence the restriction
+           to exact tuples here.  In contrast, lists always make
+           a copy, so there's no need for exactness below. */
+        Py_INCREF(v);
+        return v;
+    }
+    if (PyList_Check(v))
+        return PyList_AsTuple(v);
+
+    /* Get iterator. */
+    it = PyObject_GetIter(v);
+    if (it == NULL)
+        return NULL;
+
+    /* Guess result size and allocate space. */
+    n = _PyObject_LengthHint(v, 10);
+    if (n == -1)
+        goto Fail;
+    result = PyTuple_New(n);
+    if (result == NULL)
+        goto Fail;
+
+    /* Fill the tuple. */
+    for (j = 0;; ++j) {
+        PyObject* item = PyIter_Next(it);
+        if (item == NULL) {
+            if (PyErr_Occurred())
+                goto Fail;
+            break;
+        }
+        if (j >= n) {
+            Py_ssize_t oldn = n;
+            /* The over-allocation strategy can grow a bit faster
+               than for lists because unlike lists the
+               over-allocation isn't permanent -- we reclaim
+               the excess before the end of this routine.
+               So, grow by ten and then add 25%.
+            */
+            n += 10;
+            n += n >> 2;
+            if (n < oldn) {
+                /* Check for overflow */
+                PyErr_NoMemory();
+                Py_DECREF(item);
+                goto Fail;
+            }
+            if (_PyTuple_Resize(&result, n) != 0) {
+                Py_DECREF(item);
+                goto Fail;
+            }
+        }
+        PyTuple_SET_ITEM(result, j, item);
+    }
+
+    /* Cut tuple back if guess was too large. */
+    if (j < n && _PyTuple_Resize(&result, j) != 0)
+        goto Fail;
+
+    Py_DECREF(it);
+    return result;
+
+Fail:
+    Py_XDECREF(result);
+    Py_DECREF(it);
+    return NULL;
+}
+
 extern "C" PyObject* PyObject_CallFunction(PyObject* callable, const char* format, ...) noexcept {
     va_list va;
     PyObject* args;
@@ -1318,14 +1430,20 @@ extern "C" PyObject* PyNumber_And(PyObject* lhs, PyObject* rhs) noexcept {
     }
 }
 
-extern "C" PyObject* PyNumber_Xor(PyObject*, PyObject*) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return nullptr;
+extern "C" PyObject* PyNumber_Xor(PyObject* lhs, PyObject* rhs) noexcept {
+    try {
+        return binop(lhs, rhs, AST_TYPE::BitXor);
+    } catch (ExcInfo e) {
+        Py_FatalError("unimplemented");
+    }
 }
 
-extern "C" PyObject* PyNumber_Or(PyObject*, PyObject*) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return nullptr;
+extern "C" PyObject* PyNumber_Or(PyObject* lhs, PyObject* rhs) noexcept {
+    try {
+        return binop(lhs, rhs, AST_TYPE::BitOr);
+    } catch (ExcInfo e) {
+        Py_FatalError("unimplemented");
+    }
 }
 
 extern "C" PyObject* PyNumber_InPlaceAdd(PyObject*, PyObject*) noexcept {

src/capi/modsupport.cpp

@@ -381,8 +381,29 @@ extern "C" PyObject* Py_BuildValue(const char* fmt, ...) noexcept {
     return r;
 }
 
+extern "C" {
+char* _Py_PackageContext = NULL;
+}
 extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, const char* doc, PyObject* self,
                                     int apiver) noexcept {
+    // Comment from CPython:
+    /* Make sure name is fully qualified.
+
+       This is a bit of a hack: when the shared library is loaded,
+       the module name is "package.module", but the module calls
+       Py_InitModule*() with just "module" for the name.  The shared
+       library loader squirrels away the true name of the module in
+       _Py_PackageContext, and Py_InitModule*() will substitute this
+       (if the name actually matches).
+    */
+    if (_Py_PackageContext != NULL) {
+        const char* p = strrchr(_Py_PackageContext, '.');
+        if (p != NULL && strcmp(name, p + 1) == 0) {
+            name = _Py_PackageContext;
+            _Py_PackageContext = NULL;
+        }
+    }
+
     BoxedModule* module = createModule(name, "__builtin__", doc);
 
     // Pass self as is, even if NULL we are not allowed to change it to None

src/codegen/ast_interpreter.cpp

@@ -64,7 +64,7 @@ union Value {
     Value(double d) : d(d) {}
     Value(Box* o) : o(o) {
         if (DEBUG >= 2)
-            assert(gc::isValidGCObject(o));
+            ASSERT(gc::isValidGCObject(o), "%p", o);
     }
 };
 
@@ -532,8 +532,15 @@ Value ASTInterpreter::visit_jump(AST_Jump* node) {
 
             CompiledFunction* partial_func = compilePartialFuncInternal(&exit);
             auto arg_tuple = getTupleFromArgsArray(&arg_array[0], arg_array.size());
-            return partial_func->call(std::get<0>(arg_tuple), std::get<1>(arg_tuple), std::get<2>(arg_tuple),
-                                      std::get<3>(arg_tuple));
+            Box* r = partial_func->call(std::get<0>(arg_tuple), std::get<1>(arg_tuple), std::get<2>(arg_tuple),
+                                        std::get<3>(arg_tuple));
+
+            // This is one of the few times that we are allowed to have an invalid value in a Box* Value.
+            // Check for it, and return as an int so that we don't trigger a potential assert when
+            // creating the Value.
+            if (compiled_func->getReturnType() != VOID)
+                assert(r);
+            return (intptr_t)r;
         }
     }
 
@@ -859,13 +866,18 @@ Value ASTInterpreter::visit_delete(AST_Delete* node) {
                     continue;
                 } else if (vst == ScopeInfo::VarScopeType::NAME) {
                     assert(frame_info.boxedLocals != NULL);
-                    assert(frame_info.boxedLocals->cls == dict_cls);
-                    auto& d = static_cast<BoxedDict*>(frame_info.boxedLocals)->d;
-                    auto it = d.find(boxString(target->id.str()));
-                    if (it == d.end()) {
-                        assertNameDefined(0, target->id.c_str(), NameError, false /* local_var_msg */);
+                    if (frame_info.boxedLocals->cls == dict_cls) {
+                        auto& d = static_cast<BoxedDict*>(frame_info.boxedLocals)->d;
+                        auto it = d.find(boxString(target->id.str()));
+                        if (it == d.end()) {
+                            assertNameDefined(0, target->id.c_str(), NameError, false /* local_var_msg */);
+                        }
+                        d.erase(it);
+                    } else if (frame_info.boxedLocals->cls == attrwrapper_cls) {
+                        attrwrapperDel(frame_info.boxedLocals, target->id.str());
+                    } else {
+                        RELEASE_ASSERT(0, "%s", frame_info.boxedLocals->cls->tp_name);
                     }
-                    d.erase(it);
                 } else {
                     assert(vst == ScopeInfo::VarScopeType::FAST);
 

src/codegen/irgen/hooks.cpp

@@ -382,7 +382,17 @@ Box* eval(Box* boxedCode) {
     // TODO this memory leaks
     RELEASE_ASSERT(boxedCode->cls == str_cls, "%s", boxedCode->cls->tp_name);
     const char* code = static_cast<BoxedString*>(boxedCode)->s.data();
+
+    // Hack: we need to support things like `eval(" 2")`.
+    // This is over-accepting since it will accept things like `eval("\n 2")`
+    while (*code == ' ' || *code == '\t' || *code == '\n' || *code == '\r')
+        code++;
+
     AST_Module* parsedModule = parse_string(code);
+    if (parsedModule->body.size() == 0)
+        raiseSyntaxError("unexpected EOF while parsing", 0, 0, "<string>", "");
+
+    RELEASE_ASSERT(parsedModule->body.size() == 1, "");
     RELEASE_ASSERT(parsedModule->body[0]->type == AST_TYPE::Expr, "");
     AST_Expression* parsedExpr = new AST_Expression(std::move(parsedModule->interned_strings));
     parsedExpr->body = static_cast<AST_Expr*>(parsedModule->body[0])->value;

src/codegen/parser.cpp

@@ -985,7 +985,9 @@ AST_Module* parse_file(const char* fn) {
     Timer _t("parsing");
 
     if (ENABLE_PYPA_PARSER) {
-        return pypa_parse(fn);
+        AST_Module* rtn = pypa_parse(fn);
+        assert(rtn);
+        return rtn;
     }
 
     FILE* fp = popen(getParserCommandLine(fn).c_str(), "r");

src/codegen/unwinding.cpp

@@ -699,7 +699,7 @@ FrameStackState getFrameStackState() {
 
                         Box* v = e.type->deserializeFromFrame(vals);
                         // printf("%s: (pp id %ld) %p\n", p.first.c_str(), e._debug_pp_id, v);
-                        assert(gc::isValidGCObject(v));
+                        ASSERT(gc::isValidGCObject(v), "%p", v);
                         d->d[boxString(p.first)] = v;
                     }
                 }

src/core/types.h

@@ -591,4 +591,9 @@ struct CallattrFlags {
 };
 }
 
+namespace std {
+template <> std::pair<pyston::Box**, std::ptrdiff_t> get_temporary_buffer<pyston::Box*>(std::ptrdiff_t count) noexcept;
+template <> void return_temporary_buffer<pyston::Box*>(pyston::Box** p);
+}
+
 #endif

src/gc/heap.cpp

@@ -30,6 +30,16 @@
 //#undef VERBOSITY
 //#define VERBOSITY(x) 2
 
+namespace std {
+template <> std::pair<pyston::Box**, std::ptrdiff_t> get_temporary_buffer<pyston::Box*>(std::ptrdiff_t count) noexcept {
+    void* r = pyston::gc::gc_alloc(sizeof(pyston::Box*) * count, pyston::gc::GCKind::CONSERVATIVE);
+    return std::make_pair((pyston::Box**)r, count);
+}
+template <> void return_temporary_buffer<pyston::Box*>(pyston::Box** p) {
+    pyston::gc::gc_free(p);
+}
+}
+
 namespace pyston {
 namespace gc {
 

src/jit.cpp

@@ -56,15 +56,31 @@ extern void setEncodingAndErrors();
 // return code in `*retcode`. does not touch `*retcode* if it returns false.
 static bool handle_toplevel_exn(const ExcInfo& e, int* retcode) {
     if (e.matches(SystemExit)) {
-        Box* code = e.value->getattr("code");
+        Box* value = e.value;
 
-        if (!code || code == None)
+        if (value && PyExceptionInstance_Check(value)) {
+            Box* code = getattr(value, "code");
+            if (code)
+                value = code;
+        }
+
+        if (!value || value == None)
             *retcode = 0;
-        else if (isSubclass(code->cls, int_cls))
-            *retcode = static_cast<BoxedInt*>(code)->n;
-        else
+        else if (isSubclass(value->cls, int_cls))
+            *retcode = static_cast<BoxedInt*>(value)->n;
+        else {
             *retcode = 1;
 
+            PyObject* sys_stderr = PySys_GetObject("stderr");
+            if (sys_stderr != NULL && sys_stderr != Py_None) {
+                PyFile_WriteObject(value, sys_stderr, Py_PRINT_RAW);
+            } else {
+                PyObject_Print(value, stderr, Py_PRINT_RAW);
+                fflush(stderr);
+            }
+            PySys_WriteStderr("\n");
+        }
+
         return true;
     }
     e.printExcAndTraceback();
@@ -82,7 +98,10 @@ static int main(int argc, char** argv) {
     bool stats = false;
     bool unbuffered = false;
     const char* command = NULL;
-    while ((code = getopt(argc, argv, "+OqdIibpjtrsSvnxc:FuP")) != -1) {
+
+    // Suppress getopt errors so we can throw them ourselves
+    opterr = 0;
+    while ((code = getopt(argc, argv, "+:OqdIibpjtrsSvnxEc:FuP")) != -1) {
         if (code == 'O')
             FORCE_OPTIMIZE = true;
         else if (code == 't')
@@ -115,16 +134,31 @@ static int main(int argc, char** argv) {
             USE_REGALLOC_BASIC = false;
         } else if (code == 'x') {
             ENABLE_PYPA_PARSER = false;
+        } else if (code == 'E') {
+            Py_IgnoreEnvironmentFlag = 1;
         } else if (code == 'P') {
             PAUSE_AT_ABORT = true;
         } else if (code == 'F') {
             CONTINUE_AFTER_FATAL = true;
         } else if (code == 'c') {
+            assert(optarg);
             command = optarg;
             // no more option parsing; the rest of our arguments go into sys.argv.
             break;
-        } else
+        } else {
+            if (code == ':') {
+                fprintf(stderr, "Argument expected for the -%c option\n", optopt);
+                return 2;
+            }
+
+            if (code == '?') {
+                fprintf(stderr, "Unknown option: -%c\n", optopt);
+                return 2;
+            }
+
+            fprintf(stderr, "Unknown getopt() error.  '%c' '%c'\n", code, optopt);
             abort();
+        }
     }
 
     const char* fn = NULL;

src/runtime/capi.cpp

@@ -258,11 +258,16 @@ extern "C" PyObject* PyObject_GetAttr(PyObject* o, PyObject* attr_name) noexcept
 }
 
 extern "C" PyObject* PyObject_GenericGetAttr(PyObject* o, PyObject* name) noexcept {
-    Box* r = getattrInternalGeneric(o, static_cast<BoxedString*>(name)->data(), NULL, false, false, NULL, NULL);
-    if (!r)
-        PyErr_Format(PyExc_AttributeError, "'%.50s' object has no attribute '%.400s'", o->cls->tp_name,
-                     PyString_AS_STRING(name));
-    return r;
+    try {
+        Box* r = getattrInternalGeneric(o, static_cast<BoxedString*>(name)->s.data(), NULL, false, false, NULL, NULL);
+        if (!r)
+            PyErr_Format(PyExc_AttributeError, "'%.50s' object has no attribute '%.400s'", o->cls->tp_name,
+                         PyString_AS_STRING(name));
+        return r;
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
 }
 
 extern "C" PyObject* _PyObject_GenericGetAttrWithDict(PyObject* obj, PyObject* name, PyObject* dict) noexcept {
@@ -399,7 +404,6 @@ extern "C" PyObject* PyObject_Call(PyObject* callable_object, PyObject* args, Py
 
 extern "C" int PyObject_GetBuffer(PyObject* obj, Py_buffer* view, int flags) noexcept {
     if (!PyObject_CheckBuffer(obj)) {
-        printf("%s\n", obj->cls->tp_name);
         PyErr_Format(PyExc_TypeError, "'%100s' does not have the buffer interface", Py_TYPE(obj)->tp_name);
         return -1;
     }
@@ -520,16 +524,6 @@ extern "C" Py_ssize_t PySequence_Index(PyObject* o, PyObject* value) noexcept {
     return -1;
 }
 
-extern "C" PyObject* PySequence_Tuple(PyObject* o) noexcept {
-    if (o->cls == tuple_cls)
-        return o;
-    if (PyList_Check(o))
-        return PyList_AsTuple(o);
-
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return nullptr;
-}
-
 extern "C" PyObject* PyIter_Next(PyObject* iter) noexcept {
     try {
         Box* hasnext = iter->hasnextOrNullIC();
@@ -1239,6 +1233,14 @@ extern "C" PyObject* PyCFunction_NewEx(PyMethodDef* ml, PyObject* self, PyObject
     return new BoxedCApiFunction(ml->ml_flags, self, ml->ml_name, ml->ml_meth);
 }
 
+extern "C" PyCFunction PyCFunction_GetFunction(PyObject* op) noexcept {
+    if (!PyCFunction_Check(op)) {
+        PyErr_BadInternalCall();
+        return NULL;
+    }
+    return ((PyCFunctionObject*)op)->m_ml->ml_meth;
+}
+
 extern "C" int _PyEval_SliceIndex(PyObject* v, Py_ssize_t* pi) noexcept {
     if (v != NULL) {
         Py_ssize_t x;
@@ -1278,6 +1280,7 @@ extern "C" char* PyModule_GetName(PyObject* m) noexcept {
 BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path) {
     void* handle = dlopen(path.c_str(), RTLD_NOW);
     if (!handle) {
+        // raiseExcHelper(ImportError, "%s", dlerror());
         fprintf(stderr, "%s\n", dlerror());
         exit(1);
     }
@@ -1288,17 +1291,26 @@ BoxedModule* importCExtension(const std::string& full_name, const std::string& l
 
     char* error;
     if ((error = dlerror()) != NULL) {
+        // raiseExcHelper(ImportError, "%s", error);
         fprintf(stderr, "%s\n", error);
         exit(1);
     }
 
     assert(init);
+
+    char* packagecontext = strdup(full_name.c_str());
+    char* oldcontext = _Py_PackageContext;
+    _Py_PackageContext = packagecontext;
     (*init)();
+    _Py_PackageContext = oldcontext;
+    free(packagecontext);
+
+    checkAndThrowCAPIException();
 
     BoxedDict* sys_modules = getSysModulesDict();
     Box* s = boxStrConstant(full_name.c_str());
     Box* _m = sys_modules->d[s];
-    RELEASE_ASSERT(_m, "module failed to initialize properly?");
+    RELEASE_ASSERT(_m, "dynamic module not initialized properly");
     assert(_m->cls == module_cls);
 
     BoxedModule* m = static_cast<BoxedModule*>(_m);

src/runtime/file.cpp

@@ -480,7 +480,15 @@ static PyObject* file_write(BoxedFile* f, Box* arg) noexcept {
     if (!f->writable)
         return err_mode("writing");
     if (f->f_binary) {
-        if (PyObject_GetBuffer(arg, &pbuf, 0))
+        // In CPython, this branch calls PyArg_ParseTuple for all types, but we never created
+        // the "args" tuple so we have to do some of the work that ParseTuple does.
+        // Mostly it's easy since we've already unpacked the args, but there is some unicode-specific
+        // code in it that is better not to duplicate.
+        // So, if it's unicode, just make the tuple for now and send it through PyArg_ParseTuple.
+        if (PyUnicode_Check(arg)) {
+            if (!PyArg_ParseTuple(BoxedTuple::create({ arg }), "s*", &pbuf))
+                return NULL;
+        } else if (PyObject_GetBuffer(arg, &pbuf, 0))
             return NULL;
 
         s = (const char*)pbuf.buf;
@@ -537,6 +545,124 @@ static PyObject* file_write(BoxedFile* f, Box* arg) noexcept {
     return Py_None;
 }
 
+static PyObject* file_writelines(BoxedFile* f, PyObject* seq) noexcept {
+#define CHUNKSIZE 1000
+    PyObject* list, *line;
+    PyObject* it; /* iter(seq) */
+    PyObject* result;
+    int index, islist;
+    Py_ssize_t i, j, nwritten, len;
+
+    assert(seq != NULL);
+    if (f->f_fp == NULL)
+        return err_closed();
+    if (!f->writable)
+        return err_mode("writing");
+
+    result = NULL;
+    list = NULL;
+    islist = PyList_Check(seq);
+    if (islist)
+        it = NULL;
+    else {
+        it = PyObject_GetIter(seq);
+        if (it == NULL) {
+            PyErr_SetString(PyExc_TypeError, "writelines() requires an iterable argument");
+            return NULL;
+        }
+        /* From here on, fail by going to error, to reclaim "it". */
+        list = PyList_New(CHUNKSIZE);
+        if (list == NULL)
+            goto error;
+    }
+
+    /* Strategy: slurp CHUNKSIZE lines into a private list,
+       checking that they are all strings, then write that list
+       without holding the interpreter lock, then come back for more. */
+    for (index = 0;; index += CHUNKSIZE) {
+        if (islist) {
+            Py_XDECREF(list);
+            list = PyList_GetSlice(seq, index, index + CHUNKSIZE);
+            if (list == NULL)
+                goto error;
+            j = PyList_GET_SIZE(list);
+        } else {
+            for (j = 0; j < CHUNKSIZE; j++) {
+                line = PyIter_Next(it);
+                if (line == NULL) {
+                    if (PyErr_Occurred())
+                        goto error;
+                    break;
+                }
+                PyList_SetItem(list, j, line);
+            }
+            /* The iterator might have closed the file on us. */
+            if (f->f_fp == NULL) {
+                err_closed();
+                goto error;
+            }
+        }
+        if (j == 0)
+            break;
+
+        /* Check that all entries are indeed strings. If not,
+           apply the same rules as for file.write() and
+           convert the results to strings. This is slow, but
+           seems to be the only way since all conversion APIs
+           could potentially execute Python code. */
+        for (i = 0; i < j; i++) {
+            PyObject* v = PyList_GET_ITEM(list, i);
+            if (!PyString_Check(v)) {
+                const char* buffer;
+                int res;
+                if (f->f_binary) {
+                    res = PyObject_AsReadBuffer(v, (const void**)&buffer, &len);
+                } else {
+                    res = PyObject_AsCharBuffer(v, &buffer, &len);
+                }
+                if (res) {
+                    PyErr_SetString(PyExc_TypeError, "writelines() argument must be a sequence of strings");
+                    goto error;
+                }
+                line = PyString_FromStringAndSize(buffer, len);
+                if (line == NULL)
+                    goto error;
+                Py_DECREF(v);
+                PyList_SET_ITEM(list, i, line);
+            }
+        }
+
+        /* Since we are releasing the global lock, the
+           following code may *not* execute Python code. */
+        f->f_softspace = 0;
+        FILE_BEGIN_ALLOW_THREADS(f)
+        errno = 0;
+        for (i = 0; i < j; i++) {
+            line = PyList_GET_ITEM(list, i);
+            len = PyString_GET_SIZE(line);
+            nwritten = fwrite(PyString_AS_STRING(line), 1, len, f->f_fp);
+            if (nwritten != len) {
+                FILE_ABORT_ALLOW_THREADS(f)
+                PyErr_SetFromErrno(PyExc_IOError);
+                clearerr(f->f_fp);
+                goto error;
+            }
+        }
+        FILE_END_ALLOW_THREADS(f)
+
+        if (j < CHUNKSIZE)
+            break;
+    }
+
+    Py_INCREF(Py_None);
+    result = Py_None;
+error:
+    Py_XDECREF(list);
+    Py_XDECREF(it);
+    return result;
+#undef CHUNKSIZE
+}
+
 Box* fileWrite(BoxedFile* self, Box* val) {
     assert(self->cls == file_cls);
 
@@ -1469,6 +1595,7 @@ PyDoc_STRVAR(isatty_doc, "isatty() -> true or false.  True if the file is connec
 PyMethodDef file_methods[] = {
     { "seek", (PyCFunction)file_seek, METH_VARARGS, seek_doc },
     { "readlines", (PyCFunction)file_readlines, METH_VARARGS, readlines_doc },
+    { "writelines", (PyCFunction)file_writelines, METH_O, NULL },
     { "isatty", (PyCFunction)file_isatty, METH_NOARGS, isatty_doc },
 };
 

src/runtime/float.cpp

@@ -27,6 +27,10 @@ extern "C" PyObject* PyFloat_FromDouble(double d) noexcept {
     return boxFloat(d);
 }
 
+extern "C" PyObject* PyFloat_FromString(PyObject* v, char** pend) noexcept {
+    Py_FatalError("unimplemented");
+}
+
 extern "C" double PyFloat_AsDouble(PyObject* o) noexcept {
     if (o->cls == float_cls)
         return static_cast<BoxedFloat*>(o)->d;

src/runtime/import.cpp

@@ -274,7 +274,14 @@ static Box* getParent(Box* globals, int level, std::string& buf) {
         if (modname == NULL || modname->cls != str_cls)
             return None;
 
-        Box* modpath = getattrInternal(globals, path_str, NULL);
+        Box* modpath = NULL;
+        try {
+            modpath = getattrInternal(globals, path_str, NULL);
+        } catch (ExcInfo e) {
+            if (!e.matches(AttributeError))
+                raiseRaw(e);
+        }
+
         if (modpath != NULL) {
             /* __path__ is set, so modname is already the package name */
             if (modname->size() > PATH_MAX) {
@@ -516,7 +523,15 @@ extern "C" PyObject* PyImport_ImportModuleLevel(const char* name, PyObject* glob
 }
 
 static void ensureFromlist(Box* module, Box* fromlist, std::string& buf, bool recursive) {
-    if (getattrInternal(module, path_str, NULL) == NULL) {
+    Box* pathlist = NULL;
+    try {
+        pathlist = getattrInternal(module, path_str, NULL);
+    } catch (ExcInfo e) {
+        if (!e.matches(AttributeError))
+            raiseRaw(e);
+    }
+
+    if (pathlist == NULL) {
         // If it's not a package, then there's no sub-importing to do
         return;
     }

src/runtime/list.cpp

@@ -246,6 +246,138 @@ static void sliceIndex(Box* b, int64_t* out) {
     *out = static_cast<BoxedInt*>(b)->n;
 }
 
+// Copied from CPython's list_ass_subscript
+int list_ass_ext_slice(BoxedList* self, PyObject* item, PyObject* value) {
+    Py_ssize_t start, stop, step, slicelength;
+
+    if (PySlice_GetIndicesEx((PySliceObject*)item, Py_SIZE(self), &start, &stop, &step, &slicelength) < 0) {
+        return -1;
+    }
+
+    RELEASE_ASSERT(step != 1, "should have handled this elsewhere");
+    // if (step == 1)
+    // return list_ass_slice(self, start, stop, value);
+
+    /* Make sure s[5:2] = [..] inserts at the right place:
+       before 5, not before 2. */
+    if ((step < 0 && start < stop) || (step > 0 && start > stop))
+        stop = start;
+
+    if (value == NULL) {
+        /* delete slice */
+        PyObject** garbage;
+        size_t cur;
+        Py_ssize_t i;
+
+        if (slicelength <= 0)
+            return 0;
+
+        if (step < 0) {
+            stop = start + 1;
+            start = stop + step * (slicelength - 1) - 1;
+            step = -step;
+        }
+
+        assert((size_t)slicelength <= PY_SIZE_MAX / sizeof(PyObject*));
+
+        garbage = (PyObject**)PyMem_MALLOC(slicelength * sizeof(PyObject*));
+        if (!garbage) {
+            PyErr_NoMemory();
+            return -1;
+        }
+
+        /* drawing pictures might help understand these for
+           loops. Basically, we memmove the parts of the
+           list that are *not* part of the slice: step-1
+           items for each item that is part of the slice,
+           and then tail end of the list that was not
+           covered by the slice */
+        for (cur = start, i = 0; cur < (size_t)stop; cur += step, i++) {
+            Py_ssize_t lim = step - 1;
+
+            garbage[i] = PyList_GET_ITEM(self, cur);
+
+            if (cur + step >= self->size) {
+                lim = self->size - cur - 1;
+            }
+
+            memmove(self->elts->elts + cur - i, self->elts->elts + cur + 1, lim * sizeof(PyObject*));
+        }
+        cur = start + slicelength * step;
+        if (cur < self->size) {
+            memmove(self->elts->elts + cur - slicelength, self->elts->elts + cur,
+                    (self->size - cur) * sizeof(PyObject*));
+        }
+
+        self->size -= slicelength;
+
+        // list_resize(self, Py_SIZE(self));
+
+        for (i = 0; i < slicelength; i++) {
+            Py_DECREF(garbage[i]);
+        }
+        PyMem_FREE(garbage);
+
+        return 0;
+    } else {
+        /* assign slice */
+        PyObject* ins, *seq;
+        PyObject** garbage, **seqitems, **selfitems;
+        Py_ssize_t cur, i;
+
+        /* protect against a[::-1] = a */
+        if (self == value) {
+            abort();
+            // seq = list_slice((PyListObject*)value, 0,
+            // PyList_GET_SIZE(value));
+        } else {
+            seq = PySequence_Fast(value, "must assign iterable "
+                                         "to extended slice");
+        }
+        if (!seq)
+            return -1;
+
+        if (PySequence_Fast_GET_SIZE(seq) != slicelength) {
+            PyErr_Format(PyExc_ValueError, "attempt to assign sequence of "
+                                           "size %zd to extended slice of "
+                                           "size %zd",
+                         PySequence_Fast_GET_SIZE(seq), slicelength);
+            Py_DECREF(seq);
+            return -1;
+        }
+
+        if (!slicelength) {
+            Py_DECREF(seq);
+            return 0;
+        }
+
+        garbage = (PyObject**)PyMem_MALLOC(slicelength * sizeof(PyObject*));
+        if (!garbage) {
+            Py_DECREF(seq);
+            PyErr_NoMemory();
+            return -1;
+        }
+
+        selfitems = self->elts->elts;
+        seqitems = PySequence_Fast_ITEMS(seq);
+        for (cur = start, i = 0; i < slicelength; cur += step, i++) {
+            garbage[i] = selfitems[cur];
+            ins = seqitems[i];
+            Py_INCREF(ins);
+            selfitems[cur] = ins;
+        }
+
+        for (i = 0; i < slicelength; i++) {
+            Py_DECREF(garbage[i]);
+        }
+
+        PyMem_FREE(garbage);
+        Py_DECREF(seq);
+
+        return 0;
+    }
+}
+
 extern "C" Box* listSetitemSlice(BoxedList* self, BoxedSlice* slice, Box* v) {
     LOCK_REGION(self->lock.asWrite());
 
@@ -258,6 +390,12 @@ extern "C" Box* listSetitemSlice(BoxedList* self, BoxedSlice* slice, Box* v) {
     sliceIndex(slice->stop, &stop);
     sliceIndex(slice->step, &step);
 
+    if (step != 1) {
+        int r = list_ass_ext_slice(self, slice, v);
+        if (r)
+            throwCAPIException();
+        return None;
+    }
     RELEASE_ASSERT(step == 1, "step sizes must be 1 for now");
 
     // Logic from PySequence_GetSlice:
@@ -566,6 +704,18 @@ extern "C" int PyList_Sort(PyObject* v) noexcept {
     return 0;
 }
 
+extern "C" Box* PyList_GetSlice(PyObject* a, Py_ssize_t ilow, Py_ssize_t ihigh) noexcept {
+    assert(isSubclass(a->cls, list_cls));
+    BoxedList* self = static_cast<BoxedList*>(a);
+    try {
+        // Lots of extra copies here; we can do better if we need to:
+        return listGetitemSlice(self, new BoxedSlice(boxInt(ilow), boxInt(ihigh), boxInt(1)));
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+}
+
 Box* listContains(BoxedList* self, Box* elt) {
     LOCK_REGION(self->lock.asRead());
 
@@ -775,6 +925,46 @@ Box* listNe(BoxedList* self, Box* rhs) {
     return _listCmp(self, static_cast<BoxedList*>(rhs), AST_TYPE::NotEq);
 }
 
+Box* listLt(BoxedList* self, Box* rhs) {
+    if (rhs->cls != list_cls) {
+        return NotImplemented;
+    }
+
+    LOCK_REGION(self->lock.asRead());
+
+    return _listCmp(self, static_cast<BoxedList*>(rhs), AST_TYPE::Lt);
+}
+
+Box* listLe(BoxedList* self, Box* rhs) {
+    if (rhs->cls != list_cls) {
+        return NotImplemented;
+    }
+
+    LOCK_REGION(self->lock.asRead());
+
+    return _listCmp(self, static_cast<BoxedList*>(rhs), AST_TYPE::LtE);
+}
+
+Box* listGt(BoxedList* self, Box* rhs) {
+    if (rhs->cls != list_cls) {
+        return NotImplemented;
+    }
+
+    LOCK_REGION(self->lock.asRead());
+
+    return _listCmp(self, static_cast<BoxedList*>(rhs), AST_TYPE::Gt);
+}
+
+Box* listGe(BoxedList* self, Box* rhs) {
+    if (rhs->cls != list_cls) {
+        return NotImplemented;
+    }
+
+    LOCK_REGION(self->lock.asRead());
+
+    return _listCmp(self, static_cast<BoxedList*>(rhs), AST_TYPE::GtE);
+}
+
 extern "C" PyObject* _PyList_Extend(PyListObject* self, PyObject* b) noexcept {
     BoxedList* l = (BoxedList*)self;
     assert(isSubclass(l->cls, list_cls));
@@ -831,6 +1021,10 @@ void setupList() {
 
     list_cls->giveAttr("__eq__", new BoxedFunction(boxRTFunction((void*)listEq, UNKNOWN, 2)));
     list_cls->giveAttr("__ne__", new BoxedFunction(boxRTFunction((void*)listNe, UNKNOWN, 2)));
+    list_cls->giveAttr("__lt__", new BoxedFunction(boxRTFunction((void*)listLt, UNKNOWN, 2)));
+    list_cls->giveAttr("__le__", new BoxedFunction(boxRTFunction((void*)listLe, UNKNOWN, 2)));
+    list_cls->giveAttr("__gt__", new BoxedFunction(boxRTFunction((void*)listGt, UNKNOWN, 2)));
+    list_cls->giveAttr("__ge__", new BoxedFunction(boxRTFunction((void*)listGe, UNKNOWN, 2)));
 
     list_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)listRepr, STR, 1)));
     list_cls->giveAttr("__nonzero__", new BoxedFunction(boxRTFunction((void*)listNonzero, BOXED_BOOL, 1)));

src/runtime/objmodel.cpp

@@ -1579,7 +1579,6 @@ Box* getattrInternalGeneric(Box* obj, const std::string& attr, GetattrRewriteArg
         return getattrInternalEx(static_cast<BoxedInstanceMethod*>(obj)->func, attr, NULL, cls_only, for_call,
                                  bind_obj_out, NULL);
     }
-    // Finally, check __getattr__
 
     if (rewrite_args) {
         rewrite_args->out_success = true;
@@ -2160,7 +2159,7 @@ extern "C" i64 unboxedLen(Box* obj) {
     return rtn;
 }
 
-extern "C" void dump(void* p) {
+extern "C" void dumpEx(void* p, int levels) {
     printf("\n");
     printf("Raw address: %p\n", p);
 
@@ -2233,7 +2232,32 @@ extern "C" void dump(void* p) {
         }
 
         if (isSubclass(b->cls, tuple_cls)) {
-            printf("%ld elements\n", static_cast<BoxedTuple*>(b)->size());
+            BoxedTuple* t = static_cast<BoxedTuple*>(b);
+            printf("%ld elements\n", t->size());
+
+            if (levels > 0) {
+                int i = 0;
+                for (auto e : *t) {
+                    printf("\nElement %d:", i);
+                    i++;
+                    dumpEx(e, levels - 1);
+                }
+            }
+        }
+
+        if (isSubclass(b->cls, dict_cls)) {
+            BoxedDict* d = static_cast<BoxedDict*>(b);
+            printf("%ld elements\n", d->d.size());
+
+            if (levels > 0) {
+                int i = 0;
+                for (auto t : d->d) {
+                    printf("\nKey:");
+                    dumpEx(t.first, levels - 1);
+                    printf("Value:");
+                    dumpEx(t.second, levels - 1);
+                }
+            }
         }
 
         if (isSubclass(b->cls, int_cls)) {
@@ -2269,6 +2293,10 @@ extern "C" void dump(void* p) {
     RELEASE_ASSERT(0, "%d", (int)al->kind_id);
 }
 
+extern "C" void dump(void* p) {
+    dumpEx(p, 0);
+}
+
 // For rewriting purposes, this function assumes that nargs will be constant.
 // That's probably fine for some uses (ex binops), but otherwise it should be guarded on beforehand.
 extern "C" Box* callattrInternal(Box* obj, const std::string* attr, LookupScope scope, CallRewriteArgs* rewrite_args,
@@ -2441,7 +2469,7 @@ extern "C" Box* callattrInternal(Box* obj, const std::string* attr, LookupScope
 
 extern "C" Box* callattr(Box* obj, const std::string* attr, CallattrFlags flags, ArgPassSpec argspec, Box* arg1,
                          Box* arg2, Box* arg3, Box** args, const std::vector<const std::string*>* keyword_names) {
-    assert(gc::isValidGCObject(obj));
+    ASSERT(gc::isValidGCObject(obj), "%p", obj);
 
     int npassed_args = argspec.totalPassed();
 
@@ -4381,11 +4409,7 @@ extern "C" Box* getGlobal(Box* globals, const std::string* name) {
 }
 
 extern "C" Box* importFrom(Box* _m, const std::string* name) {
-    assert(isSubclass(_m->cls, module_cls));
-
-    BoxedModule* m = static_cast<BoxedModule*>(_m);
-
-    Box* r = getattrInternal(m, *name, NULL);
+    Box* r = getattrInternal(_m, *name, NULL);
     if (r)
         return r;
 

src/runtime/objmodel.h

@@ -94,6 +94,7 @@ extern "C" Box* createBoxedIterWrapperIfNeeded(Box* o);
 extern "C" bool hasnext(Box* o);
 
 extern "C" void dump(void* p);
+extern "C" void dumpEx(void* p, int levels = 0);
 
 struct SetattrRewriteArgs;
 void setattrGeneric(Box* obj, const std::string& attr, Box* val, SetattrRewriteArgs* rewrite_args);

src/runtime/stacktrace.cpp

@@ -159,7 +159,8 @@ extern "C" void abort() {
 
     // In case something calls abort down the line:
     static bool recursive = false;
-    if (!recursive) {
+    // If object_cls is NULL, then we somehow died early on, and won't be able to display a traceback.
+    if (!recursive && object_cls) {
         recursive = true;
 
         fprintf(stderr, "Someone called abort!\n");

src/runtime/types.cpp

@@ -1431,6 +1431,10 @@ Box* attrwrapperKeys(Box* b) {
     return AttrWrapper::keys(b);
 }
 
+void attrwrapperDel(Box* b, const std::string& attr) {
+    AttrWrapper::delitem(b, boxString(attr));
+}
+
 Box* objectNewNoArgs(BoxedClass* cls) {
     assert(isSubclass(cls->cls, type_cls));
     assert(typeLookup(cls, "__new__", NULL) == typeLookup(object_cls, "__new__", NULL)
@@ -2336,13 +2340,21 @@ void setupRuntime() {
 
 BoxedModule* createModule(const std::string& name, const std::string& fn, const char* doc) {
     assert(fn.size() && "probably wanted to set the fn to <stdin>?");
-    BoxedModule* module = new BoxedModule(name, fn, doc);
 
     BoxedDict* d = getSysModulesDict();
     Box* b_name = boxStringPtr(&name);
-    ASSERT(d->d.count(b_name) == 0, "%s", name.c_str());
-    d->d[b_name] = module;
 
+    // Surprisingly, there are times that we need to return the existing module if
+    // one exists:
+    Box*& ptr = d->d[b_name];
+    if (ptr && isSubclass(ptr->cls, module_cls)) {
+        return static_cast<BoxedModule*>(ptr);
+    } else {
+        ptr = NULL;
+    }
+
+    BoxedModule* module = new BoxedModule(name, fn, doc);
+    ptr = module;
     return module;
 }
 

src/runtime/types.h

@@ -655,6 +655,10 @@ public:
 
     DEFAULT_CLASS_SIMPLE(slice_cls);
 };
+static_assert(sizeof(BoxedSlice) == sizeof(PySliceObject), "");
+static_assert(offsetof(BoxedSlice, start) == offsetof(PySliceObject, start), "");
+static_assert(offsetof(BoxedSlice, stop) == offsetof(PySliceObject, stop), "");
+static_assert(offsetof(BoxedSlice, step) == offsetof(PySliceObject, step), "");
 
 class BoxedMemberDescriptor : public Box {
 public:
@@ -786,6 +790,7 @@ Box* objectSetattr(Box* obj, Box* attr, Box* value);
 Box* makeAttrWrapper(Box* b);
 Box* unwrapAttrWrapper(Box* b);
 Box* attrwrapperKeys(Box* b);
+void attrwrapperDel(Box* b, const std::string& attr);
 
 #define SystemError ((BoxedClass*)PyExc_SystemError)
 #define StopIteration ((BoxedClass*)PyExc_StopIteration)

test/tests/dash_c.py

@@ -7,7 +7,7 @@ with open('/dev/null')as ignore:
     # We don't (yet?) require exact stderr or return code compatibility w/
     # python. So we just check that we succeed or fail as appropriate.
     def run(args):
-        print 0 == subprocess.call([me] + args, stderr=ignore)
+        print subprocess.call([me] + args, stderr=ignore)
 
     run(["-c", "print 2 + 2"])
     run(["-c", "import sys; print sys.argv", "hello", "world"])
@@ -16,3 +16,6 @@ with open('/dev/null')as ignore:
     run(["-c"])
     run(["-c", "-c"])
     run(["-c", "this should not work"])
+
+    run(["-c", ";"])
+    run(["-cprint 1"])

test/tests/eval_bad_indentation.py

+# expected: fail
+
+try:
+    eval("\n 2")
+    print "bad, should have thrown an exception"
+except SyntaxError:
+    print "good, threw exception"

test/tests/eval_test.py

@@ -157,3 +157,9 @@ def wrap():
     inner2()
 
 wrap()
+
+try:
+    eval(" ")
+    print "worked?"
+except SyntaxError:
+    pass

test/tests/exec_syntaxerror.py

+# fail-if: '-x' in EXTRA_JIT_ARGS
+# - we don't get syntax errors through the old parser correctly
+
+try:
+    exec ";"
+    print "worked?"
+except SyntaxError:
+    pass
+

test/tests/file_writing.py

@@ -4,13 +4,15 @@ import tempfile
 fd, fn = tempfile.mkstemp()
 
 with open(fn, "wb") as f:
-    f.write("hello world!")
+    f.write("hello world!\n")
+    f.write(u"hello world2")
 
 with open(fn) as f:
     print repr(f.read())
 
 with open(fn, "w") as f:
     f.write("hello world!")
+    f.writelines(["hi", "world"])
 
 with open(fn) as f:
     print repr(f.read())

test/tests/getattr_exceptions.py

+
+# This throws an exception in the import machinery when we try to access __path__,
+# but that should get caught.
+# Also, email.MIMEText isn't even a Module or a subclass of Module...
+from email.MIMEText import MIMEText
+
+from email.MIMEMultipart import MIMEMultipart
+
+# TODO check similar cases with descriptors

test/tests/list.py

@@ -153,3 +153,21 @@ print l
 l = range(5)
 l[2:4] = tuple(range(2))
 print l
+
+
+l = [None]*4
+try:
+    l[::-1] = range(5)
+except ValueError as e:
+    print e
+l[::-1] = range(4)
+print l
+del l[::2]
+print l
+
+for i in xrange(3):
+    for j in xrange(3):
+        for k in xrange(3):
+            l1 = [i]
+            l2 = [j, k]
+            print l1 < l2, l1 <= l2, l1 > l2, l1 >= l2

test/tests/list_sorting_gc.py

+# regression test: list sorting had a gc bug
+
+import gc
+class C(object):
+    def __init__(self, n):
+        self.n = range(n)
+
+    def __eq__(self, rhs):
+        # print "eq"
+        gc.collect()
+        return self.n == rhs.n
+
+    def __lt__(self, rhs):
+        # print "lt"
+        gc.collect()
+        return self.n < rhs.n
+
+def keyfunc(c):
+    return c
+
+def f():
+    for i in xrange(10):
+        print i
+        l = [C(i % 5) for i in xrange(10)]
+        l.sort(key=keyfunc)
+f()

test/tests/locals_test.py

@@ -29,6 +29,7 @@ def f3():
     s = "hello world"
     t = (1.0, "asdf")
     print sorted(locals().items())
+    print sorted(vars().items())
 f3()
 
 def f4(t):
@@ -38,6 +39,7 @@ def f4(t):
     else:
         y = 2
     print sorted(locals().items())
+    print sorted(vars().items())
 f4(0)
 f4(1)
 
@@ -52,6 +54,7 @@ def f5():
                 print a
                 print b
                 print sorted(locals().items())
+                print sorted(vars().items())
             i()
         h()
     g()

test/tests/sys_exit_str.py

+# should_error
+# no-collect-stats
+
+try:
+    raise SystemExit, "hello"
+except Exception as e:
+    pass