Merge pull request #464 from kmod/virtualenv

Some misc virtualenv + library support

CMakeLists.txt

@@ -206,8 +206,8 @@ add_test(NAME gc_unittest COMMAND gc_unittest)
 add_test(NAME analysis_unittest COMMAND analysis_unittest)
 add_test(NAME pyston_defaults COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
 # we pass -I to cpython tests and skip failing ones b/c they are slooow otherwise
-add_test(NAME pyston_defaults_cpython_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing ${CMAKE_SOURCE_DIR}/test/cpython)
-add_test(NAME pyston_defaults_integration_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t60 ${CMAKE_SOURCE_DIR}/test/integration)
+add_test(NAME pyston_defaults_cpython_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t30 ${CMAKE_SOURCE_DIR}/test/cpython)
+add_test(NAME pyston_defaults_integration_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t120 ${CMAKE_SOURCE_DIR}/test/integration)
 add_test(NAME pyston_max_compilation_tier COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-O -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
 add_test(NAME pyston_old_parser COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -a=-x -R ./pyston -j1 -a=-n -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
 

Makefile

@@ -493,8 +493,8 @@ check:
 
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests & skip failing ones because they are sloooow otherwise
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t60 $(TEST_DIR)/integration $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t30 $(TEST_DIR)/cpython $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t120 $(TEST_DIR)/integration $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-n -a=-x -a=-S $(TESTS_DIR) $(ARGS)
 	@# skip -O for dbg
 
@@ -955,8 +955,8 @@ $(eval \
 check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1 ext_pyston
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t=60 $(TEST_DIR)/integration $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing -t30 $(TEST_DIR)/cpython $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t120 $(TEST_DIR)/integration $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -a=-S -k $(TESTS_DIR) $(ARGS)
 

from_cpython/Lib/distutils/sysconfig.py

@@ -69,7 +69,7 @@ def get_python_version():
 
 def get_python_inc(plat_specific=0, prefix=None):
     # Pyston change: this is the way we layout things internally:
-    return os.path.join(os.path.dirname(sys.executable), "from_cpython/Include")
+    return os.path.join(sys.prefix, "from_cpython/Include")
 
     """Return the directory containing installed Python header files.
 

from_cpython/Lib/modulefinder.py

@@ -3,11 +3,6 @@
 
 from __future__ import generators
 
-# Pyston change:
-import sys
-del sys.modules['modulefinder']
-raise ImportError("This isn't really supported in Pyston yet")
-
 import dis
 import imp
 import marshal

from_cpython/Objects/stringobject.c

@@ -206,3 +206,65 @@ string_splitlines(PyStringObject *self, PyObject *args)
         keepends
     );
 }
+
+PyObject *PyString_AsDecodedObject(PyObject *str,
+                                   const char *encoding,
+                                   const char *errors)
+{
+    PyObject *v;
+
+    if (!PyString_Check(str)) {
+        PyErr_BadArgument();
+        goto onError;
+    }
+
+    if (encoding == NULL) {
+#ifdef Py_USING_UNICODE
+        encoding = PyUnicode_GetDefaultEncoding();
+#else
+        PyErr_SetString(PyExc_ValueError, "no encoding specified");
+        goto onError;
+#endif
+    }
+
+    /* Decode via the codec registry */
+    v = PyCodec_Decode(str, encoding, errors);
+    if (v == NULL)
+        goto onError;
+
+    return v;
+
+ onError:
+    return NULL;
+}
+
+PyObject *PyString_AsEncodedObject(PyObject *str,
+                                   const char *encoding,
+                                   const char *errors)
+{
+    PyObject *v;
+
+    if (!PyString_Check(str)) {
+        PyErr_BadArgument();
+        goto onError;
+    }
+
+    if (encoding == NULL) {
+#ifdef Py_USING_UNICODE
+        encoding = PyUnicode_GetDefaultEncoding();
+#else
+        PyErr_SetString(PyExc_ValueError, "no encoding specified");
+        goto onError;
+#endif
+    }
+
+    /* Encode via the codec registry */
+    v = PyCodec_Encode(str, encoding, errors);
+    if (v == NULL)
+        goto onError;
+
+    return v;
+
+ onError:
+    return NULL;
+}

src/runtime/import.cpp

@@ -32,11 +32,22 @@ static const std::string path_str("__path__");
 static const std::string package_str("__package__");
 static BoxedClass* null_importer_cls;
 
+static void removeModule(const std::string& name) {
+    BoxedDict* d = getSysModulesDict();
+    Box* b_name = boxString(name);
+    d->d.erase(b_name);
+}
+
 BoxedModule* createAndRunModule(const std::string& name, const std::string& fn) {
     BoxedModule* module = createModule(name, fn);
 
     AST_Module* ast = caching_parse_file(fn.c_str());
-    compileAndRunModule(ast, module);
+    try {
+        compileAndRunModule(ast, module);
+    } catch (ExcInfo e) {
+        removeModule(name);
+        raiseRaw(e);
+    }
     return module;
 }
 
@@ -51,7 +62,12 @@ static BoxedModule* createAndRunModule(const std::string& name, const std::strin
     module->setattr(path_str, path_list, NULL);
 
     AST_Module* ast = caching_parse_file(fn.c_str());
-    compileAndRunModule(ast, module);
+    try {
+        compileAndRunModule(ast, module);
+    } catch (ExcInfo e) {
+        removeModule(name);
+        raiseRaw(e);
+    }
     return module;
 }
 
@@ -363,19 +379,24 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box
     if (sr.type != SearchResult::SEARCH_ERROR) {
         Box* module;
 
-        if (sr.type == SearchResult::PY_SOURCE)
-            module = createAndRunModule(full_name, sr.path);
-        else if (sr.type == SearchResult::PKG_DIRECTORY)
-            module = createAndRunModule(full_name, sr.path + "/__init__.py", sr.path);
-        else if (sr.type == SearchResult::C_EXTENSION)
-            module = importCExtension(full_name, name, sr.path);
-        else if (sr.type == SearchResult::IMP_HOOK) {
-            const static std::string load_module_str("load_module");
-            module = callattr(sr.loader, &load_module_str,
-                              CallattrFlags({.cls_only = false, .null_on_nonexistent = false }), ArgPassSpec(1),
-                              boxString(full_name), NULL, NULL, NULL, NULL);
-        } else
-            RELEASE_ASSERT(0, "%d", sr.type);
+        try {
+            if (sr.type == SearchResult::PY_SOURCE)
+                module = createAndRunModule(full_name, sr.path);
+            else if (sr.type == SearchResult::PKG_DIRECTORY)
+                module = createAndRunModule(full_name, sr.path + "/__init__.py", sr.path);
+            else if (sr.type == SearchResult::C_EXTENSION)
+                module = importCExtension(full_name, name, sr.path);
+            else if (sr.type == SearchResult::IMP_HOOK) {
+                const static std::string load_module_str("load_module");
+                module = callattr(sr.loader, &load_module_str,
+                                  CallattrFlags({.cls_only = false, .null_on_nonexistent = false }), ArgPassSpec(1),
+                                  boxString(full_name), NULL, NULL, NULL, NULL);
+            } else
+                RELEASE_ASSERT(0, "%d", sr.type);
+        } catch (ExcInfo e) {
+            removeModule(name);
+            raiseRaw(e);
+        }
 
         if (parent_module && parent_module != None)
             parent_module->setattr(name, module, NULL);
@@ -609,6 +630,7 @@ extern "C" PyObject* PyImport_ExecCodeModuleEx(char* name, PyObject* co, char* p
         module->setattr("__file__", boxString(pathname), NULL);
         return module;
     } catch (ExcInfo e) {
+        removeModule(name);
         setCAPIException(e);
         return NULL;
     }

src/runtime/list.cpp

@@ -630,53 +630,79 @@ extern "C" int PyList_Reverse(PyObject* v) noexcept {
     return 0;
 }
 
+class PyCmpComparer {
+private:
+    Box* cmp;
+
+public:
+    PyCmpComparer(Box* cmp) : cmp(cmp) {}
+    bool operator()(Box* lhs, Box* rhs) {
+        Box* r = runtimeCallInternal(cmp, NULL, ArgPassSpec(2), lhs, rhs, NULL, NULL, NULL);
+        if (!isSubclass(r->cls, int_cls))
+            raiseExcHelper(TypeError, "comparison function must return int, not %.200s", r->cls->tp_name);
+        return static_cast<BoxedInt*>(r)->n < 0;
+    }
+};
+
 void listSort(BoxedList* self, Box* cmp, Box* key, Box* reverse) {
     LOCK_REGION(self->lock.asWrite());
     assert(isSubclass(self->cls, list_cls));
 
-    RELEASE_ASSERT(cmp == None, "The 'cmp' keyword is currently not supported");
+    if (cmp == None)
+        cmp = NULL;
 
     if (key == None)
         key = NULL;
 
-    int num_keys_added = 0;
-    auto remove_keys = [&]() {
-        for (int i = 0; i < num_keys_added; i++) {
-            Box** obj_loc = &self->elts->elts[i];
-            assert((*obj_loc)->cls == tuple_cls);
-            *obj_loc = static_cast<BoxedTuple*>(*obj_loc)->elts[2];
-        }
-    };
-
-    try {
-        if (key) {
-            for (int i = 0; i < self->size; i++) {
-                Box** obj_loc = &self->elts->elts[i];
+    RELEASE_ASSERT(!cmp || !key, "Specifying both the 'cmp' and 'key' keywords is currently not supported");
 
-                Box* key_val = runtimeCall(key, ArgPassSpec(1), *obj_loc, NULL, NULL, NULL, NULL);
-                // Add the index as part of the new tuple so that the comparison never hits the
-                // original object.
-                // TODO we could potentially make this faster by copying the CPython approach of
-                // creating special sortwrapper objects that compare only based on the key.
-                Box* new_obj = BoxedTuple::create({ key_val, boxInt(i), *obj_loc });
+    // TODO(kmod): maybe we should just switch to CPython's sort.  not sure how the algorithms compare,
+    // but they specifically try to support cases where __lt__ or the cmp function might end up inspecting
+    // the current list being sorted.
+    // I also don't know if std::stable_sort is exception-safe.
 
-                *obj_loc = new_obj;
-                num_keys_added++;
+    if (cmp) {
+        std::stable_sort<Box**, PyCmpComparer>(self->elts->elts, self->elts->elts + self->size, PyCmpComparer(cmp));
+    } else {
+        int num_keys_added = 0;
+        auto remove_keys = [&]() {
+            for (int i = 0; i < num_keys_added; i++) {
+                Box** obj_loc = &self->elts->elts[i];
+                assert((*obj_loc)->cls == tuple_cls);
+                *obj_loc = static_cast<BoxedTuple*>(*obj_loc)->elts[2];
+            }
+        };
+
+        try {
+            if (key) {
+                for (int i = 0; i < self->size; i++) {
+                    Box** obj_loc = &self->elts->elts[i];
+
+                    Box* key_val = runtimeCall(key, ArgPassSpec(1), *obj_loc, NULL, NULL, NULL, NULL);
+                    // Add the index as part of the new tuple so that the comparison never hits the
+                    // original object.
+                    // TODO we could potentially make this faster by copying the CPython approach of
+                    // creating special sortwrapper objects that compare only based on the key.
+                    Box* new_obj = BoxedTuple::create({ key_val, boxInt(i), *obj_loc });
+
+                    *obj_loc = new_obj;
+                    num_keys_added++;
+                }
             }
+
+            // We don't need to do a stable sort if there's a keyfunc, since we explicitly added the index
+            // as part of the sort key.
+            // But we might want to get rid of that approach?  CPython doesn't do that (they create special
+            // wrapper objects that compare only based on the key).
+            std::stable_sort<Box**, PyLt>(self->elts->elts, self->elts->elts + self->size, PyLt());
+        } catch (ExcInfo e) {
+            remove_keys();
+            raiseRaw(e);
         }
 
-        // We don't need to do a stable sort if there's a keyfunc, since we explicitly added the index
-        // as part of the sort key.
-        // But we might want to get rid of that approach?  CPython doesn't do that (they create special
-        // wrapper objects that compare only based on the key).
-        std::stable_sort<Box**, PyLt>(self->elts->elts, self->elts->elts + self->size, PyLt());
-    } catch (ExcInfo e) {
         remove_keys();
-        throw e;
     }
 
-    remove_keys();
-
     if (nonzero(reverse)) {
         listReverse(self);
     }

src/runtime/objmodel.cpp

@@ -99,8 +99,6 @@ void REWRITE_ABORTED(const char* reason) {
 #define REWRITE_ABORTED(reason) ((void)(reason))
 #endif
 
-Box* runtimeCallInternal(Box* obj, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3,
-                         Box** args, const std::vector<const std::string*>* keyword_names);
 static Box* (*runtimeCallInternal0)(Box*, CallRewriteArgs*, ArgPassSpec)
     = (Box * (*)(Box*, CallRewriteArgs*, ArgPassSpec))runtimeCallInternal;
 static Box* (*runtimeCallInternal1)(Box*, CallRewriteArgs*, ArgPassSpec, Box*)

src/runtime/objmodel.h

@@ -103,6 +103,9 @@ struct BinopRewriteArgs;
 extern "C" Box* binopInternal(Box* lhs, Box* rhs, int op_type, bool inplace, BinopRewriteArgs* rewrite_args);
 
 struct CallRewriteArgs;
+Box* runtimeCallInternal(Box* obj, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3,
+                         Box** args, const std::vector<const std::string*>* keyword_names);
+
 Box* lenCallInternal(BoxedFunctionBase* f, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2,
                      Box* arg3, Box** args, const std::vector<const std::string*>* keyword_names);
 Box* typeCallInternal(BoxedFunctionBase* f, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2,

src/runtime/str.cpp

@@ -343,11 +343,23 @@ extern "C" Box* strAdd(BoxedString* lhs, Box* _rhs) {
     return new (lhs->size() + rhs->size()) BoxedString(lhs->s, rhs->s);
 }
 
+static llvm::StringMap<Box*> interned_strings;
+
 extern "C" PyObject* PyString_InternFromString(const char* s) noexcept {
-    Py_FatalError("unimplemented");
+    RELEASE_ASSERT(s, "");
+    auto it = interned_strings.find(s);
+    if (it == interned_strings.end()) {
+        Box* b = PyGC_AddRoot(boxString(s));
+        assert(b);
+        interned_strings[s] = b;
+        return b;
+    } else {
+        assert(it->second);
+        return it->second;
+    }
 }
 
-extern "C" void PyString_InternInPlace(PyObject**) noexcept {
+extern "C" void PyString_InternInPlace(PyObject** o) noexcept {
     Py_FatalError("unimplemented");
 }
 
@@ -2194,8 +2206,8 @@ Box* strDecode(BoxedString* self, Box* encoding, Box* error) {
     if (error_str && !isSubclass(error_str->cls, str_cls))
         raiseExcHelper(TypeError, "decode() argument 2 must be string, not '%s'", getTypeName(error_str));
 
-    Box* result
-        = PyCodec_Decode(self, encoding_str ? encoding_str->data() : NULL, error_str ? error_str->data() : NULL);
+    Box* result = PyString_AsDecodedObject(self, encoding_str ? encoding_str->data() : NULL,
+                                           error_str ? error_str->data() : NULL);
     checkAndThrowCAPIException();
     return result;
 }
@@ -2219,8 +2231,8 @@ Box* strEncode(BoxedString* self, Box* encoding, Box* error) {
     if (error_str && !isSubclass(error_str->cls, str_cls))
         raiseExcHelper(TypeError, "encode() argument 2 must be string, not '%s'", getTypeName(error_str));
 
-    Box* result = PyCodec_Encode(self, encoding_str ? encoding_str->data() : PyUnicode_GetDefaultEncoding(),
-                                 error_str ? error_str->data() : NULL);
+    Box* result = PyString_AsEncodedObject(self, encoding_str ? encoding_str->data() : PyUnicode_GetDefaultEncoding(),
+                                           error_str ? error_str->data() : NULL);
     checkAndThrowCAPIException();
     return result;
 }

virtualenv @ 0c0974d5

-Subproject commit ee62ccfda4950352bcad9612f0951fb38d805350
+Subproject commit 0c0974d501ffeb0ec29ead9eba75e51959dce616

test/integration/virtualenv_test.py

@@ -14,3 +14,16 @@ if os.path.exists("test_env"):
 args = [sys.executable, VIRTUALENV_SCRIPT, "-p", sys.executable, "test_env"]
 print "Running", args
 subprocess.check_call(args)
+
+sh_script = """
+set -e
+. test_env/bin/activate
+set -ux
+python -c 'import __future__'
+python -c 'import sys; print sys.executable'
+pip install six==1.9.0 cffi==0.9.2
+python -c 'import six; print six.__version__'
+""".strip()
+
+# print sh_script
+subprocess.check_call(["sh", "-c", sh_script])

test/tests/import_failure_target.py

+# skip-if: True
+
+import sys
+print "import_failure_target" in sys.modules
+raise Exception("pretending submodule didn't import")

test/tests/import_failure_test.py

@@ -62,3 +62,15 @@ def f2():
     except NameError, e:
         print e
 f2()
+
+try:
+    import import_failure_target
+    raise Exception("This should not be importable")
+except Exception, e:
+    print type(e), e
+
+try:
+    import import_failure_target
+    raise Exception("This should not be importable if we tried it again")
+except Exception, e:
+    print type(e), e

test/tests/list.py

@@ -171,3 +171,26 @@ for i in xrange(3):
             l1 = [i]
             l2 = [j, k]
             print l1 < l2, l1 <= l2, l1 > l2, l1 >= l2
+
+
+def mycmp(k1, k2):
+    types_seen.add((type(k1), type(k2)))
+    if k1 == k2:
+        return 0
+    if k1 < k2:
+        return -1
+    return 1
+
+types_seen = set()
+l = ["%d" for i in xrange(20)]
+l.sort(cmp=mycmp)
+print types_seen
+print l
+
+"""
+types_seen = set()
+l = range(20)
+l.sort(cmp=mycmp, key=str)
+print types_seen
+print l
+"""

test/tests/str_encode_decode.py

@@ -7,3 +7,7 @@ test("hello world", "hex")
 test("hello world", "base64")
 test("\r\n\\", "string-escape")
 
+"".encode()
+"".decode()
+u"".encode()
+u"".decode()