Commit 7b71c6f8 authored by Kevin Modzelewski's avatar Kevin Modzelewski

Merge pull request #464 from kmod/virtualenv

Some misc virtualenv + library support
parents eb48bede a722d9c6
......@@ -206,8 +206,8 @@ add_test(NAME gc_unittest COMMAND gc_unittest)
add_test(NAME analysis_unittest COMMAND analysis_unittest)
add_test(NAME pyston_defaults COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
# we pass -I to cpython tests and skip failing ones b/c they are slooow otherwise
add_test(NAME pyston_defaults_cpython_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing ${CMAKE_SOURCE_DIR}/test/cpython)
add_test(NAME pyston_defaults_integration_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t60 ${CMAKE_SOURCE_DIR}/test/integration)
add_test(NAME pyston_defaults_cpython_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t30 ${CMAKE_SOURCE_DIR}/test/cpython)
add_test(NAME pyston_defaults_integration_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t120 ${CMAKE_SOURCE_DIR}/test/integration)
add_test(NAME pyston_max_compilation_tier COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-O -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
add_test(NAME pyston_old_parser COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -a=-x -R ./pyston -j1 -a=-n -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
......
......@@ -493,8 +493,8 @@ check:
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S $(TESTS_DIR) $(ARGS)
@# we pass -I to cpython tests & skip failing ones because they are sloooow otherwise
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t60 $(TEST_DIR)/integration $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t30 $(TEST_DIR)/cpython $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t120 $(TEST_DIR)/integration $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-n -a=-x -a=-S $(TESTS_DIR) $(ARGS)
@# skip -O for dbg
......@@ -955,8 +955,8 @@ $(eval \
check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1 ext_pyston
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k $(TESTS_DIR) $(ARGS)
@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t=60 $(TEST_DIR)/integration $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing -t30 $(TEST_DIR)/cpython $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t120 $(TEST_DIR)/integration $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -a=-S -k $(TESTS_DIR) $(ARGS)
......
......@@ -69,7 +69,7 @@ def get_python_version():
def get_python_inc(plat_specific=0, prefix=None):
# Pyston change: this is the way we layout things internally:
return os.path.join(os.path.dirname(sys.executable), "from_cpython/Include")
return os.path.join(sys.prefix, "from_cpython/Include")
"""Return the directory containing installed Python header files.
......
......@@ -3,11 +3,6 @@
from __future__ import generators
# Pyston change:
import sys
del sys.modules['modulefinder']
raise ImportError("This isn't really supported in Pyston yet")
import dis
import imp
import marshal
......
......@@ -206,3 +206,65 @@ string_splitlines(PyStringObject *self, PyObject *args)
keepends
);
}
PyObject *PyString_AsDecodedObject(PyObject *str,
const char *encoding,
const char *errors)
{
PyObject *v;
if (!PyString_Check(str)) {
PyErr_BadArgument();
goto onError;
}
if (encoding == NULL) {
#ifdef Py_USING_UNICODE
encoding = PyUnicode_GetDefaultEncoding();
#else
PyErr_SetString(PyExc_ValueError, "no encoding specified");
goto onError;
#endif
}
/* Decode via the codec registry */
v = PyCodec_Decode(str, encoding, errors);
if (v == NULL)
goto onError;
return v;
onError:
return NULL;
}
PyObject *PyString_AsEncodedObject(PyObject *str,
const char *encoding,
const char *errors)
{
PyObject *v;
if (!PyString_Check(str)) {
PyErr_BadArgument();
goto onError;
}
if (encoding == NULL) {
#ifdef Py_USING_UNICODE
encoding = PyUnicode_GetDefaultEncoding();
#else
PyErr_SetString(PyExc_ValueError, "no encoding specified");
goto onError;
#endif
}
/* Encode via the codec registry */
v = PyCodec_Encode(str, encoding, errors);
if (v == NULL)
goto onError;
return v;
onError:
return NULL;
}
......@@ -32,11 +32,22 @@ static const std::string path_str("__path__");
static const std::string package_str("__package__");
static BoxedClass* null_importer_cls;
static void removeModule(const std::string& name) {
BoxedDict* d = getSysModulesDict();
Box* b_name = boxString(name);
d->d.erase(b_name);
}
BoxedModule* createAndRunModule(const std::string& name, const std::string& fn) {
BoxedModule* module = createModule(name, fn);
AST_Module* ast = caching_parse_file(fn.c_str());
try {
compileAndRunModule(ast, module);
} catch (ExcInfo e) {
removeModule(name);
raiseRaw(e);
}
return module;
}
......@@ -51,7 +62,12 @@ static BoxedModule* createAndRunModule(const std::string& name, const std::strin
module->setattr(path_str, path_list, NULL);
AST_Module* ast = caching_parse_file(fn.c_str());
try {
compileAndRunModule(ast, module);
} catch (ExcInfo e) {
removeModule(name);
raiseRaw(e);
}
return module;
}
......@@ -363,6 +379,7 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box
if (sr.type != SearchResult::SEARCH_ERROR) {
Box* module;
try {
if (sr.type == SearchResult::PY_SOURCE)
module = createAndRunModule(full_name, sr.path);
else if (sr.type == SearchResult::PKG_DIRECTORY)
......@@ -376,6 +393,10 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box
boxString(full_name), NULL, NULL, NULL, NULL);
} else
RELEASE_ASSERT(0, "%d", sr.type);
} catch (ExcInfo e) {
removeModule(name);
raiseRaw(e);
}
if (parent_module && parent_module != None)
parent_module->setattr(name, module, NULL);
......@@ -609,6 +630,7 @@ extern "C" PyObject* PyImport_ExecCodeModuleEx(char* name, PyObject* co, char* p
module->setattr("__file__", boxString(pathname), NULL);
return module;
} catch (ExcInfo e) {
removeModule(name);
setCAPIException(e);
return NULL;
}
......
......@@ -630,15 +630,40 @@ extern "C" int PyList_Reverse(PyObject* v) noexcept {
return 0;
}
class PyCmpComparer {
private:
Box* cmp;
public:
PyCmpComparer(Box* cmp) : cmp(cmp) {}
bool operator()(Box* lhs, Box* rhs) {
Box* r = runtimeCallInternal(cmp, NULL, ArgPassSpec(2), lhs, rhs, NULL, NULL, NULL);
if (!isSubclass(r->cls, int_cls))
raiseExcHelper(TypeError, "comparison function must return int, not %.200s", r->cls->tp_name);
return static_cast<BoxedInt*>(r)->n < 0;
}
};
void listSort(BoxedList* self, Box* cmp, Box* key, Box* reverse) {
LOCK_REGION(self->lock.asWrite());
assert(isSubclass(self->cls, list_cls));
RELEASE_ASSERT(cmp == None, "The 'cmp' keyword is currently not supported");
if (cmp == None)
cmp = NULL;
if (key == None)
key = NULL;
RELEASE_ASSERT(!cmp || !key, "Specifying both the 'cmp' and 'key' keywords is currently not supported");
// TODO(kmod): maybe we should just switch to CPython's sort. not sure how the algorithms compare,
// but they specifically try to support cases where __lt__ or the cmp function might end up inspecting
// the current list being sorted.
// I also don't know if std::stable_sort is exception-safe.
if (cmp) {
std::stable_sort<Box**, PyCmpComparer>(self->elts->elts, self->elts->elts + self->size, PyCmpComparer(cmp));
} else {
int num_keys_added = 0;
auto remove_keys = [&]() {
for (int i = 0; i < num_keys_added; i++) {
......@@ -672,10 +697,11 @@ void listSort(BoxedList* self, Box* cmp, Box* key, Box* reverse) {
std::stable_sort<Box**, PyLt>(self->elts->elts, self->elts->elts + self->size, PyLt());
} catch (ExcInfo e) {
remove_keys();
throw e;
raiseRaw(e);
}
remove_keys();
}
if (nonzero(reverse)) {
listReverse(self);
......
......@@ -99,8 +99,6 @@ void REWRITE_ABORTED(const char* reason) {
#define REWRITE_ABORTED(reason) ((void)(reason))
#endif
Box* runtimeCallInternal(Box* obj, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3,
Box** args, const std::vector<const std::string*>* keyword_names);
static Box* (*runtimeCallInternal0)(Box*, CallRewriteArgs*, ArgPassSpec)
= (Box * (*)(Box*, CallRewriteArgs*, ArgPassSpec))runtimeCallInternal;
static Box* (*runtimeCallInternal1)(Box*, CallRewriteArgs*, ArgPassSpec, Box*)
......
......@@ -103,6 +103,9 @@ struct BinopRewriteArgs;
extern "C" Box* binopInternal(Box* lhs, Box* rhs, int op_type, bool inplace, BinopRewriteArgs* rewrite_args);
struct CallRewriteArgs;
Box* runtimeCallInternal(Box* obj, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3,
Box** args, const std::vector<const std::string*>* keyword_names);
Box* lenCallInternal(BoxedFunctionBase* f, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2,
Box* arg3, Box** args, const std::vector<const std::string*>* keyword_names);
Box* typeCallInternal(BoxedFunctionBase* f, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2,
......
......@@ -343,11 +343,23 @@ extern "C" Box* strAdd(BoxedString* lhs, Box* _rhs) {
return new (lhs->size() + rhs->size()) BoxedString(lhs->s, rhs->s);
}
static llvm::StringMap<Box*> interned_strings;
extern "C" PyObject* PyString_InternFromString(const char* s) noexcept {
Py_FatalError("unimplemented");
RELEASE_ASSERT(s, "");
auto it = interned_strings.find(s);
if (it == interned_strings.end()) {
Box* b = PyGC_AddRoot(boxString(s));
assert(b);
interned_strings[s] = b;
return b;
} else {
assert(it->second);
return it->second;
}
}
extern "C" void PyString_InternInPlace(PyObject**) noexcept {
extern "C" void PyString_InternInPlace(PyObject** o) noexcept {
Py_FatalError("unimplemented");
}
......@@ -2194,8 +2206,8 @@ Box* strDecode(BoxedString* self, Box* encoding, Box* error) {
if (error_str && !isSubclass(error_str->cls, str_cls))
raiseExcHelper(TypeError, "decode() argument 2 must be string, not '%s'", getTypeName(error_str));
Box* result
= PyCodec_Decode(self, encoding_str ? encoding_str->data() : NULL, error_str ? error_str->data() : NULL);
Box* result = PyString_AsDecodedObject(self, encoding_str ? encoding_str->data() : NULL,
error_str ? error_str->data() : NULL);
checkAndThrowCAPIException();
return result;
}
......@@ -2219,7 +2231,7 @@ Box* strEncode(BoxedString* self, Box* encoding, Box* error) {
if (error_str && !isSubclass(error_str->cls, str_cls))
raiseExcHelper(TypeError, "encode() argument 2 must be string, not '%s'", getTypeName(error_str));
Box* result = PyCodec_Encode(self, encoding_str ? encoding_str->data() : PyUnicode_GetDefaultEncoding(),
Box* result = PyString_AsEncodedObject(self, encoding_str ? encoding_str->data() : PyUnicode_GetDefaultEncoding(),
error_str ? error_str->data() : NULL);
checkAndThrowCAPIException();
return result;
......
Subproject commit ee62ccfda4950352bcad9612f0951fb38d805350
Subproject commit 0c0974d501ffeb0ec29ead9eba75e51959dce616
......@@ -14,3 +14,16 @@ if os.path.exists("test_env"):
args = [sys.executable, VIRTUALENV_SCRIPT, "-p", sys.executable, "test_env"]
print "Running", args
subprocess.check_call(args)
sh_script = """
set -e
. test_env/bin/activate
set -ux
python -c 'import __future__'
python -c 'import sys; print sys.executable'
pip install six==1.9.0 cffi==0.9.2
python -c 'import six; print six.__version__'
""".strip()
# print sh_script
subprocess.check_call(["sh", "-c", sh_script])
# skip-if: True
import sys
print "import_failure_target" in sys.modules
raise Exception("pretending submodule didn't import")
......@@ -62,3 +62,15 @@ def f2():
except NameError, e:
print e
f2()
try:
import import_failure_target
raise Exception("This should not be importable")
except Exception, e:
print type(e), e
try:
import import_failure_target
raise Exception("This should not be importable if we tried it again")
except Exception, e:
print type(e), e
......@@ -171,3 +171,26 @@ for i in xrange(3):
l1 = [i]
l2 = [j, k]
print l1 < l2, l1 <= l2, l1 > l2, l1 >= l2
def mycmp(k1, k2):
types_seen.add((type(k1), type(k2)))
if k1 == k2:
return 0
if k1 < k2:
return -1
return 1
types_seen = set()
l = ["%d" for i in xrange(20)]
l.sort(cmp=mycmp)
print types_seen
print l
"""
types_seen = set()
l = range(20)
l.sort(cmp=mycmp, key=str)
print types_seen
print l
"""
......@@ -7,3 +7,7 @@ test("hello world", "hex")
test("hello world", "base64")
test("\r\n\\", "string-escape")
"".encode()
"".decode()
u"".encode()
u"".decode()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment