Merge branch 'testsuite'

Beginnings of a testsuite, starting with a simple django test.
Increases testing time by about 45 seconds which isn't great, but
tests a lot of stuff which is good.

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "libpypa"]
 	path = libpypa
 	url = git://github.com/vinzenz/libpypa.git
+[submodule "test/integration/django"]
+	path = test/integration/django
+	url = https://github.com/django/django

Makefile

@@ -492,6 +492,7 @@ check:
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests & skip failing ones because they are sloooow otherwise
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S -a=-I --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only -t60 $(TEST_DIR)/integration $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-n -a=-x -a=-S $(TESTS_DIR) $(ARGS)
 	@# skip -O for dbg
 
@@ -508,6 +509,7 @@ check:
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-S $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-S -a=-I --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-S --exit-code-only -t60 $(TEST_DIR)/integration $(ARGS)
 	@# skip -n for dbg
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-O -a=-x -a=-S $(TESTS_DIR) $(ARGS)
 
@@ -951,6 +953,7 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1 ext_pyston
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -a=-I -k --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only -t=60 $(TEST_DIR)/integration $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -a=-S -k $(TESTS_DIR) $(ARGS)
 

src/codegen/irgen/hooks.cpp

@@ -368,6 +368,9 @@ Box* eval(Box* boxedCode) {
     if (globals == module)
         globals = NULL;
 
+    if (globals && globals->cls == attrwrapper_cls && unwrapAttrWrapper(globals) == module)
+        globals = NULL;
+
     // TODO error message if parse fails or if it isn't an expr
     // TODO should have a cleaner interface that can parse the Expression directly
     // TODO this memory leaks
@@ -414,7 +417,10 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) {
     if (globals == module)
         globals = NULL;
 
-    assert(!globals || globals->cls == dict_cls);
+    if (globals && globals->cls == attrwrapper_cls && unwrapAttrWrapper(globals) == module)
+        globals = NULL;
+
+    ASSERT(!globals || globals->cls == dict_cls, "%s", globals->cls->tp_name);
 
     if (globals) {
         // From CPython (they set it to be f->f_builtins):

src/runtime/str.cpp

@@ -44,6 +44,11 @@ extern "C" PyObject* string_rindex(PyStringObject* self, PyObject* args) noexcep
 extern "C" PyObject* string_rfind(PyStringObject* self, PyObject* args) noexcept;
 extern "C" PyObject* string_splitlines(PyStringObject* self, PyObject* args) noexcept;
 
+// from cpython's stringobject.c:
+#define LEFTSTRIP 0
+#define RIGHTSTRIP 1
+#define BOTHSTRIP 2
+
 namespace pyston {
 
 BoxedString::BoxedString(const char* s, size_t n) : s(storage(), n) {
@@ -1848,6 +1853,16 @@ Box* strStrip(BoxedString* self, Box* chars) {
         return boxStringRef(str.trim(chars_str));
     } else if (chars->cls == none_cls) {
         return boxStringRef(str.trim(" \t\n\r\f\v"));
+    } else if (isSubclass(chars->cls, unicode_cls)) {
+        PyObject* uniself = PyUnicode_FromObject((PyObject*)self);
+        PyObject* res;
+        if (uniself == NULL)
+            throwCAPIException();
+        res = _PyUnicode_XStrip((PyUnicodeObject*)uniself, BOTHSTRIP, chars);
+        if (!res)
+            throwCAPIException();
+        Py_DECREF(uniself);
+        return res;
     } else {
         raiseExcHelper(TypeError, "strip arg must be None, str or unicode");
     }
@@ -1862,6 +1877,16 @@ Box* strLStrip(BoxedString* self, Box* chars) {
         return boxStringRef(str.ltrim(chars_str));
     } else if (chars->cls == none_cls) {
         return boxStringRef(str.ltrim(" \t\n\r\f\v"));
+    } else if (isSubclass(chars->cls, unicode_cls)) {
+        PyObject* uniself = PyUnicode_FromObject((PyObject*)self);
+        PyObject* res;
+        if (uniself == NULL)
+            throwCAPIException();
+        res = _PyUnicode_XStrip((PyUnicodeObject*)uniself, LEFTSTRIP, chars);
+        if (!res)
+            throwCAPIException();
+        Py_DECREF(uniself);
+        return res;
     } else {
         raiseExcHelper(TypeError, "lstrip arg must be None, str or unicode");
     }
@@ -1876,6 +1901,16 @@ Box* strRStrip(BoxedString* self, Box* chars) {
         return boxStringRef(str.rtrim(chars_str));
     } else if (chars->cls == none_cls) {
         return boxStringRef(str.rtrim(" \t\n\r\f\v"));
+    } else if (isSubclass(chars->cls, unicode_cls)) {
+        PyObject* uniself = PyUnicode_FromObject((PyObject*)self);
+        PyObject* res;
+        if (uniself == NULL)
+            throwCAPIException();
+        res = _PyUnicode_XStrip((PyUnicodeObject*)uniself, RIGHTSTRIP, chars);
+        if (!res)
+            throwCAPIException();
+        Py_DECREF(uniself);
+        return res;
     } else {
         raiseExcHelper(TypeError, "rstrip arg must be None, str or unicode");
     }
@@ -2670,9 +2705,12 @@ void setupStr() {
     str_cls->giveAttr("swapcase", new BoxedFunction(boxRTFunction((void*)strSwapcase, STR, 1)));
     str_cls->giveAttr("upper", new BoxedFunction(boxRTFunction((void*)strUpper, STR, 1)));
 
-    str_cls->giveAttr("strip", new BoxedFunction(boxRTFunction((void*)strStrip, STR, 2, 1, false, false), { None }));
-    str_cls->giveAttr("lstrip", new BoxedFunction(boxRTFunction((void*)strLStrip, STR, 2, 1, false, false), { None }));
-    str_cls->giveAttr("rstrip", new BoxedFunction(boxRTFunction((void*)strRStrip, STR, 2, 1, false, false), { None }));
+    str_cls->giveAttr("strip",
+                      new BoxedFunction(boxRTFunction((void*)strStrip, UNKNOWN, 2, 1, false, false), { None }));
+    str_cls->giveAttr("lstrip",
+                      new BoxedFunction(boxRTFunction((void*)strLStrip, UNKNOWN, 2, 1, false, false), { None }));
+    str_cls->giveAttr("rstrip",
+                      new BoxedFunction(boxRTFunction((void*)strRStrip, UNKNOWN, 2, 1, false, false), { None }));
 
     str_cls->giveAttr("capitalize", new BoxedFunction(boxRTFunction((void*)strCapitalize, STR, 1)));
     str_cls->giveAttr("title", new BoxedFunction(boxRTFunction((void*)strTitle, STR, 1)));

src/runtime/types.cpp

@@ -1153,6 +1153,8 @@ public:
 
     DEFAULT_CLASS(attrwrapper_cls);
 
+    Box* getUnderlying() { return b; }
+
     static void gcHandler(GCVisitor* v, Box* b) {
         boxGCHandler(v, b);
 
@@ -1215,6 +1217,25 @@ public:
         return r;
     }
 
+    static Box* pop(Box* _self, Box* _key, Box* default_) {
+        RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
+        AttrWrapper* self = static_cast<AttrWrapper*>(_self);
+
+        _key = coerceUnicodeToStr(_key);
+
+        RELEASE_ASSERT(_key->cls == str_cls, "");
+        BoxedString* key = static_cast<BoxedString*>(_key);
+        Box* r = self->b->getattr(key->s);
+        if (r) {
+            self->b->delattr(key->s, NULL);
+            return r;
+        } else {
+            if (default_)
+                return default_;
+            raiseExcHelper(KeyError, "'%s'", key->data());
+        }
+    }
+
     static Box* delitem(Box* _self, Box* _key) {
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
@@ -1401,6 +1422,11 @@ Box* makeAttrWrapper(Box* b) {
     return new AttrWrapper(b);
 }
 
+Box* unwrapAttrWrapper(Box* b) {
+    assert(b->cls == attrwrapper_cls);
+    return static_cast<AttrWrapper*>(b)->getUnderlying();
+}
+
 Box* attrwrapperKeys(Box* b) {
     return AttrWrapper::keys(b);
 }
@@ -2201,6 +2227,8 @@ void setupRuntime() {
     slice_cls->freeze();
 
     attrwrapper_cls->giveAttr("__setitem__", new BoxedFunction(boxRTFunction((void*)AttrWrapper::setitem, UNKNOWN, 3)));
+    attrwrapper_cls->giveAttr(
+        "pop", new BoxedFunction(boxRTFunction((void*)AttrWrapper::pop, UNKNOWN, 3, 1, false, false), { NULL }));
     attrwrapper_cls->giveAttr("__getitem__", new BoxedFunction(boxRTFunction((void*)AttrWrapper::getitem, UNKNOWN, 2)));
     attrwrapper_cls->giveAttr("__delitem__", new BoxedFunction(boxRTFunction((void*)AttrWrapper::delitem, UNKNOWN, 2)));
     attrwrapper_cls->giveAttr("setdefault",

src/runtime/types.h

@@ -784,6 +784,7 @@ Box* objectNewNoArgs(BoxedClass* cls);
 Box* objectSetattr(Box* obj, Box* attr, Box* value);
 
 Box* makeAttrWrapper(Box* b);
+Box* unwrapAttrWrapper(Box* b);
 Box* attrwrapperKeys(Box* b);
 
 #define SystemError ((BoxedClass*)PyExc_SystemError)

django @ 634f4229

+Subproject commit 634f4229c5cafeb3a1c03e5deb9434d7c0f74ebe

test/integration/django_test.py

+# run_args: -x
+# - pypa currently has a couple issues with django (set literals with trailing commas, nested attribute names)
+# TODO remove that directive, and also remove it from the subprocess commands down below.
+
+import os
+import subprocess
+import sys
+
+EXTRA_PATH = os.path.dirname(__file__) + "/django"
+sys.path.insert(0, EXTRA_PATH)
+
+from django.core.management import execute_from_command_line
+
+import os
+import shutil
+
+if os.path.exists("testsite"):
+    print "Removing the existing 'testsite/' directory"
+    shutil.rmtree("testsite")
+
+try:
+    sys.argv += ["startproject", "testsite"]
+    print "Running 'startproject testsite'"
+    r = execute_from_command_line()
+    assert not r
+
+    print "Running testsite/manage.py migrate"
+    env = dict(os.environ)
+    env["PYTHONPATH"] = env.get("PYTHONPATH", "") + ":" + EXTRA_PATH
+    subprocess.check_call([sys.executable, "-x", "testsite/manage.py", "migrate"], env=env)
+    # subprocess.check_call([sys.executable, "testsite/manage.py", "runserver", "--noreload"])
+
+finally:
+    pass
+    # shutil.rmtree("testsite")

test/tests/unicode_test.py

@@ -144,3 +144,8 @@ class C(object):
 print [C()], set([C()]), {1:C()}
 
 print "hello" + u"world"
+
+s = " \thello world\t "
+for m in str.strip, str.lstrip, str.rstrip:
+    for args in [], [" "], [u" "]:
+        print repr(m(s, *args))