Merge pull request #524 from kmod/sqlalchemy_tests2

Sqlalchemy tests2

from_cpython/Lib/inspect.py

@@ -818,6 +818,11 @@ def getargspec(func):
         func = func.im_func
     if not isfunction(func):
         raise TypeError('{!r} is not a Python function'.format(func))
+    # Pyston change: many of our builtin functions are of type FunctionType, but
+    # don't have full introspection available.  I think this check catches most of them,
+    # though I think it allows some cases that CPython does not:
+    if func.func_code.co_argcount > len(func.func_code.co_varnames):
+        raise TypeError('{!r} is not a Python function'.format(func))
     args, varargs, varkw = getargs(func.func_code)
     return ArgSpec(args, varargs, varkw, func.func_defaults)
 

from_cpython/Lib/pickle.py

@@ -651,6 +651,8 @@ class Pickler:
     dispatch[DictionaryType] = save_dict
     if not PyStringMap is None:
         dispatch[PyStringMap] = save_dict
+    # Pyston change:
+    dispatch[AttrwrapperType] = save_dict
 
     def _batch_setitems(self, items):
         # Helper to batch up SETITEMS sequences; proto >= 1 only
@@ -772,6 +774,11 @@ class Pickler:
     dispatch[BuiltinFunctionType] = save_global
     dispatch[TypeType] = save_global
 
+    # Pyston change: extension functions have a different type from our
+    # builtin functions (which is BuiltinFunctionType):
+    import math
+    dispatch[type(math.sin)] = save_global
+
 # Pickling helpers
 
 def _keep_alive(x, memo):

from_cpython/Lib/types.py

@@ -81,6 +81,9 @@ EllipsisType = type(Ellipsis)
 # DictProxyType = type(TypeType.__dict__)
 NotImplementedType = type(NotImplemented)
 
+# Pyston change:
+AttrwrapperType = type(_C().__dict__)
+
 # For Jython, the following two types are identical
 # Pyston change: don't support these yet
 # GetSetDescriptorType = type(FunctionType.func_code)

src/capi/abstract.cpp

@@ -1621,8 +1621,12 @@ extern "C" PyObject* PyNumber_Absolute(PyObject* o) noexcept {
 }
 
 extern "C" PyObject* PyNumber_Invert(PyObject* o) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return nullptr;
+    try {
+        return unaryop(o, AST_TYPE::Invert);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return nullptr;
+    }
 }
 
 extern "C" PyObject* PyNumber_Lshift(PyObject* lhs, PyObject* rhs) noexcept {

src/capi/types.h

@@ -89,6 +89,14 @@ public:
         return rtn;
     }
 
+    static Box* getname(Box* b, void*) {
+        RELEASE_ASSERT(b->cls == capifunc_cls, "");
+        const char* s = static_cast<BoxedCApiFunction*>(b)->name;
+        if (s)
+            return boxStrConstant(s);
+        return None;
+    }
+
     static Box* callInternal(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1,
                              Box* arg2, Box* arg3, Box** args, const std::vector<const std::string*>* keyword_names);
 };

src/codegen/ast_interpreter.cpp

@@ -814,7 +814,14 @@ Value ASTInterpreter::visit_makeClass(AST_MakeClass* mkclass) {
     for (AST_expr* d : node->decorator_list)
         decorators.push_back(visit_expr(d).o);
 
-    BoxedClosure* closure = scope_info->takesClosure() ? created_closure : 0;
+    BoxedClosure* closure = NULL;
+    if (scope_info->takesClosure()) {
+        if (this->scope_info->passesThroughClosure())
+            closure = passed_closure;
+        else
+            closure = created_closure;
+        assert(closure);
+    }
     CLFunction* cl = wrapFunction(node, nullptr, node->body, source_info);
 
     Box* passed_globals = NULL;

src/codegen/irgen/irgenerator.cpp

@@ -1273,7 +1273,12 @@ private:
         // TODO duplication with _createFunction:
         CompilerVariable* created_closure = NULL;
         if (scope_info->takesClosure()) {
-            created_closure = symbol_table[internString(CREATED_CLOSURE_NAME)];
+            if (irstate->getScopeInfo()->createsClosure()) {
+                created_closure = symbol_table[internString(CREATED_CLOSURE_NAME)];
+            } else {
+                assert(irstate->getScopeInfo()->passesThroughClosure());
+                created_closure = symbol_table[internString(PASSED_CLOSURE_NAME)];
+            }
             assert(created_closure);
         }
 

src/gc/heap.cpp

@@ -95,12 +95,13 @@ inline void sweepList(ListT* head, std::list<Box*, StlCompatAllocator<Box*>>& we
             cur = cur->next;
         } else {
             if (_doFree(al, &weakly_referenced)) {
-
                 removeFromLL(cur);
 
                 auto to_free = cur;
                 cur = cur->next;
                 free_func(to_free);
+            } else {
+                cur = cur->next;
             }
         }
     }

src/runtime/capi.cpp

@@ -1392,6 +1392,8 @@ void setupCAPI() {
     auto capi_call = new BoxedFunction(boxRTFunction((void*)BoxedCApiFunction::__call__, UNKNOWN, 1, 0, true, true));
     capi_call->f->internal_callable = BoxedCApiFunction::callInternal;
     capifunc_cls->giveAttr("__call__", capi_call);
+    capifunc_cls->giveAttr("__name__",
+                           new (pyston_getset_cls) BoxedGetsetDescriptor(BoxedCApiFunction::getname, NULL, NULL));
 
     capifunc_cls->freeze();
 

src/runtime/dict.cpp

@@ -250,10 +250,7 @@ extern "C" PyObject* PyDict_GetItem(PyObject* dict, PyObject* key) noexcept {
     ASSERT(isSubclass(dict->cls, dict_cls) || dict->cls == attrwrapper_cls, "%s", getTypeName(dict));
     if (isSubclass(dict->cls, dict_cls)) {
         BoxedDict* d = static_cast<BoxedDict*>(dict);
-        auto it = d->d.find(key);
-        if (it != d->d.end())
-            return it->second;
-        return NULL;
+        return d->getOrNull(key);
     }
 
     // This path doesn't exist in CPython; we have it to support extension modules that do

src/runtime/objmodel.cpp

@@ -141,6 +141,9 @@ size_t PyHasher::operator()(Box* b) const {
 bool PyEq::operator()(Box* lhs, Box* rhs) const {
     STAT_TIMER(t0, "us_timer_PyEq");
 
+    if (lhs == rhs)
+        return true;
+
     if (lhs->cls == rhs->cls) {
         if (lhs->cls == str_cls) {
             return static_cast<BoxedString*>(lhs)->s == static_cast<BoxedString*>(rhs)->s;
@@ -149,8 +152,7 @@ bool PyEq::operator()(Box* lhs, Box* rhs) const {
 
     // TODO fix this
     Box* cmp = compareInternal(lhs, rhs, AST_TYPE::Eq, NULL);
-    assert(cmp->cls == bool_cls);
-    return cmp == True;
+    return cmp->nonzeroIC();
 }
 
 bool PyLt::operator()(Box* lhs, Box* rhs) const {
@@ -158,8 +160,7 @@ bool PyLt::operator()(Box* lhs, Box* rhs) const {
 
     // TODO fix this
     Box* cmp = compareInternal(lhs, rhs, AST_TYPE::Lt, NULL);
-    assert(cmp->cls == bool_cls);
-    return cmp == True;
+    return cmp->nonzeroIC();
 }
 
 extern "C" Box* deopt(AST_expr* expr, Box* value) {

test/integration/sqlalchemy_smalltest.py

+# run_args: -x
+
+import os
+import sys
+import subprocess
+import traceback
+
+ENV_NAME = "sqlalchemy_test_env_" + os.path.basename(sys.executable)
+
+if not os.path.exists(ENV_NAME) or os.stat(sys.executable).st_mtime > os.stat(ENV_NAME + "/bin/python").st_mtime:
+    print "Creating virtualenv to install testing dependencies..."
+    VIRTUALENV_SCRIPT = os.path.dirname(__file__) + "/virtualenv/virtualenv.py"
+
+    try:
+        args = [sys.executable, VIRTUALENV_SCRIPT, "-p", sys.executable, ENV_NAME]
+        print "Running", args
+        subprocess.check_call(args)
+
+        subprocess.check_call([ENV_NAME + "/bin/pip", "install", "mock", "pytest"])
+    except:
+        print "Error occurred; trying to remove partially-created directory"
+        ei = sys.exc_info()
+        try:
+            subprocess.check_call(["rm", "-rf", ENV_NAME])
+        except Exception as e:
+            print e
+        raise ei[0], ei[1], ei[2]
+
+# subprocess.check_call([os.path.abspath("sqlalchemy_test_env/bin/python"), "-c", "import py; print type(py); print py.builtin"])
+
+SQLALCHEMY_DIR = os.path.dirname(__file__) + "/sqlalchemy"
+TEST_DIR = SQLALCHEMY_DIR + "/test"
+python_exe = os.path.abspath(ENV_NAME + "/bin/python")
+
+sys.path.append(SQLALCHEMY_DIR + "/lib")
+sys.path.insert(0, SQLALCHEMY_DIR)
+sys.path.append(ENV_NAME + "/site-packages")
+sys.path.append(ENV_NAME + "/lib/python2.7/site-packages")
+
+# make sure this is importable:
+import mock
+
+import sqlalchemy.testing
+class Requirements(object):
+    def __getattr__(self, n):
+        def inner(f):
+            return f
+        inner.not_ = lambda: inner
+        return inner
+sqlalchemy.testing.config.requirements = sqlalchemy.testing.requires = Requirements()
+
+import glob
+test_files = glob.glob(TEST_DIR + "/test*.py") + glob.glob(TEST_DIR + "/*/test*.py")
+
+# These are the ones that pass on CPython (ie that we've stubbed enough of their testing
+# infrastructure to run):
+MODULES_TO_TEST = ['test.engine.test_parseconnect', 'test.ext.test_compiler', 'test.dialect.test_pyodbc', 'test.dialect.test_sybase', 'test.dialect.test_mxodbc', 'test.sql.test_inspect', 'test.sql.test_operators', 'test.sql.test_ddlemit', 'test.sql.test_cte', 'test.base.test_dependency', 'test.base.test_except', 'test.base.test_inspect', 'test.base.test_events', 'test.orm.test_inspect', 'test.orm.test_descriptor']
+
+# These are currently broken on Pyston:
+MODULES_TO_TEST.remove("test.sql.test_operators")
+MODULES_TO_TEST.remove("test.base.test_events")
+MODULES_TO_TEST.remove("test.orm.test_descriptor")
+
+passed = []
+failed = []
+
+for fn in test_files:
+    assert fn.startswith(TEST_DIR + '/')
+    assert fn.endswith(".py")
+    mname = fn[len(SQLALCHEMY_DIR) + 1:-3].replace('/', '.')
+    if mname not in MODULES_TO_TEST:
+        continue
+    print
+    print mname
+
+    try:
+        m = __import__(mname, fromlist=["__all__"])
+        for nname in dir(m):
+            n = getattr(m, nname)
+            if not nname.endswith("Test") or not isinstance(n, type):
+                continue
+            print "Running", n
+
+            n = n()
+            for t in dir(n):
+                if not t.startswith("test_"):
+                    continue
+                print "Running", t
+                n.setup()
+                getattr(n, t)()
+                n.teardown()
+    except Exception:
+        print mname, "FAILED"
+        traceback.print_exc()
+        failed.append(mname)
+    else:
+        print mname, "PASSED"
+        passed.append(mname)
+
+print "passing:", passed
+print "failing:", failed
+
+print
+if failed:
+    print "FAILED"
+    sys.exit(1)
+else:
+    print "PASSED"

test/tests/closure_test.py

@@ -69,3 +69,16 @@ def f6():
 
     print (lambda a=1: x)()
 f6()
+
+
+# Regression test: make sure we can handle two pass-through-closure frames in a row
+def f7(n):
+    class C(object):
+        class D(object):
+            def foo(self):
+                return n
+
+    return C.D
+
+c = f7(5)()
+print c.foo()

test/tests/comparisons.py

@@ -96,3 +96,20 @@ class EqOnly(object):
 
 print EqOnly() == 1
 print EqOnly() != 1
+
+
+class NonboolEq(object):
+    def __init__(self, n):
+        self.n = n
+    def __eq__(self, rhs):
+        return 2 if self.n == rhs.n else ()
+    def __hash__(self):
+        return 0
+
+print NonboolEq(1) == NonboolEq(2)
+print NonboolEq(1) == NonboolEq(True)
+
+d = {}
+for i in xrange(20):
+    d[NonboolEq(i % 10)] = i
+print len(d), sorted(d.values())

test/tests/dict_internals.py

@@ -28,3 +28,13 @@ d.__setitem__(c2, 2)
 d.__setitem__(c3, 3)
 
 print d
+
+
+# dicts need to check identify and not just equality.
+# This is important for sqlalchemy where equality constructs a sql equals clause and doesn't
+# do comparison of the objects at hand.
+d = {}
+nan = float('nan')
+d[nan] = "hello world"
+print d[nan]
+

test/tests/inspect_test.py

@@ -5,8 +5,21 @@ def f1(a, b=2, *args, **kw):
 def f2():
     pass
 
+class C(object):
+    def __init__(self):
+        pass
+
+class D(object):
+    pass
+
 print inspect.getargspec(f1)
 print inspect.getargspec(f2)
+print inspect.getargspec(C.__init__)
+print inspect.getargspec(C().__init__)
+try:
+    print inspect.getargspec(D.__init__)
+except Exception as e:
+    print type(e)
 
 def G():
     yield 1

test/tests/math_test.py

@@ -18,3 +18,5 @@ print min(range(5))
 
 for x in [float("inf"), math.pi]:
     print x, math.isinf(x), math.fabs(x), math.ceil(x), math.log(x), math.log10(x)
+
+print math.sqrt.__name__

test/tests/no_newline.py

+# fail-if: '-x' not in EXTRA_JIT_ARGS
+class C(object):
+    """
+    """
\ No newline at end of file

test/tests/operator__name__.py

-# expected: fail
-import operator
-
-for op in sorted(dir(operator)):
-    if op.startswith("_"):
-        continue
-    print getattr(operator, op).__name__

test/tests/operator_test.py

@@ -16,3 +16,7 @@ f = operator.attrgetter("count")
 print f(["a", "a"])("a")
 print f("ababa")("a")
 
+for op in sorted(dir(operator)):
+    if op.startswith("_"):
+        continue
+    print getattr(operator, op).__name__

test/tests/pickle_test.py

@@ -10,3 +10,15 @@ l3 = l2.pop()
 print l2, l3, l2 is l3
 
 print pickle.loads(pickle.dumps("hello world"))
+
+# Sqlalchemy wants this:
+import operator
+print repr(pickle.dumps(len))
+print repr(pickle.dumps(operator.and_))
+
+class C(object):
+    pass
+c = C()
+c.a = 1
+print repr(pickle.dumps(c))
+print pickle.loads(pickle.dumps(c)).a

test/tests/set.py

@@ -104,7 +104,32 @@ print s
 s.discard(1)
 print s
 
-s = set(range(5))
+
+s = set(range(10))
+print s.difference_update(range(-3, 2), range(7, 23))
+print sorted(s)
+
+
+# Check set subclassing:
+
+class MySet(set):
+    pass
+class MyFrozenset(frozenset):
+    pass
+
+compare_to = []
 for i in xrange(10):
-    s2 = set(range(i))
-    print s.issubset(s2), s.issuperset(s2), s == s2, s != s2, s.difference(s2), s.issubset(range(i)), s.issuperset(range(i))
+    compare_to.append(set(range(i)))
+    compare_to.append(frozenset(range(i)))
+    compare_to.append(MySet(range(i)))
+    compare_to.append(MyFrozenset(range(i)))
+    compare_to.append(range(i))
+    compare_to.append(range(i, 10))
+
+for s1 in set(range(5)), frozenset(range(5)):
+    for s2 in compare_to:
+        print type(s2), sorted(s2), s.issubset(s2), s.issuperset(s2), s == s2, s != s2, s.difference(s2), s.isdisjoint(s2), sorted(s1.union(s2)), sorted(s1.intersection(s2))
+
+f = float('nan')
+s = set([f])
+print f in s, f == list(s)[0]

test/tests/str_adding.py

+# expected: fail
+
+# str has some weird adding behavior that cannot be replicated with Python code.
+# Usually, if an __add__ throws an error, then __radd__ isn't tried.
+# But string addition is defined in sq_concat, which is tried after __add__ and
+# __radd__.  And string addition will throw a TypeError.
+
+class C(object):
+    def __add__(self, rhs):
+        raise TypeError("dont support add")
+
+class D(object):
+    def __radd__(self, lhs):
+        return 3.14
+
+try:
+    print "".__add__(D())
+except TypeError as e:
+    print e
+try:
+    print C() + D()
+except TypeError as e:
+    print e
+print "" + D()

test/tests/weakref4.py

 import weakref
-import gc
 
 def doStuff():
   def meth():
@@ -16,5 +15,18 @@ def fact(n):
 
 w = doStuff()
 print fact(10) # try to clear some memory
+
+# Try creating a large object to make sure we can handle them:
+def f():
+    class C(object):
+        # Adding a __slots__ directive increases the size of the type object:
+        __slots__ = ['a' + str(i) for i in xrange(1000)]
+    return weakref.ref(C)
+
+
+r = f()
+
+import gc
 gc.collect()
-print w()
+assert r() is None, "object was not collected"
+assert w() is None, "object was not collected"