Remove "skipped" tests and instead make them all expected failures.

Except for the few that were actually working, which became
normal tests again.

src/codegen/irgen/irgenerator.cpp

@@ -1242,7 +1242,7 @@ class IRGeneratorImpl : public IRGenerator {
             if (state == PARTIAL)
                 return;
 
-            assert(node->dest == NULL);
+            RELEASE_ASSERT(node->dest == NULL, "");
             for (int i = 0; i < node->values.size(); i++) {
                 if (i > 0) {
                     emitter.getBuilder()->CreateCall(g.funcs.printf, getStringConstantPtr(" "));

test/tests/30.py

 # run_args: -n
-# statcheck: 1 <= stats["OSR exits"] <= 2
+# statcheck: ("-O" in EXTRA_JIT_ARGS) or (1 <= stats["OSR exits"] <= 2)
 # statcheck: stats['slowpath_binop'] <= 10
 
 x = 100000

test/tests/46.py

+# expected: fail
+# - metaclasses
+
 # Metaclass test:
 
 class M(type):

test/tests/54.py

@@ -22,7 +22,7 @@ print type(1)
 
 # As evidenced by the following real + important example:
 # type() is a special function that gets the type of an object:
-print type(C) # prints "<type 'type'>"
+print type(C) is type
 # but type.__call__() is the clsattr for any (non-metaclassed) class,
 # which is how object creation gets handled:
-print repr(type.__call__(C))[:20] # prints "<__main__.C object at 0xXXXXXXX>"
+print type.__call__(C) is type

test/tests/55.py

+# expected: fail
+# - metaclasses
+
 # attr-getting resolution.
 
 class M(type):

test/tests/56.py

+# expected: fail
+# - arbitrary stuff in classes
+
 # I guess type.__name__ works specially:
 
 class C(object):

test/tests/57.py

+# expected: fail
+# - lambdas, varargs
+
 # Testing im-boxing:
 
 class C(object):

test/tests/61.py

+# expected: fail
+# - varargs
+
 # varargs (but not starargs yet)
 
 def f(*args):

test/tests/66.py

 # Make sure that overriding __file__ doesn't change the traceback
+# TODO the tester doesn't currently check the traceback
 
 import sys
 m = sys.modules['__main__']

test/tests/69.py

+# expected: fail
+# - I'm not sure how this is supposed to work
+
 # I guess the __name__ attribute on classes is weird?
 
 class C(object):

test/tests/70.py

+# expected: fail
+
 print type.__call__.__name__ # should print "__call__" (w/o quotes)

test/tests/72.py

+# expected: fail
+# - arbitrary stuff in classdefs
+
 # objmodel classattrs (like __add__) can be non-functions, so might not get bound into instancemethods:
 
 class Adder(object):

test/tests/binops_subclass.py

+# expected: fail
+# - this particular check isn't implemented yet
+
 class M(type):
     def __instancecheck__(self, rhs):
         print "M.instancecheck",

test/tests/class_changing.py

+# expected: fail
+# - class changing not supported yet
+
 # Tests to make sure that setting __class__ changes the class, and that it's ok to disallow
 # having anything other than a type as the class
 class C(object):

test/tests/class_freeing_time.py

+# expected: fail
+# - __del__ not supported
+# probably need to have some gc collections going on as well
+
 # Classes should be freed right away
 
 class DeallocShower(object):

test/tests/class_noctor.py

+# expected: fail
+# - don't support object.__new__ yet
+
 # Regression test:
 # If the init function doesn't exist, shouldn't just silently ignore any args
 # that got passed

test/tests/classdef_arbitrary.py

+# expected: fail
+# - arbitrary stuff in classes
+
 # You can put arbitrary stuff in class definitions, which end up being added as class attributes
 
 class C(object):

test/tests/comparisons_more.py

 print (1, 2) < (1, 3)
+print (1, 4) < (1, 3)
 print [1, 2] < [1, 3]
 print {1:2} < {1:3}

test/tests/finalization_cycles.expected

+[1, 2]

test/tests/finalization_cycles.py

+# expected: fail
+#  - finalization not implemented yet
+
 # Finalizers should be called before any objects are deallocated
 # Note: the behavior here will differ from cPython and maybe PyPy
 

test/tests/for_iter.py

+# expected: fail
+# - real iteration protocol is unsupported (no exceptions yet)
+
 class C(object):
     def __iter__(self):
         print "orig iter"

test/tests/global_and_local.py

+# expected: fail
+# - this particular check isn't implemented yet
+
 # I would have expected this to be valid, but cPython and pypy err out saying "name 'x' is local and global"
-# Interestingly, this SyntaxError gets thrown *after* AST creation, which I guess makes sense but I wasn't
-# expecting
+
+print "first"
 
 x = 1
 def f(x):

test/tests/listcomp_osr.py

 # While perhaps not required in practice, we should have the ability to
 # OSR from inside a list comprehension.
-# statcheck: stats['OSR exits'] in (1, 2)
+# statcheck: ("-O" in EXTRA_JIT_ARGS) or (1 <= stats["OSR exits"] <= 2)
 
 def p(i):
     print i

test/tests/many_attrs_setattr.py

+# expected: fail
+# - setattr() not supported
+
 class C(object):
     pass
 

test/tests/math_more.py

+# expected: fail
+# - min/max on iterables
+
+import math
 
 print max(range(5))
 print min(range(5))

test/tests/metaclass_parent.py

+# expected: fail
+# - metaclasses, inheritance
+
 # This would make a good Python quiz:
 
 sl = slice(1,2)

test/tests/non_function_ctors.py

+# expected: fail
+# - not implemented yet
+
 class CallableNew(object):
     def __call__(self, cls, arg):
         print "new", cls, arg

test/tests/nondirectly_callable_ics.py

+# expected: statfail
+# - callable not being patched like this test checks for
+#
 # run_args: -n
 # statcheck: stats['slowpath_runtimecall'] < 10
 

test/tests/none_not_settable.py

+# expected: fail
+# setattr() not implemented
+
 class C(object):
     def print_none(self):
         print None

test/tests/nonself_resurrection.py

+# expected: fail
+#  - finalization (let alone resurrection) not implemented yet
+
 # Objects are allowed to resurrect other objects too, I guess
 
 class C(object):

test/tests/resurrection.py

+# expected: fail
+#  - finalization (let alone resurrection) not implemented yet
+
 # Objects are allowed to resurrect themselves in their __del__ methods...
 # Note: the behavior here will differ from cPython and maybe PyPy
 

test/tests/return_in_class.py

+# expected: fail
+# - arbitrary stuff in classes
+
 class C(object):
     return

test/tests/scoping_classes.py

+# expected: fail
+# - arbitrary stuff in classes
 
 X = 0
 Y = 0

test/tests/setattr_patching_under.py

+# expected: fail
+# - finalization not implemented yet
+# This test might also be broken in the presence of GC
+
 class C(object):
     pass
 

test/tests/xrange.py

 # run_args: -n
-# statcheck: stats['slowpath_getclsattrint'] <= 20
+#
+# statcheck: "-O" in EXTRA_JIT_ARGS or 'slowpath_getclsattr' in stats or 'slowpath_callattr' in stats
+# statcheck: stats.get('slowpath_getclsattr', 0) <= 20
+# statcheck: stats.get('slowpath_callattr', 0) <= 20
 
 for i in xrange(1000):
     print i

tools/tester.py

@@ -91,7 +91,7 @@ def run_test(fn, check_stats, run_memcheck):
         elif l.startswith("# expected:"):
             expected = l[len("# run_args:"):].strip()
 
-    assert expected in ("success", "fail"), expected
+    assert expected in ("success", "fail", "statfail"), expected
 
     run_args = ["./%s" % IMAGE] + jit_args + ["-q", fn]
     start = time.time()
@@ -173,7 +173,10 @@ def run_test(fn, check_stats, run_memcheck):
         for l in statchecks:
             test = eval(l)
             if not test:
-                if KEEP_GOING:
+                if expected == "statfail":
+                    r += "    (expected statfailure)"
+                    break
+                elif KEEP_GOING:
                     r += "    \033[31mFailed statcheck\033[0m"
                     failed.append(fn)
                     return r
@@ -183,6 +186,15 @@ def run_test(fn, check_stats, run_memcheck):
                         statname = m.group(1)
                         raise Exception((l, statname, stats[statname]))
                     raise Exception((l, stats))
+        else:
+            # only can get here if all statchecks passed
+            if expected == "statfail":
+                if KEEP_GOING:
+                    r += "    \033[31mUnexpected statcheck success\033[0m"
+                    failed.append(fn)
+                    return r
+                else:
+                    raise Exception(("Unexpected statcheck success!", statchecks, stats))
     else:
         r += "    (ignoring stats)"
 
@@ -310,34 +322,6 @@ if __name__ == "__main__":
     skip = functools.partial(_addto, TOSKIP)
     nostat = functools.partial(_addto, IGNORE_STATS)
 
-    if '-O' in EXTRA_JIT_ARGS:
-        # OSR tests, doesn't make sense with -O
-        skip(["30", "listcomp_osr"])
-
-    if datetime.datetime.now() < datetime.datetime(2014,4,29):
-        nostat(["nondirectly_callable_ics"]) # WIP
-        skip(["finalization_cycles", "resurrection", "nonself_resurrection"]) # WIP
-
-    if datetime.datetime.now() < datetime.datetime(2014,4,29):
-        skip(["class_noctor", "non_function_ctors"]) # object.__new__
-        skip(["setattr_patching_under"]) # requires __del__
-
-    if datetime.datetime.now() < datetime.datetime(2014,5,1):
-        # varargs
-        skip([57, 61])
-        # arbitrary stuff in classes
-        skip([56, "classdef_arbitrary", "scoping_classes", "return_in_class"])
-        # sequence comparisons
-        skip(["comparisons_more"])
-
-    if datetime.datetime.now() < datetime.datetime(2014,5,1):
-        # Metaclass tests
-        skip([46, 55])
-
-    if datetime.datetime.now() < datetime.datetime(2014,5,1):
-        # random tests that aren't too important right now:
-        skip([54, 65, 66, 69, 70, 72, "many_attrs_setattr", "for_iter", "none_not_settable", "math_more", "global_and_local", "metaclass_parent", "class_freeing_time", "xrange", "binops_subclass", "class_changing"])
-
     if not patterns:
         skip(["t", "t2"])