Update our CPython testsuite notes

We added a couple of features that the tests used to be running
into, such as exec and extslice.  Now a lot of things are blocking
on doctest, for which the main difficulty is compile().

libpypa @ 8f62d5d7

-Subproject commit 98d769dcd3a792f70b5e9c9bb9afa7544a50ae4a
+Subproject commit 8f62d5d7440cff89f98ed5a2e7756321811a0f2d

src/codegen/irgen/hooks.cpp

@@ -382,7 +382,7 @@ Box* eval(Box* boxedCode) {
     // TODO error message if parse fails or if it isn't an expr
     // TODO should have a cleaner interface that can parse the Expression directly
     // TODO this memory leaks
-    RELEASE_ASSERT(boxedCode->cls == str_cls, "");
+    RELEASE_ASSERT(boxedCode->cls == str_cls, "%s", boxedCode->cls->tp_name);
     const char* code = static_cast<BoxedString*>(boxedCode)->s.data();
     AST_Module* parsedModule = parse_string(code);
     RELEASE_ASSERT(parsedModule->body[0]->type == AST_TYPE::Expr, "");

src/runtime/builtin_modules/sys.cpp

@@ -206,7 +206,7 @@ void prependToSysPath(const std::string& path) {
 static BoxedClass* sys_flags_cls;
 class BoxedSysFlags : public Box {
 public:
-    Box* division_warning, *bytes_warning, *no_user_site;
+    Box* division_warning, *bytes_warning, *no_user_site, *optimize;
 
     BoxedSysFlags() {
         auto zero = boxInt(0);
@@ -214,6 +214,7 @@ public:
         division_warning = zero;
         bytes_warning = zero;
         no_user_site = zero;
+        optimize = zero;
     }
 
     DEFAULT_CLASS(sys_flags_cls);
@@ -226,6 +227,7 @@ public:
         v->visit(self->division_warning);
         v->visit(self->bytes_warning);
         v->visit(self->no_user_site);
+        v->visit(self->optimize);
     }
 
     static Box* __new__(Box* cls, Box* args, Box* kwargs) {
@@ -347,6 +349,7 @@ void setupSys() {
     ADD(division_warning);
     ADD(bytes_warning);
     ADD(no_user_site);
+    ADD(optimize);
 #undef ADD
 
     sys_flags_cls->tp_mro = BoxedTuple::create({ sys_flags_cls, object_cls });

src/runtime/capi.cpp

@@ -1370,11 +1370,4 @@ void setupCAPI() {
 
 void teardownCAPI() {
 }
-
-void fatalOrError(PyObject* exception, const char* message) noexcept {
-    if (CONTINUE_AFTER_FATAL)
-        PyErr_SetString(exception, message);
-    else
-        Py_FatalError(message);
-}
 }

src/runtime/capi.h

@@ -28,8 +28,13 @@ void throwCAPIException() __attribute__((noreturn));
 struct ExcInfo;
 void setCAPIException(const ExcInfo& e);
 
-// TODO: not sure whether this belongs here
-void fatalOrError(Box* object, const char* message) noexcept;
+#define fatalOrError(exception, message)                                                                               \
+    do {                                                                                                               \
+        if (CONTINUE_AFTER_FATAL)                                                                                      \
+            PyErr_SetString((exception), (message));                                                                   \
+        else                                                                                                           \
+            Py_FatalError((message));                                                                                  \
+    } while (0)
 }
 
 #endif

src/runtime/objmodel.cpp

@@ -1961,7 +1961,7 @@ extern "C" bool nonzero(Box* obj) {
         ASSERT(isUserDefined(obj->cls) || obj->cls == classobj_cls || obj->cls == type_cls
                    || isSubclass(obj->cls, Exception) || obj->cls == file_cls || obj->cls == traceback_cls
                    || obj->cls == instancemethod_cls || obj->cls == module_cls || obj->cls == capifunc_cls
-                   || obj->cls == builtin_function_or_method_cls,
+                   || obj->cls == builtin_function_or_method_cls || obj->cls == method_cls,
                "%s.__nonzero__", getTypeName(obj)); // TODO
 
         // TODO should rewrite these?
@@ -4389,7 +4389,7 @@ extern "C" Box* importStar(Box* _from_module, BoxedModule* to_module) {
     // it looks like mostly a matter of changing the getattr calls to getitem.
     RELEASE_ASSERT(getGlobals() == to_module, "importStar doesn't support custom globals yet");
 
-    assert(_from_module->cls == module_cls);
+    ASSERT(isSubclass(_from_module->cls, module_cls), "%s", _from_module->cls->tp_name);
     BoxedModule* from_module = static_cast<BoxedModule*>(_from_module);
 
     Box* all = from_module->getattr(all_str);

test/cpython/NOTES.org

@@ -38,11 +38,7 @@ disable this, pass --all-cpython-tests to tester.py.
 
 * bugs uncovered
 The CPython tests I've included fail for various reasons. Recurring issues include:
-- Use of sys.flags.optimize, to test whether we kept docstrings around; I
-  commented these out, since we always keep docstrings around.
 - use of compile()
-- `exec' support
-- the doctest module imports bdb, which trips an ASTVisitor assert(0) for visit_extslice
 - missing __hash__ implementations for some builtin types
 - we don't have imp.get_magic()
 - segfaults
@@ -58,41 +54,41 @@ FILE                    REASONS
 test_abstract_numbers   missing .real attribute
 test_augassign          bugs in +=, compile()
 test_bisect             somehow sys.modules['_bisect'] is getting set to 0
-test_builtin            exec
-test_coercion           serialize_ast bug
-test_collections        extslice in bdb in doctest
+test_builtin            execfile scoping issue
+test_coercion           serialize_ast bug (AST node not getting correct lineno or col_offset set)
+test_collections        compile()
 test_compare            segfault
 test_complex            serialize_ast assert
-test_complex_args       exec
+test_complex_args       we apparently don't always unpack args correctly
 test_contains           TypeError not raised
 test_contextlib         file.closed, lock.locked attributes
 test_datetime           kwargs bug in BoxedMethodDescriptor
 test_decimal            float.__getformat__
 test_decorators         compile(), func_name attribute, and another bug in test_eval_order
-test_defaultdict        assert failure in nonzero
+test_defaultdict        "False is not true"
 test_deque              assert in _collectionsmodule.c
-test_descr              pypa assert
-test_descrtut           extslice in bdb in doctest
-test_dict               exec
+test_descr              PySequence_Tuple
+test_descrtut           doctest, compile()
+test_dict               infinite recursion in dict.__repr__
 test_dictcomps          compile()
 test_dictviews          various unique bugs
-test_doctest            extslice in bdb in doctest
-test_doctest2           extslice in bdb in doctest
+test_doctest            code.co_firstlineno, compile()
+test_doctest2           code.co_firstlineno, compile()
 test_enumerate          wrong assert in BoxedEnumerate
-test_exceptions         exec
-test_extcall            extslice in bdb in doctest
+test_exceptions         segfault in dictIterNext
+test_extcall            doctest (compile())
 test_file               segfaults
 test_file2k             "Someone called abort!"
 test_file_eintr         TypeError
-test_filecmp            extslice in filecmp
+test_filecmp            float.__hash__
 test_fileinput          UnboundLocalError (try/finally bug?)
 test_float              capifunc.__name__ in fractions
 test_format             can't float(long)
-test_funcattrs          exec
+test_funcattrs          segfault
 test_functools          segfault
-test_generators         extslice in bdb in doctest
-test_genexps            extslice in bdb in doctest
-test_getopt             extslice in bdb in doctest
+test_generators         doctest (compile())
+test_genexps            doctest (compile())
+test_getopt             doctest (compile())
 test_global             compile()
 test_grammar            bug in our tokenizer
 test_hash               float.__hash__
@@ -100,45 +96,44 @@ test_index              parseSlice assert
 test_int                float.__getformat__
 test_io                 parseSlice assert
 test_isinstance         sys.getrecursionlimit
-test_json               extslice in bdb in doctest
+test_json               doctest (compile())
 test_list               assert in sliceIndex to do with a[1L:2L]
 test_long               float.__getformat__
 test_math               float.__getformat__
-test_module             exec
-test_mutants            segfault
+test_module             exec in globals_
+test_mutants            needs cmp()
 test_opcodes            TypeError: exceptions must be old-style classes or derived from BaseException, not instance
 test_operator           BoxedCApiFunction::__call__: assert(varargs->elts.size() == 1)
 test_optparse           long.cpp: _longNew: assert(r == 0)
 test_pep277             segfaults
 test_pep352             various unique bugs
-test_pkg                exec
+test_pkg                unknown bug
 test_popen              objmodel.cpp: callCLFunc: assert(chosen_cfg->spec->rtn_type->isFitBy(r->cls))
 test_pow                global name `pow' is not defined
-test_property           pypa assert
+test_property           unknown bug
 test_random             floats unhashable
 test_repr               complex.__hash__
-test_richcmp            segfaults
-test_scope              exec
+test_richcmp            some missing C API functions
+test_scope              eval of code object
 test_set                list.__hash__
-test_setcomps           extslice in bdb in doctest
-test_sets               extslice in bdb in doctest
+test_setcomps           doctest (compile())
+test_sets               doctest (compile())
 test_slice              slice.__hash__
 test_sort               listSort(): cmp not supported
 test_stat               16384 != 40960; I guess we implement stat wrong or something?
-test_str                segfault
+test_str                memory leak?
 test_string             infinite loops in test_replace
 test_subprocess         exit code 141, no error message
 test_tuple              parseSlice assert
 test_types              func_name attribute
-test_ucn                can't eval unicode objects
 test_unary              objmodel.cpp: unaryop: Assertion `attr_func' failed: str.__pos__
-test_undocumented_details   segfault
+test_undocumented_details   cmp()
 test_unicode            sys.maxunicode
 test_unicode_file       exit code 139, no error message
 test_unittest           serialize_ast assert
-test_unpack             extslice in bdb in doctest
-test_urllib             'ascii' codec can't encode characters in position 2-3: ordinal not in range(128)
-test_urllib2            extslice in bdb in doctest
+test_unpack             doctest (compile())
+test_urllib             str.join(unicode)
+test_urllib2            doctest (compile())
 test_userdict           segfault
 test_userlist           sliceIndex assert
 test_userstring         std::length_error
@@ -147,4 +142,4 @@ test_weakref            collector.cpp: runCollection: isValidGCObject(head) fail
 test_weakset            set.cpp: setIssubset: assert(container->cls == set_cls)
 test_with               objmodel.cpp: getclsattr: Assertion `gotten' failed: LacksExit:__exit__
 test_wsgiref            ast_interpreter.cpp: createFunction: Assertion `closure' failed.
-test_xrange             exit code 139, no error message
+test_xrange             xrange of long

test/tests/many_attrs_setattr.py

+# skip-if: True
+# - this test just runs until it times out and bloats test time
 # expected: fail
 # - memory explosion