Misc support for cffi and bcrypt

- Py_FatalError() has to be an expression (previously was multi-line macro)
- Add definitions of some PyObject_GC_* functions
- imp.load_dynamic

With these changes, bcrypt installs and runs

CMakeLists.txt

@@ -207,7 +207,7 @@ add_test(NAME analysis_unittest COMMAND analysis_unittest)
 add_test(NAME pyston_defaults COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
 # we pass -I to cpython tests and skip failing ones b/c they are slooow otherwise
 add_test(NAME pyston_defaults_cpython_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t30 ${CMAKE_SOURCE_DIR}/test/cpython)
-add_test(NAME pyston_defaults_integration_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t120 ${CMAKE_SOURCE_DIR}/test/integration)
+add_test(NAME pyston_defaults_integration_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t180 ${CMAKE_SOURCE_DIR}/test/integration)
 add_test(NAME pyston_max_compilation_tier COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-O -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
 add_test(NAME pyston_old_parser COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -a=-x -R ./pyston -j1 -a=-n -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
 

Makefile

@@ -495,7 +495,7 @@ check:
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests & skip failing ones because they are sloooow otherwise
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t30 $(TEST_DIR)/cpython $(ARGS)
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t120 $(TEST_DIR)/integration $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t180 $(TEST_DIR)/integration $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-n -a=-x -a=-S $(TESTS_DIR) $(ARGS)
 	@# skip -O for dbg
 
@@ -512,7 +512,7 @@ check:
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-S $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t60 $(TEST_DIR)/integration $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t120 $(TEST_DIR)/integration $(ARGS)
 	@# skip -n for dbg
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_release -j$(TEST_THREADS) -k -a=-O -a=-x -a=-S $(TESTS_DIR) $(ARGS)
 
@@ -957,7 +957,7 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1 ext_pyston
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing -t30 $(TEST_DIR)/cpython $(ARGS)
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t120 $(TEST_DIR)/integration $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t180 $(TEST_DIR)/integration $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -a=-S -k $(TESTS_DIR) $(ARGS)
 

from_cpython/Include/pydebug.h

@@ -36,13 +36,16 @@ PyAPI_DATA(int) Py_HashRandomizationFlag;
 
 // Pyston change: make Py_FatalError a macro so that it can access linenumber info, similar to assert:
 //PyAPI_FUNC(void) Py_FatalError(const char *message) __attribute__((__noreturn__)) PYSTON_NOEXCEPT;
+PyAPI_FUNC(void) _Py_FatalError(const char *fmt, const char *function, const char *message)
+    __attribute__((__noreturn__));
 #define _PYSTON_STRINGIFY(N) #N
 #define PYSTON_STRINGIFY(N) _PYSTON_STRINGIFY(N)
-#define Py_FatalError(message)                                                                                          \
-do {                                                                                                                    \
-    fprintf(stderr, __FILE__ ":" PYSTON_STRINGIFY(__LINE__) ": %s: Fatal Python error: %s\n", __PRETTY_FUNCTION__, message);   \
-    abort();                                                                                                            \
-} while (0)
+// Defer the real work of Py_FatalError to a function, since some users expect it to be a real function,
+// ie can be used as an expression (a do-while(0) loop would fail that).
+#define Py_FatalError(message)                                                 \
+  _Py_FatalError(__FILE__                                                      \
+                 ":" PYSTON_STRINGIFY(__LINE__) ": %s: Fatal Python error: %s\n",        \
+                 __PRETTY_FUNCTION__, message)
 
 #ifdef __cplusplus
 }

src/runtime/capi.cpp

@@ -1336,6 +1336,44 @@ static Box* methodGetDoc(Box* b, void*) {
     return None;
 }
 
+/* extension modules might be compiled with GC support so these
+   functions must always be available */
+
+#undef PyObject_GC_Track
+#undef PyObject_GC_UnTrack
+#undef PyObject_GC_Del
+#undef _PyObject_GC_Malloc
+
+extern "C" PyObject* _PyObject_GC_Malloc(size_t basicsize) noexcept {
+    Box* r = ((PyObject*)PyObject_MALLOC(basicsize));
+    RELEASE_ASSERT(gc::isValidGCObject(r), "");
+    return r;
+}
+
+#undef _PyObject_GC_New
+extern "C" PyObject* _PyObject_GC_New(PyTypeObject* tp) noexcept {
+    PyObject* op = _PyObject_GC_Malloc(_PyObject_SIZE(tp));
+    if (op != NULL)
+        op = PyObject_INIT(op, tp);
+    RELEASE_ASSERT(gc::isValidGCObject(op), "");
+    return op;
+}
+
+extern "C" PyVarObject* _PyObject_GC_NewVar(PyTypeObject* tp, Py_ssize_t nitems) noexcept {
+    const size_t size = _PyObject_VAR_SIZE(tp, nitems);
+    PyVarObject* op = (PyVarObject*)_PyObject_GC_Malloc(size);
+    if (op != NULL)
+        op = PyObject_INIT_VAR(op, tp, nitems);
+    RELEASE_ASSERT(gc::isValidGCObject(op), "");
+    return op;
+}
+
+extern "C" void _Py_FatalError(const char* fmt, const char* function, const char* message) {
+    fprintf(stderr, fmt, function, message);
+    fflush(stderr); /* it helps in Windows debug build */
+    abort();
+}
+
 void setupCAPI() {
     capifunc_cls->giveAttr("__repr__",
                            new BoxedFunction(boxRTFunction((void*)BoxedCApiFunction::__repr__, UNKNOWN, 1)));

src/runtime/import.cpp

@@ -690,7 +690,14 @@ Box* impFindModule(Box* _name, BoxedList* path) {
         return BoxedTuple::create({ None, path, BoxedTuple::create({ mode, mode, boxInt(sr.type) }) });
     }
 
-    Py_FatalError("unimplemented");
+    if (sr.type == SearchResult::C_EXTENSION) {
+        Box* path = boxString(sr.path);
+        Box* mode = boxStrConstant("rb");
+        Box* f = runtimeCall(file_cls, ArgPassSpec(2), path, mode, NULL, NULL, NULL);
+        return BoxedTuple::create({ f, path, BoxedTuple::create({ boxStrConstant(".so"), mode, boxInt(sr.type) }) });
+    }
+
+    RELEASE_ASSERT(0, "unknown type: %d", sr.type);
 }
 
 Box* impLoadModule(Box* _name, Box* _file, Box* _pathname, Box** args) {
@@ -724,6 +731,26 @@ Box* impLoadModule(Box* _name, Box* _file, Box* _pathname, Box** args) {
     Py_FatalError("unimplemented");
 }
 
+Box* impLoadDynamic(Box* _name, Box* _pathname, Box* _file) {
+    RELEASE_ASSERT(_name->cls == str_cls, "");
+    RELEASE_ASSERT(_pathname->cls == str_cls, "");
+    RELEASE_ASSERT(_file == None, "");
+
+    BoxedString* name = (BoxedString*)_name;
+    BoxedString* pathname = (BoxedString*)_pathname;
+
+    const char* lastdot = strrchr(name->s.data(), '.');
+
+    const char* shortname;
+    if (lastdot == NULL) {
+        shortname = name->s.data();
+    } else {
+        shortname = lastdot + 1;
+    }
+
+    return importCExtension(name->s, shortname, pathname->s);
+}
+
 Box* impGetSuffixes() {
     BoxedList* list = new BoxedList;
     // For now only add *.py
@@ -772,9 +799,12 @@ void setupImport() {
 
     CLFunction* load_module_func = boxRTFunction((void*)impLoadModule, UNKNOWN, 4,
                                                  ParamNames({ "name", "file", "pathname", "description" }, "", ""));
-
     imp_module->giveAttr("load_module", new BoxedBuiltinFunctionOrMethod(load_module_func, "load_module"));
 
+    CLFunction* load_dynamic_func = boxRTFunction((void*)impLoadDynamic, UNKNOWN, 3, 1, false, false,
+                                                  ParamNames({ "name", "pathname", "file" }, "", ""));
+    imp_module->giveAttr("load_dynamic", new BoxedBuiltinFunctionOrMethod(load_dynamic_func, "load_dynamic", { None }));
+
     imp_module->giveAttr("get_suffixes", new BoxedBuiltinFunctionOrMethod(
                                              boxRTFunction((void*)impGetSuffixes, UNKNOWN, 0), "get_suffixes"));
     imp_module->giveAttr("acquire_lock", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)impAcquireLock, NONE, 0),

test/integration/virtualenv_bcrypt_test.py

-# expected: fail
-# - misc issues
-
-import os
-import sys
-import subprocess
-import shutil
-
-VIRTUALENV_SCRIPT = os.path.dirname(__file__) + "/virtualenv/virtualenv.py"
-
-if os.path.exists("test_env"):
-    print "Removing the existing 'test_env/' directory"
-    subprocess.check_call(["rm", "-rf", "test_env"])
-    # shutil follows symlinks to directories, and deletes whatever those contain.
-    # shutil.rmtree("test_env")
-
-args = [sys.executable, VIRTUALENV_SCRIPT, "-p", sys.executable, "test_env"]
-print "Running", args
-subprocess.check_call(args)
-
-sh_script = """
-set -e
-. test_env/bin/activate
-set -ux
-python -c 'import __future__'
-python -c 'import sys; print sys.executable'
-pip install bcrypt==1.1.0
-python -c 'import bcrypt; assert bcrypt.__version__ == "1.1.0"; assert bcrypt.hashpw("password1", "$2a$12$0123456789012345678901").endswith("I1hdtg4K"); print "bcrypt seems to work"'
-""".strip()
-
-# print sh_script
-subprocess.check_call(["sh", "-c", sh_script])
-

test/integration/virtualenv_test.py

@@ -21,9 +21,12 @@ set -e
 set -ux
 python -c 'import __future__'
 python -c 'import sys; print sys.executable'
-pip install six==1.9.0 cffi==0.9.2 pycparser==2.12
-python -c 'import six; print six.__version__'
+pip install bcrypt==1.1.0
+python -c 'import bcrypt; assert bcrypt.__version__ == "1.1.0"; assert bcrypt.hashpw("password1", "$2a$12$0123456789012345678901").endswith("I1hdtg4K"); print "bcrypt seems to work"'
 """.strip()
 
 # print sh_script
 subprocess.check_call(["sh", "-c", sh_script])
+
+print
+print "PASSED"