Merge pull request #540 from kmod/full_check

Add full_check and fix a couple misc things

.gitmodules

@@ -26,3 +26,4 @@
 [submodule "test/integration/pyxl"]
 	path = test/integration/pyxl
 	url = https://github.com/dropbox/pyxl.git
+	ignore = untracked

CMakeLists.txt

@@ -207,7 +207,7 @@ add_test(NAME analysis_unittest COMMAND analysis_unittest)
 add_test(NAME pyston_defaults COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
 # we pass -I to cpython tests and skip failing ones b/c they are slooow otherwise
 add_test(NAME pyston_defaults_cpython_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t30 ${CMAKE_SOURCE_DIR}/test/cpython)
-add_test(NAME pyston_defaults_integration_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t180 ${CMAKE_SOURCE_DIR}/test/integration)
+add_test(NAME pyston_defaults_integration_tests COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k --exit-code-only --skip-failing -t300 ${CMAKE_SOURCE_DIR}/test/integration)
 add_test(NAME pyston_max_compilation_tier COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-O -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
 add_test(NAME pyston_old_parser COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -a=-x -R ./pyston -j1 -a=-n -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
 

Makefile

@@ -522,7 +522,7 @@ lint: $(PYTHON_EXE_DEPS)
 cpplint:
 	$(VERB) $(PYTHON) $(TOOLS_DIR)/cpplint.py --filter=-whitespace,-build/header_guard,-build/include_order,-readability/todo $(SRCS)
 
-.PHONY: check quick_check
+.PHONY: check
 check:
 	@# These are ordered roughly in decreasing order of (chance will expose issue) / (time to run test)
 	$(MAKE) lint
@@ -532,7 +532,7 @@ check:
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests & skip failing ones because they are sloooow otherwise
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t30 $(TEST_DIR)/cpython $(ARGS)
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t180 $(TEST_DIR)/integration $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t300 $(TEST_DIR)/integration $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -k -a=-n -a=-x -a=-S $(TESTS_DIR) $(ARGS)
 	@# skip -O for dbg
 
@@ -555,6 +555,9 @@ check:
 
 	echo "All tests passed"
 
+# A stripped down set of tests, meant as a quick smoke test to run before submitting a PR and having
+# Travis-CI do the full test.
+.PHONY: quick_check
 quick_check:
 	$(MAKE) pyston_dbg $(CHECK_DEPS)
 	$(MAKE) check_format
@@ -562,6 +565,24 @@ quick_check:
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -a=-S -k --order-by-mtime $(TESTS_DIR) $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_dbg -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing $(TEST_DIR)/cpython $(ARGS)
 
+# A comprehensive test that checks that all three build formats (makefile, makefile->cmake shim, cmake) work.
+# This should only be necessary when doing work on the build system; it shouldn't be necessary for normal development.
+.PHONY: full_check
+full_check:
+	$(MAKE) clean
+	rm -rfv $(CMAKE_DIR_DBG)
+	$(MAKE) llvm_quick llvm_release USE_CMAKE=0
+	$(MAKE) check_dbg USE_CMAKE=0
+	\
+	$(MAKE) clean
+	rm -rfv $(CMAKE_DIR_DBG)
+	$(MAKE) check_dbg USE_CMAKE=0
+	\
+	$(MAKE) clean
+	rm -rfv $(CMAKE_DIR_DBG)
+	$(MAKE) $(CMAKE_SETUP_DBG)
+	ninja -C $(CMAKE_DIR_DBG) check-pyston
+
 Makefile.local:
 	echo "Creating default Makefile.local"
 	python -c 'import sys; v = sys.version_info; print "PYTHON_MAJOR_VERSION:=%d\nPYTHON_MINOR_VERSION:=%d\nPYTHON_MICRO_VERSION:=%d" % (v[0], v[1], v[2])' > Makefile.local || (rm $@; false)
@@ -952,6 +973,7 @@ pyston_gcc: $(CMAKE_SETUP_GCC)
 clean:
 	@ find src $(TOOLS_DIR) $(TEST_DIR) ./from_cpython ./lib_pyston \( -name '*.o' -o -name '*.d' -o -name '*.py_cache' -o -name '*.bc' -o -name '*.o.ll' -o -name '*.pub.ll' -o -name '*.cache' -o -name 'stdlib*.ll' -o -name '*.pyc' -o -name '*.so' -o -name '*.a' -o -name '*.expected_cache' -o -name '*.pch' \) -print -delete
 	@ find \( -name 'pyston*' -executable -type f \) -print -delete
+	@ rm -vf pyston_dbg pyston_release pyston_gcc
 	@ find $(TOOLS_DIR) -maxdepth 0 -executable -type f -print -delete
 	@ rm -rf oprofile_data
 	@ rm -f *_unittest
@@ -987,7 +1009,7 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1 $(CHECK_DEPS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k $(TESTS_DIR) $(ARGS)
 	@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing -t30 $(TEST_DIR)/cpython $(ARGS)
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t180 $(TEST_DIR)/integration $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t300 $(TEST_DIR)/integration $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -a=-S -k $(TESTS_DIR) $(ARGS)
 

from_cpython/Include/pyconfig.h

@@ -138,6 +138,7 @@
 #define HAVE_WCHAR_H 1
 #define HAVE_WORKING_TZSET 1
 #define HAVE_PUTENV 1
+#define HAVE_LSTAT 1
 
 // Added this for some Pyston modifications:
 #define MAX_PYSTRING_SIZE (PY_SSIZE_T_MAX/2 - (1<<20))

src/runtime/builtin_modules/sys.cpp

@@ -433,10 +433,7 @@ void setupSys() {
 
     sys_module->giveAttr("platform", boxStrConstant(Py_GetPlatform()));
 
-    llvm::SmallString<128> main_fn;
-    // TODO supposed to pass argv0, main_addr to this function:
-    main_fn = llvm::sys::fs::getMainExecutable(NULL, NULL);
-    sys_module->giveAttr("executable", boxString(main_fn.str()));
+    sys_module->giveAttr("executable", boxString(Py_GetProgramFullPath()));
 
     sys_module->giveAttr("_getframe",
                          new BoxedFunction(boxRTFunction((void*)sysGetFrame, UNKNOWN, 1, 1, false, false), { NULL }));

src/runtime/types.cpp

@@ -856,6 +856,7 @@ static Box* functionGlobals(Box* self, void*) {
         return func->globals;
     }
     assert(func->f->source);
+    assert(func->f->source->scoping->areGlobalsFromModule());
     return getattr(func->f->source->parent_module, "__dict__");
 }