Build _multiprocessing as a shared library

by adding from_cpython/setup.py.

This way we can build _multiprocessing as a shared module to be
loaded on-demand, since it's quite expensive right now for us to
import it.

CPython has a similar setup.py, but theirs is pretty large (2kloc)
so I don't feel like we need to try copying yet.

We could/should move other modules to use this strategy.

I'm not 100% happy with the build system support, but we can iterate
on that.

CMakeLists.txt

@@ -220,7 +220,7 @@ add_custom_target(check-format ${CMAKE_SOURCE_DIR}/tools/check_format.sh ${LLVM_
 add_custom_target(lint ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/lint.py WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src)
 
 # check
-add_custom_target(check-pyston COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure DEPENDS pyston copy_stdlib copy_libpyston clang-format ext_cpython ext_pyston unittests) 
+add_custom_target(check-pyston COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure DEPENDS pyston copy_stdlib copy_libpyston clang-format ext_cpython ext_pyston unittests sharedmods)
 
 # {run,dbg,perf,memcheck,memleaks,cachegrind}_TESTNAME
 file(GLOB RUNTARGETS ${CMAKE_SOURCE_DIR}/test/tests/*.py ${CMAKE_SOURCE_DIR}/microbenchmarks/*.py ${CMAKE_SOURCE_DIR}/minibenchmarks/*.py)

Makefile

@@ -420,16 +420,23 @@ UNITTEST_SRCS := $(wildcard $(UNITTEST_DIR)/*.cpp)
 
 NONSTDLIB_SRCS := $(MAIN_SRCS) $(OPTIONAL_SRCS) $(TOOL_SRCS) $(UNITTEST_SRCS)
 
-.DEFAULT_GOAL := pyston_dbg
-# _ :
-	# $(MAKE) pyston_dbg || (clear; $(MAKE) pyston_dbg -j1 ERROR_LIMIT=1)
+.DEFAULT_GOAL := small_all
+
+RUN_DEPS := ext_pyston
+
+ifneq ($(USE_CMAKE),1)
+	RUN_DEPS := $(RUN_DEPS) sharedmods
+endif
+
+.PHONY: small_all
+small_all: pyston_dbg $(RUN_DEPS)
 
 .PHONY: all _all
 # all: llvm
 	# @# have to do this in a recursive make so that dependency is enforced:
 	# $(MAKE) pyston_all
 # all: pyston_dbg pyston_release pyston_oprof pyston_prof $(OPTIONAL_SRCS:.cpp=.o) ext_python ext_pyston
-all: pyston_dbg pyston_release pyston_prof ext_python ext_pyston unittests
+all: pyston_dbg pyston_release pyston_gcc ext_python ext_pyston unittests sharedmods
 
 ALL_HEADERS := $(wildcard src/*/*.h) $(wildcard src/*/*/*.h) $(wildcard from_cpython/Include/*.h)
 tags: $(SRCS) $(OPTIONAL_SRCS) $(FROM_CPYTHON_SRCS) $(ALL_HEADERS)
@@ -913,10 +920,10 @@ $(CMAKE_SETUP_RELEASE):
 
 .PHONY: pyston_dbg pyston_release
 pyston_dbg: $(CMAKE_SETUP_DBG)
-	$(NINJA) -C $(HOME)/pyston-build-dbg pyston copy_stdlib copy_libpyston ext_pyston $(NINJAFLAGS)
+	$(NINJA) -C $(HOME)/pyston-build-dbg pyston copy_stdlib copy_libpyston sharedmods ext_pyston $(NINJAFLAGS)
 	ln -sf $(HOME)/pyston-build-dbg/pyston pyston_dbg
 pyston_release: $(CMAKE_SETUP_RELEASE)
-	$(NINJA) -C $(HOME)/pyston-build-release pyston copy_stdlib copy_libpyston ext_pyston $(NINJAFLAGS)
+	$(NINJA) -C $(HOME)/pyston-build-release pyston copy_stdlib copy_libpyston sharedmods ext_pyston $(NINJAFLAGS)
 	ln -sf $(HOME)/pyston-build-release/pyston pyston_release
 endif
 CMAKE_DIR_GCC := $(HOME)/pyston-build-gcc
@@ -927,14 +934,14 @@ $(CMAKE_SETUP_GCC):
 	cd $(CMAKE_DIR_GCC); CC='$(GCC)' CXX='$(GPP)' cmake -GNinja $(HOME)/pyston -DCMAKE_BUILD_TYPE=Debug
 .PHONY: pyston_gcc
 pyston_gcc: $(CMAKE_SETUP_GCC)
-	$(NINJA) -C $(HOME)/pyston-build-gcc pyston copy_stdlib copy_libpyston ext_pyston $(NINJAFLAGS)
+	$(NINJA) -C $(HOME)/pyston-build-gcc pyston copy_stdlib copy_libpyston sharedmods ext_pyston $(NINJAFLAGS)
 	ln -sf $(HOME)/pyston-build-gcc/pyston pyston_gcc
 
 -include $(wildcard src/*.d) $(wildcard src/*/*.d) $(wildcard src/*/*/*.d) $(wildcard $(UNITTEST_DIR)/*.d) $(wildcard from_cpython/*/*.d) $(wildcard from_cpython/*/*/*.d)
 
 .PHONY: clean
 clean:
-	@ find src $(TOOLS_DIR) $(TEST_DIR) ./from_cpython/Modules \( -name '*.o' -o -name '*.d' -o -name '*.py_cache' -o -name '*.bc' -o -name '*.o.ll' -o -name '*.pub.ll' -o -name '*.cache' -o -name 'stdlib*.ll' -o -name '*.pyc' -o -name '*.so' -o -name '*.a' -o -name '*.expected_cache' -o -name '*.pch' \) -print -delete
+	@ find src $(TOOLS_DIR) $(TEST_DIR) ./from_cpython ./lib_pyston \( -name '*.o' -o -name '*.d' -o -name '*.py_cache' -o -name '*.bc' -o -name '*.o.ll' -o -name '*.pub.ll' -o -name '*.cache' -o -name 'stdlib*.ll' -o -name '*.pyc' -o -name '*.so' -o -name '*.a' -o -name '*.expected_cache' -o -name '*.pch' \) -print -delete
 	@ find \( -name 'pyston*' -executable -type f \) -print -delete
 	@ find $(TOOLS_DIR) -maxdepth 0 -executable -type f -print -delete
 	@ rm -rf oprofile_data
@@ -963,8 +970,6 @@ $(patsubst %, $$1: %/nosearch_$$1 ;,$(EXTRA_SEARCH_DIRS))
 )
 endef
 
-RUN_DEPS :=
-
 define make_target
 $(eval \
 .PHONY: test$1 check$1
@@ -1149,47 +1154,39 @@ bench_exceptions:
 	rm bench_exceptions
 
 TEST_EXT_MODULE_NAMES := basic_test descr_test slots_test
+TEST_EXT_MODULE_SRCS := $(TEST_EXT_MODULE_NAMES:%=test/test_extension/%.c)
+TEST_EXT_MODULE_OBJS := $(TEST_EXT_MODULE_NAMES:%=test/test_extension/%.pyston.so)
 
-# SELF_HOST_EXTENSIONS = SELF_HOST or USE_CMAKE
-# - cmake doesn't support non-self-hosting extensions
-SELF_HOST_EXTENSIONS := $(SELF_HOST)
-ifeq ($(USE_CMAKE),1)
-	SELF_HOST_EXTENSIONS := 1
-endif
+SHAREDMODS_NAMES := _multiprocessing
+SHAREDMODS_SRCS := \
+	_multiprocessing/multiprocessing.c \
+	_multiprocessing/semaphore.c \
+	_multiprocessing/socket_connection.c
+SHAREDMODS_SRCS := $(SHAREDMODS_SRCS:%=from_cpython/Modules/%)
+SHAREDMODS_OBJS := $(SHAREDMODS_NAMES:%=lib_pyston/%.pyston.so)
+
+.PHONY: sharedmods
+sharedmods: $(SHAREDMODS_OBJS)
 
 .PHONY: ext_pyston
-ext_pyston: $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/%.pyston.so)
-ifneq ($(SELF_HOST_EXTENSIONS),1)
-$(TEST_DIR)/test_extension/%.pyston.so: $(TEST_DIR)/test_extension/%.o
-	$(CC) -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-z,relro $< -o $@ -g
-$(TEST_DIR)/test_extension/%.o: $(TEST_DIR)/test_extension/%.c $(wildcard from_cpython/Include/*.h)
-	$(CC) -pthread $(EXT_CFLAGS) -c $< -o $@
-else
+ext_pyston: $(TEST_EXT_MODULE_OBJS)
+
 # Hax: we want to generate multiple targets from a single rule, and run the rule only if the
 # dependencies have been updated, and only run it once for all the targets.
 # So just tell make to generate the first extension module, and that the non-first ones just
 # depend on the first one.
-$(TEST_DIR)/test_extension/$(firstword $(TEST_EXT_MODULE_NAMES)).pyston.so: $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/%.c) | pyston_dbg
-	$(MAKE) ext_pyston_selfhost
-NONFIRST_EXT := $(wordlist 2,9999,$(TEST_EXT_MODULE_NAMES))
-$(NONFIRST_EXT:%=$(TEST_DIR)/test_extension/%.pyston.so): $(TEST_DIR)/test_extension/$(firstword $(TEST_EXT_MODULE_NAMES)).pyston.so
-endif
-
-.PHONY: ext_pyston_selfhost dbg_ext_pyston_selfhost ext_pyston_selfhost_release
-ext_pyston_selfhost: pyston_dbg $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/*.c)
-	cd $(TEST_DIR)/test_extension; DISTUTILS_DEBUG=1 time ../../pyston_dbg setup.py build
-	cd $(TEST_DIR)/test_extension; ln -sf $(TEST_EXT_MODULE_NAMES:%=build/lib.linux2-2.7/%.pyston.so) .
-dbg_ext_pyston_selfhost: pyston_dbg $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/*.c)
-	cd $(TEST_DIR)/test_extension; DISTUTILS_DEBUG=1 $(GDB) $(GDB_CMDS) --args ../../pyston_dbg setup.py build
-	cd $(TEST_DIR)/test_extension; ln -sf $(TEST_EXT_MODULE_NAMES:%=build/lib.linux2-2.7/%.pyston.so) .
-ext_pyston_selfhost_release: pyston_release $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/*.c)
-	cd $(TEST_DIR)/test_extension; DISTUTILS_DEBUG=1 time ../../pyston_release setup.py build
-	cd $(TEST_DIR)/test_extension; ln -sf $(TEST_EXT_MODULE_NAMES:%=build/lib.linux2-2.7/%.pyston.so) .
+$(firstword $(TEST_EXT_MODULE_OBJS)): $(TEST_EXT_MODULE_SRCS) pyston_dbg
+	$(VERB) cd $(TEST_DIR)/test_extension; time ../../pyston_dbg setup.py build
+	$(VERB) cd $(TEST_DIR)/test_extension; ln -sf $(TEST_EXT_MODULE_NAMES:%=build/lib.linux2-2.7/%.pyston.so) .
+$(wordlist 2,9999,$(TEST_EXT_MODULE_OBJS)): $(firstword $(TEST_EXT_MODULE_OBJS))
+$(firstword $(SHAREDMODS_OBJS)): $(SHAREDMODS_SRCS) pyston_dbg
+	$(VERB) cd $(TEST_DIR)/test_extension; time ../../pyston_dbg ../../from_cpython/setup.py build --build-lib ../../lib_pyston
+$(wordlist 2,9999,$(SHAREDMODS_OBJS)): $(firstword $(SHAREDMODS_OBJS))
 
 .PHONY: ext_python ext_pythondbg
-ext_python: $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/*.c)
+ext_python: $(TEST_EXT_MODULE_SRCS)
 	cd $(TEST_DIR)/test_extension; python setup.py build
-ext_pythondbg: $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/*.c)
+ext_pythondbg: $(TEST_EXT_MODULE_SRCS)
 	cd $(TEST_DIR)/test_extension; python2.7-dbg setup.py build
 
 $(FROM_CPYTHON_SRCS:.c=.o): %.o: %.c $(BUILD_SYSTEM_DEPS)

from_cpython/CMakeLists.txt

@@ -48,7 +48,6 @@ file(GLOB_RECURSE STDMODULE_SRCS Modules
     md5module.c
     microprotocols.c
     module.c
-    multiprocessing.c
     operator.c
     posixmodule.c
     prepare_protocol.c
@@ -56,12 +55,10 @@ file(GLOB_RECURSE STDMODULE_SRCS Modules
     resource.c
     row.c
     selectmodule.c
-    semaphore.c
     sha256module.c
     sha512module.c
     shamodule.c
     signalmodule.c
-    socket_connection.c
     socketmodule.c
     statement.c
     stringio.c
@@ -111,3 +108,15 @@ file(GLOB_RECURSE STDPARSER_SRCS Parser
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-tautological-compare -Wno-type-limits -Wno-unused-result -Wno-strict-aliasing")
 add_library(FROM_CPYTHON OBJECT ${STDMODULE_SRCS} ${STDOBJECT_SRCS} ${STDPYTHON_SRCS} ${STDPARSER_SRCS})
+
+add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/lib_pyston/_multiprocessing.pyston.so
+                   COMMAND ${CMAKE_BINARY_DIR}/pyston setup.py build --build-lib ${CMAKE_BINARY_DIR}/lib_pyston
+                   DEPENDS
+                       pyston
+                       copy_stdlib
+                       copy_libpyston
+                       Modules/_multiprocessing/multiprocessing.c
+                       Modules/_multiprocessing/semaphore.c
+                       Modules/_multiprocessing/socket_connection.c
+                   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+add_custom_target(sharedmods DEPENDS ${CMAKE_BINARY_DIR}/lib_pyston/_multiprocessing.pyston.so)

from_cpython/setup.py

+# CPython has a 2kloc version of this file
+
+from distutils.core import setup, Extension
+import os
+
+def relpath(fn):
+    r =  os.path.join(os.path.dirname(__file__), fn)
+    return r
+
+setup(name="Pyston",
+        version="1.0",
+        description="Pyston shared modules",
+        ext_modules=[Extension("_multiprocessing", sources = map(relpath, [
+                "Modules/_multiprocessing/multiprocessing.c",
+                "Modules/_multiprocessing/socket_connection.c",
+                "Modules/_multiprocessing/semaphore.c",
+            ]),
+        )],
+    )

lib_pyston/CMakeLists.txt

 # Copy any changed lib_pyston sources:
-file(GLOB_RECURSE LIBPYSTON_SRCS . "*")
+file(GLOB_RECURSE LIBPYSTON_SRCS . "*.py")
 set(LIBPYSTON_TARGETS "")
 foreach(STDLIB_FILE ${LIBPYSTON_SRCS})
     file(RELATIVE_PATH FN_REL ${CMAKE_SOURCE_DIR} ${STDLIB_FILE})

src/jit.cpp

@@ -51,8 +51,6 @@
 #error
 #endif
 
-extern "C" void init_multiprocessing();
-
 namespace pyston {
 
 extern void setEncodingAndErrors();
@@ -352,10 +350,6 @@ static int main(int argc, char** argv) {
         // encodings module.
         setEncodingAndErrors();
 
-        // _multiprocessing relies on a bit more state being set up than the other modules.
-        // At some point we should try to make our initialization closer to CPython's
-        init_multiprocessing();
-
         Stats::endOfInit();
 
         _t.split("to run");

test/test_extension/CMakeLists.txt

-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers")
-set(CMAKE_SHARED_LIBRARY_PREFIX "")
-set(CMAKE_SHARED_LIBRARY_SUFFIX ".pyston${CMAKE_SHARED_LIBRARY_SUFFIX}")
-
-add_custom_command(OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/build/lib.linux-x86_64-2.7/basic_test.so
-                   COMMAND python setup.py build
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/build/lib.linux-x86_64-2.7/basic_test.so
+                   COMMAND python setup.py build --build-lib ${CMAKE_CURRENT_BINARY_DIR}/build/lib.linux-x86_64-2.7
                    DEPENDS basic_test.c descr_test.c slots_test.c
                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
-add_library(basic_test SHARED basic_test.c)
-add_library(descr_test SHARED descr_test.c)
-add_library(slots_test SHARED slots_test.c)
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/basic_test.pyston.so
+                   COMMAND ${CMAKE_BINARY_DIR}/pyston setup.py build --build-lib ${CMAKE_CURRENT_BINARY_DIR}
+                   DEPENDS pyston copy_stdlib copy_libpyston basic_test.c descr_test.c slots_test.c
+                   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
-add_custom_target(ext_cpython DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/build/lib.linux-x86_64-2.7/basic_test.so)
-add_custom_target(ext_pyston DEPENDS basic_test descr_test slots_test)
+add_custom_target(ext_cpython DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/build/lib.linux-x86_64-2.7/basic_test.so)
+add_custom_target(ext_pyston DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/basic_test.pyston.so)

tools/tester.py

@@ -59,7 +59,7 @@ def set_ulimits():
     MAX_MEM_MB = 100
     resource.setrlimit(resource.RLIMIT_RSS, (MAX_MEM_MB * 1024 * 1024, MAX_MEM_MB * 1024 * 1024))
 
-EXTMODULE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../test/test_extension/build/lib.linux-x86_64-2.7/")
+EXTMODULE_DIR = None
 EXTMODULE_DIR_PYSTON = None
 THIS_FILE = os.path.abspath(__file__)
 
@@ -72,6 +72,7 @@ def get_global_mtime():
     # Start off by depending on the tester itself
     rtn = os.stat(THIS_FILE).st_mtime
 
+    assert os.listdir(EXTMODULE_DIR), EXTMODULE_DIR
     for fn in os.listdir(EXTMODULE_DIR):
         if not fn.endswith(".so"):
             continue
@@ -462,6 +463,7 @@ def main(orig_dir):
     global SKIP_FAILING_TESTS
     global VERBOSE
     global EXTMODULE_DIR_PYSTON
+    global EXTMODULE_DIR
 
     run_memcheck = False
 
@@ -479,6 +481,7 @@ def main(orig_dir):
 
     TEST_DIR = os.path.join(orig_dir, opts.test_dir)
     EXTMODULE_DIR_PYSTON = os.path.abspath(os.path.dirname(os.path.realpath(IMAGE)) + "/test/test_extension/")
+    EXTMODULE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(IMAGE)) + "/test/test_extension/build/lib.linux-x86_64-2.7/")
     patterns = opts.pattern
 
     if not patterns and not TESTS_TO_SKIP: