multiprocessing module

Our initialization is a bit different from CPython's, and multiprocessing
seems pretty sensitive to that.

So initialize it in a different place than we load the rest of the builtin
modules.

Some of the functionality relies on ctypes so it doesn't work, but it looks
like ctypes doesn't get imported until you try to use those features.

from_cpython/CMakeLists.txt

@@ -48,6 +48,7 @@ file(GLOB_RECURSE STDMODULE_SRCS Modules
     md5module.c
     microprotocols.c
     module.c
+    multiprocessing.c
     operator.c
     posixmodule.c
     prepare_protocol.c
@@ -55,10 +56,12 @@ file(GLOB_RECURSE STDMODULE_SRCS Modules
     resource.c
     row.c
     selectmodule.c
+    semaphore.c
     sha256module.c
     sha512module.c
     shamodule.c
     signalmodule.c
+    socket_connection.c
     socketmodule.c
     statement.c
     stringio.c

from_cpython/Include/pyconfig.h

@@ -62,6 +62,10 @@
 #define HAVE_MKTIME 1
 #define HAVE_PROTOTYPES 1
 #define STDC_HEADERS 1
+#define HAVE_SEM_GETVALUE 1
+#define HAVE_SEM_OPEN 1
+#define HAVE_SEM_TIMEDWAIT 1
+#define HAVE_SEM_UNLINK 1
 
 #define TIME_WITH_SYS_TIME
 #define HAVE_GETTIMEOFDAY 1

from_cpython/Modules/_multiprocessing/connection.h

@@ -358,7 +358,7 @@ connection_poll(ConnectionObject *self, PyObject *args)
     }
 
     Py_BEGIN_ALLOW_THREADS
-    res = conn_poll(self, timeout, _save);
+    res = conn_poll(self, timeout /* Pyston change, don't need this argument: , _save */);
     Py_END_ALLOW_THREADS
 
     switch (res) {

from_cpython/Modules/_multiprocessing/socket_connection.c

@@ -185,7 +185,7 @@ conn_recv_string(ConnectionObject *conn, char *buffer,
  */
 
 static int
-conn_poll(ConnectionObject *conn, double timeout, PyThreadState *_save)
+conn_poll(ConnectionObject *conn, double timeout /* Pyston change: don't need this argument: , PyThreadState *_save */)
 {
 #if defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)
     int res;

src/jit.cpp

@@ -51,6 +51,8 @@
 #error
 #endif
 
+extern "C" void init_multiprocessing();
+
 namespace pyston {
 
 extern void setEncodingAndErrors();
@@ -350,7 +352,11 @@ static int main(int argc, char** argv) {
         // encodings module.
         setEncodingAndErrors();
 
-        // end of argument parsing
+        // _multiprocessing relies on a bit more state being set up than the other modules.
+        // At some point we should try to make our initialization closer to CPython's
+        init_multiprocessing();
+
+        Stats::endOfInit();
 
         _t.split("to run");
         BoxedModule* main_module = NULL;

src/runtime/objmodel.cpp

@@ -2439,7 +2439,16 @@ extern "C" void dumpEx(void* p, int levels) {
         }
 
         if (isSubclass(b->cls, list_cls)) {
-            printf("%ld elements\n", static_cast<BoxedList*>(b)->size);
+            auto l = static_cast<BoxedList*>(b);
+            printf("%ld elements\n", l->size);
+
+            if (levels > 0) {
+                int i = 0;
+                for (int i = 0; i < l->size; i++) {
+                    printf("\nElement %d:", i);
+                    dumpEx(l->elts->elts[i], levels - 1);
+                }
+            }
         }
 
         if (isSubclass(b->cls, module_cls)) {

src/runtime/types.cpp

@@ -2589,8 +2589,6 @@ void setupRuntime() {
 
     setupSysEnd();
 
-    Stats::endOfInit();
-
     TRACK_ALLOCATIONS = true;
 }
 

test/tests/multiprocessing_ctypes_test.py

+# expected: fail
+# - relies on ctypes
+
+import multiprocessing
+
+# from https://docs.python.org/2/library/multiprocessing.html
+
+def f(n, a):
+    n.value = 3.1415927
+    for i in range(len(a)):
+        a[i] = -a[i]
+
+if __name__ == '__main__':
+    num = multiprocessing.Value('d', 0.0)
+    arr = multiprocessing.Array('i', range(10))
+
+    p = multiprocessing.Process(target=f, args=(num, arr))
+    p.start()
+    p.join()
+
+    print num.value
+    print arr[:]

test/tests/multiprocessing_test.py

+import multiprocessing
+
+# from https://docs.python.org/2/library/multiprocessing.html
+
+def f(x):
+    return x*x
+
+if __name__ == '__main__':
+    p = multiprocessing.Pool(5)
+    print(p.map(f, [1, 2, 3]))
+
+
+def f(name):
+    print 'hello', name
+
+if __name__ == '__main__':
+    p = multiprocessing.Process(target=f, args=('bob',))
+    p.start()
+    p.join()
+
+
+def f(q):
+    q.put([42, None, 'hello'])
+
+if __name__ == '__main__':
+    q = multiprocessing.Queue()
+    p = multiprocessing.Process(target=f, args=(q,))
+    p.start()
+    print q.get()    # prints "[42, None, 'hello']"
+    p.join()