Commit 82e3709a authored by Jason Madden's avatar Jason Madden

Parrellize cythoncpp.py.

On my machine, this cuts the build time by a fourth to a third.

Not ready to merge, submitting for testing on CI (esp Windows).

Cython calls are threaded because they release the GIL.

The most expensive part was merging results, which holds the GIL, so use
multiprocessing for that. Pre-combine the identical results that we
expect to get for two sets of defines to reduce the number of merges.

Change internal data structures to be immutable to make debugging this
easier (and substantially faster when pickling through multiprocessing.)
parent 4fe26b0d
/* Copyright (c) 2011-2012 Denis Bilenko. See LICENSE for details. */
#ifdef Py_PYTHON_H
#define _GEVENTLOOP struct __pyx_vtabstruct_8corecext_loop
static void gevent_handle_error(struct PyGeventLoopObject* loop, PyObject* context) {
PyThreadState *tstate;
PyObject *type, *value, *traceback, *result;
......@@ -19,7 +21,7 @@ static void gevent_handle_error(struct PyGeventLoopObject* loop, PyObject* conte
PyErr_Clear();
result = ((struct __pyx_vtabstruct_6gevent_8corecext_loop *)loop->__pyx_vtab)->handle_error(loop, context, type, value, traceback, 0);
result = ((_GEVENTLOOP *)loop->__pyx_vtab)->handle_error(loop, context, type, value, traceback, 0);
if (result) {
Py_DECREF(result);
......@@ -195,7 +197,7 @@ static void gevent_run_callbacks(struct ev_loop *_loop, void *watcher, int reven
loop = GET_OBJECT(PyGeventLoopObject, watcher, _prepare);
Py_INCREF(loop);
gevent_check_signals(loop);
result = ((struct __pyx_vtabstruct_6gevent_8corecext_loop *)loop->__pyx_vtab)->_run_callbacks(loop);
result = ((_GEVENTLOOP *)loop->__pyx_vtab)->_run_callbacks(loop);
if (result) {
Py_DECREF(result);
}
......
......@@ -9,12 +9,13 @@ import time
from datetime import timedelta
from multiprocessing.pool import ThreadPool
from multiprocessing import cpu_count
import util
from util import log
TIMEOUT = 180
NWORKERS = int(os.environ.get('NWORKERS') or 4)
NWORKERS = int(os.environ.get('NWORKERS') or max(cpu_count() - 1, 4))
# tests that don't do well when run on busy box
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment