time: Factor-out PyFunc into shared library libpyxruntime.so

Commit b073f6df (time: Move/Port timers to C++/Pyx nogil) added C++
level PyFunc as a way to call Python function from under nogil code.
We will soon need this functionality for sync.WorkGroup too, so move it
into shared place.

We move into regular library - not a pyx module (e.g. golang.pyx) -
because for pyx modules linking is done at import time by specially
generated code, while we need the linker support to automatically
resolve e.g. calls to PyFunc::operator() from inside std::function.

MANIFEST.in

 include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun
 include golang/libgolang.h
 include golang/runtime/libgolang.cpp
+include golang/runtime/libpyxruntime.cpp
+include golang/pyx/runtime.h
+include golang/pyx/runtime.cpp
 include golang/context.h
 include golang/context.cpp
 include golang/cxx.h

golang/_golang.pyx

@@ -30,8 +30,9 @@ See _golang.pxd for package overview.
 
 from __future__ import print_function, absolute_import
 
-# init libgolang runtime early
+# init libgolang runtime & friends early
 _init_libgolang()
+_init_libpyxruntime()
 
 from cpython cimport PyObject, Py_INCREF, Py_DECREF, PY_MAJOR_VERSION
 ctypedef PyObject *pPyObject # https://github.com/cython/cython/issues/534
@@ -526,6 +527,10 @@ cdef void _init_libgolang() except*:
     _libgolang_init(runtime_ops)
 
 
+cdef void _init_libpyxruntime() except*:
+    # this initializes libpyxruntime and registers its pyatexit hook
+    import golang.pyx.runtime
+
 
 # ---- misc ----
 

golang/_time.pyx

@@ -23,11 +23,10 @@ from __future__ import print_function, absolute_import
 
 from golang cimport pychan, topyexc
 from golang cimport sync
+from golang.pyx cimport runtime
 from cpython cimport PyObject
 from cython cimport final
 
-import atexit as pyatexit
-
 
 def pynow(): # -> t
     return now_pyexc()
@@ -133,142 +132,6 @@ cdef class PyTimer:
             timer_reset_pyexc(pyt.t, dt)
 
 
-# _PyFunc represents python function scheduled to be called via PyTimer(f=...).
-# _PyFunc can be used from nogil code.
-# _PyFunc is safe to use wrt race to python interpreter shutdown.
-cdef extern from * nogil:
-    """
-    #include <golang/sync.h>
-    using namespace golang;
-
-    #include <utility>
-    using std::tuple;
-    using std::make_tuple;
-    using std::tie;
-
-    // pyexited indicates whether Python interpreter exited.
-    static sync::Mutex  *pyexitedMu = new sync::Mutex(); // never freed not race at exit
-    static bool         pyexited = false;                // on mu dtor vs mu use
-    """
-    sync.Mutex *pyexitedMu
-    bint       pyexited
-
-cdef _time_pyatexit():
-    global pyexited
-    with nogil:
-        pyexitedMu.lock()
-        pyexited = True
-        pyexitedMu.unlock()
-
-pyatexit.register(_time_pyatexit)
-
-cdef extern from * nogil:
-    """
-    // pygil_ensure is like `with gil` but also takes into account possibility
-    // of python interpreter shutdown.
-    static tuple<PyGILState_STATE, bool> pygil_ensure() {
-        PyGILState_STATE gstate;
-
-        // A C++ thread might still be running while python interpreter is stopped.
-        // Verify it is not the case not to crash in PyGILState_Ensure().
-        //
-        // Tell caller not to run any py code if python interpreter is gone and ignore any error.
-        // Python itself behaves the same way on threading cleanup - see e.g.
-        // comments in our _golang.pyx::__goviac() about that and also e.g.
-        // https://www.riverbankcomputing.com/pipermail/pyqt/2004-July/008196.html
-        pyexitedMu->lock();
-        if (pyexited) {
-            pyexitedMu->unlock();
-            return make_tuple(PyGILState_STATE(0), false);
-        }
-
-        gstate = PyGILState_Ensure();
-        pyexitedMu->unlock();
-        return make_tuple(gstate, true);
-    }
-
-    struct _PyFunc {
-        PyObject *pyf;  // function to call; _PyFunc keeps 1 reference to f
-
-        // ctor.
-        // _PyFunc must be constructed while Python interpreter is alive.
-        _PyFunc(PyObject *pyf) {
-            PyGILState_STATE gstate = PyGILState_Ensure();
-                Py_INCREF(pyf);
-                this->pyf = pyf;
-            PyGILState_Release(gstate);
-        }
-
-        // all other methods may be called at any time, including when python
-        // interpreter is gone.
-
-        // copy
-        _PyFunc(const _PyFunc& from) {
-            PyGILState_STATE gstate;
-            bool ok;
-
-            tie(gstate, ok) = pygil_ensure();
-            if (!ok) {
-                pyf = NULL; // won't be used
-                return;
-            }
-
-                pyf = from.pyf;
-                Py_INCREF(pyf);
-            PyGILState_Release(gstate);
-        }
-
-        // dtor
-        ~_PyFunc() {
-            PyGILState_STATE gstate;
-            bool ok;
-
-            tie(gstate, ok) = pygil_ensure();
-            PyObject *pyf = this->pyf;
-            this->pyf = NULL;
-            if (!ok) {
-                return;
-            }
-
-                Py_DECREF(pyf);
-            PyGILState_Release(gstate);
-        }
-
-        // call
-        void operator() () const {
-            PyGILState_STATE gstate;
-            bool ok;
-
-            // C++ timer thread might still be running while python interpreter is stopped.
-            // Verify it is not the case not to crash in PyGILState_Ensure().
-            //
-            // Don't call the function if python interpreter is gone - i.e. ignore error here.
-            // Python itself behaves the same way on threading cleanup - see
-            // _golang.pyx::__goviac and pygil_ensure for details.
-            tie(gstate, ok) = pygil_ensure();
-            if (!ok) {
-                return;
-            }
-
-                ok = true;
-                PyObject *ret = PyObject_CallFunction(pyf, NULL);
-                if (ret == NULL && !pyexited) {
-                    PyErr_PrintEx(0);
-                    ok = false;
-                }
-                Py_XDECREF(ret);
-            PyGILState_Release(gstate);
-
-            // XXX exception -> exit program with traceback (same as in go) ?
-            //if (!ok)
-            //    panic("pycall failed");
-        }
-    };
-    """
-    cppclass _PyFunc:
-        _PyFunc(PyObject *pyf)
-
-
 # ---- misc ----
 pysecond        = second
 pynanosecond    = nanosecond
@@ -291,7 +154,7 @@ cdef nogil:
     Timer new_timer_pyexc(double dt)                        except +topyexc:
         return new_timer(dt)
     Timer _new_timer_pyfunc_pyexc(double dt, PyObject *pyf) except +topyexc:
-        return after_func(dt, _PyFunc(pyf))
+        return after_func(dt, runtime.PyFunc(pyf))
 
     cbool timer_stop_pyexc(Timer t)                         except +topyexc:
         return t.stop()

golang/pyx/.gitignore

+/runtime.cpp

golang/pyx/build.py

@@ -180,6 +180,8 @@ def Extension(name, sources, **kw):
     dependv.append('%s/golang/time.h'       % pygo)
     dependv.append('%s/golang/time.pxd'     % pygo)
     dependv.append('%s/golang/_time.pxd'    % pygo)
+    dependv.append('%s/golang/pyx/runtime.h'    % pygo)
+    dependv.append('%s/golang/pyx/runtime.pxd'  % pygo)
     kw['depends'] = dependv
 
     # workaround pip bug that for virtualenv case headers are installed into

golang/pyx/runtime.h

+#ifndef _NXD_LIBGOLANG_PYX_RUNTIME_H
+#define _NXD_LIBGOLANG_PYX_RUNTIME_H
+
+// Copyright (C) 2018-2019  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Library Libpyxruntime complements Libgolang and provides support for
+// Python/Cython runtimes that can be used from nogil code.
+//
+//  - `PyFunc` represents Python function that can be called from nogil code.
+
+#include <golang/libgolang.h>
+#include <Python.h>
+
+#if BUILDING_LIBPYXRUNTIME
+#  define LIBPYXRUNTIME_API LIBGOLANG_DSO_EXPORT
+#else
+#  define LIBPYXRUNTIME_API LIBGOLANG_DSO_IMPORT
+#endif
+
+
+// golang::pyx::runtime::
+namespace golang {
+namespace pyx {
+namespace runtime {
+
+// PyFunc represents python function that can be called.
+// PyFunc can be used from nogil code.
+// PyFunc is safe to use wrt race to python interpreter shutdown.
+class PyFunc {
+    PyObject *pyf;  // function to call; PyFunc keeps 1 reference to f
+
+public:
+    // ctor.
+    // PyFunc must be constructed while Python interpreter is alive.
+    LIBPYXRUNTIME_API PyFunc(PyObject *pyf);
+
+    // all other methods may be called at any time, including when python
+    // interpreter is gone.
+
+    LIBPYXRUNTIME_API PyFunc(const PyFunc& from);   // copy
+    LIBPYXRUNTIME_API ~PyFunc();                    // dtor
+
+    // call.
+    // returned error is either PyError or ErrPyStopped.
+    LIBPYXRUNTIME_API void operator() () const;
+};
+
+// libpyxruntime must be initialized before use via _init.
+// _pyatexit_nogil must be called when python interpreter is shut down.
+//
+// These are established by golang/pyx/runtime.pyx initialization, who, in
+// turn, is initialized at `import golang` time.
+LIBPYXRUNTIME_API void _init();
+LIBPYXRUNTIME_API void _pyatexit_nogil();
+
+}}} // golang::pyx::runtime::
+
+#endif  // _NXD_LIBGOLANG_PYX_RUNTIME_H

golang/pyx/runtime.pxd

+# cython: language_level=2
+# Copyright (C) 2019  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""Package pyx/runtime.pyx complements golang.pyx and provides support for
+Python/Cython runtimes that can be used from nogil code.
+
+ - `PyFunc` represents Python function that can be called from nogil code.
+"""
+
+from cpython cimport PyObject
+
+
+cdef extern from "golang/pyx/runtime.h" namespace "golang::pyx::runtime" nogil:
+    cppclass PyFunc:
+        __init__(PyObject *pyf)
+        void operator() ()

golang/pyx/runtime.pyx

+# cython: language_level=2
+# Copyright (C) 2019  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+# pyx/runtime.pyx implementation - see pyx/runtime.pxd for package overview.
+
+from __future__ import print_function, absolute_import
+
+# initialize libpyxruntime at import time.
+# NOTE golang.pyx imports us right after initializing libgolang.
+
+import atexit as pyatexit
+
+cdef extern from "golang/pyx/runtime.h" namespace "golang::pyx::runtime" nogil:
+    void _init()
+    void _pyatexit_nogil()
+
+# init libpyxruntime
+_init()
+
+# register its pyatexit hook
+cdef void _pyatexit():
+    with nogil:
+        _pyatexit_nogil()
+
+pyatexit.register(_pyatexit)

golang/runtime/libpyxruntime.cpp

+// Copyright (C) 2019  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Library Libpyxruntime complements Libgolang and provides support for
+// Python/Cython runtimes that can be used from nogil code.
+//
+// See pyx/runtime.h for library overview.
+
+#include "golang/pyx/runtime.h"
+#include "golang/sync.h"
+using namespace golang;
+
+#include <utility>
+using std::tuple;
+using std::make_tuple;
+using std::tie;
+
+// golang::pyx::runtime::
+namespace golang {
+namespace pyx {
+namespace runtime {
+
+
+// pyexited indicates whether Python interpreter exited.
+static sync::Mutex     *pyexitedMu  = NULL; // allocated in _init and never freed not to race
+static bool            pyexited = false;    // at exit on mu dtor vs mu use
+
+void _init() {
+    pyexitedMu  = new sync::Mutex();
+}
+
+void _pyatexit_nogil() {
+    pyexitedMu->lock();
+    pyexited = true;
+    pyexitedMu->unlock();
+}
+
+
+// pygil_ensure is like `with gil` but also takes into account possibility
+// of python interpreter shutdown.
+static tuple<PyGILState_STATE, bool> pygil_ensure() {
+    PyGILState_STATE gstate;
+
+    // A C++ thread might still be running while python interpreter is stopped.
+    // Verify it is not the case not to crash in PyGILState_Ensure().
+    //
+    // Tell caller not to run any py code if python interpreter is gone and ignore any error.
+    // Python itself behaves the same way on threading cleanup - see e.g.
+    // comments in our _golang.pyx::__goviac() about that and also e.g.
+    // https://www.riverbankcomputing.com/pipermail/pyqt/2004-July/008196.html
+    pyexitedMu->lock();
+    if (pyexited) {
+        pyexitedMu->unlock();
+        return make_tuple(PyGILState_STATE(0), false);
+    }
+
+    gstate = PyGILState_Ensure();
+    pyexitedMu->unlock();
+    return make_tuple(gstate, true);
+}
+
+
+
+// PyFunc
+PyFunc::PyFunc(PyObject *pyf) {
+    PyGILState_STATE gstate = PyGILState_Ensure();
+        Py_INCREF(pyf);
+        this->pyf = pyf;
+    PyGILState_Release(gstate);
+}
+
+PyFunc::PyFunc(const PyFunc& from) {
+    PyGILState_STATE gstate;
+    bool ok;
+
+    tie(gstate, ok) = pygil_ensure();
+    if (!ok) {
+        pyf = NULL; // won't be used
+        return;
+    }
+
+        pyf = from.pyf;
+        Py_INCREF(pyf);
+    PyGILState_Release(gstate);
+}
+
+PyFunc::~PyFunc() {
+    PyGILState_STATE gstate;
+    bool ok;
+
+    tie(gstate, ok) = pygil_ensure();
+    PyObject *pyf = this->pyf;
+    this->pyf = NULL;
+    if (!ok) {
+        return;
+    }
+
+        Py_DECREF(pyf);
+    PyGILState_Release(gstate);
+}
+
+void PyFunc::operator() () const {
+    PyGILState_STATE gstate;
+    bool ok;
+
+    // e.g. C++ timer thread might still be running while python interpreter is stopped.
+    // Verify it is not the case not to crash in PyGILState_Ensure().
+    //
+    // Don't call the function if python interpreter is gone - i.e. ignore error here.
+    // Python itself behaves the same way on threading cleanup - see
+    // _golang.pyx::__goviac and pygil_ensure for details.
+    tie(gstate, ok) = pygil_ensure();
+    if (!ok) {
+        return;
+    }
+
+        ok = true;
+        PyObject *ret = PyObject_CallFunction(pyf, NULL);
+        if (ret == NULL && !pyexited) {
+            PyErr_PrintEx(0);
+            ok = false;
+        }
+        Py_XDECREF(ret);
+    PyGILState_Release(gstate);
+
+    // XXX exception -> exit program with traceback (same as in go) ?
+    //if (!ok)
+    //    panic("pycall failed");
+}
+
+
+}}} // golang::pyx::runtime

setup.py

@@ -25,6 +25,7 @@ from setuptools import find_packages
 from setuptools_dso import DSO
 from setuptools.command.install_scripts import install_scripts as _install_scripts
 from setuptools.command.develop import develop as _develop
+from distutils import sysconfig
 from os.path import dirname, join
 import sys, re
 
@@ -207,7 +208,16 @@ setup(
                         include_dirs    = ['.', '3rdparty/include'],
                         define_macros   = [('BUILDING_LIBGOLANG', None)],
                         extra_compile_args = ['-std=gnu++11'], # not c++11 as linux/list.h uses typeof
+                        soversion       = '0.1'),
+
+                    DSO('golang.runtime.libpyxruntime',
+                        ['golang/runtime/libpyxruntime.cpp'],
+                        depends = ['golang/pyx/runtime.h'],
+                        include_dirs    = ['.', sysconfig.get_python_inc()],
+                        define_macros   = [('BUILDING_LIBPYXRUNTIME', None)],
+                        extra_compile_args = ['-std=c++11'],
                         soversion       = '0.1')],
+
     ext_modules = [
                     Ext('golang._golang',
                         ['golang/_golang.pyx']),
@@ -220,6 +230,10 @@ setup(
                         ['golang/runtime/_runtime_gevent.pyx'],
                         language = 'c'),
 
+                    Ext('golang.pyx.runtime',
+                        ['golang/pyx/runtime.pyx'],
+                        dsos = ['golang.runtime.libpyxruntime']),
+
                     Ext('golang._golang_test',
                         ['golang/_golang_test.pyx',
                          'golang/runtime/libgolang_test_c.c',
@@ -234,7 +248,8 @@ setup(
                         ['golang/_sync_test.pyx']),
 
                     Ext('golang._time',
-                        ['golang/_time.pyx']),
+                        ['golang/_time.pyx'],
+                        dsos = ['golang.runtime.libpyxruntime']),
                   ],
     include_package_data = True,