Sync with master

.gitignore

@@ -10,6 +10,8 @@ build/
 *.so.*
 *.dylib
 *.dll
+*.lib
+*.exp
 *.pyd
 *_dsoinfo.py
 

.gitmodules

+[submodule "3rdparty/ratas"]
+	path = 3rdparty/ratas
+	url = https://github.com/jsnell/ratas.git

.lsan-ignore.txt

+# .lsan-ignore.txt lists memory leak events that LeakSanitizer should not
+# report when running pygolang tests.
+#
+# Many python allocations, whose lifetime coincides with python interpreter
+# lifetime, and which are not explicitly freed on python shutdown, are
+# reported as leaks by default. Disable leak reporting for those to avoid
+# non-pygolang related printouts.
+
+
+# >>> Everything created when initializing python, e.g. sys.stderr
+#   #0 0x7f21e74f3bd7 in malloc .../asan_malloc_linux.cpp:69
+#   #1 0x555f361ff9a4 in PyThread_allocate_lock Python/thread_pthread.h:385
+#   #2 0x555f3623f72a in _buffered_init Modules/_io/bufferedio.c:725
+#   #3 0x555f3623ff7e in _io_BufferedWriter___init___impl Modules/_io/bufferedio.c:1803
+#   #4 0x555f3623ff7e in _io_BufferedWriter___init__ Modules/_io/clinic/bufferedio.c.h:489
+#   #5 0x555f3610c086 in type_call Objects/typeobject.c:1103
+#   #6 0x555f3609cdcc in _PyObject_MakeTpCall Objects/call.c:214
+#   #7 0x555f3609d6a8 in _PyObject_VectorcallTstate Include/internal/pycore_call.h:90
+#   #8 0x555f3609d6a8 in _PyObject_VectorcallTstate Include/internal/pycore_call.h:77
+#   #9 0x555f3609d6a8 in _PyObject_CallFunctionVa Objects/call.c:536
+#   #10 0x555f3609e89c in _PyObject_CallFunction_SizeT Objects/call.c:590
+#   #11 0x555f3623a0df in _io_open_impl Modules/_io/_iomodule.c:407
+#   #12 0x555f3623a0df in _io_open Modules/_io/clinic/_iomodule.c.h:264
+#   #13 0x555f360f17da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
+#   #14 0x555f3609d54c in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
+#   #15 0x555f3609d54c in _PyObject_CallFunctionVa Objects/call.c:536
+#   #16 0x555f3609ec34 in callmethod Objects/call.c:608
+#   #17 0x555f3609ec34 in _PyObject_CallMethod Objects/call.c:677
+#   #18 0x555f361e60cf in create_stdio Python/pylifecycle.c:2244
+#   #19 0x555f361e6523 in init_sys_streams Python/pylifecycle.c:2431
+#   #20 0x555f361e6523 in init_interp_main Python/pylifecycle.c:1154
+#   #21 0x555f361e7204 in pyinit_main Python/pylifecycle.c:1230
+#   #22 0x555f361e85ba in Py_InitializeFromConfig Python/pylifecycle.c:1261
+#   #23 0x555f3621010a in pymain_init Modules/main.c:67
+#   #24 0x555f362113de in pymain_main Modules/main.c:701
+#   #25 0x555f362113de in Py_BytesMain Modules/main.c:734
+leak:^pymain_init$
+
+# >>> Everything created when importing py modules, e.g.
+#   #0 0x7f18c86f3bd7 in malloc .../asan_malloc_linux.cpp:69
+#   #1 0x55b971430acf in PyMem_RawMalloc Objects/obmalloc.c:586
+#   #2 0x55b971430acf in _PyObject_Malloc Objects/obmalloc.c:2003
+#   #3 0x55b971430acf in _PyObject_Malloc Objects/obmalloc.c:1996
+#   #4 0x55b971415696 in new_keys_object Objects/dictobject.c:632
+#   #5 0x55b971415716 in dictresize Objects/dictobject.c:1429
+#   #6 0x55b97141961a in insertion_resize Objects/dictobject.c:1183
+#   #7 0x55b97141961a in insertdict Objects/dictobject.c:1248
+#   #8 0x55b97143eb7b in add_subclass Objects/typeobject.c:6547
+#   #9 0x55b97144ca52 in type_ready_add_subclasses Objects/typeobject.c:6345
+#   #10 0x55b97144ca52 in type_ready Objects/typeobject.c:6476
+#   #11 0x55b971451a1f in PyType_Ready Objects/typeobject.c:6508
+#   #12 0x55b971451a1f in type_new_impl Objects/typeobject.c:3189
+#   #13 0x55b971451a1f in type_new Objects/typeobject.c:3323
+#   #14 0x55b971443014 in type_call Objects/typeobject.c:1091
+#   #15 0x55b9713d3dcc in _PyObject_MakeTpCall Objects/call.c:214
+#   #16 0x55b9713d47bd in _PyObject_FastCallDictTstate Objects/call.c:141
+#   #17 0x55b9713d47bd in PyObject_VectorcallDict Objects/call.c:165
+#   #18 0x55b9714d14c2 in builtin___build_class__ Python/bltinmodule.c:209
+#   #19 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
+#   #20 0x55b9713d4a7b in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
+#   #21 0x55b9713d4a7b in PyObject_Vectorcall Objects/call.c:299
+#   #22 0x55b97137666e in _PyEval_EvalFrameDefault Python/ceval.c:4769
+#   #23 0x55b9714d7e6b in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
+#   #24 0x55b9714d7e6b in _PyEval_Vector Python/ceval.c:6434
+#   #25 0x55b9714d7e6b in PyEval_EvalCode Python/ceval.c:1148
+#   #26 0x55b9714d2e1f in builtin_exec_impl Python/bltinmodule.c:1077
+#   #27 0x55b9714d2e1f in builtin_exec Python/clinic/bltinmodule.c.h:465
+#   #28 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
+#   #29 0x55b971376dcb in do_call_core Python/ceval.c:7349
+#   #30 0x55b971376dcb in _PyEval_EvalFrameDefault Python/ceval.c:5376
+#   #31 0x55b9714d7faf in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
+#   #32 0x55b9714d7faf in _PyEval_Vector Python/ceval.c:6434
+#   #33 0x55b9713d436e in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
+#   #34 0x55b9713d436e in object_vacall Objects/call.c:819
+#   #35 0x55b9713d63cf in PyObject_CallMethodObjArgs Objects/call.c:879
+#   #36 0x55b9715080e1 in import_find_and_load Python/import.c:1748
+#   #37 0x55b9715080e1 in PyImport_ImportModuleLevelObject Python/import.c:1847
+#   #38 0x55b97137de9c in import_name Python/ceval.c:7422
+#   #39 0x55b97137de9c in _PyEval_EvalFrameDefault Python/ceval.c:3946
+#   #40 0x55b9714d7e6b in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
+#   #41 0x55b9714d7e6b in _PyEval_Vector Python/ceval.c:6434
+#   #42 0x55b9714d7e6b in PyEval_EvalCode Python/ceval.c:1148
+#   #43 0x55b9714d2e1f in builtin_exec_impl Python/bltinmodule.c:1077
+#   #44 0x55b9714d2e1f in builtin_exec Python/clinic/bltinmodule.c.h:465
+#   #45 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
+#   #46 0x55b971376dcb in do_call_core Python/ceval.c:7349
+#   #47 0x55b971376dcb in _PyEval_EvalFrameDefault Python/ceval.c:5376
+leak:^PyImport_Import
+#  importlib.import_module leads to
+#   #0 0x7f1951ef3bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
+#   #1 0x55f399e8cacf in PyMem_RawMalloc Objects/obmalloc.c:586
+#   #2 0x55f399e8cacf in _PyObject_Malloc Objects/obmalloc.c:2003
+#   #3 0x55f399e8cacf in _PyObject_Malloc Objects/obmalloc.c:1996
+#   #4 0x55f399e86344 in PyModule_ExecDef Objects/moduleobject.c:400
+#   #5 0x55f399f6178a in exec_builtin_or_dynamic Python/import.c:2345
+#   #6 0x55f399f6178a in _imp_exec_dynamic_impl Python/import.c:2419
+#   #7 0x55f399f6178a in _imp_exec_dynamic Python/clinic/import.c.h:474
+#   #8 0x55f399e8438a in cfunction_vectorcall_O Objects/methodobject.c:514
+leak:^_imp_exec_dynamic
+
+
+# >>> Everything allocated at DSO initialization, e.g.
+#   #0 0x7f35d2af46c8 in operator new(unsigned long) .../asan_new_delete.cpp:95
+#   #1 0x7f35ce897e9f in __static_initialization_and_destruction_0 golang/context.cpp:61
+#   #2 0x7f35ce8982ef in _GLOBAL__sub_I_context.cpp golang/context.cpp:380
+#   #3 0x7f35d32838bd in call_init elf/dl-init.c:90
+#   #4 0x7f35d32838bd in call_init elf/dl-init.c:27
+#   #5 0x7f35d32839a3 in _dl_init elf/dl-init.c:137
+#   #6 0x7f35d256e023 in __GI__dl_catch_exception elf/dl-error-skeleton.c:182
+#   #7 0x7f35d328a09d in dl_open_worker elf/dl-open.c:808
+#   #8 0x7f35d256dfc9 in __GI__dl_catch_exception elf/dl-error-skeleton.c:208
+#   #9 0x7f35d328a437 in _dl_open elf/dl-open.c:884
+#   #10 0x7f35d24a4437 in dlopen_doit dlfcn/dlopen.c:56
+#   #11 0x7f35d256dfc9 in __GI__dl_catch_exception elf/dl-error-skeleton.c:208
+#   #12 0x7f35d256e07e in __GI__dl_catch_error elf/dl-error-skeleton.c:227
+#   #13 0x7f35d24a3f26 in _dlerror_run dlfcn/dlerror.c:138
+#   #14 0x7f35d24a44e8 in dlopen_implementation dlfcn/dlopen.c:71
+#   #15 0x7f35d24a44e8 in ___dlopen dlfcn/dlopen.c:81
+#   #16 0x7f35d2a77ff9 in dlopen .../sanitizer_common_interceptors.inc:6341
+leak:^_GLOBAL_
+
+
+# global<> does not deallocate its reference on purpose
+leak:^_test_global()$

ratas @ becd5fc5

+Subproject commit becd5fc5c1e9ea600cd8b3b1c24d564794fedac4

MANIFEST.in

-include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .nxdtest
+include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .lsan-ignore.txt .nxdtest conftest.py
 include golang/libgolang.h
 include golang/runtime/libgolang.cpp
 include golang/runtime/libpyxruntime.cpp
 include golang/pyx/runtime.h
-include golang/pyx/runtime.cpp
+include golang/pyx/testprog/golang_dso_user/dsouser/dso.h
 include golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
 include golang/runtime/internal.h
 include golang/runtime/internal/atomic.h
@@ -34,6 +34,9 @@ include golang/sync_test.cpp
 include golang/time.h
 include golang/time.cpp
 include golang/_testing.h
+include golang/_compat/windows/strings.h
+include golang/_compat/windows/unistd.h
 recursive-include golang    *.py *.pxd *.pyx *.toml *.txt*
 recursive-include gpython   *.py
 recursive-include 3rdparty  *.h
+recursive-exclude golang    *_dsoinfo.py

conftest.py

+# pygolang | pytest config
+# Copyright (C) 2021-2024  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+from __future__ import print_function, absolute_import
+
+import gc
+
+
+# Do full GC before pytest exits, to avoid false positives in the leak detector.
+def pytest_unconfigure():
+    gc.collect()

golang/__init__.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2018-2022  Nexedi SA and Contributors.
+# Copyright (C) 2018-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -42,6 +42,9 @@ from golang._gopath import gimport  # make gimport available from golang
 import inspect, sys
 import decorator, six
 
+import setuptools_dso
+setuptools_dso.dylink_prepare_dso('golang.runtime.libgolang')
+
 from golang._golang import _pysys_exc_clear as _sys_exc_clear
 
 # @func is a necessary decorator for functions for selected golang features to work.
@@ -70,10 +73,25 @@ def _meth(cls, fcall):
         # wrap f with @_func, so that e.g. defer works automatically.
         f = _func(f)
 
-        if isinstance(f, (staticmethod, classmethod)):
-            func_name = f.__func__.__name__
+        # property is special - it has up to 3 functions inside
+        # but the only practical case is when it starts with
+        #
+        #   @func(Class)
+        #   @property
+        #   def ...
+        #
+        # which means that .fget should be set and so we use get name as the
+        # name of the method.
+        f_ = f
+        if isinstance(f, property):
+            f_ = f.fget
+            if f_ is None:
+                raise ValueError("func(cls) used on property without getter")
+
+        if isinstance(f_, (staticmethod, classmethod)):
+            func_name = f_.__func__.__name__
         else:
-            func_name = f.__name__
+            func_name = f_.__name__
         setattr(cls, func_name, f)
 
         # if `@func(cls) def name` caller already has `name` set, don't override it
@@ -82,28 +100,57 @@ def _meth(cls, fcall):
         if already is not missing:
             return already
 
-        # FIXME try to arrange so that python does not set anything on caller's
-        # namespace[func_name]  (currently it sets that to implicitly returned None)
+        # arrange so that python eventually does not set anything on caller's
+        # namespace[func_name]  (it unconditionally sets what decorator returns, even implicit None)
+        #
+        # _DelAttrAfterMeth.__del__ is invoked:
+        # * on cpython: right after  namespace[func_name] = returned _meth_leftover
+        # * on pypy:    eventually on next GC
+        fcall.f_locals[func_name] = _DelAttrAfterMeth(fcall.f_locals, func_name)
+        return _meth_leftover
 
     return deco
 
+# _DelAttrAfterMeth serves _meth by unsetting f_locals[meth] that python
+# unconditionally sets after `@func(cls) def meth()`.
+_meth_leftover = object()
+class _DelAttrAfterMeth(object):
+    def __init__(self, f_locals, name):
+        self.f_locals = f_locals
+        self.name     = name
+    def __del__(self):
+        obj = self.f_locals.get(self.name)
+        if obj is _meth_leftover:
+            del self.f_locals[self.name]
+
 # _func serves @func.
 def _func(f):
-    # @staticmethod & friends require special care:
+    # @property is special: there are 3 functions inside and we need to wrap
+    # them all with repacking back into property.
+    if isinstance(f, property):
+        fget = fset = fdel = None
+        if f.fget is not None:
+            fget = _func(f.fget)
+        if f.fset is not None:
+            fset = _func(f.fset)
+        if f.fdel is not None:
+            fdel = _func(f.fdel)
+        return type(f)(fget, fset, fdel, f.__doc__)
+
+    # @staticmethod & friends also require special care:
     # unpack f first to original func and then repack back after wrapping.
     fclass = None
     if isinstance(f, (staticmethod, classmethod)):
         fclass = type(f)
         f = f.__func__
 
-    def _(f, *argv, **kw):
-        # run f under separate frame, where defer will register calls.
-        __goframe__ = _GoFrame()
-        with __goframe__:
-            return f(*argv, **kw)
-
-    # keep all f attributes, like __name__, __doc__, etc on _
-    _ = decorator.decorate(f, _)
+    # prepare function that runs f under separate frame, where defer will register calls
+    # keep all f attributes, like __name__, __doc__, etc on the wrapper
+    # if f was already wrapped with _func - no need to wrap it again
+    _ = f
+    if getattr(f, '__go_wrapper__', None) is not _goframe:
+        _ = decorator.decorate(f, _goframe)
+        _.__go_wrapper__ = _goframe
 
     # repack _ into e.g. @staticmethod if that was used on f.
     if fclass is not None:
@@ -111,7 +158,13 @@ def _func(f):
 
     return _
 
-# _GoFrame serves __goframe__ that is setup by @func.
+# _goframe is used by @func to run f under separate frame.
+def _goframe(f, *argv, **kw):
+    __goframe__ = _GoFrame()
+    with __goframe__:
+        return f(*argv, **kw)
+
+# _GoFrame serves __goframe__ that is setup by _goframe.
 class _GoFrame:
     def __init__(self):
         self.deferv    = []     # defer registers funcs here

golang/_compat/windows/strings.h

+#ifndef _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
+#define _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
+//
+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+#include <string.h>
+
+static inline void bzero(void *p, size_t n) {
+    memset(p, '\0', n);
+}
+
+#endif  // _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
+

golang/_compat/windows/unistd.h

+#ifndef _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
+#define _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
+//
+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// stub unistd.h to be used on Windows where it is absent.
+// we need this because e.g. `cimport posix.stat` forces inclusion of unistd.h
+// even if we use part of posix.stat that is available everywhere.
+
+#include <io.h>
+
+#define O_CLOEXEC  _O_NOINHERIT
+
+#endif  // _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H

golang/_golang.pyx

@@ -5,7 +5,7 @@
 # distutils: language = c++
 # distutils: depends = libgolang.h os/signal.h _golang_str.pyx
 #
-# Copyright (C) 2018-2022  Nexedi SA and Contributors.
+# Copyright (C) 2018-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -173,11 +173,18 @@ cdef void __goviac(void *arg) nogil:
 
 # ---- channels ----
 
+# _frompyx indicates that a constructor is called from pyx code
+cdef object _frompyx = object()
+
 @final
 cdef class pychan:
     def __cinit__(pychan pych, size=0, dtype=object):
-        pych.dtype = parse_dtype(dtype)
-        pych._ch = _makechan_pyexc(dtypeRegistry[<int>pych.dtype].size, size)
+        if dtype is _frompyx:
+            pych.dtype = DTYPE_STRUCTZ  # anything
+            pych._ch   = NULL
+        else:
+            pych.dtype = parse_dtype(dtype)
+            pych._ch = _makechan_pyexc(dtypeRegistry[<int>pych.dtype].size, size)
 
     # pychan.nil(X) creates new nil pychan with specified dtype.
     # TODO try to avoid exposing .nil on pychan instances, and expose only pychan.nil
@@ -370,7 +377,7 @@ cdef void pychan_asserttype(pychan pych, DType dtype) nogil:
         panic("pychan: channel type mismatch")
 
 cdef pychan pychan_from_raw(_chan *_ch, DType dtype):
-    cdef pychan pych = pychan.__new__(pychan)
+    cdef pychan pych = pychan.__new__(pychan, dtype=_frompyx)
     pych.dtype = dtype
     pych._ch   = _ch; _chanxincref(_ch)
     return pych
@@ -469,7 +476,7 @@ def pyselect(*pycasev):
                 casev[i].user = pych.dtype
 
         with nogil:
-            selected = _chanselect_pyexc(&casev[0], casev.size())
+            selected = _chanselect_pyexc(casev.data(), casev.size())
 
     finally:
         # decref not sent tx (see ^^^ send prepare)
@@ -626,9 +633,7 @@ cdef object c_to_py(DType dtype, const chanElemBuf *cfrom):
 
 # mkpynil creates pychan instance that represents nil[dtype].
 cdef PyObject *mkpynil(DType dtype):
-    cdef pychan pynil = pychan.__new__(pychan)
-    pynil.dtype = dtype
-    pynil._ch   = NULL   # should be already NULL
+    cdef pychan pynil = pychan_from_raw(NULL, dtype)
     Py_INCREF(pynil)
     return <PyObject *>pynil
 
@@ -818,9 +823,6 @@ from libcpp.typeinfo cimport type_info
 from cython.operator cimport typeid
 from libc.string cimport strcmp
 
-# _frompyx indicates that a constructor is called from pyx code
-cdef object _frompyx = object()
-
 cdef class pyerror(Exception):
     # pyerror <- error
     @staticmethod

golang/_golang_str.pyx

 # -*- coding: utf-8 -*-
-# Copyright (C) 2018-2022  Nexedi SA and Contributors.
+# Copyright (C) 2018-2023  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -1791,8 +1791,9 @@ cdef (int, int) _utf8_decode_rune(const uint8_t[::1] s):
         if _ucs2_build and len(r) == 2:
             try:
                 return _xuniord(r), l
-            # e.g. TypeError: ord() expected a character, but string of length 2 found
-            except TypeError:
+            # py: TypeError: ord() expected a character, but string of length 2 found
+            # cy: ValueError: only single character unicode strings can be converted to Py_UCS4, got length 2
+            except (TypeError, ValueError):
                 l -= 1
                 continue
 

golang/_golang_test.pyx

@@ -2,7 +2,7 @@
 # cython: language_level=2
 # distutils: language=c++
 #
-# Copyright (C) 2018-2020  Nexedi SA and Contributors.
+# Copyright (C) 2018-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -344,6 +344,25 @@ cdef nogil:
         pych.chan_double().close()
 
 
+# verify that pychan_from_raw is not leaking C channel.
+def test_pychan_from_raw_noleak():
+    # pychan_from_raw used to create another channel and leak it
+    #
+    # this test _implicitly_ verifies that it is no longer the case - if it is,
+    # LSAN will report a memory leak after running the test.
+    #
+    # TODO consider adding explicit verification effective even under regular
+    #      builds. Possible options:
+    #
+    #      * verify malloc totals before and after tested code
+    #        see e.g. https://stackoverflow.com/q/1761125/9456786
+    #      * hook _makechan and verify that it is not invoked from under
+    #        pychan_from_raw. Depends on funchook integration.
+    cdef chan[int] ch   = makechan[int]()
+    cdef pychan    pych = pychan.from_chan_int(ch)  # uses pychan_from_raw internally
+    # pych and ch are freed automatically
+
+
 # ---- benchmarks ----
 
 # bench_go_nogil mirrors golang_test.py:bench_go

golang/_gopath.py

-# Copyright (C) 2018-2019  Nexedi SA and Contributors.
+# Copyright (C) 2018-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -34,11 +34,7 @@ from __future__ import print_function, absolute_import
 
 import os, os.path
 import sys
-
-import warnings
-with warnings.catch_warnings():
-    warnings.simplefilter('ignore', DeprecationWarning)
-    import imp
+import six
 
 # _gopathv returns $GOPATH vector.
 def _gopathv():
@@ -51,11 +47,25 @@ def _gopathv():
 
 # gimport imports python module or package from fully-qualified module name under $GOPATH.
 def gimport(name):
-    imp.acquire_lock()
+    _gimport_lock()
     try:
         return _gimport(name)
     finally:
-        imp.release_lock()
+        _gimport_unlock()
+
+# on py2 there is global import lock
+# on py3 we need to organize our own gimport synchronization
+if six.PY2:
+    import imp
+    _gimport_lock   = imp.acquire_lock
+    _gimport_unlock = imp.release_lock
+else:
+    from importlib import machinery as imp_machinery
+    from importlib import util      as imp_util
+    from golang import sync
+    _gimport_mu = sync.Mutex()
+    _gimport_lock   = _gimport_mu.lock
+    _gimport_unlock = _gimport_mu.unlock
 
 def _gimport(name):
     # we will register imported module into sys.modules with adjusted path.
@@ -93,4 +103,16 @@ def _gimport(name):
 
 
     # https://stackoverflow.com/a/67692
-    return imp.load_source(modname, modpath)
+    return _imp_load_source(modname, modpath)
+
+def _imp_load_source(modname, modpath):
+    if six.PY2:
+        return imp.load_source(modname, modpath)
+
+    # https://docs.python.org/3/whatsnew/3.12.html#imp
+    loader = imp_machinery.SourceFileLoader(modname, modpath)
+    spec   = imp_util.spec_from_file_location(modname, modpath, loader=loader)
+    mod    = imp_util.module_from_spec(spec)
+    sys.modules[modname] = mod
+    loader.exec_module(mod)
+    return mod

golang/fmt.cpp

-// Copyright (C) 2019-2020  Nexedi SA and Contributors.
+// Copyright (C) 2019-2023  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -48,7 +48,7 @@ string _vsprintf(const char *format, va_list argp) {
     return string(buf.get(), buf.get() + nchar); // without trailing '\0'
 }
 
-string sprintf(const string &format, ...) {
+string sprintf(const string format, ...) {
     va_list argp;
     va_start(argp, format);
     string str = fmt::_vsprintf(format.c_str(), argp);
@@ -64,7 +64,7 @@ string sprintf(const char *format, ...) {
     return str;
 }
 
-error ___errorf(const string &format, ...) {
+error ___errorf(const string format, ...) {
     va_list argp;
     va_start(argp, format);
     error err = errors::New(fmt::_vsprintf(format.c_str(), argp));

golang/fmt.h

 #ifndef _NXD_LIBGOLANG_FMT_H
 #define _NXD_LIBGOLANG_FMT_H
 
-// Copyright (C) 2019-2020  Nexedi SA and Contributors.
+// Copyright (C) 2019-2023  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -40,7 +40,7 @@ namespace golang {
 namespace fmt {
 
 // sprintf formats text into string.
-LIBGOLANG_API string sprintf(const string &format, ...);
+LIBGOLANG_API string sprintf(const string format, ...);
 
 
 // intseq<i1, i2, ...> and intrange<n> are used by errorf to handle %w.
@@ -75,7 +75,7 @@ namespace {
 //
 // format suffix ": %w" is additionally handled as in Go with
 // `errorf("... : %w", ..., err)` creating error that can be unwrapped back to err.
-LIBGOLANG_API error ___errorf(const string& format, ...);
+LIBGOLANG_API error ___errorf(const string format, ...);
 LIBGOLANG_API error ___errorfTryWrap(const string& format, error last_err, ...);
 LIBGOLANG_API string ___error_str(error err);
 
@@ -111,7 +111,10 @@ inline error errorf(const string& format, Argv... argv) {
 // `const char *` overloads just to catch format mistakes as
 // __attribute__(format) does not work with std::string.
 LIBGOLANG_API string sprintf(const char *format, ...)
-                                __attribute__ ((format (printf, 1, 2)));
+#ifndef _MSC_VER
+                                __attribute__ ((format (printf, 1, 2)))
+#endif
+	;
 
 // cannot use __attribute__(format) for errorf as we add %w handling.
 // still `const char *` overload is useful for performance.

golang/libgolang.h

 #ifndef _NXD_LIBGOLANG_H
 #define _NXD_LIBGOLANG_H
 
-// Copyright (C) 2018-2022  Nexedi SA and Contributors.
+// Copyright (C) 2018-2024  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -176,6 +176,12 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 
+#include <fcntl.h>
+#ifdef _MSC_VER // no mode_t on msvc
+typedef int mode_t;
+#endif
+
+
 // DSO symbols visibility (based on https://gcc.gnu.org/wiki/Visibility)
 #if defined _WIN32 || defined __CYGWIN__
   #define LIBGOLANG_DSO_EXPORT __declspec(dllexport)
@@ -340,8 +346,13 @@ typedef struct _libgolang_runtime_ops {
     // previously successfully allocated via sema_alloc.
     void             (*sema_free)   (_libgolang_sema*);
 
-    // sema_acquire/sema_release should acquire/release live semaphore allocated via sema_alloc.
-    void             (*sema_acquire)(_libgolang_sema*);
+    // sema_acquire should try to acquire live semaphore allocated via sema_alloc during given time.
+    // it returns whether acquisition succeeded or timed out.
+    // the timeout is specified in nanoseconds.
+    // UINT64_MAX means no timeout.
+    bool             (*sema_acquire)(_libgolang_sema*, uint64_t timeout_ns);
+
+    // sema_release should release live semaphore allocated via sema_alloc.
     void             (*sema_release)(_libgolang_sema*);
 
     // nanosleep should pause current goroutine for at least dt nanoseconds.
@@ -577,7 +588,7 @@ int select(const _selcase (&casev)[N]) {
 
 static inline                       // select(vector<casev>)
 int select(const std::vector<_selcase> &casev) {
-    return _chanselect(&casev[0], casev.size());
+    return _chanselect(casev.data(), casev.size());
 }
 
 // defer(f) mimics `defer f()` from golang.
@@ -829,7 +840,7 @@ struct _interface {
 
 protected:
     // don't use destructor -> use decref
-    ~_interface();
+    LIBGOLANG_API ~_interface();
 };
 typedef refptr<_interface> interface;
 

golang/os.cpp

-// Copyright (C) 2019-2022  Nexedi SA and Contributors.
+// Copyright (C) 2019-2023  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -37,7 +37,8 @@
 // GLIBC <  2.32 provides sys_siglist but not sigdescr_np in its headers
 // cut this short
 // (on darwing sys_siglist declaration is normally provided)
-#ifndef __APPLE__
+// (on windows sys_siglist is not available at all)
+#if !(defined(__APPLE__) || defined(_WIN32))
 extern "C" {
     extern const char * const sys_siglist[];
 }
@@ -226,8 +227,8 @@ tuple<string, error> ReadFile(const string& path) {
 
     while (1) {
         int n;
-        tie(n, err) = f->Read(&buf[0], buf.size());
-        data.append(&buf[0], n);
+        tie(n, err) = f->Read(buf.data(), buf.size());
+        data.append(buf.data(), n);
         if (err != nil) {
             if (err == io::EOF_)
                 err = nil;
@@ -248,7 +249,7 @@ tuple<string, error> ReadFile(const string& path) {
 
 tuple<File, File, error> Pipe() {
     int vfd[2], syserr;
-    syserr = sys::Pipe(vfd);
+    syserr = sys::Pipe(vfd, 0);
     if (syserr != 0)
         return make_tuple(nil, nil, fmt::errorf("pipe: %w", sys::NewErrno(syserr)));
 
@@ -286,8 +287,20 @@ string Signal::String() const {
     const Signal& sig = *this;
     const char *sigstr = nil;
 
+#ifdef _WIN32
+    switch (sig.signo) {
+    case SIGABRT:   return "Aborted";
+    case SIGBREAK:  return "Break";
+    case SIGFPE:    return "Floating point exception";
+    case SIGILL:    return "Illegal instruction";
+    case SIGINT:    return "Interrupt";
+    case SIGSEGV:   return "Segmentation fault";
+    case SIGTERM:   return "Terminated";
+    }
+#else
     if (0 <= sig.signo && sig.signo < NSIG)
         sigstr = ::sys_siglist[sig.signo]; // might be nil as well
+#endif
 
     if (sigstr != nil)
         return string(sigstr);

golang/os.h

 #ifndef _NXD_LIBGOLANG_OS_H
 #define _NXD_LIBGOLANG_OS_H
 //
-// Copyright (C) 2019-2022  Nexedi SA and Contributors.
+// Copyright (C) 2019-2023  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -61,7 +61,7 @@ private:
     ~_File();
     friend File _newFile(_libgolang_ioh* ioh, const string& name);
 public:
-    void decref();
+    LIBGOLANG_API void decref();
 
 public:
     LIBGOLANG_API string  Name()  const;
@@ -95,9 +95,15 @@ private:
 
 // Open opens file @path.
 LIBGOLANG_API std::tuple<File, error> Open(const string &path, int flags = O_RDONLY,
-        mode_t mode = S_IRUSR | S_IWUSR | S_IXUSR |
+        mode_t mode =
+#if !defined(_MSC_VER)
+                      S_IRUSR | S_IWUSR | S_IXUSR |
                       S_IRGRP | S_IWGRP | S_IXGRP |
-                      S_IROTH | S_IWOTH | S_IXOTH);
+                      S_IROTH | S_IWOTH | S_IXOTH
+#else
+                      _S_IREAD | _S_IWRITE
+#endif
+        );
 
 // NewFile wraps OS-level file-descriptor into File.
 // The ownership of sysfd is transferred to File.

golang/os/signal.cpp

-// Copyright (C) 2021-2022  Nexedi SA and Contributors.
+// Copyright (C) 2021-2023  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -89,6 +89,10 @@
 #include <atomic>
 #include <tuple>
 
+#if defined(_WIN32)
+# include <windows.h>
+#endif
+
 
 #define DEBUG 0
 #if DEBUG
@@ -97,6 +101,12 @@
 #  define debugf(format, ...) do {} while (0)
 #endif
 
+#if defined(_MSC_VER)
+# define HAVE_SIGACTION 0
+#else
+# define HAVE_SIGACTION 1
+#endif
+
 // golang::os::signal::
 namespace golang {
 namespace os {
@@ -109,13 +119,30 @@ using std::tie;
 using std::vector;
 using cxx::set;
 
+#if HAVE_SIGACTION
+static void xsigemptyset(sigset_t *sa_mask);
+#else
+// custom `struct sigaction` emulated via signal(2)
+#define SA_SIGINFO  1
+typedef struct {} siginfo_t;
+struct sigaction {
+    union {
+        void (*sa_handler)(int);
+        void (*sa_sigaction)(int, siginfo_t*, void*);
+    };
+    int  sa_flags;
+};
+static void _os_sighandler_nosigaction(int sig);
+#endif
+
 static void _os_sighandler(int sig, siginfo_t *info, void *ucontext);
 static void _notify(int signo);
 static void _checksig(int signo);
 static void _checkActEqual(const struct sigaction *a, const struct sigaction *b);
 static void _spinwaitNextQueueCycle();
+
+static int  sys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact);
 static void xsys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact);
-static void xsigemptyset(sigset_t *sa_mask);
 static bool _sigact_equal(const struct sigaction *a, const struct sigaction *b);
 
 
@@ -160,27 +187,31 @@ void _init() {
 
     // create _wakerx <-> _waketx pipe; set _waketx to nonblocking mode
     int vfd[2];
-    if (sys::Pipe(vfd) < 0)
+    if (sys::Pipe(vfd, O_CLOEXEC) < 0)
         panic("pipe(_wakerx, _waketx)");        // TODO +syserr
-    if (sys::Fcntl(vfd[0], F_SETFD, FD_CLOEXEC) < 0)
-        panic("fcntl(_wakerx, FD_CLOEXEC)");    // TODO +syserr
     error err;
     tie(_wakerx, err) = os::NewFile(vfd[0], "_wakerx");
     if (err != nil)
         panic("os::newFile(_wakerx");
     _waketx = vfd[1];
+#ifndef _WIN32
     if (sys::Fcntl(_waketx, F_SETFL, O_NONBLOCK) < 0)
         panic("fcntl(_waketx, O_NONBLOCK)");    // TODO +syserr
-    if (sys::Fcntl(_waketx, F_SETFD, FD_CLOEXEC) < 0)
-        panic("fcntl(_waketx, FD_CLOEXEC)");    // TODO +syserr
+#else
+    HANDLE hwaketx = (HANDLE)_get_osfhandle(_waketx);
+    DWORD  mode    = PIPE_READMODE_BYTE | PIPE_NOWAIT;
+    if (!SetNamedPipeHandleState(hwaketx, &mode, NULL, NULL))
+        panic("SetNamedPipeHandleState(hwaketx, PIPE_NOWAIT)"); // TODO +syserr
+#endif
 
     _actIgnore.sa_handler = SIG_IGN;
     _actIgnore.sa_flags   = 0;
-    xsigemptyset(&_actIgnore.sa_mask);
-
     _actNotify.sa_sigaction = _os_sighandler;
     _actNotify.sa_flags     = SA_SIGINFO;
+#if HAVE_SIGACTION
+    xsigemptyset(&_actIgnore.sa_mask);
     xsigemptyset(&_actNotify.sa_mask);
+#endif
 }
 
 
@@ -248,6 +279,14 @@ done:
             if (sah != SIG_IGN) {
                 if (sah != SIG_DFL) {
                     sah(sig);
+                    // handlers installed via signal usually reinstall themselves
+                    // return it back to us
+                    struct sigaction after;
+                    xsys_sigaction(sig, &_actNotify, &after);
+                    if (! (_sigact_equal(&after, &_actNotify)   ||
+                           ( ((after.sa_flags & SA_SIGINFO) == 0)   &&
+                             ((after.sa_handler == SIG_DFL) || (after.sa_handler == sah)))))
+                        panic("collision detected wrt thirdparty signal usage");
                 }
                 else {
                     // SIG_DFL && _SigReset - reraise to die if the signal is fatal
@@ -343,7 +382,7 @@ static int/*syserr*/ _Notify1(chan<os::Signal> ch, os::Signal sig) {
 
     // retrieve current signal action
     struct sigaction cur;
-    int syserr = sys::Sigaction(sig.signo, nil, &cur);
+    int syserr = sys_sigaction(sig.signo, nil, &cur);
     if (syserr < 0) {
         // TODO reenable once we can panic with any object
         //return fmt::errorf("sigaction sig%d: %w", sig.signo, sys::NewErrno(syserr);
@@ -389,7 +428,7 @@ static int/*syserr*/ _Notify1(chan<os::Signal> ch, os::Signal sig) {
 
         // register our sigaction
         struct sigaction old;
-        syserr = sys::Sigaction(sig.signo, &_actNotify, &old);
+        syserr = sys_sigaction(sig.signo, &_actNotify, &old);
         if (syserr < 0) {
             // TODO reenable once we can panic with any object
             //return fmt::errorf("sigaction sig%d: %w", sig.signo, sys::NewErrno(syserr);
@@ -469,7 +508,7 @@ static int/*syserr*/ _Ignore1(os::Signal sig) {
     if (h == nil) {
         h = new _SigHandler();
         h->sigstate.store(_SigIgnoring);
-        int syserr = sys::Sigaction(sig.signo, nil, &h->prev_act);
+        int syserr = sys_sigaction(sig.signo, nil, &h->prev_act);
         if (syserr < 0) {
             delete h;
             return syserr; // TODO errctx
@@ -481,7 +520,7 @@ static int/*syserr*/ _Ignore1(os::Signal sig) {
     h->sigstate.store(_SigIgnoring);
     h->subscribers.clear();
 
-    int syserr = sys::Sigaction(sig.signo, &_actIgnore, nil);
+    int syserr = sys_sigaction(sig.signo, &_actIgnore, nil);
     if (syserr < 0)
         return syserr; // TODO errctx
 
@@ -509,7 +548,7 @@ static int/*syserr*/ _Reset1(os::Signal sig) {
     h->sigstate.store(_SigReset);
 
     struct sigaction act;
-    int syserr = sys::Sigaction(sig.signo, &h->prev_act, &act);
+    int syserr = sys_sigaction(sig.signo, &h->prev_act, &act);
     if (syserr < 0)
         return syserr; // TODO errctx
     if (sigstate == _SigNotifying)
@@ -617,16 +656,75 @@ static void _checksig(int signo) {
 }
 
 
+#if HAVE_SIGACTION
 static void xsigemptyset(sigset_t *sa_mask) {
     if (sigemptyset(sa_mask) < 0)
         panic("sigemptyset failed"); // must always succeed
 }
+#endif
 
 static void xsys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact) {
-    int syserr = sys::Sigaction(signo, act, oldact);
+    int syserr = sys_sigaction(signo, act, oldact);
     if (syserr != 0)
         panic("sigaction failed");   // TODO add errno detail
 }
+static int sys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact) {
+#if HAVE_SIGACTION
+    return sys::Sigaction(signo, act, oldact);
+#else
+    sys::sighandler_t oldh, newh;
+    int syserr;
+
+    if (act == nil) {
+        newh = SIG_GET;
+    }
+    else if (act->sa_flags & SA_SIGINFO) {
+        if (act->sa_sigaction != _os_sighandler)
+            panic("BUG: compat sigaction: act->sa_sigaction != _os_sighandler");
+        newh = _os_sighandler_nosigaction;
+    }
+    else {
+        newh = act->sa_handler;
+    }
+
+    oldh = sys::Signal(signo, newh, &syserr);
+    if (oldh == SIG_ERR)
+        return syserr;
+    if (oldact != nil) {
+        if (oldh == _os_sighandler_nosigaction) {
+            oldact->sa_sigaction = _os_sighandler;
+            oldact->sa_flags     = SA_SIGINFO;
+        }
+        else {
+            oldact->sa_handler   = oldh;
+            oldact->sa_flags     = 0;
+        }
+    }
+    return 0;
+#endif
+}
+
+#if !HAVE_SIGACTION
+static void _os_sighandler_nosigaction(int sig) {
+    // reinstall sighandler since on handler invocation it is reset to SIG_DFL.
+    // do it as fast as we can as the first thing - to minimize window of race
+    // while signal handler is reset from being our _os_sighandler.
+    //
+    // NOTE it is ok to reinstall before invoking _os_sighandler because
+    // _os_sighandler uses only atomics and pipe write and so is reentrant.
+    sys::sighandler_t h;
+    int syserr;
+    h = sys::Signal(sig, _os_sighandler_nosigaction, &syserr);
+    if (h == SIG_ERR)
+        panic("signal(reinstall) failed");
+    if ( !(h == SIG_DFL /*reset*/ ||
+           h == _os_sighandler_nosigaction /*concurrent sighandler*/) )
+        panic("collision detected wrt thirdparty signal usage");
+
+    // invoke _os_sighandler as if it was setup by sigaction.
+    _os_sighandler(sig, NULL, NULL);
+}
+#endif
 
 static bool _sigact_equal(const struct sigaction *a, const struct sigaction *b) {
     // don't compare sigaction by memcmp - it will fail because struct sigaction

golang/os/testprog/signal_test_all.py

 #!/usr/bin/env python
-# Copyright (C) 2021-2022  Nexedi SA and Contributors.
+# Copyright (C) 2021-2023  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -24,7 +24,8 @@ from __future__ import print_function, absolute_import
 from golang import chan
 from golang import os as gos, syscall, time
 from golang.os import signal
-import os, sys
+from golang.os.signal_test import killme
+import sys
 
 def main():
     # build "all signals" list
@@ -35,11 +36,17 @@ def main():
             if sig not in allsigv: # avoid e.g. SIGCHLD/SIGCLD dups
                 allsigv.append(sig)
     allsigv.sort(key=lambda sig: sig.signo)
-    allsigv.remove(syscall.SIGKILL) # SIGKILL/SIGSTOP cannot be caught
-    allsigv.remove(syscall.SIGSTOP)
-    allsigv.remove(syscall.SIGBUS)  # AddressSanitizer crashes on SIGBUS/SIGFPE/SIGSEGV
-    allsigv.remove(syscall.SIGFPE)
-    allsigv.remove(syscall.SIGSEGV)
+    def without(signame):
+        sig = getattr(syscall, signame, None)
+        if sig is not None:
+            allsigv.remove(sig)
+    without('SIGKILL') # SIGKILL/SIGSTOP cannot be caught
+    without('SIGSTOP')
+    without('SIGBUS')  # AddressSanitizer crashes on SIGBUS/SIGFPE/SIGSEGV
+    without('SIGFPE')
+    without('SIGSEGV')
+    without('SIGILL')  # SIGILL/SIGABRT cause termination on windows
+    without('SIGABRT')
 
     # Notify() -> kill * -> should be notified
     ch = chan(10, dtype=gos.Signal)
@@ -72,11 +79,6 @@ def main():
     killme(syscall.SIGTERM)
     raise AssertionError("not terminated")
 
-# killme sends signal sig to own process.
-def killme(sig):
-    mypid = os.getpid()
-    os.kill(mypid, sig.signo)
-
 def emit(msg=''):
     print(msg)
     sys.stdout.flush()

golang/pyx/build.py

-# Copyright (C) 2019-2022  Nexedi SA and Contributors.
+# Copyright (C) 2019-2023  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -73,8 +73,10 @@ _dso_build_ext = setuptools_dso.build_ext
 class build_ext(_dso_build_ext):
     def build_extension(self, ext):
         # wrap _compiler <src> -> <obj> with our code
+
+        # ._compile is used on gcc/clang but not with msvc
         _compile = self.compiler._compile
-        def _(obj, src, ext, cc_args, extra_postargs, pp_opts):
+        def xcompile(obj, src, ext, cc_args, extra_postargs, pp_opts):
             # filter_out removes arguments that start with argprefix
             cc_args         = cc_args[:]
             extra_postargs  = extra_postargs[:]
@@ -101,11 +103,30 @@ class build_ext(_dso_build_ext):
                 filter_out('-std=gnu++')
 
             _compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
-        self.compiler._compile = _
+
+        # msvc handles all sources directly in the loop in .compile and we can
+        # do per-source adjustsment only in .spawn .
+        spawn = self.compiler.spawn
+        def xspawn(argv):
+            c = False
+            for arg in argv:
+                if arg.startswith('/Tc'):
+                    c = True
+            if c:
+                argv = argv[:]
+                for i in range(len(argv)):
+                    if argv[i] == '/std:c++20':
+                        argv[i] = '/std:c11'
+
+            return spawn(argv)
+
+        self.compiler._compile = xcompile
+        self.compiler.spawn    = xspawn
         try:
             _dso_build_ext.build_extension(self, ext) # super doesn't work for _dso_build_ext
         finally:
             self.compiler._compile = _compile
+            self.compiler.spawn    = spawn
 
 
 # setup should be used instead of setuptools.setup
@@ -128,14 +149,14 @@ def setup(**kw):
 #       x_dsos = [DSO('mypkg.mydso', ['mypkg/mydso.cpp'])],
 #   )
 def DSO(name, sources, **kw):
-    _, kw = _with_build_defaults(kw)
+    _, kw = _with_build_defaults(name, kw)
     dso = setuptools_dso.DSO(name, sources, **kw)
     return dso
 
 
 # _with_build_defaults returns copy of kw amended with build options common for
 # both DSO and Extension.
-def _with_build_defaults(kw):   # -> (pygo, kw')
+def _with_build_defaults(name, kw):   # -> (pygo, kw')
     # find pygolang root
     gopkg = _findpkg("golang")
     pygo  = dirname(gopkg.path) # .../pygolang/golang -> .../pygolang
@@ -144,15 +165,22 @@ def _with_build_defaults(kw):   # -> (pygo, kw')
 
     kw = kw.copy()
 
-    # prepend -I<pygolang> so that e.g. golang/libgolang.h is found
+    sysname = platform.system().lower()
+    win  = (sysname == 'windows')
+    msvc = win  # TODO also support mingw ?
+
+    # prepend   -I<pygolang> so that e.g. golang/libgolang.h is found
+    # same with -I<pygolang>/golang/_compat/<os>
     incv = kw.get('include_dirs', [])[:]
     incv.insert(0, pygo)
+    incv.insert(1, join(pygo, 'golang', '_compat', sysname))
     kw['include_dirs'] = incv
 
-    # link with libgolang.so
-    dsov = kw.get('dsos', [])[:]
-    dsov.insert(0, 'golang.runtime.libgolang')
-    kw['dsos'] = dsov
+    # link with libgolang.so  if it is not libgolang itself
+    if name != 'golang.runtime.libgolang':
+        dsov = kw.get('dsos', [])[:]
+        dsov.insert(0, 'golang.runtime.libgolang')
+        kw['dsos'] = dsov
 
     # default language to C++ (chan[T] & co are accessible only via C++)
     lang = kw.setdefault('language', 'c++')
@@ -160,9 +188,20 @@ def _with_build_defaults(kw):   # -> (pygo, kw')
     # default to C++11 (chan[T] & co require C++11 features)
     ccdefault = []
     if lang == 'c++':
-        ccdefault.append('-std=c++11')
+        if not msvc:
+            if name == 'golang.runtime.libgolang':
+                ccdefault.append('-std=gnu++11') # not c++11 as linux/list.h uses typeof
+            else:
+                ccdefault.append('-std=c++11')
+        else:
+            # MSVC requries C++20 for designated struct initializers
+            ccdefault.append('/std:c++20')
+            # and make exception handling: "fully conformant", so that e.g. extern "C" `panic` works
+            ccdefault.extend(['/EHc-', '/EHsr'])
+
     # default to no strict-aliasing
-    ccdefault.append('-fno-strict-aliasing')
+    if not msvc:
+        ccdefault.append('-fno-strict-aliasing') # use only on gcc/clang - msvc does it by default
 
     _ = kw.get('extra_compile_args', [])[:]
     _[0:0] = ccdefault              # if another e.g. -std=... was already there -
@@ -188,6 +227,8 @@ def _with_build_defaults(kw):   # -> (pygo, kw')
         'os/signal.h',
         'pyx/runtime.h',
         '_testing.h',
+        '_compat/windows/strings.h',
+        '_compat/windows/unistd.h',
     ]])
     kw['depends'] = dependv
 
@@ -203,7 +244,7 @@ def _with_build_defaults(kw):   # -> (pygo, kw')
 #       ext_modules = [Extension('mypkg.mymod', ['mypkg/mymod.pyx'])],
 #   )
 def Extension(name, sources, **kw):
-    pygo, kw = _with_build_defaults(kw)
+    pygo, kw = _with_build_defaults(name, kw)
 
     # some pyx-level depends to workaround a bit lack of proper dependency
     # tracking in setuptools/distutils.

golang/pyx/build_test.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2019  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+# Copyright (C) 2019-2024  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -28,7 +28,8 @@ testprog = dirname(__file__) + "/testprog"
 # verify that we can build/run external package that uses pygolang in pyx mode.
 def test_pyx_build():
     pyxuser = testprog + "/golang_pyx_user"
-    pyrun(["setup.py", "build_ext", "-i"], cwd=pyxuser)
+    pyrun(["setup.py", "build_ext", "-i"], cwd=pyxuser,
+          lsan=False)   # gcc leaks
 
     # run built test.
     _ = pyout(["-c",
@@ -44,8 +45,8 @@ def test_pyx_build():
 # verify that we can build/run external dso that uses libgolang.
 def test_dso_build():
     dsouser = testprog + "/golang_dso_user"
-    pyrun(["setup.py", "build_dso", "-i"], cwd=dsouser)
-    pyrun(["setup.py", "build_ext", "-i"], cwd=dsouser)
+    pyrun(["setup.py", "build_dso", "-i"], cwd=dsouser,  lsan=False) # gcc leaks
+    pyrun(["setup.py", "build_ext", "-i"], cwd=dsouser,  lsan=False) # gcc leaks
 
     # run built test.
     _ = pyout(["-c",

golang/pyx/runtime.h

 #ifndef _NXD_LIBGOLANG_PYX_RUNTIME_H
 #define _NXD_LIBGOLANG_PYX_RUNTIME_H
 
-// Copyright (C) 2018-2019  Nexedi SA and Contributors.
+// Copyright (C) 2018-2023  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -58,8 +58,8 @@ class _PyError final : public _error, object {
 private:
     _PyError();
     ~_PyError();
-    friend error PyErr_Fetch();
-    friend void  PyErr_ReRaise(PyError pyerr);
+    friend LIBPYXRUNTIME_API error PyErr_Fetch();
+    friend LIBPYXRUNTIME_API void  PyErr_ReRaise(PyError pyerr);
 public:
     LIBPYXRUNTIME_API void incref();
     LIBPYXRUNTIME_API void decref();

golang/pyx/testprog/cmdclass_custom.py

-# Copyright (C) 2019  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+# Copyright (C) 2019-2022  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -29,4 +29,9 @@ class mybuild_ext(build_ext):
 
 setup(
     cmdclass    = {'build_ext': mybuild_ext},
+
+    # avoid setuptools thinking nearby golang_pyx_user/ golang_dso_user/ also
+    # relate to hereby setup and rejecting the build.
+    # https://stackoverflow.com/questions/72294299/multiple-top-level-packages-discovered-in-a-flat-layout
+    py_modules  = [],
 )

golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp

-// Copyright (C) 2019  Nexedi SA and Contributors.
-//                     Kirill Smelkov <kirr@nexedi.com>
+// Copyright (C) 2019-2023  Nexedi SA and Contributors.
+//                         Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
 // it under the terms of the GNU General Public License version 3, or (at your
@@ -20,7 +20,7 @@
 // Small library that uses a bit of libgolang features, mainly to verify
 // that external project can build against libgolang.
 
-#include <golang/libgolang.h>
+#include "dso.h"
 using namespace golang;
 
 #include <stdio.h>

golang/pyx/testprog/golang_dso_user/dsouser/dso.h

+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+#ifndef _NXD_GOLANG_DSOUSER_DSO_H
+#define _NXD_GOLANG_DSOUSER_DSO_H
+
+#include <golang/libgolang.h>
+
+#if BUILDING_DSOUSER_DSO
+#  define DSOUSER_DSO_API LIBGOLANG_DSO_EXPORT
+#else
+#  define DSOUSER_DSO_API LIBGOLANG_DSO_IMPORT
+#endif
+
+DSOUSER_DSO_API void dsotest();
+
+#endif // _NXD_GOLANG_DSOUSER_DSO_H

golang/pyx/testprog/golang_dso_user/dsouser/test.pyx

 # cython: language_level=2
 #
-# Copyright (C) 2019  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+# Copyright (C) 2019-2023  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -21,10 +21,7 @@
 
 # Small test driver that calls dso.so .
 
-cdef extern from * nogil:
-    """
-    void dsotest();
-    """
+cdef extern from "dso.h" nogil:
     void dsotest()
 
 def main():

golang/pyx/testprog/golang_dso_user/setup.py

-# Copyright (C) 2019  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+# Copyright (C) 2019-2023  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -24,7 +24,9 @@ setup(
     name        = 'golang_dso_user',
     description = 'test project that uses libgolang from a dso',
 
-    x_dsos      = [DSO('dsouser.dso', ['dsouser/dso.cpp'])],
+    x_dsos      = [DSO('dsouser.dso',
+                       ['dsouser/dso.cpp'],
+                       define_macros = [('BUILDING_DSOUSER_DSO', None)])],
     ext_modules = [Extension('dsouser.test',
                     ['dsouser/test.pyx'],
                     dsos = ['dsouser.dso'])],

golang/runtime/_libgolang.pxd

 # cython: language_level=2
-# Copyright (C) 2019-2022  Nexedi SA and Contributors.
+# Copyright (C) 2019-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -36,7 +36,7 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil:
 
         _libgolang_sema* (*sema_alloc)  ()
         void             (*sema_free)   (_libgolang_sema*)
-        void             (*sema_acquire)(_libgolang_sema*)
+        bint             (*sema_acquire)(_libgolang_sema*, uint64_t timeout_ns)
         void             (*sema_release)(_libgolang_sema*)
 
         void        (*nanosleep)(uint64_t)

golang/runtime/_runtime_gevent.pyx

 # cython: language_level=2
-# Copyright (C) 2019-2022  Nexedi SA and Contributors.
+# Copyright (C) 2019-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -40,7 +40,10 @@ ELSE:
 
 from gevent import sleep as pygsleep
 
-from libc.stdint cimport uint8_t, uint64_t
+from libc.stdint cimport uint8_t, uint64_t, UINT64_MAX
+cdef extern from *:
+    ctypedef bint cbool "bool"
+
 from cpython cimport PyObject, Py_INCREF, Py_DECREF
 from cython cimport final
 
@@ -57,7 +60,7 @@ from posix.fcntl cimport mode_t, F_GETFL, F_SETFL, O_NONBLOCK, O_ACCMODE, O_RDON
 from posix.stat cimport struct_stat, S_ISREG, S_ISDIR, S_ISBLK
 from posix.strings cimport bzero
 
-from gevent.fileobject import FileObjectThread, FileObjectPosix
+from gevent import fileobject as gfileobj
 
 
 # _goviapy & _togo serve go
@@ -95,9 +98,12 @@ cdef:
         Py_DECREF(pygsema)
         return True
 
-    bint _sema_acquire(_libgolang_sema *gsema):
+    bint _sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns, cbool* pacq):
         pygsema = <PYGSema>gsema
-        pygsema.acquire()
+        timeout = None
+        if timeout_ns != UINT64_MAX:
+            timeout = float(timeout_ns) * 1e-9
+        pacq[0] = pygsema.acquire(timeout=timeout)
         return True
 
     bint _sema_release(_libgolang_sema *gsema):
@@ -142,14 +148,16 @@ cdef nogil:
         if not ok:
             panic("pyxgo: gevent: sema: free: failed")
 
-    void sema_acquire(_libgolang_sema *gsema):
+    cbool sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns):
         cdef PyExc exc
+        cdef cbool acq
         with gil:
             pyexc_fetch(&exc)
-            ok = _sema_acquire(gsema)
+            ok = _sema_acquire(gsema, timeout_ns, &acq)
             pyexc_restore(exc)
         if not ok:
             panic("pyxgo: gevent: sema: acquire: failed")
+        return acq
 
     void sema_release(_libgolang_sema *gsema):
         cdef PyExc exc
@@ -194,31 +202,39 @@ cdef nogil:
         return ioh
 
     _libgolang_ioh* _io_fdopen(int *out_syserr, int sysfd):
-        # check if we should enable O_NONBLOCK on this file-descriptor
-        # even though we could enable O_NONBLOCK for regular files, it does not
-        # work as expected as most unix'es report regular files as always read
-        # and write ready.
-        cdef struct_stat st
-        cdef int syserr = syscall.Fstat(sysfd, &st)
-        if syserr < 0:
-            out_syserr[0] = syserr
-            return NULL
-        m = st.st_mode
-        blocking = (S_ISREG(m) or S_ISDIR(m) or S_ISBLK(m)) # fd cannot refer to symlink
-
-        # retrieve current sysfd flags and access mode
-        flags = syscall.Fcntl(sysfd, F_GETFL, 0)
-        if flags < 0:
-            out_syserr[0] = flags
-            return NULL
-        acc = (flags & O_ACCMODE)
-
-        # enable O_NONBLOCK if needed
-        if not blocking:
-            syserr = syscall.Fcntl(sysfd, F_SETFL, flags | O_NONBLOCK)
+        IF POSIX:
+            # check if we should enable O_NONBLOCK on this file-descriptor
+            # even though we could enable O_NONBLOCK for regular files, it does not
+            # work as expected as most unix'es report regular files as always read
+            # and write ready.
+            cdef struct_stat st
+            cdef int syserr = syscall.Fstat(sysfd, &st)
             if syserr < 0:
                 out_syserr[0] = syserr
                 return NULL
+            m = st.st_mode
+            blocking = (S_ISREG(m) or S_ISDIR(m) or S_ISBLK(m)) # fd cannot refer to symlink
+
+            # retrieve current sysfd flags and access mode
+            flags = syscall.Fcntl(sysfd, F_GETFL, 0)
+            if flags < 0:
+                out_syserr[0] = flags
+                return NULL
+            acc = (flags & O_ACCMODE)
+
+            # enable O_NONBLOCK if needed
+            if not blocking:
+                syserr = syscall.Fcntl(sysfd, F_SETFL, flags | O_NONBLOCK)
+                if syserr < 0:
+                    out_syserr[0] = syserr
+                    return NULL
+
+        ELSE: # !POSIX (windows)
+            cdef bint blocking = True
+            # FIXME acc: use GetFileInformationByHandleEx to determine whether
+            # HANDLE(sysfd) was opened for reading or writing.
+            # https://stackoverflow.com/q/9442436/9456786
+            cdef int acc = O_RDWR
 
         # create IOH backed by FileObjectThread or FileObjectPosix
         ioh = <IOH*>calloc(1, sizeof(IOH))
@@ -253,9 +269,9 @@ cdef:
         pygfobj = None
         try:
             if blocking:
-                pygfobj = FileObjectThread(sysfd, mode=mode, buffering=0)
+                pygfobj = gfileobj.FileObjectThread(sysfd, mode=mode, buffering=0)
             else:
-                pygfobj = FileObjectPosix(sysfd, mode=mode, buffering=0)
+                pygfobj = gfileobj.FileObjectPosix(sysfd, mode=mode, buffering=0)
         except OSError as e:
             out_syserr[0] = -e.errno
         else:
@@ -338,7 +354,19 @@ cdef:
         cdef uint8_t[::1] mem = <uint8_t[:count]>buf
         xmem = memoryview(mem) # to avoid https://github.com/cython/cython/issues/3900 on mem[:0]=b''
         try:
-            n = pygfobj.readinto(xmem)
+            # NOTE buf might be on stack, so it must not be accessed, e.g. from
+            # FileObjectThread, while our greenlet is parked (see STACK_DEAD_WHILE_PARKED
+            # for details). -> Use intermediate on-heap buffer to protect from that.
+            #
+            # Also: we cannot use pygfobj.readinto due to
+            # https://github.com/gevent/gevent/pull/1948
+            #
+            # TODO use .readinto() when buf is not on stack and gevent is
+            # recent enough or pygfobj is not FileObjectThread.
+            #n = pygfobj.readinto(xmem)
+            buf2 = pygfobj.read(count)
+            n = len(buf2)
+            xmem[:n] = buf2
         except OSError as e:
             n = -e.errno
         out_n[0] = n
@@ -361,8 +389,14 @@ cdef:
     bint _io_write(IOH* ioh, int* out_n, const void *buf, size_t count):
         pygfobj = <object>ioh.pygfobj
         cdef const uint8_t[::1] mem = <const uint8_t[:count]>buf
+
+        # NOTE buf might be on stack, so it must not be accessed, e.g. from
+        # FileObjectThread, while our greenlet is parked (see STACK_DEAD_WHILE_PARKED
+        # for details). -> Use intermediate on-heap buffer to protect from that.
+        buf2 = bytearray(mem)
+
         try:
-            n = pygfobj.write(mem)
+            n = pygfobj.write(buf2)
         except OSError as e:
             n = -e.errno
         out_n[0] = n

golang/runtime/_runtime_thread.pyx

 # cython: language_level=2
-# Copyright (C) 2019-2022  Nexedi SA and Contributors.
+# Copyright (C) 2019-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -35,7 +35,12 @@ from __future__ import print_function, absolute_import
 #
 # NOTE Cython declares PyThread_acquire_lock/PyThread_release_lock as nogil
 from cpython.pythread cimport PyThread_acquire_lock, PyThread_release_lock, \
-        PyThread_type_lock, WAIT_LOCK
+        PyThread_type_lock, WAIT_LOCK, NOWAIT_LOCK, PyLockStatus, PY_LOCK_ACQUIRED, PY_LOCK_FAILURE
+
+cdef extern from * nogil:
+    ctypedef int PY_TIMEOUT_T   # long long there
+    PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock, PY_TIMEOUT_T timeout_us, int intr_flag)
+
 
 # NOTE On Darwin, even though this is considered as POSIX, Python uses
 # mutex+condition variable to implement its lock, and, as of 20190828, Py2.7
@@ -98,6 +103,9 @@ from libc.errno  cimport errno, EINTR, EBADF
 from posix.fcntl cimport mode_t
 from posix.stat cimport struct_stat
 from posix.strings cimport bzero
+cdef extern from *:
+    ctypedef bint cbool "bool"
+
 IF POSIX:
     from posix.time cimport clock_gettime, nanosleep as posix_nanosleep, timespec, CLOCK_REALTIME
 ELSE:
@@ -138,11 +146,46 @@ cdef nogil:
         pysema = <PyThread_type_lock>gsema
         PyThread_free_lock(pysema)
 
-    void sema_acquire(_libgolang_sema *gsema):
+    cbool sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns):
         pysema = <PyThread_type_lock>gsema
-        ok = PyThread_acquire_lock(pysema, WAIT_LOCK)
-        if ok == 0:
-            panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock failed")
+        IF PY3:
+            cdef PY_TIMEOUT_T timeout_us
+        ELSE:
+            cdef uint64_t tprev, t, tsleep
+        if timeout_ns == UINT64_MAX:
+            ok = PyThread_acquire_lock(pysema, WAIT_LOCK)
+            if ok == 0:
+                panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock failed")
+            return 1
+        else:
+            IF PY3:
+                timeout_us = timeout_ns // 1000
+                lkok = PyThread_acquire_lock_timed(pysema, timeout_us, 0)
+                if lkok == PY_LOCK_FAILURE:
+                    return 0
+                elif lkok == PY_LOCK_ACQUIRED:
+                    return 1
+                else:
+                    panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock_timed failed")
+            ELSE:
+                # py2 misses PyThread_acquire_lock_timed - provide fallback ourselves
+                tprev = nanotime()
+                while 1:
+                     ok = PyThread_acquire_lock(pysema, NOWAIT_LOCK)
+                     if ok:
+                         return 1
+                     tsleep = min(timeout_ns, 50*1000)    # poll every 50 μs = 20 Hz
+                     if tsleep == 0:
+                         break
+                     nanosleep(tsleep)
+                     t = nanotime()
+                     if t < tprev:
+                         break  # clock skew
+                     if t - tprev >= timeout_ns:
+                         break
+                     timeout_ns -= t - tprev
+                     tprev = t
+                return 0
 
     void sema_release(_libgolang_sema *gsema):
         pysema = <PyThread_type_lock>gsema

golang/runtime/internal/atomic.cpp

-// Copyright (C) 2022  Nexedi SA and Contributors.
-//                     Kirill Smelkov <kirr@nexedi.com>
+// Copyright (C) 2022-2023  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
 // it under the terms of the GNU General Public License version 3, or (at your
@@ -20,7 +20,9 @@
 #include "golang/runtime/internal/atomic.h"
 #include "golang/libgolang.h"
 
+#ifndef _WIN32
 #include <pthread.h>
+#endif
 
 // golang::internal::atomic::
 namespace golang {
@@ -41,9 +43,12 @@ static void _forkNewEpoch() {
 }
 
 void _init() {
+// there is no fork on windows
+#ifndef _WIN32
     int e = pthread_atfork(/*prepare*/nil, /*inparent*/nil, /*inchild*/_forkNewEpoch);
     if (e != 0)
         panic("pthread_atfork failed");
+#endif
 }
 
 

golang/runtime/internal/syscall.cpp

-// Copyright (C) 2021-2022  Nexedi SA and Contributors.
+// Copyright (C) 2021-2023  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -57,15 +57,18 @@ string _Errno::Error() {
     _Errno& e = *this;
 
     char ebuf[128];
+    bool ok;
 #if __APPLE__
-    int x = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
-    if (x == 0)
-        return string(ebuf);
-    return "unknown error " + std::to_string(-e.syserr);
+    ok = (::strerror_r(-e.syserr, ebuf, sizeof(ebuf)) == 0);
+#elif defined(_WIN32)
+    ok = (::strerror_s(ebuf, sizeof(ebuf), -e.syserr) == 0);
 #else
     char *estr = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
     return string(estr);
 #endif
+    if (ok)
+        return string(ebuf);
+    return "unknown error " + std::to_string(-e.syserr);
 }
 
 
@@ -99,6 +102,7 @@ __Errno Close(int fd) {
     return err;
 }
 
+#ifndef _WIN32
 __Errno Fcntl(int fd, int cmd, int arg) {
     int save_errno = errno;
     int err = ::fcntl(fd, cmd, arg);
@@ -107,6 +111,7 @@ __Errno Fcntl(int fd, int cmd, int arg) {
     errno = save_errno;
     return err;
 }
+#endif
 
 __Errno Fstat(int fd, struct ::stat *out_st) {
     int save_errno = errno;
@@ -119,6 +124,10 @@ __Errno Fstat(int fd, struct ::stat *out_st) {
 
 int Open(const char *path, int flags, mode_t mode) {
     int save_errno = errno;
+#ifdef _WIN32  // default to open files in binary mode
+    if ((flags & (_O_TEXT | _O_BINARY)) == 0)
+        flags |= _O_BINARY;
+#endif
     int fd = ::open(path, flags, mode);
     if (fd < 0)
         fd = -errno;
@@ -126,15 +135,39 @@ int Open(const char *path, int flags, mode_t mode) {
     return fd;
 }
 
-__Errno Pipe(int vfd[2]) {
+__Errno Pipe(int vfd[2], int flags) {
+    // supported flags: O_CLOEXEC
+    if (flags & ~(O_CLOEXEC))
+        return -EINVAL;
     int save_errno = errno;
-    int err = ::pipe(vfd);
+    int err;
+#ifdef __linux__
+    err = ::pipe2(vfd, flags);
+#elif defined(_WIN32)
+    err = ::_pipe(vfd, 4096, flags | _O_BINARY);
+#else
+    err = ::pipe(vfd);
+    if (err)
+        goto out;
+    if (flags & O_CLOEXEC) {
+        for (int i=0; i<2; i++) {
+            err = Fcntl(vfd[i], F_SETFD, FD_CLOEXEC);
+            if (err) {
+                Close(vfd[0]);
+                Close(vfd[1]);
+                goto out;
+            }
+        }
+    }
+out:
+#endif
     if (err == -1)
         err = -errno;
     errno = save_errno;
     return err;
 }
 
+#ifndef _WIN32
 __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact) {
     int save_errno = errno;
     int err = ::sigaction(signo, act, oldact);
@@ -143,6 +176,17 @@ __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *
     errno = save_errno;
     return err;
 }
+#endif
+
+sighandler_t Signal(int signo, sighandler_t handler, int *out_psyserr) {
+    int save_errno = errno;
+    sighandler_t oldh = ::signal(signo, handler);
+    *out_psyserr = (oldh == SIG_ERR
+        ? (errno ? -errno : -EINVAL) // windows returns SIG_ERR/errno=0 for invalid signo
+        : 0);
+    errno = save_errno;
+    return oldh;
+}
 
 
 }}} // golang::internal::syscall::

golang/runtime/internal/syscall.h

 #ifndef _NXD_LIBGOLANG_RUNTIME_INTERNAL_SYSCALL_H
 #define _NXD_LIBGOLANG_RUNTIME_INTERNAL_SYSCALL_H
 
-// Copyright (C) 2021-2022  Nexedi SA and Contributors.
+// Copyright (C) 2021-2023  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -59,15 +59,21 @@ error NewErrno(__Errno syserr); // TODO better return Errno directly.
 
 // system calls
 
-int/*n|err*/ Read(int fd, void *buf, size_t count);
-int/*n|err*/ Write(int fd, const void *buf, size_t count);
-
-__Errno Close(int fd);
-__Errno Fcntl(int fd, int cmd, int arg);
-__Errno Fstat(int fd, struct ::stat *out_st);
-int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
-__Errno Pipe(int vfd[2]);
-__Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
+LIBGOLANG_API int/*n|err*/ Read(int fd, void *buf, size_t count);
+LIBGOLANG_API int/*n|err*/ Write(int fd, const void *buf, size_t count);
+
+LIBGOLANG_API __Errno Close(int fd);
+#ifndef _WIN32
+LIBGOLANG_API __Errno Fcntl(int fd, int cmd, int arg);
+#endif
+LIBGOLANG_API __Errno Fstat(int fd, struct ::stat *out_st);
+LIBGOLANG_API int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
+LIBGOLANG_API __Errno Pipe(int vfd[2], int flags);
+#ifndef _WIN32
+LIBGOLANG_API __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
+#endif
+typedef void (*sighandler_t)(int);
+LIBGOLANG_API sighandler_t /*sigh|SIG_ERR*/ Signal(int signo, sighandler_t handler, int *out_psyserr);
 
 
 }}} // golang::internal::syscall::

golang/runtime/libgolang.cpp

-// Copyright (C) 2018-2022  Nexedi SA and Contributors.
+// Copyright (C) 2018-2024  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -43,12 +43,23 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <strings.h>
 
 // linux/list.h needs ARRAY_SIZE    XXX -> better use c.h or ccan/array_size.h ?
 #ifndef ARRAY_SIZE
 # define ARRAY_SIZE(A) (sizeof(A) / sizeof((A)[0]))
 #endif
 #include <linux/list.h>
+// MSVC does not support statement expressions and typeof
+// -> redo list_entry via C++ lambda.
+#ifdef _MSC_VER
+# undef list_entry
+# define list_entry(ptr, type, member) [&]() {                      \
+        const decltype( ((type *)0)->member ) *__mptr = (ptr);      \
+        return (type *)( (char *)__mptr - offsetof(type,member) );  \
+    }()
+#endif
+
 
 using std::atomic;
 using std::bad_alloc;
@@ -120,6 +131,7 @@ using internal::_runtime;
 
 namespace internal { namespace atomic { extern void _init(); } }
 namespace os { namespace signal { extern void _init(); } }
+namespace time { extern void _init(); }
 void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) {
     if (_runtime != nil) // XXX better check atomically
         panic("libgolang: double init");
@@ -127,6 +139,7 @@ void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) {
 
     internal::atomic::_init();
     os::signal::_init();
+    time::_init();
 }
 
 void _taskgo(void (*f)(void *), void *arg) {
@@ -155,7 +168,15 @@ void _semafree(_sema *sema) {
 }
 
 void _semaacquire(_sema *sema) {
-    _runtime->sema_acquire((_libgolang_sema *)sema);
+    bool ok;
+    ok = _runtime->sema_acquire((_libgolang_sema *)sema, UINT64_MAX);
+    if (!ok)
+        panic("semaacquire: failed");
+}
+
+// NOTE not currently exposed in public API
+bool _semaacquire_timed(_sema *sema, uint64_t timeout_ns) {
+    return _runtime->sema_acquire((_libgolang_sema *)sema, timeout_ns);
 }
 
 void _semarelease(_sema *sema) {
@@ -899,6 +920,13 @@ template<> int _chanselect2</*onstack=*/true> (const _selcase *, int, const vect
 template<> int _chanselect2</*onstack=*/false>(const _selcase *, int, const vector<int>&);
 static int __chanselect2(const _selcase *, int, const vector<int>&, _WaitGroup*);
 
+// PRNG for select.
+// TODO consider switching to xoroshiro or wyrand if speed is an issue
+// https://thompsonsed.co.uk/random-number-generators-for-c-performance-tested
+// https://nullprogram.com/blog/2017/09/21/
+static std::random_device        _devrand;
+static thread_local std::mt19937 _t_rng(_devrand());
+
 // _chanselect executes one ready send or receive channel case.
 //
 // if no case is ready and default case was provided, select chooses default.
@@ -925,7 +953,7 @@ int _chanselect(const _selcase *casev, int casec) {
     vector<int> nv(casec); // n -> n(case)      TODO -> caller stack-allocate nv
     for (int i=0; i <casec; i++)
         nv[i] = i;
-    std::random_shuffle(nv.begin(), nv.end());
+    std::shuffle(nv.begin(), nv.end(), _t_rng);
 
     // first pass: poll all cases and bail out in the end if default was provided
     int  ndefault = -1;

golang/sync.h

 #ifndef _NXD_LIBGOLANG_SYNC_H
 #define	_NXD_LIBGOLANG_SYNC_H
 
-// Copyright (C) 2018-2020  Nexedi SA and Contributors.
+// Copyright (C) 2018-2023  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -212,7 +212,7 @@ class _WorkGroup : public object {
 private:
     _WorkGroup();
     ~_WorkGroup();
-    friend WorkGroup NewWorkGroup(context::Context ctx);
+    friend LIBGOLANG_API WorkGroup NewWorkGroup(context::Context ctx);
 public:
     LIBGOLANG_API void decref();
 

golang/testprog/golang_test_defer_excchain.txt

 Traceback (most recent call last):
-  File "PYGOLANG/golang/__init__.py", line ..., in _
+  File "PYGOLANG/golang/__init__.py", line ..., in _goframe
     return f(*argv, **kw)
+           ^^^^^^^^^^^^^^                                       +PY311
   File "PY39(PYGOLANG/golang/testprog/)golang_test_defer_excchain.py", line 42, in main
     raise RuntimeError("err")
 RuntimeError: err
@@ -21,6 +22,7 @@ Traceback (most recent call last):
     ...
   File "PY39(PYGOLANG/golang/testprog/)golang_test_defer_excchain.py", line 33, in d2
     1/0
+    ~^~                                                         +PY311
 ZeroDivisionError: ...
 
 During handling of the above exception, another exception occurred:
@@ -29,8 +31,9 @@ Traceback (most recent call last):
   ... "PY39(PYGOLANG/golang/testprog/)golang_test_defer_excchain.py", line 45, in <module>
     main()
   ...
-  File "PYGOLANG/golang/__init__.py", line ..., in _
-    return f(*argv, **kw)
+  File "PYGOLANG/golang/__init__.py", line ..., in _goframe
+    return f(*argv, **kw)                                       -PY310
+    with __goframe__:                                           +PY310
   File "PYGOLANG/golang/__init__.py", line ..., in __exit__
     ...
   File "PYGOLANG/golang/__init__.py", line ..., in __exit__

golang/testprog/golang_test_defer_excchain.txt-ipython

 ...
 RuntimeError                              Traceback (most recent call last)
-PYGOLANG/golang/__init__.py in _(f, *argv, **kw)
+PYGOLANG/golang/__init__.py in _goframe(f, *argv, **kw)
 ...
---> ...             return f(*argv, **kw)
+--> ...         return f(*argv, **kw)
 ...
 
 PYGOLANG/golang/testprog/golang_test_defer_excchain.py in main()
@@ -50,9 +50,9 @@ PYGOLANG/golang/testprog/golang_test_defer_excchain.py in ...
 
 ...
 
-PYGOLANG/golang/__init__.py in _(f, *argv, **kw)
+PYGOLANG/golang/__init__.py in _goframe(f, *argv, **kw)
 ...
---> ...             return f(*argv, **kw)
+--> ...         return f(*argv, **kw)
 ...
 
 PYGOLANG/golang/__init__.py in __exit__(__goframe__, exc_type, exc_val, exc_tb)

golang/testprog/golang_test_defer_excchain.txt-pytest

 ...
-_____________________________________ main _____________________________________
-../__init__.py:...: in _
+...________________ main ________________...
+../__init__.py:...: in _goframe
     return f(*argv, **kw)
 golang_test_defer_excchain.py:42: in main
     raise RuntimeError("err")

golang/testprog/golang_test_str.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2022  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+# Copyright (C) 2022-2023  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -29,8 +29,8 @@ from golang import b, u
 from golang.gcompat import qq
 
 def main():
-    sb = b("привет b")
-    su = u("привет u")
+    sb = b("привет αβγ b")
+    su = u("привет αβγ u")
     print("print(b):", sb)
     print("print(u):", su)
     print("print(qq(b)):", qq(sb))

golang/testprog/golang_test_str.txt

-print(b): привет b
-print(u): привет u
-print(qq(b)): "привет b"
-print(qq(u)): "привет u"
-print(repr(b)): b('привет b')
-print(repr(u)): u('привет u')
-print({b: u}): {b('привет b'): u('привет u')}
+print(b): привет αβγ b
+print(u): привет αβγ u
+print(qq(b)): "привет αβγ b"
+print(qq(u)): "привет αβγ u"
+print(repr(b)): b('привет αβγ b')
+print(repr(u)): u('привет αβγ u')
+print({b: u}): {b('привет αβγ b'): u('привет αβγ u')}

golang/time.h

 #ifndef _NXD_LIBGOLANG_TIME_H
 #define	_NXD_LIBGOLANG_TIME_H
 
-// Copyright (C) 2019-2020  Nexedi SA and Contributors.
+// Copyright (C) 2019-2024  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -118,12 +118,13 @@ private:
     double      _dt;
     sync::Mutex _mu;
     bool        _stop;
+    Timer       _timer;
 
     // don't new - create only via new_ticker()
 private:
     _Ticker();
     ~_Ticker();
-    friend Ticker new_ticker(double dt);
+    friend Ticker LIBGOLANG_API new_ticker(double dt);
 public:
     LIBGOLANG_API void decref();
 
@@ -147,18 +148,12 @@ LIBGOLANG_API Timer new_timer(double dt);
 struct _Timer : object {
     chan<double> c;
 
-private:
-    func<void()> _f;
-
-    sync::Mutex  _mu;
-    double       _dt;  // +inf - stopped, otherwise - armed
-    int          _ver; // current timer was armed by n'th reset
-
     // don't new - create only via new_timer() & co
 private:
     _Timer();
     ~_Timer();
     friend Timer _new_timer(double dt, func<void()> f);
+    friend class _TimerImpl;
 public:
     LIBGOLANG_API void decref();
 
@@ -182,9 +177,6 @@ public:
     //
     // the timer must be either already stopped or expired.
     LIBGOLANG_API void reset(double dt);
-
-private:
-    void _fire(double dt, int ver);
 };
 
 

golang/time_test.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2019  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+# Copyright (C) 2019-2024  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -20,9 +20,10 @@
 
 from __future__ import print_function, absolute_import
 
-from golang import select
-from golang import time
+from golang import select, func, defer
+from golang import time, sync
 from golang.golang_test import panics
+from six.moves import range as xrange
 
 # all timer tests operate in dt units
 dt = 10*time.millisecond
@@ -65,6 +66,7 @@ def test_ticker_time():
 
 
 # test_timer verifies that Timer/Ticker fire as expected.
+@func
 def test_timer():
     # start timers at x5, x7 and x11 intervals an verify that the timers fire
     # in expected sequence. The times when the timers fire do not overlap in
@@ -73,15 +75,15 @@ def test_timer():
     tv = [] # timer events
     Tstart = time.now()
 
-    t23 = time.Timer(23*dt)
-    t5  = time.Timer( 5*dt)
+    t23 = time.Timer(23*dt);        defer(t23.stop)
+    t5  = time.Timer( 5*dt);        defer(t5 .stop)
 
     def _():
         tv.append(7)
         t7f.reset(7*dt)
-    t7f = time.Timer( 7*dt, f=_)
+    t7f = time.Timer( 7*dt, f=_);   defer(t7f.stop)
 
-    tx11 = time.Ticker(11*dt)
+    tx11 = time.Ticker(11*dt);      defer(tx11.stop)
 
     while 1:
         _, _rx = select(
@@ -108,19 +110,20 @@ def test_timer():
 
 
 # test_timer_misc, similarly to test_timer, verifies misc timer convenience functions.
+@func
 def test_timer_misc():
     tv = []
     Tstart = time.now()
 
-    c23 = time.after(23*dt)
-    c5  = time.after( 5*dt)
+    c23 = time.after(23*dt)         # cannot stop
+    c5  = time.after( 5*dt)         # cannot stop
 
     def _():
         tv.append(7)
         t7f.reset(7*dt)
-    t7f = time.after_func(7*dt, _)
+    t7f = time.after_func(7*dt, _); defer(t7f.stop)
 
-    cx11 = time.tick(11*dt)
+    cx11 = time.tick(11*dt)         # cannot stop
 
     while 1:
         _, _rx = select(
@@ -148,13 +151,14 @@ def test_timer_misc():
 
 
 # test_timer_stop verifies that .stop() cancels Timer or Ticker.
+@func
 def test_timer_stop():
     tv = []
 
-    t10 = time.Timer (10*dt)
-    t2  = time.Timer ( 2*dt)    # will fire and cancel t3, tx5
-    t3  = time.Timer ( 3*dt)    # will be canceled
-    tx5 = time.Ticker( 5*dt)    # will be canceled
+    t10 = time.Timer (10*dt);   defer(t10.stop)
+    t2  = time.Timer ( 2*dt);   defer(t2 .stop) # will fire and cancel t3, tx5
+    t3  = time.Timer ( 3*dt);   defer(t3 .stop) # will be canceled
+    tx5 = time.Ticker( 5*dt);   defer(tx5.stop) # will be canceled
 
     while 1:
         _, _rx = select(
@@ -180,9 +184,10 @@ def test_timer_stop():
 
 
 # test_timer_stop_drain verifies that Timer/Ticker .stop() drains timer channel.
+@func
 def test_timer_stop_drain():
-    t  = time.Timer (1*dt)
-    tx = time.Ticker(1*dt)
+    t  = time.Timer (1*dt);     defer(t.stop)
+    tx = time.Ticker(1*dt);     defer(tx.stop)
 
     time.sleep(2*dt)
     assert len(t.c)  == 1
@@ -195,9 +200,45 @@ def test_timer_stop_drain():
     assert len(tx.c) == 0
 
 
+# test_timer_stop_vs_func verifies that Timer .stop() works correctly with func-timer.
+@func
+def test_timer_stop_vs_func():
+    tv = []
+    def _1(): tv.append(1)
+    def _2(): tv.append(2)
+
+    t1 = time.after_func(1e6*dt, _1);   defer(t1.stop)
+    t2 = time.after_func(  1*dt, _2);   defer(t2.stop)
+
+    time.sleep(2*dt)
+    assert t1.stop() == True
+    assert t2.stop() == False
+    assert tv == [2]
+
+
 # test_timer_reset_armed verifies that .reset() panics if called on armed timer.
+@func
 def test_timer_reset_armed():
     # reset while armed
-    t = time.Timer(10*dt)
+    t = time.Timer(10*dt);  defer(t.stop)
     with panics("Timer.reset: the timer is armed; must be stopped or expired"):
         t.reset(5*dt)
+
+
+# bench_timer_arm_cancel benchmarks arming timers that do not fire.
+# it shows how cheap or expensive it is to use timers to implement timeouts.
+def bench_timer_arm_cancel(b):
+    for i in xrange(b.N):
+        t = time.Timer(10*time.second)
+        _ = t.stop()
+        assert _ is True
+
+
+# bench_timer_arm_fire benchmarks arming timers that do fire.
+# it shows what it costs to go through all steps related to timer loop and firing timers.
+def bench_timer_arm_fire(b):
+    wg = sync.WaitGroup()
+    wg.add(b.N)
+    for i in xrange(b.N):
+        t = time.after_func(1*time.millisecond, wg.done)
+    wg.wait()

gpython/__init__.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2018-2021  Nexedi SA and Contributors.
+# Copyright (C) 2018-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -41,7 +41,7 @@ $GPYTHON_RUNTIME=threads.
 from __future__ import print_function, absolute_import
 
 
-_pyopt = "c:im:OVW:X:"
+_pyopt = "c:Eim:OvVW:X:"
 _pyopt_long = ('version',)
 
 # pymain mimics `python ...`
@@ -50,19 +50,27 @@ _pyopt_long = ('version',)
 # init, if provided, is called after options are parsed, but before interpreter start.
 def pymain(argv, init=None):
     import sys
-    from os.path import dirname, realpath
+    import os
+    from os.path import dirname, realpath, splitext
 
     # sys.executable
     # on windows there are
-    #   gpython-script.py
     #   gpython.exe
+    #   gpython-script.py
     #   gpython.manifest
-    # and argv[0] is gpython-script.py
+    # with gpython-script.py sometimes embedded into gpython.exe (pip/distlib)
+    # and argv[0] is 'gpython-script.py' (setuptools)  or  'gpython' (pip/distlib; note no .exe)
     exe  = realpath(argv[0])
     argv = argv[1:]
-    if exe.endswith('-script.py'):
-        exe = exe[:-len('-script.py')]
-        exe = exe + '.exe'
+    if os.name == 'nt':
+        if exe.endswith('-script.py'):
+            exe = exe[:-len('-script.py')]  # gpython-script.py  ->  gpython.exe
+            exe = exe + '.exe'
+        else:
+            _, ext = splitext(exe)          # gpython            ->  gpython.exe
+            if not ext:
+                exe += '.exe'
+
     sys._gpy_underlying_executable = sys.executable
     sys.executable  = exe
 
@@ -70,20 +78,26 @@ def pymain(argv, init=None):
     # `gpython file` will add path-to-file to sys.path[0] by itself, and
     # /path/to/gpython is unnecessary and would create difference in behaviour
     # in between gpython and python.
+    #
+    # on windows when gpython.exe comes with embedded __main__.py, it is
+    # gpython.exe that is installed into sys.path[0] .
     exedir = dirname(exe)
-    if sys.path[0] == exedir:
+    if sys.path[0] in (exedir, exe):
         del sys.path[0]
     else:
         # buildout injects `sys.path[0:0] = eggs` into python scripts.
         # detect that and remove sys.path entry corresponding to exedir.
         if not _is_buildout_script(exe):
-            raise RuntimeError('pymain: internal error: sys.path[0] was not set by underlying python to dirname(exe):'
+            raise RuntimeError('pymain: internal error: sys.path[0] was not set by underlying python to dirname(exe) or exe:'
                     '\n\n\texe:\t%s\n\tsys.path[0]:\t%s' % (exe, sys.path[0]))
         else:
-            if exedir in sys.path:
-                sys.path.remove(exedir)
-            else:
-                raise RuntimeError('pymain: internal error: sys.path does not contain dirname(exe):'
+            ok = False
+            for _ in (exedir, exe):
+                if _ in sys.path:
+                    sys.path.remove(_)
+                    ok = True
+            if not ok:
+                raise RuntimeError('pymain: internal error: sys.path does not contain dirname(exe) or exe:'
                     '\n\n\texe:\t%s\n\tsys.path:\t%s' % (exe, sys.path))
 
 
@@ -99,11 +113,24 @@ def pymain(argv, init=None):
     for (opt, arg) in igetopt:
         # options that require reexecuting through underlying python with that -<opt>
         if opt in (
+                '-E',   # ignore $PYTHON*
                 '-O',   # optimize
+                '-v',   # trace import statements
+                '-X',   # set implementation-specific option
             ):
-            reexec_with.append(opt)
-            if arg is not None:
-                reexec_with.append(arg)
+
+            # but keep `-X gpython.*` in user part of argv in case of reexec
+            # leaving it for main to handle. If it is only pymain to run, then
+            # we will be ignoring `-X gpython.*` which goes in line with builtin
+            # py3 behaviour to ignore any unknown -X option.
+            if opt == '-X' and arg is not None and arg.startswith('gpython.'):
+                reexec_argv.append(opt)
+                reexec_argv.append(arg)
+
+            else:
+                reexec_with.append(opt)
+                if arg is not None:
+                    reexec_with.append(arg)
             continue
 
         reexec_argv.append(opt)
@@ -202,7 +229,6 @@ def pymain(argv, init=None):
     #
     #   python -O gpython file.py
     if len(reexec_with) > 0:
-        import os
         argv = [sys._gpy_underlying_executable] + reexec_with + [sys.executable] + reexec_argv
         os.execv(argv[0], argv)
 
@@ -226,11 +252,12 @@ def pymain(argv, init=None):
         pyimpl = platform.python_implementation()
 
         v = _version_info_str
+        pyver  = platform.python_version()  # ~ v(sys.version_info) but might also have e.g. '+' at tail
         if pyimpl == 'CPython':
-            ver.append('CPython %s' % v(sys.version_info))
+            ver.append('CPython %s' % pyver)
         elif pyimpl == 'PyPy':
             ver.append('PyPy %s'   % v(sys.pypy_version_info))
-            ver.append('Python %s' % v(sys.version_info))
+            ver.append('Python %s' % pyver)
         else:
             ver = [] # unknown
 
@@ -367,33 +394,35 @@ def main():
     # no harm wrt gevent monkey-patching even if we import os first.
     import os
 
-    # extract and process `-X gpython.*`
+    # process `-X gpython.*`
     # -X gpython.runtime=(gevent|threads)    + $GPYTHON_RUNTIME
     sys._xoptions = getattr(sys, '_xoptions', {})
-    argv_ = []
     gpy_runtime = os.getenv('GPYTHON_RUNTIME', 'gevent')
     igetopt = _IGetOpt(sys.argv[1:], _pyopt, _pyopt_long)
     for (opt, arg) in igetopt:
         if opt == '-X':
-            if arg.startswith('gpython.'):
-                if arg.startswith('gpython.runtime='):
-                    gpy_runtime = arg[len('gpython.runtime='):]
-                    sys._xoptions['gpython.runtime'] = gpy_runtime
+            # any non gpython -X option is handled by pymain; ignore them here
+            if not arg.startswith('gpython.'):
+                continue
 
-                else:
-                    raise RuntimeError('gpython: unknown -X option %s' % opt)
+            if arg.startswith('gpython.runtime='):
+                gpy_runtime = arg[len('gpython.runtime='):]
+                sys._xoptions['gpython.runtime'] = gpy_runtime
 
-                continue
+            else:
+                raise RuntimeError('gpython: unknown -X option %s' % arg)
 
-        argv_.append(opt)
-        if arg is not None:
-            argv_.append(arg)
+            continue
 
         # options after -c / -m are not for python itself
         if opt in ('-c', '-m'):
             break
 
-    argv = [sys.argv[0]] + argv_ + igetopt.argv
+
+    # propagate those settings as defaults to subinterpreters, so that e.g.
+    # sys.executable spawned from under `gpython -X gpython.runtime=threads`
+    # also uses "threads" runtime by default.
+    os.environ['GPYTHON_RUNTIME'] = gpy_runtime
 
     # init initializes according to selected runtime
     # it is called after options are parsed and sys.path is setup correspondingly.
@@ -434,18 +463,18 @@ def main():
         sys.version += (' [GPython %s] [%s]' % (golang.__version__, gpy_verextra))
 
     # tail to pymain
-    pymain(argv, init)
+    pymain(sys.argv, init)
 
 
 # _is_buildout_script returns whether file @path is generated as python buildout script.
 def _is_buildout_script(path):
-    with open(path, 'r') as f:
+    with open(path, 'rb') as f:
         src = f.read()
     # buildout injects the following prologues into python scripts:
     #   sys.path[0:0] = [
     #     ...
     #   ]
-    return ('\nsys.path[0:0] = [\n' in src)
+    return (b'\nsys.path[0:0] = [\n' in src)
 
 
 # _IGetOpt provides getopt-style incremental options parsing.

gpython/gpython_test.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2019-2022  Nexedi SA and Contributors.
+# Copyright (C) 2019-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -184,7 +184,7 @@ def test_pymain():
     _ = pyout(['-m', 'hello', 'abc', 'def'], cwd=testdata)
     # realpath rewrites e.g. `local/lib -> lib` if local/lib is symlink
     hellopy = realpath(join(testdata, 'hello.py'))
-    assert _ == b"hello\nworld\n['%s', 'abc', 'def']\n" % b(hellopy)
+    assert _ == b"hello\nworld\n[%s, 'abc', 'def']\n" % b(repr(hellopy))
     # -m<module>
     __ = pyout(['-mhello', 'abc', 'def'], cwd=testdata)
     assert __ == _
@@ -195,8 +195,8 @@ def test_pymain():
 
     # -i after stdin (also tests interactive mode as -i forces interactive even on non-tty)
     d = {
-        b'hellopy': b(hellopy),
-        b'ps1':     b'' # cpython emits prompt to stderr
+        b'repr(hellopy)': b(repr(hellopy)),
+        b'ps1':           b'' # cpython emits prompt to stderr
     }
     if is_pypy and not is_gpython:
         d[b'ps1'] = b'>>>> ' # native pypy emits prompt to stdout and >>>> instead of >>>
@@ -212,7 +212,7 @@ def test_pymain():
     assert _ == b"hello\nworld\n['-c']\n%(ps1)s'~~HELLO~~'\n%(ps1)s"    % d
     # -i after -m
     _ = pyout(['-i', '-m', 'hello'], stdin=b'world.tag', cwd=testdata)
-    assert _ == b"hello\nworld\n['%(hellopy)s']\n%(ps1)s'~~WORLD~~'\n%(ps1)s"  % d
+    assert _ == b"hello\nworld\n[%(repr(hellopy))s]\n%(ps1)s'~~WORLD~~'\n%(ps1)s"  % d
     # -i after file
     _ = pyout(['-i', 'testdata/hello.py'], stdin=b'tag', cwd=here)
     assert _ == b"hello\nworld\n['testdata/hello.py']\n%(ps1)s'~~HELLO~~'\n%(ps1)s" % d
@@ -221,26 +221,27 @@ def test_pymain():
     # -W <opt>
     _ = pyout(['-Werror', '-Whello', '-W', 'ignore::DeprecationWarning',
                'testprog/print_warnings_setup.py'], cwd=here)
-    if PY2:
-        # py2 threading, which is imported after gpython startup, adds ignore
-        # for sys.exc_clear
-        _ = grepv(r'ignore:sys.exc_clear:DeprecationWarning:threading:*', _)
-    assert _.startswith(
-        b"sys.warnoptions: ['error', 'hello', 'ignore::DeprecationWarning']\n\n" + \
-        b"warnings.filters:\n" + \
-        b"- ignore::DeprecationWarning::*\n" + \
-        b"- error::Warning::*\n"), _
+    assert re.match(
+        br"sys\.warnoptions: \['error', 'hello', 'ignore::DeprecationWarning'\]\n\n"
+        br"warnings\.filters:\n"
+        br"(- [^\n]+\n)*" # Additional filters added by automatically imported modules
+        br"- ignore::DeprecationWarning::\*\n"
+        br"- error::Warning::\*\n"
+        br"(- [^\n]+\n)*", # Remaining filters
+        _,
+    )
     # $PYTHONWARNINGS
     _ = pyout(['testprog/print_warnings_setup.py'], cwd=here,
               envadj={'PYTHONWARNINGS': 'ignore,world,error::SyntaxWarning'})
-    if PY2:
-        # see ^^^
-        _ = grepv(r'ignore:sys.exc_clear:DeprecationWarning:threading:*', _)
-    assert _.startswith(
-        b"sys.warnoptions: ['ignore', 'world', 'error::SyntaxWarning']\n\n" + \
-        b"warnings.filters:\n" + \
-        b"- error::SyntaxWarning::*\n" + \
-        b"- ignore::Warning::*\n"), _
+    assert re.match(
+        br"sys\.warnoptions: \['ignore', 'world', 'error::SyntaxWarning'\]\n\n"
+        br"warnings\.filters:\n"
+        br"(- [^\n]+\n)*" # Additional filters added by automatically imported modules
+        br"- error::SyntaxWarning::\*\n"
+        br"- ignore::Warning::\*\n"
+        br"(- [^\n]+\n)*", # Remaining filters
+        _,
+    )
 
 
 def test_pymain_print_function_future():
@@ -318,6 +319,60 @@ def test_pymain_opt():
         check(["-O", "-O"])
         check(["-O", "-O", "-O"])
 
+# verify that pymain handles -E in exactly the same way as underlying python does.
+@gpython_only
+def test_pymain_E():
+    envadj = {'PYTHONOPTIMIZE': '1'}
+    def sys_flags_optimize(level):
+        return 'sys.flags.optimize:   %s' % level
+
+    # without -E $PYTHONOPTIMIZE should be taken into account
+    def _(gpyoutv, stdpyoutv):
+        assert sys_flags_optimize(0) not in stdpyoutv
+        assert sys_flags_optimize(0) not in gpyoutv
+        assert sys_flags_optimize(1)     in stdpyoutv
+        assert sys_flags_optimize(1)     in gpyoutv
+    check_gpy_vs_py(['testprog/print_opt.py'], _, envadj=envadj, cwd=here)
+
+    # with -E not
+    def _(gpyoutv, stdpyoutv):
+        assert sys_flags_optimize(0)     in stdpyoutv
+        assert sys_flags_optimize(0)     in gpyoutv
+        assert sys_flags_optimize(1) not in stdpyoutv
+        assert sys_flags_optimize(1) not in gpyoutv
+    check_gpy_vs_py(['-E', 'testprog/print_opt.py'], _, envadj=envadj, cwd=here)
+
+
+# verify that pymain handles -X non-gpython-option in exactly the same way as underlying python does.
+@pytest.mark.skipif(PY2, reason="-X does not work at all on plain cpython2")
+@gpython_only
+def test_pymain_X():
+    check_gpy_vs_py(['testprog/print_faulthandler.py'], cwd=here)
+    check_gpy_vs_py(['-X', 'faulthandler', 'testprog/print_faulthandler.py'], cwd=here)
+
+
+# pymain -v
+@gpython_only
+def test_pymain_v():
+    def nimport(argv, **kw):
+        argv = argv + ['testdata/hello.py']
+        kw.setdefault('cwd', here)
+        ret, out, err = _pyrun(argv, stdout=PIPE, stderr=PIPE, **kw)
+        assert ret == 0,    (out, err)
+        n = 0
+        for _ in u(err).splitlines():
+            if _.startswith("import "):
+                n += 1
+        return n
+
+    # without -v there must be no "import ..." messages
+    assert nimport([])                                              == 0
+    assert nimport([], pyexe=sys._gpy_underlying_executable)        == 0
+
+    # with    -v there must be many "import ..." messages
+    assert nimport(['-v'])                                          >  10
+    assert nimport(['-v'], pyexe=sys._gpy_underlying_executable)    >  10
+
 
 # pymain -V/--version
 # gpython_only because output differs from !gpython.
@@ -359,20 +414,31 @@ def test_pymain_run_via_relpath():
     out2 = pyout(['./__init__.py'] + argv, pyexe=sys._gpy_underlying_executable, cwd=here)
     assert out1 == out2
 
+
 # verify -X gpython.runtime=...
 @gpython_only
 def test_Xruntime(runtime):
+    _xopt_assert_in_subprocess('gpython.runtime', runtime,
+                                assert_gevent_activated  if runtime != 'threads'  else \
+                                assert_gevent_not_activated)
+
+# _xopt_assert_in_subprocess runs tfunc in subprocess interpreter spawned with
+# `-X xopt=xval` and checks that there is no error.
+#
+# It is also verified that tfunc runs ok in sub-subprocess interpreter spawned
+# _without_ `-X ...`, i.e. once given -X setting is inherited by spawned interpreters.
+def _xopt_assert_in_subprocess(xopt, xval, tfunc):
+    XOPT = xopt.upper().replace('.','_')    # gpython.runtime -> GPYTHON_RUNTIME
     env = os.environ.copy()
-    env.pop('GPYTHON_RUNTIME', None) # del
+    env.pop(XOPT, None) # del
 
     argv = []
-    if runtime != '':
-        argv += ['-X', 'gpython.runtime='+runtime]
-    prog = 'from gpython import gpython_test as t; '
-    if runtime != 'threads':
-        prog += 't.assert_gevent_activated(); '
-    else:
-        prog += 't.assert_gevent_not_activated(); '
+    if xval != '':
+        argv += ['-X', xopt+'='+xval]
+    prog = import_t = 'from gpython import gpython_test as t; '
+    prog += 't.%s(); ' % tfunc.__name__
+    prog += import_t  # + same in subprocess
+    prog += "t.pyrun(['-c', '%s t.%s(); ']); " % (import_t, tfunc.__name__)
     prog += 'print("ok")'
     argv += ['-c', prog]
 
@@ -382,20 +448,6 @@ def test_Xruntime(runtime):
 
 # ---- misc ----
 
-# grepv filters out lines matching pattern from text.
-def grepv(pattern, text): # -> text
-    if isinstance(text, bytes):
-        t = b''
-    else:
-        t = ''
-    p = re.compile(pattern)
-    v = []
-    for l in text.splitlines(True):
-        m = p.search(l)
-        if not m:
-            v.append(l)
-    return t.join(v)
-
 # check_gpy_vs_py verifies that gpython output matches underlying python output.
 def check_gpy_vs_py(argv, postprocessf=None, **kw):
         gpyout   = u(pyout(argv, **kw))

gpython/testprog/print_faulthandler.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2024  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""Program print_faulthandler prints information about faulthandler settings."""
+
+from __future__ import print_function, absolute_import
+
+import sys
+
+def main():
+    if 'faulthandler' not in sys.modules:
+        print('faulthandler is not imported')
+        return
+
+    fh = sys.modules['faulthandler']
+    print('faulthandler imported')
+    print('faulthandler %s' % ('enabled' if fh.is_enabled() else 'disabled'))
+
+
+if __name__ == '__main__':
+    main()

gpython/testprog/print_opt.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2020  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+# Copyright (C) 2020-2023  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -61,9 +61,10 @@ def modpy_imports_from():
     else:
         raise AssertionError("module 'mod' is already there")
 
-    tmpd = tempfile.mkdtemp('', 'modpy_imports_from')
+    tmpd  = tempfile.mkdtemp('', 'modpy_imports_from')
+    tmpd_ = tmpd + os.path.sep
     try:
-        pymod = "%s/mod.py" % tmpd
+        pymod = tmpd_ + "mod.py"
         with open(pymod, "w") as f:
             f.write("# hello up there\n")
 
@@ -73,9 +74,9 @@ def modpy_imports_from():
         files = set()
         for dirpath, dirnames, filenames in os.walk(tmpd):
             for _ in filenames:
-                f = '%s/%s' % (dirpath, _)
-                if f.startswith(tmpd+'/'):
-                    f = f[len(tmpd+'/'):]
+                f = os.path.join(dirpath, _)
+                if f.startswith(tmpd_):
+                    f = f[len(tmpd_):]
                 files.add(f)
 
 

pyproject.toml

 [build-system]
-requires = ["setuptools", "wheel", "setuptools_dso >= 1.7", "cython", "gevent"]
+requires = ["setuptools", "wheel", "setuptools_dso >= 2.8", "cython < 3", "gevent"]

setup.py

+# -*- coding: utf-8 -*-
 # pygolang | pythonic package setup
-# Copyright (C) 2018-2022  Nexedi SA and Contributors.
+# Copyright (C) 2018-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -19,28 +20,27 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 from setuptools import find_packages
-# setuptools has Library but this days it is not well supported and test for it
-# has been killed https://github.com/pypa/setuptools/commit/654c26f78a30
-# -> use setuptools_dso instead.
-from setuptools_dso import DSO
 from setuptools.command.install_scripts import install_scripts as _install_scripts
 from setuptools.command.develop import develop as _develop
 from distutils import sysconfig
 from os.path import dirname, join
-import sys, re
+import sys, os, re
 
 # read file content
-def readfile(path):
-    with open(path, 'r') as f:
-        return f.read()
-
-# reuse golang.pyx.build to build pygolang extensions.
+def readfile(path): # -> str
+    with open(path, 'rb') as f:
+        data = f.read()
+        if not isinstance(data, str):   # py3
+            data = data.decode('utf-8')
+        return data
+
+# reuse golang.pyx.build to build pygolang dso and extensions.
 # we have to be careful and inject synthetic golang package in order to be
 # able to import golang.pyx.build without built/working golang.
 trun = {}
 exec(readfile('trun'), trun)
 trun['ximport_empty_golangmod']()
-from golang.pyx.build import setup, Extension as Ext
+from golang.pyx.build import setup, DSO, Extension as Ext
 
 
 # grep searches text for pattern.
@@ -155,8 +155,8 @@ class develop(XInstallGPython, _develop):
 
 # requirements of packages under "golang." namespace
 R = {
-    'cmd.pybench':      {'pytest'},
-    'pyx.build':        {'setuptools', 'wheel', 'cython', 'setuptools_dso >= 1.7'},
+    'cmd.pybench':      {'pytest', 'py ; python_version >= "3"'},
+    'pyx.build':        {'setuptools', 'wheel', 'cython < 3', 'setuptools_dso >= 2.8'},
     'x.perf.benchlib':  {'numpy'},
 }
 # TODO generate `a.b -> a`, e.g. x.perf = join(x.perf.*); x = join(x.*)
@@ -174,6 +174,14 @@ for k in sorted(R.keys()):
     extras_require[k] = list(sorted(R[k]))
 
 
+# get_python_libdir() returns path where libpython is located
+def get_python_libdir():
+    # mimic what distutils.command.build_ext does
+    if os.name == 'nt':
+        return join(sysconfig.get_config_var('installed_platbase'), 'libs')
+    else:
+        return sysconfig.get_config_var('LIBDIR')
+
 setup(
     name        = 'pygolang',
     version     = version,
@@ -222,20 +230,21 @@ setup(
                             'golang/os/signal.h',
                             'golang/strings.h',
                             'golang/sync.h',
-                            'golang/time.h'],
-                        include_dirs    = ['.', '3rdparty/include'],
+                            'golang/time.h',
+                            '3rdparty/ratas/src/timer-wheel.h'],
+                        include_dirs    = [
+                            '3rdparty/include',
+                            '3rdparty/ratas/src'],
                         define_macros   = [('BUILDING_LIBGOLANG', None)],
-                        extra_compile_args = ['-std=gnu++11'], # not c++11 as linux/list.h uses typeof
                         soversion       = '0.1'),
 
                     DSO('golang.runtime.libpyxruntime',
                         ['golang/runtime/libpyxruntime.cpp'],
                         depends = ['golang/pyx/runtime.h'],
-                        include_dirs    = ['.', sysconfig.get_python_inc()],
+                        include_dirs    = [sysconfig.get_python_inc()],
+                        library_dirs    = [get_python_libdir()],
                         define_macros   = [('BUILDING_LIBPYXRUNTIME', None)],
-                        extra_compile_args = ['-std=c++11'],
-                        soversion       = '0.1',
-                        dsos = ['golang.runtime.libgolang'])],
+                        soversion       = '0.1')],
 
     ext_modules = [
                     Ext('golang._golang',
@@ -311,11 +320,15 @@ setup(
     include_package_data = True,
 
     install_requires = ['gevent', 'six', 'decorator', 'Importing;python_version<="2.7"',
+                        # only runtime part: for dylink_prepare_dso
+                        # also need to pin setuptools ≥ 60.2 because else wheel configures logging
+                        # to go to stdout and so dylink_prepare_dso garbles program output
+                        'setuptools_dso >= 2.8',
+                        'setuptools >= 60.2 ; python_version>="3"',
                         # pyx.build -> setuptools_dso uses multiprocessing
-                        # FIXME geventmp fails on python2, but setuptools_dso
-                        # uses multiprocessing only on Python3, so for now we
-                        # are ok. https://github.com/karellen/geventmp/pull/2
-                        'geventmp;python_version>="3"',
+                        # setuptools_dso uses multiprocessing only on Python3, and only on systems where
+                        # mp.get_start_method()!='fork', while geventmp does not work on windows.
+                        'geventmp ; python_version>="3" and platform_system != "Windows" ',
                        ],
     extras_require   = extras_require,
 
@@ -340,13 +353,18 @@ setup(
         Programming Language :: Python :: 2
         Programming Language :: Python :: 2.7
         Programming Language :: Python :: 3
-        Programming Language :: Python :: 3.5
-        Programming Language :: Python :: 3.6
-        Programming Language :: Python :: 3.7
         Programming Language :: Python :: 3.8
         Programming Language :: Python :: 3.9
+        Programming Language :: Python :: 3.10
+        Programming Language :: Python :: 3.11
+        Programming Language :: Python :: 3.12
         Programming Language :: Python :: Implementation :: CPython
         Programming Language :: Python :: Implementation :: PyPy
+        Operating System :: POSIX
+        Operating System :: POSIX :: Linux
+        Operating System :: Unix
+        Operating System :: MacOS
+        Operating System :: Microsoft :: Windows
         Topic :: Software Development :: Interpreters
         Topic :: Software Development :: Libraries :: Python Modules\
     """.splitlines()]

tox.ini

 [tox]
 envlist =
-    {py27d,py27,py37,py38d,py38,py39d,py39,pypy,pypy3}-{thread,gevent}
+    {py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread,gevent}
 
 
 # ThreadSanitizer
@@ -10,44 +10,48 @@ envlist =
 # (*) PyPy locks its GIL (see RPyGilAcquire) by manually doing atomic cmpxchg
 # and other games, which TSAN cannot see if PyPy itself was not compiled with
 # -fsanitize=thread.
-    {py27d,py27,py37,py38d,py38,py39d,py39           }-{thread       }-tsan
+    {py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312           }-{thread       }-tsan
 # XXX py*-gevent-tsan would be nice to have, but at present TSAN is not
 # effective with gevent, because it does not understand greenlet "thread"
 # switching and so perceives the program as having only one thread where races
 # are impossible. Disabled to save time.
-#   {py27d,py27,py37,py38d,py38,py39d,py39           }-{       gevent}-tsan
+#   {py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312           }-{       gevent}-tsan
 
 
 # AddressSanitizer
 
 # XXX asan does not work with gevent: https://github.com/python-greenlet/greenlet/issues/113
-    {py27d,py27,py37,py38d,py38,py39d,py39,pypy,pypy3}-{thread       }-asan
+    {py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread       }-asan
 
 [testenv]
 basepython =
     py27d:  python2.7-dbg
     py27:   python2.7
-    py37:   python3.7
-    py38d:  python3.8-dbg
     py38:   python3.8
     py39d:  python3.9-dbg
     py39:   python3.9
+    py310d: python3.10-dbg
+    py310:  python3.10
+    py311d: python3.11-dbg
+    py311:  python3.11
+    py312:  python3.12
+    py312d: python3.12-dbg
     pypy:   pypy
     pypy3:  pypy3
 
 setenv =
 # distutils take CFLAGS for both C and C++.
 # distutils use  CFLAGS also at link stage -> we don't need to set LDFLAGS separately.
-    tsan: CFLAGS=-g -fsanitize=thread
-    asan: CFLAGS=-g -fsanitize=address
+    tsan: CFLAGS=-g -fsanitize=thread   -fno-omit-frame-pointer
+    asan: CFLAGS=-g -fsanitize=address  -fno-omit-frame-pointer
 # XXX however distutils' try_link, which is used by numpy.distutils use only CC
 # as linker without CFLAGS and _without_ LDFLAGS, which fails if *.o were
 # compiled with -fsanitize=X and linked without that option. Work it around
 # with also adjusting CC.
 # XXX better arrange to pass CFLAGS to pygolang only, e.g. by adding --race or
 # --sanitize=thread to `setup.py build_ext`.
-    tsan: CC=cc -fsanitize=thread
-    asan: CC=cc -fsanitize=address
+    tsan: CC=cc -fsanitize=thread   -fno-omit-frame-pointer
+    asan: CC=cc -fsanitize=address  -fno-omit-frame-pointer
 
 # always compile pygolang from source and don't reuse binary pygolang wheels as
 # we compile each case with different CFLAGS.
@@ -69,5 +73,7 @@ commands=
 # asan/tsan: tell pytest not to capture output - else it is not possible to see
 # reports from sanitizers because they crash tested process on error.
 # likewise for python debug builds.
-        asan,tsan,py{27,37}d: -s    \
+        asan,tsan,py{27,39,310,311,312}d: -s    \
         gpython/ golang/
+
+allowlist_externals={toxinidir}/trun

trun

 #!/usr/bin/env python
-# Copyright (C) 2019-2020  Nexedi SA and Contributors.
+# -*- coding: utf-8 -*-
+# Copyright (C) 2019-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -34,12 +35,13 @@ trun cares to run python with LD_PRELOAD set appropriately to /path/to/libtsan.s
 
 from __future__ import print_function, absolute_import
 
-import os, sys, re, subprocess, pkgutil
-import warnings
-with warnings.catch_warnings():
-    warnings.simplefilter('ignore', DeprecationWarning)
-    import imp
+import os, os.path, sys, re, subprocess, platform, types
 PY3 = (bytes is not str)
+if PY3:
+    from importlib import machinery as imp_machinery
+else:
+    import imp, pkgutil
+
 
 # env_prepend prepends value to ${name} environment variable.
 #
@@ -64,12 +66,15 @@ def grep1(pattern, text): # -> re.Match|None
 # to import e.g. golang.pyx.build, or locate golang._golang, without built/working golang.
 def ximport_empty_golangmod():
     assert 'golang' not in sys.modules
-    golang = imp.new_module('golang')
+    golang = types.ModuleType('golang')
     golang.__package__ = 'golang'
     golang.__path__    = ['golang']
     golang.__file__    = 'golang/__init__.py'
-    golang.__loader__  = pkgutil.ImpLoader('golang', None, 'golang/__init__.py',
-                                           [None, None, imp.PY_SOURCE])
+    if PY3:
+        golang.__loader__ = imp_machinery.SourceFileLoader('golang', 'golang/__init__.py')
+    else:
+        golang.__loader__ = pkgutil.ImpLoader('golang', None, 'golang/__init__.py',
+                                              [None, None, imp.PY_SOURCE])
     sys.modules['golang'] = golang
 
 
@@ -83,6 +88,7 @@ def main():
 
     # determine if _golang.so is linked to a sanitizer, and if yes, to which
     # particular sanitizer DSO. Set LD_PRELOAD appropriately.
+    libxsan    = None
     ld_preload = None
     if 'linux' in sys.platform:
         p = subprocess.Popen(["ldd", _golang_so.path], stdout=subprocess.PIPE)
@@ -123,7 +129,8 @@ def main():
 
             _ = grep1("DYLD_INSERT_LIBRARIES=(.*)$", err)
             if _ is not None:
-                ld_preload = ("DYLD_INSERT_LIBRARIES", _.group(1))
+                libxsan = _.group(1)
+                ld_preload = ("DYLD_INSERT_LIBRARIES", libxsan)
             else:
                 print("trun %r: `import golang` failed with unexpected error:" % sys.argv[1:], file=sys.stderr)
                 print(err, file=sys.stderr)
@@ -140,7 +147,7 @@ def main():
     env_prepend("TSAN_OPTIONS", "halt_on_error=1")
     env_prepend("ASAN_OPTIONS", "halt_on_error=1")
 
-    # tweak TSAN/ASAN defaults:
+    # tweak TSAN/ASAN/LSAN defaults:
 
     # enable TSAN deadlock detector
     # (unfortunately it caughts only few _potential_ deadlocks and actually
@@ -148,15 +155,49 @@ def main():
     env_prepend("TSAN_OPTIONS", "detect_deadlocks=1")
     env_prepend("TSAN_OPTIONS", "second_deadlock_stack=1")
 
-    # many python allocations, whose lifetime coincides with python interpreter
-    # lifetime and which are not explicitly freed on python shutdown, are
-    # reported as leaks. Disable leak reporting to avoid huge non-pygolang
-    # related printouts.
-    env_prepend("ASAN_OPTIONS", "detect_leaks=0")
-
     # tune ASAN to check more aggressively by default
     env_prepend("ASAN_OPTIONS", "detect_stack_use_after_return=1")
 
+    # enable ASAN/LSAN leak detector.
+    #
+    # Do it only on CPython ≥ 3.11 because on py2 and on earlier py3 versions
+    # there are many many python allocations, whose lifetime coincide with
+    # python interpreter lifetime, and which are not explicitly freed on python
+    # shutdown. For py3 they significantly improved this step by step and
+    # starting from 3.11 it becomes practical to silence some still-leaks with
+    # suppressions, while for earlier py3 versions and especially for py2 it
+    # is, unfortunately, not manageable. Do not spend engineering time with
+    # activating LSAN on PyPy as that is tier 2 platform and bug tail history
+    # of memory leaks is very long even only on cpython.
+    if sys.version_info < (3,11):
+        env_prepend("ASAN_OPTIONS", "detect_leaks=0")
+        if libxsan is not None:
+            if 'asan' in libxsan.lower():
+                print("W: trun %r: asan: leak detection deactivated on %s %s" % (
+                    sys.argv[1:], platform.python_implementation(), platform.python_version()),
+                    file=sys.stderr)
+    else:
+        env_prepend("ASAN_OPTIONS", "detect_leaks=1")
+        env_prepend("LSAN_OPTIONS", "suppressions=%s" % os.path.abspath(os.path.join(
+                                        os.path.dirname(__file__), ".lsan-ignore.txt")))
+        # do not print statistics for suppressed leaks - else it breaks tests that verify program output
+        env_prepend("LSAN_OPTIONS", "print_suppressions=0")
+
+        # enable DWARF-based unwinding.
+        # else, if python is not compiled with -fno-omit-frame-pointer, it can show
+        # the whole traceback as e.g. just
+        #   Direct leak of 32 byte(s) in 1 object(s) allocated from:
+        #     #0 0x7f88522f3bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
+        #     #1 0x55f910a3d9a4 in PyThread_allocate_lock Python/thread_pthread.h:385
+        # and our leak suppressions won't work.
+        # this is slower compared to default frame-pointer based unwinding, but
+        # still works reasonably timely when run with just tests.
+        env_prepend("ASAN_OPTIONS", "fast_unwind_on_malloc=0")
+        # leak suppression also needs full tracebacks to work correctly, since with
+        # python there are many levels of call nesting at C level, and to filter-out e.g.
+        # top-level PyImport_Import we need to go really deep.
+        env_prepend("ASAN_OPTIONS", "malloc_context_size=255")
+
     # exec `...`
     os.execvp(sys.argv[1], sys.argv[1:])