Skip to content

pygolang
pygolang
Commit fdd73156 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

Sync with master

parents 300d7dfa 91a434d5
Hide whitespace changes
Inline Side-by-side
Showing with 1931 additions and 540 deletions
.gitignore
View file @ fdd73156
......@@ -10,6 +10,8 @@ build/
*.so.*
*.dylib
*.dll
*.lib
*.exp
*.pyd
*_dsoinfo.py
......
.gitmodules 0 → 100644
View file @ fdd73156
[submodule "3rdparty/ratas"]
path = 3rdparty/ratas
url = https://github.com/jsnell/ratas.git
.lsan-ignore.txt 0 → 100644
View file @ fdd73156
# .lsan-ignore.txt lists memory leak events that LeakSanitizer should not
# report when running pygolang tests.
#
# Many python allocations, whose lifetime coincides with python interpreter
# lifetime, and which are not explicitly freed on python shutdown, are
# reported as leaks by default. Disable leak reporting for those to avoid
# non-pygolang related printouts.
# >>> Everything created when initializing python, e.g. sys.stderr
# #0 0x7f21e74f3bd7 in malloc .../asan_malloc_linux.cpp:69
# #1 0x555f361ff9a4 in PyThread_allocate_lock Python/thread_pthread.h:385
# #2 0x555f3623f72a in _buffered_init Modules/_io/bufferedio.c:725
# #3 0x555f3623ff7e in _io_BufferedWriter___init___impl Modules/_io/bufferedio.c:1803
# #4 0x555f3623ff7e in _io_BufferedWriter___init__ Modules/_io/clinic/bufferedio.c.h:489
# #5 0x555f3610c086 in type_call Objects/typeobject.c:1103
# #6 0x555f3609cdcc in _PyObject_MakeTpCall Objects/call.c:214
# #7 0x555f3609d6a8 in _PyObject_VectorcallTstate Include/internal/pycore_call.h:90
# #8 0x555f3609d6a8 in _PyObject_VectorcallTstate Include/internal/pycore_call.h:77
# #9 0x555f3609d6a8 in _PyObject_CallFunctionVa Objects/call.c:536
# #10 0x555f3609e89c in _PyObject_CallFunction_SizeT Objects/call.c:590
# #11 0x555f3623a0df in _io_open_impl Modules/_io/_iomodule.c:407
# #12 0x555f3623a0df in _io_open Modules/_io/clinic/_iomodule.c.h:264
# #13 0x555f360f17da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #14 0x555f3609d54c in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
# #15 0x555f3609d54c in _PyObject_CallFunctionVa Objects/call.c:536
# #16 0x555f3609ec34 in callmethod Objects/call.c:608
# #17 0x555f3609ec34 in _PyObject_CallMethod Objects/call.c:677
# #18 0x555f361e60cf in create_stdio Python/pylifecycle.c:2244
# #19 0x555f361e6523 in init_sys_streams Python/pylifecycle.c:2431
# #20 0x555f361e6523 in init_interp_main Python/pylifecycle.c:1154
# #21 0x555f361e7204 in pyinit_main Python/pylifecycle.c:1230
# #22 0x555f361e85ba in Py_InitializeFromConfig Python/pylifecycle.c:1261
# #23 0x555f3621010a in pymain_init Modules/main.c:67
# #24 0x555f362113de in pymain_main Modules/main.c:701
# #25 0x555f362113de in Py_BytesMain Modules/main.c:734
leak:^pymain_init$
# >>> Everything created when importing py modules, e.g.
# #0 0x7f18c86f3bd7 in malloc .../asan_malloc_linux.cpp:69
# #1 0x55b971430acf in PyMem_RawMalloc Objects/obmalloc.c:586
# #2 0x55b971430acf in _PyObject_Malloc Objects/obmalloc.c:2003
# #3 0x55b971430acf in _PyObject_Malloc Objects/obmalloc.c:1996
# #4 0x55b971415696 in new_keys_object Objects/dictobject.c:632
# #5 0x55b971415716 in dictresize Objects/dictobject.c:1429
# #6 0x55b97141961a in insertion_resize Objects/dictobject.c:1183
# #7 0x55b97141961a in insertdict Objects/dictobject.c:1248
# #8 0x55b97143eb7b in add_subclass Objects/typeobject.c:6547
# #9 0x55b97144ca52 in type_ready_add_subclasses Objects/typeobject.c:6345
# #10 0x55b97144ca52 in type_ready Objects/typeobject.c:6476
# #11 0x55b971451a1f in PyType_Ready Objects/typeobject.c:6508
# #12 0x55b971451a1f in type_new_impl Objects/typeobject.c:3189
# #13 0x55b971451a1f in type_new Objects/typeobject.c:3323
# #14 0x55b971443014 in type_call Objects/typeobject.c:1091
# #15 0x55b9713d3dcc in _PyObject_MakeTpCall Objects/call.c:214
# #16 0x55b9713d47bd in _PyObject_FastCallDictTstate Objects/call.c:141
# #17 0x55b9713d47bd in PyObject_VectorcallDict Objects/call.c:165
# #18 0x55b9714d14c2 in builtin___build_class__ Python/bltinmodule.c:209
# #19 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #20 0x55b9713d4a7b in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
# #21 0x55b9713d4a7b in PyObject_Vectorcall Objects/call.c:299
# #22 0x55b97137666e in _PyEval_EvalFrameDefault Python/ceval.c:4769
# #23 0x55b9714d7e6b in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
# #24 0x55b9714d7e6b in _PyEval_Vector Python/ceval.c:6434
# #25 0x55b9714d7e6b in PyEval_EvalCode Python/ceval.c:1148
# #26 0x55b9714d2e1f in builtin_exec_impl Python/bltinmodule.c:1077
# #27 0x55b9714d2e1f in builtin_exec Python/clinic/bltinmodule.c.h:465
# #28 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #29 0x55b971376dcb in do_call_core Python/ceval.c:7349
# #30 0x55b971376dcb in _PyEval_EvalFrameDefault Python/ceval.c:5376
# #31 0x55b9714d7faf in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
# #32 0x55b9714d7faf in _PyEval_Vector Python/ceval.c:6434
# #33 0x55b9713d436e in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
# #34 0x55b9713d436e in object_vacall Objects/call.c:819
# #35 0x55b9713d63cf in PyObject_CallMethodObjArgs Objects/call.c:879
# #36 0x55b9715080e1 in import_find_and_load Python/import.c:1748
# #37 0x55b9715080e1 in PyImport_ImportModuleLevelObject Python/import.c:1847
# #38 0x55b97137de9c in import_name Python/ceval.c:7422
# #39 0x55b97137de9c in _PyEval_EvalFrameDefault Python/ceval.c:3946
# #40 0x55b9714d7e6b in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
# #41 0x55b9714d7e6b in _PyEval_Vector Python/ceval.c:6434
# #42 0x55b9714d7e6b in PyEval_EvalCode Python/ceval.c:1148
# #43 0x55b9714d2e1f in builtin_exec_impl Python/bltinmodule.c:1077
# #44 0x55b9714d2e1f in builtin_exec Python/clinic/bltinmodule.c.h:465
# #45 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #46 0x55b971376dcb in do_call_core Python/ceval.c:7349
# #47 0x55b971376dcb in _PyEval_EvalFrameDefault Python/ceval.c:5376
leak:^PyImport_Import
# importlib.import_module leads to
# #0 0x7f1951ef3bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
# #1 0x55f399e8cacf in PyMem_RawMalloc Objects/obmalloc.c:586
# #2 0x55f399e8cacf in _PyObject_Malloc Objects/obmalloc.c:2003
# #3 0x55f399e8cacf in _PyObject_Malloc Objects/obmalloc.c:1996
# #4 0x55f399e86344 in PyModule_ExecDef Objects/moduleobject.c:400
# #5 0x55f399f6178a in exec_builtin_or_dynamic Python/import.c:2345
# #6 0x55f399f6178a in _imp_exec_dynamic_impl Python/import.c:2419
# #7 0x55f399f6178a in _imp_exec_dynamic Python/clinic/import.c.h:474
# #8 0x55f399e8438a in cfunction_vectorcall_O Objects/methodobject.c:514
leak:^_imp_exec_dynamic
# >>> Everything allocated at DSO initialization, e.g.
# #0 0x7f35d2af46c8 in operator new(unsigned long) .../asan_new_delete.cpp:95
# #1 0x7f35ce897e9f in __static_initialization_and_destruction_0 golang/context.cpp:61
# #2 0x7f35ce8982ef in _GLOBAL__sub_I_context.cpp golang/context.cpp:380
# #3 0x7f35d32838bd in call_init elf/dl-init.c:90
# #4 0x7f35d32838bd in call_init elf/dl-init.c:27
# #5 0x7f35d32839a3 in _dl_init elf/dl-init.c:137
# #6 0x7f35d256e023 in __GI__dl_catch_exception elf/dl-error-skeleton.c:182
# #7 0x7f35d328a09d in dl_open_worker elf/dl-open.c:808
# #8 0x7f35d256dfc9 in __GI__dl_catch_exception elf/dl-error-skeleton.c:208
# #9 0x7f35d328a437 in _dl_open elf/dl-open.c:884
# #10 0x7f35d24a4437 in dlopen_doit dlfcn/dlopen.c:56
# #11 0x7f35d256dfc9 in __GI__dl_catch_exception elf/dl-error-skeleton.c:208
# #12 0x7f35d256e07e in __GI__dl_catch_error elf/dl-error-skeleton.c:227
# #13 0x7f35d24a3f26 in _dlerror_run dlfcn/dlerror.c:138
# #14 0x7f35d24a44e8 in dlopen_implementation dlfcn/dlopen.c:71
# #15 0x7f35d24a44e8 in ___dlopen dlfcn/dlopen.c:81
# #16 0x7f35d2a77ff9 in dlopen .../sanitizer_common_interceptors.inc:6341
leak:^_GLOBAL_
# global<> does not deallocate its reference on purpose
leak:^_test_global()$
ratas @ becd5fc5
Subproject commit becd5fc5c1e9ea600cd8b3b1c24d564794fedac4
MANIFEST.in
View file @ fdd73156
include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .nxdtest
include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .lsan-ignore.txt .nxdtest conftest.py
include golang/libgolang.h
include golang/runtime/libgolang.cpp
include golang/runtime/libpyxruntime.cpp
include golang/pyx/runtime.h
include golang/pyx/runtime.cpp
include golang/pyx/testprog/golang_dso_user/dsouser/dso.h
include golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
include golang/runtime/internal.h
include golang/runtime/internal/atomic.h
......@@ -34,6 +34,9 @@ include golang/sync_test.cpp
include golang/time.h
include golang/time.cpp
include golang/_testing.h
include golang/_compat/windows/strings.h
include golang/_compat/windows/unistd.h
recursive-include golang *.py *.pxd *.pyx *.toml *.txt*
recursive-include gpython *.py
recursive-include 3rdparty *.h
recursive-exclude golang *_dsoinfo.py
conftest.py 0 → 100644
View file @ fdd73156
# pygolang | pytest config
# Copyright (C) 2021-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from __future__ import print_function, absolute_import
import gc
# Do full GC before pytest exits, to avoid false positives in the leak detector.
def pytest_unconfigure():
gc.collect()
golang/__init__.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -42,6 +42,9 @@ from golang._gopath import gimport # make gimport available from golang
import inspect, sys
import decorator, six
import setuptools_dso
setuptools_dso.dylink_prepare_dso('golang.runtime.libgolang')
from golang._golang import _pysys_exc_clear as _sys_exc_clear
# @func is a necessary decorator for functions for selected golang features to work.
......@@ -70,10 +73,25 @@ def _meth(cls, fcall):
# wrap f with @_func, so that e.g. defer works automatically.
f = _func(f)
if isinstance(f, (staticmethod, classmethod)):
func_name = f.__func__.__name__
# property is special - it has up to 3 functions inside
# but the only practical case is when it starts with
#
# @func(Class)
# @property
# def ...
#
# which means that .fget should be set and so we use get name as the
# name of the method.
f_ = f
if isinstance(f, property):
f_ = f.fget
if f_ is None:
raise ValueError("func(cls) used on property without getter")
if isinstance(f_, (staticmethod, classmethod)):
func_name = f_.__func__.__name__
else:
func_name = f.__name__
func_name = f_.__name__
setattr(cls, func_name, f)
# if `@func(cls) def name` caller already has `name` set, don't override it
......@@ -82,28 +100,57 @@ def _meth(cls, fcall):
if already is not missing:
return already
# FIXME try to arrange so that python does not set anything on caller's
# namespace[func_name] (currently it sets that to implicitly returned None)
# arrange so that python eventually does not set anything on caller's
# namespace[func_name] (it unconditionally sets what decorator returns, even implicit None)
#
# _DelAttrAfterMeth.__del__ is invoked:
# * on cpython: right after namespace[func_name] = returned _meth_leftover
# * on pypy: eventually on next GC
fcall.f_locals[func_name] = _DelAttrAfterMeth(fcall.f_locals, func_name)
return _meth_leftover
return deco
# _DelAttrAfterMeth serves _meth by unsetting f_locals[meth] that python
# unconditionally sets after `@func(cls) def meth()`.
_meth_leftover = object()
class _DelAttrAfterMeth(object):
def __init__(self, f_locals, name):
self.f_locals = f_locals
self.name = name
def __del__(self):
obj = self.f_locals.get(self.name)
if obj is _meth_leftover:
del self.f_locals[self.name]
# _func serves @func.
def _func(f):
# @staticmethod & friends require special care:
# @property is special: there are 3 functions inside and we need to wrap
# them all with repacking back into property.
if isinstance(f, property):
fget = fset = fdel = None
if f.fget is not None:
fget = _func(f.fget)
if f.fset is not None:
fset = _func(f.fset)
if f.fdel is not None:
fdel = _func(f.fdel)
return type(f)(fget, fset, fdel, f.__doc__)
# @staticmethod & friends also require special care:
# unpack f first to original func and then repack back after wrapping.
fclass = None
if isinstance(f, (staticmethod, classmethod)):
fclass = type(f)
f = f.__func__
def _(f, *argv, **kw):
# run f under separate frame, where defer will register calls.
__goframe__ = _GoFrame()
with __goframe__:
return f(*argv, **kw)
# keep all f attributes, like __name__, __doc__, etc on _
_ = decorator.decorate(f, _)
# prepare function that runs f under separate frame, where defer will register calls
# keep all f attributes, like __name__, __doc__, etc on the wrapper
# if f was already wrapped with _func - no need to wrap it again
_ = f
if getattr(f, '__go_wrapper__', None) is not _goframe:
_ = decorator.decorate(f, _goframe)
_.__go_wrapper__ = _goframe
# repack _ into e.g. @staticmethod if that was used on f.
if fclass is not None:
......@@ -111,7 +158,13 @@ def _func(f):
return _
# _GoFrame serves __goframe__ that is setup by @func.
# _goframe is used by @func to run f under separate frame.
def _goframe(f, *argv, **kw):
__goframe__ = _GoFrame()
with __goframe__:
return f(*argv, **kw)
# _GoFrame serves __goframe__ that is setup by _goframe.
class _GoFrame:
def __init__(self):
self.deferv = [] # defer registers funcs here
......
golang/_compat/windows/strings.h 0 → 100644
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
#define _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
//
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
#include <string.h>
static inline void bzero(void *p, size_t n) {
memset(p, '\0', n);
}
#endif // _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
golang/_compat/windows/unistd.h 0 → 100644
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
#define _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
//
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// stub unistd.h to be used on Windows where it is absent.
// we need this because e.g. `cimport posix.stat` forces inclusion of unistd.h
// even if we use part of posix.stat that is available everywhere.
#include <io.h>
#define O_CLOEXEC _O_NOINHERIT
#endif // _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
golang/_golang.pyx
View file @ fdd73156
......@@ -5,7 +5,7 @@
# distutils: language = c++
# distutils: depends = libgolang.h os/signal.h _golang_str.pyx
#
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -173,11 +173,18 @@ cdef void __goviac(void *arg) nogil:
# ---- channels ----
# _frompyx indicates that a constructor is called from pyx code
cdef object _frompyx = object()
@final
cdef class pychan:
def __cinit__(pychan pych, size=0, dtype=object):
pych.dtype = parse_dtype(dtype)
pych._ch = _makechan_pyexc(dtypeRegistry[<int>pych.dtype].size, size)
if dtype is _frompyx:
pych.dtype = DTYPE_STRUCTZ # anything
pych._ch = NULL
else:
pych.dtype = parse_dtype(dtype)
pych._ch = _makechan_pyexc(dtypeRegistry[<int>pych.dtype].size, size)
# pychan.nil(X) creates new nil pychan with specified dtype.
# TODO try to avoid exposing .nil on pychan instances, and expose only pychan.nil
......@@ -370,7 +377,7 @@ cdef void pychan_asserttype(pychan pych, DType dtype) nogil:
panic("pychan: channel type mismatch")
cdef pychan pychan_from_raw(_chan *_ch, DType dtype):
cdef pychan pych = pychan.__new__(pychan)
cdef pychan pych = pychan.__new__(pychan, dtype=_frompyx)
pych.dtype = dtype
pych._ch = _ch; _chanxincref(_ch)
return pych
......@@ -469,7 +476,7 @@ def pyselect(*pycasev):
casev[i].user = pych.dtype
with nogil:
selected = _chanselect_pyexc(&casev[0], casev.size())
selected = _chanselect_pyexc(casev.data(), casev.size())
finally:
# decref not sent tx (see ^^^ send prepare)
......@@ -626,9 +633,7 @@ cdef object c_to_py(DType dtype, const chanElemBuf *cfrom):
# mkpynil creates pychan instance that represents nil[dtype].
cdef PyObject *mkpynil(DType dtype):
cdef pychan pynil = pychan.__new__(pychan)
pynil.dtype = dtype
pynil._ch = NULL # should be already NULL
cdef pychan pynil = pychan_from_raw(NULL, dtype)
Py_INCREF(pynil)
return <PyObject *>pynil
......@@ -818,9 +823,6 @@ from libcpp.typeinfo cimport type_info
from cython.operator cimport typeid
from libc.string cimport strcmp
# _frompyx indicates that a constructor is called from pyx code
cdef object _frompyx = object()
cdef class pyerror(Exception):
# pyerror <- error
@staticmethod
......
golang/_golang_str.pyx
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -1791,8 +1791,9 @@ cdef (int, int) _utf8_decode_rune(const uint8_t[::1] s):
if _ucs2_build and len(r) == 2:
try:
return _xuniord(r), l
# e.g. TypeError: ord() expected a character, but string of length 2 found
except TypeError:
# py: TypeError: ord() expected a character, but string of length 2 found
# cy: ValueError: only single character unicode strings can be converted to Py_UCS4, got length 2
except (TypeError, ValueError):
l -= 1
continue
......
golang/_golang_test.pyx
View file @ fdd73156
......@@ -2,7 +2,7 @@
# cython: language_level=2
# distutils: language=c++
#
# Copyright (C) 2018-2020 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -344,6 +344,25 @@ cdef nogil:
pych.chan_double().close()
# verify that pychan_from_raw is not leaking C channel.
def test_pychan_from_raw_noleak():
# pychan_from_raw used to create another channel and leak it
#
# this test _implicitly_ verifies that it is no longer the case - if it is,
# LSAN will report a memory leak after running the test.
#
# TODO consider adding explicit verification effective even under regular
# builds. Possible options:
#
# * verify malloc totals before and after tested code
# see e.g. https://stackoverflow.com/q/1761125/9456786
# * hook _makechan and verify that it is not invoked from under
# pychan_from_raw. Depends on funchook integration.
cdef chan[int] ch = makechan[int]()
cdef pychan pych = pychan.from_chan_int(ch) # uses pychan_from_raw internally
# pych and ch are freed automatically
# ---- benchmarks ----
# bench_go_nogil mirrors golang_test.py:bench_go
......
golang/_gopath.py
View file @ fdd73156
# Copyright (C) 2018-2019 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -34,11 +34,7 @@ from __future__ import print_function, absolute_import
import os, os.path
import sys
import warnings
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import imp
import six
# _gopathv returns $GOPATH vector.
def _gopathv():
......@@ -51,11 +47,25 @@ def _gopathv():
# gimport imports python module or package from fully-qualified module name under $GOPATH.
def gimport(name):
imp.acquire_lock()
_gimport_lock()
try:
return _gimport(name)
finally:
imp.release_lock()
_gimport_unlock()
# on py2 there is global import lock
# on py3 we need to organize our own gimport synchronization
if six.PY2:
import imp
_gimport_lock = imp.acquire_lock
_gimport_unlock = imp.release_lock
else:
from importlib import machinery as imp_machinery
from importlib import util as imp_util
from golang import sync
_gimport_mu = sync.Mutex()
_gimport_lock = _gimport_mu.lock
_gimport_unlock = _gimport_mu.unlock
def _gimport(name):
# we will register imported module into sys.modules with adjusted path.
......@@ -93,4 +103,16 @@ def _gimport(name):
# https://stackoverflow.com/a/67692
return imp.load_source(modname, modpath)
return _imp_load_source(modname, modpath)
def _imp_load_source(modname, modpath):
if six.PY2:
return imp.load_source(modname, modpath)
# https://docs.python.org/3/whatsnew/3.12.html#imp
loader = imp_machinery.SourceFileLoader(modname, modpath)
spec = imp_util.spec_from_file_location(modname, modpath, loader=loader)
mod = imp_util.module_from_spec(spec)
sys.modules[modname] = mod
loader.exec_module(mod)
return mod
golang/fmt.cpp
View file @ fdd73156
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -48,7 +48,7 @@ string _vsprintf(const char *format, va_list argp) {
return string(buf.get(), buf.get() + nchar); // without trailing '\0'
}
string sprintf(const string &format, ...) {
string sprintf(const string format, ...) {
va_list argp;
va_start(argp, format);
string str = fmt::_vsprintf(format.c_str(), argp);
......@@ -64,7 +64,7 @@ string sprintf(const char *format, ...) {
return str;
}
error ___errorf(const string &format, ...) {
error ___errorf(const string format, ...) {
va_list argp;
va_start(argp, format);
error err = errors::New(fmt::_vsprintf(format.c_str(), argp));
......
golang/fmt.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_FMT_H
#define _NXD_LIBGOLANG_FMT_H
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -40,7 +40,7 @@ namespace golang {
namespace fmt {
// sprintf formats text into string.
LIBGOLANG_API string sprintf(const string &format, ...);
LIBGOLANG_API string sprintf(const string format, ...);
// intseq<i1, i2, ...> and intrange<n> are used by errorf to handle %w.
......@@ -75,7 +75,7 @@ namespace {
//
// format suffix ": %w" is additionally handled as in Go with
// `errorf("... : %w", ..., err)` creating error that can be unwrapped back to err.
LIBGOLANG_API error ___errorf(const string& format, ...);
LIBGOLANG_API error ___errorf(const string format, ...);
LIBGOLANG_API error ___errorfTryWrap(const string& format, error last_err, ...);
LIBGOLANG_API string ___error_str(error err);
......@@ -111,7 +111,10 @@ inline error errorf(const string& format, Argv... argv) {
// `const char *` overloads just to catch format mistakes as
// __attribute__(format) does not work with std::string.
LIBGOLANG_API string sprintf(const char *format, ...)
__attribute__ ((format (printf, 1, 2)));
#ifndef _MSC_VER
__attribute__ ((format (printf, 1, 2)))
#endif
;
// cannot use __attribute__(format) for errorf as we add %w handling.
// still `const char *` overload is useful for performance.
......
golang/golang_test.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -23,14 +23,14 @@ from __future__ import print_function, absolute_import
from golang import go, chan, select, default, nilchan, _PanicError, func, panic, \
defer, recover, u
from golang import sync
from pytest import raises, mark, fail
from pytest import raises, mark, fail, skip
from _pytest._code import Traceback
from os.path import dirname
import os, sys, inspect, importlib, traceback, doctest
from subprocess import Popen, PIPE
import six
from six.moves import range as xrange
import gc, weakref, warnings
import gc, weakref
import re
from golang import _golang_test
......@@ -74,7 +74,8 @@ import_pyx_tests("golang._golang_test")
# leaked goroutine behaviour check: done in separate process because we need
# to test process termination exit there.
def test_go_leaked():
pyrun([dir_testprog + "/golang_test_goleaked.py"])
pyrun([dir_testprog + "/golang_test_goleaked.py"],
lsan=False) # there are on-purpose leaks in this test
# benchmark go+join a thread/coroutine.
# pyx/nogil mirror is in _golang_test.pyx
......@@ -972,7 +973,7 @@ def test_func():
# test how @func(cls) works
# this also implicitly tests just @func, since @func(cls) uses that.
class MyClass:
class MyClass(object):
def __init__(self, v):
self.v = v
......@@ -1003,12 +1004,57 @@ def test_func():
assert mcls is mcls_orig
assert mcls == 'mcls'
# FIXME undefined var after `@func(cls) def var` should be not set
# undefined var after `@func(cls) def var` should be not set
assert 'var' not in locals()
@func(MyClass)
def var(self, v):
assert v == 8
return v + 1
gc.collect() # pypy needs this to trigger _DelAttrAfterMeth GC
assert 'var' not in locals()
vproperty = mproperty_orig = 'vproperty'
@func(MyClass)
@property
def vproperty(self):
"""documentation for vproperty"""
assert isinstance(self, MyClass)
return 'v%s' % self.v
assert vproperty is mproperty_orig
assert vproperty == 'vproperty'
@func(MyClass)
@MyClass.vproperty.setter
def _(self, v):
assert isinstance(self, MyClass)
self.v = v
assert vproperty is mproperty_orig
assert vproperty == 'vproperty'
@func(MyClass)
@MyClass.vproperty.deleter
def _(self):
assert isinstance(self, MyClass)
self.v = 'deleted'
assert vproperty is mproperty_orig
assert vproperty == 'vproperty'
obj = MyClass(4)
assert obj.zzz(4) == 4 + 1
assert obj.mstatic(5) == 5 + 1
assert obj.mcls(7) == 7 + 1
assert obj.var(8) == 8 + 1
assert obj.v == 4 # set by .zzz
assert obj.vproperty == 'v4'
obj.vproperty = 5
assert obj.v == 5
assert obj.vproperty == 'v5'
del obj.vproperty
assert obj.v == 'deleted'
assert obj.vproperty == 'vdeleted'
assert MyClass.vproperty.__doc__ == "documentation for vproperty"""
# this tests that @func (used by @func(cls)) preserves decorated function signature
assert fmtargspec(MyClass.zzz) == '(self, v, x=2, **kkkkwww)'
......@@ -1022,6 +1068,23 @@ def test_func():
assert MyClass.mcls.__module__ == __name__
assert MyClass.mcls.__name__ == 'mcls'
assert MyClass.var.__module__ == __name__
assert MyClass.var.__name__ == 'var'
assert MyClass.vproperty.fget.__module__ == __name__
assert MyClass.vproperty.fset.__module__ == __name__
assert MyClass.vproperty.fdel.__module__ == __name__
assert MyClass.vproperty.fget.__name__ == 'vproperty'
assert MyClass.vproperty.fset.__name__ == '_'
assert MyClass.vproperty.fdel.__name__ == '_'
# test that func·func = func (double _func calls are done internally for
# getter when handling @func(@MyClass.vproperty.setter)
def f(): pass
g = func(f)
h = func(g)
assert h is g
# @func overhead at def time.
def bench_def(b):
......@@ -1246,6 +1309,45 @@ def test_deferrecover():
assert v == [7, 2, 1]
# defer in std @property
v = []
class MyClass(object):
@func
@property
def vproperty(self):
"""vproperty doc"""
defer(lambda: v.append(1))
defer(lambda: v.append(3))
defer(lambda: v.append(4))
return 'v'
@func
@vproperty.setter
def vproperty(self, val):
defer(lambda: v.append(1))
defer(lambda: v.append(4))
defer(lambda: v.append(val))
@func
@vproperty.deleter
def vproperty(self):
defer(lambda: v.append(1))
defer(lambda: v.append(5))
defer(lambda: v.append('del'))
obj = MyClass()
assert MyClass.vproperty.__doc__ == "vproperty doc"
assert obj.vproperty == 'v'
assert v == [4, 3, 1]
v = []
obj.vproperty = 'q'
assert v == ['q', 4, 1]
v = []
del obj.vproperty
assert v == ['del', 5, 1]
# verify that defer correctly establishes exception chain (even on py2).
def test_defer_excchain():
# just @func/raise embeds traceback and adds ø chain
......@@ -1557,8 +1659,9 @@ RuntimeError: gamma
assertDoc("""\
Traceback (most recent call last):
File "PYGOLANG/golang/__init__.py", line ..., in _
File "PYGOLANG/golang/__init__.py", line ..., in _goframe
return f(*argv, **kw)
^^^^^^^^^^^^^^ +PY311
File "PYGOLANG/golang/golang_test.py", line ..., in caller
raise RuntimeError("ccc")
RuntimeError: ccc
......@@ -1578,10 +1681,12 @@ Traceback (most recent call last):
File "PYGOLANG/golang/golang_test.py", line ..., in test_defer_excchain_traceback
caller()
...
File "PYGOLANG/golang/__init__.py", line ..., in _
return f(*argv, **kw)
File "PYGOLANG/golang/__init__.py", line ..., in _goframe
return f(*argv, **kw) -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
d()
d() -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
d()
File "PYGOLANG/golang/golang_test.py", line ..., in q1
......@@ -1595,10 +1700,12 @@ Traceback (most recent call last):
File "PYGOLANG/golang/golang_test.py", line ..., in test_defer_excchain_traceback
caller()
...
File "PYGOLANG/golang/__init__.py", line ..., in _
return f(*argv, **kw)
File "PYGOLANG/golang/__init__.py", line ..., in _goframe
return f(*argv, **kw) -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
d()
d() -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
d()
File "PYGOLANG/golang/golang_test.py", line ..., in q1
......@@ -1609,8 +1716,9 @@ RuntimeError: aaa
e.__cause__ = e.__context__
assertDoc("""\
Traceback (most recent call last):
File "PYGOLANG/golang/__init__.py", line ..., in _
File "PYGOLANG/golang/__init__.py", line ..., in _goframe
return f(*argv, **kw)
^^^^^^^^^^^^^^ +PY311
File "PYGOLANG/golang/golang_test.py", line ..., in caller
raise RuntimeError("ccc")
RuntimeError: ccc
......@@ -1630,10 +1738,12 @@ Traceback (most recent call last):
File "PYGOLANG/golang/golang_test.py", line ..., in test_defer_excchain_traceback
caller()
...
File "PYGOLANG/golang/__init__.py", line ..., in _
return f(*argv, **kw)
File "PYGOLANG/golang/__init__.py", line ..., in _goframe
return f(*argv, **kw) -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
d()
d() -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
d()
File "PYGOLANG/golang/golang_test.py", line ..., in q1
......@@ -1654,6 +1764,13 @@ def test_defer_excchain_dump():
# ----//---- (ipython)
def test_defer_excchain_dump_ipython():
# ipython 8 changed traceback output significantly
# we do not need to test it because we acticate ipython-related patch only
# on py2 for which latest ipython version is 5.
import IPython
if six.PY3 and IPython.version_info >= (8,0):
skip("ipython is patched only on py2; ipython8 changed traceback format")
tbok = readfile(dir_testprog + "/golang_test_defer_excchain.txt-ipython")
retcode, stdout, stderr = _pyrun(["-m", "IPython", "--quick", "--colors=NoColor",
"-m", "golang_test_defer_excchain"],
......@@ -1667,6 +1784,12 @@ def test_defer_excchain_dump_ipython():
# ----//---- (pytest)
def test_defer_excchain_dump_pytest():
# pytest 7.4 also changed traceback output format
# similarly to ipython we do not need to test it becase we activate
# pytest-related patch only on py2 for which latest pytest version is 4.6.11 .
import pytest
if six.PY3 and getattr(pytest, 'version_tuple', (0,)) >= (7,4):
skip("pytest is patched only on py2; pytest7.4 changed traceback format")
tbok = readfile(dir_testprog + "/golang_test_defer_excchain.txt-pytest")
retcode, stdout, stderr = _pyrun([
# don't let pytest emit internal deprecation warnings to stderr
......@@ -1724,8 +1847,33 @@ def _pyrun(argv, stdin=None, stdout=None, stderr=None, **kw): # -> retcode, st
pathv.extend(envpath.split(os.pathsep))
env['PYTHONPATH'] = os.pathsep.join(pathv)
# set $PYTHONIOENCODING to encoding of stdin/stdout/stderr
# we need to do it because on Windows `python x.py | ...` runs with stdio
# encoding set to cp125X even if just `python x.py` runs with stdio
# encoding=UTF-8.
if 'PYTHONIOENCODING' not in env:
enc = set([_.encoding for _ in (sys.stdin, sys.stdout, sys.stderr)])
if None in enc: # without -s pytest uses _pytest.capture.DontReadFromInput
enc.remove(None) # with None .encoding
assert len(enc) == 1
env['PYTHONIOENCODING'] = enc.pop()
# disable LeakSanitizer if requested, e.g. when test is known to leak something on purpose
lsan = kw.pop('lsan', True)
if not lsan:
env['ASAN_OPTIONS'] = env.get('ASAN_OPTIONS', '') + ',detect_leaks=0'
p = Popen(argv, stdin=(PIPE if stdin else None), stdout=stdout, stderr=stderr, env=env, **kw)
stdout, stderr = p.communicate(stdin)
# on windows print emits \r\n instead of just \n
# normalize that to \n in *out
if os.name == 'nt':
if stdout is not None:
stdout = stdout.replace(b'\r\n', b'\n')
if stderr is not None:
stderr = stderr.replace(b'\r\n', b'\n')
return p.returncode, stdout, stderr
# pyrun runs `sys.executable argv... <stdin`.
......@@ -1792,10 +1940,33 @@ def assertDoc(want, got):
got = got.replace(dir_pygolang, "PYGOLANG") # /home/x/.../pygolang -> PYGOLANG
got = got.replace(udir_pygolang, "PYGOLANG") # ~/.../pygolang -> PYGOLANG
# got: normalize PYGOLANG\a\b\c -> PYGOLANG/a/b/c
# a\b\c\d.py -> a/b/c/d.py
def _(m):
return m.group(0).replace(os.path.sep, '/')
got = re.sub(r"(?<=PYGOLANG)[^\s]+(?=\s)", _, got)
got = re.sub(r"([\w\\\.]+)(?=\.py)", _, got)
# want: process conditionals
# PY39(...) -> ... if py39 else ø
py39 = sys.version_info >= (3, 9)
want = re.sub(r"PY39\((.*)\)", r"\1" if py39 else "", want)
# PY39(...) -> ... if py ≥ 3.9 else ø (inline)
# `... +PY39` -> ... if py ≥ 3.9 else ø (whole line)
# `... -PY39` -> ... if py < 3.9 else ø (whole line)
have = {} # 'PYxy' -> y/n
for minor in (9,10,11):
have['PY3%d' % minor] = (sys.version_info >= (3, minor))
for x, havex in have.items():
want = re.sub(r"%s\((.*)\)" % x, r"\1" if havex else "", want)
r = re.compile(r'^(?P<main>.*?) +(?P<y>(\+|-))%s$' % x)
v = []
for l in want.splitlines():
m = r.match(l)
if m is not None:
l = m.group('main')
y = {'+':True, '-':False}[m.group('y')]
if (y and not havex) or (havex and not y):
continue
v.append(l)
want = '\n'.join(v)+'\n'
# want: ^$ -> <BLANKLINE>
while "\n\n" in want:
......@@ -1818,8 +1989,11 @@ def assertDoc(want, got):
# ...
# fmtargspec(f) -> '(x, y=3)'
def fmtargspec(f): # -> str
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
# inspect.formatargspec is deprecated since py3.5 and was removed in py3.11
# -> use inspect.signature instead.
if six.PY3:
return str(inspect.signature(f))
else:
return inspect.formatargspec(*inspect.getargspec(f))
def test_fmtargspec():
......@@ -1828,8 +2002,10 @@ def test_fmtargspec():
# readfile returns content of file @path.
def readfile(path):
with open(path, "r") as f:
def readfile(path): # -> bytes
# on windows in text mode files are opened with encoding=locale.getdefaultlocale()
# which is CP125X instead of UTF-8. -> manually decode as 'UTF-8'
with open(path, "rb") as f:
return f.read()
# abbrev_home returns path with user home prefix abbreviated with ~.
......
golang/libgolang.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_H
#define _NXD_LIBGOLANG_H
// Copyright (C) 2018-2022 Nexedi SA and Contributors.
// Copyright (C) 2018-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -176,6 +176,12 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#ifdef _MSC_VER // no mode_t on msvc
typedef int mode_t;
#endif
// DSO symbols visibility (based on https://gcc.gnu.org/wiki/Visibility)
#if defined _WIN32 || defined __CYGWIN__
#define LIBGOLANG_DSO_EXPORT __declspec(dllexport)
......@@ -340,8 +346,13 @@ typedef struct _libgolang_runtime_ops {
// previously successfully allocated via sema_alloc.
void (*sema_free) (_libgolang_sema*);
// sema_acquire/sema_release should acquire/release live semaphore allocated via sema_alloc.
void (*sema_acquire)(_libgolang_sema*);
// sema_acquire should try to acquire live semaphore allocated via sema_alloc during given time.
// it returns whether acquisition succeeded or timed out.
// the timeout is specified in nanoseconds.
// UINT64_MAX means no timeout.
bool (*sema_acquire)(_libgolang_sema*, uint64_t timeout_ns);
// sema_release should release live semaphore allocated via sema_alloc.
void (*sema_release)(_libgolang_sema*);
// nanosleep should pause current goroutine for at least dt nanoseconds.
......@@ -577,7 +588,7 @@ int select(const _selcase (&casev)[N]) {
static inline // select(vector<casev>)
int select(const std::vector<_selcase> &casev) {
return _chanselect(&casev[0], casev.size());
return _chanselect(casev.data(), casev.size());
}
// defer(f) mimics `defer f()` from golang.
......@@ -829,7 +840,7 @@ struct _interface {
protected:
// don't use destructor -> use decref
~_interface();
LIBGOLANG_API ~_interface();
};
typedef refptr<_interface> interface;
......
golang/os.cpp
View file @ fdd73156
// Copyright (C) 2019-2022 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -37,7 +37,8 @@
// GLIBC < 2.32 provides sys_siglist but not sigdescr_np in its headers
// cut this short
// (on darwing sys_siglist declaration is normally provided)
#ifndef __APPLE__
// (on windows sys_siglist is not available at all)
#if !(defined(__APPLE__) || defined(_WIN32))
extern "C" {
extern const char * const sys_siglist[];
}
......@@ -226,8 +227,8 @@ tuple<string, error> ReadFile(const string& path) {
while (1) {
int n;
tie(n, err) = f->Read(&buf[0], buf.size());
data.append(&buf[0], n);
tie(n, err) = f->Read(buf.data(), buf.size());
data.append(buf.data(), n);
if (err != nil) {
if (err == io::EOF_)
err = nil;
......@@ -248,7 +249,7 @@ tuple<string, error> ReadFile(const string& path) {
tuple<File, File, error> Pipe() {
int vfd[2], syserr;
syserr = sys::Pipe(vfd);
syserr = sys::Pipe(vfd, 0);
if (syserr != 0)
return make_tuple(nil, nil, fmt::errorf("pipe: %w", sys::NewErrno(syserr)));
......@@ -286,8 +287,20 @@ string Signal::String() const {
const Signal& sig = *this;
const char *sigstr = nil;
#ifdef _WIN32
switch (sig.signo) {
case SIGABRT: return "Aborted";
case SIGBREAK: return "Break";
case SIGFPE: return "Floating point exception";
case SIGILL: return "Illegal instruction";
case SIGINT: return "Interrupt";
case SIGSEGV: return "Segmentation fault";
case SIGTERM: return "Terminated";
}
#else
if (0 <= sig.signo && sig.signo < NSIG)
sigstr = ::sys_siglist[sig.signo]; // might be nil as well
#endif
if (sigstr != nil)
return string(sigstr);
......
golang/os.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_OS_H
#define _NXD_LIBGOLANG_OS_H
//
// Copyright (C) 2019-2022 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -61,7 +61,7 @@ private:
~_File();
friend File _newFile(_libgolang_ioh* ioh, const string& name);
public:
void decref();
LIBGOLANG_API void decref();
public:
LIBGOLANG_API string Name() const;
......@@ -95,9 +95,15 @@ private:
// Open opens file @path.
LIBGOLANG_API std::tuple<File, error> Open(const string &path, int flags = O_RDONLY,
mode_t mode = S_IRUSR | S_IWUSR | S_IXUSR |
mode_t mode =
#if !defined(_MSC_VER)
S_IRUSR | S_IWUSR | S_IXUSR |
S_IRGRP | S_IWGRP | S_IXGRP |
S_IROTH | S_IWOTH | S_IXOTH);
S_IROTH | S_IWOTH | S_IXOTH
#else
_S_IREAD | _S_IWRITE
#endif
);
// NewFile wraps OS-level file-descriptor into File.
// The ownership of sysfd is transferred to File.
......
golang/os/signal.cpp
View file @ fdd73156
// Copyright (C) 2021-2022 Nexedi SA and Contributors.
// Copyright (C) 2021-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -89,6 +89,10 @@
#include <atomic>
#include <tuple>
#if defined(_WIN32)
# include <windows.h>
#endif
#define DEBUG 0
#if DEBUG
......@@ -97,6 +101,12 @@
# define debugf(format, ...) do {} while (0)
#endif
#if defined(_MSC_VER)
# define HAVE_SIGACTION 0
#else
# define HAVE_SIGACTION 1
#endif
// golang::os::signal::
namespace golang {
namespace os {
......@@ -109,13 +119,30 @@ using std::tie;
using std::vector;
using cxx::set;
#if HAVE_SIGACTION
static void xsigemptyset(sigset_t *sa_mask);
#else
// custom `struct sigaction` emulated via signal(2)
#define SA_SIGINFO 1
typedef struct {} siginfo_t;
struct sigaction {
union {
void (*sa_handler)(int);
void (*sa_sigaction)(int, siginfo_t*, void*);
};
int sa_flags;
};
static void _os_sighandler_nosigaction(int sig);
#endif
static void _os_sighandler(int sig, siginfo_t *info, void *ucontext);
static void _notify(int signo);
static void _checksig(int signo);
static void _checkActEqual(const struct sigaction *a, const struct sigaction *b);
static void _spinwaitNextQueueCycle();
static int sys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact);
static void xsys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact);
static void xsigemptyset(sigset_t *sa_mask);
static bool _sigact_equal(const struct sigaction *a, const struct sigaction *b);
......@@ -160,27 +187,31 @@ void _init() {
// create _wakerx <-> _waketx pipe; set _waketx to nonblocking mode
int vfd[2];
if (sys::Pipe(vfd) < 0)
if (sys::Pipe(vfd, O_CLOEXEC) < 0)
panic("pipe(_wakerx, _waketx)"); // TODO +syserr
if (sys::Fcntl(vfd[0], F_SETFD, FD_CLOEXEC) < 0)
panic("fcntl(_wakerx, FD_CLOEXEC)"); // TODO +syserr
error err;
tie(_wakerx, err) = os::NewFile(vfd[0], "_wakerx");
if (err != nil)
panic("os::newFile(_wakerx");
_waketx = vfd[1];
#ifndef _WIN32
if (sys::Fcntl(_waketx, F_SETFL, O_NONBLOCK) < 0)
panic("fcntl(_waketx, O_NONBLOCK)"); // TODO +syserr
if (sys::Fcntl(_waketx, F_SETFD, FD_CLOEXEC) < 0)
panic("fcntl(_waketx, FD_CLOEXEC)"); // TODO +syserr
#else
HANDLE hwaketx = (HANDLE)_get_osfhandle(_waketx);
DWORD mode = PIPE_READMODE_BYTE | PIPE_NOWAIT;
if (!SetNamedPipeHandleState(hwaketx, &mode, NULL, NULL))
panic("SetNamedPipeHandleState(hwaketx, PIPE_NOWAIT)"); // TODO +syserr
#endif
_actIgnore.sa_handler = SIG_IGN;
_actIgnore.sa_flags = 0;
xsigemptyset(&_actIgnore.sa_mask);
_actNotify.sa_sigaction = _os_sighandler;
_actNotify.sa_flags = SA_SIGINFO;
#if HAVE_SIGACTION
xsigemptyset(&_actIgnore.sa_mask);
xsigemptyset(&_actNotify.sa_mask);
#endif
}
......@@ -248,6 +279,14 @@ done:
if (sah != SIG_IGN) {
if (sah != SIG_DFL) {
sah(sig);
// handlers installed via signal usually reinstall themselves
// return it back to us
struct sigaction after;
xsys_sigaction(sig, &_actNotify, &after);
if (! (_sigact_equal(&after, &_actNotify) ||
( ((after.sa_flags & SA_SIGINFO) == 0) &&
((after.sa_handler == SIG_DFL) || (after.sa_handler == sah)))))
panic("collision detected wrt thirdparty signal usage");
}
else {
// SIG_DFL && _SigReset - reraise to die if the signal is fatal
......@@ -343,7 +382,7 @@ static int/*syserr*/ _Notify1(chan<os::Signal> ch, os::Signal sig) {
// retrieve current signal action
struct sigaction cur;
int syserr = sys::Sigaction(sig.signo, nil, &cur);
int syserr = sys_sigaction(sig.signo, nil, &cur);
if (syserr < 0) {
// TODO reenable once we can panic with any object
//return fmt::errorf("sigaction sig%d: %w", sig.signo, sys::NewErrno(syserr);
......@@ -389,7 +428,7 @@ static int/*syserr*/ _Notify1(chan<os::Signal> ch, os::Signal sig) {
// register our sigaction
struct sigaction old;
syserr = sys::Sigaction(sig.signo, &_actNotify, &old);
syserr = sys_sigaction(sig.signo, &_actNotify, &old);
if (syserr < 0) {
// TODO reenable once we can panic with any object
//return fmt::errorf("sigaction sig%d: %w", sig.signo, sys::NewErrno(syserr);
......@@ -469,7 +508,7 @@ static int/*syserr*/ _Ignore1(os::Signal sig) {
if (h == nil) {
h = new _SigHandler();
h->sigstate.store(_SigIgnoring);
int syserr = sys::Sigaction(sig.signo, nil, &h->prev_act);
int syserr = sys_sigaction(sig.signo, nil, &h->prev_act);
if (syserr < 0) {
delete h;
return syserr; // TODO errctx
......@@ -481,7 +520,7 @@ static int/*syserr*/ _Ignore1(os::Signal sig) {
h->sigstate.store(_SigIgnoring);
h->subscribers.clear();
int syserr = sys::Sigaction(sig.signo, &_actIgnore, nil);
int syserr = sys_sigaction(sig.signo, &_actIgnore, nil);
if (syserr < 0)
return syserr; // TODO errctx
......@@ -509,7 +548,7 @@ static int/*syserr*/ _Reset1(os::Signal sig) {
h->sigstate.store(_SigReset);
struct sigaction act;
int syserr = sys::Sigaction(sig.signo, &h->prev_act, &act);
int syserr = sys_sigaction(sig.signo, &h->prev_act, &act);
if (syserr < 0)
return syserr; // TODO errctx
if (sigstate == _SigNotifying)
......@@ -617,16 +656,75 @@ static void _checksig(int signo) {
}
#if HAVE_SIGACTION
static void xsigemptyset(sigset_t *sa_mask) {
if (sigemptyset(sa_mask) < 0)
panic("sigemptyset failed"); // must always succeed
}
#endif
static void xsys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact) {
int syserr = sys::Sigaction(signo, act, oldact);
int syserr = sys_sigaction(signo, act, oldact);
if (syserr != 0)
panic("sigaction failed"); // TODO add errno detail
}
static int sys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact) {
#if HAVE_SIGACTION
return sys::Sigaction(signo, act, oldact);
#else
sys::sighandler_t oldh, newh;
int syserr;
if (act == nil) {
newh = SIG_GET;
}
else if (act->sa_flags & SA_SIGINFO) {
if (act->sa_sigaction != _os_sighandler)
panic("BUG: compat sigaction: act->sa_sigaction != _os_sighandler");
newh = _os_sighandler_nosigaction;
}
else {
newh = act->sa_handler;
}
oldh = sys::Signal(signo, newh, &syserr);
if (oldh == SIG_ERR)
return syserr;
if (oldact != nil) {
if (oldh == _os_sighandler_nosigaction) {
oldact->sa_sigaction = _os_sighandler;
oldact->sa_flags = SA_SIGINFO;
}
else {
oldact->sa_handler = oldh;
oldact->sa_flags = 0;
}
}
return 0;
#endif
}
#if !HAVE_SIGACTION
static void _os_sighandler_nosigaction(int sig) {
// reinstall sighandler since on handler invocation it is reset to SIG_DFL.
// do it as fast as we can as the first thing - to minimize window of race
// while signal handler is reset from being our _os_sighandler.
//
// NOTE it is ok to reinstall before invoking _os_sighandler because
// _os_sighandler uses only atomics and pipe write and so is reentrant.
sys::sighandler_t h;
int syserr;
h = sys::Signal(sig, _os_sighandler_nosigaction, &syserr);
if (h == SIG_ERR)
panic("signal(reinstall) failed");
if ( !(h == SIG_DFL /*reset*/ ||
h == _os_sighandler_nosigaction /*concurrent sighandler*/) )
panic("collision detected wrt thirdparty signal usage");
// invoke _os_sighandler as if it was setup by sigaction.
_os_sighandler(sig, NULL, NULL);
}
#endif
static bool _sigact_equal(const struct sigaction *a, const struct sigaction *b) {
// don't compare sigaction by memcmp - it will fail because struct sigaction
......
golang/os/signal_test.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2021-2022 Nexedi SA and Contributors.
# Copyright (C) 2021-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -30,12 +30,27 @@ from golang.golang_test import panics, _pyrun
from pytest import raises
from subprocess import PIPE
try:
from signal import raise_signal
except ImportError: # py2
from _testcapi import raise_signal
# directories
dir_os = dirname(__file__) # .../pygolang/os
dir_testprog = dir_os + "/testprog" # .../pygolang/os/testprog
# default to use SIGUSR1/SIGUSR2 in tests.
# but use SIGTERM/SIGINT if those are not available (windows).
try:
SIG1 = getattr(syscall, 'SIGUSR1')
SIG2 = getattr(syscall, 'SIGUSR2')
except AttributeError:
SIG1 = syscall.SIGTERM
SIG2 = syscall.SIGINT
N = 1000
# test_signal verifies signal delivery to channels controlled by Notify/Stop/Ignore/Reset.
......@@ -43,9 +58,9 @@ N = 1000
def test_signal():
# Notify/Stop with wrong chan dtype -> panic
_ = panics("pychan: channel type mismatch")
with _: signal.Notify(chan(2), syscall.SIGUSR1)
with _: signal.Notify(chan(2), SIG1)
with _: signal.Stop (chan(2))
with _: signal.Notify(chan(2, dtype='C.int'), syscall.SIGUSR1)
with _: signal.Notify(chan(2, dtype='C.int'), SIG1)
with _: signal.Stop (chan(2, dtype='C.int'))
# Notify/Ignore/Reset with wrong signal type
......@@ -54,109 +69,109 @@ def test_signal():
with _: signal.Ignore(None)
with _: signal.Reset(None)
# subscribe ch1(USR1), ch12(USR1,USR2) and ch2(USR2)
# subscribe ch1(SIG1), ch12(SIG1,SIG2) and ch2(SIG2)
ch1 = chan(2, dtype=gos.Signal)
ch12 = chan(2, dtype=gos.Signal)
ch2 = chan(2, dtype=gos.Signal)
signal.Notify(ch1, syscall.SIGUSR1)
signal.Notify(ch12, syscall.SIGUSR1, syscall.SIGUSR2)
signal.Notify(ch2, syscall.SIGUSR2)
signal.Notify(ch1, SIG1)
signal.Notify(ch12, SIG1, SIG2)
signal.Notify(ch2, SIG2)
def _():
signal.Reset()
defer(_)
for i in range(N):
# raise SIGUSR1 -> should be delivered to ch1 and ch12
# raise SIG1 -> should be delivered to ch1 and ch12
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1 and len(ch12) == 1)
sig1 = ch1.recv()
sig12 = ch12.recv()
assert sig1 == syscall.SIGUSR1
assert sig12 == syscall.SIGUSR1
assert sig1 == SIG1
assert sig12 == SIG1
# raise SIGUSR2 -> should be delivered to ch12 and ch2
# raise SIG2 -> should be delivered to ch12 and ch2
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR2)
killme(SIG2)
waitfor(lambda: len(ch12) == 1 and len(ch2) == 1)
sig12 = ch12.recv()
sig2 = ch2.recv()
assert sig12 == syscall.SIGUSR2
assert sig2 == syscall.SIGUSR2
# if SIGUSR2 will be eventually delivered to ch1 - it will break
# in SIGUSR1 check on next iteration.
assert sig12 == SIG2
assert sig2 == SIG2
# if SIG2 will be eventually delivered to ch1 - it will break
# in SIG1 check on next iteration.
# Stop(ch2) -> signals should not be delivered to ch2 anymore
signal.Stop(ch2)
for i in range(N):
# USR1 -> ch1, ch12
# SIG1 -> ch1, ch12
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1 and len(ch12) == 1)
sig1 = ch1.recv()
sig12 = ch12.recv()
assert sig1 == syscall.SIGUSR1
assert sig12 == syscall.SIGUSR1
assert sig1 == SIG1
assert sig12 == SIG1
# USR2 -> ch12, !ch2
# SIG2 -> ch12, !ch2
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR2)
killme(SIG2)
waitfor(lambda: len(ch12) == 1)
sig12 = ch12.recv()
assert sig12 == syscall.SIGUSR2
# if SIGUSR2 will be eventually delivered to ch2 - it will break on
assert sig12 == SIG2
# if SIG2 will be eventually delivered to ch2 - it will break on
# next iteration.
# Ignore(USR1) -> ch1 should not be delivered to anymore, ch12 should be delivered only USR2
signal.Ignore(syscall.SIGUSR1)
# Ignore(SIG1) -> ch1 should not be delivered to anymore, ch12 should be delivered only SIG2
signal.Ignore(SIG1)
for i in range(N):
# USR1 -> ø
# SIG1 -> ø
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
time.sleep(1E-6)
# USR2 -> ch12
# SIG2 -> ch12
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR2)
killme(SIG2)
waitfor(lambda: len(ch12) == 1)
sig12 = ch12.recv()
assert sig12 == syscall.SIGUSR2
# if SIGUSR1 or SIGUSR2 will be eventually delivered to ch1 or ch2 - it
assert sig12 == SIG2
# if SIG1 or SIG2 will be eventually delivered to ch1 or ch2 - it
# will break on next iteration.
# Notify after Ignore
signal.Notify(ch1, syscall.SIGUSR1)
signal.Notify(ch1, SIG1)
for i in range(N):
# USR1 -> ch1
# SIG1 -> ch1
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1)
sig1 = ch1.recv()
assert sig1 == syscall.SIGUSR1
assert sig1 == SIG1
# USR2 -> ch12
# SIG2 -> ch12
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR2)
killme(SIG2)
waitfor(lambda: len(ch12) == 1)
sig12 = ch12.recv()
assert sig12 == syscall.SIGUSR2
# if SIGUSR1 or SIGUSR2 will be eventually delivered to wrong place -
assert sig12 == SIG2
# if SIG1 or SIG2 will be eventually delivered to wrong place -
# it will break on next iteration.
# Reset is tested in test_stdlib_interop (it needs non-terminating default
......@@ -173,11 +188,11 @@ def test_notify_reinstall():
for i in range(N):
signal.Stop(ch)
signal.Notify(ch, syscall.SIGUSR1)
signal.Notify(ch, SIG1)
time.sleep(0.1*time.second)
assert len(ch) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
time.sleep(0.1*time.second)
assert len(ch) == 1
......@@ -188,7 +203,7 @@ def test_signal_all():
assert b"ok (notify)" in out
assert b"ok (ignore)" in out
assert b"terminating ..." in out
assert retcode == -syscall.SIGTERM.signo
assert retcode == (-syscall.SIGTERM.signo if os.name != 'nt' else 3)
# test_stdlib_interop verifies that there is minimal compatibility in between
......@@ -206,23 +221,23 @@ def test_stdlib_interop():
ch1 = chan(2, dtype=object) # NOTE not gos.Signal nor 'C.os::Signal'
def _(signo, frame):
ch1.send("USR1")
pysig.signal(pysig.SIGUSR1, _)
ch1.send("SIG1")
pysig.signal(SIG1.signo, _)
def _():
pysig.signal(pysig.SIGUSR1, pysig.SIG_IGN)
pysig.signal(SIG1.signo, pysig.SIG_IGN)
defer(_)
# verify that plain pysig delivery works
for i in range(N):
assert len(ch1) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1)
obj1 = ch1.recv()
assert obj1 == "USR1"
assert obj1 == "SIG1"
# verify that combined pysig + golang.os.signal delivery works
ch2 = chan(2, dtype=gos.Signal)
signal.Notify(ch2, syscall.SIGUSR1)
signal.Notify(ch2, SIG1)
def _():
signal.Stop(ch2)
defer(_)
......@@ -230,46 +245,46 @@ def test_stdlib_interop():
for i in range(N):
assert len(ch1) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1 and len(ch2) == 1)
obj1 = ch1.recv()
sig2 = ch2.recv()
assert obj1 == "USR1"
assert sig2 == syscall.SIGUSR1
assert obj1 == "SIG1"
assert sig2 == SIG1
# Ignore stops delivery to both pysig and golang.os.signal
signal.Ignore(syscall.SIGUSR1)
signal.Ignore(SIG1)
for i in range(N):
assert len(ch1) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
time.sleep(1E-6)
time.sleep(0.1) # just in case
assert len(ch1) == 0
assert len(ch2) == 0
# after Reset pysig delivery is restored even after Ignore
signal.Reset(syscall.SIGUSR1)
signal.Reset(SIG1)
for i in range(N):
assert len(ch1) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1)
assert len(ch2) == 0
obj1 = ch1.recv()
assert obj1 == "USR1"
assert obj1 == "SIG1"
# Reset stops delivery to golang.os.signal and restores pysig delivery
signal.Notify(ch2, syscall.SIGUSR1)
signal.Reset(syscall.SIGUSR1)
signal.Notify(ch2, SIG1)
signal.Reset(SIG1)
for i in range(N):
assert len(ch1) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1)
assert len(ch2) == 0
obj1 = ch1.recv()
assert obj1 == "USR1"
assert obj1 == "SIG1"
# test_stdlib_interop_KeyboardInterrupt verifies that signal.{Notify,Ignore} disable
......@@ -353,8 +368,9 @@ def test_stdlib_interop_KeyboardInterrupt():
# killme sends signal sig to own process.
def killme(sig):
mypid = os.getpid()
os.kill(mypid, sig.signo)
# use raise(sig) instead of kill(mypid, sig) so that it works on windows,
# where os.kill unconditionally terminates target process.
raise_signal(sig.signo)
# wait for waits until cond() becomes true or timeout.
def waitfor(cond):
......
golang/os/testprog/signal_test_all.py
View file @ fdd73156
#!/usr/bin/env python
# Copyright (C) 2021-2022 Nexedi SA and Contributors.
# Copyright (C) 2021-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -24,7 +24,8 @@ from __future__ import print_function, absolute_import
from golang import chan
from golang import os as gos, syscall, time
from golang.os import signal
import os, sys
from golang.os.signal_test import killme
import sys
def main():
# build "all signals" list
......@@ -35,11 +36,17 @@ def main():
if sig not in allsigv: # avoid e.g. SIGCHLD/SIGCLD dups
allsigv.append(sig)
allsigv.sort(key=lambda sig: sig.signo)
allsigv.remove(syscall.SIGKILL) # SIGKILL/SIGSTOP cannot be caught
allsigv.remove(syscall.SIGSTOP)
allsigv.remove(syscall.SIGBUS) # AddressSanitizer crashes on SIGBUS/SIGFPE/SIGSEGV
allsigv.remove(syscall.SIGFPE)
allsigv.remove(syscall.SIGSEGV)
def without(signame):
sig = getattr(syscall, signame, None)
if sig is not None:
allsigv.remove(sig)
without('SIGKILL') # SIGKILL/SIGSTOP cannot be caught
without('SIGSTOP')
without('SIGBUS') # AddressSanitizer crashes on SIGBUS/SIGFPE/SIGSEGV
without('SIGFPE')
without('SIGSEGV')
without('SIGILL') # SIGILL/SIGABRT cause termination on windows
without('SIGABRT')
# Notify() -> kill * -> should be notified
ch = chan(10, dtype=gos.Signal)
......@@ -72,11 +79,6 @@ def main():
killme(syscall.SIGTERM)
raise AssertionError("not terminated")
# killme sends signal sig to own process.
def killme(sig):
mypid = os.getpid()
os.kill(mypid, sig.signo)
def emit(msg=''):
print(msg)
sys.stdout.flush()
......
golang/pyx/build.py
View file @ fdd73156
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -73,8 +73,10 @@ _dso_build_ext = setuptools_dso.build_ext
class build_ext(_dso_build_ext):
def build_extension(self, ext):
# wrap _compiler <src> -> <obj> with our code
# ._compile is used on gcc/clang but not with msvc
_compile = self.compiler._compile
def _(obj, src, ext, cc_args, extra_postargs, pp_opts):
def xcompile(obj, src, ext, cc_args, extra_postargs, pp_opts):
# filter_out removes arguments that start with argprefix
cc_args = cc_args[:]
extra_postargs = extra_postargs[:]
......@@ -101,11 +103,30 @@ class build_ext(_dso_build_ext):
filter_out('-std=gnu++')
_compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
self.compiler._compile = _
# msvc handles all sources directly in the loop in .compile and we can
# do per-source adjustsment only in .spawn .
spawn = self.compiler.spawn
def xspawn(argv):
c = False
for arg in argv:
if arg.startswith('/Tc'):
c = True
if c:
argv = argv[:]
for i in range(len(argv)):
if argv[i] == '/std:c++20':
argv[i] = '/std:c11'
return spawn(argv)
self.compiler._compile = xcompile
self.compiler.spawn = xspawn
try:
_dso_build_ext.build_extension(self, ext) # super doesn't work for _dso_build_ext
finally:
self.compiler._compile = _compile
self.compiler.spawn = spawn
# setup should be used instead of setuptools.setup
......@@ -128,14 +149,14 @@ def setup(**kw):
# x_dsos = [DSO('mypkg.mydso', ['mypkg/mydso.cpp'])],
# )
def DSO(name, sources, **kw):
_, kw = _with_build_defaults(kw)
_, kw = _with_build_defaults(name, kw)
dso = setuptools_dso.DSO(name, sources, **kw)
return dso
# _with_build_defaults returns copy of kw amended with build options common for
# both DSO and Extension.
def _with_build_defaults(kw): # -> (pygo, kw')
def _with_build_defaults(name, kw): # -> (pygo, kw')
# find pygolang root
gopkg = _findpkg("golang")
pygo = dirname(gopkg.path) # .../pygolang/golang -> .../pygolang
......@@ -144,15 +165,22 @@ def _with_build_defaults(kw): # -> (pygo, kw')
kw = kw.copy()
# prepend -I<pygolang> so that e.g. golang/libgolang.h is found
sysname = platform.system().lower()
win = (sysname == 'windows')
msvc = win # TODO also support mingw ?
# prepend -I<pygolang> so that e.g. golang/libgolang.h is found
# same with -I<pygolang>/golang/_compat/<os>
incv = kw.get('include_dirs', [])[:]
incv.insert(0, pygo)
incv.insert(1, join(pygo, 'golang', '_compat', sysname))
kw['include_dirs'] = incv
# link with libgolang.so
dsov = kw.get('dsos', [])[:]
dsov.insert(0, 'golang.runtime.libgolang')
kw['dsos'] = dsov
# link with libgolang.so if it is not libgolang itself
if name != 'golang.runtime.libgolang':
dsov = kw.get('dsos', [])[:]
dsov.insert(0, 'golang.runtime.libgolang')
kw['dsos'] = dsov
# default language to C++ (chan[T] & co are accessible only via C++)
lang = kw.setdefault('language', 'c++')
......@@ -160,9 +188,20 @@ def _with_build_defaults(kw): # -> (pygo, kw')
# default to C++11 (chan[T] & co require C++11 features)
ccdefault = []
if lang == 'c++':
ccdefault.append('-std=c++11')
if not msvc:
if name == 'golang.runtime.libgolang':
ccdefault.append('-std=gnu++11') # not c++11 as linux/list.h uses typeof
else:
ccdefault.append('-std=c++11')
else:
# MSVC requries C++20 for designated struct initializers
ccdefault.append('/std:c++20')
# and make exception handling: "fully conformant", so that e.g. extern "C" `panic` works
ccdefault.extend(['/EHc-', '/EHsr'])
# default to no strict-aliasing
ccdefault.append('-fno-strict-aliasing')
if not msvc:
ccdefault.append('-fno-strict-aliasing') # use only on gcc/clang - msvc does it by default
_ = kw.get('extra_compile_args', [])[:]
_[0:0] = ccdefault # if another e.g. -std=... was already there -
......@@ -188,6 +227,8 @@ def _with_build_defaults(kw): # -> (pygo, kw')
'os/signal.h',
'pyx/runtime.h',
'_testing.h',
'_compat/windows/strings.h',
'_compat/windows/unistd.h',
]])
kw['depends'] = dependv
......@@ -203,7 +244,7 @@ def _with_build_defaults(kw): # -> (pygo, kw')
# ext_modules = [Extension('mypkg.mymod', ['mypkg/mymod.pyx'])],
# )
def Extension(name, sources, **kw):
pygo, kw = _with_build_defaults(kw)
pygo, kw = _with_build_defaults(name, kw)
# some pyx-level depends to workaround a bit lack of proper dependency
# tracking in setuptools/distutils.
......
golang/pyx/build_test.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -28,7 +28,8 @@ testprog = dirname(__file__) + "/testprog"
# verify that we can build/run external package that uses pygolang in pyx mode.
def test_pyx_build():
pyxuser = testprog + "/golang_pyx_user"
pyrun(["setup.py", "build_ext", "-i"], cwd=pyxuser)
pyrun(["setup.py", "build_ext", "-i"], cwd=pyxuser,
lsan=False) # gcc leaks
# run built test.
_ = pyout(["-c",
......@@ -44,8 +45,8 @@ def test_pyx_build():
# verify that we can build/run external dso that uses libgolang.
def test_dso_build():
dsouser = testprog + "/golang_dso_user"
pyrun(["setup.py", "build_dso", "-i"], cwd=dsouser)
pyrun(["setup.py", "build_ext", "-i"], cwd=dsouser)
pyrun(["setup.py", "build_dso", "-i"], cwd=dsouser, lsan=False) # gcc leaks
pyrun(["setup.py", "build_ext", "-i"], cwd=dsouser, lsan=False) # gcc leaks
# run built test.
_ = pyout(["-c",
......
golang/pyx/runtime.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_PYX_RUNTIME_H
#define _NXD_LIBGOLANG_PYX_RUNTIME_H
// Copyright (C) 2018-2019 Nexedi SA and Contributors.
// Copyright (C) 2018-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -58,8 +58,8 @@ class _PyError final : public _error, object {
private:
_PyError();
~_PyError();
friend error PyErr_Fetch();
friend void PyErr_ReRaise(PyError pyerr);
friend LIBPYXRUNTIME_API error PyErr_Fetch();
friend LIBPYXRUNTIME_API void PyErr_ReRaise(PyError pyerr);
public:
LIBPYXRUNTIME_API void incref();
LIBPYXRUNTIME_API void decref();
......
golang/pyx/testprog/cmdclass_custom.py
View file @ fdd73156
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -29,4 +29,9 @@ class mybuild_ext(build_ext):
setup(
cmdclass = {'build_ext': mybuild_ext},
# avoid setuptools thinking nearby golang_pyx_user/ golang_dso_user/ also
# relate to hereby setup and rejecting the build.
# https://stackoverflow.com/questions/72294299/multiple-top-level-packages-discovered-in-a-flat-layout
py_modules = [],
)
golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
View file @ fdd73156
// Copyright (C) 2019 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
......@@ -20,7 +20,7 @@
// Small library that uses a bit of libgolang features, mainly to verify
// that external project can build against libgolang.
#include <golang/libgolang.h>
#include "dso.h"
using namespace golang;
#include <stdio.h>
......
golang/pyx/testprog/golang_dso_user/dsouser/dso.h 0 → 100644
View file @ fdd73156
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
#ifndef _NXD_GOLANG_DSOUSER_DSO_H
#define _NXD_GOLANG_DSOUSER_DSO_H
#include <golang/libgolang.h>
#if BUILDING_DSOUSER_DSO
# define DSOUSER_DSO_API LIBGOLANG_DSO_EXPORT
#else
# define DSOUSER_DSO_API LIBGOLANG_DSO_IMPORT
#endif
DSOUSER_DSO_API void dsotest();
#endif // _NXD_GOLANG_DSOUSER_DSO_H
golang/pyx/testprog/golang_dso_user/dsouser/test.pyx
View file @ fdd73156
# cython: language_level=2
#
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -21,10 +21,7 @@
# Small test driver that calls dso.so .
cdef extern from * nogil:
"""
void dsotest();
"""
cdef extern from "dso.h" nogil:
void dsotest()
def main():
......
golang/pyx/testprog/golang_dso_user/setup.py
View file @ fdd73156
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -24,7 +24,9 @@ setup(
name = 'golang_dso_user',
description = 'test project that uses libgolang from a dso',
x_dsos = [DSO('dsouser.dso', ['dsouser/dso.cpp'])],
x_dsos = [DSO('dsouser.dso',
['dsouser/dso.cpp'],
define_macros = [('BUILDING_DSOUSER_DSO', None)])],
ext_modules = [Extension('dsouser.test',
['dsouser/test.pyx'],
dsos = ['dsouser.dso'])],
......
golang/runtime/_libgolang.pxd
View file @ fdd73156
# cython: language_level=2
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -36,7 +36,7 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil:
_libgolang_sema* (*sema_alloc) ()
void (*sema_free) (_libgolang_sema*)
void (*sema_acquire)(_libgolang_sema*)
bint (*sema_acquire)(_libgolang_sema*, uint64_t timeout_ns)
void (*sema_release)(_libgolang_sema*)
void (*nanosleep)(uint64_t)
......
golang/runtime/_runtime_gevent.pyx
View file @ fdd73156
# cython: language_level=2
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -40,7 +40,10 @@ ELSE:
from gevent import sleep as pygsleep
from libc.stdint cimport uint8_t, uint64_t
from libc.stdint cimport uint8_t, uint64_t, UINT64_MAX
cdef extern from *:
ctypedef bint cbool "bool"
from cpython cimport PyObject, Py_INCREF, Py_DECREF
from cython cimport final
......@@ -57,7 +60,7 @@ from posix.fcntl cimport mode_t, F_GETFL, F_SETFL, O_NONBLOCK, O_ACCMODE, O_RDON
from posix.stat cimport struct_stat, S_ISREG, S_ISDIR, S_ISBLK
from posix.strings cimport bzero
from gevent.fileobject import FileObjectThread, FileObjectPosix
from gevent import fileobject as gfileobj
# _goviapy & _togo serve go
......@@ -95,9 +98,12 @@ cdef:
Py_DECREF(pygsema)
return True
bint _sema_acquire(_libgolang_sema *gsema):
bint _sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns, cbool* pacq):
pygsema = <PYGSema>gsema
pygsema.acquire()
timeout = None
if timeout_ns != UINT64_MAX:
timeout = float(timeout_ns) * 1e-9
pacq[0] = pygsema.acquire(timeout=timeout)
return True
bint _sema_release(_libgolang_sema *gsema):
......@@ -142,14 +148,16 @@ cdef nogil:
if not ok:
panic("pyxgo: gevent: sema: free: failed")
void sema_acquire(_libgolang_sema *gsema):
cbool sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns):
cdef PyExc exc
cdef cbool acq
with gil:
pyexc_fetch(&exc)
ok = _sema_acquire(gsema)
ok = _sema_acquire(gsema, timeout_ns, &acq)
pyexc_restore(exc)
if not ok:
panic("pyxgo: gevent: sema: acquire: failed")
return acq
void sema_release(_libgolang_sema *gsema):
cdef PyExc exc
......@@ -194,31 +202,39 @@ cdef nogil:
return ioh
_libgolang_ioh* _io_fdopen(int *out_syserr, int sysfd):
# check if we should enable O_NONBLOCK on this file-descriptor
# even though we could enable O_NONBLOCK for regular files, it does not
# work as expected as most unix'es report regular files as always read
# and write ready.
cdef struct_stat st
cdef int syserr = syscall.Fstat(sysfd, &st)
if syserr < 0:
out_syserr[0] = syserr
return NULL
m = st.st_mode
blocking = (S_ISREG(m) or S_ISDIR(m) or S_ISBLK(m)) # fd cannot refer to symlink
# retrieve current sysfd flags and access mode
flags = syscall.Fcntl(sysfd, F_GETFL, 0)
if flags < 0:
out_syserr[0] = flags
return NULL
acc = (flags & O_ACCMODE)
# enable O_NONBLOCK if needed
if not blocking:
syserr = syscall.Fcntl(sysfd, F_SETFL, flags | O_NONBLOCK)
IF POSIX:
# check if we should enable O_NONBLOCK on this file-descriptor
# even though we could enable O_NONBLOCK for regular files, it does not
# work as expected as most unix'es report regular files as always read
# and write ready.
cdef struct_stat st
cdef int syserr = syscall.Fstat(sysfd, &st)
if syserr < 0:
out_syserr[0] = syserr
return NULL
m = st.st_mode
blocking = (S_ISREG(m) or S_ISDIR(m) or S_ISBLK(m)) # fd cannot refer to symlink
# retrieve current sysfd flags and access mode
flags = syscall.Fcntl(sysfd, F_GETFL, 0)
if flags < 0:
out_syserr[0] = flags
return NULL
acc = (flags & O_ACCMODE)
# enable O_NONBLOCK if needed
if not blocking:
syserr = syscall.Fcntl(sysfd, F_SETFL, flags | O_NONBLOCK)
if syserr < 0:
out_syserr[0] = syserr
return NULL
ELSE: # !POSIX (windows)
cdef bint blocking = True
# FIXME acc: use GetFileInformationByHandleEx to determine whether
# HANDLE(sysfd) was opened for reading or writing.
# https://stackoverflow.com/q/9442436/9456786
cdef int acc = O_RDWR
# create IOH backed by FileObjectThread or FileObjectPosix
ioh = <IOH*>calloc(1, sizeof(IOH))
......@@ -253,9 +269,9 @@ cdef:
pygfobj = None
try:
if blocking:
pygfobj = FileObjectThread(sysfd, mode=mode, buffering=0)
pygfobj = gfileobj.FileObjectThread(sysfd, mode=mode, buffering=0)
else:
pygfobj = FileObjectPosix(sysfd, mode=mode, buffering=0)
pygfobj = gfileobj.FileObjectPosix(sysfd, mode=mode, buffering=0)
except OSError as e:
out_syserr[0] = -e.errno
else:
......@@ -338,7 +354,19 @@ cdef:
cdef uint8_t[::1] mem = <uint8_t[:count]>buf
xmem = memoryview(mem) # to avoid https://github.com/cython/cython/issues/3900 on mem[:0]=b''
try:
n = pygfobj.readinto(xmem)
# NOTE buf might be on stack, so it must not be accessed, e.g. from
# FileObjectThread, while our greenlet is parked (see STACK_DEAD_WHILE_PARKED
# for details). -> Use intermediate on-heap buffer to protect from that.
#
# Also: we cannot use pygfobj.readinto due to
# https://github.com/gevent/gevent/pull/1948
#
# TODO use .readinto() when buf is not on stack and gevent is
# recent enough or pygfobj is not FileObjectThread.
#n = pygfobj.readinto(xmem)
buf2 = pygfobj.read(count)
n = len(buf2)
xmem[:n] = buf2
except OSError as e:
n = -e.errno
out_n[0] = n
......@@ -361,8 +389,14 @@ cdef:
bint _io_write(IOH* ioh, int* out_n, const void *buf, size_t count):
pygfobj = <object>ioh.pygfobj
cdef const uint8_t[::1] mem = <const uint8_t[:count]>buf
# NOTE buf might be on stack, so it must not be accessed, e.g. from
# FileObjectThread, while our greenlet is parked (see STACK_DEAD_WHILE_PARKED
# for details). -> Use intermediate on-heap buffer to protect from that.
buf2 = bytearray(mem)
try:
n = pygfobj.write(mem)
n = pygfobj.write(buf2)
except OSError as e:
n = -e.errno
out_n[0] = n
......
golang/runtime/_runtime_thread.pyx
View file @ fdd73156
# cython: language_level=2
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -35,7 +35,12 @@ from __future__ import print_function, absolute_import
#
# NOTE Cython declares PyThread_acquire_lock/PyThread_release_lock as nogil
from cpython.pythread cimport PyThread_acquire_lock, PyThread_release_lock, \
PyThread_type_lock, WAIT_LOCK
PyThread_type_lock, WAIT_LOCK, NOWAIT_LOCK, PyLockStatus, PY_LOCK_ACQUIRED, PY_LOCK_FAILURE
cdef extern from * nogil:
ctypedef int PY_TIMEOUT_T # long long there
PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock, PY_TIMEOUT_T timeout_us, int intr_flag)
# NOTE On Darwin, even though this is considered as POSIX, Python uses
# mutex+condition variable to implement its lock, and, as of 20190828, Py2.7
......@@ -98,6 +103,9 @@ from libc.errno cimport errno, EINTR, EBADF
from posix.fcntl cimport mode_t
from posix.stat cimport struct_stat
from posix.strings cimport bzero
cdef extern from *:
ctypedef bint cbool "bool"
IF POSIX:
from posix.time cimport clock_gettime, nanosleep as posix_nanosleep, timespec, CLOCK_REALTIME
ELSE:
......@@ -138,11 +146,46 @@ cdef nogil:
pysema = <PyThread_type_lock>gsema
PyThread_free_lock(pysema)
void sema_acquire(_libgolang_sema *gsema):
cbool sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns):
pysema = <PyThread_type_lock>gsema
ok = PyThread_acquire_lock(pysema, WAIT_LOCK)
if ok == 0:
panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock failed")
IF PY3:
cdef PY_TIMEOUT_T timeout_us
ELSE:
cdef uint64_t tprev, t, tsleep
if timeout_ns == UINT64_MAX:
ok = PyThread_acquire_lock(pysema, WAIT_LOCK)
if ok == 0:
panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock failed")
return 1
else:
IF PY3:
timeout_us = timeout_ns // 1000
lkok = PyThread_acquire_lock_timed(pysema, timeout_us, 0)
if lkok == PY_LOCK_FAILURE:
return 0
elif lkok == PY_LOCK_ACQUIRED:
return 1
else:
panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock_timed failed")
ELSE:
# py2 misses PyThread_acquire_lock_timed - provide fallback ourselves
tprev = nanotime()
while 1:
ok = PyThread_acquire_lock(pysema, NOWAIT_LOCK)
if ok:
return 1
tsleep = min(timeout_ns, 50*1000) # poll every 50 μs = 20 Hz
if tsleep == 0:
break
nanosleep(tsleep)
t = nanotime()
if t < tprev:
break # clock skew
if t - tprev >= timeout_ns:
break
timeout_ns -= t - tprev
tprev = t
return 0
void sema_release(_libgolang_sema *gsema):
pysema = <PyThread_type_lock>gsema
......
golang/runtime/internal/atomic.cpp
View file @ fdd73156
// Copyright (C) 2022 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
// Copyright (C) 2022-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
......@@ -20,7 +20,9 @@
#include "golang/runtime/internal/atomic.h"
#include "golang/libgolang.h"
#ifndef _WIN32
#include <pthread.h>
#endif
// golang::internal::atomic::
namespace golang {
......@@ -41,9 +43,12 @@ static void _forkNewEpoch() {
}
void _init() {
// there is no fork on windows
#ifndef _WIN32
int e = pthread_atfork(/*prepare*/nil, /*inparent*/nil, /*inchild*/_forkNewEpoch);
if (e != 0)
panic("pthread_atfork failed");
#endif
}
......
golang/runtime/internal/syscall.cpp
View file @ fdd73156
// Copyright (C) 2021-2022 Nexedi SA and Contributors.
// Copyright (C) 2021-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -57,15 +57,18 @@ string _Errno::Error() {
_Errno& e = *this;
char ebuf[128];
bool ok;
#if __APPLE__
int x = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
if (x == 0)
return string(ebuf);
return "unknown error " + std::to_string(-e.syserr);
ok = (::strerror_r(-e.syserr, ebuf, sizeof(ebuf)) == 0);
#elif defined(_WIN32)
ok = (::strerror_s(ebuf, sizeof(ebuf), -e.syserr) == 0);
#else
char *estr = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
return string(estr);
#endif
if (ok)
return string(ebuf);
return "unknown error " + std::to_string(-e.syserr);
}
......@@ -99,6 +102,7 @@ __Errno Close(int fd) {
return err;
}
#ifndef _WIN32
__Errno Fcntl(int fd, int cmd, int arg) {
int save_errno = errno;
int err = ::fcntl(fd, cmd, arg);
......@@ -107,6 +111,7 @@ __Errno Fcntl(int fd, int cmd, int arg) {
errno = save_errno;
return err;
}
#endif
__Errno Fstat(int fd, struct ::stat *out_st) {
int save_errno = errno;
......@@ -119,6 +124,10 @@ __Errno Fstat(int fd, struct ::stat *out_st) {
int Open(const char *path, int flags, mode_t mode) {
int save_errno = errno;
#ifdef _WIN32 // default to open files in binary mode
if ((flags & (_O_TEXT | _O_BINARY)) == 0)
flags |= _O_BINARY;
#endif
int fd = ::open(path, flags, mode);
if (fd < 0)
fd = -errno;
......@@ -126,15 +135,39 @@ int Open(const char *path, int flags, mode_t mode) {
return fd;
}
__Errno Pipe(int vfd[2]) {
__Errno Pipe(int vfd[2], int flags) {
// supported flags: O_CLOEXEC
if (flags & ~(O_CLOEXEC))
return -EINVAL;
int save_errno = errno;
int err = ::pipe(vfd);
int err;
#ifdef __linux__
err = ::pipe2(vfd, flags);
#elif defined(_WIN32)
err = ::_pipe(vfd, 4096, flags | _O_BINARY);
#else
err = ::pipe(vfd);
if (err)
goto out;
if (flags & O_CLOEXEC) {
for (int i=0; i<2; i++) {
err = Fcntl(vfd[i], F_SETFD, FD_CLOEXEC);
if (err) {
Close(vfd[0]);
Close(vfd[1]);
goto out;
}
}
}
out:
#endif
if (err == -1)
err = -errno;
errno = save_errno;
return err;
}
#ifndef _WIN32
__Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact) {
int save_errno = errno;
int err = ::sigaction(signo, act, oldact);
......@@ -143,6 +176,17 @@ __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *
errno = save_errno;
return err;
}
#endif
sighandler_t Signal(int signo, sighandler_t handler, int *out_psyserr) {
int save_errno = errno;
sighandler_t oldh = ::signal(signo, handler);
*out_psyserr = (oldh == SIG_ERR
? (errno ? -errno : -EINVAL) // windows returns SIG_ERR/errno=0 for invalid signo
: 0);
errno = save_errno;
return oldh;
}
}}} // golang::internal::syscall::
golang/runtime/internal/syscall.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_RUNTIME_INTERNAL_SYSCALL_H
#define _NXD_LIBGOLANG_RUNTIME_INTERNAL_SYSCALL_H
// Copyright (C) 2021-2022 Nexedi SA and Contributors.
// Copyright (C) 2021-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -59,15 +59,21 @@ error NewErrno(__Errno syserr); // TODO better return Errno directly.
// system calls
int/*n|err*/ Read(int fd, void *buf, size_t count);
int/*n|err*/ Write(int fd, const void *buf, size_t count);
__Errno Close(int fd);
__Errno Fcntl(int fd, int cmd, int arg);
__Errno Fstat(int fd, struct ::stat *out_st);
int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
__Errno Pipe(int vfd[2]);
__Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
LIBGOLANG_API int/*n|err*/ Read(int fd, void *buf, size_t count);
LIBGOLANG_API int/*n|err*/ Write(int fd, const void *buf, size_t count);
LIBGOLANG_API __Errno Close(int fd);
#ifndef _WIN32
LIBGOLANG_API __Errno Fcntl(int fd, int cmd, int arg);
#endif
LIBGOLANG_API __Errno Fstat(int fd, struct ::stat *out_st);
LIBGOLANG_API int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
LIBGOLANG_API __Errno Pipe(int vfd[2], int flags);
#ifndef _WIN32
LIBGOLANG_API __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
#endif
typedef void (*sighandler_t)(int);
LIBGOLANG_API sighandler_t /*sigh|SIG_ERR*/ Signal(int signo, sighandler_t handler, int *out_psyserr);
}}} // golang::internal::syscall::
......
golang/runtime/libgolang.cpp
View file @ fdd73156
// Copyright (C) 2018-2022 Nexedi SA and Contributors.
// Copyright (C) 2018-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -43,12 +43,23 @@
#include <stdlib.h>
#include <string.h>
#include <strings.h>
// linux/list.h needs ARRAY_SIZE XXX -> better use c.h or ccan/array_size.h ?
#ifndef ARRAY_SIZE
# define ARRAY_SIZE(A) (sizeof(A) / sizeof((A)[0]))
#endif
#include <linux/list.h>
// MSVC does not support statement expressions and typeof
// -> redo list_entry via C++ lambda.
#ifdef _MSC_VER
# undef list_entry
# define list_entry(ptr, type, member) [&]() { \
const decltype( ((type *)0)->member ) *__mptr = (ptr); \
return (type *)( (char *)__mptr - offsetof(type,member) ); \
}()
#endif
using std::atomic;
using std::bad_alloc;
......@@ -120,6 +131,7 @@ using internal::_runtime;
namespace internal { namespace atomic { extern void _init(); } }
namespace os { namespace signal { extern void _init(); } }
namespace time { extern void _init(); }
void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) {
if (_runtime != nil) // XXX better check atomically
panic("libgolang: double init");
......@@ -127,6 +139,7 @@ void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) {
internal::atomic::_init();
os::signal::_init();
time::_init();
}
void _taskgo(void (*f)(void *), void *arg) {
......@@ -155,7 +168,15 @@ void _semafree(_sema *sema) {
}
void _semaacquire(_sema *sema) {
_runtime->sema_acquire((_libgolang_sema *)sema);
bool ok;
ok = _runtime->sema_acquire((_libgolang_sema *)sema, UINT64_MAX);
if (!ok)
panic("semaacquire: failed");
}
// NOTE not currently exposed in public API
bool _semaacquire_timed(_sema *sema, uint64_t timeout_ns) {
return _runtime->sema_acquire((_libgolang_sema *)sema, timeout_ns);
}
void _semarelease(_sema *sema) {
......@@ -899,6 +920,13 @@ template<> int _chanselect2</*onstack=*/true> (const _selcase *, int, const vect
template<> int _chanselect2</*onstack=*/false>(const _selcase *, int, const vector<int>&);
static int __chanselect2(const _selcase *, int, const vector<int>&, _WaitGroup*);
// PRNG for select.
// TODO consider switching to xoroshiro or wyrand if speed is an issue
// https://thompsonsed.co.uk/random-number-generators-for-c-performance-tested
// https://nullprogram.com/blog/2017/09/21/
static std::random_device _devrand;
static thread_local std::mt19937 _t_rng(_devrand());
// _chanselect executes one ready send or receive channel case.
//
// if no case is ready and default case was provided, select chooses default.
......@@ -925,7 +953,7 @@ int _chanselect(const _selcase *casev, int casec) {
vector<int> nv(casec); // n -> n(case) TODO -> caller stack-allocate nv
for (int i=0; i <casec; i++)
nv[i] = i;
std::random_shuffle(nv.begin(), nv.end());
std::shuffle(nv.begin(), nv.end(), _t_rng);
// first pass: poll all cases and bail out in the end if default was provided
int ndefault = -1;
......
golang/sync.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_SYNC_H
#define _NXD_LIBGOLANG_SYNC_H
// Copyright (C) 2018-2020 Nexedi SA and Contributors.
// Copyright (C) 2018-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -212,7 +212,7 @@ class _WorkGroup : public object {
private:
_WorkGroup();
~_WorkGroup();
friend WorkGroup NewWorkGroup(context::Context ctx);
friend LIBGOLANG_API WorkGroup NewWorkGroup(context::Context ctx);
public:
LIBGOLANG_API void decref();
......
golang/testprog/golang_test_defer_excchain.txt
View file @ fdd73156
Traceback (most recent call last):
File "PYGOLANG/golang/__init__.py", line ..., in _
File "PYGOLANG/golang/__init__.py", line ..., in _goframe
return f(*argv, **kw)
^^^^^^^^^^^^^^ +PY311
File "PY39(PYGOLANG/golang/testprog/)golang_test_defer_excchain.py", line 42, in main
raise RuntimeError("err")
RuntimeError: err
......@@ -21,6 +22,7 @@ Traceback (most recent call last):
...
File "PY39(PYGOLANG/golang/testprog/)golang_test_defer_excchain.py", line 33, in d2
1/0
~^~ +PY311
ZeroDivisionError: ...
During handling of the above exception, another exception occurred:
......@@ -29,8 +31,9 @@ Traceback (most recent call last):
... "PY39(PYGOLANG/golang/testprog/)golang_test_defer_excchain.py", line 45, in <module>
main()
...
File "PYGOLANG/golang/__init__.py", line ..., in _
return f(*argv, **kw)
File "PYGOLANG/golang/__init__.py", line ..., in _goframe
return f(*argv, **kw) -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
...
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
......
golang/testprog/golang_test_defer_excchain.txt-ipython
View file @ fdd73156
...
RuntimeError Traceback (most recent call last)
PYGOLANG/golang/__init__.py in _(f, *argv, **kw)
PYGOLANG/golang/__init__.py in _goframe(f, *argv, **kw)
...
--> ... return f(*argv, **kw)
--> ... return f(*argv, **kw)
...
PYGOLANG/golang/testprog/golang_test_defer_excchain.py in main()
......@@ -50,9 +50,9 @@ PYGOLANG/golang/testprog/golang_test_defer_excchain.py in ...
...
PYGOLANG/golang/__init__.py in _(f, *argv, **kw)
PYGOLANG/golang/__init__.py in _goframe(f, *argv, **kw)
...
--> ... return f(*argv, **kw)
--> ... return f(*argv, **kw)
...
PYGOLANG/golang/__init__.py in __exit__(__goframe__, exc_type, exc_val, exc_tb)
......
golang/testprog/golang_test_defer_excchain.txt-pytest
View file @ fdd73156
...
_____________________________________ main _____________________________________
../__init__.py:...: in _
...________________ main ________________...
../__init__.py:...: in _goframe
return f(*argv, **kw)
golang_test_defer_excchain.py:42: in main
raise RuntimeError("err")
......
golang/testprog/golang_test_str.py
View file @ fdd73156
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2022 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2022-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -29,8 +29,8 @@ from golang import b, u
from golang.gcompat import qq
def main():
sb = b("привет b")
su = u("привет u")
sb = b("привет αβγ b")
su = u("привет αβγ u")
print("print(b):", sb)
print("print(u):", su)
print("print(qq(b)):", qq(sb))
......
golang/testprog/golang_test_str.txt
View file @ fdd73156
print(b): привет b
print(u): привет u
print(qq(b)): "привет b"
print(qq(u)): "привет u"
print(repr(b)): b('привет b')
print(repr(u)): u('привет u')
print({b: u}): {b('привет b'): u('привет u')}
print(b): привет αβγ b
print(u): привет αβγ u
print(qq(b)): "привет αβγ b"
print(qq(u)): "привет αβγ u"
print(repr(b)): b('привет αβγ b')
print(repr(u)): u('привет αβγ u')
print({b: u}): {b('привет αβγ b'): u('привет αβγ u')}
golang/time.cpp
View file @ fdd73156
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -21,8 +21,24 @@
// See time.h for package overview.
#include "golang/time.h"
#include "timer-wheel.h"
#include <math.h>
#define DEBUG 0
#if DEBUG
# define debugf(format, ...) fprintf(stderr, format, ##__VA_ARGS__)
#else
# define debugf(format, ...) do {} while (0)
#endif
// golang::sync:: (private imports)
namespace golang {
namespace sync {
bool _semaacquire_timed(_sema *sema, uint64_t timeout_ns);
}} // golang::sync::
// golang::time:: (except sleep and now)
......@@ -30,7 +46,6 @@ namespace golang {
namespace time {
// ---- timers ----
// FIXME timers are implemented very inefficiently - each timer currently consumes a goroutine.
Ticker new_ticker(double dt);
Timer new_timer (double dt);
......@@ -51,7 +66,12 @@ Timer after_func(double dt, func<void()> f) {
return _new_timer(dt, f);
}
// Ticker
Timer new_timer(double dt) {
return _new_timer(dt, nil);
}
// Ticker (small wrapper around Timer)
_Ticker::_Ticker() {}
_Ticker::~_Ticker() {}
void _Ticker::decref() {
......@@ -67,9 +87,7 @@ Ticker new_ticker(double dt) {
tx->c = makechan<double>(1); // 1-buffer -- same as in Go
tx->_dt = dt;
tx->_stop = false;
go([tx]() {
tx->_tick();
});
tx->_timer = after_func(dt, [tx]() { tx ->_tick(); });
return tx;
}
......@@ -78,6 +96,10 @@ void _Ticker::stop() {
tx._mu.lock();
tx._stop = true;
if (tx._timer != nil) {
tx._timer->stop();
tx._timer = nil; // break Ticker -> Timer -> _tick -> Ticker cycle
}
// drain what _tick could have been queued already
while (tx.c.len() > 0)
......@@ -88,113 +110,379 @@ void _Ticker::stop() {
void _Ticker::_tick() {
_Ticker &tx = *this;
while (1) {
// XXX adjust for accumulated error δ?
sleep(tx._dt);
tx._mu.lock();
if (tx._stop) {
tx._mu.unlock();
return;
}
// send from under ._mu so that .stop can be sure there is no
// ongoing send while it drains the channel.
double t = now();
select({
_default,
tx.c.sends(&t),
});
tx._mu.lock();
if (tx._stop) {
tx._mu.unlock();
return;
}
// XXX adjust for accumulated error δ?
tx._timer->reset(tx._dt);
// send from under ._mu so that .stop can be sure there is no
// ongoing send while it drains the channel.
double t = now();
select({
_default,
tx.c.sends(&t),
});
tx._mu.unlock();
}
// Timer
// Timers
//
// Timers are implemented via Timer Wheel.
// For this time arrow is divided into equal periods named ticks, and Ratas
// library[1] is used to manage timers with granularity of ticks. We employ
// ticks to avoid unnecessary overhead of managing timeout-style timers with
// nanosecond precision.
//
// Let g denote tick granularity.
//
// The timers are provided with guaranty that their expiration happens after
// requested expiration time. In other words the following invariant is always true:
//
// t(exp) ≤ t(fire)
//
// we also want that firing _ideally_ happens not much far away from requested
// expiration time, meaning that the following property is aimed for, but not guaranteed:
//
// t(fire) < t(exp) + g
//
// a tick Ti is associated with [i-1,i)·g time range. It is said that tick Ti
// "happens" at i·g point in time. Firing of timers associated with tick Ti is
// done when Ti happens - ideally at i·g time or strictly speaking ≥ that point.
//
// When timers are armed their expiration tick is set as Texp = ⌊t(exp)/g+1⌋ to
// be in time range that tick Texp covers.
//
//
// A special goroutine, _timer_loop, is dedicated to advance time of the
// timer-wheel as ticks happen, and to run expired timers. When there is
// nothing to do that goroutine pauses itself and goes to sleep until either
// next expiration moment, or until new timer with earlier expiration time is
// armed. To be able to simultaneously select on those two condition a
// semaphore with acquisition timeout is employed. Please see _tSema for
// details.
//
//
// [1] Ratas - A hierarchical timer wheel.
// https://www.snellman.net/blog/archive/2016-07-27-ratas-hierarchical-timer-wheel,
// https://github.com/jsnell/ratas
// Tns indicates time measured in nanoseconds.
// It is used for documentation purposes mainly to distinguish from the time measured in ticks.
typedef uint64_t Tns;
// _tick_g is ticks granularity in nanoseconds.
static const Tns _tick_g = 1024; // 1 tick is ~ 1 μs
// timer-wheel holds registry of all timers and manages them.
static sync::Mutex* _tWheelMu; // lock for timer wheel + sleep/wakeup channel (see _tSema & co below)
static TimerWheel* _tWheel; // for each timer the wheel holds 1 reference to _TimerImpl object
// _TimerImpl amends _Timer with timer-wheel entry and implementation-specific state.
enum _TimerState {
_TimerDisarmed, // timer is not registered to timer wheel and is not firing
_TimerArmed, // timer is registered to timer wheel and is not firing
_TimerFiring // timer is currently firing (and not on the timer wheel)
};
struct _TimerImpl : _Timer {
void _fire();
void _queue_fire();
MemberTimerEvent<_TimerImpl, &_TimerImpl::_queue_fire> _tWheelEntry;
func<void()> _f;
sync::Mutex _mu;
_TimerState _state;
// entry on "firing" list; see _tFiring for details
_TimerImpl* _tFiringNext; // TODO could reuse _tWheelEntry.{next_,prev_} for "firing" list
_TimerImpl();
~_TimerImpl();
};
_TimerImpl::_TimerImpl() : _tWheelEntry(this) {}
_TimerImpl::~_TimerImpl() {}
_Timer::_Timer() {}
_Timer::~_Timer() {}
void _Timer::decref() {
if (__decref())
delete this;
delete static_cast<_TimerImpl*>(this);
}
// _tSema and _tSleeping + _tWaking organize sleep/wakeup channel.
//
// Timer loop uses wakeup sema to both:
// * sleep until next timer expires, and
// * become woken up earlier if new timer with earlier expiration time is armed
//
// _tSleeping + _tWaking are used by the timer loop and clients to coordinate
// _tSema operations, so that the value of sema is always 0 or 1, and that
// every new loop cycle starts with sema=0, meaning that sema.Acquire will block.
//
// Besides same.Acquire, all operations on the sleep/wakeup channel are done under _tWheelMu.
static sync::_sema* _tSema;
static bool _tSleeping; // 1 iff timer loop:
// \/ decided to go to sleep on wakeup sema
// \/ sleeps on wakeup sema via Acquire
// \/ woken up after Acquire before setting _tSleeping=0 back
static bool _tWaking; // 1 iff client timer arm:
// /\ saw _tSleeping=1 && _tWaking=0 and decided to do wakeup
// /\ (did Release \/ will do Release)
// /\ until timer loop set back _tWaking=0
static Tns _tSleeping_until; // until when timer loop is sleeping if _tSleeping=1
// _timer_loop implements timer loop: it runs in dedicated goroutine ticking the
// timer-wheel and sleeping in between ticks.
static void _timer_loop();
static void _timer_loop_fire_queued();
void _init() {
_tWheelMu = new sync::Mutex();
_tWheel = new TimerWheel(_nanotime() / _tick_g);
_tSema = sync::_makesema(); sync::_semaacquire(_tSema); // 1 -> 0
_tSleeping = false;
_tWaking = false;
_tSleeping_until = 0;
go(_timer_loop);
}
static void _timer_loop() {
while (1) {
// tick the wheel. This puts expired timers on firing list but delays
// really firing them until we release _tWheelMu.
_tWheelMu->lock();
Tick now_t = _nanotime() / _tick_g;
Tick wnow_t = _tWheel->now();
Tick wdt_t = now_t - wnow_t;
debugf("LOOP: now_t: %lu wnow_t: %lu δ_t %lu ...\n", now_t, wnow_t, wdt_t);
if (now_t > wnow_t) // advance(0) panics. Avoid that if we wake up earlier
_tWheel->advance(wdt_t); // inside the same tick, e.g. due to signal.
_tWheelMu->unlock();
// fire the timers queued on the firing list
_timer_loop_fire_queued();
// go to sleep until next timer expires or wakeup comes from new arming.
//
// limit max sleeping time because contrary to other wheel operations -
// - e.g. insert and delete which are O(1), the complexity of
// ticks_to_next_event is O(time till next expiry).
Tns tsleep_max = 1*1E9; // 1s
bool sleeping = false;
_tWheelMu->lock();
Tick wsleep_t = _tWheel->ticks_to_next_event(tsleep_max / _tick_g);
Tick wnext_t = _tWheel->now() + wsleep_t;
Tns tnext = wnext_t * _tick_g;
Tns tnow = _nanotime();
if (tnext > tnow) {
_tSleeping = sleeping = true;
_tSleeping_until = tnext;
}
_tWheelMu->unlock();
if (!sleeping)
continue;
Tns tsleep = tnext - tnow;
debugf("LOOP: sleeping %.3f μs ...\n", tsleep / 1e3);
bool acq = sync::_semaacquire_timed(_tSema, tsleep);
// bring sleep/wakeup channel back into reset state with S=0
_tWheelMu->lock();
// acq ^ waking Release was done while Acquire was blocked S=0
// acq ^ !waking impossible
// !acq ^ waking Acquire finished due to timeout; Release was done after that S=1
// !acq ^ !waking Acquire finished due to timeout; no Release was done at all S=0
debugf("LOOP: woken up acq=%d waking=%d\n", acq, _tWaking);
if ( acq && !_tWaking) {
_tWheelMu->unlock();
panic("BUG: timer loop: woken up with acq ^ !waking");
}
if (!acq && _tWaking) {
acq = sync::_semaacquire_timed(_tSema, 0); // S=1 -> acquire should be immediate
if (!acq) {
_tWheelMu->unlock();
panic("BUG: timer loop: reacquire after acq ^ waking failed");
}
}
_tSleeping = false;
_tWaking = false;
_tSleeping_until = 0;
_tWheelMu->unlock();
}
}
Timer _new_timer(double dt, func<void()> f) {
Timer t = adoptref(new _Timer());
t->c = (f == nil ? makechan<double>(1) : nil);
t->_f = f;
t->_dt = INFINITY;
t->_ver = 0;
_TimerImpl* _t = new _TimerImpl();
_t->c = (f == nil ? makechan<double>(1) : nil);
_t->_f = f;
_t->_state = _TimerDisarmed;
_t->_tFiringNext = nil;
Timer t = adoptref(static_cast<_Timer*>(_t));
t->reset(dt);
return t;
}
Timer new_timer(double dt) {
return _new_timer(dt, nil);
void _Timer::reset(double dt) {
_TimerImpl& t = *static_cast<_TimerImpl*>(this);
if (dt <= 0)
dt = 0;
Tns when = _nanotime() + Tns(dt*1e9);
Tick when_t = when / _tick_g + 1; // Ti covers [i-1,i)·g
_tWheelMu->lock();
t._mu.lock();
if (t._state != _TimerDisarmed) {
t._mu.unlock();
_tWheelMu->unlock();
panic("Timer.reset: the timer is armed; must be stopped or expired");
}
t._state = _TimerArmed;
Tick wnow_t = _tWheel->now();
Tick wdt_t;
if (when_t > wnow_t)
wdt_t = when_t - wnow_t;
else
wdt_t = 1; // schedule(0) panics
// the wheel will keep a reference to the timer
t.incref();
_tWheel->schedule(&t._tWheelEntry, wdt_t);
t._mu.unlock();
// wakeup timer loop if it is sleeping until later than new timer expiry
if (_tSleeping) {
if ((when < _tSleeping_until) && !_tWaking) {
debugf("USER: waking up loop\n");
_tWaking = true;
sync::_semarelease(_tSema);
}
}
_tWheelMu->unlock();
}
bool _Timer::stop() {
_Timer &t = *this;
_TimerImpl& t = *static_cast<_TimerImpl*>(this);
bool canceled;
_tWheelMu->lock();
t._mu.lock();
if (t._dt == INFINITY) {
switch (t._state) {
case _TimerDisarmed:
canceled = false;
}
else {
t._dt = INFINITY;
t._ver += 1;
break;
case _TimerArmed:
// timer wheel is holding this timer entry. Remove it from there.
t._tWheelEntry.cancel();
t.decref();
canceled = true;
break;
case _TimerFiring:
// the timer is on "firing" list. Timer loop will process it and skip
// upon seeing ._state = _TimerDisarmed. It will also be the timer loop
// to drop the reference to the timer that timer-wheel was holding.
canceled = true;
break;
default:
panic("invalid timer state");
}
if (canceled)
t._state = _TimerDisarmed;
// drain what _fire could have been queued already
while (t.c.len() > 0)
t.c.recv();
t._mu.unlock();
_tWheelMu->unlock();
return canceled;
}
void _Timer::reset(double dt) {
_Timer &t = *this;
// when timers are fired by _tWheel.advance(), they are first popped from _tWheel and put on
// _tFiring list, so that the real firing could be done without holding _tWheelMu.
static _TimerImpl* _tFiring = nil;
static _TimerImpl* _tFiringLast = nil;
void _TimerImpl::_queue_fire() {
_TimerImpl& t = *this;
t._mu.lock();
if (t._dt != INFINITY) {
t._mu.unlock();
panic("Timer.reset: the timer is armed; must be stopped or expired");
}
t._dt = dt;
t._ver += 1;
// TODO rework timers so that new timer does not spawn new goroutine.
Timer tref = newref(&t); // pass t reference to spawned goroutine
go([tref, dt](int ver) {
tref->_fire(dt, ver);
}, t._ver);
assert(t._state == _TimerArmed);
t._state = _TimerFiring;
t._mu.unlock();
t._tFiringNext = nil;
if (_tFiring == nil)
_tFiring = &t;
if (_tFiringLast != nil)
_tFiringLast->_tFiringNext = &t;
_tFiringLast = &t;
}
void _Timer::_fire(double dt, int ver) {
_Timer &t = *this;
static void _timer_loop_fire_queued() {
for (_TimerImpl* t = _tFiring; t != nil;) {
_TimerImpl* fnext = t->_tFiringNext;
t->_tFiringNext = nil;
t->_fire();
sleep(dt);
t._mu.lock();
if (t._ver != ver) {
t._mu.unlock();
return; // the timer was stopped/resetted - don't fire it
t->decref(); // wheel was holding a reference to the timer
t = fnext;
}
t._dt = INFINITY;
_tFiring = nil;
_tFiringLast = nil;
}
// send under ._mu so that .stop can be sure that if it sees
// ._dt = INFINITY, there is no ongoing .c send.
if (t._f == nil) {
t.c.send(now());
t._mu.unlock();
return;
void _TimerImpl::_fire() {
_TimerImpl& t = *this;
bool fire = false;
t._mu.lock();
if (t._state == _TimerFiring) { // stop could disarm the timer in the meantime
t._state = _TimerDisarmed;
fire = true;
debugf("LOOP: firing @ %lu ...\n", t._tWheelEntry.scheduled_at());
// send under ._mu so that .stop can be sure that if it sees
// ._state = _TimerDisarmed, there is no ongoing .c send.
if (t._f == nil)
t.c.send(now());
}
t._mu.unlock();
// call ._f not from under ._mu not to deadlock e.g. if ._f wants to reset the timer.
t._f();
if (fire && t._f != nil)
t._f();
}
}} // golang::time::
golang/time.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_TIME_H
#define _NXD_LIBGOLANG_TIME_H
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -118,12 +118,13 @@ private:
double _dt;
sync::Mutex _mu;
bool _stop;
Timer _timer;
// don't new - create only via new_ticker()
private:
_Ticker();
~_Ticker();
friend Ticker new_ticker(double dt);
friend Ticker LIBGOLANG_API new_ticker(double dt);
public:
LIBGOLANG_API void decref();
......@@ -147,18 +148,12 @@ LIBGOLANG_API Timer new_timer(double dt);
struct _Timer : object {
chan<double> c;
private:
func<void()> _f;
sync::Mutex _mu;
double _dt; // +inf - stopped, otherwise - armed
int _ver; // current timer was armed by n'th reset
// don't new - create only via new_timer() & co
private:
_Timer();
~_Timer();
friend Timer _new_timer(double dt, func<void()> f);
friend class _TimerImpl;
public:
LIBGOLANG_API void decref();
......@@ -182,9 +177,6 @@ public:
//
// the timer must be either already stopped or expired.
LIBGOLANG_API void reset(double dt);
private:
void _fire(double dt, int ver);
};
......
golang/time_test.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -20,9 +20,10 @@
from __future__ import print_function, absolute_import
from golang import select
from golang import time
from golang import select, func, defer
from golang import time, sync
from golang.golang_test import panics
from six.moves import range as xrange
# all timer tests operate in dt units
dt = 10*time.millisecond
......@@ -65,6 +66,7 @@ def test_ticker_time():
# test_timer verifies that Timer/Ticker fire as expected.
@func
def test_timer():
# start timers at x5, x7 and x11 intervals an verify that the timers fire
# in expected sequence. The times when the timers fire do not overlap in
......@@ -73,15 +75,15 @@ def test_timer():
tv = [] # timer events
Tstart = time.now()
t23 = time.Timer(23*dt)
t5 = time.Timer( 5*dt)
t23 = time.Timer(23*dt); defer(t23.stop)
t5 = time.Timer( 5*dt); defer(t5 .stop)
def _():
tv.append(7)
t7f.reset(7*dt)
t7f = time.Timer( 7*dt, f=_)
t7f = time.Timer( 7*dt, f=_); defer(t7f.stop)
tx11 = time.Ticker(11*dt)
tx11 = time.Ticker(11*dt); defer(tx11.stop)
while 1:
_, _rx = select(
......@@ -108,19 +110,20 @@ def test_timer():
# test_timer_misc, similarly to test_timer, verifies misc timer convenience functions.
@func
def test_timer_misc():
tv = []
Tstart = time.now()
c23 = time.after(23*dt)
c5 = time.after( 5*dt)
c23 = time.after(23*dt) # cannot stop
c5 = time.after( 5*dt) # cannot stop
def _():
tv.append(7)
t7f.reset(7*dt)
t7f = time.after_func(7*dt, _)
t7f = time.after_func(7*dt, _); defer(t7f.stop)
cx11 = time.tick(11*dt)
cx11 = time.tick(11*dt) # cannot stop
while 1:
_, _rx = select(
......@@ -148,13 +151,14 @@ def test_timer_misc():
# test_timer_stop verifies that .stop() cancels Timer or Ticker.
@func
def test_timer_stop():
tv = []
t10 = time.Timer (10*dt)
t2 = time.Timer ( 2*dt) # will fire and cancel t3, tx5
t3 = time.Timer ( 3*dt) # will be canceled
tx5 = time.Ticker( 5*dt) # will be canceled
t10 = time.Timer (10*dt); defer(t10.stop)
t2 = time.Timer ( 2*dt); defer(t2 .stop) # will fire and cancel t3, tx5
t3 = time.Timer ( 3*dt); defer(t3 .stop) # will be canceled
tx5 = time.Ticker( 5*dt); defer(tx5.stop) # will be canceled
while 1:
_, _rx = select(
......@@ -180,9 +184,10 @@ def test_timer_stop():
# test_timer_stop_drain verifies that Timer/Ticker .stop() drains timer channel.
@func
def test_timer_stop_drain():
t = time.Timer (1*dt)
tx = time.Ticker(1*dt)
t = time.Timer (1*dt); defer(t.stop)
tx = time.Ticker(1*dt); defer(tx.stop)
time.sleep(2*dt)
assert len(t.c) == 1
......@@ -195,9 +200,45 @@ def test_timer_stop_drain():
assert len(tx.c) == 0
# test_timer_stop_vs_func verifies that Timer .stop() works correctly with func-timer.
@func
def test_timer_stop_vs_func():
tv = []
def _1(): tv.append(1)
def _2(): tv.append(2)
t1 = time.after_func(1e6*dt, _1); defer(t1.stop)
t2 = time.after_func( 1*dt, _2); defer(t2.stop)
time.sleep(2*dt)
assert t1.stop() == True
assert t2.stop() == False
assert tv == [2]
# test_timer_reset_armed verifies that .reset() panics if called on armed timer.
@func
def test_timer_reset_armed():
# reset while armed
t = time.Timer(10*dt)
t = time.Timer(10*dt); defer(t.stop)
with panics("Timer.reset: the timer is armed; must be stopped or expired"):
t.reset(5*dt)
# bench_timer_arm_cancel benchmarks arming timers that do not fire.
# it shows how cheap or expensive it is to use timers to implement timeouts.
def bench_timer_arm_cancel(b):
for i in xrange(b.N):
t = time.Timer(10*time.second)
_ = t.stop()
assert _ is True
# bench_timer_arm_fire benchmarks arming timers that do fire.
# it shows what it costs to go through all steps related to timer loop and firing timers.
def bench_timer_arm_fire(b):
wg = sync.WaitGroup()
wg.add(b.N)
for i in xrange(b.N):
t = time.after_func(1*time.millisecond, wg.done)
wg.wait()
gpython/__init__.py
View file @ fdd73156
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -41,7 +41,7 @@ $GPYTHON_RUNTIME=threads.
from __future__ import print_function, absolute_import
_pyopt = "c:im:OVW:X:"
_pyopt = "c:Eim:OvVW:X:"
_pyopt_long = ('version',)
# pymain mimics `python ...`
......@@ -50,19 +50,27 @@ _pyopt_long = ('version',)
# init, if provided, is called after options are parsed, but before interpreter start.
def pymain(argv, init=None):
import sys
from os.path import dirname, realpath
import os
from os.path import dirname, realpath, splitext
# sys.executable
# on windows there are
# gpython-script.py
# gpython.exe
# gpython-script.py
# gpython.manifest
# and argv[0] is gpython-script.py
# with gpython-script.py sometimes embedded into gpython.exe (pip/distlib)
# and argv[0] is 'gpython-script.py' (setuptools) or 'gpython' (pip/distlib; note no .exe)
exe = realpath(argv[0])
argv = argv[1:]
if exe.endswith('-script.py'):
exe = exe[:-len('-script.py')]
exe = exe + '.exe'
if os.name == 'nt':
if exe.endswith('-script.py'):
exe = exe[:-len('-script.py')] # gpython-script.py -> gpython.exe
exe = exe + '.exe'
else:
_, ext = splitext(exe) # gpython -> gpython.exe
if not ext:
exe += '.exe'
sys._gpy_underlying_executable = sys.executable
sys.executable = exe
......@@ -70,20 +78,26 @@ def pymain(argv, init=None):
# `gpython file` will add path-to-file to sys.path[0] by itself, and
# /path/to/gpython is unnecessary and would create difference in behaviour
# in between gpython and python.
#
# on windows when gpython.exe comes with embedded __main__.py, it is
# gpython.exe that is installed into sys.path[0] .
exedir = dirname(exe)
if sys.path[0] == exedir:
if sys.path[0] in (exedir, exe):
del sys.path[0]
else:
# buildout injects `sys.path[0:0] = eggs` into python scripts.
# detect that and remove sys.path entry corresponding to exedir.
if not _is_buildout_script(exe):
raise RuntimeError('pymain: internal error: sys.path[0] was not set by underlying python to dirname(exe):'
raise RuntimeError('pymain: internal error: sys.path[0] was not set by underlying python to dirname(exe) or exe:'
'\n\n\texe:\t%s\n\tsys.path[0]:\t%s' % (exe, sys.path[0]))
else:
if exedir in sys.path:
sys.path.remove(exedir)
else:
raise RuntimeError('pymain: internal error: sys.path does not contain dirname(exe):'
ok = False
for _ in (exedir, exe):
if _ in sys.path:
sys.path.remove(_)
ok = True
if not ok:
raise RuntimeError('pymain: internal error: sys.path does not contain dirname(exe) or exe:'
'\n\n\texe:\t%s\n\tsys.path:\t%s' % (exe, sys.path))
......@@ -99,11 +113,24 @@ def pymain(argv, init=None):
for (opt, arg) in igetopt:
# options that require reexecuting through underlying python with that -<opt>
if opt in (
'-E', # ignore $PYTHON*
'-O', # optimize
'-v', # trace import statements
'-X', # set implementation-specific option
):
reexec_with.append(opt)
if arg is not None:
reexec_with.append(arg)
# but keep `-X gpython.*` in user part of argv in case of reexec
# leaving it for main to handle. If it is only pymain to run, then
# we will be ignoring `-X gpython.*` which goes in line with builtin
# py3 behaviour to ignore any unknown -X option.
if opt == '-X' and arg is not None and arg.startswith('gpython.'):
reexec_argv.append(opt)
reexec_argv.append(arg)
else:
reexec_with.append(opt)
if arg is not None:
reexec_with.append(arg)
continue
reexec_argv.append(opt)
......@@ -202,7 +229,6 @@ def pymain(argv, init=None):
#
# python -O gpython file.py
if len(reexec_with) > 0:
import os
argv = [sys._gpy_underlying_executable] + reexec_with + [sys.executable] + reexec_argv
os.execv(argv[0], argv)
......@@ -226,11 +252,12 @@ def pymain(argv, init=None):
pyimpl = platform.python_implementation()
v = _version_info_str
pyver = platform.python_version() # ~ v(sys.version_info) but might also have e.g. '+' at tail
if pyimpl == 'CPython':
ver.append('CPython %s' % v(sys.version_info))
ver.append('CPython %s' % pyver)
elif pyimpl == 'PyPy':
ver.append('PyPy %s' % v(sys.pypy_version_info))
ver.append('Python %s' % v(sys.version_info))
ver.append('Python %s' % pyver)
else:
ver = [] # unknown
......@@ -367,33 +394,35 @@ def main():
# no harm wrt gevent monkey-patching even if we import os first.
import os
# extract and process `-X gpython.*`
# process `-X gpython.*`
# -X gpython.runtime=(gevent|threads) + $GPYTHON_RUNTIME
sys._xoptions = getattr(sys, '_xoptions', {})
argv_ = []
gpy_runtime = os.getenv('GPYTHON_RUNTIME', 'gevent')
igetopt = _IGetOpt(sys.argv[1:], _pyopt, _pyopt_long)
for (opt, arg) in igetopt:
if opt == '-X':
if arg.startswith('gpython.'):
if arg.startswith('gpython.runtime='):
gpy_runtime = arg[len('gpython.runtime='):]
sys._xoptions['gpython.runtime'] = gpy_runtime
# any non gpython -X option is handled by pymain; ignore them here
if not arg.startswith('gpython.'):
continue
else:
raise RuntimeError('gpython: unknown -X option %s' % opt)
if arg.startswith('gpython.runtime='):
gpy_runtime = arg[len('gpython.runtime='):]
sys._xoptions['gpython.runtime'] = gpy_runtime
continue
else:
raise RuntimeError('gpython: unknown -X option %s' % arg)
argv_.append(opt)
if arg is not None:
argv_.append(arg)
continue
# options after -c / -m are not for python itself
if opt in ('-c', '-m'):
break
argv = [sys.argv[0]] + argv_ + igetopt.argv
# propagate those settings as defaults to subinterpreters, so that e.g.
# sys.executable spawned from under `gpython -X gpython.runtime=threads`
# also uses "threads" runtime by default.
os.environ['GPYTHON_RUNTIME'] = gpy_runtime
# init initializes according to selected runtime
# it is called after options are parsed and sys.path is setup correspondingly.
......@@ -434,18 +463,18 @@ def main():
sys.version += (' [GPython %s] [%s]' % (golang.__version__, gpy_verextra))
# tail to pymain
pymain(argv, init)
pymain(sys.argv, init)
# _is_buildout_script returns whether file @path is generated as python buildout script.
def _is_buildout_script(path):
with open(path, 'r') as f:
with open(path, 'rb') as f:
src = f.read()
# buildout injects the following prologues into python scripts:
# sys.path[0:0] = [
# ...
# ]
return ('\nsys.path[0:0] = [\n' in src)
return (b'\nsys.path[0:0] = [\n' in src)
# _IGetOpt provides getopt-style incremental options parsing.
......
gpython/gpython_test.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -184,7 +184,7 @@ def test_pymain():
_ = pyout(['-m', 'hello', 'abc', 'def'], cwd=testdata)
# realpath rewrites e.g. `local/lib -> lib` if local/lib is symlink
hellopy = realpath(join(testdata, 'hello.py'))
assert _ == b"hello\nworld\n['%s', 'abc', 'def']\n" % b(hellopy)
assert _ == b"hello\nworld\n[%s, 'abc', 'def']\n" % b(repr(hellopy))
# -m<module>
__ = pyout(['-mhello', 'abc', 'def'], cwd=testdata)
assert __ == _
......@@ -195,8 +195,8 @@ def test_pymain():
# -i after stdin (also tests interactive mode as -i forces interactive even on non-tty)
d = {
b'hellopy': b(hellopy),
b'ps1': b'' # cpython emits prompt to stderr
b'repr(hellopy)': b(repr(hellopy)),
b'ps1': b'' # cpython emits prompt to stderr
}
if is_pypy and not is_gpython:
d[b'ps1'] = b'>>>> ' # native pypy emits prompt to stdout and >>>> instead of >>>
......@@ -212,7 +212,7 @@ def test_pymain():
assert _ == b"hello\nworld\n['-c']\n%(ps1)s'~~HELLO~~'\n%(ps1)s" % d
# -i after -m
_ = pyout(['-i', '-m', 'hello'], stdin=b'world.tag', cwd=testdata)
assert _ == b"hello\nworld\n['%(hellopy)s']\n%(ps1)s'~~WORLD~~'\n%(ps1)s" % d
assert _ == b"hello\nworld\n[%(repr(hellopy))s]\n%(ps1)s'~~WORLD~~'\n%(ps1)s" % d
# -i after file
_ = pyout(['-i', 'testdata/hello.py'], stdin=b'tag', cwd=here)
assert _ == b"hello\nworld\n['testdata/hello.py']\n%(ps1)s'~~HELLO~~'\n%(ps1)s" % d
......@@ -221,26 +221,27 @@ def test_pymain():
# -W <opt>
_ = pyout(['-Werror', '-Whello', '-W', 'ignore::DeprecationWarning',
'testprog/print_warnings_setup.py'], cwd=here)
if PY2:
# py2 threading, which is imported after gpython startup, adds ignore
# for sys.exc_clear
_ = grepv(r'ignore:sys.exc_clear:DeprecationWarning:threading:*', _)
assert _.startswith(
b"sys.warnoptions: ['error', 'hello', 'ignore::DeprecationWarning']\n\n" + \
b"warnings.filters:\n" + \
b"- ignore::DeprecationWarning::*\n" + \
b"- error::Warning::*\n"), _
assert re.match(
br"sys\.warnoptions: \['error', 'hello', 'ignore::DeprecationWarning'\]\n\n"
br"warnings\.filters:\n"
br"(- [^\n]+\n)*" # Additional filters added by automatically imported modules
br"- ignore::DeprecationWarning::\*\n"
br"- error::Warning::\*\n"
br"(- [^\n]+\n)*", # Remaining filters
_,
)
# $PYTHONWARNINGS
_ = pyout(['testprog/print_warnings_setup.py'], cwd=here,
envadj={'PYTHONWARNINGS': 'ignore,world,error::SyntaxWarning'})
if PY2:
# see ^^^
_ = grepv(r'ignore:sys.exc_clear:DeprecationWarning:threading:*', _)
assert _.startswith(
b"sys.warnoptions: ['ignore', 'world', 'error::SyntaxWarning']\n\n" + \
b"warnings.filters:\n" + \
b"- error::SyntaxWarning::*\n" + \
b"- ignore::Warning::*\n"), _
assert re.match(
br"sys\.warnoptions: \['ignore', 'world', 'error::SyntaxWarning'\]\n\n"
br"warnings\.filters:\n"
br"(- [^\n]+\n)*" # Additional filters added by automatically imported modules
br"- error::SyntaxWarning::\*\n"
br"- ignore::Warning::\*\n"
br"(- [^\n]+\n)*", # Remaining filters
_,
)
def test_pymain_print_function_future():
......@@ -318,6 +319,60 @@ def test_pymain_opt():
check(["-O", "-O"])
check(["-O", "-O", "-O"])
# verify that pymain handles -E in exactly the same way as underlying python does.
@gpython_only
def test_pymain_E():
envadj = {'PYTHONOPTIMIZE': '1'}
def sys_flags_optimize(level):
return 'sys.flags.optimize: %s' % level
# without -E $PYTHONOPTIMIZE should be taken into account
def _(gpyoutv, stdpyoutv):
assert sys_flags_optimize(0) not in stdpyoutv
assert sys_flags_optimize(0) not in gpyoutv
assert sys_flags_optimize(1) in stdpyoutv
assert sys_flags_optimize(1) in gpyoutv
check_gpy_vs_py(['testprog/print_opt.py'], _, envadj=envadj, cwd=here)
# with -E not
def _(gpyoutv, stdpyoutv):
assert sys_flags_optimize(0) in stdpyoutv
assert sys_flags_optimize(0) in gpyoutv
assert sys_flags_optimize(1) not in stdpyoutv
assert sys_flags_optimize(1) not in gpyoutv
check_gpy_vs_py(['-E', 'testprog/print_opt.py'], _, envadj=envadj, cwd=here)
# verify that pymain handles -X non-gpython-option in exactly the same way as underlying python does.
@pytest.mark.skipif(PY2, reason="-X does not work at all on plain cpython2")
@gpython_only
def test_pymain_X():
check_gpy_vs_py(['testprog/print_faulthandler.py'], cwd=here)
check_gpy_vs_py(['-X', 'faulthandler', 'testprog/print_faulthandler.py'], cwd=here)
# pymain -v
@gpython_only
def test_pymain_v():
def nimport(argv, **kw):
argv = argv + ['testdata/hello.py']
kw.setdefault('cwd', here)
ret, out, err = _pyrun(argv, stdout=PIPE, stderr=PIPE, **kw)
assert ret == 0, (out, err)
n = 0
for _ in u(err).splitlines():
if _.startswith("import "):
n += 1
return n
# without -v there must be no "import ..." messages
assert nimport([]) == 0
assert nimport([], pyexe=sys._gpy_underlying_executable) == 0
# with -v there must be many "import ..." messages
assert nimport(['-v']) > 10
assert nimport(['-v'], pyexe=sys._gpy_underlying_executable) > 10
# pymain -V/--version
# gpython_only because output differs from !gpython.
......@@ -359,20 +414,31 @@ def test_pymain_run_via_relpath():
out2 = pyout(['./__init__.py'] + argv, pyexe=sys._gpy_underlying_executable, cwd=here)
assert out1 == out2
# verify -X gpython.runtime=...
@gpython_only
def test_Xruntime(runtime):
_xopt_assert_in_subprocess('gpython.runtime', runtime,
assert_gevent_activated if runtime != 'threads' else \
assert_gevent_not_activated)
# _xopt_assert_in_subprocess runs tfunc in subprocess interpreter spawned with
# `-X xopt=xval` and checks that there is no error.
#
# It is also verified that tfunc runs ok in sub-subprocess interpreter spawned
# _without_ `-X ...`, i.e. once given -X setting is inherited by spawned interpreters.
def _xopt_assert_in_subprocess(xopt, xval, tfunc):
XOPT = xopt.upper().replace('.','_') # gpython.runtime -> GPYTHON_RUNTIME
env = os.environ.copy()
env.pop('GPYTHON_RUNTIME', None) # del
env.pop(XOPT, None) # del
argv = []
if runtime != '':
argv += ['-X', 'gpython.runtime='+runtime]
prog = 'from gpython import gpython_test as t; '
if runtime != 'threads':
prog += 't.assert_gevent_activated(); '
else:
prog += 't.assert_gevent_not_activated(); '
if xval != '':
argv += ['-X', xopt+'='+xval]
prog = import_t = 'from gpython import gpython_test as t; '
prog += 't.%s(); ' % tfunc.__name__
prog += import_t # + same in subprocess
prog += "t.pyrun(['-c', '%s t.%s(); ']); " % (import_t, tfunc.__name__)
prog += 'print("ok")'
argv += ['-c', prog]
......@@ -382,20 +448,6 @@ def test_Xruntime(runtime):
# ---- misc ----
# grepv filters out lines matching pattern from text.
def grepv(pattern, text): # -> text
if isinstance(text, bytes):
t = b''
else:
t = ''
p = re.compile(pattern)
v = []
for l in text.splitlines(True):
m = p.search(l)
if not m:
v.append(l)
return t.join(v)
# check_gpy_vs_py verifies that gpython output matches underlying python output.
def check_gpy_vs_py(argv, postprocessf=None, **kw):
gpyout = u(pyout(argv, **kw))
......
gpython/testprog/print_faulthandler.py 0 → 100644
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""Program print_faulthandler prints information about faulthandler settings."""
from __future__ import print_function, absolute_import
import sys
def main():
if 'faulthandler' not in sys.modules:
print('faulthandler is not imported')
return
fh = sys.modules['faulthandler']
print('faulthandler imported')
print('faulthandler %s' % ('enabled' if fh.is_enabled() else 'disabled'))
if __name__ == '__main__':
main()
gpython/testprog/print_opt.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2020-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -61,9 +61,10 @@ def modpy_imports_from():
else:
raise AssertionError("module 'mod' is already there")
tmpd = tempfile.mkdtemp('', 'modpy_imports_from')
tmpd = tempfile.mkdtemp('', 'modpy_imports_from')
tmpd_ = tmpd + os.path.sep
try:
pymod = "%s/mod.py" % tmpd
pymod = tmpd_ + "mod.py"
with open(pymod, "w") as f:
f.write("# hello up there\n")
......@@ -73,9 +74,9 @@ def modpy_imports_from():
files = set()
for dirpath, dirnames, filenames in os.walk(tmpd):
for _ in filenames:
f = '%s/%s' % (dirpath, _)
if f.startswith(tmpd+'/'):
f = f[len(tmpd+'/'):]
f = os.path.join(dirpath, _)
if f.startswith(tmpd_):
f = f[len(tmpd_):]
files.add(f)
......
pyproject.toml
View file @ fdd73156
[build-system]
requires = ["setuptools", "wheel", "setuptools_dso >= 1.7", "cython", "gevent"]
requires = ["setuptools", "wheel", "setuptools_dso >= 2.8", "cython < 3", "gevent"]
setup.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# pygolang | pythonic package setup
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -19,28 +20,27 @@
# See https://www.nexedi.com/licensing for rationale and options.
from setuptools import find_packages
# setuptools has Library but this days it is not well supported and test for it
# has been killed https://github.com/pypa/setuptools/commit/654c26f78a30
# -> use setuptools_dso instead.
from setuptools_dso import DSO
from setuptools.command.install_scripts import install_scripts as _install_scripts
from setuptools.command.develop import develop as _develop
from distutils import sysconfig
from os.path import dirname, join
import sys, re
import sys, os, re
# read file content
def readfile(path):
with open(path, 'r') as f:
return f.read()
# reuse golang.pyx.build to build pygolang extensions.
def readfile(path): # -> str
with open(path, 'rb') as f:
data = f.read()
if not isinstance(data, str): # py3
data = data.decode('utf-8')
return data
# reuse golang.pyx.build to build pygolang dso and extensions.
# we have to be careful and inject synthetic golang package in order to be
# able to import golang.pyx.build without built/working golang.
trun = {}
exec(readfile('trun'), trun)
trun['ximport_empty_golangmod']()
from golang.pyx.build import setup, Extension as Ext
from golang.pyx.build import setup, DSO, Extension as Ext
# grep searches text for pattern.
......@@ -155,8 +155,8 @@ class develop(XInstallGPython, _develop):
# requirements of packages under "golang." namespace
R = {
'cmd.pybench': {'pytest'},
'pyx.build': {'setuptools', 'wheel', 'cython', 'setuptools_dso >= 1.7'},
'cmd.pybench': {'pytest', 'py ; python_version >= "3"'},
'pyx.build': {'setuptools', 'wheel', 'cython < 3', 'setuptools_dso >= 2.8'},
'x.perf.benchlib': {'numpy'},
}
# TODO generate `a.b -> a`, e.g. x.perf = join(x.perf.*); x = join(x.*)
......@@ -174,6 +174,14 @@ for k in sorted(R.keys()):
extras_require[k] = list(sorted(R[k]))
# get_python_libdir() returns path where libpython is located
def get_python_libdir():
# mimic what distutils.command.build_ext does
if os.name == 'nt':
return join(sysconfig.get_config_var('installed_platbase'), 'libs')
else:
return sysconfig.get_config_var('LIBDIR')
setup(
name = 'pygolang',
version = version,
......@@ -222,20 +230,21 @@ setup(
'golang/os/signal.h',
'golang/strings.h',
'golang/sync.h',
'golang/time.h'],
include_dirs = ['.', '3rdparty/include'],
'golang/time.h',
'3rdparty/ratas/src/timer-wheel.h'],
include_dirs = [
'3rdparty/include',
'3rdparty/ratas/src'],
define_macros = [('BUILDING_LIBGOLANG', None)],
extra_compile_args = ['-std=gnu++11'], # not c++11 as linux/list.h uses typeof
soversion = '0.1'),
DSO('golang.runtime.libpyxruntime',
['golang/runtime/libpyxruntime.cpp'],
depends = ['golang/pyx/runtime.h'],
include_dirs = ['.', sysconfig.get_python_inc()],
include_dirs = [sysconfig.get_python_inc()],
library_dirs = [get_python_libdir()],
define_macros = [('BUILDING_LIBPYXRUNTIME', None)],
extra_compile_args = ['-std=c++11'],
soversion = '0.1',
dsos = ['golang.runtime.libgolang'])],
soversion = '0.1')],
ext_modules = [
Ext('golang._golang',
......@@ -311,11 +320,15 @@ setup(
include_package_data = True,
install_requires = ['gevent', 'six', 'decorator', 'Importing;python_version<="2.7"',
# only runtime part: for dylink_prepare_dso
# also need to pin setuptools ≥ 60.2 because else wheel configures logging
# to go to stdout and so dylink_prepare_dso garbles program output
'setuptools_dso >= 2.8',
'setuptools >= 60.2 ; python_version>="3"',
# pyx.build -> setuptools_dso uses multiprocessing
# FIXME geventmp fails on python2, but setuptools_dso
# uses multiprocessing only on Python3, so for now we
# are ok. https://github.com/karellen/geventmp/pull/2
'geventmp;python_version>="3"',
# setuptools_dso uses multiprocessing only on Python3, and only on systems where
# mp.get_start_method()!='fork', while geventmp does not work on windows.
'geventmp ; python_version>="3" and platform_system != "Windows" ',
],
extras_require = extras_require,
......@@ -340,13 +353,18 @@ setup(
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.5
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: 3.11
Programming Language :: Python :: 3.12
Programming Language :: Python :: Implementation :: CPython
Programming Language :: Python :: Implementation :: PyPy
Operating System :: POSIX
Operating System :: POSIX :: Linux
Operating System :: Unix
Operating System :: MacOS
Operating System :: Microsoft :: Windows
Topic :: Software Development :: Interpreters
Topic :: Software Development :: Libraries :: Python Modules\
""".splitlines()]
......
tox.ini
View file @ fdd73156
[tox]
envlist =
{py27d,py27,py37,py38d,py38,py39d,py39,pypy,pypy3}-{thread,gevent}
{py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread,gevent}
# ThreadSanitizer
......@@ -10,44 +10,48 @@ envlist =
# (*) PyPy locks its GIL (see RPyGilAcquire) by manually doing atomic cmpxchg
# and other games, which TSAN cannot see if PyPy itself was not compiled with
# -fsanitize=thread.
{py27d,py27,py37,py38d,py38,py39d,py39 }-{thread }-tsan
{py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312 }-{thread }-tsan
# XXX py*-gevent-tsan would be nice to have, but at present TSAN is not
# effective with gevent, because it does not understand greenlet "thread"
# switching and so perceives the program as having only one thread where races
# are impossible. Disabled to save time.
# {py27d,py27,py37,py38d,py38,py39d,py39 }-{ gevent}-tsan
# {py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312 }-{ gevent}-tsan
# AddressSanitizer
# XXX asan does not work with gevent: https://github.com/python-greenlet/greenlet/issues/113
{py27d,py27,py37,py38d,py38,py39d,py39,pypy,pypy3}-{thread }-asan
{py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread }-asan
[testenv]
basepython =
py27d: python2.7-dbg
py27: python2.7
py37: python3.7
py38d: python3.8-dbg
py38: python3.8
py39d: python3.9-dbg
py39: python3.9
py310d: python3.10-dbg
py310: python3.10
py311d: python3.11-dbg
py311: python3.11
py312: python3.12
py312d: python3.12-dbg
pypy: pypy
pypy3: pypy3
setenv =
# distutils take CFLAGS for both C and C++.
# distutils use CFLAGS also at link stage -> we don't need to set LDFLAGS separately.
tsan: CFLAGS=-g -fsanitize=thread
asan: CFLAGS=-g -fsanitize=address
tsan: CFLAGS=-g -fsanitize=thread -fno-omit-frame-pointer
asan: CFLAGS=-g -fsanitize=address -fno-omit-frame-pointer
# XXX however distutils' try_link, which is used by numpy.distutils use only CC
# as linker without CFLAGS and _without_ LDFLAGS, which fails if *.o were
# compiled with -fsanitize=X and linked without that option. Work it around
# with also adjusting CC.
# XXX better arrange to pass CFLAGS to pygolang only, e.g. by adding --race or
# --sanitize=thread to `setup.py build_ext`.
tsan: CC=cc -fsanitize=thread
asan: CC=cc -fsanitize=address
tsan: CC=cc -fsanitize=thread -fno-omit-frame-pointer
asan: CC=cc -fsanitize=address -fno-omit-frame-pointer
# always compile pygolang from source and don't reuse binary pygolang wheels as
# we compile each case with different CFLAGS.
......@@ -69,5 +73,7 @@ commands=
# asan/tsan: tell pytest not to capture output - else it is not possible to see
# reports from sanitizers because they crash tested process on error.
# likewise for python debug builds.
asan,tsan,py{27,37}d: -s \
asan,tsan,py{27,39,310,311,312}d: -s \
gpython/ golang/
allowlist_externals={toxinidir}/trun
trun
View file @ fdd73156
#!/usr/bin/env python
# Copyright (C) 2019-2020 Nexedi SA and Contributors.
# -*- coding: utf-8 -*-
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -34,12 +35,13 @@ trun cares to run python with LD_PRELOAD set appropriately to /path/to/libtsan.s
from __future__ import print_function, absolute_import
import os, sys, re, subprocess, pkgutil
import warnings
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import imp
import os, os.path, sys, re, subprocess, platform, types
PY3 = (bytes is not str)
if PY3:
from importlib import machinery as imp_machinery
else:
import imp, pkgutil
# env_prepend prepends value to ${name} environment variable.
#
......@@ -64,12 +66,15 @@ def grep1(pattern, text): # -> re.Match|None
# to import e.g. golang.pyx.build, or locate golang._golang, without built/working golang.
def ximport_empty_golangmod():
assert 'golang' not in sys.modules
golang = imp.new_module('golang')
golang = types.ModuleType('golang')
golang.__package__ = 'golang'
golang.__path__ = ['golang']
golang.__file__ = 'golang/__init__.py'
golang.__loader__ = pkgutil.ImpLoader('golang', None, 'golang/__init__.py',
[None, None, imp.PY_SOURCE])
if PY3:
golang.__loader__ = imp_machinery.SourceFileLoader('golang', 'golang/__init__.py')
else:
golang.__loader__ = pkgutil.ImpLoader('golang', None, 'golang/__init__.py',
[None, None, imp.PY_SOURCE])
sys.modules['golang'] = golang
......@@ -83,6 +88,7 @@ def main():
# determine if _golang.so is linked to a sanitizer, and if yes, to which
# particular sanitizer DSO. Set LD_PRELOAD appropriately.
libxsan = None
ld_preload = None
if 'linux' in sys.platform:
p = subprocess.Popen(["ldd", _golang_so.path], stdout=subprocess.PIPE)
......@@ -123,7 +129,8 @@ def main():
_ = grep1("DYLD_INSERT_LIBRARIES=(.*)$", err)
if _ is not None:
ld_preload = ("DYLD_INSERT_LIBRARIES", _.group(1))
libxsan = _.group(1)
ld_preload = ("DYLD_INSERT_LIBRARIES", libxsan)
else:
print("trun %r: `import golang` failed with unexpected error:" % sys.argv[1:], file=sys.stderr)
print(err, file=sys.stderr)
......@@ -140,7 +147,7 @@ def main():
env_prepend("TSAN_OPTIONS", "halt_on_error=1")
env_prepend("ASAN_OPTIONS", "halt_on_error=1")
# tweak TSAN/ASAN defaults:
# tweak TSAN/ASAN/LSAN defaults:
# enable TSAN deadlock detector
# (unfortunately it caughts only few _potential_ deadlocks and actually
......@@ -148,15 +155,49 @@ def main():
env_prepend("TSAN_OPTIONS", "detect_deadlocks=1")
env_prepend("TSAN_OPTIONS", "second_deadlock_stack=1")
# many python allocations, whose lifetime coincides with python interpreter
# lifetime and which are not explicitly freed on python shutdown, are
# reported as leaks. Disable leak reporting to avoid huge non-pygolang
# related printouts.
env_prepend("ASAN_OPTIONS", "detect_leaks=0")
# tune ASAN to check more aggressively by default
env_prepend("ASAN_OPTIONS", "detect_stack_use_after_return=1")
# enable ASAN/LSAN leak detector.
#
# Do it only on CPython ≥ 3.11 because on py2 and on earlier py3 versions
# there are many many python allocations, whose lifetime coincide with
# python interpreter lifetime, and which are not explicitly freed on python
# shutdown. For py3 they significantly improved this step by step and
# starting from 3.11 it becomes practical to silence some still-leaks with
# suppressions, while for earlier py3 versions and especially for py2 it
# is, unfortunately, not manageable. Do not spend engineering time with
# activating LSAN on PyPy as that is tier 2 platform and bug tail history
# of memory leaks is very long even only on cpython.
if sys.version_info < (3,11):
env_prepend("ASAN_OPTIONS", "detect_leaks=0")
if libxsan is not None:
if 'asan' in libxsan.lower():
print("W: trun %r: asan: leak detection deactivated on %s %s" % (
sys.argv[1:], platform.python_implementation(), platform.python_version()),
file=sys.stderr)
else:
env_prepend("ASAN_OPTIONS", "detect_leaks=1")
env_prepend("LSAN_OPTIONS", "suppressions=%s" % os.path.abspath(os.path.join(
os.path.dirname(__file__), ".lsan-ignore.txt")))
# do not print statistics for suppressed leaks - else it breaks tests that verify program output
env_prepend("LSAN_OPTIONS", "print_suppressions=0")
# enable DWARF-based unwinding.
# else, if python is not compiled with -fno-omit-frame-pointer, it can show
# the whole traceback as e.g. just
# Direct leak of 32 byte(s) in 1 object(s) allocated from:
# #0 0x7f88522f3bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
# #1 0x55f910a3d9a4 in PyThread_allocate_lock Python/thread_pthread.h:385
# and our leak suppressions won't work.
# this is slower compared to default frame-pointer based unwinding, but
# still works reasonably timely when run with just tests.
env_prepend("ASAN_OPTIONS", "fast_unwind_on_malloc=0")
# leak suppression also needs full tracebacks to work correctly, since with
# python there are many levels of call nesting at C level, and to filter-out e.g.
# top-level PyImport_Import we need to go really deep.
env_prepend("ASAN_OPTIONS", "malloc_context_size=255")
# exec `...`
os.execvp(sys.argv[1], sys.argv[1:])
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7