Skip to content

pygolang
pygolang
Commit fdd73156 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

Sync with master

parents 300d7dfa 91a434d5
Expand all Show whitespace changes
Inline Side-by-side
Showing with 1931 additions and 540 deletions
.gitignore
View file @ fdd73156
......@@ -10,6 +10,8 @@ build/
*.so.*
*.dylib
*.dll
*.lib
*.exp
*.pyd
*_dsoinfo.py
......
.gitmodules 0 → 100644
View file @ fdd73156
[submodule "3rdparty/ratas"]
path = 3rdparty/ratas
url = https://github.com/jsnell/ratas.git
.lsan-ignore.txt 0 → 100644
View file @ fdd73156
# .lsan-ignore.txt lists memory leak events that LeakSanitizer should not
# report when running pygolang tests.
#
# Many python allocations, whose lifetime coincides with python interpreter
# lifetime, and which are not explicitly freed on python shutdown, are
# reported as leaks by default. Disable leak reporting for those to avoid
# non-pygolang related printouts.
# >>> Everything created when initializing python, e.g. sys.stderr
# #0 0x7f21e74f3bd7 in malloc .../asan_malloc_linux.cpp:69
# #1 0x555f361ff9a4 in PyThread_allocate_lock Python/thread_pthread.h:385
# #2 0x555f3623f72a in _buffered_init Modules/_io/bufferedio.c:725
# #3 0x555f3623ff7e in _io_BufferedWriter___init___impl Modules/_io/bufferedio.c:1803
# #4 0x555f3623ff7e in _io_BufferedWriter___init__ Modules/_io/clinic/bufferedio.c.h:489
# #5 0x555f3610c086 in type_call Objects/typeobject.c:1103
# #6 0x555f3609cdcc in _PyObject_MakeTpCall Objects/call.c:214
# #7 0x555f3609d6a8 in _PyObject_VectorcallTstate Include/internal/pycore_call.h:90
# #8 0x555f3609d6a8 in _PyObject_VectorcallTstate Include/internal/pycore_call.h:77
# #9 0x555f3609d6a8 in _PyObject_CallFunctionVa Objects/call.c:536
# #10 0x555f3609e89c in _PyObject_CallFunction_SizeT Objects/call.c:590
# #11 0x555f3623a0df in _io_open_impl Modules/_io/_iomodule.c:407
# #12 0x555f3623a0df in _io_open Modules/_io/clinic/_iomodule.c.h:264
# #13 0x555f360f17da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #14 0x555f3609d54c in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
# #15 0x555f3609d54c in _PyObject_CallFunctionVa Objects/call.c:536
# #16 0x555f3609ec34 in callmethod Objects/call.c:608
# #17 0x555f3609ec34 in _PyObject_CallMethod Objects/call.c:677
# #18 0x555f361e60cf in create_stdio Python/pylifecycle.c:2244
# #19 0x555f361e6523 in init_sys_streams Python/pylifecycle.c:2431
# #20 0x555f361e6523 in init_interp_main Python/pylifecycle.c:1154
# #21 0x555f361e7204 in pyinit_main Python/pylifecycle.c:1230
# #22 0x555f361e85ba in Py_InitializeFromConfig Python/pylifecycle.c:1261
# #23 0x555f3621010a in pymain_init Modules/main.c:67
# #24 0x555f362113de in pymain_main Modules/main.c:701
# #25 0x555f362113de in Py_BytesMain Modules/main.c:734
leak:^pymain_init$
# >>> Everything created when importing py modules, e.g.
# #0 0x7f18c86f3bd7 in malloc .../asan_malloc_linux.cpp:69
# #1 0x55b971430acf in PyMem_RawMalloc Objects/obmalloc.c:586
# #2 0x55b971430acf in _PyObject_Malloc Objects/obmalloc.c:2003
# #3 0x55b971430acf in _PyObject_Malloc Objects/obmalloc.c:1996
# #4 0x55b971415696 in new_keys_object Objects/dictobject.c:632
# #5 0x55b971415716 in dictresize Objects/dictobject.c:1429
# #6 0x55b97141961a in insertion_resize Objects/dictobject.c:1183
# #7 0x55b97141961a in insertdict Objects/dictobject.c:1248
# #8 0x55b97143eb7b in add_subclass Objects/typeobject.c:6547
# #9 0x55b97144ca52 in type_ready_add_subclasses Objects/typeobject.c:6345
# #10 0x55b97144ca52 in type_ready Objects/typeobject.c:6476
# #11 0x55b971451a1f in PyType_Ready Objects/typeobject.c:6508
# #12 0x55b971451a1f in type_new_impl Objects/typeobject.c:3189
# #13 0x55b971451a1f in type_new Objects/typeobject.c:3323
# #14 0x55b971443014 in type_call Objects/typeobject.c:1091
# #15 0x55b9713d3dcc in _PyObject_MakeTpCall Objects/call.c:214
# #16 0x55b9713d47bd in _PyObject_FastCallDictTstate Objects/call.c:141
# #17 0x55b9713d47bd in PyObject_VectorcallDict Objects/call.c:165
# #18 0x55b9714d14c2 in builtin___build_class__ Python/bltinmodule.c:209
# #19 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #20 0x55b9713d4a7b in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
# #21 0x55b9713d4a7b in PyObject_Vectorcall Objects/call.c:299
# #22 0x55b97137666e in _PyEval_EvalFrameDefault Python/ceval.c:4769
# #23 0x55b9714d7e6b in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
# #24 0x55b9714d7e6b in _PyEval_Vector Python/ceval.c:6434
# #25 0x55b9714d7e6b in PyEval_EvalCode Python/ceval.c:1148
# #26 0x55b9714d2e1f in builtin_exec_impl Python/bltinmodule.c:1077
# #27 0x55b9714d2e1f in builtin_exec Python/clinic/bltinmodule.c.h:465
# #28 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #29 0x55b971376dcb in do_call_core Python/ceval.c:7349
# #30 0x55b971376dcb in _PyEval_EvalFrameDefault Python/ceval.c:5376
# #31 0x55b9714d7faf in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
# #32 0x55b9714d7faf in _PyEval_Vector Python/ceval.c:6434
# #33 0x55b9713d436e in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
# #34 0x55b9713d436e in object_vacall Objects/call.c:819
# #35 0x55b9713d63cf in PyObject_CallMethodObjArgs Objects/call.c:879
# #36 0x55b9715080e1 in import_find_and_load Python/import.c:1748
# #37 0x55b9715080e1 in PyImport_ImportModuleLevelObject Python/import.c:1847
# #38 0x55b97137de9c in import_name Python/ceval.c:7422
# #39 0x55b97137de9c in _PyEval_EvalFrameDefault Python/ceval.c:3946
# #40 0x55b9714d7e6b in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
# #41 0x55b9714d7e6b in _PyEval_Vector Python/ceval.c:6434
# #42 0x55b9714d7e6b in PyEval_EvalCode Python/ceval.c:1148
# #43 0x55b9714d2e1f in builtin_exec_impl Python/bltinmodule.c:1077
# #44 0x55b9714d2e1f in builtin_exec Python/clinic/bltinmodule.c.h:465
# #45 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #46 0x55b971376dcb in do_call_core Python/ceval.c:7349
# #47 0x55b971376dcb in _PyEval_EvalFrameDefault Python/ceval.c:5376
leak:^PyImport_Import
# importlib.import_module leads to
# #0 0x7f1951ef3bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
# #1 0x55f399e8cacf in PyMem_RawMalloc Objects/obmalloc.c:586
# #2 0x55f399e8cacf in _PyObject_Malloc Objects/obmalloc.c:2003
# #3 0x55f399e8cacf in _PyObject_Malloc Objects/obmalloc.c:1996
# #4 0x55f399e86344 in PyModule_ExecDef Objects/moduleobject.c:400
# #5 0x55f399f6178a in exec_builtin_or_dynamic Python/import.c:2345
# #6 0x55f399f6178a in _imp_exec_dynamic_impl Python/import.c:2419
# #7 0x55f399f6178a in _imp_exec_dynamic Python/clinic/import.c.h:474
# #8 0x55f399e8438a in cfunction_vectorcall_O Objects/methodobject.c:514
leak:^_imp_exec_dynamic
# >>> Everything allocated at DSO initialization, e.g.
# #0 0x7f35d2af46c8 in operator new(unsigned long) .../asan_new_delete.cpp:95
# #1 0x7f35ce897e9f in __static_initialization_and_destruction_0 golang/context.cpp:61
# #2 0x7f35ce8982ef in _GLOBAL__sub_I_context.cpp golang/context.cpp:380
# #3 0x7f35d32838bd in call_init elf/dl-init.c:90
# #4 0x7f35d32838bd in call_init elf/dl-init.c:27
# #5 0x7f35d32839a3 in _dl_init elf/dl-init.c:137
# #6 0x7f35d256e023 in __GI__dl_catch_exception elf/dl-error-skeleton.c:182
# #7 0x7f35d328a09d in dl_open_worker elf/dl-open.c:808
# #8 0x7f35d256dfc9 in __GI__dl_catch_exception elf/dl-error-skeleton.c:208
# #9 0x7f35d328a437 in _dl_open elf/dl-open.c:884
# #10 0x7f35d24a4437 in dlopen_doit dlfcn/dlopen.c:56
# #11 0x7f35d256dfc9 in __GI__dl_catch_exception elf/dl-error-skeleton.c:208
# #12 0x7f35d256e07e in __GI__dl_catch_error elf/dl-error-skeleton.c:227
# #13 0x7f35d24a3f26 in _dlerror_run dlfcn/dlerror.c:138
# #14 0x7f35d24a44e8 in dlopen_implementation dlfcn/dlopen.c:71
# #15 0x7f35d24a44e8 in ___dlopen dlfcn/dlopen.c:81
# #16 0x7f35d2a77ff9 in dlopen .../sanitizer_common_interceptors.inc:6341
leak:^_GLOBAL_
# global<> does not deallocate its reference on purpose
leak:^_test_global()$
ratas @ becd5fc5
Subproject commit becd5fc5c1e9ea600cd8b3b1c24d564794fedac4
MANIFEST.in
View file @ fdd73156
include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .nxdtest
include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .lsan-ignore.txt .nxdtest conftest.py
include golang/libgolang.h
include golang/runtime/libgolang.cpp
include golang/runtime/libpyxruntime.cpp
include golang/pyx/runtime.h
include golang/pyx/runtime.cpp
include golang/pyx/testprog/golang_dso_user/dsouser/dso.h
include golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
include golang/runtime/internal.h
include golang/runtime/internal/atomic.h
......@@ -34,6 +34,9 @@ include golang/sync_test.cpp
include golang/time.h
include golang/time.cpp
include golang/_testing.h
include golang/_compat/windows/strings.h
include golang/_compat/windows/unistd.h
recursive-include golang *.py *.pxd *.pyx *.toml *.txt*
recursive-include gpython *.py
recursive-include 3rdparty *.h
recursive-exclude golang *_dsoinfo.py
conftest.py 0 → 100644
View file @ fdd73156
# pygolang | pytest config
# Copyright (C) 2021-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from __future__ import print_function, absolute_import
import gc
# Do full GC before pytest exits, to avoid false positives in the leak detector.
def pytest_unconfigure():
gc.collect()
golang/__init__.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -42,6 +42,9 @@ from golang._gopath import gimport # make gimport available from golang
import inspect, sys
import decorator, six
import setuptools_dso
setuptools_dso.dylink_prepare_dso('golang.runtime.libgolang')
from golang._golang import _pysys_exc_clear as _sys_exc_clear
# @func is a necessary decorator for functions for selected golang features to work.
......@@ -70,10 +73,25 @@ def _meth(cls, fcall):
# wrap f with @_func, so that e.g. defer works automatically.
f = _func(f)
if isinstance(f, (staticmethod, classmethod)):
func_name = f.__func__.__name__
# property is special - it has up to 3 functions inside
# but the only practical case is when it starts with
#
# @func(Class)
# @property
# def ...
#
# which means that .fget should be set and so we use get name as the
# name of the method.
f_ = f
if isinstance(f, property):
f_ = f.fget
if f_ is None:
raise ValueError("func(cls) used on property without getter")
if isinstance(f_, (staticmethod, classmethod)):
func_name = f_.__func__.__name__
else:
func_name = f.__name__
func_name = f_.__name__
setattr(cls, func_name, f)
# if `@func(cls) def name` caller already has `name` set, don't override it
......@@ -82,28 +100,57 @@ def _meth(cls, fcall):
if already is not missing:
return already
# FIXME try to arrange so that python does not set anything on caller's
# namespace[func_name] (currently it sets that to implicitly returned None)
# arrange so that python eventually does not set anything on caller's
# namespace[func_name] (it unconditionally sets what decorator returns, even implicit None)
#
# _DelAttrAfterMeth.__del__ is invoked:
# * on cpython: right after namespace[func_name] = returned _meth_leftover
# * on pypy: eventually on next GC
fcall.f_locals[func_name] = _DelAttrAfterMeth(fcall.f_locals, func_name)
return _meth_leftover
return deco
# _DelAttrAfterMeth serves _meth by unsetting f_locals[meth] that python
# unconditionally sets after `@func(cls) def meth()`.
_meth_leftover = object()
class _DelAttrAfterMeth(object):
def __init__(self, f_locals, name):
self.f_locals = f_locals
self.name = name
def __del__(self):
obj = self.f_locals.get(self.name)
if obj is _meth_leftover:
del self.f_locals[self.name]
# _func serves @func.
def _func(f):
# @staticmethod & friends require special care:
# @property is special: there are 3 functions inside and we need to wrap
# them all with repacking back into property.
if isinstance(f, property):
fget = fset = fdel = None
if f.fget is not None:
fget = _func(f.fget)
if f.fset is not None:
fset = _func(f.fset)
if f.fdel is not None:
fdel = _func(f.fdel)
return type(f)(fget, fset, fdel, f.__doc__)
# @staticmethod & friends also require special care:
# unpack f first to original func and then repack back after wrapping.
fclass = None
if isinstance(f, (staticmethod, classmethod)):
fclass = type(f)
f = f.__func__
def _(f, *argv, **kw):
# run f under separate frame, where defer will register calls.
__goframe__ = _GoFrame()
with __goframe__:
return f(*argv, **kw)
# keep all f attributes, like __name__, __doc__, etc on _
_ = decorator.decorate(f, _)
# prepare function that runs f under separate frame, where defer will register calls
# keep all f attributes, like __name__, __doc__, etc on the wrapper
# if f was already wrapped with _func - no need to wrap it again
_ = f
if getattr(f, '__go_wrapper__', None) is not _goframe:
_ = decorator.decorate(f, _goframe)
_.__go_wrapper__ = _goframe
# repack _ into e.g. @staticmethod if that was used on f.
if fclass is not None:
......@@ -111,7 +158,13 @@ def _func(f):
return _
# _GoFrame serves __goframe__ that is setup by @func.
# _goframe is used by @func to run f under separate frame.
def _goframe(f, *argv, **kw):
__goframe__ = _GoFrame()
with __goframe__:
return f(*argv, **kw)
# _GoFrame serves __goframe__ that is setup by _goframe.
class _GoFrame:
def __init__(self):
self.deferv = [] # defer registers funcs here
......
golang/_compat/windows/strings.h 0 → 100644
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
#define _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
//
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
#include <string.h>
static inline void bzero(void *p, size_t n) {
memset(p, '\0', n);
}
#endif // _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
golang/_compat/windows/unistd.h 0 → 100644
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
#define _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
//
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// stub unistd.h to be used on Windows where it is absent.
// we need this because e.g. `cimport posix.stat` forces inclusion of unistd.h
// even if we use part of posix.stat that is available everywhere.
#include <io.h>
#define O_CLOEXEC _O_NOINHERIT
#endif // _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
golang/_golang.pyx
View file @ fdd73156
......@@ -5,7 +5,7 @@
# distutils: language = c++
# distutils: depends = libgolang.h os/signal.h _golang_str.pyx
#
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -173,9 +173,16 @@ cdef void __goviac(void *arg) nogil:
# ---- channels ----
# _frompyx indicates that a constructor is called from pyx code
cdef object _frompyx = object()
@final
cdef class pychan:
def __cinit__(pychan pych, size=0, dtype=object):
if dtype is _frompyx:
pych.dtype = DTYPE_STRUCTZ # anything
pych._ch = NULL
else:
pych.dtype = parse_dtype(dtype)
pych._ch = _makechan_pyexc(dtypeRegistry[<int>pych.dtype].size, size)
......@@ -370,7 +377,7 @@ cdef void pychan_asserttype(pychan pych, DType dtype) nogil:
panic("pychan: channel type mismatch")
cdef pychan pychan_from_raw(_chan *_ch, DType dtype):
cdef pychan pych = pychan.__new__(pychan)
cdef pychan pych = pychan.__new__(pychan, dtype=_frompyx)
pych.dtype = dtype
pych._ch = _ch; _chanxincref(_ch)
return pych
......@@ -469,7 +476,7 @@ def pyselect(*pycasev):
casev[i].user = pych.dtype
with nogil:
selected = _chanselect_pyexc(&casev[0], casev.size())
selected = _chanselect_pyexc(casev.data(), casev.size())
finally:
# decref not sent tx (see ^^^ send prepare)
......@@ -626,9 +633,7 @@ cdef object c_to_py(DType dtype, const chanElemBuf *cfrom):
# mkpynil creates pychan instance that represents nil[dtype].
cdef PyObject *mkpynil(DType dtype):
cdef pychan pynil = pychan.__new__(pychan)
pynil.dtype = dtype
pynil._ch = NULL # should be already NULL
cdef pychan pynil = pychan_from_raw(NULL, dtype)
Py_INCREF(pynil)
return <PyObject *>pynil
......@@ -818,9 +823,6 @@ from libcpp.typeinfo cimport type_info
from cython.operator cimport typeid
from libc.string cimport strcmp
# _frompyx indicates that a constructor is called from pyx code
cdef object _frompyx = object()
cdef class pyerror(Exception):
# pyerror <- error
@staticmethod
......
golang/_golang_str.pyx
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -1791,8 +1791,9 @@ cdef (int, int) _utf8_decode_rune(const uint8_t[::1] s):
if _ucs2_build and len(r) == 2:
try:
return _xuniord(r), l
# e.g. TypeError: ord() expected a character, but string of length 2 found
except TypeError:
# py: TypeError: ord() expected a character, but string of length 2 found
# cy: ValueError: only single character unicode strings can be converted to Py_UCS4, got length 2
except (TypeError, ValueError):
l -= 1
continue
......
golang/_golang_test.pyx
View file @ fdd73156
......@@ -2,7 +2,7 @@
# cython: language_level=2
# distutils: language=c++
#
# Copyright (C) 2018-2020 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -344,6 +344,25 @@ cdef nogil:
pych.chan_double().close()
# verify that pychan_from_raw is not leaking C channel.
def test_pychan_from_raw_noleak():
# pychan_from_raw used to create another channel and leak it
#
# this test _implicitly_ verifies that it is no longer the case - if it is,
# LSAN will report a memory leak after running the test.
#
# TODO consider adding explicit verification effective even under regular
# builds. Possible options:
#
# * verify malloc totals before and after tested code
# see e.g. https://stackoverflow.com/q/1761125/9456786
# * hook _makechan and verify that it is not invoked from under
# pychan_from_raw. Depends on funchook integration.
cdef chan[int] ch = makechan[int]()
cdef pychan pych = pychan.from_chan_int(ch) # uses pychan_from_raw internally
# pych and ch are freed automatically
# ---- benchmarks ----
# bench_go_nogil mirrors golang_test.py:bench_go
......
golang/_gopath.py
View file @ fdd73156
# Copyright (C) 2018-2019 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -34,11 +34,7 @@ from __future__ import print_function, absolute_import
import os, os.path
import sys
import warnings
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import imp
import six
# _gopathv returns $GOPATH vector.
def _gopathv():
......@@ -51,11 +47,25 @@ def _gopathv():
# gimport imports python module or package from fully-qualified module name under $GOPATH.
def gimport(name):
imp.acquire_lock()
_gimport_lock()
try:
return _gimport(name)
finally:
imp.release_lock()
_gimport_unlock()
# on py2 there is global import lock
# on py3 we need to organize our own gimport synchronization
if six.PY2:
import imp
_gimport_lock = imp.acquire_lock
_gimport_unlock = imp.release_lock
else:
from importlib import machinery as imp_machinery
from importlib import util as imp_util
from golang import sync
_gimport_mu = sync.Mutex()
_gimport_lock = _gimport_mu.lock
_gimport_unlock = _gimport_mu.unlock
def _gimport(name):
# we will register imported module into sys.modules with adjusted path.
......@@ -93,4 +103,16 @@ def _gimport(name):
# https://stackoverflow.com/a/67692
return _imp_load_source(modname, modpath)
def _imp_load_source(modname, modpath):
if six.PY2:
return imp.load_source(modname, modpath)
# https://docs.python.org/3/whatsnew/3.12.html#imp
loader = imp_machinery.SourceFileLoader(modname, modpath)
spec = imp_util.spec_from_file_location(modname, modpath, loader=loader)
mod = imp_util.module_from_spec(spec)
sys.modules[modname] = mod
loader.exec_module(mod)
return mod
golang/fmt.cpp
View file @ fdd73156
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -48,7 +48,7 @@ string _vsprintf(const char *format, va_list argp) {
return string(buf.get(), buf.get() + nchar); // without trailing '\0'
}
string sprintf(const string &format, ...) {
string sprintf(const string format, ...) {
va_list argp;
va_start(argp, format);
string str = fmt::_vsprintf(format.c_str(), argp);
......@@ -64,7 +64,7 @@ string sprintf(const char *format, ...) {
return str;
}
error ___errorf(const string &format, ...) {
error ___errorf(const string format, ...) {
va_list argp;
va_start(argp, format);
error err = errors::New(fmt::_vsprintf(format.c_str(), argp));
......
golang/fmt.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_FMT_H
#define _NXD_LIBGOLANG_FMT_H
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -40,7 +40,7 @@ namespace golang {
namespace fmt {
// sprintf formats text into string.
LIBGOLANG_API string sprintf(const string &format, ...);
LIBGOLANG_API string sprintf(const string format, ...);
// intseq<i1, i2, ...> and intrange<n> are used by errorf to handle %w.
......@@ -75,7 +75,7 @@ namespace {
//
// format suffix ": %w" is additionally handled as in Go with
// `errorf("... : %w", ..., err)` creating error that can be unwrapped back to err.
LIBGOLANG_API error ___errorf(const string& format, ...);
LIBGOLANG_API error ___errorf(const string format, ...);
LIBGOLANG_API error ___errorfTryWrap(const string& format, error last_err, ...);
LIBGOLANG_API string ___error_str(error err);
......@@ -111,7 +111,10 @@ inline error errorf(const string& format, Argv... argv) {
// `const char *` overloads just to catch format mistakes as
// __attribute__(format) does not work with std::string.
LIBGOLANG_API string sprintf(const char *format, ...)
__attribute__ ((format (printf, 1, 2)));
#ifndef _MSC_VER
__attribute__ ((format (printf, 1, 2)))
#endif
;
// cannot use __attribute__(format) for errorf as we add %w handling.
// still `const char *` overload is useful for performance.
......
golang/golang_test.py
View file @ fdd73156
This diff is collapsed.
golang/libgolang.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_H
#define _NXD_LIBGOLANG_H
// Copyright (C) 2018-2022 Nexedi SA and Contributors.
// Copyright (C) 2018-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -176,6 +176,12 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#ifdef _MSC_VER // no mode_t on msvc
typedef int mode_t;
#endif
// DSO symbols visibility (based on https://gcc.gnu.org/wiki/Visibility)
#if defined _WIN32 || defined __CYGWIN__
#define LIBGOLANG_DSO_EXPORT __declspec(dllexport)
......@@ -340,8 +346,13 @@ typedef struct _libgolang_runtime_ops {
// previously successfully allocated via sema_alloc.
void (*sema_free) (_libgolang_sema*);
// sema_acquire/sema_release should acquire/release live semaphore allocated via sema_alloc.
void (*sema_acquire)(_libgolang_sema*);
// sema_acquire should try to acquire live semaphore allocated via sema_alloc during given time.
// it returns whether acquisition succeeded or timed out.
// the timeout is specified in nanoseconds.
// UINT64_MAX means no timeout.
bool (*sema_acquire)(_libgolang_sema*, uint64_t timeout_ns);
// sema_release should release live semaphore allocated via sema_alloc.
void (*sema_release)(_libgolang_sema*);
// nanosleep should pause current goroutine for at least dt nanoseconds.
......@@ -577,7 +588,7 @@ int select(const _selcase (&casev)[N]) {
static inline // select(vector<casev>)
int select(const std::vector<_selcase> &casev) {
return _chanselect(&casev[0], casev.size());
return _chanselect(casev.data(), casev.size());
}
// defer(f) mimics `defer f()` from golang.
......@@ -829,7 +840,7 @@ struct _interface {
protected:
// don't use destructor -> use decref
~_interface();
LIBGOLANG_API ~_interface();
};
typedef refptr<_interface> interface;
......
golang/os.cpp
View file @ fdd73156
// Copyright (C) 2019-2022 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -37,7 +37,8 @@
// GLIBC < 2.32 provides sys_siglist but not sigdescr_np in its headers
// cut this short
// (on darwing sys_siglist declaration is normally provided)
#ifndef __APPLE__
// (on windows sys_siglist is not available at all)
#if !(defined(__APPLE__) || defined(_WIN32))
extern "C" {
extern const char * const sys_siglist[];
}
......@@ -226,8 +227,8 @@ tuple<string, error> ReadFile(const string& path) {
while (1) {
int n;
tie(n, err) = f->Read(&buf[0], buf.size());
data.append(&buf[0], n);
tie(n, err) = f->Read(buf.data(), buf.size());
data.append(buf.data(), n);
if (err != nil) {
if (err == io::EOF_)
err = nil;
......@@ -248,7 +249,7 @@ tuple<string, error> ReadFile(const string& path) {
tuple<File, File, error> Pipe() {
int vfd[2], syserr;
syserr = sys::Pipe(vfd);
syserr = sys::Pipe(vfd, 0);
if (syserr != 0)
return make_tuple(nil, nil, fmt::errorf("pipe: %w", sys::NewErrno(syserr)));
......@@ -286,8 +287,20 @@ string Signal::String() const {
const Signal& sig = *this;
const char *sigstr = nil;
#ifdef _WIN32
switch (sig.signo) {
case SIGABRT: return "Aborted";
case SIGBREAK: return "Break";
case SIGFPE: return "Floating point exception";
case SIGILL: return "Illegal instruction";
case SIGINT: return "Interrupt";
case SIGSEGV: return "Segmentation fault";
case SIGTERM: return "Terminated";
}
#else
if (0 <= sig.signo && sig.signo < NSIG)
sigstr = ::sys_siglist[sig.signo]; // might be nil as well
#endif
if (sigstr != nil)
return string(sigstr);
......
golang/os.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_OS_H
#define _NXD_LIBGOLANG_OS_H
//
// Copyright (C) 2019-2022 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -61,7 +61,7 @@ private:
~_File();
friend File _newFile(_libgolang_ioh* ioh, const string& name);
public:
void decref();
LIBGOLANG_API void decref();
public:
LIBGOLANG_API string Name() const;
......@@ -95,9 +95,15 @@ private:
// Open opens file @path.
LIBGOLANG_API std::tuple<File, error> Open(const string &path, int flags = O_RDONLY,
mode_t mode = S_IRUSR | S_IWUSR | S_IXUSR |
mode_t mode =
#if !defined(_MSC_VER)
S_IRUSR | S_IWUSR | S_IXUSR |
S_IRGRP | S_IWGRP | S_IXGRP |
S_IROTH | S_IWOTH | S_IXOTH);
S_IROTH | S_IWOTH | S_IXOTH
#else
_S_IREAD | _S_IWRITE
#endif
);
// NewFile wraps OS-level file-descriptor into File.
// The ownership of sysfd is transferred to File.
......
golang/os/signal.cpp
View file @ fdd73156
// Copyright (C) 2021-2022 Nexedi SA and Contributors.
// Copyright (C) 2021-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -89,6 +89,10 @@
#include <atomic>
#include <tuple>
#if defined(_WIN32)
# include <windows.h>
#endif
#define DEBUG 0
#if DEBUG
......@@ -97,6 +101,12 @@
# define debugf(format, ...) do {} while (0)
#endif
#if defined(_MSC_VER)
# define HAVE_SIGACTION 0
#else
# define HAVE_SIGACTION 1
#endif
// golang::os::signal::
namespace golang {
namespace os {
......@@ -109,13 +119,30 @@ using std::tie;
using std::vector;
using cxx::set;
#if HAVE_SIGACTION
static void xsigemptyset(sigset_t *sa_mask);
#else
// custom `struct sigaction` emulated via signal(2)
#define SA_SIGINFO 1
typedef struct {} siginfo_t;
struct sigaction {
union {
void (*sa_handler)(int);
void (*sa_sigaction)(int, siginfo_t*, void*);
};
int sa_flags;
};
static void _os_sighandler_nosigaction(int sig);
#endif
static void _os_sighandler(int sig, siginfo_t *info, void *ucontext);
static void _notify(int signo);
static void _checksig(int signo);
static void _checkActEqual(const struct sigaction *a, const struct sigaction *b);
static void _spinwaitNextQueueCycle();
static int sys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact);
static void xsys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact);
static void xsigemptyset(sigset_t *sa_mask);
static bool _sigact_equal(const struct sigaction *a, const struct sigaction *b);
......@@ -160,27 +187,31 @@ void _init() {
// create _wakerx <-> _waketx pipe; set _waketx to nonblocking mode
int vfd[2];
if (sys::Pipe(vfd) < 0)
if (sys::Pipe(vfd, O_CLOEXEC) < 0)
panic("pipe(_wakerx, _waketx)"); // TODO +syserr
if (sys::Fcntl(vfd[0], F_SETFD, FD_CLOEXEC) < 0)
panic("fcntl(_wakerx, FD_CLOEXEC)"); // TODO +syserr
error err;
tie(_wakerx, err) = os::NewFile(vfd[0], "_wakerx");
if (err != nil)
panic("os::newFile(_wakerx");
_waketx = vfd[1];
#ifndef _WIN32
if (sys::Fcntl(_waketx, F_SETFL, O_NONBLOCK) < 0)
panic("fcntl(_waketx, O_NONBLOCK)"); // TODO +syserr
if (sys::Fcntl(_waketx, F_SETFD, FD_CLOEXEC) < 0)
panic("fcntl(_waketx, FD_CLOEXEC)"); // TODO +syserr
#else
HANDLE hwaketx = (HANDLE)_get_osfhandle(_waketx);
DWORD mode = PIPE_READMODE_BYTE | PIPE_NOWAIT;
if (!SetNamedPipeHandleState(hwaketx, &mode, NULL, NULL))
panic("SetNamedPipeHandleState(hwaketx, PIPE_NOWAIT)"); // TODO +syserr
#endif
_actIgnore.sa_handler = SIG_IGN;
_actIgnore.sa_flags = 0;
xsigemptyset(&_actIgnore.sa_mask);
_actNotify.sa_sigaction = _os_sighandler;
_actNotify.sa_flags = SA_SIGINFO;
#if HAVE_SIGACTION
xsigemptyset(&_actIgnore.sa_mask);
xsigemptyset(&_actNotify.sa_mask);
#endif
}
......@@ -248,6 +279,14 @@ done:
if (sah != SIG_IGN) {
if (sah != SIG_DFL) {
sah(sig);
// handlers installed via signal usually reinstall themselves
// return it back to us
struct sigaction after;
xsys_sigaction(sig, &_actNotify, &after);
if (! (_sigact_equal(&after, &_actNotify) ||
( ((after.sa_flags & SA_SIGINFO) == 0) &&
((after.sa_handler == SIG_DFL) || (after.sa_handler == sah)))))
panic("collision detected wrt thirdparty signal usage");
}
else {
// SIG_DFL && _SigReset - reraise to die if the signal is fatal
......@@ -343,7 +382,7 @@ static int/*syserr*/ _Notify1(chan<os::Signal> ch, os::Signal sig) {
// retrieve current signal action
struct sigaction cur;
int syserr = sys::Sigaction(sig.signo, nil, &cur);
int syserr = sys_sigaction(sig.signo, nil, &cur);
if (syserr < 0) {
// TODO reenable once we can panic with any object
//return fmt::errorf("sigaction sig%d: %w", sig.signo, sys::NewErrno(syserr);
......@@ -389,7 +428,7 @@ static int/*syserr*/ _Notify1(chan<os::Signal> ch, os::Signal sig) {
// register our sigaction
struct sigaction old;
syserr = sys::Sigaction(sig.signo, &_actNotify, &old);
syserr = sys_sigaction(sig.signo, &_actNotify, &old);
if (syserr < 0) {
// TODO reenable once we can panic with any object
//return fmt::errorf("sigaction sig%d: %w", sig.signo, sys::NewErrno(syserr);
......@@ -469,7 +508,7 @@ static int/*syserr*/ _Ignore1(os::Signal sig) {
if (h == nil) {
h = new _SigHandler();
h->sigstate.store(_SigIgnoring);
int syserr = sys::Sigaction(sig.signo, nil, &h->prev_act);
int syserr = sys_sigaction(sig.signo, nil, &h->prev_act);
if (syserr < 0) {
delete h;
return syserr; // TODO errctx
......@@ -481,7 +520,7 @@ static int/*syserr*/ _Ignore1(os::Signal sig) {
h->sigstate.store(_SigIgnoring);
h->subscribers.clear();
int syserr = sys::Sigaction(sig.signo, &_actIgnore, nil);
int syserr = sys_sigaction(sig.signo, &_actIgnore, nil);
if (syserr < 0)
return syserr; // TODO errctx
......@@ -509,7 +548,7 @@ static int/*syserr*/ _Reset1(os::Signal sig) {
h->sigstate.store(_SigReset);
struct sigaction act;
int syserr = sys::Sigaction(sig.signo, &h->prev_act, &act);
int syserr = sys_sigaction(sig.signo, &h->prev_act, &act);
if (syserr < 0)
return syserr; // TODO errctx
if (sigstate == _SigNotifying)
......@@ -617,16 +656,75 @@ static void _checksig(int signo) {
}
#if HAVE_SIGACTION
static void xsigemptyset(sigset_t *sa_mask) {
if (sigemptyset(sa_mask) < 0)
panic("sigemptyset failed"); // must always succeed
}
#endif
static void xsys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact) {
int syserr = sys::Sigaction(signo, act, oldact);
int syserr = sys_sigaction(signo, act, oldact);
if (syserr != 0)
panic("sigaction failed"); // TODO add errno detail
}
static int sys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact) {
#if HAVE_SIGACTION
return sys::Sigaction(signo, act, oldact);
#else
sys::sighandler_t oldh, newh;
int syserr;
if (act == nil) {
newh = SIG_GET;
}
else if (act->sa_flags & SA_SIGINFO) {
if (act->sa_sigaction != _os_sighandler)
panic("BUG: compat sigaction: act->sa_sigaction != _os_sighandler");
newh = _os_sighandler_nosigaction;
}
else {
newh = act->sa_handler;
}
oldh = sys::Signal(signo, newh, &syserr);
if (oldh == SIG_ERR)
return syserr;
if (oldact != nil) {
if (oldh == _os_sighandler_nosigaction) {
oldact->sa_sigaction = _os_sighandler;
oldact->sa_flags = SA_SIGINFO;
}
else {
oldact->sa_handler = oldh;
oldact->sa_flags = 0;
}
}
return 0;
#endif
}
#if !HAVE_SIGACTION
static void _os_sighandler_nosigaction(int sig) {
// reinstall sighandler since on handler invocation it is reset to SIG_DFL.
// do it as fast as we can as the first thing - to minimize window of race
// while signal handler is reset from being our _os_sighandler.
//
// NOTE it is ok to reinstall before invoking _os_sighandler because
// _os_sighandler uses only atomics and pipe write and so is reentrant.
sys::sighandler_t h;
int syserr;
h = sys::Signal(sig, _os_sighandler_nosigaction, &syserr);
if (h == SIG_ERR)
panic("signal(reinstall) failed");
if ( !(h == SIG_DFL /*reset*/ ||
h == _os_sighandler_nosigaction /*concurrent sighandler*/) )
panic("collision detected wrt thirdparty signal usage");
// invoke _os_sighandler as if it was setup by sigaction.
_os_sighandler(sig, NULL, NULL);
}
#endif
static bool _sigact_equal(const struct sigaction *a, const struct sigaction *b) {
// don't compare sigaction by memcmp - it will fail because struct sigaction
......
golang/os/signal_test.py
View file @ fdd73156
This diff is collapsed.
golang/os/testprog/signal_test_all.py
View file @ fdd73156
#!/usr/bin/env python
# Copyright (C) 2021-2022 Nexedi SA and Contributors.
# Copyright (C) 2021-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -24,7 +24,8 @@ from __future__ import print_function, absolute_import
from golang import chan
from golang import os as gos, syscall, time
from golang.os import signal
import os, sys
from golang.os.signal_test import killme
import sys
def main():
# build "all signals" list
......@@ -35,11 +36,17 @@ def main():
if sig not in allsigv: # avoid e.g. SIGCHLD/SIGCLD dups
allsigv.append(sig)
allsigv.sort(key=lambda sig: sig.signo)
allsigv.remove(syscall.SIGKILL) # SIGKILL/SIGSTOP cannot be caught
allsigv.remove(syscall.SIGSTOP)
allsigv.remove(syscall.SIGBUS) # AddressSanitizer crashes on SIGBUS/SIGFPE/SIGSEGV
allsigv.remove(syscall.SIGFPE)
allsigv.remove(syscall.SIGSEGV)
def without(signame):
sig = getattr(syscall, signame, None)
if sig is not None:
allsigv.remove(sig)
without('SIGKILL') # SIGKILL/SIGSTOP cannot be caught
without('SIGSTOP')
without('SIGBUS') # AddressSanitizer crashes on SIGBUS/SIGFPE/SIGSEGV
without('SIGFPE')
without('SIGSEGV')
without('SIGILL') # SIGILL/SIGABRT cause termination on windows
without('SIGABRT')
# Notify() -> kill * -> should be notified
ch = chan(10, dtype=gos.Signal)
......@@ -72,11 +79,6 @@ def main():
killme(syscall.SIGTERM)
raise AssertionError("not terminated")
# killme sends signal sig to own process.
def killme(sig):
mypid = os.getpid()
os.kill(mypid, sig.signo)
def emit(msg=''):
print(msg)
sys.stdout.flush()
......
golang/pyx/build.py
View file @ fdd73156
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -73,8 +73,10 @@ _dso_build_ext = setuptools_dso.build_ext
class build_ext(_dso_build_ext):
def build_extension(self, ext):
# wrap _compiler <src> -> <obj> with our code
# ._compile is used on gcc/clang but not with msvc
_compile = self.compiler._compile
def _(obj, src, ext, cc_args, extra_postargs, pp_opts):
def xcompile(obj, src, ext, cc_args, extra_postargs, pp_opts):
# filter_out removes arguments that start with argprefix
cc_args = cc_args[:]
extra_postargs = extra_postargs[:]
......@@ -101,11 +103,30 @@ class build_ext(_dso_build_ext):
filter_out('-std=gnu++')
_compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
self.compiler._compile = _
# msvc handles all sources directly in the loop in .compile and we can
# do per-source adjustsment only in .spawn .
spawn = self.compiler.spawn
def xspawn(argv):
c = False
for arg in argv:
if arg.startswith('/Tc'):
c = True
if c:
argv = argv[:]
for i in range(len(argv)):
if argv[i] == '/std:c++20':
argv[i] = '/std:c11'
return spawn(argv)
self.compiler._compile = xcompile
self.compiler.spawn = xspawn
try:
_dso_build_ext.build_extension(self, ext) # super doesn't work for _dso_build_ext
finally:
self.compiler._compile = _compile
self.compiler.spawn = spawn
# setup should be used instead of setuptools.setup
......@@ -128,14 +149,14 @@ def setup(**kw):
# x_dsos = [DSO('mypkg.mydso', ['mypkg/mydso.cpp'])],
# )
def DSO(name, sources, **kw):
_, kw = _with_build_defaults(kw)
_, kw = _with_build_defaults(name, kw)
dso = setuptools_dso.DSO(name, sources, **kw)
return dso
# _with_build_defaults returns copy of kw amended with build options common for
# both DSO and Extension.
def _with_build_defaults(kw): # -> (pygo, kw')
def _with_build_defaults(name, kw): # -> (pygo, kw')
# find pygolang root
gopkg = _findpkg("golang")
pygo = dirname(gopkg.path) # .../pygolang/golang -> .../pygolang
......@@ -144,12 +165,19 @@ def _with_build_defaults(kw): # -> (pygo, kw')
kw = kw.copy()
sysname = platform.system().lower()
win = (sysname == 'windows')
msvc = win # TODO also support mingw ?
# prepend -I<pygolang> so that e.g. golang/libgolang.h is found
# same with -I<pygolang>/golang/_compat/<os>
incv = kw.get('include_dirs', [])[:]
incv.insert(0, pygo)
incv.insert(1, join(pygo, 'golang', '_compat', sysname))
kw['include_dirs'] = incv
# link with libgolang.so
# link with libgolang.so if it is not libgolang itself
if name != 'golang.runtime.libgolang':
dsov = kw.get('dsos', [])[:]
dsov.insert(0, 'golang.runtime.libgolang')
kw['dsos'] = dsov
......@@ -160,9 +188,20 @@ def _with_build_defaults(kw): # -> (pygo, kw')
# default to C++11 (chan[T] & co require C++11 features)
ccdefault = []
if lang == 'c++':
if not msvc:
if name == 'golang.runtime.libgolang':
ccdefault.append('-std=gnu++11') # not c++11 as linux/list.h uses typeof
else:
ccdefault.append('-std=c++11')
else:
# MSVC requries C++20 for designated struct initializers
ccdefault.append('/std:c++20')
# and make exception handling: "fully conformant", so that e.g. extern "C" `panic` works
ccdefault.extend(['/EHc-', '/EHsr'])
# default to no strict-aliasing
ccdefault.append('-fno-strict-aliasing')
if not msvc:
ccdefault.append('-fno-strict-aliasing') # use only on gcc/clang - msvc does it by default
_ = kw.get('extra_compile_args', [])[:]
_[0:0] = ccdefault # if another e.g. -std=... was already there -
......@@ -188,6 +227,8 @@ def _with_build_defaults(kw): # -> (pygo, kw')
'os/signal.h',
'pyx/runtime.h',
'_testing.h',
'_compat/windows/strings.h',
'_compat/windows/unistd.h',
]])
kw['depends'] = dependv
......@@ -203,7 +244,7 @@ def _with_build_defaults(kw): # -> (pygo, kw')
# ext_modules = [Extension('mypkg.mymod', ['mypkg/mymod.pyx'])],
# )
def Extension(name, sources, **kw):
pygo, kw = _with_build_defaults(kw)
pygo, kw = _with_build_defaults(name, kw)
# some pyx-level depends to workaround a bit lack of proper dependency
# tracking in setuptools/distutils.
......
golang/pyx/build_test.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -28,7 +28,8 @@ testprog = dirname(__file__) + "/testprog"
# verify that we can build/run external package that uses pygolang in pyx mode.
def test_pyx_build():
pyxuser = testprog + "/golang_pyx_user"
pyrun(["setup.py", "build_ext", "-i"], cwd=pyxuser)
pyrun(["setup.py", "build_ext", "-i"], cwd=pyxuser,
lsan=False) # gcc leaks
# run built test.
_ = pyout(["-c",
......@@ -44,8 +45,8 @@ def test_pyx_build():
# verify that we can build/run external dso that uses libgolang.
def test_dso_build():
dsouser = testprog + "/golang_dso_user"
pyrun(["setup.py", "build_dso", "-i"], cwd=dsouser)
pyrun(["setup.py", "build_ext", "-i"], cwd=dsouser)
pyrun(["setup.py", "build_dso", "-i"], cwd=dsouser, lsan=False) # gcc leaks
pyrun(["setup.py", "build_ext", "-i"], cwd=dsouser, lsan=False) # gcc leaks
# run built test.
_ = pyout(["-c",
......
golang/pyx/runtime.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_PYX_RUNTIME_H
#define _NXD_LIBGOLANG_PYX_RUNTIME_H
// Copyright (C) 2018-2019 Nexedi SA and Contributors.
// Copyright (C) 2018-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -58,8 +58,8 @@ class _PyError final : public _error, object {
private:
_PyError();
~_PyError();
friend error PyErr_Fetch();
friend void PyErr_ReRaise(PyError pyerr);
friend LIBPYXRUNTIME_API error PyErr_Fetch();
friend LIBPYXRUNTIME_API void PyErr_ReRaise(PyError pyerr);
public:
LIBPYXRUNTIME_API void incref();
LIBPYXRUNTIME_API void decref();
......
golang/pyx/testprog/cmdclass_custom.py
View file @ fdd73156
# Copyright (C) 2019 Nexedi SA and Contributors.
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -29,4 +29,9 @@ class mybuild_ext(build_ext):
setup(
cmdclass = {'build_ext': mybuild_ext},
# avoid setuptools thinking nearby golang_pyx_user/ golang_dso_user/ also
# relate to hereby setup and rejecting the build.
# https://stackoverflow.com/questions/72294299/multiple-top-level-packages-discovered-in-a-flat-layout
py_modules = [],
)
golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
View file @ fdd73156
// Copyright (C) 2019 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -20,7 +20,7 @@
// Small library that uses a bit of libgolang features, mainly to verify
// that external project can build against libgolang.
#include <golang/libgolang.h>
#include "dso.h"
using namespace golang;
#include <stdio.h>
......
golang/pyx/testprog/golang_dso_user/dsouser/dso.h 0 → 100644
View file @ fdd73156
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
#ifndef _NXD_GOLANG_DSOUSER_DSO_H
#define _NXD_GOLANG_DSOUSER_DSO_H
#include <golang/libgolang.h>
#if BUILDING_DSOUSER_DSO
# define DSOUSER_DSO_API LIBGOLANG_DSO_EXPORT
#else
# define DSOUSER_DSO_API LIBGOLANG_DSO_IMPORT
#endif
DSOUSER_DSO_API void dsotest();
#endif // _NXD_GOLANG_DSOUSER_DSO_H
golang/pyx/testprog/golang_dso_user/dsouser/test.pyx
View file @ fdd73156
# cython: language_level=2
#
# Copyright (C) 2019 Nexedi SA and Contributors.
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -21,10 +21,7 @@
# Small test driver that calls dso.so .
cdef extern from * nogil:
"""
void dsotest();
"""
cdef extern from "dso.h" nogil:
void dsotest()
def main():
......
golang/pyx/testprog/golang_dso_user/setup.py
View file @ fdd73156
# Copyright (C) 2019 Nexedi SA and Contributors.
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -24,7 +24,9 @@ setup(
name = 'golang_dso_user',
description = 'test project that uses libgolang from a dso',
x_dsos = [DSO('dsouser.dso', ['dsouser/dso.cpp'])],
x_dsos = [DSO('dsouser.dso',
['dsouser/dso.cpp'],
define_macros = [('BUILDING_DSOUSER_DSO', None)])],
ext_modules = [Extension('dsouser.test',
['dsouser/test.pyx'],
dsos = ['dsouser.dso'])],
......
golang/runtime/_libgolang.pxd
View file @ fdd73156
# cython: language_level=2
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -36,7 +36,7 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil:
_libgolang_sema* (*sema_alloc) ()
void (*sema_free) (_libgolang_sema*)
void (*sema_acquire)(_libgolang_sema*)
bint (*sema_acquire)(_libgolang_sema*, uint64_t timeout_ns)
void (*sema_release)(_libgolang_sema*)
void (*nanosleep)(uint64_t)
......
golang/runtime/_runtime_gevent.pyx
View file @ fdd73156
# cython: language_level=2
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -40,7 +40,10 @@ ELSE:
from gevent import sleep as pygsleep
from libc.stdint cimport uint8_t, uint64_t
from libc.stdint cimport uint8_t, uint64_t, UINT64_MAX
cdef extern from *:
ctypedef bint cbool "bool"
from cpython cimport PyObject, Py_INCREF, Py_DECREF
from cython cimport final
......@@ -57,7 +60,7 @@ from posix.fcntl cimport mode_t, F_GETFL, F_SETFL, O_NONBLOCK, O_ACCMODE, O_RDON
from posix.stat cimport struct_stat, S_ISREG, S_ISDIR, S_ISBLK
from posix.strings cimport bzero
from gevent.fileobject import FileObjectThread, FileObjectPosix
from gevent import fileobject as gfileobj
# _goviapy & _togo serve go
......@@ -95,9 +98,12 @@ cdef:
Py_DECREF(pygsema)
return True
bint _sema_acquire(_libgolang_sema *gsema):
bint _sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns, cbool* pacq):
pygsema = <PYGSema>gsema
pygsema.acquire()
timeout = None
if timeout_ns != UINT64_MAX:
timeout = float(timeout_ns) * 1e-9
pacq[0] = pygsema.acquire(timeout=timeout)
return True
bint _sema_release(_libgolang_sema *gsema):
......@@ -142,14 +148,16 @@ cdef nogil:
if not ok:
panic("pyxgo: gevent: sema: free: failed")
void sema_acquire(_libgolang_sema *gsema):
cbool sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns):
cdef PyExc exc
cdef cbool acq
with gil:
pyexc_fetch(&exc)
ok = _sema_acquire(gsema)
ok = _sema_acquire(gsema, timeout_ns, &acq)
pyexc_restore(exc)
if not ok:
panic("pyxgo: gevent: sema: acquire: failed")
return acq
void sema_release(_libgolang_sema *gsema):
cdef PyExc exc
......@@ -194,6 +202,7 @@ cdef nogil:
return ioh
_libgolang_ioh* _io_fdopen(int *out_syserr, int sysfd):
IF POSIX:
# check if we should enable O_NONBLOCK on this file-descriptor
# even though we could enable O_NONBLOCK for regular files, it does not
# work as expected as most unix'es report regular files as always read
......@@ -220,6 +229,13 @@ cdef nogil:
out_syserr[0] = syserr
return NULL
ELSE: # !POSIX (windows)
cdef bint blocking = True
# FIXME acc: use GetFileInformationByHandleEx to determine whether
# HANDLE(sysfd) was opened for reading or writing.
# https://stackoverflow.com/q/9442436/9456786
cdef int acc = O_RDWR
# create IOH backed by FileObjectThread or FileObjectPosix
ioh = <IOH*>calloc(1, sizeof(IOH))
if ioh == NULL:
......@@ -253,9 +269,9 @@ cdef:
pygfobj = None
try:
if blocking:
pygfobj = FileObjectThread(sysfd, mode=mode, buffering=0)
pygfobj = gfileobj.FileObjectThread(sysfd, mode=mode, buffering=0)
else:
pygfobj = FileObjectPosix(sysfd, mode=mode, buffering=0)
pygfobj = gfileobj.FileObjectPosix(sysfd, mode=mode, buffering=0)
except OSError as e:
out_syserr[0] = -e.errno
else:
......@@ -338,7 +354,19 @@ cdef:
cdef uint8_t[::1] mem = <uint8_t[:count]>buf
xmem = memoryview(mem) # to avoid https://github.com/cython/cython/issues/3900 on mem[:0]=b''
try:
n = pygfobj.readinto(xmem)
# NOTE buf might be on stack, so it must not be accessed, e.g. from
# FileObjectThread, while our greenlet is parked (see STACK_DEAD_WHILE_PARKED
# for details). -> Use intermediate on-heap buffer to protect from that.
#
# Also: we cannot use pygfobj.readinto due to
# https://github.com/gevent/gevent/pull/1948
#
# TODO use .readinto() when buf is not on stack and gevent is
# recent enough or pygfobj is not FileObjectThread.
#n = pygfobj.readinto(xmem)
buf2 = pygfobj.read(count)
n = len(buf2)
xmem[:n] = buf2
except OSError as e:
n = -e.errno
out_n[0] = n
......@@ -361,8 +389,14 @@ cdef:
bint _io_write(IOH* ioh, int* out_n, const void *buf, size_t count):
pygfobj = <object>ioh.pygfobj
cdef const uint8_t[::1] mem = <const uint8_t[:count]>buf
# NOTE buf might be on stack, so it must not be accessed, e.g. from
# FileObjectThread, while our greenlet is parked (see STACK_DEAD_WHILE_PARKED
# for details). -> Use intermediate on-heap buffer to protect from that.
buf2 = bytearray(mem)
try:
n = pygfobj.write(mem)
n = pygfobj.write(buf2)
except OSError as e:
n = -e.errno
out_n[0] = n
......
golang/runtime/_runtime_thread.pyx
View file @ fdd73156
# cython: language_level=2
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -35,7 +35,12 @@ from __future__ import print_function, absolute_import
#
# NOTE Cython declares PyThread_acquire_lock/PyThread_release_lock as nogil
from cpython.pythread cimport PyThread_acquire_lock, PyThread_release_lock, \
PyThread_type_lock, WAIT_LOCK
PyThread_type_lock, WAIT_LOCK, NOWAIT_LOCK, PyLockStatus, PY_LOCK_ACQUIRED, PY_LOCK_FAILURE
cdef extern from * nogil:
ctypedef int PY_TIMEOUT_T # long long there
PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock, PY_TIMEOUT_T timeout_us, int intr_flag)
# NOTE On Darwin, even though this is considered as POSIX, Python uses
# mutex+condition variable to implement its lock, and, as of 20190828, Py2.7
......@@ -98,6 +103,9 @@ from libc.errno cimport errno, EINTR, EBADF
from posix.fcntl cimport mode_t
from posix.stat cimport struct_stat
from posix.strings cimport bzero
cdef extern from *:
ctypedef bint cbool "bool"
IF POSIX:
from posix.time cimport clock_gettime, nanosleep as posix_nanosleep, timespec, CLOCK_REALTIME
ELSE:
......@@ -138,11 +146,46 @@ cdef nogil:
pysema = <PyThread_type_lock>gsema
PyThread_free_lock(pysema)
void sema_acquire(_libgolang_sema *gsema):
cbool sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns):
pysema = <PyThread_type_lock>gsema
IF PY3:
cdef PY_TIMEOUT_T timeout_us
ELSE:
cdef uint64_t tprev, t, tsleep
if timeout_ns == UINT64_MAX:
ok = PyThread_acquire_lock(pysema, WAIT_LOCK)
if ok == 0:
panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock failed")
return 1
else:
IF PY3:
timeout_us = timeout_ns // 1000
lkok = PyThread_acquire_lock_timed(pysema, timeout_us, 0)
if lkok == PY_LOCK_FAILURE:
return 0
elif lkok == PY_LOCK_ACQUIRED:
return 1
else:
panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock_timed failed")
ELSE:
# py2 misses PyThread_acquire_lock_timed - provide fallback ourselves
tprev = nanotime()
while 1:
ok = PyThread_acquire_lock(pysema, NOWAIT_LOCK)
if ok:
return 1
tsleep = min(timeout_ns, 50*1000) # poll every 50 μs = 20 Hz
if tsleep == 0:
break
nanosleep(tsleep)
t = nanotime()
if t < tprev:
break # clock skew
if t - tprev >= timeout_ns:
break
timeout_ns -= t - tprev
tprev = t
return 0
void sema_release(_libgolang_sema *gsema):
pysema = <PyThread_type_lock>gsema
......
golang/runtime/internal/atomic.cpp
View file @ fdd73156
// Copyright (C) 2022 Nexedi SA and Contributors.
// Copyright (C) 2022-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -20,7 +20,9 @@
#include "golang/runtime/internal/atomic.h"
#include "golang/libgolang.h"
#ifndef _WIN32
#include <pthread.h>
#endif
// golang::internal::atomic::
namespace golang {
......@@ -41,9 +43,12 @@ static void _forkNewEpoch() {
}
void _init() {
// there is no fork on windows
#ifndef _WIN32
int e = pthread_atfork(/*prepare*/nil, /*inparent*/nil, /*inchild*/_forkNewEpoch);
if (e != 0)
panic("pthread_atfork failed");
#endif
}
......
golang/runtime/internal/syscall.cpp
View file @ fdd73156
// Copyright (C) 2021-2022 Nexedi SA and Contributors.
// Copyright (C) 2021-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -57,15 +57,18 @@ string _Errno::Error() {
_Errno& e = *this;
char ebuf[128];
bool ok;
#if __APPLE__
int x = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
if (x == 0)
return string(ebuf);
return "unknown error " + std::to_string(-e.syserr);
ok = (::strerror_r(-e.syserr, ebuf, sizeof(ebuf)) == 0);
#elif defined(_WIN32)
ok = (::strerror_s(ebuf, sizeof(ebuf), -e.syserr) == 0);
#else
char *estr = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
return string(estr);
#endif
if (ok)
return string(ebuf);
return "unknown error " + std::to_string(-e.syserr);
}
......@@ -99,6 +102,7 @@ __Errno Close(int fd) {
return err;
}
#ifndef _WIN32
__Errno Fcntl(int fd, int cmd, int arg) {
int save_errno = errno;
int err = ::fcntl(fd, cmd, arg);
......@@ -107,6 +111,7 @@ __Errno Fcntl(int fd, int cmd, int arg) {
errno = save_errno;
return err;
}
#endif
__Errno Fstat(int fd, struct ::stat *out_st) {
int save_errno = errno;
......@@ -119,6 +124,10 @@ __Errno Fstat(int fd, struct ::stat *out_st) {
int Open(const char *path, int flags, mode_t mode) {
int save_errno = errno;
#ifdef _WIN32 // default to open files in binary mode
if ((flags & (_O_TEXT | _O_BINARY)) == 0)
flags |= _O_BINARY;
#endif
int fd = ::open(path, flags, mode);
if (fd < 0)
fd = -errno;
......@@ -126,15 +135,39 @@ int Open(const char *path, int flags, mode_t mode) {
return fd;
}
__Errno Pipe(int vfd[2]) {
__Errno Pipe(int vfd[2], int flags) {
// supported flags: O_CLOEXEC
if (flags & ~(O_CLOEXEC))
return -EINVAL;
int save_errno = errno;
int err = ::pipe(vfd);
int err;
#ifdef __linux__
err = ::pipe2(vfd, flags);
#elif defined(_WIN32)
err = ::_pipe(vfd, 4096, flags | _O_BINARY);
#else
err = ::pipe(vfd);
if (err)
goto out;
if (flags & O_CLOEXEC) {
for (int i=0; i<2; i++) {
err = Fcntl(vfd[i], F_SETFD, FD_CLOEXEC);
if (err) {
Close(vfd[0]);
Close(vfd[1]);
goto out;
}
}
}
out:
#endif
if (err == -1)
err = -errno;
errno = save_errno;
return err;
}
#ifndef _WIN32
__Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact) {
int save_errno = errno;
int err = ::sigaction(signo, act, oldact);
......@@ -143,6 +176,17 @@ __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *
errno = save_errno;
return err;
}
#endif
sighandler_t Signal(int signo, sighandler_t handler, int *out_psyserr) {
int save_errno = errno;
sighandler_t oldh = ::signal(signo, handler);
*out_psyserr = (oldh == SIG_ERR
? (errno ? -errno : -EINVAL) // windows returns SIG_ERR/errno=0 for invalid signo
: 0);
errno = save_errno;
return oldh;
}
}}} // golang::internal::syscall::
golang/runtime/internal/syscall.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_RUNTIME_INTERNAL_SYSCALL_H
#define _NXD_LIBGOLANG_RUNTIME_INTERNAL_SYSCALL_H
// Copyright (C) 2021-2022 Nexedi SA and Contributors.
// Copyright (C) 2021-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -59,15 +59,21 @@ error NewErrno(__Errno syserr); // TODO better return Errno directly.
// system calls
int/*n|err*/ Read(int fd, void *buf, size_t count);
int/*n|err*/ Write(int fd, const void *buf, size_t count);
__Errno Close(int fd);
__Errno Fcntl(int fd, int cmd, int arg);
__Errno Fstat(int fd, struct ::stat *out_st);
int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
__Errno Pipe(int vfd[2]);
__Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
LIBGOLANG_API int/*n|err*/ Read(int fd, void *buf, size_t count);
LIBGOLANG_API int/*n|err*/ Write(int fd, const void *buf, size_t count);
LIBGOLANG_API __Errno Close(int fd);
#ifndef _WIN32
LIBGOLANG_API __Errno Fcntl(int fd, int cmd, int arg);
#endif
LIBGOLANG_API __Errno Fstat(int fd, struct ::stat *out_st);
LIBGOLANG_API int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
LIBGOLANG_API __Errno Pipe(int vfd[2], int flags);
#ifndef _WIN32
LIBGOLANG_API __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
#endif
typedef void (*sighandler_t)(int);
LIBGOLANG_API sighandler_t /*sigh|SIG_ERR*/ Signal(int signo, sighandler_t handler, int *out_psyserr);
}}} // golang::internal::syscall::
......
golang/runtime/libgolang.cpp
View file @ fdd73156
// Copyright (C) 2018-2022 Nexedi SA and Contributors.
// Copyright (C) 2018-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -43,12 +43,23 @@
#include <stdlib.h>
#include <string.h>
#include <strings.h>
// linux/list.h needs ARRAY_SIZE XXX -> better use c.h or ccan/array_size.h ?
#ifndef ARRAY_SIZE
# define ARRAY_SIZE(A) (sizeof(A) / sizeof((A)[0]))
#endif
#include <linux/list.h>
// MSVC does not support statement expressions and typeof
// -> redo list_entry via C++ lambda.
#ifdef _MSC_VER
# undef list_entry
# define list_entry(ptr, type, member) [&]() { \
const decltype( ((type *)0)->member ) *__mptr = (ptr); \
return (type *)( (char *)__mptr - offsetof(type,member) ); \
}()
#endif
using std::atomic;
using std::bad_alloc;
......@@ -120,6 +131,7 @@ using internal::_runtime;
namespace internal { namespace atomic { extern void _init(); } }
namespace os { namespace signal { extern void _init(); } }
namespace time { extern void _init(); }
void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) {
if (_runtime != nil) // XXX better check atomically
panic("libgolang: double init");
......@@ -127,6 +139,7 @@ void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) {
internal::atomic::_init();
os::signal::_init();
time::_init();
}
void _taskgo(void (*f)(void *), void *arg) {
......@@ -155,7 +168,15 @@ void _semafree(_sema *sema) {
}
void _semaacquire(_sema *sema) {
_runtime->sema_acquire((_libgolang_sema *)sema);
bool ok;
ok = _runtime->sema_acquire((_libgolang_sema *)sema, UINT64_MAX);
if (!ok)
panic("semaacquire: failed");
}
// NOTE not currently exposed in public API
bool _semaacquire_timed(_sema *sema, uint64_t timeout_ns) {
return _runtime->sema_acquire((_libgolang_sema *)sema, timeout_ns);
}
void _semarelease(_sema *sema) {
......@@ -899,6 +920,13 @@ template<> int _chanselect2</*onstack=*/true> (const _selcase *, int, const vect
template<> int _chanselect2</*onstack=*/false>(const _selcase *, int, const vector<int>&);
static int __chanselect2(const _selcase *, int, const vector<int>&, _WaitGroup*);
// PRNG for select.
// TODO consider switching to xoroshiro or wyrand if speed is an issue
// https://thompsonsed.co.uk/random-number-generators-for-c-performance-tested
// https://nullprogram.com/blog/2017/09/21/
static std::random_device _devrand;
static thread_local std::mt19937 _t_rng(_devrand());
// _chanselect executes one ready send or receive channel case.
//
// if no case is ready and default case was provided, select chooses default.
......@@ -925,7 +953,7 @@ int _chanselect(const _selcase *casev, int casec) {
vector<int> nv(casec); // n -> n(case) TODO -> caller stack-allocate nv
for (int i=0; i <casec; i++)
nv[i] = i;
std::random_shuffle(nv.begin(), nv.end());
std::shuffle(nv.begin(), nv.end(), _t_rng);
// first pass: poll all cases and bail out in the end if default was provided
int ndefault = -1;
......
golang/sync.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_SYNC_H
#define _NXD_LIBGOLANG_SYNC_H
// Copyright (C) 2018-2020 Nexedi SA and Contributors.
// Copyright (C) 2018-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -212,7 +212,7 @@ class _WorkGroup : public object {
private:
_WorkGroup();
~_WorkGroup();
friend WorkGroup NewWorkGroup(context::Context ctx);
friend LIBGOLANG_API WorkGroup NewWorkGroup(context::Context ctx);
public:
LIBGOLANG_API void decref();
......
golang/testprog/golang_test_defer_excchain.txt
View file @ fdd73156
Traceback (most recent call last):
File "PYGOLANG/golang/__init__.py", line ..., in _
File "PYGOLANG/golang/__init__.py", line ..., in _goframe
return f(*argv, **kw)
^^^^^^^^^^^^^^ +PY311
File "PY39(PYGOLANG/golang/testprog/)golang_test_defer_excchain.py", line 42, in main
raise RuntimeError("err")
RuntimeError: err
......@@ -21,6 +22,7 @@ Traceback (most recent call last):
...
File "PY39(PYGOLANG/golang/testprog/)golang_test_defer_excchain.py", line 33, in d2
1/0
~^~ +PY311
ZeroDivisionError: ...
During handling of the above exception, another exception occurred:
......@@ -29,8 +31,9 @@ Traceback (most recent call last):
... "PY39(PYGOLANG/golang/testprog/)golang_test_defer_excchain.py", line 45, in <module>
main()
...
File "PYGOLANG/golang/__init__.py", line ..., in _
return f(*argv, **kw)
File "PYGOLANG/golang/__init__.py", line ..., in _goframe
return f(*argv, **kw) -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
...
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
......
golang/testprog/golang_test_defer_excchain.txt-ipython
View file @ fdd73156
...
RuntimeError Traceback (most recent call last)
PYGOLANG/golang/__init__.py in _(f, *argv, **kw)
PYGOLANG/golang/__init__.py in _goframe(f, *argv, **kw)
...
--> ... return f(*argv, **kw)
...
......@@ -50,7 +50,7 @@ PYGOLANG/golang/testprog/golang_test_defer_excchain.py in ...
...
PYGOLANG/golang/__init__.py in _(f, *argv, **kw)
PYGOLANG/golang/__init__.py in _goframe(f, *argv, **kw)
...
--> ... return f(*argv, **kw)
...
......
golang/testprog/golang_test_defer_excchain.txt-pytest
View file @ fdd73156
...
_____________________________________ main _____________________________________
../__init__.py:...: in _
...________________ main ________________...
../__init__.py:...: in _goframe
return f(*argv, **kw)
golang_test_defer_excchain.py:42: in main
raise RuntimeError("err")
......
golang/testprog/golang_test_str.py
View file @ fdd73156
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2022 Nexedi SA and Contributors.
# Copyright (C) 2022-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -29,8 +29,8 @@ from golang import b, u
from golang.gcompat import qq
def main():
sb = b("привет b")
su = u("привет u")
sb = b("привет αβγ b")
su = u("привет αβγ u")
print("print(b):", sb)
print("print(u):", su)
print("print(qq(b)):", qq(sb))
......
golang/testprog/golang_test_str.txt
View file @ fdd73156
print(b): привет b
print(u): привет u
print(qq(b)): "привет b"
print(qq(u)): "привет u"
print(repr(b)): b('привет b')
print(repr(u)): u('привет u')
print({b: u}): {b('привет b'): u('привет u')}
print(b): привет αβγ b
print(u): привет αβγ u
print(qq(b)): "привет αβγ b"
print(qq(u)): "привет αβγ u"
print(repr(b)): b('привет αβγ b')
print(repr(u)): u('привет αβγ u')
print({b: u}): {b('привет αβγ b'): u('привет αβγ u')}
golang/time.cpp
View file @ fdd73156
This diff is collapsed.
golang/time.h
View file @ fdd73156
#ifndef _NXD_LIBGOLANG_TIME_H
#define _NXD_LIBGOLANG_TIME_H
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -118,12 +118,13 @@ private:
double _dt;
sync::Mutex _mu;
bool _stop;
Timer _timer;
// don't new - create only via new_ticker()
private:
_Ticker();
~_Ticker();
friend Ticker new_ticker(double dt);
friend Ticker LIBGOLANG_API new_ticker(double dt);
public:
LIBGOLANG_API void decref();
......@@ -147,18 +148,12 @@ LIBGOLANG_API Timer new_timer(double dt);
struct _Timer : object {
chan<double> c;
private:
func<void()> _f;
sync::Mutex _mu;
double _dt; // +inf - stopped, otherwise - armed
int _ver; // current timer was armed by n'th reset
// don't new - create only via new_timer() & co
private:
_Timer();
~_Timer();
friend Timer _new_timer(double dt, func<void()> f);
friend class _TimerImpl;
public:
LIBGOLANG_API void decref();
......@@ -182,9 +177,6 @@ public:
//
// the timer must be either already stopped or expired.
LIBGOLANG_API void reset(double dt);
private:
void _fire(double dt, int ver);
};
......
golang/time_test.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -20,9 +20,10 @@
from __future__ import print_function, absolute_import
from golang import select
from golang import time
from golang import select, func, defer
from golang import time, sync
from golang.golang_test import panics
from six.moves import range as xrange
# all timer tests operate in dt units
dt = 10*time.millisecond
......@@ -65,6 +66,7 @@ def test_ticker_time():
# test_timer verifies that Timer/Ticker fire as expected.
@func
def test_timer():
# start timers at x5, x7 and x11 intervals an verify that the timers fire
# in expected sequence. The times when the timers fire do not overlap in
......@@ -73,15 +75,15 @@ def test_timer():
tv = [] # timer events
Tstart = time.now()
t23 = time.Timer(23*dt)
t5 = time.Timer( 5*dt)
t23 = time.Timer(23*dt); defer(t23.stop)
t5 = time.Timer( 5*dt); defer(t5 .stop)
def _():
tv.append(7)
t7f.reset(7*dt)
t7f = time.Timer( 7*dt, f=_)
t7f = time.Timer( 7*dt, f=_); defer(t7f.stop)
tx11 = time.Ticker(11*dt)
tx11 = time.Ticker(11*dt); defer(tx11.stop)
while 1:
_, _rx = select(
......@@ -108,19 +110,20 @@ def test_timer():
# test_timer_misc, similarly to test_timer, verifies misc timer convenience functions.
@func
def test_timer_misc():
tv = []
Tstart = time.now()
c23 = time.after(23*dt)
c5 = time.after( 5*dt)
c23 = time.after(23*dt) # cannot stop
c5 = time.after( 5*dt) # cannot stop
def _():
tv.append(7)
t7f.reset(7*dt)
t7f = time.after_func(7*dt, _)
t7f = time.after_func(7*dt, _); defer(t7f.stop)
cx11 = time.tick(11*dt)
cx11 = time.tick(11*dt) # cannot stop
while 1:
_, _rx = select(
......@@ -148,13 +151,14 @@ def test_timer_misc():
# test_timer_stop verifies that .stop() cancels Timer or Ticker.
@func
def test_timer_stop():
tv = []
t10 = time.Timer (10*dt)
t2 = time.Timer ( 2*dt) # will fire and cancel t3, tx5
t3 = time.Timer ( 3*dt) # will be canceled
tx5 = time.Ticker( 5*dt) # will be canceled
t10 = time.Timer (10*dt); defer(t10.stop)
t2 = time.Timer ( 2*dt); defer(t2 .stop) # will fire and cancel t3, tx5
t3 = time.Timer ( 3*dt); defer(t3 .stop) # will be canceled
tx5 = time.Ticker( 5*dt); defer(tx5.stop) # will be canceled
while 1:
_, _rx = select(
......@@ -180,9 +184,10 @@ def test_timer_stop():
# test_timer_stop_drain verifies that Timer/Ticker .stop() drains timer channel.
@func
def test_timer_stop_drain():
t = time.Timer (1*dt)
tx = time.Ticker(1*dt)
t = time.Timer (1*dt); defer(t.stop)
tx = time.Ticker(1*dt); defer(tx.stop)
time.sleep(2*dt)
assert len(t.c) == 1
......@@ -195,9 +200,45 @@ def test_timer_stop_drain():
assert len(tx.c) == 0
# test_timer_stop_vs_func verifies that Timer .stop() works correctly with func-timer.
@func
def test_timer_stop_vs_func():
tv = []
def _1(): tv.append(1)
def _2(): tv.append(2)
t1 = time.after_func(1e6*dt, _1); defer(t1.stop)
t2 = time.after_func( 1*dt, _2); defer(t2.stop)
time.sleep(2*dt)
assert t1.stop() == True
assert t2.stop() == False
assert tv == [2]
# test_timer_reset_armed verifies that .reset() panics if called on armed timer.
@func
def test_timer_reset_armed():
# reset while armed
t = time.Timer(10*dt)
t = time.Timer(10*dt); defer(t.stop)
with panics("Timer.reset: the timer is armed; must be stopped or expired"):
t.reset(5*dt)
# bench_timer_arm_cancel benchmarks arming timers that do not fire.
# it shows how cheap or expensive it is to use timers to implement timeouts.
def bench_timer_arm_cancel(b):
for i in xrange(b.N):
t = time.Timer(10*time.second)
_ = t.stop()
assert _ is True
# bench_timer_arm_fire benchmarks arming timers that do fire.
# it shows what it costs to go through all steps related to timer loop and firing timers.
def bench_timer_arm_fire(b):
wg = sync.WaitGroup()
wg.add(b.N)
for i in xrange(b.N):
t = time.after_func(1*time.millisecond, wg.done)
wg.wait()
gpython/__init__.py
View file @ fdd73156
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -41,7 +41,7 @@ $GPYTHON_RUNTIME=threads.
from __future__ import print_function, absolute_import
_pyopt = "c:im:OVW:X:"
_pyopt = "c:Eim:OvVW:X:"
_pyopt_long = ('version',)
# pymain mimics `python ...`
......@@ -50,19 +50,27 @@ _pyopt_long = ('version',)
# init, if provided, is called after options are parsed, but before interpreter start.
def pymain(argv, init=None):
import sys
from os.path import dirname, realpath
import os
from os.path import dirname, realpath, splitext
# sys.executable
# on windows there are
# gpython-script.py
# gpython.exe
# gpython-script.py
# gpython.manifest
# and argv[0] is gpython-script.py
# with gpython-script.py sometimes embedded into gpython.exe (pip/distlib)
# and argv[0] is 'gpython-script.py' (setuptools) or 'gpython' (pip/distlib; note no .exe)
exe = realpath(argv[0])
argv = argv[1:]
if os.name == 'nt':
if exe.endswith('-script.py'):
exe = exe[:-len('-script.py')]
exe = exe[:-len('-script.py')] # gpython-script.py -> gpython.exe
exe = exe + '.exe'
else:
_, ext = splitext(exe) # gpython -> gpython.exe
if not ext:
exe += '.exe'
sys._gpy_underlying_executable = sys.executable
sys.executable = exe
......@@ -70,20 +78,26 @@ def pymain(argv, init=None):
# `gpython file` will add path-to-file to sys.path[0] by itself, and
# /path/to/gpython is unnecessary and would create difference in behaviour
# in between gpython and python.
#
# on windows when gpython.exe comes with embedded __main__.py, it is
# gpython.exe that is installed into sys.path[0] .
exedir = dirname(exe)
if sys.path[0] == exedir:
if sys.path[0] in (exedir, exe):
del sys.path[0]
else:
# buildout injects `sys.path[0:0] = eggs` into python scripts.
# detect that and remove sys.path entry corresponding to exedir.
if not _is_buildout_script(exe):
raise RuntimeError('pymain: internal error: sys.path[0] was not set by underlying python to dirname(exe):'
raise RuntimeError('pymain: internal error: sys.path[0] was not set by underlying python to dirname(exe) or exe:'
'\n\n\texe:\t%s\n\tsys.path[0]:\t%s' % (exe, sys.path[0]))
else:
if exedir in sys.path:
sys.path.remove(exedir)
else:
raise RuntimeError('pymain: internal error: sys.path does not contain dirname(exe):'
ok = False
for _ in (exedir, exe):
if _ in sys.path:
sys.path.remove(_)
ok = True
if not ok:
raise RuntimeError('pymain: internal error: sys.path does not contain dirname(exe) or exe:'
'\n\n\texe:\t%s\n\tsys.path:\t%s' % (exe, sys.path))
......@@ -99,8 +113,21 @@ def pymain(argv, init=None):
for (opt, arg) in igetopt:
# options that require reexecuting through underlying python with that -<opt>
if opt in (
'-E', # ignore $PYTHON*
'-O', # optimize
'-v', # trace import statements
'-X', # set implementation-specific option
):
# but keep `-X gpython.*` in user part of argv in case of reexec
# leaving it for main to handle. If it is only pymain to run, then
# we will be ignoring `-X gpython.*` which goes in line with builtin
# py3 behaviour to ignore any unknown -X option.
if opt == '-X' and arg is not None and arg.startswith('gpython.'):
reexec_argv.append(opt)
reexec_argv.append(arg)
else:
reexec_with.append(opt)
if arg is not None:
reexec_with.append(arg)
......@@ -202,7 +229,6 @@ def pymain(argv, init=None):
#
# python -O gpython file.py
if len(reexec_with) > 0:
import os
argv = [sys._gpy_underlying_executable] + reexec_with + [sys.executable] + reexec_argv
os.execv(argv[0], argv)
......@@ -226,11 +252,12 @@ def pymain(argv, init=None):
pyimpl = platform.python_implementation()
v = _version_info_str
pyver = platform.python_version() # ~ v(sys.version_info) but might also have e.g. '+' at tail
if pyimpl == 'CPython':
ver.append('CPython %s' % v(sys.version_info))
ver.append('CPython %s' % pyver)
elif pyimpl == 'PyPy':
ver.append('PyPy %s' % v(sys.pypy_version_info))
ver.append('Python %s' % v(sys.version_info))
ver.append('Python %s' % pyver)
else:
ver = [] # unknown
......@@ -367,33 +394,35 @@ def main():
# no harm wrt gevent monkey-patching even if we import os first.
import os
# extract and process `-X gpython.*`
# process `-X gpython.*`
# -X gpython.runtime=(gevent|threads) + $GPYTHON_RUNTIME
sys._xoptions = getattr(sys, '_xoptions', {})
argv_ = []
gpy_runtime = os.getenv('GPYTHON_RUNTIME', 'gevent')
igetopt = _IGetOpt(sys.argv[1:], _pyopt, _pyopt_long)
for (opt, arg) in igetopt:
if opt == '-X':
if arg.startswith('gpython.'):
# any non gpython -X option is handled by pymain; ignore them here
if not arg.startswith('gpython.'):
continue
if arg.startswith('gpython.runtime='):
gpy_runtime = arg[len('gpython.runtime='):]
sys._xoptions['gpython.runtime'] = gpy_runtime
else:
raise RuntimeError('gpython: unknown -X option %s' % opt)
raise RuntimeError('gpython: unknown -X option %s' % arg)
continue
argv_.append(opt)
if arg is not None:
argv_.append(arg)
# options after -c / -m are not for python itself
if opt in ('-c', '-m'):
break
argv = [sys.argv[0]] + argv_ + igetopt.argv
# propagate those settings as defaults to subinterpreters, so that e.g.
# sys.executable spawned from under `gpython -X gpython.runtime=threads`
# also uses "threads" runtime by default.
os.environ['GPYTHON_RUNTIME'] = gpy_runtime
# init initializes according to selected runtime
# it is called after options are parsed and sys.path is setup correspondingly.
......@@ -434,18 +463,18 @@ def main():
sys.version += (' [GPython %s] [%s]' % (golang.__version__, gpy_verextra))
# tail to pymain
pymain(argv, init)
pymain(sys.argv, init)
# _is_buildout_script returns whether file @path is generated as python buildout script.
def _is_buildout_script(path):
with open(path, 'r') as f:
with open(path, 'rb') as f:
src = f.read()
# buildout injects the following prologues into python scripts:
# sys.path[0:0] = [
# ...
# ]
return ('\nsys.path[0:0] = [\n' in src)
return (b'\nsys.path[0:0] = [\n' in src)
# _IGetOpt provides getopt-style incremental options parsing.
......
gpython/gpython_test.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -184,7 +184,7 @@ def test_pymain():
_ = pyout(['-m', 'hello', 'abc', 'def'], cwd=testdata)
# realpath rewrites e.g. `local/lib -> lib` if local/lib is symlink
hellopy = realpath(join(testdata, 'hello.py'))
assert _ == b"hello\nworld\n['%s', 'abc', 'def']\n" % b(hellopy)
assert _ == b"hello\nworld\n[%s, 'abc', 'def']\n" % b(repr(hellopy))
# -m<module>
__ = pyout(['-mhello', 'abc', 'def'], cwd=testdata)
assert __ == _
......@@ -195,7 +195,7 @@ def test_pymain():
# -i after stdin (also tests interactive mode as -i forces interactive even on non-tty)
d = {
b'hellopy': b(hellopy),
b'repr(hellopy)': b(repr(hellopy)),
b'ps1': b'' # cpython emits prompt to stderr
}
if is_pypy and not is_gpython:
......@@ -212,7 +212,7 @@ def test_pymain():
assert _ == b"hello\nworld\n['-c']\n%(ps1)s'~~HELLO~~'\n%(ps1)s" % d
# -i after -m
_ = pyout(['-i', '-m', 'hello'], stdin=b'world.tag', cwd=testdata)
assert _ == b"hello\nworld\n['%(hellopy)s']\n%(ps1)s'~~WORLD~~'\n%(ps1)s" % d
assert _ == b"hello\nworld\n[%(repr(hellopy))s]\n%(ps1)s'~~WORLD~~'\n%(ps1)s" % d
# -i after file
_ = pyout(['-i', 'testdata/hello.py'], stdin=b'tag', cwd=here)
assert _ == b"hello\nworld\n['testdata/hello.py']\n%(ps1)s'~~HELLO~~'\n%(ps1)s" % d
......@@ -221,26 +221,27 @@ def test_pymain():
# -W <opt>
_ = pyout(['-Werror', '-Whello', '-W', 'ignore::DeprecationWarning',
'testprog/print_warnings_setup.py'], cwd=here)
if PY2:
# py2 threading, which is imported after gpython startup, adds ignore
# for sys.exc_clear
_ = grepv(r'ignore:sys.exc_clear:DeprecationWarning:threading:*', _)
assert _.startswith(
b"sys.warnoptions: ['error', 'hello', 'ignore::DeprecationWarning']\n\n" + \
b"warnings.filters:\n" + \
b"- ignore::DeprecationWarning::*\n" + \
b"- error::Warning::*\n"), _
assert re.match(
br"sys\.warnoptions: \['error', 'hello', 'ignore::DeprecationWarning'\]\n\n"
br"warnings\.filters:\n"
br"(- [^\n]+\n)*" # Additional filters added by automatically imported modules
br"- ignore::DeprecationWarning::\*\n"
br"- error::Warning::\*\n"
br"(- [^\n]+\n)*", # Remaining filters
_,
)
# $PYTHONWARNINGS
_ = pyout(['testprog/print_warnings_setup.py'], cwd=here,
envadj={'PYTHONWARNINGS': 'ignore,world,error::SyntaxWarning'})
if PY2:
# see ^^^
_ = grepv(r'ignore:sys.exc_clear:DeprecationWarning:threading:*', _)
assert _.startswith(
b"sys.warnoptions: ['ignore', 'world', 'error::SyntaxWarning']\n\n" + \
b"warnings.filters:\n" + \
b"- error::SyntaxWarning::*\n" + \
b"- ignore::Warning::*\n"), _
assert re.match(
br"sys\.warnoptions: \['ignore', 'world', 'error::SyntaxWarning'\]\n\n"
br"warnings\.filters:\n"
br"(- [^\n]+\n)*" # Additional filters added by automatically imported modules
br"- error::SyntaxWarning::\*\n"
br"- ignore::Warning::\*\n"
br"(- [^\n]+\n)*", # Remaining filters
_,
)
def test_pymain_print_function_future():
......@@ -318,6 +319,60 @@ def test_pymain_opt():
check(["-O", "-O"])
check(["-O", "-O", "-O"])
# verify that pymain handles -E in exactly the same way as underlying python does.
@gpython_only
def test_pymain_E():
envadj = {'PYTHONOPTIMIZE': '1'}
def sys_flags_optimize(level):
return 'sys.flags.optimize: %s' % level
# without -E $PYTHONOPTIMIZE should be taken into account
def _(gpyoutv, stdpyoutv):
assert sys_flags_optimize(0) not in stdpyoutv
assert sys_flags_optimize(0) not in gpyoutv
assert sys_flags_optimize(1) in stdpyoutv
assert sys_flags_optimize(1) in gpyoutv
check_gpy_vs_py(['testprog/print_opt.py'], _, envadj=envadj, cwd=here)
# with -E not
def _(gpyoutv, stdpyoutv):
assert sys_flags_optimize(0) in stdpyoutv
assert sys_flags_optimize(0) in gpyoutv
assert sys_flags_optimize(1) not in stdpyoutv
assert sys_flags_optimize(1) not in gpyoutv
check_gpy_vs_py(['-E', 'testprog/print_opt.py'], _, envadj=envadj, cwd=here)
# verify that pymain handles -X non-gpython-option in exactly the same way as underlying python does.
@pytest.mark.skipif(PY2, reason="-X does not work at all on plain cpython2")
@gpython_only
def test_pymain_X():
check_gpy_vs_py(['testprog/print_faulthandler.py'], cwd=here)
check_gpy_vs_py(['-X', 'faulthandler', 'testprog/print_faulthandler.py'], cwd=here)
# pymain -v
@gpython_only
def test_pymain_v():
def nimport(argv, **kw):
argv = argv + ['testdata/hello.py']
kw.setdefault('cwd', here)
ret, out, err = _pyrun(argv, stdout=PIPE, stderr=PIPE, **kw)
assert ret == 0, (out, err)
n = 0
for _ in u(err).splitlines():
if _.startswith("import "):
n += 1
return n
# without -v there must be no "import ..." messages
assert nimport([]) == 0
assert nimport([], pyexe=sys._gpy_underlying_executable) == 0
# with -v there must be many "import ..." messages
assert nimport(['-v']) > 10
assert nimport(['-v'], pyexe=sys._gpy_underlying_executable) > 10
# pymain -V/--version
# gpython_only because output differs from !gpython.
......@@ -359,20 +414,31 @@ def test_pymain_run_via_relpath():
out2 = pyout(['./__init__.py'] + argv, pyexe=sys._gpy_underlying_executable, cwd=here)
assert out1 == out2
# verify -X gpython.runtime=...
@gpython_only
def test_Xruntime(runtime):
_xopt_assert_in_subprocess('gpython.runtime', runtime,
assert_gevent_activated if runtime != 'threads' else \
assert_gevent_not_activated)
# _xopt_assert_in_subprocess runs tfunc in subprocess interpreter spawned with
# `-X xopt=xval` and checks that there is no error.
#
# It is also verified that tfunc runs ok in sub-subprocess interpreter spawned
# _without_ `-X ...`, i.e. once given -X setting is inherited by spawned interpreters.
def _xopt_assert_in_subprocess(xopt, xval, tfunc):
XOPT = xopt.upper().replace('.','_') # gpython.runtime -> GPYTHON_RUNTIME
env = os.environ.copy()
env.pop('GPYTHON_RUNTIME', None) # del
env.pop(XOPT, None) # del
argv = []
if runtime != '':
argv += ['-X', 'gpython.runtime='+runtime]
prog = 'from gpython import gpython_test as t; '
if runtime != 'threads':
prog += 't.assert_gevent_activated(); '
else:
prog += 't.assert_gevent_not_activated(); '
if xval != '':
argv += ['-X', xopt+'='+xval]
prog = import_t = 'from gpython import gpython_test as t; '
prog += 't.%s(); ' % tfunc.__name__
prog += import_t # + same in subprocess
prog += "t.pyrun(['-c', '%s t.%s(); ']); " % (import_t, tfunc.__name__)
prog += 'print("ok")'
argv += ['-c', prog]
......@@ -382,20 +448,6 @@ def test_Xruntime(runtime):
# ---- misc ----
# grepv filters out lines matching pattern from text.
def grepv(pattern, text): # -> text
if isinstance(text, bytes):
t = b''
else:
t = ''
p = re.compile(pattern)
v = []
for l in text.splitlines(True):
m = p.search(l)
if not m:
v.append(l)
return t.join(v)
# check_gpy_vs_py verifies that gpython output matches underlying python output.
def check_gpy_vs_py(argv, postprocessf=None, **kw):
gpyout = u(pyout(argv, **kw))
......
gpython/testprog/print_faulthandler.py 0 → 100644
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""Program print_faulthandler prints information about faulthandler settings."""
from __future__ import print_function, absolute_import
import sys
def main():
if 'faulthandler' not in sys.modules:
print('faulthandler is not imported')
return
fh = sys.modules['faulthandler']
print('faulthandler imported')
print('faulthandler %s' % ('enabled' if fh.is_enabled() else 'disabled'))
if __name__ == '__main__':
main()
gpython/testprog/print_opt.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# Copyright (C) 2020 Nexedi SA and Contributors.
# Copyright (C) 2020-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -62,8 +62,9 @@ def modpy_imports_from():
raise AssertionError("module 'mod' is already there")
tmpd = tempfile.mkdtemp('', 'modpy_imports_from')
tmpd_ = tmpd + os.path.sep
try:
pymod = "%s/mod.py" % tmpd
pymod = tmpd_ + "mod.py"
with open(pymod, "w") as f:
f.write("# hello up there\n")
......@@ -73,9 +74,9 @@ def modpy_imports_from():
files = set()
for dirpath, dirnames, filenames in os.walk(tmpd):
for _ in filenames:
f = '%s/%s' % (dirpath, _)
if f.startswith(tmpd+'/'):
f = f[len(tmpd+'/'):]
f = os.path.join(dirpath, _)
if f.startswith(tmpd_):
f = f[len(tmpd_):]
files.add(f)
......
pyproject.toml
View file @ fdd73156
[build-system]
requires = ["setuptools", "wheel", "setuptools_dso >= 1.7", "cython", "gevent"]
requires = ["setuptools", "wheel", "setuptools_dso >= 2.8", "cython < 3", "gevent"]
setup.py
View file @ fdd73156
# -*- coding: utf-8 -*-
# pygolang | pythonic package setup
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -19,28 +20,27 @@
# See https://www.nexedi.com/licensing for rationale and options.
from setuptools import find_packages
# setuptools has Library but this days it is not well supported and test for it
# has been killed https://github.com/pypa/setuptools/commit/654c26f78a30
# -> use setuptools_dso instead.
from setuptools_dso import DSO
from setuptools.command.install_scripts import install_scripts as _install_scripts
from setuptools.command.develop import develop as _develop
from distutils import sysconfig
from os.path import dirname, join
import sys, re
import sys, os, re
# read file content
def readfile(path):
with open(path, 'r') as f:
return f.read()
# reuse golang.pyx.build to build pygolang extensions.
def readfile(path): # -> str
with open(path, 'rb') as f:
data = f.read()
if not isinstance(data, str): # py3
data = data.decode('utf-8')
return data
# reuse golang.pyx.build to build pygolang dso and extensions.
# we have to be careful and inject synthetic golang package in order to be
# able to import golang.pyx.build without built/working golang.
trun = {}
exec(readfile('trun'), trun)
trun['ximport_empty_golangmod']()
from golang.pyx.build import setup, Extension as Ext
from golang.pyx.build import setup, DSO, Extension as Ext
# grep searches text for pattern.
......@@ -155,8 +155,8 @@ class develop(XInstallGPython, _develop):
# requirements of packages under "golang." namespace
R = {
'cmd.pybench': {'pytest'},
'pyx.build': {'setuptools', 'wheel', 'cython', 'setuptools_dso >= 1.7'},
'cmd.pybench': {'pytest', 'py ; python_version >= "3"'},
'pyx.build': {'setuptools', 'wheel', 'cython < 3', 'setuptools_dso >= 2.8'},
'x.perf.benchlib': {'numpy'},
}
# TODO generate `a.b -> a`, e.g. x.perf = join(x.perf.*); x = join(x.*)
......@@ -174,6 +174,14 @@ for k in sorted(R.keys()):
extras_require[k] = list(sorted(R[k]))
# get_python_libdir() returns path where libpython is located
def get_python_libdir():
# mimic what distutils.command.build_ext does
if os.name == 'nt':
return join(sysconfig.get_config_var('installed_platbase'), 'libs')
else:
return sysconfig.get_config_var('LIBDIR')
setup(
name = 'pygolang',
version = version,
......@@ -222,20 +230,21 @@ setup(
'golang/os/signal.h',
'golang/strings.h',
'golang/sync.h',
'golang/time.h'],
include_dirs = ['.', '3rdparty/include'],
'golang/time.h',
'3rdparty/ratas/src/timer-wheel.h'],
include_dirs = [
'3rdparty/include',
'3rdparty/ratas/src'],
define_macros = [('BUILDING_LIBGOLANG', None)],
extra_compile_args = ['-std=gnu++11'], # not c++11 as linux/list.h uses typeof
soversion = '0.1'),
DSO('golang.runtime.libpyxruntime',
['golang/runtime/libpyxruntime.cpp'],
depends = ['golang/pyx/runtime.h'],
include_dirs = ['.', sysconfig.get_python_inc()],
include_dirs = [sysconfig.get_python_inc()],
library_dirs = [get_python_libdir()],
define_macros = [('BUILDING_LIBPYXRUNTIME', None)],
extra_compile_args = ['-std=c++11'],
soversion = '0.1',
dsos = ['golang.runtime.libgolang'])],
soversion = '0.1')],
ext_modules = [
Ext('golang._golang',
......@@ -311,11 +320,15 @@ setup(
include_package_data = True,
install_requires = ['gevent', 'six', 'decorator', 'Importing;python_version<="2.7"',
# only runtime part: for dylink_prepare_dso
# also need to pin setuptools ≥ 60.2 because else wheel configures logging
# to go to stdout and so dylink_prepare_dso garbles program output
'setuptools_dso >= 2.8',
'setuptools >= 60.2 ; python_version>="3"',
# pyx.build -> setuptools_dso uses multiprocessing
# FIXME geventmp fails on python2, but setuptools_dso
# uses multiprocessing only on Python3, so for now we
# are ok. https://github.com/karellen/geventmp/pull/2
'geventmp;python_version>="3"',
# setuptools_dso uses multiprocessing only on Python3, and only on systems where
# mp.get_start_method()!='fork', while geventmp does not work on windows.
'geventmp ; python_version>="3" and platform_system != "Windows" ',
],
extras_require = extras_require,
......@@ -340,13 +353,18 @@ setup(
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.5
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: 3.11
Programming Language :: Python :: 3.12
Programming Language :: Python :: Implementation :: CPython
Programming Language :: Python :: Implementation :: PyPy
Operating System :: POSIX
Operating System :: POSIX :: Linux
Operating System :: Unix
Operating System :: MacOS
Operating System :: Microsoft :: Windows
Topic :: Software Development :: Interpreters
Topic :: Software Development :: Libraries :: Python Modules\
""".splitlines()]
......
tox.ini
View file @ fdd73156
[tox]
envlist =
{py27d,py27,py37,py38d,py38,py39d,py39,pypy,pypy3}-{thread,gevent}
{py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread,gevent}
# ThreadSanitizer
......@@ -10,44 +10,48 @@ envlist =
# (*) PyPy locks its GIL (see RPyGilAcquire) by manually doing atomic cmpxchg
# and other games, which TSAN cannot see if PyPy itself was not compiled with
# -fsanitize=thread.
{py27d,py27,py37,py38d,py38,py39d,py39 }-{thread }-tsan
{py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312 }-{thread }-tsan
# XXX py*-gevent-tsan would be nice to have, but at present TSAN is not
# effective with gevent, because it does not understand greenlet "thread"
# switching and so perceives the program as having only one thread where races
# are impossible. Disabled to save time.
# {py27d,py27,py37,py38d,py38,py39d,py39 }-{ gevent}-tsan
# {py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312 }-{ gevent}-tsan
# AddressSanitizer
# XXX asan does not work with gevent: https://github.com/python-greenlet/greenlet/issues/113
{py27d,py27,py37,py38d,py38,py39d,py39,pypy,pypy3}-{thread }-asan
{py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread }-asan
[testenv]
basepython =
py27d: python2.7-dbg
py27: python2.7
py37: python3.7
py38d: python3.8-dbg
py38: python3.8
py39d: python3.9-dbg
py39: python3.9
py310d: python3.10-dbg
py310: python3.10
py311d: python3.11-dbg
py311: python3.11
py312: python3.12
py312d: python3.12-dbg
pypy: pypy
pypy3: pypy3
setenv =
# distutils take CFLAGS for both C and C++.
# distutils use CFLAGS also at link stage -> we don't need to set LDFLAGS separately.
tsan: CFLAGS=-g -fsanitize=thread
asan: CFLAGS=-g -fsanitize=address
tsan: CFLAGS=-g -fsanitize=thread -fno-omit-frame-pointer
asan: CFLAGS=-g -fsanitize=address -fno-omit-frame-pointer
# XXX however distutils' try_link, which is used by numpy.distutils use only CC
# as linker without CFLAGS and _without_ LDFLAGS, which fails if *.o were
# compiled with -fsanitize=X and linked without that option. Work it around
# with also adjusting CC.
# XXX better arrange to pass CFLAGS to pygolang only, e.g. by adding --race or
# --sanitize=thread to `setup.py build_ext`.
tsan: CC=cc -fsanitize=thread
asan: CC=cc -fsanitize=address
tsan: CC=cc -fsanitize=thread -fno-omit-frame-pointer
asan: CC=cc -fsanitize=address -fno-omit-frame-pointer
# always compile pygolang from source and don't reuse binary pygolang wheels as
# we compile each case with different CFLAGS.
......@@ -69,5 +73,7 @@ commands=
# asan/tsan: tell pytest not to capture output - else it is not possible to see
# reports from sanitizers because they crash tested process on error.
# likewise for python debug builds.
asan,tsan,py{27,37}d: -s \
asan,tsan,py{27,39,310,311,312}d: -s \
gpython/ golang/
allowlist_externals={toxinidir}/trun
trun
View file @ fdd73156
#!/usr/bin/env python
# Copyright (C) 2019-2020 Nexedi SA and Contributors.
# -*- coding: utf-8 -*-
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -34,12 +35,13 @@ trun cares to run python with LD_PRELOAD set appropriately to /path/to/libtsan.s
from __future__ import print_function, absolute_import
import os, sys, re, subprocess, pkgutil
import warnings
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import imp
import os, os.path, sys, re, subprocess, platform, types
PY3 = (bytes is not str)
if PY3:
from importlib import machinery as imp_machinery
else:
import imp, pkgutil
# env_prepend prepends value to ${name} environment variable.
#
......@@ -64,10 +66,13 @@ def grep1(pattern, text): # -> re.Match|None
# to import e.g. golang.pyx.build, or locate golang._golang, without built/working golang.
def ximport_empty_golangmod():
assert 'golang' not in sys.modules
golang = imp.new_module('golang')
golang = types.ModuleType('golang')
golang.__package__ = 'golang'
golang.__path__ = ['golang']
golang.__file__ = 'golang/__init__.py'
if PY3:
golang.__loader__ = imp_machinery.SourceFileLoader('golang', 'golang/__init__.py')
else:
golang.__loader__ = pkgutil.ImpLoader('golang', None, 'golang/__init__.py',
[None, None, imp.PY_SOURCE])
sys.modules['golang'] = golang
......@@ -83,6 +88,7 @@ def main():
# determine if _golang.so is linked to a sanitizer, and if yes, to which
# particular sanitizer DSO. Set LD_PRELOAD appropriately.
libxsan = None
ld_preload = None
if 'linux' in sys.platform:
p = subprocess.Popen(["ldd", _golang_so.path], stdout=subprocess.PIPE)
......@@ -123,7 +129,8 @@ def main():
_ = grep1("DYLD_INSERT_LIBRARIES=(.*)$", err)
if _ is not None:
ld_preload = ("DYLD_INSERT_LIBRARIES", _.group(1))
libxsan = _.group(1)
ld_preload = ("DYLD_INSERT_LIBRARIES", libxsan)
else:
print("trun %r: `import golang` failed with unexpected error:" % sys.argv[1:], file=sys.stderr)
print(err, file=sys.stderr)
......@@ -140,7 +147,7 @@ def main():
env_prepend("TSAN_OPTIONS", "halt_on_error=1")
env_prepend("ASAN_OPTIONS", "halt_on_error=1")
# tweak TSAN/ASAN defaults:
# tweak TSAN/ASAN/LSAN defaults:
# enable TSAN deadlock detector
# (unfortunately it caughts only few _potential_ deadlocks and actually
......@@ -148,15 +155,49 @@ def main():
env_prepend("TSAN_OPTIONS", "detect_deadlocks=1")
env_prepend("TSAN_OPTIONS", "second_deadlock_stack=1")
# many python allocations, whose lifetime coincides with python interpreter
# lifetime and which are not explicitly freed on python shutdown, are
# reported as leaks. Disable leak reporting to avoid huge non-pygolang
# related printouts.
env_prepend("ASAN_OPTIONS", "detect_leaks=0")
# tune ASAN to check more aggressively by default
env_prepend("ASAN_OPTIONS", "detect_stack_use_after_return=1")
# enable ASAN/LSAN leak detector.
#
# Do it only on CPython ≥ 3.11 because on py2 and on earlier py3 versions
# there are many many python allocations, whose lifetime coincide with
# python interpreter lifetime, and which are not explicitly freed on python
# shutdown. For py3 they significantly improved this step by step and
# starting from 3.11 it becomes practical to silence some still-leaks with
# suppressions, while for earlier py3 versions and especially for py2 it
# is, unfortunately, not manageable. Do not spend engineering time with
# activating LSAN on PyPy as that is tier 2 platform and bug tail history
# of memory leaks is very long even only on cpython.
if sys.version_info < (3,11):
env_prepend("ASAN_OPTIONS", "detect_leaks=0")
if libxsan is not None:
if 'asan' in libxsan.lower():
print("W: trun %r: asan: leak detection deactivated on %s %s" % (
sys.argv[1:], platform.python_implementation(), platform.python_version()),
file=sys.stderr)
else:
env_prepend("ASAN_OPTIONS", "detect_leaks=1")
env_prepend("LSAN_OPTIONS", "suppressions=%s" % os.path.abspath(os.path.join(
os.path.dirname(__file__), ".lsan-ignore.txt")))
# do not print statistics for suppressed leaks - else it breaks tests that verify program output
env_prepend("LSAN_OPTIONS", "print_suppressions=0")
# enable DWARF-based unwinding.
# else, if python is not compiled with -fno-omit-frame-pointer, it can show
# the whole traceback as e.g. just
# Direct leak of 32 byte(s) in 1 object(s) allocated from:
# #0 0x7f88522f3bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
# #1 0x55f910a3d9a4 in PyThread_allocate_lock Python/thread_pthread.h:385
# and our leak suppressions won't work.
# this is slower compared to default frame-pointer based unwinding, but
# still works reasonably timely when run with just tests.
env_prepend("ASAN_OPTIONS", "fast_unwind_on_malloc=0")
# leak suppression also needs full tracebacks to work correctly, since with
# python there are many levels of call nesting at C level, and to filter-out e.g.
# top-level PyImport_Import we need to go really deep.
env_prepend("ASAN_OPTIONS", "malloc_context_size=255")
# exec `...`
os.execvp(sys.argv[1], sys.argv[1:])
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7