Skip to content

pygolang
pygolang
Commit 2b56ce71 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

Sync with master

parents 300d7dfa 55d39d4d
Expand all Show whitespace changes
Inline Side-by-side
Showing with 770 additions and 284 deletions
.gitignore
View file @ 2b56ce71
......@@ -10,6 +10,8 @@ build/
*.so.*
*.dylib
*.dll
*.lib
*.exp
*.pyd
*_dsoinfo.py
......
MANIFEST.in
View file @ 2b56ce71
......@@ -3,7 +3,7 @@ include golang/libgolang.h
include golang/runtime/libgolang.cpp
include golang/runtime/libpyxruntime.cpp
include golang/pyx/runtime.h
include golang/pyx/runtime.cpp
include golang/pyx/testprog/golang_dso_user/dsouser/dso.h
include golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
include golang/runtime/internal.h
include golang/runtime/internal/atomic.h
......@@ -34,6 +34,9 @@ include golang/sync_test.cpp
include golang/time.h
include golang/time.cpp
include golang/_testing.h
include golang/_compat/windows/strings.h
include golang/_compat/windows/unistd.h
recursive-include golang *.py *.pxd *.pyx *.toml *.txt*
recursive-include gpython *.py
recursive-include 3rdparty *.h
recursive-exclude golang *_dsoinfo.py
golang/__init__.py
View file @ 2b56ce71
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -42,6 +42,9 @@ from golang._gopath import gimport # make gimport available from golang
import inspect, sys
import decorator, six
import setuptools_dso
setuptools_dso.dylink_prepare_dso('golang.runtime.libgolang')
from golang._golang import _pysys_exc_clear as _sys_exc_clear
# @func is a necessary decorator for functions for selected golang features to work.
......
golang/_compat/windows/strings.h 0 → 100644
View file @ 2b56ce71
#ifndef _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
#define _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
//
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
#include <string.h>
static inline void bzero(void *p, size_t n) {
memset(p, '\0', n);
}
#endif // _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
golang/_compat/windows/unistd.h 0 → 100644
View file @ 2b56ce71
#ifndef _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
#define _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
//
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// stub unistd.h to be used on Windows where it is absent.
// we need this because e.g. `cimport posix.stat` forces inclusion of unistd.h
// even if we use part of posix.stat that is available everywhere.
#include <io.h>
#define O_CLOEXEC _O_NOINHERIT
#endif // _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
golang/_golang.pyx
View file @ 2b56ce71
......@@ -5,7 +5,7 @@
# distutils: language = c++
# distutils: depends = libgolang.h os/signal.h _golang_str.pyx
#
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -469,7 +469,7 @@ def pyselect(*pycasev):
casev[i].user = pych.dtype
with nogil:
selected = _chanselect_pyexc(&casev[0], casev.size())
selected = _chanselect_pyexc(casev.data(), casev.size())
finally:
# decref not sent tx (see ^^^ send prepare)
......
golang/fmt.cpp
View file @ 2b56ce71
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -48,7 +48,7 @@ string _vsprintf(const char *format, va_list argp) {
return string(buf.get(), buf.get() + nchar); // without trailing '\0'
}
string sprintf(const string &format, ...) {
string sprintf(const string format, ...) {
va_list argp;
va_start(argp, format);
string str = fmt::_vsprintf(format.c_str(), argp);
......@@ -64,7 +64,7 @@ string sprintf(const char *format, ...) {
return str;
}
error ___errorf(const string &format, ...) {
error ___errorf(const string format, ...) {
va_list argp;
va_start(argp, format);
error err = errors::New(fmt::_vsprintf(format.c_str(), argp));
......
golang/fmt.h
View file @ 2b56ce71
#ifndef _NXD_LIBGOLANG_FMT_H
#define _NXD_LIBGOLANG_FMT_H
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -40,7 +40,7 @@ namespace golang {
namespace fmt {
// sprintf formats text into string.
LIBGOLANG_API string sprintf(const string &format, ...);
LIBGOLANG_API string sprintf(const string format, ...);
// intseq<i1, i2, ...> and intrange<n> are used by errorf to handle %w.
......@@ -75,7 +75,7 @@ namespace {
//
// format suffix ": %w" is additionally handled as in Go with
// `errorf("... : %w", ..., err)` creating error that can be unwrapped back to err.
LIBGOLANG_API error ___errorf(const string& format, ...);
LIBGOLANG_API error ___errorf(const string format, ...);
LIBGOLANG_API error ___errorfTryWrap(const string& format, error last_err, ...);
LIBGOLANG_API string ___error_str(error err);
......@@ -111,7 +111,10 @@ inline error errorf(const string& format, Argv... argv) {
// `const char *` overloads just to catch format mistakes as
// __attribute__(format) does not work with std::string.
LIBGOLANG_API string sprintf(const char *format, ...)
__attribute__ ((format (printf, 1, 2)));
#ifndef _MSC_VER
__attribute__ ((format (printf, 1, 2)))
#endif
;
// cannot use __attribute__(format) for errorf as we add %w handling.
// still `const char *` overload is useful for performance.
......
golang/golang_test.py
View file @ 2b56ce71
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -23,14 +23,14 @@ from __future__ import print_function, absolute_import
from golang import go, chan, select, default, nilchan, _PanicError, func, panic, \
defer, recover, u
from golang import sync
from pytest import raises, mark, fail
from pytest import raises, mark, fail, skip
from _pytest._code import Traceback
from os.path import dirname
import os, sys, inspect, importlib, traceback, doctest
from subprocess import Popen, PIPE
import six
from six.moves import range as xrange
import gc, weakref, warnings
import gc, weakref
import re
from golang import _golang_test
......@@ -1559,6 +1559,7 @@ RuntimeError: gamma
Traceback (most recent call last):
File "PYGOLANG/golang/__init__.py", line ..., in _
return f(*argv, **kw)
^^^^^^^^^^^^^^ +PY311
File "PYGOLANG/golang/golang_test.py", line ..., in caller
raise RuntimeError("ccc")
RuntimeError: ccc
......@@ -1579,9 +1580,11 @@ Traceback (most recent call last):
caller()
...
File "PYGOLANG/golang/__init__.py", line ..., in _
return f(*argv, **kw)
return f(*argv, **kw) -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
d()
d() -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
d()
File "PYGOLANG/golang/golang_test.py", line ..., in q1
......@@ -1596,9 +1599,11 @@ Traceback (most recent call last):
caller()
...
File "PYGOLANG/golang/__init__.py", line ..., in _
return f(*argv, **kw)
return f(*argv, **kw) -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
d()
d() -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
d()
File "PYGOLANG/golang/golang_test.py", line ..., in q1
......@@ -1611,6 +1616,7 @@ RuntimeError: aaa
Traceback (most recent call last):
File "PYGOLANG/golang/__init__.py", line ..., in _
return f(*argv, **kw)
^^^^^^^^^^^^^^ +PY311
File "PYGOLANG/golang/golang_test.py", line ..., in caller
raise RuntimeError("ccc")
RuntimeError: ccc
......@@ -1631,9 +1637,11 @@ Traceback (most recent call last):
caller()
...
File "PYGOLANG/golang/__init__.py", line ..., in _
return f(*argv, **kw)
return f(*argv, **kw) -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
d()
d() -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
d()
File "PYGOLANG/golang/golang_test.py", line ..., in q1
......@@ -1654,6 +1662,13 @@ def test_defer_excchain_dump():
# ----//---- (ipython)
def test_defer_excchain_dump_ipython():
# ipython 8 changed traceback output significantly
# we do not need to test it because we acticate ipython-related patch only
# on py2 for which latest ipython version is 5.
import IPython
if six.PY3 and IPython.version_info >= (8,0):
skip("ipython is patched only on py2; ipython8 changed traceback format")
tbok = readfile(dir_testprog + "/golang_test_defer_excchain.txt-ipython")
retcode, stdout, stderr = _pyrun(["-m", "IPython", "--quick", "--colors=NoColor",
"-m", "golang_test_defer_excchain"],
......@@ -1724,8 +1739,28 @@ def _pyrun(argv, stdin=None, stdout=None, stderr=None, **kw): # -> retcode, st
pathv.extend(envpath.split(os.pathsep))
env['PYTHONPATH'] = os.pathsep.join(pathv)
# set $PYTHONIOENCODING to encoding of stdin/stdout/stderr
# we need to do it because on Windows `python x.py | ...` runs with stdio
# encoding set to cp125X even if just `python x.py` runs with stdio
# encoding=UTF-8.
if 'PYTHONIOENCODING' not in env:
enc = set([_.encoding for _ in (sys.stdin, sys.stdout, sys.stderr)])
if None in enc: # without -s pytest uses _pytest.capture.DontReadFromInput
enc.remove(None) # with None .encoding
assert len(enc) == 1
env['PYTHONIOENCODING'] = enc.pop()
p = Popen(argv, stdin=(PIPE if stdin else None), stdout=stdout, stderr=stderr, env=env, **kw)
stdout, stderr = p.communicate(stdin)
# on windows print emits \r\n instead of just \n
# normalize that to \n in *out
if os.name == 'nt':
if stdout is not None:
stdout = stdout.replace(b'\r\n', b'\n')
if stderr is not None:
stderr = stderr.replace(b'\r\n', b'\n')
return p.returncode, stdout, stderr
# pyrun runs `sys.executable argv... <stdin`.
......@@ -1792,10 +1827,33 @@ def assertDoc(want, got):
got = got.replace(dir_pygolang, "PYGOLANG") # /home/x/.../pygolang -> PYGOLANG
got = got.replace(udir_pygolang, "PYGOLANG") # ~/.../pygolang -> PYGOLANG
# got: normalize PYGOLANG\a\b\c -> PYGOLANG/a/b/c
# a\b\c\d.py -> a/b/c/d.py
def _(m):
return m.group(0).replace(os.path.sep, '/')
got = re.sub(r"(?<=PYGOLANG)[^\s]+(?=\s)", _, got)
got = re.sub(r"([\w\\\.]+)(?=\.py)", _, got)
# want: process conditionals
# PY39(...) -> ... if py39 else ø
py39 = sys.version_info >= (3, 9)
want = re.sub(r"PY39\((.*)\)", r"\1" if py39 else "", want)
# PY39(...) -> ... if py ≥ 3.9 else ø (inline)
# `... +PY39` -> ... if py ≥ 3.9 else ø (whole line)
# `... -PY39` -> ... if py < 3.9 else ø (whole line)
have = {} # 'PYxy' -> y/n
for minor in (9,10,11):
have['PY3%d' % minor] = (sys.version_info >= (3, minor))
for x, havex in have.items():
want = re.sub(r"%s\((.*)\)" % x, r"\1" if havex else "", want)
r = re.compile(r'^(?P<main>.*?) +(?P<y>(\+|-))%s$' % x)
v = []
for l in want.splitlines():
m = r.match(l)
if m is not None:
l = m.group('main')
y = {'+':True, '-':False}[m.group('y')]
if (y and not havex) or (havex and not y):
continue
v.append(l)
want = '\n'.join(v)+'\n'
# want: ^$ -> <BLANKLINE>
while "\n\n" in want:
......@@ -1818,8 +1876,11 @@ def assertDoc(want, got):
# ...
# fmtargspec(f) -> '(x, y=3)'
def fmtargspec(f): # -> str
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
# inspect.formatargspec is deprecated since py3.5 and was removed in py3.11
# -> use inspect.signature instead.
if six.PY3:
return str(inspect.signature(f))
else:
return inspect.formatargspec(*inspect.getargspec(f))
def test_fmtargspec():
......@@ -1828,8 +1889,10 @@ def test_fmtargspec():
# readfile returns content of file @path.
def readfile(path):
with open(path, "r") as f:
def readfile(path): # -> bytes
# on windows in text mode files are opened with encoding=locale.getdefaultlocale()
# which is CP125X instead of UTF-8. -> manually decode as 'UTF-8'
with open(path, "rb") as f:
return f.read()
# abbrev_home returns path with user home prefix abbreviated with ~.
......
golang/libgolang.h
View file @ 2b56ce71
#ifndef _NXD_LIBGOLANG_H
#define _NXD_LIBGOLANG_H
// Copyright (C) 2018-2022 Nexedi SA and Contributors.
// Copyright (C) 2018-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -176,6 +176,12 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#ifdef _MSC_VER // no mode_t on msvc
typedef int mode_t;
#endif
// DSO symbols visibility (based on https://gcc.gnu.org/wiki/Visibility)
#if defined _WIN32 || defined __CYGWIN__
#define LIBGOLANG_DSO_EXPORT __declspec(dllexport)
......@@ -577,7 +583,7 @@ int select(const _selcase (&casev)[N]) {
static inline // select(vector<casev>)
int select(const std::vector<_selcase> &casev) {
return _chanselect(&casev[0], casev.size());
return _chanselect(casev.data(), casev.size());
}
// defer(f) mimics `defer f()` from golang.
......@@ -829,7 +835,7 @@ struct _interface {
protected:
// don't use destructor -> use decref
~_interface();
LIBGOLANG_API ~_interface();
};
typedef refptr<_interface> interface;
......
golang/os.cpp
View file @ 2b56ce71
// Copyright (C) 2019-2022 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -37,7 +37,8 @@
// GLIBC < 2.32 provides sys_siglist but not sigdescr_np in its headers
// cut this short
// (on darwing sys_siglist declaration is normally provided)
#ifndef __APPLE__
// (on windows sys_siglist is not available at all)
#if !(defined(__APPLE__) || defined(_WIN32))
extern "C" {
extern const char * const sys_siglist[];
}
......@@ -226,8 +227,8 @@ tuple<string, error> ReadFile(const string& path) {
while (1) {
int n;
tie(n, err) = f->Read(&buf[0], buf.size());
data.append(&buf[0], n);
tie(n, err) = f->Read(buf.data(), buf.size());
data.append(buf.data(), n);
if (err != nil) {
if (err == io::EOF_)
err = nil;
......@@ -248,7 +249,7 @@ tuple<string, error> ReadFile(const string& path) {
tuple<File, File, error> Pipe() {
int vfd[2], syserr;
syserr = sys::Pipe(vfd);
syserr = sys::Pipe(vfd, 0);
if (syserr != 0)
return make_tuple(nil, nil, fmt::errorf("pipe: %w", sys::NewErrno(syserr)));
......@@ -286,8 +287,20 @@ string Signal::String() const {
const Signal& sig = *this;
const char *sigstr = nil;
#ifdef _WIN32
switch (sig.signo) {
case SIGABRT: return "Aborted";
case SIGBREAK: return "Break";
case SIGFPE: return "Floating point exception";
case SIGILL: return "Illegal instruction";
case SIGINT: return "Interrupt";
case SIGSEGV: return "Segmentation fault";
case SIGTERM: return "Terminated";
}
#else
if (0 <= sig.signo && sig.signo < NSIG)
sigstr = ::sys_siglist[sig.signo]; // might be nil as well
#endif
if (sigstr != nil)
return string(sigstr);
......
golang/os.h
View file @ 2b56ce71
#ifndef _NXD_LIBGOLANG_OS_H
#define _NXD_LIBGOLANG_OS_H
//
// Copyright (C) 2019-2022 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -61,7 +61,7 @@ private:
~_File();
friend File _newFile(_libgolang_ioh* ioh, const string& name);
public:
void decref();
LIBGOLANG_API void decref();
public:
LIBGOLANG_API string Name() const;
......@@ -95,9 +95,15 @@ private:
// Open opens file @path.
LIBGOLANG_API std::tuple<File, error> Open(const string &path, int flags = O_RDONLY,
mode_t mode = S_IRUSR | S_IWUSR | S_IXUSR |
mode_t mode =
#if !defined(_MSC_VER)
S_IRUSR | S_IWUSR | S_IXUSR |
S_IRGRP | S_IWGRP | S_IXGRP |
S_IROTH | S_IWOTH | S_IXOTH);
S_IROTH | S_IWOTH | S_IXOTH
#else
_S_IREAD | _S_IWRITE
#endif
);
// NewFile wraps OS-level file-descriptor into File.
// The ownership of sysfd is transferred to File.
......
golang/os/signal.cpp
View file @ 2b56ce71
// Copyright (C) 2021-2022 Nexedi SA and Contributors.
// Copyright (C) 2021-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -89,6 +89,10 @@
#include <atomic>
#include <tuple>
#if defined(_WIN32)
# include <windows.h>
#endif
#define DEBUG 0
#if DEBUG
......@@ -97,6 +101,12 @@
# define debugf(format, ...) do {} while (0)
#endif
#if defined(_MSC_VER)
# define HAVE_SIGACTION 0
#else
# define HAVE_SIGACTION 1
#endif
// golang::os::signal::
namespace golang {
namespace os {
......@@ -109,13 +119,30 @@ using std::tie;
using std::vector;
using cxx::set;
#if HAVE_SIGACTION
static void xsigemptyset(sigset_t *sa_mask);
#else
// custom `struct sigaction` emulated via signal(2)
#define SA_SIGINFO 1
typedef struct {} siginfo_t;
struct sigaction {
union {
void (*sa_handler)(int);
void (*sa_sigaction)(int, siginfo_t*, void*);
};
int sa_flags;
};
static void _os_sighandler_nosigaction(int sig);
#endif
static void _os_sighandler(int sig, siginfo_t *info, void *ucontext);
static void _notify(int signo);
static void _checksig(int signo);
static void _checkActEqual(const struct sigaction *a, const struct sigaction *b);
static void _spinwaitNextQueueCycle();
static int sys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact);
static void xsys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact);
static void xsigemptyset(sigset_t *sa_mask);
static bool _sigact_equal(const struct sigaction *a, const struct sigaction *b);
......@@ -160,27 +187,31 @@ void _init() {
// create _wakerx <-> _waketx pipe; set _waketx to nonblocking mode
int vfd[2];
if (sys::Pipe(vfd) < 0)
if (sys::Pipe(vfd, O_CLOEXEC) < 0)
panic("pipe(_wakerx, _waketx)"); // TODO +syserr
if (sys::Fcntl(vfd[0], F_SETFD, FD_CLOEXEC) < 0)
panic("fcntl(_wakerx, FD_CLOEXEC)"); // TODO +syserr
error err;
tie(_wakerx, err) = os::NewFile(vfd[0], "_wakerx");
if (err != nil)
panic("os::newFile(_wakerx");
_waketx = vfd[1];
#ifndef _WIN32
if (sys::Fcntl(_waketx, F_SETFL, O_NONBLOCK) < 0)
panic("fcntl(_waketx, O_NONBLOCK)"); // TODO +syserr
if (sys::Fcntl(_waketx, F_SETFD, FD_CLOEXEC) < 0)
panic("fcntl(_waketx, FD_CLOEXEC)"); // TODO +syserr
#else
HANDLE hwaketx = (HANDLE)_get_osfhandle(_waketx);
DWORD mode = PIPE_READMODE_BYTE | PIPE_NOWAIT;
if (!SetNamedPipeHandleState(hwaketx, &mode, NULL, NULL))
panic("SetNamedPipeHandleState(hwaketx, PIPE_NOWAIT)"); // TODO +syserr
#endif
_actIgnore.sa_handler = SIG_IGN;
_actIgnore.sa_flags = 0;
xsigemptyset(&_actIgnore.sa_mask);
_actNotify.sa_sigaction = _os_sighandler;
_actNotify.sa_flags = SA_SIGINFO;
#if HAVE_SIGACTION
xsigemptyset(&_actIgnore.sa_mask);
xsigemptyset(&_actNotify.sa_mask);
#endif
}
......@@ -248,6 +279,14 @@ done:
if (sah != SIG_IGN) {
if (sah != SIG_DFL) {
sah(sig);
// handlers installed via signal usually reinstall themselves
// return it back to us
struct sigaction after;
xsys_sigaction(sig, &_actNotify, &after);
if (! (_sigact_equal(&after, &_actNotify) ||
( ((after.sa_flags & SA_SIGINFO) == 0) &&
((after.sa_handler == SIG_DFL) || (after.sa_handler == sah)))))
panic("collision detected wrt thirdparty signal usage");
}
else {
// SIG_DFL && _SigReset - reraise to die if the signal is fatal
......@@ -343,7 +382,7 @@ static int/*syserr*/ _Notify1(chan<os::Signal> ch, os::Signal sig) {
// retrieve current signal action
struct sigaction cur;
int syserr = sys::Sigaction(sig.signo, nil, &cur);
int syserr = sys_sigaction(sig.signo, nil, &cur);
if (syserr < 0) {
// TODO reenable once we can panic with any object
//return fmt::errorf("sigaction sig%d: %w", sig.signo, sys::NewErrno(syserr);
......@@ -389,7 +428,7 @@ static int/*syserr*/ _Notify1(chan<os::Signal> ch, os::Signal sig) {
// register our sigaction
struct sigaction old;
syserr = sys::Sigaction(sig.signo, &_actNotify, &old);
syserr = sys_sigaction(sig.signo, &_actNotify, &old);
if (syserr < 0) {
// TODO reenable once we can panic with any object
//return fmt::errorf("sigaction sig%d: %w", sig.signo, sys::NewErrno(syserr);
......@@ -469,7 +508,7 @@ static int/*syserr*/ _Ignore1(os::Signal sig) {
if (h == nil) {
h = new _SigHandler();
h->sigstate.store(_SigIgnoring);
int syserr = sys::Sigaction(sig.signo, nil, &h->prev_act);
int syserr = sys_sigaction(sig.signo, nil, &h->prev_act);
if (syserr < 0) {
delete h;
return syserr; // TODO errctx
......@@ -481,7 +520,7 @@ static int/*syserr*/ _Ignore1(os::Signal sig) {
h->sigstate.store(_SigIgnoring);
h->subscribers.clear();
int syserr = sys::Sigaction(sig.signo, &_actIgnore, nil);
int syserr = sys_sigaction(sig.signo, &_actIgnore, nil);
if (syserr < 0)
return syserr; // TODO errctx
......@@ -509,7 +548,7 @@ static int/*syserr*/ _Reset1(os::Signal sig) {
h->sigstate.store(_SigReset);
struct sigaction act;
int syserr = sys::Sigaction(sig.signo, &h->prev_act, &act);
int syserr = sys_sigaction(sig.signo, &h->prev_act, &act);
if (syserr < 0)
return syserr; // TODO errctx
if (sigstate == _SigNotifying)
......@@ -617,16 +656,75 @@ static void _checksig(int signo) {
}
#if HAVE_SIGACTION
static void xsigemptyset(sigset_t *sa_mask) {
if (sigemptyset(sa_mask) < 0)
panic("sigemptyset failed"); // must always succeed
}
#endif
static void xsys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact) {
int syserr = sys::Sigaction(signo, act, oldact);
int syserr = sys_sigaction(signo, act, oldact);
if (syserr != 0)
panic("sigaction failed"); // TODO add errno detail
}
static int sys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact) {
#if HAVE_SIGACTION
return sys::Sigaction(signo, act, oldact);
#else
sys::sighandler_t oldh, newh;
int syserr;
if (act == nil) {
newh = SIG_GET;
}
else if (act->sa_flags & SA_SIGINFO) {
if (act->sa_sigaction != _os_sighandler)
panic("BUG: compat sigaction: act->sa_sigaction != _os_sighandler");
newh = _os_sighandler_nosigaction;
}
else {
newh = act->sa_handler;
}
oldh = sys::Signal(signo, newh, &syserr);
if (oldh == SIG_ERR)
return syserr;
if (oldact != nil) {
if (oldh == _os_sighandler_nosigaction) {
oldact->sa_sigaction = _os_sighandler;
oldact->sa_flags = SA_SIGINFO;
}
else {
oldact->sa_handler = oldh;
oldact->sa_flags = 0;
}
}
return 0;
#endif
}
#if !HAVE_SIGACTION
static void _os_sighandler_nosigaction(int sig) {
// reinstall sighandler since on handler invocation it is reset to SIG_DFL.
// do it as fast as we can as the first thing - to minimize window of race
// while signal handler is reset from being our _os_sighandler.
//
// NOTE it is ok to reinstall before invoking _os_sighandler because
// _os_sighandler uses only atomics and pipe write and so is reentrant.
sys::sighandler_t h;
int syserr;
h = sys::Signal(sig, _os_sighandler_nosigaction, &syserr);
if (h == SIG_ERR)
panic("signal(reinstall) failed");
if ( !(h == SIG_DFL /*reset*/ ||
h == _os_sighandler_nosigaction /*concurrent sighandler*/) )
panic("collision detected wrt thirdparty signal usage");
// invoke _os_sighandler as if it was setup by sigaction.
_os_sighandler(sig, NULL, NULL);
}
#endif
static bool _sigact_equal(const struct sigaction *a, const struct sigaction *b) {
// don't compare sigaction by memcmp - it will fail because struct sigaction
......
golang/os/signal_test.py
View file @ 2b56ce71
This diff is collapsed.
golang/os/testprog/signal_test_all.py
View file @ 2b56ce71
#!/usr/bin/env python
# Copyright (C) 2021-2022 Nexedi SA and Contributors.
# Copyright (C) 2021-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -24,7 +24,8 @@ from __future__ import print_function, absolute_import
from golang import chan
from golang import os as gos, syscall, time
from golang.os import signal
import os, sys
from golang.os.signal_test import killme
import sys
def main():
# build "all signals" list
......@@ -35,11 +36,17 @@ def main():
if sig not in allsigv: # avoid e.g. SIGCHLD/SIGCLD dups
allsigv.append(sig)
allsigv.sort(key=lambda sig: sig.signo)
allsigv.remove(syscall.SIGKILL) # SIGKILL/SIGSTOP cannot be caught
allsigv.remove(syscall.SIGSTOP)
allsigv.remove(syscall.SIGBUS) # AddressSanitizer crashes on SIGBUS/SIGFPE/SIGSEGV
allsigv.remove(syscall.SIGFPE)
allsigv.remove(syscall.SIGSEGV)
def without(signame):
sig = getattr(syscall, signame, None)
if sig is not None:
allsigv.remove(sig)
without('SIGKILL') # SIGKILL/SIGSTOP cannot be caught
without('SIGSTOP')
without('SIGBUS') # AddressSanitizer crashes on SIGBUS/SIGFPE/SIGSEGV
without('SIGFPE')
without('SIGSEGV')
without('SIGILL') # SIGILL/SIGABRT cause termination on windows
without('SIGABRT')
# Notify() -> kill * -> should be notified
ch = chan(10, dtype=gos.Signal)
......@@ -72,11 +79,6 @@ def main():
killme(syscall.SIGTERM)
raise AssertionError("not terminated")
# killme sends signal sig to own process.
def killme(sig):
mypid = os.getpid()
os.kill(mypid, sig.signo)
def emit(msg=''):
print(msg)
sys.stdout.flush()
......
golang/pyx/build.py
View file @ 2b56ce71
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -73,8 +73,10 @@ _dso_build_ext = setuptools_dso.build_ext
class build_ext(_dso_build_ext):
def build_extension(self, ext):
# wrap _compiler <src> -> <obj> with our code
# ._compile is used on gcc/clang but not with msvc
_compile = self.compiler._compile
def _(obj, src, ext, cc_args, extra_postargs, pp_opts):
def xcompile(obj, src, ext, cc_args, extra_postargs, pp_opts):
# filter_out removes arguments that start with argprefix
cc_args = cc_args[:]
extra_postargs = extra_postargs[:]
......@@ -101,11 +103,30 @@ class build_ext(_dso_build_ext):
filter_out('-std=gnu++')
_compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
self.compiler._compile = _
# msvc handles all sources directly in the loop in .compile and we can
# do per-source adjustsment only in .spawn .
spawn = self.compiler.spawn
def xspawn(argv):
c = False
for arg in argv:
if arg.startswith('/Tc'):
c = True
if c:
argv = argv[:]
for i in range(len(argv)):
if argv[i] == '/std:c++20':
argv[i] = '/std:c11'
return spawn(argv)
self.compiler._compile = xcompile
self.compiler.spawn = xspawn
try:
_dso_build_ext.build_extension(self, ext) # super doesn't work for _dso_build_ext
finally:
self.compiler._compile = _compile
self.compiler.spawn = spawn
# setup should be used instead of setuptools.setup
......@@ -128,14 +149,14 @@ def setup(**kw):
# x_dsos = [DSO('mypkg.mydso', ['mypkg/mydso.cpp'])],
# )
def DSO(name, sources, **kw):
_, kw = _with_build_defaults(kw)
_, kw = _with_build_defaults(name, kw)
dso = setuptools_dso.DSO(name, sources, **kw)
return dso
# _with_build_defaults returns copy of kw amended with build options common for
# both DSO and Extension.
def _with_build_defaults(kw): # -> (pygo, kw')
def _with_build_defaults(name, kw): # -> (pygo, kw')
# find pygolang root
gopkg = _findpkg("golang")
pygo = dirname(gopkg.path) # .../pygolang/golang -> .../pygolang
......@@ -144,12 +165,19 @@ def _with_build_defaults(kw): # -> (pygo, kw')
kw = kw.copy()
sysname = platform.system().lower()
win = (sysname == 'windows')
msvc = win # TODO also support mingw ?
# prepend -I<pygolang> so that e.g. golang/libgolang.h is found
# same with -I<pygolang>/golang/_compat/<os>
incv = kw.get('include_dirs', [])[:]
incv.insert(0, pygo)
incv.insert(1, join(pygo, 'golang', '_compat', sysname))
kw['include_dirs'] = incv
# link with libgolang.so
# link with libgolang.so if it is not libgolang itself
if name != 'golang.runtime.libgolang':
dsov = kw.get('dsos', [])[:]
dsov.insert(0, 'golang.runtime.libgolang')
kw['dsos'] = dsov
......@@ -160,9 +188,20 @@ def _with_build_defaults(kw): # -> (pygo, kw')
# default to C++11 (chan[T] & co require C++11 features)
ccdefault = []
if lang == 'c++':
if not msvc:
if name == 'golang.runtime.libgolang':
ccdefault.append('-std=gnu++11') # not c++11 as linux/list.h uses typeof
else:
ccdefault.append('-std=c++11')
else:
# MSVC requries C++20 for designated struct initializers
ccdefault.append('/std:c++20')
# and make exception handling: "fully conformant", so that e.g. extern "C" `panic` works
ccdefault.extend(['/EHc-', '/EHsr'])
# default to no strict-aliasing
ccdefault.append('-fno-strict-aliasing')
if not msvc:
ccdefault.append('-fno-strict-aliasing') # use only on gcc/clang - msvc does it by default
_ = kw.get('extra_compile_args', [])[:]
_[0:0] = ccdefault # if another e.g. -std=... was already there -
......@@ -188,6 +227,8 @@ def _with_build_defaults(kw): # -> (pygo, kw')
'os/signal.h',
'pyx/runtime.h',
'_testing.h',
'_compat/windows/strings.h',
'_compat/windows/unistd.h',
]])
kw['depends'] = dependv
......@@ -203,7 +244,7 @@ def _with_build_defaults(kw): # -> (pygo, kw')
# ext_modules = [Extension('mypkg.mymod', ['mypkg/mymod.pyx'])],
# )
def Extension(name, sources, **kw):
pygo, kw = _with_build_defaults(kw)
pygo, kw = _with_build_defaults(name, kw)
# some pyx-level depends to workaround a bit lack of proper dependency
# tracking in setuptools/distutils.
......
golang/pyx/runtime.h
View file @ 2b56ce71
#ifndef _NXD_LIBGOLANG_PYX_RUNTIME_H
#define _NXD_LIBGOLANG_PYX_RUNTIME_H
// Copyright (C) 2018-2019 Nexedi SA and Contributors.
// Copyright (C) 2018-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -58,8 +58,8 @@ class _PyError final : public _error, object {
private:
_PyError();
~_PyError();
friend error PyErr_Fetch();
friend void PyErr_ReRaise(PyError pyerr);
friend LIBPYXRUNTIME_API error PyErr_Fetch();
friend LIBPYXRUNTIME_API void PyErr_ReRaise(PyError pyerr);
public:
LIBPYXRUNTIME_API void incref();
LIBPYXRUNTIME_API void decref();
......
golang/pyx/testprog/cmdclass_custom.py
View file @ 2b56ce71
# Copyright (C) 2019 Nexedi SA and Contributors.
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -29,4 +29,9 @@ class mybuild_ext(build_ext):
setup(
cmdclass = {'build_ext': mybuild_ext},
# avoid setuptools thinking nearby golang_pyx_user/ golang_dso_user/ also
# relate to hereby setup and rejecting the build.
# https://stackoverflow.com/questions/72294299/multiple-top-level-packages-discovered-in-a-flat-layout
py_modules = [],
)
golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
View file @ 2b56ce71
// Copyright (C) 2019 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -20,7 +20,7 @@
// Small library that uses a bit of libgolang features, mainly to verify
// that external project can build against libgolang.
#include <golang/libgolang.h>
#include "dso.h"
using namespace golang;
#include <stdio.h>
......
golang/pyx/testprog/golang_dso_user/dsouser/dso.h 0 → 100644
View file @ 2b56ce71
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
#ifndef _NXD_GOLANG_DSOUSER_DSO_H
#define _NXD_GOLANG_DSOUSER_DSO_H
#include <golang/libgolang.h>
#if BUILDING_DSOUSER_DSO
# define DSOUSER_DSO_API LIBGOLANG_DSO_EXPORT
#else
# define DSOUSER_DSO_API LIBGOLANG_DSO_IMPORT
#endif
DSOUSER_DSO_API void dsotest();
#endif // _NXD_GOLANG_DSOUSER_DSO_H
golang/pyx/testprog/golang_dso_user/dsouser/test.pyx
View file @ 2b56ce71
# cython: language_level=2
#
# Copyright (C) 2019 Nexedi SA and Contributors.
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -21,10 +21,7 @@
# Small test driver that calls dso.so .
cdef extern from * nogil:
"""
void dsotest();
"""
cdef extern from "dso.h" nogil:
void dsotest()
def main():
......
golang/pyx/testprog/golang_dso_user/setup.py
View file @ 2b56ce71
# Copyright (C) 2019 Nexedi SA and Contributors.
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -24,7 +24,9 @@ setup(
name = 'golang_dso_user',
description = 'test project that uses libgolang from a dso',
x_dsos = [DSO('dsouser.dso', ['dsouser/dso.cpp'])],
x_dsos = [DSO('dsouser.dso',
['dsouser/dso.cpp'],
define_macros = [('BUILDING_DSOUSER_DSO', None)])],
ext_modules = [Extension('dsouser.test',
['dsouser/test.pyx'],
dsos = ['dsouser.dso'])],
......
golang/runtime/_runtime_gevent.pyx
View file @ 2b56ce71
# cython: language_level=2
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -57,7 +57,7 @@ from posix.fcntl cimport mode_t, F_GETFL, F_SETFL, O_NONBLOCK, O_ACCMODE, O_RDON
from posix.stat cimport struct_stat, S_ISREG, S_ISDIR, S_ISBLK
from posix.strings cimport bzero
from gevent.fileobject import FileObjectThread, FileObjectPosix
from gevent import fileobject as gfileobj
# _goviapy & _togo serve go
......@@ -194,6 +194,7 @@ cdef nogil:
return ioh
_libgolang_ioh* _io_fdopen(int *out_syserr, int sysfd):
IF POSIX:
# check if we should enable O_NONBLOCK on this file-descriptor
# even though we could enable O_NONBLOCK for regular files, it does not
# work as expected as most unix'es report regular files as always read
......@@ -220,6 +221,13 @@ cdef nogil:
out_syserr[0] = syserr
return NULL
ELSE: # !POSIX (windows)
cdef bint blocking = True
# FIXME acc: use GetFileInformationByHandleEx to determine whether
# HANDLE(sysfd) was opened for reading or writing.
# https://stackoverflow.com/q/9442436/9456786
cdef int acc = O_RDWR
# create IOH backed by FileObjectThread or FileObjectPosix
ioh = <IOH*>calloc(1, sizeof(IOH))
if ioh == NULL:
......@@ -253,9 +261,9 @@ cdef:
pygfobj = None
try:
if blocking:
pygfobj = FileObjectThread(sysfd, mode=mode, buffering=0)
pygfobj = gfileobj.FileObjectThread(sysfd, mode=mode, buffering=0)
else:
pygfobj = FileObjectPosix(sysfd, mode=mode, buffering=0)
pygfobj = gfileobj.FileObjectPosix(sysfd, mode=mode, buffering=0)
except OSError as e:
out_syserr[0] = -e.errno
else:
......@@ -338,7 +346,19 @@ cdef:
cdef uint8_t[::1] mem = <uint8_t[:count]>buf
xmem = memoryview(mem) # to avoid https://github.com/cython/cython/issues/3900 on mem[:0]=b''
try:
n = pygfobj.readinto(xmem)
# NOTE buf might be on stack, so it must not be accessed, e.g. from
# FileObjectThread, while our greenlet is parked (see STACK_DEAD_WHILE_PARKED
# for details). -> Use intermediate on-heap buffer to protect from that.
#
# Also: we cannot use pygfobj.readinto due to
# https://github.com/gevent/gevent/pull/1948
#
# TODO use .readinto() when buf is not on stack and gevent is
# recent enough or pygfobj is not FileObjectThread.
#n = pygfobj.readinto(xmem)
buf2 = pygfobj.read(count)
n = len(buf2)
xmem[:n] = buf2
except OSError as e:
n = -e.errno
out_n[0] = n
......@@ -361,8 +381,14 @@ cdef:
bint _io_write(IOH* ioh, int* out_n, const void *buf, size_t count):
pygfobj = <object>ioh.pygfobj
cdef const uint8_t[::1] mem = <const uint8_t[:count]>buf
# NOTE buf might be on stack, so it must not be accessed, e.g. from
# FileObjectThread, while our greenlet is parked (see STACK_DEAD_WHILE_PARKED
# for details). -> Use intermediate on-heap buffer to protect from that.
buf2 = bytearray(mem)
try:
n = pygfobj.write(mem)
n = pygfobj.write(buf2)
except OSError as e:
n = -e.errno
out_n[0] = n
......
golang/runtime/internal/atomic.cpp
View file @ 2b56ce71
// Copyright (C) 2022 Nexedi SA and Contributors.
// Copyright (C) 2022-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -20,7 +20,9 @@
#include "golang/runtime/internal/atomic.h"
#include "golang/libgolang.h"
#ifndef _WIN32
#include <pthread.h>
#endif
// golang::internal::atomic::
namespace golang {
......@@ -41,9 +43,12 @@ static void _forkNewEpoch() {
}
void _init() {
// there is no fork on windows
#ifndef _WIN32
int e = pthread_atfork(/*prepare*/nil, /*inparent*/nil, /*inchild*/_forkNewEpoch);
if (e != 0)
panic("pthread_atfork failed");
#endif
}
......
golang/runtime/internal/syscall.cpp
View file @ 2b56ce71
// Copyright (C) 2021-2022 Nexedi SA and Contributors.
// Copyright (C) 2021-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -57,15 +57,18 @@ string _Errno::Error() {
_Errno& e = *this;
char ebuf[128];
bool ok;
#if __APPLE__
int x = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
if (x == 0)
return string(ebuf);
return "unknown error " + std::to_string(-e.syserr);
ok = (::strerror_r(-e.syserr, ebuf, sizeof(ebuf)) == 0);
#elif defined(_WIN32)
ok = (::strerror_s(ebuf, sizeof(ebuf), -e.syserr) == 0);
#else
char *estr = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
return string(estr);
#endif
if (ok)
return string(ebuf);
return "unknown error " + std::to_string(-e.syserr);
}
......@@ -99,6 +102,7 @@ __Errno Close(int fd) {
return err;
}
#ifndef _WIN32
__Errno Fcntl(int fd, int cmd, int arg) {
int save_errno = errno;
int err = ::fcntl(fd, cmd, arg);
......@@ -107,6 +111,7 @@ __Errno Fcntl(int fd, int cmd, int arg) {
errno = save_errno;
return err;
}
#endif
__Errno Fstat(int fd, struct ::stat *out_st) {
int save_errno = errno;
......@@ -119,6 +124,10 @@ __Errno Fstat(int fd, struct ::stat *out_st) {
int Open(const char *path, int flags, mode_t mode) {
int save_errno = errno;
#ifdef _WIN32 // default to open files in binary mode
if ((flags & (_O_TEXT | _O_BINARY)) == 0)
flags |= _O_BINARY;
#endif
int fd = ::open(path, flags, mode);
if (fd < 0)
fd = -errno;
......@@ -126,15 +135,39 @@ int Open(const char *path, int flags, mode_t mode) {
return fd;
}
__Errno Pipe(int vfd[2]) {
__Errno Pipe(int vfd[2], int flags) {
// supported flags: O_CLOEXEC
if (flags & ~(O_CLOEXEC))
return -EINVAL;
int save_errno = errno;
int err = ::pipe(vfd);
int err;
#ifdef __linux__
err = ::pipe2(vfd, flags);
#elif defined(_WIN32)
err = ::_pipe(vfd, 4096, flags | _O_BINARY);
#else
err = ::pipe(vfd);
if (err)
goto out;
if (flags & O_CLOEXEC) {
for (int i=0; i<2; i++) {
err = Fcntl(vfd[i], F_SETFD, FD_CLOEXEC);
if (err) {
Close(vfd[0]);
Close(vfd[1]);
goto out;
}
}
}
out:
#endif
if (err == -1)
err = -errno;
errno = save_errno;
return err;
}
#ifndef _WIN32
__Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact) {
int save_errno = errno;
int err = ::sigaction(signo, act, oldact);
......@@ -143,6 +176,17 @@ __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *
errno = save_errno;
return err;
}
#endif
sighandler_t Signal(int signo, sighandler_t handler, int *out_psyserr) {
int save_errno = errno;
sighandler_t oldh = ::signal(signo, handler);
*out_psyserr = (oldh == SIG_ERR
? (errno ? -errno : -EINVAL) // windows returns SIG_ERR/errno=0 for invalid signo
: 0);
errno = save_errno;
return oldh;
}
}}} // golang::internal::syscall::
golang/runtime/internal/syscall.h
View file @ 2b56ce71
#ifndef _NXD_LIBGOLANG_RUNTIME_INTERNAL_SYSCALL_H
#define _NXD_LIBGOLANG_RUNTIME_INTERNAL_SYSCALL_H
// Copyright (C) 2021-2022 Nexedi SA and Contributors.
// Copyright (C) 2021-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -59,15 +59,21 @@ error NewErrno(__Errno syserr); // TODO better return Errno directly.
// system calls
int/*n|err*/ Read(int fd, void *buf, size_t count);
int/*n|err*/ Write(int fd, const void *buf, size_t count);
__Errno Close(int fd);
__Errno Fcntl(int fd, int cmd, int arg);
__Errno Fstat(int fd, struct ::stat *out_st);
int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
__Errno Pipe(int vfd[2]);
__Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
LIBGOLANG_API int/*n|err*/ Read(int fd, void *buf, size_t count);
LIBGOLANG_API int/*n|err*/ Write(int fd, const void *buf, size_t count);
LIBGOLANG_API __Errno Close(int fd);
#ifndef _WIN32
LIBGOLANG_API __Errno Fcntl(int fd, int cmd, int arg);
#endif
LIBGOLANG_API __Errno Fstat(int fd, struct ::stat *out_st);
LIBGOLANG_API int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
LIBGOLANG_API __Errno Pipe(int vfd[2], int flags);
#ifndef _WIN32
LIBGOLANG_API __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
#endif
typedef void (*sighandler_t)(int);
LIBGOLANG_API sighandler_t /*sigh|SIG_ERR*/ Signal(int signo, sighandler_t handler, int *out_psyserr);
}}} // golang::internal::syscall::
......
golang/runtime/libgolang.cpp
View file @ 2b56ce71
// Copyright (C) 2018-2022 Nexedi SA and Contributors.
// Copyright (C) 2018-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -43,12 +43,23 @@
#include <stdlib.h>
#include <string.h>
#include <strings.h>
// linux/list.h needs ARRAY_SIZE XXX -> better use c.h or ccan/array_size.h ?
#ifndef ARRAY_SIZE
# define ARRAY_SIZE(A) (sizeof(A) / sizeof((A)[0]))
#endif
#include <linux/list.h>
// MSVC does not support statement expressions and typeof
// -> redo list_entry via C++ lambda.
#ifdef _MSC_VER
# undef list_entry
# define list_entry(ptr, type, member) [&]() { \
const decltype( ((type *)0)->member ) *__mptr = (ptr); \
return (type *)( (char *)__mptr - offsetof(type,member) ); \
}()
#endif
using std::atomic;
using std::bad_alloc;
......@@ -899,6 +910,13 @@ template<> int _chanselect2</*onstack=*/true> (const _selcase *, int, const vect
template<> int _chanselect2</*onstack=*/false>(const _selcase *, int, const vector<int>&);
static int __chanselect2(const _selcase *, int, const vector<int>&, _WaitGroup*);
// PRNG for select.
// TODO consider switching to xoroshiro or wyrand if speed is an issue
// https://thompsonsed.co.uk/random-number-generators-for-c-performance-tested
// https://nullprogram.com/blog/2017/09/21/
static std::random_device _devrand;
static thread_local std::mt19937 _t_rng(_devrand());
// _chanselect executes one ready send or receive channel case.
//
// if no case is ready and default case was provided, select chooses default.
......@@ -925,7 +943,7 @@ int _chanselect(const _selcase *casev, int casec) {
vector<int> nv(casec); // n -> n(case) TODO -> caller stack-allocate nv
for (int i=0; i <casec; i++)
nv[i] = i;
std::random_shuffle(nv.begin(), nv.end());
std::shuffle(nv.begin(), nv.end(), _t_rng);
// first pass: poll all cases and bail out in the end if default was provided
int ndefault = -1;
......
golang/sync.h
View file @ 2b56ce71
#ifndef _NXD_LIBGOLANG_SYNC_H
#define _NXD_LIBGOLANG_SYNC_H
// Copyright (C) 2018-2020 Nexedi SA and Contributors.
// Copyright (C) 2018-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -212,7 +212,7 @@ class _WorkGroup : public object {
private:
_WorkGroup();
~_WorkGroup();
friend WorkGroup NewWorkGroup(context::Context ctx);
friend LIBGOLANG_API WorkGroup NewWorkGroup(context::Context ctx);
public:
LIBGOLANG_API void decref();
......
golang/testprog/golang_test_defer_excchain.txt
View file @ 2b56ce71
Traceback (most recent call last):
File "PYGOLANG/golang/__init__.py", line ..., in _
return f(*argv, **kw)
^^^^^^^^^^^^^^ +PY311
File "PY39(PYGOLANG/golang/testprog/)golang_test_defer_excchain.py", line 42, in main
raise RuntimeError("err")
RuntimeError: err
......@@ -21,6 +22,7 @@ Traceback (most recent call last):
...
File "PY39(PYGOLANG/golang/testprog/)golang_test_defer_excchain.py", line 33, in d2
1/0
~^~ +PY311
ZeroDivisionError: ...
During handling of the above exception, another exception occurred:
......@@ -30,7 +32,8 @@ Traceback (most recent call last):
main()
...
File "PYGOLANG/golang/__init__.py", line ..., in _
return f(*argv, **kw)
return f(*argv, **kw) -PY310
with __goframe__: +PY310
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
...
File "PYGOLANG/golang/__init__.py", line ..., in __exit__
......
golang/testprog/golang_test_defer_excchain.txt-pytest
View file @ 2b56ce71
...
_____________________________________ main _____________________________________
...________________ main ________________...
../__init__.py:...: in _
return f(*argv, **kw)
golang_test_defer_excchain.py:42: in main
......
golang/testprog/golang_test_str.py
View file @ 2b56ce71
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2022 Nexedi SA and Contributors.
# Copyright (C) 2022-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -29,8 +29,8 @@ from golang import b, u
from golang.gcompat import qq
def main():
sb = b("привет b")
su = u("привет u")
sb = b("привет αβγ b")
su = u("привет αβγ u")
print("print(b):", sb)
print("print(u):", su)
print("print(qq(b)):", qq(sb))
......
golang/testprog/golang_test_str.txt
View file @ 2b56ce71
print(b): привет b
print(u): привет u
print(qq(b)): "привет b"
print(qq(u)): "привет u"
print(repr(b)): b('привет b')
print(repr(u)): u('привет u')
print({b: u}): {b('привет b'): u('привет u')}
print(b): привет αβγ b
print(u): привет αβγ u
print(qq(b)): "привет αβγ b"
print(qq(u)): "привет αβγ u"
print(repr(b)): b('привет αβγ b')
print(repr(u)): u('привет αβγ u')
print({b: u}): {b('привет αβγ b'): u('привет αβγ u')}
golang/time.h
View file @ 2b56ce71
#ifndef _NXD_LIBGOLANG_TIME_H
#define _NXD_LIBGOLANG_TIME_H
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -123,7 +123,7 @@ private:
private:
_Ticker();
~_Ticker();
friend Ticker new_ticker(double dt);
friend Ticker LIBGOLANG_API new_ticker(double dt);
public:
LIBGOLANG_API void decref();
......
gpython/__init__.py
View file @ 2b56ce71
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Copyright (C) 2018-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -50,19 +50,27 @@ _pyopt_long = ('version',)
# init, if provided, is called after options are parsed, but before interpreter start.
def pymain(argv, init=None):
import sys
from os.path import dirname, realpath
import os
from os.path import dirname, realpath, splitext
# sys.executable
# on windows there are
# gpython-script.py
# gpython.exe
# gpython-script.py
# gpython.manifest
# and argv[0] is gpython-script.py
# with gpython-script.py sometimes embedded into gpython.exe (pip/distlib)
# and argv[0] is 'gpython-script.py' (setuptools) or 'gpython' (pip/distlib; note no .exe)
exe = realpath(argv[0])
argv = argv[1:]
if os.name == 'nt':
if exe.endswith('-script.py'):
exe = exe[:-len('-script.py')]
exe = exe[:-len('-script.py')] # gpython-script.py -> gpython.exe
exe = exe + '.exe'
else:
_, ext = splitext(exe) # gpython -> gpython.exe
if not ext:
exe += '.exe'
sys._gpy_underlying_executable = sys.executable
sys.executable = exe
......@@ -70,20 +78,26 @@ def pymain(argv, init=None):
# `gpython file` will add path-to-file to sys.path[0] by itself, and
# /path/to/gpython is unnecessary and would create difference in behaviour
# in between gpython and python.
#
# on windows when gpython.exe comes with embedded __main__.py, it is
# gpython.exe that is installed into sys.path[0] .
exedir = dirname(exe)
if sys.path[0] == exedir:
if sys.path[0] in (exedir, exe):
del sys.path[0]
else:
# buildout injects `sys.path[0:0] = eggs` into python scripts.
# detect that and remove sys.path entry corresponding to exedir.
if not _is_buildout_script(exe):
raise RuntimeError('pymain: internal error: sys.path[0] was not set by underlying python to dirname(exe):'
raise RuntimeError('pymain: internal error: sys.path[0] was not set by underlying python to dirname(exe) or exe:'
'\n\n\texe:\t%s\n\tsys.path[0]:\t%s' % (exe, sys.path[0]))
else:
if exedir in sys.path:
sys.path.remove(exedir)
else:
raise RuntimeError('pymain: internal error: sys.path does not contain dirname(exe):'
ok = False
for _ in (exedir, exe):
if _ in sys.path:
sys.path.remove(_)
ok = True
if not ok:
raise RuntimeError('pymain: internal error: sys.path does not contain dirname(exe) or exe:'
'\n\n\texe:\t%s\n\tsys.path:\t%s' % (exe, sys.path))
......@@ -202,7 +216,6 @@ def pymain(argv, init=None):
#
# python -O gpython file.py
if len(reexec_with) > 0:
import os
argv = [sys._gpy_underlying_executable] + reexec_with + [sys.executable] + reexec_argv
os.execv(argv[0], argv)
......@@ -439,13 +452,13 @@ def main():
# _is_buildout_script returns whether file @path is generated as python buildout script.
def _is_buildout_script(path):
with open(path, 'r') as f:
with open(path, 'rb') as f:
src = f.read()
# buildout injects the following prologues into python scripts:
# sys.path[0:0] = [
# ...
# ]
return ('\nsys.path[0:0] = [\n' in src)
return (b'\nsys.path[0:0] = [\n' in src)
# _IGetOpt provides getopt-style incremental options parsing.
......
gpython/gpython_test.py
View file @ 2b56ce71
# -*- coding: utf-8 -*-
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -184,7 +184,7 @@ def test_pymain():
_ = pyout(['-m', 'hello', 'abc', 'def'], cwd=testdata)
# realpath rewrites e.g. `local/lib -> lib` if local/lib is symlink
hellopy = realpath(join(testdata, 'hello.py'))
assert _ == b"hello\nworld\n['%s', 'abc', 'def']\n" % b(hellopy)
assert _ == b"hello\nworld\n[%s, 'abc', 'def']\n" % b(repr(hellopy))
# -m<module>
__ = pyout(['-mhello', 'abc', 'def'], cwd=testdata)
assert __ == _
......@@ -195,7 +195,7 @@ def test_pymain():
# -i after stdin (also tests interactive mode as -i forces interactive even on non-tty)
d = {
b'hellopy': b(hellopy),
b'repr(hellopy)': b(repr(hellopy)),
b'ps1': b'' # cpython emits prompt to stderr
}
if is_pypy and not is_gpython:
......@@ -212,7 +212,7 @@ def test_pymain():
assert _ == b"hello\nworld\n['-c']\n%(ps1)s'~~HELLO~~'\n%(ps1)s" % d
# -i after -m
_ = pyout(['-i', '-m', 'hello'], stdin=b'world.tag', cwd=testdata)
assert _ == b"hello\nworld\n['%(hellopy)s']\n%(ps1)s'~~WORLD~~'\n%(ps1)s" % d
assert _ == b"hello\nworld\n[%(repr(hellopy))s]\n%(ps1)s'~~WORLD~~'\n%(ps1)s" % d
# -i after file
_ = pyout(['-i', 'testdata/hello.py'], stdin=b'tag', cwd=here)
assert _ == b"hello\nworld\n['testdata/hello.py']\n%(ps1)s'~~HELLO~~'\n%(ps1)s" % d
......
gpython/testprog/print_opt.py
View file @ 2b56ce71
# -*- coding: utf-8 -*-
# Copyright (C) 2020 Nexedi SA and Contributors.
# Copyright (C) 2020-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -62,8 +62,9 @@ def modpy_imports_from():
raise AssertionError("module 'mod' is already there")
tmpd = tempfile.mkdtemp('', 'modpy_imports_from')
tmpd_ = tmpd + os.path.sep
try:
pymod = "%s/mod.py" % tmpd
pymod = tmpd_ + "mod.py"
with open(pymod, "w") as f:
f.write("# hello up there\n")
......@@ -73,9 +74,9 @@ def modpy_imports_from():
files = set()
for dirpath, dirnames, filenames in os.walk(tmpd):
for _ in filenames:
f = '%s/%s' % (dirpath, _)
if f.startswith(tmpd+'/'):
f = f[len(tmpd+'/'):]
f = os.path.join(dirpath, _)
if f.startswith(tmpd_):
f = f[len(tmpd_):]
files.add(f)
......
pyproject.toml
View file @ 2b56ce71
[build-system]
requires = ["setuptools", "wheel", "setuptools_dso >= 1.7", "cython", "gevent"]
requires = ["setuptools", "wheel", "setuptools_dso >= 2.7", "cython", "gevent"]
setup.py
View file @ 2b56ce71
# pygolang | pythonic package setup
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -19,28 +19,27 @@
# See https://www.nexedi.com/licensing for rationale and options.
from setuptools import find_packages
# setuptools has Library but this days it is not well supported and test for it
# has been killed https://github.com/pypa/setuptools/commit/654c26f78a30
# -> use setuptools_dso instead.
from setuptools_dso import DSO
from setuptools.command.install_scripts import install_scripts as _install_scripts
from setuptools.command.develop import develop as _develop
from distutils import sysconfig
from os.path import dirname, join
import sys, re
import sys, os, re
# read file content
def readfile(path):
with open(path, 'r') as f:
return f.read()
# reuse golang.pyx.build to build pygolang extensions.
def readfile(path): # -> str
with open(path, 'rb') as f:
data = f.read()
if not isinstance(data, str): # py3
data = data.decode('utf-8')
return data
# reuse golang.pyx.build to build pygolang dso and extensions.
# we have to be careful and inject synthetic golang package in order to be
# able to import golang.pyx.build without built/working golang.
trun = {}
exec(readfile('trun'), trun)
trun['ximport_empty_golangmod']()
from golang.pyx.build import setup, Extension as Ext
from golang.pyx.build import setup, DSO, Extension as Ext
# grep searches text for pattern.
......@@ -156,7 +155,7 @@ class develop(XInstallGPython, _develop):
# requirements of packages under "golang." namespace
R = {
'cmd.pybench': {'pytest'},
'pyx.build': {'setuptools', 'wheel', 'cython', 'setuptools_dso >= 1.7'},
'pyx.build': {'setuptools', 'wheel', 'cython', 'setuptools_dso >= 2.7'},
'x.perf.benchlib': {'numpy'},
}
# TODO generate `a.b -> a`, e.g. x.perf = join(x.perf.*); x = join(x.*)
......@@ -174,6 +173,14 @@ for k in sorted(R.keys()):
extras_require[k] = list(sorted(R[k]))
# get_python_libdir() returns path where libpython is located
def get_python_libdir():
# mimic what distutils.command.build_ext does
if os.name == 'nt':
return join(sysconfig.get_config_var('installed_platbase'), 'libs')
else:
return sysconfig.get_config_var('LIBDIR')
setup(
name = 'pygolang',
version = version,
......@@ -223,19 +230,17 @@ setup(
'golang/strings.h',
'golang/sync.h',
'golang/time.h'],
include_dirs = ['.', '3rdparty/include'],
include_dirs = ['3rdparty/include'],
define_macros = [('BUILDING_LIBGOLANG', None)],
extra_compile_args = ['-std=gnu++11'], # not c++11 as linux/list.h uses typeof
soversion = '0.1'),
DSO('golang.runtime.libpyxruntime',
['golang/runtime/libpyxruntime.cpp'],
depends = ['golang/pyx/runtime.h'],
include_dirs = ['.', sysconfig.get_python_inc()],
include_dirs = [sysconfig.get_python_inc()],
library_dirs = [get_python_libdir()],
define_macros = [('BUILDING_LIBPYXRUNTIME', None)],
extra_compile_args = ['-std=c++11'],
soversion = '0.1',
dsos = ['golang.runtime.libgolang'])],
soversion = '0.1')],
ext_modules = [
Ext('golang._golang',
......@@ -311,11 +316,12 @@ setup(
include_package_data = True,
install_requires = ['gevent', 'six', 'decorator', 'Importing;python_version<="2.7"',
# only runtime part: for dylink_prepare_dso
'setuptools_dso >= 2.7',
# pyx.build -> setuptools_dso uses multiprocessing
# FIXME geventmp fails on python2, but setuptools_dso
# uses multiprocessing only on Python3, so for now we
# are ok. https://github.com/karellen/geventmp/pull/2
'geventmp;python_version>="3"',
# setuptools_dso uses multiprocessing only on Python3, and only on systems where
# mp.get_start_method()!='fork', while geventmp does not work on windows.
'geventmp ; python_version>="3" and platform_system != "Windows" ',
],
extras_require = extras_require,
......@@ -345,8 +351,15 @@ setup(
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: 3.11
Programming Language :: Python :: Implementation :: CPython
Programming Language :: Python :: Implementation :: PyPy
Operating System :: POSIX
Operating System :: POSIX :: Linux
Operating System :: Unix
Operating System :: MacOS
Operating System :: Microsoft :: Windows
Topic :: Software Development :: Interpreters
Topic :: Software Development :: Libraries :: Python Modules\
""".splitlines()]
......
tox.ini
View file @ 2b56ce71
[tox]
envlist =
{py27d,py27,py37,py38d,py38,py39d,py39,pypy,pypy3}-{thread,gevent}
{py27d,py27,py37,py38,py39d,py39,py310d,py310,py311d,py311,pypy,pypy3}-{thread,gevent}
# ThreadSanitizer
......@@ -10,28 +10,31 @@ envlist =
# (*) PyPy locks its GIL (see RPyGilAcquire) by manually doing atomic cmpxchg
# and other games, which TSAN cannot see if PyPy itself was not compiled with
# -fsanitize=thread.
{py27d,py27,py37,py38d,py38,py39d,py39 }-{thread }-tsan
{py27d,py27,py37,py38,py39d,py39,py310d,py310,py311d,py311 }-{thread }-tsan
# XXX py*-gevent-tsan would be nice to have, but at present TSAN is not
# effective with gevent, because it does not understand greenlet "thread"
# switching and so perceives the program as having only one thread where races
# are impossible. Disabled to save time.
# {py27d,py27,py37,py38d,py38,py39d,py39 }-{ gevent}-tsan
# {py27d,py27,py37,py38,py39d,py39,py310d,py310,py311d,py311 }-{ gevent}-tsan
# AddressSanitizer
# XXX asan does not work with gevent: https://github.com/python-greenlet/greenlet/issues/113
{py27d,py27,py37,py38d,py38,py39d,py39,pypy,pypy3}-{thread }-asan
{py27d,py27,py37,py38,py39d,py39,py310d,py310,py311d,py311,pypy,pypy3}-{thread }-asan
[testenv]
basepython =
py27d: python2.7-dbg
py27: python2.7
py37: python3.7
py38d: python3.8-dbg
py38: python3.8
py39d: python3.9-dbg
py39: python3.9
py310d: python3.10-dbg
py310: python3.10
py311d: python3.11-dbg
py311: python3.11
pypy: pypy
pypy3: pypy3
......@@ -69,5 +72,5 @@ commands=
# asan/tsan: tell pytest not to capture output - else it is not possible to see
# reports from sanitizers because they crash tested process on error.
# likewise for python debug builds.
asan,tsan,py{27,37}d: -s \
asan,tsan,py{27,39,310,311}d: -s \
gpython/ golang/
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7