Commit 17dbfbac authored by Kirill Smelkov's avatar Kirill Smelkov

X My draft state of x/gpystr work; py2/py3 pickle problem should be essentially solved

parent ac751a56
[submodule "3rdparty/funchook"]
path = 3rdparty/funchook
url = https://github.com/kubo/funchook.git
[submodule "3rdparty/capstone"]
path = 3rdparty/capstone
url = https://github.com/capstone-engine/capstone.git
Subproject commit 097c04d9413c59a58b00d4d1c8d5dc0ac158ffaa
Subproject commit 88388db3c69e16c1560fee65c6857d75f5ce6fd5
......@@ -2,6 +2,9 @@ include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .nxdtest
include golang/libgolang.h
include golang/runtime/libgolang.cpp
include golang/runtime/libpyxruntime.cpp
include golang/runtime/platform.h
include golang/runtime.h
include golang/runtime.cpp
include golang/pyx/runtime.h
include golang/pyx/testprog/golang_dso_user/dsouser/dso.h
include golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
......@@ -36,7 +39,10 @@ include golang/time.cpp
include golang/_testing.h
include golang/_compat/windows/strings.h
include golang/_compat/windows/unistd.h
include gpython/_gpython_c.cpp
recursive-include golang *.py *.pxd *.pyx *.toml *.txt*
recursive-include gpython *.py
recursive-include 3rdparty *.h
recursive-include gpython *.py *.pyx
recursive-include 3rdparty *.h *.c *.cpp *.S *.py *.cmake *.cs *.java
recursive-include 3rdparty LICENSE README.md README COPYING Makefile CMakeLists.txt
recursive-exclude golang *_dsoinfo.py
include conftest.py
......@@ -4,7 +4,7 @@
Package `golang` provides Go-like features for Python:
- `gpython` is Python interpreter with support for lightweight threads.
- `gpython` is Python interpreter with support for lightweight threads and uniform UTF8-based approach to strings.
- `go` spawns lightweight thread.
- `chan` and `select` provide channels with Go semantic.
- `func` allows to define methods separate from class.
......@@ -46,15 +46,16 @@ __ http://libuv.org/
__ http://software.schmorp.de/pkg/libev.html
Additionally GPython sets UTF-8 to be default encoding always, and puts `go`,
`chan`, `select` etc into builtin namespace.
Additionally GPython sets UTF-8 to be default encoding always, puts `go`,
`chan`, `select` etc into builtin namespace, and makes `bstr`/`ustr` to be used
instead of builtin string types.
.. note::
GPython is optional and the rest of Pygolang can be used from under standard Python too.
However without gevent integration `go` spawns full - not lightweight - OS thread.
GPython can be also used with threads - not gevent - runtime. Please see
`GPython options`_ for details.
GPython can be also used with threads - not gevent - runtime and with builtin string types.
Please see `GPython options`_ for details.
Goroutines and channels
......@@ -571,3 +572,9 @@ GPython-specific options and environment variables are listed below:
coroutines, while with `threads` `go` spawns full OS thread. `gevent` is
default. The runtime to use can be also specified via `$GPYTHON_RUNTIME`
environment variable.
`-X gpython.strings=(bstr+ustr|pystd)`
Specify which string types GPython should use. `bstr+ustr` provide
uniform UTF8-based approach to strings, while `pystd` selects regular
`str` and `unicode`. `bstr+ustr` is default. String types to use can be
also specified via `$GPYTHON_STRINGS` environment variable.
# ignore tests in distorm - else it breaks as e.g.
#
# 3rdparty/funchook/distorm/python/test_distorm3.py:15: in <module>
# import distorm3
# 3rdparty/funchook/distorm/python/distorm3/__init__.py:57: in <module>
# _distorm = _load_distorm()
# 3rdparty/funchook/distorm/python/distorm3/__init__.py:55: in _load_distorm
# raise ImportError("Error loading the diStorm dynamic library (or cannot load library into process).")
# E ImportError: Error loading the diStorm dynamic library (or cannot load library into process).
collect_ignore = ["3rdparty"]
......@@ -3,7 +3,7 @@
# cython: binding=False
# cython: c_string_type=str, c_string_encoding=utf8
# distutils: language = c++
# distutils: depends = libgolang.h os/signal.h unicode/utf8.h _golang_str.pyx
# distutils: depends = libgolang.h os/signal.h unicode/utf8.h _golang_str.pyx _golang_str_pickle.pyx
#
# Copyright (C) 2018-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
......@@ -34,7 +34,7 @@ from __future__ import print_function, absolute_import
_init_libgolang()
_init_libpyxruntime()
from cpython cimport PyObject, Py_INCREF, Py_DECREF, PY_MAJOR_VERSION
from cpython cimport PyObject, Py_INCREF, Py_DECREF, Py_CLEAR, PY_MAJOR_VERSION
ctypedef PyObject *pPyObject # https://github.com/cython/cython/issues/534
cdef extern from "Python.h":
ctypedef struct PyTupleObject:
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
# -*- coding: utf-8 -*-
# Copyright (C) 2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
# test for inside_counted
def _test_inside_counted(): # -> outok
outok = ''
outok += '\n\n\nBEFORE PATCH\n'
print('\n\n\nBEFORE PATCH')
tfunc(3)
t0 = ''
for i in range(3,0-1,-1):
t0 += '> tfunc(%d)\tinside_counter: 0\n' % i
for i in range(0,3+1,+1):
t0 += '< tfunc(%d)\tinside_counter: 0\n' % i
outok += t0
outok += '\n\n\nPATCHED\n'
print('\n\n\nPATCHED')
_patch = xfunchook_create()
global inside_counted_func
inside_counted_func = <void*>&tfunc
xfunchook_prepare(_patch, &inside_counted_func, <void*>inside_counted)
xfunchook_install(_patch, 0)
tfunc(12)
stk_size = 8 # = STK_SIZE from _golang_str_pickle.S
for i in range(12,0-1,-1):
outok += '> tfunc(%d)\tinside_counter: %d\n' % (i, min(12-i+1, stk_size))
for i in range(0,12+1,+1):
outok += '< tfunc(%d)\tinside_counter: %d\n' % (i, min(12-i+1, stk_size))
outok += '\n\n\nUNPATCHED\n'
print('\n\n\nUNPATCHED')
xfunchook_uninstall(_patch, 0)
tfunc(3)
outok += t0
return outok
cdef void tfunc(int x):
print('> tfunc(%d)\tinside_counter: %d' % (x, inside_counter))
if x > 0:
tfunc(x-1)
print('< tfunc(%d)\tinside_counter: %d' % (x, inside_counter))
def _test_cfunc_is_callee_cleanup():
for t in _cfunc_is_callee_cleanup_testv:
stkclean = cfunc_is_callee_cleanup(t.cfunc)
assert stkclean == t.stkclean_by_callee_ok, (t.cfunc_name, stkclean, t.stkclean_by_callee_ok)
cdef extern from * nogil:
r"""
struct _Test_cfunc_is_callee_clenup {
const char* cfunc_name;
void* cfunc;
int stkclean_by_callee_ok;
};
#define CASE(func, stkclean_ok) \
_Test_cfunc_is_callee_clenup{#func, (void*)func, stkclean_ok}
#if defined(LIBGOLANG_ARCH_386)
int CALLCONV(cdecl)
tfunc_cdecl1(int x) { return x; }
int CALLCONV(cdecl)
tfunc_cdecl2(int x, int y) { return x; }
int CALLCONV(cdecl)
tfunc_cdecl3(int x, int y, int z) { return x; }
int CALLCONV(stdcall)
tfunc_stdcall1(int x) { return x; }
int CALLCONV(stdcall)
tfunc_stdcall2(int x, int y) { return x; }
int CALLCONV(stdcall)
tfunc_stdcall3(int x, int y, int z) { return x; }
int CALLCONV(fastcall)
tfunc_fastcall1(int x) { return x; }
int CALLCONV(fastcall)
tfunc_fastcall2(int x, int y) { return x; }
int CALLCONV(fastcall)
tfunc_fastcall3(int x, int y, int z) { return x; }
#ifndef LIBGOLANG_CC_msc // see note about C3865 in FOR_EACH_CALLCONV
int CALLCONV(thiscall)
tfunc_thiscall1(int x) { return x; }
int CALLCONV(thiscall)
tfunc_thiscall2(int x, int y) { return x; }
int CALLCONV(thiscall)
tfunc_thiscall3(int x, int y, int z) { return x; }
#endif
#ifndef LIBGOLANG_CC_msc // no regparm on MSCV
int CALLCONV(regparm(1))
tfunc_regparm1_1(int x) { return x; }
int CALLCONV(regparm(1))
tfunc_regparm1_2(int x, int y) { return x; }
int CALLCONV(regparm(1))
tfunc_regparm1_3(int x, int y, int z) { return x; }
int CALLCONV(regparm(2))
tfunc_regparm2_1(int x) { return x; }
int CALLCONV(regparm(2))
tfunc_regparm2_2(int x, int y) { return x; }
int CALLCONV(regparm(2))
tfunc_regparm2_3(int x, int y, int z) { return x; }
int CALLCONV(regparm(3))
tfunc_regparm3_1(int x) { return x; }
int CALLCONV(regparm(3))
tfunc_regparm3_2(int x, int y) { return x; }
int CALLCONV(regparm(3))
tfunc_regparm3_3(int x, int y, int z) { return x; }
#endif
static std::vector<_Test_cfunc_is_callee_clenup> _cfunc_is_callee_cleanup_testv = {
CASE(tfunc_cdecl1 , 0 * 4),
CASE(tfunc_cdecl2 , 0 * 4),
CASE(tfunc_cdecl3 , 0 * 4),
CASE(tfunc_stdcall1 , 1 * 4),
CASE(tfunc_stdcall2 , 2 * 4),
CASE(tfunc_stdcall3 , 3 * 4),
CASE(tfunc_fastcall1 , 0 * 4),
CASE(tfunc_fastcall2 , 0 * 4),
CASE(tfunc_fastcall3 , 1 * 4),
#ifndef LIBGOLANG_CC_msc
CASE(tfunc_thiscall1 , 0 * 4),
CASE(tfunc_thiscall2 , 1 * 4),
CASE(tfunc_thiscall3 , 2 * 4),
#endif
#ifndef LIBGOLANG_CC_msc
CASE(tfunc_regparm1_1 , 0 * 4),
CASE(tfunc_regparm1_2 , 0 * 4),
CASE(tfunc_regparm1_3 , 0 * 4),
CASE(tfunc_regparm2_1 , 0 * 4),
CASE(tfunc_regparm2_2 , 0 * 4),
CASE(tfunc_regparm2_3 , 0 * 4),
CASE(tfunc_regparm3_1 , 0 * 4),
CASE(tfunc_regparm3_2 , 0 * 4),
CASE(tfunc_regparm3_3 , 0 * 4),
#endif
};
#else
// only i386 has many calling conventions
int tfunc_default(int x, int y, int z) { return x; }
static std::vector<_Test_cfunc_is_callee_clenup> _cfunc_is_callee_cleanup_testv = {
CASE(tfunc_default, 0),
};
#endif
#undef CASE
"""
struct _Test_cfunc_is_callee_clenup:
const char* cfunc_name
void* cfunc
int stkclean_by_callee_ok
vector[_Test_cfunc_is_callee_clenup] _cfunc_is_callee_cleanup_testv
......@@ -28,12 +28,11 @@ from golang cimport pyb, byte, rune
from golang cimport _utf8_decode_rune, _xunichr
from golang.unicode cimport utf8
from cpython cimport PyObject
from cpython cimport PyObject, _PyBytes_Resize
cdef extern from "Python.h":
PyObject* PyBytes_FromStringAndSize(char*, Py_ssize_t) except NULL
char* PyBytes_AS_STRING(PyObject*)
int _PyBytes_Resize(PyObject**, Py_ssize_t) except -1
void Py_DECREF(PyObject*)
......@@ -65,7 +64,7 @@ cdef bytes _quote(const byte[::1] s, char quote, bint* out_nonascii_escape): # -
cdef byte c
q[0] = quote; q += 1
while i < len(s):
c = s[i]
c = s[i] # XXX -> use raw pointer in the loop
# fast path - ASCII only
if c < 0x80:
if c in (ord('\\'), quote):
......@@ -104,7 +103,8 @@ cdef bytes _quote(const byte[::1] s, char quote, bint* out_nonascii_escape): # -
# slow path - full UTF-8 decoding + unicodedata
else:
r, size = _utf8_decode_rune(s[i:])
# XXX optimize non-ascii case
r, size = _utf8_decode_rune(s[i:]) # XXX -> raw pointer
isize = i + size
# decode error - just emit raw byte as escaped
......@@ -117,6 +117,9 @@ cdef bytes _quote(const byte[::1] s, char quote, bint* out_nonascii_escape): # -
q += 4
# printable utf-8 characters go as is
# XXX ? use Py_UNICODE_ISPRINTABLE (py3, not available on py2) ?
# XXX ? and generate C table based on unicodedata for py2 ?
# XXX -> generate table based on unicodedata for both py2/py3 because Py_UNICODE_ISPRINTABLE is not exactly what matches strconv.IsPrint (i.e. cat starts from LNPS)
elif _unicodedata_category(_xunichr(r))[0] in 'LNPS': # letters, numbers, punctuation, symbols
for j in range(i, isize):
q[0] = s[j]
......
......@@ -111,7 +111,7 @@ inline error errorf(const string& format, Argv... argv) {
// `const char *` overloads just to catch format mistakes as
// __attribute__(format) does not work with std::string.
LIBGOLANG_API string sprintf(const char *format, ...)
#ifndef _MSC_VER
#ifndef LIBGOLANG_CC_msc
__attribute__ ((format (printf, 1, 2)))
#endif
;
......
This diff is collapsed.
This diff is collapsed.
......@@ -169,6 +169,8 @@
// [1] Libtask: a Coroutine Library for C and Unix. https://swtch.com/libtask.
// [2] http://9p.io/magic/man2html/2/thread.
#include "golang/runtime/platform.h"
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
......@@ -177,21 +179,18 @@
#include <sys/stat.h>
#include <fcntl.h>
#ifdef _MSC_VER // no mode_t on msvc
#ifdef LIBGOLANG_CC_msc // no mode_t on msvc
typedef int mode_t;
#endif
// DSO symbols visibility (based on https://gcc.gnu.org/wiki/Visibility)
#if defined _WIN32 || defined __CYGWIN__
#ifdef LIBGOLANG_OS_windows
#define LIBGOLANG_DSO_EXPORT __declspec(dllexport)
#define LIBGOLANG_DSO_IMPORT __declspec(dllimport)
#elif __GNUC__ >= 4
#else
#define LIBGOLANG_DSO_EXPORT __attribute__ ((visibility ("default")))
#define LIBGOLANG_DSO_IMPORT __attribute__ ((visibility ("default")))
#else
#define LIBGOLANG_DSO_EXPORT
#define LIBGOLANG_DSO_IMPORT
#endif
#if BUILDING_LIBGOLANG
......
......@@ -38,7 +38,7 @@
// cut this short
// (on darwing sys_siglist declaration is normally provided)
// (on windows sys_siglist is not available at all)
#if !(defined(__APPLE__) || defined(_WIN32))
#if !(defined(LIBGOLANG_OS_darwin) || defined(LIBGOLANG_OS_windows))
extern "C" {
extern const char * const sys_siglist[];
}
......@@ -287,7 +287,7 @@ string Signal::String() const {
const Signal& sig = *this;
const char *sigstr = nil;
#ifdef _WIN32
#ifdef LIBGOLANG_OS_windows
switch (sig.signo) {
case SIGABRT: return "Aborted";
case SIGBREAK: return "Break";
......
......@@ -96,7 +96,7 @@ private:
// Open opens file @path.
LIBGOLANG_API std::tuple<File, error> Open(const string &path, int flags = O_RDONLY,
mode_t mode =
#if !defined(_MSC_VER)
#if !defined(LIBGOLANG_CC_msc)
S_IRUSR | S_IWUSR | S_IXUSR |
S_IRGRP | S_IWGRP | S_IXGRP |
S_IROTH | S_IWOTH | S_IXOTH
......
......@@ -89,7 +89,7 @@
#include <atomic>
#include <tuple>
#if defined(_WIN32)
#if defined(LIBGOLANG_OS_windows)
# include <windows.h>
#endif
......@@ -101,7 +101,7 @@
# define debugf(format, ...) do {} while (0)
#endif
#if defined(_MSC_VER)
#ifdef LIBGOLANG_CC_msc
# define HAVE_SIGACTION 0
#else
# define HAVE_SIGACTION 1
......@@ -194,7 +194,7 @@ void _init() {
if (err != nil)
panic("os::newFile(_wakerx");
_waketx = vfd[1];
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
if (sys::Fcntl(_waketx, F_SETFL, O_NONBLOCK) < 0)
panic("fcntl(_waketx, O_NONBLOCK)"); // TODO +syserr
#else
......
......@@ -35,7 +35,7 @@ from __future__ import print_function, absolute_import
# pygolang uses setuptools_dso.DSO to build libgolang; all extensions link to it.
import setuptools_dso
import sys, pkgutil, platform, sysconfig
import os, sys, pkgutil, platform, sysconfig
from os.path import dirname, join, exists
from distutils.errors import DistutilsError
......@@ -68,7 +68,7 @@ def _findpkg(pkgname): # -> _PyPkg
# build_ext amends setuptools_dso.build_ext to allow combining C and C++
# sources in one extension without hitting `error: invalid argument
# '-std=c++11' not allowed with 'C'`.
# '-std=c++11' not allowed with 'C'`. XXX + asm
_dso_build_ext = setuptools_dso.build_ext
class build_ext(_dso_build_ext):
def build_extension(self, ext):
......@@ -108,12 +108,33 @@ class build_ext(_dso_build_ext):
# do per-source adjustsment only in .spawn .
spawn = self.compiler.spawn
def xspawn(argv):
argv = argv[:]
c = False
for arg in argv:
S = False
for i,arg in enumerate(argv):
if arg.startswith('/Tc'):
if arg.endswith('.S'):
argv[i] = arg[3:] # /Tcabc.S -> abc.S
S = True
else:
c = True
if c:
argv = argv[:]
# change cl.exe -> clang-cl.exe for assembly files so that assembler dialect is the same everywhere
if S:
assert argv[0] == self.compiler.cc, (argv, self.compiler.cc)
argv[0] = self.compiler.clang_cl
# clang-cl fails on *.S if also given /EH... -> remove /EH...
while 1:
for i in range(len(argv)):
if argv[i].startswith('/EH'):
del argv[i]
break
else:
break
if c or S:
for i in range(len(argv)):
if argv[i] == '/std:c++20':
argv[i] = '/std:c11'
......@@ -128,6 +149,22 @@ class build_ext(_dso_build_ext):
self.compiler._compile = _compile
self.compiler.spawn = spawn
def build_extensions(self):
# adjust .compiler to support assembly sources
cc = self.compiler
if '.S' not in cc.src_extensions:
cc.src_extensions.append('.S')
cc.language_map['.S'] = 'asm'
cc.language_order.append('asm')
# XXX refer to https://blog.mozilla.org/nfroyd/2019/04/25/an-unexpected-benefit-of-standardizing-on-clang-cl/
if cc.compiler_type == 'msvc':
if not cc.initialized:
cc.initialize()
ccmod = sys.modules[cc.__module__]
cc.clang_cl = ccmod._find_exe('clang-cl.exe', cc._paths.split(os.pathsep))
cc._c_extensions.append('.S') # MSVCCompiler thinks it is C, but xspawn handles .S specially
_dso_build_ext.build_extensions(self)
# setup should be used instead of setuptools.setup
def setup(**kw):
......@@ -176,8 +213,8 @@ def _with_build_defaults(name, kw): # -> (pygo, kw')
incv.insert(1, join(pygo, 'golang', '_compat', sysname))
kw['include_dirs'] = incv
# link with libgolang.so if it is not libgolang itself
if name != 'golang.runtime.libgolang':
# link with libgolang.so if it is not libgolang itself, or another internal DSO
if name not in ('golang.runtime.libgolang', 'golang.runtime.funchook'):
dsov = kw.get('dsos', [])[:]
dsov.insert(0, 'golang.runtime.libgolang')
kw['dsos'] = dsov
......@@ -212,9 +249,11 @@ def _with_build_defaults(name, kw): # -> (pygo, kw')
dependv = kw.get('depends', [])[:]
dependv.extend(['%s/golang/%s' % (pygo, _) for _ in [
'libgolang.h',
'runtime.h',
'runtime/internal.h',
'runtime/internal/atomic.h',
'runtime/internal/syscall.h',
'runtime/platform.h',
'context.h',
'cxx.h',
'errors.h',
......
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package runtime mirrors Go package runtime.
// See runtime.h for package overview.
#include "golang/runtime.h"
// golang::runtime::
namespace golang {
namespace runtime {
const string ARCH =
#ifdef LIBGOLANG_ARCH_386
"386"
#elif defined(LIBGOLANG_ARCH_amd64)
"amd64"
#elif defined(LIBGOLANG_ARCH_arm64)
"arm64"
#else
# error
#endif
;
const string OS =
#ifdef LIBGOLANG_OS_linux
"linux"
#elif defined(LIBGOLANG_OS_darwin)
"darwin"
#elif defined(LIBGOLANG_OS_windows)
"windows"
#else
# error
#endif
;
const string CC =
#ifdef LIBGOLANG_CC_gcc
"gcc"
#elif defined(LIBGOLANG_CC_clang)
"clang"
#elif defined(LIBGOLANG_CC_msc)
"msc"
#else
# error
#endif
;
}} // golang::runtime::
#ifndef _NXD_LIBGOLANG_RUNTIME_H
#define _NXD_LIBGOLANG_RUNTIME_H
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package runtime mirrors Go package runtime.
#include "golang/libgolang.h"
// golang::runtime::
namespace golang {
namespace runtime {
// ARCH indicates processor architecture, that is running the program.
//
// e.g. "386", "amd64", "arm64", ...
extern LIBGOLANG_API const string ARCH;
// OS indicates operating system, that is running the program.
//
// e.g. "linux", "darwin", "windows", ...
extern LIBGOLANG_API const string OS;
// CC indicates C/C++ compiler, that compiled the program.
//
// e.g. "gcc", "clang", "msc", ...
extern LIBGOLANG_API const string CC;
}} // golang::runtime::
#endif // _NXD_LIBGOLANG_RUNTIME_H
......@@ -20,7 +20,7 @@
#include "golang/runtime/internal/atomic.h"
#include "golang/libgolang.h"
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
#include <pthread.h>
#endif
......@@ -44,7 +44,7 @@ static void _forkNewEpoch() {
void _init() {
// there is no fork on windows
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
int e = pthread_atfork(/*prepare*/nil, /*inparent*/nil, /*inchild*/_forkNewEpoch);
if (e != 0)
panic("pthread_atfork failed");
......
......@@ -58,9 +58,9 @@ string _Errno::Error() {
char ebuf[128];
bool ok;
#if __APPLE__
#ifdef LIBGOLANG_OS_darwin
ok = (::strerror_r(-e.syserr, ebuf, sizeof(ebuf)) == 0);
#elif defined(_WIN32)
#elif defined(LIBGOLANG_OS_windows)
ok = (::strerror_s(ebuf, sizeof(ebuf), -e.syserr) == 0);
#else
char *estr = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
......@@ -102,7 +102,7 @@ __Errno Close(int fd) {
return err;
}
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
__Errno Fcntl(int fd, int cmd, int arg) {
int save_errno = errno;
int err = ::fcntl(fd, cmd, arg);
......@@ -124,7 +124,7 @@ __Errno Fstat(int fd, struct ::stat *out_st) {
int Open(const char *path, int flags, mode_t mode) {
int save_errno = errno;
#ifdef _WIN32 // default to open files in binary mode
#ifdef LIBGOLANG_OS_windows // default to open files in binary mode
if ((flags & (_O_TEXT | _O_BINARY)) == 0)
flags |= _O_BINARY;
#endif
......@@ -141,9 +141,9 @@ __Errno Pipe(int vfd[2], int flags) {
return -EINVAL;
int save_errno = errno;
int err;
#ifdef __linux__
#ifdef LIBGOLANG_OS_linux
err = ::pipe2(vfd, flags);
#elif defined(_WIN32)
#elif defined(LIBGOLANG_OS_windows)
err = ::_pipe(vfd, 4096, flags | _O_BINARY);
#else
err = ::pipe(vfd);
......@@ -167,7 +167,7 @@ out:
return err;
}
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
__Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact) {
int save_errno = errno;
int err = ::sigaction(signo, act, oldact);
......
......@@ -63,13 +63,13 @@ LIBGOLANG_API int/*n|err*/ Read(int fd, void *buf, size_t count);
LIBGOLANG_API int/*n|err*/ Write(int fd, const void *buf, size_t count);
LIBGOLANG_API __Errno Close(int fd);
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
LIBGOLANG_API __Errno Fcntl(int fd, int cmd, int arg);
#endif
LIBGOLANG_API __Errno Fstat(int fd, struct ::stat *out_st);
LIBGOLANG_API int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
LIBGOLANG_API __Errno Pipe(int vfd[2], int flags);
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
LIBGOLANG_API __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
#endif
typedef void (*sighandler_t)(int);
......
......@@ -52,7 +52,7 @@
#include <linux/list.h>
// MSVC does not support statement expressions and typeof
// -> redo list_entry via C++ lambda.
#ifdef _MSC_VER
#ifdef LIBGOLANG_CC_msc
# undef list_entry
# define list_entry(ptr, type, member) [&]() { \
const decltype( ((type *)0)->member ) *__mptr = (ptr); \
......
#ifndef _NXD_LIBGOLANG_RUNTIME_PLATFORM_H
#define _NXD_LIBGOLANG_RUNTIME_PLATFORM_H
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Header platform.h provides preprocessor defines that describe target platform.
// LIBGOLANG_ARCH_<X> is defined on architecture X.
//
// List of supported architectures: 386, amd64, arm64.
#if defined(__i386__) || defined(_M_IX86)
# define LIBGOLANG_ARCH_386 1
#elif defined(__x86_64__) || defined(_M_X64)
# define LIBGOLANG_ARCH_amd64 1
#elif defined(__aarch64__) || defined(_M_ARM64)
# define LIBGOLANG_ARCH_arm64 1
#else
# error "unsupported architecture"
#endif
// LIBGOLANG_OS_<X> is defined on operating system X.
//
// List of supported operating systems: linux, darwin, windows.
#ifdef __linux__
# define LIBGOLANG_OS_linux 1
#elif defined(__APPLE__)
# define LIBGOLANG_OS_darwin 1
#elif defined(_WIN32) || defined(__CYGWIN__)
# define LIBGOLANG_OS_windows 1
#else
# error "unsupported operating system"
#endif
// LIBGOLANG_CC_<X> is defined on C/C++ compiler X.
//
// List of supported compilers: gcc, clang, msc.
#ifdef __clang__
# define LIBGOLANG_CC_clang 1
#elif defined(_MSC_VER)
# define LIBGOLANG_CC_msc 1
// NOTE gcc comes last because e.g. clang and icc define __GNUC__ as well
#elif __GNUC__
# define LIBGOLANG_CC_gcc 1
#else
# error "unsupported compiler"
#endif
#endif // _NXD_LIBGOLANG_RUNTIME_PLATFORM_H
......@@ -25,10 +25,14 @@ differences:
- gevent is pre-activated and stdlib is patched to be gevent aware;
- go, chan, select etc are put into builtin namespace;
- default string encoding is always set to UTF-8.
- default string encoding is always set to UTF-8;
- bstr/ustr replace builtin str/unicode types.
Gevent activation can be disabled via `-X gpython.runtime=threads`, or
$GPYTHON_RUNTIME=threads.
String types replacement can be disabled via `-X gpython.strings=pystd`, or
$GPYTHON_STRINGS=pystd.
"""
# NOTE gpython is kept out of golang/ , since even just importing e.g. golang.cmd.gpython,
......@@ -230,9 +234,13 @@ def pymain(argv, init=None):
gevent = sys.modules.get('gevent', None)
gpyver = 'GPython %s' % golang.__version__
if gevent is not None:
gpyver += ' [gevent %s]' % gevent.__version__
gpyver += ' [runtime gevent %s]' % gevent.__version__
else:
gpyver += ' [runtime threads]'
if type(u'') is golang.ustr:
gpyver += ' [strings bstr+ustr]'
else:
gpyver += ' [threads]'
gpyver += ' [strings pystd]'
ver.append(gpyver)
import platform
......@@ -344,6 +352,9 @@ def main():
# imported first, e.g. to support sys.modules.
import sys
# import pyx/c part of gpython
from gpython import _gpython
# safety check that we are not running from a setuptools entrypoint, where
# it would be too late to monkey-patch stdlib.
#
......@@ -372,6 +383,7 @@ def main():
reload(sys)
sys.setdefaultencoding('utf-8')
delattr(sys, 'setdefaultencoding')
_gpython.set_utf8_as_default_src_encoding()
# import os to get access to environment.
......@@ -382,9 +394,11 @@ def main():
# extract and process `-X gpython.*`
# -X gpython.runtime=(gevent|threads) + $GPYTHON_RUNTIME
# -X gpython.strings=(bstr+ustr|pystd) + $GPYTHON_STRINGS
sys._xoptions = getattr(sys, '_xoptions', {})
argv_ = []
gpy_runtime = os.getenv('GPYTHON_RUNTIME', 'gevent')
gpy_strings = os.getenv('GPYTHON_STRINGS', 'bstr+ustr')
igetopt = _IGetOpt(sys.argv[1:], _pyopt, _pyopt_long)
for (opt, arg) in igetopt:
if opt == '-X':
......@@ -393,6 +407,10 @@ def main():
gpy_runtime = arg[len('gpython.runtime='):]
sys._xoptions['gpython.runtime'] = gpy_runtime
elif arg.startswith('gpython.strings='):
gpy_strings = arg[len('gpython.strings='):]
sys._xoptions['gpython.strings'] = gpy_strings
else:
raise RuntimeError('gpython: unknown -X option %s' % arg)
......@@ -412,13 +430,15 @@ def main():
# sys.executable spawned from under `gpython -X gpython.runtime=threads`
# also uses "threads" runtime by default.
os.environ['GPYTHON_RUNTIME'] = gpy_runtime
os.environ['GPYTHON_STRINGS'] = gpy_strings
# init initializes according to selected runtime
# init initializes according to selected runtime and strings
# it is called after options are parsed and sys.path is setup correspondingly.
# this way golang and gevent are imported from exactly the same place as
# they would be in standard python after regular import (ex from golang/
# under cwd if run under `python -c ...` or interactive console.
def init():
gpy_runtime_ver = gpy_runtime
if gpy_runtime == 'gevent':
# make gevent pre-available & stdlib patched
import gevent
......@@ -434,22 +454,30 @@ def main():
if _ not in (True, None): # patched or nothing to do
# XXX provide details
raise RuntimeError('gevent monkey-patching failed')
gpy_verextra = 'gevent %s' % gevent.__version__
gpy_runtime_ver += ' %s' % gevent.__version__
elif gpy_runtime == 'threads':
gpy_verextra = 'threads'
pass
else:
raise RuntimeError('gpython: invalid runtime %s' % gpy_runtime)
raise RuntimeError('gpython: invalid runtime %r' % gpy_runtime)
# put go, chan, select, ... into builtin namespace
if gpy_strings not in ('bstr+ustr', 'pystd'):
raise RuntimeError('gpython: invalid strings %r' % gpy_strings)
# import golang
# this will activate selected runtime and strings
sys._gpy_runtime = gpy_runtime
sys._gpy_strings = gpy_strings
import golang
# put go, chan, select, ... into builtin namespace
from six.moves import builtins
for k in golang.__all__:
setattr(builtins, k, getattr(golang, k))
setattr(builtins, 'CCC', CCC)
# sys.version
sys.version += (' [GPython %s] [%s]' % (golang.__version__, gpy_verextra))
sys.version += (' [GPython %s] [runtime %s] [strings %s]' % (golang.__version__, gpy_runtime_ver, gpy_strings))
# tail to pymain
pymain(argv, init)
......@@ -567,5 +595,11 @@ class _IGetOpt:
next = __next__ # for py2
# for tests XXX continue by first writing test XXX
1/0
class _tEarlyStrSubclass(str):
pass
if __name__ == '__main__':
main()
# -*- coding: utf-8 -*-
# cython: language_level=2
# Copyright (C) 2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""_gpython.pyx ... XXX
"""
cdef extern from *:
"""
void _set_utf8_as_default_src_encoding();
"""
void _set_utf8_as_default_src_encoding() except *
def set_utf8_as_default_src_encoding():
_set_utf8_as_default_src_encoding()
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// XXX doctitle
#include <Python.h>
#if PY_MAJOR_VERSION < 3
#include <Python-ast.h> // mod_ty & co
#include <node.h> // node
#include <graminit.h> // encoding_decl & co
#include <ast.h> // PyAST_FromNode & co
#endif
#include <funchook.h>
// py2: wrap PyAST_FromNode so that "utf-8" becomes the default encoding
#if PY_MAJOR_VERSION < 3
static auto _py_PyAST_FromNode = &PyAST_FromNode;
static mod_ty gpy_PyAST_FromNode(const node* n, PyCompilerFlags* flags,
const char* filename, PyArena* arena)
{
// fprintf(stderr, "gpy_PyAST_FromNode...\n");
PyCompilerFlags gflags = {.cf_flags = 0};
if (flags)
gflags = *flags;
if (TYPE(n) != encoding_decl)
gflags.cf_flags |= PyCF_SOURCE_IS_UTF8;
return _py_PyAST_FromNode(n, &gflags, filename, arena);
}
static funchook_t* gpy_PyAST_FromNode_hook;
void _set_utf8_as_default_src_encoding() {
funchook_t *h;
int err;
// funchook_set_debug_file("/dev/stderr");
gpy_PyAST_FromNode_hook = h = funchook_create();
if (h == NULL) {
PyErr_NoMemory();
return;
}
err = funchook_prepare(h, (void**)&_py_PyAST_FromNode, (void*)gpy_PyAST_FromNode);
if (err != 0) {
PyErr_SetString(PyExc_RuntimeError, funchook_error_message(h));
return;
}
err = funchook_install(h, 0);
if (err != 0) {
PyErr_SetString(PyExc_RuntimeError, funchook_error_message(h));
return;
}
// ok
}
#else
void _set_utf8_as_default_src_encoding() {}
#endif
......@@ -47,20 +47,34 @@ gpython_only = pytest.mark.skipif(not is_gpython, reason="gpython-only test")
def runtime(request):
yield request.param
# strings is pytest fixture that yields all variants of should be supported gpython strings:
# '' - not specified (gpython should autoselect)
# 'bstr+ustr'
# 'pystd'
@pytest.fixture(scope="function", params=['', 'bstr+ustr', 'pystd'])
def strings(request):
yield request.param
# gpyenv returns environment appropriate for spawning gpython with
# specified runtime.
def gpyenv(runtime): # -> env
# specified runtime and strings.
def gpyenv(runtime, strings): # -> env
env = os.environ.copy()
if runtime != '':
env['GPYTHON_RUNTIME'] = runtime
else:
env.pop('GPYTHON_RUNTIME', None)
if strings != '':
env['GPYTHON_STRINGS'] = strings
else:
env.pop('GPYTHON_STRINGS', None)
return env
@gpython_only
def test_defaultencoding_utf8():
assert sys.getdefaultencoding() == 'utf-8'
assert eval("u'αβγ'") == u'αβγ' # FIXME fails on py2 which uses hardcoded default latin1
# XXX +exec, +run file
@gpython_only
def test_golang_builtins():
......@@ -143,19 +157,42 @@ def assert_gevent_not_activated():
@gpython_only
def test_executable(runtime):
def test_str_patched():
# gpython, by default, patches str/unicode to be bstr/ustr.
# handling of various string modes is explicitly tested in test_Xstrings.
assert_str_patched()
def assert_str_patched():
#assert str.__name__ == ('bstr' if PY2 else 'ustr')
assert str.__name__ == 'str'
assert str is (bstr if PY2 else ustr)
if PY2:
assert unicode.__name__ == 'unicode'
assert unicode is ustr
assert type('') is str
assert type(b'') is (bstr if PY2 else bytes)
assert type(u'') is ustr
def assert_str_not_patched():
assert str.__name__ == 'str'
assert str is not bstr
assert str is not ustr
if PY2:
assert unicode.__name__ == 'unicode'
assert unicode is not bstr
assert unicode is not ustr
assert type('') is str
assert type(b'') is bytes
assert type(u'') is (unicode if PY2 else str)
@gpython_only
def test_executable():
# sys.executable must point to gpython and we must be able to execute it.
import gevent
assert 'gpython' in sys.executable
ver = pyout(['-c', 'import sys; print(sys.version)'], env=gpyenv(runtime))
ver = pyout(['-c', 'import sys; print(sys.version)'], env=gpyenv('', ''))
ver = str(ver)
assert ('[GPython %s]' % golang.__version__) in ver
if runtime != 'threads':
assert ('[gevent %s]' % gevent.__version__) in ver
assert ('[threads]') not in ver
else:
assert ('[gevent ') not in ver
assert ('[threads]') in ver
# verify pymain.
......@@ -322,15 +359,20 @@ def test_pymain_opt():
# pymain -V/--version
# gpython_only because output differs from !gpython.
@gpython_only
def test_pymain_ver(runtime):
def test_pymain_ver(runtime, strings):
from golang import b
from gpython import _version_info_str as V
import gevent
vok = 'GPython %s' % golang.__version__
if runtime != 'threads':
vok += ' [gevent %s]' % gevent.__version__
vok += ' [runtime gevent %s]' % gevent.__version__
else:
vok += ' [threads]'
vok += ' [runtime threads]'
if strings != 'pystd':
vok += ' [strings bstr+ustr]'
else:
vok += ' [strings pystd]'
if is_cpython:
vok += ' / CPython %s' % platform.python_version()
......@@ -341,10 +383,12 @@ def test_pymain_ver(runtime):
vok += '\n'
ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
env = gpyenv(runtime, strings)
ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE, env=env)
assert (ret, out, b(err)) == (0, b'', b(vok))
ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE, env=env)
assert (ret, out, b(err)) == (0, b'', b(vok))
# verify that ./bin/gpython runs ok.
......
[build-system]
requires = ["setuptools", "wheel", "setuptools_dso >= 2.7", "cython", "gevent"]
requires = ["setuptools", "wheel", "setuptools_dso >= 2.7", "cython < 3", "gevent"]
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment