X My draft state of x/gpystr work; py2/py3 pickle problem should be essentially solved

.gitmodules

+[submodule "3rdparty/funchook"]
+	path = 3rdparty/funchook
+	url = https://github.com/kubo/funchook.git
+[submodule "3rdparty/capstone"]
+	path = 3rdparty/capstone
+	url = https://github.com/capstone-engine/capstone.git

capstone @ 097c04d9

+Subproject commit 097c04d9413c59a58b00d4d1c8d5dc0ac158ffaa

funchook @ 88388db3

+Subproject commit 88388db3c69e16c1560fee65c6857d75f5ce6fd5

MANIFEST.in

@@ -2,6 +2,9 @@ include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .nxdtest
 include golang/libgolang.h
 include golang/runtime/libgolang.cpp
 include golang/runtime/libpyxruntime.cpp
+include golang/runtime/platform.h
+include golang/runtime.h
+include golang/runtime.cpp
 include golang/pyx/runtime.h
 include golang/pyx/testprog/golang_dso_user/dsouser/dso.h
 include golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
@@ -36,7 +39,10 @@ include golang/time.cpp
 include golang/_testing.h
 include golang/_compat/windows/strings.h
 include golang/_compat/windows/unistd.h
+include gpython/_gpython_c.cpp
 recursive-include golang    *.py *.pxd *.pyx *.toml *.txt*
-recursive-include gpython   *.py
-recursive-include 3rdparty  *.h
+recursive-include gpython   *.py       *.pyx
+recursive-include 3rdparty  *.h *.c *.cpp *.S *.py *.cmake *.cs *.java
+recursive-include 3rdparty  LICENSE README.md README COPYING Makefile CMakeLists.txt
 recursive-exclude golang    *_dsoinfo.py
+include conftest.py

README.rst

@@ -4,7 +4,7 @@
 
 Package `golang` provides Go-like features for Python:
 
-- `gpython` is Python interpreter with support for lightweight threads.
+- `gpython` is Python interpreter with support for lightweight threads and uniform UTF8-based approach to strings.
 - `go` spawns lightweight thread.
 - `chan` and `select` provide channels with Go semantic.
 - `func` allows to define methods separate from class.
@@ -46,15 +46,16 @@ __ http://libuv.org/
 __ http://software.schmorp.de/pkg/libev.html
 
 
-Additionally GPython sets UTF-8 to be default encoding always, and puts `go`,
-`chan`, `select` etc into builtin namespace.
+Additionally GPython sets UTF-8 to be default encoding always, puts `go`,
+`chan`, `select` etc into builtin namespace, and makes `bstr`/`ustr` to be used
+instead of builtin string types.
 
 .. note::
 
    GPython is optional and the rest of Pygolang can be used from under standard Python too.
    However without gevent integration `go` spawns full - not lightweight - OS thread.
-   GPython can be also used with threads - not gevent - runtime. Please see
-   `GPython options`_ for details.
+   GPython can be also used with threads - not gevent - runtime and with builtin string types.
+   Please see `GPython options`_ for details.
 
 
 Goroutines and channels
@@ -571,3 +572,9 @@ GPython-specific options and environment variables are listed below:
     coroutines, while with `threads` `go` spawns full OS thread. `gevent` is
     default. The runtime to use can be also specified via `$GPYTHON_RUNTIME`
     environment variable.
+
+`-X gpython.strings=(bstr+ustr|pystd)`
+    Specify which string types GPython should use. `bstr+ustr` provide
+    uniform UTF8-based approach to strings, while `pystd` selects regular
+    `str` and `unicode`. `bstr+ustr` is default. String types to use can be
+    also specified via `$GPYTHON_STRINGS` environment variable.

conftest.py

+# ignore tests in distorm - else it breaks as e.g.
+#
+# 3rdparty/funchook/distorm/python/test_distorm3.py:15: in <module>
+#     import distorm3
+# 3rdparty/funchook/distorm/python/distorm3/__init__.py:57: in <module>
+#     _distorm = _load_distorm()
+# 3rdparty/funchook/distorm/python/distorm3/__init__.py:55: in _load_distorm
+#     raise ImportError("Error loading the diStorm dynamic library (or cannot load library into process).")
+# E   ImportError: Error loading the diStorm dynamic library (or cannot load library into process).
+collect_ignore = ["3rdparty"]

golang/_golang.pyx

@@ -3,7 +3,7 @@
 # cython: binding=False
 # cython: c_string_type=str, c_string_encoding=utf8
 # distutils: language = c++
-# distutils: depends = libgolang.h os/signal.h unicode/utf8.h _golang_str.pyx
+# distutils: depends = libgolang.h os/signal.h unicode/utf8.h _golang_str.pyx _golang_str_pickle.pyx
 #
 # Copyright (C) 2018-2023  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
@@ -34,7 +34,7 @@ from __future__ import print_function, absolute_import
 _init_libgolang()
 _init_libpyxruntime()
 
-from cpython cimport PyObject, Py_INCREF, Py_DECREF, PY_MAJOR_VERSION
+from cpython cimport PyObject, Py_INCREF, Py_DECREF, Py_CLEAR, PY_MAJOR_VERSION
 ctypedef PyObject *pPyObject # https://github.com/cython/cython/issues/534
 cdef extern from "Python.h":
     ctypedef struct PyTupleObject:

golang/_golang_str_pickle_test.pyx

+# -*- coding: utf-8 -*-
+# Copyright (C) 2023  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+# test for inside_counted
+def _test_inside_counted(): # -> outok
+    outok = ''
+
+    outok += '\n\n\nBEFORE PATCH\n'
+    print('\n\n\nBEFORE PATCH')
+    tfunc(3)
+
+    t0 = ''
+    for i in range(3,0-1,-1):
+        t0 += '> tfunc(%d)\tinside_counter: 0\n' % i
+    for i in range(0,3+1,+1):
+        t0 += '< tfunc(%d)\tinside_counter: 0\n' % i
+    outok += t0
+
+    outok += '\n\n\nPATCHED\n'
+    print('\n\n\nPATCHED')
+    _patch = xfunchook_create()
+    global inside_counted_func
+    inside_counted_func = <void*>&tfunc
+    xfunchook_prepare(_patch, &inside_counted_func, <void*>inside_counted)
+    xfunchook_install(_patch, 0)
+
+    tfunc(12)
+
+    stk_size = 8  # = STK_SIZE from _golang_str_pickle.S
+    for i in range(12,0-1,-1):
+        outok += '> tfunc(%d)\tinside_counter: %d\n' % (i, min(12-i+1, stk_size))
+    for i in range(0,12+1,+1):
+        outok += '< tfunc(%d)\tinside_counter: %d\n' % (i, min(12-i+1, stk_size))
+
+    outok += '\n\n\nUNPATCHED\n'
+    print('\n\n\nUNPATCHED')
+    xfunchook_uninstall(_patch, 0)
+    tfunc(3)
+    outok += t0
+
+    return outok
+
+cdef void tfunc(int x):
+    print('> tfunc(%d)\tinside_counter: %d' % (x, inside_counter))
+    if x > 0:
+        tfunc(x-1)
+    print('< tfunc(%d)\tinside_counter: %d' % (x, inside_counter))
+
+
+def _test_cfunc_is_callee_cleanup():
+    for t in _cfunc_is_callee_cleanup_testv:
+        stkclean = cfunc_is_callee_cleanup(t.cfunc)
+        assert stkclean == t.stkclean_by_callee_ok, (t.cfunc_name, stkclean, t.stkclean_by_callee_ok)
+
+cdef extern from * nogil:
+    r"""
+    struct _Test_cfunc_is_callee_clenup {
+        const char* cfunc_name;
+        void*       cfunc;
+        int         stkclean_by_callee_ok;
+    };
+
+    #define CASE(func, stkclean_ok) \
+        _Test_cfunc_is_callee_clenup{#func, (void*)func, stkclean_ok}
+
+    #if defined(LIBGOLANG_ARCH_386)
+    int CALLCONV(cdecl)
+    tfunc_cdecl1(int x)                     { return x; }
+    int CALLCONV(cdecl)
+    tfunc_cdecl2(int x, int y)              { return x; }
+    int CALLCONV(cdecl)
+    tfunc_cdecl3(int x, int y, int z)       { return x; }
+
+    int CALLCONV(stdcall)
+    tfunc_stdcall1(int x)                   { return x; }
+    int CALLCONV(stdcall)
+    tfunc_stdcall2(int x, int y)            { return x; }
+    int CALLCONV(stdcall)
+    tfunc_stdcall3(int x, int y, int z)     { return x; }
+
+    int CALLCONV(fastcall)
+    tfunc_fastcall1(int x)                  { return x; }
+    int CALLCONV(fastcall)
+    tfunc_fastcall2(int x, int y)           { return x; }
+    int CALLCONV(fastcall)
+    tfunc_fastcall3(int x, int y, int z)    { return x; }
+
+    #ifndef LIBGOLANG_CC_msc    // see note about C3865 in FOR_EACH_CALLCONV
+    int CALLCONV(thiscall)
+    tfunc_thiscall1(int x)                  { return x; }
+    int CALLCONV(thiscall)
+    tfunc_thiscall2(int x, int y)           { return x; }
+    int CALLCONV(thiscall)
+    tfunc_thiscall3(int x, int y, int z)    { return x; }
+    #endif
+
+    #ifndef LIBGOLANG_CC_msc    // no regparm on MSCV
+    int CALLCONV(regparm(1))
+    tfunc_regparm1_1(int x)                 { return x; }
+    int CALLCONV(regparm(1))
+    tfunc_regparm1_2(int x, int y)          { return x; }
+    int CALLCONV(regparm(1))
+    tfunc_regparm1_3(int x, int y, int z)   { return x; }
+
+    int CALLCONV(regparm(2))
+    tfunc_regparm2_1(int x)                 { return x; }
+    int CALLCONV(regparm(2))
+    tfunc_regparm2_2(int x, int y)          { return x; }
+    int CALLCONV(regparm(2))
+    tfunc_regparm2_3(int x, int y, int z)   { return x; }
+
+    int CALLCONV(regparm(3))
+    tfunc_regparm3_1(int x)                 { return x; }
+    int CALLCONV(regparm(3))
+    tfunc_regparm3_2(int x, int y)          { return x; }
+    int CALLCONV(regparm(3))
+    tfunc_regparm3_3(int x, int y, int z)   { return x; }
+    #endif
+
+    static std::vector<_Test_cfunc_is_callee_clenup> _cfunc_is_callee_cleanup_testv = {
+        CASE(tfunc_cdecl1     , 0 * 4),
+        CASE(tfunc_cdecl2     , 0 * 4),
+        CASE(tfunc_cdecl3     , 0 * 4),
+        CASE(tfunc_stdcall1   , 1 * 4),
+        CASE(tfunc_stdcall2   , 2 * 4),
+        CASE(tfunc_stdcall3   , 3 * 4),
+        CASE(tfunc_fastcall1  , 0 * 4),
+        CASE(tfunc_fastcall2  , 0 * 4),
+        CASE(tfunc_fastcall3  , 1 * 4),
+    #ifndef LIBGOLANG_CC_msc
+        CASE(tfunc_thiscall1  , 0 * 4),
+        CASE(tfunc_thiscall2  , 1 * 4),
+        CASE(tfunc_thiscall3  , 2 * 4),
+    #endif
+    #ifndef LIBGOLANG_CC_msc
+        CASE(tfunc_regparm1_1 , 0 * 4),
+        CASE(tfunc_regparm1_2 , 0 * 4),
+        CASE(tfunc_regparm1_3 , 0 * 4),
+        CASE(tfunc_regparm2_1 , 0 * 4),
+        CASE(tfunc_regparm2_2 , 0 * 4),
+        CASE(tfunc_regparm2_3 , 0 * 4),
+        CASE(tfunc_regparm3_1 , 0 * 4),
+        CASE(tfunc_regparm3_2 , 0 * 4),
+        CASE(tfunc_regparm3_3 , 0 * 4),
+    #endif
+    };
+
+    #else
+    // only i386 has many calling conventions
+    int tfunc_default(int x, int y, int z)      { return x; }
+
+    static std::vector<_Test_cfunc_is_callee_clenup> _cfunc_is_callee_cleanup_testv = {
+        CASE(tfunc_default, 0),
+    };
+    #endif
+
+    #undef CASE
+    """
+    struct _Test_cfunc_is_callee_clenup:
+        const char* cfunc_name
+        void*       cfunc
+        int         stkclean_by_callee_ok
+
+    vector[_Test_cfunc_is_callee_clenup] _cfunc_is_callee_cleanup_testv

golang/_strconv.pyx

@@ -28,12 +28,11 @@ from golang cimport pyb, byte, rune
 from golang cimport _utf8_decode_rune, _xunichr
 from golang.unicode cimport utf8
 
-from cpython cimport PyObject
+from cpython cimport PyObject, _PyBytes_Resize
 
 cdef extern from "Python.h":
     PyObject* PyBytes_FromStringAndSize(char*, Py_ssize_t) except NULL
     char* PyBytes_AS_STRING(PyObject*)
-    int _PyBytes_Resize(PyObject**, Py_ssize_t) except -1
     void Py_DECREF(PyObject*)
 
 
@@ -65,7 +64,7 @@ cdef bytes _quote(const byte[::1] s, char quote, bint* out_nonascii_escape): # -
     cdef byte c
     q[0] = quote;  q += 1
     while i < len(s):
-        c = s[i]
+        c = s[i]        # XXX -> use raw pointer in the loop
         # fast path - ASCII only
         if c < 0x80:
             if c in (ord('\\'), quote):
@@ -104,7 +103,8 @@ cdef bytes _quote(const byte[::1] s, char quote, bint* out_nonascii_escape): # -
 
         # slow path - full UTF-8 decoding + unicodedata
         else:
-            r, size = _utf8_decode_rune(s[i:])
+            # XXX optimize non-ascii case
+            r, size = _utf8_decode_rune(s[i:])  # XXX -> raw pointer
             isize = i + size
 
             # decode error - just emit raw byte as escaped
@@ -117,6 +117,9 @@ cdef bytes _quote(const byte[::1] s, char quote, bint* out_nonascii_escape): # -
                 q += 4
 
             # printable utf-8 characters go as is
+            # XXX ? use Py_UNICODE_ISPRINTABLE (py3, not available on py2)  ?
+            # XXX ? and generate C table based on unicodedata for py2 ?
+            # XXX -> generate table based on unicodedata for both py2/py3 because Py_UNICODE_ISPRINTABLE is not exactly what matches strconv.IsPrint  (i.e. cat starts from LNPS)
             elif _unicodedata_category(_xunichr(r))[0] in 'LNPS': # letters, numbers, punctuation, symbols
                 for j in range(i, isize):
                     q[0] = s[j]

golang/fmt.h

@@ -111,7 +111,7 @@ inline error errorf(const string& format, Argv... argv) {
 // `const char *` overloads just to catch format mistakes as
 // __attribute__(format) does not work with std::string.
 LIBGOLANG_API string sprintf(const char *format, ...)
-#ifndef _MSC_VER
+#ifndef LIBGOLANG_CC_msc
                                 __attribute__ ((format (printf, 1, 2)))
 #endif
 	;

golang/libgolang.h

@@ -169,6 +169,8 @@
 // [1] Libtask: a Coroutine Library for C and Unix. https://swtch.com/libtask.
 // [2] http://9p.io/magic/man2html/2/thread.
 
+#include "golang/runtime/platform.h"
+
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -177,21 +179,18 @@
 #include <sys/stat.h>
 
 #include <fcntl.h>
-#ifdef _MSC_VER // no mode_t on msvc
+#ifdef LIBGOLANG_CC_msc // no mode_t on msvc
 typedef int mode_t;
 #endif
 
 
 // DSO symbols visibility (based on https://gcc.gnu.org/wiki/Visibility)
-#if defined _WIN32 || defined __CYGWIN__
+#ifdef LIBGOLANG_OS_windows
   #define LIBGOLANG_DSO_EXPORT __declspec(dllexport)
   #define LIBGOLANG_DSO_IMPORT __declspec(dllimport)
-#elif __GNUC__ >= 4
+#else
   #define LIBGOLANG_DSO_EXPORT __attribute__ ((visibility ("default")))
   #define LIBGOLANG_DSO_IMPORT __attribute__ ((visibility ("default")))
-#else
-  #define LIBGOLANG_DSO_EXPORT
-  #define LIBGOLANG_DSO_IMPORT
 #endif
 
 #if BUILDING_LIBGOLANG

golang/os.cpp

@@ -38,7 +38,7 @@
 // cut this short
 // (on darwing sys_siglist declaration is normally provided)
 // (on windows sys_siglist is not available at all)
-#if !(defined(__APPLE__) || defined(_WIN32))
+#if !(defined(LIBGOLANG_OS_darwin) || defined(LIBGOLANG_OS_windows))
 extern "C" {
     extern const char * const sys_siglist[];
 }
@@ -287,7 +287,7 @@ string Signal::String() const {
     const Signal& sig = *this;
     const char *sigstr = nil;
 
-#ifdef _WIN32
+#ifdef LIBGOLANG_OS_windows
     switch (sig.signo) {
     case SIGABRT:   return "Aborted";
     case SIGBREAK:  return "Break";

golang/os.h

@@ -96,7 +96,7 @@ private:
 // Open opens file @path.
 LIBGOLANG_API std::tuple<File, error> Open(const string &path, int flags = O_RDONLY,
         mode_t mode =
-#if !defined(_MSC_VER)
+#if !defined(LIBGOLANG_CC_msc)
                       S_IRUSR | S_IWUSR | S_IXUSR |
                       S_IRGRP | S_IWGRP | S_IXGRP |
                       S_IROTH | S_IWOTH | S_IXOTH

golang/os/signal.cpp

@@ -89,7 +89,7 @@
 #include <atomic>
 #include <tuple>
 
-#if defined(_WIN32)
+#if defined(LIBGOLANG_OS_windows)
 # include <windows.h>
 #endif
 
@@ -101,7 +101,7 @@
 #  define debugf(format, ...) do {} while (0)
 #endif
 
-#if defined(_MSC_VER)
+#ifdef LIBGOLANG_CC_msc
 # define HAVE_SIGACTION 0
 #else
 # define HAVE_SIGACTION 1
@@ -194,7 +194,7 @@ void _init() {
     if (err != nil)
         panic("os::newFile(_wakerx");
     _waketx = vfd[1];
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
     if (sys::Fcntl(_waketx, F_SETFL, O_NONBLOCK) < 0)
         panic("fcntl(_waketx, O_NONBLOCK)");    // TODO +syserr
 #else

golang/pyx/build.py

@@ -35,7 +35,7 @@ from __future__ import print_function, absolute_import
 # pygolang uses setuptools_dso.DSO to build libgolang; all extensions link to it.
 import setuptools_dso
 
-import sys, pkgutil, platform, sysconfig
+import os, sys, pkgutil, platform, sysconfig
 from os.path import dirname, join, exists
 from distutils.errors import DistutilsError
 
@@ -68,7 +68,7 @@ def _findpkg(pkgname):  # -> _PyPkg
 
 # build_ext amends setuptools_dso.build_ext to allow combining C and C++
 # sources in one extension without hitting `error: invalid argument
-# '-std=c++11' not allowed with 'C'`.
+# '-std=c++11' not allowed with 'C'`. XXX + asm
 _dso_build_ext = setuptools_dso.build_ext
 class build_ext(_dso_build_ext):
     def build_extension(self, ext):
@@ -108,12 +108,33 @@ class build_ext(_dso_build_ext):
         # do per-source adjustsment only in .spawn .
         spawn = self.compiler.spawn
         def xspawn(argv):
+            argv = argv[:]
+
             c = False
-            for arg in argv:
+            S = False
+            for i,arg in enumerate(argv):
                 if arg.startswith('/Tc'):
+                    if arg.endswith('.S'):
+                        argv[i] = arg[3:]   # /Tcabc.S -> abc.S
+                        S = True
+                    else:
                         c = True
-            if c:
-                argv = argv[:]
+
+            # change cl.exe -> clang-cl.exe for assembly files so that assembler dialect is the same everywhere
+            if S:
+                assert argv[0] == self.compiler.cc, (argv, self.compiler.cc)
+                argv[0] = self.compiler.clang_cl
+
+                # clang-cl fails on *.S if also given /EH... -> remove /EH...
+                while 1:
+                    for i in range(len(argv)):
+                        if argv[i].startswith('/EH'):
+                            del argv[i]
+                            break
+                    else:
+                        break
+
+            if c or S:
                 for i in range(len(argv)):
                     if argv[i] == '/std:c++20':
                         argv[i] = '/std:c11'
@@ -128,6 +149,22 @@ class build_ext(_dso_build_ext):
             self.compiler._compile = _compile
             self.compiler.spawn    = spawn
 
+    def build_extensions(self):
+        # adjust .compiler to support assembly sources
+        cc = self.compiler
+        if '.S' not in cc.src_extensions:
+            cc.src_extensions.append('.S')
+            cc.language_map['.S'] = 'asm'
+            cc.language_order.append('asm')
+            # XXX refer to https://blog.mozilla.org/nfroyd/2019/04/25/an-unexpected-benefit-of-standardizing-on-clang-cl/
+            if cc.compiler_type == 'msvc':
+                if not cc.initialized:
+                    cc.initialize()
+                ccmod = sys.modules[cc.__module__]
+                cc.clang_cl = ccmod._find_exe('clang-cl.exe', cc._paths.split(os.pathsep))
+                cc._c_extensions.append('.S')   # MSVCCompiler thinks it is C, but xspawn handles .S specially
+        _dso_build_ext.build_extensions(self)
+
 
 # setup should be used instead of setuptools.setup
 def setup(**kw):
@@ -176,8 +213,8 @@ def _with_build_defaults(name, kw):   # -> (pygo, kw')
     incv.insert(1, join(pygo, 'golang', '_compat', sysname))
     kw['include_dirs'] = incv
 
-    # link with libgolang.so  if it is not libgolang itself
-    if name != 'golang.runtime.libgolang':
+    # link with libgolang.so  if it is not libgolang itself, or another internal DSO
+    if name not in ('golang.runtime.libgolang', 'golang.runtime.funchook'):
         dsov = kw.get('dsos', [])[:]
         dsov.insert(0, 'golang.runtime.libgolang')
         kw['dsos'] = dsov
@@ -212,9 +249,11 @@ def _with_build_defaults(name, kw):   # -> (pygo, kw')
     dependv = kw.get('depends', [])[:]
     dependv.extend(['%s/golang/%s' % (pygo, _) for _ in [
         'libgolang.h',
+        'runtime.h',
         'runtime/internal.h',
         'runtime/internal/atomic.h',
         'runtime/internal/syscall.h',
+        'runtime/platform.h',
         'context.h',
         'cxx.h',
         'errors.h',

golang/runtime.cpp

+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package runtime mirrors Go package runtime.
+// See runtime.h for package overview.
+
+#include "golang/runtime.h"
+
+
+// golang::runtime::
+namespace golang {
+namespace runtime {
+
+const string ARCH =
+#ifdef LIBGOLANG_ARCH_386
+    "386"
+#elif defined(LIBGOLANG_ARCH_amd64)
+    "amd64"
+#elif defined(LIBGOLANG_ARCH_arm64)
+    "arm64"
+#else
+# error
+#endif
+    ;
+
+
+const string OS =
+#ifdef LIBGOLANG_OS_linux
+    "linux"
+#elif defined(LIBGOLANG_OS_darwin)
+    "darwin"
+#elif defined(LIBGOLANG_OS_windows)
+    "windows"
+#else
+# error
+#endif
+    ;
+
+
+const string CC =
+#ifdef LIBGOLANG_CC_gcc
+    "gcc"
+#elif defined(LIBGOLANG_CC_clang)
+    "clang"
+#elif defined(LIBGOLANG_CC_msc)
+    "msc"
+#else
+# error
+#endif
+    ;
+
+
+}}  // golang::runtime::

golang/runtime.h

+#ifndef _NXD_LIBGOLANG_RUNTIME_H
+#define _NXD_LIBGOLANG_RUNTIME_H
+
+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package runtime mirrors Go package runtime.
+
+#include "golang/libgolang.h"
+
+
+// golang::runtime::
+namespace golang {
+namespace runtime {
+
+// ARCH indicates processor architecture, that is running the program.
+//
+// e.g. "386", "amd64", "arm64", ...
+extern LIBGOLANG_API const string ARCH;
+
+// OS indicates operating system, that is running the program.
+//
+// e.g. "linux", "darwin", "windows", ...
+extern LIBGOLANG_API const string OS;
+
+// CC indicates C/C++ compiler, that compiled the program.
+//
+// e.g. "gcc", "clang", "msc", ...
+extern LIBGOLANG_API const string CC;
+
+
+}} // golang::runtime::
+
+#endif  // _NXD_LIBGOLANG_RUNTIME_H

golang/runtime/internal/atomic.cpp

@@ -20,7 +20,7 @@
 #include "golang/runtime/internal/atomic.h"
 #include "golang/libgolang.h"
 
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
 #include <pthread.h>
 #endif
 
@@ -44,7 +44,7 @@ static void _forkNewEpoch() {
 
 void _init() {
 // there is no fork on windows
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
     int e = pthread_atfork(/*prepare*/nil, /*inparent*/nil, /*inchild*/_forkNewEpoch);
     if (e != 0)
         panic("pthread_atfork failed");

golang/runtime/internal/syscall.cpp

@@ -58,9 +58,9 @@ string _Errno::Error() {
 
     char ebuf[128];
     bool ok;
-#if __APPLE__
+#ifdef LIBGOLANG_OS_darwin
     ok = (::strerror_r(-e.syserr, ebuf, sizeof(ebuf)) == 0);
-#elif defined(_WIN32)
+#elif defined(LIBGOLANG_OS_windows)
     ok = (::strerror_s(ebuf, sizeof(ebuf), -e.syserr) == 0);
 #else
     char *estr = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
@@ -102,7 +102,7 @@ __Errno Close(int fd) {
     return err;
 }
 
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
 __Errno Fcntl(int fd, int cmd, int arg) {
     int save_errno = errno;
     int err = ::fcntl(fd, cmd, arg);
@@ -124,7 +124,7 @@ __Errno Fstat(int fd, struct ::stat *out_st) {
 
 int Open(const char *path, int flags, mode_t mode) {
     int save_errno = errno;
-#ifdef _WIN32  // default to open files in binary mode
+#ifdef LIBGOLANG_OS_windows  // default to open files in binary mode
     if ((flags & (_O_TEXT | _O_BINARY)) == 0)
         flags |= _O_BINARY;
 #endif
@@ -141,9 +141,9 @@ __Errno Pipe(int vfd[2], int flags) {
         return -EINVAL;
     int save_errno = errno;
     int err;
-#ifdef __linux__
+#ifdef LIBGOLANG_OS_linux
     err = ::pipe2(vfd, flags);
-#elif defined(_WIN32)
+#elif defined(LIBGOLANG_OS_windows)
     err = ::_pipe(vfd, 4096, flags | _O_BINARY);
 #else
     err = ::pipe(vfd);
@@ -167,7 +167,7 @@ out:
     return err;
 }
 
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
 __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact) {
     int save_errno = errno;
     int err = ::sigaction(signo, act, oldact);

golang/runtime/internal/syscall.h

@@ -63,13 +63,13 @@ LIBGOLANG_API int/*n|err*/ Read(int fd, void *buf, size_t count);
 LIBGOLANG_API int/*n|err*/ Write(int fd, const void *buf, size_t count);
 
 LIBGOLANG_API __Errno Close(int fd);
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
 LIBGOLANG_API __Errno Fcntl(int fd, int cmd, int arg);
 #endif
 LIBGOLANG_API __Errno Fstat(int fd, struct ::stat *out_st);
 LIBGOLANG_API int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
 LIBGOLANG_API __Errno Pipe(int vfd[2], int flags);
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
 LIBGOLANG_API __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
 #endif
 typedef void (*sighandler_t)(int);

golang/runtime/libgolang.cpp

@@ -52,7 +52,7 @@
 #include <linux/list.h>
 // MSVC does not support statement expressions and typeof
 // -> redo list_entry via C++ lambda.
-#ifdef _MSC_VER
+#ifdef LIBGOLANG_CC_msc
 # undef list_entry
 # define list_entry(ptr, type, member) [&]() {                      \
         const decltype( ((type *)0)->member ) *__mptr = (ptr);      \

golang/runtime/platform.h

+#ifndef _NXD_LIBGOLANG_RUNTIME_PLATFORM_H
+#define _NXD_LIBGOLANG_RUNTIME_PLATFORM_H
+
+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Header platform.h provides preprocessor defines that describe target platform.
+
+// LIBGOLANG_ARCH_<X> is defined on architecture X.
+//
+// List of supported architectures: 386, amd64, arm64.
+#if defined(__i386__) || defined(_M_IX86)
+# define LIBGOLANG_ARCH_386     1
+#elif defined(__x86_64__) || defined(_M_X64)
+# define LIBGOLANG_ARCH_amd64   1
+#elif defined(__aarch64__) || defined(_M_ARM64)
+# define LIBGOLANG_ARCH_arm64   1
+#else
+# error "unsupported architecture"
+#endif
+
+// LIBGOLANG_OS_<X> is defined on operating system X.
+//
+// List of supported operating systems: linux, darwin, windows.
+#ifdef __linux__
+# define LIBGOLANG_OS_linux     1
+#elif defined(__APPLE__)
+# define LIBGOLANG_OS_darwin    1
+#elif defined(_WIN32) || defined(__CYGWIN__)
+# define LIBGOLANG_OS_windows   1
+#else
+# error "unsupported operating system"
+#endif
+
+// LIBGOLANG_CC_<X> is defined on C/C++ compiler X.
+//
+// List of supported compilers: gcc, clang, msc.
+#ifdef __clang__
+# define LIBGOLANG_CC_clang     1
+#elif defined(_MSC_VER)
+# define LIBGOLANG_CC_msc       1
+// NOTE gcc comes last because e.g. clang and icc define __GNUC__ as well
+#elif __GNUC__
+# define LIBGOLANG_CC_gcc       1
+#else
+# error "unsupported compiler"
+#endif
+
+#endif  // _NXD_LIBGOLANG_RUNTIME_PLATFORM_H

gpython/.gitignore

+/_gpython.cpp

gpython/__init__.py

@@ -25,10 +25,14 @@ differences:
 
 - gevent is pre-activated and stdlib is patched to be gevent aware;
 - go, chan, select etc are put into builtin namespace;
-- default string encoding is always set to UTF-8.
+- default string encoding is always set to UTF-8;
+- bstr/ustr replace builtin str/unicode types.
 
 Gevent activation can be disabled via `-X gpython.runtime=threads`, or
 $GPYTHON_RUNTIME=threads.
+
+String types replacement can be disabled via `-X gpython.strings=pystd`, or
+$GPYTHON_STRINGS=pystd.
 """
 
 # NOTE gpython is kept out of golang/ , since even just importing e.g. golang.cmd.gpython,
@@ -230,9 +234,13 @@ def pymain(argv, init=None):
             gevent = sys.modules.get('gevent', None)
             gpyver = 'GPython %s' % golang.__version__
             if gevent is not None:
-                gpyver += ' [gevent %s]' % gevent.__version__
+                gpyver += ' [runtime gevent %s]' % gevent.__version__
+            else:
+                gpyver += ' [runtime threads]'
+            if type(u'') is golang.ustr:
+                gpyver += ' [strings bstr+ustr]'
             else:
-                gpyver += ' [threads]'
+                gpyver += ' [strings pystd]'
             ver.append(gpyver)
 
         import platform
@@ -344,6 +352,9 @@ def main():
     # imported first, e.g. to support sys.modules.
     import sys
 
+    # import pyx/c part of gpython
+    from gpython import _gpython
+
     # safety check that we are not running from a setuptools entrypoint, where
     # it would be too late to monkey-patch stdlib.
     #
@@ -372,6 +383,7 @@ def main():
         reload(sys)
         sys.setdefaultencoding('utf-8')
         delattr(sys, 'setdefaultencoding')
+        _gpython.set_utf8_as_default_src_encoding()
 
 
     # import os to get access to environment.
@@ -382,9 +394,11 @@ def main():
 
     # extract and process `-X gpython.*`
     # -X gpython.runtime=(gevent|threads)       + $GPYTHON_RUNTIME
+    # -X gpython.strings=(bstr+ustr|pystd)      + $GPYTHON_STRINGS
     sys._xoptions = getattr(sys, '_xoptions', {})
     argv_ = []
     gpy_runtime = os.getenv('GPYTHON_RUNTIME', 'gevent')
+    gpy_strings = os.getenv('GPYTHON_STRINGS', 'bstr+ustr')
     igetopt = _IGetOpt(sys.argv[1:], _pyopt, _pyopt_long)
     for (opt, arg) in igetopt:
         if opt == '-X':
@@ -393,6 +407,10 @@ def main():
                     gpy_runtime = arg[len('gpython.runtime='):]
                     sys._xoptions['gpython.runtime'] = gpy_runtime
 
+                elif arg.startswith('gpython.strings='):
+                    gpy_strings = arg[len('gpython.strings='):]
+                    sys._xoptions['gpython.strings'] = gpy_strings
+
                 else:
                     raise RuntimeError('gpython: unknown -X option %s' % arg)
 
@@ -412,13 +430,15 @@ def main():
     # sys.executable spawned from under `gpython -X gpython.runtime=threads`
     # also uses "threads" runtime by default.
     os.environ['GPYTHON_RUNTIME'] = gpy_runtime
+    os.environ['GPYTHON_STRINGS'] = gpy_strings
 
-    # init initializes according to selected runtime
+    # init initializes according to selected runtime and strings
     # it is called after options are parsed and sys.path is setup correspondingly.
     # this way golang and gevent are imported from exactly the same place as
     # they would be in standard python after regular import (ex from golang/
     # under cwd if run under `python -c ...` or interactive console.
     def init():
+        gpy_runtime_ver = gpy_runtime
         if gpy_runtime == 'gevent':
             # make gevent pre-available & stdlib patched
             import gevent
@@ -434,22 +454,30 @@ def main():
             if _ not in (True, None):   # patched or nothing to do
                 # XXX provide details
                 raise RuntimeError('gevent monkey-patching failed')
-            gpy_verextra = 'gevent %s' % gevent.__version__
+            gpy_runtime_ver += ' %s' % gevent.__version__
 
         elif gpy_runtime == 'threads':
-            gpy_verextra = 'threads'
-
+            pass
         else:
-            raise RuntimeError('gpython: invalid runtime %s' % gpy_runtime)
+            raise RuntimeError('gpython: invalid runtime %r' % gpy_runtime)
 
-        # put go, chan, select, ... into builtin namespace
+        if gpy_strings not in ('bstr+ustr', 'pystd'):
+            raise RuntimeError('gpython: invalid strings %r' % gpy_strings)
+
+        # import golang
+        # this will activate selected runtime and strings
+        sys._gpy_runtime = gpy_runtime
+        sys._gpy_strings = gpy_strings
         import golang
+
+        # put go, chan, select, ... into builtin namespace
         from six.moves import builtins
         for k in golang.__all__:
             setattr(builtins, k, getattr(golang, k))
+        setattr(builtins, 'CCC', CCC)
 
         # sys.version
-        sys.version += (' [GPython %s] [%s]' % (golang.__version__, gpy_verextra))
+        sys.version += (' [GPython %s] [runtime %s] [strings %s]' % (golang.__version__, gpy_runtime_ver, gpy_strings))
 
     # tail to pymain
     pymain(argv, init)
@@ -567,5 +595,11 @@ class _IGetOpt:
     next = __next__ # for py2
 
 
+# for tests XXX continue by first writing test  XXX
+1/0
+class _tEarlyStrSubclass(str):
+    pass
+
+
 if __name__ == '__main__':
     main()

gpython/_gpython.pyx

+# -*- coding: utf-8 -*-
+# cython: language_level=2
+# Copyright (C) 2023  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""_gpython.pyx ... XXX
+"""
+
+cdef extern from *:
+    """
+    void _set_utf8_as_default_src_encoding();
+    """
+    void _set_utf8_as_default_src_encoding() except *
+
+def set_utf8_as_default_src_encoding():
+    _set_utf8_as_default_src_encoding()

gpython/_gpython_c.cpp

+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// XXX doctitle
+
+#include <Python.h>
+#if PY_MAJOR_VERSION < 3
+#include <Python-ast.h> // mod_ty & co
+#include <node.h>       // node
+#include <graminit.h>   // encoding_decl & co
+#include <ast.h>        // PyAST_FromNode & co
+#endif
+
+#include <funchook.h>
+
+// py2: wrap PyAST_FromNode so that "utf-8" becomes the default encoding
+#if PY_MAJOR_VERSION < 3
+static auto   _py_PyAST_FromNode = &PyAST_FromNode;
+static mod_ty gpy_PyAST_FromNode(const node* n, PyCompilerFlags* flags,
+                                 const char* filename, PyArena* arena)
+{
+//  fprintf(stderr, "gpy_PyAST_FromNode...\n");
+    PyCompilerFlags gflags = {.cf_flags = 0};
+    if (flags)
+        gflags = *flags;
+    if (TYPE(n) != encoding_decl)
+        gflags.cf_flags |= PyCF_SOURCE_IS_UTF8;
+    return _py_PyAST_FromNode(n, &gflags, filename, arena);
+}
+
+static funchook_t* gpy_PyAST_FromNode_hook;
+void _set_utf8_as_default_src_encoding() {
+    funchook_t *h;
+    int err;
+
+//  funchook_set_debug_file("/dev/stderr");
+
+    gpy_PyAST_FromNode_hook = h = funchook_create();
+    if (h == NULL) {
+        PyErr_NoMemory();
+        return;
+    }
+
+    err = funchook_prepare(h, (void**)&_py_PyAST_FromNode, (void*)gpy_PyAST_FromNode);
+    if (err != 0) {
+        PyErr_SetString(PyExc_RuntimeError, funchook_error_message(h));
+        return;
+    }
+
+    err = funchook_install(h, 0);
+    if (err != 0) {
+        PyErr_SetString(PyExc_RuntimeError, funchook_error_message(h));
+        return;
+    }
+
+    // ok
+}
+#else
+void _set_utf8_as_default_src_encoding() {}
+#endif

gpython/gpython_test.py

@@ -47,20 +47,34 @@ gpython_only = pytest.mark.skipif(not is_gpython, reason="gpython-only test")
 def runtime(request):
     yield request.param
 
+# strings is pytest fixture that yields all variants of should be supported gpython strings:
+# '' - not specified (gpython should autoselect)
+# 'bstr+ustr'
+# 'pystd'
+@pytest.fixture(scope="function", params=['', 'bstr+ustr', 'pystd'])
+def strings(request):
+    yield request.param
+
 # gpyenv returns environment appropriate for spawning gpython with
-# specified runtime.
-def gpyenv(runtime): # -> env
+# specified runtime and strings.
+def gpyenv(runtime, strings): # -> env
     env = os.environ.copy()
     if runtime != '':
         env['GPYTHON_RUNTIME'] = runtime
     else:
         env.pop('GPYTHON_RUNTIME', None)
+    if strings != '':
+        env['GPYTHON_STRINGS'] = strings
+    else:
+        env.pop('GPYTHON_STRINGS', None)
     return env
 
 
 @gpython_only
 def test_defaultencoding_utf8():
     assert sys.getdefaultencoding() == 'utf-8'
+    assert eval("u'αβγ'") == u'αβγ'     # FIXME fails on py2 which uses hardcoded default latin1
+    # XXX +exec, +run file
 
 @gpython_only
 def test_golang_builtins():
@@ -143,19 +157,42 @@ def assert_gevent_not_activated():
 
 
 @gpython_only
-def test_executable(runtime):
+def test_str_patched():
+    # gpython, by default, patches str/unicode to be bstr/ustr.
+    # handling of various string modes is explicitly tested in test_Xstrings.
+    assert_str_patched()
+
+def assert_str_patched():
+    #assert str.__name__ == ('bstr'  if PY2 else  'ustr')
+    assert str.__name__ == 'str'
+    assert str          is (bstr    if PY2 else  ustr)
+    if PY2:
+        assert unicode.__name__ == 'unicode'
+        assert unicode  is ustr
+    assert type('')     is str
+    assert type(b'')    is (bstr    if PY2 else  bytes)
+    assert type(u'')    is ustr
+
+def assert_str_not_patched():
+    assert str.__name__ == 'str'
+    assert str is not bstr
+    assert str is not ustr
+    if PY2:
+        assert unicode.__name__ == 'unicode'
+        assert unicode is not bstr
+        assert unicode is not ustr
+    assert type('')     is str
+    assert type(b'')    is bytes
+    assert type(u'')    is (unicode if PY2 else str)
+
+
+@gpython_only
+def test_executable():
     # sys.executable must point to gpython and we must be able to execute it.
-    import gevent
     assert 'gpython' in sys.executable
-    ver = pyout(['-c', 'import sys; print(sys.version)'], env=gpyenv(runtime))
+    ver = pyout(['-c', 'import sys; print(sys.version)'], env=gpyenv('', ''))
     ver = str(ver)
     assert ('[GPython %s]' % golang.__version__) in ver
-    if runtime != 'threads':
-        assert ('[gevent %s]'  % gevent.__version__)     in ver
-        assert ('[threads]')                         not in ver
-    else:
-        assert ('[gevent ')                          not in ver
-        assert ('[threads]')                             in ver
 
 
 # verify pymain.
@@ -322,15 +359,20 @@ def test_pymain_opt():
 # pymain -V/--version
 # gpython_only because output differs from !gpython.
 @gpython_only
-def test_pymain_ver(runtime):
+def test_pymain_ver(runtime, strings):
     from golang import b
     from gpython import _version_info_str as V
     import gevent
     vok = 'GPython %s' % golang.__version__
     if runtime != 'threads':
-        vok += ' [gevent %s]' % gevent.__version__
+        vok += ' [runtime gevent %s]' % gevent.__version__
     else:
-        vok += ' [threads]'
+        vok += ' [runtime threads]'
+
+    if strings != 'pystd':
+        vok += ' [strings bstr+ustr]'
+    else:
+        vok += ' [strings pystd]'
 
     if is_cpython:
         vok += ' / CPython %s' % platform.python_version()
@@ -341,10 +383,12 @@ def test_pymain_ver(runtime):
 
     vok += '\n'
 
-    ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
+    env = gpyenv(runtime, strings)
+
+    ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE, env=env)
     assert (ret, out, b(err)) == (0, b'', b(vok))
 
-    ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
+    ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE, env=env)
     assert (ret, out, b(err)) == (0, b'', b(vok))
 
 # verify that ./bin/gpython runs ok.

pyproject.toml

 [build-system]
-requires = ["setuptools", "wheel", "setuptools_dso >= 2.7", "cython", "gevent"]
+requires = ["setuptools", "wheel", "setuptools_dso >= 2.7", "cython < 3", "gevent"]