X My draft state of x/gpystr work; py2/py3 pickle problem should be essentially solved

.gitmodules

+[submodule "3rdparty/funchook"]
+	path = 3rdparty/funchook
+	url = https://github.com/kubo/funchook.git
+[submodule "3rdparty/capstone"]
+	path = 3rdparty/capstone
+	url = https://github.com/capstone-engine/capstone.git

capstone @ 097c04d9

+Subproject commit 097c04d9413c59a58b00d4d1c8d5dc0ac158ffaa

funchook @ 88388db3

+Subproject commit 88388db3c69e16c1560fee65c6857d75f5ce6fd5

MANIFEST.in

@@ -2,6 +2,9 @@ include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .nxdtest
 include golang/libgolang.h
 include golang/runtime/libgolang.cpp
 include golang/runtime/libpyxruntime.cpp
+include golang/runtime/platform.h
+include golang/runtime.h
+include golang/runtime.cpp
 include golang/pyx/runtime.h
 include golang/pyx/testprog/golang_dso_user/dsouser/dso.h
 include golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
@@ -36,7 +39,10 @@ include golang/time.cpp
 include golang/_testing.h
 include golang/_compat/windows/strings.h
 include golang/_compat/windows/unistd.h
+include gpython/_gpython_c.cpp
 recursive-include golang    *.py *.pxd *.pyx *.toml *.txt*
-recursive-include gpython   *.py
-recursive-include 3rdparty  *.h
+recursive-include gpython   *.py       *.pyx
+recursive-include 3rdparty  *.h *.c *.cpp *.S *.py *.cmake *.cs *.java
+recursive-include 3rdparty  LICENSE README.md README COPYING Makefile CMakeLists.txt
 recursive-exclude golang    *_dsoinfo.py
+include conftest.py

README.rst

@@ -4,7 +4,7 @@
 
 Package `golang` provides Go-like features for Python:
 
-- `gpython` is Python interpreter with support for lightweight threads.
+- `gpython` is Python interpreter with support for lightweight threads and uniform UTF8-based approach to strings.
 - `go` spawns lightweight thread.
 - `chan` and `select` provide channels with Go semantic.
 - `func` allows to define methods separate from class.
@@ -46,15 +46,16 @@ __ http://libuv.org/
 __ http://software.schmorp.de/pkg/libev.html
 
 
-Additionally GPython sets UTF-8 to be default encoding always, and puts `go`,
-`chan`, `select` etc into builtin namespace.
+Additionally GPython sets UTF-8 to be default encoding always, puts `go`,
+`chan`, `select` etc into builtin namespace, and makes `bstr`/`ustr` to be used
+instead of builtin string types.
 
 .. note::
 
    GPython is optional and the rest of Pygolang can be used from under standard Python too.
    However without gevent integration `go` spawns full - not lightweight - OS thread.
-   GPython can be also used with threads - not gevent - runtime. Please see
-   `GPython options`_ for details.
+   GPython can be also used with threads - not gevent - runtime and with builtin string types.
+   Please see `GPython options`_ for details.
 
 
 Goroutines and channels
@@ -571,3 +572,9 @@ GPython-specific options and environment variables are listed below:
     coroutines, while with `threads` `go` spawns full OS thread. `gevent` is
     default. The runtime to use can be also specified via `$GPYTHON_RUNTIME`
     environment variable.
+
+`-X gpython.strings=(bstr+ustr|pystd)`
+    Specify which string types GPython should use. `bstr+ustr` provide
+    uniform UTF8-based approach to strings, while `pystd` selects regular
+    `str` and `unicode`. `bstr+ustr` is default. String types to use can be
+    also specified via `$GPYTHON_STRINGS` environment variable.

conftest.py

+# ignore tests in distorm - else it breaks as e.g.
+#
+# 3rdparty/funchook/distorm/python/test_distorm3.py:15: in <module>
+#     import distorm3
+# 3rdparty/funchook/distorm/python/distorm3/__init__.py:57: in <module>
+#     _distorm = _load_distorm()
+# 3rdparty/funchook/distorm/python/distorm3/__init__.py:55: in _load_distorm
+#     raise ImportError("Error loading the diStorm dynamic library (or cannot load library into process).")
+# E   ImportError: Error loading the diStorm dynamic library (or cannot load library into process).
+collect_ignore = ["3rdparty"]

golang/_golang.pyx

@@ -3,7 +3,7 @@
 # cython: binding=False
 # cython: c_string_type=str, c_string_encoding=utf8
 # distutils: language = c++
-# distutils: depends = libgolang.h os/signal.h unicode/utf8.h _golang_str.pyx
+# distutils: depends = libgolang.h os/signal.h unicode/utf8.h _golang_str.pyx _golang_str_pickle.pyx
 #
 # Copyright (C) 2018-2023  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
@@ -34,7 +34,7 @@ from __future__ import print_function, absolute_import
 _init_libgolang()
 _init_libpyxruntime()
 
-from cpython cimport PyObject, Py_INCREF, Py_DECREF, PY_MAJOR_VERSION
+from cpython cimport PyObject, Py_INCREF, Py_DECREF, Py_CLEAR, PY_MAJOR_VERSION
 ctypedef PyObject *pPyObject # https://github.com/cython/cython/issues/534
 cdef extern from "Python.h":
     ctypedef struct PyTupleObject:

golang/_golang_str.pyx

@@ -22,6 +22,8 @@
 It is included from _golang.pyx .
 """
 
+from libc.stdio cimport fprintf, stderr # XXX kill
+
 from golang.unicode cimport utf8
 
 from cpython cimport PyUnicode_AsUnicode, PyUnicode_GetSize, PyUnicode_FromUnicode
@@ -31,11 +33,13 @@ from cpython cimport Py_EQ, Py_NE, Py_LT, Py_GT, Py_LE, Py_GE
 from cpython.iterobject cimport PySeqIter_New
 from cpython cimport PyThreadState_GetDict, PyDict_SetItem
 from cpython cimport PyObject_CheckBuffer
+from cpython cimport Py_TPFLAGS_HAVE_GC, Py_TPFLAGS_HEAPTYPE, Py_TPFLAGS_READY, PyType_Ready
+from cpython cimport PyBytes_Format, PyUnicode_Format, PyObject_Str
 
 cdef extern from "Python.h":
     PyTypeObject PyBytes_Type
     ctypedef struct PyBytesObject:
-        pass
+        char *ob_sval
 
 cdef extern from "Python.h":
     PyTypeObject PyUnicode_Type
@@ -60,13 +64,40 @@ cdef extern from "Python.h":
     ctypedef struct _XPyTypeObject "PyTypeObject":
         PyObject* tp_new(PyTypeObject*, PyObject*, PyObject*) except NULL
         initproc  tp_init
+
+        Py_ssize_t tp_vectorcall_offset
+        Py_ssize_t tp_weaklistoffset
+
+        PyObject *tp_bases
+        PyObject *tp_mro
+        PyObject *tp_cache
+        PyObject *tp_weaklist
+        PyObject *tp_subclasses
+
         PySequenceMethods *tp_as_sequence
+        PyMethodDef       *tp_methods
+        PyMemberDef       *tp_members
 
     ctypedef struct PySequenceMethods:
         binaryfunc sq_concat
         binaryfunc sq_inplace_concat
         object (*sq_slice) (object, Py_ssize_t, Py_ssize_t)     # present only on py2
 
+cdef extern from "Python.h":
+    ctypedef struct PyVarObject:
+        Py_ssize_t ob_size
+
+
+cdef extern from "funchook.h" nogil:
+    ctypedef struct funchook_t
+    funchook_t* funchook_create()
+    int funchook_prepare(funchook_t* h, void** target_func, void* hook_func)
+    int funchook_install(funchook_t* h, int flags)
+    int funchook_uninstall(funchook_t* h, int flags)
+    int funchook_destroy(funchook_t*)
+    const char* funchook_error_message(const funchook_t*)
+    int funchook_set_debug_file(const char* name)
+
 
 from cython cimport no_gc
 
@@ -77,10 +108,6 @@ import string as pystring
 import types as pytypes
 import functools as pyfunctools
 import re as pyre
-if PY_MAJOR_VERSION >= 3:
-    import copyreg as pycopyreg
-else:
-    import copy_reg as pycopyreg
 
 
 # zbytes/zunicode point to original std bytes/unicode types even if they will be patched.
@@ -250,6 +277,8 @@ cdef __pystr(object obj): # -> ~str
         return pyb(obj)
 
 
+# XXX -> bchr ?  (not good as "character" means "unicode character")
+#     -> bstr.chr ?
 def pybbyte(int i): # -> 1-byte bstr
     """bbyte(i) returns 1-byte bstr with ordinal i."""
     return pyb(bytearray([i]))
@@ -259,6 +288,22 @@ def pyuchr(int i):  # -> 1-character ustr
     return pyu(unichr(i))
 
 
+# XXX due to "cannot `cdef class` with __new__" (https://github.com/cython/cython/issues/799)                   XXX review text
+# _pybstr.__new__ is hand-made in _pybstr_tp_new which invokes ↓ ._pybstr__new__() .
+# we keep it out of class instead of cdef @staticmethod due to https://github.com/cython/cython/issues/5337
+# XXX def instead of cdef due to ""Non-trivial keyword arguments and starred arguments not allowed in cdef functions
+def _pybstr__new__(cls, object='', encoding=None, errors=None):
+    # encoding or errors  ->  object must expose buffer interface
+    if not (encoding is None and errors is None):
+        object = _buffer_decode(object, encoding, errors)
+
+    # _bstringify. Note: it handles bstr/ustr / unicode/bytes/bytearray as documented
+    object = _bstringify(object)
+    assert isinstance(object, (unicode, bytes)), object
+    bobj = _pyb(cls, object)
+    assert bobj is not None
+    return bobj
+
 @no_gc                       # note setup.py assist this to compile despite
 cdef class _pybstr(bytes):   # https://github.com/cython/cython/issues/711
     """bstr is byte-string.
@@ -293,34 +338,26 @@ cdef class _pybstr(bytes):   # https://github.com/cython/cython/issues/711
     """
 
     # XXX due to "cannot `cdef class` with __new__" (https://github.com/cython/cython/issues/799)
-    # _pybstr.__new__ is hand-made in _pybstr_tp_new which invokes ↓ .____new__() .
-    @staticmethod
-    def ____new__(cls, object='', encoding=None, errors=None):
-        # encoding or errors  ->  object must expose buffer interface
-        if not (encoding is None and errors is None):
-            object = _buffer_decode(object, encoding, errors)
+    # _pybstr.__new__ is hand-made in _pybstr_tp_new which invokes ↑ _pybstr__new__() .
 
-        # _bstringify. Note: it handles bstr/ustr / unicode/bytes/bytearray as documented
-        object = _bstringify(object)
-        assert isinstance(object, (unicode, bytes)), object
-        bobj = _pyb(cls, object)
-        assert bobj is not None
-        return bobj
 
-
-    def __bytes__(self):    return self
+    def __bytes__(self):    return pyb(self)  # see __str__
     def __unicode__(self):  return pyu(self)
 
     def __str__(self):
         if PY_MAJOR_VERSION >= 3:
             return pyu(self)
         else:
-            return self
+            return pyb(self)  # self  or  pybstr if it was subclass
 
     def __repr__(self):
         qself, nonascii_escape = _bpysmartquote_u3b2(self)
         bs = _inbstringify_get()
         if bs.inbstringify == 0  or  bs.inrepr:
+            if pybstr is bytes:     # don't wrap with b(...) when bstr replaces builtin str
+                if PY_MAJOR_VERSION >= 3:
+                    qself = 'b' + qself
+                return qself
             if nonascii_escape:         # so that e.g. b(u'\x80') is represented as
                 qself = 'b' + qself     # b(b'\xc2\x80'),  not as b('\xc2\x80')
             return "b(" + qself + ")"
@@ -328,18 +365,8 @@ cdef class _pybstr(bytes):   # https://github.com/cython/cython/issues/711
             # [b('β')] goes as ['β'] when under _bstringify for %s
             return qself
 
-
-    # override reduce for protocols < 2. Builtin handler for that goes through
-    # copyreg._reduce_ex which eventually calls bytes(bstr-instance) to
-    # retrieve state, which gives bstr, not bytes. Fix state to be bytes ourselves.
     def __reduce_ex__(self, protocol):
-        if protocol >= 2:
-            return zbytes.__reduce_ex__(self, protocol)
-        return (
-            pycopyreg._reconstructor,
-            (self.__class__, self.__class__, _bdata(self))
-        )
-
+        return _bstr__reduce_ex__(self, protocol)
 
     def __hash__(self):
         # hash of the same unicode and UTF-8 encoded bytes is generally different
@@ -381,6 +408,7 @@ cdef class _pybstr(bytes):   # https://github.com/cython/cython/issues/711
             else:
                 return pyb(x)
 
+    # XXX temp disabled
     # __iter__  - yields unicode characters
     def __iter__(self):
         # TODO iterate without converting self to u
@@ -575,7 +603,7 @@ cdef PyObject* _pybstr_tp_new(PyTypeObject* _cls, PyObject* _argv, PyObject* _kw
     if _kw != NULL:
         kw = <object>_kw
 
-    cdef object x = _pybstr.____new__(<object>_cls, *argv, **kw)
+    cdef object x = _pybstr__new__(<object>_cls, *argv, **kw)
     Py_INCREF(x)
     return <PyObject*>x
 (<_XPyTypeObject*>_pybstr).tp_new   = &_pybstr_tp_new
@@ -592,6 +620,18 @@ cdef PyObject* _pybstr_tp_new(PyTypeObject* _cls, PyObject* _argv, PyObject* _kw
 # and bytes are completely the same.
 assert sizeof(_pybstr) == sizeof(PyBytesObject)
 
+# XXX text
+def _pyustr__new__(cls, object='', encoding=None, errors=None):
+    # encoding or errors  ->  object must expose buffer interface
+    if not (encoding is None and errors is None):
+        object = _buffer_decode(object, encoding, errors)
+
+    # _bstringify. Note: it handles bstr/ustr / unicode/bytes/bytearray as documented
+    object = _bstringify(object)
+    assert isinstance(object, (unicode, bytes)), object
+    uobj = _pyu(cls, object)
+    assert uobj is not None
+    return uobj
 
 @no_gc
 cdef class _pyustr(unicode):
@@ -622,27 +662,15 @@ cdef class _pyustr(unicode):
     """
 
     # XXX due to "cannot `cdef class` with __new__" (https://github.com/cython/cython/issues/799)
-    # _pyustr.__new__ is hand-made in _pyustr_tp_new which invokes ↓ .____new__() .
-    @staticmethod
-    def ____new__(cls, object='', encoding=None, errors=None):
-        # encoding or errors  ->  object must expose buffer interface
-        if not (encoding is None and errors is None):
-            object = _buffer_decode(object, encoding, errors)
-
-        # _bstringify. Note: it handles bstr/ustr / unicode/bytes/bytearray as documented
-        object = _bstringify(object)
-        assert isinstance(object, (unicode, bytes)), object
-        uobj = _pyu(cls, object)
-        assert uobj is not None
-        return uobj
+    # _pyustr.__new__ is hand-made in _pyustr_tp_new which invokes ↑ _pyustr__new__() .
 
 
     def __bytes__(self):    return pyb(self)
-    def __unicode__(self):  return self
+    def __unicode__(self):  return pyu(self)  # see __str__
 
     def __str__(self):
         if PY_MAJOR_VERSION >= 3:
-            return self
+            return pyu(self)  # = self  or  pyustr if it was subclass
         else:
             return pyb(self)
 
@@ -650,6 +678,11 @@ cdef class _pyustr(unicode):
         qself, nonascii_escape = _upysmartquote_u3b2(self)
         bs = _inbstringify_get()
         if bs.inbstringify == 0  or  bs.inrepr:
+            if pyustr is unicode:   # don't wrap with u(...) when ustr replaces builtin str/unicode
+                if not nonascii_escape: # but only if the string is valid utf-8
+                    if PY_MAJOR_VERSION < 3:
+                        qself = 'u'+qself
+                    return qself
             if nonascii_escape:
                 qself = 'b'+qself       # see bstr.__repr__
             return "u(" + qself + ")"
@@ -657,18 +690,8 @@ cdef class _pyustr(unicode):
             # [u('β')] goes as ['β'] when under _bstringify for %s
             return qself
 
-
-    # override reduce for protocols < 2. Builtin handler for that goes through
-    # copyreg._reduce_ex which eventually calls unicode(ustr-instance) to
-    # retrieve state, which gives ustr, not unicode. Fix state to be unicode ourselves.
     def __reduce_ex__(self, protocol):
-        if protocol >= 2:
-            return zunicode.__reduce_ex__(self, protocol)
-        return (
-            pycopyreg._reconstructor,
-            (self.__class__, self.__class__, _udata(self))
-        )
-
+        return _ustr__reduce_ex__(self, protocol)
 
     def __hash__(self):
         # see _pybstr.__hash__ for why we stick to hash of current str
@@ -718,7 +741,7 @@ cdef class _pyustr(unicode):
         # https://cython.readthedocs.io/en/latest/src/userguide/migrating_to_cy30.html#arithmetic-special-methods
         # see also https://github.com/cython/cython/issues/4750
         if type(a) is not pyustr:
-            assert type(b) is pyustr
+            assert type(b) is pyustr,  type(b)
             return b.__radd__(a)
         return pyu(zunicode.__add__(a, _pyu_coerce(b)))
 
@@ -738,7 +761,7 @@ cdef class _pyustr(unicode):
     # __mul__, __rmul__     (no need to override __imul__)
     def __mul__(a, b):
         if type(a) is not pyustr:
-            assert type(b) is pyustr
+            assert type(b) is pyustr,  type(b)
             return b.__rmul__(a)
         return pyu(zunicode.__mul__(a, b))
     def __rmul__(b, a):
@@ -939,7 +962,7 @@ cdef PyObject* _pyustr_tp_new(PyTypeObject* _cls, PyObject* _argv, PyObject* _kw
     if _kw != NULL:
         kw = <object>_kw
 
-    cdef object x = _pyustr.____new__(<object>_cls, *argv, **kw)
+    cdef object x = _pyustr__new__(<object>_cls, *argv, **kw)
     Py_INCREF(x)
     return <PyObject*>x
 (<_XPyTypeObject*>_pyustr).tp_new   = &_pyustr_tp_new
@@ -963,9 +986,10 @@ cdef class _pyustrIter:
 # _bdata/_udata retrieve raw data from bytes/unicode.
 def _bdata(obj): # -> bytes
     assert isinstance(obj, bytes)
-    _ = obj.__getnewargs__()[0] # (`bytes-data`,)
-    assert type(_) is bytes
-    return _
+    if type(obj) is not bytes:
+        obj = obj.__getnewargs__()[0] # (`bytes-data`,)
+    assert type(obj) is bytes
+    return obj
     """
     bcopy = bytes(memoryview(obj))
     assert type(bcopy) is bytes
@@ -973,9 +997,10 @@ def _bdata(obj): # -> bytes
     """
 def _udata(obj): # -> unicode
     assert isinstance(obj, unicode)
-    _ = obj.__getnewargs__()[0] # (`unicode-data`,)
-    assert type(_) is unicode
-    return _
+    if type(obj) is not unicode:
+        obj = obj.__getnewargs__()[0] # (`unicode-data`,)
+    assert type(obj) is unicode
+    return obj
     """
     cdef Py_UNICODE* u     = PyUnicode_AsUnicode(obj)
     cdef Py_ssize_t  size  = PyUnicode_GetSize(obj)
@@ -1027,6 +1052,22 @@ if PY2:
 
 # ---- adjust bstr/ustr classes after what cython generated ----
 
+# for pybstr/pyustr cython generates .tp_dealloc that refer to bytes/unicode types directly.
+# override that to refer to zbytes/zunicode to avoid infinite recursion on free.
+cdef void _pybstr_tp_dealloc(PyObject *self):   (<PyTypeObject*>zbytes)   .tp_dealloc(self)
+cdef void _pyustr_tp_dealloc(PyObject *self):   (<PyTypeObject*>zunicode) .tp_dealloc(self)
+(<PyTypeObject*>pybstr).tp_dealloc = &_pybstr_tp_dealloc
+(<PyTypeObject*>pyustr).tp_dealloc = &_pyustr_tp_dealloc
+
+# change names of bstr/ustr to be e.g. "golang.bstr" instead of "golang._golang._bstr"  XXX adjust after .name=str
+# this makes sure that unpickling saved bstr does not load via unpatched origin
+# class, and is also generally good for saving pickle size and for reducing _golang exposure.
+# XXX -> _golang_str_pickle.pyx ?
+(<PyTypeObject*>pybstr).tp_name = "golang.bstr"
+(<PyTypeObject*>pyustr).tp_name = "golang.ustr"
+assert pybstr.__module__ == "golang";  assert pybstr.__name__ == "bstr"
+assert pyustr.__module__ == "golang";  assert pyustr.__name__ == "ustr"
+
 # remove unsupported bstr/ustr methods. do it outside of `cdef class` to
 # workaround https://github.com/cython/cython/issues/4556 (`if ...` during
 # `cdef class` is silently handled wrongly)
@@ -1039,12 +1080,11 @@ cdef _bstrustr_remove_unsupported_slots():
         'removesuffix', # py3.9     TODO provide fallback implementation
     )
     for slot in vslot:
-        if not hasattr(unicode, slot):
-            _patch_slot(<PyTypeObject*>pybstr, slot, DEL)
-            try:
+        if not hasattr(zunicode, slot):
+            if hasattr(pybstr, slot):   # we might have already removed it on previous call
+                _patch_slot(<PyTypeObject*>pybstr, slot, DEL)
+            if hasattr(pyustr, slot):   # e.g. we do not define ustr.isprintable ourselves
                 _patch_slot(<PyTypeObject*>pyustr, slot, DEL)
-            except KeyError:    # e.g. we do not define ustr.isprintable ourselves
-                pass
 _bstrustr_remove_unsupported_slots()
 
 
@@ -1105,7 +1145,7 @@ cdef _bstringify(object obj): # -> unicode|bytes
     _bstringify_enter()
 
     try:
-        if PY_MAJOR_VERSION >= 3:
+        if False:   # PY_MAJOR_VERSION >= 3:
             # NOTE this depends on patches to bytes.{__repr__,__str__} below
             return unicode(obj)
 
@@ -1118,10 +1158,12 @@ cdef _bstringify(object obj): # -> unicode|bytes
             #
             # NOTE this depends on patches to bytes.{__repr__,__str__} and
             #      unicode.{__repr__,__str__} below.
-            if hasattr(obj, '__unicode__'):
-                return obj.__unicode__()
-            elif hasattr(obj, '__str__'):
-                return obj.__str__()
+            if False:   # PY_MAJOR_VERSION < 3  and  hasattr(obj, '__unicode__'):
+                return obj.__unicode__()    # XXX needed ?
+            elif Py_TYPE(obj).tp_str != NULL:
+                return Py_TYPE(obj).tp_str(obj)
+            #elif hasattr(obj, '__str__'):
+            #    return obj.__str__()
             else:
                 return repr(obj)
 
@@ -1422,19 +1464,24 @@ cdef _InBStringify _inbstringify_get():
     return ts_inbstringify
 
 
+# XXX text
+cdef _get_slot(PyTypeObject* typ, str name):
+    typdict = <dict>(typ.tp_dict)
+    return typdict[name]
+
 # _patch_slot installs func_or_descr into typ's __dict__ as name.
 #
-# if func_or_descr is descriptor (has __get__), it is installed as is.
+# if func_or_descr is descriptor (has __get__), or asis=True, it is installed as is.
 # otherwise it is wrapped with "unbound method" descriptor.
 #
 # if func_or_descr is DEL the slot is removed from typ's __dict__.
 cdef DEL = object()
-cdef _patch_slot(PyTypeObject* typ, str name, object func_or_descr):
+cdef _patch_slot(PyTypeObject* typ, str name, object func_or_descr, asis=False):
     typdict = <dict>(typ.tp_dict)
     #print("\npatching %s.%s  with  %r" % (typ.tp_name, name, func_or_descr))
     #print("old:  %r" % typdict.get(name))
 
-    if hasattr(func_or_descr, '__get__')  or  func_or_descr is DEL:
+    if hasattr(func_or_descr, '__get__')  or  func_or_descr is DEL  or  asis:
         descr = func_or_descr
     else:
         func = func_or_descr
@@ -1498,7 +1545,7 @@ cdef object _atidx_re = pyre.compile('.* at index ([0-9]+)$')
 cdef _bprintf(const byte[::1] fmt, xarg): # -> pybstr
     cdef bytearray out = bytearray()
 
-    cdef tuple  argv = None  # if xarg is tuple
+    cdef object argv = None  # if xarg is tuple or subclass
     cdef object argm = None  # if xarg is mapping
 
     # https://github.com/python/cpython/blob/2.7-0-g8d21aa21f2c/Objects/stringobject.c#L4298-L4300
@@ -1704,7 +1751,11 @@ cdef _bprintf(const byte[::1] fmt, xarg): # -> pybstr
 
         #print('--> __mod__ ', repr(fmt1), ' % ', repr(arg))
         try:
-            s = zbytes.__mod__(fmt1, arg)
+            IF PY2:
+                # NOTE not zbytes.__mod__ because underlying PyBytes_Format is patched
+                s = _pbytes_Format(fmt1, arg)
+            ELSE:
+                s = zbytes.__mod__(fmt1, arg)
         except ValueError as e:
             # adjust position in '... at index <idx>' from fmt1 to fmt
             if len(e.args) == 1:
@@ -1795,6 +1846,50 @@ class _BFormatter(pystring.Formatter):
             return super(_BFormatter, self).get_field(field_name, args, kwargs)
 
 
+# XXX place, comments
+# str % ... : ceval on py2 and py3 < 3.11 invokes PyString_Format / PyUnicode_Format
+#   directly upon seeing BINARY_MODULO. This leads to bstr.__mod__ not being called.
+ctypedef unicode uformatfunc(object, object)
+ctypedef bytes   bformatfunc(object, object)
+cdef uformatfunc* _punicode_Format = PyUnicode_Format
+cdef unicode _unicode_xFormat(object s, object args):
+    return pyustr.__mod__(s, args)
+
+IF PY2:
+    cdef bformatfunc* _pbytes_Format = PyBytes_Format
+    cdef _bytes_xFormat(object s, object args):
+        return pybstr.__mod__(s, args)
+
+cdef _patch_capi_str_format():
+    cpatch(<void**>&_punicode_Format, <void*>_unicode_xFormat)
+    IF PY2:
+        cpatch(<void**>&_pbytes_Format, <void*>_bytes_xFormat)
+
+
+# XXX place, comments, test
+#py3.11: specializes instructions. e.g. ustr(obj) will specialize (after
+#    executing 8 times) to directly invoke
+#
+#   PyObject_Str(obj)
+#
+#    which, if obj is e.g. b'123' will return "b'123'" instead of "123".
+#
+#    -> if we patch str=ustr, we need to patch PyObject_Str as well.
+#    -> XXX and check all other specializations.
+#
+# NOTE also good to just do
+cdef _object_xStr(object s):
+    IF PY2:
+        return pybstr(s)
+    ELSE:
+        return pyustr(s)
+ctypedef object objstrfunc(object)
+cdef objstrfunc* _pobject_Str = PyObject_Str
+cdef  _patch_capi_object_str():
+    cpatch(<void**>&_pobject_Str, <void*>_object_xStr)
+
+
+
 # ---- misc ----
 
 cdef object _xpyu_coerce(obj):
@@ -1871,6 +1966,7 @@ cdef extern from "Python.h":
 from six import unichr                      # py2: unichr       py3: chr
 from six import int2byte as bchr            # py2: chr          py3: lambda x: bytes((x,))
 
+# XXX turn vvv into compile-time constant
 cdef bint _ucs2_build = (sys.maxunicode ==     0xffff)      #    ucs2
 assert    _ucs2_build or sys.maxunicode >= 0x0010ffff       # or ucs4
 
@@ -1910,7 +2006,7 @@ cdef (rune, int) _utf8_decode_rune(const byte[::1] s):
 
 
 # _utf8_decode_surrogateescape mimics s.decode('utf-8', 'surrogateescape') from py3.
-def _utf8_decode_surrogateescape(const byte[::1] s): # -> unicode
+cdef _utf8_decode_surrogateescape(const byte[::1] s): # -> unicode
     if PY_MAJOR_VERSION >= 3:
         if len(s) == 0:
             return u''  # avoid out-of-bounds slice access on &s[0]
@@ -1950,7 +2046,7 @@ def _utf8_decode_surrogateescape(const byte[::1] s): # -> unicode
 
 
 # _utf8_encode_surrogateescape mimics s.encode('utf-8', 'surrogateescape') from py3.
-def _utf8_encode_surrogateescape(s): # -> bytes
+cdef _utf8_encode_surrogateescape(s): # -> bytes
     assert isinstance(s, unicode)
     if PY_MAJOR_VERSION >= 3:
         return zunicode.encode(s, 'UTF-8', 'surrogateescape')
@@ -2032,3 +2128,289 @@ cdef unicode _xunichr(rune i):
         uh = i - 0x10000
         return unichr(0xd800 + (uh >> 10)) + \
                unichr(0xdc00 + (uh & 0x3ff))
+
+
+# ---- funchook wrappers -----
+
+cdef funchook_t* xfunchook_create() except NULL:
+    h = funchook_create()
+    if h == NULL:
+        raise MemoryError()
+    return h
+
+cdef xfunchook_destroy(funchook_t* h):
+    err = funchook_destroy(h)
+    if err != 0:
+        raise RuntimeError(funchook_error_message(h))
+
+cdef xfunchook_prepare(funchook_t* h, void** target_func, void* hook_func):
+    err = funchook_prepare(h, target_func, hook_func)
+    if err != 0:
+        raise RuntimeError(funchook_error_message(h))
+
+cdef xfunchook_install(funchook_t* h, int flags):
+    err = funchook_install(h, flags)
+    if err != 0:
+        raise RuntimeError(funchook_error_message(h))
+
+cdef xfunchook_uninstall(funchook_t* h, int flags):
+    err = funchook_uninstall(h, flags)
+    if err != 0:
+        raise RuntimeError(funchook_error_message(h))
+
+# cpatch = xfunchook_prepare on _patch_capi_hook
+cdef cpatch(void** target_func, void* hook_func):
+    assert target_func[0] != NULL
+    xfunchook_prepare(_patch_capi_hook, target_func, hook_func)
+
+
+# ---- patch unicode/str types to be ustr/bstr under gpython ----
+# XXX make sure original _pybstr/_pyustr cannot be used after patching      XXX right ?
+# XXX and make sure golang._golang._pybstr cannot be imported as well  (ex pickle)
+# XXX ._pyustr.__module__ = 'builtins' after patch      - why?
+
+def _():
+    gpy_strings = getattr(sys, '_gpy_strings', None)
+    if gpy_strings == 'bstr+ustr':
+        _patch_str()
+    elif gpy_strings in ('pystd', None):
+        pass
+    else:
+        raise AssertionError("invalid sys._gpy_strings: %r" % (gpy_strings,))
+_()
+
+# _patch_str is invoked when gpython imports golang and instructs to replace
+# builtin str/unicode types with bstr/ustr.
+#
+# After the patch is applied all existing objects that have e.g. unicode type
+# will switch to having ustr type.
+cdef PyTypeObject _unicode_orig
+cdef PyTypeObject _bytes_orig
+cdef funchook_t* _patch_capi_hook
+cdef _patch_str():
+    global zbytes,   _bytes_orig,   pybstr
+    global zunicode, _unicode_orig, pyustr
+    global _patch_capi_hook
+
+    #print('\n\nPATCH\n\n')
+
+    # XXX explain
+    bpreserve_slots = upreserve_slots = ("maketrans",)
+    if PY_MAJOR_VERSION < 3:
+        bpreserve_slots += ("encode",) # @property'ies
+        upreserve_slots += ("decode",)
+
+    # patch unicode to be pyustr. This patches
+    # - unicode (py2)
+    # - str     (py3)
+    _pytype_clone(<PyTypeObject*>unicode, &_unicode_orig, "unicode(pystd)")
+    Py_INCREF(unicode)  # XXX needed?
+    zunicode = <object>&_unicode_orig
+
+    _pytype_replace_by_child(
+            <PyTypeObject*>unicode, &_unicode_orig,
+            <PyTypeObject*>pyustr, "ustr(origin)",
+            upreserve_slots)
+    pyustr = unicode    # retarget pyustr -> unicode to where it was copied
+    # XXX vvv needed so that patched unicode could be saved by py2:cPickle at all
+    (<PyTypeObject*>pyustr).tp_name = ("unicode" if PY_MAJOR_VERSION < 3  else "str")
+
+    # py2: patch str to be pybstr
+    if PY_MAJOR_VERSION < 3:
+        _pytype_clone(<PyTypeObject*>bytes, &_bytes_orig, "bytes(pystd)")
+        Py_INCREF(bytes)    # XXX needed?
+        zbytes = <object>&_bytes_orig
+
+        _pytype_replace_by_child(
+                <PyTypeObject*>bytes, &_bytes_orig,
+                <PyTypeObject*>_pybstr, "bstr(origin)",
+                bpreserve_slots)
+        pybstr = bytes  # retarget pybstr -> bytes to where it was copied
+        (<PyTypeObject*>pybstr).tp_name = ("str" if PY_MAJOR_VERSION < 3  else "bytes")
+
+    # need to remove unsupported slots in cloned bstr/ustr again since PyType_Ready might have recreated them
+    _bstrustr_remove_unsupported_slots()
+
+
+    # also patch UserString to have methods that bstr/ustr have
+    # else e.g. IPython's guarded_eval.py fails in `_list_methods(collections.UserString, dir(str))`
+    from six.moves import UserString
+    def userstr__bytes__(s):    return bytes(s.data)
+    def userstr__unicode__(s):  return unicode(s.data)
+    assert not hasattr(UserString, '__bytes__')         # XXX test
+    assert not hasattr(UserString, '__unicode__')
+    UserString.__bytes__   = userstr__bytes__
+    UserString.__unicode__ = userstr__unicode__
+
+    # XXX also patch CAPI functions ... XXX explain
+    #funchook_set_debug_file("/dev/stderr")
+    _patch_capi_hook = xfunchook_create()
+
+    _patch_capi_str_format()
+    _patch_capi_object_str()
+    _patch_capi_unicode_decode_as_bstr()
+    _patch_str_pickle()
+    # ...
+
+    xfunchook_install(_patch_capi_hook, 0)
+
+
+# XXX place ok ?
+include '_golang_str_pickle.pyx'
+
+# _pytype_clone clones PyTypeObject src into dst.
+# dst must not be previously initialized.
+#
+# dst will have reference-count = 1 meaning new reference to it is returned.
+cdef _pytype_clone(PyTypeObject *src, PyTypeObject *dst, const char* new_name):
+    assert (src.tp_flags & Py_TPFLAGS_READY) != 0
+    assert (src.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0    # src is not allocated on heap
+    #assert not PyType_IS_GC((<PyObject*>src).ob_type)  # XXX not true as unicode.ob_type is PyType_Type
+                                                        #     which generally has GC support, but
+                                                        #     GC is deactivated for non-heap types.
+    # copy the struct   XXX + .ob_next / .ob_prev (Py_TRACE_REFS)
+    dst[0] = src[0]
+    (<PyObject*>dst).ob_refcnt = 1
+
+    if new_name != NULL:
+        dst.tp_name = new_name
+
+    # now reinitialize things like .tp_dict etc, where PyType_Ready built slots that point to src.
+    # we want all those slots to be rebuilt and point to dst instead.
+    _dst = <_XPyTypeObject*>dst
+    dst .tp_flags &= ~Py_TPFLAGS_READY
+    dst .tp_dict     = NULL
+    _dst.tp_bases    = NULL
+    _dst.tp_mro      = NULL
+    _dst.tp_cache    = NULL
+    _dst.tp_weaklist = NULL
+
+    # dst.__subclasses__ will be empty because existing children inherit from src, not from dst.
+    _dst.tp_subclasses = NULL
+
+    PyType_Ready(<object>dst)
+    assert (dst.tp_flags & Py_TPFLAGS_READY) != 0
+
+# _pytype_replace_by_child replaces typ by its child egg.
+#
+# All existing objects that have type typ will switch to having type egg' .
+# The instance/inheritance diagram for existing objects and types will switch
+# as depicted below:
+#
+#           base                    base
+#            ↑                           ↖
+#           typ        ------>      egg' → typ_clone
+#          ↗ ↑ ↖                   ↗ ↑       ↗
+#   objects  X  egg         objects  X   egg
+#            ↑                       ↑
+#            Y                       Y
+#
+# typ_clone must be initialized via _pytype_clone(typ, typ_clone).
+# egg' is egg clone put inplace of typ
+#
+# XXX preserve_slots - describe
+cdef _pytype_replace_by_child(PyTypeObject *typ, PyTypeObject *typ_clone,
+                              PyTypeObject *egg, const char* egg_old_name,
+                              preserve_slots):
+    otyp = <PyObject*>typ           ; oegg = <PyObject*>egg
+    vtyp = <PyVarObject*>typ        ; vegg = <PyVarObject*>egg
+    _typ = <_XPyTypeObject*>typ     ; _egg = <_XPyTypeObject*>egg
+
+    assert egg.tp_base == typ
+    assert _egg.tp_subclasses == NULL
+
+    assert (typ.tp_flags & Py_TPFLAGS_READY)  != 0
+    assert (egg.tp_flags & Py_TPFLAGS_READY)  != 0
+
+    assert (typ.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0
+    assert (egg.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0 # XXX will be not true
+                                                     # -> ! Py_TPFLAGS_HAVE_GC
+                                                     # -> ? set Py_TPFLAGS_HEAPTYPE back on typ' ?
+
+    # (generally not required)
+    assert (typ.tp_flags & Py_TPFLAGS_HAVE_GC) == 0
+    assert (egg.tp_flags & Py_TPFLAGS_HAVE_GC) == 0
+    # XXX also check PyObject_IS_GC  (verifies .tp_is_gc() = n)  ?
+
+
+    assert vtyp.ob_size               ==  vegg.ob_size
+    assert typ .tp_basicsize          ==  egg .tp_basicsize
+    assert typ .tp_itemsize           ==  egg .tp_itemsize
+    IF PY3:
+        assert _typ.tp_vectorcall_offset  ==  _egg.tp_vectorcall_offset
+    assert _typ.tp_weaklistoffset     ==  _egg.tp_weaklistoffset
+    assert typ .tp_dictoffset         ==  egg .tp_dictoffset
+
+    # since egg will change .tp_base it will also need to reinitialize
+    # .tp_bases, .tp_mro and friends. Retrieve egg slots to preserve before we
+    # clear egg.__dict__ . This covers e.g. @staticmethod and @property.
+    keep_slots = {}  # name -> slot
+    for name in preserve_slots:
+        keep_slots[name] = _get_slot(egg, name)
+
+    # egg: clear what PyType_Ready will recompute
+    Py_CLEAR(egg .tp_dict)
+    Py_CLEAR(_egg.tp_bases)
+    Py_CLEAR(_egg.tp_mro)
+    Py_CLEAR(_egg.tp_cache)
+
+    # typ <- egg  preserving original typ's refcnt, weak references and subclasses\egg.
+    # typ will be now playing the role of egg
+    typ_refcnt     = otyp.ob_refcnt
+    typ_weaklist   = _typ.tp_weaklist
+    typ_subclasses = _typ.tp_subclasses
+    typ[0] = egg[0]
+    otyp.ob_refcnt     = typ_refcnt
+    _typ.tp_weaklist   = typ_weaklist
+    _typ.tp_subclasses = typ_subclasses # XXX need to remove egg from here
+
+    # adjust .tp_base
+    typ.tp_base = typ_clone
+    egg.tp_base = typ_clone
+
+    # adjust egg.tp_name
+    if egg_old_name != NULL:
+        egg.tp_name = egg_old_name
+
+    # reinitialize .tp_bases, .tp_mro. .tp_cache, and recompute slots that
+    # live in .tp_dict and point to their type. Do it for both typ (new egg)
+    # and origin egg for generality, even though original egg won't be used
+    # anymore.
+    typ.tp_flags &= ~Py_TPFLAGS_READY
+    egg.tp_flags &= ~Py_TPFLAGS_READY
+    PyType_Ready(<object>typ)
+    PyType_Ready(<object>egg)
+    assert (typ.tp_flags & Py_TPFLAGS_READY) != 0
+    assert (egg.tp_flags & Py_TPFLAGS_READY) != 0
+
+    # restore slots we were asked to preserve as is
+    # since those slots are e.g. @staticmethods they go to both egg' and egg.
+    for name, slot in keep_slots.items():
+        _patch_slot(typ, name, slot, asis=True)
+        _patch_slot(egg, name, slot, asis=True)
+
+    # XXX remove egg from typ.tp_subclasses     (also possible via setting .__bases__)
+    # XXX remove typ from base.tp_subclasses
+    #     else e.g. ustr(origin) is reported to be subclass of ustr by help()
+    #     (pyustr.__subclasses__()  give it)
+
+    # rebuild .tp_mro of all other typ's children
+    # initially X.__mro__ = (X, typ, base) and without rebuilding it would
+    # remain (X, egg', base) instead of correct (X, egg' typ_clone, base)
+    # XXX py3 does this automatically?  XXX -> no, it can invalidate .__mro__, but not .tp_mro
+    def refresh(x):
+        assert isinstance(x, type)
+        xtyp  = <PyTypeObject*>x
+        _xtyp = <_XPyTypeObject*>x
+        fprintf(stderr, 'refreshing %s\n', xtyp.tp_name)
+        assert (xtyp.tp_flags & Py_TPFLAGS_READY) != 0
+        xtyp.tp_flags &= ~Py_TPFLAGS_READY
+        Py_CLEAR(_xtyp.tp_mro)
+        PyType_Ready(x)
+        assert (xtyp.tp_flags & Py_TPFLAGS_READY) != 0
+        for _ in x.__subclasses__():
+            refresh(_)
+    for _ in (<object>typ).__subclasses__():
+        refresh(_)
+
+    # XXX also preserve ._ob_next + ._ob_prev  (present in Py_TRACE_REFS builds)

golang/_golang_str_pickle.S

+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// _golang_str_pickle.S complements _golang_str_pickle.pyx with assembly routines.
+
+#include "golang/runtime/platform.h"
+
+    .text
+    .p2align 4
+
+// CSYM returns assembler symbol for C-symbol name
+#if defined(LIBGOLANG_OS_darwin) || \
+    (defined(LIBGOLANG_OS_windows) && defined(LIBGOLANG_ARCH_386))
+# define CSYM(name)  _ ## name
+#else
+# define CSYM(name)       name
+#endif
+
+// _TYPE emits `.type sym, symtype` on systems where .type directive is supported
+// _SIZE emits `.size sym, symsize` on systems where .size directive is supported
+#ifdef LIBGOLANG_OS_linux
+# define _TYPE(sym, symtype) .type sym, symtype
+# define _SIZE(sym, symsize) .size sym, symsize
+#else
+# define _TYPE(sym, type)
+# define _SIZE(sym, size)
+#endif
+
+// inside_counted provides trampoline to call *inside_counted_func with
+// counting how many times that function entered inside and exited.
+//
+// Each enter increments inside_counter, while each exit decrements it.
+// Recursion is supported up to STK_SIZE times with counter stopping to be
+// adjusted at deeper recursion levels.
+//
+// inside_counted can be used on functions with arbitrary signatures because
+// all registers and stack arguments are preserved exactly as is on the call(*).
+//
+// (*) NOTE₁ on-stack return address / link-register is adjusted during the call.
+//           this prevents inside_counted to be used with e.g. x86.get_pc_thunk.ax .
+//     NOTE₂ on ARM64 xip0 (x16) is clobbered.
+#define inside_counted          CSYM(inside_counted)
+#define inside_counted_func     CSYM(inside_counted_func)
+#define inside_counter          CSYM(inside_counter)
+#define inside_counted_stk      CSYM(inside_counted_stk)
+    .globl  inside_counted
+    _TYPE(  inside_counted, @function   )
+inside_counted:
+#define STK_SIZE 8
+
+// ---- X86_64 / i386 ----
+
+#if defined(LIBGOLANG_ARCH_amd64) || defined(LIBGOLANG_ARCH_386)
+#if defined(LIBGOLANG_ARCH_amd64)
+# define REGSIZE  8
+# define rAX    rax
+# define rPCNT  rbx
+# define rCNT   rcx
+# define rPSTK  rdx
+# define rSP    rsp
+# ifndef LIBGOLANG_OS_windows
+   .macro LEAGOT sym, reg
+     movq   \sym@GOTPCREL(%rip), %\reg
+   .endm
+# else
+   // windows does not use PIC and relocates DLLs when loading them
+   // there is no GOT and we need to access in-DLL symbols directly
+   // see e.g. https://stackoverflow.com/q/13309662/9456786 for details.
+   .macro LEAGOT sym, reg
+     leaq   \sym(%rip), %\reg   // NOTE pc-relative addressing used to avoid LNK2017:
+   .endm                        // 'ADDR32' relocation ... invalid without /LARGEADDRESSAWARE:NO
+# endif
+#else
+# define REGSIZE  4
+# define rAX    eax
+# define rPCNT  ebx
+# define rCNT   ecx
+# define rPSTK  edx
+# define rSP    esp
+# ifndef LIBGOLANG_OS_windows
+   .macro LEAGOT sym, reg
+     call   .Lget_pc_\reg
+     addl   $_GLOBAL_OFFSET_TABLE_, %\reg
+     movl   \sym@GOT(%\reg), %\reg
+   .endm
+# else
+   // windows does not use PIC - see details in ^^^ amd64 case
+   .macro LEAGOT sym, reg
+     leal \sym, %\reg
+   .endm
+# endif
+#endif
+
+    sub     $REGSIZE, %rSP    // make place for jmp-via-ret to *inside_counted_func
+
+    // TODO consider adding cfi_* annotations, but probably it won't be simple
+    //      since we manipulate retaddr on the stack
+
+    push    %rAX        // save registers we'll use
+    push    %rPCNT
+    push    %rCNT
+    push    %rPSTK
+#define SP_JMPVIARET    (4*REGSIZE)
+#define SP_RETORIG      (5*REGSIZE)
+
+    // jmp-via-ret = *inside_counted_func
+    LEAGOT  inside_counted_func, rAX                    // &inside_counted_func
+    mov     (%rAX), %rAX                                //  inside_counted_func
+    mov     %rAX, SP_JMPVIARET(%rSP)
+
+    // check whether altstk is overflowed
+    // if it is - invoke the func without counting
+    LEAGOT  inside_counter, rPCNT                       // &inside_counter
+    mov     (%rPCNT), %rCNT                             //  inside_counter
+    cmp     $STK_SIZE, %rCNT
+    jge     .Lcall
+
+    // altstk is not overflowed
+    // push original ret to altstk and replace the ret to return to us after the call
+    LEAGOT  inside_counted_stk, rPSTK                   // &inside_counted_stk
+    mov     SP_RETORIG(%rSP), %rAX                      // original ret address
+    mov     %rAX, (%rPSTK,%rCNT,REGSIZE)                // inside_counted_stk[inside_counter] = retorig
+    add     $1, %rCNT                                   // inside_counter++
+    mov     %rCNT, (%rPCNT)
+
+#if defined(LIBGOLANG_ARCH_amd64)
+    lea     .Laftercall(%rip), %rAX
+#else
+    call    .Lget_pc_eax
+    add     $(.Laftercall-.), %rAX
+#endif
+    mov     %rAX, SP_RETORIG(%rSP)                      // replace ret addr on stack to .Laftercall
+
+.Lcall:
+    // restore registers and invoke the func through jmp-via-ret
+    pop     %rPSTK
+    pop     %rCNT
+    pop     %rPCNT
+    pop     %rAX
+    ret
+
+.Laftercall:
+    // we get here after invoked func returns if altstk was not overflowed
+    // decrement inside_counter and return to original ret address
+    sub     $REGSIZE, %rSP  // make place for original ret
+    push    %rAX            // save registers
+    push    %rPCNT
+    push    %rCNT
+    push    %rPSTK
+#undef  SP_RETORIG
+#define SP_RETORIG      (4*REGSIZE)
+
+    LEAGOT  inside_counter, rPCNT                       // &inside_counter
+    mov     (%rPCNT), %rCNT                             //  inside_counter
+    sub     $1, %rCNT
+    mov     %rCNT, (%rPCNT)                             //  inside_counter--
+    LEAGOT  inside_counted_stk,  rPSTK                  // &inside_counted_stk
+    mov     (%rPSTK,%rCNT,REGSIZE), %rAX                // retorig = inside_counted_stk[inside_counter]
+    mov     %rAX, SP_RETORIG(%rSP)
+
+    // restore registers and return to original caller
+    pop     %rPSTK
+    pop     %rCNT
+    pop     %rPCNT
+    pop     %rAX
+    ret
+
+#if defined(LIBGOLANG_ARCH_386)
+.macro DEF_get_pc reg
+ .Lget_pc_\reg:
+    mov     (%esp), %\reg
+    ret
+.endm
+DEF_get_pc eax
+DEF_get_pc ebx
+DEF_get_pc ecx
+DEF_get_pc edx
+#endif
+
+// ---- ARM64 ----
+
+#elif defined(LIBGOLANG_ARCH_arm64)
+#define REGSIZE 8
+#define rPCNT   x0
+#define rCNT    x1
+#define rPSTK   x2
+#define rXIP0   x16
+    stp     rPCNT, rCNT, [sp, -16]!     // save registers we'll use
+    stp     rPSTK, xzr,  [sp, -16]!     // NOTE xip0 is clobbered
+
+    // xip0 = *inside_counted_func
+    adrp    rXIP0, :got:inside_counted_func
+    ldr     rXIP0, [rXIP0, :got_lo12:inside_counted_func]   // &inside_counted_func
+    ldr     rXIP0, [rXIP0]                                  //  inside_counted_func
+
+    // check whether altstk is overflowed
+    // if it is - invoke the func without counting
+    adrp    rPCNT, :got:inside_counter
+    ldr     rPCNT, [rPCNT, :got_lo12:inside_counter]        // &inside_counter
+    ldr     rCNT, [rPCNT]                                   //  inside_counter
+    cmp     rCNT, STK_SIZE
+    bge     .Lcall
+
+    // altstk is not overflowed
+    // push original ret to altstk and replace the ret to return to us after the call
+    adrp    rPSTK, :got:inside_counted_stk
+    ldr     rPSTK, [rPSTK, :got_lo12:inside_counted_stk]    // &inside_counted_stk
+    str     lr, [rPSTK, rCNT, lsl 3]                        // inside_counted_stk[inside_counter] = retorig
+    add     rCNT, rCNT, 1                                   // inside_counter++
+    str     rCNT, [rPCNT]
+
+    adr     lr, .Laftercall                                 // replace ret addr to .Laftercall
+
+.Lcall:
+    // restore registers and invoke the func via xip0
+    ldp     rPSTK, xzr,  [sp], 16
+    ldp     rPCNT, rCNT, [sp], 16
+    br      rXIP0
+
+.Laftercall:
+    // we get here after invoked func returns if altstk was not overflowed
+    // decrement inside_counter and return to original ret address
+    stp     rPCNT, rCNT, [sp, -16]!     // save registers
+    stp     rPSTK, xzr,  [sp, -16]!
+
+    adrp    rPCNT, :got:inside_counter
+    ldr     rPCNT, [rPCNT, :got_lo12:inside_counter]        // &inside_counter
+    ldr     rCNT, [rPCNT]                                   //  inside_counter
+    sub     rCNT, rCNT, 1
+    str     rCNT, [rPCNT]                                   //  inside_counter--
+    adrp    rPSTK, :got:inside_counted_stk
+    ldr     rPSTK, [rPSTK, :got_lo12:inside_counted_stk]    // &inside_counted_stk
+    ldr     lr, [rPSTK, rCNT, lsl 3]                        // lr = inside_counted_stk[inside_counter]
+
+    // restore registers and return to original caller
+    ldp     rPSTK, xzr,  [sp], 16
+    ldp     rPCNT, rCNT, [sp], 16
+    ret
+
+#else
+# error "unsupported architecture"
+#endif
+
+    _SIZE(  inside_counted, .-inside_counted    )
+
+// ---- data ---
+    .bss
+
+// void* inside_counted_func
+    .globl  inside_counted_func
+    .p2align 3  // 8
+    _TYPE(  inside_counted_func, @object    )
+    _SIZE(  inside_counted_func, REGSIZE    )
+inside_counted_func:
+    .zero   REGSIZE
+
+// long inside_counter
+    .globl  inside_counter
+    .p2align 3  // 8
+    _TYPE(  inside_counter, @object )
+    _SIZE(  inside_counter, REGSIZE )
+inside_counter:
+    .zero   REGSIZE
+
+// void* inside_counted_stk[STK_SIZE]
+    .globl  inside_counted_stk
+    .p2align 5  // 32
+    _TYPE(  inside_counted_stk, @object             )
+    _SIZE(  inside_counted_stk, STK_SIZE*REGSIZE    )
+inside_counted_stk:
+    .zero   STK_SIZE*REGSIZE
+
+
+// disable executable stack
+#ifndef LIBGOLANG_OS_windows
+    .section        .note.GNU-stack,"",@progbits
+#endif
+
+
+// ---- custom callconv proxies ----
+    .text
+    .p2align 4
+
+// saveprobe_<callconv>            (self, obj, pers_save)  input callconv, proxy to saveprobe
+// _pickle_Pickler_xsave_<callconv>(self, obj, pers_save)  input callconv, proxy to _pickle_Pickler_xsave
+// save_invoke_as_<callconv> (save, self, obj, pers_save)  input std, proxy to save invoked via callconv
+
+
+#if defined(LIBGOLANG_ARCH_386)
+
+#ifdef LIBGOLANG_CC_msc
+# define CSYM_FASTCALL3(name)   @name@12     // MSVC mangles __fastcall
+# define CSYM_FASTCALL4(name)   @name@16
+#else
+# define CSYM_FASTCALL3(name)   CSYM(name)
+# define CSYM_FASTCALL4(name)   CSYM(name)
+#endif
+
+// python-3.11.5.exe has _pickle.save accepting arguments in ecx,edx,stack but
+// contrary to fastcall the callee does not cleanup the stack.
+// Handle this as fastcall_nostkclean
+
+.macro FUNC_fastcall_nostkclean name
+    .globl  CSYM(\name\()_fastcall_nostkclean)
+    _TYPE(  CSYM(\name\()_fastcall_nostkclean), @function    )
+CSYM(\name\()_fastcall_nostkclean):
+    // we are proxying to fastcall - ecx and edx are already setup and we
+    // need to only duplicate the 3rd argument on the stack. Do this without
+    // clobbering any register.
+    sub     $4, %esp        // place to copy on-stack argument to
+    push    %eax
+    mov     12(%esp), %eax  // original on-stack arg
+    mov     %eax, 4(%esp)   // dup to copy
+    pop     %eax
+
+    call    CSYM_FASTCALL3(\name\()_ifastcall)
+    // ^^^ cleaned up the stack from our copy
+    // nothing to do anymore
+    ret
+    _SIZE(  CSYM(\name\()_fastcall_nostkclean), .-CSYM(\name\()_fastcall_nostkclean)  )
+.endm
+FUNC_fastcall_nostkclean  saveprobe
+FUNC_fastcall_nostkclean  _pickle_Pickler_xsave
+FUNC_fastcall_nostkclean  _zpickle_Pickler_xsave
+
+#define save_invoke_as_fastcall_nostkclean  CSYM_FASTCALL4(save_invoke_as_fastcall_nostkclean)
+    .globl  save_invoke_as_fastcall_nostkclean
+    _TYPE(  save_invoke_as_fastcall_nostkclean, @function   )
+save_invoke_as_fastcall_nostkclean:
+    // input:
+    //      ecx:     save
+    //      edx:     self
+    //      stk[1]:  obj
+    //      stk[2]:  pers_save
+    //
+    // invoke save as:
+    //      ecx:     self
+    //      edx:     obj
+    //      stk*[1]: pers_save
+
+    mov     8(%esp), %eax       // pers_save
+    push    %eax                // stk*[1] <- per_save
+
+    mov     %ecx, %eax          // eax <- save
+    mov     %edx, %ecx          // ecx <- self
+    mov     (4+4)(%esp), %edx   // edx <- obj
+
+    call    *%eax
+
+    // return with cleaning up stack
+    add     $4, %esp    // pers_save copy we created
+    ret     $8          // original arguments
+    _SIZE(  save_invoke_as_fastcall_nostkclean, .-save_invoke_as_fastcall_nostkclean)
+
+#endif  // 386

golang/_golang_str_pickle.pyx

+# -*- coding: utf-8 -*-
+# Copyright (C) 2023  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""_golang_str_pickle.pyx complements _golang_str.pyx and keeps everything
+related to pickling strings.
+
+It is included from _golang_str.pyx .
+
+The main entry-points are _patch_str_pickle and _patch_capi_unicode_decode_as_bstr.
+"""
+
+from cpython cimport PyUnicode_Decode
+from cpython cimport PyBytes_FromStringAndSize, _PyBytes_Resize
+
+cdef extern from "Python.h":
+    char* PyBytes_AS_STRING(PyObject*)
+    Py_ssize_t PyBytes_GET_SIZE(PyObject*)
+
+cdef extern from "Python.h":
+    ctypedef PyObject* (*PyCFunction)(PyObject*, PyObject*)
+    ctypedef struct PyMethodDef:
+        const char* ml_name
+        PyCFunction ml_meth
+    ctypedef struct PyCFunctionObject:
+        PyMethodDef *m_ml
+        PyObject*    m_self
+        PyObject*    m_module
+
+cdef extern from "structmember.h":
+    ctypedef struct PyMemberDef:
+        const char* name
+        int         type
+        Py_ssize_t  offset
+
+    enum:
+        T_INT
+
+from libc.stdlib cimport malloc, free
+from libc.string cimport memcpy, memcmp
+
+if PY_MAJOR_VERSION >= 3:
+    import copyreg as pycopyreg
+else:
+    import copy_reg as pycopyreg
+
+cdef object zbinary  # = zodbpickle.binary | None
+try:
+    import zodbpickle
+except ImportError:
+    zbinary = None
+else:
+    zbinary = zodbpickle.binary
+
+
+# support for pickling bstr/ustr as standalone types.
+#
+# pickling is organized in such a way that
+# - what is saved by py2 can be loaded correctly on both py2/py3,  and similarly
+# - what is saved by py3 can be loaded correctly on both py2/py3   as well.
+#
+# XXX place
+cdef _bstr__reduce_ex__(self, protocol):
+    # Ideally we want to emit bstr(BYTES), but BYTES is not available for
+    # protocol < 3. And for protocol < 3 emitting bstr(STRING) is not an
+    # option because plain py3 raises UnicodeDecodeError on loading arbitrary
+    # STRING data. However emitting bstr(UNICODE) works universally because
+    # pickle supports arbitrary unicode - including invalid unicode - out of
+    # the box and in exactly the same way on both py2 and py3. For the
+    # reference upstream py3 uses surrogatepass on encode/decode UNICODE data
+    # to achieve that.
+    if protocol < 3:
+        # use UNICODE for data
+        udata = _udata(pyu(self))
+        if protocol < 2:
+            return (self.__class__, (udata,))   # bstr UNICODE REDUCE
+        else:
+            return (pycopyreg.__newobj__,
+                    (self.__class__, udata))    # bstr UNICODE NEWOBJ
+    else:
+        # use BYTES for data
+        bdata = _bdata(self)
+        if PY_MAJOR_VERSION < 3:
+            # the only way we can get here on py2 and protocol >= 3 is zodbpickle
+            # -> similarly to py3 save bdata as BYTES
+            assert zbinary is not None
+            bdata = zbinary(bdata)
+        return (
+            pycopyreg.__newobj__,               # bstr BYTES   NEWOBJ
+            (self.__class__, bdata))
+
+cdef _ustr__reduce_ex__(self, protocol):
+    # emit ustr(UNICODE).
+    # TODO later we might want to switch to emitting ustr(BYTES)
+    #      even if we do this, it should be backward compatible
+    if protocol < 2:
+        return (self.__class__, (_udata(self),))# ustr UNICODE REDUCE
+    else:
+        return (pycopyreg.__newobj__,           # ustr UNICODE NEWOBJ
+                (self.__class__, _udata(self)))
+
+
+
+# types used while patching
+cdef extern from *:
+    """
+    struct PicklerObject;
+    """
+    struct PicklerObject:
+        pass
+
+cdef struct PicklerTypeInfo:
+    Py_ssize_t size                 # sizeof(PicklerObject)
+    Py_ssize_t off_bin              # offsetof `int bin`
+    Py_ssize_t off_poutput_buffer   # offsetof `PyObject *output_buffer`
+    Py_ssize_t off_output_len       # offsetof `Py_ssize_t output_len`
+    Py_ssize_t off_max_output_len   # offsetof `Py_ssize_t max_output_len`
+
+
+# XXX place ?
+cdef extern from * nogil:
+    r"""
+    // CALLCONV instructs compiler to use specified builtin calling convention.
+    // it should be used like this:
+    //
+    //  int CALLCONV(stdcall) myfunc(...)
+    #ifndef LIBGOLANG_CC_msc
+    # define CALLCONV(callconv) __attribute__((callconv))
+    #else // MSC
+    # define CALLCONV(callconv) __##callconv
+    #endif
+
+
+    // FOR_EACH_CALLCONV invokes macro X(ccname, callconv, cckind) for every supported calling convention.
+    // cckind is one of `builtin` or `custom`.
+    #ifdef LIBGOLANG_ARCH_386
+    # ifndef LIBGOLANG_CC_msc
+    #  define FOR_EACH_CALLCONV(X)  \
+         X(default,,                            builtin)    \
+         X(cdecl,       CALLCONV(cdecl),        builtin)    \
+         X(stdcall,     CALLCONV(stdcall),      builtin)    \
+         X(fastcall,    CALLCONV(fastcall),     builtin)    \
+         X(thiscall,    CALLCONV(thiscall),     builtin)    \
+         X(regparm1,    CALLCONV(regparm(1)),   builtin)    \
+         X(regparm2,    CALLCONV(regparm(2)),   builtin)    \
+         X(regparm3,    CALLCONV(regparm(3)),   builtin)    \
+         X(fastcall_nostkclean,  na,            custom )
+    # else // MSC
+    #  define FOR_EACH_CALLCONV(X)  \
+         X(default,,                            builtin)    \
+         X(cdecl,       CALLCONV(cdecl),        builtin)    \
+         X(stdcall,     CALLCONV(stdcall),      builtin)    \
+         X(fastcall,    CALLCONV(fastcall),     builtin)    \
+         /* X(CALLCONV(thiscall),   thiscall)   MSVC emits "C3865: '__thiscall': can only be used on native member functions"       */ \
+         /* in theory we can emulate thiscall via fastcall https://tresp4sser.wordpress.com/2012/10/06/how-to-hook-thiscall-functions/ */ \
+         X(vectorcall,  CALLCONV(vectorcall),   builtin)    \
+         X(fastcall_nostkclean,  na,            custom )
+    # endif
+    #elif defined(LIBGOLANG_ARCH_amd64)
+    # define FOR_EACH_CALLCONV(X)   \
+        X(default,,                             builtin)
+    #elif defined(LIBGOLANG_ARCH_arm64)
+    # define FOR_EACH_CALLCONV(X)   \
+        X(default,,             builtin)
+    #else
+    # error "unsupported architecture"
+    #endif
+
+    // Callconv denotes calling convention of a function.
+    enum Callconv {
+    #define CC_ENUM1(ccname, _, __) \
+        CALLCONV_##ccname,
+    FOR_EACH_CALLCONV(CC_ENUM1)
+    };
+
+    const char* callconv_str(Callconv cconv) {
+        using namespace golang;
+        switch(cconv) {
+        #define CC_STR1(ccname, _, __)  \
+        case CALLCONV_##ccname:       \
+            return #ccname;
+        FOR_EACH_CALLCONV(CC_STR1)
+        default:
+            panic("bug");
+        }
+    }
+
+    // SaveFunc represents a save function - its address and calling convention.
+    struct SaveFunc {
+        void*     addr;
+        Callconv  cconv;
+    };
+    """
+    enum Callconv: pass
+    const char* callconv_str(Callconv)
+    struct SaveFunc:
+        void*    addr
+        Callconv cconv
+
+# XXX doc
+cdef struct _pickle_PatchCtx:
+    initproc Unpickler_tp_xinit             # func to replace Unpickler.tp_init
+    initproc Unpickler_tp_init_orig         # what was there before
+
+    vector[SaveFunc]  Pickler_xsave_ccv     # func to replace _Pickler_save  (all callconv variants)
+    SaveFunc          Pickler_save_orig     # what was there before
+
+    PicklerTypeInfo iPickler                # information detected about PicklerObject type
+
+
+# patch contexts for _pickle and _zodbpickle modules
+cdef _pickle_PatchCtx _pickle_patchctx
+cdef _pickle_PatchCtx _zpickle_patchctx
+
+
+# _patch_str_pickle patches *pickle modules to support bstr/ustr and UTF-8 properly.
+#
+# STRING opcodes are handled in backward-compatible way:
+#
+# - *STRING are loaded as bstr
+# - bstr is saved as *STRING
+# - pickletools decodes *STRING as UTF-8
+cdef _patch_str_pickle():
+    try:
+        import zodbpickle
+    except ImportError:
+        zodbpickle = None
+
+    # py3: pickletools.dis raises UnicodeDecodeError on non-ascii STRING and treats *BINSTRING as latin1
+    #      -> decode as UTF8b instead
+    if PY_MAJOR_VERSION >= 3:
+        import pickletools, codecs
+        _codecs_escape_decode = codecs.escape_decode
+        def xread_stringnl(f):
+            data = _codecs_escape_decode(pickletools.read_stringnl(f, decode=False))[0]
+            return pybstr(data)
+        def xread_string1(f):
+            data = pickletools.read_string1(f).encode('latin1')
+            return pybstr(data)
+        def xread_string4(f):
+            data = pickletools.read_string4(f).encode('latin1')
+            return pybstr(data)
+
+        pickletools.stringnl.reader = xread_stringnl
+        pickletools.string1.reader  = xread_string1
+        pickletools.string4.reader  = xread_string4
+
+        if zodbpickle:
+            from zodbpickle import pickletools_3 as zpickletools
+            zpickletools.stringnl.reader = xread_stringnl   # was same logic as in std pickletools
+            zpickletools.string1.reader  = xread_string1
+            zpickletools.string4.reader  = xread_string4
+
+    # py3: pickle.load wants to treat *STRING as bytes and decode it as ASCII
+    #      -> adjust to decode to bstr instead
+    #      -> also save bstr via *STRING opcodes so that load/save is identity
+        import pickle, _pickle
+        # TODO _pickle not available (pypy)
+        _pickle_patchctx.Unpickler_tp_xinit = _pickle_Unpickler_xinit
+        _pickle_patchctx.Pickler_xsave_ccv  = _pickle_Pickler_xsave_ccv
+        _patch_pickle(pickle, _pickle, &_pickle_patchctx)
+
+        if zodbpickle:
+            from zodbpickle import pickle as zpickle, _pickle as _zpickle
+            from zodbpickle import slowpickle as zslowPickle, fastpickle as zfastPickle
+            # TODO _pickle / fastpickle not available (pypy)
+            for x in 'load', 'loads', 'Unpickler', 'dump', 'dumps', 'Pickler':
+                assert getattr(_zpickle, x) is getattr(zfastPickle, x)
+                assert getattr(zpickle, x)  is getattr(_zpickle, x)
+            _patch_pickle(zslowPickle, None, NULL)
+            _zpickle_patchctx.Unpickler_tp_xinit = _zpickle_Unpickler_xinit
+            _zpickle_patchctx.Pickler_xsave_ccv  = _zpickle_Pickler_xsave_ccv
+            _patch_pickle(None, zfastPickle, &_zpickle_patchctx)
+            # propagate changes from fastpickle -> _zpickle -> zpickle
+            _zpickle.load  = zfastPickle.load
+            _zpickle.loads = zfastPickle.loads
+            _zpickle.dump  = zfastPickle.dump
+            _zpickle.dumps = zfastPickle.dumps
+            assert _zpickle.Unpickler is zfastPickle.Unpickler
+            assert _zpickle.Pickler   is zfastPickle.Pickler
+            zpickle.load   = zfastPickle.load
+            zpickle.loads  = zfastPickle.loads
+            zpickle.dump   = zfastPickle.dump
+            zpickle.dumps  = zfastPickle.dumps
+            assert zpickle.Unpickler  is zfastPickle.Unpickler
+            assert zpickle.Pickler    is zfastPickle.Pickler
+
+# _patch_pickle serves _patch_str_pickle by patching pair of py-by-default and
+# C implementations of a pickle module.
+#
+# pickle or _pickle being None indicates that corresponding module version is not available.
+cdef _patch_pickle(pickle, _pickle, _pickle_PatchCtx* _pctx):
+    # if C module is available - it should shadow default py implementation
+    if _pickle is not None  and  pickle is not None:
+        assert pickle.load      is  _pickle.load
+        assert pickle.loads     is  _pickle.loads
+        assert pickle.Unpickler is  _pickle.Unpickler
+        assert pickle.dump      is  _pickle.dump
+        assert pickle.dumps     is  _pickle.dumps
+        assert pickle.Pickler   is  _pickle.Pickler
+
+    # patch C
+    if _pickle is not None:
+        _patch_cpickle(_pickle, _pctx)
+        # propagate C updates to py
+        if pickle is not None:
+            pickle.load      = _pickle.load
+            pickle.loads     = _pickle.loads
+            pickle.Unpickler = _pickle.Unpickler
+            pickle.dump      = _pickle.dump
+            pickle.dumps     = _pickle.dumps        # XXX needed?
+            pickle.Pickler   = _pickle.Pickler
+
+    # patch py
+    if pickle is not None:
+        _patch_pypickle(pickle, shadowed = (_pickle is not None))
+
+# _patch_pypickle serves _patch_pickle for py version.
+cdef _patch_pypickle(pickle, shadowed):
+    def pyattr(name):
+        if shadowed:
+            name = '_'+name
+        return getattr(pickle, name)
+
+    # adjust load / loads / Unpickler to use 'bstr' encoding by default
+    Unpickler = pyattr('Unpickler')
+    for f in pyattr('load'), pyattr('loads'), Unpickler.__init__:
+        f.__kwdefaults__['encoding'] = 'bstr'
+
+    # patch Unpickler._decode_string to handle 'bstr' encoding
+    # zodbpickle uses .decode_string from first version of patch from bugs.python.org/issue6784
+    has__decode = hasattr(Unpickler, '_decode_string')
+    has_decode  = hasattr(Unpickler, 'decode_string')
+    assert has__decode or has_decode
+    assert not (has__decode and has_decode)
+    _decode_string = '_decode_string'  if has__decode  else  'decode_string'
+
+    Unpickler_decode_string = getattr(Unpickler, _decode_string)
+    def _xdecode_string(self, value):
+        if self.encoding == 'bstr':
+            return pyb(value)
+        else:
+            return Unpickler_decode_string(self, value)
+    setattr(Unpickler, _decode_string, _xdecode_string)
+
+    # adjust Pickler to save bstr as STRING
+    from struct import pack
+    Pickler = pyattr('Pickler')
+    def save_bstr(self, obj):
+        cdef bint nonascii_escape  # unused
+        if self.proto >= 1:
+            n = len(obj)
+            if n < 256:
+                op = b'U' + bytes((n,)) + _bdata(obj)   # SHORT_BINSTRING
+            else:
+                op = b'T' + pack('<i', n) + _bdata(obj) # BINSTRING
+        else:
+            qobj = strconv._quote(obj, b"'", &nonascii_escape)
+            op = b'S' + qobj + b'\n'                    # STRING
+        self.write(op)
+        self.memoize(obj)
+    Pickler.dispatch[pybstr] = save_bstr
+
+# _patch_cpickle serves _patch_pickle for C version.
+cdef _patch_cpickle(_pickle, _pickle_PatchCtx *pctx):
+    # adjust load / loads to use 'bstr' encoding by default
+    # builtin_function_or_method does not have __kwdefaults__  (defaults for
+    # arguments are hardcoded in generated C code)
+    # -> wrap functions
+    _pickle_load  = _pickle.load
+    _pickle_loads = _pickle.loads
+    def load (file,    *, **kw):
+        kw.setdefault('encoding', 'bstr')
+        return _pickle_load (file, **kw)
+    def loads(data,    *, **kw):
+        kw.setdefault('encoding', 'bstr')
+        return _pickle_loads(data, **kw)
+    _pickle.load  = load
+    _pickle.loads = loads
+
+    # adjust Unpickler to use 'bstr' encoding by default
+    assert isinstance(_pickle.Unpickler, type)
+    cdef _XPyTypeObject* Unpickler = <_XPyTypeObject*>(_pickle.Unpickler)
+
+    pctx.Unpickler_tp_init_orig = Unpickler.tp_init
+    Unpickler.tp_init = pctx.Unpickler_tp_xinit
+
+    def Unpickler_x__init__(self, *argv, **kw):
+        # NOTE don't return - just call: __init__ should return None
+        pctx.Unpickler_tp_xinit(self, <PyObject*>argv, <PyObject*>kw)
+
+    _patch_slot(<PyTypeObject*>Unpickler, '__init__', Unpickler_x__init__)
+    # decoding to bstr relies on _patch_capi_unicode_decode_as_bstr
+
+    # adjust Pickler to save bstr as *STRING
+    # it is a bit involved because:
+    # - save function, that we need to patch, is not exported.
+    # - _Pickle_Write, that we need to use from patched save, is not exported neither.
+    pctx.iPickler = _detect_Pickler_typeinfo(_pickle.Pickler)
+    pctx.Pickler_save_orig = save = _find_Pickler_save(_pickle.Pickler)
+    xsave = pctx.Pickler_xsave_ccv[save.cconv]
+    assert xsave.cconv == save.cconv, (callconv_str(xsave.cconv), callconv_str(save.cconv))
+    cpatch(&pctx.Pickler_save_orig.addr, xsave.addr)
+
+    # XXX test at runtime that we hooked save correctly
+
+
+# ---- adjusted C bits for loading ----
+
+# adjust Unpickler to use 'bstr' encoding by default and handle that encoding
+# in PyUnicode_Decode by returning bstr instead of unicode. This mirrors
+# corresponding py loading adjustments.
+
+cdef int _pickle_Unpickler_xinit(object self, PyObject* args, PyObject* kw) except -1:
+    xkw = {'encoding': 'bstr'}
+    if kw != NULL:
+        xkw.update(<object>kw)
+    return _pickle_patchctx.Unpickler_tp_init_orig(self, args, <PyObject*>xkw)
+
+cdef int _zpickle_Unpickler_xinit(object self, PyObject* args, PyObject* kw) except -1:
+    xkw = {'encoding': 'bstr'}
+    if kw != NULL:
+        xkw.update(<object>kw)
+    return _zpickle_patchctx.Unpickler_tp_init_orig(self, args, <PyObject*>xkw)
+
+ctypedef object unicode_decodefunc(const char*, Py_ssize_t, const char* encoding, const char* errors)
+cdef unicode_decodefunc* _punicode_Decode
+cdef object _unicode_xDecode(const char *s, Py_ssize_t size, const char* encoding, const char* errors):
+    if encoding != NULL  and  strcmp(encoding, 'bstr') == 0:
+        bobj = PyBytes_FromStringAndSize(s, size)  # TODO -> PyBSTR_FromStringAndSize directly
+        return pyb(bobj)
+    return _punicode_Decode(s, size, encoding, errors)
+
+cdef _patch_capi_unicode_decode_as_bstr():
+    global _punicode_Decode
+    _punicode_Decode = PyUnicode_Decode
+    cpatch(<void**>&_punicode_Decode, <void*>_unicode_xDecode)
+
+
+# ---- adjusted C bits for saving ----
+
+# adjust Pickler save to save bstr via *STRING opcodes.
+# This mirrors corresponding py saving adjustments, but is more involved to implement.
+
+cdef int _pickle_Pickler_xsave(PicklerObject* self, PyObject* obj, int pers_save) except -1:
+    return __Pickler_xsave(&_pickle_patchctx, self, obj, pers_save)
+
+cdef int _zpickle_Pickler_xsave(PicklerObject* self, PyObject* obj, int pers_save) except -1:
+    return __Pickler_xsave(&_zpickle_patchctx, self, obj, pers_save)
+
+# callconv wrappers XXX place
+cdef extern from *:
+    r"""
+    static int __pyx_f_6golang_7_golang__pickle_Pickler_xsave(PicklerObject*, PyObject*, int);
+    static int __pyx_f_6golang_7_golang__zpickle_Pickler_xsave(PicklerObject*, PyObject*, int);
+
+    #define DEF_PICKLE_XSAVE_builtin(ccname, callconv)                                      \
+    static int callconv                                                                     \
+    _pickle_Pickler_xsave_##ccname(PicklerObject* self, PyObject* obj, int pers_save) {     \
+        return __pyx_f_6golang_7_golang__pickle_Pickler_xsave(self, obj, pers_save);        \
+    }
+    #define DEF_ZPICKLE_XSAVE_builtin(ccname, callconv)                                     \
+    static int callconv                                                                     \
+    _zpickle_Pickler_xsave_##ccname(PicklerObject* self, PyObject* obj, int pers_save) {    \
+        return __pyx_f_6golang_7_golang__zpickle_Pickler_xsave(self, obj, pers_save);       \
+    }
+
+    #define DEF_PICKLE_XSAVE_custom(ccname, _)                                              \
+        extern "C" char _pickle_Pickler_xsave_##ccname;
+    #define DEF_ZPICKLE_XSAVE_custom(ccname, _)                                             \
+        extern "C" char _zpickle_Pickler_xsave_##ccname;
+
+    #define DEF_PICKLE_XSAVE(ccname, callconv, cckind)  DEF_PICKLE_XSAVE_##cckind(ccname, callconv)
+    #define DEF_ZPICKLE_XSAVE(ccname, callconv, cckind) DEF_ZPICKLE_XSAVE_##cckind(ccname, callconv)
+
+    FOR_EACH_CALLCONV(DEF_PICKLE_XSAVE)
+    FOR_EACH_CALLCONV(DEF_ZPICKLE_XSAVE)
+
+    static std::vector<SaveFunc> _pickle_Pickler_xsave_ccv = {
+    #define PICKLE_CC_XSAVE(ccname, _, __)  \
+        SaveFunc{(void*)&_pickle_Pickler_xsave_##ccname, CALLCONV_##ccname},
+    FOR_EACH_CALLCONV(PICKLE_CC_XSAVE)
+    };
+
+    static std::vector<SaveFunc> _zpickle_Pickler_xsave_ccv = {
+    #define ZPICKLE_CC_XSAVE(ccname, _, __) \
+        SaveFunc{(void*)&_zpickle_Pickler_xsave_##ccname, CALLCONV_##ccname},
+    FOR_EACH_CALLCONV(ZPICKLE_CC_XSAVE)
+    };
+
+    // proxy for asm routines to invoke _pickle_Pickler_xsave and _zpickle_Pickler_xsave
+    #ifdef LIBGOLANG_ARCH_386
+    extern "C" int CALLCONV(fastcall)
+    _pickle_Pickler_xsave_ifastcall(PicklerObject* self, PyObject* obj, int pers_save) {
+        return __pyx_f_6golang_7_golang__pickle_Pickler_xsave(self, obj, pers_save);
+    }
+    extern "C" int CALLCONV(fastcall)
+    _zpickle_Pickler_xsave_ifastcall(PicklerObject* self, PyObject* obj, int pers_save) {
+        return __pyx_f_6golang_7_golang__zpickle_Pickler_xsave(self, obj, pers_save);
+    }
+    #endif
+    """
+    vector[SaveFunc] _pickle_Pickler_xsave_ccv
+    vector[SaveFunc] _zpickle_Pickler_xsave_ccv
+
+
+cdef int __Pickler_xsave(_pickle_PatchCtx* pctx, PicklerObject* self, PyObject* obj, int pers_save) except -1:
+    # !bstr -> use builtin pickle code
+    if obj.ob_type != <PyTypeObject*>pybstr:
+        return save_invoke(pctx.Pickler_save_orig.addr, pctx.Pickler_save_orig.cconv,
+                                self, obj, pers_save)
+
+    # bstr  -> pickle it as *STRING
+    cdef const char* s
+    cdef Py_ssize_t  l
+    cdef byte[5]     h
+    cdef Py_ssize_t  lh = 1;
+    cdef bint nonascii_escape
+
+    cdef int bin = (<int*>((<byte*>self) + pctx.iPickler.off_bin))[0]
+    if bin == 0:
+        esc = strconv._quote(<object>obj, "'", &nonascii_escape)
+        assert type(esc) is bytes
+        s = PyBytes_AS_STRING(<PyObject*>esc)
+        l = PyBytes_GET_SIZE(<PyObject*>esc)
+        __Pickler_xWrite(pctx, self, b'S', 1)   # STRING
+        __Pickler_xWrite(pctx, self, s, l)
+        __Pickler_xWrite(pctx, self, b'\n', 1)
+
+    else:
+        s = PyBytes_AS_STRING(obj)
+        l = PyBytes_GET_SIZE(obj)
+        if l < 0x100:
+            h[0] = b'U'     # SHORT_BINSTRING
+            h[1] = <byte>l
+            lh += 1
+        elif l < 0x7fffffff:
+            h[0] = b'T'     # BINSTRING
+            h[1] = <byte>(l >> 0)
+            h[2] = <byte>(l >> 8)
+            h[3] = <byte>(l >> 16)
+            h[4] = <byte>(l >> 24)
+            lh += 4
+        else:
+            raise OverflowError("cannot serialize a string larger than 2 GiB")
+
+        __Pickler_xWrite(pctx, self, <char*>h, lh)
+        __Pickler_xWrite(pctx, self, s, l)
+
+    return 0
+
+
+# __Pickler_xWrite mimics original _Pickler_Write.
+#
+# we have to implement it ourselves because there is no way to discover
+# original _Pickler_Write address: contrary to `save` function _Pickler_Write
+# is small and is not recursive. A compiler is thus free to create many
+# versions of it with e.g. constant propagation and to inline it freely. The
+# latter actually happens for real on LLVM which for py3.11 inlines
+# _Pickler_Write fully without leaving any single freestanding instance of it.
+#
+# XXX explain why we can skip flush in zpickle case
+# XXX explain that we do not emit FRAME
+cdef int __Pickler_xWrite(_pickle_PatchCtx* pctx, PicklerObject* self, const char* s, Py_ssize_t l) except -1:
+    ppoutput_buffer = <PyObject**> (<byte*>self + pctx.iPickler.off_poutput_buffer)
+    poutput_len     = <Py_ssize_t*>(<byte*>self + pctx.iPickler.off_output_len)
+    pmax_output_len = <Py_ssize_t*>(<byte*>self + pctx.iPickler.off_max_output_len)
+
+    assert ppoutput_buffer[0].ob_type == &PyBytes_Type
+    assert l >= 0
+    assert poutput_len[0] >= 0
+
+    if l > PY_SSIZE_T_MAX - poutput_len[0]:
+        raise MemoryError() # overflow
+
+    need = poutput_len[0] + l
+    if need > pmax_output_len[0]:
+        if need >= PY_SSIZE_T_MAX // 2:
+            raise MemoryError()
+        pmax_output_len[0] = need // 2 * 3
+        _PyBytes_Resize(ppoutput_buffer, pmax_output_len[0])
+
+    buf = PyBytes_AS_STRING(ppoutput_buffer[0])
+    memcpy(buf + poutput_len[0], s, l)
+    poutput_len[0] += l
+
+    return 0
+
+
+# ---- infrastructure to assist patching C saving codepath ----
+
+# _detect_Pickler_typeinfo detects information about PicklerObject type
+# through runtime introspection.
+#
+# This information is used mainly by __Pickler_xWrite.
+cdef PicklerTypeInfo _detect_Pickler_typeinfo(pyPickler) except *:
+    cdef PicklerTypeInfo t
+
+    cdef bint debug = False
+    def trace(*argv):
+        if debug:
+            print(*argv)
+    trace()
+
+    assert isinstance(pyPickler, type)
+    cdef PyTypeObject*   Pickler  = <PyTypeObject*>   pyPickler
+    cdef _XPyTypeObject* xPickler = <_XPyTypeObject*> pyPickler
+
+    # sizeof
+    assert Pickler.tp_basicsize  > 0
+    assert Pickler.tp_itemsize  == 0
+    t.size = Pickler.tp_basicsize
+    trace('size:\t', t.size)
+
+    # busy keeps offsets of all bytes for already detected fields
+    busy = set()
+    def markbusy(off, size):
+        for _ in range(off, off+size):
+            assert _ not in busy,  (_, busy)
+            assert 0 < off <= t.size
+            busy.add(_)
+
+    # .bin
+    cdef PyMemberDef* mbin = tp_members_lookup(xPickler.tp_members, 'bin')
+    assert mbin.type == T_INT, (mbin.type,)
+    t.off_bin = mbin.offset
+    markbusy(t.off_bin, sizeof(int))
+    trace('.bin:\t', t.off_bin)
+
+    # .output_buffer
+    #
+    #   1) new Pickler
+    #   2) .memo = {}    - the only pointer that changes is .memo (PyMemoTable* - not pyobject)
+    #   3) .tp_clear()   - all changed words are changed to 0 and cover non-optional PyObject* and memo
+    #   4) .__init__()
+    #   5) go through offsets of all pyobjects and find the one with .ob_type = PyBytes_Type
+    #      -> that is .output_buffer
+
+    #       1)
+    class Null:
+        def write(self, data): pass
+    pyobj = pyPickler(Null())
+    cdef PyObject* obj = <PyObject*>pyobj
+    assert obj.ob_type == Pickler
+
+    cdef byte* bobj  = <byte*>obj
+    cdef byte* bobj2 = <byte*>malloc(t.size)
+    # obj_copy copies obj to obj2.
+    def obj_copy():
+        memcpy(bobj2, bobj, t.size)
+    # obj_diff finds difference in between obj2 and obj.
+    def obj_diff(Py_ssize_t elemsize): # -> []offset
+        assert (elemsize & (elemsize - 1)) == 0,  elemsize # elemsize is 2^x
+        cdef Py_ssize_t off
+
+        # skip PyObject_HEAD
+        off = sizeof(PyObject)
+        off = (off + elemsize - 1) & (~(elemsize - 1))
+        assert off % elemsize == 0
+
+        # find out offsets of different elements
+        vdelta = []
+        while off + elemsize <= t.size:
+            if memcmp(bobj + off, bobj2 + off, elemsize):
+                vdelta.append(off)
+            off += elemsize
+
+        return vdelta
+
+    #       2)
+    obj_copy()
+    pyobj.memo = {}
+    dmemo = obj_diff(sizeof(void*))
+    assert len(dmemo) == 1,  dmemo
+    off_memo = dmemo[0]
+    markbusy(off_memo, sizeof(void*))
+    trace('.memo:\t', off_memo)
+
+    #       3)
+    assert Pickler.tp_clear != NULL
+    obj_copy()
+    Pickler.tp_clear(pyobj)
+    pointers = obj_diff(sizeof(void*))
+    for poff in pointers:
+        assert (<void**>(bobj + <Py_ssize_t>poff))[0] == NULL
+    assert off_memo in pointers
+    pyobjects = pointers[:]
+    pyobjects.remove(off_memo)
+    trace('pyobjects:\t', pyobjects)
+
+    #       4)
+    pyobj.__init__(Null())
+
+    #       5)
+    cdef PyObject* bout = NULL
+    t.off_poutput_buffer = 0
+    for poff in pyobjects:
+        x = (<PyObject**>(bobj + <Py_ssize_t>poff))[0]
+        if x.ob_type == &PyBytes_Type:
+            if t.off_poutput_buffer == 0:
+                t.off_poutput_buffer = poff
+            else:
+                raise AssertionError("found several <bytes> inside Pickler")
+    assert t.off_poutput_buffer != 0
+    markbusy(t.off_poutput_buffer, sizeof(PyObject*))
+    trace(".output_buffer:\t", t.off_poutput_buffer)
+
+    # .output_len  +  .max_output_len
+    # dump something small and expected -> find out which field changes correspondingly
+    import io
+    output_len     = None
+    max_output_len = None
+    for n in range(1,10):
+        f = io.BytesIO()
+        pyobj.__init__(f, 0)
+        o = (None,)*n
+        pyobj.dump(o)
+        p = f.getvalue()
+        phok = b'(' + b'N'*n + b't'  # full trails with "p0\n." but "p0\n" is optional
+        assert p.startswith(phok), p
+
+        # InspectWhilePickling observes obj while the pickling is going on:
+        # - sees which fields have changes
+        # - sees which fields are candidates for max_output_len
+        class InspectWhilePickling:
+            def __init__(self):
+                self.diff     = None  # what changes
+                self.doff2val = {}    # off from .diff  ->  Py_ssize_t read from it
+                self.max_output_len = set() # offsets that are candidates for .max_output_len
+
+            def __reduce__(self):
+                self.diff = obj_diff(sizeof(Py_ssize_t))
+                for off in self.diff:
+                    self.doff2val[off] = (<Py_ssize_t*>(bobj + <Py_ssize_t>off))[0]
+
+                cdef PyObject* output_buffer = \
+                        (<PyObject**>(bobj + t.off_poutput_buffer))[0]
+                assert output_buffer.ob_type == &PyBytes_Type
+                off = sizeof(PyObject)
+                off = (off + sizeof(Py_ssize_t) - 1) & (~(sizeof(Py_ssize_t) - 1))
+                assert off % sizeof(Py_ssize_t) == 0
+                while off + sizeof(Py_ssize_t) <= t.size:
+                    v = (<Py_ssize_t*>(bobj + <Py_ssize_t>off))[0]
+                    if v == PyBytes_GET_SIZE(output_buffer):
+                        self.max_output_len.add(off)
+                    off += sizeof(Py_ssize_t)
+
+                return (int, ())    # arbitrary
+
+        pyobj.__init__(Null(), 0)
+        i = InspectWhilePickling()
+        o += (i,)
+        obj_copy()
+        pyobj.dump(o)
+        assert i.diff is not None
+        #trace('n%d  diff: %r\toff2val: %r' % (n, i.diff, i.doff2val))
+        #trace('     ', busy)
+
+        noutput_len = set()
+        for off in i.diff:
+            if off not in busy:
+                if i.doff2val[off] == (len(phok)-1): # (NNNN without t yet
+                    noutput_len.add(off)
+        assert len(noutput_len) >= 1, noutput_len
+        if output_len is None:
+            output_len = noutput_len
+        else:
+            output_len.intersection_update(noutput_len)
+
+        nmax_output_len = set()
+        for off in i.max_output_len:
+            if off not in busy:
+                nmax_output_len.add(off)
+        assert len(nmax_output_len) >= 1, nmax_output_len
+        if max_output_len is None:
+            max_output_len = nmax_output_len
+        else:
+            max_output_len.intersection_update(nmax_output_len)
+
+    if len(output_len) != 1:
+        raise AssertionError("cannot find .output_len")
+    if len(max_output_len) != 1:
+        raise AssertionError("cannot find .max_output_len")
+
+    t.off_output_len = output_len.pop()
+    markbusy(t.off_output_len, sizeof(Py_ssize_t))
+    trace(".output_len:\t", t.off_output_len)
+
+    t.off_max_output_len = max_output_len.pop()
+    markbusy(t.off_max_output_len, sizeof(Py_ssize_t))
+    trace(".max_output_len:\t", t.off_max_output_len)
+
+    free(bobj2)
+    return t
+
+
+# _find_Pickler_save determines address and calling convention of `save` C
+# function associated with specified Pickler.
+#
+# Address and calling convention of `save` are needed to be able to patch it.
+cdef SaveFunc _find_Pickler_save(pyPickler) except *:
+    cdef SaveFunc save
+    save.addr  = __find_Pickler_save(pyPickler)
+    save.cconv = __detect_save_callconv(pyPickler, save.addr)
+    #fprintf(stderr, "save.addr:  %p\n", save.addr)
+    #fprintf(stderr, "save.cconv: %s\n", callconv_str(save.cconv))
+    return save
+
+cdef void* __find_Pickler_save(pyPickler) except NULL:
+    assert isinstance(pyPickler, type)
+
+    # start from _pickle_Pickler_dump as root and analyze how called functions
+    # behave wrt pickling deep chain of objects. We know whether a callee leads
+    # to save if, upon receiving control in our __reduce__, we see that the
+    # callee was entered and did not exited yet. If we find such a callee, we
+    # recourse the process and start to analyze functions that the callee invokes
+    # itself. We detect reaching save when we see that a callee was entered
+    # many times recursively. That happens because we feed deep recursive
+    # structure to the pickle, and because save itself is organized to invoke
+    # itself recursively - e.g. (obj,) is pickled via save -> save_tuple -> save.
+    cdef _XPyTypeObject* Pickler = <_XPyTypeObject*>(pyPickler)
+    cdef PyMethodDef*    mdump   = tp_methods_lookup(Pickler.tp_methods, 'dump')
+    #print("%s _pickle_Pickler_dump:" % pyPickler)
+    addr = <void*>mdump.ml_meth  # = _pickle_Pickler_dump
+    while 1:
+        vcallee = cfunc_direct_callees(addr)
+        ok = False
+        for i in range(vcallee.size()):
+            callee = vcallee[i]
+            #fprintf(stderr, "checking %p ...\n", callee)
+            nentry = _nentry_on_deep_save(pyPickler, callee)
+            #fprintf(stderr, "%p  - %ld\n", callee, nentry)
+            assert nentry in (0, 1)  or  nentry > 5,  nentry
+            if nentry > 5:
+                return callee   # found save
+            if nentry == 1:
+                addr = callee   # found path that will lead to save
+                ok = True
+                break
+
+        if not ok:
+            raise AssertionError('cannot find path leading to save')
+
+# _nentry_on_deep_save tests how addr is related to `save` via inspecting
+# addr entry count when Pickler is feed deep recursive structure.
+#
+# if #entry is 0   - addr is unrelated to save
+# if #entry is 1   - addr is related to save and calls it
+# if #entry is big - addr is save
+cdef long _nentry_on_deep_save(pyPickler, void* addr) except -1: # -> nentry
+    # below we rely on inside_counted which alters return address during the
+    # call to wrapped func. In practice this does not create problems on x86_64
+    # and arm64, but on i386 there are many calls to functions like
+    # x86.get_pc_thunk.ax which are used to implement PC-relative addressing.
+    # If we let inside_counted to hook such a func it will result in a crash
+    # because returned address will be different from real PC of the caller.
+    # Try to protect us from entering into such situation by detecting leaf
+    # functions and not hooking them. For the reference x86.get_pc_thunk.ax is:
+    #
+    #       movl (%esp), %eax
+    #       ret
+    vcallee = cfunc_direct_callees(addr)
+    if vcallee.size() == 0:
+        return 0
+
+    # InspectWhilePickling observes how many times currently considered
+    # function was entered at the point of deep recursion inside save.
+    class InspectWhilePickling:
+        def __init__(self):
+            self.inside_counter = None
+        def __reduce__(self):
+            self.inside_counter = inside_counter
+            return (int, ())    # arbitrary
+
+    class Null:
+        def write(self, data): pass
+
+    i = InspectWhilePickling()
+    obj = (i,)
+    for _ in range(20):
+        obj = (obj,)
+
+    p = pyPickler(Null(), 0)
+
+    h = xfunchook_create()
+    global inside_counted_func
+    inside_counted_func = addr
+    xfunchook_prepare(h, &inside_counted_func, <void*>inside_counted)
+    xfunchook_install(h, 0)
+    p.dump(obj)
+    xfunchook_uninstall(h, 0)
+    xfunchook_destroy(h)
+
+    assert i.inside_counter is not None
+    return i.inside_counter
+
+
+# inside_counted is used to patch a function to count how many times that
+# function is entered/leaved.
+cdef extern from * nogil: # see _golang_str_pickle.S for details
+    """
+    extern "C" {
+         extern void  inside_counted();
+         extern void* inside_counted_func;
+         extern long  inside_counter;
+    }
+    """
+    void  inside_counted()
+    void* inside_counted_func
+    long  inside_counter
+
+
+# __detect_save_callconv determines calling convention that compiler used for save.
+#
+# On architectures with many registers - e.g. x86_64 and arm64 - the calling
+# convention is usually the same as default, but on e.g. i386 - where the
+# default cdecl means to put arguments on the stack, the compiler usually
+# changes calling convention to use registers instead.
+cdef Callconv __detect_save_callconv(pyPickler, void* save) except *:
+    for p in saveprobe_test_ccv:
+        #print("save: probing %s" % callconv_str(p.cconv))
+        good = __save_probe1(pyPickler, save, p.addr)
+        #print("  ->", good)
+        if good:
+            return p.cconv
+    bad = "cannot determine save calling convention\n\n"
+    bad += "probed:\n"
+    for p in saveprobe_test_ccv:
+        bad += "  - %s\t; callee_stkcleanup: %d\n" % (callconv_str(p.cconv), cfunc_is_callee_cleanup(p.addr))
+    bad += "\n"
+    bad += "save callee_stkcleanup: %d\n" % cfunc_is_callee_cleanup(save)
+    bad += "save disassembly:\n%s" % cfunc_disasm(save)
+    raise AssertionError(bad)
+
+cdef bint __save_probe1(pyPickler, void* save, void* cfunc) except *:
+    # first see whether stack is cleaned up by caller or callee and how much.
+    # we need to do this first to avoid segfault if we patch save with cfunc
+    # with different stack cleanup as the probe.
+    save_stkclean  = cfunc_is_callee_cleanup(save)
+    cfunc_stkclean = cfunc_is_callee_cleanup(cfunc)
+    if save_stkclean != cfunc_stkclean:
+        return False
+
+    # now when we know that save and cfunc have the same stack cleanup protocol, we can start probing
+    global saveprobe_ncall, saveprobe_self, saveprobe_obj, saveprobe_pers_save
+    saveprobe_ncall = 0
+    saveprobe_self  = NULL
+    saveprobe_obj   = NULL
+    saveprobe_pers_save = 0xdeafbeaf
+
+    class Null:
+        def write(self, data): pass
+    p = pyPickler(Null(), 0)
+    obj = object()
+
+    h = xfunchook_create()
+    xfunchook_prepare(h, &save, cfunc)
+    xfunchook_install(h, 0)
+    p.dump(obj)
+    xfunchook_uninstall(h, 0)
+    xfunchook_destroy(h)
+
+    assert saveprobe_ncall == 1, saveprobe_ncall
+    good = (saveprobe_self == <void*>p    and \
+            saveprobe_obj  == <void*>obj  and \
+            saveprobe_pers_save == 0)
+    return good
+
+cdef extern from * nogil:
+    r"""
+    static int    saveprobe_ncall;
+    static void*  saveprobe_self;
+    static void*  saveprobe_obj;
+    static int    saveprobe_pers_save;
+
+    static int saveprobe(void* self, PyObject* obj, int pers_save) {
+        saveprobe_ncall++;
+        saveprobe_self = self;
+        saveprobe_obj  = obj;
+        saveprobe_pers_save = pers_save;
+        return 0; // do nothing
+    }
+
+    #define DEF_SAVEPROBE_builtin(ccname, callconv)                     \
+        static int callconv                                             \
+        saveprobe_##ccname(void* self, PyObject* obj, int pers_save) {  \
+            return saveprobe(self, obj, pers_save);                     \
+        }
+    #define DEF_SAVEPROBE_custom(ccname, _)                             \
+        extern "C" char saveprobe_##ccname;
+    #define DEF_SAVEPROBE(ccname, callconv, cckind) DEF_SAVEPROBE_##cckind(ccname, callconv)
+    FOR_EACH_CALLCONV(DEF_SAVEPROBE)
+
+    static std::vector<SaveFunc> saveprobe_test_ccv = {
+    #define CC_SAVEPROBE(ccname, _, __) \
+        SaveFunc{(void*)&saveprobe_##ccname, CALLCONV_##ccname},
+    FOR_EACH_CALLCONV(CC_SAVEPROBE)
+    };
+
+    // proxy for asm routines to invoke saveprobe
+    #ifdef LIBGOLANG_ARCH_386
+    extern "C" int CALLCONV(fastcall)
+    saveprobe_ifastcall(void* self, PyObject* obj, int pers_save) { \
+        return saveprobe(self, obj, pers_save);                     \
+    }
+    #endif
+    """
+    int   saveprobe_ncall
+    void* saveprobe_self
+    void* saveprobe_obj
+    int   saveprobe_pers_save
+
+    vector[SaveFunc] saveprobe_test_ccv
+
+
+# XXX doc save_invoke ...
+# XXX place
+cdef extern from *:
+    r"""
+    #define CC_SAVE_DEFCALL1_builtin(ccname, callconv)
+    #define CC_SAVE_DEFCALL1_custom(ccname, _)  \
+        extern "C" int CALLCONV(fastcall)       \
+        save_invoke_as_##ccname(void* save, void* self, PyObject* obj, int pers_save);
+    #define CC_SAVE_DEFCALL1(ccname, callconv, cckind)  CC_SAVE_DEFCALL1_##cckind(ccname, callconv)
+    FOR_EACH_CALLCONV(CC_SAVE_DEFCALL1)
+
+    static int save_invoke(void* save, Callconv cconv, void* self, PyObject* obj, int pers_save) {
+        using namespace golang;
+
+        switch(cconv) {
+    #define CC_SAVE_CALL1_builtin(ccname, callconv)     \
+        case CALLCONV_ ## ccname:                                   \
+            return ((int (callconv *)(void*, PyObject*, int))save)  \
+                    (self, obj, pers_save);
+    #define CC_SAVE_CALL1_custom(ccname, _)             \
+        case CALLCONV_ ## ccname:                                   \
+            return save_invoke_as_##ccname(save, self, obj, pers_save);
+    #define CC_SAVE_CALL1(ccname, callconv, cckind) CC_SAVE_CALL1_##cckind(ccname, callconv)
+    FOR_EACH_CALLCONV(CC_SAVE_CALL1)
+        default:
+            panic("unreachable");
+        }
+    }
+    """
+    int save_invoke(void* save, Callconv cconv, void* self, PyObject* obj, int pers_save) except -1
+
+
+# - cfunc_direct_callees returns addresses of functions that cfunc calls directly.
+#
+# - cfunc_is_callee_cleanup determines whether cfunc does stack cleanup by
+#   itself and for how much.
+#
+# - cfunc_disassembly returns disassembly of cfunc.
+#
+# XXX dedup iterating instructions -> DisasmIter
+cdef extern from "capstone/capstone.h" nogil:
+    r"""
+    #include <algorithm>
+    #include "golang/fmt.h"
+
+    #if defined(LIBGOLANG_ARCH_amd64)
+    # define MY_ARCH    CS_ARCH_X86
+    # define MY_MODE    CS_MODE_64
+    #elif defined(LIBGOLANG_ARCH_386)
+    # define MY_ARCH    CS_ARCH_X86
+    # define MY_MODE    CS_MODE_32
+    #elif defined(LIBGOLANG_ARCH_arm64)
+    # define MY_ARCH    CS_ARCH_ARM64
+    # define MY_MODE    CS_MODE_LITTLE_ENDIAN
+    #else
+    # error "unsupported architecture"
+    #endif
+
+    static std::tuple<uint64_t, bool> _insn_getimm1(cs_arch arch, cs_insn* ins);
+    std::vector<void*> cfunc_direct_callees(void *cfunc) {
+        const bool debug = false;
+
+        using namespace golang;
+        using std::tie;
+        using std::max;
+
+        std::vector<void*> vcallee;
+
+        csh       h;
+        cs_insn*  ins;
+        cs_err    err;
+
+        cs_arch arch = MY_ARCH;
+        err = cs_open(arch, MY_MODE, &h);
+        if (err) {
+            fprintf(stderr, "cs_open: %s\n", cs_strerror(err));
+            panic(cs_strerror(err));
+        }
+
+        err = cs_option(h, CS_OPT_DETAIL, CS_OPT_ON);
+        if (err) {
+            fprintf(stderr, "cs_option: %s\n", cs_strerror(err));
+            panic(cs_strerror(err));
+        }
+
+        ins = cs_malloc(h);
+        if (ins == nil)
+            panic("cs_malloc failed");
+
+        const byte* code = (const byte*)cfunc;
+        size_t      size = 10*1024; // something sane and limited
+        uint64_t    addr = (uint64_t)cfunc;
+        uint64_t    maxjump = addr;
+        while (cs_disasm_iter(h, &code, &size, &addr, ins)) {
+            if (debug)
+                fprintf(stderr, "0x%" PRIx64 ":\t%s\t\t%s\n", ins->address, ins->mnemonic, ins->op_str);
+
+            if (cs_insn_group(h, ins, CS_GRP_RET)) {
+                if (ins->address >= maxjump)
+                    break;
+                continue;
+            }
+
+            uint64_t imm1;
+            bool     imm1ok;
+            tie(imm1, imm1ok) = _insn_getimm1(arch, ins);
+
+            bool call = cs_insn_group(h, ins, CS_GRP_CALL);
+            bool jump = cs_insn_group(h, ins, CS_GRP_JUMP) && !call;  // e.g. BL on arm64 is both jump and call
+
+            if (jump && imm1ok) {
+                maxjump = max(maxjump, imm1);
+                continue;
+            }
+
+            if (call && imm1ok) {
+                void* callee = (void*)imm1;
+                if (debug)
+                    fprintf(stderr, "  *** DIRECT CALL -> %p\n", callee);
+                if (!std::count(vcallee.begin(), vcallee.end(), callee))
+                    vcallee.push_back(callee);
+            }
+        }
+
+        if (debug)
+            fprintf(stderr, "\n");
+
+        cs_free(ins, 1);
+        cs_close(&h);
+        return vcallee;
+    }
+
+    // _insn_getimm1 checks whether instruction comes with the sole immediate operand and returns it.
+    static std::tuple<uint64_t, bool> _insn_getimm1(cs_arch arch, cs_insn* ins) {
+        using namespace golang;
+        using std::make_tuple;
+
+        switch (arch) {
+        case CS_ARCH_X86: {
+            cs_x86* x86 = &(ins->detail->x86);
+            if (x86->op_count == 1) {
+                cs_x86_op* op = &(x86->operands[0]);
+                if (op->type == X86_OP_IMM)
+                    return make_tuple(op->imm, true);
+            }
+            break;
+        }
+
+        case CS_ARCH_ARM64: {
+            cs_arm64* arm64 = &(ins->detail->arm64);
+            if (arm64->op_count == 1) {
+                cs_arm64_op* op = &(arm64->operands[0]);
+                if (op->type == ARM64_OP_IMM)
+                    return make_tuple(op->imm, true);
+            }
+            break;
+        }
+
+        default:
+            panic("TODO");
+        }
+
+        return make_tuple(0, false);
+    }
+
+
+    int cfunc_is_callee_cleanup(void *cfunc) {
+        // only i386 might have callee-cleanup
+        // https://en.wikipedia.org/wiki/X86_calling_conventions#List_of_x86_calling_conventions
+        if (!(MY_ARCH == CS_ARCH_X86 && MY_MODE == CS_MODE_32))
+            return 0;
+
+        const bool debug = false;
+
+        int stkclean_by_callee = 0;
+        using namespace golang;
+
+        csh       h;
+        cs_insn*  ins;
+        cs_err    err;
+
+        err = cs_open(MY_ARCH, MY_MODE, &h);
+        if (err) {
+            fprintf(stderr, "cs_open: %s\n", cs_strerror(err));
+            panic(cs_strerror(err));
+        }
+
+        err = cs_option(h, CS_OPT_DETAIL, CS_OPT_ON);
+        if (err) {
+            fprintf(stderr, "cs_option: %s\n", cs_strerror(err));
+            panic(cs_strerror(err));
+        }
+
+        ins = cs_malloc(h);
+        if (ins == nil)
+            panic("cs_malloc failed");
+
+        const byte* code = (const byte*)cfunc;
+        size_t      size = 10*1024; // something sane and limited
+        uint64_t    addr = (uint64_t)cfunc;
+        while (cs_disasm_iter(h, &code, &size, &addr, ins)) {
+            if (debug)
+                fprintf(stderr, "0x%" PRIx64 ":\t%s\t\t%s\n", ins->address, ins->mnemonic, ins->op_str);
+
+            if (!cs_insn_group(h, ins, CS_GRP_RET))
+                continue;
+
+            assert(ins->id == X86_INS_RET);
+            cs_x86* x86 =  &(ins->detail->x86);
+            if (x86->op_count > 0) {
+                cs_x86_op* op = &(x86->operands[0]);
+                if (op->type == X86_OP_IMM)
+                    stkclean_by_callee = op->imm;
+            }
+
+            break;
+        }
+
+        if (debug)
+            fprintf(stderr, "  *** CLEANUP BY: %s  (%d)\n", (stkclean_by_callee ? "callee" : "caller"), stkclean_by_callee);
+
+        cs_free(ins, 1);
+        cs_close(&h);
+        return stkclean_by_callee;
+    }
+
+    std::string cfunc_disasm(void *cfunc) {
+        using namespace golang;
+        string disasm;
+
+        csh       h;
+        cs_insn*  ins;
+        cs_err    err;
+
+        err = cs_open(MY_ARCH, MY_MODE, &h);
+        if (err) {
+            fprintf(stderr, "cs_open: %s\n", cs_strerror(err));
+            panic(cs_strerror(err));
+        }
+
+        err = cs_option(h, CS_OPT_DETAIL, CS_OPT_ON);
+        if (err) {
+            fprintf(stderr, "cs_option: %s\n", cs_strerror(err));
+            panic(cs_strerror(err));
+        }
+
+        ins = cs_malloc(h);
+        if (ins == nil)
+            panic("cs_malloc failed");
+
+        const byte* code = (const byte*)cfunc;
+        size_t      size = 10*1024; // something sane and limited
+        uint64_t    addr = (uint64_t)cfunc;
+        while (cs_disasm_iter(h, &code, &size, &addr, ins)) {
+            disasm += fmt::sprintf("0x%" PRIx64 ":\t%s\t\t%s\n", ins->address, ins->mnemonic, ins->op_str);
+
+            // FIXME also handle forward jump like cfunc_direct_callees does
+            //       should be done automatically after DisasmIter dedup
+            if (cs_insn_group(h, ins, CS_GRP_RET))
+                break;
+        }
+
+        cs_free(ins, 1);
+        cs_close(&h);
+
+        return disasm;
+    }
+    """
+    vector[void*] cfunc_direct_callees(void* cfunc)
+    int cfunc_is_callee_cleanup(void* cfunc)
+    string cfunc_disasm(void* cfunc)
+
+
+# _test_inside_counted depends on inside_counted and funchook, which we don't want to expose.
+# -> include the test from here. Do the same for other low-level tests.
+include '_golang_str_pickle_test.pyx'
+
+
+# ---- misc ----
+
+cdef PyMethodDef* tp_methods_lookup(PyMethodDef* methv, str name) except NULL:
+    m = &methv[0]
+    while m.ml_name != NULL:
+        if str(m.ml_name) == name:
+            return m
+        m += 1
+    raise KeyError("method %s not found" % name)
+
+cdef PyMemberDef* tp_members_lookup(PyMemberDef* membv, str name) except NULL:
+    m = &membv[0]
+    while m.name != NULL:
+        if str(m.name) == name:
+            return m
+        m += 1
+    raise KeyError("member %s not found" % name)

golang/_golang_str_pickle_test.pyx

+# -*- coding: utf-8 -*-
+# Copyright (C) 2023  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+# test for inside_counted
+def _test_inside_counted(): # -> outok
+    outok = ''
+
+    outok += '\n\n\nBEFORE PATCH\n'
+    print('\n\n\nBEFORE PATCH')
+    tfunc(3)
+
+    t0 = ''
+    for i in range(3,0-1,-1):
+        t0 += '> tfunc(%d)\tinside_counter: 0\n' % i
+    for i in range(0,3+1,+1):
+        t0 += '< tfunc(%d)\tinside_counter: 0\n' % i
+    outok += t0
+
+    outok += '\n\n\nPATCHED\n'
+    print('\n\n\nPATCHED')
+    _patch = xfunchook_create()
+    global inside_counted_func
+    inside_counted_func = <void*>&tfunc
+    xfunchook_prepare(_patch, &inside_counted_func, <void*>inside_counted)
+    xfunchook_install(_patch, 0)
+
+    tfunc(12)
+
+    stk_size = 8  # = STK_SIZE from _golang_str_pickle.S
+    for i in range(12,0-1,-1):
+        outok += '> tfunc(%d)\tinside_counter: %d\n' % (i, min(12-i+1, stk_size))
+    for i in range(0,12+1,+1):
+        outok += '< tfunc(%d)\tinside_counter: %d\n' % (i, min(12-i+1, stk_size))
+
+    outok += '\n\n\nUNPATCHED\n'
+    print('\n\n\nUNPATCHED')
+    xfunchook_uninstall(_patch, 0)
+    tfunc(3)
+    outok += t0
+
+    return outok
+
+cdef void tfunc(int x):
+    print('> tfunc(%d)\tinside_counter: %d' % (x, inside_counter))
+    if x > 0:
+        tfunc(x-1)
+    print('< tfunc(%d)\tinside_counter: %d' % (x, inside_counter))
+
+
+def _test_cfunc_is_callee_cleanup():
+    for t in _cfunc_is_callee_cleanup_testv:
+        stkclean = cfunc_is_callee_cleanup(t.cfunc)
+        assert stkclean == t.stkclean_by_callee_ok, (t.cfunc_name, stkclean, t.stkclean_by_callee_ok)
+
+cdef extern from * nogil:
+    r"""
+    struct _Test_cfunc_is_callee_clenup {
+        const char* cfunc_name;
+        void*       cfunc;
+        int         stkclean_by_callee_ok;
+    };
+
+    #define CASE(func, stkclean_ok) \
+        _Test_cfunc_is_callee_clenup{#func, (void*)func, stkclean_ok}
+
+    #if defined(LIBGOLANG_ARCH_386)
+    int CALLCONV(cdecl)
+    tfunc_cdecl1(int x)                     { return x; }
+    int CALLCONV(cdecl)
+    tfunc_cdecl2(int x, int y)              { return x; }
+    int CALLCONV(cdecl)
+    tfunc_cdecl3(int x, int y, int z)       { return x; }
+
+    int CALLCONV(stdcall)
+    tfunc_stdcall1(int x)                   { return x; }
+    int CALLCONV(stdcall)
+    tfunc_stdcall2(int x, int y)            { return x; }
+    int CALLCONV(stdcall)
+    tfunc_stdcall3(int x, int y, int z)     { return x; }
+
+    int CALLCONV(fastcall)
+    tfunc_fastcall1(int x)                  { return x; }
+    int CALLCONV(fastcall)
+    tfunc_fastcall2(int x, int y)           { return x; }
+    int CALLCONV(fastcall)
+    tfunc_fastcall3(int x, int y, int z)    { return x; }
+
+    #ifndef LIBGOLANG_CC_msc    // see note about C3865 in FOR_EACH_CALLCONV
+    int CALLCONV(thiscall)
+    tfunc_thiscall1(int x)                  { return x; }
+    int CALLCONV(thiscall)
+    tfunc_thiscall2(int x, int y)           { return x; }
+    int CALLCONV(thiscall)
+    tfunc_thiscall3(int x, int y, int z)    { return x; }
+    #endif
+
+    #ifndef LIBGOLANG_CC_msc    // no regparm on MSCV
+    int CALLCONV(regparm(1))
+    tfunc_regparm1_1(int x)                 { return x; }
+    int CALLCONV(regparm(1))
+    tfunc_regparm1_2(int x, int y)          { return x; }
+    int CALLCONV(regparm(1))
+    tfunc_regparm1_3(int x, int y, int z)   { return x; }
+
+    int CALLCONV(regparm(2))
+    tfunc_regparm2_1(int x)                 { return x; }
+    int CALLCONV(regparm(2))
+    tfunc_regparm2_2(int x, int y)          { return x; }
+    int CALLCONV(regparm(2))
+    tfunc_regparm2_3(int x, int y, int z)   { return x; }
+
+    int CALLCONV(regparm(3))
+    tfunc_regparm3_1(int x)                 { return x; }
+    int CALLCONV(regparm(3))
+    tfunc_regparm3_2(int x, int y)          { return x; }
+    int CALLCONV(regparm(3))
+    tfunc_regparm3_3(int x, int y, int z)   { return x; }
+    #endif
+
+    static std::vector<_Test_cfunc_is_callee_clenup> _cfunc_is_callee_cleanup_testv = {
+        CASE(tfunc_cdecl1     , 0 * 4),
+        CASE(tfunc_cdecl2     , 0 * 4),
+        CASE(tfunc_cdecl3     , 0 * 4),
+        CASE(tfunc_stdcall1   , 1 * 4),
+        CASE(tfunc_stdcall2   , 2 * 4),
+        CASE(tfunc_stdcall3   , 3 * 4),
+        CASE(tfunc_fastcall1  , 0 * 4),
+        CASE(tfunc_fastcall2  , 0 * 4),
+        CASE(tfunc_fastcall3  , 1 * 4),
+    #ifndef LIBGOLANG_CC_msc
+        CASE(tfunc_thiscall1  , 0 * 4),
+        CASE(tfunc_thiscall2  , 1 * 4),
+        CASE(tfunc_thiscall3  , 2 * 4),
+    #endif
+    #ifndef LIBGOLANG_CC_msc
+        CASE(tfunc_regparm1_1 , 0 * 4),
+        CASE(tfunc_regparm1_2 , 0 * 4),
+        CASE(tfunc_regparm1_3 , 0 * 4),
+        CASE(tfunc_regparm2_1 , 0 * 4),
+        CASE(tfunc_regparm2_2 , 0 * 4),
+        CASE(tfunc_regparm2_3 , 0 * 4),
+        CASE(tfunc_regparm3_1 , 0 * 4),
+        CASE(tfunc_regparm3_2 , 0 * 4),
+        CASE(tfunc_regparm3_3 , 0 * 4),
+    #endif
+    };
+
+    #else
+    // only i386 has many calling conventions
+    int tfunc_default(int x, int y, int z)      { return x; }
+
+    static std::vector<_Test_cfunc_is_callee_clenup> _cfunc_is_callee_cleanup_testv = {
+        CASE(tfunc_default, 0),
+    };
+    #endif
+
+    #undef CASE
+    """
+    struct _Test_cfunc_is_callee_clenup:
+        const char* cfunc_name
+        void*       cfunc
+        int         stkclean_by_callee_ok
+
+    vector[_Test_cfunc_is_callee_clenup] _cfunc_is_callee_cleanup_testv

golang/_strconv.pyx

@@ -28,12 +28,11 @@ from golang cimport pyb, byte, rune
 from golang cimport _utf8_decode_rune, _xunichr
 from golang.unicode cimport utf8
 
-from cpython cimport PyObject
+from cpython cimport PyObject, _PyBytes_Resize
 
 cdef extern from "Python.h":
     PyObject* PyBytes_FromStringAndSize(char*, Py_ssize_t) except NULL
     char* PyBytes_AS_STRING(PyObject*)
-    int _PyBytes_Resize(PyObject**, Py_ssize_t) except -1
     void Py_DECREF(PyObject*)
 
 
@@ -65,7 +64,7 @@ cdef bytes _quote(const byte[::1] s, char quote, bint* out_nonascii_escape): # -
     cdef byte c
     q[0] = quote;  q += 1
     while i < len(s):
-        c = s[i]
+        c = s[i]        # XXX -> use raw pointer in the loop
         # fast path - ASCII only
         if c < 0x80:
             if c in (ord('\\'), quote):
@@ -104,7 +103,8 @@ cdef bytes _quote(const byte[::1] s, char quote, bint* out_nonascii_escape): # -
 
         # slow path - full UTF-8 decoding + unicodedata
         else:
-            r, size = _utf8_decode_rune(s[i:])
+            # XXX optimize non-ascii case
+            r, size = _utf8_decode_rune(s[i:])  # XXX -> raw pointer
             isize = i + size
 
             # decode error - just emit raw byte as escaped
@@ -117,6 +117,9 @@ cdef bytes _quote(const byte[::1] s, char quote, bint* out_nonascii_escape): # -
                 q += 4
 
             # printable utf-8 characters go as is
+            # XXX ? use Py_UNICODE_ISPRINTABLE (py3, not available on py2)  ?
+            # XXX ? and generate C table based on unicodedata for py2 ?
+            # XXX -> generate table based on unicodedata for both py2/py3 because Py_UNICODE_ISPRINTABLE is not exactly what matches strconv.IsPrint  (i.e. cat starts from LNPS)
             elif _unicodedata_category(_xunichr(r))[0] in 'LNPS': # letters, numbers, punctuation, symbols
                 for j in range(i, isize):
                     q[0] = s[j]

golang/fmt.h

@@ -111,7 +111,7 @@ inline error errorf(const string& format, Argv... argv) {
 // `const char *` overloads just to catch format mistakes as
 // __attribute__(format) does not work with std::string.
 LIBGOLANG_API string sprintf(const char *format, ...)
-#ifndef _MSC_VER
+#ifndef LIBGOLANG_CC_msc
                                 __attribute__ ((format (printf, 1, 2)))
 #endif
 	;

golang/golang_str_pickle_test.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2022-2023  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+from __future__ import print_function, absolute_import
+
+from golang import b, u, bstr, ustr
+from golang.golang_str_test import xbytes, x32, unicode
+from golang._golang import _test_inside_counted, _test_cfunc_is_callee_cleanup
+from gpython.gpython_test import is_gpython
+from pytest import raises, fixture, mark
+import sys, io, struct
+import six
+
+# run all tests on all py/c pickle modules we aim to support
+import pickle as stdPickle
+if six.PY2:
+    import cPickle
+else:
+    import _pickle as cPickle
+from zodbpickle import slowpickle as zslowPickle
+from zodbpickle import fastpickle as zfastPickle
+from zodbpickle import pickle  as zpickle
+from zodbpickle import _pickle as _zpickle
+import pickletools as stdpickletools
+if six.PY2:
+    from zodbpickle import pickletools_2 as zpickletools
+else:
+    from zodbpickle import pickletools_3 as zpickletools
+
+
+# pickle is pytest fixture that yields all variants of pickle module.
+@fixture(scope="function", params=[stdPickle, cPickle,
+                                   zslowPickle, zfastPickle, zpickle, _zpickle])
+def pickle(request):
+    yield request.param
+
+# pickletools is pytest fixture that yields all variants of pickletools module.
+@fixture(scope="function", params=[stdpickletools, zpickletools])
+def pickletools(request):
+    yield request.param
+
+# pickle2tools returns pickletools module that corresponds to module pickle.
+def pickle2tools(pickle):
+    if pickle in (stdPickle, cPickle):
+        return stdpickletools
+    else:
+        return zpickletools
+
+# @gpystr_only is marker to run a test only under gpython -X gpython.strings=bstr+ustr
+is_gpystr = type(u'') is ustr
+gpystr_only = mark.skipif(not is_gpystr, reason="gpystr-only test")
+
+
+# ---- pickling/unpickling under gpystr ----
+
+# verify that loading *STRING opcodes loads them as bstr on gpython by default.
+# TODO or with encoding='bstr' under plain py
+@gpystr_only
+def test_string_pickle_load_STRING(pickle):
+    p_str   = b"S'\\xd0\\xbc\\xd0\\xb8\\xd1\\x80\\xff'\n."      # STRING 'мир\xff'
+    p_utf8  = b"S'"+xbytes('мир')+b"\\xff'\n."                  # STRING 'мир\xff'
+    p_sbins = b'U\x07\xd0\xbc\xd0\xb8\xd1\x80\xff.'             # SHORT_BINSTRING 'мир\xff'
+    p_bins  = b'T\x07\x00\x00\x00\xd0\xbc\xd0\xb8\xd1\x80\xff.' # BINSTRING 'мир\xff'
+
+    p_bytes = xbytes('мир')+b'\xff'
+
+    # check invokes f on all test pickles
+    def check(f):
+        f(p_str)
+        f(p_utf8)
+        f(p_sbins)
+        f(p_bins)
+
+    # default -> bstr  on both py2 and py3
+    # TODO only this check is gpystr_only -> remove whole-func @gpystr_only
+    def _(p):
+        obj = xloads(pickle, p)
+        assert type(obj) is bstr
+        assert obj == p_bytes
+    check(_)
+
+    # also test bstr inside tuple (for symmetry with save)
+    def _(p):
+        p_ = b'(' + p[:-1] + b't.'
+        tobj = xloads(pickle, p_)
+        assert type(tobj) is tuple
+        assert len(tobj) == 1
+        obj = tobj[0]
+        assert type(obj) is bstr
+        assert obj == p_bytes
+    check(_)
+
+    # pickle supports encoding=... only on py3
+    if six.PY3:
+        # encoding='bstr'  -> bstr
+        def _(p):
+            obj = xloads(pickle, p, encoding='bstr')
+            assert type(obj) is bstr
+            assert obj == p_bytes
+        check(_)
+
+        # encoding='bytes' -> bytes
+        def _(p):
+            obj = xloads(pickle, p, encoding='bytes')
+            assert type(obj) is bytes
+            assert obj == p_bytes
+        check(_)
+
+        # encoding='utf-8' -> UnicodeDecodeError
+        def _(p):
+            with raises(UnicodeDecodeError):
+                xloads(pickle, p, encoding='utf-8')
+        check(_)
+
+        # encoding='utf-8', errors=... -> unicode
+        def _(p):
+            obj = xloads(pickle, p, encoding='utf-8', errors='backslashreplace')
+            assert type(obj) is unicode
+            assert obj == u'мир\\xff'
+        check(_)
+
+
+# verify that saving bstr results in *STRING opcodes on gpython.
+@gpystr_only
+def test_strings_pickle_save_STRING(pickle):
+    s = s0 = b(xbytes('мир')+b'\xff')
+    assert type(s) is bstr
+
+    p_utf8  = b"S'"+xbytes('мир')+b"\\xff'\n."                  # STRING 'мир\xff'
+    p_sbins = b'U\x07\xd0\xbc\xd0\xb8\xd1\x80\xff.'             # SHORT_BINSTRING 'мир\xff'
+    p_bins  = b'T\x07\x00\x00\x00\xd0\xbc\xd0\xb8\xd1\x80\xff.' # BINSTRING 'мир\xff'
+
+    def dumps(proto):
+        return xdumps(pickle, s, proto)
+
+    assert dumps(0) == p_utf8
+    for proto in range(1, HIGHEST_PROTOCOL(pickle)+1):
+        assert dumps(proto) == p_sbins
+
+    # BINSTRING
+    s += b'\x55'*0x100
+    p_bins_ = p_bins[:2] + b'\x01' + p_bins[3:-1] + b'\x55'*0x100 + b'.'
+    for proto in range(1, HIGHEST_PROTOCOL(pickle)+1):
+        assert dumps(proto) == p_bins_
+
+    # also test bstr inside tuple to verify that what we patched is actually
+    # _pickle.save that is invoked from inside other save_X functions.
+    s = (s0,)
+    p_tutf8  = b'(' + p_utf8[:-1]  + b't.'
+    p_tsbins = b'(' + p_sbins[:-1] + b't.'
+    assert dumps(0) == p_tutf8
+    assert dumps(1) == p_tsbins
+    # don't test proto ≥ 2 because they start to use TUPLE1 instead of TUPLE
+
+
+# verify that loading *UNICODE opcodes loads them as unicode/ustr.
+# this is standard behaviour but we verify it since we patch pickle's strings processing.
+# also verify save lightly for symmetry.
+# NOTE not @gpystr_only
+def test_string_pickle_loadsave_UNICODE(pickle):
+    # NOTE builtin pickle behaviour is to save unicode via 'surrogatepass' error handler
+    #      this means that b'мир\xff' -> ustr/unicode -> save will emit *UNICODE with
+    #      b'мир\xed\xb3\xbf' instead of b'мир\xff' as data.
+    p_uni   = b'V\\u043c\\u0438\\u0440\\udcff\n.'                       # UNICODE 'мир\uDCFF'
+    p_binu  = b'X\x09\x00\x00\x00\xd0\xbc\xd0\xb8\xd1\x80\xed\xb3\xbf.' # BINUNICODE  NOTE ...edb3bf not ...ff
+    p_sbinu = b'\x8c\x09\xd0\xbc\xd0\xb8\xd1\x80\xed\xb3\xbf.'          # SHORT_BINUNICODE
+    p_binu8 = b'\x8d\x09\x00\x00\x00\x00\x00\x00\x00\xd0\xbc\xd0\xb8\xd1\x80\xed\xb3\xbf.' # BINUNICODE8
+
+    u_obj = u'мир\uDCFF'; assert type(u_obj) is unicode
+
+    # load: check invokes f on all test pickles that pickle should support
+    def check(f):
+        f(p_uni)
+        f(p_binu)
+        if HIGHEST_PROTOCOL(pickle) >= 4:
+            f(p_sbinu)
+            f(p_binu8)
+
+    def _(p):
+        obj = xloads(pickle, p)
+        assert type(obj) is unicode
+        assert obj == u_obj
+    check(_)
+
+    # save
+    def dumps(proto):
+        return xdumps(pickle, u_obj, proto)
+    assert dumps(0) == p_uni
+    assert dumps(1) == p_binu
+    assert dumps(2) == p_binu
+    if HIGHEST_PROTOCOL(pickle) >= 3:
+        assert dumps(3) == p_binu
+    if HIGHEST_PROTOCOL(pickle) >= 4:
+        assert dumps(4) == p_sbinu
+
+
+# ---- pickling/unpickling generally without gpystr ----
+
+# verify that bstr/ustr can be pickled/unpickled correctly on !gpystr.
+# gpystr should also load ok what was pickled on !gpystr.
+# for uniformity gpystr is also verified to save/load objects correctly.
+# However the main gpystr tests are load/save tests for *STRING and *UNICODE above.
+def test_strings_pickle_bstr_ustr(pickle):
+    bs = b(xbytes('мир')+b'\xff')
+    us = u(xbytes('май')+b'\xff')
+
+    def diss(p): return xdiss(pickle2tools(pickle), p)
+    def dis(p): print(diss(p))
+
+    # assert_pickle verifies that pickling obj results in
+    #
+    #   - dumps_ok_gpystr  (when run under gpython with gpython.string=bstr+ustr),  or
+    #   - dumps_ok_stdstr  (when run under plain python or gpython with gpython.strings=pystd)
+    #
+    # and that unpickling results back in obj.
+    #
+    # gpystr should also unpickle !gpystr pickle correctly.
+    assert HIGHEST_PROTOCOL(pickle) <= 5
+    def assert_pickle(obj, proto, dumps_ok_gpystr, dumps_ok_stdstr):
+        if proto > HIGHEST_PROTOCOL(pickle):
+            with raises(ValueError):
+                xdumps(pickle, obj, proto)
+            return
+        p = xdumps(pickle, obj, proto)
+        if not is_gpystr:
+            assert p == dumps_ok_stdstr, diss(p)
+            dumps_okv = [dumps_ok_stdstr]
+        else:
+            assert p == dumps_ok_gpystr, diss(p)
+            dumps_okv = [dumps_ok_gpystr, dumps_ok_stdstr]
+        for p in dumps_okv:
+            #dis(p)
+            obj2 = xloads(pickle, p)
+            assert type(obj2) is type(obj)
+            assert obj2 == obj
+
+    _ = assert_pickle
+
+    _(bs, 0, xbytes("S'мир\\xff'\n."),                                      # STRING
+             b"cgolang\nbstr\n(V\\u043c\\u0438\\u0440\\udcff\ntR.")         # bstr(UNICODE)
+
+    _(us, 0, b'V\\u043c\\u0430\\u0439\\udcff\n.',                           # UNICODE
+             b'cgolang\nustr\n(V\\u043c\\u0430\\u0439\\udcff\ntR.')         # ustr(UNICODE)
+
+    _(bs, 1, b'U\x07\xd0\xbc\xd0\xb8\xd1\x80\xff.',                         # SHORT_BINSTRING
+             b'cgolang\nbstr\n(X\x09\x00\x00\x00'                           # bstr(BINUNICODE)
+                        b'\xd0\xbc\xd0\xb8\xd1\x80\xed\xb3\xbftR.')
+
+    # NOTE BINUNICODE ...edb3bf not ...ff  (see test_string_pickle_loadsave_UNICODE for details)
+    _(us, 1, b'X\x09\x00\x00\x00\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf.',     # BINUNICODE
+             b'cgolang\nustr\n(X\x09\x00\x00\x00'                           # bstr(BINUNICODE)
+                        b'\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbftR.')
+
+    _(bs, 2, b'U\x07\xd0\xbc\xd0\xb8\xd1\x80\xff.',                         # SHORT_BINSTRING
+             b'cgolang\nbstr\nX\x09\x00\x00\x00'                            # bstr(BINUNICODE)
+                        b'\xd0\xbc\xd0\xb8\xd1\x80\xed\xb3\xbf\x85\x81.')
+
+    _(us, 2, b'X\x09\x00\x00\x00\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf.',     # BINUNICODE
+             b'cgolang\nustr\nX\x09\x00\x00\x00'                            # ustr(BINUNICODE)
+                        b'\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf\x85\x81.')
+
+    _(bs, 3, b'U\x07\xd0\xbc\xd0\xb8\xd1\x80\xff.',                         # SHORT_BINSTRING
+             b'cgolang\nbstr\nC\x07\xd0\xbc\xd0\xb8\xd1\x80\xff\x85\x81.')  # bstr(SHORT_BINBYTES)
+
+    _(us, 3, b'X\x09\x00\x00\x00\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf.',     # BINUNICODE
+             b'cgolang\nustr\nX\x09\x00\x00\x00'                            # ustr(BINUNICODE)
+                        b'\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf\x85\x81.')
+
+    for p in (4,5):
+        _(bs, p,
+             b'U\x07\xd0\xbc\xd0\xb8\xd1\x80\xff.',                         # SHORT_BINSTRING
+             b'\x8c\x06golang\x8c\x04bstr\x93C\x07'                         # bstr(SHORT_BINBYTES)
+                        b'\xd0\xbc\xd0\xb8\xd1\x80\xff\x85\x81.')
+        _(us, p,
+             b'\x8c\x09\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf.',              # SHORT_BINUNICODE
+             b'\x8c\x06golang\x8c\x04ustr\x93\x8c\x09'                      # ustr(SHORT_BINUNICODE)
+                        b'\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf\x85\x81.')
+
+
+# ---- disassembly ----
+
+# xdiss returns disassembly of a pickle as string.
+def xdiss(pickletools, p): # -> str
+    out = six.StringIO()
+    pickletools.dis(p, out)
+    return out.getvalue()
+
+# verify that disassembling *STRING opcodes works with treating strings as UTF8b.
+@gpystr_only
+def test_string_pickle_dis_STRING(pickletools):
+    p_str   = b"S'\\xd0\\xbc\\xd0\\xb8\\xd1\\x80'\n."       # STRING 'мир'
+    p_sbins = b'U\x06\xd0\xbc\xd0\xb8\xd1\x80.'             # SHORT_BINSTRING 'мир'
+    p_bins  = b'T\x06\x00\x00\x00\xd0\xbc\xd0\xb8\xd1\x80.' # BINSTRING 'мир'
+
+    bmir = x32("b('мир')", "'мир'")
+
+    assert xdiss(pickletools, p_str) == """\
+    0: S    STRING     %s
+   28: .    STOP
+highest protocol among opcodes = 0
+""" % bmir
+
+    assert xdiss(pickletools, p_sbins) == """\
+    0: U    SHORT_BINSTRING %s
+    8: .    STOP
+highest protocol among opcodes = 1
+""" % bmir
+
+    assert xdiss(pickletools, p_bins) == """\
+    0: T    BINSTRING  %s
+   11: .    STOP
+highest protocol among opcodes = 1
+""" % bmir
+
+
+# ---- loads and normalized dumps ----
+
+# xloads loads pickle p via pickle.loads
+# it also verifies that .load and Unpickler.load give the same result.
+def xloads(pickle, p, **kw):
+    obj1 = _xpickle_attr(pickle, 'loads')(p, **kw)
+    obj2 = _xpickle_attr(pickle, 'load') (io.BytesIO(p), **kw)
+    obj3 = _xpickle_attr(pickle, 'Unpickler')(io.BytesIO(p), **kw).load()
+    assert type(obj2) is type(obj1)
+    assert type(obj3) is type(obj1)
+    assert obj1 == obj2 == obj3
+    return obj1
+
+# xdumps dumps obj via pickle.dumps
+# it also verifies that .dump and Pickler.dump give the same.
+# the pickle is returned in normalized form - see pickle_normalize for details.
+def xdumps(pickle, obj, proto, **kw):
+    p1 = _xpickle_attr(pickle, 'dumps')(obj, proto, **kw)
+    f2 = io.BytesIO();  _xpickle_attr(pickle, 'dump')(obj, f2, proto, **kw)
+    p2 = f2.getvalue()
+    f3 = io.BytesIO();  _xpickle_attr(pickle, 'Pickler')(f3, proto, **kw).dump(obj)
+    p3 = f3.getvalue()
+    assert type(p1) is bytes
+    assert type(p2) is bytes
+    assert type(p3) is bytes
+    assert p1 == p2 == p3
+
+    # remove not interesting parts: PROTO / FRAME header and unused PUTs
+    if proto >= 2:
+        protover = PROTO(proto)
+        assert p1.startswith(protover)
+    return pickle_normalize(pickle2tools(pickle), p1)
+
+def _xpickle_attr(pickle, name):
+    # on py3 pickle.py tries to import from C _pickle to optimize by default
+    # -> verify py version if we are asked to test pickle.py
+    if six.PY3 and (pickle is stdPickle):
+        assert getattr(pickle, name) is getattr(cPickle, name)
+        name = '_'+name
+    return getattr(pickle, name)
+
+# pickle_normalize returns normalized version of pickle p.
+#
+# - PROTO and FRAME opcodes are removed from header,
+# - unused PUT, BINPUT and MEMOIZE opcodes - those without corresponding GET are removed,
+# - *PUT indices start from 0 (this unifies cPickle with pickle).
+def pickle_normalize(pickletools, p):
+    def iter_pickle(p): # -> i(op, arg, pdata)
+        op_prev  = None
+        arg_prev = None
+        pos_prev = None
+        for op, arg, pos in pickletools.genops(p):
+            if op_prev is not None:
+                pdata_prev = p[pos_prev:pos]
+                yield (op_prev, arg_prev, pdata_prev)
+            op_prev  = op
+            arg_prev = arg
+            pos_prev = pos
+        if op_prev is not None:
+            yield (op_prev, arg_prev, p[pos_prev:])
+
+    memo_oldnew = {} # idx used in original pop/get -> new index | None if not get
+    idx = 0
+    for op, arg, pdata in iter_pickle(p):
+        if 'PUT' in op.name:
+            memo_oldnew.setdefault(arg, None)
+        elif 'MEMOIZE' in op.name:
+            memo_oldnew.setdefault(len(memo_oldnew), None)
+        elif 'GET' in op.name:
+            if memo_oldnew.get(arg) is None:
+                memo_oldnew[arg] = idx
+                idx += 1
+
+    pout = b''
+    memo_old = set() # idx used in original pop
+    for op, arg, pdata in iter_pickle(p):
+        if op.name in ('PROTO', 'FRAME'):
+            continue
+        if 'PUT' in op.name:
+            memo_old.add(arg)
+            newidx = memo_oldnew.get(arg)
+            if newidx is None:
+                continue
+            pdata = globals()[op.name](newidx)
+        if 'MEMOIZE' in op.name:
+            idx = len(memo_old)
+            memo_old.add(idx)
+            newidx = memo_oldnew.get(idx)
+            if newidx is None:
+                continue
+        if 'GET' in op.name:
+            newidx = memo_oldnew[arg]
+            assert newidx is not None
+            pdata = globals()[op.name](newidx)
+        pout += pdata
+    return pout
+
+P = struct.pack
+def PROTO(version):     return b'\x80'  + P('<B', version)
+def FRAME(size):        return b'\x95'  + P('<Q', size)
+def GET(idx):           return b'g%d\n' % (idx,)
+def PUT(idx):           return b'p%d\n' % (idx,)
+def BINPUT(idx):        return b'q'     + P('<B', idx)
+def BINGET(idx):        return b'h'     + P('<B', idx)
+def LONG_BINPUT(idx):   return b'r'     + P('<I', idx)
+def LONG_BINGET(idx):   return b'j'     + P('<I', idx)
+MEMOIZE =                      b'\x94'
+
+def test_pickle_normalize(pickletools):
+    def diss(p):
+        return xdiss(pickletools, p)
+
+    proto = 0
+    for op in pickletools.opcodes:
+        proto = max(proto, op.proto)
+    assert proto >= 2
+
+    def _(p, p_normok):
+        p_norm = pickle_normalize(pickletools, p)
+        assert p_norm == p_normok, diss(p_norm)
+
+    _(b'.', b'.')
+    _(b'I1\n.', b'I1\n.')
+    _(PROTO(2)+b'I1\n.', b'I1\n.')
+
+    putgetv = [(PUT,GET), (BINPUT, BINGET)]
+    if proto >= 4:
+        putgetv.append((LONG_BINPUT, LONG_BINGET))
+    for (put,get) in putgetv:
+        _(b'(I1\n'+put(1) + b'I2\n'+put(2) +b't'+put(3)+b'0'+get(3)+put(4)+b'.',
+          b'(I1\nI2\nt'+put(0)+b'0'+get(0)+b'.')
+
+    if proto >= 4:
+        _(FRAME(4)+b'I1\n.', b'I1\n.')
+        _(b'I1\n'+MEMOIZE+b'I2\n'+MEMOIZE+GET(0)+b'.',
+          b'I1\n'+MEMOIZE+b'I2\n'+GET(0)+b'.')
+
+
+# ---- internals of patching ----
+
+# being able to cPickle bstr as STRING depends on proper working of inside_counted function.
+# Verify it with dedicated unit test.
+def test_inside_counted(capsys):
+    outok = _test_inside_counted()
+    _ = capsys.readouterr()
+    if _.err:
+        print(_.err, file=sys.stderr)
+    assert _.out == outok
+
+def test_cfunc_is_callee_cleanup():
+    _test_cfunc_is_callee_cleanup()
+
+# verify that what we patched - e.g. PyUnicode_Decode - stay unaffected when
+# called outside of bstr/ustr context.
+# NOTE this test complements test_strings_patched_transparently in golang_str_test.py
+def test_pickle_strings_patched_transparently():
+    # PyUnicode_Decode stays working and unaffected
+    b_  = xbytes("abc")
+    _ = b_.decode();         assert type(_) is unicode;  assert _ == u"abc"
+    _ = b_.decode("utf8");   assert type(_) is unicode;  assert _ == u"abc"
+    _ = b_.decode("ascii");  assert type(_) is unicode;  assert _ == u"abc"
+
+    b_  = xbytes("мир")
+    _ = b_.decode("utf8");   assert type(_) is unicode;  assert _ == u"мир"
+    with raises(UnicodeDecodeError):
+        b_.decode("ascii")
+
+
+# ---- misc ----
+
+# HIGHEST_PROTOCOL returns highest protocol supported by pickle.
+def HIGHEST_PROTOCOL(pickle):
+    if   six.PY3  and  pickle is cPickle:
+        pmax = stdPickle.HIGHEST_PROTOCOL  # py3: _pickle has no .HIGHEST_PROTOCOL
+    elif six.PY3  and  pickle is _zpickle:
+        pmax = zpickle.HIGHEST_PROTOCOL    # ----//---- for _zpickle
+    else:
+        pmax = pickle.HIGHEST_PROTOCOL
+    assert pmax >= 2
+    return pmax

golang/golang_str_test.py

@@ -146,9 +146,17 @@ def test_strings_basic():
     _ = ustr(123);      assert type(_) is ustr;  assert _ == '123'
     _ = bstr([1,'β']);  assert type(_) is bstr;  assert _ == "[1, 'β']"
     _ = ustr([1,'β']);  assert type(_) is ustr;  assert _ == "[1, 'β']"
-    obj = object()
-    _ = bstr(obj);      assert type(_) is bstr;  assert _ == str(obj)  # <object ...>
-    _ = ustr(obj);      assert type(_) is ustr;  assert _ == str(obj)  # <object ...>
+    obj = object();  assert str(obj).startswith('<object object at 0x')
+    _ = bstr(obj);      assert type(_) is bstr;  assert _ == str(obj)
+    _ = ustr(obj);      assert type(_) is ustr;  assert _ == str(obj)
+    ecls = RuntimeError;  assert str(ecls) == x32("<class 'RuntimeError'>",
+                                                 "<type 'exceptions.RuntimeError'>")
+    _ = bstr(ecls);     assert type(_) is bstr;  assert _ == str(ecls)
+    _ = ustr(ecls);     assert type(_) is ustr;  assert _ == str(ecls)
+    exc = RuntimeError('zzz');  assert str(exc) == 'zzz'
+    _ = bstr(exc);      assert type(_) is bstr;  assert _ == str(exc)
+    _ = ustr(exc);      assert type(_) is ustr;  assert _ == str(exc)
+
 
     # when stringifying they also handle bytes/bytearray inside containers as UTF-8 strings
     _ = bstr([xunicode(  'β')]);   assert type(_) is bstr;  assert _ == "['β']"
@@ -246,10 +254,12 @@ def test_strings_basic():
     assert hash(bs) == hash("мир");  assert bs == "мир"
 
     # str/repr
+    def rb(x,y): return xb32(x, 'b'+y,y)
+    def ru(x,y): return xu32(x, y,'u'+y)
     _ = str(us);   assert isinstance(_, str);  assert _ == "мир"
     _ = str(bs);   assert isinstance(_, str);  assert _ == "мир"
-    _ = repr(us);  assert isinstance(_, str);  assert _ == "u('мир')"
-    _ = repr(bs);  assert isinstance(_, str);  assert _ == "b('мир')"
+    _ = repr(us);  assert isinstance(_, str);  assert _ == ru("u('мир')",  "'мир'")
+    _ = repr(bs);  assert isinstance(_, str);  assert _ == rb("b('мир')",  "'мир'")
 
     # str/repr of non-valid utf8
     b_hik8 = xbytes  ('привет ')+b(k8mir_bytes);  assert type(b_hik8) is bstr
@@ -259,11 +269,17 @@ def test_strings_basic():
 
     _ = str(u_hik8);   assert isinstance(_, str);  assert _ == xbytes('привет ')+b'\xcd\xc9\xd2'
     _ = str(b_hik8);   assert isinstance(_, str);  assert _ == xbytes('привет ')+b'\xcd\xc9\xd2'
-    _ = repr(u_hik8);  assert isinstance(_, str);  assert _ == r"u(b'привет \xcd\xc9\xd2')"
-    _ = repr(b_hik8);  assert isinstance(_, str);  assert _ == r"b(b'привет \xcd\xc9\xd2')"
+    _ = repr(u_hik8);  assert isinstance(_, str);  assert _ ==      r"u(b'привет \xcd\xc9\xd2')"
+                                                                    # NOTE ^^^ same for u,3/2
+    _ = repr(b_hik8);  assert isinstance(_, str);  assert _ == rb(r"b(b'привет \xcd\xc9\xd2')",
+                                                                     r"'привет \xcd\xc9\xd2'")
 
     # str/repr of quotes
     def _(text, breprok, ureprok):
+        assert breprok[:2] == "b(";  assert breprok[-1] == ")"
+        assert ureprok[:2] == "u(";  assert ureprok[-1] == ")"
+        breprok = rb(breprok, breprok[2:-1])  # b('...')  or '...' if bytes   patched
+        ureprok = ru(ureprok, ureprok[2:-1])  # u('...')  or '...' if unicode patched
         bt = b(text);  assert type(bt) is bstr
         ut = u(text);  assert type(ut) is ustr
         _ = str(bt);   assert isinstance(_, str);  assert _ == text
@@ -286,20 +302,26 @@ def test_strings_basic():
 
 # verify that bstr/ustr are created with correct refcount.
 def test_strings_refcount():
+    # buffer with string data - not bytes nor unicode so that when builting
+    # string types are patched no case where bytes is created from the same
+    # bytes, or unicode is created from the same unicode - only increasing
+    # refcount of original object.
+    data = bytearray([ord('a'), ord('b'), ord('c'), ord('4')])
+
     # first verify our logic on std type
-    obj = xbytes(u'abc');   assert type(obj) is bytes
+    obj = bytes(data);      assert type(obj) is bytes
     gc.collect();   assert sys.getrefcount(obj) == 1+1   # +1 due to obj passed to getrefcount call
 
     # bstr
-    obj = b('abc');         assert type(obj) is bstr
+    obj = b(data);          assert type(obj) is bstr
     gc.collect();           assert sys.getrefcount(obj) == 1+1
-    obj = bstr('abc');      assert type(obj) is bstr
+    obj = bstr(data);       assert type(obj) is bstr
     gc.collect();           assert sys.getrefcount(obj) == 1+1
 
     # ustr
-    obj = u('abc');         assert type(obj) is ustr
+    obj = u(data);          assert type(obj) is ustr
     gc.collect();           assert sys.getrefcount(obj) == 1+1
-    obj = ustr('abc');      assert type(obj) is ustr
+    obj = ustr(data);       assert type(obj) is ustr
     gc.collect();           assert sys.getrefcount(obj) == 1+1
 
 
@@ -326,26 +348,6 @@ def test_strings_memoryview():
     assert _(5) == 0x80
 
 
-# verify that bstr/ustr can be pickled/unpickled correctly.
-def test_strings_pickle():
-    bs = b("мир")
-    us = u("май")
-
-    #from pickletools import dis
-    for proto in range(0, pickle.HIGHEST_PROTOCOL+1):
-        p_bs = pickle.dumps(bs, proto)
-        #dis(p_bs)
-        bs_ = pickle.loads(p_bs)
-        assert type(bs_) is bstr
-        assert bs_ == bs
-
-        p_us = pickle.dumps(us, proto)
-        #dis(p_us)
-        us_ = pickle.loads(p_us)
-        assert type(us_) is ustr
-        assert us_ == us
-
-
 # verify that ord on bstr/ustr works as expected.
 def test_strings_ord():
     with raises(TypeError): ord(b(''))
@@ -617,7 +619,8 @@ def test_strings_iter():
 
     # iter( b/u/unicode ) -> iterate unicode characters
     # NOTE that iter(b) too yields unicode characters - not integers or bytes
-    bi  = iter(bs)
+    #bi  = iter(bs)         # XXX temp disabled
+    bi  = iter(us)
     ui  = iter(us)
     ui_ = iter(u_)
     class XIter:
@@ -1100,64 +1103,65 @@ def test_strings_mod_and_format():
 
     # _bprintf parses %-format ourselves. Verify that parsing first
     # NOTE here all strings are plain ASCII.
-    def _(fmt, args):
+    def _(fmt, args, ok):
         fmt = '*str '+fmt
-        for l in range(len(fmt), -1, -1):
-            # [:len(fmt)] verifies original case
-            # [:l<len]    should verify "incomplete format" parsing
-            verify_fmt_all_types(lambda fmt, args: fmt % args,
-                                 fmt[:l], args, excok=True)
-
-    _('%(name)s',   {'name': 123})
-    _('%x',         123)        # flags
-    _('%#x',        123)
-    _('%05d',       123)
-    _('%-5d',       123)
-    _('% d',        123)
-    _('% d',       -123)
-    _('%+d',       -123)
-    _('%5d',        123)        # width
-    _('%*d',        (5,123))
-    _('%f',         1.234)      # .prec
-    _('%.f',        1.234)
-    _('%.1f',       1.234)
-    _('%.2f',       1.234)
-    _('%*f',        (2,1.234))
-    _('%hi',        123)        # len
-    _('%li',        123)
-    _('%Li',        123)
-    _('%%',         ())         # %%
-    _('%10.4f',     1.234)      # multiple features
-    _('%(x)10.4f',  {'y':0, 'x':1.234})
-    _('%*.*f',      (10,4,1.234))
-
-    _('',           {})         # not all arguments converted
-    _('',           [])
-    _('',           123)
-    _('',           '123')
-    _('%s',         ())         # not enough arguments to format
-    _('%s %s',      123)
-    _('%s %s',      (123,))
-
-    _('%(x)s',      123)        # format requires a mapping
-    _('%(x)s',      (123,))
-    _('%s %(x)s',   (123,4))
-    _('%(x)s %s',   (123,4))
-
-    _('%(x)s %s',   {'x':1})    # mixing tuple/dict
-    _('%s %(x)s',   {'x':1})
-
-    _('abc %z',     1)          # unsupported format character
-    _('abc %44z',   1)
+        if isinstance(ok, Exception):
+            excok = True
+        else:
+            ok  = '*str '+ok
+            excok = False
+        verify_fmt_all_types(lambda fmt, args: fmt % args, fmt, args, ok, excok=excok)
+        # also automatically verify "incomplete format" parsing via fmt[:l<len]
+        # this works effectively only when run under std python though.
+        for l in range(len(fmt)-1, -1, -1):
+            verify_fmt_all_types(lambda fmt, args: fmt % args, fmt[:l], args, excok=True)
+
+    _('%(name)s',   {'name': 123}   ,   '123')
+    _('%x',         123             ,   '7b')           # flags
+    _('%#x',        123             ,   '0x7b')
+    _('%05d',       123             ,   '00123')
+    _('%-5d',       123             ,   '123  ')
+    _('% d',        123             ,   ' 123')
+    _('% d',       -123             ,   '-123')
+    _('%+d',        123             ,   '+123')
+    _('%+d',       -123             ,   '-123')
+    _('%5d',        123             ,   '  123')        # width
+    _('%*d',        (5,123)         ,   '  123')
+    _('%f',         1.234           ,   '1.234000')     # .prec
+    _('%.f',        1.234           ,   '1')
+    _('%.1f',       1.234           ,   '1.2')
+    _('%.2f',       1.234           ,   '1.23')
+    _('%*f',        (2,1.234)       ,   '1.234000')
+    _('%.*f',       (2,1.234)       ,   '1.23')
+    _('%hi',        123             ,   '123')          # len
+    _('%li',        123             ,   '123')
+    _('%Li',        123             ,   '123')
+    _('%%',         ()              ,   '%')            # %%
+    _('%10.4f',     1.234           ,   '    1.2340')   # multiple features
+    _('%(x)10.4f',  {'y':0, 'x':1.234}, '    1.2340')
+    _('%*.*f',      (10,4,1.234)    ,   '    1.2340')
+
+    _('',           {}      ,   '')                     # errors
+    _('',           []      ,   '')
+    _('',           123     ,   TypeError('not all arguments converted during string formatting'))
+    _('',           '123'   ,   TypeError('not all arguments converted during string formatting'))
+    _('%s',         ()      ,   TypeError('not enough arguments for format string'))
+    _('%s %s',      123     ,   TypeError('not enough arguments for format string'))
+    _('%s %s',      (123,)  ,   TypeError('not enough arguments for format string'))
+
+    _('%(x)s',      123     ,   TypeError('format requires a mapping'))
+    _('%(x)s',      (123,)  ,   TypeError('format requires a mapping'))
+    _('%s %(x)s',   (123,4) ,   TypeError('format requires a mapping'))
+    _('%(x)s %s',   (123,4) ,   TypeError('format requires a mapping'))
+
+    _('%(x)s %s',   {'x':1} ,   TypeError('not enough arguments for format string'))    # mixing tuple/dict
+    _('%s %(x)s',   {'x':1} ,   "{'x': 1} 1")
 
     # for `'%4%' % ()` py2 gives '   %', but we stick to more reasonable py3 semantic
-    def _(fmt, args, ok):
-        return verify_fmt_all_types(lambda fmt, args: fmt % args,
-                                    fmt, args, ok, excok=True)
-    _('*str %4%',   (),      TypeError("not enough arguments for format string"))
-    _('*str %4%',   1,       ValueError("unsupported format character '%' (0x25) at index 7"))
-    _('*str %4%',   (1,),    ValueError("unsupported format character '%' (0x25) at index 7"))
-    _('*str %(x)%', {'x':1}, ValueError("unsupported format character '%' (0x25) at index 9"))
+    _('%4%',        ()      ,   TypeError("not enough arguments for format string"))
+    _('%4%',        1       ,   ValueError("unsupported format character '%' (0x25) at index 7"))
+    _('%4%',        (1,)    ,   ValueError("unsupported format character '%' (0x25) at index 7"))
+    _('%(x)%',      {'x':1} ,   ValueError("unsupported format character '%' (0x25) at index 9"))
 
 
     # parse checking complete. now verify actual %- and format- formatting
@@ -1211,40 +1215,42 @@ def test_strings_mod_and_format():
             fmt_ = fmt
         verify_fmt_all_types(xformat, fmt_, args, *okv)
 
-    _("*str a %s z",  123)      # NOTE *str to force str -> bstr/ustr even for ASCII string
-    _("*str a %s z",  '*str \'"\x7f')
-    _("*str a %s z",  'β')
-    _("*str a %s z",  ('β',))
+    # NOTE *str to force str -> bstr/ustr even for ASCII string
+    _("*str a %s z",  123                         , "*str a 123 z")
+    _("*str a %s z",  '*str \'"\x7f'              , "*str a *str '\"\x7f z")
+    _("*str a %s z",  'β'                         , "*str a β z")
+    _("*str a %s z",  ('β',)                      , "*str a β z")
     _("*str a %s z",  ['β']                       , "*str a ['β'] z")
 
-    _("a %s π",  123)
-    _("a %s π",  '*str \'"\x7f')
-    _("a %s π",  'β')
-    _("a %s π",  ('β',))
+    _("a %s π",  123                              , "a 123 π")
+    _("a %s π",  '*str \'"\x7f'                   , "a *str '\"\x7f π")
+    _("a %s π",  'β'                              , "a β π")
+    _("a %s π",  ('β',)                           , "a β π")
     _("a %s π",  ['β']                            , "a ['β'] π")
 
-    _("α %s z",  123)
-    _("α %s z",  '*str \'"\x7f')
-    _("α %s z",  'β')
-    _("α %s z",  ('β',))
+    _("α %s z",  123                              , "α 123 z")
+    _("α %s z",  '*str \'"\x7f'                   , "α *str '\"\x7f z")
+    _("α %s z",  'β'                              , "α β z")
+    _("α %s z",  ('β',)                           , "α β z")
     _("α %s z",  ['β']                            , "α ['β'] z")
 
-    _("α %s π",  123)
-    _("α %s π",  '*str \'"\x7f')
-    _("α %s π",  'β')
-    _("α %s π",  ('β',))
-    _("α %s π",  ('β',))
-    _("α %s %s π",  ('β', 'γ'))
-    _("α %s %s %s π",  ('β', 'γ', 'δ'))
-    _("α %s %s %s %s %s %s %s π",  (1, 'β', 2, 'γ', 3, 'δ', 4))
-    _("α %s π",  [])
-    _("α %s π",  ([],))
-    _("α %s π",  ((),))
-    _("α %s π",  set())
-    _("α %s π",  (set(),))
-    _("α %s π",  frozenset())
-    _("α %s π",  (frozenset(),))
-    _("α %s π",  ({},))
+    _("α %s π",  123                              , "α 123 π")
+    _("α %s π",  '*str \'"\x7f'                   , "α *str '\"\x7f π")
+    _("α %s π",  'β'                              , "α β π")
+    _("α %s π",  ('β',)                           , "α β π")
+    _("α %s π",  ('β',)                           , "α β π")
+    _("α %s %s π",  ('β', 'γ')                    , "α β γ π")
+    _("α %s %s %s π",  ('β', 'γ', 'δ')            , "α β γ δ π")
+    _("α %s %s %s %s %s %s %s π",  (1, 'β', 2, 'γ', 3, 'δ', 4),
+                                                    "α 1 β 2 γ 3 δ 4 π")
+    _("α %s π",  []                               , "α [] π")
+    _("α %s π",  ([],)                            , "α [] π")
+    _("α %s π",  ((),)                            , "α () π")
+    _("α %s π",  set()                            , x32("α set() π", "α set([]) π"))
+    _("α %s π",  (set(),)                         , x32("α set() π", "α set([]) π"))
+    _("α %s π",  frozenset()                      , x32("α frozenset() π", "α frozenset([]) π"))
+    _("α %s π",  (frozenset(),)                   , x32("α frozenset() π", "α frozenset([]) π"))
+    _("α %s π",  ({},)                            , "α {} π")
     _("α %s π",  ['β']                            , "α ['β'] π")
     _("α %s π",  (['β'],)                         , "α ['β'] π")
     _("α %s π",  (('β',),)                        , "α ('β',) π")
@@ -1279,7 +1285,8 @@ def test_strings_mod_and_format():
     # recursive frozenset
     l = hlist()
     f = frozenset({1, l}); l.append(f)
-    _('α %s π', (f,))
+    _('α %s π', (f,)                              , *x32(("α frozenset({1, [frozenset(...)]}) π", "α frozenset({[frozenset(...)], 1}) π"),
+                                                         ("α frozenset([1, [frozenset(...)]]) π", "α frozenset([[frozenset(...)], 1]) π")))
 
     # recursive dict (via value)
     d = {1:'мир'}; d.update({2:d})
@@ -1296,15 +1303,15 @@ def test_strings_mod_and_format():
     class Cold:
         def __repr__(self): return "Cold()"
         def __str__(self):  return u"Класс (old)"
-    _('α %s π', Cold())
-    _('α %s π', (Cold(),))
+    _('α %s π', Cold()                            , "α Класс (old) π")
+    _('α %s π', (Cold(),)                         , "α Класс (old) π")
 
     # new-style class with __str__
     class Cnew(object):
         def __repr__(self): return "Cnew()"
         def __str__(self):  return u"Класс (new)"
-    _('α %s π', Cnew())
-    _('α %s π', (Cnew(),))
+    _('α %s π', Cnew()                            , "α Класс (new) π")
+    _('α %s π', (Cnew(),)                         , "α Класс (new) π")
 
 
     # custom classes inheriting from set/list/tuple/dict/frozenset
@@ -1334,7 +1341,10 @@ def test_strings_mod_and_format():
     # namedtuple
     cc = collections; xcc = six.moves
     Point = cc.namedtuple('Point', ['x', 'y'])
-    _('α %s π', (Point('β','γ'),)             , "α Point(x='β', y='γ') π")
+    verify_fmt_all_types(lambda fmt, args: fmt % args,
+      'α %s π',   Point('β','γ')              , TypeError("not all arguments converted during string formatting"), excok=True)
+    _('α %s %s π',Point('β','γ')              , "α β γ π")
+    _('α %s π',  (Point('β','γ'),)            , "α Point(x='β', y='γ') π")
     # deque
     _('α %s π', cc.deque(['β','γ'])           , "α deque(['β', 'γ']) π")
     _('α %s π', (cc.deque(['β','γ']),)        , "α deque(['β', 'γ']) π")
@@ -1536,6 +1546,14 @@ def test_strings__format__():
 # verify print for bstr/ustr.
 def test_strings_print():
     outok = readfile(dir_testprog + "/golang_test_str.txt")
+    # repr(bstr|ustr) is changed if string types are patched:
+    # b('...') ->  '...'  if bstr is patched in
+    # u('...') -> u'...'  if ustr is patched in  (here we assume it is all valid utf8 there)
+    if bstr is bytes:
+        outok = re.sub(br"b\((.*?)\)", x32(r"b\1", r"\1"), outok)
+    if ustr is unicode:
+        outok = re.sub(br"u\((.*?)\)", x32(r"\1", r"u\1"), outok)
+
     retcode, stdout, stderr = _pyrun(["golang_test_str.py"],
                                 cwd=dir_testprog, stdout=PIPE, stderr=PIPE)
     assert retcode == 0, (stdout, stderr)
@@ -1578,7 +1596,11 @@ def test_strings_methods():
         ur = xcall(us, meth, *argv, **kw)
 
         def assertDeepEQ(a, b, bstrtype):
-            assert not isinstance(a, (bstr, ustr))
+            # `assert not isinstance(a, (bstr, ustr))` done carefully not to
+            # break when bytes/unicode are patched with bstr/ustr
+            if isinstance(a, bytes):    assert type(a) is bytes
+            if isinstance(a, unicode):  assert type(a) is unicode
+
             if type(a) is unicode:
                 assert type(b) is bstrtype
                 assert a == b
@@ -1841,6 +1863,26 @@ def test_strings_subclasses(tx):
     _  = b(xx);     assert type(_)  is bstr ; assert _ == 'мир'
     _  = u(xx);     assert type(_)  is ustr ; assert _ == 'мир'
 
+    # __str__ returns *str, not MyStr
+    txstr = {
+        unicode: str,
+        bstr:    x32(ustr, bstr),
+        ustr:    x32(ustr, bstr),
+    }[tx]
+    if six.PY2  and  tx is unicode: # on py2 unicode.__str__ raises UnicodeEncodeError:
+        aa = u'mir'                 # `'ascii' codec can't encode ...` -> do the test on ascii
+        _  = aa.__str__();  assert _ == 'mir'
+    else:
+        _  = xx.__str__();  assert _ == 'мир'
+    assert type(_) is txstr
+
+    # for bstr/ustr  __bytes__/__unicode__ return *str, never MyStr
+    # (builtin unicode has no __bytes__/__unicode__)
+    if tx is not unicode:
+        _ = xx.__bytes__();    assert type(_) is bstr; assert _ == 'мир'
+        _ = xx.__unicode__();  assert type(_) is ustr; assert _ == 'мир'
+
+
     # subclass with __str__
     class MyStr(tx):
         def __str__(self): return u'αβγ'
@@ -1864,6 +1906,17 @@ def test_strings_subclasses(tx):
     with raises(TypeError): u(xx)
 
 
+# verify that bstr/ustr has no extra attributes compared to str and UserString.
+# (else e.g. IPython's guarded_eval.py fails when doing `_list_methods(collections.UserString, dir(str)`.
+# XXX gpython-only ?
+@mark.parametrize('tx', (bstr, ustr))
+def _test_strings_no_extra_methods(tx):     # XXX reenable  (str does not have __bytes__)
+    from six.moves import UserString
+    for attr in dir(tx):
+        assert hasattr(str, attr)
+        assert hasattr(UserString, attr)
+
+
 def test_qq():
     # NOTE qq is also tested as part of strconv.quote
 
@@ -2417,20 +2470,24 @@ def test_deepreplace_str():
 
 # verify that what we patched - e.g. bytes.__repr__ - stay unaffected when
 # called outside of bstr/ustr context.
+# NOTE this test is complemented by test_pickle_strings_patched_transparently in golang_str_pickle_test.py
 def test_strings_patched_transparently():
     b_  = xbytes    ("мир");  assert type(b_)  is bytes
     u_  = xunicode  ("мир");  assert type(u_)  is unicode
     ba_ = xbytearray("мир");  assert type(ba_) is bytearray
 
     # standard {repr,str}(bytes|unicode|bytearray) stay unaffected
-    assert repr(b_)  == x32(r"b'\xd0\xbc\xd0\xb8\xd1\x80'",
-                             r"'\xd0\xbc\xd0\xb8\xd1\x80'")
-    assert repr(u_)  == x32(r"'мир'",
-                            r"u'\u043c\u0438\u0440'")
+    assert repr(b_)  == xB32(x32("b'мир'", "'мир'"),
+                             r"b'\xd0\xbc\xd0\xb8\xd1\x80'",
+                              r"'\xd0\xbc\xd0\xb8\xd1\x80'")
+    assert repr(u_)  == xU32(x32("'мир'", "u'мир'"),
+                             r"'мир'",
+                             r"u'\u043c\u0438\u0440'")
     assert repr(ba_) == r"bytearray(b'\xd0\xbc\xd0\xb8\xd1\x80')"
 
-    assert str(b_)   == x32(r"b'\xd0\xbc\xd0\xb8\xd1\x80'",
-                               "\xd0\xbc\xd0\xb8\xd1\x80")
+    assert str(b_)   == xS32("мир",
+                             r"b'\xd0\xbc\xd0\xb8\xd1\x80'",
+                                "\xd0\xbc\xd0\xb8\xd1\x80")
     if six.PY3  or  sys.getdefaultencoding() == 'utf-8': # py3 or gpython/py2
         assert str(u_) == "мир"
     else:
@@ -2438,8 +2495,9 @@ def test_strings_patched_transparently():
         with raises(UnicodeEncodeError): str(u_)  # 'ascii' codec can't encode ...
         assert str(u'abc') == "abc"
 
-    assert str(ba_)  == x32(r"bytearray(b'\xd0\xbc\xd0\xb8\xd1\x80')",
-                                        b'\xd0\xbc\xd0\xb8\xd1\x80')
+    assert str(ba_)  == xS32("мир",
+                             r"bytearray(b'\xd0\xbc\xd0\xb8\xd1\x80')",
+                                         b'\xd0\xbc\xd0\xb8\xd1\x80')
 
     # unicode comparison stay unaffected
     assert (u_ == u_)  is True
@@ -2458,9 +2516,10 @@ def test_strings_patched_transparently():
     assert (u_ >= u2)  is True      ; assert (u2 >= u_)  is False
 
     # bytearray.__init__ stay unaffected
-    with raises(TypeError): bytearray(u'мир')
-    a = bytearray()
-    with raises(TypeError): a.__init__(u'мир')
+    if ustr is not unicode:
+        with raises(TypeError): bytearray(u'мир')
+        a = bytearray()
+        with raises(TypeError): a.__init__(u'мир')
 
     def _(*argv):
         a = bytearray(*argv)
@@ -2530,9 +2589,29 @@ def bench_bencode(b):
 
 # xbytes/xunicode/xbytearray convert provided bytes/unicode object to bytes,
 # unicode or bytearray correspondingly to function name.
-def xbytes(x):     return x.encode('utf-8') if type(x) is unicode else x
-def xunicode(x):   return x.decode('utf-8') if type(x) is bytes   else x
-def xbytearray(x): return bytearray(xbytes(x))
+def xbytes(x):
+    assert isinstance(x, (bytes,unicode))
+    if isinstance(x, unicode):
+        x = x.encode('utf-8')
+    assert isinstance(x, bytes)
+    x = _bdata(x)
+    assert type(x) is bytes
+    return x
+
+def xunicode(x):
+    assert isinstance(x, (bytes,unicode))
+    if isinstance(x, bytes):
+        x = x.decode('utf-8')
+    assert isinstance(x, unicode)
+    x = _udata(x)
+    assert type(x) is unicode
+    return x
+
+def xbytearray(x):
+    assert isinstance(x, (bytes,unicode))
+    x = bytearray(xbytes(x))
+    assert type(x) is bytearray
+    return x
 
 # deepReplaceStr2Bytearray replaces str to bytearray, or hashable-version of
 # bytearray, if str objects are detected to be present inside set or dict keys.
@@ -2625,3 +2704,29 @@ class hlist(list):
 # x32(a,b) returns a on py3, or b on py2
 def x32(a, b):
     return a if six.PY3 else b
+
+# xb32(x, y, z) returns x if (bstr is not bytes)    or  x32(y,z)
+# xu32(x, y, z) returns x if (ustr is not unicode)  or  x32(y,z)
+def xb32(x, y, z):
+    return x if (bstr is not bytes)   else x32(y,z)
+def xu32(x, y, z):
+    return x if (ustr is not unicode) else x32(y,z)
+
+# xB32(x, y, z) returns x if (bstr is     bytes)    or  x32(y,z)
+# xU32(x, y, z) returns x if (ustr is     unicode)  or  x32(y,z)
+# xS32(x, y, z) returns x if (str  is bstr|ustr)    or  x32(y,z)
+# XXX replace usage of xB32 to directly via xB ?
+def xB32(x, y, z): return xB(x, x32(y,z))
+def xU32(x, y, z): return xU(x, x32(y,z))
+def xS32(x, y, z): return xS(x, x32(y,z))
+
+
+# xB(x, y) returns x if (bstr is     bytes)    or  y
+# xU(x, y) returns x if (ustr is     unicode)  or  y
+# xS(x, y) returns x if (str  is bstr|ustr)    or  y
+def xB(x, y):
+    return x if (bstr is     bytes)   else y
+def xU(x, y):
+    return x if (ustr is     unicode) else y
+def xS(x, y):
+    return x if (str is bstr  or  str is ustr) else y

golang/libgolang.h

@@ -169,6 +169,8 @@
 // [1] Libtask: a Coroutine Library for C and Unix. https://swtch.com/libtask.
 // [2] http://9p.io/magic/man2html/2/thread.
 
+#include "golang/runtime/platform.h"
+
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -177,21 +179,18 @@
 #include <sys/stat.h>
 
 #include <fcntl.h>
-#ifdef _MSC_VER // no mode_t on msvc
+#ifdef LIBGOLANG_CC_msc // no mode_t on msvc
 typedef int mode_t;
 #endif
 
 
 // DSO symbols visibility (based on https://gcc.gnu.org/wiki/Visibility)
-#if defined _WIN32 || defined __CYGWIN__
+#ifdef LIBGOLANG_OS_windows
   #define LIBGOLANG_DSO_EXPORT __declspec(dllexport)
   #define LIBGOLANG_DSO_IMPORT __declspec(dllimport)
-#elif __GNUC__ >= 4
+#else
   #define LIBGOLANG_DSO_EXPORT __attribute__ ((visibility ("default")))
   #define LIBGOLANG_DSO_IMPORT __attribute__ ((visibility ("default")))
-#else
-  #define LIBGOLANG_DSO_EXPORT
-  #define LIBGOLANG_DSO_IMPORT
 #endif
 
 #if BUILDING_LIBGOLANG

golang/os.cpp

@@ -38,7 +38,7 @@
 // cut this short
 // (on darwing sys_siglist declaration is normally provided)
 // (on windows sys_siglist is not available at all)
-#if !(defined(__APPLE__) || defined(_WIN32))
+#if !(defined(LIBGOLANG_OS_darwin) || defined(LIBGOLANG_OS_windows))
 extern "C" {
     extern const char * const sys_siglist[];
 }
@@ -287,7 +287,7 @@ string Signal::String() const {
     const Signal& sig = *this;
     const char *sigstr = nil;
 
-#ifdef _WIN32
+#ifdef LIBGOLANG_OS_windows
     switch (sig.signo) {
     case SIGABRT:   return "Aborted";
     case SIGBREAK:  return "Break";

golang/os.h

@@ -96,7 +96,7 @@ private:
 // Open opens file @path.
 LIBGOLANG_API std::tuple<File, error> Open(const string &path, int flags = O_RDONLY,
         mode_t mode =
-#if !defined(_MSC_VER)
+#if !defined(LIBGOLANG_CC_msc)
                       S_IRUSR | S_IWUSR | S_IXUSR |
                       S_IRGRP | S_IWGRP | S_IXGRP |
                       S_IROTH | S_IWOTH | S_IXOTH

golang/os/signal.cpp

@@ -89,7 +89,7 @@
 #include <atomic>
 #include <tuple>
 
-#if defined(_WIN32)
+#if defined(LIBGOLANG_OS_windows)
 # include <windows.h>
 #endif
 
@@ -101,7 +101,7 @@
 #  define debugf(format, ...) do {} while (0)
 #endif
 
-#if defined(_MSC_VER)
+#ifdef LIBGOLANG_CC_msc
 # define HAVE_SIGACTION 0
 #else
 # define HAVE_SIGACTION 1
@@ -194,7 +194,7 @@ void _init() {
     if (err != nil)
         panic("os::newFile(_wakerx");
     _waketx = vfd[1];
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
     if (sys::Fcntl(_waketx, F_SETFL, O_NONBLOCK) < 0)
         panic("fcntl(_waketx, O_NONBLOCK)");    // TODO +syserr
 #else

golang/pyx/build.py

@@ -35,7 +35,7 @@ from __future__ import print_function, absolute_import
 # pygolang uses setuptools_dso.DSO to build libgolang; all extensions link to it.
 import setuptools_dso
 
-import sys, pkgutil, platform, sysconfig
+import os, sys, pkgutil, platform, sysconfig
 from os.path import dirname, join, exists
 from distutils.errors import DistutilsError
 
@@ -68,7 +68,7 @@ def _findpkg(pkgname):  # -> _PyPkg
 
 # build_ext amends setuptools_dso.build_ext to allow combining C and C++
 # sources in one extension without hitting `error: invalid argument
-# '-std=c++11' not allowed with 'C'`.
+# '-std=c++11' not allowed with 'C'`. XXX + asm
 _dso_build_ext = setuptools_dso.build_ext
 class build_ext(_dso_build_ext):
     def build_extension(self, ext):
@@ -108,12 +108,33 @@ class build_ext(_dso_build_ext):
         # do per-source adjustsment only in .spawn .
         spawn = self.compiler.spawn
         def xspawn(argv):
+            argv = argv[:]
+
             c = False
-            for arg in argv:
+            S = False
+            for i,arg in enumerate(argv):
                 if arg.startswith('/Tc'):
-                    c = True
-            if c:
-                argv = argv[:]
+                    if arg.endswith('.S'):
+                        argv[i] = arg[3:]   # /Tcabc.S -> abc.S
+                        S = True
+                    else:
+                        c = True
+
+            # change cl.exe -> clang-cl.exe for assembly files so that assembler dialect is the same everywhere
+            if S:
+                assert argv[0] == self.compiler.cc, (argv, self.compiler.cc)
+                argv[0] = self.compiler.clang_cl
+
+                # clang-cl fails on *.S if also given /EH... -> remove /EH...
+                while 1:
+                    for i in range(len(argv)):
+                        if argv[i].startswith('/EH'):
+                            del argv[i]
+                            break
+                    else:
+                        break
+
+            if c or S:
                 for i in range(len(argv)):
                     if argv[i] == '/std:c++20':
                         argv[i] = '/std:c11'
@@ -128,6 +149,22 @@ class build_ext(_dso_build_ext):
             self.compiler._compile = _compile
             self.compiler.spawn    = spawn
 
+    def build_extensions(self):
+        # adjust .compiler to support assembly sources
+        cc = self.compiler
+        if '.S' not in cc.src_extensions:
+            cc.src_extensions.append('.S')
+            cc.language_map['.S'] = 'asm'
+            cc.language_order.append('asm')
+            # XXX refer to https://blog.mozilla.org/nfroyd/2019/04/25/an-unexpected-benefit-of-standardizing-on-clang-cl/
+            if cc.compiler_type == 'msvc':
+                if not cc.initialized:
+                    cc.initialize()
+                ccmod = sys.modules[cc.__module__]
+                cc.clang_cl = ccmod._find_exe('clang-cl.exe', cc._paths.split(os.pathsep))
+                cc._c_extensions.append('.S')   # MSVCCompiler thinks it is C, but xspawn handles .S specially
+        _dso_build_ext.build_extensions(self)
+
 
 # setup should be used instead of setuptools.setup
 def setup(**kw):
@@ -176,8 +213,8 @@ def _with_build_defaults(name, kw):   # -> (pygo, kw')
     incv.insert(1, join(pygo, 'golang', '_compat', sysname))
     kw['include_dirs'] = incv
 
-    # link with libgolang.so  if it is not libgolang itself
-    if name != 'golang.runtime.libgolang':
+    # link with libgolang.so  if it is not libgolang itself, or another internal DSO
+    if name not in ('golang.runtime.libgolang', 'golang.runtime.funchook'):
         dsov = kw.get('dsos', [])[:]
         dsov.insert(0, 'golang.runtime.libgolang')
         kw['dsos'] = dsov
@@ -212,9 +249,11 @@ def _with_build_defaults(name, kw):   # -> (pygo, kw')
     dependv = kw.get('depends', [])[:]
     dependv.extend(['%s/golang/%s' % (pygo, _) for _ in [
         'libgolang.h',
+        'runtime.h',
         'runtime/internal.h',
         'runtime/internal/atomic.h',
         'runtime/internal/syscall.h',
+        'runtime/platform.h',
         'context.h',
         'cxx.h',
         'errors.h',

golang/runtime.cpp

+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package runtime mirrors Go package runtime.
+// See runtime.h for package overview.
+
+#include "golang/runtime.h"
+
+
+// golang::runtime::
+namespace golang {
+namespace runtime {
+
+const string ARCH =
+#ifdef LIBGOLANG_ARCH_386
+    "386"
+#elif defined(LIBGOLANG_ARCH_amd64)
+    "amd64"
+#elif defined(LIBGOLANG_ARCH_arm64)
+    "arm64"
+#else
+# error
+#endif
+    ;
+
+
+const string OS =
+#ifdef LIBGOLANG_OS_linux
+    "linux"
+#elif defined(LIBGOLANG_OS_darwin)
+    "darwin"
+#elif defined(LIBGOLANG_OS_windows)
+    "windows"
+#else
+# error
+#endif
+    ;
+
+
+const string CC =
+#ifdef LIBGOLANG_CC_gcc
+    "gcc"
+#elif defined(LIBGOLANG_CC_clang)
+    "clang"
+#elif defined(LIBGOLANG_CC_msc)
+    "msc"
+#else
+# error
+#endif
+    ;
+
+
+}}  // golang::runtime::

golang/runtime.h

+#ifndef _NXD_LIBGOLANG_RUNTIME_H
+#define _NXD_LIBGOLANG_RUNTIME_H
+
+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package runtime mirrors Go package runtime.
+
+#include "golang/libgolang.h"
+
+
+// golang::runtime::
+namespace golang {
+namespace runtime {
+
+// ARCH indicates processor architecture, that is running the program.
+//
+// e.g. "386", "amd64", "arm64", ...
+extern LIBGOLANG_API const string ARCH;
+
+// OS indicates operating system, that is running the program.
+//
+// e.g. "linux", "darwin", "windows", ...
+extern LIBGOLANG_API const string OS;
+
+// CC indicates C/C++ compiler, that compiled the program.
+//
+// e.g. "gcc", "clang", "msc", ...
+extern LIBGOLANG_API const string CC;
+
+
+}} // golang::runtime::
+
+#endif  // _NXD_LIBGOLANG_RUNTIME_H

golang/runtime/internal/atomic.cpp

@@ -20,7 +20,7 @@
 #include "golang/runtime/internal/atomic.h"
 #include "golang/libgolang.h"
 
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
 #include <pthread.h>
 #endif
 
@@ -44,7 +44,7 @@ static void _forkNewEpoch() {
 
 void _init() {
 // there is no fork on windows
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
     int e = pthread_atfork(/*prepare*/nil, /*inparent*/nil, /*inchild*/_forkNewEpoch);
     if (e != 0)
         panic("pthread_atfork failed");

golang/runtime/internal/syscall.cpp

@@ -58,9 +58,9 @@ string _Errno::Error() {
 
     char ebuf[128];
     bool ok;
-#if __APPLE__
+#ifdef LIBGOLANG_OS_darwin
     ok = (::strerror_r(-e.syserr, ebuf, sizeof(ebuf)) == 0);
-#elif defined(_WIN32)
+#elif defined(LIBGOLANG_OS_windows)
     ok = (::strerror_s(ebuf, sizeof(ebuf), -e.syserr) == 0);
 #else
     char *estr = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
@@ -102,7 +102,7 @@ __Errno Close(int fd) {
     return err;
 }
 
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
 __Errno Fcntl(int fd, int cmd, int arg) {
     int save_errno = errno;
     int err = ::fcntl(fd, cmd, arg);
@@ -124,7 +124,7 @@ __Errno Fstat(int fd, struct ::stat *out_st) {
 
 int Open(const char *path, int flags, mode_t mode) {
     int save_errno = errno;
-#ifdef _WIN32  // default to open files in binary mode
+#ifdef LIBGOLANG_OS_windows  // default to open files in binary mode
     if ((flags & (_O_TEXT | _O_BINARY)) == 0)
         flags |= _O_BINARY;
 #endif
@@ -141,9 +141,9 @@ __Errno Pipe(int vfd[2], int flags) {
         return -EINVAL;
     int save_errno = errno;
     int err;
-#ifdef __linux__
+#ifdef LIBGOLANG_OS_linux
     err = ::pipe2(vfd, flags);
-#elif defined(_WIN32)
+#elif defined(LIBGOLANG_OS_windows)
     err = ::_pipe(vfd, 4096, flags | _O_BINARY);
 #else
     err = ::pipe(vfd);
@@ -167,7 +167,7 @@ out:
     return err;
 }
 
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
 __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact) {
     int save_errno = errno;
     int err = ::sigaction(signo, act, oldact);

golang/runtime/internal/syscall.h

@@ -63,13 +63,13 @@ LIBGOLANG_API int/*n|err*/ Read(int fd, void *buf, size_t count);
 LIBGOLANG_API int/*n|err*/ Write(int fd, const void *buf, size_t count);
 
 LIBGOLANG_API __Errno Close(int fd);
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
 LIBGOLANG_API __Errno Fcntl(int fd, int cmd, int arg);
 #endif
 LIBGOLANG_API __Errno Fstat(int fd, struct ::stat *out_st);
 LIBGOLANG_API int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
 LIBGOLANG_API __Errno Pipe(int vfd[2], int flags);
-#ifndef _WIN32
+#ifndef LIBGOLANG_OS_windows
 LIBGOLANG_API __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
 #endif
 typedef void (*sighandler_t)(int);

golang/runtime/libgolang.cpp

@@ -52,7 +52,7 @@
 #include <linux/list.h>
 // MSVC does not support statement expressions and typeof
 // -> redo list_entry via C++ lambda.
-#ifdef _MSC_VER
+#ifdef LIBGOLANG_CC_msc
 # undef list_entry
 # define list_entry(ptr, type, member) [&]() {                      \
         const decltype( ((type *)0)->member ) *__mptr = (ptr);      \

golang/runtime/platform.h

+#ifndef _NXD_LIBGOLANG_RUNTIME_PLATFORM_H
+#define _NXD_LIBGOLANG_RUNTIME_PLATFORM_H
+
+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Header platform.h provides preprocessor defines that describe target platform.
+
+// LIBGOLANG_ARCH_<X> is defined on architecture X.
+//
+// List of supported architectures: 386, amd64, arm64.
+#if defined(__i386__) || defined(_M_IX86)
+# define LIBGOLANG_ARCH_386     1
+#elif defined(__x86_64__) || defined(_M_X64)
+# define LIBGOLANG_ARCH_amd64   1
+#elif defined(__aarch64__) || defined(_M_ARM64)
+# define LIBGOLANG_ARCH_arm64   1
+#else
+# error "unsupported architecture"
+#endif
+
+// LIBGOLANG_OS_<X> is defined on operating system X.
+//
+// List of supported operating systems: linux, darwin, windows.
+#ifdef __linux__
+# define LIBGOLANG_OS_linux     1
+#elif defined(__APPLE__)
+# define LIBGOLANG_OS_darwin    1
+#elif defined(_WIN32) || defined(__CYGWIN__)
+# define LIBGOLANG_OS_windows   1
+#else
+# error "unsupported operating system"
+#endif
+
+// LIBGOLANG_CC_<X> is defined on C/C++ compiler X.
+//
+// List of supported compilers: gcc, clang, msc.
+#ifdef __clang__
+# define LIBGOLANG_CC_clang     1
+#elif defined(_MSC_VER)
+# define LIBGOLANG_CC_msc       1
+// NOTE gcc comes last because e.g. clang and icc define __GNUC__ as well
+#elif __GNUC__
+# define LIBGOLANG_CC_gcc       1
+#else
+# error "unsupported compiler"
+#endif
+
+#endif  // _NXD_LIBGOLANG_RUNTIME_PLATFORM_H

gpython/.gitignore

+/_gpython.cpp

gpython/__init__.py

@@ -25,10 +25,14 @@ differences:
 
 - gevent is pre-activated and stdlib is patched to be gevent aware;
 - go, chan, select etc are put into builtin namespace;
-- default string encoding is always set to UTF-8.
+- default string encoding is always set to UTF-8;
+- bstr/ustr replace builtin str/unicode types.
 
 Gevent activation can be disabled via `-X gpython.runtime=threads`, or
 $GPYTHON_RUNTIME=threads.
+
+String types replacement can be disabled via `-X gpython.strings=pystd`, or
+$GPYTHON_STRINGS=pystd.
 """
 
 # NOTE gpython is kept out of golang/ , since even just importing e.g. golang.cmd.gpython,
@@ -230,9 +234,13 @@ def pymain(argv, init=None):
             gevent = sys.modules.get('gevent', None)
             gpyver = 'GPython %s' % golang.__version__
             if gevent is not None:
-                gpyver += ' [gevent %s]' % gevent.__version__
+                gpyver += ' [runtime gevent %s]' % gevent.__version__
+            else:
+                gpyver += ' [runtime threads]'
+            if type(u'') is golang.ustr:
+                gpyver += ' [strings bstr+ustr]'
             else:
-                gpyver += ' [threads]'
+                gpyver += ' [strings pystd]'
             ver.append(gpyver)
 
         import platform
@@ -344,6 +352,9 @@ def main():
     # imported first, e.g. to support sys.modules.
     import sys
 
+    # import pyx/c part of gpython
+    from gpython import _gpython
+
     # safety check that we are not running from a setuptools entrypoint, where
     # it would be too late to monkey-patch stdlib.
     #
@@ -372,6 +383,7 @@ def main():
         reload(sys)
         sys.setdefaultencoding('utf-8')
         delattr(sys, 'setdefaultencoding')
+        _gpython.set_utf8_as_default_src_encoding()
 
 
     # import os to get access to environment.
@@ -381,10 +393,12 @@ def main():
     import os
 
     # extract and process `-X gpython.*`
-    # -X gpython.runtime=(gevent|threads)    + $GPYTHON_RUNTIME
+    # -X gpython.runtime=(gevent|threads)       + $GPYTHON_RUNTIME
+    # -X gpython.strings=(bstr+ustr|pystd)      + $GPYTHON_STRINGS
     sys._xoptions = getattr(sys, '_xoptions', {})
     argv_ = []
     gpy_runtime = os.getenv('GPYTHON_RUNTIME', 'gevent')
+    gpy_strings = os.getenv('GPYTHON_STRINGS', 'bstr+ustr')
     igetopt = _IGetOpt(sys.argv[1:], _pyopt, _pyopt_long)
     for (opt, arg) in igetopt:
         if opt == '-X':
@@ -393,6 +407,10 @@ def main():
                     gpy_runtime = arg[len('gpython.runtime='):]
                     sys._xoptions['gpython.runtime'] = gpy_runtime
 
+                elif arg.startswith('gpython.strings='):
+                    gpy_strings = arg[len('gpython.strings='):]
+                    sys._xoptions['gpython.strings'] = gpy_strings
+
                 else:
                     raise RuntimeError('gpython: unknown -X option %s' % arg)
 
@@ -412,13 +430,15 @@ def main():
     # sys.executable spawned from under `gpython -X gpython.runtime=threads`
     # also uses "threads" runtime by default.
     os.environ['GPYTHON_RUNTIME'] = gpy_runtime
+    os.environ['GPYTHON_STRINGS'] = gpy_strings
 
-    # init initializes according to selected runtime
+    # init initializes according to selected runtime and strings
     # it is called after options are parsed and sys.path is setup correspondingly.
     # this way golang and gevent are imported from exactly the same place as
     # they would be in standard python after regular import (ex from golang/
     # under cwd if run under `python -c ...` or interactive console.
     def init():
+        gpy_runtime_ver = gpy_runtime
         if gpy_runtime == 'gevent':
             # make gevent pre-available & stdlib patched
             import gevent
@@ -434,22 +454,30 @@ def main():
             if _ not in (True, None):   # patched or nothing to do
                 # XXX provide details
                 raise RuntimeError('gevent monkey-patching failed')
-            gpy_verextra = 'gevent %s' % gevent.__version__
+            gpy_runtime_ver += ' %s' % gevent.__version__
 
         elif gpy_runtime == 'threads':
-            gpy_verextra = 'threads'
-
+            pass
         else:
-            raise RuntimeError('gpython: invalid runtime %s' % gpy_runtime)
+            raise RuntimeError('gpython: invalid runtime %r' % gpy_runtime)
 
-        # put go, chan, select, ... into builtin namespace
+        if gpy_strings not in ('bstr+ustr', 'pystd'):
+            raise RuntimeError('gpython: invalid strings %r' % gpy_strings)
+
+        # import golang
+        # this will activate selected runtime and strings
+        sys._gpy_runtime = gpy_runtime
+        sys._gpy_strings = gpy_strings
         import golang
+
+        # put go, chan, select, ... into builtin namespace
         from six.moves import builtins
         for k in golang.__all__:
             setattr(builtins, k, getattr(golang, k))
+        setattr(builtins, 'CCC', CCC)
 
         # sys.version
-        sys.version += (' [GPython %s] [%s]' % (golang.__version__, gpy_verextra))
+        sys.version += (' [GPython %s] [runtime %s] [strings %s]' % (golang.__version__, gpy_runtime_ver, gpy_strings))
 
     # tail to pymain
     pymain(argv, init)
@@ -567,5 +595,11 @@ class _IGetOpt:
     next = __next__ # for py2
 
 
+# for tests XXX continue by first writing test  XXX
+1/0
+class _tEarlyStrSubclass(str):
+    pass
+
+
 if __name__ == '__main__':
     main()

gpython/_gpython.pyx

+# -*- coding: utf-8 -*-
+# cython: language_level=2
+# Copyright (C) 2023  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""_gpython.pyx ... XXX
+"""
+
+cdef extern from *:
+    """
+    void _set_utf8_as_default_src_encoding();
+    """
+    void _set_utf8_as_default_src_encoding() except *
+
+def set_utf8_as_default_src_encoding():
+    _set_utf8_as_default_src_encoding()

gpython/_gpython_c.cpp

+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// XXX doctitle
+
+#include <Python.h>
+#if PY_MAJOR_VERSION < 3
+#include <Python-ast.h> // mod_ty & co
+#include <node.h>       // node
+#include <graminit.h>   // encoding_decl & co
+#include <ast.h>        // PyAST_FromNode & co
+#endif
+
+#include <funchook.h>
+
+// py2: wrap PyAST_FromNode so that "utf-8" becomes the default encoding
+#if PY_MAJOR_VERSION < 3
+static auto   _py_PyAST_FromNode = &PyAST_FromNode;
+static mod_ty gpy_PyAST_FromNode(const node* n, PyCompilerFlags* flags,
+                                 const char* filename, PyArena* arena)
+{
+//  fprintf(stderr, "gpy_PyAST_FromNode...\n");
+    PyCompilerFlags gflags = {.cf_flags = 0};
+    if (flags)
+        gflags = *flags;
+    if (TYPE(n) != encoding_decl)
+        gflags.cf_flags |= PyCF_SOURCE_IS_UTF8;
+    return _py_PyAST_FromNode(n, &gflags, filename, arena);
+}
+
+static funchook_t* gpy_PyAST_FromNode_hook;
+void _set_utf8_as_default_src_encoding() {
+    funchook_t *h;
+    int err;
+
+//  funchook_set_debug_file("/dev/stderr");
+
+    gpy_PyAST_FromNode_hook = h = funchook_create();
+    if (h == NULL) {
+        PyErr_NoMemory();
+        return;
+    }
+
+    err = funchook_prepare(h, (void**)&_py_PyAST_FromNode, (void*)gpy_PyAST_FromNode);
+    if (err != 0) {
+        PyErr_SetString(PyExc_RuntimeError, funchook_error_message(h));
+        return;
+    }
+
+    err = funchook_install(h, 0);
+    if (err != 0) {
+        PyErr_SetString(PyExc_RuntimeError, funchook_error_message(h));
+        return;
+    }
+
+    // ok
+}
+#else
+void _set_utf8_as_default_src_encoding() {}
+#endif

gpython/gpython_test.py

@@ -47,20 +47,34 @@ gpython_only = pytest.mark.skipif(not is_gpython, reason="gpython-only test")
 def runtime(request):
     yield request.param
 
+# strings is pytest fixture that yields all variants of should be supported gpython strings:
+# '' - not specified (gpython should autoselect)
+# 'bstr+ustr'
+# 'pystd'
+@pytest.fixture(scope="function", params=['', 'bstr+ustr', 'pystd'])
+def strings(request):
+    yield request.param
+
 # gpyenv returns environment appropriate for spawning gpython with
-# specified runtime.
-def gpyenv(runtime): # -> env
+# specified runtime and strings.
+def gpyenv(runtime, strings): # -> env
     env = os.environ.copy()
     if runtime != '':
         env['GPYTHON_RUNTIME'] = runtime
     else:
         env.pop('GPYTHON_RUNTIME', None)
+    if strings != '':
+        env['GPYTHON_STRINGS'] = strings
+    else:
+        env.pop('GPYTHON_STRINGS', None)
     return env
 
 
 @gpython_only
 def test_defaultencoding_utf8():
     assert sys.getdefaultencoding() == 'utf-8'
+    assert eval("u'αβγ'") == u'αβγ'     # FIXME fails on py2 which uses hardcoded default latin1
+    # XXX +exec, +run file
 
 @gpython_only
 def test_golang_builtins():
@@ -143,19 +157,42 @@ def assert_gevent_not_activated():
 
 
 @gpython_only
-def test_executable(runtime):
+def test_str_patched():
+    # gpython, by default, patches str/unicode to be bstr/ustr.
+    # handling of various string modes is explicitly tested in test_Xstrings.
+    assert_str_patched()
+
+def assert_str_patched():
+    #assert str.__name__ == ('bstr'  if PY2 else  'ustr')
+    assert str.__name__ == 'str'
+    assert str          is (bstr    if PY2 else  ustr)
+    if PY2:
+        assert unicode.__name__ == 'unicode'
+        assert unicode  is ustr
+    assert type('')     is str
+    assert type(b'')    is (bstr    if PY2 else  bytes)
+    assert type(u'')    is ustr
+
+def assert_str_not_patched():
+    assert str.__name__ == 'str'
+    assert str is not bstr
+    assert str is not ustr
+    if PY2:
+        assert unicode.__name__ == 'unicode'
+        assert unicode is not bstr
+        assert unicode is not ustr
+    assert type('')     is str
+    assert type(b'')    is bytes
+    assert type(u'')    is (unicode if PY2 else str)
+
+
+@gpython_only
+def test_executable():
     # sys.executable must point to gpython and we must be able to execute it.
-    import gevent
     assert 'gpython' in sys.executable
-    ver = pyout(['-c', 'import sys; print(sys.version)'], env=gpyenv(runtime))
+    ver = pyout(['-c', 'import sys; print(sys.version)'], env=gpyenv('', ''))
     ver = str(ver)
     assert ('[GPython %s]' % golang.__version__) in ver
-    if runtime != 'threads':
-        assert ('[gevent %s]'  % gevent.__version__)     in ver
-        assert ('[threads]')                         not in ver
-    else:
-        assert ('[gevent ')                          not in ver
-        assert ('[threads]')                             in ver
 
 
 # verify pymain.
@@ -322,15 +359,20 @@ def test_pymain_opt():
 # pymain -V/--version
 # gpython_only because output differs from !gpython.
 @gpython_only
-def test_pymain_ver(runtime):
+def test_pymain_ver(runtime, strings):
     from golang import b
     from gpython import _version_info_str as V
     import gevent
     vok = 'GPython %s' % golang.__version__
     if runtime != 'threads':
-        vok += ' [gevent %s]' % gevent.__version__
+        vok += ' [runtime gevent %s]' % gevent.__version__
     else:
-        vok += ' [threads]'
+        vok += ' [runtime threads]'
+
+    if strings != 'pystd':
+        vok += ' [strings bstr+ustr]'
+    else:
+        vok += ' [strings pystd]'
 
     if is_cpython:
         vok += ' / CPython %s' % platform.python_version()
@@ -341,10 +383,12 @@ def test_pymain_ver(runtime):
 
     vok += '\n'
 
-    ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
+    env = gpyenv(runtime, strings)
+
+    ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE, env=env)
     assert (ret, out, b(err)) == (0, b'', b(vok))
 
-    ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
+    ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE, env=env)
     assert (ret, out, b(err)) == (0, b'', b(vok))
 
 # verify that ./bin/gpython runs ok.

pyproject.toml

 [build-system]
-requires = ["setuptools", "wheel", "setuptools_dso >= 2.7", "cython", "gevent"]
+requires = ["setuptools", "wheel", "setuptools_dso >= 2.7", "cython < 3", "gevent"]

setup.py

@@ -42,9 +42,9 @@ from setuptools.command.install_scripts import install_scripts as _install_scrip
 from setuptools.command.develop import develop as _develop
 from distutils import sysconfig
 from os.path import dirname, join
-import sys, os, re
+import sys, os, re, platform, errno
 
-# read file content
+# read/write file content
 def readfile(path): # -> str
     with open(path, 'rb') as f:
         data = f.read()
@@ -52,6 +52,20 @@ def readfile(path): # -> str
             data = data.decode('utf-8')
         return data
 
+def writefile(path, data):
+    if not isinstance(data, bytes):
+        data = data.encode('utf-8')
+    with open(path, 'wb') as f:
+        f.write(data)
+
+# mkdir -p
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+
 # reuse golang.pyx.build to build pygolang dso and extensions.
 # we have to be careful and inject synthetic golang package in order to be
 # able to import golang.pyx.build without built/working golang.
@@ -59,6 +73,7 @@ trun = {}
 exec(readfile('trun'), trun)
 trun['ximport_empty_golangmod']()
 from golang.pyx.build import setup, DSO, Extension as Ext
+from setuptools_dso import ProbeToolchain
 
 
 # grep searches text for pattern.
@@ -104,7 +119,7 @@ class XInstallGPython:
     # (script_name, script) -> (script_name, script)
     def transform_script(self, script_name, script):
         # on windows setuptools installs 3 files:
-        #   gpython-script.py
+        #   gpython-script.py           XXX do we need to adjust this similarly to pymain?
         #   gpython.exe
         #   gpython.exe.manifest
         # we want to override .py only.
@@ -173,8 +188,8 @@ class develop(XInstallGPython, _develop):
 
 # requirements of packages under "golang." namespace
 R = {
-    'cmd.pybench':      {'pytest'},
-    'pyx.build':        {'setuptools', 'wheel', 'cython', 'setuptools_dso >= 2.7'},
+    'cmd.pybench':      {'pytest', 'py'},
+    'pyx.build':        {'setuptools', 'wheel', 'cython < 3', 'setuptools_dso >= 2.7'},
     'x.perf.benchlib':  {'numpy'},
 }
 # TODO generate `a.b -> a`, e.g. x.perf = join(x.perf.*); x = join(x.*)
@@ -184,7 +199,8 @@ for pkg in R:
 R['all'] = Rall
 
 # ipython/pytest are required to test py2 integration patches
-R['all_test'] = Rall.union(['ipython', 'pytest']) # pip does not like "+" in all+test
+# zodbpickle is used to test pickle support for bstr/ustr
+R['all_test'] = Rall.union(['ipython', 'pytest', 'zodbpickle']) # pip does not like "+" in all+test
 
 # extras_require <- R
 extras_require = {}
@@ -200,6 +216,206 @@ def get_python_libdir():
     else:
         return sysconfig.get_config_var('LIBDIR')
 
+# funchook_dso is DSO for libfunchook.so or None if CPU is not supported.
+def _():
+    cpu = platform.machine()
+    if re.match('x86|i.86|x86_64|amd64', cpu, re.I):
+        cpu = 'x86'
+        disasm = 'distorm'
+    elif re.match('aarch64|arm64', cpu, re.I):
+        cpu = 'arm64'
+        disasm = 'capstone'
+    else:
+        return None # no funchook support
+
+    # XXX temp test XXX no -> we need capstone for disasm
+    disasm = 'capstone'
+
+    if platform.system() == 'Windows':
+        os   = 'windows'
+        libv = ['psapi']
+    else:
+        os   = 'unix'
+        libv = ['dl']
+
+    FH = '3rdparty/funchook/'
+    srcv = [FH+'src/funchook.c',
+            FH+'src/funchook_%s.c' % cpu,
+            FH+'src/funchook_%s.c' % os,
+            FH+'src/disasm_%s.c'   % disasm]
+    depv = [FH+'include/funchook.h',
+            FH+'src/disasm.h',
+            FH+'src/funchook_arm64.h',
+            FH+'src/funchook_internal.h',
+            FH+'src/funchook_x86.h']
+    incv = [FH+'include']
+    defv = ['FUNCHOOK_EXPORTS']
+
+    if disasm == 'distorm':
+        D3 = '3rdparty/funchook/distorm/'
+        srcv += [D3+'src/decoder.c',
+                 D3+'src/distorm.c',
+                 D3+'src/instructions.c',
+                 D3+'src/insts.c',
+                 D3+'src/mnemonics.c',
+                 D3+'src/operands.c',
+                 D3+'src/prefix.c',
+                 D3+'src/textdefs.c']
+        depv += [D3+'include/distorm.h',
+                 D3+'include/mnemonics.h',
+                 D3+'src/config.h',
+                 D3+'src/decoder.h',
+                 D3+'src/instructions.h',
+                 D3+'src/insts.h',
+                 D3+'src/operands.h',
+                 D3+'src/prefix.h',
+                 D3+'src/textdefs.h',
+                 D3+'src/wstring.h',
+                 D3+'src/x86defs.h']
+        incv += [D3+'include']
+
+    if disasm == 'capstone':
+        CS = '3rdparty/capstone/'
+        srcv += [CS+'cs.c',
+                 CS+'Mapping.c',
+                 CS+'MCInst.c',
+                 CS+'MCInstrDesc.c',
+                 CS+'MCRegisterInfo.c',
+                 CS+'SStream.c',
+                 CS+'utils.c']
+        depv += [CS+'cs_simple_types.h',
+                 CS+'cs_priv.h',
+                 CS+'LEB128.h',
+                 CS+'Mapping.h',
+                 CS+'MathExtras.h',
+                 CS+'MCDisassembler.h',
+                 CS+'MCFixedLenDisassembler.h',
+                 CS+'MCInst.h',
+                 CS+'MCInstrDesc.h',
+                 CS+'MCRegisterInfo.h',
+                 CS+'SStream.h',
+                 CS+'utils.h']
+        incv += [CS+'include']
+
+        depv += [CS+'include/capstone/arm64.h',
+                 CS+'include/capstone/arm.h',
+                 CS+'include/capstone/capstone.h',
+                 CS+'include/capstone/evm.h',
+                 CS+'include/capstone/wasm.h',
+                 CS+'include/capstone/mips.h',
+                 CS+'include/capstone/ppc.h',
+                 CS+'include/capstone/x86.h',
+                 CS+'include/capstone/sparc.h',
+                 CS+'include/capstone/systemz.h',
+                 CS+'include/capstone/xcore.h',
+                 CS+'include/capstone/m68k.h',
+                 CS+'include/capstone/tms320c64x.h',
+                 CS+'include/capstone/m680x.h',
+                 CS+'include/capstone/mos65xx.h',
+                 CS+'include/capstone/bpf.h',
+                 CS+'include/capstone/riscv.h',
+                 CS+'include/capstone/sh.h',
+                 CS+'include/capstone/tricore.h',
+                 CS+'include/capstone/platform.h']
+
+        defv += ['CAPSTONE_SHARED', 'CAPSTONE_USE_SYS_DYN_MEM']
+
+        if cpu == 'arm64':
+            defv += ['CAPSTONE_HAS_ARM64']
+            srcv += [CS+'arch/AArch64/AArch64BaseInfo.c',
+                     CS+'arch/AArch64/AArch64Disassembler.c',
+                     CS+'arch/AArch64/AArch64InstPrinter.c',
+                     CS+'arch/AArch64/AArch64Mapping.c',
+                     CS+'arch/AArch64/AArch64Module.c']
+            depv += [CS+'arch/AArch64/AArch64AddressingModes.h',
+                     CS+'arch/AArch64/AArch64BaseInfo.h',
+                     CS+'arch/AArch64/AArch64Disassembler.h',
+                     CS+'arch/AArch64/AArch64InstPrinter.h',
+                     CS+'arch/AArch64/AArch64Mapping.h',
+                     CS+'arch/AArch64/AArch64GenAsmWriter.inc',
+                     CS+'arch/AArch64/AArch64GenDisassemblerTables.inc',
+                     CS+'arch/AArch64/AArch64GenInstrInfo.inc',
+                     CS+'arch/AArch64/AArch64GenRegisterInfo.inc',
+                     CS+'arch/AArch64/AArch64GenRegisterName.inc',
+                     CS+'arch/AArch64/AArch64GenRegisterV.inc',
+                     CS+'arch/AArch64/AArch64GenSubtargetInfo.inc',
+                     CS+'arch/AArch64/AArch64GenSystemOperands.inc',
+                     CS+'arch/AArch64/AArch64GenSystemOperands_enum.inc',
+                     CS+'arch/AArch64/AArch64MappingInsn.inc',
+                     CS+'arch/AArch64/AArch64MappingInsnName.inc',
+                     CS+'arch/AArch64/AArch64MappingInsnOp.inc']
+
+        if cpu == 'x86':
+            defv += ['CAPSTONE_HAS_X86']
+            srcv += [CS+'arch/X86/X86ATTInstPrinter.c',     # !diet
+                     CS+'arch/X86/X86Disassembler.c',
+                     CS+'arch/X86/X86DisassemblerDecoder.c',
+                     CS+'arch/X86/X86IntelInstPrinter.c',
+                     CS+'arch/X86/X86InstPrinterCommon.c',
+                     CS+'arch/X86/X86Mapping.c',
+                     CS+'arch/X86/X86Module.c']
+            depv += [CS+'arch/X86/X86BaseInfo.h',
+                     CS+'arch/X86/X86Disassembler.h',
+                     CS+'arch/X86/X86DisassemblerDecoder.h',
+                     CS+'arch/X86/X86DisassemblerDecoderCommon.h',
+                     CS+'arch/X86/X86GenAsmWriter.inc',
+                     CS+'arch/X86/X86GenAsmWriter1.inc',
+                     CS+'arch/X86/X86GenAsmWriter1_reduce.inc',
+                     CS+'arch/X86/X86GenAsmWriter_reduce.inc',
+                     CS+'arch/X86/X86GenDisassemblerTables.inc',
+                     CS+'arch/X86/X86GenDisassemblerTables_reduce.inc',
+                     CS+'arch/X86/X86GenInstrInfo.inc',
+                     CS+'arch/X86/X86GenInstrInfo_reduce.inc',
+                     CS+'arch/X86/X86GenRegisterInfo.inc',
+                     CS+'arch/X86/X86InstPrinter.h',
+                     CS+'arch/X86/X86Mapping.h',
+                     CS+'arch/X86/X86MappingInsn.inc',
+                     CS+'arch/X86/X86MappingInsnOp.inc',
+                     CS+'arch/X86/X86MappingInsnOp_reduce.inc',
+                     CS+'arch/X86/X86MappingInsn_reduce.inc']
+
+    # config.h
+    probe = ProbeToolchain()
+    config_h = []
+    def cfgemit(line):
+        config_h.append(line+'\n')
+    def defif(name, ok):
+        if ok:
+            cfgemit('#define %s 1' % name)
+        else:
+            cfgemit('#undef  %s'   % name)
+
+    for d in ('capstone', 'distorm', 'zydis'):
+        defif('DISASM_%s' % d.upper(), d == disasm)
+
+    cfgemit('#define SIZEOF_VOID_P %d' % probe.sizeof('void*'))
+
+    defif('_GNU_SOURCE', 1)
+    defif('GNU_SPECIFIC_STRERROR_R', probe.try_compile("""
+#define _GNU_SOURCE 1
+#include <string.h>
+int main()
+{
+    char dummy[128];
+    return *strerror_r(0, dummy, sizeof(dummy));
+}
+"""))
+
+    fbuild_src = 'build/3rdparty/funchook/src'
+    mkdir_p(fbuild_src)
+    writefile(fbuild_src+'/config.h', ''.join(config_h))
+    incv  += [fbuild_src]
+
+    return DSO('golang.runtime.funchook', srcv,
+               depends         = depv,
+               language        = 'c',
+               include_dirs    = incv,
+               define_macros   = [(_, None) for _ in defv],
+               libraries       = libv,
+               soversion       = '1.1')
+funchook_dso = _()
+
+
 setup(
     name        = 'pygolang',
     version     = version,
@@ -225,6 +441,7 @@ setup(
                         ['golang/runtime/libgolang.cpp',
                          'golang/runtime/internal/atomic.cpp',
                          'golang/runtime/internal/syscall.cpp',
+                         'golang/runtime.cpp',
                          'golang/context.cpp',
                          'golang/errors.cpp',
                          'golang/fmt.cpp',
@@ -236,9 +453,11 @@ setup(
                          'golang/time.cpp'],
                         depends = [
                             'golang/libgolang.h',
+                            'golang/runtime.h',
                             'golang/runtime/internal.h',
                             'golang/runtime/internal/atomic.h',
                             'golang/runtime/internal/syscall.h',
+                            'golang/runtime/platform.h',
                             'golang/context.h',
                             'golang/cxx.h',
                             'golang/errors.h',
@@ -259,12 +478,21 @@ setup(
                         include_dirs    = [sysconfig.get_python_inc()],
                         library_dirs    = [get_python_libdir()],
                         define_macros   = [('BUILDING_LIBPYXRUNTIME', None)],
-                        soversion       = '0.1')],
+                        soversion       = '0.1')]
+                    + ([funchook_dso] if funchook_dso else []),
 
     ext_modules = [
                     Ext('golang._golang',
-                        ['golang/_golang.pyx'],
-                        depends = ['golang/_golang_str.pyx']),
+                        ['golang/_golang.pyx',
+                         'golang/_golang_str_pickle.S'],
+                        depends = [
+                            'golang/_golang_str.pyx',
+                            'golang/_golang_str_pickle.pyx',
+                            'golang/_golang_str_pickle_test.pyx',
+                            'golang/_golang_str_pickle.S'],
+                        dsos = ['golang.runtime.funchook'], # XXX only if available
+                        include_dirs = ['3rdparty/funchook/include',
+                                        '3rdparty/capstone/include']),
 
                     Ext('golang.runtime._runtime_thread',
                         ['golang/runtime/_runtime_thread.pyx']),
@@ -334,6 +562,14 @@ setup(
                     Ext('golang._time',
                         ['golang/_time.pyx'],
                         dsos = ['golang.runtime.libpyxruntime']),
+
+                    # XXX consider putting everything into just gpython.pyx + .c
+                    Ext('gpython._gpython',
+                        ['gpython/_gpython.pyx',
+                         'gpython/_gpython_c.cpp'],    # XXX do we need C++ here?
+                        include_dirs =  ['3rdparty/funchook/include'],
+                        dsos = ['golang.runtime.funchook'], # XXX only if available
+                    ),
                   ],
     include_package_data = True,