Skip to content

pygolang
pygolang
Commit 17dbfbac authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

X My draft state of x/gpystr work; py2/py3 pickle problem should be essentially solved

parent ac751a56
Hide whitespace changes
Inline Side-by-side
Showing with 3858 additions and 304 deletions
.gitmodules 0 → 100644
View file @ 17dbfbac
[submodule "3rdparty/funchook"]
path = 3rdparty/funchook
url = https://github.com/kubo/funchook.git
[submodule "3rdparty/capstone"]
path = 3rdparty/capstone
url = https://github.com/capstone-engine/capstone.git
capstone @ 097c04d9
Subproject commit 097c04d9413c59a58b00d4d1c8d5dc0ac158ffaa
funchook @ 88388db3
Subproject commit 88388db3c69e16c1560fee65c6857d75f5ce6fd5
MANIFEST.in
View file @ 17dbfbac
......@@ -2,6 +2,9 @@ include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .nxdtest
include golang/libgolang.h
include golang/runtime/libgolang.cpp
include golang/runtime/libpyxruntime.cpp
include golang/runtime/platform.h
include golang/runtime.h
include golang/runtime.cpp
include golang/pyx/runtime.h
include golang/pyx/testprog/golang_dso_user/dsouser/dso.h
include golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
......@@ -36,7 +39,10 @@ include golang/time.cpp
include golang/_testing.h
include golang/_compat/windows/strings.h
include golang/_compat/windows/unistd.h
include gpython/_gpython_c.cpp
recursive-include golang *.py *.pxd *.pyx *.toml *.txt*
recursive-include gpython *.py
recursive-include 3rdparty *.h
recursive-include gpython *.py *.pyx
recursive-include 3rdparty *.h *.c *.cpp *.S *.py *.cmake *.cs *.java
recursive-include 3rdparty LICENSE README.md README COPYING Makefile CMakeLists.txt
recursive-exclude golang *_dsoinfo.py
include conftest.py
README.rst
View file @ 17dbfbac
......@@ -4,7 +4,7 @@
Package `golang` provides Go-like features for Python:
- `gpython` is Python interpreter with support for lightweight threads.
- `gpython` is Python interpreter with support for lightweight threads and uniform UTF8-based approach to strings.
- `go` spawns lightweight thread.
- `chan` and `select` provide channels with Go semantic.
- `func` allows to define methods separate from class.
......@@ -46,15 +46,16 @@ __ http://libuv.org/
__ http://software.schmorp.de/pkg/libev.html
Additionally GPython sets UTF-8 to be default encoding always, and puts `go`,
`chan`, `select` etc into builtin namespace.
Additionally GPython sets UTF-8 to be default encoding always, puts `go`,
`chan`, `select` etc into builtin namespace, and makes `bstr`/`ustr` to be used
instead of builtin string types.
.. note::
GPython is optional and the rest of Pygolang can be used from under standard Python too.
However without gevent integration `go` spawns full - not lightweight - OS thread.
GPython can be also used with threads - not gevent - runtime. Please see
`GPython options`_ for details.
GPython can be also used with threads - not gevent - runtime and with builtin string types.
Please see `GPython options`_ for details.
Goroutines and channels
......@@ -571,3 +572,9 @@ GPython-specific options and environment variables are listed below:
coroutines, while with `threads` `go` spawns full OS thread. `gevent` is
default. The runtime to use can be also specified via `$GPYTHON_RUNTIME`
environment variable.
`-X gpython.strings=(bstr+ustr|pystd)`
Specify which string types GPython should use. `bstr+ustr` provide
uniform UTF8-based approach to strings, while `pystd` selects regular
`str` and `unicode`. `bstr+ustr` is default. String types to use can be
also specified via `$GPYTHON_STRINGS` environment variable.
conftest.py 0 → 100644
View file @ 17dbfbac
# ignore tests in distorm - else it breaks as e.g.
#
# 3rdparty/funchook/distorm/python/test_distorm3.py:15: in <module>
# import distorm3
# 3rdparty/funchook/distorm/python/distorm3/__init__.py:57: in <module>
# _distorm = _load_distorm()
# 3rdparty/funchook/distorm/python/distorm3/__init__.py:55: in _load_distorm
# raise ImportError("Error loading the diStorm dynamic library (or cannot load library into process).")
# E ImportError: Error loading the diStorm dynamic library (or cannot load library into process).
collect_ignore = ["3rdparty"]
golang/_golang.pyx
View file @ 17dbfbac
......@@ -3,7 +3,7 @@
# cython: binding=False
# cython: c_string_type=str, c_string_encoding=utf8
# distutils: language = c++
# distutils: depends = libgolang.h os/signal.h unicode/utf8.h _golang_str.pyx
# distutils: depends = libgolang.h os/signal.h unicode/utf8.h _golang_str.pyx _golang_str_pickle.pyx
#
# Copyright (C) 2018-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
......@@ -34,7 +34,7 @@ from __future__ import print_function, absolute_import
_init_libgolang()
_init_libpyxruntime()
from cpython cimport PyObject, Py_INCREF, Py_DECREF, PY_MAJOR_VERSION
from cpython cimport PyObject, Py_INCREF, Py_DECREF, Py_CLEAR, PY_MAJOR_VERSION
ctypedef PyObject *pPyObject # https://github.com/cython/cython/issues/534
cdef extern from "Python.h":
ctypedef struct PyTupleObject:
......
golang/_golang_str.pyx
View file @ 17dbfbac
......@@ -22,6 +22,8 @@
It is included from _golang.pyx .
"""
from libc.stdio cimport fprintf, stderr # XXX kill
from golang.unicode cimport utf8
from cpython cimport PyUnicode_AsUnicode, PyUnicode_GetSize, PyUnicode_FromUnicode
......@@ -31,11 +33,13 @@ from cpython cimport Py_EQ, Py_NE, Py_LT, Py_GT, Py_LE, Py_GE
from cpython.iterobject cimport PySeqIter_New
from cpython cimport PyThreadState_GetDict, PyDict_SetItem
from cpython cimport PyObject_CheckBuffer
from cpython cimport Py_TPFLAGS_HAVE_GC, Py_TPFLAGS_HEAPTYPE, Py_TPFLAGS_READY, PyType_Ready
from cpython cimport PyBytes_Format, PyUnicode_Format, PyObject_Str
cdef extern from "Python.h":
PyTypeObject PyBytes_Type
ctypedef struct PyBytesObject:
pass
char *ob_sval
cdef extern from "Python.h":
PyTypeObject PyUnicode_Type
......@@ -60,13 +64,40 @@ cdef extern from "Python.h":
ctypedef struct _XPyTypeObject "PyTypeObject":
PyObject* tp_new(PyTypeObject*, PyObject*, PyObject*) except NULL
initproc tp_init
Py_ssize_t tp_vectorcall_offset
Py_ssize_t tp_weaklistoffset
PyObject *tp_bases
PyObject *tp_mro
PyObject *tp_cache
PyObject *tp_weaklist
PyObject *tp_subclasses
PySequenceMethods *tp_as_sequence
PyMethodDef *tp_methods
PyMemberDef *tp_members
ctypedef struct PySequenceMethods:
binaryfunc sq_concat
binaryfunc sq_inplace_concat
object (*sq_slice) (object, Py_ssize_t, Py_ssize_t) # present only on py2
cdef extern from "Python.h":
ctypedef struct PyVarObject:
Py_ssize_t ob_size
cdef extern from "funchook.h" nogil:
ctypedef struct funchook_t
funchook_t* funchook_create()
int funchook_prepare(funchook_t* h, void** target_func, void* hook_func)
int funchook_install(funchook_t* h, int flags)
int funchook_uninstall(funchook_t* h, int flags)
int funchook_destroy(funchook_t*)
const char* funchook_error_message(const funchook_t*)
int funchook_set_debug_file(const char* name)
from cython cimport no_gc
......@@ -77,10 +108,6 @@ import string as pystring
import types as pytypes
import functools as pyfunctools
import re as pyre
if PY_MAJOR_VERSION >= 3:
import copyreg as pycopyreg
else:
import copy_reg as pycopyreg
# zbytes/zunicode point to original std bytes/unicode types even if they will be patched.
......@@ -250,6 +277,8 @@ cdef __pystr(object obj): # -> ~str
return pyb(obj)
# XXX -> bchr ? (not good as "character" means "unicode character")
# -> bstr.chr ?
def pybbyte(int i): # -> 1-byte bstr
"""bbyte(i) returns 1-byte bstr with ordinal i."""
return pyb(bytearray([i]))
......@@ -259,6 +288,22 @@ def pyuchr(int i): # -> 1-character ustr
return pyu(unichr(i))
# XXX due to "cannot `cdef class` with __new__" (https://github.com/cython/cython/issues/799) XXX review text
# _pybstr.__new__ is hand-made in _pybstr_tp_new which invokes ↓ ._pybstr__new__() .
# we keep it out of class instead of cdef @staticmethod due to https://github.com/cython/cython/issues/5337
# XXX def instead of cdef due to ""Non-trivial keyword arguments and starred arguments not allowed in cdef functions
def _pybstr__new__(cls, object='', encoding=None, errors=None):
# encoding or errors -> object must expose buffer interface
if not (encoding is None and errors is None):
object = _buffer_decode(object, encoding, errors)
# _bstringify. Note: it handles bstr/ustr / unicode/bytes/bytearray as documented
object = _bstringify(object)
assert isinstance(object, (unicode, bytes)), object
bobj = _pyb(cls, object)
assert bobj is not None
return bobj
@no_gc # note setup.py assist this to compile despite
cdef class _pybstr(bytes): # https://github.com/cython/cython/issues/711
"""bstr is byte-string.
......@@ -293,34 +338,26 @@ cdef class _pybstr(bytes): # https://github.com/cython/cython/issues/711
"""
# XXX due to "cannot `cdef class` with __new__" (https://github.com/cython/cython/issues/799)
# _pybstr.__new__ is hand-made in _pybstr_tp_new which invokes ↓ .____new__() .
@staticmethod
def ____new__(cls, object='', encoding=None, errors=None):
# encoding or errors -> object must expose buffer interface
if not (encoding is None and errors is None):
object = _buffer_decode(object, encoding, errors)
# _pybstr.__new__ is hand-made in _pybstr_tp_new which invokes ↑ _pybstr__new__() .
# _bstringify. Note: it handles bstr/ustr / unicode/bytes/bytearray as documented
object = _bstringify(object)
assert isinstance(object, (unicode, bytes)), object
bobj = _pyb(cls, object)
assert bobj is not None
return bobj
def __bytes__(self): return self
def __bytes__(self): return pyb(self) # see __str__
def __unicode__(self): return pyu(self)
def __str__(self):
if PY_MAJOR_VERSION >= 3:
return pyu(self)
else:
return self
return pyb(self) # self or pybstr if it was subclass
def __repr__(self):
qself, nonascii_escape = _bpysmartquote_u3b2(self)
bs = _inbstringify_get()
if bs.inbstringify == 0 or bs.inrepr:
if pybstr is bytes: # don't wrap with b(...) when bstr replaces builtin str
if PY_MAJOR_VERSION >= 3:
qself = 'b' + qself
return qself
if nonascii_escape: # so that e.g. b(u'\x80') is represented as
qself = 'b' + qself # b(b'\xc2\x80'), not as b('\xc2\x80')
return "b(" + qself + ")"
......@@ -328,18 +365,8 @@ cdef class _pybstr(bytes): # https://github.com/cython/cython/issues/711
# [b('β')] goes as ['β'] when under _bstringify for %s
return qself
# override reduce for protocols < 2. Builtin handler for that goes through
# copyreg._reduce_ex which eventually calls bytes(bstr-instance) to
# retrieve state, which gives bstr, not bytes. Fix state to be bytes ourselves.
def __reduce_ex__(self, protocol):
if protocol >= 2:
return zbytes.__reduce_ex__(self, protocol)
return (
pycopyreg._reconstructor,
(self.__class__, self.__class__, _bdata(self))
)
return _bstr__reduce_ex__(self, protocol)
def __hash__(self):
# hash of the same unicode and UTF-8 encoded bytes is generally different
......@@ -381,6 +408,7 @@ cdef class _pybstr(bytes): # https://github.com/cython/cython/issues/711
else:
return pyb(x)
# XXX temp disabled
# __iter__ - yields unicode characters
def __iter__(self):
# TODO iterate without converting self to u
......@@ -575,7 +603,7 @@ cdef PyObject* _pybstr_tp_new(PyTypeObject* _cls, PyObject* _argv, PyObject* _kw
if _kw != NULL:
kw = <object>_kw
cdef object x = _pybstr.____new__(<object>_cls, *argv, **kw)
cdef object x = _pybstr__new__(<object>_cls, *argv, **kw)
Py_INCREF(x)
return <PyObject*>x
(<_XPyTypeObject*>_pybstr).tp_new = &_pybstr_tp_new
......@@ -592,6 +620,18 @@ cdef PyObject* _pybstr_tp_new(PyTypeObject* _cls, PyObject* _argv, PyObject* _kw
# and bytes are completely the same.
assert sizeof(_pybstr) == sizeof(PyBytesObject)
# XXX text
def _pyustr__new__(cls, object='', encoding=None, errors=None):
# encoding or errors -> object must expose buffer interface
if not (encoding is None and errors is None):
object = _buffer_decode(object, encoding, errors)
# _bstringify. Note: it handles bstr/ustr / unicode/bytes/bytearray as documented
object = _bstringify(object)
assert isinstance(object, (unicode, bytes)), object
uobj = _pyu(cls, object)
assert uobj is not None
return uobj
@no_gc
cdef class _pyustr(unicode):
......@@ -622,27 +662,15 @@ cdef class _pyustr(unicode):
"""
# XXX due to "cannot `cdef class` with __new__" (https://github.com/cython/cython/issues/799)
# _pyustr.__new__ is hand-made in _pyustr_tp_new which invokes ↓ .____new__() .
@staticmethod
def ____new__(cls, object='', encoding=None, errors=None):
# encoding or errors -> object must expose buffer interface
if not (encoding is None and errors is None):
object = _buffer_decode(object, encoding, errors)
# _bstringify. Note: it handles bstr/ustr / unicode/bytes/bytearray as documented
object = _bstringify(object)
assert isinstance(object, (unicode, bytes)), object
uobj = _pyu(cls, object)
assert uobj is not None
return uobj
# _pyustr.__new__ is hand-made in _pyustr_tp_new which invokes ↑ _pyustr__new__() .
def __bytes__(self): return pyb(self)
def __unicode__(self): return self
def __unicode__(self): return pyu(self) # see __str__
def __str__(self):
if PY_MAJOR_VERSION >= 3:
return self
return pyu(self) # = self or pyustr if it was subclass
else:
return pyb(self)
......@@ -650,6 +678,11 @@ cdef class _pyustr(unicode):
qself, nonascii_escape = _upysmartquote_u3b2(self)
bs = _inbstringify_get()
if bs.inbstringify == 0 or bs.inrepr:
if pyustr is unicode: # don't wrap with u(...) when ustr replaces builtin str/unicode
if not nonascii_escape: # but only if the string is valid utf-8
if PY_MAJOR_VERSION < 3:
qself = 'u'+qself
return qself
if nonascii_escape:
qself = 'b'+qself # see bstr.__repr__
return "u(" + qself + ")"
......@@ -657,18 +690,8 @@ cdef class _pyustr(unicode):
# [u('β')] goes as ['β'] when under _bstringify for %s
return qself
# override reduce for protocols < 2. Builtin handler for that goes through
# copyreg._reduce_ex which eventually calls unicode(ustr-instance) to
# retrieve state, which gives ustr, not unicode. Fix state to be unicode ourselves.
def __reduce_ex__(self, protocol):
if protocol >= 2:
return zunicode.__reduce_ex__(self, protocol)
return (
pycopyreg._reconstructor,
(self.__class__, self.__class__, _udata(self))
)
return _ustr__reduce_ex__(self, protocol)
def __hash__(self):
# see _pybstr.__hash__ for why we stick to hash of current str
......@@ -718,7 +741,7 @@ cdef class _pyustr(unicode):
# https://cython.readthedocs.io/en/latest/src/userguide/migrating_to_cy30.html#arithmetic-special-methods
# see also https://github.com/cython/cython/issues/4750
if type(a) is not pyustr:
assert type(b) is pyustr
assert type(b) is pyustr, type(b)
return b.__radd__(a)
return pyu(zunicode.__add__(a, _pyu_coerce(b)))
......@@ -738,7 +761,7 @@ cdef class _pyustr(unicode):
# __mul__, __rmul__ (no need to override __imul__)
def __mul__(a, b):
if type(a) is not pyustr:
assert type(b) is pyustr
assert type(b) is pyustr, type(b)
return b.__rmul__(a)
return pyu(zunicode.__mul__(a, b))
def __rmul__(b, a):
......@@ -939,7 +962,7 @@ cdef PyObject* _pyustr_tp_new(PyTypeObject* _cls, PyObject* _argv, PyObject* _kw
if _kw != NULL:
kw = <object>_kw
cdef object x = _pyustr.____new__(<object>_cls, *argv, **kw)
cdef object x = _pyustr__new__(<object>_cls, *argv, **kw)
Py_INCREF(x)
return <PyObject*>x
(<_XPyTypeObject*>_pyustr).tp_new = &_pyustr_tp_new
......@@ -963,9 +986,10 @@ cdef class _pyustrIter:
# _bdata/_udata retrieve raw data from bytes/unicode.
def _bdata(obj): # -> bytes
assert isinstance(obj, bytes)
_ = obj.__getnewargs__()[0] # (`bytes-data`,)
assert type(_) is bytes
return _
if type(obj) is not bytes:
obj = obj.__getnewargs__()[0] # (`bytes-data`,)
assert type(obj) is bytes
return obj
"""
bcopy = bytes(memoryview(obj))
assert type(bcopy) is bytes
......@@ -973,9 +997,10 @@ def _bdata(obj): # -> bytes
"""
def _udata(obj): # -> unicode
assert isinstance(obj, unicode)
_ = obj.__getnewargs__()[0] # (`unicode-data`,)
assert type(_) is unicode
return _
if type(obj) is not unicode:
obj = obj.__getnewargs__()[0] # (`unicode-data`,)
assert type(obj) is unicode
return obj
"""
cdef Py_UNICODE* u = PyUnicode_AsUnicode(obj)
cdef Py_ssize_t size = PyUnicode_GetSize(obj)
......@@ -1027,6 +1052,22 @@ if PY2:
# ---- adjust bstr/ustr classes after what cython generated ----
# for pybstr/pyustr cython generates .tp_dealloc that refer to bytes/unicode types directly.
# override that to refer to zbytes/zunicode to avoid infinite recursion on free.
cdef void _pybstr_tp_dealloc(PyObject *self): (<PyTypeObject*>zbytes) .tp_dealloc(self)
cdef void _pyustr_tp_dealloc(PyObject *self): (<PyTypeObject*>zunicode) .tp_dealloc(self)
(<PyTypeObject*>pybstr).tp_dealloc = &_pybstr_tp_dealloc
(<PyTypeObject*>pyustr).tp_dealloc = &_pyustr_tp_dealloc
# change names of bstr/ustr to be e.g. "golang.bstr" instead of "golang._golang._bstr" XXX adjust after .name=str
# this makes sure that unpickling saved bstr does not load via unpatched origin
# class, and is also generally good for saving pickle size and for reducing _golang exposure.
# XXX -> _golang_str_pickle.pyx ?
(<PyTypeObject*>pybstr).tp_name = "golang.bstr"
(<PyTypeObject*>pyustr).tp_name = "golang.ustr"
assert pybstr.__module__ == "golang"; assert pybstr.__name__ == "bstr"
assert pyustr.__module__ == "golang"; assert pyustr.__name__ == "ustr"
# remove unsupported bstr/ustr methods. do it outside of `cdef class` to
# workaround https://github.com/cython/cython/issues/4556 (`if ...` during
# `cdef class` is silently handled wrongly)
......@@ -1039,12 +1080,11 @@ cdef _bstrustr_remove_unsupported_slots():
'removesuffix', # py3.9 TODO provide fallback implementation
)
for slot in vslot:
if not hasattr(unicode, slot):
_patch_slot(<PyTypeObject*>pybstr, slot, DEL)
try:
if not hasattr(zunicode, slot):
if hasattr(pybstr, slot): # we might have already removed it on previous call
_patch_slot(<PyTypeObject*>pybstr, slot, DEL)
if hasattr(pyustr, slot): # e.g. we do not define ustr.isprintable ourselves
_patch_slot(<PyTypeObject*>pyustr, slot, DEL)
except KeyError: # e.g. we do not define ustr.isprintable ourselves
pass
_bstrustr_remove_unsupported_slots()
......@@ -1105,7 +1145,7 @@ cdef _bstringify(object obj): # -> unicode|bytes
_bstringify_enter()
try:
if PY_MAJOR_VERSION >= 3:
if False: # PY_MAJOR_VERSION >= 3:
# NOTE this depends on patches to bytes.{__repr__,__str__} below
return unicode(obj)
......@@ -1118,10 +1158,12 @@ cdef _bstringify(object obj): # -> unicode|bytes
#
# NOTE this depends on patches to bytes.{__repr__,__str__} and
# unicode.{__repr__,__str__} below.
if hasattr(obj, '__unicode__'):
return obj.__unicode__()
elif hasattr(obj, '__str__'):
return obj.__str__()
if False: # PY_MAJOR_VERSION < 3 and hasattr(obj, '__unicode__'):
return obj.__unicode__() # XXX needed ?
elif Py_TYPE(obj).tp_str != NULL:
return Py_TYPE(obj).tp_str(obj)
#elif hasattr(obj, '__str__'):
# return obj.__str__()
else:
return repr(obj)
......@@ -1422,19 +1464,24 @@ cdef _InBStringify _inbstringify_get():
return ts_inbstringify
# XXX text
cdef _get_slot(PyTypeObject* typ, str name):
typdict = <dict>(typ.tp_dict)
return typdict[name]
# _patch_slot installs func_or_descr into typ's __dict__ as name.
#
# if func_or_descr is descriptor (has __get__), it is installed as is.
# if func_or_descr is descriptor (has __get__), or asis=True, it is installed as is.
# otherwise it is wrapped with "unbound method" descriptor.
#
# if func_or_descr is DEL the slot is removed from typ's __dict__.
cdef DEL = object()
cdef _patch_slot(PyTypeObject* typ, str name, object func_or_descr):
cdef _patch_slot(PyTypeObject* typ, str name, object func_or_descr, asis=False):
typdict = <dict>(typ.tp_dict)
#print("\npatching %s.%s with %r" % (typ.tp_name, name, func_or_descr))
#print("old: %r" % typdict.get(name))
if hasattr(func_or_descr, '__get__') or func_or_descr is DEL:
if hasattr(func_or_descr, '__get__') or func_or_descr is DEL or asis:
descr = func_or_descr
else:
func = func_or_descr
......@@ -1498,7 +1545,7 @@ cdef object _atidx_re = pyre.compile('.* at index ([0-9]+)$')
cdef _bprintf(const byte[::1] fmt, xarg): # -> pybstr
cdef bytearray out = bytearray()
cdef tuple argv = None # if xarg is tuple
cdef object argv = None # if xarg is tuple or subclass
cdef object argm = None # if xarg is mapping
# https://github.com/python/cpython/blob/2.7-0-g8d21aa21f2c/Objects/stringobject.c#L4298-L4300
......@@ -1704,7 +1751,11 @@ cdef _bprintf(const byte[::1] fmt, xarg): # -> pybstr
#print('--> __mod__ ', repr(fmt1), ' % ', repr(arg))
try:
s = zbytes.__mod__(fmt1, arg)
IF PY2:
# NOTE not zbytes.__mod__ because underlying PyBytes_Format is patched
s = _pbytes_Format(fmt1, arg)
ELSE:
s = zbytes.__mod__(fmt1, arg)
except ValueError as e:
# adjust position in '... at index <idx>' from fmt1 to fmt
if len(e.args) == 1:
......@@ -1795,6 +1846,50 @@ class _BFormatter(pystring.Formatter):
return super(_BFormatter, self).get_field(field_name, args, kwargs)
# XXX place, comments
# str % ... : ceval on py2 and py3 < 3.11 invokes PyString_Format / PyUnicode_Format
# directly upon seeing BINARY_MODULO. This leads to bstr.__mod__ not being called.
ctypedef unicode uformatfunc(object, object)
ctypedef bytes bformatfunc(object, object)
cdef uformatfunc* _punicode_Format = PyUnicode_Format
cdef unicode _unicode_xFormat(object s, object args):
return pyustr.__mod__(s, args)
IF PY2:
cdef bformatfunc* _pbytes_Format = PyBytes_Format
cdef _bytes_xFormat(object s, object args):
return pybstr.__mod__(s, args)
cdef _patch_capi_str_format():
cpatch(<void**>&_punicode_Format, <void*>_unicode_xFormat)
IF PY2:
cpatch(<void**>&_pbytes_Format, <void*>_bytes_xFormat)
# XXX place, comments, test
#py3.11: specializes instructions. e.g. ustr(obj) will specialize (after
# executing 8 times) to directly invoke
#
# PyObject_Str(obj)
#
# which, if obj is e.g. b'123' will return "b'123'" instead of "123".
#
# -> if we patch str=ustr, we need to patch PyObject_Str as well.
# -> XXX and check all other specializations.
#
# NOTE also good to just do
cdef _object_xStr(object s):
IF PY2:
return pybstr(s)
ELSE:
return pyustr(s)
ctypedef object objstrfunc(object)
cdef objstrfunc* _pobject_Str = PyObject_Str
cdef _patch_capi_object_str():
cpatch(<void**>&_pobject_Str, <void*>_object_xStr)
# ---- misc ----
cdef object _xpyu_coerce(obj):
......@@ -1871,6 +1966,7 @@ cdef extern from "Python.h":
from six import unichr # py2: unichr py3: chr
from six import int2byte as bchr # py2: chr py3: lambda x: bytes((x,))
# XXX turn vvv into compile-time constant
cdef bint _ucs2_build = (sys.maxunicode == 0xffff) # ucs2
assert _ucs2_build or sys.maxunicode >= 0x0010ffff # or ucs4
......@@ -1910,7 +2006,7 @@ cdef (rune, int) _utf8_decode_rune(const byte[::1] s):
# _utf8_decode_surrogateescape mimics s.decode('utf-8', 'surrogateescape') from py3.
def _utf8_decode_surrogateescape(const byte[::1] s): # -> unicode
cdef _utf8_decode_surrogateescape(const byte[::1] s): # -> unicode
if PY_MAJOR_VERSION >= 3:
if len(s) == 0:
return u'' # avoid out-of-bounds slice access on &s[0]
......@@ -1950,7 +2046,7 @@ def _utf8_decode_surrogateescape(const byte[::1] s): # -> unicode
# _utf8_encode_surrogateescape mimics s.encode('utf-8', 'surrogateescape') from py3.
def _utf8_encode_surrogateescape(s): # -> bytes
cdef _utf8_encode_surrogateescape(s): # -> bytes
assert isinstance(s, unicode)
if PY_MAJOR_VERSION >= 3:
return zunicode.encode(s, 'UTF-8', 'surrogateescape')
......@@ -2032,3 +2128,289 @@ cdef unicode _xunichr(rune i):
uh = i - 0x10000
return unichr(0xd800 + (uh >> 10)) + \
unichr(0xdc00 + (uh & 0x3ff))
# ---- funchook wrappers -----
cdef funchook_t* xfunchook_create() except NULL:
h = funchook_create()
if h == NULL:
raise MemoryError()
return h
cdef xfunchook_destroy(funchook_t* h):
err = funchook_destroy(h)
if err != 0:
raise RuntimeError(funchook_error_message(h))
cdef xfunchook_prepare(funchook_t* h, void** target_func, void* hook_func):
err = funchook_prepare(h, target_func, hook_func)
if err != 0:
raise RuntimeError(funchook_error_message(h))
cdef xfunchook_install(funchook_t* h, int flags):
err = funchook_install(h, flags)
if err != 0:
raise RuntimeError(funchook_error_message(h))
cdef xfunchook_uninstall(funchook_t* h, int flags):
err = funchook_uninstall(h, flags)
if err != 0:
raise RuntimeError(funchook_error_message(h))
# cpatch = xfunchook_prepare on _patch_capi_hook
cdef cpatch(void** target_func, void* hook_func):
assert target_func[0] != NULL
xfunchook_prepare(_patch_capi_hook, target_func, hook_func)
# ---- patch unicode/str types to be ustr/bstr under gpython ----
# XXX make sure original _pybstr/_pyustr cannot be used after patching XXX right ?
# XXX and make sure golang._golang._pybstr cannot be imported as well (ex pickle)
# XXX ._pyustr.__module__ = 'builtins' after patch - why?
def _():
gpy_strings = getattr(sys, '_gpy_strings', None)
if gpy_strings == 'bstr+ustr':
_patch_str()
elif gpy_strings in ('pystd', None):
pass
else:
raise AssertionError("invalid sys._gpy_strings: %r" % (gpy_strings,))
_()
# _patch_str is invoked when gpython imports golang and instructs to replace
# builtin str/unicode types with bstr/ustr.
#
# After the patch is applied all existing objects that have e.g. unicode type
# will switch to having ustr type.
cdef PyTypeObject _unicode_orig
cdef PyTypeObject _bytes_orig
cdef funchook_t* _patch_capi_hook
cdef _patch_str():
global zbytes, _bytes_orig, pybstr
global zunicode, _unicode_orig, pyustr
global _patch_capi_hook
#print('\n\nPATCH\n\n')
# XXX explain
bpreserve_slots = upreserve_slots = ("maketrans",)
if PY_MAJOR_VERSION < 3:
bpreserve_slots += ("encode",) # @property'ies
upreserve_slots += ("decode",)
# patch unicode to be pyustr. This patches
# - unicode (py2)
# - str (py3)
_pytype_clone(<PyTypeObject*>unicode, &_unicode_orig, "unicode(pystd)")
Py_INCREF(unicode) # XXX needed?
zunicode = <object>&_unicode_orig
_pytype_replace_by_child(
<PyTypeObject*>unicode, &_unicode_orig,
<PyTypeObject*>pyustr, "ustr(origin)",
upreserve_slots)
pyustr = unicode # retarget pyustr -> unicode to where it was copied
# XXX vvv needed so that patched unicode could be saved by py2:cPickle at all
(<PyTypeObject*>pyustr).tp_name = ("unicode" if PY_MAJOR_VERSION < 3 else "str")
# py2: patch str to be pybstr
if PY_MAJOR_VERSION < 3:
_pytype_clone(<PyTypeObject*>bytes, &_bytes_orig, "bytes(pystd)")
Py_INCREF(bytes) # XXX needed?
zbytes = <object>&_bytes_orig
_pytype_replace_by_child(
<PyTypeObject*>bytes, &_bytes_orig,
<PyTypeObject*>_pybstr, "bstr(origin)",
bpreserve_slots)
pybstr = bytes # retarget pybstr -> bytes to where it was copied
(<PyTypeObject*>pybstr).tp_name = ("str" if PY_MAJOR_VERSION < 3 else "bytes")
# need to remove unsupported slots in cloned bstr/ustr again since PyType_Ready might have recreated them
_bstrustr_remove_unsupported_slots()
# also patch UserString to have methods that bstr/ustr have
# else e.g. IPython's guarded_eval.py fails in `_list_methods(collections.UserString, dir(str))`
from six.moves import UserString
def userstr__bytes__(s): return bytes(s.data)
def userstr__unicode__(s): return unicode(s.data)
assert not hasattr(UserString, '__bytes__') # XXX test
assert not hasattr(UserString, '__unicode__')
UserString.__bytes__ = userstr__bytes__
UserString.__unicode__ = userstr__unicode__
# XXX also patch CAPI functions ... XXX explain
#funchook_set_debug_file("/dev/stderr")
_patch_capi_hook = xfunchook_create()
_patch_capi_str_format()
_patch_capi_object_str()
_patch_capi_unicode_decode_as_bstr()
_patch_str_pickle()
# ...
xfunchook_install(_patch_capi_hook, 0)
# XXX place ok ?
include '_golang_str_pickle.pyx'
# _pytype_clone clones PyTypeObject src into dst.
# dst must not be previously initialized.
#
# dst will have reference-count = 1 meaning new reference to it is returned.
cdef _pytype_clone(PyTypeObject *src, PyTypeObject *dst, const char* new_name):
assert (src.tp_flags & Py_TPFLAGS_READY) != 0
assert (src.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0 # src is not allocated on heap
#assert not PyType_IS_GC((<PyObject*>src).ob_type) # XXX not true as unicode.ob_type is PyType_Type
# which generally has GC support, but
# GC is deactivated for non-heap types.
# copy the struct XXX + .ob_next / .ob_prev (Py_TRACE_REFS)
dst[0] = src[0]
(<PyObject*>dst).ob_refcnt = 1
if new_name != NULL:
dst.tp_name = new_name
# now reinitialize things like .tp_dict etc, where PyType_Ready built slots that point to src.
# we want all those slots to be rebuilt and point to dst instead.
_dst = <_XPyTypeObject*>dst
dst .tp_flags &= ~Py_TPFLAGS_READY
dst .tp_dict = NULL
_dst.tp_bases = NULL
_dst.tp_mro = NULL
_dst.tp_cache = NULL
_dst.tp_weaklist = NULL
# dst.__subclasses__ will be empty because existing children inherit from src, not from dst.
_dst.tp_subclasses = NULL
PyType_Ready(<object>dst)
assert (dst.tp_flags & Py_TPFLAGS_READY) != 0
# _pytype_replace_by_child replaces typ by its child egg.
#
# All existing objects that have type typ will switch to having type egg' .
# The instance/inheritance diagram for existing objects and types will switch
# as depicted below:
#
# base base
# ↑ ↖
# typ ------> egg' → typ_clone
# ↗ ↑ ↖ ↗ ↑ ↗
# objects X egg objects X egg
# ↑ ↑
# Y Y
#
# typ_clone must be initialized via _pytype_clone(typ, typ_clone).
# egg' is egg clone put inplace of typ
#
# XXX preserve_slots - describe
cdef _pytype_replace_by_child(PyTypeObject *typ, PyTypeObject *typ_clone,
PyTypeObject *egg, const char* egg_old_name,
preserve_slots):
otyp = <PyObject*>typ ; oegg = <PyObject*>egg
vtyp = <PyVarObject*>typ ; vegg = <PyVarObject*>egg
_typ = <_XPyTypeObject*>typ ; _egg = <_XPyTypeObject*>egg
assert egg.tp_base == typ
assert _egg.tp_subclasses == NULL
assert (typ.tp_flags & Py_TPFLAGS_READY) != 0
assert (egg.tp_flags & Py_TPFLAGS_READY) != 0
assert (typ.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0
assert (egg.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0 # XXX will be not true
# -> ! Py_TPFLAGS_HAVE_GC
# -> ? set Py_TPFLAGS_HEAPTYPE back on typ' ?
# (generally not required)
assert (typ.tp_flags & Py_TPFLAGS_HAVE_GC) == 0
assert (egg.tp_flags & Py_TPFLAGS_HAVE_GC) == 0
# XXX also check PyObject_IS_GC (verifies .tp_is_gc() = n) ?
assert vtyp.ob_size == vegg.ob_size
assert typ .tp_basicsize == egg .tp_basicsize
assert typ .tp_itemsize == egg .tp_itemsize
IF PY3:
assert _typ.tp_vectorcall_offset == _egg.tp_vectorcall_offset
assert _typ.tp_weaklistoffset == _egg.tp_weaklistoffset
assert typ .tp_dictoffset == egg .tp_dictoffset
# since egg will change .tp_base it will also need to reinitialize
# .tp_bases, .tp_mro and friends. Retrieve egg slots to preserve before we
# clear egg.__dict__ . This covers e.g. @staticmethod and @property.
keep_slots = {} # name -> slot
for name in preserve_slots:
keep_slots[name] = _get_slot(egg, name)
# egg: clear what PyType_Ready will recompute
Py_CLEAR(egg .tp_dict)
Py_CLEAR(_egg.tp_bases)
Py_CLEAR(_egg.tp_mro)
Py_CLEAR(_egg.tp_cache)
# typ <- egg preserving original typ's refcnt, weak references and subclasses\egg.
# typ will be now playing the role of egg
typ_refcnt = otyp.ob_refcnt
typ_weaklist = _typ.tp_weaklist
typ_subclasses = _typ.tp_subclasses
typ[0] = egg[0]
otyp.ob_refcnt = typ_refcnt
_typ.tp_weaklist = typ_weaklist
_typ.tp_subclasses = typ_subclasses # XXX need to remove egg from here
# adjust .tp_base
typ.tp_base = typ_clone
egg.tp_base = typ_clone
# adjust egg.tp_name
if egg_old_name != NULL:
egg.tp_name = egg_old_name
# reinitialize .tp_bases, .tp_mro. .tp_cache, and recompute slots that
# live in .tp_dict and point to their type. Do it for both typ (new egg)
# and origin egg for generality, even though original egg won't be used
# anymore.
typ.tp_flags &= ~Py_TPFLAGS_READY
egg.tp_flags &= ~Py_TPFLAGS_READY
PyType_Ready(<object>typ)
PyType_Ready(<object>egg)
assert (typ.tp_flags & Py_TPFLAGS_READY) != 0
assert (egg.tp_flags & Py_TPFLAGS_READY) != 0
# restore slots we were asked to preserve as is
# since those slots are e.g. @staticmethods they go to both egg' and egg.
for name, slot in keep_slots.items():
_patch_slot(typ, name, slot, asis=True)
_patch_slot(egg, name, slot, asis=True)
# XXX remove egg from typ.tp_subclasses (also possible via setting .__bases__)
# XXX remove typ from base.tp_subclasses
# else e.g. ustr(origin) is reported to be subclass of ustr by help()
# (pyustr.__subclasses__() give it)
# rebuild .tp_mro of all other typ's children
# initially X.__mro__ = (X, typ, base) and without rebuilding it would
# remain (X, egg', base) instead of correct (X, egg' typ_clone, base)
# XXX py3 does this automatically? XXX -> no, it can invalidate .__mro__, but not .tp_mro
def refresh(x):
assert isinstance(x, type)
xtyp = <PyTypeObject*>x
_xtyp = <_XPyTypeObject*>x
fprintf(stderr, 'refreshing %s\n', xtyp.tp_name)
assert (xtyp.tp_flags & Py_TPFLAGS_READY) != 0
xtyp.tp_flags &= ~Py_TPFLAGS_READY
Py_CLEAR(_xtyp.tp_mro)
PyType_Ready(x)
assert (xtyp.tp_flags & Py_TPFLAGS_READY) != 0
for _ in x.__subclasses__():
refresh(_)
for _ in (<object>typ).__subclasses__():
refresh(_)
# XXX also preserve ._ob_next + ._ob_prev (present in Py_TRACE_REFS builds)
golang/_golang_str_pickle.S 0 → 100644
View file @ 17dbfbac
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// _golang_str_pickle.S complements _golang_str_pickle.pyx with assembly routines.
#include "golang/runtime/platform.h"
.text
.p2align 4
// CSYM returns assembler symbol for C-symbol name
#if defined(LIBGOLANG_OS_darwin) || \
(defined(LIBGOLANG_OS_windows) && defined(LIBGOLANG_ARCH_386))
# define CSYM(name) _ ## name
#else
# define CSYM(name) name
#endif
// _TYPE emits `.type sym, symtype` on systems where .type directive is supported
// _SIZE emits `.size sym, symsize` on systems where .size directive is supported
#ifdef LIBGOLANG_OS_linux
# define _TYPE(sym, symtype) .type sym, symtype
# define _SIZE(sym, symsize) .size sym, symsize
#else
# define _TYPE(sym, type)
# define _SIZE(sym, size)
#endif
// inside_counted provides trampoline to call *inside_counted_func with
// counting how many times that function entered inside and exited.
//
// Each enter increments inside_counter, while each exit decrements it.
// Recursion is supported up to STK_SIZE times with counter stopping to be
// adjusted at deeper recursion levels.
//
// inside_counted can be used on functions with arbitrary signatures because
// all registers and stack arguments are preserved exactly as is on the call(*).
//
// (*) NOTE₁ on-stack return address / link-register is adjusted during the call.
// this prevents inside_counted to be used with e.g. x86.get_pc_thunk.ax .
// NOTE₂ on ARM64 xip0 (x16) is clobbered.
#define inside_counted CSYM(inside_counted)
#define inside_counted_func CSYM(inside_counted_func)
#define inside_counter CSYM(inside_counter)
#define inside_counted_stk CSYM(inside_counted_stk)
.globl inside_counted
_TYPE( inside_counted, @function )
inside_counted:
#define STK_SIZE 8
// ---- X86_64 / i386 ----
#if defined(LIBGOLANG_ARCH_amd64) || defined(LIBGOLANG_ARCH_386)
#if defined(LIBGOLANG_ARCH_amd64)
# define REGSIZE 8
# define rAX rax
# define rPCNT rbx
# define rCNT rcx
# define rPSTK rdx
# define rSP rsp
# ifndef LIBGOLANG_OS_windows
.macro LEAGOT sym, reg
movq \sym@GOTPCREL(%rip), %\reg
.endm
# else
// windows does not use PIC and relocates DLLs when loading them
// there is no GOT and we need to access in-DLL symbols directly
// see e.g. https://stackoverflow.com/q/13309662/9456786 for details.
.macro LEAGOT sym, reg
leaq \sym(%rip), %\reg // NOTE pc-relative addressing used to avoid LNK2017:
.endm // 'ADDR32' relocation ... invalid without /LARGEADDRESSAWARE:NO
# endif
#else
# define REGSIZE 4
# define rAX eax
# define rPCNT ebx
# define rCNT ecx
# define rPSTK edx
# define rSP esp
# ifndef LIBGOLANG_OS_windows
.macro LEAGOT sym, reg
call .Lget_pc_\reg
addl $_GLOBAL_OFFSET_TABLE_, %\reg
movl \sym@GOT(%\reg), %\reg
.endm
# else
// windows does not use PIC - see details in ^^^ amd64 case
.macro LEAGOT sym, reg
leal \sym, %\reg
.endm
# endif
#endif
sub $REGSIZE, %rSP // make place for jmp-via-ret to *inside_counted_func
// TODO consider adding cfi_* annotations, but probably it won't be simple
// since we manipulate retaddr on the stack
push %rAX // save registers we'll use
push %rPCNT
push %rCNT
push %rPSTK
#define SP_JMPVIARET (4*REGSIZE)
#define SP_RETORIG (5*REGSIZE)
// jmp-via-ret = *inside_counted_func
LEAGOT inside_counted_func, rAX // &inside_counted_func
mov (%rAX), %rAX // inside_counted_func
mov %rAX, SP_JMPVIARET(%rSP)
// check whether altstk is overflowed
// if it is - invoke the func without counting
LEAGOT inside_counter, rPCNT // &inside_counter
mov (%rPCNT), %rCNT // inside_counter
cmp $STK_SIZE, %rCNT
jge .Lcall
// altstk is not overflowed
// push original ret to altstk and replace the ret to return to us after the call
LEAGOT inside_counted_stk, rPSTK // &inside_counted_stk
mov SP_RETORIG(%rSP), %rAX // original ret address
mov %rAX, (%rPSTK,%rCNT,REGSIZE) // inside_counted_stk[inside_counter] = retorig
add $1, %rCNT // inside_counter++
mov %rCNT, (%rPCNT)
#if defined(LIBGOLANG_ARCH_amd64)
lea .Laftercall(%rip), %rAX
#else
call .Lget_pc_eax
add $(.Laftercall-.), %rAX
#endif
mov %rAX, SP_RETORIG(%rSP) // replace ret addr on stack to .Laftercall
.Lcall:
// restore registers and invoke the func through jmp-via-ret
pop %rPSTK
pop %rCNT
pop %rPCNT
pop %rAX
ret
.Laftercall:
// we get here after invoked func returns if altstk was not overflowed
// decrement inside_counter and return to original ret address
sub $REGSIZE, %rSP // make place for original ret
push %rAX // save registers
push %rPCNT
push %rCNT
push %rPSTK
#undef SP_RETORIG
#define SP_RETORIG (4*REGSIZE)
LEAGOT inside_counter, rPCNT // &inside_counter
mov (%rPCNT), %rCNT // inside_counter
sub $1, %rCNT
mov %rCNT, (%rPCNT) // inside_counter--
LEAGOT inside_counted_stk, rPSTK // &inside_counted_stk
mov (%rPSTK,%rCNT,REGSIZE), %rAX // retorig = inside_counted_stk[inside_counter]
mov %rAX, SP_RETORIG(%rSP)
// restore registers and return to original caller
pop %rPSTK
pop %rCNT
pop %rPCNT
pop %rAX
ret
#if defined(LIBGOLANG_ARCH_386)
.macro DEF_get_pc reg
.Lget_pc_\reg:
mov (%esp), %\reg
ret
.endm
DEF_get_pc eax
DEF_get_pc ebx
DEF_get_pc ecx
DEF_get_pc edx
#endif
// ---- ARM64 ----
#elif defined(LIBGOLANG_ARCH_arm64)
#define REGSIZE 8
#define rPCNT x0
#define rCNT x1
#define rPSTK x2
#define rXIP0 x16
stp rPCNT, rCNT, [sp, -16]! // save registers we'll use
stp rPSTK, xzr, [sp, -16]! // NOTE xip0 is clobbered
// xip0 = *inside_counted_func
adrp rXIP0, :got:inside_counted_func
ldr rXIP0, [rXIP0, :got_lo12:inside_counted_func] // &inside_counted_func
ldr rXIP0, [rXIP0] // inside_counted_func
// check whether altstk is overflowed
// if it is - invoke the func without counting
adrp rPCNT, :got:inside_counter
ldr rPCNT, [rPCNT, :got_lo12:inside_counter] // &inside_counter
ldr rCNT, [rPCNT] // inside_counter
cmp rCNT, STK_SIZE
bge .Lcall
// altstk is not overflowed
// push original ret to altstk and replace the ret to return to us after the call
adrp rPSTK, :got:inside_counted_stk
ldr rPSTK, [rPSTK, :got_lo12:inside_counted_stk] // &inside_counted_stk
str lr, [rPSTK, rCNT, lsl 3] // inside_counted_stk[inside_counter] = retorig
add rCNT, rCNT, 1 // inside_counter++
str rCNT, [rPCNT]
adr lr, .Laftercall // replace ret addr to .Laftercall
.Lcall:
// restore registers and invoke the func via xip0
ldp rPSTK, xzr, [sp], 16
ldp rPCNT, rCNT, [sp], 16
br rXIP0
.Laftercall:
// we get here after invoked func returns if altstk was not overflowed
// decrement inside_counter and return to original ret address
stp rPCNT, rCNT, [sp, -16]! // save registers
stp rPSTK, xzr, [sp, -16]!
adrp rPCNT, :got:inside_counter
ldr rPCNT, [rPCNT, :got_lo12:inside_counter] // &inside_counter
ldr rCNT, [rPCNT] // inside_counter
sub rCNT, rCNT, 1
str rCNT, [rPCNT] // inside_counter--
adrp rPSTK, :got:inside_counted_stk
ldr rPSTK, [rPSTK, :got_lo12:inside_counted_stk] // &inside_counted_stk
ldr lr, [rPSTK, rCNT, lsl 3] // lr = inside_counted_stk[inside_counter]
// restore registers and return to original caller
ldp rPSTK, xzr, [sp], 16
ldp rPCNT, rCNT, [sp], 16
ret
#else
# error "unsupported architecture"
#endif
_SIZE( inside_counted, .-inside_counted )
// ---- data ---
.bss
// void* inside_counted_func
.globl inside_counted_func
.p2align 3 // 8
_TYPE( inside_counted_func, @object )
_SIZE( inside_counted_func, REGSIZE )
inside_counted_func:
.zero REGSIZE
// long inside_counter
.globl inside_counter
.p2align 3 // 8
_TYPE( inside_counter, @object )
_SIZE( inside_counter, REGSIZE )
inside_counter:
.zero REGSIZE
// void* inside_counted_stk[STK_SIZE]
.globl inside_counted_stk
.p2align 5 // 32
_TYPE( inside_counted_stk, @object )
_SIZE( inside_counted_stk, STK_SIZE*REGSIZE )
inside_counted_stk:
.zero STK_SIZE*REGSIZE
// disable executable stack
#ifndef LIBGOLANG_OS_windows
.section .note.GNU-stack,"",@progbits
#endif
// ---- custom callconv proxies ----
.text
.p2align 4
// saveprobe_<callconv> (self, obj, pers_save) input callconv, proxy to saveprobe
// _pickle_Pickler_xsave_<callconv>(self, obj, pers_save) input callconv, proxy to _pickle_Pickler_xsave
// save_invoke_as_<callconv> (save, self, obj, pers_save) input std, proxy to save invoked via callconv
#if defined(LIBGOLANG_ARCH_386)
#ifdef LIBGOLANG_CC_msc
# define CSYM_FASTCALL3(name) @name@12 // MSVC mangles __fastcall
# define CSYM_FASTCALL4(name) @name@16
#else
# define CSYM_FASTCALL3(name) CSYM(name)
# define CSYM_FASTCALL4(name) CSYM(name)
#endif
// python-3.11.5.exe has _pickle.save accepting arguments in ecx,edx,stack but
// contrary to fastcall the callee does not cleanup the stack.
// Handle this as fastcall_nostkclean
.macro FUNC_fastcall_nostkclean name
.globl CSYM(\name\()_fastcall_nostkclean)
_TYPE( CSYM(\name\()_fastcall_nostkclean), @function )
CSYM(\name\()_fastcall_nostkclean):
// we are proxying to fastcall - ecx and edx are already setup and we
// need to only duplicate the 3rd argument on the stack. Do this without
// clobbering any register.
sub $4, %esp // place to copy on-stack argument to
push %eax
mov 12(%esp), %eax // original on-stack arg
mov %eax, 4(%esp) // dup to copy
pop %eax
call CSYM_FASTCALL3(\name\()_ifastcall)
// ^^^ cleaned up the stack from our copy
// nothing to do anymore
ret
_SIZE( CSYM(\name\()_fastcall_nostkclean), .-CSYM(\name\()_fastcall_nostkclean) )
.endm
FUNC_fastcall_nostkclean saveprobe
FUNC_fastcall_nostkclean _pickle_Pickler_xsave
FUNC_fastcall_nostkclean _zpickle_Pickler_xsave
#define save_invoke_as_fastcall_nostkclean CSYM_FASTCALL4(save_invoke_as_fastcall_nostkclean)
.globl save_invoke_as_fastcall_nostkclean
_TYPE( save_invoke_as_fastcall_nostkclean, @function )
save_invoke_as_fastcall_nostkclean:
// input:
// ecx: save
// edx: self
// stk[1]: obj
// stk[2]: pers_save
//
// invoke save as:
// ecx: self
// edx: obj
// stk*[1]: pers_save
mov 8(%esp), %eax // pers_save
push %eax // stk*[1] <- per_save
mov %ecx, %eax // eax <- save
mov %edx, %ecx // ecx <- self
mov (4+4)(%esp), %edx // edx <- obj
call *%eax
// return with cleaning up stack
add $4, %esp // pers_save copy we created
ret $8 // original arguments
_SIZE( save_invoke_as_fastcall_nostkclean, .-save_invoke_as_fastcall_nostkclean)
#endif // 386
golang/_golang_str_pickle.pyx 0 → 100644
View file @ 17dbfbac
# -*- coding: utf-8 -*-
# Copyright (C) 2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""_golang_str_pickle.pyx complements _golang_str.pyx and keeps everything
related to pickling strings.
It is included from _golang_str.pyx .
The main entry-points are _patch_str_pickle and _patch_capi_unicode_decode_as_bstr.
"""
from cpython cimport PyUnicode_Decode
from cpython cimport PyBytes_FromStringAndSize, _PyBytes_Resize
cdef extern from "Python.h":
char* PyBytes_AS_STRING(PyObject*)
Py_ssize_t PyBytes_GET_SIZE(PyObject*)
cdef extern from "Python.h":
ctypedef PyObject* (*PyCFunction)(PyObject*, PyObject*)
ctypedef struct PyMethodDef:
const char* ml_name
PyCFunction ml_meth
ctypedef struct PyCFunctionObject:
PyMethodDef *m_ml
PyObject* m_self
PyObject* m_module
cdef extern from "structmember.h":
ctypedef struct PyMemberDef:
const char* name
int type
Py_ssize_t offset
enum:
T_INT
from libc.stdlib cimport malloc, free
from libc.string cimport memcpy, memcmp
if PY_MAJOR_VERSION >= 3:
import copyreg as pycopyreg
else:
import copy_reg as pycopyreg
cdef object zbinary # = zodbpickle.binary | None
try:
import zodbpickle
except ImportError:
zbinary = None
else:
zbinary = zodbpickle.binary
# support for pickling bstr/ustr as standalone types.
#
# pickling is organized in such a way that
# - what is saved by py2 can be loaded correctly on both py2/py3, and similarly
# - what is saved by py3 can be loaded correctly on both py2/py3 as well.
#
# XXX place
cdef _bstr__reduce_ex__(self, protocol):
# Ideally we want to emit bstr(BYTES), but BYTES is not available for
# protocol < 3. And for protocol < 3 emitting bstr(STRING) is not an
# option because plain py3 raises UnicodeDecodeError on loading arbitrary
# STRING data. However emitting bstr(UNICODE) works universally because
# pickle supports arbitrary unicode - including invalid unicode - out of
# the box and in exactly the same way on both py2 and py3. For the
# reference upstream py3 uses surrogatepass on encode/decode UNICODE data
# to achieve that.
if protocol < 3:
# use UNICODE for data
udata = _udata(pyu(self))
if protocol < 2:
return (self.__class__, (udata,)) # bstr UNICODE REDUCE
else:
return (pycopyreg.__newobj__,
(self.__class__, udata)) # bstr UNICODE NEWOBJ
else:
# use BYTES for data
bdata = _bdata(self)
if PY_MAJOR_VERSION < 3:
# the only way we can get here on py2 and protocol >= 3 is zodbpickle
# -> similarly to py3 save bdata as BYTES
assert zbinary is not None
bdata = zbinary(bdata)
return (
pycopyreg.__newobj__, # bstr BYTES NEWOBJ
(self.__class__, bdata))
cdef _ustr__reduce_ex__(self, protocol):
# emit ustr(UNICODE).
# TODO later we might want to switch to emitting ustr(BYTES)
# even if we do this, it should be backward compatible
if protocol < 2:
return (self.__class__, (_udata(self),))# ustr UNICODE REDUCE
else:
return (pycopyreg.__newobj__, # ustr UNICODE NEWOBJ
(self.__class__, _udata(self)))
# types used while patching
cdef extern from *:
"""
struct PicklerObject;
"""
struct PicklerObject:
pass
cdef struct PicklerTypeInfo:
Py_ssize_t size # sizeof(PicklerObject)
Py_ssize_t off_bin # offsetof `int bin`
Py_ssize_t off_poutput_buffer # offsetof `PyObject *output_buffer`
Py_ssize_t off_output_len # offsetof `Py_ssize_t output_len`
Py_ssize_t off_max_output_len # offsetof `Py_ssize_t max_output_len`
# XXX place ?
cdef extern from * nogil:
r"""
// CALLCONV instructs compiler to use specified builtin calling convention.
// it should be used like this:
//
// int CALLCONV(stdcall) myfunc(...)
#ifndef LIBGOLANG_CC_msc
# define CALLCONV(callconv) __attribute__((callconv))
#else // MSC
# define CALLCONV(callconv) __##callconv
#endif
// FOR_EACH_CALLCONV invokes macro X(ccname, callconv, cckind) for every supported calling convention.
// cckind is one of `builtin` or `custom`.
#ifdef LIBGOLANG_ARCH_386
# ifndef LIBGOLANG_CC_msc
# define FOR_EACH_CALLCONV(X) \
X(default,, builtin) \
X(cdecl, CALLCONV(cdecl), builtin) \
X(stdcall, CALLCONV(stdcall), builtin) \
X(fastcall, CALLCONV(fastcall), builtin) \
X(thiscall, CALLCONV(thiscall), builtin) \
X(regparm1, CALLCONV(regparm(1)), builtin) \
X(regparm2, CALLCONV(regparm(2)), builtin) \
X(regparm3, CALLCONV(regparm(3)), builtin) \
X(fastcall_nostkclean, na, custom )
# else // MSC
# define FOR_EACH_CALLCONV(X) \
X(default,, builtin) \
X(cdecl, CALLCONV(cdecl), builtin) \
X(stdcall, CALLCONV(stdcall), builtin) \
X(fastcall, CALLCONV(fastcall), builtin) \
/* X(CALLCONV(thiscall), thiscall) MSVC emits "C3865: '__thiscall': can only be used on native member functions" */ \
/* in theory we can emulate thiscall via fastcall https://tresp4sser.wordpress.com/2012/10/06/how-to-hook-thiscall-functions/ */ \
X(vectorcall, CALLCONV(vectorcall), builtin) \
X(fastcall_nostkclean, na, custom )
# endif
#elif defined(LIBGOLANG_ARCH_amd64)
# define FOR_EACH_CALLCONV(X) \
X(default,, builtin)
#elif defined(LIBGOLANG_ARCH_arm64)
# define FOR_EACH_CALLCONV(X) \
X(default,, builtin)
#else
# error "unsupported architecture"
#endif
// Callconv denotes calling convention of a function.
enum Callconv {
#define CC_ENUM1(ccname, _, __) \
CALLCONV_##ccname,
FOR_EACH_CALLCONV(CC_ENUM1)
};
const char* callconv_str(Callconv cconv) {
using namespace golang;
switch(cconv) {
#define CC_STR1(ccname, _, __) \
case CALLCONV_##ccname: \
return #ccname;
FOR_EACH_CALLCONV(CC_STR1)
default:
panic("bug");
}
}
// SaveFunc represents a save function - its address and calling convention.
struct SaveFunc {
void* addr;
Callconv cconv;
};
"""
enum Callconv: pass
const char* callconv_str(Callconv)
struct SaveFunc:
void* addr
Callconv cconv
# XXX doc
cdef struct _pickle_PatchCtx:
initproc Unpickler_tp_xinit # func to replace Unpickler.tp_init
initproc Unpickler_tp_init_orig # what was there before
vector[SaveFunc] Pickler_xsave_ccv # func to replace _Pickler_save (all callconv variants)
SaveFunc Pickler_save_orig # what was there before
PicklerTypeInfo iPickler # information detected about PicklerObject type
# patch contexts for _pickle and _zodbpickle modules
cdef _pickle_PatchCtx _pickle_patchctx
cdef _pickle_PatchCtx _zpickle_patchctx
# _patch_str_pickle patches *pickle modules to support bstr/ustr and UTF-8 properly.
#
# STRING opcodes are handled in backward-compatible way:
#
# - *STRING are loaded as bstr
# - bstr is saved as *STRING
# - pickletools decodes *STRING as UTF-8
cdef _patch_str_pickle():
try:
import zodbpickle
except ImportError:
zodbpickle = None
# py3: pickletools.dis raises UnicodeDecodeError on non-ascii STRING and treats *BINSTRING as latin1
# -> decode as UTF8b instead
if PY_MAJOR_VERSION >= 3:
import pickletools, codecs
_codecs_escape_decode = codecs.escape_decode
def xread_stringnl(f):
data = _codecs_escape_decode(pickletools.read_stringnl(f, decode=False))[0]
return pybstr(data)
def xread_string1(f):
data = pickletools.read_string1(f).encode('latin1')
return pybstr(data)
def xread_string4(f):
data = pickletools.read_string4(f).encode('latin1')
return pybstr(data)
pickletools.stringnl.reader = xread_stringnl
pickletools.string1.reader = xread_string1
pickletools.string4.reader = xread_string4
if zodbpickle:
from zodbpickle import pickletools_3 as zpickletools
zpickletools.stringnl.reader = xread_stringnl # was same logic as in std pickletools
zpickletools.string1.reader = xread_string1
zpickletools.string4.reader = xread_string4
# py3: pickle.load wants to treat *STRING as bytes and decode it as ASCII
# -> adjust to decode to bstr instead
# -> also save bstr via *STRING opcodes so that load/save is identity
import pickle, _pickle
# TODO _pickle not available (pypy)
_pickle_patchctx.Unpickler_tp_xinit = _pickle_Unpickler_xinit
_pickle_patchctx.Pickler_xsave_ccv = _pickle_Pickler_xsave_ccv
_patch_pickle(pickle, _pickle, &_pickle_patchctx)
if zodbpickle:
from zodbpickle import pickle as zpickle, _pickle as _zpickle
from zodbpickle import slowpickle as zslowPickle, fastpickle as zfastPickle
# TODO _pickle / fastpickle not available (pypy)
for x in 'load', 'loads', 'Unpickler', 'dump', 'dumps', 'Pickler':
assert getattr(_zpickle, x) is getattr(zfastPickle, x)
assert getattr(zpickle, x) is getattr(_zpickle, x)
_patch_pickle(zslowPickle, None, NULL)
_zpickle_patchctx.Unpickler_tp_xinit = _zpickle_Unpickler_xinit
_zpickle_patchctx.Pickler_xsave_ccv = _zpickle_Pickler_xsave_ccv
_patch_pickle(None, zfastPickle, &_zpickle_patchctx)
# propagate changes from fastpickle -> _zpickle -> zpickle
_zpickle.load = zfastPickle.load
_zpickle.loads = zfastPickle.loads
_zpickle.dump = zfastPickle.dump
_zpickle.dumps = zfastPickle.dumps
assert _zpickle.Unpickler is zfastPickle.Unpickler
assert _zpickle.Pickler is zfastPickle.Pickler
zpickle.load = zfastPickle.load
zpickle.loads = zfastPickle.loads
zpickle.dump = zfastPickle.dump
zpickle.dumps = zfastPickle.dumps
assert zpickle.Unpickler is zfastPickle.Unpickler
assert zpickle.Pickler is zfastPickle.Pickler
# _patch_pickle serves _patch_str_pickle by patching pair of py-by-default and
# C implementations of a pickle module.
#
# pickle or _pickle being None indicates that corresponding module version is not available.
cdef _patch_pickle(pickle, _pickle, _pickle_PatchCtx* _pctx):
# if C module is available - it should shadow default py implementation
if _pickle is not None and pickle is not None:
assert pickle.load is _pickle.load
assert pickle.loads is _pickle.loads
assert pickle.Unpickler is _pickle.Unpickler
assert pickle.dump is _pickle.dump
assert pickle.dumps is _pickle.dumps
assert pickle.Pickler is _pickle.Pickler
# patch C
if _pickle is not None:
_patch_cpickle(_pickle, _pctx)
# propagate C updates to py
if pickle is not None:
pickle.load = _pickle.load
pickle.loads = _pickle.loads
pickle.Unpickler = _pickle.Unpickler
pickle.dump = _pickle.dump
pickle.dumps = _pickle.dumps # XXX needed?
pickle.Pickler = _pickle.Pickler
# patch py
if pickle is not None:
_patch_pypickle(pickle, shadowed = (_pickle is not None))
# _patch_pypickle serves _patch_pickle for py version.
cdef _patch_pypickle(pickle, shadowed):
def pyattr(name):
if shadowed:
name = '_'+name
return getattr(pickle, name)
# adjust load / loads / Unpickler to use 'bstr' encoding by default
Unpickler = pyattr('Unpickler')
for f in pyattr('load'), pyattr('loads'), Unpickler.__init__:
f.__kwdefaults__['encoding'] = 'bstr'
# patch Unpickler._decode_string to handle 'bstr' encoding
# zodbpickle uses .decode_string from first version of patch from bugs.python.org/issue6784
has__decode = hasattr(Unpickler, '_decode_string')
has_decode = hasattr(Unpickler, 'decode_string')
assert has__decode or has_decode
assert not (has__decode and has_decode)
_decode_string = '_decode_string' if has__decode else 'decode_string'
Unpickler_decode_string = getattr(Unpickler, _decode_string)
def _xdecode_string(self, value):
if self.encoding == 'bstr':
return pyb(value)
else:
return Unpickler_decode_string(self, value)
setattr(Unpickler, _decode_string, _xdecode_string)
# adjust Pickler to save bstr as STRING
from struct import pack
Pickler = pyattr('Pickler')
def save_bstr(self, obj):
cdef bint nonascii_escape # unused
if self.proto >= 1:
n = len(obj)
if n < 256:
op = b'U' + bytes((n,)) + _bdata(obj) # SHORT_BINSTRING
else:
op = b'T' + pack('<i', n) + _bdata(obj) # BINSTRING
else:
qobj = strconv._quote(obj, b"'", &nonascii_escape)
op = b'S' + qobj + b'\n' # STRING
self.write(op)
self.memoize(obj)
Pickler.dispatch[pybstr] = save_bstr
# _patch_cpickle serves _patch_pickle for C version.
cdef _patch_cpickle(_pickle, _pickle_PatchCtx *pctx):
# adjust load / loads to use 'bstr' encoding by default
# builtin_function_or_method does not have __kwdefaults__ (defaults for
# arguments are hardcoded in generated C code)
# -> wrap functions
_pickle_load = _pickle.load
_pickle_loads = _pickle.loads
def load (file, *, **kw):
kw.setdefault('encoding', 'bstr')
return _pickle_load (file, **kw)
def loads(data, *, **kw):
kw.setdefault('encoding', 'bstr')
return _pickle_loads(data, **kw)
_pickle.load = load
_pickle.loads = loads
# adjust Unpickler to use 'bstr' encoding by default
assert isinstance(_pickle.Unpickler, type)
cdef _XPyTypeObject* Unpickler = <_XPyTypeObject*>(_pickle.Unpickler)
pctx.Unpickler_tp_init_orig = Unpickler.tp_init
Unpickler.tp_init = pctx.Unpickler_tp_xinit
def Unpickler_x__init__(self, *argv, **kw):
# NOTE don't return - just call: __init__ should return None
pctx.Unpickler_tp_xinit(self, <PyObject*>argv, <PyObject*>kw)
_patch_slot(<PyTypeObject*>Unpickler, '__init__', Unpickler_x__init__)
# decoding to bstr relies on _patch_capi_unicode_decode_as_bstr
# adjust Pickler to save bstr as *STRING
# it is a bit involved because:
# - save function, that we need to patch, is not exported.
# - _Pickle_Write, that we need to use from patched save, is not exported neither.
pctx.iPickler = _detect_Pickler_typeinfo(_pickle.Pickler)
pctx.Pickler_save_orig = save = _find_Pickler_save(_pickle.Pickler)
xsave = pctx.Pickler_xsave_ccv[save.cconv]
assert xsave.cconv == save.cconv, (callconv_str(xsave.cconv), callconv_str(save.cconv))
cpatch(&pctx.Pickler_save_orig.addr, xsave.addr)
# XXX test at runtime that we hooked save correctly
# ---- adjusted C bits for loading ----
# adjust Unpickler to use 'bstr' encoding by default and handle that encoding
# in PyUnicode_Decode by returning bstr instead of unicode. This mirrors
# corresponding py loading adjustments.
cdef int _pickle_Unpickler_xinit(object self, PyObject* args, PyObject* kw) except -1:
xkw = {'encoding': 'bstr'}
if kw != NULL:
xkw.update(<object>kw)
return _pickle_patchctx.Unpickler_tp_init_orig(self, args, <PyObject*>xkw)
cdef int _zpickle_Unpickler_xinit(object self, PyObject* args, PyObject* kw) except -1:
xkw = {'encoding': 'bstr'}
if kw != NULL:
xkw.update(<object>kw)
return _zpickle_patchctx.Unpickler_tp_init_orig(self, args, <PyObject*>xkw)
ctypedef object unicode_decodefunc(const char*, Py_ssize_t, const char* encoding, const char* errors)
cdef unicode_decodefunc* _punicode_Decode
cdef object _unicode_xDecode(const char *s, Py_ssize_t size, const char* encoding, const char* errors):
if encoding != NULL and strcmp(encoding, 'bstr') == 0:
bobj = PyBytes_FromStringAndSize(s, size) # TODO -> PyBSTR_FromStringAndSize directly
return pyb(bobj)
return _punicode_Decode(s, size, encoding, errors)
cdef _patch_capi_unicode_decode_as_bstr():
global _punicode_Decode
_punicode_Decode = PyUnicode_Decode
cpatch(<void**>&_punicode_Decode, <void*>_unicode_xDecode)
# ---- adjusted C bits for saving ----
# adjust Pickler save to save bstr via *STRING opcodes.
# This mirrors corresponding py saving adjustments, but is more involved to implement.
cdef int _pickle_Pickler_xsave(PicklerObject* self, PyObject* obj, int pers_save) except -1:
return __Pickler_xsave(&_pickle_patchctx, self, obj, pers_save)
cdef int _zpickle_Pickler_xsave(PicklerObject* self, PyObject* obj, int pers_save) except -1:
return __Pickler_xsave(&_zpickle_patchctx, self, obj, pers_save)
# callconv wrappers XXX place
cdef extern from *:
r"""
static int __pyx_f_6golang_7_golang__pickle_Pickler_xsave(PicklerObject*, PyObject*, int);
static int __pyx_f_6golang_7_golang__zpickle_Pickler_xsave(PicklerObject*, PyObject*, int);
#define DEF_PICKLE_XSAVE_builtin(ccname, callconv) \
static int callconv \
_pickle_Pickler_xsave_##ccname(PicklerObject* self, PyObject* obj, int pers_save) { \
return __pyx_f_6golang_7_golang__pickle_Pickler_xsave(self, obj, pers_save); \
}
#define DEF_ZPICKLE_XSAVE_builtin(ccname, callconv) \
static int callconv \
_zpickle_Pickler_xsave_##ccname(PicklerObject* self, PyObject* obj, int pers_save) { \
return __pyx_f_6golang_7_golang__zpickle_Pickler_xsave(self, obj, pers_save); \
}
#define DEF_PICKLE_XSAVE_custom(ccname, _) \
extern "C" char _pickle_Pickler_xsave_##ccname;
#define DEF_ZPICKLE_XSAVE_custom(ccname, _) \
extern "C" char _zpickle_Pickler_xsave_##ccname;
#define DEF_PICKLE_XSAVE(ccname, callconv, cckind) DEF_PICKLE_XSAVE_##cckind(ccname, callconv)
#define DEF_ZPICKLE_XSAVE(ccname, callconv, cckind) DEF_ZPICKLE_XSAVE_##cckind(ccname, callconv)
FOR_EACH_CALLCONV(DEF_PICKLE_XSAVE)
FOR_EACH_CALLCONV(DEF_ZPICKLE_XSAVE)
static std::vector<SaveFunc> _pickle_Pickler_xsave_ccv = {
#define PICKLE_CC_XSAVE(ccname, _, __) \
SaveFunc{(void*)&_pickle_Pickler_xsave_##ccname, CALLCONV_##ccname},
FOR_EACH_CALLCONV(PICKLE_CC_XSAVE)
};
static std::vector<SaveFunc> _zpickle_Pickler_xsave_ccv = {
#define ZPICKLE_CC_XSAVE(ccname, _, __) \
SaveFunc{(void*)&_zpickle_Pickler_xsave_##ccname, CALLCONV_##ccname},
FOR_EACH_CALLCONV(ZPICKLE_CC_XSAVE)
};
// proxy for asm routines to invoke _pickle_Pickler_xsave and _zpickle_Pickler_xsave
#ifdef LIBGOLANG_ARCH_386
extern "C" int CALLCONV(fastcall)
_pickle_Pickler_xsave_ifastcall(PicklerObject* self, PyObject* obj, int pers_save) {
return __pyx_f_6golang_7_golang__pickle_Pickler_xsave(self, obj, pers_save);
}
extern "C" int CALLCONV(fastcall)
_zpickle_Pickler_xsave_ifastcall(PicklerObject* self, PyObject* obj, int pers_save) {
return __pyx_f_6golang_7_golang__zpickle_Pickler_xsave(self, obj, pers_save);
}
#endif
"""
vector[SaveFunc] _pickle_Pickler_xsave_ccv
vector[SaveFunc] _zpickle_Pickler_xsave_ccv
cdef int __Pickler_xsave(_pickle_PatchCtx* pctx, PicklerObject* self, PyObject* obj, int pers_save) except -1:
# !bstr -> use builtin pickle code
if obj.ob_type != <PyTypeObject*>pybstr:
return save_invoke(pctx.Pickler_save_orig.addr, pctx.Pickler_save_orig.cconv,
self, obj, pers_save)
# bstr -> pickle it as *STRING
cdef const char* s
cdef Py_ssize_t l
cdef byte[5] h
cdef Py_ssize_t lh = 1;
cdef bint nonascii_escape
cdef int bin = (<int*>((<byte*>self) + pctx.iPickler.off_bin))[0]
if bin == 0:
esc = strconv._quote(<object>obj, "'", &nonascii_escape)
assert type(esc) is bytes
s = PyBytes_AS_STRING(<PyObject*>esc)
l = PyBytes_GET_SIZE(<PyObject*>esc)
__Pickler_xWrite(pctx, self, b'S', 1) # STRING
__Pickler_xWrite(pctx, self, s, l)
__Pickler_xWrite(pctx, self, b'\n', 1)
else:
s = PyBytes_AS_STRING(obj)
l = PyBytes_GET_SIZE(obj)
if l < 0x100:
h[0] = b'U' # SHORT_BINSTRING
h[1] = <byte>l
lh += 1
elif l < 0x7fffffff:
h[0] = b'T' # BINSTRING
h[1] = <byte>(l >> 0)
h[2] = <byte>(l >> 8)
h[3] = <byte>(l >> 16)
h[4] = <byte>(l >> 24)
lh += 4
else:
raise OverflowError("cannot serialize a string larger than 2 GiB")
__Pickler_xWrite(pctx, self, <char*>h, lh)
__Pickler_xWrite(pctx, self, s, l)
return 0
# __Pickler_xWrite mimics original _Pickler_Write.
#
# we have to implement it ourselves because there is no way to discover
# original _Pickler_Write address: contrary to `save` function _Pickler_Write
# is small and is not recursive. A compiler is thus free to create many
# versions of it with e.g. constant propagation and to inline it freely. The
# latter actually happens for real on LLVM which for py3.11 inlines
# _Pickler_Write fully without leaving any single freestanding instance of it.
#
# XXX explain why we can skip flush in zpickle case
# XXX explain that we do not emit FRAME
cdef int __Pickler_xWrite(_pickle_PatchCtx* pctx, PicklerObject* self, const char* s, Py_ssize_t l) except -1:
ppoutput_buffer = <PyObject**> (<byte*>self + pctx.iPickler.off_poutput_buffer)
poutput_len = <Py_ssize_t*>(<byte*>self + pctx.iPickler.off_output_len)
pmax_output_len = <Py_ssize_t*>(<byte*>self + pctx.iPickler.off_max_output_len)
assert ppoutput_buffer[0].ob_type == &PyBytes_Type
assert l >= 0
assert poutput_len[0] >= 0
if l > PY_SSIZE_T_MAX - poutput_len[0]:
raise MemoryError() # overflow
need = poutput_len[0] + l
if need > pmax_output_len[0]:
if need >= PY_SSIZE_T_MAX // 2:
raise MemoryError()
pmax_output_len[0] = need // 2 * 3
_PyBytes_Resize(ppoutput_buffer, pmax_output_len[0])
buf = PyBytes_AS_STRING(ppoutput_buffer[0])
memcpy(buf + poutput_len[0], s, l)
poutput_len[0] += l
return 0
# ---- infrastructure to assist patching C saving codepath ----
# _detect_Pickler_typeinfo detects information about PicklerObject type
# through runtime introspection.
#
# This information is used mainly by __Pickler_xWrite.
cdef PicklerTypeInfo _detect_Pickler_typeinfo(pyPickler) except *:
cdef PicklerTypeInfo t
cdef bint debug = False
def trace(*argv):
if debug:
print(*argv)
trace()
assert isinstance(pyPickler, type)
cdef PyTypeObject* Pickler = <PyTypeObject*> pyPickler
cdef _XPyTypeObject* xPickler = <_XPyTypeObject*> pyPickler
# sizeof
assert Pickler.tp_basicsize > 0
assert Pickler.tp_itemsize == 0
t.size = Pickler.tp_basicsize
trace('size:\t', t.size)
# busy keeps offsets of all bytes for already detected fields
busy = set()
def markbusy(off, size):
for _ in range(off, off+size):
assert _ not in busy, (_, busy)
assert 0 < off <= t.size
busy.add(_)
# .bin
cdef PyMemberDef* mbin = tp_members_lookup(xPickler.tp_members, 'bin')
assert mbin.type == T_INT, (mbin.type,)
t.off_bin = mbin.offset
markbusy(t.off_bin, sizeof(int))
trace('.bin:\t', t.off_bin)
# .output_buffer
#
# 1) new Pickler
# 2) .memo = {} - the only pointer that changes is .memo (PyMemoTable* - not pyobject)
# 3) .tp_clear() - all changed words are changed to 0 and cover non-optional PyObject* and memo
# 4) .__init__()
# 5) go through offsets of all pyobjects and find the one with .ob_type = PyBytes_Type
# -> that is .output_buffer
# 1)
class Null:
def write(self, data): pass
pyobj = pyPickler(Null())
cdef PyObject* obj = <PyObject*>pyobj
assert obj.ob_type == Pickler
cdef byte* bobj = <byte*>obj
cdef byte* bobj2 = <byte*>malloc(t.size)
# obj_copy copies obj to obj2.
def obj_copy():
memcpy(bobj2, bobj, t.size)
# obj_diff finds difference in between obj2 and obj.
def obj_diff(Py_ssize_t elemsize): # -> []offset
assert (elemsize & (elemsize - 1)) == 0, elemsize # elemsize is 2^x
cdef Py_ssize_t off
# skip PyObject_HEAD
off = sizeof(PyObject)
off = (off + elemsize - 1) & (~(elemsize - 1))
assert off % elemsize == 0
# find out offsets of different elements
vdelta = []
while off + elemsize <= t.size:
if memcmp(bobj + off, bobj2 + off, elemsize):
vdelta.append(off)
off += elemsize
return vdelta
# 2)
obj_copy()
pyobj.memo = {}
dmemo = obj_diff(sizeof(void*))
assert len(dmemo) == 1, dmemo
off_memo = dmemo[0]
markbusy(off_memo, sizeof(void*))
trace('.memo:\t', off_memo)
# 3)
assert Pickler.tp_clear != NULL
obj_copy()
Pickler.tp_clear(pyobj)
pointers = obj_diff(sizeof(void*))
for poff in pointers:
assert (<void**>(bobj + <Py_ssize_t>poff))[0] == NULL
assert off_memo in pointers
pyobjects = pointers[:]
pyobjects.remove(off_memo)
trace('pyobjects:\t', pyobjects)
# 4)
pyobj.__init__(Null())
# 5)
cdef PyObject* bout = NULL
t.off_poutput_buffer = 0
for poff in pyobjects:
x = (<PyObject**>(bobj + <Py_ssize_t>poff))[0]
if x.ob_type == &PyBytes_Type:
if t.off_poutput_buffer == 0:
t.off_poutput_buffer = poff
else:
raise AssertionError("found several <bytes> inside Pickler")
assert t.off_poutput_buffer != 0
markbusy(t.off_poutput_buffer, sizeof(PyObject*))
trace(".output_buffer:\t", t.off_poutput_buffer)
# .output_len + .max_output_len
# dump something small and expected -> find out which field changes correspondingly
import io
output_len = None
max_output_len = None
for n in range(1,10):
f = io.BytesIO()
pyobj.__init__(f, 0)
o = (None,)*n
pyobj.dump(o)
p = f.getvalue()
phok = b'(' + b'N'*n + b't' # full trails with "p0\n." but "p0\n" is optional
assert p.startswith(phok), p
# InspectWhilePickling observes obj while the pickling is going on:
# - sees which fields have changes
# - sees which fields are candidates for max_output_len
class InspectWhilePickling:
def __init__(self):
self.diff = None # what changes
self.doff2val = {} # off from .diff -> Py_ssize_t read from it
self.max_output_len = set() # offsets that are candidates for .max_output_len
def __reduce__(self):
self.diff = obj_diff(sizeof(Py_ssize_t))
for off in self.diff:
self.doff2val[off] = (<Py_ssize_t*>(bobj + <Py_ssize_t>off))[0]
cdef PyObject* output_buffer = \
(<PyObject**>(bobj + t.off_poutput_buffer))[0]
assert output_buffer.ob_type == &PyBytes_Type
off = sizeof(PyObject)
off = (off + sizeof(Py_ssize_t) - 1) & (~(sizeof(Py_ssize_t) - 1))
assert off % sizeof(Py_ssize_t) == 0
while off + sizeof(Py_ssize_t) <= t.size:
v = (<Py_ssize_t*>(bobj + <Py_ssize_t>off))[0]
if v == PyBytes_GET_SIZE(output_buffer):
self.max_output_len.add(off)
off += sizeof(Py_ssize_t)
return (int, ()) # arbitrary
pyobj.__init__(Null(), 0)
i = InspectWhilePickling()
o += (i,)
obj_copy()
pyobj.dump(o)
assert i.diff is not None
#trace('n%d diff: %r\toff2val: %r' % (n, i.diff, i.doff2val))
#trace(' ', busy)
noutput_len = set()
for off in i.diff:
if off not in busy:
if i.doff2val[off] == (len(phok)-1): # (NNNN without t yet
noutput_len.add(off)
assert len(noutput_len) >= 1, noutput_len
if output_len is None:
output_len = noutput_len
else:
output_len.intersection_update(noutput_len)
nmax_output_len = set()
for off in i.max_output_len:
if off not in busy:
nmax_output_len.add(off)
assert len(nmax_output_len) >= 1, nmax_output_len
if max_output_len is None:
max_output_len = nmax_output_len
else:
max_output_len.intersection_update(nmax_output_len)
if len(output_len) != 1:
raise AssertionError("cannot find .output_len")
if len(max_output_len) != 1:
raise AssertionError("cannot find .max_output_len")
t.off_output_len = output_len.pop()
markbusy(t.off_output_len, sizeof(Py_ssize_t))
trace(".output_len:\t", t.off_output_len)
t.off_max_output_len = max_output_len.pop()
markbusy(t.off_max_output_len, sizeof(Py_ssize_t))
trace(".max_output_len:\t", t.off_max_output_len)
free(bobj2)
return t
# _find_Pickler_save determines address and calling convention of `save` C
# function associated with specified Pickler.
#
# Address and calling convention of `save` are needed to be able to patch it.
cdef SaveFunc _find_Pickler_save(pyPickler) except *:
cdef SaveFunc save
save.addr = __find_Pickler_save(pyPickler)
save.cconv = __detect_save_callconv(pyPickler, save.addr)
#fprintf(stderr, "save.addr: %p\n", save.addr)
#fprintf(stderr, "save.cconv: %s\n", callconv_str(save.cconv))
return save
cdef void* __find_Pickler_save(pyPickler) except NULL:
assert isinstance(pyPickler, type)
# start from _pickle_Pickler_dump as root and analyze how called functions
# behave wrt pickling deep chain of objects. We know whether a callee leads
# to save if, upon receiving control in our __reduce__, we see that the
# callee was entered and did not exited yet. If we find such a callee, we
# recourse the process and start to analyze functions that the callee invokes
# itself. We detect reaching save when we see that a callee was entered
# many times recursively. That happens because we feed deep recursive
# structure to the pickle, and because save itself is organized to invoke
# itself recursively - e.g. (obj,) is pickled via save -> save_tuple -> save.
cdef _XPyTypeObject* Pickler = <_XPyTypeObject*>(pyPickler)
cdef PyMethodDef* mdump = tp_methods_lookup(Pickler.tp_methods, 'dump')
#print("%s _pickle_Pickler_dump:" % pyPickler)
addr = <void*>mdump.ml_meth # = _pickle_Pickler_dump
while 1:
vcallee = cfunc_direct_callees(addr)
ok = False
for i in range(vcallee.size()):
callee = vcallee[i]
#fprintf(stderr, "checking %p ...\n", callee)
nentry = _nentry_on_deep_save(pyPickler, callee)
#fprintf(stderr, "%p - %ld\n", callee, nentry)
assert nentry in (0, 1) or nentry > 5, nentry
if nentry > 5:
return callee # found save
if nentry == 1:
addr = callee # found path that will lead to save
ok = True
break
if not ok:
raise AssertionError('cannot find path leading to save')
# _nentry_on_deep_save tests how addr is related to `save` via inspecting
# addr entry count when Pickler is feed deep recursive structure.
#
# if #entry is 0 - addr is unrelated to save
# if #entry is 1 - addr is related to save and calls it
# if #entry is big - addr is save
cdef long _nentry_on_deep_save(pyPickler, void* addr) except -1: # -> nentry
# below we rely on inside_counted which alters return address during the
# call to wrapped func. In practice this does not create problems on x86_64
# and arm64, but on i386 there are many calls to functions like
# x86.get_pc_thunk.ax which are used to implement PC-relative addressing.
# If we let inside_counted to hook such a func it will result in a crash
# because returned address will be different from real PC of the caller.
# Try to protect us from entering into such situation by detecting leaf
# functions and not hooking them. For the reference x86.get_pc_thunk.ax is:
#
# movl (%esp), %eax
# ret
vcallee = cfunc_direct_callees(addr)
if vcallee.size() == 0:
return 0
# InspectWhilePickling observes how many times currently considered
# function was entered at the point of deep recursion inside save.
class InspectWhilePickling:
def __init__(self):
self.inside_counter = None
def __reduce__(self):
self.inside_counter = inside_counter
return (int, ()) # arbitrary
class Null:
def write(self, data): pass
i = InspectWhilePickling()
obj = (i,)
for _ in range(20):
obj = (obj,)
p = pyPickler(Null(), 0)
h = xfunchook_create()
global inside_counted_func
inside_counted_func = addr
xfunchook_prepare(h, &inside_counted_func, <void*>inside_counted)
xfunchook_install(h, 0)
p.dump(obj)
xfunchook_uninstall(h, 0)
xfunchook_destroy(h)
assert i.inside_counter is not None
return i.inside_counter
# inside_counted is used to patch a function to count how many times that
# function is entered/leaved.
cdef extern from * nogil: # see _golang_str_pickle.S for details
"""
extern "C" {
extern void inside_counted();
extern void* inside_counted_func;
extern long inside_counter;
}
"""
void inside_counted()
void* inside_counted_func
long inside_counter
# __detect_save_callconv determines calling convention that compiler used for save.
#
# On architectures with many registers - e.g. x86_64 and arm64 - the calling
# convention is usually the same as default, but on e.g. i386 - where the
# default cdecl means to put arguments on the stack, the compiler usually
# changes calling convention to use registers instead.
cdef Callconv __detect_save_callconv(pyPickler, void* save) except *:
for p in saveprobe_test_ccv:
#print("save: probing %s" % callconv_str(p.cconv))
good = __save_probe1(pyPickler, save, p.addr)
#print(" ->", good)
if good:
return p.cconv
bad = "cannot determine save calling convention\n\n"
bad += "probed:\n"
for p in saveprobe_test_ccv:
bad += " - %s\t; callee_stkcleanup: %d\n" % (callconv_str(p.cconv), cfunc_is_callee_cleanup(p.addr))
bad += "\n"
bad += "save callee_stkcleanup: %d\n" % cfunc_is_callee_cleanup(save)
bad += "save disassembly:\n%s" % cfunc_disasm(save)
raise AssertionError(bad)
cdef bint __save_probe1(pyPickler, void* save, void* cfunc) except *:
# first see whether stack is cleaned up by caller or callee and how much.
# we need to do this first to avoid segfault if we patch save with cfunc
# with different stack cleanup as the probe.
save_stkclean = cfunc_is_callee_cleanup(save)
cfunc_stkclean = cfunc_is_callee_cleanup(cfunc)
if save_stkclean != cfunc_stkclean:
return False
# now when we know that save and cfunc have the same stack cleanup protocol, we can start probing
global saveprobe_ncall, saveprobe_self, saveprobe_obj, saveprobe_pers_save
saveprobe_ncall = 0
saveprobe_self = NULL
saveprobe_obj = NULL
saveprobe_pers_save = 0xdeafbeaf
class Null:
def write(self, data): pass
p = pyPickler(Null(), 0)
obj = object()
h = xfunchook_create()
xfunchook_prepare(h, &save, cfunc)
xfunchook_install(h, 0)
p.dump(obj)
xfunchook_uninstall(h, 0)
xfunchook_destroy(h)
assert saveprobe_ncall == 1, saveprobe_ncall
good = (saveprobe_self == <void*>p and \
saveprobe_obj == <void*>obj and \
saveprobe_pers_save == 0)
return good
cdef extern from * nogil:
r"""
static int saveprobe_ncall;
static void* saveprobe_self;
static void* saveprobe_obj;
static int saveprobe_pers_save;
static int saveprobe(void* self, PyObject* obj, int pers_save) {
saveprobe_ncall++;
saveprobe_self = self;
saveprobe_obj = obj;
saveprobe_pers_save = pers_save;
return 0; // do nothing
}
#define DEF_SAVEPROBE_builtin(ccname, callconv) \
static int callconv \
saveprobe_##ccname(void* self, PyObject* obj, int pers_save) { \
return saveprobe(self, obj, pers_save); \
}
#define DEF_SAVEPROBE_custom(ccname, _) \
extern "C" char saveprobe_##ccname;
#define DEF_SAVEPROBE(ccname, callconv, cckind) DEF_SAVEPROBE_##cckind(ccname, callconv)
FOR_EACH_CALLCONV(DEF_SAVEPROBE)
static std::vector<SaveFunc> saveprobe_test_ccv = {
#define CC_SAVEPROBE(ccname, _, __) \
SaveFunc{(void*)&saveprobe_##ccname, CALLCONV_##ccname},
FOR_EACH_CALLCONV(CC_SAVEPROBE)
};
// proxy for asm routines to invoke saveprobe
#ifdef LIBGOLANG_ARCH_386
extern "C" int CALLCONV(fastcall)
saveprobe_ifastcall(void* self, PyObject* obj, int pers_save) { \
return saveprobe(self, obj, pers_save); \
}
#endif
"""
int saveprobe_ncall
void* saveprobe_self
void* saveprobe_obj
int saveprobe_pers_save
vector[SaveFunc] saveprobe_test_ccv
# XXX doc save_invoke ...
# XXX place
cdef extern from *:
r"""
#define CC_SAVE_DEFCALL1_builtin(ccname, callconv)
#define CC_SAVE_DEFCALL1_custom(ccname, _) \
extern "C" int CALLCONV(fastcall) \
save_invoke_as_##ccname(void* save, void* self, PyObject* obj, int pers_save);
#define CC_SAVE_DEFCALL1(ccname, callconv, cckind) CC_SAVE_DEFCALL1_##cckind(ccname, callconv)
FOR_EACH_CALLCONV(CC_SAVE_DEFCALL1)
static int save_invoke(void* save, Callconv cconv, void* self, PyObject* obj, int pers_save) {
using namespace golang;
switch(cconv) {
#define CC_SAVE_CALL1_builtin(ccname, callconv) \
case CALLCONV_ ## ccname: \
return ((int (callconv *)(void*, PyObject*, int))save) \
(self, obj, pers_save);
#define CC_SAVE_CALL1_custom(ccname, _) \
case CALLCONV_ ## ccname: \
return save_invoke_as_##ccname(save, self, obj, pers_save);
#define CC_SAVE_CALL1(ccname, callconv, cckind) CC_SAVE_CALL1_##cckind(ccname, callconv)
FOR_EACH_CALLCONV(CC_SAVE_CALL1)
default:
panic("unreachable");
}
}
"""
int save_invoke(void* save, Callconv cconv, void* self, PyObject* obj, int pers_save) except -1
# - cfunc_direct_callees returns addresses of functions that cfunc calls directly.
#
# - cfunc_is_callee_cleanup determines whether cfunc does stack cleanup by
# itself and for how much.
#
# - cfunc_disassembly returns disassembly of cfunc.
#
# XXX dedup iterating instructions -> DisasmIter
cdef extern from "capstone/capstone.h" nogil:
r"""
#include <algorithm>
#include "golang/fmt.h"
#if defined(LIBGOLANG_ARCH_amd64)
# define MY_ARCH CS_ARCH_X86
# define MY_MODE CS_MODE_64
#elif defined(LIBGOLANG_ARCH_386)
# define MY_ARCH CS_ARCH_X86
# define MY_MODE CS_MODE_32
#elif defined(LIBGOLANG_ARCH_arm64)
# define MY_ARCH CS_ARCH_ARM64
# define MY_MODE CS_MODE_LITTLE_ENDIAN
#else
# error "unsupported architecture"
#endif
static std::tuple<uint64_t, bool> _insn_getimm1(cs_arch arch, cs_insn* ins);
std::vector<void*> cfunc_direct_callees(void *cfunc) {
const bool debug = false;
using namespace golang;
using std::tie;
using std::max;
std::vector<void*> vcallee;
csh h;
cs_insn* ins;
cs_err err;
cs_arch arch = MY_ARCH;
err = cs_open(arch, MY_MODE, &h);
if (err) {
fprintf(stderr, "cs_open: %s\n", cs_strerror(err));
panic(cs_strerror(err));
}
err = cs_option(h, CS_OPT_DETAIL, CS_OPT_ON);
if (err) {
fprintf(stderr, "cs_option: %s\n", cs_strerror(err));
panic(cs_strerror(err));
}
ins = cs_malloc(h);
if (ins == nil)
panic("cs_malloc failed");
const byte* code = (const byte*)cfunc;
size_t size = 10*1024; // something sane and limited
uint64_t addr = (uint64_t)cfunc;
uint64_t maxjump = addr;
while (cs_disasm_iter(h, &code, &size, &addr, ins)) {
if (debug)
fprintf(stderr, "0x%" PRIx64 ":\t%s\t\t%s\n", ins->address, ins->mnemonic, ins->op_str);
if (cs_insn_group(h, ins, CS_GRP_RET)) {
if (ins->address >= maxjump)
break;
continue;
}
uint64_t imm1;
bool imm1ok;
tie(imm1, imm1ok) = _insn_getimm1(arch, ins);
bool call = cs_insn_group(h, ins, CS_GRP_CALL);
bool jump = cs_insn_group(h, ins, CS_GRP_JUMP) && !call; // e.g. BL on arm64 is both jump and call
if (jump && imm1ok) {
maxjump = max(maxjump, imm1);
continue;
}
if (call && imm1ok) {
void* callee = (void*)imm1;
if (debug)
fprintf(stderr, " *** DIRECT CALL -> %p\n", callee);
if (!std::count(vcallee.begin(), vcallee.end(), callee))
vcallee.push_back(callee);
}
}
if (debug)
fprintf(stderr, "\n");
cs_free(ins, 1);
cs_close(&h);
return vcallee;
}
// _insn_getimm1 checks whether instruction comes with the sole immediate operand and returns it.
static std::tuple<uint64_t, bool> _insn_getimm1(cs_arch arch, cs_insn* ins) {
using namespace golang;
using std::make_tuple;
switch (arch) {
case CS_ARCH_X86: {
cs_x86* x86 = &(ins->detail->x86);
if (x86->op_count == 1) {
cs_x86_op* op = &(x86->operands[0]);
if (op->type == X86_OP_IMM)
return make_tuple(op->imm, true);
}
break;
}
case CS_ARCH_ARM64: {
cs_arm64* arm64 = &(ins->detail->arm64);
if (arm64->op_count == 1) {
cs_arm64_op* op = &(arm64->operands[0]);
if (op->type == ARM64_OP_IMM)
return make_tuple(op->imm, true);
}
break;
}
default:
panic("TODO");
}
return make_tuple(0, false);
}
int cfunc_is_callee_cleanup(void *cfunc) {
// only i386 might have callee-cleanup
// https://en.wikipedia.org/wiki/X86_calling_conventions#List_of_x86_calling_conventions
if (!(MY_ARCH == CS_ARCH_X86 && MY_MODE == CS_MODE_32))
return 0;
const bool debug = false;
int stkclean_by_callee = 0;
using namespace golang;
csh h;
cs_insn* ins;
cs_err err;
err = cs_open(MY_ARCH, MY_MODE, &h);
if (err) {
fprintf(stderr, "cs_open: %s\n", cs_strerror(err));
panic(cs_strerror(err));
}
err = cs_option(h, CS_OPT_DETAIL, CS_OPT_ON);
if (err) {
fprintf(stderr, "cs_option: %s\n", cs_strerror(err));
panic(cs_strerror(err));
}
ins = cs_malloc(h);
if (ins == nil)
panic("cs_malloc failed");
const byte* code = (const byte*)cfunc;
size_t size = 10*1024; // something sane and limited
uint64_t addr = (uint64_t)cfunc;
while (cs_disasm_iter(h, &code, &size, &addr, ins)) {
if (debug)
fprintf(stderr, "0x%" PRIx64 ":\t%s\t\t%s\n", ins->address, ins->mnemonic, ins->op_str);
if (!cs_insn_group(h, ins, CS_GRP_RET))
continue;
assert(ins->id == X86_INS_RET);
cs_x86* x86 = &(ins->detail->x86);
if (x86->op_count > 0) {
cs_x86_op* op = &(x86->operands[0]);
if (op->type == X86_OP_IMM)
stkclean_by_callee = op->imm;
}
break;
}
if (debug)
fprintf(stderr, " *** CLEANUP BY: %s (%d)\n", (stkclean_by_callee ? "callee" : "caller"), stkclean_by_callee);
cs_free(ins, 1);
cs_close(&h);
return stkclean_by_callee;
}
std::string cfunc_disasm(void *cfunc) {
using namespace golang;
string disasm;
csh h;
cs_insn* ins;
cs_err err;
err = cs_open(MY_ARCH, MY_MODE, &h);
if (err) {
fprintf(stderr, "cs_open: %s\n", cs_strerror(err));
panic(cs_strerror(err));
}
err = cs_option(h, CS_OPT_DETAIL, CS_OPT_ON);
if (err) {
fprintf(stderr, "cs_option: %s\n", cs_strerror(err));
panic(cs_strerror(err));
}
ins = cs_malloc(h);
if (ins == nil)
panic("cs_malloc failed");
const byte* code = (const byte*)cfunc;
size_t size = 10*1024; // something sane and limited
uint64_t addr = (uint64_t)cfunc;
while (cs_disasm_iter(h, &code, &size, &addr, ins)) {
disasm += fmt::sprintf("0x%" PRIx64 ":\t%s\t\t%s\n", ins->address, ins->mnemonic, ins->op_str);
// FIXME also handle forward jump like cfunc_direct_callees does
// should be done automatically after DisasmIter dedup
if (cs_insn_group(h, ins, CS_GRP_RET))
break;
}
cs_free(ins, 1);
cs_close(&h);
return disasm;
}
"""
vector[void*] cfunc_direct_callees(void* cfunc)
int cfunc_is_callee_cleanup(void* cfunc)
string cfunc_disasm(void* cfunc)
# _test_inside_counted depends on inside_counted and funchook, which we don't want to expose.
# -> include the test from here. Do the same for other low-level tests.
include '_golang_str_pickle_test.pyx'
# ---- misc ----
cdef PyMethodDef* tp_methods_lookup(PyMethodDef* methv, str name) except NULL:
m = &methv[0]
while m.ml_name != NULL:
if str(m.ml_name) == name:
return m
m += 1
raise KeyError("method %s not found" % name)
cdef PyMemberDef* tp_members_lookup(PyMemberDef* membv, str name) except NULL:
m = &membv[0]
while m.name != NULL:
if str(m.name) == name:
return m
m += 1
raise KeyError("member %s not found" % name)
golang/_golang_str_pickle_test.pyx 0 → 100644
View file @ 17dbfbac
# -*- coding: utf-8 -*-
# Copyright (C) 2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
# test for inside_counted
def _test_inside_counted(): # -> outok
outok = ''
outok += '\n\n\nBEFORE PATCH\n'
print('\n\n\nBEFORE PATCH')
tfunc(3)
t0 = ''
for i in range(3,0-1,-1):
t0 += '> tfunc(%d)\tinside_counter: 0\n' % i
for i in range(0,3+1,+1):
t0 += '< tfunc(%d)\tinside_counter: 0\n' % i
outok += t0
outok += '\n\n\nPATCHED\n'
print('\n\n\nPATCHED')
_patch = xfunchook_create()
global inside_counted_func
inside_counted_func = <void*>&tfunc
xfunchook_prepare(_patch, &inside_counted_func, <void*>inside_counted)
xfunchook_install(_patch, 0)
tfunc(12)
stk_size = 8 # = STK_SIZE from _golang_str_pickle.S
for i in range(12,0-1,-1):
outok += '> tfunc(%d)\tinside_counter: %d\n' % (i, min(12-i+1, stk_size))
for i in range(0,12+1,+1):
outok += '< tfunc(%d)\tinside_counter: %d\n' % (i, min(12-i+1, stk_size))
outok += '\n\n\nUNPATCHED\n'
print('\n\n\nUNPATCHED')
xfunchook_uninstall(_patch, 0)
tfunc(3)
outok += t0
return outok
cdef void tfunc(int x):
print('> tfunc(%d)\tinside_counter: %d' % (x, inside_counter))
if x > 0:
tfunc(x-1)
print('< tfunc(%d)\tinside_counter: %d' % (x, inside_counter))
def _test_cfunc_is_callee_cleanup():
for t in _cfunc_is_callee_cleanup_testv:
stkclean = cfunc_is_callee_cleanup(t.cfunc)
assert stkclean == t.stkclean_by_callee_ok, (t.cfunc_name, stkclean, t.stkclean_by_callee_ok)
cdef extern from * nogil:
r"""
struct _Test_cfunc_is_callee_clenup {
const char* cfunc_name;
void* cfunc;
int stkclean_by_callee_ok;
};
#define CASE(func, stkclean_ok) \
_Test_cfunc_is_callee_clenup{#func, (void*)func, stkclean_ok}
#if defined(LIBGOLANG_ARCH_386)
int CALLCONV(cdecl)
tfunc_cdecl1(int x) { return x; }
int CALLCONV(cdecl)
tfunc_cdecl2(int x, int y) { return x; }
int CALLCONV(cdecl)
tfunc_cdecl3(int x, int y, int z) { return x; }
int CALLCONV(stdcall)
tfunc_stdcall1(int x) { return x; }
int CALLCONV(stdcall)
tfunc_stdcall2(int x, int y) { return x; }
int CALLCONV(stdcall)
tfunc_stdcall3(int x, int y, int z) { return x; }
int CALLCONV(fastcall)
tfunc_fastcall1(int x) { return x; }
int CALLCONV(fastcall)
tfunc_fastcall2(int x, int y) { return x; }
int CALLCONV(fastcall)
tfunc_fastcall3(int x, int y, int z) { return x; }
#ifndef LIBGOLANG_CC_msc // see note about C3865 in FOR_EACH_CALLCONV
int CALLCONV(thiscall)
tfunc_thiscall1(int x) { return x; }
int CALLCONV(thiscall)
tfunc_thiscall2(int x, int y) { return x; }
int CALLCONV(thiscall)
tfunc_thiscall3(int x, int y, int z) { return x; }
#endif
#ifndef LIBGOLANG_CC_msc // no regparm on MSCV
int CALLCONV(regparm(1))
tfunc_regparm1_1(int x) { return x; }
int CALLCONV(regparm(1))
tfunc_regparm1_2(int x, int y) { return x; }
int CALLCONV(regparm(1))
tfunc_regparm1_3(int x, int y, int z) { return x; }
int CALLCONV(regparm(2))
tfunc_regparm2_1(int x) { return x; }
int CALLCONV(regparm(2))
tfunc_regparm2_2(int x, int y) { return x; }
int CALLCONV(regparm(2))
tfunc_regparm2_3(int x, int y, int z) { return x; }
int CALLCONV(regparm(3))
tfunc_regparm3_1(int x) { return x; }
int CALLCONV(regparm(3))
tfunc_regparm3_2(int x, int y) { return x; }
int CALLCONV(regparm(3))
tfunc_regparm3_3(int x, int y, int z) { return x; }
#endif
static std::vector<_Test_cfunc_is_callee_clenup> _cfunc_is_callee_cleanup_testv = {
CASE(tfunc_cdecl1 , 0 * 4),
CASE(tfunc_cdecl2 , 0 * 4),
CASE(tfunc_cdecl3 , 0 * 4),
CASE(tfunc_stdcall1 , 1 * 4),
CASE(tfunc_stdcall2 , 2 * 4),
CASE(tfunc_stdcall3 , 3 * 4),
CASE(tfunc_fastcall1 , 0 * 4),
CASE(tfunc_fastcall2 , 0 * 4),
CASE(tfunc_fastcall3 , 1 * 4),
#ifndef LIBGOLANG_CC_msc
CASE(tfunc_thiscall1 , 0 * 4),
CASE(tfunc_thiscall2 , 1 * 4),
CASE(tfunc_thiscall3 , 2 * 4),
#endif
#ifndef LIBGOLANG_CC_msc
CASE(tfunc_regparm1_1 , 0 * 4),
CASE(tfunc_regparm1_2 , 0 * 4),
CASE(tfunc_regparm1_3 , 0 * 4),
CASE(tfunc_regparm2_1 , 0 * 4),
CASE(tfunc_regparm2_2 , 0 * 4),
CASE(tfunc_regparm2_3 , 0 * 4),
CASE(tfunc_regparm3_1 , 0 * 4),
CASE(tfunc_regparm3_2 , 0 * 4),
CASE(tfunc_regparm3_3 , 0 * 4),
#endif
};
#else
// only i386 has many calling conventions
int tfunc_default(int x, int y, int z) { return x; }
static std::vector<_Test_cfunc_is_callee_clenup> _cfunc_is_callee_cleanup_testv = {
CASE(tfunc_default, 0),
};
#endif
#undef CASE
"""
struct _Test_cfunc_is_callee_clenup:
const char* cfunc_name
void* cfunc
int stkclean_by_callee_ok
vector[_Test_cfunc_is_callee_clenup] _cfunc_is_callee_cleanup_testv
golang/_strconv.pyx
View file @ 17dbfbac
......@@ -28,12 +28,11 @@ from golang cimport pyb, byte, rune
from golang cimport _utf8_decode_rune, _xunichr
from golang.unicode cimport utf8
from cpython cimport PyObject
from cpython cimport PyObject, _PyBytes_Resize
cdef extern from "Python.h":
PyObject* PyBytes_FromStringAndSize(char*, Py_ssize_t) except NULL
char* PyBytes_AS_STRING(PyObject*)
int _PyBytes_Resize(PyObject**, Py_ssize_t) except -1
void Py_DECREF(PyObject*)
......@@ -65,7 +64,7 @@ cdef bytes _quote(const byte[::1] s, char quote, bint* out_nonascii_escape): # -
cdef byte c
q[0] = quote; q += 1
while i < len(s):
c = s[i]
c = s[i] # XXX -> use raw pointer in the loop
# fast path - ASCII only
if c < 0x80:
if c in (ord('\\'), quote):
......@@ -104,7 +103,8 @@ cdef bytes _quote(const byte[::1] s, char quote, bint* out_nonascii_escape): # -
# slow path - full UTF-8 decoding + unicodedata
else:
r, size = _utf8_decode_rune(s[i:])
# XXX optimize non-ascii case
r, size = _utf8_decode_rune(s[i:]) # XXX -> raw pointer
isize = i + size
# decode error - just emit raw byte as escaped
......@@ -117,6 +117,9 @@ cdef bytes _quote(const byte[::1] s, char quote, bint* out_nonascii_escape): # -
q += 4
# printable utf-8 characters go as is
# XXX ? use Py_UNICODE_ISPRINTABLE (py3, not available on py2) ?
# XXX ? and generate C table based on unicodedata for py2 ?
# XXX -> generate table based on unicodedata for both py2/py3 because Py_UNICODE_ISPRINTABLE is not exactly what matches strconv.IsPrint (i.e. cat starts from LNPS)
elif _unicodedata_category(_xunichr(r))[0] in 'LNPS': # letters, numbers, punctuation, symbols
for j in range(i, isize):
q[0] = s[j]
......
golang/fmt.h
View file @ 17dbfbac
......@@ -111,7 +111,7 @@ inline error errorf(const string& format, Argv... argv) {
// `const char *` overloads just to catch format mistakes as
// __attribute__(format) does not work with std::string.
LIBGOLANG_API string sprintf(const char *format, ...)
#ifndef _MSC_VER
#ifndef LIBGOLANG_CC_msc
__attribute__ ((format (printf, 1, 2)))
#endif
;
......
golang/golang_str_pickle_test.py 0 → 100644
View file @ 17dbfbac
# -*- coding: utf-8 -*-
# Copyright (C) 2022-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from __future__ import print_function, absolute_import
from golang import b, u, bstr, ustr
from golang.golang_str_test import xbytes, x32, unicode
from golang._golang import _test_inside_counted, _test_cfunc_is_callee_cleanup
from gpython.gpython_test import is_gpython
from pytest import raises, fixture, mark
import sys, io, struct
import six
# run all tests on all py/c pickle modules we aim to support
import pickle as stdPickle
if six.PY2:
import cPickle
else:
import _pickle as cPickle
from zodbpickle import slowpickle as zslowPickle
from zodbpickle import fastpickle as zfastPickle
from zodbpickle import pickle as zpickle
from zodbpickle import _pickle as _zpickle
import pickletools as stdpickletools
if six.PY2:
from zodbpickle import pickletools_2 as zpickletools
else:
from zodbpickle import pickletools_3 as zpickletools
# pickle is pytest fixture that yields all variants of pickle module.
@fixture(scope="function", params=[stdPickle, cPickle,
zslowPickle, zfastPickle, zpickle, _zpickle])
def pickle(request):
yield request.param
# pickletools is pytest fixture that yields all variants of pickletools module.
@fixture(scope="function", params=[stdpickletools, zpickletools])
def pickletools(request):
yield request.param
# pickle2tools returns pickletools module that corresponds to module pickle.
def pickle2tools(pickle):
if pickle in (stdPickle, cPickle):
return stdpickletools
else:
return zpickletools
# @gpystr_only is marker to run a test only under gpython -X gpython.strings=bstr+ustr
is_gpystr = type(u'') is ustr
gpystr_only = mark.skipif(not is_gpystr, reason="gpystr-only test")
# ---- pickling/unpickling under gpystr ----
# verify that loading *STRING opcodes loads them as bstr on gpython by default.
# TODO or with encoding='bstr' under plain py
@gpystr_only
def test_string_pickle_load_STRING(pickle):
p_str = b"S'\\xd0\\xbc\\xd0\\xb8\\xd1\\x80\\xff'\n." # STRING 'мир\xff'
p_utf8 = b"S'"+xbytes('мир')+b"\\xff'\n." # STRING 'мир\xff'
p_sbins = b'U\x07\xd0\xbc\xd0\xb8\xd1\x80\xff.' # SHORT_BINSTRING 'мир\xff'
p_bins = b'T\x07\x00\x00\x00\xd0\xbc\xd0\xb8\xd1\x80\xff.' # BINSTRING 'мир\xff'
p_bytes = xbytes('мир')+b'\xff'
# check invokes f on all test pickles
def check(f):
f(p_str)
f(p_utf8)
f(p_sbins)
f(p_bins)
# default -> bstr on both py2 and py3
# TODO only this check is gpystr_only -> remove whole-func @gpystr_only
def _(p):
obj = xloads(pickle, p)
assert type(obj) is bstr
assert obj == p_bytes
check(_)
# also test bstr inside tuple (for symmetry with save)
def _(p):
p_ = b'(' + p[:-1] + b't.'
tobj = xloads(pickle, p_)
assert type(tobj) is tuple
assert len(tobj) == 1
obj = tobj[0]
assert type(obj) is bstr
assert obj == p_bytes
check(_)
# pickle supports encoding=... only on py3
if six.PY3:
# encoding='bstr' -> bstr
def _(p):
obj = xloads(pickle, p, encoding='bstr')
assert type(obj) is bstr
assert obj == p_bytes
check(_)
# encoding='bytes' -> bytes
def _(p):
obj = xloads(pickle, p, encoding='bytes')
assert type(obj) is bytes
assert obj == p_bytes
check(_)
# encoding='utf-8' -> UnicodeDecodeError
def _(p):
with raises(UnicodeDecodeError):
xloads(pickle, p, encoding='utf-8')
check(_)
# encoding='utf-8', errors=... -> unicode
def _(p):
obj = xloads(pickle, p, encoding='utf-8', errors='backslashreplace')
assert type(obj) is unicode
assert obj == u'мир\\xff'
check(_)
# verify that saving bstr results in *STRING opcodes on gpython.
@gpystr_only
def test_strings_pickle_save_STRING(pickle):
s = s0 = b(xbytes('мир')+b'\xff')
assert type(s) is bstr
p_utf8 = b"S'"+xbytes('мир')+b"\\xff'\n." # STRING 'мир\xff'
p_sbins = b'U\x07\xd0\xbc\xd0\xb8\xd1\x80\xff.' # SHORT_BINSTRING 'мир\xff'
p_bins = b'T\x07\x00\x00\x00\xd0\xbc\xd0\xb8\xd1\x80\xff.' # BINSTRING 'мир\xff'
def dumps(proto):
return xdumps(pickle, s, proto)
assert dumps(0) == p_utf8
for proto in range(1, HIGHEST_PROTOCOL(pickle)+1):
assert dumps(proto) == p_sbins
# BINSTRING
s += b'\x55'*0x100
p_bins_ = p_bins[:2] + b'\x01' + p_bins[3:-1] + b'\x55'*0x100 + b'.'
for proto in range(1, HIGHEST_PROTOCOL(pickle)+1):
assert dumps(proto) == p_bins_
# also test bstr inside tuple to verify that what we patched is actually
# _pickle.save that is invoked from inside other save_X functions.
s = (s0,)
p_tutf8 = b'(' + p_utf8[:-1] + b't.'
p_tsbins = b'(' + p_sbins[:-1] + b't.'
assert dumps(0) == p_tutf8
assert dumps(1) == p_tsbins
# don't test proto ≥ 2 because they start to use TUPLE1 instead of TUPLE
# verify that loading *UNICODE opcodes loads them as unicode/ustr.
# this is standard behaviour but we verify it since we patch pickle's strings processing.
# also verify save lightly for symmetry.
# NOTE not @gpystr_only
def test_string_pickle_loadsave_UNICODE(pickle):
# NOTE builtin pickle behaviour is to save unicode via 'surrogatepass' error handler
# this means that b'мир\xff' -> ustr/unicode -> save will emit *UNICODE with
# b'мир\xed\xb3\xbf' instead of b'мир\xff' as data.
p_uni = b'V\\u043c\\u0438\\u0440\\udcff\n.' # UNICODE 'мир\uDCFF'
p_binu = b'X\x09\x00\x00\x00\xd0\xbc\xd0\xb8\xd1\x80\xed\xb3\xbf.' # BINUNICODE NOTE ...edb3bf not ...ff
p_sbinu = b'\x8c\x09\xd0\xbc\xd0\xb8\xd1\x80\xed\xb3\xbf.' # SHORT_BINUNICODE
p_binu8 = b'\x8d\x09\x00\x00\x00\x00\x00\x00\x00\xd0\xbc\xd0\xb8\xd1\x80\xed\xb3\xbf.' # BINUNICODE8
u_obj = u'мир\uDCFF'; assert type(u_obj) is unicode
# load: check invokes f on all test pickles that pickle should support
def check(f):
f(p_uni)
f(p_binu)
if HIGHEST_PROTOCOL(pickle) >= 4:
f(p_sbinu)
f(p_binu8)
def _(p):
obj = xloads(pickle, p)
assert type(obj) is unicode
assert obj == u_obj
check(_)
# save
def dumps(proto):
return xdumps(pickle, u_obj, proto)
assert dumps(0) == p_uni
assert dumps(1) == p_binu
assert dumps(2) == p_binu
if HIGHEST_PROTOCOL(pickle) >= 3:
assert dumps(3) == p_binu
if HIGHEST_PROTOCOL(pickle) >= 4:
assert dumps(4) == p_sbinu
# ---- pickling/unpickling generally without gpystr ----
# verify that bstr/ustr can be pickled/unpickled correctly on !gpystr.
# gpystr should also load ok what was pickled on !gpystr.
# for uniformity gpystr is also verified to save/load objects correctly.
# However the main gpystr tests are load/save tests for *STRING and *UNICODE above.
def test_strings_pickle_bstr_ustr(pickle):
bs = b(xbytes('мир')+b'\xff')
us = u(xbytes('май')+b'\xff')
def diss(p): return xdiss(pickle2tools(pickle), p)
def dis(p): print(diss(p))
# assert_pickle verifies that pickling obj results in
#
# - dumps_ok_gpystr (when run under gpython with gpython.string=bstr+ustr), or
# - dumps_ok_stdstr (when run under plain python or gpython with gpython.strings=pystd)
#
# and that unpickling results back in obj.
#
# gpystr should also unpickle !gpystr pickle correctly.
assert HIGHEST_PROTOCOL(pickle) <= 5
def assert_pickle(obj, proto, dumps_ok_gpystr, dumps_ok_stdstr):
if proto > HIGHEST_PROTOCOL(pickle):
with raises(ValueError):
xdumps(pickle, obj, proto)
return
p = xdumps(pickle, obj, proto)
if not is_gpystr:
assert p == dumps_ok_stdstr, diss(p)
dumps_okv = [dumps_ok_stdstr]
else:
assert p == dumps_ok_gpystr, diss(p)
dumps_okv = [dumps_ok_gpystr, dumps_ok_stdstr]
for p in dumps_okv:
#dis(p)
obj2 = xloads(pickle, p)
assert type(obj2) is type(obj)
assert obj2 == obj
_ = assert_pickle
_(bs, 0, xbytes("S'мир\\xff'\n."), # STRING
b"cgolang\nbstr\n(V\\u043c\\u0438\\u0440\\udcff\ntR.") # bstr(UNICODE)
_(us, 0, b'V\\u043c\\u0430\\u0439\\udcff\n.', # UNICODE
b'cgolang\nustr\n(V\\u043c\\u0430\\u0439\\udcff\ntR.') # ustr(UNICODE)
_(bs, 1, b'U\x07\xd0\xbc\xd0\xb8\xd1\x80\xff.', # SHORT_BINSTRING
b'cgolang\nbstr\n(X\x09\x00\x00\x00' # bstr(BINUNICODE)
b'\xd0\xbc\xd0\xb8\xd1\x80\xed\xb3\xbftR.')
# NOTE BINUNICODE ...edb3bf not ...ff (see test_string_pickle_loadsave_UNICODE for details)
_(us, 1, b'X\x09\x00\x00\x00\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf.', # BINUNICODE
b'cgolang\nustr\n(X\x09\x00\x00\x00' # bstr(BINUNICODE)
b'\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbftR.')
_(bs, 2, b'U\x07\xd0\xbc\xd0\xb8\xd1\x80\xff.', # SHORT_BINSTRING
b'cgolang\nbstr\nX\x09\x00\x00\x00' # bstr(BINUNICODE)
b'\xd0\xbc\xd0\xb8\xd1\x80\xed\xb3\xbf\x85\x81.')
_(us, 2, b'X\x09\x00\x00\x00\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf.', # BINUNICODE
b'cgolang\nustr\nX\x09\x00\x00\x00' # ustr(BINUNICODE)
b'\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf\x85\x81.')
_(bs, 3, b'U\x07\xd0\xbc\xd0\xb8\xd1\x80\xff.', # SHORT_BINSTRING
b'cgolang\nbstr\nC\x07\xd0\xbc\xd0\xb8\xd1\x80\xff\x85\x81.') # bstr(SHORT_BINBYTES)
_(us, 3, b'X\x09\x00\x00\x00\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf.', # BINUNICODE
b'cgolang\nustr\nX\x09\x00\x00\x00' # ustr(BINUNICODE)
b'\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf\x85\x81.')
for p in (4,5):
_(bs, p,
b'U\x07\xd0\xbc\xd0\xb8\xd1\x80\xff.', # SHORT_BINSTRING
b'\x8c\x06golang\x8c\x04bstr\x93C\x07' # bstr(SHORT_BINBYTES)
b'\xd0\xbc\xd0\xb8\xd1\x80\xff\x85\x81.')
_(us, p,
b'\x8c\x09\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf.', # SHORT_BINUNICODE
b'\x8c\x06golang\x8c\x04ustr\x93\x8c\x09' # ustr(SHORT_BINUNICODE)
b'\xd0\xbc\xd0\xb0\xd0\xb9\xed\xb3\xbf\x85\x81.')
# ---- disassembly ----
# xdiss returns disassembly of a pickle as string.
def xdiss(pickletools, p): # -> str
out = six.StringIO()
pickletools.dis(p, out)
return out.getvalue()
# verify that disassembling *STRING opcodes works with treating strings as UTF8b.
@gpystr_only
def test_string_pickle_dis_STRING(pickletools):
p_str = b"S'\\xd0\\xbc\\xd0\\xb8\\xd1\\x80'\n." # STRING 'мир'
p_sbins = b'U\x06\xd0\xbc\xd0\xb8\xd1\x80.' # SHORT_BINSTRING 'мир'
p_bins = b'T\x06\x00\x00\x00\xd0\xbc\xd0\xb8\xd1\x80.' # BINSTRING 'мир'
bmir = x32("b('мир')", "'мир'")
assert xdiss(pickletools, p_str) == """\
0: S STRING %s
28: . STOP
highest protocol among opcodes = 0
""" % bmir
assert xdiss(pickletools, p_sbins) == """\
0: U SHORT_BINSTRING %s
8: . STOP
highest protocol among opcodes = 1
""" % bmir
assert xdiss(pickletools, p_bins) == """\
0: T BINSTRING %s
11: . STOP
highest protocol among opcodes = 1
""" % bmir
# ---- loads and normalized dumps ----
# xloads loads pickle p via pickle.loads
# it also verifies that .load and Unpickler.load give the same result.
def xloads(pickle, p, **kw):
obj1 = _xpickle_attr(pickle, 'loads')(p, **kw)
obj2 = _xpickle_attr(pickle, 'load') (io.BytesIO(p), **kw)
obj3 = _xpickle_attr(pickle, 'Unpickler')(io.BytesIO(p), **kw).load()
assert type(obj2) is type(obj1)
assert type(obj3) is type(obj1)
assert obj1 == obj2 == obj3
return obj1
# xdumps dumps obj via pickle.dumps
# it also verifies that .dump and Pickler.dump give the same.
# the pickle is returned in normalized form - see pickle_normalize for details.
def xdumps(pickle, obj, proto, **kw):
p1 = _xpickle_attr(pickle, 'dumps')(obj, proto, **kw)
f2 = io.BytesIO(); _xpickle_attr(pickle, 'dump')(obj, f2, proto, **kw)
p2 = f2.getvalue()
f3 = io.BytesIO(); _xpickle_attr(pickle, 'Pickler')(f3, proto, **kw).dump(obj)
p3 = f3.getvalue()
assert type(p1) is bytes
assert type(p2) is bytes
assert type(p3) is bytes
assert p1 == p2 == p3
# remove not interesting parts: PROTO / FRAME header and unused PUTs
if proto >= 2:
protover = PROTO(proto)
assert p1.startswith(protover)
return pickle_normalize(pickle2tools(pickle), p1)
def _xpickle_attr(pickle, name):
# on py3 pickle.py tries to import from C _pickle to optimize by default
# -> verify py version if we are asked to test pickle.py
if six.PY3 and (pickle is stdPickle):
assert getattr(pickle, name) is getattr(cPickle, name)
name = '_'+name
return getattr(pickle, name)
# pickle_normalize returns normalized version of pickle p.
#
# - PROTO and FRAME opcodes are removed from header,
# - unused PUT, BINPUT and MEMOIZE opcodes - those without corresponding GET are removed,
# - *PUT indices start from 0 (this unifies cPickle with pickle).
def pickle_normalize(pickletools, p):
def iter_pickle(p): # -> i(op, arg, pdata)
op_prev = None
arg_prev = None
pos_prev = None
for op, arg, pos in pickletools.genops(p):
if op_prev is not None:
pdata_prev = p[pos_prev:pos]
yield (op_prev, arg_prev, pdata_prev)
op_prev = op
arg_prev = arg
pos_prev = pos
if op_prev is not None:
yield (op_prev, arg_prev, p[pos_prev:])
memo_oldnew = {} # idx used in original pop/get -> new index | None if not get
idx = 0
for op, arg, pdata in iter_pickle(p):
if 'PUT' in op.name:
memo_oldnew.setdefault(arg, None)
elif 'MEMOIZE' in op.name:
memo_oldnew.setdefault(len(memo_oldnew), None)
elif 'GET' in op.name:
if memo_oldnew.get(arg) is None:
memo_oldnew[arg] = idx
idx += 1
pout = b''
memo_old = set() # idx used in original pop
for op, arg, pdata in iter_pickle(p):
if op.name in ('PROTO', 'FRAME'):
continue
if 'PUT' in op.name:
memo_old.add(arg)
newidx = memo_oldnew.get(arg)
if newidx is None:
continue
pdata = globals()[op.name](newidx)
if 'MEMOIZE' in op.name:
idx = len(memo_old)
memo_old.add(idx)
newidx = memo_oldnew.get(idx)
if newidx is None:
continue
if 'GET' in op.name:
newidx = memo_oldnew[arg]
assert newidx is not None
pdata = globals()[op.name](newidx)
pout += pdata
return pout
P = struct.pack
def PROTO(version): return b'\x80' + P('<B', version)
def FRAME(size): return b'\x95' + P('<Q', size)
def GET(idx): return b'g%d\n' % (idx,)
def PUT(idx): return b'p%d\n' % (idx,)
def BINPUT(idx): return b'q' + P('<B', idx)
def BINGET(idx): return b'h' + P('<B', idx)
def LONG_BINPUT(idx): return b'r' + P('<I', idx)
def LONG_BINGET(idx): return b'j' + P('<I', idx)
MEMOIZE = b'\x94'
def test_pickle_normalize(pickletools):
def diss(p):
return xdiss(pickletools, p)
proto = 0
for op in pickletools.opcodes:
proto = max(proto, op.proto)
assert proto >= 2
def _(p, p_normok):
p_norm = pickle_normalize(pickletools, p)
assert p_norm == p_normok, diss(p_norm)
_(b'.', b'.')
_(b'I1\n.', b'I1\n.')
_(PROTO(2)+b'I1\n.', b'I1\n.')
putgetv = [(PUT,GET), (BINPUT, BINGET)]
if proto >= 4:
putgetv.append((LONG_BINPUT, LONG_BINGET))
for (put,get) in putgetv:
_(b'(I1\n'+put(1) + b'I2\n'+put(2) +b't'+put(3)+b'0'+get(3)+put(4)+b'.',
b'(I1\nI2\nt'+put(0)+b'0'+get(0)+b'.')
if proto >= 4:
_(FRAME(4)+b'I1\n.', b'I1\n.')
_(b'I1\n'+MEMOIZE+b'I2\n'+MEMOIZE+GET(0)+b'.',
b'I1\n'+MEMOIZE+b'I2\n'+GET(0)+b'.')
# ---- internals of patching ----
# being able to cPickle bstr as STRING depends on proper working of inside_counted function.
# Verify it with dedicated unit test.
def test_inside_counted(capsys):
outok = _test_inside_counted()
_ = capsys.readouterr()
if _.err:
print(_.err, file=sys.stderr)
assert _.out == outok
def test_cfunc_is_callee_cleanup():
_test_cfunc_is_callee_cleanup()
# verify that what we patched - e.g. PyUnicode_Decode - stay unaffected when
# called outside of bstr/ustr context.
# NOTE this test complements test_strings_patched_transparently in golang_str_test.py
def test_pickle_strings_patched_transparently():
# PyUnicode_Decode stays working and unaffected
b_ = xbytes("abc")
_ = b_.decode(); assert type(_) is unicode; assert _ == u"abc"
_ = b_.decode("utf8"); assert type(_) is unicode; assert _ == u"abc"
_ = b_.decode("ascii"); assert type(_) is unicode; assert _ == u"abc"
b_ = xbytes("мир")
_ = b_.decode("utf8"); assert type(_) is unicode; assert _ == u"мир"
with raises(UnicodeDecodeError):
b_.decode("ascii")
# ---- misc ----
# HIGHEST_PROTOCOL returns highest protocol supported by pickle.
def HIGHEST_PROTOCOL(pickle):
if six.PY3 and pickle is cPickle:
pmax = stdPickle.HIGHEST_PROTOCOL # py3: _pickle has no .HIGHEST_PROTOCOL
elif six.PY3 and pickle is _zpickle:
pmax = zpickle.HIGHEST_PROTOCOL # ----//---- for _zpickle
else:
pmax = pickle.HIGHEST_PROTOCOL
assert pmax >= 2
return pmax
golang/golang_str_test.py
View file @ 17dbfbac
......@@ -146,9 +146,17 @@ def test_strings_basic():
_ = ustr(123); assert type(_) is ustr; assert _ == '123'
_ = bstr([1,'β']); assert type(_) is bstr; assert _ == "[1, 'β']"
_ = ustr([1,'β']); assert type(_) is ustr; assert _ == "[1, 'β']"
obj = object()
_ = bstr(obj); assert type(_) is bstr; assert _ == str(obj) # <object ...>
_ = ustr(obj); assert type(_) is ustr; assert _ == str(obj) # <object ...>
obj = object(); assert str(obj).startswith('<object object at 0x')
_ = bstr(obj); assert type(_) is bstr; assert _ == str(obj)
_ = ustr(obj); assert type(_) is ustr; assert _ == str(obj)
ecls = RuntimeError; assert str(ecls) == x32("<class 'RuntimeError'>",
"<type 'exceptions.RuntimeError'>")
_ = bstr(ecls); assert type(_) is bstr; assert _ == str(ecls)
_ = ustr(ecls); assert type(_) is ustr; assert _ == str(ecls)
exc = RuntimeError('zzz'); assert str(exc) == 'zzz'
_ = bstr(exc); assert type(_) is bstr; assert _ == str(exc)
_ = ustr(exc); assert type(_) is ustr; assert _ == str(exc)
# when stringifying they also handle bytes/bytearray inside containers as UTF-8 strings
_ = bstr([xunicode( 'β')]); assert type(_) is bstr; assert _ == "['β']"
......@@ -246,10 +254,12 @@ def test_strings_basic():
assert hash(bs) == hash("мир"); assert bs == "мир"
# str/repr
def rb(x,y): return xb32(x, 'b'+y,y)
def ru(x,y): return xu32(x, y,'u'+y)
_ = str(us); assert isinstance(_, str); assert _ == "мир"
_ = str(bs); assert isinstance(_, str); assert _ == "мир"
_ = repr(us); assert isinstance(_, str); assert _ == "u('мир')"
_ = repr(bs); assert isinstance(_, str); assert _ == "b('мир')"
_ = repr(us); assert isinstance(_, str); assert _ == ru("u('мир')", "'мир'")
_ = repr(bs); assert isinstance(_, str); assert _ == rb("b('мир')", "'мир'")
# str/repr of non-valid utf8
b_hik8 = xbytes ('привет ')+b(k8mir_bytes); assert type(b_hik8) is bstr
......@@ -259,11 +269,17 @@ def test_strings_basic():
_ = str(u_hik8); assert isinstance(_, str); assert _ == xbytes('привет ')+b'\xcd\xc9\xd2'
_ = str(b_hik8); assert isinstance(_, str); assert _ == xbytes('привет ')+b'\xcd\xc9\xd2'
_ = repr(u_hik8); assert isinstance(_, str); assert _ == r"u(b'привет \xcd\xc9\xd2')"
_ = repr(b_hik8); assert isinstance(_, str); assert _ == r"b(b'привет \xcd\xc9\xd2')"
_ = repr(u_hik8); assert isinstance(_, str); assert _ == r"u(b'привет \xcd\xc9\xd2')"
# NOTE ^^^ same for u,3/2
_ = repr(b_hik8); assert isinstance(_, str); assert _ == rb(r"b(b'привет \xcd\xc9\xd2')",
r"'привет \xcd\xc9\xd2'")
# str/repr of quotes
def _(text, breprok, ureprok):
assert breprok[:2] == "b("; assert breprok[-1] == ")"
assert ureprok[:2] == "u("; assert ureprok[-1] == ")"
breprok = rb(breprok, breprok[2:-1]) # b('...') or '...' if bytes patched
ureprok = ru(ureprok, ureprok[2:-1]) # u('...') or '...' if unicode patched
bt = b(text); assert type(bt) is bstr
ut = u(text); assert type(ut) is ustr
_ = str(bt); assert isinstance(_, str); assert _ == text
......@@ -286,20 +302,26 @@ def test_strings_basic():
# verify that bstr/ustr are created with correct refcount.
def test_strings_refcount():
# buffer with string data - not bytes nor unicode so that when builting
# string types are patched no case where bytes is created from the same
# bytes, or unicode is created from the same unicode - only increasing
# refcount of original object.
data = bytearray([ord('a'), ord('b'), ord('c'), ord('4')])
# first verify our logic on std type
obj = xbytes(u'abc'); assert type(obj) is bytes
obj = bytes(data); assert type(obj) is bytes
gc.collect(); assert sys.getrefcount(obj) == 1+1 # +1 due to obj passed to getrefcount call
# bstr
obj = b('abc'); assert type(obj) is bstr
obj = b(data); assert type(obj) is bstr
gc.collect(); assert sys.getrefcount(obj) == 1+1
obj = bstr('abc'); assert type(obj) is bstr
obj = bstr(data); assert type(obj) is bstr
gc.collect(); assert sys.getrefcount(obj) == 1+1
# ustr
obj = u('abc'); assert type(obj) is ustr
obj = u(data); assert type(obj) is ustr
gc.collect(); assert sys.getrefcount(obj) == 1+1
obj = ustr('abc'); assert type(obj) is ustr
obj = ustr(data); assert type(obj) is ustr
gc.collect(); assert sys.getrefcount(obj) == 1+1
......@@ -326,26 +348,6 @@ def test_strings_memoryview():
assert _(5) == 0x80
# verify that bstr/ustr can be pickled/unpickled correctly.
def test_strings_pickle():
bs = b("мир")
us = u("май")
#from pickletools import dis
for proto in range(0, pickle.HIGHEST_PROTOCOL+1):
p_bs = pickle.dumps(bs, proto)
#dis(p_bs)
bs_ = pickle.loads(p_bs)
assert type(bs_) is bstr
assert bs_ == bs
p_us = pickle.dumps(us, proto)
#dis(p_us)
us_ = pickle.loads(p_us)
assert type(us_) is ustr
assert us_ == us
# verify that ord on bstr/ustr works as expected.
def test_strings_ord():
with raises(TypeError): ord(b(''))
......@@ -617,7 +619,8 @@ def test_strings_iter():
# iter( b/u/unicode ) -> iterate unicode characters
# NOTE that iter(b) too yields unicode characters - not integers or bytes
bi = iter(bs)
#bi = iter(bs) # XXX temp disabled
bi = iter(us)
ui = iter(us)
ui_ = iter(u_)
class XIter:
......@@ -1100,64 +1103,65 @@ def test_strings_mod_and_format():
# _bprintf parses %-format ourselves. Verify that parsing first
# NOTE here all strings are plain ASCII.
def _(fmt, args):
def _(fmt, args, ok):
fmt = '*str '+fmt
for l in range(len(fmt), -1, -1):
# [:len(fmt)] verifies original case
# [:l<len] should verify "incomplete format" parsing
verify_fmt_all_types(lambda fmt, args: fmt % args,
fmt[:l], args, excok=True)
_('%(name)s', {'name': 123})
_('%x', 123) # flags
_('%#x', 123)
_('%05d', 123)
_('%-5d', 123)
_('% d', 123)
_('% d', -123)
_('%+d', -123)
_('%5d', 123) # width
_('%*d', (5,123))
_('%f', 1.234) # .prec
_('%.f', 1.234)
_('%.1f', 1.234)
_('%.2f', 1.234)
_('%*f', (2,1.234))
_('%hi', 123) # len
_('%li', 123)
_('%Li', 123)
_('%%', ()) # %%
_('%10.4f', 1.234) # multiple features
_('%(x)10.4f', {'y':0, 'x':1.234})
_('%*.*f', (10,4,1.234))
_('', {}) # not all arguments converted
_('', [])
_('', 123)
_('', '123')
_('%s', ()) # not enough arguments to format
_('%s %s', 123)
_('%s %s', (123,))
_('%(x)s', 123) # format requires a mapping
_('%(x)s', (123,))
_('%s %(x)s', (123,4))
_('%(x)s %s', (123,4))
_('%(x)s %s', {'x':1}) # mixing tuple/dict
_('%s %(x)s', {'x':1})
_('abc %z', 1) # unsupported format character
_('abc %44z', 1)
if isinstance(ok, Exception):
excok = True
else:
ok = '*str '+ok
excok = False
verify_fmt_all_types(lambda fmt, args: fmt % args, fmt, args, ok, excok=excok)
# also automatically verify "incomplete format" parsing via fmt[:l<len]
# this works effectively only when run under std python though.
for l in range(len(fmt)-1, -1, -1):
verify_fmt_all_types(lambda fmt, args: fmt % args, fmt[:l], args, excok=True)
_('%(name)s', {'name': 123} , '123')
_('%x', 123 , '7b') # flags
_('%#x', 123 , '0x7b')
_('%05d', 123 , '00123')
_('%-5d', 123 , '123 ')
_('% d', 123 , ' 123')
_('% d', -123 , '-123')
_('%+d', 123 , '+123')
_('%+d', -123 , '-123')
_('%5d', 123 , ' 123') # width
_('%*d', (5,123) , ' 123')
_('%f', 1.234 , '1.234000') # .prec
_('%.f', 1.234 , '1')
_('%.1f', 1.234 , '1.2')
_('%.2f', 1.234 , '1.23')
_('%*f', (2,1.234) , '1.234000')
_('%.*f', (2,1.234) , '1.23')
_('%hi', 123 , '123') # len
_('%li', 123 , '123')
_('%Li', 123 , '123')
_('%%', () , '%') # %%
_('%10.4f', 1.234 , ' 1.2340') # multiple features
_('%(x)10.4f', {'y':0, 'x':1.234}, ' 1.2340')
_('%*.*f', (10,4,1.234) , ' 1.2340')
_('', {} , '') # errors
_('', [] , '')
_('', 123 , TypeError('not all arguments converted during string formatting'))
_('', '123' , TypeError('not all arguments converted during string formatting'))
_('%s', () , TypeError('not enough arguments for format string'))
_('%s %s', 123 , TypeError('not enough arguments for format string'))
_('%s %s', (123,) , TypeError('not enough arguments for format string'))
_('%(x)s', 123 , TypeError('format requires a mapping'))
_('%(x)s', (123,) , TypeError('format requires a mapping'))
_('%s %(x)s', (123,4) , TypeError('format requires a mapping'))
_('%(x)s %s', (123,4) , TypeError('format requires a mapping'))
_('%(x)s %s', {'x':1} , TypeError('not enough arguments for format string')) # mixing tuple/dict
_('%s %(x)s', {'x':1} , "{'x': 1} 1")
# for `'%4%' % ()` py2 gives ' %', but we stick to more reasonable py3 semantic
def _(fmt, args, ok):
return verify_fmt_all_types(lambda fmt, args: fmt % args,
fmt, args, ok, excok=True)
_('*str %4%', (), TypeError("not enough arguments for format string"))
_('*str %4%', 1, ValueError("unsupported format character '%' (0x25) at index 7"))
_('*str %4%', (1,), ValueError("unsupported format character '%' (0x25) at index 7"))
_('*str %(x)%', {'x':1}, ValueError("unsupported format character '%' (0x25) at index 9"))
_('%4%', () , TypeError("not enough arguments for format string"))
_('%4%', 1 , ValueError("unsupported format character '%' (0x25) at index 7"))
_('%4%', (1,) , ValueError("unsupported format character '%' (0x25) at index 7"))
_('%(x)%', {'x':1} , ValueError("unsupported format character '%' (0x25) at index 9"))
# parse checking complete. now verify actual %- and format- formatting
......@@ -1211,40 +1215,42 @@ def test_strings_mod_and_format():
fmt_ = fmt
verify_fmt_all_types(xformat, fmt_, args, *okv)
_("*str a %s z", 123) # NOTE *str to force str -> bstr/ustr even for ASCII string
_("*str a %s z", '*str \'"\x7f')
_("*str a %s z", 'β')
_("*str a %s z", ('β',))
# NOTE *str to force str -> bstr/ustr even for ASCII string
_("*str a %s z", 123 , "*str a 123 z")
_("*str a %s z", '*str \'"\x7f' , "*str a *str '\"\x7f z")
_("*str a %s z", 'β' , "*str a β z")
_("*str a %s z", ('β',) , "*str a β z")
_("*str a %s z", ['β'] , "*str a ['β'] z")
_("a %s π", 123)
_("a %s π", '*str \'"\x7f')
_("a %s π", 'β')
_("a %s π", ('β',))
_("a %s π", 123 , "a 123 π")
_("a %s π", '*str \'"\x7f' , "a *str '\"\x7f π")
_("a %s π", 'β' , "a β π")
_("a %s π", ('β',) , "a β π")
_("a %s π", ['β'] , "a ['β'] π")
_("α %s z", 123)
_("α %s z", '*str \'"\x7f')
_("α %s z", 'β')
_("α %s z", ('β',))
_("α %s z", 123 , "α 123 z")
_("α %s z", '*str \'"\x7f' , "α *str '\"\x7f z")
_("α %s z", 'β' , "α β z")
_("α %s z", ('β',) , "α β z")
_("α %s z", ['β'] , "α ['β'] z")
_("α %s π", 123)
_("α %s π", '*str \'"\x7f')
_("α %s π", 'β')
_("α %s π", ('β',))
_("α %s π", ('β',))
_("α %s %s π", ('β', 'γ'))
_("α %s %s %s π", ('β', 'γ', 'δ'))
_("α %s %s %s %s %s %s %s π", (1, 'β', 2, 'γ', 3, 'δ', 4))
_("α %s π", [])
_("α %s π", ([],))
_("α %s π", ((),))
_("α %s π", set())
_("α %s π", (set(),))
_("α %s π", frozenset())
_("α %s π", (frozenset(),))
_("α %s π", ({},))
_("α %s π", 123 , "α 123 π")
_("α %s π", '*str \'"\x7f' , "α *str '\"\x7f π")
_("α %s π", 'β' , "α β π")
_("α %s π", ('β',) , "α β π")
_("α %s π", ('β',) , "α β π")
_("α %s %s π", ('β', 'γ') , "α β γ π")
_("α %s %s %s π", ('β', 'γ', 'δ') , "α β γ δ π")
_("α %s %s %s %s %s %s %s π", (1, 'β', 2, 'γ', 3, 'δ', 4),
"α 1 β 2 γ 3 δ 4 π")
_("α %s π", [] , "α [] π")
_("α %s π", ([],) , "α [] π")
_("α %s π", ((),) , "α () π")
_("α %s π", set() , x32("α set() π", "α set([]) π"))
_("α %s π", (set(),) , x32("α set() π", "α set([]) π"))
_("α %s π", frozenset() , x32("α frozenset() π", "α frozenset([]) π"))
_("α %s π", (frozenset(),) , x32("α frozenset() π", "α frozenset([]) π"))
_("α %s π", ({},) , "α {} π")
_("α %s π", ['β'] , "α ['β'] π")
_("α %s π", (['β'],) , "α ['β'] π")
_("α %s π", (('β',),) , "α ('β',) π")
......@@ -1279,7 +1285,8 @@ def test_strings_mod_and_format():
# recursive frozenset
l = hlist()
f = frozenset({1, l}); l.append(f)
_('α %s π', (f,))
_('α %s π', (f,) , *x32(("α frozenset({1, [frozenset(...)]}) π", "α frozenset({[frozenset(...)], 1}) π"),
("α frozenset([1, [frozenset(...)]]) π", "α frozenset([[frozenset(...)], 1]) π")))
# recursive dict (via value)
d = {1:'мир'}; d.update({2:d})
......@@ -1296,15 +1303,15 @@ def test_strings_mod_and_format():
class Cold:
def __repr__(self): return "Cold()"
def __str__(self): return u"Класс (old)"
_('α %s π', Cold())
_('α %s π', (Cold(),))
_('α %s π', Cold() , "α Класс (old) π")
_('α %s π', (Cold(),) , "α Класс (old) π")
# new-style class with __str__
class Cnew(object):
def __repr__(self): return "Cnew()"
def __str__(self): return u"Класс (new)"
_('α %s π', Cnew())
_('α %s π', (Cnew(),))
_('α %s π', Cnew() , "α Класс (new) π")
_('α %s π', (Cnew(),) , "α Класс (new) π")
# custom classes inheriting from set/list/tuple/dict/frozenset
......@@ -1334,7 +1341,10 @@ def test_strings_mod_and_format():
# namedtuple
cc = collections; xcc = six.moves
Point = cc.namedtuple('Point', ['x', 'y'])
_('α %s π', (Point('β','γ'),) , "α Point(x='β', y='γ') π")
verify_fmt_all_types(lambda fmt, args: fmt % args,
'α %s π', Point('β','γ') , TypeError("not all arguments converted during string formatting"), excok=True)
_('α %s %s π',Point('β','γ') , "α β γ π")
_('α %s π', (Point('β','γ'),) , "α Point(x='β', y='γ') π")
# deque
_('α %s π', cc.deque(['β','γ']) , "α deque(['β', 'γ']) π")
_('α %s π', (cc.deque(['β','γ']),) , "α deque(['β', 'γ']) π")
......@@ -1536,6 +1546,14 @@ def test_strings__format__():
# verify print for bstr/ustr.
def test_strings_print():
outok = readfile(dir_testprog + "/golang_test_str.txt")
# repr(bstr|ustr) is changed if string types are patched:
# b('...') -> '...' if bstr is patched in
# u('...') -> u'...' if ustr is patched in (here we assume it is all valid utf8 there)
if bstr is bytes:
outok = re.sub(br"b\((.*?)\)", x32(r"b\1", r"\1"), outok)
if ustr is unicode:
outok = re.sub(br"u\((.*?)\)", x32(r"\1", r"u\1"), outok)
retcode, stdout, stderr = _pyrun(["golang_test_str.py"],
cwd=dir_testprog, stdout=PIPE, stderr=PIPE)
assert retcode == 0, (stdout, stderr)
......@@ -1578,7 +1596,11 @@ def test_strings_methods():
ur = xcall(us, meth, *argv, **kw)
def assertDeepEQ(a, b, bstrtype):
assert not isinstance(a, (bstr, ustr))
# `assert not isinstance(a, (bstr, ustr))` done carefully not to
# break when bytes/unicode are patched with bstr/ustr
if isinstance(a, bytes): assert type(a) is bytes
if isinstance(a, unicode): assert type(a) is unicode
if type(a) is unicode:
assert type(b) is bstrtype
assert a == b
......@@ -1841,6 +1863,26 @@ def test_strings_subclasses(tx):
_ = b(xx); assert type(_) is bstr ; assert _ == 'мир'
_ = u(xx); assert type(_) is ustr ; assert _ == 'мир'
# __str__ returns *str, not MyStr
txstr = {
unicode: str,
bstr: x32(ustr, bstr),
ustr: x32(ustr, bstr),
}[tx]
if six.PY2 and tx is unicode: # on py2 unicode.__str__ raises UnicodeEncodeError:
aa = u'mir' # `'ascii' codec can't encode ...` -> do the test on ascii
_ = aa.__str__(); assert _ == 'mir'
else:
_ = xx.__str__(); assert _ == 'мир'
assert type(_) is txstr
# for bstr/ustr __bytes__/__unicode__ return *str, never MyStr
# (builtin unicode has no __bytes__/__unicode__)
if tx is not unicode:
_ = xx.__bytes__(); assert type(_) is bstr; assert _ == 'мир'
_ = xx.__unicode__(); assert type(_) is ustr; assert _ == 'мир'
# subclass with __str__
class MyStr(tx):
def __str__(self): return u'αβγ'
......@@ -1864,6 +1906,17 @@ def test_strings_subclasses(tx):
with raises(TypeError): u(xx)
# verify that bstr/ustr has no extra attributes compared to str and UserString.
# (else e.g. IPython's guarded_eval.py fails when doing `_list_methods(collections.UserString, dir(str)`.
# XXX gpython-only ?
@mark.parametrize('tx', (bstr, ustr))
def _test_strings_no_extra_methods(tx): # XXX reenable (str does not have __bytes__)
from six.moves import UserString
for attr in dir(tx):
assert hasattr(str, attr)
assert hasattr(UserString, attr)
def test_qq():
# NOTE qq is also tested as part of strconv.quote
......@@ -2417,20 +2470,24 @@ def test_deepreplace_str():
# verify that what we patched - e.g. bytes.__repr__ - stay unaffected when
# called outside of bstr/ustr context.
# NOTE this test is complemented by test_pickle_strings_patched_transparently in golang_str_pickle_test.py
def test_strings_patched_transparently():
b_ = xbytes ("мир"); assert type(b_) is bytes
u_ = xunicode ("мир"); assert type(u_) is unicode
ba_ = xbytearray("мир"); assert type(ba_) is bytearray
# standard {repr,str}(bytes|unicode|bytearray) stay unaffected
assert repr(b_) == x32(r"b'\xd0\xbc\xd0\xb8\xd1\x80'",
r"'\xd0\xbc\xd0\xb8\xd1\x80'")
assert repr(u_) == x32(r"'мир'",
r"u'\u043c\u0438\u0440'")
assert repr(b_) == xB32(x32("b'мир'", "'мир'"),
r"b'\xd0\xbc\xd0\xb8\xd1\x80'",
r"'\xd0\xbc\xd0\xb8\xd1\x80'")
assert repr(u_) == xU32(x32("'мир'", "u'мир'"),
r"'мир'",
r"u'\u043c\u0438\u0440'")
assert repr(ba_) == r"bytearray(b'\xd0\xbc\xd0\xb8\xd1\x80')"
assert str(b_) == x32(r"b'\xd0\xbc\xd0\xb8\xd1\x80'",
"\xd0\xbc\xd0\xb8\xd1\x80")
assert str(b_) == xS32("мир",
r"b'\xd0\xbc\xd0\xb8\xd1\x80'",
"\xd0\xbc\xd0\xb8\xd1\x80")
if six.PY3 or sys.getdefaultencoding() == 'utf-8': # py3 or gpython/py2
assert str(u_) == "мир"
else:
......@@ -2438,8 +2495,9 @@ def test_strings_patched_transparently():
with raises(UnicodeEncodeError): str(u_) # 'ascii' codec can't encode ...
assert str(u'abc') == "abc"
assert str(ba_) == x32(r"bytearray(b'\xd0\xbc\xd0\xb8\xd1\x80')",
b'\xd0\xbc\xd0\xb8\xd1\x80')
assert str(ba_) == xS32("мир",
r"bytearray(b'\xd0\xbc\xd0\xb8\xd1\x80')",
b'\xd0\xbc\xd0\xb8\xd1\x80')
# unicode comparison stay unaffected
assert (u_ == u_) is True
......@@ -2458,9 +2516,10 @@ def test_strings_patched_transparently():
assert (u_ >= u2) is True ; assert (u2 >= u_) is False
# bytearray.__init__ stay unaffected
with raises(TypeError): bytearray(u'мир')
a = bytearray()
with raises(TypeError): a.__init__(u'мир')
if ustr is not unicode:
with raises(TypeError): bytearray(u'мир')
a = bytearray()
with raises(TypeError): a.__init__(u'мир')
def _(*argv):
a = bytearray(*argv)
......@@ -2530,9 +2589,29 @@ def bench_bencode(b):
# xbytes/xunicode/xbytearray convert provided bytes/unicode object to bytes,
# unicode or bytearray correspondingly to function name.
def xbytes(x): return x.encode('utf-8') if type(x) is unicode else x
def xunicode(x): return x.decode('utf-8') if type(x) is bytes else x
def xbytearray(x): return bytearray(xbytes(x))
def xbytes(x):
assert isinstance(x, (bytes,unicode))
if isinstance(x, unicode):
x = x.encode('utf-8')
assert isinstance(x, bytes)
x = _bdata(x)
assert type(x) is bytes
return x
def xunicode(x):
assert isinstance(x, (bytes,unicode))
if isinstance(x, bytes):
x = x.decode('utf-8')
assert isinstance(x, unicode)
x = _udata(x)
assert type(x) is unicode
return x
def xbytearray(x):
assert isinstance(x, (bytes,unicode))
x = bytearray(xbytes(x))
assert type(x) is bytearray
return x
# deepReplaceStr2Bytearray replaces str to bytearray, or hashable-version of
# bytearray, if str objects are detected to be present inside set or dict keys.
......@@ -2625,3 +2704,29 @@ class hlist(list):
# x32(a,b) returns a on py3, or b on py2
def x32(a, b):
return a if six.PY3 else b
# xb32(x, y, z) returns x if (bstr is not bytes) or x32(y,z)
# xu32(x, y, z) returns x if (ustr is not unicode) or x32(y,z)
def xb32(x, y, z):
return x if (bstr is not bytes) else x32(y,z)
def xu32(x, y, z):
return x if (ustr is not unicode) else x32(y,z)
# xB32(x, y, z) returns x if (bstr is bytes) or x32(y,z)
# xU32(x, y, z) returns x if (ustr is unicode) or x32(y,z)
# xS32(x, y, z) returns x if (str is bstr|ustr) or x32(y,z)
# XXX replace usage of xB32 to directly via xB ?
def xB32(x, y, z): return xB(x, x32(y,z))
def xU32(x, y, z): return xU(x, x32(y,z))
def xS32(x, y, z): return xS(x, x32(y,z))
# xB(x, y) returns x if (bstr is bytes) or y
# xU(x, y) returns x if (ustr is unicode) or y
# xS(x, y) returns x if (str is bstr|ustr) or y
def xB(x, y):
return x if (bstr is bytes) else y
def xU(x, y):
return x if (ustr is unicode) else y
def xS(x, y):
return x if (str is bstr or str is ustr) else y
golang/libgolang.h
View file @ 17dbfbac
......@@ -169,6 +169,8 @@
// [1] Libtask: a Coroutine Library for C and Unix. https://swtch.com/libtask.
// [2] http://9p.io/magic/man2html/2/thread.
#include "golang/runtime/platform.h"
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
......@@ -177,21 +179,18 @@
#include <sys/stat.h>
#include <fcntl.h>
#ifdef _MSC_VER // no mode_t on msvc
#ifdef LIBGOLANG_CC_msc // no mode_t on msvc
typedef int mode_t;
#endif
// DSO symbols visibility (based on https://gcc.gnu.org/wiki/Visibility)
#if defined _WIN32 || defined __CYGWIN__
#ifdef LIBGOLANG_OS_windows
#define LIBGOLANG_DSO_EXPORT __declspec(dllexport)
#define LIBGOLANG_DSO_IMPORT __declspec(dllimport)
#elif __GNUC__ >= 4
#else
#define LIBGOLANG_DSO_EXPORT __attribute__ ((visibility ("default")))
#define LIBGOLANG_DSO_IMPORT __attribute__ ((visibility ("default")))
#else
#define LIBGOLANG_DSO_EXPORT
#define LIBGOLANG_DSO_IMPORT
#endif
#if BUILDING_LIBGOLANG
......
golang/os.cpp
View file @ 17dbfbac
......@@ -38,7 +38,7 @@
// cut this short
// (on darwing sys_siglist declaration is normally provided)
// (on windows sys_siglist is not available at all)
#if !(defined(__APPLE__) || defined(_WIN32))
#if !(defined(LIBGOLANG_OS_darwin) || defined(LIBGOLANG_OS_windows))
extern "C" {
extern const char * const sys_siglist[];
}
......@@ -287,7 +287,7 @@ string Signal::String() const {
const Signal& sig = *this;
const char *sigstr = nil;
#ifdef _WIN32
#ifdef LIBGOLANG_OS_windows
switch (sig.signo) {
case SIGABRT: return "Aborted";
case SIGBREAK: return "Break";
......
golang/os.h
View file @ 17dbfbac
......@@ -96,7 +96,7 @@ private:
// Open opens file @path.
LIBGOLANG_API std::tuple<File, error> Open(const string &path, int flags = O_RDONLY,
mode_t mode =
#if !defined(_MSC_VER)
#if !defined(LIBGOLANG_CC_msc)
S_IRUSR | S_IWUSR | S_IXUSR |
S_IRGRP | S_IWGRP | S_IXGRP |
S_IROTH | S_IWOTH | S_IXOTH
......
golang/os/signal.cpp
View file @ 17dbfbac
......@@ -89,7 +89,7 @@
#include <atomic>
#include <tuple>
#if defined(_WIN32)
#if defined(LIBGOLANG_OS_windows)
# include <windows.h>
#endif
......@@ -101,7 +101,7 @@
# define debugf(format, ...) do {} while (0)
#endif
#if defined(_MSC_VER)
#ifdef LIBGOLANG_CC_msc
# define HAVE_SIGACTION 0
#else
# define HAVE_SIGACTION 1
......@@ -194,7 +194,7 @@ void _init() {
if (err != nil)
panic("os::newFile(_wakerx");
_waketx = vfd[1];
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
if (sys::Fcntl(_waketx, F_SETFL, O_NONBLOCK) < 0)
panic("fcntl(_waketx, O_NONBLOCK)"); // TODO +syserr
#else
......
golang/pyx/build.py
View file @ 17dbfbac
......@@ -35,7 +35,7 @@ from __future__ import print_function, absolute_import
# pygolang uses setuptools_dso.DSO to build libgolang; all extensions link to it.
import setuptools_dso
import sys, pkgutil, platform, sysconfig
import os, sys, pkgutil, platform, sysconfig
from os.path import dirname, join, exists
from distutils.errors import DistutilsError
......@@ -68,7 +68,7 @@ def _findpkg(pkgname): # -> _PyPkg
# build_ext amends setuptools_dso.build_ext to allow combining C and C++
# sources in one extension without hitting `error: invalid argument
# '-std=c++11' not allowed with 'C'`.
# '-std=c++11' not allowed with 'C'`. XXX + asm
_dso_build_ext = setuptools_dso.build_ext
class build_ext(_dso_build_ext):
def build_extension(self, ext):
......@@ -108,12 +108,33 @@ class build_ext(_dso_build_ext):
# do per-source adjustsment only in .spawn .
spawn = self.compiler.spawn
def xspawn(argv):
argv = argv[:]
c = False
for arg in argv:
S = False
for i,arg in enumerate(argv):
if arg.startswith('/Tc'):
c = True
if c:
argv = argv[:]
if arg.endswith('.S'):
argv[i] = arg[3:] # /Tcabc.S -> abc.S
S = True
else:
c = True
# change cl.exe -> clang-cl.exe for assembly files so that assembler dialect is the same everywhere
if S:
assert argv[0] == self.compiler.cc, (argv, self.compiler.cc)
argv[0] = self.compiler.clang_cl
# clang-cl fails on *.S if also given /EH... -> remove /EH...
while 1:
for i in range(len(argv)):
if argv[i].startswith('/EH'):
del argv[i]
break
else:
break
if c or S:
for i in range(len(argv)):
if argv[i] == '/std:c++20':
argv[i] = '/std:c11'
......@@ -128,6 +149,22 @@ class build_ext(_dso_build_ext):
self.compiler._compile = _compile
self.compiler.spawn = spawn
def build_extensions(self):
# adjust .compiler to support assembly sources
cc = self.compiler
if '.S' not in cc.src_extensions:
cc.src_extensions.append('.S')
cc.language_map['.S'] = 'asm'
cc.language_order.append('asm')
# XXX refer to https://blog.mozilla.org/nfroyd/2019/04/25/an-unexpected-benefit-of-standardizing-on-clang-cl/
if cc.compiler_type == 'msvc':
if not cc.initialized:
cc.initialize()
ccmod = sys.modules[cc.__module__]
cc.clang_cl = ccmod._find_exe('clang-cl.exe', cc._paths.split(os.pathsep))
cc._c_extensions.append('.S') # MSVCCompiler thinks it is C, but xspawn handles .S specially
_dso_build_ext.build_extensions(self)
# setup should be used instead of setuptools.setup
def setup(**kw):
......@@ -176,8 +213,8 @@ def _with_build_defaults(name, kw): # -> (pygo, kw')
incv.insert(1, join(pygo, 'golang', '_compat', sysname))
kw['include_dirs'] = incv
# link with libgolang.so if it is not libgolang itself
if name != 'golang.runtime.libgolang':
# link with libgolang.so if it is not libgolang itself, or another internal DSO
if name not in ('golang.runtime.libgolang', 'golang.runtime.funchook'):
dsov = kw.get('dsos', [])[:]
dsov.insert(0, 'golang.runtime.libgolang')
kw['dsos'] = dsov
......@@ -212,9 +249,11 @@ def _with_build_defaults(name, kw): # -> (pygo, kw')
dependv = kw.get('depends', [])[:]
dependv.extend(['%s/golang/%s' % (pygo, _) for _ in [
'libgolang.h',
'runtime.h',
'runtime/internal.h',
'runtime/internal/atomic.h',
'runtime/internal/syscall.h',
'runtime/platform.h',
'context.h',
'cxx.h',
'errors.h',
......
golang/runtime.cpp 0 → 100644
View file @ 17dbfbac
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package runtime mirrors Go package runtime.
// See runtime.h for package overview.
#include "golang/runtime.h"
// golang::runtime::
namespace golang {
namespace runtime {
const string ARCH =
#ifdef LIBGOLANG_ARCH_386
"386"
#elif defined(LIBGOLANG_ARCH_amd64)
"amd64"
#elif defined(LIBGOLANG_ARCH_arm64)
"arm64"
#else
# error
#endif
;
const string OS =
#ifdef LIBGOLANG_OS_linux
"linux"
#elif defined(LIBGOLANG_OS_darwin)
"darwin"
#elif defined(LIBGOLANG_OS_windows)
"windows"
#else
# error
#endif
;
const string CC =
#ifdef LIBGOLANG_CC_gcc
"gcc"
#elif defined(LIBGOLANG_CC_clang)
"clang"
#elif defined(LIBGOLANG_CC_msc)
"msc"
#else
# error
#endif
;
}} // golang::runtime::
golang/runtime.h 0 → 100644
View file @ 17dbfbac
#ifndef _NXD_LIBGOLANG_RUNTIME_H
#define _NXD_LIBGOLANG_RUNTIME_H
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package runtime mirrors Go package runtime.
#include "golang/libgolang.h"
// golang::runtime::
namespace golang {
namespace runtime {
// ARCH indicates processor architecture, that is running the program.
//
// e.g. "386", "amd64", "arm64", ...
extern LIBGOLANG_API const string ARCH;
// OS indicates operating system, that is running the program.
//
// e.g. "linux", "darwin", "windows", ...
extern LIBGOLANG_API const string OS;
// CC indicates C/C++ compiler, that compiled the program.
//
// e.g. "gcc", "clang", "msc", ...
extern LIBGOLANG_API const string CC;
}} // golang::runtime::
#endif // _NXD_LIBGOLANG_RUNTIME_H
golang/runtime/internal/atomic.cpp
View file @ 17dbfbac
......@@ -20,7 +20,7 @@
#include "golang/runtime/internal/atomic.h"
#include "golang/libgolang.h"
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
#include <pthread.h>
#endif
......@@ -44,7 +44,7 @@ static void _forkNewEpoch() {
void _init() {
// there is no fork on windows
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
int e = pthread_atfork(/*prepare*/nil, /*inparent*/nil, /*inchild*/_forkNewEpoch);
if (e != 0)
panic("pthread_atfork failed");
......
golang/runtime/internal/syscall.cpp
View file @ 17dbfbac
......@@ -58,9 +58,9 @@ string _Errno::Error() {
char ebuf[128];
bool ok;
#if __APPLE__
#ifdef LIBGOLANG_OS_darwin
ok = (::strerror_r(-e.syserr, ebuf, sizeof(ebuf)) == 0);
#elif defined(_WIN32)
#elif defined(LIBGOLANG_OS_windows)
ok = (::strerror_s(ebuf, sizeof(ebuf), -e.syserr) == 0);
#else
char *estr = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
......@@ -102,7 +102,7 @@ __Errno Close(int fd) {
return err;
}
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
__Errno Fcntl(int fd, int cmd, int arg) {
int save_errno = errno;
int err = ::fcntl(fd, cmd, arg);
......@@ -124,7 +124,7 @@ __Errno Fstat(int fd, struct ::stat *out_st) {
int Open(const char *path, int flags, mode_t mode) {
int save_errno = errno;
#ifdef _WIN32 // default to open files in binary mode
#ifdef LIBGOLANG_OS_windows // default to open files in binary mode
if ((flags & (_O_TEXT | _O_BINARY)) == 0)
flags |= _O_BINARY;
#endif
......@@ -141,9 +141,9 @@ __Errno Pipe(int vfd[2], int flags) {
return -EINVAL;
int save_errno = errno;
int err;
#ifdef __linux__
#ifdef LIBGOLANG_OS_linux
err = ::pipe2(vfd, flags);
#elif defined(_WIN32)
#elif defined(LIBGOLANG_OS_windows)
err = ::_pipe(vfd, 4096, flags | _O_BINARY);
#else
err = ::pipe(vfd);
......@@ -167,7 +167,7 @@ out:
return err;
}
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
__Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact) {
int save_errno = errno;
int err = ::sigaction(signo, act, oldact);
......
golang/runtime/internal/syscall.h
View file @ 17dbfbac
......@@ -63,13 +63,13 @@ LIBGOLANG_API int/*n|err*/ Read(int fd, void *buf, size_t count);
LIBGOLANG_API int/*n|err*/ Write(int fd, const void *buf, size_t count);
LIBGOLANG_API __Errno Close(int fd);
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
LIBGOLANG_API __Errno Fcntl(int fd, int cmd, int arg);
#endif
LIBGOLANG_API __Errno Fstat(int fd, struct ::stat *out_st);
LIBGOLANG_API int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
LIBGOLANG_API __Errno Pipe(int vfd[2], int flags);
#ifndef _WIN32
#ifndef LIBGOLANG_OS_windows
LIBGOLANG_API __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
#endif
typedef void (*sighandler_t)(int);
......
golang/runtime/libgolang.cpp
View file @ 17dbfbac
......@@ -52,7 +52,7 @@
#include <linux/list.h>
// MSVC does not support statement expressions and typeof
// -> redo list_entry via C++ lambda.
#ifdef _MSC_VER
#ifdef LIBGOLANG_CC_msc
# undef list_entry
# define list_entry(ptr, type, member) [&]() { \
const decltype( ((type *)0)->member ) *__mptr = (ptr); \
......
golang/runtime/platform.h 0 → 100644
View file @ 17dbfbac
#ifndef _NXD_LIBGOLANG_RUNTIME_PLATFORM_H
#define _NXD_LIBGOLANG_RUNTIME_PLATFORM_H
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Header platform.h provides preprocessor defines that describe target platform.
// LIBGOLANG_ARCH_<X> is defined on architecture X.
//
// List of supported architectures: 386, amd64, arm64.
#if defined(__i386__) || defined(_M_IX86)
# define LIBGOLANG_ARCH_386 1
#elif defined(__x86_64__) || defined(_M_X64)
# define LIBGOLANG_ARCH_amd64 1
#elif defined(__aarch64__) || defined(_M_ARM64)
# define LIBGOLANG_ARCH_arm64 1
#else
# error "unsupported architecture"
#endif
// LIBGOLANG_OS_<X> is defined on operating system X.
//
// List of supported operating systems: linux, darwin, windows.
#ifdef __linux__
# define LIBGOLANG_OS_linux 1
#elif defined(__APPLE__)
# define LIBGOLANG_OS_darwin 1
#elif defined(_WIN32) || defined(__CYGWIN__)
# define LIBGOLANG_OS_windows 1
#else
# error "unsupported operating system"
#endif
// LIBGOLANG_CC_<X> is defined on C/C++ compiler X.
//
// List of supported compilers: gcc, clang, msc.
#ifdef __clang__
# define LIBGOLANG_CC_clang 1
#elif defined(_MSC_VER)
# define LIBGOLANG_CC_msc 1
// NOTE gcc comes last because e.g. clang and icc define __GNUC__ as well
#elif __GNUC__
# define LIBGOLANG_CC_gcc 1
#else
# error "unsupported compiler"
#endif
#endif // _NXD_LIBGOLANG_RUNTIME_PLATFORM_H
gpython/__init__.py
View file @ 17dbfbac
......@@ -25,10 +25,14 @@ differences:
- gevent is pre-activated and stdlib is patched to be gevent aware;
- go, chan, select etc are put into builtin namespace;
- default string encoding is always set to UTF-8.
- default string encoding is always set to UTF-8;
- bstr/ustr replace builtin str/unicode types.
Gevent activation can be disabled via `-X gpython.runtime=threads`, or
$GPYTHON_RUNTIME=threads.
String types replacement can be disabled via `-X gpython.strings=pystd`, or
$GPYTHON_STRINGS=pystd.
"""
# NOTE gpython is kept out of golang/ , since even just importing e.g. golang.cmd.gpython,
......@@ -230,9 +234,13 @@ def pymain(argv, init=None):
gevent = sys.modules.get('gevent', None)
gpyver = 'GPython %s' % golang.__version__
if gevent is not None:
gpyver += ' [gevent %s]' % gevent.__version__
gpyver += ' [runtime gevent %s]' % gevent.__version__
else:
gpyver += ' [runtime threads]'
if type(u'') is golang.ustr:
gpyver += ' [strings bstr+ustr]'
else:
gpyver += ' [threads]'
gpyver += ' [strings pystd]'
ver.append(gpyver)
import platform
......@@ -344,6 +352,9 @@ def main():
# imported first, e.g. to support sys.modules.
import sys
# import pyx/c part of gpython
from gpython import _gpython
# safety check that we are not running from a setuptools entrypoint, where
# it would be too late to monkey-patch stdlib.
#
......@@ -372,6 +383,7 @@ def main():
reload(sys)
sys.setdefaultencoding('utf-8')
delattr(sys, 'setdefaultencoding')
_gpython.set_utf8_as_default_src_encoding()
# import os to get access to environment.
......@@ -381,10 +393,12 @@ def main():
import os
# extract and process `-X gpython.*`
# -X gpython.runtime=(gevent|threads) + $GPYTHON_RUNTIME
# -X gpython.runtime=(gevent|threads) + $GPYTHON_RUNTIME
# -X gpython.strings=(bstr+ustr|pystd) + $GPYTHON_STRINGS
sys._xoptions = getattr(sys, '_xoptions', {})
argv_ = []
gpy_runtime = os.getenv('GPYTHON_RUNTIME', 'gevent')
gpy_strings = os.getenv('GPYTHON_STRINGS', 'bstr+ustr')
igetopt = _IGetOpt(sys.argv[1:], _pyopt, _pyopt_long)
for (opt, arg) in igetopt:
if opt == '-X':
......@@ -393,6 +407,10 @@ def main():
gpy_runtime = arg[len('gpython.runtime='):]
sys._xoptions['gpython.runtime'] = gpy_runtime
elif arg.startswith('gpython.strings='):
gpy_strings = arg[len('gpython.strings='):]
sys._xoptions['gpython.strings'] = gpy_strings
else:
raise RuntimeError('gpython: unknown -X option %s' % arg)
......@@ -412,13 +430,15 @@ def main():
# sys.executable spawned from under `gpython -X gpython.runtime=threads`
# also uses "threads" runtime by default.
os.environ['GPYTHON_RUNTIME'] = gpy_runtime
os.environ['GPYTHON_STRINGS'] = gpy_strings
# init initializes according to selected runtime
# init initializes according to selected runtime and strings
# it is called after options are parsed and sys.path is setup correspondingly.
# this way golang and gevent are imported from exactly the same place as
# they would be in standard python after regular import (ex from golang/
# under cwd if run under `python -c ...` or interactive console.
def init():
gpy_runtime_ver = gpy_runtime
if gpy_runtime == 'gevent':
# make gevent pre-available & stdlib patched
import gevent
......@@ -434,22 +454,30 @@ def main():
if _ not in (True, None): # patched or nothing to do
# XXX provide details
raise RuntimeError('gevent monkey-patching failed')
gpy_verextra = 'gevent %s' % gevent.__version__
gpy_runtime_ver += ' %s' % gevent.__version__
elif gpy_runtime == 'threads':
gpy_verextra = 'threads'
pass
else:
raise RuntimeError('gpython: invalid runtime %s' % gpy_runtime)
raise RuntimeError('gpython: invalid runtime %r' % gpy_runtime)
# put go, chan, select, ... into builtin namespace
if gpy_strings not in ('bstr+ustr', 'pystd'):
raise RuntimeError('gpython: invalid strings %r' % gpy_strings)
# import golang
# this will activate selected runtime and strings
sys._gpy_runtime = gpy_runtime
sys._gpy_strings = gpy_strings
import golang
# put go, chan, select, ... into builtin namespace
from six.moves import builtins
for k in golang.__all__:
setattr(builtins, k, getattr(golang, k))
setattr(builtins, 'CCC', CCC)
# sys.version
sys.version += (' [GPython %s] [%s]' % (golang.__version__, gpy_verextra))
sys.version += (' [GPython %s] [runtime %s] [strings %s]' % (golang.__version__, gpy_runtime_ver, gpy_strings))
# tail to pymain
pymain(argv, init)
......@@ -567,5 +595,11 @@ class _IGetOpt:
next = __next__ # for py2
# for tests XXX continue by first writing test XXX
1/0
class _tEarlyStrSubclass(str):
pass
if __name__ == '__main__':
main()
gpython/_gpython.pyx 0 → 100644
View file @ 17dbfbac
# -*- coding: utf-8 -*-
# cython: language_level=2
# Copyright (C) 2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""_gpython.pyx ... XXX
"""
cdef extern from *:
"""
void _set_utf8_as_default_src_encoding();
"""
void _set_utf8_as_default_src_encoding() except *
def set_utf8_as_default_src_encoding():
_set_utf8_as_default_src_encoding()
gpython/_gpython_c.cpp 0 → 100644
View file @ 17dbfbac
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// XXX doctitle
#include <Python.h>
#if PY_MAJOR_VERSION < 3
#include <Python-ast.h> // mod_ty & co
#include <node.h> // node
#include <graminit.h> // encoding_decl & co
#include <ast.h> // PyAST_FromNode & co
#endif
#include <funchook.h>
// py2: wrap PyAST_FromNode so that "utf-8" becomes the default encoding
#if PY_MAJOR_VERSION < 3
static auto _py_PyAST_FromNode = &PyAST_FromNode;
static mod_ty gpy_PyAST_FromNode(const node* n, PyCompilerFlags* flags,
const char* filename, PyArena* arena)
{
// fprintf(stderr, "gpy_PyAST_FromNode...\n");
PyCompilerFlags gflags = {.cf_flags = 0};
if (flags)
gflags = *flags;
if (TYPE(n) != encoding_decl)
gflags.cf_flags |= PyCF_SOURCE_IS_UTF8;
return _py_PyAST_FromNode(n, &gflags, filename, arena);
}
static funchook_t* gpy_PyAST_FromNode_hook;
void _set_utf8_as_default_src_encoding() {
funchook_t *h;
int err;
// funchook_set_debug_file("/dev/stderr");
gpy_PyAST_FromNode_hook = h = funchook_create();
if (h == NULL) {
PyErr_NoMemory();
return;
}
err = funchook_prepare(h, (void**)&_py_PyAST_FromNode, (void*)gpy_PyAST_FromNode);
if (err != 0) {
PyErr_SetString(PyExc_RuntimeError, funchook_error_message(h));
return;
}
err = funchook_install(h, 0);
if (err != 0) {
PyErr_SetString(PyExc_RuntimeError, funchook_error_message(h));
return;
}
// ok
}
#else
void _set_utf8_as_default_src_encoding() {}
#endif
gpython/gpython_test.py
View file @ 17dbfbac
......@@ -47,20 +47,34 @@ gpython_only = pytest.mark.skipif(not is_gpython, reason="gpython-only test")
def runtime(request):
yield request.param
# strings is pytest fixture that yields all variants of should be supported gpython strings:
# '' - not specified (gpython should autoselect)
# 'bstr+ustr'
# 'pystd'
@pytest.fixture(scope="function", params=['', 'bstr+ustr', 'pystd'])
def strings(request):
yield request.param
# gpyenv returns environment appropriate for spawning gpython with
# specified runtime.
def gpyenv(runtime): # -> env
# specified runtime and strings.
def gpyenv(runtime, strings): # -> env
env = os.environ.copy()
if runtime != '':
env['GPYTHON_RUNTIME'] = runtime
else:
env.pop('GPYTHON_RUNTIME', None)
if strings != '':
env['GPYTHON_STRINGS'] = strings
else:
env.pop('GPYTHON_STRINGS', None)
return env
@gpython_only
def test_defaultencoding_utf8():
assert sys.getdefaultencoding() == 'utf-8'
assert eval("u'αβγ'") == u'αβγ' # FIXME fails on py2 which uses hardcoded default latin1
# XXX +exec, +run file
@gpython_only
def test_golang_builtins():
......@@ -143,19 +157,42 @@ def assert_gevent_not_activated():
@gpython_only
def test_executable(runtime):
def test_str_patched():
# gpython, by default, patches str/unicode to be bstr/ustr.
# handling of various string modes is explicitly tested in test_Xstrings.
assert_str_patched()
def assert_str_patched():
#assert str.__name__ == ('bstr' if PY2 else 'ustr')
assert str.__name__ == 'str'
assert str is (bstr if PY2 else ustr)
if PY2:
assert unicode.__name__ == 'unicode'
assert unicode is ustr
assert type('') is str
assert type(b'') is (bstr if PY2 else bytes)
assert type(u'') is ustr
def assert_str_not_patched():
assert str.__name__ == 'str'
assert str is not bstr
assert str is not ustr
if PY2:
assert unicode.__name__ == 'unicode'
assert unicode is not bstr
assert unicode is not ustr
assert type('') is str
assert type(b'') is bytes
assert type(u'') is (unicode if PY2 else str)
@gpython_only
def test_executable():
# sys.executable must point to gpython and we must be able to execute it.
import gevent
assert 'gpython' in sys.executable
ver = pyout(['-c', 'import sys; print(sys.version)'], env=gpyenv(runtime))
ver = pyout(['-c', 'import sys; print(sys.version)'], env=gpyenv('', ''))
ver = str(ver)
assert ('[GPython %s]' % golang.__version__) in ver
if runtime != 'threads':
assert ('[gevent %s]' % gevent.__version__) in ver
assert ('[threads]') not in ver
else:
assert ('[gevent ') not in ver
assert ('[threads]') in ver
# verify pymain.
......@@ -322,15 +359,20 @@ def test_pymain_opt():
# pymain -V/--version
# gpython_only because output differs from !gpython.
@gpython_only
def test_pymain_ver(runtime):
def test_pymain_ver(runtime, strings):
from golang import b
from gpython import _version_info_str as V
import gevent
vok = 'GPython %s' % golang.__version__
if runtime != 'threads':
vok += ' [gevent %s]' % gevent.__version__
vok += ' [runtime gevent %s]' % gevent.__version__
else:
vok += ' [threads]'
vok += ' [runtime threads]'
if strings != 'pystd':
vok += ' [strings bstr+ustr]'
else:
vok += ' [strings pystd]'
if is_cpython:
vok += ' / CPython %s' % platform.python_version()
......@@ -341,10 +383,12 @@ def test_pymain_ver(runtime):
vok += '\n'
ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
env = gpyenv(runtime, strings)
ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE, env=env)
assert (ret, out, b(err)) == (0, b'', b(vok))
ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE, env=env)
assert (ret, out, b(err)) == (0, b'', b(vok))
# verify that ./bin/gpython runs ok.
......
pyproject.toml
View file @ 17dbfbac
[build-system]
requires = ["setuptools", "wheel", "setuptools_dso >= 2.7", "cython", "gevent"]
requires = ["setuptools", "wheel", "setuptools_dso >= 2.7", "cython < 3", "gevent"]
setup.py
View file @ 17dbfbac
......@@ -42,9 +42,9 @@ from setuptools.command.install_scripts import install_scripts as _install_scrip
from setuptools.command.develop import develop as _develop
from distutils import sysconfig
from os.path import dirname, join
import sys, os, re
import sys, os, re, platform, errno
# read file content
# read/write file content
def readfile(path): # -> str
with open(path, 'rb') as f:
data = f.read()
......@@ -52,6 +52,20 @@ def readfile(path): # -> str
data = data.decode('utf-8')
return data
def writefile(path, data):
if not isinstance(data, bytes):
data = data.encode('utf-8')
with open(path, 'wb') as f:
f.write(data)
# mkdir -p
def mkdir_p(path):
try:
os.makedirs(path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# reuse golang.pyx.build to build pygolang dso and extensions.
# we have to be careful and inject synthetic golang package in order to be
# able to import golang.pyx.build without built/working golang.
......@@ -59,6 +73,7 @@ trun = {}
exec(readfile('trun'), trun)
trun['ximport_empty_golangmod']()
from golang.pyx.build import setup, DSO, Extension as Ext
from setuptools_dso import ProbeToolchain
# grep searches text for pattern.
......@@ -104,7 +119,7 @@ class XInstallGPython:
# (script_name, script) -> (script_name, script)
def transform_script(self, script_name, script):
# on windows setuptools installs 3 files:
# gpython-script.py
# gpython-script.py XXX do we need to adjust this similarly to pymain?
# gpython.exe
# gpython.exe.manifest
# we want to override .py only.
......@@ -173,8 +188,8 @@ class develop(XInstallGPython, _develop):
# requirements of packages under "golang." namespace
R = {
'cmd.pybench': {'pytest'},
'pyx.build': {'setuptools', 'wheel', 'cython', 'setuptools_dso >= 2.7'},
'cmd.pybench': {'pytest', 'py'},
'pyx.build': {'setuptools', 'wheel', 'cython < 3', 'setuptools_dso >= 2.7'},
'x.perf.benchlib': {'numpy'},
}
# TODO generate `a.b -> a`, e.g. x.perf = join(x.perf.*); x = join(x.*)
......@@ -184,7 +199,8 @@ for pkg in R:
R['all'] = Rall
# ipython/pytest are required to test py2 integration patches
R['all_test'] = Rall.union(['ipython', 'pytest']) # pip does not like "+" in all+test
# zodbpickle is used to test pickle support for bstr/ustr
R['all_test'] = Rall.union(['ipython', 'pytest', 'zodbpickle']) # pip does not like "+" in all+test
# extras_require <- R
extras_require = {}
......@@ -200,6 +216,206 @@ def get_python_libdir():
else:
return sysconfig.get_config_var('LIBDIR')
# funchook_dso is DSO for libfunchook.so or None if CPU is not supported.
def _():
cpu = platform.machine()
if re.match('x86|i.86|x86_64|amd64', cpu, re.I):
cpu = 'x86'
disasm = 'distorm'
elif re.match('aarch64|arm64', cpu, re.I):
cpu = 'arm64'
disasm = 'capstone'
else:
return None # no funchook support
# XXX temp test XXX no -> we need capstone for disasm
disasm = 'capstone'
if platform.system() == 'Windows':
os = 'windows'
libv = ['psapi']
else:
os = 'unix'
libv = ['dl']
FH = '3rdparty/funchook/'
srcv = [FH+'src/funchook.c',
FH+'src/funchook_%s.c' % cpu,
FH+'src/funchook_%s.c' % os,
FH+'src/disasm_%s.c' % disasm]
depv = [FH+'include/funchook.h',
FH+'src/disasm.h',
FH+'src/funchook_arm64.h',
FH+'src/funchook_internal.h',
FH+'src/funchook_x86.h']
incv = [FH+'include']
defv = ['FUNCHOOK_EXPORTS']
if disasm == 'distorm':
D3 = '3rdparty/funchook/distorm/'
srcv += [D3+'src/decoder.c',
D3+'src/distorm.c',
D3+'src/instructions.c',
D3+'src/insts.c',
D3+'src/mnemonics.c',
D3+'src/operands.c',
D3+'src/prefix.c',
D3+'src/textdefs.c']
depv += [D3+'include/distorm.h',
D3+'include/mnemonics.h',
D3+'src/config.h',
D3+'src/decoder.h',
D3+'src/instructions.h',
D3+'src/insts.h',
D3+'src/operands.h',
D3+'src/prefix.h',
D3+'src/textdefs.h',
D3+'src/wstring.h',
D3+'src/x86defs.h']
incv += [D3+'include']
if disasm == 'capstone':
CS = '3rdparty/capstone/'
srcv += [CS+'cs.c',
CS+'Mapping.c',
CS+'MCInst.c',
CS+'MCInstrDesc.c',
CS+'MCRegisterInfo.c',
CS+'SStream.c',
CS+'utils.c']
depv += [CS+'cs_simple_types.h',
CS+'cs_priv.h',
CS+'LEB128.h',
CS+'Mapping.h',
CS+'MathExtras.h',
CS+'MCDisassembler.h',
CS+'MCFixedLenDisassembler.h',
CS+'MCInst.h',
CS+'MCInstrDesc.h',
CS+'MCRegisterInfo.h',
CS+'SStream.h',
CS+'utils.h']
incv += [CS+'include']
depv += [CS+'include/capstone/arm64.h',
CS+'include/capstone/arm.h',
CS+'include/capstone/capstone.h',
CS+'include/capstone/evm.h',
CS+'include/capstone/wasm.h',
CS+'include/capstone/mips.h',
CS+'include/capstone/ppc.h',
CS+'include/capstone/x86.h',
CS+'include/capstone/sparc.h',
CS+'include/capstone/systemz.h',
CS+'include/capstone/xcore.h',
CS+'include/capstone/m68k.h',
CS+'include/capstone/tms320c64x.h',
CS+'include/capstone/m680x.h',
CS+'include/capstone/mos65xx.h',
CS+'include/capstone/bpf.h',
CS+'include/capstone/riscv.h',
CS+'include/capstone/sh.h',
CS+'include/capstone/tricore.h',
CS+'include/capstone/platform.h']
defv += ['CAPSTONE_SHARED', 'CAPSTONE_USE_SYS_DYN_MEM']
if cpu == 'arm64':
defv += ['CAPSTONE_HAS_ARM64']
srcv += [CS+'arch/AArch64/AArch64BaseInfo.c',
CS+'arch/AArch64/AArch64Disassembler.c',
CS+'arch/AArch64/AArch64InstPrinter.c',
CS+'arch/AArch64/AArch64Mapping.c',
CS+'arch/AArch64/AArch64Module.c']
depv += [CS+'arch/AArch64/AArch64AddressingModes.h',
CS+'arch/AArch64/AArch64BaseInfo.h',
CS+'arch/AArch64/AArch64Disassembler.h',
CS+'arch/AArch64/AArch64InstPrinter.h',
CS+'arch/AArch64/AArch64Mapping.h',
CS+'arch/AArch64/AArch64GenAsmWriter.inc',
CS+'arch/AArch64/AArch64GenDisassemblerTables.inc',
CS+'arch/AArch64/AArch64GenInstrInfo.inc',
CS+'arch/AArch64/AArch64GenRegisterInfo.inc',
CS+'arch/AArch64/AArch64GenRegisterName.inc',
CS+'arch/AArch64/AArch64GenRegisterV.inc',
CS+'arch/AArch64/AArch64GenSubtargetInfo.inc',
CS+'arch/AArch64/AArch64GenSystemOperands.inc',
CS+'arch/AArch64/AArch64GenSystemOperands_enum.inc',
CS+'arch/AArch64/AArch64MappingInsn.inc',
CS+'arch/AArch64/AArch64MappingInsnName.inc',
CS+'arch/AArch64/AArch64MappingInsnOp.inc']
if cpu == 'x86':
defv += ['CAPSTONE_HAS_X86']
srcv += [CS+'arch/X86/X86ATTInstPrinter.c', # !diet
CS+'arch/X86/X86Disassembler.c',
CS+'arch/X86/X86DisassemblerDecoder.c',
CS+'arch/X86/X86IntelInstPrinter.c',
CS+'arch/X86/X86InstPrinterCommon.c',
CS+'arch/X86/X86Mapping.c',
CS+'arch/X86/X86Module.c']
depv += [CS+'arch/X86/X86BaseInfo.h',
CS+'arch/X86/X86Disassembler.h',
CS+'arch/X86/X86DisassemblerDecoder.h',
CS+'arch/X86/X86DisassemblerDecoderCommon.h',
CS+'arch/X86/X86GenAsmWriter.inc',
CS+'arch/X86/X86GenAsmWriter1.inc',
CS+'arch/X86/X86GenAsmWriter1_reduce.inc',
CS+'arch/X86/X86GenAsmWriter_reduce.inc',
CS+'arch/X86/X86GenDisassemblerTables.inc',
CS+'arch/X86/X86GenDisassemblerTables_reduce.inc',
CS+'arch/X86/X86GenInstrInfo.inc',
CS+'arch/X86/X86GenInstrInfo_reduce.inc',
CS+'arch/X86/X86GenRegisterInfo.inc',
CS+'arch/X86/X86InstPrinter.h',
CS+'arch/X86/X86Mapping.h',
CS+'arch/X86/X86MappingInsn.inc',
CS+'arch/X86/X86MappingInsnOp.inc',
CS+'arch/X86/X86MappingInsnOp_reduce.inc',
CS+'arch/X86/X86MappingInsn_reduce.inc']
# config.h
probe = ProbeToolchain()
config_h = []
def cfgemit(line):
config_h.append(line+'\n')
def defif(name, ok):
if ok:
cfgemit('#define %s 1' % name)
else:
cfgemit('#undef %s' % name)
for d in ('capstone', 'distorm', 'zydis'):
defif('DISASM_%s' % d.upper(), d == disasm)
cfgemit('#define SIZEOF_VOID_P %d' % probe.sizeof('void*'))
defif('_GNU_SOURCE', 1)
defif('GNU_SPECIFIC_STRERROR_R', probe.try_compile("""
#define _GNU_SOURCE 1
#include <string.h>
int main()
{
char dummy[128];
return *strerror_r(0, dummy, sizeof(dummy));
}
"""))
fbuild_src = 'build/3rdparty/funchook/src'
mkdir_p(fbuild_src)
writefile(fbuild_src+'/config.h', ''.join(config_h))
incv += [fbuild_src]
return DSO('golang.runtime.funchook', srcv,
depends = depv,
language = 'c',
include_dirs = incv,
define_macros = [(_, None) for _ in defv],
libraries = libv,
soversion = '1.1')
funchook_dso = _()
setup(
name = 'pygolang',
version = version,
......@@ -225,6 +441,7 @@ setup(
['golang/runtime/libgolang.cpp',
'golang/runtime/internal/atomic.cpp',
'golang/runtime/internal/syscall.cpp',
'golang/runtime.cpp',
'golang/context.cpp',
'golang/errors.cpp',
'golang/fmt.cpp',
......@@ -236,9 +453,11 @@ setup(
'golang/time.cpp'],
depends = [
'golang/libgolang.h',
'golang/runtime.h',
'golang/runtime/internal.h',
'golang/runtime/internal/atomic.h',
'golang/runtime/internal/syscall.h',
'golang/runtime/platform.h',
'golang/context.h',
'golang/cxx.h',
'golang/errors.h',
......@@ -259,12 +478,21 @@ setup(
include_dirs = [sysconfig.get_python_inc()],
library_dirs = [get_python_libdir()],
define_macros = [('BUILDING_LIBPYXRUNTIME', None)],
soversion = '0.1')],
soversion = '0.1')]
+ ([funchook_dso] if funchook_dso else []),
ext_modules = [
Ext('golang._golang',
['golang/_golang.pyx'],
depends = ['golang/_golang_str.pyx']),
['golang/_golang.pyx',
'golang/_golang_str_pickle.S'],
depends = [
'golang/_golang_str.pyx',
'golang/_golang_str_pickle.pyx',
'golang/_golang_str_pickle_test.pyx',
'golang/_golang_str_pickle.S'],
dsos = ['golang.runtime.funchook'], # XXX only if available
include_dirs = ['3rdparty/funchook/include',
'3rdparty/capstone/include']),
Ext('golang.runtime._runtime_thread',
['golang/runtime/_runtime_thread.pyx']),
......@@ -334,6 +562,14 @@ setup(
Ext('golang._time',
['golang/_time.pyx'],
dsos = ['golang.runtime.libpyxruntime']),
# XXX consider putting everything into just gpython.pyx + .c
Ext('gpython._gpython',
['gpython/_gpython.pyx',
'gpython/_gpython_c.cpp'], # XXX do we need C++ here?
include_dirs = ['3rdparty/funchook/include'],
dsos = ['golang.runtime.funchook'], # XXX only if available
),
],
include_package_data = True,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7