Commit ccb85991 authored by Kirill Smelkov's avatar Kirill Smelkov

commit: Fix it wrt ZEO

Since 7ae5ff82 (Port zodbtools to py3) zodbtools/py3 requires
nexedi/pygolang!21 with which
running `zodb commit` wrt ZEO, as demonstrates by added
test_zodbcommit_cmd, yields:

    test/test_commit.py::test_zodbcommit_cmd[ZEO-!zext] Traceback (most recent call last):
      File "/usr/lib/python2.7/runpy.py", line 174, in _run_module_as_main
        "__main__", fname, loader, pkg_name)
      File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
        exec code in run_globals
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodb.py", line 133, in <module>
        main()
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodb.py", line 129, in main
        return command_module.main(argv)
      File "<decorator-gen-4>", line 2, in main
      File "/home/kirr/src/tools/go/pygolang/golang/__init__.py", line 125, in _
        return f(*argv, **kw)
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodbcommit.py", line 232, in main
        tid = zodbcommit(stor, at, txn)
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodbcommit.py", line 131, in zodbcommit
        _()
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodbcommit.py", line 128, in _
        stor.store(obj.oid, current_serial(obj.oid), data, '', txn)
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodbcommit.py", line 87, in current_serial
        return _serial_at(stor, oid, at)
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodbcommit.py", line 152, in _serial_at
        xdata = stor.loadBefore(oid, before)
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/ClientStorage.py", line 616, in loadBefore
        return self._server.load_before(oid, tid)
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/client.py", line 935, in load_before
        return self.io_call(self.load_before_co(oid, tid, self.timeout))
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/client.py", line 862, in io_call
        return future.result() if wait else future
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/futures.py", line 230, in result
        return Future.result(self)
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/futures.py", line 81, in result
        raise self._result
    ClientDisconnected: [Errno 104] Connection reset by peer
    FAILED()
    zeo.log:

    2024-07-16T21:13:45 ERROR ZEO.asyncio.server Can't deserialize message
    Traceback (most recent call last):
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/server.py", line 100, in message_received
        message_id, async_, name, args = self.decode(message)
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/marshal.py", line 119, in pickle_server_decode
        return unpickler.load()  # msgid, flags, name, args
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/marshal.py", line 176, in server_find_global
        raise ImportError("Module not allowed: %s" % (module,))
    ImportError: Module not allowed: golang

This happens because zodbcommit.main and zodbdump.DumpReader use bstr to
handle binary data and further pass as IStorageTransactionInformation
instance to ZEO via its RPC, but ZEO server explicitly checks safe to
import modules and rejects golang.

One way to fix would be to patch ZEO to allow golang imports, but in
general it is better to follow the interface strictly and stay 100%
compatible with its users. That's why the fix converts fields covered by
IStorageTransactionInformation to exact types specified by that
interface to stay 100% compatible with users of the interface because on
py3 e.g. (b' ' == ' ') returns False and so we need to be careful to
provide .status as exactly str instead bytes, and do the similar for
other fields.
parent 8eebbe62
......@@ -21,11 +21,16 @@
from zodbtools.zodbcommit import zodbcommit
from zodbtools.zodbdump import zodbdump, Transaction, ObjectData, ObjectDelete, ObjectCopy
from zodbtools.util import storageFromURL, sha1
from zodbtools.util import storageFromURL, sha1, ashex, fromhex
from ZODB.utils import p64, u64, z64
from ZODB._compat import BytesIO, dumps, _protocol # XXX can't yet commit with arbitrary ext.bytes
from golang import func, defer, b
from golang.gcompat import qq
import sys
from subprocess import Popen, PIPE
# verify zodbcommit.
@func
......@@ -66,3 +71,76 @@ def test_zodbcommit(zsrv, zext):
data1_3, _, _ = stor.loadBefore(p64(1), p64(u64(t3.tid)+1))
assert data1_1 == data1_3
assert data1_1 == b'data1' # just in case
# verify zodbcommit via commandline / stdin.
def test_zodbcommit_cmd(zsrv, zext):
# for ZEO sync is noop unless server_sync=True is specified in options,
# but server_sync is available only on ZEO5, not ZEO4. Work it around with
# opening/closing the storage on every query.
@func
def zsrv_do(f):
stor = storageFromURL(zsrv.zurl)
defer(stor.close)
return f(stor)
at0 = zsrv_do(lambda stor: stor.lastTransaction())
# zodbcommit_cmd does `zodb commit` via command line and returns TID of
# committed transction.
def zodbcommit_cmd(at, stdin): # -> tid
p = Popen([sys.executable, '-m', 'zodbtools.zodb', 'commit',
zsrv.zurl, ashex(at)], stdin=PIPE, stdout=PIPE)
stdout, _ = p.communicate(stdin)
assert p.returncode == 0, stdout
return fromhex(stdout.rstrip())
t1 = b'''\
user "user name"
description "description ..."
extension %s
obj 0000000000000001 5 sha1:%s
data1
obj 0000000000000002 5 sha1:%s
data2
''' % (qq(zext(dumps({'a': 'b'}, _protocol))), ashex(sha1(b'data1')), ashex(sha1(b'data2')))
t2 = b'''\
user "user2"
description "desc2"
extension ""
obj 0000000000000002 delete
'''
at1 = zodbcommit_cmd(at0, t1)
at2 = zodbcommit_cmd(at1, t2)
t1 = (b'txn %s " "\n' % ashex(at1)) + t1
t2 = (b'txn %s " "\n' % ashex(at2)) + t2
def _(stor):
buf = BytesIO()
zodbdump(stor, p64(u64(at0)+1), None, out=buf)
return buf.getvalue()
dumped = zsrv_do(_)
assert dumped == b''.join([t1, t2])
t3 = b'''\
user "user3"
description "desc3"
extension ""
obj 0000000000000001 from %s
''' % ashex(at1)
# XXX see note about ObjectCopy in test_zodbcommit
at3 = zodbcommit_cmd(at1, t3)
def _(stor):
data1_1, _, _ = stor.loadBefore(p64(1), p64(u64(at1)+1))
data1_3, _, _ = stor.loadBefore(p64(1), p64(u64(at3)+1))
assert data1_1 == data1_3
assert data1_1 == b'data1' # just in case
zsrv_do(_)
......@@ -491,11 +491,19 @@ class Transaction(object):
# .extension_bytes bytes transaction extension
# .objv []Object* objects changed by transaction
def __init__(self, tid, status, user, description, extension, objv):
self.tid = tid
self.status = status
self.user = user
self.description = description
self.extension_bytes = extension
# NOTE we convert fields covered by IStorageTransactionInformation to
# exact types specified by that interface to stay 100% compatible with
# users of the interface because on py3 e.g. (b' ' == ' ') returns
# False and so we need to be careful to provide .status as exactly str
# instead bytes, and do the similar for other fields. It also would be
# generally incorrect to use bstr/ustr for the fields, because e.g. ZEO
# rejects messages with golang.bstr/ustr objects on the basis that they
# do not come from allowed list of modules.
self.tid = _exactly_bytes(tid)
self.status = _exactly_str(status)
self.user = _exactly_bytes(user)
self.description = _exactly_bytes(description)
self.extension_bytes = _exactly_bytes(extension)
self.objv = objv
# ZODB wants to work with extension as {} - try to convert it on the fly.
......@@ -593,3 +601,28 @@ class HashOnly(object):
def __eq__(a, b):
return isinstance(b, HashOnly) and a.size == b.size
# _exactly_bytes returns obj as an instance of exactly type bytes.
#
# obj must be initially an instance of bytes, e.g. bstr.
def _exactly_bytes(obj): # -> bytes
assert isinstance(obj, bytes), type(obj)
if type(obj) is not bytes:
obj = b(obj) # bstr
obj = obj.encode() # bytes
assert type(obj) is bytes, type(obj)
return obj
# _exactly_str returns obj as an instance of exactly type str.
#
# obj must be initially an instance of bytes or str/unicode, e.g. bstr or ustr.
def _exactly_str(obj): # -> str
if type(obj) is not str:
obj = b(obj) # bstr
obj = obj.encode() # bytes
if str is not bytes:
obj = obj.decode('UTF-8')
assert type(obj) is str, type(obj)
return obj
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment