Commit b44f9c0d authored by Kirill Smelkov's avatar Kirill Smelkov

More python3 compatibility

@jerome, I was trying to make zodbtools work with Python3 and along that road picked some bits of your work from nexedi/zodbtools!12. At present the migration to Python3 is not complete, and even though now I have the answer to how handle strings in both python2/3 in compatible and reasonable way (I can share details if you are interested), I have to put that work on hold for some time and use https://pypi.org/project/pep3134 directly in wcfs tests, since getting all string details right, even after figuring on how to do it, will take time. Anyway the bits presented here should be ready for master and could be merged now. Could you please have a look?

Thanks beforehand,  
Kirill

/reviewed-on nexedi/zodbtools!13
parents bcaf3984 c5f20201
...@@ -23,7 +23,7 @@ setup( ...@@ -23,7 +23,7 @@ setup(
install_requires = ['ZODB', 'zodburi', 'zope.interface', 'pygolang >= 0.0.0.dev6', 'six', 'dateparser'], install_requires = ['ZODB', 'zodburi', 'zope.interface', 'pygolang >= 0.0.0.dev6', 'six', 'dateparser'],
extras_require = { extras_require = {
'test': ['pytest', 'freezegun', 'pytz'], 'test': ['pytest', 'freezegun', 'pytz', 'mock;python_version<="2.7"'],
}, },
entry_points= {'console_scripts': ['zodb = zodbtools.zodb:main']}, entry_points= {'console_scripts': ['zodb = zodbtools.zodb:main']},
......
...@@ -8,6 +8,7 @@ deps = ...@@ -8,6 +8,7 @@ deps =
pytest pytest
freezegun freezegun
pytz pytz
mock;python_version<="2.7"
# latest ZODB from ZODB3 series # latest ZODB from ZODB3 series
ZODB3: ZODB3 >=3.10, <3.11dev ZODB3: ZODB3 >=3.10, <3.11dev
......
# Copyright (C) 2018 Nexedi SA and Contributors. # -*- coding: utf-8 -*-
# Kirill Smelkov <kirr@nexedi.com> # Copyright (C) 2018-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -41,8 +43,8 @@ def test_zodbcommit(zext): ...@@ -41,8 +43,8 @@ def test_zodbcommit(zext):
# commit some transactions via zodbcommit and verify if storage dump gives # commit some transactions via zodbcommit and verify if storage dump gives
# what is expected. # what is expected.
t1 = Transaction(z64, ' ', b'user name', b'description ...', zext(dumps({'a': 'b'}, _protocol)), [ t1 = Transaction(z64, ' ', b'user name', b'description ...', zext(dumps({'a': 'b'}, _protocol)), [
ObjectData(p64(1), b'data1', 'sha1', sha1('data1')), ObjectData(p64(1), b'data1', 'sha1', sha1(b'data1')),
ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))]) ObjectData(p64(2), b'data2', 'sha1', sha1(b'data2'))])
t1.tid = zodbcommit(stor, head, t1) t1.tid = zodbcommit(stor, head, t1)
......
# Copyright (C) 2017-2018 Nexedi SA and Contributors. # -*- coding: utf-8 -*-
# Copyright (C) 2017-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -21,9 +23,10 @@ from zodbtools.zodbdump import ( ...@@ -21,9 +23,10 @@ from zodbtools.zodbdump import (
zodbdump, DumpReader, Transaction, ObjectDelete, ObjectCopy, zodbdump, DumpReader, Transaction, ObjectDelete, ObjectCopy,
ObjectData, HashOnly ObjectData, HashOnly
) )
from zodbtools.util import fromhex
from ZODB.FileStorage import FileStorage from ZODB.FileStorage import FileStorage
from ZODB.utils import p64 from ZODB.utils import p64
from cStringIO import StringIO from io import BytesIO
from os.path import dirname from os.path import dirname
...@@ -39,7 +42,7 @@ def test_zodbdump(zext): ...@@ -39,7 +42,7 @@ def test_zodbdump(zext):
with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f: with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f:
dumpok = f.read() dumpok = f.read()
out = StringIO() out = BytesIO()
zodbdump(stor, None, None, out=out) zodbdump(stor, None, None, out=out)
assert out.getvalue() == dumpok assert out.getvalue() == dumpok
...@@ -69,10 +72,10 @@ extension "qqq" ...@@ -69,10 +72,10 @@ extension "qqq"
""" """
r = DumpReader(StringIO(in_)) r = DumpReader(BytesIO(in_))
t1 = r.readtxn() t1 = r.readtxn()
assert isinstance(t1, Transaction) assert isinstance(t1, Transaction)
assert t1.tid == '0123456789abcdef'.decode('hex') assert t1.tid == fromhex('0123456789abcdef')
assert t1.user == b'my name' assert t1.user == b'my name'
assert t1.description == b'o la-la...' assert t1.description == b'o la-la...'
assert t1.extension_bytes == b'zzz123 def' assert t1.extension_bytes == b'zzz123 def'
...@@ -83,29 +86,29 @@ extension "qqq" ...@@ -83,29 +86,29 @@ extension "qqq"
_ = t1.objv[1] _ = t1.objv[1]
assert isinstance(_, ObjectCopy) assert isinstance(_, ObjectCopy)
assert _.oid == p64(2) assert _.oid == p64(2)
assert _.copy_from == '0123456789abcdee'.decode('hex') assert _.copy_from == fromhex('0123456789abcdee')
_ = t1.objv[2] _ = t1.objv[2]
assert isinstance(_, ObjectData) assert isinstance(_, ObjectData)
assert _.oid == p64(3) assert _.oid == p64(3)
assert _.data == HashOnly(54) assert _.data == HashOnly(54)
assert _.hashfunc == 'adler32' assert _.hashfunc == 'adler32'
assert _.hash_ == '01234567'.decode('hex') assert _.hash_ == fromhex('01234567')
_ = t1.objv[3] _ = t1.objv[3]
assert isinstance(_, ObjectData) assert isinstance(_, ObjectData)
assert _.oid == p64(4) assert _.oid == p64(4)
assert _.data == b'ZZZZ' assert _.data == b'ZZZZ'
assert _.hashfunc == 'sha1' assert _.hashfunc == 'sha1'
assert _.hash_ == '9865d483bc5a94f2e30056fc256ed3066af54d04'.decode('hex') assert _.hash_ == fromhex('9865d483bc5a94f2e30056fc256ed3066af54d04')
_ = t1.objv[4] _ = t1.objv[4]
assert isinstance(_, ObjectData) assert isinstance(_, ObjectData)
assert _.oid == p64(5) assert _.oid == p64(5)
assert _.data == b'ABC\n\nDEF!' assert _.data == b'ABC\n\nDEF!'
assert _.hashfunc == 'crc32' assert _.hashfunc == 'crc32'
assert _.hash_ == '52fdeac5'.decode('hex') assert _.hash_ == fromhex('52fdeac5')
t2 = r.readtxn() t2 = r.readtxn()
assert isinstance(t2, Transaction) assert isinstance(t2, Transaction)
assert t2.tid == '0123456789abcdf0'.decode('hex') assert t2.tid == fromhex('0123456789abcdf0')
assert t2.user == b'author2' assert t2.user == b'author2'
assert t2.description == b'zzz' assert t2.description == b'zzz'
assert t2.extension_bytes == b'qqq' assert t2.extension_bytes == b'qqq'
...@@ -117,7 +120,7 @@ extension "qqq" ...@@ -117,7 +120,7 @@ extension "qqq"
assert z == in_ assert z == in_
# unknown hash function # unknown hash function
r = DumpReader(StringIO("""\ r = DumpReader(BytesIO("""\
txn 0000000000000000 " " txn 0000000000000000 " "
user "" user ""
description "" description ""
...@@ -130,7 +133,7 @@ obj 0000000000000001 1 xyz:0123 - ...@@ -130,7 +133,7 @@ obj 0000000000000001 1 xyz:0123 -
assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ('obj 0000000000000001 1 xyz:0123 -')""",) assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ('obj 0000000000000001 1 xyz:0123 -')""",)
# data integrity error # data integrity error
r = DumpReader(StringIO("""\ r = DumpReader(BytesIO("""\
txn 0000000000000000 " " txn 0000000000000000 " "
user "" user ""
description "" description ""
......
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
# Jérome Perrin <jerome@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import sys
try:
from unittest import mock
except ImportError:
# BBB python2
import mock
import pytest
from zodbtools import zodb
from zodbtools import help as help_module
# zodbrun runs zodb.main with argv and returns exit code + captured stdout/stderr.
def zodbrun(capsys, *argv):
with mock.patch.object(sys, 'argv', ('zodb',) + argv), \
pytest.raises(SystemExit) as excinfo:
zodb.main()
assert len(excinfo.value.args) == 1
ecode = excinfo.value.args[0]
return ecode, capsys.readouterr()
def test_main(capsys):
e, _ = zodbrun(capsys)
assert e == 2
assert "" == _.out
assert "Zodb is a tool for managing ZODB databases." in _.err
e, _ = zodbrun(capsys, '-h')
assert e == 0
assert "Zodb is a tool for managing ZODB databases." in _.out
assert "" == _.err
@pytest.mark.parametrize(
"help_topic",
tuple(zodb.command_dict) + tuple(help_module.topic_dict))
def test_help(capsys, help_topic):
e, _ = zodbrun(capsys, 'help', help_topic)
assert e == 0
assert _.err == ""
assert _.out != ""
# -*- coding: utf-8 -*-
# zodbtools - various utility routines # zodbtools - various utility routines
# Copyright (C) 2016-2018 Nexedi SA and Contributors. # Copyright (C) 2016-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -18,7 +20,7 @@ ...@@ -18,7 +20,7 @@
# See COPYING file for full licensing terms. # See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options. # See https://www.nexedi.com/licensing for rationale and options.
import hashlib, struct, codecs import hashlib, struct, codecs, io
import zodburi import zodburi
from six.moves.urllib_parse import urlsplit, urlunsplit from six.moves.urllib_parse import urlsplit, urlunsplit
from zlib import crc32, adler32 from zlib import crc32, adler32
...@@ -26,12 +28,15 @@ from ZODB.TimeStamp import TimeStamp ...@@ -26,12 +28,15 @@ from ZODB.TimeStamp import TimeStamp
import dateparser import dateparser
def ashex(s): def ashex(s):
return s.encode('hex') # type: (bytes) -> bytes
return codecs.encode(s, 'hex')
def fromhex(s): def fromhex(s):
# type: (Union[str,bytes]) -> bytes
return codecs.decode(s, 'hex') return codecs.decode(s, 'hex')
def sha1(data): def sha1(data):
# type: (bytes) -> bytes
m = hashlib.sha1() m = hashlib.sha1()
m.update(data) m.update(data)
return m.digest() return m.digest()
...@@ -183,7 +188,7 @@ class Adler32Hasher: ...@@ -183,7 +188,7 @@ class Adler32Hasher:
digest_size = 4 digest_size = 4
def __init__(self): def __init__(self):
self._h = adler32('') self._h = adler32(b'')
def update(self, data): def update(self, data):
self._h = adler32(data, self._h) self._h = adler32(data, self._h)
...@@ -200,7 +205,7 @@ class CRC32Hasher: ...@@ -200,7 +205,7 @@ class CRC32Hasher:
digest_size = 4 digest_size = 4
def __init__(self): def __init__(self):
self._h = crc32('') self._h = crc32(b'')
def update(self, data): def update(self, data):
self._h = crc32(data, self._h) self._h = crc32(data, self._h)
...@@ -220,3 +225,13 @@ hashRegistry = { ...@@ -220,3 +225,13 @@ hashRegistry = {
"sha256": hashlib.sha256, "sha256": hashlib.sha256,
"sha512": hashlib.sha512, "sha512": hashlib.sha512,
} }
# ---- IO ----
# asbinstream return binary stream associated with stream.
# For example on py3 sys.stdout is io.TextIO which does not allow to write binary data to it.
def asbinstream(stream):
# type: (IO) -> BinaryIO
if isinstance(stream, io.TextIOBase):
return stream.buffer
return stream
#!/usr/bin/env python #!/usr/bin/env python
# Copyright (C) 2017-2018 Nexedi SA and Contributors. # -*- coding: utf-8 -*-
# Copyright (C) 2017-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -51,10 +53,7 @@ Usage: ...@@ -51,10 +53,7 @@ Usage:
The commands are: The commands are:
""", file=out) """, file=out)
cmdv = command_dict.keys() for cmd, cmd_module in sorted(command_dict.items()):
cmdv.sort()
for cmd in cmdv:
cmd_module = command_dict[cmd]
print(" %-11s %s" % (cmd, cmd_module.summary), file=out) print(" %-11s %s" % (cmd, cmd_module.summary), file=out)
print("""\ print("""\
......
# Copyright (C) 2018 Nexedi SA and Contributors. # Copyright (C) 2018-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -40,7 +40,7 @@ can query current database head (last_tid) with `zodb info <stor> last_tid`. ...@@ -40,7 +40,7 @@ can query current database head (last_tid) with `zodb info <stor> last_tid`.
from __future__ import print_function from __future__ import print_function
from zodbtools import zodbdump from zodbtools import zodbdump
from zodbtools.util import ashex, storageFromURL from zodbtools.util import ashex, fromhex, storageFromURL
from ZODB.utils import p64, u64, z64 from ZODB.utils import p64, u64, z64
from ZODB.POSException import POSKeyError from ZODB.POSException import POSKeyError
from ZODB._compat import BytesIO from ZODB._compat import BytesIO
...@@ -154,7 +154,7 @@ def main(argv): ...@@ -154,7 +154,7 @@ def main(argv):
sys.exit(2) sys.exit(2)
storurl = argv[0] storurl = argv[0]
at = argv[1].decode('hex') at = fromhex(argv[1])
stor = storageFromURL(storurl) stor = storageFromURL(storurl)
defer(stor.close) defer(stor.close)
......
# Copyright (C) 2016-2018 Nexedi SA and Contributors. # -*- coding: utf-8 -*-
# Copyright (C) 2016-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -54,7 +56,7 @@ TODO also protect txn record by hash. ...@@ -54,7 +56,7 @@ TODO also protect txn record by hash.
from __future__ import print_function from __future__ import print_function
from zodbtools.util import ashex, fromhex, sha1, txnobjv, parse_tidrange, TidRangeInvalid, \ from zodbtools.util import ashex, fromhex, sha1, txnobjv, parse_tidrange, TidRangeInvalid, \
storageFromURL, hashRegistry storageFromURL, hashRegistry, asbinstream
from ZODB._compat import loads, _protocol, BytesIO from ZODB._compat import loads, _protocol, BytesIO
from zodbpickle.slowpickle import Pickler as pyPickler from zodbpickle.slowpickle import Pickler as pyPickler
#import pickletools #import pickletools
...@@ -90,7 +92,7 @@ _already_warned_notxnraw = set() ...@@ -90,7 +92,7 @@ _already_warned_notxnraw = set()
# zodbdump dumps content of a ZODB storage to a file. # zodbdump dumps content of a ZODB storage to a file.
# please see module doc-string for dump format and details # please see module doc-string for dump format and details
def zodbdump(stor, tidmin, tidmax, hashonly=False, out=sys.stdout): def zodbdump(stor, tidmin, tidmax, hashonly=False, out=asbinstream(sys.stdout)):
for txn in stor.iterator(tidmin, tidmax): for txn in stor.iterator(tidmin, tidmax):
# XXX .status not covered by IStorageTransactionInformation # XXX .status not covered by IStorageTransactionInformation
# XXX but covered by BaseStorage.TransactionRecord # XXX but covered by BaseStorage.TransactionRecord
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment