From d8b712d7f87355fe8dbeda47cfdbfee9be209108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Wed, 30 Jan 2019 08:54:32 +0100 Subject: [PATCH 1/9] tox: run tests for python 3 also simplify a bit definition as ZODB is common in all versions --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f8ff560..446355f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ # zodbtools | tox setup [tox] -envlist = py27-{ZODB3,ZODB4,ZODB5} +envlist = py27-ZODB{3,4,5}, py3{6,7,8}-ZODB{4,5} [testenv] deps = -- 2.30.9 From 1f46e389bf4c89881303c88b6d13c6436b0919e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Wed, 30 Jan 2019 08:56:16 +0100 Subject: [PATCH 2/9] tox: run tests against ZODB #183 until https://github.com/zopefoundation/ZODB/pull/183 gets merged, let's run also the tests for this, since we have support for this extension. --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 446355f..9483889 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ # zodbtools | tox setup [tox] -envlist = py27-ZODB{3,4,5}, py3{6,7,8}-ZODB{4,5} +envlist = py27-ZODB{3,4,5,devPR183}, py3{6,7,8}-ZODB{4,5,devPR183} [testenv] deps = @@ -26,5 +26,8 @@ deps = ZODB5: ZODB >=5.0, <6.0dev ZODB5: ZEO >=5.0, <6.0dev + # with the patches from https://github.com/zopefoundation/ZODB/pull/183 + ZODBdevPR183: -e git+https://github.com/navytux/ZODB@7f5a67b3e0d2dcf2d13f18a42a82e526f6d4055c#egg=ZODB + ZODBdevPR183: ZEO >=5.0, <6.0dev commands= {envpython} -m pytest -- 2.30.9 From 8d1b444206aeab7a7ac591a7e5fafdb7aef7fe1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Wed, 30 Jan 2019 09:56:03 +0100 Subject: [PATCH 3/9] tox: fix outdated comment --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9483889..6956149 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = py27-ZODB{3,4,5,devPR183}, py3{6,7,8}-ZODB{4,5,devPR183} [testenv] deps = - # XXX tox does not consult extras_require['test'] -> install pytest explicitly + # XXX tox does not consult extras_require['test'] -> install test dependencies explicitly pytest freezegun pytz -- 2.30.9 From 19f230c28df9a42e578c40ee92d3a010ca97e3db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Wed, 30 Jan 2019 10:12:28 +0100 Subject: [PATCH 4/9] zodbdump: use r-strings for regexp this silents a warning about \w being unknown escape sequence --- zodbtools/zodbdump.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zodbtools/zodbdump.py b/zodbtools/zodbdump.py index 5164c4c..75234f8 100644 --- a/zodbtools/zodbdump.py +++ b/zodbtools/zodbdump.py @@ -283,8 +283,11 @@ def main(argv): # ---------------------------------------- # dump reading/parsing -_txn_re = re.compile(b'^txn (?P[0-9a-f]{16}) "(?P.)"$') -_obj_re = re.compile(b'^obj (?P[0-9a-f]{16}) ((?Pdelete)|from (?P[0-9a-f]{16})|(?P[0-9]+) (?P\w+):(?P[0-9a-f]+)(?P -)?)') +_txn_re = re.compile(br'^txn (?P[0-9a-f]{16}) "(?P.)"$') +_obj_re = re.compile( + br'^obj (?P[0-9a-f]{16}) ((?Pdelete)|from (?P[0-9a-f]{16})|(?P[0-9]+) (?P\w+):(?P[0-9a-f]+)(?P -)?)' +) + # _ioname returns name of the reader r, if it has one. # if there is no name - '' is returned. -- 2.30.9 From efa9e9451d28e8937056ff3561552fafdf4881b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Wed, 30 Jan 2019 10:14:45 +0100 Subject: [PATCH 5/9] zodbdump: use logging.warning instead of deprecated logging.warn --- zodbtools/zodbdump.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/zodbtools/zodbdump.py b/zodbtools/zodbdump.py index 75234f8..657a9fe 100644 --- a/zodbtools/zodbdump.py +++ b/zodbtools/zodbdump.py @@ -78,9 +78,15 @@ def txn_raw_extension(stor, txn): # in a rational way stor_name = "(%s, %s)" % (type(stor).__name__, stor.getName()) if stor_name not in _already_warned_notxnraw: - logging.warn("%s: storage does not provide IStorageTransactionInformationRaw ...", stor_name) - logging.warn("... will do best-effort to dump pickles in stable order but this cannot be done 100% correctly") - logging.warn("... please upgrade your ZODB & storage: see https://github.com/zopefoundation/ZODB/pull/183 for details.") + logging.warning( + "%s: storage does not provide IStorageTransactionInformationRaw ...", + stor_name) + logging.warning( + "... will do best-effort to dump pickles in stable order but this cannot be done 100% correctly" + ) + logging.warning( + "... please upgrade your ZODB & storage: see https://github.com/zopefoundation/ZODB/pull/183 for details." + ) _already_warned_notxnraw.add(stor_name) return serializeext(txn.extension) -- 2.30.9 From 1cab40946c2bec211937759204174e895cfa3804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Wed, 30 Jan 2019 14:15:31 +0100 Subject: [PATCH 6/9] zodbdump: fix invalid escape \- warning --- zodbtools/zodbdump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zodbtools/zodbdump.py b/zodbtools/zodbdump.py index 657a9fe..8a5549b 100644 --- a/zodbtools/zodbdump.py +++ b/zodbtools/zodbdump.py @@ -16,7 +16,7 @@ # # See COPYING file for full licensing terms. # See https://www.nexedi.com/licensing for rationale and options. -"""Zodbdump - Tool to dump content of a ZODB database +r"""Zodbdump - Tool to dump content of a ZODB database This program dumps content of a ZODB database. It uses ZODB Storage iteration API to get list of transactions and for every -- 2.30.9 From 27064917707d7bb6a023811ff0896de5ab2bfdf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Wed, 30 Jan 2019 12:30:58 +0100 Subject: [PATCH 7/9] test: add a test for zodb commmad and help driver --- zodbtools/test/test_zodb.py | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 zodbtools/test/test_zodb.py diff --git a/zodbtools/test/test_zodb.py b/zodbtools/test/test_zodb.py new file mode 100644 index 0000000..cda6f54 --- /dev/null +++ b/zodbtools/test/test_zodb.py @@ -0,0 +1,51 @@ +# Copyright (C) 2019 Nexedi SA and Contributors. +# Jerome Perrin +# +# This program is free software: you can Use, Study, Modify and Redistribute +# it under the terms of the GNU General Public License version 3, or (at your +# option) any later version, as published by the Free Software Foundation. +# +# You can also Link and Combine this program with other software covered by +# the terms of any of the Free Software licenses or any of the Open Source +# Initiative approved licenses and Convey the resulting work. Corresponding +# source of such a combination shall include the source code for all other +# software used. +# +# This program is distributed WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See COPYING file for full licensing terms. +# See https://www.nexedi.com/licensing for rationale and options. + +import sys +try: + from unittest import mock +except ImportError: + # BBB python2 + import mock + +import pytest + +from zodbtools import zodb +from zodbtools import help as help_module + + +def test_main(capsys): + with mock.patch.object(sys, 'argv', ('zodb', )), \ + pytest.raises(SystemExit) as excinfo: + zodb.main() + assert "Zodb is a tool for managing ZODB databases." in capsys.readouterr( + ).err + assert excinfo.value.args == (2, ) + + +@pytest.mark.parametrize( + "help_topic", + tuple(zodb.command_dict) + tuple(help_module.topic_dict)) +def test_help(capsys, help_topic): + with mock.patch.object(sys, 'argv', ('zodb', 'help', help_topic)), \ + pytest.raises(SystemExit) as excinfo: + zodb.main() + assert capsys.readouterr().out + assert "" == capsys.readouterr().err + assert excinfo.value.args == (0, ) -- 2.30.9 From 7201175586d494e70a37dad844287811e8c6835d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Wed, 30 Jan 2019 12:31:27 +0100 Subject: [PATCH 8/9] zodb: rework command driver for python3 compatibility --- setup.py | 2 +- tox.ini | 1 + zodbtools/zodb.py | 5 +---- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index dc58157..2788c0c 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ setup( install_requires = ['ZODB', 'zodburi', 'zope.interface', 'pygolang >= 0.0.0.dev6', 'six', 'dateparser'], extras_require = { - 'test': ['pytest', 'freezegun', 'pytz'], + 'test': ['pytest', 'freezegun', 'pytz', 'mock;python_version<="2.7"'], }, entry_points= {'console_scripts': ['zodb = zodbtools.zodb:main']}, diff --git a/tox.ini b/tox.ini index 6956149..42b3508 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ deps = pytest freezegun pytz + mock;python_version<="2.7" # latest ZODB from ZODB3 series ZODB3: ZODB3 >=3.10, <3.11dev diff --git a/zodbtools/zodb.py b/zodbtools/zodb.py index f349327..c99e8ee 100755 --- a/zodbtools/zodb.py +++ b/zodbtools/zodb.py @@ -51,10 +51,7 @@ Usage: The commands are: """, file=out) - cmdv = command_dict.keys() - cmdv.sort() - for cmd in cmdv: - cmd_module = command_dict[cmd] + for cmd, cmd_module in sorted(command_dict.items()): print(" %-11s %s" % (cmd, cmd_module.summary), file=out) print("""\ -- 2.30.9 From 347ea21dc304927370f61689cee169dce1b3f2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Wed, 30 Jan 2019 10:07:46 +0100 Subject: [PATCH 9/9] util: add type annotations and fix bytes vs str --- mypy.ini | 3 ++ zodbtools/help.py | 7 ++- zodbtools/test/test_commit.py | 6 +-- zodbtools/test/test_dump.py | 30 ++++++------- zodbtools/test/test_zodb.py | 4 +- zodbtools/util.py | 45 ++++++++++++++----- zodbtools/zodbanalyze.py | 2 +- zodbtools/zodbdump.py | 84 ++++++++++++++++++++++------------- 8 files changed, 117 insertions(+), 64 deletions(-) create mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..bab5d3a --- /dev/null +++ b/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +# XXX according to mypy doc, this is a bad idea. +ignore_missing_imports = True diff --git a/zodbtools/help.py b/zodbtools/help.py index 5a12ad2..0b4a10d 100644 --- a/zodbtools/help.py +++ b/zodbtools/help.py @@ -19,10 +19,15 @@ # See COPYING file for full licensing terms. # See https://www.nexedi.com/licensing for rationale and options. +try: + from typing import Tuple +except ImportError: + pass + from collections import OrderedDict # topic_name -> (topic_summary, topic_help) -topic_dict = OrderedDict() +topic_dict = OrderedDict() # type: OrderedDict[str, Tuple[str, str]] help_zurl = """\ Almost every zodb command works with a database. diff --git a/zodbtools/test/test_commit.py b/zodbtools/test/test_commit.py index 5cca7a0..560b33f 100644 --- a/zodbtools/test/test_commit.py +++ b/zodbtools/test/test_commit.py @@ -41,8 +41,8 @@ def test_zodbcommit(zext): # commit some transactions via zodbcommit and verify if storage dump gives # what is expected. t1 = Transaction(z64, ' ', b'user name', b'description ...', zext(dumps({'a': 'b'}, _protocol)), [ - ObjectData(p64(1), b'data1', 'sha1', sha1('data1')), - ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))]) + ObjectData(p64(1), b'data1', 'sha1', sha1(b'data1')), + ObjectData(p64(2), b'data2', 'sha1', sha1(b'data2'))]) t1.tid = zodbcommit(stor, head, t1) @@ -56,7 +56,7 @@ def test_zodbcommit(zext): zodbdump(stor, p64(u64(head)+1), None, out=buf) dumped = buf.getvalue() - assert dumped == ''.join([_.zdump() for _ in (t1, t2)]) + assert dumped == b''.join([_.zdump() for _ in (t1, t2)]) # ObjectCopy. XXX zodbcommit handled ObjectCopy by actually copying data, # not referencing previous transaction via backpointer. diff --git a/zodbtools/test/test_dump.py b/zodbtools/test/test_dump.py index ec317f0..0000afc 100644 --- a/zodbtools/test/test_dump.py +++ b/zodbtools/test/test_dump.py @@ -21,14 +21,14 @@ from zodbtools.zodbdump import ( zodbdump, DumpReader, Transaction, ObjectDelete, ObjectCopy, ObjectData, HashOnly ) +from zodbtools.util import fromhex from ZODB.FileStorage import FileStorage from ZODB.utils import p64 -from cStringIO import StringIO +from io import BytesIO from os.path import dirname -from zodbtools.test.testutil import zext_supported -from pytest import raises, xfail +from pytest import raises # verify zodbdump output against golden def test_zodbdump(zext): @@ -39,7 +39,7 @@ def test_zodbdump(zext): with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f: dumpok = f.read() - out = StringIO() + out = BytesIO() zodbdump(stor, None, None, out=out) assert out.getvalue() == dumpok @@ -69,10 +69,10 @@ extension "qqq" """ - r = DumpReader(StringIO(in_)) + r = DumpReader(BytesIO(in_)) t1 = r.readtxn() assert isinstance(t1, Transaction) - assert t1.tid == '0123456789abcdef'.decode('hex') + assert t1.tid == fromhex('0123456789abcdef') assert t1.user == b'my name' assert t1.description == b'o la-la...' assert t1.extension_bytes == b'zzz123 def' @@ -83,29 +83,29 @@ extension "qqq" _ = t1.objv[1] assert isinstance(_, ObjectCopy) assert _.oid == p64(2) - assert _.copy_from == '0123456789abcdee'.decode('hex') + assert _.copy_from == fromhex('0123456789abcdee') _ = t1.objv[2] assert isinstance(_, ObjectData) assert _.oid == p64(3) assert _.data == HashOnly(54) assert _.hashfunc == 'adler32' - assert _.hash_ == '01234567'.decode('hex') + assert _.hash_ == fromhex('01234567') _ = t1.objv[3] assert isinstance(_, ObjectData) assert _.oid == p64(4) assert _.data == b'ZZZZ' assert _.hashfunc == 'sha1' - assert _.hash_ == '9865d483bc5a94f2e30056fc256ed3066af54d04'.decode('hex') + assert _.hash_ == fromhex('9865d483bc5a94f2e30056fc256ed3066af54d04') _ = t1.objv[4] assert isinstance(_, ObjectData) assert _.oid == p64(5) assert _.data == b'ABC\n\nDEF!' assert _.hashfunc == 'crc32' - assert _.hash_ == '52fdeac5'.decode('hex') + assert _.hash_ == fromhex('52fdeac5') t2 = r.readtxn() assert isinstance(t2, Transaction) - assert t2.tid == '0123456789abcdf0'.decode('hex') + assert t2.tid == fromhex('0123456789abcdf0') assert t2.user == b'author2' assert t2.description == b'zzz' assert t2.extension_bytes == b'qqq' @@ -113,11 +113,11 @@ extension "qqq" assert r.readtxn() == None - z = ''.join([_.zdump() for _ in (t1, t2)]) + z = b''.join([_.zdump() for _ in (t1, t2)]) assert z == in_ # unknown hash function - r = DumpReader(StringIO("""\ + r = DumpReader(BytesIO(b"""\ txn 0000000000000000 " " user "" description "" @@ -130,7 +130,7 @@ obj 0000000000000001 1 xyz:0123 - assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ('obj 0000000000000001 1 xyz:0123 -')""",) # data integrity error - r = DumpReader(StringIO("""\ + r = DumpReader(BytesIO(b"""\ txn 0000000000000000 " " user "" description "" @@ -141,4 +141,4 @@ hello """)) with raises(RuntimeError) as exc: r.readtxn() - assert exc.value.args == ("""+6: data corrupt: crc32 = 3610a686, expected 01234567""",) + assert exc.value.args == ("""+6: data corrupt: crc32 = 3610a686, expected 01234567""",) \ No newline at end of file diff --git a/zodbtools/test/test_zodb.py b/zodbtools/test/test_zodb.py index cda6f54..b2310c1 100644 --- a/zodbtools/test/test_zodb.py +++ b/zodbtools/test/test_zodb.py @@ -22,7 +22,9 @@ try: from unittest import mock except ImportError: # BBB python2 - import mock + import mock # type: ignore + # mypy complains: error: Name 'mock' already defined (by an import) + # https://github.com/python/mypy/issues/1153#issuecomment-253842414 import pytest diff --git a/zodbtools/util.py b/zodbtools/util.py index 172fca4..e930c35 100644 --- a/zodbtools/util.py +++ b/zodbtools/util.py @@ -18,20 +18,32 @@ # See COPYING file for full licensing terms. # See https://www.nexedi.com/licensing for rationale and options. +try: + from typing import Tuple, Optional, Union, Iterable, Any, Mapping, Callable +except ImportError: + pass + import hashlib, struct, codecs -import zodburi +import zodburi # type: ignore from six.moves.urllib_parse import urlsplit, urlunsplit from zlib import crc32, adler32 from ZODB.TimeStamp import TimeStamp import dateparser +# XXX note that for ashex and fromhex I run mypy with a typeshed patch +# https://github.com/python/typeshed/issues/300#issuecomment-459151016 + def ashex(s): - return s.encode('hex') + # type: (bytes) -> str + return codecs.encode(s, 'hex').decode() def fromhex(s): + # type: (Union[str,bytes]) -> bytes return codecs.decode(s, 'hex') + def sha1(data): + # type: (bytes) -> bytes m = hashlib.sha1() m.update(data) return m.digest() @@ -53,6 +65,8 @@ def nextitem(it): # objects of a IStorageTransactionInformation def txnobjv(txn): + # type: (Any) -> Iterable[Any] + # XXX type ? objv = [] for obj in txn: assert obj.tid == txn.tid @@ -72,6 +86,7 @@ class TidRangeInvalid(ValueError): def parse_tid(tid_string, raw_only=False): + # type: (str, bool) -> bytes """Try to parse `tid_string` as a time and returns the corresponding raw TID. If `tid_string` cannot be parsed as a time, assume it was @@ -121,19 +136,16 @@ def parse_tid(tid_string, raw_only=False): # # see `zodb help tidrange` for accepted tidrange syntax. def parse_tidrange(tidrange): + # type: (str) -> Tuple[Optional[bytes], Optional[bytes]] try: tidmin, tidmax = tidrange.split("..") except ValueError: # not exactly 2 parts in between ".." raise TidRangeInvalid(tidrange) - if tidmin: - tidmin = parse_tid(tidmin) - if tidmax: - tidmax = parse_tid(tidmax) - # empty tid means -inf / +inf respectively # ( which is None in IStorage.iterator() ) - return (tidmin or None, tidmax or None) + return (parse_tid(tidmin) if tidmin else None, + parse_tid(tidmax) if tidmax else None) # storageFromURL opens a ZODB-storage specified by url @@ -169,12 +181,15 @@ class NullHasher: digest_size = 1 def update(self, data): + # type: (bytes) -> None pass def digest(self): + # type: () -> bytes return b'\0' def hexdigest(self): + # type: () -> str return "00" # adler32 in hashlib interface @@ -183,15 +198,19 @@ class Adler32Hasher: digest_size = 4 def __init__(self): - self._h = adler32('') + # type: () -> None + self._h = adler32(b'') def update(self, data): + # type: (bytes) -> None self._h = adler32(data, self._h) def digest(self): + # type: () -> bytes return struct.pack('>I', self._h & 0xffffffff) def hexdigest(self): + # type: () -> str return '%08x' % (self._h & 0xffffffff) # crc32 in hashlib interface @@ -200,15 +219,19 @@ class CRC32Hasher: digest_size = 4 def __init__(self): - self._h = crc32('') + # type: () -> None + self._h = crc32(b'') def update(self, data): + # type: (bytes) -> None self._h = crc32(data, self._h) def digest(self): + # type: () -> bytes return struct.pack('>I', self._h & 0xffffffff) def hexdigest(self): + # type: () -> str return '%08x' % (self._h & 0xffffffff) # {} name -> hasher @@ -219,4 +242,4 @@ hashRegistry = { "sha1": hashlib.sha1, "sha256": hashlib.sha256, "sha512": hashlib.sha512, -} +} # type: Mapping[str, Callable] # XXX "Callable" is a bit too wide typing diff --git a/zodbtools/zodbanalyze.py b/zodbtools/zodbanalyze.py index 04bacb1..f4b6fef 100644 --- a/zodbtools/zodbanalyze.py +++ b/zodbtools/zodbanalyze.py @@ -9,7 +9,7 @@ from __future__ import print_function import sys import os import getopt -from six.moves import dbm_gnu as dbm +from six.moves import dbm_gnu as dbm # type: ignore import tempfile import shutil from ZODB.FileStorage import FileIterator, packed_version diff --git a/zodbtools/zodbdump.py b/zodbtools/zodbdump.py index 8a5549b..e8d77d3 100644 --- a/zodbtools/zodbdump.py +++ b/zodbtools/zodbdump.py @@ -53,6 +53,10 @@ TODO also protect txn record by hash. """ from __future__ import print_function +try: + from typing import Any, Set, Optional, BinaryIO, NoReturn, Union +except ImportError: + pass from zodbtools.util import ashex, fromhex, sha1, txnobjv, parse_tidrange, TidRangeInvalid, \ storageFromURL, hashRegistry from ZODB._compat import loads, _protocol, BytesIO @@ -61,6 +65,7 @@ from zodbpickle.slowpickle import Pickler as pyPickler from ZODB.interfaces import IStorageTransactionInformation from zope.interface import implementer +import six import sys import logging import re @@ -92,50 +97,53 @@ def txn_raw_extension(stor, txn): return serializeext(txn.extension) # set of storage names already warned for not providing IStorageTransactionInformationRaw -_already_warned_notxnraw = set() +_already_warned_notxnraw = set() # type: Set[str] # zodbdump dumps content of a ZODB storage to a file. # please see module doc-string for dump format and details -def zodbdump(stor, tidmin, tidmax, hashonly=False, out=sys.stdout): +def zodbdump(stor, tidmin, tidmax, hashonly=False, out=sys.stdout.buffer): + # type: (Any, Optional[bytes], Optional[bytes], bool, BinaryIO) -> None for txn in stor.iterator(tidmin, tidmax): # XXX .status not covered by IStorageTransactionInformation # XXX but covered by BaseStorage.TransactionRecord - out.write("txn %s %s\nuser %s\ndescription %s\nextension %s\n" % ( + out.write(("txn %s %s\nuser %s\ndescription %s\nextension %s\n" % ( ashex(txn.tid), qq(txn.status), qq(txn.user), qq(txn.description), - qq(txn_raw_extension(stor, txn)) )) + qq(txn_raw_extension(stor, txn)) )).encode()) objv = txnobjv(txn) for obj in objv: - entry = "obj %s " % ashex(obj.oid) + entry = b"obj %s " % ashex(obj.oid).encode() write_data = False if obj.data is None: - entry += "delete" + entry += b"delete" # was undo and data taken from obj.data_txn elif obj.data_txn is not None: - entry += "from %s" % ashex(obj.data_txn) + entry += b"from %s" % ashex(obj.data_txn) else: # XXX sha1 is hardcoded for now. Dump format allows other hashes. - entry += "%i sha1:%s" % (len(obj.data), ashex(sha1(obj.data))) + entry += b"%i sha1:%s" % (len(obj.data), ashex(sha1(obj.data)).encode()) write_data = True + if six.PY2: + entry = entry.encode('utf-8') out.write(entry) if write_data: if hashonly: - out.write(" -") + out.write(b" -") else: - out.write("\n") - out.write(obj.data) + out.write(b"\n") + out.write(obj.data or b"") - out.write("\n") + out.write(b"\n") - out.write("\n") + out.write(b"\n") # ---------------------------------------- # XPickler is Pickler that tries to save objects stably @@ -309,13 +317,15 @@ class DumpReader(object): # .lineno - line number position in read stream def __init__(self, r): - self._r = r + # type (BinaryIO) -> None + self._r = r # type: BinaryIO self._line = None # last read line self.lineno = 0 def _readline(self): + # type: () -> Optional[bytes] l = self._r.readline() - if l == '': + if l == b'': self._line = None return None # EOF @@ -326,11 +336,17 @@ class DumpReader(object): # report a problem found around currently-read line def _badline(self, msg): - raise RuntimeError("%s+%d: invalid line: %s (%r)" % (_ioname(self._r), self.lineno, msg, self._line)) + # type: (str) -> NoReturn + raise RuntimeError("%s+%d: invalid line: %s (%r)" % ( + _ioname(self._r), + self.lineno, msg, + # BBB produce same output in python 2 and 3 + self._line.decode() if six.PY3 else self._line.encode())) # readtxn reads one transaction record from input stream and returns # Transaction instance or None at EOF. def readtxn(self): + # type: () -> Optional[Transaction] # header l = self._readline() if l is None: @@ -356,7 +372,7 @@ class DumpReader(object): objv = [] while 1: l = self._readline() - if l == '': + if l == b'': break # empty line - end of transaction if l is None or not l.startswith(b'obj '): @@ -366,7 +382,7 @@ class DumpReader(object): if m is None: self._badline('invalid obj entry') - obj = None # will be Object* + obj = None # type: Optional[Union[ObjectDelete, ObjectCopy, ObjectData]] oid = fromhex(m.group('oid')) from_ = m.group('from') @@ -380,10 +396,10 @@ class DumpReader(object): else: size = int(m.group('size')) - hashfunc = m.group('hashfunc') + hashfunc = m.group('hashfunc').decode() hashok = fromhex(m.group('hash')) hashonly = m.group('hashonly') is not None - data = None # see vvv + data = None # type: Optional[Union[HashOnly, bytes]] # see vvv hcls = hashRegistry.get(hashfunc) if hcls is None: @@ -399,7 +415,7 @@ class DumpReader(object): chunk = self._r.read(n) data += chunk n -= len(chunk) - self.lineno += data.count('\n') + self.lineno += data.count(b'\n') self._line = None if data[-1:] != b'\n': raise RuntimeError('%s+%d: no LF after obj data' % (_ioname(self._r), self.lineno)) @@ -460,13 +476,14 @@ class Transaction(object): # zdump returns text representation of a record in zodbdump format. def zdump(self): - z = 'txn %s %s\n' % (ashex(self.tid), qq(self.status)) - z += 'user %s\n' % qq(self.user) - z += 'description %s\n' % qq(self.description) - z += 'extension %s\n' % qq(self.extension_bytes) + # type: () -> bytes + z = b'txn %s %s\n' % (ashex(self.tid).encode(), qq(self.status).encode()) + z += b'user %s\n' % qq(self.user).encode() + z += b'description %s\n' % qq(self.description).encode() + z += b'extension %s\n' % qq(self.extension_bytes).encode() for obj in self.objv: z += obj.zdump() - z += '\n' + z += b'\n' return z @@ -483,7 +500,8 @@ class ObjectDelete(Object): super(ObjectDelete, self).__init__(oid) def zdump(self): - return 'obj %s delete\n' % (ashex(self.oid)) + # type: () -> bytes + return b'obj %s delete\n' % (ashex(self.oid).encode()) # ObjectCopy represents object data copy. class ObjectCopy(Object): @@ -493,7 +511,8 @@ class ObjectCopy(Object): self.copy_from = copy_from def zdump(self): - return 'obj %s from %s\n' % (ashex(self.oid), ashex(self.copy_from)) + # type: () -> bytes + return b'obj %s from %s\n' % (ashex(self.oid).encode(), ashex(self.copy_from).encode()) # ObjectData represents record with object data. class ObjectData(Object): @@ -507,19 +526,20 @@ class ObjectData(Object): self.hash_ = hash_ def zdump(self): + # type: () -> bytes data = self.data hashonly = isinstance(data, HashOnly) if hashonly: size = data.size else: size = len(data) - z = 'obj %s %d %s:%s' % (ashex(self.oid), size, self.hashfunc, ashex(self.hash_)) + z = b'obj %s %d %s:%s' % (ashex(self.oid).encode(), size, self.hashfunc.encode(), ashex(self.hash_).encode()) if hashonly: - z += ' -' + z += b' -' else: - z += '\n' + z += b'\n' z += data - z += '\n' + z += b'\n' return z # HashOnly indicated that this ObjectData record contains only hash and does not contain object data. -- 2.30.9