Commit 3a0f7724 authored by Jérome Perrin's avatar Jérome Perrin

py3 wip

parent 98d84b91
......@@ -41,8 +41,8 @@ def test_zodbcommit():
# commit some transactions via zodbcommit and verify if storage dump gives
# what is expected.
t1 = Transaction(z64, ' ', b'user name', b'description ...', dumps({'a': 'b'}, _protocol), [
ObjectData(p64(1), b'data1', 'sha1', sha1('data1')),
ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))])
ObjectData(p64(1), b'data1', 'sha1', sha1(b'data1')),
ObjectData(p64(2), b'data2', 'sha1', sha1(b'data2'))])
t1.tid = zodbcommit(stor, head, t1)
......@@ -52,7 +52,9 @@ def test_zodbcommit():
t2.tid = zodbcommit(stor, t1.tid, t2)
buf = BytesIO()
# buf = BytesIO()
from io import StringIO
buf = StringIO()
zodbdump(stor, p64(u64(head)+1), maxtid, out=buf)
dumped = buf.getvalue()
......
......@@ -21,9 +21,11 @@ from zodbtools.zodbdump import (
zodbdump, DumpReader, Transaction, ObjectDelete, ObjectCopy,
ObjectData, HashOnly
)
from zodbtools.util import fromhex
from ZODB.FileStorage import FileStorage
from ZODB.utils import p64
from cStringIO import StringIO
from io import BytesIO
from os.path import dirname
......@@ -37,7 +39,7 @@ def test_zodbdump():
with open('%s/testdata/1.zdump.ok' % tdir) as f:
dumpok = f.read()
out = StringIO()
out = BytesIO()
zodbdump(stor, None, None, out=out)
assert out.getvalue() == dumpok
......@@ -67,10 +69,10 @@ extension "qqq"
"""
r = DumpReader(StringIO(in_))
r = DumpReader(BytesIO(in_))
t1 = r.readtxn()
assert isinstance(t1, Transaction)
assert t1.tid == '0123456789abcdef'.decode('hex')
assert t1.tid == fromhex('0123456789abcdef')
assert t1.user == b'my name'
assert t1.description == b'o la-la...'
assert t1.extension_bytes == b'zzz123 def'
......@@ -81,29 +83,29 @@ extension "qqq"
_ = t1.objv[1]
assert isinstance(_, ObjectCopy)
assert _.oid == p64(2)
assert _.copy_from == '0123456789abcdee'.decode('hex')
assert _.copy_from == fromhex('0123456789abcdee')
_ = t1.objv[2]
assert isinstance(_, ObjectData)
assert _.oid == p64(3)
assert _.data == HashOnly(54)
assert _.hashfunc == 'adler32'
assert _.hash_ == '01234567'.decode('hex')
assert _.hash_ == fromhex('01234567')
_ = t1.objv[3]
assert isinstance(_, ObjectData)
assert _.oid == p64(4)
assert _.data == b'ZZZZ'
assert _.hashfunc == 'sha1'
assert _.hash_ == '9865d483bc5a94f2e30056fc256ed3066af54d04'.decode('hex')
assert _.hash_ == fromhex('9865d483bc5a94f2e30056fc256ed3066af54d04')
_ = t1.objv[4]
assert isinstance(_, ObjectData)
assert _.oid == p64(5)
assert _.data == b'ABC\n\nDEF!'
assert _.hashfunc == 'crc32'
assert _.hash_ == '52fdeac5'.decode('hex')
assert _.hash_ == fromhex('52fdeac5')
t2 = r.readtxn()
assert isinstance(t2, Transaction)
assert t2.tid == '0123456789abcdf0'.decode('hex')
assert t2.tid == fromhex('0123456789abcdf0')
assert t2.user == b'author2'
assert t2.description == b'zzz'
assert t2.extension_bytes == b'qqq'
......@@ -112,10 +114,10 @@ extension "qqq"
assert r.readtxn() == None
z = ''.join([_.zdump() for _ in (t1, t2)])
assert z == in_
assert z.encode() == in_
# unknown hash function
r = DumpReader(StringIO("""\
r = DumpReader(BytesIO(b"""\
txn 0000000000000000 " "
user ""
description ""
......@@ -125,10 +127,11 @@ obj 0000000000000001 1 xyz:0123 -
"""))
with raises(RuntimeError) as exc:
r.readtxn()
assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ('obj 0000000000000001 1 xyz:0123 -')""",)
# XXX b ?
assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" (b'obj 0000000000000001 1 xyz:0123 -')""",)
# data integrity error
r = DumpReader(StringIO("""\
r = DumpReader(BytesIO(b"""\
txn 0000000000000000 " "
user ""
description ""
......
......@@ -18,16 +18,19 @@
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import hashlib, struct, codecs
import hashlib, struct, codecs, binascii
import zodburi
from six.moves.urllib_parse import urlsplit, urlunsplit
from zlib import crc32, adler32
def ashex(s):
return binascii.hexlify(s).decode()
return s.encode('hex')
#ashex = binascii.hexlify
def fromhex(s):
return codecs.decode(s, 'hex')
fromhex = binascii.unhexlify
Please register or sign in to reply
def sha1(data):
m = hashlib.sha1()
......@@ -74,8 +77,8 @@ def parse_tidrange(tidrange):
raise TidRangeInvalid(tidrange)
try:
tidmin = tidmin.decode("hex")
tidmax = tidmax.decode("hex")
tidmin = binascii.unhexlify(tidmin)
tidmax = binascii.unhexlify(tidmax)
except TypeError: # hex decoding error
raise TidRangeInvalid(tidrange)
......@@ -131,7 +134,7 @@ class Adler32Hasher:
digest_size = 4
def __init__(self):
self._h = adler32('')
self._h = adler32(b'')
def update(self, data):
self._h = adler32(data, self._h)
......@@ -148,7 +151,7 @@ class CRC32Hasher:
digest_size = 4
def __init__(self):
self._h = crc32('')
self._h = crc32(b'')
def update(self, data):
self._h = crc32(data, self._h)
......
......@@ -78,9 +78,9 @@ def txn_raw_extension(stor, txn):
# in a rational way
stor_name = "(%s, %s)" % (type(stor).__name__, stor.getName())
if stor_name not in _already_warned_notxnraw:
logging.warn("%s: storage does not provide IStorageTransactionInformationRaw ...", stor_name)
logging.warn("... will do best-effort to dump pickles in stable order but this cannot be done 100% correctly")
logging.warn("... please upgrade your ZODB & storage: see https://github.com/zopefoundation/ZODB/pull/183 for details.")
logging.warning("%s: storage does not provide IStorageTransactionInformationRaw ...", stor_name)
logging.warning("... will do best-effort to dump pickles in stable order but this cannot be done 100% correctly")
logging.warning("... please upgrade your ZODB & storage: see https://github.com/zopefoundation/ZODB/pull/183 for details.")
_already_warned_notxnraw.add(stor_name)
return serializeext(txn.extension)
......@@ -125,7 +125,7 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, out=sys.stdout):
out.write(" -")
else:
out.write("\n")
out.write(obj.data)
out.write(obj.data.decode())
out.write("\n")
......@@ -306,7 +306,7 @@ class DumpReader(object):
def _readline(self):
l = self._r.readline()
if l == '':
if l == b'':
self._line = None
return None # EOF
......@@ -347,7 +347,7 @@ class DumpReader(object):
objv = []
while 1:
l = self._readline()
if l == '':
if l == b'':
break # empty line - end of transaction
if l is None or not l.startswith(b'obj '):
......@@ -371,7 +371,7 @@ class DumpReader(object):
else:
size = int(m.group('size'))
hashfunc = m.group('hashfunc')
hashfunc = m.group('hashfunc').decode()
hashok = fromhex(m.group('hash'))
hashonly = m.group('hashonly') is not None
data = None # see vvv
......@@ -390,7 +390,7 @@ class DumpReader(object):
chunk = self._r.read(n)
data += chunk
n -= len(chunk)
self.lineno += data.count('\n')
self.lineno += data.count(b'\n')
self._line = None
if data[-1:] != b'\n':
raise RuntimeError('%s+%d: no LF after obj data' % (_ioname(self._r), self.lineno))
......@@ -504,7 +504,7 @@ class ObjectData(Object):
z += ' -'
else:
z += '\n'
z += data
z += data.decode()
z += '\n'
return z
......
  • mentioned in merge request nexedi/zodbtools!10 (closed)

    Toggle commit list
  • @jerome, thanks, I suggest we extract non-controversial bits from here (e.g. ashex, fromhex, BytesIO) and merge them to master without waiting for full py3 testing coverage. Would you please prepare the patches? Thanks beforehand, Kirill.

Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment