...
 
Commits (17)
/dist
/zodbtools.egg-info
/.tox/
Zodbtools change history
========================
0.0.0.dev7 (2019-01-11)
-----------------------
- Fix zodbtools to work with all ZODB3, ZODB4 and ZODB5 (`commit 1`__, 2__,
3__, 4__).
__ https://lab.nexedi.com/nexedi/zodbtools/commit/425e6656
__ https://lab.nexedi.com/nexedi/zodbtools/commit/0e5d2f81
__ https://lab.nexedi.com/nexedi/zodbtools/commit/7a94e312
__ https://lab.nexedi.com/nexedi/zodbtools/commit/8ff7020c
- Fix `zodb analyze` for the case when history range is empty (`commit 1`__,
2__, 3__).
__ https://lab.nexedi.com/nexedi/zodbtools/commit/b4824ad5
__ https://lab.nexedi.com/nexedi/zodbtools/commit/d37746c6
__ https://lab.nexedi.com/nexedi/zodbtools/commit/474a0559
- Zodbtools is not yet Python3-ready (commit__), but we started to fix it
step-by-step (`commit 1`__, 2__, 3__, 4__).
__ https://lab.nexedi.com/nexedi/zodbtools/commit/7c5bb0b5
__ https://lab.nexedi.com/nexedi/zodbtools/commit/7d24147b
__ https://lab.nexedi.com/nexedi/zodbtools/commit/55853615
__ https://lab.nexedi.com/nexedi/zodbtools/commit/79aa0c45
__ https://lab.nexedi.com/nexedi/zodbtools/commit/5e2ed5e7
0.0.0.dev6 (2018-12-30)
-----------------------
......
include COPYING LICENSE-ZPL.txt README.rst CHANGELOG.rst
include COPYING LICENSE-ZPL.txt README.rst CHANGELOG.rst tox.ini
recursive-include zodbtools/test/testdata *.fs *.index *.ok
......@@ -8,7 +8,7 @@ def readfile(path):
setup(
name = 'zodbtools',
version = '0.0.0.dev6',
version = '0.0.0.dev7',
description = 'ZODB-related utilities',
long_description = '%s\n----\n\n%s' % (
readfile('README.rst'), readfile('CHANGELOG.rst')),
......@@ -20,23 +20,21 @@ setup(
keywords = 'zodb utility tool',
packages = find_packages(),
install_requires = ['ZODB', 'zodburi', 'zope.interface', 'pygolang >= 0.0.0.dev6', 'six'],
install_requires = ['ZODB', 'zodburi', 'zope.interface', 'pygolang >= 0.0.0.dev6', 'six', 'dateparser'],
extras_require = {
'test': ['pytest'],
'test': ['pytest', 'freezegun', 'pytz'],
},
entry_points= {'console_scripts': ['zodb = zodbtools.zodb:main']},
# FIXME restore py3 support
classifiers = [_.strip() for _ in """\
Development Status :: 3 - Alpha
Intended Audience :: Developers
Operating System :: POSIX :: Linux
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.4
Programming Language :: Python :: 3.5
Topic :: Database
Topic :: Utilities
Framework :: ZODB\
......
# zodbtools | tox setup
[tox]
envlist = py27-{ZODB3,ZODB4,ZODB5}
[testenv]
deps =
# XXX tox does not consult extras_require['test'] -> install pytest explicitly
pytest
freezegun
pytz
# latest ZODB from ZODB3 series
ZODB3: ZODB3 >=3.10, <3.11dev
ZODB3: transaction <2.0dev
# ZConfig 3.2.0 passes filename to ZEO config as unicode which eventualy breaks in FileStorage:
# https://github.com/zopefoundation/ZODB/blob/3.10.7-4-gb8d7a8567/src/ZODB/FileStorage/FileStorage.py#L1640
ZODB3: ZConfig <3.2.0
# latest current ZODB 4
ZODB4: ZODB >=4.0, <5.0dev
ZODB4: ZEO >=4.0, <5.0dev
# ZEO4 depends on transaction <2
ZODB4: transaction <2.0dev
# latest current ZODB 5
ZODB5: ZODB >=5.0, <6.0dev
ZODB5: ZEO >=5.0, <6.0dev
commands= {envpython} -m pytest
......@@ -67,11 +67,8 @@ inclusive. Both tidmin and tidmax are optional and default to
If a tid (tidmin or tidmax) is given, it has to be specified as follows:
- a 16-digit hex number specifying transaction ID, e.g. 0285cbac258bf266
TODO (recheck what git does and use dateparser):
- absolute timestamp,
- relative timestamp, e.g. yesterday, 1.week.ago
- absolute timestamp, in RFC3339 or RFC822 formats
- relative timestamp, e.g. yesterday, 1 week ago
Example tid ranges:
......@@ -79,6 +76,11 @@ Example tid ranges:
000000000000aaaa.. transactions starting from 000000000000aaaa till latest
..000000000000bbbb transactions starting from database beginning till 000000000000bbbb
000000000000aaaa..000000000000bbbb transactions starting from 000000000000aaaa till 000000000000bbbb
1985-04-12T23:20:50.52Z..2018-01-01T10:30:00Z
transactions starting from 1985-04-12 at 23 hours
20 minutes 50 seconds and 520000000 nano seconds
in UTC till 2018-01-01 at 10 hours 30 minutes in UTC
1_week_ago..yesterday transactions from one week ago until yesterday.
In commands <tidrange> is optional - if it is not given at all, it defaults to
0..+∞, i.e. to whole database history.
......
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import pytest
from zodbtools.test.testutil import zext_supported
# zext is a test fixture function object that allows to exercise 2 cases:
#
# - when ZODB does not have txn.extension_bytes support
# - when ZODB might have txn.extension_bytes support
#
# in a test, zext should be used as as follows:
#
# def test_something(zext):
# # bytes for an extension dict
# raw_ext = dumps({...})
#
# # will be either same as raw_ext, or b'' if ZODB lacks txn.extension_bytes support
# raw_ext = zext(raw_ext)
#
# # zext.disabled indicates whether testing for non-empty extension was disabled.
# if zext.disabled:
# ...
@pytest.fixture(params=['!zext', 'zext'])
def zext(request):
if request.param == '!zext':
# txn.extension_bytes is not working - always test with empty extension
def _(ext):
return b''
_.disabled = True
return _
else:
# txn.extension_bytes might be working - test with given extension and
# xfail if ZODB does not have necessary support.
def _(ext):
return ext
_.disabled = False
if not zext_supported():
request.applymarker(pytest.mark.xfail(reason='ZODB does not have txn.extension_bytes support'))
return _
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2017 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2017-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -60,7 +60,12 @@ def hex64(packed):
return '0x%016x' % unpack64(packed)
# make time.time() predictable
_xtime = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
_xtime0 = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
def xtime_reset():
global _xtime
_xtime = _xtime0
xtime_reset()
def xtime():
global _xtime
_xtime += 1.1
......@@ -94,7 +99,7 @@ class Object(Persistent):
# prepare extension dictionary for subject
alnum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
def ext(subj):
def ext4subj(subj):
d = {"x-generator": "zodb/py%s (%s)" % (sys.version_info.major, subj)}
# also add some random 'x-cookie'
......@@ -115,8 +120,16 @@ def ext(subj):
return ext
# gen_testdb generates test FileStorage database @ outfs_path
def gen_testdb(outfs_path):
# gen_testdb generates test FileStorage database @ outfs_path.
#
# zext indicates whether or not to include non-empty extension into transactions.
def gen_testdb(outfs_path, zext=True):
xtime_reset()
ext = ext4subj
if not zext:
def ext(subj): return {}
logging.basicConfig()
# generate random changes to objects hooked to top-level root by a/b/c/... key
......@@ -196,13 +209,22 @@ def gen_testdb(outfs_path):
# ----------------------------------------
from zodbtools.zodbdump import zodbdump
from zodbtools.test.testutil import zext_supported
def main():
# check that ZODB supports txn.extension_bytes; refuse to work if not.
if not zext_supported():
raise RuntimeError("gen_testdata must be used with ZODB that supports txn.extension_bytes")
out = "testdata/1"
gen_testdb("%s.fs" % out)
stor = FileStorage("%s.fs" % out, read_only=True)
with open("%s.zdump.ok" % out, "w") as f:
zodbdump(stor, None, None, out=f)
for zext in [True, False]:
dbname = out
if not zext:
dbname += "_!zext"
gen_testdb("%s.fs" % dbname, zext=zext)
stor = FileStorage("%s.fs" % dbname, read_only=True)
with open("%s.zdump.ok" % dbname, "w") as f:
zodbdump(stor, None, None, out=f)
if __name__ == '__main__':
main()
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from zodbtools.zodbanalyze import analyze, report
import os.path
def test_zodbanalyze(capsys):
for use_dbm in (False, True):
report(
analyze(
os.path.join(os.path.dirname(__file__), "testdata", "1.fs"),
use_dbm=use_dbm,
delta_fs=False,
tidmin=None,
tidmax=None,
),
csv=False,
)
captured = capsys.readouterr()
assert "Processed 68 records in 59 transactions" in captured.out
assert captured.err == ""
# csv output
report(
analyze(
os.path.join(os.path.dirname(__file__), "testdata", "1.fs"),
use_dbm=False,
delta_fs=False,
tidmin=None,
tidmax=None,
),
csv=True,
)
captured = capsys.readouterr()
assert (
"""Class Name,T.Count,T.Bytes,Pct,AvgSize,C.Count,C.Bytes,O.Count,O.Bytes
persistent.mapping.PersistentMapping,10,1578,45.633314%,157.800000,1,213,9,1365
__main__.Object,56,1880,54.366686%,33.571429,9,303,47,1577
"""
== captured.out
)
assert captured.err == ""
# empty range
report(
analyze(
os.path.join(os.path.dirname(__file__), "testdata", "1.fs"),
use_dbm=False,
delta_fs=False,
tidmin="ffffffffffffffff",
tidmax=None,
),
csv=False,
)
captured = capsys.readouterr()
assert "# ø\nNo transactions processed\n" == captured.out.encode('utf-8')
assert captured.err == ""
......@@ -20,7 +20,7 @@
from zodbtools.zodbcommit import zodbcommit
from zodbtools.zodbdump import zodbdump, Transaction, ObjectData, ObjectDelete, ObjectCopy
from zodbtools.util import storageFromURL, sha1
from ZODB.utils import p64, u64, z64, maxtid
from ZODB.utils import p64, u64, z64
from ZODB._compat import BytesIO, dumps, _protocol # XXX can't yet commit with arbitrary ext.bytes
from tempfile import mkdtemp
......@@ -29,7 +29,7 @@ from golang import func, defer
# verify zodbcommit.
@func
def test_zodbcommit():
def test_zodbcommit(zext):
tmpd = mkdtemp('', 'zodbcommit.')
defer(lambda: rmtree(tmpd))
......@@ -40,7 +40,7 @@ def test_zodbcommit():
# commit some transactions via zodbcommit and verify if storage dump gives
# what is expected.
t1 = Transaction(z64, ' ', b'user name', b'description ...', dumps({'a': 'b'}, _protocol), [
t1 = Transaction(z64, ' ', b'user name', b'description ...', zext(dumps({'a': 'b'}, _protocol)), [
ObjectData(p64(1), b'data1', 'sha1', sha1('data1')),
ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))])
......@@ -53,7 +53,7 @@ def test_zodbcommit():
buf = BytesIO()
zodbdump(stor, p64(u64(head)+1), maxtid, out=buf)
zodbdump(stor, p64(u64(head)+1), None, out=buf)
dumped = buf.getvalue()
assert dumped == ''.join([_.zdump() for _ in (t1, t2)])
......
......@@ -27,14 +27,16 @@ from cStringIO import StringIO
from os.path import dirname
from pytest import raises
from zodbtools.test.testutil import zext_supported
from pytest import raises, xfail
# verify zodbdump output against golden
def test_zodbdump():
tdir = dirname(__file__)
stor = FileStorage('%s/testdata/1.fs' % tdir, read_only=True)
def test_zodbdump(zext):
tdir = dirname(__file__)
zkind = '_!zext' if zext.disabled else ''
stor = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True)
with open('%s/testdata/1.zdump.ok' % tdir) as f:
with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f:
dumpok = f.read()
out = StringIO()
......
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import datetime
import os
import time
import pytest
import pytz
from freezegun import freeze_time
import tzlocal
from zodbtools.util import TidRangeInvalid, TidInvalid, ashex, parse_tid, parse_tidrange
@pytest.fixture
def fake_time():
"""Pytest's fixture to run this test as if now() was 2009-08-30T19:20:00Z
and if the machine timezone was Europe/Paris
"""
initial_tz = os.environ.get("TZ")
os.environ["TZ"] = "Europe/Paris"
time.tzset()
tzlocal.reload_localzone()
reference_time = datetime.datetime(2009, 8, 30, 19, 20, 0, 0,
pytz.utc).astimezone(
pytz.timezone("Europe/Paris"))
with freeze_time(reference_time):
yield
del os.environ["TZ"]
if initial_tz:
os.environ["TZ"] = initial_tz
time.tzset()
def test_tidrange_tid():
assert (
b"\x00\x00\x00\x00\x00\x00\xaa\xaa",
b"\x00\x00\x00\x00\x00\x00\xbb\xbb",
) == parse_tidrange("000000000000aaaa..000000000000bbbb")
assert (b"\x00\x00\x00\x00\x00\x00\xaa\xaa",
None) == parse_tidrange("000000000000aaaa..")
assert (None, b"\x00\x00\x00\x00\x00\x00\xbb\xbb"
) == parse_tidrange("..000000000000bbbb")
assert (None, None) == parse_tidrange("..")
with pytest.raises(TidRangeInvalid) as exc:
parse_tidrange("inv.alid")
assert exc.value.args == ("inv.alid", )
# range is correct, but a TID is invalid
with pytest.raises(TidInvalid) as exc:
parse_tidrange("invalid..")
assert exc.value.args == ("invalid", )
def test_tidrange_date():
assert (
b"\x03\xc4\x85v\x00\x00\x00\x00",
b"\x03\xc4\x88\xa0\x00\x00\x00\x00",
) == parse_tidrange(
"2018-01-01T10:30:00Z..2018-01-02T00:00:00.000000+00:00")
def test_parse_tid():
assert b"\x00\x00\x00\x00\x00\x00\xbb\xbb" == parse_tid("000000000000bbbb")
with pytest.raises(TidInvalid) as exc:
parse_tid("invalid")
assert exc.value.args == ("invalid", )
with pytest.raises(TidInvalid) as exc:
parse_tid('')
assert exc.value.args == ('', )
test_parameters = []
with open(
os.path.join(
os.path.dirname(__file__), "testdata",
"tid-time-format.txt")) as f:
for line in f:
line = line.strip()
if line and not line.startswith("#"):
test_parameters.append(line.split(" ", 2))
@pytest.mark.parametrize("reference_time,reference_tid,input_time",
test_parameters)
def test_parse_tid_time_format(fake_time, reference_time, reference_tid,
input_time):
assert reference_tid == ashex(parse_tid(input_time))
# check that the reference_tid matches the reference time, mainly
# to check that input is defined correctly.
assert reference_tid == ashex(parse_tid(reference_time))
# This is the supported time formats for zodbutils <tidrange>
# Format of this file is:
# <reference time> <tid in hex format> <time input format>
#
# These must be run with current time: 2009-08-30T19:20:00Z
# in Europe/Paris timezone.
# ( as a timestamp: 1251660000 )
# some absolute date formats:
# RFC3339
2018-01-01T10:30:00Z 03c4857600000000 2018-01-01T10:30:00Z
1985-04-12T23:20:50.520000Z 02b914f8d78d4fdf 1985-04-12T23:20:50.52Z
1996-12-20T00:39:57Z 03189927f3333333 1996-12-19T16:39:57-08:00
2018-01-01T05:30:00Z 03c4844a00000000 2018-01-01T10:30:00+05:00
# RFC822
1976-08-26T14:29:00Z 02728aa500000000 26 Aug 76 14:29 GMT
1976-08-26T12:29:00Z 02728a2d00000000 26 Aug 76 14:29 +02:00
# RFC850 -> not supported (by go implementation)
#2006-01-02T22:04:05Z 036277cc15555555 Monday, 02-Jan-06 15:04:05 MST
# RFC1123 -> not supported (by go implementation)
#2006-01-02T22:04:05Z 036277cc15555555 Mon, 02 Jan 2006 15:04:05 MST
#2006-01-02T22:04:05Z 036277cc15555555 Mon, 02 Jan 2006 23:04:05 GMT+1
# explicit UTC timezone
2018-01-01T10:30:00Z 03c4857600000000 2018-01-01 10:30:00 UTC
2018-01-02T00:00:00Z 03c488a000000000 2018-01-02 UTC
# Relative formats, based on git's test for approxidate
# (adapted for timezone Europe/Paris and extended a bit)
2009-08-30T19:20:00Z 03805ec800000000 now
2009-08-30T19:19:55Z 03805ec7eaaaaaaa 5 seconds ago
2009-08-30T19:19:55Z 03805ec7eaaaaaaa 5.seconds.ago
2009-08-30T19:10:00Z 03805ebe00000000 10.minutes.ago
2009-08-29T19:20:00Z 0380592800000000 yesterday
2009-08-27T19:20:00Z 03804de800000000 3.days.ago
2009-08-09T19:20:00Z 037fe8a800000000 3.weeks.ago
2009-05-30T19:20:00Z 037e53a800000000 3.months.ago
2009-08-30T19:19:00Z 03805ec700000000 1 minute ago
2009-08-29T19:20:00Z 0380592800000000 1 day ago
2009-07-30T19:20:00Z 037fb06800000000 1 month ago
# go's when does not support "chaining" like this
#2007-05-30T19:20:00Z 036dfaa800000000 2.years.3.months.ago
2009-08-29T04:00:00Z 0380559000000000 6am yesterday
2009-08-29T16:00:00Z 0380586000000000 6pm yesterday
2009-08-30T01:00:00Z 03805a7c00000000 3:00
2009-08-30T13:00:00Z 03805d4c00000000 15:00
2009-08-30T10:00:00Z 03805c9800000000 noon today
2009-08-29T10:00:00Z 038056f800000000 noon yesterday
# this input is a bit weird also, what does "noon pm" mean?
# it seems to trigger a bug in python's parser
# TypeError: can't compare offset-naive and offset-aware datetimes
#2009-01-05T12:00:00Z 037b0bd000000000 January 5th noon pm
# this input is "ambiguous"
#2009-08-29T12:00:00Z 0380577000000000 10am noon
# not supported by date parser
#2009-08-25T19:20:00Z 038042a800000000 last tuesday
# non consistent behavior ( go keep current hour:minutes - python use midnight )
# this also TypeError on python
#2009-07-05T00:00:00Z 037f1f4000000000 July 5th
# parsed as month/day (at least for me ... it might depend on some locale settings other than $TZ ?)
#2009-05-06T00:00:00Z 037dc82000000000 06.05.2009
# go parser is wrong on this one
#2009-06-06T05:00:00Z 037e77ac00000000 Jun 6, 5AM
# go parser is wrong on this one
#2009-06-06T05:00:00Z 037e77ac00000000 5AM Jun 6
2009-06-07T04:00:00Z 037e7d1000000000 6AM, June 7, 2009
# python and go disagree on these two, go see them as 00:00 UTC
#2008-11-30T23:00:00Z 037a3e4400000000 2008-12-01
#2009-11-30T23:00:00Z 03826ac400000000 2009-12-01
#2009-06-04T22:00:00Z 037e706800000000 06/05/2009
# ( end of tests from git )
# more tests
### works with python implementation, but not supported:
#2018-01-01T09:30:00Z 03c4853a00000000 le 1er janvier 2018 à 10h30
#2018-01-01T23:00:00Z 03c4886400000000 2018年1月2日
### some invalid formats that "looks OK"
# wrong format on timezone (should be 2009-06-01T22:00:00+09:00)
#2009-06-01T01:00:00Z 037e5a9c00000000 2009-06-01T10:00:00:+09:00
# day is 34
# ERROR XXX 2009-06-34T22:00:00Z
# one digits hour minutes
# ERROR XXX 2009-06-01T1:2:3
# month use a captital o instead of O
# ERROR XXX 2009-O6-01T22:00:00Z
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""utilities for testing"""
from ZODB.FileStorage import FileStorage
from ZODB import DB
import transaction
from tempfile import mkdtemp
from shutil import rmtree
from golang import func, defer
# zext_supported checks whether ZODB supports txn.extension_bytes .
_zext_supported_memo = None
def zext_supported():
global _zext_supported_memo
if _zext_supported_memo is not None:
return _zext_supported_memo
_ = _zext_supported_memo = _zext_supported()
return _
@func
def _zext_supported():
tmpd = mkdtemp('', 'zext_check.')
defer(lambda: rmtree(tmpd))
dbfs = tmpd + '/1.fs'
stor = FileStorage(dbfs, create=True)
db = DB(stor)
conn = db.open()
root = conn.root()
root._p_changed = True
txn = transaction.get()
txn.setExtendedInfo('a', 'b')
txn.commit()
for last_txn in stor.iterator(start=stor.lastTransaction()):
break
else:
assert False, "cannot see details of last transaction"
assert last_txn.extension == {'a': 'b'}
return hasattr(last_txn, 'extension_bytes')
......@@ -22,6 +22,8 @@ import hashlib, struct, codecs
import zodburi
from six.moves.urllib_parse import urlsplit, urlunsplit
from zlib import crc32, adler32
from ZODB.TimeStamp import TimeStamp
import dateparser
def ashex(s):
return s.encode('hex')
......@@ -61,9 +63,60 @@ def txnobjv(txn):
return objv
# "tidmin..tidmax" -> (tidmin, tidmax)
class TidRangeInvalid(Exception):
class TidInvalid(ValueError):
pass
class TidRangeInvalid(ValueError):
pass
def parse_tid(tid_string, raw_only=False):
"""Try to parse `tid_string` as a time and returns the
corresponding raw TID.
If `tid_string` cannot be parsed as a time, assume it was
already a TID.
This function also raise TidRangeInvalid when `tid_string`
is invalid.
"""
assert isinstance(tid_string, (str, bytes))
# If it "looks like a TID", don't try to parse it as time,
# because parsing is slow.
if len(tid_string) == 16:
try:
return fromhex(tid_string)
except ValueError:
pass
if raw_only:
# either it was not 16-char string or hex decoding failed
raise TidInvalid(tid_string)
# preprocess to support `1.day.ago` style formats like git log does.
if "ago" in tid_string:
tid_string = tid_string.replace(".", " ").replace("_", " ")
parsed_time = dateparser.parse(
tid_string,
settings={
'TO_TIMEZONE': 'UTC',
'RETURN_AS_TIMEZONE_AWARE': True
})
if not parsed_time:
# parsing as date failed
raise TidInvalid(tid_string)
# build a ZODB.TimeStamp to convert as a TID
return TimeStamp(
parsed_time.year,
parsed_time.month,
parsed_time.day,
parsed_time.hour,
parsed_time.minute,
parsed_time.second + parsed_time.microsecond / 1000000.).raw()
# parse_tidrange parses a string into (tidmin, tidmax).
#
# see `zodb help tidrange` for accepted tidrange syntax.
......@@ -73,11 +126,10 @@ def parse_tidrange(tidrange):
except ValueError: # not exactly 2 parts in between ".."
raise TidRangeInvalid(tidrange)
try:
tidmin = tidmin.decode("hex")
tidmax = tidmax.decode("hex")
except TypeError: # hex decoding error
raise TidRangeInvalid(tidrange)
if tidmin:
tidmin = parse_tid(tidmin)
if tidmax:
tidmax = parse_tid(tidmax)
# empty tid means -inf / +inf respectively
# ( which is None in IStorage.iterator() )
......
......@@ -4,10 +4,12 @@
# Based on a transaction analyzer by Matt Kromer.
from __future__ import print_function
import sys
import os
import getopt
import anydbm as dbm
from six.moves import dbm_gnu as dbm
import tempfile
import shutil
from ZODB.FileStorage import FileIterator, packed_version
......@@ -29,7 +31,7 @@ class DeltaFileStorage(
pass
class DeltaFileIterator(FileIterator):
def __init__(self, filename, start=None, stop=None, pos=0L):
def __init__(self, filename, start=None, stop=None, pos=0):
assert isinstance(filename, str)
file = open(filename, 'rb')
self._file = file
......@@ -95,15 +97,17 @@ def report(rep, csv=False):
delta_fs = rep.delta_fs
if not csv:
if rep.TIDS == 0:
print "# ø"
else:
print "# %s..%s" % (ashex(rep.tidmin), ashex(rep.tidmax))
print "Processed %d records in %d transactions" % (rep.OIDS, rep.TIDS)
print "Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS)
print ("# ø")
print ("No transactions processed")
return
print ("# %s..%s" % (ashex(rep.tidmin), ashex(rep.tidmax)))
print ("Processed %d records in %d transactions" % (rep.OIDS, rep.TIDS))
print ("Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS))
print ("Average transaction size is %7.2f bytes" %
(rep.DBYTES * 1.0 / rep.TIDS))
print "Types used:"
print ("Types used:")
if delta_fs:
if csv:
fmt = "%s,%s,%s,%s,%s"
......@@ -111,9 +115,9 @@ def report(rep, csv=False):
else:
fmt = "%-46s %7s %9s %6s %7s"
fmtp = "%-46s %7d %9d %5.1f%% %7.2f" # per-class format
print fmt % ("Class Name", "T.Count", "T.Bytes", "Pct", "AvgSize")
print (fmt % ("Class Name", "T.Count", "T.Bytes", "Pct", "AvgSize"))
if not csv:
print fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7)
print (fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7))
else:
if csv:
fmt = "%s,%s,%s,%s,%s,%s,%s,%s,%s"
......@@ -121,15 +125,13 @@ def report(rep, csv=False):
else:
fmt = "%-46s %7s %9s %6s %7s %7s %9s %7s %9s"
fmtp = "%-46s %7d %9d %5.1f%% %7.2f %7d %9d %7d %9d" # per-class format
print fmt % ("Class Name", "T.Count", "T.Bytes", "Pct", "AvgSize",
"C.Count", "C.Bytes", "O.Count", "O.Bytes")
print (fmt % ("Class Name", "T.Count", "T.Bytes", "Pct", "AvgSize",
"C.Count", "C.Bytes", "O.Count", "O.Bytes"))
if not csv:
print fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7, '-'*7, '-'*9, '-'*7, '-'*9)
print (fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7, '-'*7, '-'*9, '-'*7, '-'*9))
fmts = "%46s %7d %8dk %5.1f%% %7.2f" # summary format
typemap = rep.TYPEMAP.keys()
typemap.sort(key=lambda a:rep.TYPESIZE[a])
cumpct = 0.0
for t in typemap:
for t in sorted(rep.TYPEMAP.keys(), key=lambda a:rep.TYPESIZE[a]):
pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES
cumpct += pct
if csv:
......@@ -137,37 +139,37 @@ def report(rep, csv=False):
else:
t_display = shorten(t, 46)
if delta_fs:
print fmtp % (t_display, rep.TYPEMAP[t], rep.TYPESIZE[t],
pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t])
print (fmtp % (t_display, rep.TYPEMAP[t], rep.TYPESIZE[t],
pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t]))
else:
print fmtp % (t_display, rep.TYPEMAP[t], rep.TYPESIZE[t],
pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t],
rep.COIDSMAP[t], rep.CBYTESMAP[t],
rep.FOIDSMAP.get(t, 0), rep.FBYTESMAP.get(t, 0))
print (fmtp % (t_display, rep.TYPEMAP[t], rep.TYPESIZE[t],
pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t],
rep.COIDSMAP[t], rep.CBYTESMAP[t],
rep.FOIDSMAP.get(t, 0), rep.FBYTESMAP.get(t, 0)))
if csv:
return
if delta_fs:
print fmt % ('='*46, '='*7, '='*9, '='*5, '='*7)
print "%46s %7d %9s %6s %6.2f" % ('Total Transactions', rep.TIDS, ' ',
' ', rep.DBYTES * 1.0 / rep.TIDS)
print fmts % ('Total Records', rep.OIDS, rep.DBYTES, cumpct,
rep.DBYTES * 1.0 / rep.OIDS)
print (fmt % ('='*46, '='*7, '='*9, '='*5, '='*7))
print ("%46s %7d %9s %6s %6.2f" % ('Total Transactions', rep.TIDS, ' ',
' ', rep.DBYTES * 1.0 / rep.TIDS))
print (fmts % ('Total Records', rep.OIDS, rep.DBYTES, cumpct,
rep.DBYTES * 1.0 / rep.OIDS))
else:
print fmt % ('='*46, '='*7, '='*9, '='*5, '='*7, '='*7, '='*9, '='*7, '='*9)
print "%46s %7d %9s %6s %6.2fk" % ('Total Transactions', rep.TIDS, ' ',
' ', rep.DBYTES * 1.0 / rep.TIDS / 1024.0)
print fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct,
rep.DBYTES * 1.0 / rep.OIDS)
print (fmt % ('='*46, '='*7, '='*9, '='*5, '='*7, '='*7, '='*9, '='*7, '='*9))
print ("%46s %7d %9s %6s %6.2fk" % ('Total Transactions', rep.TIDS, ' ',
' ', rep.DBYTES * 1.0 / rep.TIDS / 1024.0))
print (fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct,
rep.DBYTES * 1.0 / rep.OIDS))
print fmts % ('Current Objects', rep.COIDS, rep.CBYTES / 1024.0,
rep.CBYTES * 100.0 / rep.DBYTES,
rep.CBYTES * 1.0 / rep.COIDS)
print (fmts % ('Current Objects', rep.COIDS, rep.CBYTES / 1024.0,
rep.CBYTES * 100.0 / rep.DBYTES,
rep.CBYTES * 1.0 / rep.COIDS))
if rep.FOIDS:
print fmts % ('Old Objects', rep.FOIDS, rep.FBYTES / 1024.0,
rep.FBYTES * 100.0 / rep.DBYTES,
rep.FBYTES * 1.0 / rep.FOIDS)
print (fmts % ('Old Objects', rep.FOIDS, rep.FBYTES / 1024.0,
rep.FBYTES * 100.0 / rep.DBYTES,
rep.FBYTES * 1.0 / rep.FOIDS))
@func
def analyze(path, use_dbm, delta_fs, tidmin, tidmax):
......@@ -238,8 +240,8 @@ def analyze_rec(report, record):
report.CBYTESMAP[type] = report.CBYTESMAP.get(type, 0) + size - fsize
report.TYPEMAP[type] = report.TYPEMAP.get(type, 0) + 1
report.TYPESIZE[type] = report.TYPESIZE.get(type, 0) + size
except Exception, err:
print err
except Exception as err:
print (err, file=sys.stderr)
__doc__ = """%(program)s: Analyzer for ZODB data or repozo deltafs
......@@ -261,9 +263,9 @@ summary = "analyze ZODB database or repozo deltafs usage"
def usage(stream, msg=None):
if msg:
print >>stream, msg
print >>stream
print >>stream, __doc__ % {"program": "zodb analyze"}
print (msg, file=stream)
print (file=stream)
print (__doc__ % {"program": "zodb analyze"}, file=stream)
def main(argv):
......@@ -271,7 +273,7 @@ def main(argv):
opts, args = getopt.getopt(argv[1:],
'hcd', ['help', 'csv', 'dbm'])
path = args[0]
except (getopt.GetoptError, IndexError), msg:
except (getopt.GetoptError, IndexError) as msg:
usage(sys.stderr, msg)
sys.exit(2)
......
......@@ -58,7 +58,7 @@ from zodbtools.util import ashex, fromhex, sha1, txnobjv, parse_tidrange, TidRan
from ZODB._compat import loads, _protocol, BytesIO
from zodbpickle.slowpickle import Pickler as pyPickler
#import pickletools
from ZODB.interfaces import IStorageTransactionMetaData
from ZODB.interfaces import IStorageTransactionInformation
from zope.interface import implementer
import sys
......@@ -413,7 +413,7 @@ class DumpReader(object):
# Transaction represents one transaction record in zodbdump stream.
@implementer(IStorageTransactionMetaData)
@implementer(IStorageTransactionInformation) # TODO -> IStorageTransactionMetaData after switch to ZODB >= 5
class Transaction(object):
# .tid p64 transaction ID
# .status char status of the transaction
......@@ -444,6 +444,11 @@ class Transaction(object):
return {}
return loads(self.extension_bytes)
# ZODB < 5 wants ._extension
@property
def _extension(self):
return self.extension
# zdump returns text representation of a record in zodbdump format.
def zdump(self):
z = 'txn %s %s\n' % (ashex(self.tid), qq(self.status))
......