.gitignore

 /dist
 /zodbtools.egg-info
+
+/.tox/

CHANGELOG.rst

 Zodbtools change history
 ========================
 
+0.0.0.dev7 (2019-01-11)
+-----------------------
+
+- Fix zodbtools to work with all ZODB3, ZODB4 and ZODB5 (`commit 1`__, 2__,
+  3__, 4__).
+
+  __ https://lab.nexedi.com/nexedi/zodbtools/commit/425e6656
+  __ https://lab.nexedi.com/nexedi/zodbtools/commit/0e5d2f81
+  __ https://lab.nexedi.com/nexedi/zodbtools/commit/7a94e312
+  __ https://lab.nexedi.com/nexedi/zodbtools/commit/8ff7020c
+
+- Fix `zodb analyze` for the case when history range is empty (`commit 1`__,
+  2__, 3__).
+
+  __ https://lab.nexedi.com/nexedi/zodbtools/commit/b4824ad5
+  __ https://lab.nexedi.com/nexedi/zodbtools/commit/d37746c6
+  __ https://lab.nexedi.com/nexedi/zodbtools/commit/474a0559
+
+- Zodbtools is not yet Python3-ready (commit__), but we started to fix it
+  step-by-step (`commit 1`__, 2__, 3__, 4__).
+
+  __ https://lab.nexedi.com/nexedi/zodbtools/commit/7c5bb0b5
+  __ https://lab.nexedi.com/nexedi/zodbtools/commit/7d24147b
+  __ https://lab.nexedi.com/nexedi/zodbtools/commit/55853615
+  __ https://lab.nexedi.com/nexedi/zodbtools/commit/79aa0c45
+  __ https://lab.nexedi.com/nexedi/zodbtools/commit/5e2ed5e7
+
+
 0.0.0.dev6 (2018-12-30)
 -----------------------
 

MANIFEST.in

-include COPYING LICENSE-ZPL.txt README.rst CHANGELOG.rst
+include COPYING LICENSE-ZPL.txt README.rst CHANGELOG.rst tox.ini
 recursive-include zodbtools/test/testdata *.fs *.index *.ok

setup.py

@@ -8,7 +8,7 @@ def readfile(path):
 
 setup(
     name        = 'zodbtools',
-    version     = '0.0.0.dev6',
+    version     = '0.0.0.dev7',
     description = 'ZODB-related utilities',
     long_description = '%s\n----\n\n%s' % (
                             readfile('README.rst'), readfile('CHANGELOG.rst')),
@@ -20,23 +20,21 @@ setup(
     keywords    = 'zodb utility tool',
 
     packages    = find_packages(),
-    install_requires = ['ZODB', 'zodburi', 'zope.interface', 'pygolang >= 0.0.0.dev6', 'six'],
+    install_requires = ['ZODB', 'zodburi', 'zope.interface', 'pygolang >= 0.0.0.dev6', 'six', 'dateparser'],
 
     extras_require = {
-                  'test': ['pytest'],
+                  'test': ['pytest', 'freezegun', 'pytz'],
     },
 
     entry_points= {'console_scripts': ['zodb = zodbtools.zodb:main']},
 
+    # FIXME restore py3 support
     classifiers = [_.strip() for _ in """\
         Development Status :: 3 - Alpha
         Intended Audience :: Developers
         Operating System :: POSIX :: Linux
         Programming Language :: Python :: 2
         Programming Language :: Python :: 2.7
-        Programming Language :: Python :: 3
-        Programming Language :: Python :: 3.4
-        Programming Language :: Python :: 3.5
         Topic :: Database
         Topic :: Utilities
         Framework :: ZODB\

tox.ini

+# zodbtools | tox setup
+[tox]
+envlist = py27-{ZODB3,ZODB4,ZODB5}
+
+[testenv]
+deps	=
+    # XXX tox does not consult extras_require['test'] -> install pytest explicitly
+    pytest
+    freezegun
+    pytz
+
+    # latest ZODB from ZODB3 series
+    ZODB3:  ZODB3 >=3.10, <3.11dev
+    ZODB3:  transaction <2.0dev
+    # ZConfig 3.2.0 passes filename to ZEO config as unicode which eventualy breaks in FileStorage:
+    # https://github.com/zopefoundation/ZODB/blob/3.10.7-4-gb8d7a8567/src/ZODB/FileStorage/FileStorage.py#L1640
+    ZODB3:  ZConfig <3.2.0
+
+    # latest current ZODB 4
+    ZODB4:  ZODB >=4.0, <5.0dev
+    ZODB4:  ZEO  >=4.0, <5.0dev
+    # ZEO4 depends on transaction <2
+    ZODB4:  transaction <2.0dev
+
+    # latest current ZODB 5
+    ZODB5:  ZODB >=5.0,	<6.0dev
+    ZODB5:  ZEO	 >=5.0, <6.0dev
+
+
+commands= {envpython} -m pytest

zodbtools/help.py

@@ -67,11 +67,8 @@ inclusive. Both tidmin and tidmax are optional and default to
 If a tid (tidmin or tidmax) is given, it has to be specified as follows:
 
     - a 16-digit hex number specifying transaction ID, e.g. 0285cbac258bf266
-
-    TODO (recheck what git does and use dateparser):
-
-    - absolute timestamp,
-    - relative timestamp, e.g. yesterday, 1.week.ago
+    - absolute timestamp, in RFC3339 or RFC822 formats
+    - relative timestamp, e.g. yesterday, 1 week ago
 
 Example tid ranges:
 
@@ -79,6 +76,11 @@ Example tid ranges:
     000000000000aaaa..                  transactions starting from 000000000000aaaa till latest
     ..000000000000bbbb                  transactions starting from database beginning till 000000000000bbbb
     000000000000aaaa..000000000000bbbb  transactions starting from 000000000000aaaa till 000000000000bbbb
+    1985-04-12T23:20:50.52Z..2018-01-01T10:30:00Z
+                                        transactions starting from 1985-04-12 at 23 hours
+                                        20 minutes 50 seconds and 520000000 nano seconds
+                                        in UTC till 2018-01-01 at 10 hours 30 minutes in UTC
+    1_week_ago..yesterday               transactions from one week ago until yesterday.
 
 In commands <tidrange> is optional - if it is not given at all, it defaults to
 0..+∞, i.e. to whole database history.

zodbtools/test/conftest.py

+# Copyright (C) 2019  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+import pytest
+from zodbtools.test.testutil import zext_supported
+
+# zext is a test fixture function object that allows to exercise 2 cases:
+#
+# - when ZODB does not have txn.extension_bytes support
+# - when ZODB might have    txn.extension_bytes support
+#
+# in a test, zext should be used as as follows:
+#
+#   def test_something(zext):
+#       # bytes for an extension dict
+#       raw_ext = dumps({...})
+#
+#       # will be either same as raw_ext, or b'' if ZODB lacks txn.extension_bytes support
+#       raw_ext = zext(raw_ext)
+#
+#       # zext.disabled indicates whether testing for non-empty extension was disabled.
+#       if zext.disabled:
+#           ...
+@pytest.fixture(params=['!zext', 'zext'])
+def zext(request):
+    if request.param == '!zext':
+        # txn.extension_bytes is not working - always test with empty extension
+        def _(ext):
+            return b''
+        _.disabled = True
+        return _
+    else:
+        # txn.extension_bytes might be working - test with given extension and
+        # xfail if ZODB does not have necessary support.
+        def _(ext):
+            return ext
+        _.disabled = False
+        if not zext_supported():
+            request.applymarker(pytest.mark.xfail(reason='ZODB does not have txn.extension_bytes support'))
+        return _

zodbtools/test/gen_testdata.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2017  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+# Copyright (C) 2017-2019  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -60,7 +60,12 @@ def hex64(packed):
     return '0x%016x' % unpack64(packed)
 
 # make time.time() predictable
-_xtime = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
+_xtime0 = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
+def xtime_reset():
+    global _xtime
+    _xtime = _xtime0
+xtime_reset()
+
 def xtime():
     global _xtime
     _xtime += 1.1
@@ -94,7 +99,7 @@ class Object(Persistent):
 
 # prepare extension dictionary for subject
 alnum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-def ext(subj):
+def ext4subj(subj):
     d = {"x-generator": "zodb/py%s (%s)" % (sys.version_info.major, subj)}
 
     # also add some random 'x-cookie'
@@ -115,8 +120,16 @@ def ext(subj):
 
     return ext
 
-# gen_testdb generates test FileStorage database @ outfs_path
-def gen_testdb(outfs_path):
+# gen_testdb generates test FileStorage database @ outfs_path.
+#
+# zext indicates whether or not to include non-empty extension into transactions.
+def gen_testdb(outfs_path, zext=True):
+    xtime_reset()
+
+    ext = ext4subj
+    if not zext:
+        def ext(subj): return {}
+
     logging.basicConfig()
 
     # generate random changes to objects hooked to top-level root by a/b/c/... key
@@ -196,13 +209,22 @@ def gen_testdb(outfs_path):
 # ----------------------------------------
 
 from zodbtools.zodbdump import zodbdump
+from zodbtools.test.testutil import zext_supported
 
 def main():
+    # check that ZODB supports txn.extension_bytes; refuse to work if not.
+    if not zext_supported():
+        raise RuntimeError("gen_testdata must be used with ZODB that supports txn.extension_bytes")
+
     out = "testdata/1"
-    gen_testdb("%s.fs" % out)
-    stor = FileStorage("%s.fs" % out, read_only=True)
-    with open("%s.zdump.ok" % out, "w") as f:
-        zodbdump(stor, None, None, out=f)
+    for zext in [True, False]:
+        dbname = out
+        if not zext:
+            dbname += "_!zext"
+        gen_testdb("%s.fs" % dbname, zext=zext)
+        stor = FileStorage("%s.fs" % dbname, read_only=True)
+        with open("%s.zdump.ok" % dbname, "w") as f:
+            zodbdump(stor, None, None, out=f)
 
 if __name__ == '__main__':
     main()

zodbtools/test/test_analyze.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2019 Nexedi SA and Contributors.
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+from zodbtools.zodbanalyze import analyze, report
+import os.path
+
+
+def test_zodbanalyze(capsys):
+    for use_dbm in (False, True):
+        report(
+            analyze(
+                os.path.join(os.path.dirname(__file__), "testdata", "1.fs"),
+                use_dbm=use_dbm,
+                delta_fs=False,
+                tidmin=None,
+                tidmax=None,
+            ),
+            csv=False,
+        )
+        captured = capsys.readouterr()
+        assert "Processed 68 records in 59 transactions" in captured.out
+        assert captured.err == ""
+
+    # csv output
+    report(
+        analyze(
+            os.path.join(os.path.dirname(__file__), "testdata", "1.fs"),
+            use_dbm=False,
+            delta_fs=False,
+            tidmin=None,
+            tidmax=None,
+        ),
+        csv=True,
+    )
+    captured = capsys.readouterr()
+    assert (
+        """Class Name,T.Count,T.Bytes,Pct,AvgSize,C.Count,C.Bytes,O.Count,O.Bytes
+persistent.mapping.PersistentMapping,10,1578,45.633314%,157.800000,1,213,9,1365
+__main__.Object,56,1880,54.366686%,33.571429,9,303,47,1577
+"""
+        == captured.out
+    )
+    assert captured.err == ""
+
+    # empty range
+    report(
+        analyze(
+            os.path.join(os.path.dirname(__file__), "testdata", "1.fs"),
+            use_dbm=False,
+            delta_fs=False,
+            tidmin="ffffffffffffffff",
+            tidmax=None,
+        ),
+        csv=False,
+    )
+    captured = capsys.readouterr()
+    assert "# ø\nNo transactions processed\n" == captured.out.encode('utf-8')
+    assert captured.err == ""

zodbtools/test/test_commit.py

@@ -20,7 +20,7 @@
 from zodbtools.zodbcommit import zodbcommit
 from zodbtools.zodbdump import zodbdump, Transaction, ObjectData, ObjectDelete, ObjectCopy
 from zodbtools.util import storageFromURL, sha1
-from ZODB.utils import p64, u64, z64, maxtid
+from ZODB.utils import p64, u64, z64
 from ZODB._compat import BytesIO, dumps, _protocol   # XXX can't yet commit with arbitrary ext.bytes
 
 from tempfile import mkdtemp
@@ -29,7 +29,7 @@ from golang import func, defer
 
 # verify zodbcommit.
 @func
-def test_zodbcommit():
+def test_zodbcommit(zext):
     tmpd = mkdtemp('', 'zodbcommit.')
     defer(lambda: rmtree(tmpd))
 
@@ -40,7 +40,7 @@ def test_zodbcommit():
 
     # commit some transactions via zodbcommit and verify if storage dump gives
     # what is expected.
-    t1 = Transaction(z64, ' ', b'user name', b'description ...', dumps({'a': 'b'}, _protocol), [
+    t1 = Transaction(z64, ' ', b'user name', b'description ...', zext(dumps({'a': 'b'}, _protocol)), [
         ObjectData(p64(1), b'data1', 'sha1', sha1('data1')),
         ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))])
 
@@ -53,7 +53,7 @@ def test_zodbcommit():
 
 
     buf = BytesIO()
-    zodbdump(stor, p64(u64(head)+1), maxtid, out=buf)
+    zodbdump(stor, p64(u64(head)+1), None, out=buf)
     dumped = buf.getvalue()
 
     assert dumped == ''.join([_.zdump() for _ in (t1, t2)])

zodbtools/test/test_dump.py

@@ -27,14 +27,16 @@ from cStringIO import StringIO
 
 from os.path import dirname
 
-from pytest import raises
+from zodbtools.test.testutil import zext_supported
+from pytest import raises, xfail
 
 # verify zodbdump output against golden
-def test_zodbdump():
-    tdir = dirname(__file__)
-    stor = FileStorage('%s/testdata/1.fs' % tdir, read_only=True)
+def test_zodbdump(zext):
+    tdir  = dirname(__file__)
+    zkind = '_!zext' if zext.disabled else ''
+    stor  = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True)
 
-    with open('%s/testdata/1.zdump.ok' % tdir) as f:
+    with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f:
         dumpok = f.read()
 
     out = StringIO()

zodbtools/test/test_tidrange.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2019 Nexedi SA and Contributors.
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+import datetime
+import os
+import time
+
+import pytest
+import pytz
+from freezegun import freeze_time
+import tzlocal
+
+from zodbtools.util import TidRangeInvalid, TidInvalid, ashex, parse_tid, parse_tidrange
+
+
+@pytest.fixture
+def fake_time():
+    """Pytest's fixture to run this test as if now() was 2009-08-30T19:20:00Z
+    and if the machine timezone was Europe/Paris
+    """
+    initial_tz = os.environ.get("TZ")
+    os.environ["TZ"] = "Europe/Paris"
+    time.tzset()
+    tzlocal.reload_localzone()
+
+    reference_time = datetime.datetime(2009, 8, 30, 19, 20, 0, 0,
+                                       pytz.utc).astimezone(
+                                           pytz.timezone("Europe/Paris"))
+    with freeze_time(reference_time):
+        yield
+    del os.environ["TZ"]
+    if initial_tz:
+        os.environ["TZ"] = initial_tz
+    time.tzset()
+
+
+def test_tidrange_tid():
+    assert (
+        b"\x00\x00\x00\x00\x00\x00\xaa\xaa",
+        b"\x00\x00\x00\x00\x00\x00\xbb\xbb",
+    ) == parse_tidrange("000000000000aaaa..000000000000bbbb")
+
+    assert (b"\x00\x00\x00\x00\x00\x00\xaa\xaa",
+            None) == parse_tidrange("000000000000aaaa..")
+
+    assert (None, b"\x00\x00\x00\x00\x00\x00\xbb\xbb"
+            ) == parse_tidrange("..000000000000bbbb")
+
+    assert (None, None) == parse_tidrange("..")
+
+    with pytest.raises(TidRangeInvalid) as exc:
+        parse_tidrange("inv.alid")
+    assert exc.value.args == ("inv.alid", )
+
+    # range is correct, but a TID is invalid
+    with pytest.raises(TidInvalid) as exc:
+        parse_tidrange("invalid..")
+    assert exc.value.args == ("invalid", )
+
+
+def test_tidrange_date():
+    assert (
+        b"\x03\xc4\x85v\x00\x00\x00\x00",
+        b"\x03\xc4\x88\xa0\x00\x00\x00\x00",
+    ) == parse_tidrange(
+        "2018-01-01T10:30:00Z..2018-01-02T00:00:00.000000+00:00")
+
+
+def test_parse_tid():
+    assert b"\x00\x00\x00\x00\x00\x00\xbb\xbb" == parse_tid("000000000000bbbb")
+
+    with pytest.raises(TidInvalid) as exc:
+        parse_tid("invalid")
+    assert exc.value.args == ("invalid", )
+
+    with pytest.raises(TidInvalid) as exc:
+        parse_tid('')
+    assert exc.value.args == ('', )
+
+
+test_parameters = []
+with open(
+        os.path.join(
+            os.path.dirname(__file__), "testdata",
+            "tid-time-format.txt")) as f:
+    for line in f:
+        line = line.strip()
+        if line and not line.startswith("#"):
+            test_parameters.append(line.split(" ", 2))
+
+
+@pytest.mark.parametrize("reference_time,reference_tid,input_time",
+                         test_parameters)
+def test_parse_tid_time_format(fake_time, reference_time, reference_tid,
+                               input_time):
+    assert reference_tid == ashex(parse_tid(input_time))
+    # check that the reference_tid matches the reference time, mainly
+    # to check that input is defined correctly.
+    assert reference_tid == ashex(parse_tid(reference_time))

zodbtools/test/testdata/tid-time-format.txt

+# This is the supported time formats for zodbutils <tidrange>
+# Format of this file is:
+#   <reference time> <tid in hex format> <time input format>
+#
+# These must be run with current time: 2009-08-30T19:20:00Z
+# in Europe/Paris timezone.
+# ( as a timestamp: 1251660000 )
+
+
+# some absolute date formats:
+# RFC3339
+2018-01-01T10:30:00Z 03c4857600000000 2018-01-01T10:30:00Z
+1985-04-12T23:20:50.520000Z 02b914f8d78d4fdf 1985-04-12T23:20:50.52Z
+1996-12-20T00:39:57Z 03189927f3333333 1996-12-19T16:39:57-08:00
+2018-01-01T05:30:00Z 03c4844a00000000 2018-01-01T10:30:00+05:00
+
+# RFC822
+1976-08-26T14:29:00Z 02728aa500000000 26 Aug 76 14:29 GMT
+1976-08-26T12:29:00Z 02728a2d00000000 26 Aug 76 14:29 +02:00
+
+
+# RFC850 -> not supported (by go implementation)
+#2006-01-02T22:04:05Z 036277cc15555555 Monday, 02-Jan-06 15:04:05 MST
+
+# RFC1123 -> not supported (by go implementation)
+#2006-01-02T22:04:05Z 036277cc15555555 Mon, 02 Jan 2006 15:04:05 MST
+#2006-01-02T22:04:05Z 036277cc15555555 Mon, 02 Jan 2006 23:04:05 GMT+1
+
+# explicit UTC timezone
+2018-01-01T10:30:00Z 03c4857600000000 2018-01-01 10:30:00 UTC
+2018-01-02T00:00:00Z 03c488a000000000 2018-01-02 UTC
+
+
+# Relative formats, based on git's test for approxidate
+# (adapted for timezone Europe/Paris and extended a bit)
+2009-08-30T19:20:00Z 03805ec800000000 now
+2009-08-30T19:19:55Z 03805ec7eaaaaaaa 5 seconds ago
+2009-08-30T19:19:55Z 03805ec7eaaaaaaa 5.seconds.ago
+2009-08-30T19:10:00Z 03805ebe00000000 10.minutes.ago
+2009-08-29T19:20:00Z 0380592800000000 yesterday
+2009-08-27T19:20:00Z 03804de800000000 3.days.ago
+2009-08-09T19:20:00Z 037fe8a800000000 3.weeks.ago
+2009-05-30T19:20:00Z 037e53a800000000 3.months.ago
+2009-08-30T19:19:00Z 03805ec700000000 1 minute ago
+2009-08-29T19:20:00Z 0380592800000000 1 day ago
+2009-07-30T19:20:00Z 037fb06800000000 1 month ago
+
+# go's when does not support "chaining" like this
+#2007-05-30T19:20:00Z 036dfaa800000000 2.years.3.months.ago
+
+2009-08-29T04:00:00Z 0380559000000000 6am yesterday
+2009-08-29T16:00:00Z 0380586000000000 6pm yesterday
+
+2009-08-30T01:00:00Z 03805a7c00000000 3:00
+2009-08-30T13:00:00Z 03805d4c00000000 15:00
+2009-08-30T10:00:00Z 03805c9800000000 noon today
+2009-08-29T10:00:00Z 038056f800000000 noon yesterday
+
+# this input is a bit weird also, what does "noon pm" mean?
+# it seems to trigger a bug in python's parser
+#  TypeError: can't compare offset-naive and offset-aware datetimes
+#2009-01-05T12:00:00Z 037b0bd000000000 January 5th noon pm
+
+# this input is "ambiguous"
+#2009-08-29T12:00:00Z 0380577000000000 10am noon
+
+# not supported by date parser
+#2009-08-25T19:20:00Z 038042a800000000 last tuesday
+# non consistent behavior ( go keep current hour:minutes - python use midnight )
+#  this also TypeError on python
+#2009-07-05T00:00:00Z 037f1f4000000000 July 5th
+
+# parsed as month/day (at least for me ... it might depend on some locale settings other than $TZ ?)
+#2009-05-06T00:00:00Z 037dc82000000000 06.05.2009
+
+# go parser is wrong on this one
+#2009-06-06T05:00:00Z 037e77ac00000000 Jun 6, 5AM
+# go parser is wrong on this one
+#2009-06-06T05:00:00Z 037e77ac00000000 5AM Jun 6
+
+2009-06-07T04:00:00Z 037e7d1000000000 6AM, June 7, 2009
+
+# python and go disagree on these two, go see them as 00:00 UTC
+#2008-11-30T23:00:00Z 037a3e4400000000 2008-12-01
+#2009-11-30T23:00:00Z 03826ac400000000 2009-12-01
+#2009-06-04T22:00:00Z 037e706800000000 06/05/2009
+# ( end of tests from git )
+
+# more tests
+
+### works with python implementation, but not supported:
+#2018-01-01T09:30:00Z 03c4853a00000000 le 1er janvier 2018 à 10h30
+#2018-01-01T23:00:00Z 03c4886400000000 2018年1月2日
+
+### some invalid formats that "looks OK"
+# wrong format on timezone (should be 2009-06-01T22:00:00+09:00)
+#2009-06-01T01:00:00Z 037e5a9c00000000 2009-06-01T10:00:00:+09:00
+# day is 34
+# ERROR XXX 2009-06-34T22:00:00Z
+# one digits hour minutes
+# ERROR XXX 2009-06-01T1:2:3
+# month use a captital o instead of O
+# ERROR XXX 2009-O6-01T22:00:00Z

zodbtools/test/testutil.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2019  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""utilities for testing"""
+
+from ZODB.FileStorage import FileStorage
+from ZODB import DB
+import transaction
+
+from tempfile import mkdtemp
+from shutil import rmtree
+from golang import func, defer
+
+# zext_supported checks whether ZODB supports txn.extension_bytes .
+_zext_supported_memo = None
+def zext_supported():
+    global _zext_supported_memo
+    if _zext_supported_memo is not None:
+        return _zext_supported_memo
+
+    _ = _zext_supported_memo = _zext_supported()
+    return _
+
+@func
+def _zext_supported():
+    tmpd = mkdtemp('', 'zext_check.')
+    defer(lambda: rmtree(tmpd))
+    dbfs = tmpd + '/1.fs'
+
+    stor = FileStorage(dbfs, create=True)
+    db   = DB(stor)
+    conn = db.open()
+    root = conn.root()
+    root._p_changed = True
+
+    txn = transaction.get()
+    txn.setExtendedInfo('a', 'b')
+    txn.commit()
+
+    for last_txn in stor.iterator(start=stor.lastTransaction()):
+        break
+    else:
+        assert False, "cannot see details of last transaction"
+
+    assert last_txn.extension == {'a': 'b'}
+    return hasattr(last_txn, 'extension_bytes')

zodbtools/util.py

@@ -22,6 +22,8 @@ import hashlib, struct, codecs
 import zodburi
 from six.moves.urllib_parse import urlsplit, urlunsplit
 from zlib import crc32, adler32
+from ZODB.TimeStamp import TimeStamp
+import dateparser
 
 def ashex(s):
     return s.encode('hex')
@@ -61,9 +63,60 @@ def txnobjv(txn):
     return objv
 
 # "tidmin..tidmax" -> (tidmin, tidmax)
-class TidRangeInvalid(Exception):
+class TidInvalid(ValueError):
     pass
 
+
+class TidRangeInvalid(ValueError):
+    pass
+
+
+def parse_tid(tid_string, raw_only=False):
+    """Try to parse `tid_string` as a time and returns the
+    corresponding raw TID.
+    If `tid_string` cannot be parsed as a time, assume it was
+    already a TID.
+    This function also raise TidRangeInvalid when `tid_string`
+    is invalid.
+    """
+    assert isinstance(tid_string, (str, bytes))
+
+    # If it "looks like a TID", don't try to parse it as time,
+    # because parsing is slow.
+    if len(tid_string) == 16:
+        try:
+            return fromhex(tid_string)
+        except ValueError:
+            pass
+
+    if raw_only:
+        # either it was not 16-char string or hex decoding failed
+        raise TidInvalid(tid_string)
+
+    # preprocess to support `1.day.ago` style formats like git log does.
+    if "ago" in tid_string:
+        tid_string = tid_string.replace(".", " ").replace("_", " ")
+    parsed_time = dateparser.parse(
+        tid_string,
+        settings={
+            'TO_TIMEZONE': 'UTC',
+            'RETURN_AS_TIMEZONE_AWARE': True
+        })
+
+    if not parsed_time:
+        # parsing as date failed
+        raise TidInvalid(tid_string)
+
+    # build a ZODB.TimeStamp to convert as a TID
+    return TimeStamp(
+            parsed_time.year,
+            parsed_time.month,
+            parsed_time.day,
+            parsed_time.hour,
+            parsed_time.minute,
+            parsed_time.second + parsed_time.microsecond / 1000000.).raw()
+
+
 # parse_tidrange parses a string into (tidmin, tidmax).
 #
 # see `zodb help tidrange` for accepted tidrange syntax.
@@ -73,11 +126,10 @@ def parse_tidrange(tidrange):
     except ValueError:  # not exactly 2 parts in between ".."
         raise TidRangeInvalid(tidrange)
 
-    try:
-        tidmin = tidmin.decode("hex")
-        tidmax = tidmax.decode("hex")
-    except TypeError:   # hex decoding error
-        raise TidRangeInvalid(tidrange)
+    if tidmin:
+        tidmin = parse_tid(tidmin)
+    if tidmax:
+        tidmax = parse_tid(tidmax)
 
     # empty tid means -inf / +inf respectively
     # ( which is None in IStorage.iterator() )

zodbtools/zodbanalyze.py

@@ -4,10 +4,12 @@
 
 # Based on a transaction analyzer by Matt Kromer.
 
+from __future__ import print_function
+
 import sys
 import os
 import getopt
-import anydbm as dbm
+from six.moves import dbm_gnu as dbm
 import tempfile
 import shutil
 from ZODB.FileStorage import FileIterator, packed_version
@@ -29,7 +31,7 @@ class DeltaFileStorage(
         pass
 
 class DeltaFileIterator(FileIterator):
-    def __init__(self, filename, start=None, stop=None, pos=0L):
+    def __init__(self, filename, start=None, stop=None, pos=0):
         assert isinstance(filename, str)
         file = open(filename, 'rb')
         self._file = file
@@ -95,15 +97,17 @@ def report(rep, csv=False):
     delta_fs = rep.delta_fs
     if not csv:
         if rep.TIDS == 0:
-            print "# ø"
-        else:
-            print "# %s..%s" % (ashex(rep.tidmin), ashex(rep.tidmax))
-        print "Processed %d records in %d transactions" % (rep.OIDS, rep.TIDS)
-        print "Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS)
+            print ("# ø")
+            print ("No transactions processed")
+            return
+
+        print ("# %s..%s" % (ashex(rep.tidmin), ashex(rep.tidmax)))
+        print ("Processed %d records in %d transactions" % (rep.OIDS, rep.TIDS))
+        print ("Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS))
         print ("Average transaction size is %7.2f bytes" %
                (rep.DBYTES * 1.0 / rep.TIDS))
 
-        print "Types used:"
+        print ("Types used:")
     if delta_fs:
         if csv:
             fmt = "%s,%s,%s,%s,%s"
@@ -111,9 +115,9 @@ def report(rep, csv=False):
         else:
             fmt = "%-46s %7s %9s %6s %7s"
             fmtp = "%-46s %7d %9d %5.1f%% %7.2f" # per-class format
-        print fmt % ("Class Name", "T.Count", "T.Bytes", "Pct", "AvgSize")
+        print (fmt % ("Class Name", "T.Count", "T.Bytes", "Pct", "AvgSize"))
         if not csv:
-            print fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7)
+            print (fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7))
     else:
         if csv:
             fmt = "%s,%s,%s,%s,%s,%s,%s,%s,%s"
@@ -121,15 +125,13 @@ def report(rep, csv=False):
         else:
             fmt = "%-46s %7s %9s %6s %7s %7s %9s %7s %9s"
             fmtp = "%-46s %7d %9d %5.1f%% %7.2f %7d %9d %7d %9d" # per-class format
-        print fmt % ("Class Name", "T.Count", "T.Bytes", "Pct", "AvgSize",
-                     "C.Count", "C.Bytes", "O.Count", "O.Bytes")
+        print (fmt % ("Class Name", "T.Count", "T.Bytes", "Pct", "AvgSize",
+                      "C.Count", "C.Bytes", "O.Count", "O.Bytes"))
         if not csv:
-            print fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7, '-'*7, '-'*9, '-'*7, '-'*9)
+            print (fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7, '-'*7, '-'*9, '-'*7, '-'*9))
     fmts = "%46s %7d %8dk %5.1f%% %7.2f" # summary format
-    typemap = rep.TYPEMAP.keys()
-    typemap.sort(key=lambda a:rep.TYPESIZE[a])
     cumpct = 0.0
-    for t in typemap:
+    for t in sorted(rep.TYPEMAP.keys(), key=lambda a:rep.TYPESIZE[a]):
         pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES
         cumpct += pct
         if csv:
@@ -137,37 +139,37 @@ def report(rep, csv=False):
         else:
             t_display = shorten(t, 46)
         if delta_fs:
-            print fmtp % (t_display, rep.TYPEMAP[t], rep.TYPESIZE[t],
-                          pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t])
+            print (fmtp % (t_display, rep.TYPEMAP[t], rep.TYPESIZE[t],
+                           pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t]))
         else:
-            print fmtp % (t_display, rep.TYPEMAP[t], rep.TYPESIZE[t],
-                          pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t],
-                          rep.COIDSMAP[t], rep.CBYTESMAP[t],
-                          rep.FOIDSMAP.get(t, 0), rep.FBYTESMAP.get(t, 0))
+            print (fmtp % (t_display, rep.TYPEMAP[t], rep.TYPESIZE[t],
+                           pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t],
+                           rep.COIDSMAP[t], rep.CBYTESMAP[t],
+                           rep.FOIDSMAP.get(t, 0), rep.FBYTESMAP.get(t, 0)))
 
     if csv:
         return
 
     if delta_fs:
-        print fmt % ('='*46, '='*7, '='*9, '='*5, '='*7)
-        print "%46s %7d %9s %6s %6.2f" % ('Total Transactions', rep.TIDS, ' ',
-                                          ' ', rep.DBYTES * 1.0 / rep.TIDS)
-        print fmts % ('Total Records', rep.OIDS, rep.DBYTES, cumpct,
-                      rep.DBYTES * 1.0 / rep.OIDS)
+        print (fmt % ('='*46, '='*7, '='*9, '='*5, '='*7))
+        print ("%46s %7d %9s %6s %6.2f" % ('Total Transactions', rep.TIDS, ' ',
+                                          ' ', rep.DBYTES * 1.0 / rep.TIDS))
+        print (fmts % ('Total Records', rep.OIDS, rep.DBYTES, cumpct,
+                       rep.DBYTES * 1.0 / rep.OIDS))
     else:
-        print fmt % ('='*46, '='*7, '='*9, '='*5, '='*7, '='*7, '='*9, '='*7, '='*9)
-        print "%46s %7d %9s %6s %6.2fk" % ('Total Transactions', rep.TIDS, ' ',
-            ' ', rep.DBYTES * 1.0 / rep.TIDS / 1024.0)
-        print fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct,
-                      rep.DBYTES * 1.0 / rep.OIDS)
+        print (fmt % ('='*46, '='*7, '='*9, '='*5, '='*7, '='*7, '='*9, '='*7, '='*9))
+        print ("%46s %7d %9s %6s %6.2fk" % ('Total Transactions', rep.TIDS, ' ',
+            ' ', rep.DBYTES * 1.0 / rep.TIDS / 1024.0))
+        print (fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct,
+                       rep.DBYTES * 1.0 / rep.OIDS))
 
-        print fmts % ('Current Objects', rep.COIDS, rep.CBYTES / 1024.0,
-                      rep.CBYTES * 100.0 / rep.DBYTES,
-                      rep.CBYTES * 1.0 / rep.COIDS)
+        print (fmts % ('Current Objects', rep.COIDS, rep.CBYTES / 1024.0,
+                       rep.CBYTES * 100.0 / rep.DBYTES,
+                       rep.CBYTES * 1.0 / rep.COIDS))
         if rep.FOIDS:
-            print fmts % ('Old Objects', rep.FOIDS, rep.FBYTES / 1024.0,
-                          rep.FBYTES * 100.0 / rep.DBYTES,
-                          rep.FBYTES * 1.0 / rep.FOIDS)
+            print (fmts % ('Old Objects', rep.FOIDS, rep.FBYTES / 1024.0,
+                           rep.FBYTES * 100.0 / rep.DBYTES,
+                           rep.FBYTES * 1.0 / rep.FOIDS))
 
 @func
 def analyze(path, use_dbm, delta_fs, tidmin, tidmax):
@@ -238,8 +240,8 @@ def analyze_rec(report, record):
                 report.CBYTESMAP[type] = report.CBYTESMAP.get(type, 0) + size - fsize
             report.TYPEMAP[type] = report.TYPEMAP.get(type, 0) + 1
             report.TYPESIZE[type] = report.TYPESIZE.get(type, 0) + size
-    except Exception, err:
-        print err
+    except Exception as err:
+        print (err, file=sys.stderr)
 
 __doc__ = """%(program)s: Analyzer for ZODB data or repozo deltafs
 
@@ -261,9 +263,9 @@ summary = "analyze ZODB database or repozo deltafs usage"
 
 def usage(stream, msg=None):
     if msg:
-        print >>stream, msg
-        print >>stream
-    print >>stream, __doc__ % {"program": "zodb analyze"}
+        print (msg, file=stream)
+        print (file=stream)
+    print (__doc__ % {"program": "zodb analyze"}, file=stream)
 
 
 def main(argv):
@@ -271,7 +273,7 @@ def main(argv):
         opts, args = getopt.getopt(argv[1:],
                                    'hcd', ['help', 'csv', 'dbm'])
         path = args[0]
-    except (getopt.GetoptError, IndexError), msg:
+    except (getopt.GetoptError, IndexError) as msg:
         usage(sys.stderr, msg)
         sys.exit(2)
 

zodbtools/zodbdump.py

@@ -58,7 +58,7 @@ from zodbtools.util import ashex, fromhex, sha1, txnobjv, parse_tidrange, TidRan
 from ZODB._compat import loads, _protocol, BytesIO
 from zodbpickle.slowpickle import Pickler as pyPickler
 #import pickletools
-from ZODB.interfaces import IStorageTransactionMetaData
+from ZODB.interfaces import IStorageTransactionInformation
 from zope.interface import implementer
 
 import sys
@@ -413,7 +413,7 @@ class DumpReader(object):
 
 
 # Transaction represents one transaction record in zodbdump stream.
-@implementer(IStorageTransactionMetaData)
+@implementer(IStorageTransactionInformation)    # TODO -> IStorageTransactionMetaData after switch to ZODB >= 5
 class Transaction(object):
     # .tid              p64         transaction ID
     # .status           char        status of the transaction
@@ -444,6 +444,11 @@ class Transaction(object):
             return {}
         return loads(self.extension_bytes)
 
+    # ZODB < 5 wants ._extension
+    @property
+    def _extension(self):
+        return self.extension
+
     # zdump returns text representation of a record in zodbdump format.
     def zdump(self):
         z  = 'txn %s %s\n' % (ashex(self.tid), qq(self.status))