zodbinfo + zodb tool + open storages by URL

Hello up there.

Recently with Ivan we needed a way to obtain last transaction ID of a ZODB storage for resiliency checking. For this `zodb info` utility to print general information about a ZODB database is introduced. Along the way, as it was already planned, all zodbtools utilities are now covered by only one `zodb` command which can invoke other subcommands and show help topics. I also used the occassion to switch how storages are specified from being  ZConfig-file-always to be specified by URL e.g.

     neo://neo1@127.0.0.1:24573
     zeo://...
     file://...

without loosing generality because zconfig:// scheme is also supported.

Please find more details about all changes in individal commit messages.

Thanks for feedback,  
Kirill

/cc @kazuhiko, @jm, @vpelletier, @jerome, @Tyagov, @klaus, @alain.takoudjou, @rafael
/reviewed-on nexedi/zodbtools!2

README.rst

@@ -8,6 +8,7 @@ scripts anymore. So we are here:
 
 __ https://github.com/zopefoundation/ZODB/pull/128#issuecomment-260970932
 
-- `zodbanalyze` - analyze FileStorage or repozo deltafs usage.
-- `zodbcmp` - compare content of two ZODB databases bit-to-bit.
-- `zodbdump` - dump content of a ZODB database.
+- `zodb analyze` - analyze FileStorage or repozo deltafs usage.
+- `zodb cmp` - compare content of two ZODB databases bit-to-bit.
+- `zodb dump` - dump content of a ZODB database.
+- `zodb info` - print general information about a ZODB database.

setup.py

@@ -19,17 +19,9 @@ setup(
     keywords    = 'zodb utility tool',
 
     packages    = find_packages(),
-    install_requires = ['ZODB'],
+    install_requires = ['ZODB', 'zodburi', 'six'],
 
-    # TODO have only one console program "zodb" and then it is
-    # zodb cmd ...
-    # zodb dump ...
-    entry_points= {'console_scripts': [
-                        'zodbanalyze = zodbtools.zodbanalyze:main',
-                        'zodbcmp     = zodbtools.zodbcmp:main',
-                        'zodbdump    = zodbtools.zodbdump:main',
-                      ]
-                  },
+    entry_points= {'console_scripts': ['zodb = zodbtools.zodb:main']},
 
     classifiers = [_.strip() for _ in """\
         Development Status :: 3 - Alpha

zodbtools/help.py

+# zodbtools - help topics
+# Copyright (C) 2017  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Open Source Initiative approved licenses and Convey
+# the resulting work. Corresponding source of such a combination shall include
+# the source code for all other software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+
+from collections import OrderedDict
+
+# topic_name -> (topic_summary, topic_help)
+topic_dict = OrderedDict()
+
+help_zurl = """\
+Almost every zodb command works with a database.
+A database can be specified by way of providing URL for its storage.
+
+The most general way to specify a storage is via preparing file with
+ZConfig-based storage definition, e.g.
+
+    %import neo.client
+    <NEOStorage>
+        master_nodes    ...
+        name            ...
+    </NEOStorage>
+
+and using path to that file with zconfig:// schema:
+
+    zconfig://<path-to-zconfig-storage-definition>
+
+There are also following simpler ways:
+
+- neo://<db>@<master>   for a NEO database
+- zeo://<host>:<port>   for a ZEO database
+- /path/to/file         for a FileStorage database
+
+Please see zodburi documentation for full details:
+
+http://docs.pylonsproject.org/projects/zodburi/
+"""
+
+topic_dict['zurl'] = "specifying database URL", help_zurl

zodbtools/util.py

@@ -17,6 +17,8 @@
 # See COPYING file for full licensing terms.
 
 import hashlib
+import zodburi
+from six.moves.urllib_parse import urlsplit, urlunsplit
 
 def ashex(s):
     return s.encode('hex')
@@ -72,3 +74,29 @@ def parse_tidrange(tidrange):
     # empty tid means -inf / +inf respectively
     # ( which is None in IStorage.iterator() )
     return (tidmin or None, tidmax or None)
+
+
+# storageFromURL opens a ZODB-storage specified by url
+# read_only specifies read or read/write mode for requested access:
+# - None: use default mode specified by url
+# - True/False: explicitly request read-only / read-write mode
+def storageFromURL(url, read_only=None):
+    # no schema -> file://
+    if "://" not in url:
+        url = "file://" + url
+
+    # read_only -> url
+    if read_only is not None:
+        scheme, netloc, path, query, fragment = urlsplit(url)
+        # XXX this won't have effect with zconfig:// but for file:// neo://
+        #     zeo:// etc ... it works
+        if scheme != "zconfig":
+            if len(query) > 0:
+                query += "&"
+            query += "read_only=%s" % read_only
+            url = urlunsplit((scheme, netloc, path, query, fragment))
+
+    stor_factory, dbkw = zodburi.resolve_uri(url)
+    stor = stor_factory()
+
+    return stor

zodbtools/zodb.py

+#!/usr/bin/env python
+# Copyright (C) 2017  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Open Source Initiative approved licenses and Convey
+# the resulting work. Corresponding source of such a combination shall include
+# the source code for all other software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+"""Zodb is a driver program for invoking zodbtools subcommands"""
+
+from __future__ import print_function
+
+from zodbtools import help as help_module
+
+import getopt
+import importlib
+import sys
+
+
+# command_name -> command_module
+command_dict = {}
+
+def register_command(cmdname):
+    command_module = importlib.import_module('zodbtools.zodb' + cmdname)
+    command_dict[cmdname] = command_module
+
+for _ in ('analyze', 'cmp', 'dump', 'info'):
+    register_command(_)
+
+
+
+def usage(out):
+    print("""\
+Zodb is a tool for managing ZODB databases.
+
+Usage:
+
+    zodb command [arguments]
+
+The commands are:
+""", file=out)
+
+    cmdv = command_dict.keys()
+    cmdv.sort()
+    for cmd in cmdv:
+        cmd_module = command_dict[cmd]
+        print("    %-11s %s" % (cmd, cmd_module.summary), file=out)
+
+    print("""\
+
+Use "zodb help [command]" for more information about a command.
+
+Additional help topics:
+""", file=out)
+
+    # NOTE no sorting here - topic_dict is pre-ordered
+    for topic, (topic_summary, _) in help_module.topic_dict.items():
+        print("    %-11s %s" % (topic, topic_summary), file=out)
+
+    print("""\
+
+Use "zodb help [topic]" for more information about that topic.
+""", file=out)
+
+
+# help shows general help or help for a command/topic
+def help(argv):
+    if len(argv) < 2:   # help topic ...
+        usage(sys.stderr)
+        sys.exit(2)
+
+    topic = argv[1]
+
+    # topic can either be a command name or a help topic
+    if topic in command_dict:
+        command = command_dict[topic]
+        command.usage(sys.stdout)
+        sys.exit(0)
+
+    if topic in help_module.topic_dict:
+        _, topic_help = help_module.topic_dict[topic]
+        print(topic_help)
+        sys.exit(0)
+
+    print("Unknown help topic `%s`.  Run 'zodb help'." % topic, file=sys.stderr)
+    sys.exit(2)
+
+
+def main():
+    try:
+        optv, argv = getopt.getopt(sys.argv[1:], "h", ["help"])
+    except getopt.GetoptError as e:
+        print(e, file=sys.stderr)
+        usage(sys.stderr)
+        sys.exit(2)
+
+    for opt, _ in optv:
+        if opt in ("-h", "--help"):
+            usage(sys.stdout)
+            sys.exit(0)
+
+    if len(argv) < 1:
+        usage(sys.stderr)
+        sys.exit(2)
+
+    command = argv[0]
+
+    # help on a topic
+    if command=="help":
+        return help(argv)
+
+    # run subcommand
+    command_module = command_dict.get(command)
+    if command_module is None:
+        print('zodb: unknown subcommand "%s"' % command, file=sys.stderr)
+        print("Run 'zodb help' for usage.", file=sys.stderr)
+        sys.exit(2)
+
+    return command_module.main(argv)
+
+
+if __name__ == '__main__':
+    main()

zodbtools/zodbanalyze.py

-#!/usr/bin/env python
 # Copyright (C) 2002-2017 Zope Foundation + Nexedi + Contributors
 # See LICENSE-ZPL.txt for full licensing terms.
 
@@ -237,17 +236,18 @@ Note:
   Input deltafs file should be uncompressed.
 """
 
+summary = "analyze FileStorage or repozo deltafs usage"
+
 def usage(stream, msg=None):
     if msg:
         print >>stream, msg
         print >>stream
-    program = os.path.basename(sys.argv[0])
-    print >>stream, __doc__ % {"program": program}
+    print >>stream, __doc__ % {"program": "zodb analyze"}
 
 
-def main():
+def main(argv):
     try:
-        opts, args = getopt.getopt(sys.argv[1:],
+        opts, args = getopt.getopt(argv[1:],
                                    'hcd', ['help', 'csv', 'dbm'])
         path = args[0]
     except (getopt.GetoptError, IndexError), msg:
@@ -275,6 +275,3 @@ def main():
             return h
         FileStorageFormatter._read_data_header = _read_data_header
     report(analyze(path, use_dbm, delta_fs), csv)
-
-if __name__ == "__main__":
-    main()

zodbtools/zodbcmp.py

-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright (C) 2016-2017  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
@@ -30,7 +29,8 @@ Exit status is 0 if inputs are the same, 1 if different, 2 if error.
 """
 
 from __future__ import print_function
-from zodbtools.util import ashex, inf, nextitem, txnobjv, parse_tidrange, TidRangeInvalid
+from zodbtools.util import ashex, inf, nextitem, txnobjv, parse_tidrange, TidRangeInvalid,  \
+        storageFromURL
 from time import time
 
 # compare two storage transactions
@@ -107,22 +107,17 @@ def storcmp(stor1, stor2, tidmin, tidmax, verbose=False):
 
 
 # ----------------------------------------
-import ZODB.config
 import sys, getopt
 import traceback
 
+summary = "compare two ZODB databases"
+
 def usage(out):
     print("""\
-Usage: zodbcmp [OPTIONS] <storage1> <storage2> [tidmin..tidmax]
+Usage: zodb cmp [OPTIONS] <storage1> <storage2> [tidmin..tidmax]
 Compare two ZODB databases.
 
-<storageX> is a file with ZConfig-based storage definition, e.g.
-
-    %import neo.client
-    <NEOStorage>
-        master_nodes    ...
-        name            ...
-    </NEOStorage>
+<storageX> is an URL (see 'zodb help zurl') of a ZODB-storage.
 
 Options:
 
@@ -130,11 +125,11 @@ Options:
     -h  --help      show this help
 """, file=out)
 
-def main2():
+def main2(argv):
     verbose = False
 
     try:
-        optv, argv = getopt.getopt(sys.argv[1:], "hv", ["help", "verbose"])
+        optv, argv = getopt.getopt(argv[1:], "hv", ["help", "verbose"])
     except getopt.GetoptError as e:
         print(e, file=sys.stderr)
         usage(sys.stderr)
@@ -148,7 +143,7 @@ def main2():
             verbose = True
 
     try:
-        storconf1, storconf2 = argv[0:2]
+        storurl1, storurl2 = argv[0:2]
     except ValueError:
         usage(sys.stderr)
         sys.exit(2)
@@ -162,20 +157,17 @@ def main2():
             print("E: invalid tidrange: %s" % e, file=sys.stderr)
             sys.exit(2)
 
-    stor1 = ZODB.config.storageFromFile(open(storconf1, 'r'))
-    stor2 = ZODB.config.storageFromFile(open(storconf2, 'r'))
+    stor1 = storageFromURL(storurl1, read_only=True)
+    stor2 = storageFromURL(storurl2, read_only=True)
 
     zcmp = storcmp(stor1, stor2, tidmin, tidmax, verbose)
     sys.exit(1 if zcmp else 0)
 
-def main():
+def main(argv):
     try:
-        main2()
+        main2(argv)
     except SystemExit:
         raise   # this was sys.exit() call, not an error
     except:
         traceback.print_exc()
         sys.exit(2)
-
-if __name__ == '__main__':
-    main()

zodbtools/zodbdump.py

-#!/usr/bin/env python
 # Copyright (C) 2016-2017  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
@@ -31,8 +30,8 @@ txn ...
 """
 
 from __future__ import print_function
-from zodbtools.util import ashex, sha1, txnobjv, parse_tidrange, TidRangeInvalid
-
+from zodbtools.util import ashex, sha1, txnobjv, parse_tidrange, TidRangeInvalid,   \
+        storageFromURL
 
 def zodbdump(stor, tidmin, tidmax, hashonly=False):
     first = True
@@ -68,21 +67,16 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False):
 
 
 # ----------------------------------------
-import ZODB.config
 import sys, getopt
 
+summary = "dump content of a ZODB database"
+
 def usage(out):
     print("""\
-Usage: zodbdump [OPTIONS] <storage> [tidmin..tidmax]
+Usage: zodb dump [OPTIONS] <storage> [tidmin..tidmax]
 Dump content of a ZODB database.
 
-<storage> is a file with ZConfig-based storage definition, e.g.
-
-    %import neo.client
-    <NEOStorage>
-        master_nodes    ...
-        name            ...
-    </NEOStorage>
+<storage> is an URL (see 'zodb help zurl') of a ZODB-storage.
 
 Options:
 
@@ -90,11 +84,11 @@ Options:
     -h  --help      show this help
 """, file=out)
 
-def main():
+def main(argv):
     hashonly = False
 
     try:
-        optv, argv = getopt.getopt(sys.argv[1:], "h", ["help", "hashonly"])
+        optv, argv = getopt.getopt(argv[1:], "h", ["help", "hashonly"])
     except getopt.GetoptError as e:
         print(e, file=sys.stderr)
         usage(sys.stderr)
@@ -108,7 +102,7 @@ def main():
             hashonly = True
 
     try:
-        storconf = argv[0]
+        storurl = argv[0]
     except IndexError:
         usage(sys.stderr)
         sys.exit(2)
@@ -122,9 +116,6 @@ def main():
             print("E: invalid tidrange: %s" % e, file=sys.stderr)
             sys.exit(2)
 
-    stor = ZODB.config.storageFromFile(open(storconf, 'r'))
+    stor = storageFromURL(storurl, read_only=True)
 
     zodbdump(stor, tidmin, tidmax, hashonly)
-
-if __name__ == '__main__':
-    main()

zodbtools/zodbinfo.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2017  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Open Source Initiative approved licenses and Convey
+# the resulting work. Corresponding source of such a combination shall include
+# the source code for all other software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+"""Zodbinfo - Print general information about a ZODB database"""
+
+from __future__ import print_function
+from zodbtools.util import ashex, storageFromURL
+from collections import OrderedDict
+import sys
+
+# {} parameter_name -> get_parameter(stor)
+infoDict = OrderedDict([
+    ("name", lambda stor: stor.getName()),
+    ("size", lambda stor: stor.getSize()),
+    ("last_tid", lambda stor: ashex(stor.lastTransaction())),
+])
+
+def zodbinfo(stor, parameterv):
+    wantnames = False
+    if not parameterv:
+        parameterv = infoDict.keys()
+        wantnames = True
+
+    for parameter in parameterv:
+        get_parameter = infoDict.get(parameter)
+        if get_parameter is None:
+            print("invalid parameter: %s" % parameter, file=sys.stderr)
+            sys.exit(1)
+
+        out = ""
+        if wantnames:
+            out += parameter + "="
+        out += "%s" % (get_parameter(stor),)
+        print(out)
+
+
+# ----------------------------------------
+import getopt
+
+summary = "print general information about a ZODB database"
+
+def usage(out):
+    print("""\
+Usage: zodb info [OPTIONS] <storage> [parameter ...]
+Print general information about a ZODB database.
+
+<storage> is an URL (see 'zodb help zurl') of a ZODB-storage.
+
+By default info prints information about all storage parameters. If one or
+more parameter names are given as arguments, info prints the value of each
+named parameter on its own line.
+
+Options:
+
+    -h  --help      show this help
+""", file=out)
+
+def main(argv):
+    try:
+        optv, argv = getopt.getopt(argv[1:], "h", ["help"])
+    except getopt.GetoptError as e:
+        print(e, file=sys.stderr)
+        usage(sys.stderr)
+        sys.exit(2)
+
+    for opt, _ in optv:
+        if opt in ("-h", "--help"):
+            usage(sys.stdout)
+            sys.exit(0)
+
+    try:
+        storurl = argv[0]
+    except IndexError:
+        usage(sys.stderr)
+        sys.exit(2)
+
+    stor = storageFromURL(storurl, read_only=True)
+
+    zodbinfo(stor, argv[1:])