Commit db3b84ec authored by Kirill Smelkov's avatar Kirill Smelkov

zodbinfo + zodb tool + open storages by URL

Hello up there.

Recently with Ivan we needed a way to obtain last transaction ID of a ZODB storage for resiliency checking. For this `zodb info` utility to print general information about a ZODB database is introduced. Along the way, as it was already planned, all zodbtools utilities are now covered by only one `zodb` command which can invoke other subcommands and show help topics. I also used the occassion to switch how storages are specified from being  ZConfig-file-always to be specified by URL e.g.

     neo://neo1@127.0.0.1:24573
     zeo://...
     file://...

without loosing generality because zconfig:// scheme is also supported.

Please find more details about all changes in individal commit messages.

Thanks for feedback,  
Kirill

/cc @kazuhiko, @jm, @vpelletier, @jerome, @Tyagov, @klaus, @alain.takoudjou, @rafael
/reviewed-on !2
parents 9e4305b8 37b9fbde
......@@ -8,6 +8,7 @@ scripts anymore. So we are here:
__ https://github.com/zopefoundation/ZODB/pull/128#issuecomment-260970932
- `zodbanalyze` - analyze FileStorage or repozo deltafs usage.
- `zodbcmp` - compare content of two ZODB databases bit-to-bit.
- `zodbdump` - dump content of a ZODB database.
- `zodb analyze` - analyze FileStorage or repozo deltafs usage.
- `zodb cmp` - compare content of two ZODB databases bit-to-bit.
- `zodb dump` - dump content of a ZODB database.
- `zodb info` - print general information about a ZODB database.
......@@ -19,17 +19,9 @@ setup(
keywords = 'zodb utility tool',
packages = find_packages(),
install_requires = ['ZODB'],
install_requires = ['ZODB', 'zodburi', 'six'],
# TODO have only one console program "zodb" and then it is
# zodb cmd ...
# zodb dump ...
entry_points= {'console_scripts': [
'zodbanalyze = zodbtools.zodbanalyze:main',
'zodbcmp = zodbtools.zodbcmp:main',
'zodbdump = zodbtools.zodbdump:main',
]
},
entry_points= {'console_scripts': ['zodb = zodbtools.zodb:main']},
classifiers = [_.strip() for _ in """\
Development Status :: 3 - Alpha
......
# zodbtools - help topics
# Copyright (C) 2017 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Open Source Initiative approved licenses and Convey
# the resulting work. Corresponding source of such a combination shall include
# the source code for all other software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
from collections import OrderedDict
# topic_name -> (topic_summary, topic_help)
topic_dict = OrderedDict()
help_zurl = """\
Almost every zodb command works with a database.
A database can be specified by way of providing URL for its storage.
The most general way to specify a storage is via preparing file with
ZConfig-based storage definition, e.g.
%import neo.client
<NEOStorage>
master_nodes ...
name ...
</NEOStorage>
and using path to that file with zconfig:// schema:
zconfig://<path-to-zconfig-storage-definition>
There are also following simpler ways:
- neo://<db>@<master> for a NEO database
- zeo://<host>:<port> for a ZEO database
- /path/to/file for a FileStorage database
Please see zodburi documentation for full details:
http://docs.pylonsproject.org/projects/zodburi/
"""
topic_dict['zurl'] = "specifying database URL", help_zurl
......@@ -17,6 +17,8 @@
# See COPYING file for full licensing terms.
import hashlib
import zodburi
from six.moves.urllib_parse import urlsplit, urlunsplit
def ashex(s):
return s.encode('hex')
......@@ -72,3 +74,29 @@ def parse_tidrange(tidrange):
# empty tid means -inf / +inf respectively
# ( which is None in IStorage.iterator() )
return (tidmin or None, tidmax or None)
# storageFromURL opens a ZODB-storage specified by url
# read_only specifies read or read/write mode for requested access:
# - None: use default mode specified by url
# - True/False: explicitly request read-only / read-write mode
def storageFromURL(url, read_only=None):
# no schema -> file://
if "://" not in url:
url = "file://" + url
# read_only -> url
if read_only is not None:
scheme, netloc, path, query, fragment = urlsplit(url)
# XXX this won't have effect with zconfig:// but for file:// neo://
# zeo:// etc ... it works
if scheme != "zconfig":
if len(query) > 0:
query += "&"
query += "read_only=%s" % read_only
url = urlunsplit((scheme, netloc, path, query, fragment))
stor_factory, dbkw = zodburi.resolve_uri(url)
stor = stor_factory()
return stor
#!/usr/bin/env python
# Copyright (C) 2017 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Open Source Initiative approved licenses and Convey
# the resulting work. Corresponding source of such a combination shall include
# the source code for all other software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
"""Zodb is a driver program for invoking zodbtools subcommands"""
from __future__ import print_function
from zodbtools import help as help_module
import getopt
import importlib
import sys
# command_name -> command_module
command_dict = {}
def register_command(cmdname):
command_module = importlib.import_module('zodbtools.zodb' + cmdname)
command_dict[cmdname] = command_module
for _ in ('analyze', 'cmp', 'dump', 'info'):
register_command(_)
def usage(out):
print("""\
Zodb is a tool for managing ZODB databases.
Usage:
zodb command [arguments]
The commands are:
""", file=out)
cmdv = command_dict.keys()
cmdv.sort()
for cmd in cmdv:
cmd_module = command_dict[cmd]
print(" %-11s %s" % (cmd, cmd_module.summary), file=out)
print("""\
Use "zodb help [command]" for more information about a command.
Additional help topics:
""", file=out)
# NOTE no sorting here - topic_dict is pre-ordered
for topic, (topic_summary, _) in help_module.topic_dict.items():
print(" %-11s %s" % (topic, topic_summary), file=out)
print("""\
Use "zodb help [topic]" for more information about that topic.
""", file=out)
# help shows general help or help for a command/topic
def help(argv):
if len(argv) < 2: # help topic ...
usage(sys.stderr)
sys.exit(2)
topic = argv[1]
# topic can either be a command name or a help topic
if topic in command_dict:
command = command_dict[topic]
command.usage(sys.stdout)
sys.exit(0)
if topic in help_module.topic_dict:
_, topic_help = help_module.topic_dict[topic]
print(topic_help)
sys.exit(0)
print("Unknown help topic `%s`. Run 'zodb help'." % topic, file=sys.stderr)
sys.exit(2)
def main():
try:
optv, argv = getopt.getopt(sys.argv[1:], "h", ["help"])
except getopt.GetoptError as e:
print(e, file=sys.stderr)
usage(sys.stderr)
sys.exit(2)
for opt, _ in optv:
if opt in ("-h", "--help"):
usage(sys.stdout)
sys.exit(0)
if len(argv) < 1:
usage(sys.stderr)
sys.exit(2)
command = argv[0]
# help on a topic
if command=="help":
return help(argv)
# run subcommand
command_module = command_dict.get(command)
if command_module is None:
print('zodb: unknown subcommand "%s"' % command, file=sys.stderr)
print("Run 'zodb help' for usage.", file=sys.stderr)
sys.exit(2)
return command_module.main(argv)
if __name__ == '__main__':
main()
#!/usr/bin/env python
# Copyright (C) 2002-2017 Zope Foundation + Nexedi + Contributors
# See LICENSE-ZPL.txt for full licensing terms.
......@@ -237,17 +236,18 @@ Note:
Input deltafs file should be uncompressed.
"""
summary = "analyze FileStorage or repozo deltafs usage"
def usage(stream, msg=None):
if msg:
print >>stream, msg
print >>stream
program = os.path.basename(sys.argv[0])
print >>stream, __doc__ % {"program": program}
print >>stream, __doc__ % {"program": "zodb analyze"}
def main():
def main(argv):
try:
opts, args = getopt.getopt(sys.argv[1:],
opts, args = getopt.getopt(argv[1:],
'hcd', ['help', 'csv', 'dbm'])
path = args[0]
except (getopt.GetoptError, IndexError), msg:
......@@ -275,6 +275,3 @@ def main():
return h
FileStorageFormatter._read_data_header = _read_data_header
report(analyze(path, use_dbm, delta_fs), csv)
if __name__ == "__main__":
main()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2017 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
......@@ -30,7 +29,8 @@ Exit status is 0 if inputs are the same, 1 if different, 2 if error.
"""
from __future__ import print_function
from zodbtools.util import ashex, inf, nextitem, txnobjv, parse_tidrange, TidRangeInvalid
from zodbtools.util import ashex, inf, nextitem, txnobjv, parse_tidrange, TidRangeInvalid, \
storageFromURL
from time import time
# compare two storage transactions
......@@ -107,22 +107,17 @@ def storcmp(stor1, stor2, tidmin, tidmax, verbose=False):
# ----------------------------------------
import ZODB.config
import sys, getopt
import traceback
summary = "compare two ZODB databases"
def usage(out):
print("""\
Usage: zodbcmp [OPTIONS] <storage1> <storage2> [tidmin..tidmax]
Usage: zodb cmp [OPTIONS] <storage1> <storage2> [tidmin..tidmax]
Compare two ZODB databases.
<storageX> is a file with ZConfig-based storage definition, e.g.
%import neo.client
<NEOStorage>
master_nodes ...
name ...
</NEOStorage>
<storageX> is an URL (see 'zodb help zurl') of a ZODB-storage.
Options:
......@@ -130,11 +125,11 @@ Options:
-h --help show this help
""", file=out)
def main2():
def main2(argv):
verbose = False
try:
optv, argv = getopt.getopt(sys.argv[1:], "hv", ["help", "verbose"])
optv, argv = getopt.getopt(argv[1:], "hv", ["help", "verbose"])
except getopt.GetoptError as e:
print(e, file=sys.stderr)
usage(sys.stderr)
......@@ -148,7 +143,7 @@ def main2():
verbose = True
try:
storconf1, storconf2 = argv[0:2]
storurl1, storurl2 = argv[0:2]
except ValueError:
usage(sys.stderr)
sys.exit(2)
......@@ -162,20 +157,17 @@ def main2():
print("E: invalid tidrange: %s" % e, file=sys.stderr)
sys.exit(2)
stor1 = ZODB.config.storageFromFile(open(storconf1, 'r'))
stor2 = ZODB.config.storageFromFile(open(storconf2, 'r'))
stor1 = storageFromURL(storurl1, read_only=True)
stor2 = storageFromURL(storurl2, read_only=True)
zcmp = storcmp(stor1, stor2, tidmin, tidmax, verbose)
sys.exit(1 if zcmp else 0)
def main():
def main(argv):
try:
main2()
main2(argv)
except SystemExit:
raise # this was sys.exit() call, not an error
except:
traceback.print_exc()
sys.exit(2)
if __name__ == '__main__':
main()
#!/usr/bin/env python
# Copyright (C) 2016-2017 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
......@@ -31,8 +30,8 @@ txn ...
"""
from __future__ import print_function
from zodbtools.util import ashex, sha1, txnobjv, parse_tidrange, TidRangeInvalid
from zodbtools.util import ashex, sha1, txnobjv, parse_tidrange, TidRangeInvalid, \
storageFromURL
def zodbdump(stor, tidmin, tidmax, hashonly=False):
first = True
......@@ -68,21 +67,16 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False):
# ----------------------------------------
import ZODB.config
import sys, getopt
summary = "dump content of a ZODB database"
def usage(out):
print("""\
Usage: zodbdump [OPTIONS] <storage> [tidmin..tidmax]
Usage: zodb dump [OPTIONS] <storage> [tidmin..tidmax]
Dump content of a ZODB database.
<storage> is a file with ZConfig-based storage definition, e.g.
%import neo.client
<NEOStorage>
master_nodes ...
name ...
</NEOStorage>
<storage> is an URL (see 'zodb help zurl') of a ZODB-storage.
Options:
......@@ -90,11 +84,11 @@ Options:
-h --help show this help
""", file=out)
def main():
def main(argv):
hashonly = False
try:
optv, argv = getopt.getopt(sys.argv[1:], "h", ["help", "hashonly"])
optv, argv = getopt.getopt(argv[1:], "h", ["help", "hashonly"])
except getopt.GetoptError as e:
print(e, file=sys.stderr)
usage(sys.stderr)
......@@ -108,7 +102,7 @@ def main():
hashonly = True
try:
storconf = argv[0]
storurl = argv[0]
except IndexError:
usage(sys.stderr)
sys.exit(2)
......@@ -122,9 +116,6 @@ def main():
print("E: invalid tidrange: %s" % e, file=sys.stderr)
sys.exit(2)
stor = ZODB.config.storageFromFile(open(storconf, 'r'))
stor = storageFromURL(storurl, read_only=True)
zodbdump(stor, tidmin, tidmax, hashonly)
if __name__ == '__main__':
main()
# -*- coding: utf-8 -*-
# Copyright (C) 2017 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Open Source Initiative approved licenses and Convey
# the resulting work. Corresponding source of such a combination shall include
# the source code for all other software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
"""Zodbinfo - Print general information about a ZODB database"""
from __future__ import print_function
from zodbtools.util import ashex, storageFromURL
from collections import OrderedDict
import sys
# {} parameter_name -> get_parameter(stor)
infoDict = OrderedDict([
("name", lambda stor: stor.getName()),
("size", lambda stor: stor.getSize()),
("last_tid", lambda stor: ashex(stor.lastTransaction())),
])
def zodbinfo(stor, parameterv):
wantnames = False
if not parameterv:
parameterv = infoDict.keys()
wantnames = True
for parameter in parameterv:
get_parameter = infoDict.get(parameter)
if get_parameter is None:
print("invalid parameter: %s" % parameter, file=sys.stderr)
sys.exit(1)
out = ""
if wantnames:
out += parameter + "="
out += "%s" % (get_parameter(stor),)
print(out)
# ----------------------------------------
import getopt
summary = "print general information about a ZODB database"
def usage(out):
print("""\
Usage: zodb info [OPTIONS] <storage> [parameter ...]
Print general information about a ZODB database.
<storage> is an URL (see 'zodb help zurl') of a ZODB-storage.
By default info prints information about all storage parameters. If one or
more parameter names are given as arguments, info prints the value of each
named parameter on its own line.
Options:
-h --help show this help
""", file=out)
def main(argv):
try:
optv, argv = getopt.getopt(argv[1:], "h", ["help"])
except getopt.GetoptError as e:
print(e, file=sys.stderr)
usage(sys.stderr)
sys.exit(2)
for opt, _ in optv:
if opt in ("-h", "--help"):
usage(sys.stdout)
sys.exit(0)
try:
storurl = argv[0]
except IndexError:
usage(sys.stderr)
sys.exit(2)
stor = storageFromURL(storurl, read_only=True)
zodbinfo(stor, argv[1:])
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment