Use tagged version of src/scripts

utilities/ZODBTools/README.txt

+This directory contains a collection of utilities for managing ZODB
+databases.  Some are more useful than others.  If you install ZODB
+using distutils ("python setup.py install"), fsdump.py, fstest.py,
+repozo.py, and zeopack.py will be installed in /usr/local/bin.
+
+Unless otherwise noted, these scripts are invoked with the name of the
+Data.fs file as their only argument.  Example: checkbtrees.py data.fs.
+
+
+analyze.py -- a transaction analyzer for FileStorage
+
+Reports on the data in a FileStorage.  The report is organized by
+class.  It shows total data, as well as separate reports for current
+and historical revisions of objects.
+
+
+checkbtrees.py -- checks BTrees in a FileStorage for corruption
+
+Attempts to find all the BTrees contained in a Data.fs, calls their
+_check() methods, and runs them through BTrees.check.check().
+
+
+fsdump.py -- summarize FileStorage contents, one line per revision
+
+Prints a report of FileStorage contents, with one line for each
+transaction and one line for each data record in that transaction.
+Includes time stamps, file positions, and class names.
+
+
+fstest.py -- simple consistency checker for FileStorage
+
+usage: fstest.py [-v] data.fs
+
+The fstest tool will scan all the data in a FileStorage and report an
+error if it finds any corrupt transaction data.  The tool will print a
+message when the first error is detected an exit.
+
+The tool accepts one or more -v arguments.  If a single -v is used, it
+will print a line of text for each transaction record it encounters.
+If two -v arguments are used, it will also print a line of text for
+each object.  The objects for a transaction will be printed before the
+transaction itself.
+
+Note: It does not check the consistency of the object pickles.  It is
+possible for the damage to occur only in the part of the file that
+stores object pickles.  Those errors will go undetected.
+
+
+space.py -- report space used by objects in a FileStorage
+
+usage: space.py [-v] data.fs
+
+This ignores revisions and versions.
+
+
+netspace.py -- hackish attempt to report on size of objects
+
+usage: netspace.py [-P | -v] data.fs
+
+-P: do a pack first
+-v: print info for all objects, even if a traversal path isn't found
+
+Traverses objects from the database root and attempts to calculate
+size of object, including all reachable subobjects.
+
+
+parsezeolog.py -- parse BLATHER logs from ZEO server
+
+This script may be obsolete.  It has not been tested against the
+current log output of the ZEO server.
+
+Reports on the time and size of transactions committed by a ZEO
+server, by inspecting log messages at BLATHER level.
+
+
+repozo.py -- incremental backup utility for FileStorage
+
+Run the script with the -h option to see usage details.
+
+
+timeout.py -- script to test transaction timeout
+
+usage: timeout.py address delay [storage-name]
+
+This script connects to a storage, begins a transaction, calls store()
+and tpc_vote(), and then sleeps forever.  This should trigger the
+transaction timeout feature of the server.
+
+
+zeopack.py -- pack a ZEO server
+
+The script connects to a server and calls pack() on a specific
+storage.  See the script for usage details.
+
+
+zeoreplay.py -- experimental script to replay transactions from a ZEO log
+
+Like parsezeolog.py, this may be obsolete because it was written
+against an earlier version of the ZEO server.  See the script for
+usage details.
+
+
+zeoup.py
+
+usage: zeoup.py [options]
+
+The test will connect to a ZEO server, load the root object, and
+attempt to update the zeoup counter in the root.  It will report
+success if it updates to counter or if it gets a ConflictError.  A
+ConflictError is considered a success, because the client was able to
+start a transaction.
+
+See the script for details about the options.
+
+
+zodbload.py -- exercise ZODB under a heavy synthesized Zope-like load
+
+See the module docstring for details.  Note that this script requires
+Zope.  New in ZODB3 3.1.4.
+
+
+zeoserverlog.py -- analyze ZEO server log for performance statistics
+
+See the module docstring for details; there are a large number of
+options.  New in ZODB3 3.1.4.
+
+
+fsrefs.py -- check FileStorage for dangling references
+
+
+fstail.py -- display the most recent transactions in a FileStorage
+
+usage:  fstail.py [-n nxtn] data.fs
+
+The most recent ntxn transactions are displayed, to stdout.
+Optional argument -n specifies ntxn, and defaults to 10.
+
+
+migrate.py -- do a storage migration and gather statistics
+
+See the module docstring for details.
+
+
+zeoqueue.py -- report number of clients currently waiting in the ZEO queue
+
+See the module docstring for details.

utilities/ZODBTools/analyze.py

+#!/usr/bin/env python2.3
+
+# Based on a transaction analyzer by Matt Kromer.
+
+import pickle
+import re
+import sys
+import types
+from ZODB.FileStorage import FileStorage
+
+class Report:
+    def __init__(self):
+        self.OIDMAP = {}
+        self.TYPEMAP = {}
+        self.TYPESIZE = {}
+        self.FREEMAP = {}
+        self.USEDMAP = {}
+        self.TIDS = 0
+        self.OIDS = 0
+        self.DBYTES = 0
+        self.COIDS = 0
+        self.CBYTES = 0
+        self.FOIDS = 0
+        self.FBYTES = 0
+
+def shorten(s, n):
+    l = len(s)
+    if l <= n:
+        return s
+    while len(s) + 3 > n: # account for ...
+        i = s.find(".")
+        if i == -1:
+            # In the worst case, just return the rightmost n bytes
+            return s[-n:]
+        else:
+            s = s[i + 1:]
+            l = len(s)
+    return "..." + s
+
+def report(rep):
+    print "Processed %d records in %d transactions" % (rep.OIDS, rep.TIDS)
+    print "Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS)
+    print ("Average transaction size is %7.2f bytes" %
+           (rep.DBYTES * 1.0 / rep.TIDS))
+
+    print "Types used:"
+    fmt = "%-46s %7s %9s %6s %7s"
+    fmtp = "%-46s %7d %9d %5.1f%% %7.2f" # per-class format
+    fmts = "%46s %7d %8dk %5.1f%% %7.2f" # summary format
+    print fmt % ("Class Name", "Count", "TBytes", "Pct", "AvgSize")
+    print fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7)
+    typemap = rep.TYPEMAP.keys()
+    typemap.sort()
+    cumpct = 0.0
+    for t in typemap:
+        pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES
+        cumpct += pct
+        print fmtp % (shorten(t, 46), rep.TYPEMAP[t], rep.TYPESIZE[t],
+                      pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t])
+
+    print fmt % ('='*46, '='*7, '='*9, '='*5, '='*7)
+    print "%46s %7d %9s %6s %6.2fk" % ('Total Transactions', rep.TIDS, ' ',
+        ' ', rep.DBYTES * 1.0 / rep.TIDS / 1024.0)
+    print fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct,
+                  rep.DBYTES * 1.0 / rep.OIDS)
+
+    print fmts % ('Current Objects', rep.COIDS, rep.CBYTES / 1024.0,
+                  rep.CBYTES * 100.0 / rep.DBYTES,
+                  rep.CBYTES * 1.0 / rep.COIDS)
+    if rep.FOIDS:
+        print fmts % ('Old Objects', rep.FOIDS, rep.FBYTES / 1024.0,
+                      rep.FBYTES * 100.0 / rep.DBYTES,
+                      rep.FBYTES * 1.0 / rep.FOIDS)
+
+def analyze(path):
+    fs = FileStorage(path, read_only=1)
+    fsi = fs.iterator()
+    report = Report()
+    while 1:
+        try:
+            transaction = fsi.next()
+        except IndexError:
+            break
+        analyze_trans(report, transaction)
+    return report
+
+def analyze_trans(report, txn):
+    report.TIDS += 1
+    while 1:
+        try:
+            rec = txn.next()
+        except IndexError:
+            break
+        analyze_rec(report, rec)
+
+def get_type(record):
+    try:
+        classinfo = pickle.loads(record.data)[0]
+    except SystemError, err:
+        s = str(err)
+        mo = re.match('Failed to import class (\S+) from module (\S+)', s)
+        if mo is None:
+            raise
+        else:
+            klass, mod = mo.group(1, 2)
+            return "%s.%s" % (mod, klass)
+    if isinstance(classinfo, types.TupleType):
+        mod, klass = classinfo
+        return "%s.%s" % (mod, klass)
+    else:
+        return str(classinfo)
+
+def analyze_rec(report, record):
+    oid = record.oid
+    report.OIDS += 1
+    try:
+        size = len(record.data) # Ignores various overhead
+        report.DBYTES += size
+        if not report.OIDMAP.has_key(oid):
+            type = get_type(record)
+            report.OIDMAP[oid] = type
+            report.USEDMAP[oid] = size
+            report.COIDS += 1
+            report.CBYTES += size
+        else:
+            type = report.OIDMAP[oid]
+            fsize = report.USEDMAP[oid]
+            report.FREEMAP[oid] = report.FREEMAP.get(oid, 0) + fsize
+            report.USEDMAP[oid] = size
+            report.FOIDS += 1
+            report.FBYTES += fsize
+            report.CBYTES += size - fsize
+        report.TYPEMAP[type] = report.TYPEMAP.get(type, 0) + 1
+        report.TYPESIZE[type] = report.TYPESIZE.get(type, 0) + size
+    except Exception, err:
+        print err
+
+if __name__ == "__main__":
+    path = sys.argv[1]
+    report(analyze(path))

utilities/ZODBTools/checkbtrees.py

+#!/usr/bin/env python2.3
+
+"""Check the consistency of BTrees in a Data.fs
+
+usage: checkbtrees.py data.fs
+
+Try to find all the BTrees in a Data.fs, call their _check() methods,
+and run them through BTrees.check.check().
+"""
+
+from types import IntType
+
+import ZODB
+from ZODB.FileStorage import FileStorage
+from BTrees.check import check
+
+# Set of oids we've already visited.  Since the object structure is
+# a general graph, this is needed to prevent unbounded paths in the
+# presence of cycles.  It's also helpful in eliminating redundant
+# checking when a BTree is pointed to by many objects.
+oids_seen = {}
+
+# Append (obj, path) to L if and only if obj is a persistent object
+# and we haven't seen it before.
+def add_if_new_persistent(L, obj, path):
+    global oids_seen
+
+    getattr(obj, '_', None) # unghostify
+    if hasattr(obj, '_p_oid'):
+        oid = obj._p_oid
+        if not oids_seen.has_key(oid):
+            L.append((obj, path))
+            oids_seen[oid] = 1
+
+def get_subobjects(obj):
+    getattr(obj, '_', None) # unghostify
+    sub = []
+    try:
+        attrs = obj.__dict__.items()
+    except AttributeError:
+        attrs = ()
+    for pair in attrs:
+        sub.append(pair)
+
+    # what if it is a mapping?
+    try:
+        items = obj.items()
+    except AttributeError:
+        items = ()
+    for k, v in items:
+        if not isinstance(k, IntType):
+            sub.append(("<key>", k))
+        if not isinstance(v, IntType):
+            sub.append(("[%s]" % repr(k), v))
+
+    # what if it is a sequence?
+    i = 0
+    while 1:
+        try:
+            elt = obj[i]
+        except:
+            break
+        sub.append(("[%d]" % i, elt))
+        i += 1
+
+    return sub
+
+def main(fname):
+    fs = FileStorage(fname, read_only=1)
+    cn = ZODB.DB(fs).open()
+    rt = cn.root()
+    todo = []
+    add_if_new_persistent(todo, rt, '')
+
+    found = 0
+    while todo:
+        obj, path = todo.pop(0)
+        found += 1
+        if not path:
+            print "<root>", repr(obj)
+        else:
+            print path, repr(obj)
+
+        mod = str(obj.__class__.__module__)
+        if mod.startswith("BTrees"):
+            if hasattr(obj, "_check"):
+                try:
+                    obj._check()
+                except AssertionError, msg:
+                    print "*" * 60
+                    print msg
+                    print "*" * 60
+
+                try:
+                    check(obj)
+                except AssertionError, msg:
+                    print "*" * 60
+                    print msg
+                    print "*" * 60
+
+        if found % 100 == 0:
+            cn.cacheMinimize()
+
+        for k, v in get_subobjects(obj):
+            if k.startswith('['):
+                # getitem
+                newpath = "%s%s" % (path, k)
+            else:
+                newpath = "%s.%s" % (path, k)
+            add_if_new_persistent(todo, v, newpath)
+
+    print "total", len(fs._index), "found", found
+
+if __name__ == "__main__":
+    import sys
+    try:
+        fname, = sys.argv[1:]
+    except:
+        print __doc__
+        sys.exit(2)
+
+    main(fname)

utilities/ZODBTools/fsdump.py

+#!/usr/bin/env python2.3
+
+"""Print a text summary of the contents of a FileStorage."""
+
+from ZODB.FileStorage.fsdump import fsdump
+
+if __name__ == "__main__":
+    import sys
+    fsdump(sys.argv[1])

utilities/ZODBTools/fsrefs.py

+#!/usr/bin/env python2.3
+
+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+"""Check FileStorage for dangling references.
+
+usage: fsrefs.py data.fs
+
+This script ignores versions, which might produce incorrect results
+for storages that use versions.
+"""
+
+from ZODB.FileStorage import FileStorage
+from ZODB.TimeStamp import TimeStamp
+from ZODB.utils import u64
+from ZODB.FileStorage.fsdump import get_pickle_metadata
+
+import cPickle
+import cStringIO
+import traceback
+import types
+
+VERBOSE = 0
+
+def get_refs(pickle):
+    refs = []
+    f = cStringIO.StringIO(pickle)
+    u = cPickle.Unpickler(f)
+    u.persistent_load = refs
+    u.noload()
+    u.noload()
+    return refs
+
+def report(oid, data, serial, fs, missing):
+    from_mod, from_class = get_pickle_metadata(data)
+    if len(missing) > 1:
+        plural = "s"
+    else:
+        plural = ""
+    ts = TimeStamp(serial)
+    print "oid %s %s.%s" % (hex(u64(oid)), from_mod, from_class)
+    print "last updated: %s, tid=%s" % (ts, hex(u64(serial)))
+    print "refers to invalid object%s:" % plural
+    for oid, info, reason in missing:
+        if isinstance(info, types.TupleType):
+            description = "%s.%s" % info
+        else:
+            description = str(info)
+        print "\toid %s %s: %r" % (hex(u64(oid)), reason, description)
+    print
+
+def main(path):
+    fs = FileStorage(path, read_only=1)
+    noload = {}
+    for oid in fs._index.keys():
+        try:
+            data, serial = fs.load(oid, "")
+        except:
+            print "oid %s failed to load" % hex(u64(oid))
+            if VERBOSE:
+                traceback.print_exc()
+            noload[oid] = 1
+
+            # XXX If we get here after we've already loaded objects
+            # that refer to this one, we won't get error reports from
+            # them.  We could fix this by making two passes over the
+            # storage, but that seems like overkill.
+
+        refs = get_refs(data)
+        missing = [] # contains 3-tuples of oid, klass-metadata, reason
+        for info in refs:
+            try:
+                ref, klass = info
+            except (ValueError, TypeError):
+                # failed to unpack
+                ref = info
+                klass = '<unknown>'
+            if not fs._index.has_key(ref):
+                missing.append((ref, klass, "missing"))
+            if noload.has_key(ref):
+                missing.append((ref, klass, "failed to load"))
+        if missing:
+            report(oid, data, serial, fs, missing)
+
+if __name__ == "__main__":
+    import sys
+    import getopt
+
+    opts, args = getopt.getopt(sys.argv[1:], "v")
+    for k, v in opts:
+        if k == "-v":
+            VERBOSE += 1
+
+    path, = args
+    main(path)

utilities/ZODBTools/fsstats.py

+#!/usr/bin/env python2.3
+
+"""Print details statistics from fsdump output."""
+
+import re
+import sys
+
+rx_txn = re.compile("tid=([0-9a-f]+).*size=(\d+)")
+rx_data = re.compile("oid=([0-9a-f]+) class=(\S+) size=(\d+)")
+
+def sort_byhsize(seq, reverse=False):
+    L = [(v.size(), k, v) for k, v in seq]
+    L.sort()
+    if reverse:
+        L.reverse()
+    return [(k, v) for n, k, v in L]
+
+class Histogram(dict):
+
+    def add(self, size):
+        self[size] = self.get(size, 0) + 1
+
+    def size(self):
+        return sum(self.itervalues())
+
+    def mean(self):
+        product = sum([k * v for k, v in self.iteritems()])
+        return product / self.size()
+
+    def median(self):
+        # close enough?
+        n = self.size() / 2
+        L = self.keys()
+        L.sort()
+        L.reverse()
+        while 1:
+            k = L.pop()
+            if self[k] > n:
+                return k
+            n -= self[k]
+
+    def mode(self):
+        mode = 0
+        value = 0
+        for k, v in self.iteritems():
+            if v > value:
+                value = v
+                mode = k
+        return mode
+
+    def make_bins(self, binsize):
+        maxkey = max(self.iterkeys())
+        self.binsize = binsize
+        self.bins = [0] * (1 + maxkey / binsize)
+        for k, v in self.iteritems():
+            b = k / binsize
+            self.bins[b] += v
+
+    def report(self, name, binsize=50, usebins=False, gaps=True, skip=True):
+        if usebins:
+            # Use existing bins with whatever size they have
+            binsize = self.binsize
+        else:
+            # Make new bins
+            self.make_bins(binsize)
+        maxval = max(self.bins)
+        # Print up to 40 dots for a value
+        dot = max(maxval / 40, 1)
+        tot = sum(self.bins)
+        print name
+        print "Total", tot,
+        print "Median", self.median(),
+        print "Mean", self.mean(),
+        print "Mode", self.mode(),
+        print "Max", max(self)
+        print "One * represents", dot
+        gap = False
+        cum = 0
+        for i, n in enumerate(self.bins):
+            if gaps and (not n or (skip and not n / dot)):
+                if not gap:
+                    print "   ..."
+                gap = True
+                continue
+            gap = False
+            p = 100 * n / tot
+            cum += n
+            pc = 100 * cum / tot
+            print "%6d %6d %3d%% %3d%% %s" % (
+                i * binsize, n, p, pc, "*" * (n / dot))
+        print
+
+def class_detail(class_size):
+    # summary of classes
+    fmt = "%5s %6s %6s %6s   %-50.50s"
+    labels = ["num", "median", "mean", "mode", "class"]
+    print fmt % tuple(labels)
+    print fmt % tuple(["-" * len(s) for s in labels])
+    for klass, h in sort_byhsize(class_size.iteritems()):
+        print fmt % (h.size(), h.median(), h.mean(), h.mode(), klass)
+    print
+
+    # per class details
+    for klass, h in sort_byhsize(class_size.iteritems(), reverse=True):
+        h.make_bins(50)
+        if len(filter(None, h.bins)) == 1:
+            continue
+        h.report("Object size for %s" % klass, usebins=True)
+
+def revision_detail(lifetimes, classes):
+    # Report per-class details for any object modified more than once
+    for name, oids in classes.iteritems():
+        h = Histogram()
+        keep = False
+        for oid in dict.fromkeys(oids, 1):
+            L = lifetimes.get(oid)
+            n = len(L)
+            h.add(n)
+            if n > 1:
+                keep = True
+        if keep:
+            h.report("Number of revisions for %s" % name, binsize=10)
+
+def main(path):
+    txn_objects = Histogram() # histogram of txn size in objects
+    txn_bytes = Histogram() # histogram of txn size in bytes
+    obj_size = Histogram() # histogram of object size
+    n_updates = Histogram() # oid -> num updates
+    n_classes = Histogram() # class -> num objects
+    lifetimes = {} # oid -> list of tids
+    class_size = {} # class -> histogram of object size
+    classes = {} # class -> list of oids
+
+    MAX = 0
+    tid = None
+
+    f = open(path, "rb")
+    for i, line in enumerate(f):
+        if MAX and i > MAX:
+            break
+        if line.startswith("  data"):
+            m = rx_data.search(line)
+            if not m:
+                continue
+            oid, klass, size = m.groups()
+            size = int(size)
+
+            obj_size.add(size)
+            n_updates.add(oid)
+            n_classes.add(klass)
+
+            h = class_size.get(klass)
+            if h is None:
+                h = class_size[klass] = Histogram()
+            h.add(size)
+
+            L = lifetimes.setdefault(oid, [])
+            L.append(tid)
+
+            L = classes.setdefault(klass, [])
+            L.append(oid)
+            objects += 1
+
+        elif line.startswith("Trans"):
+
+            if tid is not None:
+                txn_objects.add(objects)
+
+            m = rx_txn.search(line)
+            if not m:
+                continue
+            tid, size = m.groups()
+            size = int(size)
+            objects = 0
+
+            txn_bytes.add(size)
+    f.close()
+
+    print "Summary: %d txns, %d objects, %d revisions" % (
+        txn_objects.size(), len(n_updates), n_updates.size())
+    print
+
+    txn_bytes.report("Transaction size (bytes)", binsize=1024)
+    txn_objects.report("Transaction size (objects)", binsize=10)
+    obj_size.report("Object size", binsize=128)
+
+    # object lifetime info
+    h = Histogram()
+    for k, v in lifetimes.items():
+        h.add(len(v))
+    h.report("Number of revisions", binsize=10, skip=False)
+
+    # details about revisions
+    revision_detail(lifetimes, classes)
+
+    class_detail(class_size)
+
+if __name__ == "__main__":
+    main(sys.argv[1])

utilities/ZODBTools/fstail.py

+#!/usr/bin/env python2.3
+
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Tool to dump the last few transactions from a FileStorage."""
+
+from ZODB.fstools import prev_txn
+
+import binascii
+import getopt
+import sha
+import sys
+
+def main(path, ntxn):
+    f = open(path, "rb")
+    f.seek(0, 2)
+    th = prev_txn(f)
+    i = ntxn
+    while th and i > 0:
+        hash = sha.sha(th.get_raw_data()).digest()
+        l = len(str(th.get_timestamp())) + 1
+        th.read_meta()
+        print "%s: hash=%s" % (th.get_timestamp(),
+                               binascii.hexlify(hash))
+        print ("user=%r description=%r length=%d"
+               % (th.user, th.descr, th.length))
+        print
+        th = th.prev_txn()
+        i -= 1
+
+if __name__ == "__main__":
+    ntxn = 10
+    opts, args = getopt.getopt(sys.argv[1:], "n:")
+    path, = args
+    for k, v in opts:
+        if k == '-n':
+            ntxn = int(v)
+    main(path, ntxn)

utilities/ZODBTools/fstest.py

+#!/usr/bin/env python2.3
+
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+"""Simple consistency checker for FileStorage.
+
+usage: fstest.py [-v] data.fs
+
+The fstest tool will scan all the data in a FileStorage and report an
+error if it finds any corrupt transaction data.  The tool will print a
+message when the first error is detected, then exit.
+
+The tool accepts one or more -v arguments.  If a single -v is used, it
+will print a line of text for each transaction record it encounters.
+If two -v arguments are used, it will also print a line of text for
+each object.  The objects for a transaction will be printed before the
+transaction itself.
+
+Note: It does not check the consistency of the object pickles.  It is
+possible for the damage to occur only in the part of the file that
+stores object pickles.  Those errors will go undetected.
+"""
+
+# The implementation is based closely on the read_index() function in
+# ZODB.FileStorage.  If anything about the FileStorage layout changes,
+# this file will need to be udpated.
+
+import string
+import struct
+import sys
+
+class FormatError(ValueError):
+    """There is a problem with the format of the FileStorage."""
+
+class Status:
+    checkpoint = 'c'
+    undone = 'u'
+
+packed_version = 'FS21'
+
+TREC_HDR_LEN = 23
+DREC_HDR_LEN = 42
+
+VERBOSE = 0
+
+def hexify(s):
+    """Format an 8-bite string as hex"""
+    l = []
+    for c in s:
+        h = hex(ord(c))
+        if h[:2] == '0x':
+            h = h[2:]
+        if len(h) == 1:
+            l.append("0")
+        l.append(h)
+    return "0x" + string.join(l, '')
+
+def chatter(msg, level=1):
+    if VERBOSE >= level:
+        sys.stdout.write(msg)
+
+def U64(v):
+    """Unpack an 8-byte string as a 64-bit long"""
+    h, l = struct.unpack(">II", v)
+    if h:
+        return (h << 32) + l
+    else:
+        return l
+
+def check(path):
+    file = open(path, 'rb')
+
+    file.seek(0, 2)
+    file_size = file.tell()
+    if file_size == 0:
+        raise FormatError("empty file")
+    file.seek(0)
+    if file.read(4) != packed_version:
+        raise FormatError("invalid file header")
+
+    pos = 4L
+    tid = '\000' * 8 # lowest possible tid to start
+    i = 0
+    while pos:
+        _pos = pos
+        pos, tid = check_trec(path, file, pos, tid, file_size)
+        if tid is not None:
+            chatter("%10d: transaction tid %s #%d \n" %
+                    (_pos, hexify(tid), i))
+            i = i + 1
+
+
+def check_trec(path, file, pos, ltid, file_size):
+    """Read an individual transaction record from file.
+
+    Returns the pos of the next transaction and the transaction id.
+    It also leaves the file pointer set to pos.  The path argument is
+    used for generating error messages.
+    """
+
+    h = file.read(TREC_HDR_LEN)
+    if not h:
+        return None, None
+    if len(h) != TREC_HDR_LEN:
+        raise FormatError("%s truncated at %s" % (path, pos))
+
+    tid, stl, status, ul, dl, el = struct.unpack(">8s8scHHH", h)
+    if el < 0:
+        el = t32 - el
+    tmeta_len = TREC_HDR_LEN + ul + dl + el
+
+    if tid <= ltid:
+        raise FormatError("%s time-stamp reduction at %s: %s <= %s" %
+                          (path, pos, hexify(tid), hexify(ltid)))
+    ltid = tid
+
+    tl = U64(stl) # transaction record length - 8
+    if pos + tl + 8 > file_size:
+        raise FormatError("%s truncated possibly because of"
+                          " damaged records at %s" % (path, pos))
+    if status == Status.checkpoint:
+        raise FormatError("%s checkpoint flag was not cleared at %s"
+                          % (path, pos))
+    if status not in ' up':
+        raise FormatError("%s has invalid status '%s' at %s" %
+                          (path, status, pos))
+
+    if tmeta_len > tl:
+        raise FormatError("%s has an invalid transaction header"
+                          " at %s" % (path, pos))
+
+    tpos = pos
+    tend = tpos + tl
+
+    if status != Status.undone:
+        pos = tpos + tmeta_len
+        file.read(ul + dl + el) # skip transaction metadata
+
+        i = 0
+        while pos < tend:
+            _pos = pos
+            pos, oid = check_drec(path, file, pos, tpos, tid)
+            if pos > tend:
+                raise FormatError("%s has data records that extend beyond"
+                                  " the transaction record; end at %s" %
+                                  (path, pos))
+            chatter("%10d: object oid %s #%d\n" % (_pos, hexify(oid), i),
+                    level=2)
+            i = i + 1
+
+    file.seek(tend)
+    rtl = file.read(8)
+    if rtl != stl:
+        raise FormatError("%s has inconsistent transaction length"
+                          " for undone transaction at %s" % (path, pos))
+    pos = tend + 8
+    return pos, tid
+
+def check_drec(path, file, pos, tpos, tid):
+    """Check a data record for the current transaction record"""
+
+    h = file.read(DREC_HDR_LEN)
+    if len(h) != DREC_HDR_LEN:
+        raise FormatError("%s truncated at %s" % (path, pos))
+    oid, serial, _prev, _tloc, vlen, _plen = (
+        struct.unpack(">8s8s8s8sH8s", h))
+    prev = U64(_prev)
+    tloc = U64(_tloc)
+    plen = U64(_plen)
+    dlen = DREC_HDR_LEN + (plen or 8)
+
+    if vlen:
+        dlen = dlen + 16 + vlen
+        file.seek(8, 1)
+        pv = U64(file.read(8))
+        file.seek(vlen, 1) # skip the version data
+
+    if tloc != tpos:
+        raise FormatError("%s data record exceeds transaction record "
+                          "at %s: tloc %d != tpos %d" %
+                          (path, pos, tloc, tpos))
+
+    pos = pos + dlen
+    # XXX is the following code necessary?
+    if plen:
+        file.seek(plen, 1)
+    else:
+        file.seek(8, 1)
+        # XXX _loadBack() ?
+
+    return pos, oid
+
+def usage():
+    print __doc__
+    sys.exit(-1)
+
+if __name__ == "__main__":
+    import getopt
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'v')
+        if len(args) != 1:
+            raise ValueError, "expected one argument"
+        for k, v in opts:
+            if k == '-v':
+                VERBOSE = VERBOSE + 1
+    except (getopt.error, ValueError):
+        usage()
+
+    try:
+        check(args[0])
+    except FormatError, msg:
+        print msg
+        sys.exit(-1)
+
+    chatter("no errors detected")

utilities/ZODBTools/netspace.py

+#!/usr/bin/env python2.3
+
+"""Report on the net size of objects counting subobjects.
+
+usage: netspace.py [-P | -v] data.fs
+
+-P: do a pack first
+-v: print info for all objects, even if a traversal path isn't found
+"""
+
+import ZODB
+from ZODB.FileStorage import FileStorage
+from ZODB.utils import U64
+from ZODB.fsdump import get_pickle_metadata
+from ZODB.referencesf import referencesf
+
+def find_paths(root, maxdist):
+    """Find Python attribute traversal paths for objects to maxdist distance.
+
+    Starting at a root object, traverse attributes up to distance levels
+    from the root, looking for persistent objects.  Return a dict
+    mapping oids to traversal paths.
+
+    XXX Assumes that the keys of the root are not themselves
+    persistent objects.
+
+    XXX Doesn't traverse containers.
+    """
+    paths = {}
+
+    # Handle the root as a special case because it's a dict
+    objs = []
+    for k, v in root.items():
+        oid = getattr(v, '_p_oid', None)
+        objs.append((k, v, oid, 0))
+
+    for path, obj, oid, dist in objs:
+        if oid is not None:
+            paths[oid] = path
+        if dist < maxdist:
+            getattr(obj, 'foo', None) # unghostify
+            try:
+                items = obj.__dict__.items()
+            except AttributeError:
+                continue
+            for k, v in items:
+                oid = getattr(v, '_p_oid', None)
+                objs.append(("%s.%s" % (path, k), v, oid, dist + 1))
+
+    return paths
+
+def main(path):
+    fs = FileStorage(path, read_only=1)
+    if PACK:
+        fs.pack()
+
+    db = ZODB.DB(fs)
+    rt = db.open().root()
+    paths = find_paths(rt, 3)
+
+    def total_size(oid):
+        cache = {}
+        cache_size = 1000
+        def _total_size(oid, seen):
+            v = cache.get(oid)
+            if v is not None:
+                return v
+            data, serialno = fs.load(oid, '')
+            size = len(data)
+            for suboid in referencesf(data):
+                if seen.has_key(suboid):
+                    continue
+                seen[suboid] = 1
+                size += _total_size(suboid, seen)
+            cache[oid] = size
+            if len(cache) == cache_size:
+                cache.popitem()
+            return size
+        return _total_size(oid, {})
+
+    keys = fs._index.keys()
+    keys.sort()
+    keys.reverse()
+
+    if not VERBOSE:
+        # If not running verbosely, don't print an entry for an object
+        # unless it has an entry in paths.
+        keys = filter(paths.has_key, keys)
+
+    fmt = "%8s %5d %8d %s %s.%s"
+
+    for oid in keys:
+        data, serialno = fs.load(oid, '')
+        mod, klass = get_pickle_metadata(data)
+        refs = referencesf(data)
+        path = paths.get(oid, '-')
+        print fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)
+
+if __name__ == "__main__":
+    import sys
+    import getopt
+
+    PACK = 0
+    VERBOSE = 0
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'Pv')
+        path, = args
+    except getopt.error, err:
+        print err
+        print __doc__
+        sys.exit(2)
+    except ValueError:
+        print "expected one argument, got", len(args)
+        print __doc__
+        sys.exit(2)
+    for o, v in opts:
+        if o == '-P':
+            PACK = 1
+        if o == '-v':
+            VERBOSE += 1
+    main(path)

utilities/ZODBTools/parsezeolog.py

+#!/usr/bin/env python2.3
+
+"""Parse the BLATHER logging generated by ZEO2.
+
+An example of the log format is:
+2002-04-15T13:05:29 BLATHER(-100) ZEO Server storea(3235680, [714], 235339406490168806) ('10.0.26.30', 45514)
+"""
+
+import re
+import time
+
+rx_time = re.compile('(\d\d\d\d-\d\d-\d\d)T(\d\d:\d\d:\d\d)')
+
+def parse_time(line):
+    """Return the time portion of a zLOG line in seconds or None."""
+    mo = rx_time.match(line)
+    if mo is None:
+        return None
+    date, time_ = mo.group(1, 2)
+    date_l = [int(elt) for elt in date.split('-')]
+    time_l = [int(elt) for elt in time_.split(':')]
+    return int(time.mktime(date_l + time_l + [0, 0, 0]))
+
+rx_meth = re.compile("zrpc:\d+ calling (\w+)\((.*)")
+
+def parse_method(line):
+    pass
+
+def parse_line(line):
+    """Parse a log entry and return time, method info, and client."""
+    t = parse_time(line)
+    if t is None:
+        return None, None
+    mo = rx_meth.search(line)
+    if mo is None:
+        return None, None
+    meth_name = mo.group(1)
+    meth_args = mo.group(2).strip()
+    if meth_args.endswith(')'):
+        meth_args = meth_args[:-1]
+    meth_args = [s.strip() for s in meth_args.split(",")]
+    m = meth_name, tuple(meth_args)
+    return t, m
+
+class TStats:
+
+    counter = 1
+
+    def __init__(self):
+        self.id = TStats.counter
+        TStats.counter += 1
+
+    fields = ("time", "vote", "done", "user", "path")
+    fmt = "%-24s %5s %5s %-15s %s"
+    hdr = fmt % fields
+
+    def report(self):
+        """Print a report about the transaction"""
+        t = time.ctime(self.begin)
+        if hasattr(self, "vote"):
+            d_vote = self.vote - self.begin
+        else:
+            d_vote = "*"
+        if hasattr(self, "finish"):
+            d_finish = self.finish - self.begin
+        else:
+            d_finish =  "*"
+        print self.fmt % (time.ctime(self.begin), d_vote, d_finish,
+                          self.user, self.url)
+
+class TransactionParser:
+
+    def __init__(self):
+        self.txns = {}
+        self.skipped = 0
+
+    def parse(self, line):
+        t, m = parse_line(line)
+        if t is None:
+            return
+        name = m[0]
+        meth = getattr(self, name, None)
+        if meth is not None:
+            meth(t, m[1])
+
+    def tpc_begin(self, time, args):
+        t = TStats()
+        t.begin = time
+        t.user = args[1]
+        t.url = args[2]
+        t.objects = []
+        tid = eval(args[0])
+        self.txns[tid] = t
+
+    def get_txn(self, args):
+        tid = eval(args[0])
+        try:
+            return self.txns[tid]
+        except KeyError:
+            print "uknown tid", repr(tid)
+            return None
+
+    def tpc_finish(self, time, args):
+        t = self.get_txn(args)
+        if t is None:
+            return
+        t.finish = time
+
+    def vote(self, time, args):
+        t = self.get_txn(args)
+        if t is None:
+            return
+        t.vote = time
+
+    def get_txns(self):
+        L = [(t.id, t) for t in self.txns.values()]
+        L.sort()
+        return [t for (id, t) in L]
+
+if __name__ == "__main__":
+    import fileinput
+
+    p = TransactionParser()
+    i = 0
+    for line in fileinput.input():
+        i += 1
+        try:
+            p.parse(line)
+        except:
+            print "line", i
+            raise
+    print "Transaction: %d" % len(p.txns)
+    print TStats.hdr
+    for txn in p.get_txns():
+        txn.report()

utilities/ZODBTools/space.py

+#!/usr/bin/env python2.3
+
+"""Report on the space used by objects in a storage.
+
+usage: space.py data.fs
+
+The current implementation only supports FileStorage.
+
+Current limitations / simplifications: Ignores revisions and versions.
+"""
+
+from ZODB.FileStorage import FileStorage
+from ZODB.utils import U64
+from ZODB.fsdump import get_pickle_metadata
+
+def run(path, v=0):
+    fs = FileStorage(path, read_only=1)
+    # break into the file implementation
+    if hasattr(fs._index, 'iterkeys'):
+        iter = fs._index.iterkeys()
+    else:
+        iter = fs._index.keys()
+    totals = {}
+    for oid in iter:
+        data, serialno = fs.load(oid, '')
+        mod, klass = get_pickle_metadata(data)
+        key = "%s.%s" % (mod, klass)
+        bytes, count = totals.get(key, (0, 0))
+        bytes += len(data)
+        count += 1
+        totals[key] = bytes, count
+        if v:
+            print "%8s %5d %s" % (U64(oid), len(data), key)
+    L = totals.items()
+    L.sort(lambda a, b: cmp(a[1], b[1]))
+    L.reverse()
+    print "Totals per object class:"
+    for key, (bytes, count) in L:
+        print "%8d %8d %s" % (count, bytes, key)
+
+def main():
+    import sys
+    import getopt
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "v")
+    except getopt.error, msg:
+        print msg
+        print "usage: space.py [-v] Data.fs"
+        sys.exit(2)
+    if len(args) != 1:
+        print "usage: space.py [-v] Data.fs"
+        sys.exit(2)
+    v = 0
+    for o, a in opts:
+        if o == "-v":
+            v += 1
+    path = args[0]
+    run(path, v)
+
+if __name__ == "__main__":
+    main()

utilities/ZODBTools/tests/testfstest.py

+"""Verify that fstest.py can find errors.
+
+XXX To run this test script fstest.py must be on your PYTHONPATH.
+"""
+
+from cStringIO import StringIO
+import os
+import re
+import struct
+import tempfile
+import unittest
+
+import fstest
+from fstest import FormatError, U64
+
+class TestCorruptedFS(unittest.TestCase):
+
+    # XXX path?
+    f = open('test-checker.fs', 'rb')
+    datafs = f.read()
+    f.close()
+    del f
+
+    def setUp(self):
+        self._temp = tempfile.mktemp()
+        self._file = open(self._temp, 'wb')
+
+    def tearDown(self):
+        if not self._file.closed:
+            self._file.close()
+        if os.path.exists(self._temp):
+            try:
+                os.remove(self._temp)
+            except os.error:
+                pass
+
+    def noError(self):
+        if not self._file.closed:
+            self._file.close()
+        fstest.check(self._temp)
+
+    def detectsError(self, rx):
+        if not self._file.closed:
+            self._file.close()
+        try:
+            fstest.check(self._temp)
+        except FormatError, msg:
+            mo = re.search(rx, str(msg))
+            self.failIf(mo is None, "unexpected error: %s" % msg)
+        else:
+            self.fail("fstest did not detect corruption")
+
+    def getHeader(self):
+        buf = self._datafs.read(16)
+        if not buf:
+            return 0, ''
+        tl = U64(buf[8:])
+        return tl, buf
+
+    def copyTransactions(self, n):
+        """Copy at most n transactions from the good data"""
+        f = self._datafs = StringIO(self.datafs)
+        self._file.write(f.read(4))
+        for i in range(n):
+            tl, data = self.getHeader()
+            if not tl:
+                return
+            self._file.write(data)
+            rec = f.read(tl - 8)
+            self._file.write(rec)
+
+    def testGood(self):
+        self._file.write(self.datafs)
+        self.noError()
+
+    def testTwoTransactions(self):
+        self.copyTransactions(2)
+        self.noError()
+
+    def testEmptyFile(self):
+        self.detectsError("empty file")
+
+    def testInvalidHeader(self):
+        self._file.write('SF12')
+        self.detectsError("invalid file header")
+
+    def testTruncatedTransaction(self):
+        self._file.write(self.datafs[:4+22])
+        self.detectsError("truncated")
+
+    def testCheckpointFlag(self):
+        self.copyTransactions(2)
+        tl, data = self.getHeader()
+        assert tl > 0, "ran out of good transaction data"
+        self._file.write(data)
+        self._file.write('c')
+        self._file.write(self._datafs.read(tl - 9))
+        self.detectsError("checkpoint flag")
+
+    def testInvalidStatus(self):
+        self.copyTransactions(2)
+        tl, data = self.getHeader()
+        assert tl > 0, "ran out of good transaction data"
+        self._file.write(data)
+        self._file.write('Z')
+        self._file.write(self._datafs.read(tl - 9))
+        self.detectsError("invalid status")
+
+    def testTruncatedRecord(self):
+        self.copyTransactions(3)
+        tl, data = self.getHeader()
+        assert tl > 0, "ran out of good transaction data"
+        self._file.write(data)
+        buf = self._datafs.read(tl / 2)
+        self._file.write(buf)
+        self.detectsError("truncated possibly")
+
+    def testBadLength(self):
+        self.copyTransactions(2)
+        tl, data = self.getHeader()
+        assert tl > 0, "ran out of good transaction data"
+        self._file.write(data)
+        buf = self._datafs.read(tl - 8)
+        self._file.write(buf[0])
+        assert tl <= 1<<16, "can't use this transaction for this test"
+        self._file.write("\777\777")
+        self._file.write(buf[3:])
+        self.detectsError("invalid transaction header")
+
+    def testDecreasingTimestamps(self):
+        self.copyTransactions(0)
+        tl, data = self.getHeader()
+        buf = self._datafs.read(tl - 8)
+        t1 = data + buf
+
+        tl, data = self.getHeader()
+        buf = self._datafs.read(tl - 8)
+        t2 = data + buf
+
+        self._file.write(t2[:8] + t1[8:])
+        self._file.write(t1[:8] + t2[8:])
+        self.detectsError("time-stamp")
+
+    def testTruncatedData(self):
+        # This test must re-write the transaction header length in
+        # order to trigger the error in check_drec().  If it doesn't,
+        # the truncated data record would also caught a truncated
+        # transaction record.
+        self.copyTransactions(1)
+        tl, data = self.getHeader()
+        pos = self._file.tell()
+        self._file.write(data)
+        buf = self._datafs.read(tl - 8)
+        hdr = buf[:15]
+        ul, dl, el = struct.unpack(">HHH", hdr[-6:])
+        self._file.write(buf[:15 + ul + dl + el])
+        data = buf[15 + ul + dl + el:]
+        self._file.write(data[:24])
+        self._file.seek(pos + 8, 0)
+        newlen = struct.pack(">II", 0, tl - (len(data) - 24))
+        self._file.write(newlen)
+        self.detectsError("truncated at")
+
+    def testBadDataLength(self):
+        self.copyTransactions(1)
+        tl, data = self.getHeader()
+        self._file.write(data)
+        buf = self._datafs.read(tl - 8)
+        hdr = buf[:7]
+        # write the transaction meta data
+        ul, dl, el = struct.unpack(">HHH", hdr[-6:])
+        self._file.write(buf[:7 + ul + dl + el])
+
+        # write the first part of the data header
+        data = buf[7 + ul + dl + el:]
+        self._file.write(data[:24])
+        self._file.write("\000" * 4 + "\077" + "\000" * 3)
+        self._file.write(data[32:])
+        self.detectsError("record exceeds transaction")
+
+if __name__ == "__main__":
+    unittest.main()

utilities/ZODBTools/tests/testzeopack.py

+# Some simple tests for zeopack.py
+# For this to work, zeopack.py must by on your PATH.
+
+from ZODB.FileStorage import FileStorage
+from ZODB.tests.StorageTestBase import StorageTestBase
+from ZEO.tests import forker
+import ZODB
+
+import os
+import socket
+import tempfile
+import threading
+import time
+import unittest
+
+# XXX The forker interface isn't clearly defined.  It's different on
+# different branches of ZEO.  This will break someday.
+
+# XXX Only handle the Unix variant of the forker.  Just to give Tim
+# something to do.
+
+class PackerTests(StorageTestBase):
+
+    def setUp(self):
+        self.started = 0
+
+    def start(self):
+        self.started =1
+        self.path = tempfile.mktemp(suffix=".fs")
+        self._storage = FileStorage(self.path)
+        self.db = ZODB.DB(self._storage)
+        self.do_updates()
+        self.pid, self.exit = forker.start_zeo_server(self._storage, self.addr)
+
+    def do_updates(self):
+        for i in range(100):
+            self._dostore()
+
+    def tearDown(self):
+        if not self.started:
+            return
+        self.db.close()
+        self._storage.close()
+        self.exit.close()
+        try:
+            os.kill(self.pid, 9)
+        except os.error:
+            pass
+        try:
+            os.waitpid(self.pid, 0)
+        except os.error, err:
+            ##print "waitpid failed", err
+            pass
+        for ext in '', '.old', '.lock', '.index', '.tmp':
+            path = self.path + ext
+            try:
+                os.remove(path)
+            except os.error:
+                pass
+
+    def set_inet_addr(self):
+        self.host = socket.gethostname()
+        self.port = forker.get_port()
+        self.addr = self.host, self.port
+
+    def testPack(self):
+        self.set_inet_addr()
+        self.start()
+        status = os.system("zeopack.py -h %s -p %s" % (self.host, self.port))
+        assert status == 0
+        assert os.path.exists(self.path + ".old")
+
+    def testPackDays(self):
+        self.set_inet_addr()
+        self.start()
+        status = os.system("zeopack.py -h %s -p %s -d 1" % (self.host,
+                                                            self.port))
+        # Since we specified one day, nothing should get packed
+        assert status == 0
+        assert not os.path.exists(self.path + ".old")
+
+    def testAF_UNIXPack(self):
+        self.addr = tempfile.mktemp(suffix=".zeo-socket")
+        self.start()
+        status = os.system("zeopack.py -U %s" % self.addr)
+        assert status == 0
+        assert os.path.exists(self.path + ".old")
+
+    def testNoServer(self):
+        status = os.system("zeopack.py -p 19")
+        assert status != 0
+
+    def testWaitForServer(self):
+        self.set_inet_addr()
+        def delayed_start():
+            time.sleep(11)
+            self.start()
+        t = threading.Thread(target=delayed_start)
+        t.start()
+        status = os.system("zeopack.py -h %s -p %s -W" % (self.host,
+                                                          self.port))
+        t.join()
+        assert status == 0
+        assert os.path.exists(self.path + ".old")
+
+class UpTest(unittest.TestCase):
+
+    def testUp(self):
+        status = os.system("zeoup.py -p 19")
+        # There is no ZEO server on port 19, so we should see non-zero
+        # exit status.
+        assert status != 0
+
+if __name__ == "__main__":
+    unittest.main()

utilities/ZODBTools/timeout.py

+#!/usr/bin/env python2.3
+
+"""Transaction timeout test script.
+
+This script connects to a storage, begins a transaction, calls store()
+and tpc_vote(), and then sleeps forever.  This should trigger the
+transaction timeout feature of the server.
+
+usage: timeout.py address delay [storage-name]
+
+"""
+
+import sys
+import time
+
+from ZODB.Transaction import Transaction
+from ZODB.tests.MinPO import MinPO
+from ZODB.tests.StorageTestBase import zodb_pickle
+from ZEO.ClientStorage import ClientStorage
+
+ZERO = '\0'*8
+
+def main():
+    if len(sys.argv) not in (3, 4):
+        sys.stderr.write("Usage: timeout.py address delay [storage-name]\n" %
+                         sys.argv[0])
+        sys.exit(2)
+
+    hostport = sys.argv[1]
+    delay = float(sys.argv[2])
+    if sys.argv[3:]:
+        name = sys.argv[3]
+    else:
+        name = "1"
+
+    if "/" in hostport:
+        address = hostport
+    else:
+        if ":" in hostport:
+            i = hostport.index(":")
+            host, port = hostport[:i], hostport[i+1:]
+        else:
+            host, port = "", hostport
+        port = int(port)
+        address = (host, port)
+
+    print "Connecting to %s..." % repr(address)
+    storage = ClientStorage(address, name)
+    print "Connected.  Now starting a transaction..."
+
+    oid = storage.new_oid()
+    version = ""
+    revid = ZERO
+    data = MinPO("timeout.py")
+    pickled_data = zodb_pickle(data)
+    t = Transaction()
+    t.user = "timeout.py"
+    storage.tpc_begin(t)
+    storage.store(oid, revid, pickled_data, version, t)
+    print "Stored.  Now voting..."
+    storage.tpc_vote(t)
+
+    print "Voted; now sleeping %s..." % delay
+    time.sleep(delay)
+    print "Done."
+
+if __name__ == "__main__":
+    main()

utilities/ZODBTools/zeopack.py

+#!/usr/bin/env python2.3
+
+"""Connect to a ZEO server and ask it to pack.
+
+Usage: zeopack.py [options]
+
+Options:
+
+    -p port -- port to connect to
+
+    -h host -- host to connect to (default is current host)
+
+    -U path -- Unix-domain socket to connect to
+
+    -S name -- storage name (default is '1')
+
+    -d days -- pack objects more than days old
+
+    -1 -- Connect to a ZEO 1 server
+
+    -W -- wait for server to come up.  Normally the script tries to
+       connect for 10 seconds, then exits with an error.  The -W
+       option is only supported with ZEO 1.
+
+You must specify either -p and -h or -U.
+"""
+
+import getopt
+import socket
+import sys
+import time
+
+from ZEO.ClientStorage import ClientStorage
+
+WAIT = 10 # wait no more than 10 seconds for client to connect
+
+def connect(storage):
+    # The connect-on-startup logic that ZEO provides isn't too useful
+    # for this script.  We'd like to client to attempt to startup, but
+    # fail if it can't get through to the server after a reasonable
+    # amount of time.  There's no external support for this, so we'll
+    # expose the ZEO 1.0 internals.  (consenting adults only)
+    t0 = time.time()
+    while t0 + WAIT > time.time():
+        storage._call.connect()
+        if storage._connected:
+            return
+    raise RuntimeError, "Unable to connect to ZEO server"
+
+def pack1(addr, storage, days, wait):
+    cs = ClientStorage(addr, storage=storage,
+                       wait_for_server_on_startup=wait)
+    if wait:
+        # _startup() is an artifact of the way ZEO 1.0 works.  The
+        # ClientStorage doesn't get fully initialized until registerDB()
+        # is called.  The only thing we care about, though, is that
+        # registerDB() calls _startup().
+        cs._startup()
+    else:
+        connect(cs)
+    cs.invalidator = None
+    cs.pack(wait=1, days=days)
+    cs.close()
+
+def pack2(addr, storage, days):
+    cs = ClientStorage(addr, storage=storage, wait=1, read_only=1)
+    cs.pack(wait=1, days=days)
+    cs.close()
+
+def usage(exit=1):
+    print __doc__
+    print " ".join(sys.argv)
+    sys.exit(exit)
+
+def main():
+    host = None
+    port = None
+    unix = None
+    storage = '1'
+    days = 0
+    wait = 0
+    zeoversion = 2
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'p:h:U:S:d:W1')
+        for o, a in opts:
+            if o == '-p':
+                port = int(a)
+            elif o == '-h':
+                host = a
+            elif o == '-U':
+                unix = a
+            elif o == '-S':
+                storage = a
+            elif o == '-d':
+                days = int(a)
+            elif o == '-W':
+                wait = 1
+            elif o == '-1':
+                zeoversion = 1
+    except Exception, err:
+        print err
+        usage()
+
+    if unix is not None:
+        addr = unix
+    else:
+        if host is None:
+            host = socket.gethostname()
+        if port is None:
+            usage()
+        addr = host, port
+
+    if zeoversion == 1:
+        pack1(addr, storage, days, wait)
+    else:
+        pack2(addr, storage, days)
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception, err:
+        print err
+        sys.exit(1)

utilities/ZODBTools/zeoreplay.py

+#!/usr/bin/env python2.3
+
+"""Parse the BLATHER logging generated by ZEO, and optionally replay it.
+
+Usage: zeointervals.py [options]
+
+Options:
+
+    --help / -h
+        Print this message and exit.
+
+    --replay=storage
+    -r storage
+        Replay the parsed transactions through the new storage
+
+    --maxtxn=count
+    -m count
+        Parse no more than count transactions.
+
+    --report / -p
+        Print a report as we're parsing.
+
+Unlike parsezeolog.py, this script generates timestamps for each transaction,
+and sub-command in the transaction.  We can use this to compare timings with
+synthesized data.
+"""
+
+import re
+import sys
+import time
+import getopt
+import operator
+# ZEO logs measure wall-clock time so for consistency we need to do the same
+#from time import clock as now
+from time import time as now
+
+from ZODB.FileStorage import FileStorage
+#from BDBStorage.BDBFullStorage import BDBFullStorage
+#from Standby.primary import PrimaryStorage
+#from Standby.config import RS_PORT
+from ZODB.Transaction import Transaction
+from ZODB.utils import p64
+
+datecre = re.compile('(\d\d\d\d-\d\d-\d\d)T(\d\d:\d\d:\d\d)')
+methcre = re.compile("ZEO Server (\w+)\((.*)\) \('(.*)', (\d+)")
+
+class StopParsing(Exception):
+    pass
+
+
+
+def usage(code, msg=''):
+    print __doc__
+    if msg:
+        print msg
+    sys.exit(code)
+
+
+
+def parse_time(line):
+    """Return the time portion of a zLOG line in seconds or None."""
+    mo = datecre.match(line)
+    if mo is None:
+        return None
+    date, time_ = mo.group(1, 2)
+    date_l = [int(elt) for elt in date.split('-')]
+    time_l = [int(elt) for elt in time_.split(':')]
+    return int(time.mktime(date_l + time_l + [0, 0, 0]))
+
+
+def parse_line(line):
+    """Parse a log entry and return time, method info, and client."""
+    t = parse_time(line)
+    if t is None:
+        return None, None, None
+    mo = methcre.search(line)
+    if mo is None:
+        return None, None, None
+    meth_name = mo.group(1)
+    meth_args = mo.group(2)
+    meth_args = [s.strip() for s in meth_args.split(',')]
+    m = meth_name, tuple(meth_args)
+    c = mo.group(3), mo.group(4)
+    return t, m, c
+
+
+
+class StoreStat:
+    def __init__(self, when, oid, size):
+        self.when = when
+        self.oid = oid
+        self.size = size
+
+    # Crufty
+    def __getitem__(self, i):
+        if i == 0: return self.oid
+        if i == 1: return self.size
+        raise IndexError
+
+
+class TxnStat:
+    def __init__(self):
+        self._begintime = None
+        self._finishtime = None
+        self._aborttime = None
+        self._url = None
+        self._objects = []
+
+    def tpc_begin(self, when, args, client):
+        self._begintime = when
+        # args are txnid, user, description (looks like it's always a url)
+        self._url = args[2]
+
+    def storea(self, when, args, client):
+        oid = int(args[0])
+        # args[1] is "[numbytes]"
+        size = int(args[1][1:-1])
+        s = StoreStat(when, oid, size)
+        self._objects.append(s)
+
+    def tpc_abort(self, when):
+        self._aborttime = when
+
+    def tpc_finish(self, when):
+        self._finishtime = when
+
+
+
+# Mapping oid -> revid
+_revids = {}
+
+class ReplayTxn(TxnStat):
+    def __init__(self, storage):
+        self._storage = storage
+        self._replaydelta = 0
+        TxnStat.__init__(self)
+
+    def replay(self):
+        ZERO = '\0'*8
+        t0 = now()
+        t = Transaction()
+        self._storage.tpc_begin(t)
+        for obj in self._objects:
+            oid = obj.oid
+            revid = _revids.get(oid, ZERO)
+            # BAW: simulate a pickle of the given size
+            data = 'x' * obj.size
+            # BAW: ignore versions for now
+            newrevid  = self._storage.store(p64(oid), revid, data, '', t)
+            _revids[oid] = newrevid
+        if self._aborttime:
+            self._storage.tpc_abort(t)
+            origdelta = self._aborttime - self._begintime
+        else:
+            self._storage.tpc_vote(t)
+            self._storage.tpc_finish(t)
+            origdelta = self._finishtime - self._begintime
+        t1 = now()
+        # Shows how many seconds behind (positive) or ahead (negative) of the
+        # original reply our local update took
+        self._replaydelta = t1 - t0 - origdelta
+
+
+
+class ZEOParser:
+    def __init__(self, maxtxns=-1, report=1, storage=None):
+        self.__txns = []
+        self.__curtxn = {}
+        self.__skipped = 0
+        self.__maxtxns = maxtxns
+        self.__finishedtxns = 0
+        self.__report = report
+        self.__storage = storage
+
+    def parse(self, line):
+        t, m, c = parse_line(line)
+        if t is None:
+            # Skip this line
+            return
+        name = m[0]
+        meth = getattr(self, name, None)
+        if meth is not None:
+            meth(t, m[1], c)
+
+    def tpc_begin(self, when, args, client):
+        txn = ReplayTxn(self.__storage)
+        self.__curtxn[client] = txn
+        meth = getattr(txn, 'tpc_begin', None)
+        if meth is not None:
+            meth(when, args, client)
+
+    def storea(self, when, args, client):
+        txn = self.__curtxn.get(client)
+        if txn is None:
+            self.__skipped += 1
+            return
+        meth = getattr(txn, 'storea', None)
+        if meth is not None:
+            meth(when, args, client)
+
+    def tpc_finish(self, when, args, client):
+        txn = self.__curtxn.get(client)
+        if txn is None:
+            self.__skipped += 1
+            return
+        meth = getattr(txn, 'tpc_finish', None)
+        if meth is not None:
+            meth(when)
+        if self.__report:
+            self.report(txn)
+        self.__txns.append(txn)
+        self.__curtxn[client] = None
+        self.__finishedtxns += 1
+        if self.__maxtxns > 0 and self.__finishedtxns >= self.__maxtxns:
+            raise StopParsing
+
+    def report(self, txn):
+        """Print a report about the transaction"""
+        if txn._objects:
+            bytes = reduce(operator.add, [size for oid, size in txn._objects])
+        else:
+            bytes = 0
+        print '%s %s %4d %10d %s %s' % (
+            txn._begintime, txn._finishtime - txn._begintime,
+            len(txn._objects),
+            bytes,
+            time.ctime(txn._begintime),
+            txn._url)
+
+    def replay(self):
+        for txn in self.__txns:
+            txn.replay()
+        # How many fell behind?
+        slower = []
+        faster = []
+        for txn in self.__txns:
+            if txn._replaydelta > 0:
+                slower.append(txn)
+            else:
+                faster.append(txn)
+        print len(slower), 'laggards,', len(faster), 'on-time or faster'
+        # Find some averages
+        if slower:
+            sum = reduce(operator.add,
+                         [txn._replaydelta for txn in slower], 0)
+            print 'average slower txn was:', float(sum) / len(slower)
+        if faster:
+            sum = reduce(operator.add,
+                         [txn._replaydelta for txn in faster], 0)
+            print 'average faster txn was:', float(sum) / len(faster)
+
+
+
+def main():
+    try:
+        opts, args = getopt.getopt(
+            sys.argv[1:],
+            'hr:pm:',
+            ['help', 'replay=', 'report', 'maxtxns='])
+    except getopt.error, e:
+        usage(1, e)
+
+    if args:
+        usage(1)
+
+    replay = 0
+    maxtxns = -1
+    report = 0
+    storagefile = None
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            usage(0)
+        elif opt in ('-r', '--replay'):
+            replay = 1
+            storagefile = arg
+        elif opt in ('-p', '--report'):
+            report = 1
+        elif opt in ('-m', '--maxtxns'):
+            try:
+                maxtxns = int(arg)
+            except ValueError:
+                usage(1, 'Bad -m argument: %s' % arg)
+
+    if replay:
+        storage = FileStorage(storagefile)
+        #storage = BDBFullStorage(storagefile)
+        #storage = PrimaryStorage('yyz', storage, RS_PORT)
+    t0 = now()
+    p = ZEOParser(maxtxns, report, storage)
+    i = 0
+    while 1:
+        line = sys.stdin.readline()
+        if not line:
+            break
+        i += 1
+        try:
+            p.parse(line)
+        except StopParsing:
+            break
+        except:
+            print 'input file line:', i
+            raise
+    t1 = now()
+    print 'total parse time:', t1-t0
+    t2 = now()
+    if replay:
+        p.replay()
+    t3 = now()
+    print 'total replay time:', t3-t2
+    print 'total time:', t3-t0
+
+
+
+if __name__ == '__main__':
+    main()

utilities/ZODBTools/zeoup.py

+#!/usr/bin/env python2.3
+
+"""Make sure a ZEO server is running.
+
+usage: zeoup.py [options]
+
+The test will connect to a ZEO server, load the root object, and attempt to
+update the zeoup counter in the root.  It will report success if it updates
+the counter or if it gets a ConflictError.  A ConflictError is considered a
+success, because the client was able to start a transaction.
+
+Options:
+
+    -p port -- port to connect to
+
+    -h host -- host to connect to (default is current host)
+
+    -S storage -- storage name (default '1')
+
+    -U path -- Unix-domain socket to connect to
+
+    --nowrite -- Do not update the zeoup counter.
+
+    -1 -- Connect to a ZEO 1.0 server.
+
+You must specify either -p and -h or -U.
+"""
+
+import getopt
+import socket
+import sys
+import time
+
+from persistent.mapping import PersistentMapping
+import transaction
+
+import ZODB
+from ZODB.POSException import ConflictError
+from ZODB.tests.MinPO import MinPO
+from ZEO.ClientStorage import ClientStorage
+
+ZEO_VERSION = 2
+
+def check_server(addr, storage, write):
+    t0 = time.time()
+    if ZEO_VERSION == 2:
+        # XXX should do retries w/ exponential backoff
+        cs = ClientStorage(addr, storage=storage, wait=0,
+                           read_only=(not write))
+    else:
+        cs = ClientStorage(addr, storage=storage, debug=1,
+                           wait_for_server_on_startup=1)
+    # _startup() is an artifact of the way ZEO 1.0 works.  The
+    # ClientStorage doesn't get fully initialized until registerDB()
+    # is called.  The only thing we care about, though, is that
+    # registerDB() calls _startup().
+
+    if write:
+        db = ZODB.DB(cs)
+        cn = db.open()
+        root = cn.root()
+        try:
+            # We store the data in a special `monitor' dict under the root,
+            # where other tools may also store such heartbeat and bookkeeping
+            # type data.
+            monitor = root.get('monitor')
+            if monitor is None:
+                monitor = root['monitor'] = PersistentMapping()
+            obj = monitor['zeoup'] = monitor.get('zeoup', MinPO(0))
+            obj.value += 1
+            transaction.commit()
+        except ConflictError:
+            pass
+        cn.close()
+        db.close()
+    else:
+        data, serial = cs.load("\0\0\0\0\0\0\0\0", "")
+        cs.close()
+    t1 = time.time()
+    print "Elapsed time: %.2f" % (t1 - t0)
+
+def usage(exit=1):
+    print __doc__
+    print " ".join(sys.argv)
+    sys.exit(exit)
+
+def main():
+    host = None
+    port = None
+    unix = None
+    write = 1
+    storage = '1'
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'p:h:U:S:1',
+                                   ['nowrite'])
+        for o, a in opts:
+            if o == '-p':
+                port = int(a)
+            elif o == '-h':
+                host = a
+            elif o == '-U':
+                unix = a
+            elif o == '-S':
+                storage = a
+            elif o == '--nowrite':
+                write = 0
+            elif o == '-1':
+                ZEO_VERSION = 1
+    except Exception, err:
+        s = str(err)
+        if s:
+            s = ": " + s
+        print err.__class__.__name__ + s
+        usage()
+
+    if unix is not None:
+        addr = unix
+    else:
+        if host is None:
+            host = socket.gethostname()
+        if port is None:
+            usage()
+        addr = host, port
+
+    check_server(addr, storage, write)
+
+if __name__ == "__main__":
+    try:
+        main()
+    except SystemExit:
+        raise
+    except Exception, err:
+        s = str(err)
+        if s:
+            s = ": " + s
+        print err.__class__.__name__ + s
+        sys.exit(1)