Commit ddf79d34 authored by Dieter Maurer's avatar Dieter Maurer Committed by GitHub

Merge pull request #355 from zopefoundation/fsdump_fsstats#354

fsdump/fsstats improvements
parents 1fb097b4 60b62fc0
......@@ -5,6 +5,12 @@
5.6.1 (unreleased)
==================
- Readd transaction size information to ``fsdump`` output;
adapt `fsstats` to ``fsdump``'s exchanged order for ``size`` and ``class``
information in data records;
(fixes `#354 <https://github.com/zopefoundation/ZODB/issues/354>_).
Make ``fsdump`` callable via Python's ``-m`` command line option.
- Fix UnboundLocalError when running fsoids.py script.
See `issue 285 <https://github.com/zopefoundation/ZODB/issues/285>`_.
......
......@@ -23,12 +23,14 @@ from ZODB.utils import u64, get_pickle_metadata
def fsdump(path, file=None, with_offset=1):
iter = FileIterator(path)
for i, trans in enumerate(iter):
size = trans._tend - trans._tpos
if with_offset:
print(("Trans #%05d tid=%016x time=%s offset=%d" %
(i, u64(trans.tid), TimeStamp(trans.tid), trans._pos)), file=file)
print(("Trans #%05d tid=%016x size=%d time=%s offset=%d" %
(i, u64(trans.tid), size,
TimeStamp(trans.tid), trans._pos)), file=file)
else:
print(("Trans #%05d tid=%016x time=%s" %
(i, u64(trans.tid), TimeStamp(trans.tid))), file=file)
print(("Trans #%05d tid=%016x size=%d time=%s" %
(i, u64(trans.tid), size, TimeStamp(trans.tid))), file=file)
print((" status=%r user=%r description=%r" %
(trans.status, trans.user, trans.description)), file=file)
......@@ -122,3 +124,7 @@ class Dumper(object):
def main():
import sys
fsdump(sys.argv[1])
if __name__ == "__main__":
main()
......@@ -7,7 +7,7 @@ import six
from six.moves import filter
rx_txn = re.compile("tid=([0-9a-f]+).*size=(\d+)")
rx_data = re.compile("oid=([0-9a-f]+) class=(\S+) size=(\d+)")
rx_data = re.compile("oid=([0-9a-f]+) size=(\d+) class=(\S+)")
def sort_byhsize(seq, reverse=False):
L = [(v.size(), k, v) for k, v in seq]
......@@ -31,8 +31,7 @@ class Histogram(dict):
def median(self):
# close enough?
n = self.size() / 2
L = self.keys()
L.sort()
L = sorted(self.keys())
L.reverse()
while 1:
k = L.pop()
......@@ -50,11 +49,14 @@ class Histogram(dict):
return mode
def make_bins(self, binsize):
try:
maxkey = max(six.iterkeys(self))
except ValueError:
maxkey = 0
self.binsize = binsize
self.bins = [0] * (1 + maxkey / binsize)
self.bins = [0] * (1 + maxkey // binsize)
for k, v in six.iteritems(self):
b = k / binsize
b = k // binsize
self.bins[b] += v
def report(self, name, binsize=50, usebins=False, gaps=True, skip=True):
......@@ -88,7 +90,7 @@ class Histogram(dict):
cum += n
pc = 100 * cum / tot
print("%6d %6d %3d%% %3d%% %s" % (
i * binsize, n, p, pc, "*" * (n / dot)))
i * binsize, n, p, pc, "*" * (n // dot)))
print()
def class_detail(class_size):
......@@ -104,7 +106,7 @@ def class_detail(class_size):
# per class details
for klass, h in sort_byhsize(six.iteritems(class_size), reverse=True):
h.make_bins(50)
if len(filter(None, h.bins)) == 1:
if len(tuple(filter(None, h.bins))) == 1:
continue
h.report("Object size for %s" % klass, usebins=True)
......@@ -138,7 +140,7 @@ def main(path=None):
objects = 0
tid = None
f = open(path, "rb")
f = open(path, "r")
for i, line in enumerate(f):
if MAX and i > MAX:
break
......@@ -146,7 +148,7 @@ def main(path=None):
m = rx_data.search(line)
if not m:
continue
oid, klass, size = m.groups()
oid, size, klass = m.groups()
size = int(size)
obj_size.add(size)
......@@ -178,6 +180,8 @@ def main(path=None):
objects = 0
txn_bytes.add(size)
if objects:
txn_objects.add(objects)
f.close()
print("Summary: %d txns, %d objects, %d revisions" % (
......
##############################################################################
#
# Copyright (c) 2021 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
from ZODB import DB
from ZODB.scripts.fsstats import rx_data
from ZODB.scripts.fsstats import rx_txn
from ZODB.tests.util import TestCase
from ZODB.tests.util import run_module_as_script
class FsdumpFsstatsTests(TestCase):
def setUp(self):
super(FsdumpFsstatsTests, self).setUp()
# create (empty) storage ``data.fs``
DB("data.fs").close()
def test_fsdump(self):
run_module_as_script("ZODB.FileStorage.fsdump", ["data.fs"])
# verify that ``fsstats`` will understand the output
with open("stdout") as f:
tno = obno = 0
for li in f:
if li.startswith(" data"):
m = rx_data.search(li)
if m is None:
continue
oid, size, klass = m.groups()
int(size)
obno += 1
elif li.startswith("Trans"):
m = rx_txn.search(li)
if not m:
continue
tid, size = m.groups()
size = int(size)
tno += 1
self.assertEqual(tno, 1)
self.assertEqual(obno, 1)
def test_fsstats(self):
# The ``fsstats`` output is complex
# currently, we just check the first (summary) line
run_module_as_script("ZODB.FileStorage.fsdump", ["data.fs"],
"data.dmp")
run_module_as_script("ZODB.scripts.fsstats", ["data.dmp"])
with open("stdout") as f:
self.assertEqual(f.readline().strip(),
"Summary: 1 txns, 1 objects, 1 revisions")
......@@ -16,9 +16,12 @@
from ZODB.MappingStorage import DB
import atexit
import doctest
import os
import pdb
import persistent
import re
import runpy
import sys
import tempfile
import time
......@@ -377,3 +380,28 @@ def with_high_concurrency(f):
restore()
return _
def run_module_as_script(mod, args, stdout="stdout", stderr="stderr"):
"""run module *mod* as script with arguments *arg*.
stdout and stderr are redirected to files given by the
correcponding parameters.
The function is usually called in a ``setUp/tearDown`` frame
which will remove the created files.
"""
sargv, sout, serr = sys.argv, sys.stdout, sys.stderr
s_set_trace = pdb.set_trace
try:
sys.argv = [sargv[0]] + args
sys.stdout = open(stdout, "w")
sys.stderr = open(stderr, "w")
# to allow debugging
pdb.set_trace = doctest._OutputRedirectingPdb(sout)
runpy.run_module(mod, run_name="__main__", alter_sys=True)
finally:
sys.stdout.close()
sys.stderr.close()
pdb.set_trace = s_set_trace
sys.argv, sys.stdout, sys.stderr = sargv, sout, serr
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment