Commit ddf79d34 authored by Dieter Maurer's avatar Dieter Maurer Committed by GitHub

Merge pull request #355 from zopefoundation/fsdump_fsstats#354

fsdump/fsstats improvements
parents 1fb097b4 60b62fc0
...@@ -5,6 +5,12 @@ ...@@ -5,6 +5,12 @@
5.6.1 (unreleased) 5.6.1 (unreleased)
================== ==================
- Readd transaction size information to ``fsdump`` output;
adapt `fsstats` to ``fsdump``'s exchanged order for ``size`` and ``class``
information in data records;
(fixes `#354 <https://github.com/zopefoundation/ZODB/issues/354>_).
Make ``fsdump`` callable via Python's ``-m`` command line option.
- Fix UnboundLocalError when running fsoids.py script. - Fix UnboundLocalError when running fsoids.py script.
See `issue 285 <https://github.com/zopefoundation/ZODB/issues/285>`_. See `issue 285 <https://github.com/zopefoundation/ZODB/issues/285>`_.
......
...@@ -23,12 +23,14 @@ from ZODB.utils import u64, get_pickle_metadata ...@@ -23,12 +23,14 @@ from ZODB.utils import u64, get_pickle_metadata
def fsdump(path, file=None, with_offset=1): def fsdump(path, file=None, with_offset=1):
iter = FileIterator(path) iter = FileIterator(path)
for i, trans in enumerate(iter): for i, trans in enumerate(iter):
size = trans._tend - trans._tpos
if with_offset: if with_offset:
print(("Trans #%05d tid=%016x time=%s offset=%d" % print(("Trans #%05d tid=%016x size=%d time=%s offset=%d" %
(i, u64(trans.tid), TimeStamp(trans.tid), trans._pos)), file=file) (i, u64(trans.tid), size,
TimeStamp(trans.tid), trans._pos)), file=file)
else: else:
print(("Trans #%05d tid=%016x time=%s" % print(("Trans #%05d tid=%016x size=%d time=%s" %
(i, u64(trans.tid), TimeStamp(trans.tid))), file=file) (i, u64(trans.tid), size, TimeStamp(trans.tid))), file=file)
print((" status=%r user=%r description=%r" % print((" status=%r user=%r description=%r" %
(trans.status, trans.user, trans.description)), file=file) (trans.status, trans.user, trans.description)), file=file)
...@@ -122,3 +124,7 @@ class Dumper(object): ...@@ -122,3 +124,7 @@ class Dumper(object):
def main(): def main():
import sys import sys
fsdump(sys.argv[1]) fsdump(sys.argv[1])
if __name__ == "__main__":
main()
...@@ -7,7 +7,7 @@ import six ...@@ -7,7 +7,7 @@ import six
from six.moves import filter from six.moves import filter
rx_txn = re.compile("tid=([0-9a-f]+).*size=(\d+)") rx_txn = re.compile("tid=([0-9a-f]+).*size=(\d+)")
rx_data = re.compile("oid=([0-9a-f]+) class=(\S+) size=(\d+)") rx_data = re.compile("oid=([0-9a-f]+) size=(\d+) class=(\S+)")
def sort_byhsize(seq, reverse=False): def sort_byhsize(seq, reverse=False):
L = [(v.size(), k, v) for k, v in seq] L = [(v.size(), k, v) for k, v in seq]
...@@ -31,8 +31,7 @@ class Histogram(dict): ...@@ -31,8 +31,7 @@ class Histogram(dict):
def median(self): def median(self):
# close enough? # close enough?
n = self.size() / 2 n = self.size() / 2
L = self.keys() L = sorted(self.keys())
L.sort()
L.reverse() L.reverse()
while 1: while 1:
k = L.pop() k = L.pop()
...@@ -50,11 +49,14 @@ class Histogram(dict): ...@@ -50,11 +49,14 @@ class Histogram(dict):
return mode return mode
def make_bins(self, binsize): def make_bins(self, binsize):
try:
maxkey = max(six.iterkeys(self)) maxkey = max(six.iterkeys(self))
except ValueError:
maxkey = 0
self.binsize = binsize self.binsize = binsize
self.bins = [0] * (1 + maxkey / binsize) self.bins = [0] * (1 + maxkey // binsize)
for k, v in six.iteritems(self): for k, v in six.iteritems(self):
b = k / binsize b = k // binsize
self.bins[b] += v self.bins[b] += v
def report(self, name, binsize=50, usebins=False, gaps=True, skip=True): def report(self, name, binsize=50, usebins=False, gaps=True, skip=True):
...@@ -88,7 +90,7 @@ class Histogram(dict): ...@@ -88,7 +90,7 @@ class Histogram(dict):
cum += n cum += n
pc = 100 * cum / tot pc = 100 * cum / tot
print("%6d %6d %3d%% %3d%% %s" % ( print("%6d %6d %3d%% %3d%% %s" % (
i * binsize, n, p, pc, "*" * (n / dot))) i * binsize, n, p, pc, "*" * (n // dot)))
print() print()
def class_detail(class_size): def class_detail(class_size):
...@@ -104,7 +106,7 @@ def class_detail(class_size): ...@@ -104,7 +106,7 @@ def class_detail(class_size):
# per class details # per class details
for klass, h in sort_byhsize(six.iteritems(class_size), reverse=True): for klass, h in sort_byhsize(six.iteritems(class_size), reverse=True):
h.make_bins(50) h.make_bins(50)
if len(filter(None, h.bins)) == 1: if len(tuple(filter(None, h.bins))) == 1:
continue continue
h.report("Object size for %s" % klass, usebins=True) h.report("Object size for %s" % klass, usebins=True)
...@@ -138,7 +140,7 @@ def main(path=None): ...@@ -138,7 +140,7 @@ def main(path=None):
objects = 0 objects = 0
tid = None tid = None
f = open(path, "rb") f = open(path, "r")
for i, line in enumerate(f): for i, line in enumerate(f):
if MAX and i > MAX: if MAX and i > MAX:
break break
...@@ -146,7 +148,7 @@ def main(path=None): ...@@ -146,7 +148,7 @@ def main(path=None):
m = rx_data.search(line) m = rx_data.search(line)
if not m: if not m:
continue continue
oid, klass, size = m.groups() oid, size, klass = m.groups()
size = int(size) size = int(size)
obj_size.add(size) obj_size.add(size)
...@@ -178,6 +180,8 @@ def main(path=None): ...@@ -178,6 +180,8 @@ def main(path=None):
objects = 0 objects = 0
txn_bytes.add(size) txn_bytes.add(size)
if objects:
txn_objects.add(objects)
f.close() f.close()
print("Summary: %d txns, %d objects, %d revisions" % ( print("Summary: %d txns, %d objects, %d revisions" % (
......
##############################################################################
#
# Copyright (c) 2021 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
from ZODB import DB
from ZODB.scripts.fsstats import rx_data
from ZODB.scripts.fsstats import rx_txn
from ZODB.tests.util import TestCase
from ZODB.tests.util import run_module_as_script
class FsdumpFsstatsTests(TestCase):
def setUp(self):
super(FsdumpFsstatsTests, self).setUp()
# create (empty) storage ``data.fs``
DB("data.fs").close()
def test_fsdump(self):
run_module_as_script("ZODB.FileStorage.fsdump", ["data.fs"])
# verify that ``fsstats`` will understand the output
with open("stdout") as f:
tno = obno = 0
for li in f:
if li.startswith(" data"):
m = rx_data.search(li)
if m is None:
continue
oid, size, klass = m.groups()
int(size)
obno += 1
elif li.startswith("Trans"):
m = rx_txn.search(li)
if not m:
continue
tid, size = m.groups()
size = int(size)
tno += 1
self.assertEqual(tno, 1)
self.assertEqual(obno, 1)
def test_fsstats(self):
# The ``fsstats`` output is complex
# currently, we just check the first (summary) line
run_module_as_script("ZODB.FileStorage.fsdump", ["data.fs"],
"data.dmp")
run_module_as_script("ZODB.scripts.fsstats", ["data.dmp"])
with open("stdout") as f:
self.assertEqual(f.readline().strip(),
"Summary: 1 txns, 1 objects, 1 revisions")
...@@ -16,9 +16,12 @@ ...@@ -16,9 +16,12 @@
from ZODB.MappingStorage import DB from ZODB.MappingStorage import DB
import atexit import atexit
import doctest
import os import os
import pdb
import persistent import persistent
import re import re
import runpy
import sys import sys
import tempfile import tempfile
import time import time
...@@ -377,3 +380,28 @@ def with_high_concurrency(f): ...@@ -377,3 +380,28 @@ def with_high_concurrency(f):
restore() restore()
return _ return _
def run_module_as_script(mod, args, stdout="stdout", stderr="stderr"):
"""run module *mod* as script with arguments *arg*.
stdout and stderr are redirected to files given by the
correcponding parameters.
The function is usually called in a ``setUp/tearDown`` frame
which will remove the created files.
"""
sargv, sout, serr = sys.argv, sys.stdout, sys.stderr
s_set_trace = pdb.set_trace
try:
sys.argv = [sargv[0]] + args
sys.stdout = open(stdout, "w")
sys.stderr = open(stderr, "w")
# to allow debugging
pdb.set_trace = doctest._OutputRedirectingPdb(sout)
runpy.run_module(mod, run_name="__main__", alter_sys=True)
finally:
sys.stdout.close()
sys.stderr.close()
pdb.set_trace = s_set_trace
sys.argv, sys.stdout, sys.stderr = sargv, sout, serr
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment