Skip to content

Z
ZEO
Commit 934650ec authored by Jim Fulton's avatar Jim Fulton Committed by GitHub
Browse files
Options

Merge pull request #46 from NextThought/argparse-cache-utils 

Convert cache_stats and cache_simul from getopt to argparse.
parents d06e32a8 834ed7d4
Hide whitespace changes
Inline Side-by-side
Showing with 103 additions and 173 deletions
src/ZEO/scripts/cache_simul.py
View file @ 934650ec
......@@ -12,14 +12,9 @@
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Cache simulation.
Usage: simul.py [-s size] tracefile
"""
Cache simulation.
Options:
-s size: cache size in MB (default 20 MB)
-i: summarizing interval in minutes (default 15; max 60)
-r: rearrange factor
Note:
......@@ -27,102 +22,52 @@ Note:
- The simulation will be far off if the trace file
was created starting with a non-empty cache
"""
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function, absolute_import
import bisect
import getopt
import struct
import re
import sys
import ZEO.cache
import argparse
from ZODB.utils import z64
from ZODB.utils import z64, u64
from .cache_stats import add_interval_argument
from .cache_stats import add_tracefile_argument
# we assign ctime locally to facilitate test replacement!
from time import ctime
import six
def usage(msg):
print(msg, file=sys.stderr)
print(__doc__, file=sys.stderr)
def main(args=None):
if args is None:
args = sys.argv[1:]
# Parse options.
MB = 1<<20
cachelimit = 20*MB
rearrange = 0.8
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--size", "-s",
default=20*MB, dest="cachelimit",
type=lambda s: int(float(s)*MB),
help="cache size in MB (default 20MB)")
add_interval_argument(parser)
parser.add_argument("--rearrange", "-r",
default=0.8, type=float,
help="rearrange factor")
add_tracefile_argument(parser)
simclass = CircularCacheSimulation
interval_step = 15
try:
opts, args = getopt.getopt(args, "s:i:r:")
except getopt.error as msg:
usage(msg)
return 2
for o, a in opts:
if o == '-s':
cachelimit = int(float(a)*MB)
elif o == '-i':
interval_step = int(a)
elif o == '-r':
rearrange = float(a)
else:
assert False, (o, a)
interval_step *= 60
if interval_step <= 0:
interval_step = 60
elif interval_step > 3600:
interval_step = 3600
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file.
if filename.endswith(".gz"):
# Open gzipped file.
try:
import gzip
except ImportError:
print("can't read gzipped files (no module gzip)", file=sys.stderr)
return 1
try:
f = gzip.open(filename, "rb")
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
elif filename == "-":
# Read from stdin.
f = sys.stdin
else:
# Open regular file.
try:
f = open(filename, "rb")
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
options = parser.parse_args(args)
f = options.tracefile
interval_step = options.interval
# Create simulation object.
sim = simclass(cachelimit, rearrange)
interval_sim = simclass(cachelimit, rearrange)
sim = simclass(options.cachelimit, options.rearrange)
interval_sim = simclass(options.cachelimit, options.rearrange)
# Print output header.
sim.printheader()
......
src/ZEO/scripts/cache_stats.py
View file @ 934650ec
......@@ -14,18 +14,7 @@ from __future__ import print_function
##############################################################################
"""Trace file statistics analyzer.
Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] [-X] tracefile
-h: print histogram of object load frequencies
-i: summarizing interval in minutes (default 15; max 60)
-q: quiet; don't print summaries
-s: print histogram of object sizes
-S: don't print statistics
-v: verbose; print each record
-X: enable heuristic checking for misaligned records: oids > 2**32
will be rejected; this requires the tracefile to be seekable
"""
"""File format:
File format:
Each record is 26 bytes, plus a variable number of bytes to store an oid,
with the following layout. Numbers are big-endian integers.
......@@ -58,84 +47,80 @@ i.e. the low bit is always zero.
"""
import sys
import time
import getopt
import argparse
import struct
import gzip
# we assign ctime locally to facilitate test replacement!
from time import ctime
import six
def usage(msg):
print(msg, file=sys.stderr)
print(__doc__, file=sys.stderr)
def add_interval_argument(parser):
def _interval(a):
interval = int(60 * float(a))
if interval <= 0:
interval = 60
elif interval > 3600:
interval = 3600
return interval
parser.add_argument("--interval", "-i",
default=15*60, type=_interval,
help="summarizing interval in minutes (default 15; max 60)")
def add_tracefile_argument(parser):
class GzipFileType(argparse.FileType):
def __init__(self):
super(GzipFileType, self).__init__(mode='rb')
def __call__(self, s):
f = super(GzipFileType, self).__call__(s)
if s.endswith(".gz"):
f = gzip.GzipFile(filename=s, fileobj=f)
return f
parser.add_argument("tracefile", type=GzipFileType(),
help="The trace to read; may be gzipped")
def main(args=None):
if args is None:
args = sys.argv[1:]
# Parse options
verbose = False
quiet = False
dostats = True
print_size_histogram = False
print_histogram = False
interval = 15*60 # Every 15 minutes
heuristic = False
try:
opts, args = getopt.getopt(args, "hi:qsSvX")
except getopt.error as msg:
usage(msg)
return 2
for o, a in opts:
if o == '-h':
print_histogram = True
elif o == "-i":
interval = int(60 * float(a))
if interval <= 0:
interval = 60
elif interval > 3600:
interval = 3600
elif o == "-q":
quiet = True
verbose = False
elif o == "-s":
print_size_histogram = True
elif o == "-S":
dostats = False
elif o == "-v":
verbose = True
elif o == '-X':
heuristic = True
else:
assert False, (o, opts)
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file
if filename.endswith(".gz"):
# Open gzipped file
try:
import gzip
except ImportError:
print("can't read gzipped files (no module gzip)", file=sys.stderr)
return 1
try:
f = gzip.open(filename, "rb")
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
elif filename == '-':
# Read from stdin
f = sys.stdin
else:
# Open regular file
try:
f = open(filename, "rb")
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
parser = argparse.ArgumentParser(description="Trace file statistics analyzer",
# Our -h, short for --load-histogram
# conflicts with default for help, so we handle
# manually.
add_help=False)
verbose_group = parser.add_mutually_exclusive_group()
verbose_group.add_argument('--verbose', '-v',
default=False, action='store_true',
help="Be verbose; print each record")
verbose_group.add_argument('--quiet', '-q',
default=False, action='store_true',
help="Reduce output; don't print summaries")
parser.add_argument("--sizes", '-s',
default=False, action="store_true", dest="print_size_histogram",
help="print histogram of object sizes")
parser.add_argument("--no-stats", '-S',
default=True, action="store_false", dest="dostats",
help="don't print statistics")
parser.add_argument("--load-histogram", "-h",
default=False, action="store_true", dest="print_histogram",
help="print histogram of object load frequencies")
parser.add_argument("--check", "-X",
default=False, action="store_true", dest="heuristic",
help=" enable heuristic checking for misaligned records: oids > 2**32"
" will be rejected; this requires the tracefile to be seekable")
add_interval_argument(parser)
add_tracefile_argument(parser)
if '--help' in args:
parser.print_help()
sys.exit(2)
options = parser.parse_args(args)
f = options.tracefile
rt0 = time.time()
bycode = {} # map code to count of occurrences
......@@ -169,7 +154,7 @@ def main(args=None):
ts, code, oidlen, start_tid, end_tid = unpack(FMT, r)
if ts == 0:
# Must be a misaligned record caused by a crash.
if not quiet:
if not options.quiet:
print("Skipping 8 bytes at offset", f.tell() - FMT_SIZE)
f.seek(f.tell() - FMT_SIZE + 8)
continue
......@@ -179,14 +164,14 @@ def main(args=None):
records += 1
if t0 is None:
t0 = ts
thisinterval = t0 // interval
thisinterval = t0 // options.interval
h0 = he = ts
te = ts
if ts // interval != thisinterval:
if not quiet:
if ts // options.interval != thisinterval:
if not options.quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts // interval
thisinterval = ts // options.interval
h0 = ts
he = ts
dlen, code = (code & 0x7fffff00) >> 8, code & 0xff
......@@ -208,7 +193,7 @@ def main(args=None):
elif code & 0x70 == 0x50: # All stores
bysizew[dlen] = d = bysizew.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1
if verbose:
if options.verbose:
print("%s %02x %s %016x %016x %c%s" % (
ctime(ts)[4:-5],
code,
......@@ -221,12 +206,12 @@ def main(args=None):
oids[oid] = oids.get(oid, 0) + 1
total_loads += 1
elif code == 0x00: # restart
if not quiet:
if not options.quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts // interval
thisinterval = ts // options.interval
h0 = he = ts
if not quiet:
if not options.quiet:
print(ctime(ts)[4:-5], end=' ')
print('='*20, "Restart", '='*20)
except KeyboardInterrupt:
......@@ -235,7 +220,7 @@ def main(args=None):
end_pos = f.tell()
f.close()
rte = time.time()
if not quiet:
if not options.quiet:
dumpbyinterval(byinterval, h0, he)
# Error if nothing was read
......@@ -244,7 +229,7 @@ def main(args=None):
return 1
# Print statistics
if dostats:
if options.dostats:
print()
print("Read %s trace records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(end_pos), rte-rt0))
......@@ -267,7 +252,7 @@ def main(args=None):
explain.get(code) or "*** unknown code ***"))
# Print histogram.
if print_histogram:
if options.print_histogram:
print()
print("Histogram of object load frequency")
total = len(oids)
......@@ -287,7 +272,7 @@ def main(args=None):
obj_percent, load_percent, cum))
# Print size histogram.
if print_size_histogram:
if options.print_size_histogram:
print()
print("Histograms of object sizes")
print()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7