Skip to content

Z
Zope
Commit c6e29c95 authored by Hanno Schlichting's avatar Hanno Schlichting
Browse files
Options

No need for top-level utilities anymore, they are inside the Zope2 package now

parent 2adae91c
Show whitespace changes
Inline Side-by-side
Showing with 0 additions and 2697 deletions
utilities/README.txt deleted 100644 → 0
View file @ 2adae91c
This directory contains utility scripts and modules that augment Zope.
To get detailed usage information, run any of these scripts without arguments:
load_site.py -- Load a Zope site from files and directories
This script illustrates used of the Zope RPC mechanism,
ZPublisher.Client. It provides some examples of pitfalls
and their work-arounds.
check_catalog.py -- Perform some consistency tests on a ZCatalog instance
mkzopeinstance.py -- create a Zope instance home
copyzopeskel.py -- copy a Zope instance home skeleton directory to target
mkzeoinstance.py -- create a ZEO instance home
requestprofiler.py -- parse and analyze the Zope "detailed" log file
zpasswd.py -- generate "access" or "inituser" files for use with Zope
compilezpy.py -- compile all .py files to .pyc files in the current
directory and below
decompilezpy.py -- remove all .py[co] files in the current directory
and below
utilities/check_catalog.py deleted 100644 → 0
View file @ 2adae91c
#!/usr/bin/env python2.4
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Script to check consistency of a ZCatalog
$Id$
"""
import Zope2
import os,sys,re,getopt
from types import IntType
from BTrees.IIBTree import IISet,difference,intersection
def checkCatalog(path,indexes):
""" perform some consistency checks on a ZCatalog instance"""
root = Zope2.app()
try:
catalog = root.unrestrictedTraverse(path)
except AttributeError:
print 'Error: catalog object not found'
sys.exit(1)
# get Catalog instance
_cat = catalog._catalog
# check Catalog internal BTrees
l_data = list(_cat.data.keys())
l_data.sort()
l_uids = list(_cat.uids.values())
l_uids.sort()
l_paths = list(_cat.data.keys())
l_paths.sort()
print "Checking catalog internal BTrees"
print "\tINFO: Mapping data: %d entries" % len(l_data)
print "\tINFO: Mapping uids: %d entries" % len(l_uids)
print "\tINFO: Mapping paths: %d entries" % len(l_paths)
if l_data == l_uids:
print "\tOK: Mapping data equals Mapping uids"
else:
print "\tERR: Mapping data does not equal Mapping uids"
if l_data == l_paths:
print "\tOK: Mapping data equals Maaping paths"
else:
print "\tERR: Mapping data does not equal Maaping paths"
# check BTrees of indexes
for id,idx in _cat.indexes.items():
if indexes and not idx.meta_type in indexes: continue
print "Checking index '%s' (type: %s)" % (id, idx.meta_type)
if idx.meta_type in ['FieldIndex','KeywordIndex']:
# check forward entries
RIDS = IISet()
for key, rids in idx._index.items():
if isinstance(rids,IntType):
RIDS.insert( rids )
else:
map(RIDS.insert , rids.keys())
diff = difference(RIDS, IISet(_cat.data.keys()))
if len(diff)!=0:
print '\tERR: Problem with forward entries'
print '\tERR: too much forward entries:', diff
else:
print '\tOK: Forward entries (%d entries)' % (len(RIDS))
elif idx.meta_type in ['PathIndex']:
RIDS = IISet()
for rids in map(None,idx._index.values()):
map(RIDS.insert , rids.values()[0])
diff = difference(RIDS, IISet(_cat.data.keys()))
if len(diff)!=0:
print '\tERR: Problem with forward entries'
print '\tERR: too much forward entries:', diff
else:
print '\tOK: Forward entries (%d entries)' % (len(RIDS))
if idx.meta_type in ['FieldIndex','KeywordIndex','PathIndex']:
# check backward entries
RIDS = IISet(idx._unindex.keys())
diff = difference(RIDS, IISet(_cat.data.keys()))
if len(diff)!=0:
print '\tERR: Problem with backward entries'
print '\tERR: too much backward entries:', diff
else:
print '\tOK: Backward entries (%d entries)' % (len(RIDS))
def usage():
print "Usage: %s [--FieldIndex|KeywordIndex|PathIndex] /path/to/ZCatalog" % \
os.path.basename(sys.argv[0])
print
print "This scripts checks the consistency of the internal"
print "BTrees of a ZCatalog and its indexes."
sys.exit(1)
def main():
opts,args = getopt.getopt(sys.argv[1:],'h',\
['help','FieldIndex','KeywordIndex','PathIndex'])
indexes = []
for o,v in opts:
if o in ['-h','--help']: usage()
if o in ['--FieldIndex','--KeywordIndex','--PathIndex']:
indexes.append(o[2:])
checkCatalog(args,indexes)
if __name__=='__main__':
main()
utilities/compilezpy.py deleted 100755 → 0
View file @ 2adae91c
#!/usr/bin/env python2.4
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import compileall, os, sys
class Shutup:
def write(*args): pass # :)
class NoteErr:
wrote = 0
def write(self, *args):
self.wrote = 1
apply(stderr.write, args)
def compile_non_test(dir):
"""Byte-compile all modules except those in test directories."""
success = compileall.compile_dir(dir, maxlevels=0)
try:
names = os.listdir(dir)
except os.error:
print "Can't list", dir
names = []
names.sort()
for name in names:
fullname = os.path.join(dir, name)
if (name != os.curdir and name != os.pardir and
os.path.isdir(fullname) and not os.path.islink(fullname) and
name != 'test' and name != 'tests' and name != 'skins'):
success = success and compile_non_test(fullname)
return success
print
print '-'*78
print 'Compiling python modules'
stdout = sys.stdout
stderr = sys.stderr
try:
try:
success = 0
sys.stdout = Shutup()
sys.stderr = NoteErr()
success = compile_non_test(os.getcwd())
finally:
success = success and not sys.stderr.wrote
sys.stdout = stdout
sys.stderr = stderr
except:
success = 0
import traceback
traceback.print_exc()
if not success:
print
print '!' * 78
print 'There were errors during Python module compilation.'
print '!' * 78
print
sys.exit(1)
utilities/copyzopeskel.py deleted 100755 → 0
View file @ 2adae91c
#!/usr/bin/env python2.4
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""%(program)s: Create a target directory from a skeleton directory.
usage: %(program)s [options]
Options:
-h/--help -- print this help text
-s/--sourcedir -- the skeleton source directory
-t/--targetdir -- the directory to which the skeleton files will be copied
-u/--uid -- the username/uid of the user who will own the target files
-g/--gid -- the groupname/gid of the group who will own the target files
-r/--replace -- specify replacement value for .in file
This script may be used to install a custom Zope instance home
skeleton directory. It is most useful when used to install a skeleton
which does not follow the standard 'one-directory-as-instance-home'
paradigm used by the stock Zope source install.
The value of --targetdir should be the directory where you would like to copy
the skeleton hierarchy. For many packagers, this will be "/" or "/usr"
or "/usr/local".
The value of --sourcedir should be a directory which contains a custom skeleton
hierarchy. For many packagers, the skeleton source directory may contain
directories like "usr" and "bin" and these directories will contain files
and other directories, comprising an installation hierarchy suitable for your
platform.
The skeleton source hierarchy may contain any kind of file. Files
in the skeleton hierarchy that end with a ".in" extension will go through
textual substitution before they are placed in the target directory. When
they are placed in the target directory, the ".in" extension is removed.
Specify textual replacement values by passing one or more --replace= options
to the script. The value of each replace option needs to be
in the following format: --replace=key:value. 'key' is the value that
will be replaced (minus the "<<" and ">>" values expected by the
replacement). 'value' is the value that should be used for replacement.
Files which do not have an ".in" extension are copied without substitution.
All file mode bits from files/dirs in the skeleton source directory are copied
along with the file/directory itself. If the --uid and/or --gid flags are
used, all directories and files created by this script will be provided with
this owner and/or group. Otherwise, the uid and group owner of the files will
be the executing user's. Existing directory structures and files are left
unchanged. If a file already exists in the target directory, it is left
unchanged and the source file is not copied. If a directory already exists in
the target directory, its ownership information and mode bit settings are left
unchanged.
"""
import os
import shutil
import sys
import getopt
def main():
try:
opts, args = getopt.getopt(sys.argv[1:],
"hs:t:u:g:r:",
["help", "sourcedir=", "targetdir=", "uid=", "gid=",
"replace="]
)
except getopt.GetoptError, msg:
usage(sys.stderr, msg)
sys.exit(2)
script = os.path.abspath(sys.argv[0])
sourcedir = None
targetdir = None
uid = None
gid = None
replacements = {}
for opt, arg in opts:
if opt in ("-h", "--help"):
usage(sys.stdout)
sys.exit()
if opt in ("-s", "--sourcedir"):
sourcedir = os.path.abspath(os.path.expanduser(arg))
if not sourcedir:
usage(sys.stderr, "sourcedir must not be empty")
sys.exit(2)
if opt in ("-t", "--targetdir"):
targetdir = os.path.abspath(os.path.expanduser(arg))
if not targetdir:
usage(sys.stderr, "targetdir must not be empty")
sys.exit(2)
if opt in ("-u", "--uid"):
if not arg:
usage(sys.stderr, "uid must not be empty")
sys.exit(2)
try:
if os.getuid() != 0:
usage(sys.stderr, "You must be root to specify a uid")
sys.exit(2)
try:
uid = int(arg)
except:
try:
import pwd
uid = pwd.getpwnam(arg)[2]
if not gid:
gid = pwd.getpwnam(arg)[3]
except KeyError:
usage(sys.stderr,
"The user indicated by uid does not exist on "
"your system")
sys.exit(2)
except (ImportError, AttributeError):
usage(sys.stderr,
"Your system does not support the gid or uid options")
sys.exit(2)
if opt in ("-g", "--gid"):
if not arg:
usage(sys.stderr, "gid must not be empty")
sys.exit(2)
try:
if os.getuid() != 0:
usage(sys.stderr, "You must be root to specify a gid")
sys.exit(2)
try:
gid = int(arg)
except:
try:
import pwd
gid = pwd.getpwnam(arg)[3]
except KeyError:
usage(sys.stderr,
"The user indicated by gid does not exist on "
"your system")
sys.exit(2)
except (ImportError, AttributeError):
usage(sys.stderr,
"Your system does not support the gid or uid options")
sys.exit(2)
if opt in ("-r", "--replace"):
if not arg:
continue
k, v = arg.split(':', 1)
replacements[k] = v
if not sourcedir:
usage(sys.stderr, "Must specify sourcedir")
sys.exit(2)
if not targetdir:
usage(sys.stderr, "Must specify targetdir")
sys.exit(2)
copyskel(sourcedir, targetdir, uid, gid, **replacements)
def copyskel(sourcedir, targetdir, uid, gid, **replacements):
""" This is an independent function because we'd like to
import and call it from mkzopeinstance """
# Create the top of the instance:
if not os.path.exists(targetdir):
os.makedirs(targetdir)
# This is fairly ugly. The chdir() makes path manipulation in the
# walk() callback a little easier (less magical), so we'll live
# with it.
pwd = os.getcwd()
os.chdir(sourcedir)
try:
try:
os.path.walk(os.curdir, copydir,
(targetdir, replacements, uid, gid))
finally:
os.chdir(pwd)
except (IOError, OSError), msg:
print >>sys.stderr, msg
sys.exit(1)
CVS_DIRS = [os.path.normcase("CVS"), os.path.normcase(".svn")]
def copydir((targetdir, replacements, uid, gid), sourcedir, names):
# Don't recurse into CVS directories:
for name in names[:]:
if os.path.normcase(name) in CVS_DIRS:
names.remove(name)
elif os.path.isfile(os.path.join(sourcedir, name)):
# Copy the file:
sn, ext = os.path.splitext(name)
if os.path.normcase(ext) == ".in":
dst = os.path.join(targetdir, sourcedir, sn)
if os.path.exists(dst):
continue
copyin(os.path.join(sourcedir, name), dst, replacements, uid,
gid)
if uid is not None:
os.chown(dst, uid, gid)
else:
src = os.path.join(sourcedir, name)
dst = os.path.join(targetdir, src)
if os.path.exists(dst):
continue
shutil.copyfile(src, dst)
shutil.copymode(src, dst)
if uid is not None:
os.chown(dst, uid, gid)
else:
# Directory:
dn = os.path.join(targetdir, sourcedir, name)
if not os.path.exists(dn):
os.mkdir(dn)
shutil.copymode(os.path.join(sourcedir, name), dn)
if uid is not None:
os.chown(dn, uid, gid)
def copyin(src, dst, replacements, uid, gid):
ifp = open(src)
text = ifp.read()
ifp.close()
for k in replacements:
text = text.replace("<<%s>>" % k, replacements[k])
ofp = open(dst, "w")
ofp.write(text)
ofp.close()
shutil.copymode(src, dst)
if uid is not None:
os.chown(dst, uid, gid)
def usage(stream, msg=None):
if msg:
print >>stream, msg
print >>stream
program = os.path.basename(sys.argv[0])
print >>stream, __doc__ % {"program": program}
if __name__ == '__main__':
main()
utilities/decompilezpy.py deleted 100755 → 0
View file @ 2adae91c
#!/usr/bin/env python2.4
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import os
import sys
def main(dirname):
os.path.walk(dirname, rmpycs, None)
def rmpycs(arg, dirname, names):
for name in names:
path = os.path.join(dirname, name)
if ( name.endswith('.pyc') or name.endswith('.pyo') and
os.path.isfile(path) ):
os.unlink(path)
if __name__ == '__main__':
main(sys.argv[1])
utilities/fixheaders.py deleted 100644 → 0
View file @ 2adae91c
# Script to fix the header files to ZPL 2.1
import os
for dirpath, dirnames, filenames in os.walk('.'):
for fname in filenames:
base,ext = os.path.splitext(fname)
if not ext in ('.py', '.c', '.h'): continue
fullname = os.path.join(dirpath, fname)
if '.svn' in fullname: continue
data = open(fullname).read()
changed = False
if 'Version 2.1 (ZPL)' in data:
data = data.replace('Version 2.1 (ZPL)', 'Version 2.1 (ZPL)')
changed = True
if '(c) 2002 Zope Corporation' in data:
data = data.replace('(c) 2002 Zope Corporation', '(c) 2002 Zope Corporation')
changed = True
print fullname, changed
if changed:
open(fullname, 'w').write(data)
utilities/load_site.py deleted 100644 → 0
View file @ 2adae91c
#!/usr/bin/env python2.4
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Load a Zope site from a collection of files or directories
"""
usage=""" [options] url file .....
where options are:
-D
For HTML documents, replace the start of the content, up to
and including the opening body tag with a DTML var tag that
inserts the standard header. Also replace the closing body
and html tag with a DTML var tag that inserts the standard
footer.
-I
For each index.html, add an index_html that redirects.
-p path
Path to ZPublisher. If not provided, load_site will
make an attempt to figure it out.
-u user:password
Credentials
-v
Run in verbose mode.
-9
Use *old* zope method names.
"""
import sys, getopt, os, string
ServerError=''
verbose=0
old=0
doctor=0
index_html=0
def main():
user, password = 'superuser', '123'
opts, args = getopt.getopt(sys.argv[1:], 'p:u:DIv9')
global verbose
global old
global doctor
global index_html
havepath=None
for o, v in opts:
if o=='-p':
d, f = os.path.split(v)
if f=='ZPublisher': sys.path.insert(0,d)
else: sys.path.insert(0,v)
havepath=1
elif o=='-u':
v = string.split(v,':')
user, password = v[0], string.join(v[1:],':')
elif o=='-D': doctor=1
elif o=='-I': index_html=1
elif o=='-v': verbose=1
elif o=='-9': old=1
if not args:
print sys.argv[0]+usage
sys.exit(1)
if not havepath:
here=os.path.split(sys.argv[0])[0]
if os.path.exists(os.path.join(here,'ZPublisher')):
sys.path.insert(0,here)
else:
here=os.path.split(here)[0]
here=os.path.join(here,'lib','python')
if os.path.exists(os.path.join(here,'ZPublisher')):
sys.path.insert(0,here)
url=args[0]
files=args[1:]
import ZPublisher.Client
global ServerError
ServerError=ZPublisher.Client.ServerError
object=ZPublisher.Client.Object(url, username=user, password=password)
for f in files: upload_file(object, f)
def call(f, *args, **kw):
# Call a function ignoring redirect bci errors.
try: apply(f,args, kw)
except ServerError, v:
if str(v)[:1] != '3':
raise sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]
def upload_file(object, f):
if os.path.isdir(f): return upload_dir(object, f)
dir, name = os.path.split(f)
root, ext = os.path.splitext(name)
if ext in ('file', 'dir'): ext=''
else:
ext=string.lower(ext)
if ext and ext[0] in '.': ext=ext[1:]
if ext and globals().has_key('upload_'+ext):
if verbose: print 'upload_'+ext, f
return globals()['upload_'+ext](object, f)
if verbose: print 'upload_file', f, ext
call(object.manage_addFile, id=name, file=open(f,'rb'))
def upload_dir(object, f):
if verbose: print 'upload_dir', f
dir, name = os.path.split(f)
call(object.manage_addFolder, id=name)
object=object.__class__(object.url+'/'+name,
username=object.username,
password=object.password)
for n in os.listdir(f):
upload_file(object, os.path.join(f,n))
# ----- phd -----
# Modified by Oleg Broytmann <phd2@earthling.net>
from sgmllib import SGMLParser
def join_attrs(attrs):
attr_list = []
for attrname, value in attrs:
attr_list.append('%s="%s"' % (attrname, string.strip(value)))
if attr_list:
s = " " + string.join(attr_list, " ")
else:
s = ""
return s
class HeadParser(SGMLParser):
def __init__(self):
SGMLParser.__init__(self)
self.seen_starthead = 0
self.seen_endhead = 0
self.seen_startbody = 0
self.head = ""
self.title = ""
self.accumulator = ""
def handle_data(self, data):
if data:
self.accumulator = self.accumulator + data
def handle_charref(self, ref):
self.handle_data("&#%s;" % ref)
def handle_entityref(self, ref):
self.handle_data("&%s;" % ref)
def handle_comment(self, data):
if data:
self.accumulator = self.accumulator + "<!--%s-->" % data
def start_head(self, attrs):
if not self.seen_starthead:
self.seen_starthead = 1
self.head = ""
self.title = ""
self.accumulator = ""
def end_head(self):
if not self.seen_endhead:
self.seen_endhead = 1
self.head = self.head + self.accumulator
self.accumulator = ""
def start_title(self, attrs):
self.head = self.head + self.accumulator
self.accumulator = ""
def end_title(self):
self.title = self.accumulator
self.accumulator = ""
def start_body(self, attrs):
if not self.seen_startbody:
self.seen_startbody = 1
self.accumulator = ""
def end_body(self): pass # Do not put </BODY> and </HTML>
def end_html(self): pass # into output stream
# Pass other tags unmodified
def unknown_starttag(self, tag, attrs):
self.accumulator = self.accumulator + "<%s%s>" % (string.upper(tag), join_attrs(attrs))
def unknown_endtag(self, tag):
self.accumulator = self.accumulator + "</%s>" % string.upper(tag)
def parse_html(infile):
parser = HeadParser()
while 1:
line = infile.readline()
if not line: break
parser.feed(line)
parser.close()
infile.close()
return (string.strip(parser.title), string.strip(parser.head),
string.strip(parser.accumulator))
def upload_html(object, f):
dir, name = os.path.split(f)
f=open(f)
if doctor:
title, head, body = parse_html(f)
if old:
body = ("<!--#var standard_html_header-->\n\n" +
body + "\n\n<!--#var standard_html_footer-->")
else:
body = ("<dtml-var standard_html_header>\n\n" +
body + "\n\n<dtml-var standard_html_footer>")
else:
if old: f=f.read()
title, head, body = '', '', f
if old:
call(object.manage_addDocument, id=name, file=body)
if index_html and name in ('index.html', 'index.htm'):
call(object.manage_addDocument, id='index_html',
file=('<!--#raise Redirect-->'
'<!--#var URL1-->/%s'
'<!--#/raise-->' % name
))
else:
call(object.manage_addDTMLDocument, id=name, title=title, file=body)
if index_html and name in ('index.html', 'index.htm'):
call(object.manage_addDTMLMethod, id='index_html',
file=('<dtml-raise Redirect>'
'<dtml-var URL1>/%s'
'</dtml-raise>' % name
))
# Now add META and other tags as property
if head:
object=object.__class__(object.url+'/'+name,
username=object.username,
password=object.password)
call(object.manage_addProperty,
id="loadsite-head", type="text", value=head)
# ----- /phd -----
upload_htm=upload_html
def upload_dtml(object, f):
dir, name = os.path.split(f)
f=open(f)
if old:
f=f.read()
call(object.manage_addDocument, id=name, file=f)
else:
call(object.manage_addDTMLMethod, id=name, file=f)
def upload_gif(object, f):
dir, name = os.path.split(f)
call(object.manage_addImage, id=name, file=open(f,'rb'))
upload_jpg=upload_gif
upload_png=upload_gif
if __name__=='__main__': main()
utilities/mkzeoinstance.py deleted 100755 → 0
View file @ 2adae91c
#!/usr/bin/env python2.4
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import os
import sys
mydir = os.path.dirname(os.path.abspath(sys.argv[0]))
zopehome = os.path.dirname(mydir)
softwarehome = os.path.join(zopehome, "lib", "python")
if softwarehome not in sys.path:
sys.path.insert(0, softwarehome)
from ZEO.mkzeoinst import ZEOInstanceBuilder
if __name__ == "__main__":
ZEOInstanceBuilder().run()
utilities/mkzopeinstance.py deleted 100755 → 0
View file @ 2adae91c
#!/usr/bin/env python2.4
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""%(program)s: Create a Zope instance home.
usage: %(program)s [options]
Options:
-h/--help -- print this help text
-d/--dir -- the dir in which the instance home should be created
-u/--user NAME:PASSWORD -- set the user name and password of the initial user
-s/--skelsrc -- the dir from which skeleton files should be copied
When run without arguments, this script will ask for the information necessary
to create a Zope instance home.
"""
import getopt
import os
import shutil
import sys
import copyzopeskel
def main():
try:
opts, args = getopt.getopt(sys.argv[1:],
"hu:d:s:",
["help", "user=", "dir=", "skelsrc="]
)
except getopt.GetoptError, msg:
usage(sys.stderr, msg)
sys.exit(2)
script = os.path.abspath(sys.argv[0])
user = None
password = None
skeltarget = None
skelsrc = None
for opt, arg in opts:
if opt in ("-d", "--dir"):
skeltarget = os.path.abspath(os.path.expanduser(arg))
if not skeltarget:
usage(sys.stderr, "dir must not be empty")
sys.exit(2)
if opt in ("-s", "--skelsrc"):
skelsrc = os.path.abspath(os.path.expanduser(arg))
if not skelsrc:
usage(sys.stderr, "skelsrc must not be empty")
sys.exit(2)
if opt in ("-h", "--help"):
usage(sys.stdout)
sys.exit()
if opt in ("-u", "--user"):
if not arg:
usage(sys.stderr, "user must not be empty")
sys.exit(2)
if not ":" in arg:
usage(sys.stderr, "user must be specified as name:password")
sys.exit(2)
user, password = arg.split(":", 1)
if not skeltarget:
# interactively ask for skeltarget and initial user name/passwd.
# cant set custom instancehome in interactive mode, we default
# to skeltarget.
skeltarget = instancehome = os.path.abspath(
os.path.expanduser(get_skeltarget())
)
instancehome = skeltarget
zopehome = os.path.dirname(os.path.dirname(script))
softwarehome = os.path.join(zopehome, "src")
configfile = os.path.join(instancehome, 'etc', 'zope.conf')
if skelsrc is None:
# default to using stock Zope skeleton source
skelsrc = os.path.join(zopehome, "skel")
inituser = os.path.join(instancehome, "inituser")
if not (user or os.path.exists(inituser)):
user, password = get_inituser()
# we need to distinguish between python.exe and pythonw.exe under
# Windows. Zope is always run using 'python.exe' (even for services),
# however, it may be installed via pythonw.exe (as a sub-process of an
# installer). Thus, sys.executable may not be the executable we use.
# We still provide both PYTHON and PYTHONW, but PYTHONW should never
# need be used.
psplit = os.path.split(sys.executable)
exedir = os.path.join(*psplit[:-1])
pythonexe = os.path.join(exedir, 'python.exe')
pythonwexe = os.path.join(exedir, 'pythonw.exe')
if ( os.path.isfile(pythonwexe) and os.path.isfile(pythonexe) and
(sys.executable in [pythonwexe, pythonexe]) ):
# we're using a Windows build with both python.exe and pythonw.exe
# in the same directory
PYTHON = pythonexe
PYTHONW = pythonwexe
else:
# we're on UNIX or we have a nonstandard Windows setup
PYTHON = PYTHONW = sys.executable
kw = {
"PYTHON":PYTHON,
"PYTHONW":PYTHONW,
"INSTANCE_HOME": instancehome,
"SOFTWARE_HOME": softwarehome,
"ZOPE_HOME": zopehome,
}
copyzopeskel.copyskel(skelsrc, skeltarget, None, None, **kw)
if user and password:
write_inituser(inituser, user, password)
def usage(stream, msg=None):
if msg:
print >>stream, msg
print >>stream
program = os.path.basename(sys.argv[0])
print >>stream, __doc__ % {"program": program}
def get_skeltarget():
print 'Please choose a directory in which you\'d like to install'
print 'Zope "instance home" files such as database files, configuration'
print 'files, etc.'
print
while 1:
skeltarget = raw_input("Directory: ").strip()
if skeltarget == '':
print 'You must specify a directory'
continue
else:
break
return skeltarget
def get_inituser():
import getpass
print 'Please choose a username and password for the initial user.'
print 'These will be the credentials you use to initially manage'
print 'your new Zope instance.'
print
user = raw_input("Username: ").strip()
if user == '':
return None, None
while 1:
passwd = getpass.getpass("Password: ")
verify = getpass.getpass("Verify password: ")
if verify == passwd:
break
else:
passwd = verify = ''
print "Password mismatch, please try again..."
return user, passwd
def write_inituser(fn, user, password):
import binascii
try:
from hashlib import sha1 as sha
except:
from sha import new as sha
fp = open(fn, "w")
pw = binascii.b2a_base64(sha(password).digest())[:-1]
fp.write('%s:{SHA}%s\n' % (user, pw))
fp.close()
os.chmod(fn, 0644)
if __name__ == "__main__":
main()
utilities/reindex_catalog.py deleted 100644 → 0
View file @ 2adae91c
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""
A utility to perform external reindex operations for ZCatalogs
Usage:
zopectl run reindex_catalog.py [options]
Use --help to get a list of all options.
Author: Andreas Jung (andreas@andreas-jung.com)
$Id$
"""
import sys
from optparse import OptionParser
import transaction
from Products.ZCatalog.ProgressHandler import StdoutHandler
def path2catalog(path):
""" lookup catalog by path """
catalog = app.restrictedTraverse(path, None)
if not catalog:
raise ValueError('No catalog found at %s' % path)
return catalog
def getHandler(options):
""" return a progress handler """
if options.silent:
return None
else:
return StdoutHandler(options.steps)
def listIndexes(options, args):
""" print a list of all indexes to stdout """
catalog = path2catalog(options.catalog)
indexes = catalog._catalog.indexes
print 'Listing of all indexes at %s' % options.catalog
print
for id, idx in indexes.items():
print '%-20s %-50s %d' % (id, idx.meta_type, idx.numObjects())
def reindexIndexes(optioons, args):
""" reindex given list of indexes """
catalog = path2catalog(options.catalog)
handler = getHandler(options)
for id in args:
print 'Reindexing index %s at %s' % (id, options.catalog)
catalog.reindexIndex(id, None, handler)
transaction.commit()
def refreshMetadata(options, args):
""" reindex metadata """
catalog = path2catalog(options.catalog)
handler = getHandler(options)
print 'Refresh metadata at %s' % options.catalog
catalog.refreshMetadata(handler)
transaction.commit()
def reindexAll(options, args):
""" reindex complete catalog """
catalog = path2catalog(options.catalog)
handler = getHandler(options)
print 'Reindexing complete ZCatalog at %s' % options.catalog
catalog.refreshCatalog(options.clearCatalog, handler)
transaction.commit()
if __name__ == '__main__':
parser = OptionParser()
parser.add_option('-c', '--catalog', dest='catalog',
help='path to ZCatalog')
parser.add_option('-i', '--indexes', action="store_true", dest="listIndexes",
help='list all indexes')
parser.add_option('-C', '--clear', action="store_true", dest="clearCatalog", default=False,
help='clear catalog before reindexing the complete catalog (--all only)')
parser.add_option('-a', '--all', action="store_true", dest="reindexAll",
help='reindex the complete catalog and update all metadata')
parser.add_option('-r', '--reindex', action="store_true", dest="reindexIndexes",
help='reindex specified indexes')
parser.add_option('-m', '--metadata', action="store_true", dest="refreshMetadata",
help='refresh all metadata')
parser.add_option('-n', '--steps', dest="steps", default=100, type="int",
help='log progress every N objects')
parser.add_option('-s', '--silent', action="store_true", dest="silent", default=False,
help='do not log reindexing progress to stdout')
options,args = parser.parse_args()
if options.listIndexes: listIndexes(options, args)
if options.reindexIndexes: reindexIndexes(options, args)
if options.refreshMetadata: refreshMetadata(options, args)
if options.reindexAll: reindexAll(options, args)
utilities/requestprofiler.py deleted 100644 → 0
View file @ 2adae91c
#!/usr/bin/env python2.4
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Request log profiler script
$Id$
"""
import sys, time, getopt, math, cPickle
from types import StringType
try: import gzip
except: pass
class ProfileException(Exception): pass
class Request:
def __init__(self):
self.url = None
self.start = None
self.method = None
self.t_recdinput = None
self.isize = None
self.t_recdoutput = None
self.osize = None
self.httpcode = None
self.t_end = None
self.elapsed = "I"
self.active = 0
def put(self, code, t, desc):
if code not in ('A', 'B', 'I', 'E'):
raise "unknown request code %s" % code
if code == 'B':
self.start = t
self.method, self.url = desc.strip().split()
elif code == "I":
self.t_recdinput = t
self.isize = desc.strip()
elif code == "A":
self.t_recdoutput = t
self.httpcode, self.osize = desc.strip().split()
elif code == 'E':
self.t_end = t
self.elapsed = int(self.t_end - self.start)
def isfinished(self):
return not self.elapsed == "I"
def prettystart(self):
if self.start is not None:
t = time.localtime(self.start)
return time.strftime('%Y-%m-%dT%H:%M:%S', t)
else:
return "NA"
def shortprettystart(self):
if self.start is not None:
t = time.localtime(self.start)
return time.strftime('%H:%M:%S', t)
else:
return "NA"
def win(self):
if self.t_recdinput is not None and self.start is not None:
return self.t_recdinput - self.start
else:
return "NA"
def wout(self):
if self.t_recdoutput is not None and self.t_recdinput is not None:
return self.t_recdoutput - self.t_recdinput
else:
return "NA"
def wend(self):
if self.t_end is not None and self.t_recdoutput is not None:
return self.t_end - self.t_recdoutput
else:
return "NA"
def endstage(self):
if self.t_end is not None:
stage = "E"
elif self.t_recdoutput is not None:
stage = "A"
elif self.t_recdinput is not None:
stage = "I"
else:
stage = "B"
return stage
def total(self):
stage = self.endstage()
if stage == "B": return 0
if stage == "I": return self.t_recdinput - self.start
if stage == "A": return self.t_recdoutput - self.start
if stage == "E": return self.elapsed
def prettyisize(self):
if self.isize is not None:
return self.isize
else:
return "NA"
def prettyosize(self):
if self.osize is not None:
return self.osize
else:
return "NA"
def prettyhttpcode(self):
if self.httpcode is not None:
return self.httpcode
else:
return "NA"
def __str__(self):
body = (
self.prettystart(), self.win(), self.wout(), self.wend(),
self.total(), self.endstage(), self.prettyosize(),
self.prettyhttpcode(), self.active, self.url
)
return self.fmt % body
fmt = "%19s %4s %4s %4s %3s %1s %7s %4s %4s %s"
def getheader(self):
body = ('Start', 'WIn', 'WOut', 'WEnd', 'Tot', 'S', 'OSize',
'Code', 'Act', 'URL')
return self.fmt % body
class StartupRequest(Request):
def endstage(self):
return "U"
def total(self):
return 0
class Cumulative:
def __init__(self, url):
self.url = url
self.times = []
self.hangs = 0
self.allelapsed = None
def put(self, request):
elapsed = request.elapsed
if elapsed == "I": self.hangs = self.hangs + 1
self.times.append(elapsed)
def all(self):
if self.allelapsed == None:
self.allelapsed = []
for elapsed in self.times:
self.allelapsed.append(elapsed)
self.allelapsed.sort()
return self.allelapsed
def __str__(self):
body = (
self.hangs, self.hits(), self.total(), self.max(), self.min(),
self.median(), self.mean(), self.url
)
return self.fmt % body
def getheader(self):
return self.fmt % ('Hangs', 'Hits', 'Total', 'Max', 'Min', 'Median',
'Mean', 'URL')
fmt = "%5s %5s %5s %5s %5s %6s %5s %s"
def hits(self):
return len(self.times)
def max(self):
return max(self.all())
def min(self):
return min(self.all())
def mean(self):
l = len(self.times)
if l == 0:
return "I"
else:
t = self.total()
if t == "I": return "I"
return t/l
def median(self):
all = self.all()
l = len(all)
if l == 0:
return "I"
else:
if l == 1:
return all[0]
elif l % 2 != 0:
i = l/2 + 1
return all[i]
else:
i = l/2 - 1
i2 = i + 1
v1 = all[i]
v2 = all[i2]
if isinstance(v1, StringType) or isinstance(v2, StringType):
return "I"
else: return (v1 + v2) / 2
def total(self):
t = 0
all = self.all()
for elapsed in all:
if elapsed == "I": continue
t = t + elapsed
return t
def parsebigmlogline(line):
tup = line.split(None, 3)
if len(tup) == 3:
code, id, timestr = tup
return code, id, timestr, ''
elif len(tup) == 4:
return tup
else:
return None
def get_earliest_file_data(files):
temp = {}
earliest_fromepoch = 0
earliest = None
retn = None
for file in files:
line = file.readline()
if not line:
continue
linelen = len(line)
line = line.strip()
tup = parsebigmlogline(line)
if tup is None:
print "Could not interpret line: %s" % line
continue
code, id, timestr, desc = tup
timestr = timestr.strip()
fromepoch = getdate(timestr)
temp[file] = linelen
if earliest_fromepoch == 0 or fromepoch < earliest_fromepoch:
earliest_fromepoch = fromepoch
earliest = file
retn = [code, id, fromepoch, desc]
for file, linelen in temp.items():
if file is not earliest:
file.seek(file.tell() - linelen)
return retn
def get_requests(files, start=None, end=None, statsfname=None,
writestats=None, readstats=None):
finished = []
unfinished = {}
if readstats:
fp = open(statsfname, 'r')
u = cPickle.Unpickler(fp)
requests = u.load()
fp.close()
del u
del fp
else:
while 1:
tup = get_earliest_file_data(files)
if tup is None:
break
code, id, fromepoch, desc = tup
if start is not None and fromepoch < start: continue
if end is not None and fromepoch > end: break
if code == 'U':
finished.extend(unfinished.values())
unfinished.clear()
request = StartupRequest()
request.url = desc
request.start = int(fromepoch)
finished.append(request)
continue
request = unfinished.get(id)
if request is None:
if code != "B": continue # garbage at beginning of file
request = Request()
for pending_req in unfinished.values():
pending_req.active = pending_req.active + 1
unfinished[id] = request
t = int(fromepoch)
try:
request.put(code, t, desc)
except:
print "Unable to handle entry: %s %s %s"%(code, t, desc)
if request.isfinished():
del unfinished[id]
finished.append(request)
finished.extend(unfinished.values())
requests = finished
if writestats:
fp = open(statsfname, 'w')
p = cPickle.Pickler(fp)
p.dump(requests)
fp.close()
del p
del fp
return requests
def analyze(requests, top, sortf, start=None, end=None, mode='cumulative',
resolution=60, urlfocusurl=None, urlfocustime=60):
if mode == 'cumulative':
cumulative = {}
for request in requests:
url = request.url
stats = cumulative.get(url)
if stats is None:
stats = Cumulative(url)
cumulative[url] = stats
stats.put(request)
requests = cumulative.values()
requests.sort(sortf)
write(requests, top)
elif mode=='timed':
computed_start = requests[0].start
computed_end = requests[-1].t_end
if start and end:
timewrite(requests,start,end,resolution)
if start and not end:
timewrite(requests,start,computed_end,resolution)
if end and not start:
timewrite(requests,computed_start,end,resolution)
if not end and not start:
timewrite(requests,computed_start,computed_end,resolution)
elif mode == 'urlfocus':
requests.sort(sortf)
urlfocuswrite(requests, urlfocusurl, urlfocustime)
else:
requests.sort(sortf)
write(requests, top)
def urlfocuswrite(requests, url, t):
l = []
i = 0
for request in requests:
if request.url == url: l.append(i)
i = i + 1
before = {}
after = {}
x = 0
for n in l:
x = x + 1
r = requests[n]
start = r.start
earliest = start - t
latest = start + t
print 'URLs invoked %s seconds before and after %s (#%s, %s)' % \
(t, url, x, r.shortprettystart())
print '---'
i = -1
for request in requests:
i = i + 1
if request.start < earliest: continue
if request.start > latest: break
if n == i: # current request
print '%3d' % (request.start - start),
print '%s' % (request.shortprettystart()),
print request.url
continue
if request.start <= start:
if before.get(i):
before[i] = before[i] + 1
else:
before[i] = 1
if request.start > start:
if after.get(i):
after[i] = after[i] + 1
else:
after[i] = 1
print '%3d' % (request.start - start),
print '%s' % (request.shortprettystart()),
print request.url
print
print ('Summary of URLs invoked before (and at the same time as) %s '
'(times, url)' % url)
before = before.items()
before.sort()
for k,v in before:
print v, requests[k].url
print
print 'Summary of URLs invoked after %s (times, url)' % url
after = after.items()
after.sort()
for k,v in after:
print v, requests[k].url
def write(requests, top=0):
if len(requests) == 0:
print "No data.\n"
return
i = 0
header = requests[0].getheader()
print header
for stat in requests:
i = i + 1
if verbose:
print str(stat)
else:
print str(stat)[:78]
if i == top:
break
def getdate(val):
try:
val = val.strip()
year, month, day = int(val[:4]), int(val[5:7]), int(val[8:10])
hour,minute,second=int(val[11:13]),int(val[14:16]),int(val[17:19])
t = time.mktime((year, month, day, hour, minute, second, 0, 0, -1))
return t
except:
raise ProfileException, "bad date %s" % val
def getTimeslice(period, utime):
low = int(math.floor(utime)) - period + 1
high = int(math.ceil(utime)) + 1
for x in range(low, high):
if x % period == 0:
return x
def timewrite(requests, start, end, resolution):
print "Start: %s End: %s Resolution: %d secs" % \
(tick2str(start), tick2str(end), resolution)
print "-" * 78
print
print "Date/Time #requests requests/second"
d = {}
max = 0
min = None
for r in requests:
t = r.start
slice = getTimeslice(resolution,t)
if slice > max: max = slice
if (min is None) or (slice < min): min = slice
if d.has_key(slice):
d[slice] = d[slice] + 1
else:
d[slice] = 1
num = 0
hits = 0
avg_requests = None
max_requests = 0
for slice in range(min, max, resolution):
num = d.get(slice, 0)
if num>max_requests: max_requests = num
hits = hits + num
if avg_requests is None:
avg_requests = num
else:
avg_requests = (avg_requests + num) / 2
s = tick2str(slice)
s = s + " %6d %4.2lf" % (num,num*1.0/resolution)
print s
print '='*78
print " Peak: %6d %4.2lf" % \
(max_requests,max_requests*1.0/resolution)
print " Avg: %6d %4.2lf" % \
(avg_requests,avg_requests*1.0/resolution)
print "Total: %6d n/a " % (hits)
def tick2str(t):
return time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(t))
def codesort(v1, v2):
v1 = v1.endstage()
v2 = v2.endstage()
if v1 == v2:
return 0
if v1 == "B":
return -1 # v1 is smaller than v2
if v1 == "I":
if v2 == "B": return 1 # v1 is larger than v2
else: return -1
if v1 == "A":
if v2 in ['B', 'I']: return 1
else: return -1
if v1 == "E":
return 1
class Sort:
def __init__(self, fname, ascending=0):
self.fname = fname
self.ascending = ascending
def __call__(self, i1, i2):
f1 = getattr(i1, self.fname)
f2 = getattr(i2, self.fname)
if callable(f1): f1 = f1()
if callable(f2): f2 = f2()
if f1 < f2:
if self.ascending: return -1
else: return 1
elif f1 == f2:
return 0
else:
if self.ascending: return 1
else: return -1
def detailedusage():
details = usage(0)
pname = sys.argv[0]
details = details + """
Reports are of four types: cumulative, detailed, timed, or urlfocus. The
default is cumulative. Data is taken from one or more Zope detailed request
logs (-M logs, aka 'big M' logs) or from a preprocessed statistics file.
For cumulative reports, each line in the profile indicates information
about a Zope method (URL) collected via a detailed request log.
For detailed reports, each line in the profile indicates information about
a single request.
For timed reports, each line in the profile indicates informations about
the number of requests and the number of requests/second for a period of time.
For urlfocus reports, ad-hoc information about requests surrounding the
specified url is given.
Each 'filename' is a path to a '-M' log that contains detailed request data.
Multiple input files can be analyzed at the same time by providing the path
to each file. (Analyzing multiple big M log files at once is useful if you
have more than one Zope client on a single machine and you'd like to
get an overview of all Zope activity on that machine).
If you wish to make multiple analysis runs against the same input data, you
may want to use the --writestats option. The --writestats option creates a
file which holds preprocessed data representing the specfified input files.
Subsequent runs (for example with a different sort spec) will be much
faster if the --readstats option is used to specify a preprocessed stats
file instead of actual input files because the logfile parse step is skipped.
If a 'sort' value is specified, sort the profile info by the spec. The sort
order is descending unless indicated. The default cumulative sort spec is
'total'. The default detailed sort spec is 'start'.
For cumulative reports, the following sort specs are accepted:
'hits' -- the number of hits against the method
'hangs' -- the number of unfinished requests to the method
'max' -- the maximum time in secs taken by a request to this method
'min' -- the minimum time in secs taken by a request to this method
'mean' -- the mean time in secs taken by a request to this method
'median' -- the median time in secs taken by a request to this method
'total' -- the total time in secs across all requests to this method
'url' -- the URL/method name (ascending)
For detailed (non-cumulative) reports, the following sort specs are accepted:
'start' -- the start time of the request to ZServer (ascending)
'win' -- the num of secs ZServer spent waiting for input from client
'wout' -- the secs ZServer spent waiting for output from ZPublisher
'wend' -- the secs ZServer spent sending data to the client
'total' -- the secs taken for the request from begin to end
'endstage' -- the last successfully completed request stage (B, I, A, E)
'osize' -- the size in bytes of output provided by ZPublisher
'httpcode' -- the HTTP response code provided by ZPublisher (ascending)
'active' -- total num of requests pending at the end of this request
'url' -- the URL/method name (ascending)
For timed and urlfocus reports, there are no sort specs allowed.
If the 'top' argument is specified, only report on the top 'n' entries in
the profile (as per the sort). The default is to show all data in the profile.
If the 'verbose' argument is specified, do not trim url to fit into 80 cols.
If the 'today' argument is specified, limit results to hits received today.
If the 'daysago' argument is specified, limit results to hits received n days ago.
The 'resolution' argument is used only for timed reports and specifies the
number of seconds between consecutive lines in the report
(default is 60 seconds).
The 'urlfocustime' argument is used only for urlfocus reports and specifies the
number of seconds to target before and after the URL provided in urlfocus mode.
(default is 10 seconds).
If the 'start' argument is specified in the form 'DD/MM/YYYY HH:MM:SS' (UTC),
limit results to hits received after this date/time.
If the 'end' argument is specified in the form 'DD/MM/YYYY HH:MM:SS' (UTC),
limit results to hits received before this date/time.
'start' and 'end' arguments are not honored when request stats are obtained
via the --readstats argument.
Examples:
%(pname)s debug.log
Show cumulative report statistics for information in the file 'debug.log',
by default sorted by 'total'.
%(pname)s debug.log --detailed
Show detailed report statistics sorted by 'start' (by default).
%(pname)s debug.log debug2.log --detailed
Show detailed report statistics for both logs sorted by 'start'
(by default).
%(pname)s debug.log --cumulative --sort=mean --today --verbose
Show cumulative report statistics sorted by mean for entries in the log
which happened today, and do not trim the URL in the resulting report.
%(pname)s debug.log --cumulative --sort=mean --daysago=3 --verbose
Show cumulative report statistics sorted by mean for entries in the log
which happened three days ago, and do not trim the URL in the resulting report.
%(pname)s debug.log --urlfocus='/manage_main' --urlfocustime=60
Show 'urlfocus' report which displays statistics about requests
surrounding the invocation of '/manage_main'. Focus on the time periods
60 seconds before and after each invocation of the '/manage_main' URL.
%(pname)s debug.log --detailed --start='2001/05/10 06:00:00'
--end='2001/05/11 23:00:00'
Show detailed report statistics for entries in 'debug.log' which
begin after 6am UTC on May 10, 2001 and which end before
11pm UTC on May 11, 2001.
%(pname)s debug.log --timed --resolution=300 --start='2001/05/10 06:00:00'
--end='2001/05/11 23:00:00'
Show timed report statistics for entries in the log for one day
with a resolution of 5 minutes
%(pname)s debug.log --top=100 --sort=max
Show cumulative report of the the 'top' 100 methods sorted by maximum
elapsed time.
%(pname)s debug.log debug2.log --writestats='requests.stat'
Write stats file for debug.log and debug2.log into 'requests.stat' and
show default report.
%(pname)s --readstats='requests.stat' --detailed
Read from 'requests.stat' stats file (instead of actual -M log files)
and show detailed report against this data.""" % {'pname':pname}
return details
def usage(basic=1):
usage = (
"""
Usage: %s filename1 [filename2 ...]
[--cumulative | --detailed | [--timed --resolution=seconds]]
[--sort=spec]
[--top=n]
[--verbose]
[--today | [--start=date] [--end=date] | --daysago=n ]
[--writestats=filename | --readstats=filename]
[--urlfocus=url]
[--urlfocustime=seconds]
[--help]
Provides a profile of one or more Zope "-M" request log files.
""" % sys.argv[0]
)
if basic == 1:
usage = usage + """
If the --help argument is given, detailed usage docs are provided."""
return usage
if __name__ == '__main__':
if len(sys.argv) == 1:
print usage()
sys.exit(0)
if sys.argv[1] == '--help': print detailedusage(); sys.exit(0)
mode = 'cumulative'
sortby = None
trim = 0
top = 0
verbose = 0
start = None
end = None
resolution=60
urlfocustime=10
urlfocusurl=None
statsfname = None
readstats = 0
writestats = 0
files = []
i = 1
for arg in sys.argv[1:]:
if arg[:2] != '--':
if arg[-3:] == '.gz' and globals().has_key('gzip'):
files.append(gzip.GzipFile(arg,'r'))
else:
files.append(open(arg))
sys.argv.remove(arg)
i = i + 1
try:
opts, extra = getopt.getopt(
sys.argv[1:], '', ['sort=', 'top=', 'help', 'verbose', 'today',
'cumulative', 'detailed', 'timed','start=',
'end=','resolution=', 'writestats=','daysago=',
'readstats=','urlfocus=','urlfocustime=']
)
for opt, val in opts:
if opt=='--readstats':
statsfname = val
readstats = 1
elif opt=='--writestats':
statsfname = val
writestats = 1
if opt=='--sort': sortby = val
if opt=='--top': top=int(val)
if opt=='--help': print detailedusage(); sys.exit(0)
if opt=='--verbose':
verbose = 1
if opt=='--resolution':
resolution=int(val)
if opt=='--today':
now = time.localtime(time.time())
# for testing - now = (2001, 04, 19, 0, 0, 0, 0, 0, -1)
start = list(now)
start[3] = start[4] = start[5] = 0
start = time.mktime(start)
end = list(now)
end[3] = 23; end[4] = 59; end[5] = 59
end = time.mktime(end)
if opt=='--daysago':
now = time.localtime(time.time() - int(val)*3600*24 )
# for testing - now = (2001, 04, 19, 0, 0, 0, 0, 0, -1)
start = list(now)
start[3] = start[4] = start[5] = 0
start = time.mktime(start)
end = list(now)
end[3] = 23; end[4] = 59; end[5] = 59
end = time.mktime(end)
if opt=='--start':
start = getdate(val)
if opt=='--end':
end = getdate(val)
if opt=='--detailed':
mode='detailed'
d_sortby = sortby
if opt=='--cumulative':
mode='cumulative'
if opt=='--timed':
mode='timed'
if opt=='--urlfocus':
mode='urlfocus'
urlfocusurl = val
if opt=='--urlfocustime':
urlfocustime=int(val)
validcumsorts = ['url', 'hits', 'hangs', 'max', 'min', 'median',
'mean', 'total']
validdetsorts = ['start', 'win', 'wout', 'wend', 'total',
'endstage', 'isize', 'osize', 'httpcode',
'active', 'url']
if mode == 'cumulative':
if sortby is None: sortby = 'total'
assert sortby in validcumsorts, (sortby, mode, validcumsorts)
if sortby in ['url']:
sortf = Sort(sortby, ascending=1)
else:
sortf = Sort(sortby)
elif mode == 'detailed':
if sortby is None: sortby = 'start'
assert sortby in validdetsorts, (sortby, mode, validdetsorts)
if sortby in ['start', 'url', 'httpcode']:
sortf = Sort(sortby, ascending=1)
elif sortby == 'endstage':
sortf = codesort
else:
sortf = Sort(sortby)
elif mode=='timed':
sortf = None
elif mode=='urlfocus':
sortf = Sort('start', ascending=1)
else:
raise 'Invalid mode'
req=get_requests(files, start, end, statsfname, writestats, readstats)
analyze(req, top, sortf, start, end, mode, resolution, urlfocusurl,
urlfocustime)
except AssertionError, val:
a = "%s is not a valid %s sort spec, use one of %s"
print a % (val[0], val[1], val[2])
sys.exit(0)
except getopt.error, val:
print val
sys.exit(0)
except ProfileException, val:
print val
sys.exit(0)
except SystemExit:
sys.exit(0)
except:
import traceback
traceback.print_exc()
print usage()
sys.exit(0)
utilities/tracelog.py deleted 100644 → 0
View file @ 2adae91c
##############################################################################
#
# Copyright (c) 2006 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Yet another lag analysis tool
$Id$
"""
import datetime, optparse, sys
class Request:
output_bytes = '-'
def __init__(self, start, method, url):
self.method = method
self.url = url
self.start = start
self.state = 'input'
def I(self, input_time, input_bytes):
self.input_time = input_time
self.input_bytes = input_bytes
self.state = 'app'
def A(self, app_time, response, output_bytes):
self.app_time = app_time
self.response = response
self.output_bytes = output_bytes
self.state = 'output'
def E(self, end):
self.end = end
@property
def app_seconds(self):
return (self.app_time - self.input_time).seconds
@property
def total_seconds(self):
return (self.end - self.start).seconds
class Times:
tid = 1l
def __init__(self):
self.times = []
self.hangs = 0
Times.tid += 1
self.tid = Times.tid # generate a unique id
def finished(self, request):
self.times.append(request.app_seconds)
def hung(self):
self.hangs += 1
def impact(self):
times = self.times
if not times:
self.median = self.mean = self.impact = 0
return 0
self.times.sort()
n = len(times)
if n % 2:
m = times[(n+1)/2-1]
else:
m = .5 * (times[n/2]+times[n/2-1])
self.median = m
self.mean = sum(times)/n
self.impact = self.mean * (n+self.hangs)
return self.impact
def __str__(self):
times = self.times
if not times:
return " 0 %5d" % (
self.hangs)
n = len(times)
m = self.median
return "%9.1f %5d %6.0f %6.2f %6.2f %6.0f %5d" % (
self.impact, n, times[0], m, self.mean, times[-1], self.hangs)
def html(self):
times = self.times
if not times:
print td('', 0, '', '', '', '', self.hangs)
else:
n = len(times)
m = self.median
impact = '<a name="u%s">%s' % (self.tid, self.impact)
print td(impact, n, times[0], m, self.mean, times[-1],
self.hangs)
def parsedt(s):
date, time = s.split('T')
return datetime.datetime(*(
map(int, date.split('-'))
+
map(int, time.split(':'))
))
def main(args=None):
if args is None:
args = sys.argv[1:]
options, args = parser.parse_args(args)
if options.event_log:
restarts = find_restarts(options.event_log)
else:
restarts = []
restarts.append(datetime.datetime.utcnow()+datetime.timedelta(1000))
if options.html:
print_app_requests = print_app_requests_html
output_minute = output_minute_html
output_stats = output_stats_html
minutes_header = minutes_header_html
minutes_footer = minutes_footer_html
print '<html title="trace log statistics"><body>'
else:
print_app_requests = print_app_requests_text
output_minute = output_minute_text
output_stats = output_stats_text
minutes_header = minutes_header_text
minutes_footer = minutes_footer_text
urls = {}
[file] = args
lmin = ldt = None
requests = {}
input = apps = output = n = 0
spr = spa = 0.0
restart = restarts.pop(0)
minutes_header()
remove_prefix = options.remove_prefix
for record in open(file):
record = record.split()
typ, rid, dt = record[:3]
min = dt[:-3]
dt = parsedt(dt)
if dt == restart:
continue
while dt > restart:
print_app_requests(requests, ldt,
options.old_requests,
options.app_requests,
urls,
"\nLeft over:")
record_hung(urls, requests)
requests = {}
input = apps = output = n = 0
spr = spa = 0.0
restart = restarts.pop(0)
ldt = dt
if min != lmin:
if lmin is not None:
output_minute(lmin, requests, input, apps, output, n, spr, spa)
if apps > options.apps:
print_app_requests(requests, dt,
options.old_requests,
options.app_requests,
urls,
)
lmin = min
spr = 0.0
spa = 0.0
n = 0
if typ == 'B':
if rid in requests:
request = requests[rid]
if request.state == 'output':
output -= 1
elif request.state == 'app':
apps -= 1
else:
input -= 1
input += 1
request = Request(dt, *record[3:5])
if remove_prefix and request.url.startswith(remove_prefix):
request.url = request.url[len(remove_prefix):]
requests[rid] = request
times = urls.get(request.url)
if times is None:
times = urls[request.url] = Times()
elif typ == 'I':
if rid in requests:
input -= 1
apps += 1
requests[rid].I(dt, record[3])
elif typ == 'A':
if rid in requests:
apps -= 1
output += 1
requests[rid].A(dt, *record[3:5])
elif typ == 'E':
if rid in requests:
output -= 1
request = requests.pop(rid)
request.E(dt)
spr += request.total_seconds
spa += request.app_seconds
n += 1
times = urls[request.url]
times.finished(request)
else:
print 'WTF', record
print_app_requests(requests, dt,
options.old_requests,
options.app_requests,
urls,
"Left over:")
minutes_footer()
output_stats(urls)
if options.html:
print '</body></html>'
def output_stats_text(urls):
print
print 'URL statistics:'
print " Impact count min median mean max hangs"
print "========= ===== ====== ====== ====== ====== ====="
urls = [(times.impact(), url, times)
for (url, times) in urls.iteritems()
]
urls.sort()
urls.reverse()
for (_, url, times) in urls:
if times.impact > 0 or times.hangs:
print times, url
def output_stats_html(urls):
print
print 'URL statistics:'
print '<table border="1">'
print '<tr><th>Impact</th><th>count</th><th>min</th>'
print '<th>median</th><th>mean</th><th>max</th><th>hangs</th></tr>'
urls = [(times.impact(), url, times)
for (url, times) in urls.iteritems()
]
urls.sort()
urls.reverse()
for (_, url, times) in urls:
if times.impact > 0 or times.hangs:
print '<tr>'
times.html()
print td(url)
print '</tr>'
print '</table>'
def minutes_header_text():
print
print " minute req input app output"
print "================ ===== ===== ===== ======"
def minutes_footer_text():
print
def minutes_header_html():
print '<table border="2">'
print "<tr>"
print '<th>Minute</th>'
print '<th>Requests</th>'
print '<th>Requests inputing</th>'
print '<th>Requests executing or waiting</th>'
print '<th>Requests outputing</th>'
print '<th>Requests completed</th>'
print '<th>Mean Seconds Per Request Total</th>'
print '<th>Mean Seconds Per Request in App</th>'
print "</tr>"
def minutes_footer_html():
print '</table>'
def output_minute_text(lmin, requests, input, apps, output, n, spr, spa):
print lmin.replace('T', ' '), "%5d I=%3d A=%3d O=%5d " % (
len(requests), input, apps, output),
if n:
print "N=%4d %10.2f %10.2f" % (n, spr/n, spa/n)
else:
print
def td(*values):
return ''.join([("<td>%s</td>" % s) for s in values])
def output_minute_html(lmin, requests, input, apps, output, n, spr, spa):
print '<tr>'
apps = '<font size="+2"><strong>%s</strong></font>' % apps
print td(lmin.replace('T', ' '), len(requests), input, apps, output)
if n:
print td(n, "%10.2f" % (spr/n), "%10.2f" % (spa/n))
print '</tr>'
def find_restarts(event_log):
result = []
for l in open(event_log):
if l.strip().endswith("Zope Ready to handle requests"):
result.append(parsedt(l.split()[0]))
return result
def record_hung(urls, requests):
for request in requests.itervalues():
times = urls.get(request.url)
if times is None:
times = urls[request.url] = Times()
times.hung()
def print_app_requests_text(requests, dt, min_seconds, max_requests, urls,
label=''):
requests = [
((dt-request.input_time).seconds, request)
for request in requests.values()
if request.state == 'app'
]
urls = {}
for s, request in requests:
urls[request.url] = urls.get(request.url, 0) + 1
requests.sort()
requests.reverse()
for s, request in requests[:max_requests]:
if s < min_seconds:
continue
if label:
print label
label = ''
url = request.url
repeat = urls[url]
if repeat > 1:
print s, "R=%d" % repeat, url
else:
print s, url
def print_app_requests_html(requests, dt, min_seconds, max_requests, allurls,
label=''):
requests = [
((dt-request.input_time).seconds, request)
for request in requests.values()
if request.state == 'app'
]
urls = {}
for s, request in requests:
urls[request.url] = urls.get(request.url, 0) + 1
requests.sort()
requests.reverse()
printed = False
for s, request in requests[:max_requests]:
if s < min_seconds:
continue
if label:
print label
label = ''
if not printed:
minutes_footer_html()
print '<table border="1">'
print '<tr><th>age</th><th>R</th><th>url</th></tr>'
printed = True
url = request.url
repeat = urls[url]
print '<tr>'
if repeat <= 1:
repeat = ''
url = '<a href="#u%s">%s</a>' % (allurls[url].tid, url)
print td(s, repeat, url)
print '</tr>'
if printed:
print '</table>'
minutes_header_html()
parser = optparse.OptionParser("""
Usage: %prog [options] trace_log_file
Output trace log data showing:
- number of active requests,
- number of input requests (requests gathering input),
- number of application requests,
- number of output requests,
- number of requests completed in the minute shown,
- mean seconds per request and
- mean application seconds per request.
Note that we don't seem to be logging when a connection to the client
is broken, so the number of active requests, and especially the number
of outputing requests tends to grow over time. This is spurious.
Also, note that, unfortunately, application requests include requests
that are running in application threads and requests waiting to get an
application thread.
When application threads get above the app request threshold, then we
show the requests that have been waiting the longest.
""")
parser.add_option("--app-request-threshold", "-a", dest='apps',
type="int", default=10,
help="""
Number of pending application requests at which detailed request information
if printed.
""")
parser.add_option("--app-requests", "-r", dest='app_requests',
type="int", default=10,
help="""
How many requests to show when the maximum number of pending
apps is exceeded.
""")
parser.add_option("--old-requests", "-o", dest='old_requests',
type="int", default=10,
help="""
Number of seconds beyond which a request is considered old.
""")
parser.add_option("--event-log", "-e", dest='event_log',
help="""
The name of an event log that goes with the trace log. This is used
to determine when the server is restarted, so that the running trace data structures can be reinitialized.
""")
parser.add_option("--html", dest='html', action='store_true',
help="""
Generate HTML output.
""")
parser.add_option("--remove-prefix", dest='remove_prefix',
help="""
A prefex to be removed from URLS.
""")
if __name__ == '__main__':
main()
utilities/zpasswd.py deleted 100755 → 0
View file @ 2adae91c
#!/usr/bin/env python2.4
##############################################################################
#
# Copyright (c) 2001,2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Zope user bootstrap system
Usage: %(PROGRAM)s [options] filename
If this program is called without command-line options, it will prompt
for all necessary information. The available options are:
-u / --username=
Set the username to be used for the initial user or the emergency user
-p / --password=
Set the password
-e / --encoding=
Set the encryption/encoding rules. Defaults to SHA-1. OPTIONAL
-d / --domains=
Set the domain names that the user user can log in from. Defaults to
any. OPTIONAL.
-h / --help
Print this help text and exit.
Filename is required and should be the name of the file to store the
information in (usually "inituser" or "access").
"""
import sys, sha, binascii, random, getopt, getpass, os
try:
from crypt import crypt
except ImportError:
crypt = None
PROGRAM = sys.argv[0]
COMMASPACE = ', '
def generate_salt():
"""Generate a salt value for the crypt function."""
salt_choices = ("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789./")
return random.choice(salt_choices)+random.choice(salt_choices)
def generate_passwd(password, encoding):
encoding=encoding.upper()
if encoding == 'SHA':
pw = '{SHA}' + binascii.b2a_base64(sha.new(password).digest())[:-1]
elif encoding == 'CRYPT':
pw = '{CRYPT}' + crypt(password, generate_salt())
elif encoding == 'CLEARTEXT':
pw = password
else:
raise ValueError('Unsupported encoding: %s' % encoding)
return pw
def write_generated_password(home, ac_path, username):
pw_choices = ("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789!")
acfile=open(ac_path, 'w')
pw = ''
for i in range(8):
pw = pw + random.choice(pw_choices)
acfile.write('%s:%s\n' % (username, generate_passwd(pw, 'SHA')))
acfile.close()
os.chmod(ac_path, 0644)
return pw
def write_access(home, user='', group=''):
ac_path=os.path.join(home, 'access')
if not os.path.exists(ac_path):
print '-'*78
print 'creating default access file'
pw = write_generated_password(home, ac_path, 'emergency')
print """Note:
The emergency user name and password are 'emergency'
and '%s'.
You can change the emergency name and password with the
zpasswd script. To find out more, type:
%s zpasswd.py
""" % (pw, sys.executable)
import do; do.ch(ac_path, user, group)
def get_password():
while 1:
password = getpass.getpass("Password: ")
verify = getpass.getpass("Verify password: ")
if verify == password:
return password
else:
password = verify = ''
print "Password mismatch, please try again..."
def write_inituser(home, user='', group=''):
ac_path=os.path.join(home, 'inituser')
if not os.path.exists(ac_path):
print '-'*78
print 'creating default inituser file'
pw = write_generated_password(home, ac_path, 'admin')
print """Note:
The initial user name and password are 'admin'
and '%s'.
You can change the name and password through the web
interface or using the 'zpasswd.py' script.
""" % pw
import do; do.ch(ac_path, user, group)
def usage(code, msg=''):
print >> sys.stderr, __doc__ % globals()
if msg:
print >> sys.stderr, msg
sys.exit(code)
def main():
shortopts = 'u:p:e:d:h'
longopts = ['username=',
'password=',
'encoding=',
'domains=',
'help']
try:
opts, args = getopt.getopt(sys.argv[1:], shortopts, longopts)
except getopt.error, msg:
usage(1, msg)
# Defaults
username = password = None
domains = ''
encoding = 'SHA'
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-u', '--username'):
username = arg
elif opt in ('-p', '--password'):
password = arg
elif opt in ('-e', '--encoding'):
encoding = arg
elif opt in ('-d', '--domains'):
domains = ':' + arg
# Extra command line arguments?
if len(args) == 0:
usage(1, 'filename is required')
elif len(args) == 1:
access_file = open(args[0], 'w')
else:
usage(1, 'Extra command line arguments: ' + COMMASPACE.join(args))
if opts:
# There were some command line args, so verify
if username is not None and password is None:
password = get_password()
else:
# No command line args, so prompt
while 1:
username = raw_input("Username: ")
if username != '':
break
password = get_password()
while 1:
print """
Please choose a format from:
SHA - SHA-1 hashed password (default)
CRYPT - UNIX-style crypt password
CLEARTEXT - no protection
"""
encoding = raw_input("Encoding: ")
if encoding == '':
encoding = 'SHA'
break
if encoding.upper() in ['SHA', 'CRYPT', 'CLEARTEXT']:
break
domains = raw_input("Domain restrictions: ")
if domains:
domains = ":" + domains
# Done with prompts and args
access_file.write(username + ":" +
generate_passwd(password, encoding) +
domains)
# If called from the command line
if __name__=='__main__':
main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7