Commit 0abe19d5 authored by Grégory Wisniewski's avatar Grégory Wisniewski

Update perf runner.

- Remove hard-coded email addresses
- Add command line options to:
  - Choose sender, recipients, mail server
  - Cluster configuration (masters, storages, replicas...)
- Include import speed and datafs size in mail subject
- Clean a bit the code structure.

git-svn-id: https://svn.erp5.org/repos/neo/trunk@1723 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent 9d0b61de
......@@ -2,6 +2,9 @@
import os
import sys
import optparse
import platform
import datetime
from time import time
from neo.tests.functional import NEOCluster
......@@ -13,7 +16,7 @@ RECIPIENTS = ['gregory@nexedi.com'] #['neo-report@erp5.org']
SMTP_SERVER = ( "mail.nexedi.com", "25")
def _runImport(filename):
def runImport(neo, datafs):
def _copyTransactionsFrom(self, other):
""" taken from ZODB.BaseStorage that build stat during import """
......@@ -24,7 +27,7 @@ def _runImport(filename):
d[t] += 1
txn = {}
obj = {}
preindex = {};
preindex = {}
fiter = other.iterator()
for transaction in fiter:
inc(txn)
......@@ -42,63 +45,58 @@ def _runImport(filename):
'Objects': obj.values(),
}
# start a NEO cluster
neo = NEOCluster(db_list=['test_import_neo1'], verbose=False)
neo.setupDB()
neo.start()
# open storage clients
# open storages clients
neo_storage = neo.getZODBStorage()
dfs_storage = FileStorage(file_name=filename)
dfs_size = os.path.getsize(filename)
dfs_storage = FileStorage(file_name=datafs)
dfs_size = os.path.getsize(datafs)
# monkey patch import method and run the import
start = time()
Storage.copyTransactionsFrom = _copyTransactionsFrom
stats = neo_storage.copyTransactionsFrom(dfs_storage)
elapsed = time() - start
neo.stop()
return (dfs_size, elapsed, stats)
return (dfs_size, elapsed, stats)
def _buildSystemInfo():
import platform
import datetime
s = """
Date : %s
Node : %s
Processor : %s (%s)
System : %s (%s)
""" % (
datetime.date.today().isoformat(),
platform.node(),
platform.processor(),
platform.architecture()[0],
platform.system(),
platform.release(),
)
return s
def _buildResult(dfs_size, elapsed, stats):
def buildReport(dfs_size, elapsed, stats):
""" build a report for the given import data """
result = '\n'
pat = '%20s | %8s | %5s | %5s | %5s \n'
sep = '%20s+%8s+%5s+%5s+%5s\n'
sep %= ('-' * 21, '-' * 10) + ('-' * 7, ) * 3
result += pat % ('', ' num ', 'min/s', 'avg/s', 'max/s')
pat = '%19s | %8s | %5s | %5s | %5s \n'
sep = '%19s+%8s+%5s+%5s+%5s\n'
sep %= ('-' * 20, '-' * 10) + ('-' * 7, ) * 3
# system informations
report = ' ' * 20 + ' NEO PERF REPORT\n\n'
report += "\tDate : %s\n" % datetime.date.today().isoformat()
report += "\tNode : %s\n" % platform.node()
report += "\tProcessor : %s (%s)\n" % (platform.processor(),
platform.architecture()[0])
report += "\tSystem : %s (%s)\n" % (platform.system(),
platform.release())
report += '\n\n'
# stats on objects and transactions
report += pat % ('', ' num ', 'min/s', 'avg/s', 'max/s')
for k, v in stats.items():
result += sep
report += sep
s = sum(v)
result += pat % (k, s, min(v), s / len(v), max(v))
report += pat % (k, s, min(v), s / len(v), max(v))
report += sep
# global results
dfs_size /= 1024
result += sep
result += '\n%20s: %6.1f MB' % ('Input size', dfs_size / 1024)
result += '\n%20s: %6d sec' % ('Import duration', elapsed)
result += '\n%20s: %6.1f KB/s\n' % ('Average speed', dfs_size / elapsed)
size = dfs_size / 1024
speed = dfs_size / elapsed
report += '\n%19s: %6.1f MB' % ('Input size', size)
report += '\n%19s: %6d sec' % ('Import duration', elapsed)
report += '\n%19s: %6.1f KB/s\n' % ('Average speed', speed)
# build summary
summary = 'Neo : %6.1f KB/s (%6.1f MB)' % (speed, size)
return result
return (summary, report)
def sendReport(report):
def sendReport(summary, report):
""" Send a mail with the report summary """
import smtplib
......@@ -107,14 +105,13 @@ def sendReport(report):
# build the email
msg = MIMEMultipart()
msg['Subject'] = 'NEO : Import report'
msg['Subject'] = summary
msg['From'] = SENDER
msg['To'] = ', '.join(RECIPIENTS)
msg.epilogue = ''
# write the body
msg.attach(MIMEText(report))
# Send the email via our own SMTP server.
# Send via smtp server
s = smtplib.SMTP()
s.connect(*SMTP_SERVER)
mail = msg.as_string()
......@@ -125,21 +122,52 @@ def sendReport(report):
print "Mail for %s fails" % recipient
s.close()
if __name__ == "__main__":
if len(sys.argv) != 2:
sys.exit('Missing data.fs argument')
result = _runImport(sys.argv[1])
result = _buildResult(*result)
# handle command line options
parser = optparse.OptionParser()
parser.add_option('-d', '--datafs')
parser.add_option('-m', '--master-count')
parser.add_option('-s', '--storage-count')
parser.add_option('-p', '--partition-count')
parser.add_option('-r', '--replica-count')
parser.add_option('', '--recipient', action='append')
parser.add_option('', '--sender')
parser.add_option('', '--server')
(options, args) = parser.parse_args()
# check arguments
if not options.datafs or not os.path.exists(options.datafs):
sys.exit('Missing or wrong data.fs argument')
if bool(options.sender) ^ bool(options.recipient):
sys.exit('Need a sender and recipients to mail report')
# load options or defaults
masters = int(options.master_count or 1)
storages = int(options.storage_count or 1)
partitions = int(options.partition_count or 10)
replicas = int(options.replica_count or 0)
datafs = options.datafs
# start neo
neo = NEOCluster(
db_list=['test_import_%d' % i for i in xrange(storages)],
clear_databases=True,
partitions=partitions,
replicas=replicas,
master_node_count=masters,
verbose=True,
)
# build the report and send it
report = ' ' * 30 + ' NEO PERF REPORT\n\n'
report += _buildSystemInfo()
report += result
# import datafs
neo.start()
summary, report = buildReport(*runImport(neo, datafs))
neo.stop()
# display and/or send the report
print summary
print report
sendReport(report)
if options.sender:
sendReport(summary, report)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment