perfs 4.44 KB
Newer Older
1
#! /usr/bin/env python
2 3 4

import os
import sys
5 6
import platform
import datetime
7
import traceback
8
from time import time
9
from ZODB.FileStorage import FileStorage
10

11
from neo.tests import DB_PREFIX
12
from neo.tests.benchmark import BenchmarkRunner
13
from neo.tests.functional import NEOCluster
14
from neo.lib.profiling import PROFILING_ENABLED, profiler_decorator, \
15
    profiler_report
16

17 18 19 20 21 22 23 24 25 26 27
class ImportBenchmark(BenchmarkRunner):
    """ Test import of a datafs """

    def add_options(self, parser):
        parser.add_option('-d', '--datafs')
        parser.add_option('-m', '--masters')
        parser.add_option('-s', '--storages')
        parser.add_option('-p', '--partitions')
        parser.add_option('-r', '--replicas')

    def load_options(self, options, args):
28
        if options.datafs and not os.path.exists(options.datafs):
29 30 31 32 33 34 35 36 37 38 39 40 41
            sys.exit('Missing or wrong data.fs argument')
        return dict(
            datafs = options.datafs,
            masters = int(options.masters or 1),
            storages = int(options.storages or 1),
            partitions = int(options.partitions or 10),
            replicas = int(options.replicas or 0),
        )

    def start(self):
        config = self._config
        # start neo
        neo = NEOCluster(
42 43
            db_list=['%s_perfs_%u' % (DB_PREFIX, i)
                     for i in xrange(config.storages)],
44 45 46
            clear_databases=True,
            partitions=config.partitions,
            replicas=config.replicas,
47
            master_count=config.masters,
48 49 50 51 52
            verbose=False,
        )

        # import datafs
        neo.start()
53
        try:
54 55 56 57 58 59
            try:
                return self.buildReport(*self.runImport(neo))
            except:
                summary = 'Perf : import failed'
                report = ''.join(traceback.format_exc())
                return summary, report
60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
        finally:
            neo.stop()

    def runImport(self, neo):

        def counter(wrapped, d):
            @profiler_decorator
            def wrapper(*args, **kw):
                # count number of tick per second
                t = int(time())
                d.setdefault(t, 0)
                d[t] += 1
                # call original method
                wrapped(*args, **kw)
            return wrapper

        # open storages clients
        datafs = self._config.datafs
        neo_storage = neo.getZODBStorage()
79 80 81 82 83 84
        if datafs:
            dfs_storage = FileStorage(file_name=datafs)
        else:
            from neo.tests.stat_zodb import PROD1
            from random import Random
            dfs_storage = PROD1(Random(0)).as_storage(10000)
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100

        # monkey patch storage
        txn_dict, obj_dict = {}, {}
        neo_storage.app.tpc_begin = counter(neo_storage.app.tpc_begin, txn_dict)
        neo_storage.app.store = counter(neo_storage.app.store, obj_dict)

        # run import
        start = time()
        stats = neo_storage.copyTransactionsFrom(dfs_storage)
        elapsed = time() - start

        # return stats
        stats = {
            'Transactions': txn_dict.values(),
            'Objects': obj_dict.values(),
        }
101
        return (dfs_storage.getSize(), elapsed, stats)
102 103 104 105

    def buildReport(self, dfs_size, elapsed, stats):
        """ build a report for the given import data """
        config = self._config
106 107
        dfs_size /= 1e3
        size = dfs_size / 1e3
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
        speed = dfs_size / elapsed

        # configuration
        self.add_status('Masters', config.masters)
        self.add_status('Storages', config.storages)
        self.add_status('Replicas', config.replicas)
        self.add_status('Partitions', config.partitions)

        # results
        self.add_status('Input size', '%-.1f MB' % size)
        self.add_status('Import duration', '%-d secs' % elapsed)
        self.add_status('Average speed', '%-.1f KB/s' % speed)

        # stats on objects and transactions
        pat = '%19s | %8s | %5s | %5s | %5s \n'
        sep = '%19s+%8s+%5s+%5s+%5s\n'
        sep %= ('-' * 20, '-' * 10) + ('-' * 7, ) * 3
        report = pat % ('', ' num ', 'min/s', 'avg/s', 'max/s')
        for k, v in stats.items():
            report += sep
            s = sum(v)
            report += pat % (k, s,  min(v), s / len(v), max(v))
        report += sep
131

132 133 134
        # build summary
        summary = 'Perf : %.1f KB/s (%.1f MB)' % (speed, size)
        return (summary, report)
135

136
def main(args=None):
137
    ImportBenchmark().run()
138 139 140
    if PROFILING_ENABLED:
        print profiler_report()

141 142
if __name__ == "__main__":
    main()