perfs 4.24 KB
Newer Older
1
#! /usr/bin/env python
2 3 4

import os
import sys
5 6
import platform
import datetime
7
import traceback
8
from time import time
9
from ZODB.FileStorage import FileStorage
10

11
from neo.tests import DB_PREFIX
12
from neo.tests.benchmark import BenchmarkRunner
13 14
from neo.tests.functional import NEOCluster

15 16 17 18
class ImportBenchmark(BenchmarkRunner):
    """ Test import of a datafs """

    def add_options(self, parser):
19 20 21 22 23 24
        _ = parser.add_argument
        _('-d', '--datafs')
        _('-m', '--masters', type=int, default=1)
        _('-s', '--storages', type=int, default=1)
        _('-p', '--partitions', type=int, default=10)
        _('-r', '--replicas', type=int, default=0)
25

26 27
    def load_options(self, args):
        if args.datafs and not os.path.exists(args.datafs):
28 29
            sys.exit('Missing or wrong data.fs argument')
        return dict(
30 31 32 33 34
            datafs = args.datafs,
            masters = args.masters,
            storages = args.storages,
            partitions = args.partitions,
            replicas = args.replicas,
35 36 37 38 39 40
        )

    def start(self):
        config = self._config
        # start neo
        neo = NEOCluster(
41 42
            db_list=['%s_perfs_%u' % (DB_PREFIX, i)
                     for i in xrange(config.storages)],
43 44 45
            clear_databases=True,
            partitions=config.partitions,
            replicas=config.replicas,
46
            master_count=config.masters,
47 48
        )
        # import datafs
49
        try:
50
            neo.start()
51
            try:
52 53 54 55
                result = self.buildReport(*self.runImport(neo))
            finally:
                neo.stop()
            # Clear DB if no error happened.
56
            neo.resetDB()
57 58 59
            return result
        except:
            return 'Perf: import failed', ''.join(traceback.format_exc())
60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75

    def runImport(self, neo):

        def counter(wrapped, d):
            def wrapper(*args, **kw):
                # count number of tick per second
                t = int(time())
                d.setdefault(t, 0)
                d[t] += 1
                # call original method
                wrapped(*args, **kw)
            return wrapper

        # open storages clients
        datafs = self._config.datafs
        neo_storage = neo.getZODBStorage()
76 77 78 79 80 81
        if datafs:
            dfs_storage = FileStorage(file_name=datafs)
        else:
            from neo.tests.stat_zodb import PROD1
            from random import Random
            dfs_storage = PROD1(Random(0)).as_storage(10000)
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97

        # monkey patch storage
        txn_dict, obj_dict = {}, {}
        neo_storage.app.tpc_begin = counter(neo_storage.app.tpc_begin, txn_dict)
        neo_storage.app.store = counter(neo_storage.app.store, obj_dict)

        # run import
        start = time()
        stats = neo_storage.copyTransactionsFrom(dfs_storage)
        elapsed = time() - start

        # return stats
        stats = {
            'Transactions': txn_dict.values(),
            'Objects': obj_dict.values(),
        }
98
        return (dfs_storage.getSize(), elapsed, stats)
99 100 101 102

    def buildReport(self, dfs_size, elapsed, stats):
        """ build a report for the given import data """
        config = self._config
103 104
        dfs_size /= 1e3
        size = dfs_size / 1e3
105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
        speed = dfs_size / elapsed

        # configuration
        self.add_status('Masters', config.masters)
        self.add_status('Storages', config.storages)
        self.add_status('Replicas', config.replicas)
        self.add_status('Partitions', config.partitions)

        # results
        self.add_status('Input size', '%-.1f MB' % size)
        self.add_status('Import duration', '%-d secs' % elapsed)
        self.add_status('Average speed', '%-.1f KB/s' % speed)

        # stats on objects and transactions
        pat = '%19s | %8s | %5s | %5s | %5s \n'
        sep = '%19s+%8s+%5s+%5s+%5s\n'
        sep %= ('-' * 20, '-' * 10) + ('-' * 7, ) * 3
        report = pat % ('', ' num ', 'min/s', 'avg/s', 'max/s')
        for k, v in stats.items():
            report += sep
            s = sum(v)
            report += pat % (k, s,  min(v), s / len(v), max(v))
        report += sep
128

129 130 131
        # build summary
        summary = 'Perf : %.1f KB/s (%.1f MB)' % (speed, size)
        return (summary, report)
132

133
def main(args=None):
134
    ImportBenchmark().run()
135

136 137
if __name__ == "__main__":
    main()