pax_global_header 0000666 0000000 0000000 00000000064 13077664403 0014524 g ustar 00root root 0000000 0000000 52 comment=b27db46f69a955b8fcfc92afa0a8c51cf457cf9e
neoppod-b27db46f69a955b8fcfc92afa0a8c51cf457cf9e-neo-scripts/ 0000775 0000000 0000000 00000000000 13077664403 0023042 5 ustar 00root root 0000000 0000000 neoppod-b27db46f69a955b8fcfc92afa0a8c51cf457cf9e-neo-scripts/neo/ 0000775 0000000 0000000 00000000000 13077664403 0023623 5 ustar 00root root 0000000 0000000 neoppod-b27db46f69a955b8fcfc92afa0a8c51cf457cf9e-neo-scripts/neo/scripts/ 0000775 0000000 0000000 00000000000 13077664403 0025312 5 ustar 00root root 0000000 0000000 neoppod-b27db46f69a955b8fcfc92afa0a8c51cf457cf9e-neo-scripts/neo/scripts/__init__.py 0000664 0000000 0000000 00000000000 13077664403 0027411 0 ustar 00root root 0000000 0000000 neoppod-b27db46f69a955b8fcfc92afa0a8c51cf457cf9e-neo-scripts/neo/scripts/neoadmin.py 0000775 0000000 0000000 00000002663 13077664403 0027470 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
#
# neoadmin - run an administrator node of NEO
#
# Copyright (C) 2009-2017 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
from neo.lib import logging
from neo.lib.config import getServerOptionParser, ConfigurationManager
parser = getServerOptionParser()
parser.add_option('-u', '--uuid', help='specify an UUID to use for this ' \
'process')
defaults = dict(
bind = '127.0.0.1:9999',
masters = '127.0.0.1:10000',
)
def main(args=None):
# build configuration dict from command line options
(options, args) = parser.parse_args(args=args)
config = ConfigurationManager(defaults, options, 'admin')
# setup custom logging
logging.setup(config.getLogfile())
# and then, load and run the application
from neo.admin.app import Application
app = Application(config)
app.run()
neoppod-b27db46f69a955b8fcfc92afa0a8c51cf457cf9e-neo-scripts/neo/scripts/neoctl.py 0000775 0000000 0000000 00000003504 13077664403 0027155 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
#
# neoadmin - run an administrator node of NEO
#
# Copyright (C) 2009-2017 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
from neo.lib import logging
from neo.lib.config import getOptionParser
from neo.lib.util import parseNodeAddress
parser = getOptionParser()
parser.add_option('-a', '--address', help = 'specify the address (ip:port) ' \
'of an admin node', default = '127.0.0.1:9999')
parser.add_option('--handler', help = 'specify the connection handler')
def main(args=None):
(options, args) = parser.parse_args(args=args)
if options.address is not None:
address = parseNodeAddress(options.address, 9999)
else:
address = ('127.0.0.1', 9999)
if options.logfile:
# Contrary to daemons, we log everything to disk automatically
# because a user using -l option here:
# - is certainly debugging an issue and wants everything,
# - would not have to time to send SIGRTMIN before neoctl exits.
logging.backlog(None)
logging.setup(options.logfile)
from neo.neoctl.app import Application
ssl = options.ca, options.cert, options.key
r = Application(address, ssl=ssl if any(ssl) else None).execute(args)
if r is not None:
print r
neoppod-b27db46f69a955b8fcfc92afa0a8c51cf457cf9e-neo-scripts/neo/scripts/neolog.py 0000775 0000000 0000000 00000020236 13077664403 0027155 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
#
# neolog - read a NEO log
#
# Copyright (C) 2012-2017 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
import bz2, gzip, errno, optparse, os, signal, sqlite3, sys, time
from bisect import insort
from logging import getLevelName
comp_dict = dict(bz2=bz2.BZ2File, gz=gzip.GzipFile)
class Log(object):
_log_id = _packet_id = -1
_protocol_date = None
def __init__(self, db_path, decode_all=False, date_format=None,
filter_from=None, node_list=None):
self._date_format = '%F %T' if date_format is None else date_format
self._decode_all = decode_all
self._filter_from = filter_from
self._node_list = node_list
name = os.path.basename(db_path)
try:
name, ext = name.rsplit(os.extsep, 1)
ZipFile = comp_dict[ext]
except (KeyError, ValueError):
# WKRD: Python does not support URI so we can't open in read-only
# mode. See http://bugs.python.org/issue13773
os.stat(db_path) # do not create empty DB if file is missing
self._db = sqlite3.connect(db_path)
else:
import shutil, tempfile
with tempfile.NamedTemporaryFile() as f:
shutil.copyfileobj(ZipFile(db_path), f)
self._db = sqlite3.connect(f.name)
name = name.rsplit(os.extsep, 1)[0]
self._default_name = name
def __iter__(self):
db = self._db
try:
db.execute("BEGIN")
yield
nl = "SELECT * FROM log WHERE id>?"
np = "SELECT * FROM packet WHERE id>?"
date = self._filter_from
if date:
date = " AND date>=%f" % date
nl += date
np += date
nl = db.execute(nl, (self._log_id,))
np = db.execute(np, (self._packet_id,))
try:
p = np.next()
self._reload(p[1])
except StopIteration:
p = None
for self._log_id, date, name, level, pathname, lineno, msg in nl:
while p and p[1] < date:
yield self._packet(*p)
p = np.fetchone()
yield date, name, getLevelName(level), msg.splitlines()
if p:
yield self._packet(*p)
for p in np:
yield self._packet(*p)
finally:
db.rollback()
def _reload(self, date):
q = self._db.execute
date, text = q("SELECT * FROM protocol WHERE date<=?"
" ORDER BY date DESC", (date,)).next()
if self._protocol_date == date:
return
self._protocol_date = date
g = {}
exec bz2.decompress(text) in g
for x in 'uuid_str', 'Packets', 'PacketMalformedError':
setattr(self, x, g[x])
try:
self._next_protocol, = q("SELECT date FROM protocol WHERE date>?",
(date,)).next()
except StopIteration:
self._next_protocol = float('inf')
def _emit(self, date, name, levelname, msg_list):
if not name:
name = self._default_name
if self._node_list and name not in self._node_list:
return
prefix = self._date_format
if prefix:
d = int(date)
prefix = '%s.%04u ' % (time.strftime(prefix, time.localtime(d)),
int((date - d) * 10000))
prefix += '%-9s %-10s ' % (levelname, name)
for msg in msg_list:
print prefix + msg
def _packet(self, id, date, name, msg_id, code, peer, body):
self._packet_id = id
if self._next_protocol <= date:
self._reload(date)
try:
p = self.Packets[code]
msg = p.__name__
except KeyError:
msg = 'UnknownPacket[%u]' % code
body = None
msg = ['#0x%04x %-30s %s' % (msg_id, msg, peer)]
if body is not None:
log = getattr(p, '_neolog', None)
if log or self._decode_all:
p = p()
p._id = msg_id
p._body = body
try:
args = p.decode()
except self.PacketMalformedError:
msg.append("Can't decode packet")
else:
if log:
args, extra = log(*args)
msg += extra
if args and self._decode_all:
msg[0] += ' \t| ' + repr(args)
return date, name, 'PACKET', msg
def emit_many(log_list):
log_list = [(log, iter(log).next) for log in log_list]
for x in log_list: # try to start all transactions at the same time
x[1]()
event_list = []
for log, next in log_list:
try:
event = next()
except StopIteration:
continue
event_list.append((-event[0], next, log._emit, event))
if event_list:
event_list.sort()
while True:
key, next, emit, event = event_list.pop()
try:
next_date = - event_list[-1][0]
except IndexError:
next_date = float('inf')
try:
while event[0] <= next_date:
emit(*event)
event = next()
except IOError, e:
if e.errno == errno.EPIPE:
sys.exit(1)
raise
except StopIteration:
if not event_list:
break
else:
insort(event_list, (-event[0], next, emit, event))
def main():
parser = optparse.OptionParser()
parser.add_option('-a', '--all', action="store_true",
help='decode all packets')
parser.add_option('-d', '--date', metavar='FORMAT',
help='custom date format, according to strftime(3)')
parser.add_option('-f', '--follow', action="store_true",
help='output appended data as the file grows')
parser.add_option('-F', '--flush', action="append", type="int",
help='with -f, tell process PID to flush logs approximately N'
' seconds (see -s)', metavar='PID')
parser.add_option('-n', '--node', action="append",
help='only show log entries from the given node'
' (only useful for logs produced by threaded tests)')
parser.add_option('-s', '--sleep-interval', type="float", default=1,
help='with -f, sleep for approximately N seconds (default 1.0)'
' between iterations', metavar='N')
parser.add_option('--from', dest='filter_from', type="float",
help='show records more recent that timestamp N if N > 0,'
' or now+N if N < 0', metavar='N')
options, args = parser.parse_args()
if options.sleep_interval <= 0:
parser.error("sleep_interval must be positive")
if not args:
parser.error("no log specified")
filter_from = options.filter_from
if filter_from and filter_from < 0:
filter_from += time.time()
log_list = [Log(db_path, options.all, options.date, filter_from,
options.node)
for db_path in args]
if options.follow:
try:
pid_list = options.flush or ()
while True:
emit_many(log_list)
for pid in pid_list:
os.kill(pid, signal.SIGRTMIN)
time.sleep(options.sleep_interval)
except KeyboardInterrupt:
pass
else:
emit_many(log_list)
if __name__ == "__main__":
main()
neoppod-b27db46f69a955b8fcfc92afa0a8c51cf457cf9e-neo-scripts/neo/scripts/neomaster.py 0000775 0000000 0000000 00000003736 13077664403 0027675 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
#
# neomaster - run a master node of NEO
#
# Copyright (C) 2006-2017 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
from neo.lib import logging
from neo.lib.config import getServerOptionParser, ConfigurationManager
parser = getServerOptionParser()
parser.add_option('-u', '--uuid', help='the node UUID (testing purpose)')
parser.add_option('-r', '--replicas', help = 'replicas number')
parser.add_option('-p', '--partitions', help = 'partitions number')
parser.add_option('-A', '--autostart',
help='minimum number of pending storage nodes to automatically start'
' new cluster (to avoid unwanted recreation of the cluster,'
' this should be the total number of storage nodes)')
parser.add_option('-C', '--upstream-cluster',
help='the name of cluster to backup')
parser.add_option('-M', '--upstream-masters',
help='list of master nodes in cluster to backup')
defaults = dict(
bind = '127.0.0.1:10000',
masters = '',
replicas = 0,
partitions = 100,
)
def main(args=None):
# build configuration dict from command line options
(options, args) = parser.parse_args(args=args)
config = ConfigurationManager(defaults, options, 'master')
# setup custom logging
logging.setup(config.getLogfile())
# and then, load and run the application
from neo.master.app import Application
app = Application(config)
app.run()
neoppod-b27db46f69a955b8fcfc92afa0a8c51cf457cf9e-neo-scripts/neo/scripts/neomigrate.py 0000775 0000000 0000000 00000005130 13077664403 0030020 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
#
# neomaster - run a master node of NEO
#
# Copyright (C) 2006-2017 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
from neo.lib.config import getOptionParser
import time
import os
# register options
parser = getOptionParser()
parser.add_option('-s', '--source', help='the source database')
parser.add_option('-d', '--destination', help='the destination database')
parser.add_option('-c', '--cluster', help='the NEO cluster name')
def main(args=None):
# parse options
(options, args) = parser.parse_args(args=args)
source = options.source or None
destination = options.destination or None
cluster = options.cluster or None
# check options
if source is None or destination is None:
raise RuntimeError('Source and destination databases must be supplied')
if cluster is None:
raise RuntimeError('The NEO cluster name must be supplied')
# open storages
from ZODB.FileStorage import FileStorage
from neo.client.Storage import Storage as NEOStorage
if os.path.exists(source):
print("WARNING: This is not the recommended way to import data to NEO:"
" you should use Imported backend instead.\n"
"NEO also does not implement IStorageRestoreable interface,"
" which means that undo information is not preserved when using"
" this tool: conflict resolution could happen when undoing an"
" old transaction.")
src = FileStorage(file_name=source, read_only=True)
dst = NEOStorage(master_nodes=destination, name=cluster,
logfile=options.logfile)
else:
src = NEOStorage(master_nodes=source, name=cluster,
logfile=options.logfile, read_only=True)
dst = FileStorage(file_name=destination)
# do the job
print "Migrating from %s to %s" % (source, destination)
start = time.time()
dst.copyTransactionsFrom(src)
elapsed = time.time() - start
print "Migration done in %3.5f" % (elapsed, )
neoppod-b27db46f69a955b8fcfc92afa0a8c51cf457cf9e-neo-scripts/neo/scripts/neostorage.py 0000775 0000000 0000000 00000004441 13077664403 0030040 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
#
# neostorage - run a storage node of NEO
#
# Copyright (C) 2006-2017 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
from neo.lib import logging
from neo.lib.config import getServerOptionParser, ConfigurationManager
parser = getServerOptionParser()
parser.add_option('-u', '--uuid', help='specify an UUID to use for this ' \
'process. Previously assigned UUID takes precedence (ie ' \
'you should always use --reset with this switch)')
parser.add_option('-a', '--adapter', help = 'database adapter to use')
parser.add_option('-d', '--database', help = 'database connections string')
parser.add_option('-e', '--engine', help = 'database engine')
parser.add_option('-w', '--wait', help='seconds to wait for backend to be '
'available, before erroring-out (-1 = infinite)', type='float', default=0)
parser.add_option('--reset', action='store_true',
help='remove an existing database if any, and exit')
defaults = dict(
bind = '127.0.0.1',
masters = '127.0.0.1:10000',
adapter = 'MySQL',
)
def main(args=None):
# TODO: Forbid using "reset" along with any unneeded argument.
# "reset" is too dangerous to let user a chance of accidentally
# letting it slip through in a long option list.
# We should drop support configuration files to make such check useful.
(options, args) = parser.parse_args(args=args)
config = ConfigurationManager(defaults, options, 'storage')
# setup custom logging
logging.setup(config.getLogfile())
# and then, load and run the application
from neo.storage.app import Application
app = Application(config)
if not config.getReset():
app.run()
neoppod-b27db46f69a955b8fcfc92afa0a8c51cf457cf9e-neo-scripts/neo/scripts/runner.py 0000775 0000000 0000000 00000032743 13077664403 0027211 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
#
# Copyright (C) 2009-2017 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
import traceback
import unittest
import time
import sys
import os
import re
from collections import Counter, defaultdict
from cStringIO import StringIO
from fnmatch import fnmatchcase
from unittest.runner import _WritelnDecorator
if filter(re.compile(r'--coverage$|-\w*c').match, sys.argv[1:]):
# Start coverage as soon as possible.
import coverage
coverage = coverage.Coverage()
coverage.neotestrunner = []
coverage.start()
from neo.lib import logging
from neo.tests import getTempDirectory, NeoTestBase, Patch, \
__dict__ as neo_tests__dict__
from neo.tests.benchmark import BenchmarkRunner
# list of test modules
# each of them have to import its TestCase classes
UNIT_TEST_MODULES = [
# generic parts
'neo.tests.testConnection',
'neo.tests.testHandler',
'neo.tests.testNodes',
'neo.tests.testUtil',
'neo.tests.testPT',
# master application
'neo.tests.master.testClientHandler',
'neo.tests.master.testMasterApp',
'neo.tests.master.testMasterPT',
'neo.tests.master.testRecovery',
'neo.tests.master.testStorageHandler',
'neo.tests.master.testTransactions',
# storage application
'neo.tests.storage.testClientHandler',
'neo.tests.storage.testMasterHandler',
'neo.tests.storage.testStorageApp',
'neo.tests.storage.testStorage' + os.getenv('NEO_TESTS_ADAPTER', 'SQLite'),
'neo.tests.storage.testTransactions',
# client application
'neo.tests.client.testClientApp',
'neo.tests.client.testMasterHandler',
'neo.tests.client.testZODBURI',
# light functional tests
'neo.tests.threaded.test',
'neo.tests.threaded.testImporter',
'neo.tests.threaded.testReplication',
'neo.tests.threaded.testSSL',
]
FUNC_TEST_MODULES = [
'neo.tests.functional.testMaster',
'neo.tests.functional.testClient',
'neo.tests.functional.testCluster',
'neo.tests.functional.testStorage',
]
ZODB_TEST_MODULES = [
('neo.tests.zodb.testBasic', 'check'),
('neo.tests.zodb.testConflict', 'check'),
('neo.tests.zodb.testHistory', 'check'),
('neo.tests.zodb.testIterator', 'check'),
('neo.tests.zodb.testMT', 'check'),
('neo.tests.zodb.testPack', 'check'),
('neo.tests.zodb.testPersistent', 'check'),
('neo.tests.zodb.testReadOnly', 'check'),
('neo.tests.zodb.testRevision', 'check'),
#('neo.tests.zodb.testRecovery', 'check'),
('neo.tests.zodb.testSynchronization', 'check'),
# ('neo.tests.zodb.testVersion', 'check'),
('neo.tests.zodb.testUndo', 'check'),
('neo.tests.zodb.testZODB', 'check'),
]
class NeoTestRunner(unittest.TextTestResult):
""" Custom result class to build report with statistics per module """
def __init__(self, title, verbosity):
super(NeoTestRunner, self).__init__(
_WritelnDecorator(sys.stderr), False, verbosity)
self._title = title
self.modulesStats = {}
self.failedImports = {}
self.run_dict = defaultdict(int)
self.time_dict = defaultdict(int)
self.temp_directory = getTempDirectory()
def wasSuccessful(self):
return not (self.failures or self.errors or self.unexpectedSuccesses)
def run(self, name, modules, only):
suite = unittest.TestSuite()
loader = unittest.TestLoader()
if only:
exclude = only[0] == '!'
test_only = only[exclude + 1:]
only = only[exclude]
if test_only:
def getTestCaseNames(testCaseClass):
tests = loader.__class__.getTestCaseNames(
loader, testCaseClass)
x = testCaseClass.__name__ + '.'
return [t for t in tests
if exclude != any(fnmatchcase(x + t, o)
for o in test_only)]
loader.getTestCaseNames = getTestCaseNames
if not only:
only = '*'
else:
print '\n', name
for test_module in modules:
# load prefix if supplied
if isinstance(test_module, tuple):
test_module, loader.testMethodPrefix = test_module
if only and not (exclude and test_only or
exclude != fnmatchcase(test_module, only)):
continue
try:
test_module = __import__(test_module, globals(), locals(), ['*'])
except ImportError, err:
self.failedImports[test_module] = err
print "Import of %s failed : %s" % (test_module, err)
traceback.print_exc()
continue
suite.addTests(loader.loadTestsFromModule(test_module))
try:
suite.run(self)
finally:
# Workaround weird behaviour of Python.
self._previousTestClass = None
def startTest(self, test):
super(NeoTestRunner, self).startTest(test)
self.run_dict[test.__class__.__module__] += 1
self.start_time = time.time()
def stopTest(self, test):
self.time_dict[test.__class__.__module__] += \
time.time() - self.start_time
super(NeoTestRunner, self).stopTest(test)
def _buildSummary(self, add_status):
unexpected_count = len(self.errors) + len(self.failures) \
+ len(self.unexpectedSuccesses)
expected_count = len(self.expectedFailures)
success = self.testsRun - unexpected_count - expected_count
add_status('Directory', self.temp_directory)
if self.testsRun:
add_status('Status', '%.3f%%' % (success * 100.0 / self.testsRun))
for var in os.environ:
if var.startswith('NEO_TEST'):
add_status(var, os.environ[var])
# visual
header = "%25s | run | unexpected | expected | skipped | time \n" % 'Test Module'
separator = "%25s-+-------+------------+----------+---------+----------\n" % ('-' * 25)
format = "%25s | %3s | %3s | %3s | %3s | %6.2fs \n"
group_f = "%25s | | | | | \n"
# header
s = ' ' * 30 + ' NEO TESTS REPORT\n\n' + header + separator
group = None
unexpected = Counter(x[0].__class__.__module__
for x in (self.errors, self.failures)
for x in x)
unexpected.update(x.__class__.__module__
for x in self.unexpectedSuccesses)
expected = Counter(x[0].__class__.__module__
for x in self.expectedFailures)
skipped = Counter(x[0].__class__.__module__
for x in self.skipped)
total_time = 0
# for each test case
for k, v in sorted(self.run_dict.iteritems()):
# display group below its content
_group, name = k.rsplit('.', 1)
if _group != group:
if group:
s += separator + group_f % group + separator
group = _group
t = self.time_dict[k]
total_time += t
s += format % (name.lstrip('test'), v, unexpected.get(k, '.'),
expected.get(k, '.'), skipped.get(k, '.'), t)
# the last group
s += separator + group_f % group + separator
# the final summary
s += format % ("Summary", self.testsRun, unexpected_count or '.',
expected_count or '.', len(self.skipped) or '.',
total_time) + separator + '\n'
return "%s Tests, %s Failed" % (self.testsRun, unexpected_count), s
def buildReport(self, add_status):
subject, summary = self._buildSummary(add_status)
body = StringIO()
body.write(summary)
for test in self.unexpectedSuccesses:
body.write("UNEXPECTED SUCCESS: %s\n" % self.getDescription(test))
self.stream = _WritelnDecorator(body)
self.printErrorList('ERROR', self.errors)
self.printErrorList('FAIL', self.failures)
return subject, body.getvalue()
class TestRunner(BenchmarkRunner):
def add_options(self, parser):
parser.add_option('-c', '--coverage', action='store_true',
help='Enable coverage')
parser.add_option('-C', '--cov-unit', action='store_true',
help='Same as -c but output 1 file per test,'
' in the temporary test directory')
parser.add_option('-L', '--log', action='store_true',
help='Force all logs to be emitted immediately and keep'
' packet body in logs of successful threaded tests')
parser.add_option('-l', '--loop', type='int', default=1,
help='Repeat tests several times')
parser.add_option('-f', '--functional', action='store_true',
help='Functional tests')
parser.add_option('-u', '--unit', action='store_true',
help='Unit & threaded tests')
parser.add_option('-z', '--zodb', action='store_true',
help='ZODB test suite running on a NEO')
parser.add_option('-v', '--verbose', action='store_true',
help='Verbose output')
parser.usage += " [[!] module [test...]]"
parser.format_epilog = lambda _: """
Positional:
Filter by given module/test. These arguments are shell patterns.
This implies -ufz if none of this option is passed.
Environment Variables:
NEO_TESTS_ADAPTER Default is SQLite for threaded clusters,
MySQL otherwise.
MySQL specific:
NEO_DB_SOCKET default: libmysqlclient.so default
NEO_DB_PREFIX default: %(DB_PREFIX)s
NEO_DB_ADMIN default: %(DB_ADMIN)s
NEO_DB_PASSWD default: %(DB_PASSWD)s
NEO_DB_USER default: %(DB_USER)s
ZODB tests:
NEO_TEST_ZODB_FUNCTIONAL Clusters are threaded by default. If true,
they are built like in functional tests.
NEO_TEST_ZODB_MASTERS default: 1
NEO_TEST_ZODB_PARTITIONS default: 1
NEO_TEST_ZODB_REPLICAS default: 0
NEO_TEST_ZODB_STORAGES default: 1
""" % neo_tests__dict__
def load_options(self, options, args):
if options.coverage and options.cov_unit:
sys.exit('-c conflicts with -C')
if not (options.unit or options.functional or options.zodb):
if not args:
sys.exit('Nothing to run, please give one of -f, -u, -z')
options.unit = options.functional = options.zodb = True
return dict(
log = options.log,
loop = options.loop,
unit = options.unit,
functional = options.functional,
zodb = options.zodb,
verbosity = 2 if options.verbose else 1,
coverage = options.coverage,
cov_unit = options.cov_unit,
only = args,
)
def start(self):
config = self._config
logging.backlog(max_packet=1<<20,
**({'max_size': None} if config.log else {}))
only = config.only
# run requested tests
runner = NeoTestRunner(config.title or 'Neo', config.verbosity)
if config.cov_unit:
from coverage import Coverage
cov_dir = runner.temp_directory + '/coverage'
os.mkdir(cov_dir)
@Patch(NeoTestBase)
def setUp(orig, self):
orig(self)
self.__coverage = Coverage('%s/%s' % (cov_dir, self.id()))
self.__coverage.start()
@Patch(NeoTestBase)
def _tearDown(orig, self, success):
self.__coverage.stop()
self.__coverage.save()
del self.__coverage
orig(self, success)
try:
for _ in xrange(config.loop):
if config.unit:
runner.run('Unit tests', UNIT_TEST_MODULES, only)
if config.functional:
runner.run('Functional tests', FUNC_TEST_MODULES, only)
if config.zodb:
runner.run('ZODB tests', ZODB_TEST_MODULES, only)
except KeyboardInterrupt:
config['mail_to'] = None
traceback.print_exc()
if config.coverage:
coverage.stop()
if coverage.neotestrunner:
coverage.combine(coverage.neotestrunner)
coverage.save()
if runner.dots:
print
# build report
if only and not config.mail_to:
runner._buildSummary = lambda *args: (
runner.__class__._buildSummary(runner, *args)[0], '')
self.build_report = str
self._successful = runner.wasSuccessful()
return runner.buildReport(self.add_status)
def main(args=None):
runner = TestRunner()
runner.run()
return sys.exit(not runner.was_successful())
if __name__ == "__main__":
main()
neoppod-b27db46f69a955b8fcfc92afa0a8c51cf457cf9e-neo-scripts/neo/scripts/simple.py 0000664 0000000 0000000 00000004267 13077664403 0027166 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
#
# Copyright (C) 2011-2017 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
import inspect, random
from logging import getLogger, INFO
from optparse import OptionParser
from neo.lib import logging
from neo.tests import functional
logging.backlog()
del logging.default_root_handler.handle
def main():
args, _, _, defaults = inspect.getargspec(functional.NEOCluster.__init__)
option_list = zip(args[-len(defaults):], defaults)
parser = OptionParser(usage="%prog [options] [db...]",
description="Quickly setup a simple NEO cluster for testing purpose.")
parser.add_option('--seed', help="settings like node ports/uuids and"
" cluster name are random: pass any string to initialize the RNG")
defaults = {}
for option, default in sorted(option_list):
kw = {}
if type(default) is bool:
kw['action'] = "store_true"
defaults[option] = False
elif default is not None:
defaults[option] = default
if isinstance(default, int):
kw['type'] = "int"
parser.add_option('--' + option, **kw)
parser.set_defaults(**defaults)
options, args = parser.parse_args()
if options.seed:
functional.random = random.Random(options.seed)
getLogger().setLevel(INFO)
cluster = functional.NEOCluster(args, **{x: getattr(options, x)
for x, _ in option_list})
try:
cluster.run()
logging.info("Cluster running ...")
cluster.waitAll()
finally:
cluster.stop()
if __name__ == "__main__":
main()