Commit 7fc4ec66 by Kirill Smelkov

tests: Allow to test with ZEO & NEO ZODB storages

Previously we were always testing with DBs backed up by FileStorage. Now
we provide a way to run the testsuite with user selected storage
backend:

    $ WENDELIN_CORE_TEST_DB="<fs>"   make test.py     # test with temporary db with FileStorage
    $ WENDELIN_CORE_TEST_DB="<zeo>"  make test.py     # ----------//---------- with ZEO
    $ WENDELIN_CORE_TEST_DB="<neo>"  make test.py     # ----------//---------- with NEO

    $ WENDELIN_CORE_TEST_DB=neo://db@master  make test.py     # test with externally provided DB

Default is still to run tests with FileStorage.

/cc @jm
1 parent 92c3bbfa
......@@ -17,13 +17,24 @@
# See COPYING file for full licensing terms.
from wendelin.bigarray.array_zodb import ZBigArray
from wendelin.bigfile.tests.test_filezodb import kkey, cacheInfo
from wendelin.lib.zodb import dbopen, dbclose
from wendelin.lib.zodb import dbclose
from wendelin.lib.testing import getTestDB
from persistent import UPTODATE
import transaction
from numpy import dtype, uint8, all, array_equal
def test_zbigarray(tmpdir):
root = dbopen('%s/1.fs' % tmpdir)
testdb = None
def setup_module():
global testdb
testdb = getTestDB()
testdb.setup()
def teardown_module():
testdb.teardown()
def test_zbigarray():
root = testdb.dbopen()
root['zarray'] = ZBigArray((16*1024*1024,), uint8)
transaction.commit()
......@@ -31,7 +42,7 @@ def test_zbigarray(tmpdir):
del root
root = dbopen('%s/1.fs' % tmpdir)
root = testdb.dbopen()
A = root['zarray']
assert isinstance(A, ZBigArray)
......@@ -75,7 +86,7 @@ def test_zbigarray(tmpdir):
del root, a,b, A
root = dbopen('%s/1.fs' % tmpdir)
root = testdb.dbopen()
A = root['zarray']
assert isinstance(A, ZBigArray)
......@@ -111,7 +122,7 @@ def test_zbigarray(tmpdir):
del root, a, A, db
root = dbopen('%s/1.fs' % tmpdir)
root = testdb.dbopen()
A = root['zarray']
assert isinstance(A, ZBigArray)
......@@ -143,7 +154,7 @@ def test_zbigarray(tmpdir):
del root, a, b, A
root = dbopen('%s/1.fs' % tmpdir)
root = testdb.dbopen()
A = root['zarray']
assert isinstance(A, ZBigArray)
......
......@@ -3,13 +3,11 @@
# TODO text
from wendelin.bigfile.file_zodb import ZBigFile
from wendelin.lib.mem import memset
from wendelin.lib.testing import Adler32, nulladler32_bysize, ffadler32_bysize
from wendelin.lib.zodb import dbopen, dbclose
from wendelin.lib.testing import getTestDB, Adler32, nulladler32_bysize, ffadler32_bysize
from wendelin.lib.zodb import dbclose
import transaction
from tempfile import mkdtemp
from shutil import rmtree
tmpd = None
testdb = None
from wendelin.bigfile.tests.bench_0virtmem import filesize, blksize # to get comparable timings
blen = filesize // blksize
nulladler32 = nulladler32_bysize(blen * blksize)
......@@ -17,10 +15,11 @@ ffadler32 = ffadler32_bysize(blen * blksize)
def setup_module():
global tmpd
tmpd = mkdtemp('', 'bigzodb.')
global testdb
testdb = getTestDB()
testdb.setup()
root = dbopen('%s/1.fs' % tmpd)
root = testdb.dbopen()
root['zfile'] = ZBigFile(blksize)
transaction.commit()
......@@ -28,14 +27,14 @@ def setup_module():
def teardown_module():
rmtree(tmpd)
testdb.teardown()
# NOTE runs before _writeff
def bench_bigz_readhole(): _bench_bigz_hash(Adler32, nulladler32)
def bench_bigz_writeff():
root = dbopen('%s/1.fs' % tmpd)
root = testdb.dbopen()
f = root['zfile']
fh = f.fileh_open() # TODO + ram
vma = fh.mmap(0, blen) # XXX assumes blksize == pagesize
......@@ -50,7 +49,7 @@ def bench_bigz_writeff():
def _bench_bigz_hash(hasher, expect):
root = dbopen('%s/1.fs' % tmpd)
root = testdb.dbopen()
f = root['zfile']
fh = f.fileh_open() # TODO + ram
vma = fh.mmap(0, blen) # XXX assumes blksize == pagesize
......
......@@ -17,31 +17,31 @@
# See COPYING file for full licensing terms.
from wendelin.bigfile.file_zodb import LivePersistent, ZBigFile
from wendelin.bigfile import ram_reclaim
from wendelin.lib.zodb import dbopen as z_dbopen, dbclose
from wendelin.lib.zodb import dbclose
from wendelin.lib.testing import getTestDB
from persistent import UPTODATE, GHOST
import transaction
from tempfile import mkdtemp
from shutil import rmtree
from numpy import ndarray, array_equal, uint8, zeros
from pytest import raises
from six.moves import range as xrange
tmpd = None
testdb = None
blksize = 2*1024*1024 # XXX hardcoded
blen = 32 # 32*2 = 64MB # TODO set it higher by default ?
def dbopen():
return z_dbopen('%s/1.fs' % tmpd)
return testdb.dbopen()
def setup_module():
global tmpd
tmpd = mkdtemp('', 'bigzodb.')
global testdb
testdb = getTestDB()
testdb.setup()
def teardown_module():
rmtree(tmpd)
testdb.teardown()
......
......@@ -16,10 +16,15 @@
#
# See COPYING file for full licensing terms.
from wendelin.lib.zodb import dbstoropen
from zlib import adler32
from struct import pack
from tempfile import mkdtemp
from shutil import rmtree
from ZODB import DB
import codecs
import math
import os
# hashlib-like interface to adler32
class Adler32:
......@@ -153,3 +158,131 @@ def nulladler32_bysize(size): return _nulladler32_byorder [ilog2_exact(size)]
def nullmd5_bysize(size): return _nullmd5_byorder [ilog2_exact(size)]
def ffadler32_bysize(size): return _ffadler32_byorder [ilog2_exact(size)]
# ----------------------------------------
# interface to setup/get to a database to use with tests
class TestDB_Base(object):
def setup(self):
raise NotImplementedError()
def teardown(self):
raise NotImplementedError()
def getZODBStorage(self):
raise NotImplementedError()
# like wendelin.lib.zodb.dbopen()
def dbopen(self):
stor = self.getZODBStorage()
db = DB(stor)
conn = db.open()
root = conn.root()
return root
# by default how db was specified is stored for reuse
def __init__(self, dburi):
self.dburi = dburi
# FileStorage for tests
class TestDB_FileStorage(TestDB_Base):
def setup(self):
self.tmpd = mkdtemp('', 'testdb_fs.')
def teardown(self):
rmtree(self.tmpd)
def getZODBStorage(self):
return dbstoropen('%s/1.fs' % self.tmpd)
# ZEO for tests
class TestDB_ZEO(TestDB_Base):
def __init__(self, dburi):
super(TestDB_ZEO, self).__init__(dburi)
from ZEO.tests import forker
self.zeo_forker = forker
def setup(self):
port = self.zeo_forker.get_port()
zconf = self.zeo_forker.ZEOConfig(('', port))
self.addr, self.adminaddr, self.pid, self.path = \
self.zeo_forker.start_zeo_server(zeo_conf=zconf, port=port)
def teardown(self):
self.zeo_forker.shutdown_zeo_server(self.adminaddr)
os.waitpid(self.pid, 0)
def getZODBStorage(self):
from ZEO.ClientStorage import ClientStorage
return ClientStorage(self.addr)
# NEO for tests
class TestDB_NEO(TestDB_Base):
def __init__(self, dburi):
super(TestDB_NEO, self).__init__(dburi)
from neo.tests.functional import NEOCluster
self.cluster = NEOCluster(['1'], adapter='SQLite')
def setup(self):
self.cluster.start()
self.cluster.expectClusterRunning()
def teardown(self):
self.cluster.stop()
def getZODBStorage(self):
return self.cluster.getZODBStorage()
# test adapter to some external database
class TestDB_External(TestDB_Base):
# we do not create/destroy it - the database managed not by us
def setup(self): pass
def teardown(self): pass
def getZODBStorage(self):
return dbstoropen(self.dburi)
# get a database for tests.
#
# either it is a temporary database with selected storage, or some other
# external database defined by its uri.
#
# db selection is done via
#
# WENDELIN_CORE_TEST_DB
#
# environment variable:
#
# <fs> temporary db
# <zeo> with corresponding
# <neo> storage
#
# everything else - considered as external db uri.
#
# default: <fs>
DB_4TESTS_REGISTRY = {
'<fs>': TestDB_FileStorage,
'<zeo>': TestDB_ZEO,
'<neo>': TestDB_NEO,
}
def getTestDB():
testdb_uri = os.environ.get('WENDELIN_CORE_TEST_DB', '<fs>')
testdb_factory = DB_4TESTS_REGISTRY.get(testdb_uri, TestDB_External)
testdb = testdb_factory(testdb_uri)
return testdb
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!