tests: Allow to test with ZEO & NEO ZODB storages

Previously we were always testing with DBs backed up by FileStorage. Now
we provide a way to run the testsuite with user selected storage
backend:

    $ WENDELIN_CORE_TEST_DB="<fs>"   make test.py     # test with temporary db with FileStorage
    $ WENDELIN_CORE_TEST_DB="<zeo>"  make test.py     # ----------//---------- with ZEO
    $ WENDELIN_CORE_TEST_DB="<neo>"  make test.py     # ----------//---------- with NEO

    $ WENDELIN_CORE_TEST_DB=neo://db@master  make test.py     # test with externally provided DB

Default is still to run tests with FileStorage.

/cc @jm

bigarray/tests/test_arrayzodb.py

@@ -17,13 +17,24 @@
 # See COPYING file for full licensing terms.
 from wendelin.bigarray.array_zodb import ZBigArray
 from wendelin.bigfile.tests.test_filezodb import kkey, cacheInfo
-from wendelin.lib.zodb import dbopen, dbclose
+from wendelin.lib.zodb import dbclose
+from wendelin.lib.testing import getTestDB
 from persistent import UPTODATE
 import transaction
 from numpy import dtype, uint8, all, array_equal
 
-def test_zbigarray(tmpdir):
-    root = dbopen('%s/1.fs' % tmpdir)
+testdb = None
+def setup_module():
+    global testdb
+    testdb = getTestDB()
+    testdb.setup()
+
+def teardown_module():
+    testdb.teardown()
+
+
+def test_zbigarray():
+    root = testdb.dbopen()
     root['zarray'] = ZBigArray((16*1024*1024,), uint8)
     transaction.commit()
 
@@ -31,7 +42,7 @@ def test_zbigarray(tmpdir):
     del root
 
 
-    root = dbopen('%s/1.fs' % tmpdir)
+    root = testdb.dbopen()
     A = root['zarray']
 
     assert isinstance(A, ZBigArray)
@@ -75,7 +86,7 @@ def test_zbigarray(tmpdir):
     del root, a,b, A
 
 
-    root = dbopen('%s/1.fs' % tmpdir)
+    root = testdb.dbopen()
     A = root['zarray']
 
     assert isinstance(A, ZBigArray)
@@ -111,7 +122,7 @@ def test_zbigarray(tmpdir):
     del root, a, A, db
 
 
-    root = dbopen('%s/1.fs' % tmpdir)
+    root = testdb.dbopen()
     A = root['zarray']
 
     assert isinstance(A, ZBigArray)
@@ -143,7 +154,7 @@ def test_zbigarray(tmpdir):
     del root, a, b, A
 
 
-    root = dbopen('%s/1.fs' % tmpdir)
+    root = testdb.dbopen()
     A = root['zarray']
 
     assert isinstance(A, ZBigArray)

bigfile/tests/bench_1filezodb.py

@@ -3,13 +3,11 @@
 # TODO text
 from wendelin.bigfile.file_zodb import ZBigFile
 from wendelin.lib.mem import memset
-from wendelin.lib.testing import Adler32, nulladler32_bysize, ffadler32_bysize
-from wendelin.lib.zodb import dbopen, dbclose
+from wendelin.lib.testing import getTestDB, Adler32, nulladler32_bysize, ffadler32_bysize
+from wendelin.lib.zodb import dbclose
 import transaction
-from tempfile import mkdtemp
-from shutil import rmtree
 
-tmpd = None
+testdb = None
 from wendelin.bigfile.tests.bench_0virtmem import filesize, blksize # to get comparable timings
 blen = filesize // blksize
 nulladler32 = nulladler32_bysize(blen * blksize)
@@ -17,10 +15,11 @@ ffadler32   = ffadler32_bysize(blen * blksize)
 
 
 def setup_module():
-    global tmpd
-    tmpd = mkdtemp('', 'bigzodb.')
+    global testdb
+    testdb = getTestDB()
+    testdb.setup()
 
-    root = dbopen('%s/1.fs' % tmpd)
+    root = testdb.dbopen()
     root['zfile'] = ZBigFile(blksize)
     transaction.commit()
 
@@ -28,14 +27,14 @@ def setup_module():
 
 
 def teardown_module():
-    rmtree(tmpd)
+    testdb.teardown()
 
 
 # NOTE runs before _writeff
 def bench_bigz_readhole():  _bench_bigz_hash(Adler32,   nulladler32)
 
 def bench_bigz_writeff():
-    root = dbopen('%s/1.fs' % tmpd)
+    root = testdb.dbopen()
     f   = root['zfile']
     fh  = f.fileh_open()    # TODO + ram
     vma = fh.mmap(0, blen)  # XXX assumes blksize == pagesize
@@ -50,7 +49,7 @@ def bench_bigz_writeff():
 
 
 def _bench_bigz_hash(hasher, expect):
-    root = dbopen('%s/1.fs' % tmpd)
+    root = testdb.dbopen()
     f   = root['zfile']
     fh  = f.fileh_open()    # TODO + ram
     vma = fh.mmap(0, blen)  # XXX assumes blksize == pagesize

bigfile/tests/test_filezodb.py

@@ -17,31 +17,31 @@
 # See COPYING file for full licensing terms.
 from wendelin.bigfile.file_zodb import LivePersistent, ZBigFile
 from wendelin.bigfile import ram_reclaim
-from wendelin.lib.zodb import dbopen as z_dbopen, dbclose
+from wendelin.lib.zodb import dbclose
+from wendelin.lib.testing import getTestDB
 from persistent import UPTODATE, GHOST
 import transaction
-from tempfile import mkdtemp
-from shutil import rmtree
 from numpy import ndarray, array_equal, uint8, zeros
 
 from pytest import raises
 from six.moves import range as xrange
 
 
-tmpd = None
+testdb = None
 blksize  = 2*1024*1024   # XXX hardcoded
 blen     = 32            # 32*2 = 64MB      # TODO set it higher by default ?
 
 
 def dbopen():
-    return z_dbopen('%s/1.fs' % tmpd)
+    return testdb.dbopen()
 
 def setup_module():
-    global tmpd
-    tmpd = mkdtemp('', 'bigzodb.')
+    global testdb
+    testdb = getTestDB()
+    testdb.setup()
 
 def teardown_module():
-    rmtree(tmpd)
+    testdb.teardown()
 
 
 

lib/testing.py

@@ -16,10 +16,15 @@
 #
 # See COPYING file for full licensing terms.
 
+from wendelin.lib.zodb import dbstoropen
 from zlib import adler32
 from struct import pack
+from tempfile import mkdtemp
+from shutil import rmtree
+from ZODB import DB
 import codecs
 import math
+import os
 
 # hashlib-like interface to adler32
 class Adler32:
@@ -153,3 +158,131 @@ def nulladler32_bysize(size):   return _nulladler32_byorder [ilog2_exact(size)]
 def nullmd5_bysize(size):       return _nullmd5_byorder     [ilog2_exact(size)]
 
 def ffadler32_bysize(size):     return _ffadler32_byorder   [ilog2_exact(size)]
+
+
+
+# ----------------------------------------
+
+
+# interface to setup/get to a database to use with tests
+class TestDB_Base(object):
+    def setup(self):
+        raise NotImplementedError()
+    def teardown(self):
+        raise NotImplementedError()
+    def getZODBStorage(self):
+        raise NotImplementedError()
+
+
+    # like wendelin.lib.zodb.dbopen()
+    def dbopen(self):
+        stor = self.getZODBStorage()
+        db   = DB(stor)
+        conn = db.open()
+        root = conn.root()
+        return root
+
+    # by default how db was specified is stored for reuse
+    def __init__(self, dburi):
+        self.dburi = dburi
+
+
+
+# FileStorage for tests
+class TestDB_FileStorage(TestDB_Base):
+
+    def setup(self):
+        self.tmpd = mkdtemp('', 'testdb_fs.')
+
+    def teardown(self):
+        rmtree(self.tmpd)
+
+    def getZODBStorage(self):
+        return dbstoropen('%s/1.fs' % self.tmpd)
+
+
+# ZEO for tests
+class TestDB_ZEO(TestDB_Base):
+
+    def __init__(self, dburi):
+        super(TestDB_ZEO, self).__init__(dburi)
+        from ZEO.tests import forker
+        self.zeo_forker = forker
+
+    def setup(self):
+        port  = self.zeo_forker.get_port()
+        zconf = self.zeo_forker.ZEOConfig(('', port))
+        self.addr, self.adminaddr, self.pid, self.path = \
+                self.zeo_forker.start_zeo_server(zeo_conf=zconf, port=port)
+
+    def teardown(self):
+        self.zeo_forker.shutdown_zeo_server(self.adminaddr)
+        os.waitpid(self.pid, 0)
+
+    def getZODBStorage(self):
+        from ZEO.ClientStorage import ClientStorage
+        return ClientStorage(self.addr)
+
+
+# NEO for tests
+class TestDB_NEO(TestDB_Base):
+
+    def __init__(self, dburi):
+        super(TestDB_NEO, self).__init__(dburi)
+        from neo.tests.functional import NEOCluster
+        self.cluster = NEOCluster(['1'], adapter='SQLite')
+
+    def setup(self):
+        self.cluster.start()
+        self.cluster.expectClusterRunning()
+
+    def teardown(self):
+        self.cluster.stop()
+
+    def getZODBStorage(self):
+        return self.cluster.getZODBStorage()
+
+
+
+# test adapter to some external database
+class TestDB_External(TestDB_Base):
+
+    # we do not create/destroy it - the database managed not by us
+    def setup(self):    pass
+    def teardown(self): pass
+
+    def getZODBStorage(self):
+        return dbstoropen(self.dburi)
+
+
+
+# get a database for tests.
+#
+# either it is a temporary database with selected storage, or some other
+# external database defined by its uri.
+#
+# db selection is done via
+#
+#   WENDELIN_CORE_TEST_DB
+#
+# environment variable:
+#
+#   <fs>        temporary db
+#   <zeo>       with corresponding
+#   <neo>       storage
+#
+#   everything else - considered as external db uri.
+#
+#   default: <fs>
+DB_4TESTS_REGISTRY = {
+    '<fs>':     TestDB_FileStorage,
+    '<zeo>':    TestDB_ZEO,
+    '<neo>':    TestDB_NEO,
+}
+
+def getTestDB():
+    testdb_uri = os.environ.get('WENDELIN_CORE_TEST_DB', '<fs>')
+    testdb_factory = DB_4TESTS_REGISTRY.get(testdb_uri, TestDB_External)
+
+    testdb = testdb_factory(testdb_uri)
+    return testdb