Back porting fixes from zodb4 to specify a `keep' flag on the

storage's artifacts.  Some tests expect to reconnect in read-only mode
to a previously created storage.

src/ZEO/tests/ConnectionTests.py

@@ -42,15 +42,12 @@ class DummyDB:
         pass
 
 
-class ConnectionTests(StorageTestBase):
-    """Tests that explicitly manage the server process.
-
-    To test the cache or re-connection, these test cases explicit
-    start and stop a ZEO storage server.
-    """
+class CommonSetupTearDown(StorageTestBase):
+    """Common boilerplate"""
 
     __super_setUp = StorageTestBase.setUp
     __super_tearDown = StorageTestBase.tearDown
+    keep = 0
 
     def setUp(self):
         """Test setup for connection tests.
@@ -119,7 +116,8 @@ class ConnectionTests(StorageTestBase):
                  (create, index, read_only, addr))
         path = "%s.%d" % (self.file, index)
         conf = self.getConfig(path, create, read_only)
-        zeoport, adminaddr, pid = forker.start_zeo_server(conf, addr, ro_svr)
+        zeoport, adminaddr, pid = forker.start_zeo_server(
+            conf, addr, ro_svr, self.keep)
         self._pids.append(pid)
         self._servers.append(adminaddr)
 
@@ -151,6 +149,14 @@ class ConnectionTests(StorageTestBase):
             if now > giveup:
                 self.fail("timed out waiting for storage to disconnect")
 
+
+class ConnectionTests(CommonSetupTearDown):
+    """Tests that explicitly manage the server process.
+
+    To test the cache or re-connection, these test cases explicit
+    start and stop a ZEO storage server.
+    """
+
     def checkMultipleAddresses(self):
         for i in range(4):
             self._newAddr()
@@ -189,21 +195,6 @@ class ConnectionTests(StorageTestBase):
         # Stores should fail here
         self.assertRaises(ReadOnlyError, self._dostore)
 
-    def checkReadOnlyStorage(self):
-        # Open a read-only client to a read-only *storage*; stores fail
-
-        # We don't want the read-write server created by setUp()
-        self.shutdownServer()
-        self._servers = []
-        self._pids = []
-
-        # Start a read-only server
-        self.startServer(create=0, index=0, read_only=1)
-        # Start a read-only client
-        self._storage = self.openClientStorage(read_only=1)
-        # Stores should fail here
-        self.assertRaises(ReadOnlyError, self._dostore)
-
     def checkReadOnlyServer(self):
         # Open a read-only client to a read-only *server*; stores fail
 
@@ -227,21 +218,6 @@ class ConnectionTests(StorageTestBase):
         # Stores should succeed here
         self._dostore()
 
-    def checkReadOnlyFallbackReadOnlyStorage(self):
-        # Open a fallback client to a read-only *storage*; stores fail
-
-        # We don't want the read-write server created by setUp()
-        self.shutdownServer()
-        self._servers = []
-        self._pids = []
-
-        # Start a read-only server
-        self.startServer(create=0, index=0, read_only=1)
-        # Start a read-only-fallback client
-        self._storage = self.openClientStorage(read_only_fallback=1)
-        # Stores should fail here
-        self.assertRaises(ReadOnlyError, self._dostore)
-
     def checkReadOnlyFallbackReadOnlyServer(self):
         # Open a fallback client to a read-only *server*; stores fail
 
@@ -285,120 +261,6 @@ class ConnectionTests(StorageTestBase):
         # Stores should succeed here
         self._dostore()
 
-    def checkReconnectReadOnly(self):
-        # A read-only client reconnects from a read-write to a
-        # read-only server
-
-        # Start a client
-        self._storage = self.openClientStorage(read_only=1)
-        # Stores should fail here
-        self.assertRaises(ReadOnlyError, self._dostore)
-
-        # Shut down the server
-        self.shutdownServer()
-        self._servers = []
-        self._pids = []
-        # Poll until the client disconnects
-        self.pollDown()
-        # Stores should still fail
-        self.assertRaises(ReadOnlyError, self._dostore)
-
-        # Restart the server
-        self.startServer(create=0, read_only=1)
-        # Poll until the client connects
-        self.pollUp()
-        # Stores should still fail
-        self.assertRaises(ReadOnlyError, self._dostore)
-
-    def checkReconnectFallback(self):
-        # A fallback client reconnects from a read-write to a
-        # read-only server
-
-        # Start a client in fallback mode
-        self._storage = self.openClientStorage(read_only_fallback=1)
-        # Stores should succeed here
-        self._dostore()
-
-        # Shut down the server
-        self.shutdownServer()
-        self._servers = []
-        self._pids = []
-        # Poll until the client disconnects
-        self.pollDown()
-        # Stores should fail now
-        self.assertRaises(Disconnected, self._dostore)
-
-        # Restart the server
-        self.startServer(create=0, read_only=1)
-        # Poll until the client connects
-        self.pollUp()
-        # Stores should fail here
-        self.assertRaises(ReadOnlyError, self._dostore)
-
-    def checkReconnectUpgrade(self):
-        # A fallback client reconnects from a read-only to a
-        # read-write server
-
-        # We don't want the read-write server created by setUp()
-        self.shutdownServer()
-        self._servers = []
-        self._pids = []
-
-        # Start a read-only server
-        self.startServer(create=0, read_only=1)
-        # Start a client in fallback mode
-        self._storage = self.openClientStorage(read_only_fallback=1)
-        # Stores should fail here
-        self.assertRaises(ReadOnlyError, self._dostore)
-
-        # Shut down the server
-        self.shutdownServer()
-        self._servers = []
-        self._pids = []
-        # Poll until the client disconnects
-        self.pollDown()
-        # Stores should fail now
-        self.assertRaises(Disconnected, self._dostore)
-
-        # Restart the server, this time read-write
-        self.startServer(create=0)
-        # Poll until the client sconnects
-        self.pollUp()
-        # Stores should now succeed
-        self._dostore()
-
-    def checkReconnectSwitch(self):
-        # A fallback client initially connects to a read-only server,
-        # then discovers a read-write server and switches to that
-
-        # We don't want the read-write server created by setUp()
-        self.shutdownServer()
-        self._servers = []
-        self._pids = []
-
-        # Allocate a second address (for the second server)
-        self._newAddr()
-
-        # Start a read-only server
-        self.startServer(create=0, index=0, read_only=1)
-        # Start a client in fallback mode
-        self._storage = self.openClientStorage(read_only_fallback=1)
-        # Stores should fail here
-        self.assertRaises(ReadOnlyError, self._dostore)
-
-        # Start a read-write server
-        self.startServer(index=1, read_only=0)
-        # After a while, stores should work
-        for i in range(300): # Try for 30 seconds
-            try:
-                self._dostore()
-                break
-            except (Disconnected, ReadOnlyError,
-                    select.error, threading.ThreadError, socket.error):
-                time.sleep(0.1)
-        else:
-            self.fail("Couldn't store after starting a read-write server")
-
     def checkDisconnectionError(self):
         # Make sure we get a Disconnected when we try to read an
         # object when we're not connected to a storage server and the
@@ -559,6 +421,155 @@ class ConnectionTests(StorageTestBase):
             for t in threads:
                 t.closeclients()
 
+
+class ReconnectionTests(CommonSetupTearDown):
+    keep = 1
+
+    def checkReadOnlyStorage(self):
+        # Open a read-only client to a read-only *storage*; stores fail
+
+        # We don't want the read-write server created by setUp()
+        self.shutdownServer()
+        self._servers = []
+        self._pids = []
+
+        # Start a read-only server
+        self.startServer(create=0, index=0, read_only=1)
+        # Start a read-only client
+        self._storage = self.openClientStorage(read_only=1)
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+    def checkReadOnlyFallbackReadOnlyStorage(self):
+        # Open a fallback client to a read-only *storage*; stores fail
+
+        # We don't want the read-write server created by setUp()
+        self.shutdownServer()
+        self._servers = []
+        self._pids = []
+
+        # Start a read-only server
+        self.startServer(create=0, index=0, read_only=1)
+        # Start a read-only-fallback client
+        self._storage = self.openClientStorage(read_only_fallback=1)
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+    def checkReconnectReadOnly(self):
+        # A read-only client reconnects from a read-write to a
+        # read-only server
+
+        # Start a client
+        self._storage = self.openClientStorage(read_only=1)
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+        # Shut down the server
+        self.shutdownServer()
+        self._servers = []
+        self._pids = []
+        # Poll until the client disconnects
+        self.pollDown()
+        # Stores should still fail
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+        # Restart the server
+        self.startServer(create=0, read_only=1)
+        # Poll until the client connects
+        self.pollUp()
+        # Stores should still fail
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+    def checkReconnectFallback(self):
+        # A fallback client reconnects from a read-write to a
+        # read-only server
+
+        # Start a client in fallback mode
+        self._storage = self.openClientStorage(read_only_fallback=1)
+        # Stores should succeed here
+        self._dostore()
+
+        # Shut down the server
+        self.shutdownServer()
+        self._servers = []
+        self._pids = []
+        # Poll until the client disconnects
+        self.pollDown()
+        # Stores should fail now
+        self.assertRaises(Disconnected, self._dostore)
+
+        # Restart the server
+        self.startServer(create=0, read_only=1)
+        # Poll until the client connects
+        self.pollUp()
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+    def checkReconnectUpgrade(self):
+        # A fallback client reconnects from a read-only to a
+        # read-write server
+
+        # We don't want the read-write server created by setUp()
+        self.shutdownServer()
+        self._servers = []
+        self._pids = []
+
+        # Start a read-only server
+        self.startServer(create=0, read_only=1)
+        # Start a client in fallback mode
+        self._storage = self.openClientStorage(read_only_fallback=1)
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+        # Shut down the server
+        self.shutdownServer()
+        self._servers = []
+        self._pids = []
+        # Poll until the client disconnects
+        self.pollDown()
+        # Stores should fail now
+        self.assertRaises(Disconnected, self._dostore)
+
+        # Restart the server, this time read-write
+        self.startServer(create=0)
+        # Poll until the client sconnects
+        self.pollUp()
+        # Stores should now succeed
+        self._dostore()
+
+    def checkReconnectSwitch(self):
+        # A fallback client initially connects to a read-only server,
+        # then discovers a read-write server and switches to that
+
+        # We don't want the read-write server created by setUp()
+        self.shutdownServer()
+        self._servers = []
+        self._pids = []
+
+        # Allocate a second address (for the second server)
+        self._newAddr()
+
+        # Start a read-only server
+        self.startServer(create=0, index=0, read_only=1)
+        # Start a client in fallback mode
+        self._storage = self.openClientStorage(read_only_fallback=1)
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+        # Start a read-write server
+        self.startServer(index=1, read_only=0)
+        # After a while, stores should work
+        for i in range(300): # Try for 30 seconds
+            try:
+                self._dostore()
+                break
+            except (Disconnected, ReadOnlyError,
+                    select.error, threading.ThreadError, socket.error):
+                time.sleep(0.1)
+        else:
+            self.fail("Couldn't store after starting a read-write server")
+
+
 class MSTThread(threading.Thread):
 
     __super_init = threading.Thread.__init__

src/ZEO/tests/forker.py

@@ -17,10 +17,8 @@ import os
 import sys
 import time
 import errno
-import types
 import random
 import socket
-import asyncore
 import tempfile
 import traceback
 
@@ -31,6 +29,7 @@ PROFILE = 0
 if PROFILE:
     import hotshot
 
+
 def get_port():
     """Return a port that is not in use.
 
@@ -53,7 +52,7 @@ def get_port():
     raise RuntimeError, "Can't find port"
 
 
-def start_zeo_server(conf, addr=None, ro_svr=0):
+def start_zeo_server(conf, addr=None, ro_svr=0, keep=0):
     """Start a ZEO server in a separate process.
 
     Returns the ZEO port, the test server port, and the pid.
@@ -76,6 +75,8 @@ def start_zeo_server(conf, addr=None, ro_svr=0):
     args = [sys.executable, script, '-C', tmpfile]
     if ro_svr:
         args.append('-r')
+    if keep:
+        args.append('-k')
     args.append(str(port))
     d = os.environ.copy()
     d['PYTHONPATH'] = os.pathsep.join(sys.path)

src/ZEO/tests/testConnection.py

@@ -20,12 +20,10 @@ platform-dependent scaffolding.
 # System imports
 import unittest
 # Import the actual test class
-from ZEO.tests.ConnectionTests import ConnectionTests
+from ZEO.tests import ConnectionTests
 
 
-class FileStorageConnectionTests(ConnectionTests):
-    """Add FileStorage-specific test."""
-
+class FileStorageConfig:
     def getConfig(self, path, create, read_only):
         return """\
         <Storage>
@@ -38,9 +36,7 @@ class FileStorageConnectionTests(ConnectionTests):
                          read_only and 'yes' or 'no')
 
 
-class BDBConnectionTests(FileStorageConnectionTests):
-    """Berkeley storage tests."""
-
+class BerkeleyStorageConfig:
     def getConfig(self, path, create, read_only):
         # Full always creates and doesn't have a read_only flag
         return """\
@@ -51,13 +47,42 @@ class BDBConnectionTests(FileStorageConnectionTests):
         </Storage>""" % (path, read_only)
 
 
-test_classes = [FileStorageConnectionTests]
+class FileStorageConnectionTests(
+    FileStorageConfig,
+    ConnectionTests.ConnectionTests
+    ):
+    """FileStorage-specific connection tests."""
+
+
+class FileStorageReconnectionTests(
+    FileStorageConfig,
+    ConnectionTests.ReconnectionTests
+    ):
+    """FileStorage-specific re-connection tests."""
+
+
+class BDBConnectionTests(
+    BerkeleyStorageConfig,
+    ConnectionTests.ConnectionTests
+    ):
+    """Berkeley storage connection tests."""
+
+
+class BDBReconnectionTests(
+    BerkeleyStorageConfig,
+    ConnectionTests.ReconnectionTests
+    ):
+    """Berkeley storage re-connection tests."""
+
+
+test_classes = [FileStorageConnectionTests, FileStorageReconnectionTests]
 try:
     from bsddb3Storage.Full import Full
 except ImportError:
     pass
 else:
     test_classes.append(BDBConnectionTests)
+    test_classes.append(BDBReconnectionTests)
 
 
 def test_suite():

src/ZEO/tests/zeoserver.py

@@ -65,13 +65,14 @@ class ZEOTestServer(asyncore.dispatcher):
     """
     __super_init = asyncore.dispatcher.__init__
 
-    def __init__(self, addr, storage):
+    def __init__(self, addr, storage, keep):
         self.__super_init()
-        self.storage = storage
+        self._storage = storage
+        self._keep = keep
         # Count down to zero, the number of connects
-        self.count = 1
+        self._count = 1
         # For zLOG
-        self.label ='zeoserver:%d @ %s' % (os.getpid(), addr)
+        self._label ='zeoserver:%d @ %s' % (os.getpid(), addr)
         self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
         # Some ZEO tests attempt a quick start of the server using the same
         # port so we have to set the reuse flag.
@@ -87,22 +88,23 @@ class ZEOTestServer(asyncore.dispatcher):
         self.log('bound and listening')
 
     def log(self, msg, *args):
-        log(self.label, msg, *args)
+        log(self._label, msg, *args)
 
     def handle_accept(self):
         sock, addr = self.accept()
         self.log('in handle_accept()')
         # When we're done with everything, close the storage.  Do not write
         # the ack character until the storage is finished closing.
-        if self.count <= 0:
+        if self._count <= 0:
             self.log('closing the storage')
-            self.storage.close()
-            cleanup(self.storage)
+            self._storage.close()
+            if not self._keep:
+                cleanup(self._storage)
             self.log('exiting')
             os._exit(0)
         self.log('continuing')
         sock.send('X')
-        self.count -= 1
+        self._count -= 1
 
 
 def main():
@@ -111,12 +113,15 @@ def main():
     # We don't do much sanity checking of the arguments, since if we get it
     # wrong, it's a bug in the test suite.
     ro_svr = 0
+    keep = 0
     configfile = None
     # Parse the arguments and let getopt.error percolate
-    opts, args = getopt.getopt(sys.argv[1:], 'rC:')
+    opts, args = getopt.getopt(sys.argv[1:], 'rkC:')
     for opt, arg in opts:
         if opt == '-r':
             ro_svr = 1
+        elif opt == '-k':
+            keep = 1
         elif opt == '-C':
             configfile = arg
     # Open the config file and let ZConfig parse the data there.  Then remove
@@ -129,8 +134,8 @@ def main():
     zeo_port = int(args[0])
     test_port = zeo_port + 1
     try:
-        log(label, 'creating the test server')
-        t = ZEOTestServer(('', test_port), storage)
+        log(label, 'creating the test server, ro: %s, keep: %s', ro_svr, keep)
+        t = ZEOTestServer(('', test_port), storage, keep)
     except socket.error, e:
         if e[0] <> errno.EADDRINUSE: raise
         log(label, 'addr in use, closing and exiting')