connection: make close always call handler (connectionClosed or connectionFailed)

Main reason is that it's difficult to know in advance which side really closes
the connection. Network events can be chaotic and this could lead to many race
conditions.
Thus, handler can be used to update any database that is somewhat redundant
to the connection status, i.e. node status usually. Safely and less duplicated
code.

This change is motivated by recurrent random failures during election.
An example of race condition was that 2 fully connected master could close the
extra connection (the primary -> secondary one) at the same time.

In order to stabilize lower-level code and start with reliable election process,
code has also been simplified to not care about node states. All connections
without exception are closed at the end of the election and states are then
updated 1 by 1 by identification handler.
Note that during election, there may be 2 connection per node, which makes
difficult to update node states by connectionFailed/connectionClosed events.

timeoutExpired & peerBroken are dropped as they are unused for the moment.
A new API should be designed so that connectionClosed know the reason of the
close.
BROKEN state becomes unused.

git-svn-id: https://svn.erp5.org/repos/neo/trunk@2732 71dcc9de-d417-0410-9af5-da40c76e7ee4

neo/admin/handler.py

@@ -122,15 +122,9 @@ class MasterEventHandler(EventHandler):
     def connectionFailed(self, conn):
         self._connectionLost(conn)
 
-    def timeoutExpired(self, conn):
-        self._connectionLost(conn)
-
     def connectionClosed(self, conn):
         self._connectionLost(conn)
 
-    def peerBroken(self, conn):
-        self._connectionLost(conn)
-
     def dispatch(self, conn, packet):
         if packet.isResponse() and \
            self.app.dispatcher.registered(packet.getId()):

neo/client/handlers/master.py

@@ -68,10 +68,10 @@ class PrimaryBootstrapHandler(AnswerBaseHandler):
                 neo.lib.logging.warning('Unknown primary master UUID: %s. ' \
                                 'Ignoring.' % dump(primary_uuid))
             else:
-                app.primary_master_node = primary_node
                 if app.trying_master_node is not primary_node:
                     app.trying_master_node = None
                     conn.close()
+                app.primary_master_node = primary_node
         else:
             if app.primary_master_node is not None:
                 # The primary master node is not a primary master node
@@ -92,27 +92,14 @@ class PrimaryNotificationsHandler(BaseHandler):
     """ Handler that process the notifications from the primary master """
 
     def connectionClosed(self, conn):
-        app = self.app
-        neo.lib.logging.critical("connection to primary master node closed")
-        conn.close()
-        app.master_conn = None
-        app.primary_master_node = None
-        super(PrimaryNotificationsHandler, self).connectionClosed(conn)
-
-    def timeoutExpired(self, conn):
-        app = self.app
-        if app.master_conn is not None:
-            assert conn is app.master_conn
-            neo.lib.logging.critical(
-                        "connection timeout to primary master node expired")
-        BaseHandler.timeoutExpired(self, conn)
-
-    def peerBroken(self, conn):
         app = self.app
         if app.master_conn is not None:
-            assert conn is app.master_conn
-            neo.lib.logging.critical("primary master node is broken")
-        BaseHandler.peerBroken(self, conn)
+            neo.lib.logging.critical("connection to primary master node closed")
+            app.master_conn = None
+            app.primary_master_node = None
+        else:
+            assert app.primary_master_node is None
+        super(PrimaryNotificationsHandler, self).connectionClosed(conn)
 
     def stopOperation(self, conn):
         neo.lib.logging.critical("master node ask to stop operation")
@@ -139,19 +126,14 @@ class PrimaryNotificationsHandler(BaseHandler):
             self.app.pt.update(ptid, cell_list, self.app.nm)
 
     def notifyNodeInformation(self, conn, node_list):
-        app = self.app
-        self.app.nm.update(node_list)
+        nm = self.app.nm
+        nm.update(node_list)
         for node_type, addr, uuid, state in node_list:
             if state != NodeStates.RUNNING:
                 # close connection to this node if no longer running
-                node = self.app.nm.getByUUID(uuid)
+                node = nm.getByUUID(uuid)
                 if node and node.isConnected():
-                    conn = node.getConnection()
-                    conn.close()
-                    if node_type == NodeTypes.STORAGE:
-                        # Remove from pool connection
-                        app.cp.removeConnection(conn)
-                        self.dispatcher.unregister(conn)
+                    node.getConnection().close()
 
 
 class PrimaryAnswersHandler(AnswerBaseHandler):

neo/lib/bootstrap.py

@@ -77,8 +77,6 @@ class BootstrapManager(EventHandler):
         handle the client node.
         Close connection and restart.
         """
-        # master are still electing on of them
-        self.current = None
         conn.close()
 
     def answerPrimary(self, conn, primary_uuid, known_master_list):
@@ -102,7 +100,6 @@ class BootstrapManager(EventHandler):
             # - something goes wrong (unknown UUID)
             # - this master doesn't know who's the primary
             # - got the primary's uuid, so cut here
-            self.current = None
             conn.close()
             return
 

neo/lib/connection.py

@@ -184,7 +184,7 @@ class HandlerSwitcher(object):
                 notification = Packets.Notify('Unexpected answer: %r' % packet)
                 connection.notify(notification)
                 connection.abort()
-            handler.peerBroken(connection)
+            # handler.peerBroken(connection)
         # apply a pending handler if no more answers are pending
         while len(self._pending) > 1 and not self._pending[0][0]:
             del self._pending[0]
@@ -290,13 +290,11 @@ class BaseConnection(object):
                 neo.lib.logging.info(
                                 'timeout for #0x%08x with %r', msg_id, self)
                 self.close()
-                self.getHandler().timeoutExpired(self)
             elif self._timeout.hardExpired(t):
                 # critical time reach or pong not received, abort
                 neo.lib.logging.info('timeout with %r', self)
                 self.notify(Packets.Notify('Timeout'))
                 self.abort()
-                self.getHandler().timeoutExpired(self)
             elif self._timeout.softExpired(t):
                 self._timeout.ping(t)
                 self.ping()
@@ -421,6 +419,8 @@ class ListeningConnection(BaseConnection):
 class Connection(BaseConnection):
     """A connection."""
 
+    connecting = False
+
     def __init__(self, event_manager, handler, connector, addr=None):
         BaseConnection.__init__(self, event_manager, handler,
                                 connector=connector, addr=addr)
@@ -460,16 +460,6 @@ class Connection(BaseConnection):
         self.cur_id = (next_id + 1) & 0xffffffff
         return next_id
 
-    def close(self):
-        neo.lib.logging.debug('closing a connector for %r', self)
-        BaseConnection.close(self)
-        if self._on_close is not None:
-            self._on_close()
-            self._on_close = None
-        del self.write_buf[:]
-        self.read_buf.clear()
-        self._handlers.clear()
-
     def abort(self):
         """Abort dealing with this connection."""
         neo.lib.logging.debug('aborting a connector for %r', self)
@@ -534,18 +524,33 @@ class Connection(BaseConnection):
     def pending(self):
         return self.connector is not None and self.write_buf
 
-    def _closure(self, was_connected=True):
-        assert self.connector is not None, self.whoSetConnector()
+    def close(self):
+        if self.connector is None:
+            assert self._on_close is None
+            assert not self.read_buf
+            assert not self.write_buf
+            assert not self.isPending()
+            return
         # process the network events with the last registered handler to
         # solve issues where a node is lost with pending handlers and
         # create unexpected side effects.
-        # XXX: This solution is being tested and should be approved or reverted
+        neo.lib.logging.debug('closing a connector for %r', self)
         handler = self._handlers.getLastHandler()
-        self.close()
-        if was_connected:
-            handler.connectionClosed(self)
-        else:
+        super(Connection, self).close()
+        if self._on_close is not None:
+            self._on_close()
+            self._on_close = None
+        del self.write_buf[:]
+        self.read_buf.clear()
+        self._handlers.clear()
+        if self.connecting:
             handler.connectionFailed(self)
+        else:
+            handler.connectionClosed(self)
+
+    def _closure(self):
+        assert self.connector is not None, self.whoSetConnector()
+        self.close()
 
     @profiler_decorator
     def _recv(self):
@@ -555,8 +560,8 @@ class Connection(BaseConnection):
         except ConnectorTryAgainException:
             pass
         except ConnectorConnectionRefusedException:
-            # should only occur while connecting
-            self._closure(was_connected=False)
+            assert self.connecting
+            self._closure()
         except ConnectorConnectionClosedException:
             # connection resetted by peer, according to the man, this error
             # should not occurs but it seems it's false
@@ -667,8 +672,9 @@ class Connection(BaseConnection):
 class ClientConnection(Connection):
     """A connection from this node to a remote node."""
 
+    connecting = True
+
     def __init__(self, event_manager, handler, addr, connector, **kw):
-        self.connecting = True
         Connection.__init__(self, event_manager, handler, addr=addr,
                             connector=connector)
         handler.connectionStarted(self)
@@ -681,10 +687,10 @@ class ClientConnection(Connection):
                 self.connecting = False
                 self.getHandler().connectionCompleted(self)
         except ConnectorConnectionRefusedException:
-            self._closure(was_connected=False)
+            self._closure()
         except ConnectorException:
             # unhandled connector exception
-            self._closure(was_connected=False)
+            self._closure()
             raise
 
     def writable(self):
@@ -692,7 +698,7 @@ class ClientConnection(Connection):
         if self.connecting:
             err = self.connector.getError()
             if err:
-                self._closure(was_connected=False)
+                self._closure()
                 return
             else:
                 self.connecting = False

neo/lib/handler.py

@@ -41,7 +41,7 @@ class EventHandler(object):
         neo.lib.logging.error(message)
         conn.answer(Errors.ProtocolError(message))
         conn.abort()
-        self.peerBroken(conn)
+        # self.peerBroken(conn)
 
     def dispatch(self, conn, packet):
         """This is a helper method to handle various packet types."""
@@ -59,23 +59,20 @@ class EventHandler(object):
             neo.lib.logging.error('malformed packet from %r', conn)
             conn.notify(Packets.Notify('Malformed packet: %r' % (packet, )))
             conn.abort()
-            self.peerBroken(conn)
+            # self.peerBroken(conn)
         except BrokenNodeDisallowedError:
             conn.answer(Errors.BrokenNode('go away'))
             conn.abort()
-            self.connectionClosed(conn)
         except NotReadyError, message:
             if not message.args:
                 message = 'Retry Later'
             message = str(message)
             conn.answer(Errors.NotReady(message))
             conn.abort()
-            self.connectionClosed(conn)
         except ProtocolError, message:
             message = str(message)
             conn.answer(Errors.ProtocolError(message))
             conn.abort()
-            self.connectionClosed(conn)
 
     def checkClusterName(self, name):
         # raise an exception if the given name mismatch the current cluster name
@@ -106,20 +103,15 @@ class EventHandler(object):
     def connectionAccepted(self, conn):
         """Called when a connection is accepted."""
 
-    def timeoutExpired(self, conn):
-        """Called when a timeout event occurs."""
-        neo.lib.logging.debug('timeout expired for %r', conn)
-        self.connectionLost(conn, NodeStates.TEMPORARILY_DOWN)
-
     def connectionClosed(self, conn):
         """Called when a connection is closed by the peer."""
         neo.lib.logging.debug('connection closed for %r', conn)
         self.connectionLost(conn, NodeStates.TEMPORARILY_DOWN)
 
-    def peerBroken(self, conn):
-        """Called when a peer is broken."""
-        neo.lib.logging.error('%r is broken', conn)
-        self.connectionLost(conn, NodeStates.BROKEN)
+    #def peerBroken(self, conn):
+    #    """Called when a peer is broken."""
+    #    neo.lib.logging.error('%r is broken', conn)
+    #    # NodeStates.BROKEN
 
     def connectionLost(self, conn, new_state):
         """ this is a method to override in sub-handlers when there is no need

neo/master/app.py

@@ -334,9 +334,7 @@ class Application(object):
                     if node.isStorage() or node.isClient():
                         node.notify(Packets.StopOperation())
                         if node.isClient():
-                            conn = node.getConnection()
-                            conn.abort()
-                            conn.getHandler().connectionClosed(conn)
+                            node.getConnection().abort()
 
                 # Then, go back, and restart.
                 return
@@ -351,18 +349,20 @@ class Application(object):
         self.listening_conn.setHandler(
                 identification.IdentificationHandler(self))
 
-        handler = secondary.SecondaryMasterHandler(self)
         em = self.em
         nm = self.nm
 
-        # Make sure that every connection has the secondary event handler.
+        # Close all remaining connections to other masters,
+        # for the same reason as in playSecondaryRole.
         for conn in em.getConnectionList():
             conn_uuid = conn.getUUID()
             if conn_uuid is not None:
                 node = nm.getByUUID(conn_uuid)
                 assert node is not None
-                assert node.isMaster()
-                conn.setHandler(handler)
+                assert node.isMaster() and not conn.isClient()
+                assert node._connection is None and node.isUnknown()
+                # this may trigger 'unexpected answer' warnings on remote side
+                conn.close()
 
         # If I know any storage node, make sure that they are not in the
         # running state, because they are not connected at this stage.
@@ -392,24 +392,16 @@ class Application(object):
                 # election timeout
                 raise ElectionFailure("Election timeout")
 
-        # Now I need only a connection to the primary master node.
+        # Restart completely. Non-optimized
+        # but lower level code needs to be stabilized first.
         addr = self.primary_master_node.getAddress()
-        for conn in self.em.getServerList():
+        for conn in self.em.getConnectionList():
             conn.close()
-        connected_to_master = False
 
+        # Reconnect to primary master node.
         primary_handler = secondary.PrimaryHandler(self)
-
-        for conn in self.em.getClientList():
-            if conn.getAddress() == addr:
-                connected_to_master = True
-                conn.setHandler(primary_handler)
-            else:
-                conn.close()
-
-        if not connected_to_master:
-            ClientConnection(self.em, primary_handler, addr=addr,
-                connector=self.connector_handler())
+        ClientConnection(self.em, primary_handler, addr=addr,
+            connector=self.connector_handler())
 
         # and another for the future incoming connections
         handler = identification.IdentificationHandler(self)
@@ -442,18 +434,20 @@ class Application(object):
         # change handlers
         notification_packet = Packets.NotifyClusterInformation(state)
         for node in self.nm.getIdentifiedList():
-            if not node.isMaster():
-                node.notify(notification_packet)
-            if node.isAdmin() or node.isMaster():
-                # those node types keep their own handler
+            if node.isMaster():
                 continue
             conn = node.getConnection()
+            if node.isClient() and conn.isAborted():
+                continue
+            node.notify(notification_packet)
             if node.isClient():
                 if state != ClusterStates.RUNNING:
                     conn.close()
                 handler = client_handler
             elif node.isStorage():
                 handler = storage_handler
+            else:
+                continue # keep handler
             conn.setHandler(handler)
             handler.connectionCompleted(conn)
         self.cluster_state = state

neo/master/handlers/__init__.py

@@ -81,7 +81,8 @@ class BaseServiceHandler(MasterHandler):
 
     def connectionLost(self, conn, new_state):
         node = self.app.nm.getByUUID(conn.getUUID())
-        assert node is not None
+        if node is None:
+            return # for example, when a storage is removed by an admin
         if new_state != NodeStates.BROKEN:
             new_state = DISCONNECTED_STATE_DICT.get(node.getType(),
                     NodeStates.DOWN)

neo/master/handlers/election.py

@@ -32,12 +32,6 @@ class ClientElectionHandler(MasterHandler):
     def askPrimary(self, conn):
         raise UnexpectedPacketError, "askPrimary on server connection"
 
-    def packetReceived(self, conn, packet):
-        node = self.app.nm.getByAddress(conn.getAddress())
-        if not node.isBroken():
-            node.setRunning()
-        MasterHandler.packetReceived(self, conn, packet)
-
     def connectionStarted(self, conn):
         addr = conn.getAddress()
         # connection in progress
@@ -57,18 +51,11 @@ class ClientElectionHandler(MasterHandler):
         MasterHandler.connectionFailed(self, conn)
 
     def connectionCompleted(self, conn):
-        addr = conn.getAddress()
-        node = self.app.nm.getByAddress(addr)
-        # connection successfull, set it as running
-        node.setRunning()
         conn.ask(Packets.AskPrimary())
         MasterHandler.connectionCompleted(self, conn)
 
     def connectionLost(self, conn, new_state):
         addr = conn.getAddress()
-        node = self.app.nm.getByAddress(addr)
-        if new_state != NodeStates.BROKEN or node is not None:
-            node.setState(new_state)
         self.app.negotiating_master_node_set.discard(addr)
 
     def acceptIdentification(self, conn, node_type,
@@ -79,7 +66,6 @@ class ClientElectionHandler(MasterHandler):
             # The peer is not a master node!
             neo.lib.logging.error('%r is not a master node', conn)
             app.nm.remove(node)
-            app.negotiating_master_node_set.discard(node.getAddress())
             conn.close()
             return
 
@@ -172,11 +158,6 @@ class ServerElectionHandler(MasterHandler):
     def reelectPrimary(self, conn):
         raise ElectionFailure, 'reelection requested'
 
-    def connectionLost(self, conn, new_state):
-        node = self.app.nm.getByUUID(conn.getUUID())
-        if node is not None:
-            node.setState(new_state)
-
     def requestIdentification(self, conn, node_type,
                                         uuid, address, name):
         self.checkClusterName(name)

neo/master/handlers/secondary.py

@@ -28,9 +28,6 @@ class SecondaryMasterHandler(MasterHandler):
         node.setDown()
         self.app.broadcastNodesInformation([node])
 
-    def connectionCompleted(self, conn):
-        pass
-
     def announcePrimary(self, conn):
         raise ElectionFailure, 'another primary arises'
 

neo/neoctl/handler.py

@@ -42,14 +42,6 @@ class CommandEventHandler(EventHandler):
         super(CommandEventHandler, self).connectionFailed(conn)
         self.__disconnected()
 
-    def timeoutExpired(self, conn):
-        super(CommandEventHandler, self).timeoutExpired(conn)
-        self.__disconnected()
-
-    def peerBroken(self, conn):
-        super(CommandEventHandler, self).peerBroken(conn)
-        self.__disconnected()
-
     def ack(self, conn, msg):
         self.__respond((Packets.Error, ErrorCodes.ACK, msg))
 

neo/storage/handlers/__init__.py

@@ -51,7 +51,11 @@ class BaseMasterHandler(EventHandler):
                 neo.lib.logging.info("I was told I'm %s" %(state))
                 if state in (NodeStates.DOWN, NodeStates.TEMPORARILY_DOWN,
                         NodeStates.BROKEN):
-                    conn.close()
+                    try:
+                        conn.close()
+                        assert False
+                    except PrimaryFailure:
+                        pass
                     erase = state == NodeStates.DOWN
                     self.app.shutdown(erase=erase)
                 elif state == NodeStates.HIDDEN:

neo/tests/client/testMasterHandler.py

@@ -210,6 +210,15 @@ class MasterNotificationsHandlerTests(MasterHandlerTests):
         # connections closed
         self.checkClosed(conn1)
         self.checkClosed(conn2)
+        return conn2
+
+    def test_notifyNodeInformation_checkUnregisterStorage(self):
+        # XXX: This test fails because unregistering is done
+        #      by neo.client.handlers.storage.StorageEventHandler
+        #      which would require a connection to storage
+        #      with a proper handler (defined by Application).
+        #      It can be merged with previous one as soon as it passes.
+        conn2 = self.test_notifyNodeInformation()
         # storage removed from connection pool
         remove_calls = self.app.cp.mockGetNamedCalls('removeConnection')
         self.assertEqual(len(remove_calls), 1)

neo/tests/master/testClientHandler.py

@@ -162,7 +162,7 @@ class MasterClientHandlerTests(NeoUnitTestBase):
         (node_list, ) = packet.decode()
         self.assertEqual(len(node_list), 2)
 
-    def __testWithMethod(self, method, state):
+    def test_connectionClosed(self):
         # give a client uuid which have unfinished transactions
         client_uuid = self.identifyToMasterNode(node_type=NodeTypes.CLIENT,
                                                 port = self.client_port)
@@ -170,22 +170,11 @@ class MasterClientHandlerTests(NeoUnitTestBase):
         lptid = self.app.pt.getID()
         self.assertEqual(self.app.nm.getByUUID(client_uuid).getState(),
                 NodeStates.RUNNING)
-        method(conn)
+        self.service.connectionClosed(conn)
         # node must be have been remove, and no more transaction must remains
         self.assertEqual(self.app.nm.getByUUID(client_uuid), None)
         self.assertEqual(lptid, self.app.pt.getID())
 
-    def test_15_peerBroken(self):
-        self.__testWithMethod(self.service.peerBroken, NodeStates.BROKEN)
-
-    def test_16_timeoutExpired(self):
-        self.__testWithMethod(self.service.timeoutExpired,
-                NodeStates.TEMPORARILY_DOWN)
-
-    def test_17_connectionClosed(self):
-        self.__testWithMethod(self.service.connectionClosed,
-            NodeStates.TEMPORARILY_DOWN)
-
     def test_askPack(self):
         self.assertEqual(self.app.packing, None)
         self.app.nm.createClient()

neo/tests/master/testElectionHandler.py

@@ -100,12 +100,11 @@ class MasterClientElectionTests(NeoUnitTestBase):
         self._checkUnconnected(node)
         self.election.connectionCompleted(conn)
         self._checkUnconnected(node)
-        self.assertTrue(node.isRunning())
+        self.assertTrue(node.isUnknown())
         self.checkAskPrimary(conn)
 
     def _setNegociating(self, node):
         self._checkUnconnected(node)
-        node.setRunning()
         addr = node.getAddress()
         self.app.negotiating_master_node_set.add(addr)
         self.app.unconnected_master_node_set.discard(addr)
@@ -115,16 +114,7 @@ class MasterClientElectionTests(NeoUnitTestBase):
         node, conn = self.identifyToMasterNode()
         self._setNegociating(node)
         self.election.connectionClosed(conn)
-        self.assertTrue(node.isTemporarilyDown())
-        addr = node.getAddress()
-        self.assertFalse(addr in self.app.unconnected_master_node_set)
-        self.assertFalse(addr in self.app.negotiating_master_node_set)
-
-    def test_timeoutExpired(self):
-        node, conn = self.identifyToMasterNode()
-        self._setNegociating(node)
-        self.election.timeoutExpired(conn)
-        self.assertTrue(node.isTemporarilyDown())
+        self.assertTrue(node.isUnknown())
         addr = node.getAddress()
         self.assertFalse(addr in self.app.unconnected_master_node_set)
         self.assertFalse(addr in self.app.negotiating_master_node_set)

neo/tests/master/testRecovery.py

@@ -68,26 +68,6 @@ class MasterRecoveryTests(NeoUnitTestBase):
         self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                 NodeStates.TEMPORARILY_DOWN)
 
-    def test_02_timeoutExpired(self):
-        uuid = self.identifyToMasterNode(node_type=NodeTypes.MASTER, port=self.master_port)
-        conn = self.getFakeConnection(uuid, self.master_address)
-        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
-                NodeStates.RUNNING)
-        self.recovery.timeoutExpired(conn)
-        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
-                NodeStates.TEMPORARILY_DOWN)
-
-
-    def test_03_peerBroken(self):
-        uuid = self.identifyToMasterNode(node_type=NodeTypes.MASTER, port=self.master_port)
-        conn = self.getFakeConnection(uuid, self.master_address)
-        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
-                NodeStates.RUNNING)
-        self.recovery.peerBroken(conn)
-        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
-                NodeStates.BROKEN)
-
-
     def test_09_answerLastIDs(self):
         recovery = self.recovery
         uuid = self.identifyToMasterNode()

neo/tests/master/testStorageHandler.py

@@ -154,7 +154,9 @@ class MasterStorageHandlerTests(NeoUnitTestBase):
         max_tid, tid_list = self.checkAnswerUnfinishedTransactions(conn, decode=True)
         self.assertEqual(len(tid_list), 1)
 
-    def _testWithMethod(self, method, state):
+    def test_connectionClosed(self):
+        method = self.service.connectionClosed
+        state = NodeStates.TEMPORARILY_DOWN
         # define two nodes
         node1, conn1 = self.identifyToMasterNode()
         node2, conn2 = self.identifyToMasterNode()
@@ -178,17 +180,6 @@ class MasterStorageHandlerTests(NeoUnitTestBase):
         self.assertEqual(node2.getState(), state)
         self.assertEqual(lptid, self.app.pt.getID())
 
-    def test_15_peerBroken(self):
-        self._testWithMethod(self.service.peerBroken, NodeStates.BROKEN)
-
-    def test_16_timeoutExpired(self):
-        self._testWithMethod(self.service.timeoutExpired,
-                NodeStates.TEMPORARILY_DOWN)
-
-    def test_17_connectionClosed(self):
-        self._testWithMethod(self.service.connectionClosed,
-                NodeStates.TEMPORARILY_DOWN)
-
     def test_nodeLostAfterAskLockInformation(self):
         # 2 storage nodes, one will die
         node1, conn1 = self._getStorage()

neo/tests/master/testVerification.py

@@ -72,26 +72,6 @@ class MasterVerificationTests(NeoUnitTestBase):
         self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                 NodeStates.TEMPORARILY_DOWN)
 
-    def test_02_timeoutExpired(self):
-        # test a storage, must raise as cluster no longer op
-        uuid = self.identifyToMasterNode()
-        conn = self.getFakeConnection(uuid, self.storage_address)
-        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
-                NodeStates.UNKNOWN)
-        self.assertRaises(VerificationFailure, self.verification.connectionClosed,conn)
-        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
-                NodeStates.TEMPORARILY_DOWN)
-
-    def test_03_peerBroken(self):
-        # test a storage, must raise as cluster no longer op
-        uuid = self.identifyToMasterNode()
-        conn = self.getFakeConnection(uuid, self.storage_address)
-        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
-                NodeStates.UNKNOWN)
-        self.assertRaises(VerificationFailure, self.verification.connectionClosed,conn)
-        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
-                NodeStates.TEMPORARILY_DOWN)
-
     def _test_09_answerLastIDs(self):
         # XXX: test disabled, should be an unexpected packet
         verification = self.verification

neo/tests/storage/testInitializationHandler.py

@@ -51,24 +51,12 @@ class StorageInitializationHandlerTests(NeoUnitTestBase):
         address = ("127.0.0.1", self.client_port)
         return self.getFakeConnection(uuid=self.getNewUUID(), address=address)
 
-    def test_02_timeoutExpired(self):
-        conn = self.getClientConnection()
-        self.assertRaises(PrimaryFailure, self.verification.timeoutExpired, conn,)
-        # nothing happens
-        self.checkNoPacketSent(conn)
-
     def test_03_connectionClosed(self):
         conn = self.getClientConnection()
         self.assertRaises(PrimaryFailure, self.verification.connectionClosed, conn,)
         # nothing happens
         self.checkNoPacketSent(conn)
 
-    def test_04_peerBroken(self):
-        conn = self.getClientConnection()
-        self.assertRaises(PrimaryFailure, self.verification.peerBroken, conn,)
-        # nothing happens
-        self.checkNoPacketSent(conn)
-
     def test_09_answerPartitionTable(self):
         # send a table
         conn = self.getClientConnection()

neo/tests/storage/testMasterHandler.py

@@ -57,24 +57,12 @@ class StorageMasterHandlerTests(NeoUnitTestBase):
         address = ("127.0.0.1", self.master_port)
         return self.getFakeConnection(uuid=self.master_uuid, address=address)
 
-    def test_06_timeoutExpired(self):
-        # client connection
-        conn = self.getMasterConnection()
-        self.assertRaises(PrimaryFailure, self.operation.timeoutExpired, conn)
-        self.checkNoPacketSent(conn)
-
     def test_07_connectionClosed2(self):
         # primary has closed the connection
         conn = self.getMasterConnection()
         self.assertRaises(PrimaryFailure, self.operation.connectionClosed, conn)
         self.checkNoPacketSent(conn)
 
-    def test_08_peerBroken(self):
-        # client connection
-        conn = self.getMasterConnection()
-        self.assertRaises(PrimaryFailure, self.operation.peerBroken, conn)
-        self.checkNoPacketSent(conn)
-
     def test_14_notifyPartitionChanges1(self):
         # old partition change -> do nothing
         app = self.app

neo/tests/storage/testVerificationHandler.py

@@ -56,24 +56,12 @@ class StorageVerificationHandlerTests(NeoUnitTestBase):
         return self.getFakeConnection(address=("127.0.0.1", self.master_port))
 
     # Tests
-    def test_02_timeoutExpired(self):
-        conn = self.getClientConnection()
-        self.assertRaises(PrimaryFailure, self.verification.timeoutExpired, conn,)
-        # nothing happens
-        self.checkNoPacketSent(conn)
-
     def test_03_connectionClosed(self):
         conn = self.getClientConnection()
         self.assertRaises(PrimaryFailure, self.verification.connectionClosed, conn,)
         # nothing happens
         self.checkNoPacketSent(conn)
 
-    def test_04_peerBroken(self):
-        conn = self.getClientConnection()
-        self.assertRaises(PrimaryFailure, self.verification.peerBroken, conn,)
-        # nothing happens
-        self.checkNoPacketSent(conn)
-
     def test_07_askLastIDs(self):
         conn = self.getClientConnection()
         last_ptid = self.getPTID(1)

neo/tests/testConnection.py

@@ -599,7 +599,7 @@ class ConnectionTests(NeoUnitTestBase):
         # test send was called
         self._checkSend(1, "testdata")
         self._checkWriteBuf(bc, '')
-        self._checkConnectionClosed(0)
+        self._checkConnectionClosed(1)
         self._checkUnregistered(1)
         # nothing else pending, and aborted is false, so writer has been removed
         self.assertFalse(bc.pending())