Send a notification when a node is lost during recovery step.

Add a test to ensure the admin is notified of connected/disconnected
nodes during recovery.

git-svn-id: https://svn.erp5.org/repos/neo/trunk@2585 71dcc9de-d417-0410-9af5-da40c76e7ee4

neo/master/recovery.py

@@ -108,6 +108,8 @@ class RecoveryManager(MasterHandler):
         if node.getState() == new_state:
             return
         node.setState(new_state)
+        # broadcast to all so that admin nodes gets informed
+        self.app.broadcastNodesInformation([node])
 
     def connectionCompleted(self, conn):
         # XXX: handler split review needed to remove this hack

neo/tests/functional/testCluster.py

@@ -127,6 +127,32 @@ class ClusterTests(NEOFunctionalTest):
         transaction.commit()
         self.assertEqual(len(self.neo.getClientlist()), 1)
 
+    def testStorageLostDuringRecovery(self):
+        """
+            Check that admin node receive notifications of storage
+            connection and disconnection during recovery
+        """
+        self.neo = NEOCluster(['test_neo%d' % i for i in xrange(2)],
+            master_node_count=1, partitions=10, replicas=1,
+            temp_dir=self.getTempDirectory(), clear_databases=True,
+        )
+        storages  = self.neo.getStorageProcessList()
+        self.neo.run(except_storages=storages)
+        self.neo.expectStorageNotKnown(storages[0])
+        self.neo.expectStorageNotKnown(storages[1])
+        storages[0].start()
+        self.neo.expectRunning(storages[0])
+        self.neo.expectStorageNotKnown(storages[1])
+        storages[1].start()
+        self.neo.expectRunning(storages[0])
+        self.neo.expectRunning(storages[1])
+        storages[0].stop()
+        self.neo.expectUnavailable(storages[0])
+        self.neo.expectRunning(storages[1])
+        storages[1].stop()
+        self.neo.expectUnavailable(storages[0])
+        self.neo.expectUnavailable(storages[1])
+
 def test_suite():
     return unittest.makeSuite(ClusterTests)