add unit test for election handler

git-svn-id: https://svn.erp5.org/repos/neo/branches/prototype3@258 71dcc9de-d417-0410-9af5-da40c76e7ee4

neo/master/tests/connector.py

+from mock import Mock
+import logging
+connector_cpt = 0
+from neo.protocol import Packet, INVALID_UUID, MASTER_NODE_TYPE
+
+import os
+
+# master node with the highest uuid will be declared as PMN
+previous_uuid = None
+def getNewUUID():
+    uuid = INVALID_UUID
+    previous = globals()["previous_uuid"]
+    while uuid == INVALID_UUID or (previous is \
+              not None and uuid > previous):
+        uuid = os.urandom(16)
+    logging.info("previous > uuid %s"%(previous > uuid)) 
+    globals()["previous_uuid"] = uuid
+    return uuid
+
+
+class DoNothingConnector(Mock):
+  def __init__(self, s=None):
+    logging.info("initializing connector")
+    self.desc = globals()['connector_cpt']
+    globals()['connector_cpt'] = globals()['connector_cpt']+ 1
+    self.packet_cpt = 0
+    Mock.__init__(self)
+
+  def getAddress(self):
+    return self.addr
+  
+  def makeClientConnection(self, addr):
+    self.addr = addr
+
+  def getDescriptor(self):
+    return self.desc
+
+
+class TestElectionConnector(DoNothingConnector):
+  def receive(self):
+    """ simulate behavior of election """
+    if self.packet_cpt == 0:
+      # first : identify
+      logging.info("in patched analyse / IDENTIFICATION")
+      p = Packet()
+      self.uuid = getNewUUID()
+      p.acceptNodeIdentification(1,
+                                 MASTER_NODE_TYPE,
+                                 self.uuid,
+                                 self.getAddress()[0],
+                                 self.getAddress()[1],
+                                 1009,
+                                 2
+                                 )
+      self.packet_cpt += 1
+      return p.encode()
+    elif self.packet_cpt == 1:
+      # second : answer primary master nodes
+      logging.info("in patched analyse / ANSWER PM")
+      p = Packet()
+      p.answerPrimaryMaster(2,
+                            INVALID_UUID,
+                            []
+                            )        
+      self.packet_cpt += 1
+      return p.encode()
+    else:
+      # then do nothing
+      from neo.connector import ConnectorTryAgainException
+      raise ConnectorTryAgainException
+    

neo/master/tests/testMasterElectionHandler.py

+#
+# Copyright (C) 2009  Nexedi SA
+# 
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+import os
+import unittest
+import logging
+from tempfile import mkstemp
+from mock import Mock
+from struct import pack, unpack
+from neo.protocol import Packet, INVALID_UUID
+from neo.master.election import ElectionEventHandler
+from neo.master.app import Application
+from neo.protocol import ERROR, REQUEST_NODE_IDENTIFICATION, ACCEPT_NODE_IDENTIFICATION, \
+     PING, PONG, ASK_PRIMARY_MASTER, ANSWER_PRIMARY_MASTER, ANNOUNCE_PRIMARY_MASTER, \
+     REELECT_PRIMARY_MASTER, NOTIFY_NODE_INFORMATION, START_OPERATION, \
+     STOP_OPERATION, ASK_LAST_IDS, ANSWER_LAST_IDS, ASK_PARTITION_TABLE, \
+     ANSWER_PARTITION_TABLE, SEND_PARTITION_TABLE, NOTIFY_PARTITION_CHANGES, \
+     ASK_UNFINISHED_TRANSACTIONS, ANSWER_UNFINISHED_TRANSACTIONS, \
+     ASK_OBJECT_PRESENT, ANSWER_OBJECT_PRESENT, \
+     DELETE_TRANSACTION, COMMIT_TRANSACTION, ASK_NEW_TID, ANSWER_NEW_TID, \
+     FINISH_TRANSACTION, NOTIFY_TRANSACTION_FINISHED, LOCK_INFORMATION, \
+     NOTIFY_INFORMATION_LOCKED, INVALIDATE_OBJECTS, UNLOCK_INFORMATION, \
+     ASK_NEW_OIDS, ANSWER_NEW_OIDS, ASK_STORE_OBJECT, ANSWER_STORE_OBJECT, \
+     ABORT_TRANSACTION, ASK_STORE_TRANSACTION, ANSWER_STORE_TRANSACTION, \
+     ASK_OBJECT, ANSWER_OBJECT, ASK_TIDS, ANSWER_TIDS, ASK_TRANSACTION_INFORMATION, \
+     ANSWER_TRANSACTION_INFORMATION, ASK_OBJECT_HISTORY, ANSWER_OBJECT_HISTORY, \
+     ASK_OIDS, ANSWER_OIDS, \
+     NOT_READY_CODE, OID_NOT_FOUND_CODE, SERIAL_NOT_FOUND_CODE, TID_NOT_FOUND_CODE, \
+     PROTOCOL_ERROR_CODE, TIMEOUT_ERROR_CODE, BROKEN_NODE_DISALLOWED_CODE, \
+     INTERNAL_ERROR_CODE, \
+     STORAGE_NODE_TYPE, CLIENT_NODE_TYPE, MASTER_NODE_TYPE, \
+     RUNNING_STATE, BROKEN_STATE, TEMPORARILY_DOWN_STATE, DOWN_STATE, \
+     UP_TO_DATE_STATE, OUT_OF_DATE_STATE, FEEDING_STATE, DISCARDED_STATE
+from neo.exception import OperationFailure, ElectionFailure     
+from neo.node import MasterNode, StorageNode
+from neo.master.tests.connector import DoNothingConnector
+from neo.connection import ClientConnection
+
+# patch connection so that we can register addPacket messages
+# in mock object
+def addPacket(self, packet):
+    if self.connector is not None:
+        self.connector.addPacket(packet)
+def expectMessage(self, packet):
+    if self.connector is not None:
+        self.connector.expectMessage(packet)
+
+ClientConnection.addPacket = addPacket
+ClientConnection.expectMessage = expectMessage
+
+
+class MasterServiceTests(unittest.TestCase):
+
+    def setUp(self):
+        logging.basicConfig(level = logging.DEBUG)
+        # create an application object
+        config_file_text = """# Default parameters.
+[DEFAULT]
+# The list of master nodes.
+master_nodes: 127.0.0.1:10010 127.0.0.1:10011
+# The number of replicas.
+replicas: 2
+# The number of partitions.
+partitions: 1009
+# The name of this cluster.
+name: main
+# The user name for the database.
+user: neo
+# The password for the database.
+password: neo
+connector : SocketConnector
+# The first master.
+[mastertest]
+server: 127.0.0.1:10010
+
+# The first storage.
+[storage1]
+database: neotest1
+server: 127.0.0.1:10020
+
+# The second storage.
+[storage2]
+database: neotest2
+server: 127.0.0.1:10021
+
+# The third storage.
+[storage3]
+database: neotest3
+server: 127.0.0.1:10022
+
+# The fourth storage.
+[storage4]
+database: neotest4
+server: 127.0.0.1:10023
+"""
+        tmp_id, self.tmp_path = mkstemp()
+        tmp_file = os.fdopen(tmp_id, "w+b")
+        tmp_file.write(config_file_text)
+        tmp_file.close()
+        self.app = Application(self.tmp_path, "mastertest")        
+        self.app.pt.clear()
+        self.app.finishing_transaction_dict = {}
+        for server in self.app.master_node_list:
+            self.app.nm.add(MasterNode(server = server))
+        self.election = ElectionEventHandler(self.app)
+        self.app.unconnected_master_node_set = set()
+        self.app.negotiating_master_node_set = set()
+        for node in self.app.nm.getMasterNodeList():
+            self.app.unconnected_master_node_set.add(node.getServer())
+            node.setState(RUNNING_STATE)
+
+        # define some variable to simulate client and storage node
+        self.client_port = 11022
+        self.storage_port = 10021
+        self.master_port = 10011
+        
+    def tearDown(self):
+        # Delete tmp file
+        os.remove(self.tmp_path)
+
+    def checkCalledAcceptNodeIdentification(self, conn, packet_number=0):
+        """ Check Accept Node Identification has been send"""
+        self.assertEquals(len(conn.mockGetNamedCalls("addPacket")), 1)
+        self.assertEquals(len(conn.mockGetNamedCalls("abort")), 0)
+        self.assertEquals(len(conn.mockGetNamedCalls("expectMessage")), 1)
+        call = conn.mockGetNamedCalls("addPacket")[packet_number]
+        packet = call.getParam(0)
+        self.assertTrue(isinstance(packet, Packet))
+        self.assertEquals(packet.getType(), ACCEPT_NODE_IDENTIFICATION)
+
+    # Common methods
+    def getNewUUID(self):
+        uuid = INVALID_UUID
+        while uuid == INVALID_UUID:
+            uuid = os.urandom(16)
+        self.uuid = uuid
+        return uuid
+
+    def getLastUUID(self):
+        return self.uuid
+
+    def identifyToMasterNode(self, node_type=STORAGE_NODE_TYPE, ip="127.0.0.1",
+                             port=10021):
+        """Do first step of identification to MN
+        """
+        packet = Packet(msg_id=1)
+        uuid = self.getNewUUID()
+        # test alien cluster
+        conn = Mock({"addPacket" : None, "abort" : None, "expectMessage" : None})
+        self.election.handleRequestNodeIdentification(conn,
+                                                packet=packet,
+                                                node_type=node_type,
+                                                uuid=uuid,
+                                                ip_address=ip,
+                                                port=port,
+                                                name=self.app.name,)
+        self.checkCalledAcceptNodeIdentification(conn)
+        return uuid
+
+    # Method to test the kind of packet returned in answer
+    def checkCalledRequestNodeIdentification(self, conn, packet_number=0):
+        """ Check Request Node Identification has been send"""
+        self.assertEquals(len(conn.mockGetNamedCalls("addPacket")), 1)
+        self.assertEquals(len(conn.mockGetNamedCalls("abort")), 0)
+        self.assertEquals(len(conn.mockGetNamedCalls("expectMessage")), 1)
+        call = conn.mockGetNamedCalls("addPacket")[packet_number]
+        packet = call.getParam(0)
+        self.assertTrue(isinstance(packet, Packet))
+        self.assertEquals(packet.getType(), REQUEST_NODE_IDENTIFICATION)
+
+        
+    def checkCalledAskPrimaryMaster(self, conn, packet_number=0):
+        """ Check ask primary master has been send"""
+        call = conn.mockGetNamedCalls("addPacket")[packet_number]
+        packet = call.getParam(0)
+        self.assertTrue(isinstance(packet, Packet))
+        self.assertEquals(packet.getType(),ASK_PRIMARY_MASTER)
+
+    def checkCalledAbort(self, conn, packet_number=0):
+        """Check the abort method has been called and an error packet has been sent"""
+        self.assertEquals(len(conn.mockGetNamedCalls("addPacket")), 1) # XXX required here ????
+        self.assertEquals(len(conn.mockGetNamedCalls("abort")), 1)
+        self.assertEquals(len(conn.mockGetNamedCalls("expectMessage")), 0)
+        call = conn.mockGetNamedCalls("addPacket")[packet_number]
+        packet = call.getParam(0)
+        self.assertTrue(isinstance(packet, Packet))
+        self.assertEquals(packet.getType(), ERROR)
+
+    # Tests
+    def test_01_connectionStarted(self):
+        uuid = self.identifyToMasterNode(node_type=MASTER_NODE_TYPE, port=self.master_port)
+        conn = Mock({"getUUID" : uuid,
+                     "getAddress" : ("127.0.0.1", self.master_port)})
+        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        self.election.connectionStarted(conn)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        
+
+    def test_02_connectionCompleted(self):
+        uuid = self.identifyToMasterNode(node_type=MASTER_NODE_TYPE, port=self.master_port)
+        conn = Mock({"getUUID" : uuid,
+                     "getAddress" : ("127.0.0.1", self.master_port)})
+        self.election.connectionCompleted(conn)
+        self.checkCalledRequestNodeIdentification(conn)
+    
+
+    def test_03_connectionFailed(self):
+        uuid = self.identifyToMasterNode(node_type=MASTER_NODE_TYPE, port=self.master_port)
+        conn = Mock({"getUUID" : uuid,
+                     "getAddress" : ("127.0.0.1", self.master_port)})
+        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        self.election.connectionStarted(conn)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getState(), RUNNING_STATE)
+        self.election.connectionFailed(conn)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getState(), TEMPORARILY_DOWN_STATE)
+
+    def test_04_connectionClosed(self):
+        uuid = self.identifyToMasterNode(node_type=MASTER_NODE_TYPE, port=self.master_port)
+        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
+                                connector_handler = DoNothingConnector)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getState(), RUNNING_STATE)
+        self.election.connectionClosed(conn)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getState(), TEMPORARILY_DOWN_STATE)
+
+    def test_05_timeoutExpired(self):
+        uuid = self.identifyToMasterNode(node_type=MASTER_NODE_TYPE, port=self.master_port)
+        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
+                                connector_handler = DoNothingConnector)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getState(), RUNNING_STATE)
+        self.election.timeoutExpired(conn)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getState(), TEMPORARILY_DOWN_STATE)
+
+    def test_06_peerBroken1(self):
+        uuid = self.identifyToMasterNode(node_type=MASTER_NODE_TYPE, port=self.master_port)
+        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
+                                connector_handler = DoNothingConnector)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getState(), RUNNING_STATE)
+        self.election.peerBroken(conn)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getState(), DOWN_STATE)
+
+    def test_06_peerBroken2(self):
+        uuid = self.identifyToMasterNode(node_type=MASTER_NODE_TYPE, port=self.master_port)
+        # Without a client connection
+        conn = Mock({"getUUID" : uuid,
+                     "getAddress" : ("127.0.0.1", self.master_port),})
+        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        self.election.connectionStarted(conn)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getState(), RUNNING_STATE)
+        self.election.peerBroken(conn)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getState(), BROKEN_STATE)
+
+    def test_07_packetReceived(self):
+        p = Packet()
+        uuid = self.identifyToMasterNode(node_type=MASTER_NODE_TYPE, port=self.master_port)
+        p.acceptNodeIdentification(1,
+                                   MASTER_NODE_TYPE,
+                                   uuid,
+                                   "127.0.0.1",
+                                   self.master_port,
+                                   1009,
+                                   2
+                                   )
+
+        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
+                                connector_handler = DoNothingConnector)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        node = self.app.nm.getNodeByServer(conn.getAddress())
+        node.setState(DOWN_STATE)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getState(), DOWN_STATE)
+        self.election.packetReceived(conn, p)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getState(), RUNNING_STATE)
+
+    def test_08_handleAcceptNodeIdentification1(self):
+        # test with storage node, must be rejected
+        uuid = self.getNewUUID()
+        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
+                                connector_handler = DoNothingConnector)
+        p = Packet()
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getUUID(), None)
+        self.assertEqual(conn.getUUID(), None)
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+        self.election.handleAcceptNodeIdentification(conn, p, STORAGE_NODE_TYPE,
+                                                     uuid, "127.0.0.1", self.master_port,
+                                                     self.app.num_partitions, self.app.num_replicas
+                                                     )
+        self.assertEqual(conn.getConnector(), None)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        
+    def test_08_handleAcceptNodeIdentification2(self):
+        # test with bad address, must be rejected
+        uuid = self.getNewUUID()
+        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
+                                connector_handler = DoNothingConnector)
+        p = Packet()
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getUUID(), None)
+        self.assertEqual(conn.getUUID(), None)
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+        self.election.handleAcceptNodeIdentification(conn, p, STORAGE_NODE_TYPE,
+                                                     uuid, "127.0.0.2", self.master_port,
+                                                     self.app.num_partitions, self.app.num_replicas
+                                                     )
+        self.assertEqual(conn.getConnector(), None)
+
+    def test_08_handleAcceptNodeIdentification3(self):
+        # test with master node, must be ok
+        uuid = self.getNewUUID()
+        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
+                                connector_handler = DoNothingConnector)
+        p = Packet()
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getUUID(), None)
+        self.assertEqual(conn.getUUID(), None)
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+
+        self.election.handleAcceptNodeIdentification(conn, p, MASTER_NODE_TYPE,
+                                                     uuid, "127.0.0.1", self.master_port,
+                                                     self.app.num_partitions, self.app.num_replicas
+                                                     )
+        self.assertEqual(self.app.nm.getNodeByServer(conn.getAddress()).getUUID(), uuid)
+        self.assertEqual(conn.getUUID(), uuid)
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),2)
+        self.checkCalledAskPrimaryMaster(conn.getConnector(), -1)
+        
+
+    def test_09_handleAnswerPrimaryMaster1(self):
+        # test with master node and greater uuid
+        uuid = self.getNewUUID()
+        while uuid < self.app.uuid:
+            uuid = self.getNewUUID()
+        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
+                                connector_handler = DoNothingConnector)
+        conn.setUUID(uuid)
+        p = Packet()
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 1)
+        self.election.handleAnswerPrimaryMaster(conn, p, INVALID_UUID, [])
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+        self.assertEqual(self.app.primary, False)
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 1)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+
+
+    def test_09_handleAnswerPrimaryMaster2(self):
+        # test with master node and lesser uuid
+        uuid = self.getNewUUID()
+        while uuid > self.app.uuid:
+            uuid = self.getNewUUID()
+        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
+                                connector_handler = DoNothingConnector)
+        conn.setUUID(uuid)
+        p = Packet()
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 1)
+        self.election.handleAnswerPrimaryMaster(conn, p, INVALID_UUID, [])
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+        self.assertEqual(self.app.primary, None)
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 1)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+
+
+    def test_09_handleAnswerPrimaryMaster3(self):
+        # test with master node and given uuid for PMN
+        uuid = self.getNewUUID()
+        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
+                                connector_handler = DoNothingConnector)
+        conn.setUUID(uuid)
+        p = Packet()
+        self.app.nm.add(MasterNode(("127.0.0.1", self.master_port), uuid))
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 2)
+        self.assertEqual(self.app.primary_master_node, None)
+        self.election.handleAnswerPrimaryMaster(conn, p, uuid, [])
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 2)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        self.assertNotEqual(self.app.primary_master_node, None)
+        self.assertEqual(self.app.primary, False)
+
+
+    def test_09_handleAnswerPrimaryMaster4(self):
+        # test with master node and unknown uuid for PMN
+        uuid = self.getNewUUID()
+        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
+                                connector_handler = DoNothingConnector)
+        conn.setUUID(uuid)
+        p = Packet()
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 1)
+        self.assertEqual(self.app.primary_master_node, None)
+        self.election.handleAnswerPrimaryMaster(conn, p, uuid, [])
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 1)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        self.assertEqual(self.app.primary_master_node, None)
+        self.assertEqual(self.app.primary, None)
+
+
+    def test_09_handleAnswerPrimaryMaster5(self):
+        # test with master node and new uuid for PMN
+        uuid = self.getNewUUID()
+        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
+                                connector_handler = DoNothingConnector)
+        conn.setUUID(uuid)
+        p = Packet()
+        self.app.nm.add(MasterNode(("127.0.0.1", self.master_port), uuid))
+        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 2)
+        self.assertEqual(self.app.primary_master_node, None)
+        master_uuid = self.getNewUUID()
+        self.election.handleAnswerPrimaryMaster(conn, p, master_uuid, [("127.0.0.1", self.master_port+1, master_uuid),])
+        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("addPacket")),1)
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 3)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        self.assertNotEqual(self.app.primary_master_node, None)
+        self.assertEqual(self.app.primary, False)
+        # Now tell it's another node which is primary, it must raise 
+        self.assertRaises(ElectionFailure, self.election.handleAnswerPrimaryMaster, conn, p, uuid, [])
+
+        
+    def test_10_handleRequestNodeIdentification(self):
+        election = self.election
+        packet = Packet(msg_id=1)
+        uuid = self.getNewUUID()
+        # test alien cluster
+        conn = Mock({"addPacket" : None, "abort" : None})
+        election.handleRequestNodeIdentification(conn,
+                                                packet=packet,
+                                                node_type=MASTER_NODE_TYPE,
+                                                uuid=uuid,
+                                                ip_address='127.0.0.1',
+                                                port=self.storage_port,
+                                                name="INVALID_NAME",)
+        self.checkCalledAbort(conn)        
+        # test connection of a storage node
+        conn = Mock({"addPacket" : None, "abort" : None, "expectMessage" : None})
+        election.handleRequestNodeIdentification(conn,
+                                                packet=packet,
+                                                node_type=STORAGE_NODE_TYPE,
+                                                uuid=uuid,
+                                                ip_address='127.0.0.1',
+                                                port=self.storage_port,
+                                                name=self.app.name,)
+        self.checkCalledAbort(conn)
+
+        # known node
+        conn = Mock({"addPacket" : None, "abort" : None, "expectMessage" : None})
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 1)
+        node = self.app.nm.getMasterNodeList()[0]
+        self.assertEqual(node.getUUID(), None)
+        self.assertEqual(node.getState(), RUNNING_STATE)
+        election.handleRequestNodeIdentification(conn,
+                                                packet=packet,
+                                                node_type=MASTER_NODE_TYPE,
+                                                uuid=uuid,
+                                                ip_address='127.0.0.1',
+                                                port=self.master_port,
+                                                name=self.app.name,)
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 1)
+        self.assertEqual(node.getUUID(), uuid)
+        self.assertEqual(node.getState(), RUNNING_STATE)
+        self.checkCalledAcceptNodeIdentification(conn)
+        # unknown node
+        conn = Mock({"addPacket" : None, "abort" : None, "expectMessage" : None})
+        new_uuid = self.getNewUUID()
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 1)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        election.handleRequestNodeIdentification(conn,
+                                                packet=packet,
+                                                node_type=MASTER_NODE_TYPE,
+                                                uuid=new_uuid,
+                                                ip_address='127.0.0.1',
+                                                port=self.master_port+1,
+                                                name=self.app.name,)
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 2)
+        self.checkCalledAcceptNodeIdentification(conn)
+        self.assertEqual(len(self.app.unconnected_master_node_set), 2)
+        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
+        # broken node
+        conn = Mock({"addPacket" : None, "abort" : None, "expectMessage" : None})
+        node = self.app.nm.getNodeByServer(("127.0.0.1", self.master_port+1))
+        self.assertEqual(node.getUUID(), new_uuid)
+        self.assertEqual(node.getState(), RUNNING_STATE)
+        node.setState(BROKEN_STATE)
+        self.assertEqual(node.getState(), BROKEN_STATE)
+        election.handleRequestNodeIdentification(conn,
+                                                 packet=packet,
+                                                 node_type=MASTER_NODE_TYPE,
+                                                 uuid=new_uuid,
+                                                 ip_address='127.0.0.1',
+                                                 port=self.master_port+1,
+                                                 name=self.app.name,)        
+        self.checkCalledAbort(conn)        
+
+
+    def test_11_handleAskPrimaryMaster(self):
+        election = self.election
+        uuid = self.identifyToMasterNode(MASTER_NODE_TYPE, port=self.master_port)
+        packet = Packet(msg_id=2)
+        conn = Mock({"addPacket" : None,
+                     "getUUID" : uuid,
+                     "getAddress" : ("127.0.0.1", self.master_port)})
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 1)
+        election.handleAskPrimaryMaster(conn, packet)        
+        self.assertEquals(len(conn.mockGetNamedCalls("addPacket")), 1)
+        self.assertEquals(len(conn.mockGetNamedCalls("abort")), 0)
+        conn.checkCalledAnswerPrimaryMaster(conn, 1)
+
+    def test_12_handleAnnouncePrimaryMaster(self):
+        election = self.election
+        uuid = self.identifyToMasterNode(MASTER_NODE_TYPE, port=self.master_port)
+        packet = Packet(msg_id=3, msg_type=ANNOUNCE_PRIMARY_MASTER)
+        # No uuid
+        conn = Mock({"addPacket" : None,
+                     "getUUID" : None,
+                     "getAddress" : ("127.0.0.1", self.master_port)})
+        self.assertEqual(len(self.app.nm.getMasterNodeList()), 1)
+        election.handleAnnouncePrimaryMaster(conn, packet)
+        self.checkCalledAbort(conn)
+        # announce
+        conn = Mock({"addPacket" : None,
+                     "getUUID" : uuid,
+                     "getAddress" : ("127.0.0.1", self.master_port)})
+        self.assertEqual(self.app.primary, None)
+        self.assertEqual(self.app.primary_master_node, None)
+        election.handleAnnouncePrimaryMaster(conn, packet)        
+        self.assertEqual(self.app.primary, False)
+        self.assertNotEqual(self.app.primary_master_node, None)
+        # set current as primary, and announce another, must raise
+        conn = Mock({"addPacket" : None,
+                     "getUUID" : uuid,
+                     "getAddress" : ("127.0.0.1", self.master_port)})
+        self.app.primary = True
+        self.assertEqual(self.app.primary, True)
+        self.assertRaises(ElectionFailure, election.handleAnnouncePrimaryMaster, conn, packet)        
+        
+
+    def test_13_handleReelectPrimaryMaster(self):
+        election = self.election
+        uuid = self.identifyToMasterNode(MASTER_NODE_TYPE, port=self.master_port)
+        packet = Packet()
+        # No uuid
+        conn = Mock({"addPacket" : None,
+                     "getUUID" : None,
+                     "getAddress" : ("127.0.0.1", self.master_port)})
+        self.assertRaises(ElectionFailure, election.handleReelectPrimaryMaster, conn, packet)
+
+    def test_14_handleNotifyNodeInformation(self):
+        election = self.election
+        uuid = self.identifyToMasterNode(MASTER_NODE_TYPE, port=self.master_port)
+        packet = Packet(msg_id=5, msg_type=NOTIFY_NODE_INFORMATION)
+        # do not answer if no uuid
+        conn = Mock({"getUUID" : None,
+                     "getAddress" : ("127.0.0.1", self.master_port)})
+        node_list = []
+        election.handleNotifyNodeInformation(conn, packet, node_list)
+        self.checkCalledAbort(conn)
+        # tell the master node about itself, must do nothing
+        conn = Mock({"getUUID" : uuid,
+                     "getAddress" : ("127.0.0.1", self.master_port)})                
+        node_list = [(MASTER_NODE_TYPE, '127.0.0.1', self.master_port - 1, self.app.uuid, DOWN_STATE),]
+        node = self.app.nm.getNodeByServer(("127.0.0.1", self.master_port-1))
+        self.assertEqual(node, None)
+        election.handleNotifyNodeInformation(conn, packet, node_list)
+        node = self.app.nm.getNodeByServer(("127.0.0.1", self.master_port-1))
+        self.assertEqual(node, None)
+        # tell about a storage node, do nothing
+        conn = Mock({"getUUID" : uuid,
+                     "getAddress" : ("127.0.0.1", self.master_port)})                
+        node_list = [(STORAGE_NODE_TYPE, '127.0.0.1', self.master_port - 1, self.getNewUUID(), DOWN_STATE),]
+        self.assertEqual(len(self.app.nm.getStorageNodeList()), 0)
+        election.handleNotifyNodeInformation(conn, packet, node_list)
+        self.assertEqual(len(self.app.nm.getStorageNodeList()), 0)
+        # tell about a client node, do nothing
+        conn = Mock({"getUUID" : uuid,
+                     "getAddress" : ("127.0.0.1", self.master_port)})                
+        node_list = [(CLIENT_NODE_TYPE, '127.0.0.1', self.master_port - 1, self.getNewUUID(), DOWN_STATE),]
+        self.assertEqual(len(self.app.nm.getClientNodeList()), 0)
+        election.handleNotifyNodeInformation(conn, packet, node_list)
+        self.assertEqual(len(self.app.nm.getClientNodeList()), 0)
+        # tell about another master node
+        conn = Mock({"getUUID" : uuid,
+                     "getAddress" : ("127.0.0.1", self.master_port)})                
+        node_list = [(MASTER_NODE_TYPE, '127.0.0.1', self.master_port + 1, self.getNewUUID(), RUNNING_STATE),]
+        node = self.app.nm.getNodeByServer(("127.0.0.1", self.master_port+1))
+        self.assertEqual(node, None)
+        election.handleNotifyNodeInformation(conn, packet, node_list)
+        node = self.app.nm.getNodeByServer(("127.0.0.1", self.master_port+1))
+        self.assertNotEqual(node, None)
+        self.assertEqual(node.getServer(), ("127.0.0.1", self.master_port+1))
+        self.assertEqual(node.getState(), RUNNING_STATE)
+        # tell that node is down
+        node_list = [(MASTER_NODE_TYPE, '127.0.0.1', self.master_port + 1, self.getNewUUID(), DOWN_STATE),]
+        election.handleNotifyNodeInformation(conn, packet, node_list)
+        node = self.app.nm.getNodeByServer(("127.0.0.1", self.master_port+1))
+        self.assertNotEqual(node, None)
+        self.assertEqual(node.getServer(), ("127.0.0.1", self.master_port+1))
+        self.assertEqual(node.getState(), DOWN_STATE)
+
+    
+if __name__ == '__main__':
+    unittest.main()
+