testElectionHandler.py 31.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
#
# Copyright (C) 2009  Nexedi SA
# 
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 18 19

import unittest
from mock import Mock
20
from neo.tests import NeoTestBase
21
from neo.protocol import Packet, Packets, NodeTypes, NodeStates, INVALID_UUID
22
from neo.master.handlers.election import ClientElectionHandler, ServerElectionHandler
23
from neo.master.app import Application
24
from neo.exception import ElectionFailure     
25
from neo.tests import DoNothingConnector
26 27
from neo.connection import ClientConnection

28
# patch connection so that we can register _addPacket messages
29
# in mock object
30
def _addPacket(self, packet):
31
    if self.connector is not None:
32
        self.connector._addPacket(packet)
33 34 35 36 37
def expectMessage(self, packet):
    if self.connector is not None:
        self.connector.expectMessage(packet)


38
class MasterClientElectionTests(NeoTestBase):
39

40 41
    def setUp(self):
        # create an application object
42
        config = self.getMasterConfiguration()
43
        self.app = Application(config)
44 45 46
        self.app.pt.clear()
        self.app.em = Mock({"getConnectionList" : []})
        self.app.finishing_transaction_dict = {}
47 48
        for address in self.app.master_node_list:
            self.app.nm.createMaster(address=address)
49 50 51
        self.election = ClientElectionHandler(self.app)
        self.app.unconnected_master_node_set = set()
        self.app.negotiating_master_node_set = set()
52
        for node in self.app.nm.getMasterList():
53
            self.app.unconnected_master_node_set.add(node.getAddress())
54
            node.setState(NodeStates.RUNNING)
55 56 57 58 59 60 61 62 63 64 65
        # define some variable to simulate client and storage node
        self.client_port = 11022
        self.storage_port = 10021
        self.master_port = 10011
        # apply monkey patches
        self._addPacket = ClientConnection._addPacket
        self.expectMessage = ClientConnection.expectMessage
        ClientConnection._addPacket = _addPacket
        ClientConnection.expectMessage = expectMessage
        
    def tearDown(self):
66 67 68
        # restore patched methods
        ClientConnection._addPacket = self._addPacket
        ClientConnection.expectMessage = self.expectMessage
69 70
        NeoTestBase.tearDown(self)

71 72 73
    def identifyToMasterNode(self, port=10000, ip='127.0.0.1'):
        uuid = self.getNewUUID()
        address = (ip, port)
74
        self.app.nm.createMaster(address=address, uuid=uuid,
75
                state=NodeStates.RUNNING)
76 77
        return uuid

78
    def test_01_connectionStarted(self):
79
        uuid = self.identifyToMasterNode(port=self.master_port)
80 81 82 83 84 85 86 87 88 89
        conn = Mock({"getUUID" : uuid,
                     "getAddress" : ("127.0.0.1", self.master_port)})
        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
        self.election.connectionStarted(conn)
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
        

    def test_02_connectionCompleted(self):
90
        uuid = self.identifyToMasterNode(port=self.master_port)
91 92 93
        conn = Mock({"getUUID" : uuid,
                     "getAddress" : ("127.0.0.1", self.master_port)})
        self.election.connectionCompleted(conn)
94
        self.checkAskPrimary(conn)
95 96 97
    

    def test_03_connectionFailed(self):
98
        uuid = self.identifyToMasterNode(port=self.master_port)
99 100 101 102 103 104 105
        conn = Mock({"getUUID" : uuid,
                     "getAddress" : ("127.0.0.1", self.master_port)})
        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
        self.election.connectionStarted(conn)
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
106 107
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                NodeStates.RUNNING)
108 109 110
        self.election.connectionFailed(conn)
        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
111 112
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                NodeStates.TEMPORARILY_DOWN)
113

114
    def test_11_askPrimary(self):
115 116
        election = self.election
        uuid = self.identifyToMasterNode(port=self.master_port)
117 118
        packet = Packets.AskPrimary()
        packet.setId(0)
119 120 121 122 123
        conn = Mock({"_addPacket" : None,
                     "getUUID" : uuid,
                     "isServer" : True,
                     "getConnector": Mock(),
                     "getAddress" : ("127.0.0.1", self.master_port)})
124
        self.assertEqual(len(self.app.nm.getMasterList()), 2)
125
        election.askPrimary(conn, packet)        
126 127
        self.assertEquals(len(conn.mockGetNamedCalls("answer")), 1)
        self.assertEquals(len(conn.mockGetNamedCalls("abort")), 0)
128
        self.checkAnswerPrimary(conn)
129

130
    def test_09_answerPrimary1(self):
131 132 133 134 135 136 137
        # test with master node and greater uuid
        uuid = self.getNewUUID()
        if uuid < self.app.uuid:
            self.app.uuid, uuid = self.app.uuid, uuid
        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
                                connector_handler = DoNothingConnector)
        conn.setUUID(uuid)
138
        p = Packets.AskPrimary()
139 140 141
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
142
        self.assertEqual(len(self.app.nm.getMasterList()), 1)
143
        self.election.answerPrimary(conn, p, INVALID_UUID, [])
144 145
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
        self.assertEqual(self.app.primary, False)
146
        self.assertEqual(len(self.app.nm.getMasterList()), 1)
147 148 149 150
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)


151
    def test_09_answerPrimary2(self):
152 153 154 155 156 157 158
        # test with master node and lesser uuid
        uuid = self.getNewUUID()
        if uuid > self.app.uuid:
            self.app.uuid, uuid = self.app.uuid, uuid
        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
                                connector_handler = DoNothingConnector)
        conn.setUUID(uuid)
159
        p = Packets.AskPrimary()
160 161 162
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
163
        self.assertEqual(len(self.app.nm.getMasterList()), 1)
164
        self.election.answerPrimary(conn, p, INVALID_UUID, [])
165 166
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
        self.assertEqual(self.app.primary, None)
167
        self.assertEqual(len(self.app.nm.getMasterList()), 1)
168 169 170 171
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)


172
    def test_09_answerPrimary3(self):
173 174 175 176 177
        # test with master node and given uuid for PMN
        uuid = self.getNewUUID()
        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
                                connector_handler = DoNothingConnector)
        conn.setUUID(uuid)
178
        p = Packets.AskPrimary()
179
        self.app.nm.createMaster(address=("127.0.0.1", self.master_port), uuid=uuid)
180 181 182
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
183
        self.assertEqual(len(self.app.nm.getMasterList()), 2)
184
        self.assertEqual(self.app.primary_master_node, None)
185
        self.election.answerPrimary(conn, p, uuid, [])
186
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
187
        self.assertEqual(len(self.app.nm.getMasterList()), 2)
188 189 190 191 192 193
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
        self.assertNotEqual(self.app.primary_master_node, None)
        self.assertEqual(self.app.primary, False)


194
    def test_09_answerPrimary4(self):
195 196 197 198 199
        # test with master node and unknown uuid for PMN
        uuid = self.getNewUUID()
        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
                                connector_handler = DoNothingConnector)
        conn.setUUID(uuid)
200
        p = Packets.AskPrimary()
201 202 203
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
204
        self.assertEqual(len(self.app.nm.getMasterList()), 1)
205
        self.assertEqual(self.app.primary_master_node, None)
206
        self.election.answerPrimary(conn, p, uuid, [])
207
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
208
        self.assertEqual(len(self.app.nm.getMasterList()), 1)
209 210 211 212 213 214
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
        self.assertEqual(self.app.primary_master_node, None)
        self.assertEqual(self.app.primary, None)


215
    def test_09_answerPrimary5(self):
216 217 218 219 220
        # test with master node and new uuid for PMN
        uuid = self.getNewUUID()
        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
                                connector_handler = DoNothingConnector)
        conn.setUUID(uuid)
221
        p = Packets.AskPrimary()
222
        self.app.nm.createMaster(address=("127.0.0.1", self.master_port), uuid=uuid)
223 224 225
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
226
        self.assertEqual(len(self.app.nm.getMasterList()), 2)
227 228
        self.assertEqual(self.app.primary_master_node, None)
        master_uuid = self.getNewUUID()
229
        self.election.answerPrimary(conn, p, master_uuid,
230 231
                [(("127.0.0.1", self.master_port+1), master_uuid,)])
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
232
        self.assertEqual(len(self.app.nm.getMasterList()), 3)
233 234 235 236 237
        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
        self.assertNotEqual(self.app.primary_master_node, None)
        self.assertEqual(self.app.primary, False)
        # Now tell it's another node which is primary, it must raise 
238
        self.assertRaises(ElectionFailure, self.election.answerPrimary, conn, p, uuid, [])
239 240

        
241 242

class MasterServerElectionTests(NeoTestBase):
243 244 245

    def setUp(self):
        # create an application object
246 247
        config = self.getMasterConfiguration()
        self.app = Application(**config)
248
        self.app.pt.clear()
249
        self.app.em = Mock({"getConnectionList" : []})
250
        self.app.finishing_transaction_dict = {}
251 252
        for address in self.app.master_node_list:
            self.app.nm.createMaster(address=address)
253
        self.election = ServerElectionHandler(self.app)
254 255
        self.app.unconnected_master_node_set = set()
        self.app.negotiating_master_node_set = set()
256
        for node in self.app.nm.getMasterList():
257
            self.app.unconnected_master_node_set.add(node.getAddress())
258
            node.setState(NodeStates.RUNNING)
259 260 261 262
        # define some variable to simulate client and storage node
        self.client_port = 11022
        self.storage_port = 10021
        self.master_port = 10011
263 264 265 266 267
        # apply monkey patches
        self._addPacket = ClientConnection._addPacket
        self.expectMessage = ClientConnection.expectMessage
        ClientConnection._addPacket = _addPacket
        ClientConnection.expectMessage = expectMessage
268 269
        
    def tearDown(self):
270
        NeoTestBase.tearDown(self)
271 272 273
        # restore environnement
        ClientConnection._addPacket = self._addPacket
        ClientConnection.expectMessage = self.expectMessage
274

275
    def identifyToMasterNode(self, node_type=NodeTypes.STORAGE, ip="127.0.0.1",
276 277 278 279 280 281 282
                             port=10021):
        """Do first step of identification to MN
        """
        uuid = self.getNewUUID()
        return uuid

        
283
    def checkCalledAskPrimary(self, conn, packet_number=0):
284
        """ Check ask primary master has been send"""
285
        call = conn.mockGetNamedCalls("_addPacket")[packet_number]
286 287
        packet = call.getParam(0)
        self.assertTrue(isinstance(packet, Packet))
288
        self.assertEquals(packet.getType(),AskPrimary)
289 290 291 292

    # Tests

    def test_04_connectionClosed(self):
293
        uuid = self.identifyToMasterNode(port=self.master_port)
294 295 296 297
        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
                                connector_handler = DoNothingConnector)
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
298 299
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                NodeStates.RUNNING)
300 301 302
        self.election.connectionClosed(conn)
        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
303 304
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                NodeStates.TEMPORARILY_DOWN)
305 306

    def test_05_timeoutExpired(self):
307
        uuid = self.identifyToMasterNode(port=self.master_port)
308 309 310 311
        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
                                connector_handler = DoNothingConnector)
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
312 313
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                NodesStates.RUNNING)
314 315 316
        self.election.timeoutExpired(conn)
        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
317 318
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                NodeStates.TEMPORARILY_DOWN)
319 320

    def test_06_peerBroken1(self):
321
        uuid = self.identifyToMasterNode(port=self.master_port)
322 323 324 325
        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
                                connector_handler = DoNothingConnector)
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
326 327
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                NodeStates.RUNNING)
328 329 330
        self.election.peerBroken(conn)
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
331 332
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                NodeStates.DOWN)
333 334

    def test_06_peerBroken2(self):
335
        uuid = self.identifyToMasterNode(port=self.master_port)
336 337
        # Without a client connection
        conn = Mock({"getUUID" : uuid,
338
                     "isServer" : True,
339 340 341 342 343 344
                     "getAddress" : ("127.0.0.1", self.master_port),})
        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
        self.election.connectionStarted(conn)
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
345 346
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                NodeStates.RUNNING)
347 348 349
        self.election.peerBroken(conn)
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
350 351
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                NodeStates.BROKEN)
352 353

    def test_07_packetReceived(self):
354
        uuid = self.identifyToMasterNode(port=self.master_port)
355
        p = Packets.AcceptIdentification(NodeTypes.MASTER, uuid,
356
                       ("127.0.0.1", self.master_port), 1009, 2, self.app.uuid)
357 358 359 360 361

        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
                                connector_handler = DoNothingConnector)
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
362
        node = self.app.nm.getByAddress(conn.getAddress())
363 364 365
        node.setState(NodeStates.DOWN)
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                NodeStates.DOWN)
366 367 368
        self.election.packetReceived(conn, p)
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
369 370
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getState(),
                NodeStates.RUNNING)
371

372
    def test_08_AcceptIdentification1(self):
373 374 375 376
        # test with storage node, must be rejected
        uuid = self.getNewUUID()
        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
                                connector_handler = DoNothingConnector)
377
        args = (NodeTypes.MASTER, uuid, ('127.0.0.1', self.master_port),
378
                self.app.pt.getPartitions(), self.app.pt.getReplicas(), self.app.uuid)
379
        p = Packets.AcceptIdentification(*args)
380 381
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
382
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getUUID(), None)
383
        self.assertEqual(conn.getUUID(), None)
384
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
385
        self.election.AcceptIdentification(conn, p, NodeTypes.STORAGE,
386
                                                     uuid, "127.0.0.1", self.master_port,
387 388
                                                     self.app.pt.getPartitions(), 
                                                     self.app.pt.getReplicas(),
389
                                                     self.app.uuid
390 391 392 393 394
                                                     )
        self.assertEqual(conn.getConnector(), None)
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
        
395
    def test_08_AcceptIdentification2(self):
396 397 398 399
        # test with bad address, must be rejected
        uuid = self.getNewUUID()
        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
                                connector_handler = DoNothingConnector)
400
        args = (NodeTypes.MASTER, uuid, ('127.0.0.1', self.master_port),
401
                self.app.pt.getPartitions(), self.app.pt.getReplicas(), self.app.uuid)
402
        p = Packets.AcceptIdentification(*args)
403 404
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
405
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getUUID(), None)
406
        self.assertEqual(conn.getUUID(), None)
407
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
408
        self.election.AcceptIdentification(conn, p, NodeTypes.STORAGE,
409
                                                     uuid, ("127.0.0.2", self.master_port),
410 411
                                                     self.app.pt.getPartitions(), 
                                                     self.app.pt.getReplicas(),
412
                                                     self.app.uuid)
413 414
        self.assertEqual(conn.getConnector(), None)

415
    def test_08_AcceptIdentification3(self):
416 417 418 419
        # test with master node, must be ok
        uuid = self.getNewUUID()
        conn = ClientConnection(self.app.em, self.election, addr = ("127.0.0.1", self.master_port),
                                connector_handler = DoNothingConnector)
420
        args = (NodeTypes.MASTER, uuid, ('127.0.0.1', self.master_port),
421
                self.app.pt.getPartitions(), self.app.pt.getReplicas(), self.app.uuid)
422
        p = Packets.AcceptIdentification(*args)
423 424
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 1)
425
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getUUID(), None)
426
        self.assertEqual(conn.getUUID(), None)
427
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),1)
428

429
        self.election.AcceptIdentification(conn, p, NodeTypes.MASTER,
430
                                                     uuid, ("127.0.0.1", self.master_port),
431 432
                                                     self.app.pt.getPartitions(), 
                                                     self.app.pt.getReplicas(),
433
                                                     self.app.uuid)
434
        self.assertEqual(self.app.nm.getByAddress(conn.getAddress()).getUUID(), uuid)
435
        self.assertEqual(conn.getUUID(), uuid)
436
        self.assertEqual(len(conn.getConnector().mockGetNamedCalls("_addPacket")),2)
437
        self.checkCalledAskPrimary(conn.getConnector(), 1)
438 439
        

440
    def test_10_RequestIdentification(self):
441 442
        election = self.election
        uuid = self.getNewUUID()
443
        args = (NodeTypes.MASTER, uuid, ('127.0.0.1', self.storage_port), 
444
                'INVALID_NAME')
445
        packet = Packets.RequestIdentification(*args)
446
        # test alien cluster
447
        conn = Mock({"_addPacket" : None, "abort" : None,
448
                     "isServer" : True})
449
        self.checkProtocolErrorRaised(
450
                election.requestIdentification,
451 452
                conn,
                packet=packet,
453
                node_type=NodeTypes.MASTER,
454
                uuid=uuid,
455
                address=('127.0.0.1', self.storage_port),
456
                name="INVALID_NAME",)
457
        # test connection of a storage node
458
        conn = Mock({"_addPacket" : None, "abort" : None, "expectMessage" : None,
459
                    "isServer" : True})
460
        self.checkNotReadyErrorRaised(
461
                election.requestIdentification,
462 463
                conn,
                packet=packet,
464
                node_type=NodeTypes.STORAGE,
465
                uuid=uuid,
466
                address=('127.0.0.1', self.storage_port),
467
                name=self.app.name,)
468 469

        # known node
470
        conn = Mock({"_addPacket" : None, "abort" : None, "expectMessage" : None,
471
                    "isServer" : True})
472 473
        self.assertEqual(len(self.app.nm.getMasterList()), 1)
        node = self.app.nm.getMasterList()[0]
474
        self.assertEqual(node.getUUID(), None)
475
        self.assertEqual(node.getState(), NodeStates.RUNNING)
476
        election.requestIdentification(conn,
477
                                                packet=packet,
478
                                                node_type=NodeTypes.MASTER,
479
                                                uuid=uuid,
480
                                                address=('127.0.0.1', self.master_port),
481
                                                name=self.app.name,)
482
        self.assertEqual(len(self.app.nm.getMasterList()), 1)
483
        self.assertEqual(node.getUUID(), uuid)
484
        self.assertEqual(node.getState(), NodeStates.RUNNING)
485
        self.checkAcceptIdentification(conn, answered_packet=packet)
486
        # unknown node
487
        conn = Mock({"_addPacket" : None, "abort" : None, "expectMessage" : None,
488
                    "isServer" : True})
489
        new_uuid = self.getNewUUID()
490
        self.assertEqual(len(self.app.nm.getMasterList()), 1)
491 492
        self.assertEqual(len(self.app.unconnected_master_node_set), 1)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
493
        election.requestIdentification(conn,
494
                                                packet=packet,
495
                                                node_type=NodeTypes.MASTER,
496
                                                uuid=new_uuid,
497 498
                                                address=('127.0.0.1',
                                                    self.master_port+1),
499
                                                name=self.app.name,)
500
        self.assertEqual(len(self.app.nm.getMasterList()), 2)
501
        self.checkAcceptIdentification(conn, answered_packet=packet)
502 503 504
        self.assertEqual(len(self.app.unconnected_master_node_set), 2)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
        # broken node
505
        conn = Mock({"_addPacket" : None, "abort" : None, "expectMessage" : None,
506
                    "isServer" : True})
507
        node = self.app.nm.getByAddress(("127.0.0.1", self.master_port+1))
508
        self.assertEqual(node.getUUID(), new_uuid)
509 510 511
        self.assertEqual(node.getState(), NodeStates.RUNNING)
        node.setState(NodeStates.BROKEN)
        self.assertEqual(node.getState(), NodeStates.BROKEN)
512
        self.checkBrokenNodeDisallowedErrorRaised(
513
                election.requestIdentification,
514 515
                conn,
                packet=packet,
516
                node_type=NodeTypes.MASTER,
517 518 519 520
                uuid=new_uuid,
                ip_address='127.0.0.1',
                port=self.master_port+1,
                name=self.app.name,)        
521 522


523
    def test_12_announcePrimary(self):
524
        election = self.election
525
        uuid = self.identifyToMasterNode(port=self.master_port)
526
        packet = Packets.AnnouncePrimary()
527
        # No uuid
528
        conn = Mock({"_addPacket" : None,
529
                     "getUUID" : None,
530
                     "isServer" : True,
531
                     "getAddress" : ("127.0.0.1", self.master_port)})
532
        self.assertEqual(len(self.app.nm.getMasterList()), 1)
533
        self.checkIdenficationRequired(election.announcePrimary, conn, packet)
534
        # announce
535
        conn = Mock({"_addPacket" : None,
536
                     "getUUID" : uuid,
537
                     "isServer" : True,
538 539 540
                     "getAddress" : ("127.0.0.1", self.master_port)})
        self.assertEqual(self.app.primary, None)
        self.assertEqual(self.app.primary_master_node, None)
541
        election.announcePrimary(conn, packet)        
542 543 544
        self.assertEqual(self.app.primary, False)
        self.assertNotEqual(self.app.primary_master_node, None)
        # set current as primary, and announce another, must raise
545
        conn = Mock({"_addPacket" : None,
546
                     "getUUID" : uuid,
547
                     "isServer" : True,
548 549 550
                     "getAddress" : ("127.0.0.1", self.master_port)})
        self.app.primary = True
        self.assertEqual(self.app.primary, True)
551
        self.assertRaises(ElectionFailure, election.announcePrimary, conn, packet)        
552 553
        

554
    def test_13_reelectPrimary(self):
555
        election = self.election
556
        uuid = self.identifyToMasterNode(port=self.master_port)
557
        packet = Packets.AskPrimary()
558
        # No uuid
559
        conn = Mock({"_addPacket" : None,
560
                     "getUUID" : None,
561
                     "isServer" : True,
562
                     "getAddress" : ("127.0.0.1", self.master_port)})
563
        self.assertRaises(ElectionFailure, election.reelectPrimary, conn, packet)
564

565
    def test_14_notifyNodeInformation(self):
566
        election = self.election
567
        uuid = self.identifyToMasterNode(port=self.master_port)
568
        packet = Packets.NotifyNodeInformation()
569 570 571 572
        # do not answer if no uuid
        conn = Mock({"getUUID" : None,
                     "getAddress" : ("127.0.0.1", self.master_port)})
        node_list = []
573
        self.checkIdenficationRequired(election.notifyNodeInformation, conn, packet, node_list)
574 575 576
        # tell the master node about itself, must do nothing
        conn = Mock({"getUUID" : uuid,
                     "getAddress" : ("127.0.0.1", self.master_port)})                
577 578
        node_list = [(NodeTypes.MASTER, ('127.0.0.1', self.master_port - 1),
            self.app.uuid, NodeStates.DOWN),]
579
        node = self.app.nm.getByAddress(("127.0.0.1", self.master_port-1))
580
        self.assertEqual(node, None)
581
        election.notifyNodeInformation(conn, packet, node_list)
582
        node = self.app.nm.getByAddress(("127.0.0.1", self.master_port-1))
583 584 585 586
        self.assertEqual(node, None)
        # tell about a storage node, do nothing
        conn = Mock({"getUUID" : uuid,
                     "getAddress" : ("127.0.0.1", self.master_port)})                
587 588
        node_list = [(NodeTypes.STORAGE, ('127.0.0.1', self.master_port - 1),
            self.getNewUUID(), NodeStates.DOWN),]
589
        self.assertEqual(len(self.app.nm.getStorageList()), 0)
590
        election.notifyNodeInformation(conn, packet, node_list)
591
        self.assertEqual(len(self.app.nm.getStorageList()), 0)
592 593 594
        # tell about a client node, do nothing
        conn = Mock({"getUUID" : uuid,
                     "getAddress" : ("127.0.0.1", self.master_port)})                
595 596
        node_list = [(NodeTypes.CLIENT, ('127.0.0.1', self.master_port - 1),
            self.getNewUUID(), NodeStates.DOWN),]
597
        self.assertEqual(len(self.app.nm.getList()), 0)
598
        election.notifyNodeInformation(conn, packet, node_list)
599
        self.assertEqual(len(self.app.nm.getNodeList()), 0)
600 601 602
        # tell about another master node
        conn = Mock({"getUUID" : uuid,
                     "getAddress" : ("127.0.0.1", self.master_port)})                
603 604
        node_list = [(NodeTypes.MASTER, ('127.0.0.1', self.master_port + 1),
            self.getNewUUID(), NodeStates.RUNNING),]
605
        node = self.app.nm.getByAddress(("127.0.0.1", self.master_port+1))
606
        self.assertEqual(node, None)
607
        election.notifyNodeInformation(conn, packet, node_list)
608
        node = self.app.nm.getByAddress(("127.0.0.1", self.master_port+1))
609
        self.assertNotEqual(node, None)
610
        self.assertEqual(node.getAddress(), ("127.0.0.1", self.master_port+1))
611
        self.assertEqual(node.getState(), NodeStates.RUNNING)
612
        # tell that node is down
613 614
        node_list = [(NodeTypes.MASTER, '127.0.0.1', self.master_port + 1,
            self.getNewUUID(), NodeStates.DOWN),]
615
        election.notifyNodeInformation(conn, packet, node_list)
616
        node = self.app.nm.getByAddress(("127.0.0.1", self.master_port+1))
617
        self.assertNotEqual(node, None)
618
        self.assertEqual(node.getAddress(), ("127.0.0.1", self.master_port+1))
619
        self.assertEqual(node.getState(), NodeStates.DOWN)
620 621 622 623 624

    
if __name__ == '__main__':
    unittest.main()