testElectionHandler.py 15.4 KB
Newer Older
1
#
Grégory Wisniewski's avatar
Grégory Wisniewski committed
2
# Copyright (C) 2009-2010  Nexedi SA
3
#
4 5 6 7
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
8
#
9 10 11 12 13 14 15
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 18 19

import unittest
from mock import Mock
20
from neo.lib import protocol
21
from neo.tests import NeoUnitTestBase
22
from neo.lib.protocol import Packet, NodeTypes, NodeStates
Olivier Cros's avatar
Olivier Cros committed
23 24
from neo.master.handlers.election import ClientElectionHandler, \
        ServerElectionHandler
25
from neo.master.app import Application
26 27
from neo.lib.exception import ElectionFailure
from neo.lib.connection import ClientConnection
28

29
# patch connection so that we can register _addPacket messages
30
# in mock object
31
def _addPacket(self, packet):
32
    if self.connector is not None:
33
        self.connector._addPacket(packet)
34

35
class MasterClientElectionTests(NeoUnitTestBase):
36

37
    def setUp(self):
38
        NeoUnitTestBase.setUp(self)
39
        # create an application object
40
        config = self.getMasterConfiguration(master_number=1)
41
        self.app = Application(config)
42
        self.app.pt.clear()
43
        self.app.em = Mock()
44
        self.app.uuid = self._makeUUID('M')
Olivier Cros's avatar
Olivier Cros committed
45
        self.app.server = (self.local_ip, 10000)
46
        self.app.name = 'NEOCLUSTER'
47 48 49
        self.election = ClientElectionHandler(self.app)
        self.app.unconnected_master_node_set = set()
        self.app.negotiating_master_node_set = set()
50
        for node in self.app.nm.getMasterList():
51
            self.app.unconnected_master_node_set.add(node.getAddress())
52 53 54 55 56 57
        # define some variable to simulate client and storage node
        self.storage_port = 10021
        self.master_port = 10011
        # apply monkey patches
        self._addPacket = ClientConnection._addPacket
        ClientConnection._addPacket = _addPacket
58

59
    def tearDown(self):
60 61
        # restore patched methods
        ClientConnection._addPacket = self._addPacket
62
        NeoUnitTestBase.tearDown(self)
63

64 65
    def identifyToMasterNode(self):
        node = self.app.nm.getMasterList()[0]
66
        node.setUUID(self.getNewUUID())
67 68
        conn = self.getFakeConnection(uuid=node.getUUID(),
                address=node.getAddress())
69
        return (node, conn)
70

71 72 73 74 75 76 77 78 79 80 81 82 83 84
    def _checkUnconnected(self, node):
        addr = node.getAddress()
        self.assertTrue(addr in self.app.unconnected_master_node_set)
        self.assertFalse(addr in self.app.negotiating_master_node_set)

    def _checkNegociating(self, node):
        addr = node.getAddress()
        self.assertTrue(addr in self.app.negotiating_master_node_set)
        self.assertFalse(addr in self.app.unconnected_master_node_set)

    def test_connectionStarted(self):
        node, conn = self.identifyToMasterNode()
        self.assertTrue(node.isUnknown())
        self._checkUnconnected(node)
85
        self.election.connectionStarted(conn)
86 87
        self.assertTrue(node.isUnknown())
        self._checkNegociating(node)
88

89 90 91 92 93 94 95
    def test_connectionFailed(self):
        node, conn = self.identifyToMasterNode()
        self.assertTrue(node.isUnknown())
        self._checkUnconnected(node)
        self.election.connectionFailed(conn)
        self._checkUnconnected(node)
        self.assertTrue(node.isUnknown())
96

97 98 99 100
    def test_connectionCompleted(self):
        node, conn = self.identifyToMasterNode()
        self.assertTrue(node.isUnknown())
        self._checkUnconnected(node)
101
        self.election.connectionCompleted(conn)
102
        self._checkUnconnected(node)
103
        self.assertTrue(node.isUnknown())
104
        self.checkAskPrimary(conn)
105

106 107 108 109 110 111
    def _setNegociating(self, node):
        self._checkUnconnected(node)
        addr = node.getAddress()
        self.app.negotiating_master_node_set.add(addr)
        self.app.unconnected_master_node_set.discard(addr)
        self._checkNegociating(node)
112

113 114 115 116
    def test_connectionClosed(self):
        node, conn = self.identifyToMasterNode()
        self._setNegociating(node)
        self.election.connectionClosed(conn)
117
        self.assertTrue(node.isUnknown())
118 119 120
        addr = node.getAddress()
        self.assertFalse(addr in self.app.unconnected_master_node_set)
        self.assertFalse(addr in self.app.negotiating_master_node_set)
121

122 123 124 125
    def test_acceptIdentification1(self):
        """ A non-master node accept identification """
        node, conn = self.identifyToMasterNode()
        args = (node.getUUID(), 0, 10, self.app.uuid)
126
        self.election.acceptIdentification(conn, 
127 128 129 130 131 132 133 134 135 136 137
            NodeTypes.CLIENT, *args)
        self.assertFalse(node in self.app.unconnected_master_node_set)
        self.assertFalse(node in self.app.negotiating_master_node_set)
        self.checkClosed(conn)

    def test_acceptIdentification2(self):
        """ UUID conflict """
        node, conn = self.identifyToMasterNode()
        new_uuid = self._makeUUID('M')
        args = (node.getUUID(), 0, 10, new_uuid)
        self.assertRaises(ElectionFailure, self.election.acceptIdentification,
138
            conn, NodeTypes.MASTER, *args)
139 140 141 142 143 144
        self.assertEqual(self.app.uuid, new_uuid)

    def test_acceptIdentification3(self):
        """ Identification accepted """
        node, conn = self.identifyToMasterNode()
        args = (node.getUUID(), 0, 10, self.app.uuid)
145
        self.election.acceptIdentification(conn, NodeTypes.MASTER, *args)
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
        self.checkUUIDSet(conn, node.getUUID())
        self.assertTrue(self.app.primary or node.getUUID() < self.app.uuid)
        self.assertFalse(node in self.app.negotiating_master_node_set)

    def _getMasterList(self, with_node=None):
        master_list = self.app.nm.getMasterList()
        return [(x.getAddress(), x.getUUID()) for x in master_list]

    def test_answerPrimary1(self):
        """ Multiple primary masters -> election failure raised """
        node, conn = self.identifyToMasterNode()
        self.app.primary = True
        self.app.primary_master_node = node
        master_list = self._getMasterList()
        self.assertRaises(ElectionFailure, self.election.answerPrimary,
161
                conn, self.app.uuid, master_list)
162

163 164 165 166
    def test_answerPrimary2(self):
        """ Don't known who's the primary """
        node, conn = self.identifyToMasterNode()
        master_list = self._getMasterList()
167
        self.election.answerPrimary(conn, None, master_list)
168
        self.assertFalse(self.app.primary)
169
        self.assertEqual(self.app.primary_master_node, None)
170
        self.checkRequestIdentification(conn)
171

172 173 174 175
    def test_answerPrimary3(self):
        """ Answer who's the primary """
        node, conn = self.identifyToMasterNode()
        master_list = self._getMasterList()
176
        self.election.answerPrimary(conn, node.getUUID(), master_list)
177 178
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
179 180 181
        self.assertFalse(self.app.primary)
        self.assertEqual(self.app.primary_master_node, node)
        self.checkRequestIdentification(conn)
182

183

184
class MasterServerElectionTests(NeoUnitTestBase):
185 186

    def setUp(self):
187
        NeoUnitTestBase.setUp(self)
188
        # create an application object
189
        config = self.getMasterConfiguration(master_number=1)
190
        self.app = Application(config)
191
        self.app.pt.clear()
192
        self.app.name = 'NEOCLUSTER'
193
        self.app.em = Mock()
194
        self.election = ServerElectionHandler(self.app)
195 196
        self.app.unconnected_master_node_set = set()
        self.app.negotiating_master_node_set = set()
197
        for node in self.app.nm.getMasterList():
198
            self.app.unconnected_master_node_set.add(node.getAddress())
199
            node.setState(NodeStates.RUNNING)
200
        # define some variable to simulate client and storage node
Olivier Cros's avatar
Olivier Cros committed
201 202 203
        self.client_address = (self.local_ip, 1000)
        self.storage_address = (self.local_ip, 2000)
        self.master_address = (self.local_ip, 3000)
204 205 206
        # apply monkey patches
        self._addPacket = ClientConnection._addPacket
        ClientConnection._addPacket = _addPacket
207

208
    def tearDown(self):
209
        NeoUnitTestBase.tearDown(self)
210 211
        # restore environnement
        ClientConnection._addPacket = self._addPacket
212

213 214
    def identifyToMasterNode(self, uuid=True):
        node = self.app.nm.getMasterList()[0]
215
        if uuid is True:
216 217
            uuid = self.getNewUUID()
        node.setUUID(uuid)
218 219 220 221
        conn = self.getFakeConnection(
                uuid=node.getUUID(),
                address=node.getAddress(),
        )
222
        return (node, conn)
223

224

225 226
    # Tests

227 228 229 230 231 232
    def test_requestIdentification1(self):
        """ A non-master node request identification """
        node, conn = self.identifyToMasterNode()
        args = (node.getUUID(), node.getAddress(), self.app.name)
        self.assertRaises(protocol.NotReadyError, 
            self.election.requestIdentification, 
233
            conn, NodeTypes.CLIENT, *args)
234 235 236 237 238 239

    def test_requestIdentification2(self):
        """ A unknown master node request identification """
        node, conn = self.identifyToMasterNode()
        args = (node.getUUID(), ('127.0.0.1', 1000), self.app.name)
        self.checkProtocolErrorRaised(self.election.requestIdentification, 
240
            conn, NodeTypes.MASTER, *args)
241 242 243 244 245 246 247 248

    def test_requestIdentification3(self):
        """ A broken master node request identification """
        node, conn = self.identifyToMasterNode()
        node.setBroken()
        args = (node.getUUID(), node.getAddress(), self.app.name)
        self.assertRaises(protocol.BrokenNodeDisallowedError, 
            self.election.requestIdentification, 
249
            conn, NodeTypes.MASTER, *args)
250 251 252 253 254

    def test_requestIdentification4(self):
        """ No conflict """
        node, conn = self.identifyToMasterNode()
        args = (node.getUUID(), node.getAddress(), self.app.name)
255
        self.election.requestIdentification(conn,
256 257 258 259
            NodeTypes.MASTER, *args)
        self.checkUUIDSet(conn, node.getUUID())
        args = self.checkAcceptIdentification(conn, decode=True)
        node_type, uuid, partitions, replicas, new_uuid = args
260
        self.assertEqual(node.getUUID(), new_uuid)
261
        self.assertNotEqual(node.getUUID(), uuid)
262

263 264 265 266
    def test_requestIdentification5(self):
        """ UUID conflict """
        node, conn = self.identifyToMasterNode()
        args = (self.app.uuid, node.getAddress(), self.app.name)
267
        self.election.requestIdentification(conn,
268 269 270 271 272 273 274 275 276
            NodeTypes.MASTER, *args)
        self.checkUUIDSet(conn)
        args = self.checkAcceptIdentification(conn, decode=True)
        node_type, uuid, partitions, replicas, new_uuid = args
        self.assertNotEqual(self.app.uuid, new_uuid)
        self.assertEqual(self.app.uuid, uuid)


    def _getNodeList(self):
277
        return [x.asTuple() for x in self.app.nm.getList()]
278

279 280
    def __getClient(self):
        uuid = self.getNewUUID()
281
        conn = self.getFakeConnection(uuid=uuid, address=self.client_address)
282 283 284 285 286 287
        self.app.nm.createClient(uuid=uuid, address=self.client_address)
        return conn

    def __getMaster(self, port=1000, register=True):
        uuid = self.getNewUUID()
        address = ('127.0.0.1', port)
288
        conn = self.getFakeConnection(uuid=uuid, address=address)
289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
        if register:
            self.app.nm.createMaster(uuid=uuid, address=address)
        return conn

    def testRequestIdentification1(self):
        """ Check with a non-master node, must be refused """
        conn = self.__getClient()
        self.checkNotReadyErrorRaised(
            self.election.requestIdentification,
            conn=conn,
            node_type=NodeTypes.CLIENT,
            uuid=conn.getUUID(),
            address=conn.getAddress(),
            name=self.app.name
        )

305
    def testRequestIdentification2(self):
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321
        """ Check with an unknown master node """
        conn = self.__getMaster(register=False)
        self.checkProtocolErrorRaised(
            self.election.requestIdentification,
            conn=conn,
            node_type=NodeTypes.MASTER,
            uuid=conn.getUUID(),
            address=conn.getAddress(),
            name=self.app.name,
        )

    def testAnnouncePrimary1(self):
        """ check the wrong cases """
        announce = self.election.announcePrimary
        # No uuid
        node, conn = self.identifyToMasterNode(uuid=None)
322
        self.checkProtocolErrorRaised(announce, conn)
323 324 325 326 327
        # Announce to a primary, raise
        self.app.primary = True
        node, conn = self.identifyToMasterNode()
        self.assertTrue(self.app.primary)
        self.assertEqual(self.app.primary_master_node, None)
328
        self.assertRaises(ElectionFailure, announce, conn)
329 330 331 332 333 334 335 336 337

    def testAnnouncePrimary2(self):
        """ Check the good case """
        announce = self.election.announcePrimary
        # Announce, must set the primary
        self.app.primary = False
        node, conn = self.identifyToMasterNode()
        self.assertFalse(self.app.primary)
        self.assertFalse(self.app.primary_master_node)
338
        announce(conn)
339 340 341
        self.assertFalse(self.app.primary)
        self.assertEqual(self.app.primary_master_node, node)

342 343 344 345
    def test_askPrimary1(self):
        """ Ask the primary to the primary """
        node, conn = self.identifyToMasterNode()
        self.app.primary = True
346
        self.election.askPrimary(conn)
347 348 349 350 351 352 353 354 355 356 357 358 359 360
        uuid, master_list = self.checkAnswerPrimary(conn, decode=True)
        self.assertEqual(uuid, self.app.uuid)
        self.assertEqual(len(master_list), 2)
        self.assertEqual(master_list[0], (self.app.server, self.app.uuid))
        master_node = self.app.nm.getMasterList()[0]
        master_node = (master_node.getAddress(), master_node.getUUID())
        self.assertEqual(master_list[1], master_node)

    def test_askPrimary2(self):
        """ Ask the primary to a secondary that known who's te primary """
        node, conn = self.identifyToMasterNode()
        self.app.primary = False
        # it will answer ourself as primary
        self.app.primary_master_node = node
361
        self.election.askPrimary(conn)
362 363 364 365 366 367 368 369 370 371 372 373
        uuid, master_list = self.checkAnswerPrimary(conn, decode=True)
        self.assertEqual(uuid, node.getUUID())
        self.assertEqual(len(master_list), 2)
        self.assertEqual(master_list[0], (self.app.server, self.app.uuid))
        master_node = (node.getAddress(), node.getUUID())
        self.assertEqual(master_list[1], master_node)

    def test_askPrimary3(self):
        """ Ask the primary to a master that don't known who's the primary """
        node, conn = self.identifyToMasterNode()
        self.app.primary = False
        self.app.primary_master_node = None
374
        self.election.askPrimary(conn)
375 376 377 378 379 380 381 382 383 384
        uuid, master_list = self.checkAnswerPrimary(conn, decode=True)
        self.assertEqual(uuid, None)
        self.assertEqual(len(master_list), 2)
        self.assertEqual(master_list[0], (self.app.server, self.app.uuid))
        master_node = self.app.nm.getMasterList()[0]
        master_node = (node.getAddress(), node.getUUID())
        self.assertEqual(master_list[1], master_node)

    def test_reelectPrimary(self):
        node, conn = self.identifyToMasterNode()
385
        self.assertRaises(ElectionFailure, self.election.reelectPrimary, conn)
386 387


388 389 390
if __name__ == '__main__':
    unittest.main()