testElectionHandler.py 14.7 KB
Newer Older
1
#
Grégory Wisniewski's avatar
Grégory Wisniewski committed
2
# Copyright (C) 2009-2010  Nexedi SA
3
#
4 5 6 7
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
8
#
9 10 11 12 13 14 15
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 18 19

import unittest
from mock import Mock
20
from neo.lib import protocol
21
from .. import NeoUnitTestBase
22
from neo.lib.protocol import Packet, NodeTypes, NodeStates
Olivier Cros's avatar
Olivier Cros committed
23 24
from neo.master.handlers.election import ClientElectionHandler, \
        ServerElectionHandler
25
from neo.master.app import Application
26 27
from neo.lib.exception import ElectionFailure
from neo.lib.connection import ClientConnection
28

29
# patch connection so that we can register _addPacket messages
30
# in mock object
31
def _addPacket(self, packet):
32
    if self.connector is not None:
33
        self.connector._addPacket(packet)
34

35
class MasterClientElectionTests(NeoUnitTestBase):
36

37
    def setUp(self):
38
        NeoUnitTestBase.setUp(self)
39
        # create an application object
40
        config = self.getMasterConfiguration(master_number=1)
41
        self.app = Application(config)
42
        self.app.pt.clear()
43
        self.app.em = Mock()
44
        self.app.uuid = self._makeUUID('M')
Olivier Cros's avatar
Olivier Cros committed
45
        self.app.server = (self.local_ip, 10000)
46
        self.app.name = 'NEOCLUSTER'
47 48 49
        self.election = ClientElectionHandler(self.app)
        self.app.unconnected_master_node_set = set()
        self.app.negotiating_master_node_set = set()
50
        for node in self.app.nm.getMasterList():
51
            self.app.unconnected_master_node_set.add(node.getAddress())
52 53 54 55 56 57
        # define some variable to simulate client and storage node
        self.storage_port = 10021
        self.master_port = 10011
        # apply monkey patches
        self._addPacket = ClientConnection._addPacket
        ClientConnection._addPacket = _addPacket
58

59
    def tearDown(self):
60 61
        # restore patched methods
        ClientConnection._addPacket = self._addPacket
62
        NeoUnitTestBase.tearDown(self)
63

64 65
    def identifyToMasterNode(self):
        node = self.app.nm.getMasterList()[0]
66
        node.setUUID(self.getNewUUID())
67 68
        conn = self.getFakeConnection(uuid=node.getUUID(),
                address=node.getAddress())
69
        return (node, conn)
70

71 72 73 74 75 76 77 78 79 80 81 82 83 84
    def _checkUnconnected(self, node):
        addr = node.getAddress()
        self.assertTrue(addr in self.app.unconnected_master_node_set)
        self.assertFalse(addr in self.app.negotiating_master_node_set)

    def _checkNegociating(self, node):
        addr = node.getAddress()
        self.assertTrue(addr in self.app.negotiating_master_node_set)
        self.assertFalse(addr in self.app.unconnected_master_node_set)

    def test_connectionStarted(self):
        node, conn = self.identifyToMasterNode()
        self.assertTrue(node.isUnknown())
        self._checkUnconnected(node)
85
        self.election.connectionStarted(conn)
86 87
        self.assertTrue(node.isUnknown())
        self._checkNegociating(node)
88

89 90 91 92 93 94 95
    def test_connectionFailed(self):
        node, conn = self.identifyToMasterNode()
        self.assertTrue(node.isUnknown())
        self._checkUnconnected(node)
        self.election.connectionFailed(conn)
        self._checkUnconnected(node)
        self.assertTrue(node.isUnknown())
96

97 98 99 100
    def test_connectionCompleted(self):
        node, conn = self.identifyToMasterNode()
        self.assertTrue(node.isUnknown())
        self._checkUnconnected(node)
101
        self.election.connectionCompleted(conn)
102
        self._checkUnconnected(node)
103
        self.assertTrue(node.isUnknown())
104
        self.checkAskPrimary(conn)
105

106 107 108 109 110
    def _setNegociating(self, node):
        self._checkUnconnected(node)
        addr = node.getAddress()
        self.app.negotiating_master_node_set.add(addr)
        self.app.unconnected_master_node_set.discard(addr)
111

112 113 114 115
    def test_connectionClosed(self):
        node, conn = self.identifyToMasterNode()
        self._setNegociating(node)
        self.election.connectionClosed(conn)
116
        self.assertTrue(node.isUnknown())
117 118 119
        addr = node.getAddress()
        self.assertFalse(addr in self.app.unconnected_master_node_set)
        self.assertFalse(addr in self.app.negotiating_master_node_set)
120

121 122 123 124
    def test_acceptIdentification1(self):
        """ A non-master node accept identification """
        node, conn = self.identifyToMasterNode()
        args = (node.getUUID(), 0, 10, self.app.uuid)
125
        self.election.acceptIdentification(conn,
126 127 128 129 130 131 132 133 134 135 136
            NodeTypes.CLIENT, *args)
        self.assertFalse(node in self.app.unconnected_master_node_set)
        self.assertFalse(node in self.app.negotiating_master_node_set)
        self.checkClosed(conn)

    def test_acceptIdentification2(self):
        """ UUID conflict """
        node, conn = self.identifyToMasterNode()
        new_uuid = self._makeUUID('M')
        args = (node.getUUID(), 0, 10, new_uuid)
        self.assertRaises(ElectionFailure, self.election.acceptIdentification,
137
            conn, NodeTypes.MASTER, *args)
138 139 140 141 142 143
        self.assertEqual(self.app.uuid, new_uuid)

    def test_acceptIdentification3(self):
        """ Identification accepted """
        node, conn = self.identifyToMasterNode()
        args = (node.getUUID(), 0, 10, self.app.uuid)
144
        self.election.acceptIdentification(conn, NodeTypes.MASTER, *args)
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
        self.checkUUIDSet(conn, node.getUUID())
        self.assertTrue(self.app.primary or node.getUUID() < self.app.uuid)
        self.assertFalse(node in self.app.negotiating_master_node_set)

    def _getMasterList(self, with_node=None):
        master_list = self.app.nm.getMasterList()
        return [(x.getAddress(), x.getUUID()) for x in master_list]

    def test_answerPrimary1(self):
        """ Multiple primary masters -> election failure raised """
        node, conn = self.identifyToMasterNode()
        self.app.primary = True
        self.app.primary_master_node = node
        master_list = self._getMasterList()
        self.assertRaises(ElectionFailure, self.election.answerPrimary,
160
                conn, self.app.uuid, master_list)
161

162 163 164 165
    def test_answerPrimary2(self):
        """ Don't known who's the primary """
        node, conn = self.identifyToMasterNode()
        master_list = self._getMasterList()
166
        self.election.answerPrimary(conn, None, master_list)
167
        self.assertFalse(self.app.primary)
168
        self.assertEqual(self.app.primary_master_node, None)
169
        self.checkRequestIdentification(conn)
170

171 172 173 174
    def test_answerPrimary3(self):
        """ Answer who's the primary """
        node, conn = self.identifyToMasterNode()
        master_list = self._getMasterList()
175
        self.election.answerPrimary(conn, node.getUUID(), master_list)
176 177
        self.assertEqual(len(self.app.unconnected_master_node_set), 0)
        self.assertEqual(len(self.app.negotiating_master_node_set), 0)
178 179 180
        self.assertFalse(self.app.primary)
        self.assertEqual(self.app.primary_master_node, node)
        self.checkRequestIdentification(conn)
181

182

183
class MasterServerElectionTests(NeoUnitTestBase):
184 185

    def setUp(self):
186
        NeoUnitTestBase.setUp(self)
187
        # create an application object
188
        config = self.getMasterConfiguration(master_number=1)
189
        self.app = Application(config)
190
        self.app.pt.clear()
191
        self.app.name = 'NEOCLUSTER'
192
        self.app.em = Mock()
193
        self.election = ServerElectionHandler(self.app)
194 195
        self.app.unconnected_master_node_set = set()
        self.app.negotiating_master_node_set = set()
196
        for node in self.app.nm.getMasterList():
197
            self.app.unconnected_master_node_set.add(node.getAddress())
198
            node.setState(NodeStates.RUNNING)
199
        # define some variable to simulate client and storage node
Olivier Cros's avatar
Olivier Cros committed
200 201 202
        self.client_address = (self.local_ip, 1000)
        self.storage_address = (self.local_ip, 2000)
        self.master_address = (self.local_ip, 3000)
203 204 205
        # apply monkey patches
        self._addPacket = ClientConnection._addPacket
        ClientConnection._addPacket = _addPacket
206

207
    def tearDown(self):
208
        NeoUnitTestBase.tearDown(self)
209 210
        # restore environnement
        ClientConnection._addPacket = self._addPacket
211

212 213
    def identifyToMasterNode(self, uuid=True):
        node = self.app.nm.getMasterList()[0]
214
        if uuid is True:
215 216
            uuid = self.getNewUUID()
        node.setUUID(uuid)
217 218 219 220
        conn = self.getFakeConnection(
                uuid=node.getUUID(),
                address=node.getAddress(),
        )
221
        return (node, conn)
222

223

224 225
    # Tests

226 227 228 229
    def test_requestIdentification1(self):
        """ A non-master node request identification """
        node, conn = self.identifyToMasterNode()
        args = (node.getUUID(), node.getAddress(), self.app.name)
230 231
        self.assertRaises(protocol.NotReadyError,
            self.election.requestIdentification,
232
            conn, NodeTypes.CLIENT, *args)
233 234 235 236 237 238

    def test_requestIdentification3(self):
        """ A broken master node request identification """
        node, conn = self.identifyToMasterNode()
        node.setBroken()
        args = (node.getUUID(), node.getAddress(), self.app.name)
239 240
        self.assertRaises(protocol.BrokenNodeDisallowedError,
            self.election.requestIdentification,
241
            conn, NodeTypes.MASTER, *args)
242 243 244 245 246

    def test_requestIdentification4(self):
        """ No conflict """
        node, conn = self.identifyToMasterNode()
        args = (node.getUUID(), node.getAddress(), self.app.name)
247
        self.election.requestIdentification(conn,
248 249 250 251
            NodeTypes.MASTER, *args)
        self.checkUUIDSet(conn, node.getUUID())
        args = self.checkAcceptIdentification(conn, decode=True)
        node_type, uuid, partitions, replicas, new_uuid = args
252
        self.assertEqual(node.getUUID(), new_uuid)
253
        self.assertNotEqual(node.getUUID(), uuid)
254

255 256 257 258
    def test_requestIdentification5(self):
        """ UUID conflict """
        node, conn = self.identifyToMasterNode()
        args = (self.app.uuid, node.getAddress(), self.app.name)
259
        self.election.requestIdentification(conn,
260 261 262 263 264 265 266 267 268
            NodeTypes.MASTER, *args)
        self.checkUUIDSet(conn)
        args = self.checkAcceptIdentification(conn, decode=True)
        node_type, uuid, partitions, replicas, new_uuid = args
        self.assertNotEqual(self.app.uuid, new_uuid)
        self.assertEqual(self.app.uuid, uuid)


    def _getNodeList(self):
269
        return [x.asTuple() for x in self.app.nm.getList()]
270

271 272
    def __getClient(self):
        uuid = self.getNewUUID()
273
        conn = self.getFakeConnection(uuid=uuid, address=self.client_address)
274 275 276 277 278 279
        self.app.nm.createClient(uuid=uuid, address=self.client_address)
        return conn

    def __getMaster(self, port=1000, register=True):
        uuid = self.getNewUUID()
        address = ('127.0.0.1', port)
280
        conn = self.getFakeConnection(uuid=uuid, address=address)
281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
        if register:
            self.app.nm.createMaster(uuid=uuid, address=address)
        return conn

    def testRequestIdentification1(self):
        """ Check with a non-master node, must be refused """
        conn = self.__getClient()
        self.checkNotReadyErrorRaised(
            self.election.requestIdentification,
            conn=conn,
            node_type=NodeTypes.CLIENT,
            uuid=conn.getUUID(),
            address=conn.getAddress(),
            name=self.app.name
        )

    def testAnnouncePrimary1(self):
        """ check the wrong cases """
        announce = self.election.announcePrimary
        # No uuid
        node, conn = self.identifyToMasterNode(uuid=None)
302
        self.checkProtocolErrorRaised(announce, conn)
303 304 305 306 307
        # Announce to a primary, raise
        self.app.primary = True
        node, conn = self.identifyToMasterNode()
        self.assertTrue(self.app.primary)
        self.assertEqual(self.app.primary_master_node, None)
308
        self.assertRaises(ElectionFailure, announce, conn)
309 310 311 312 313 314 315 316 317

    def testAnnouncePrimary2(self):
        """ Check the good case """
        announce = self.election.announcePrimary
        # Announce, must set the primary
        self.app.primary = False
        node, conn = self.identifyToMasterNode()
        self.assertFalse(self.app.primary)
        self.assertFalse(self.app.primary_master_node)
318
        announce(conn)
319 320 321
        self.assertFalse(self.app.primary)
        self.assertEqual(self.app.primary_master_node, node)

322 323 324 325
    def test_askPrimary1(self):
        """ Ask the primary to the primary """
        node, conn = self.identifyToMasterNode()
        self.app.primary = True
326
        self.election.askPrimary(conn)
327 328 329 330 331 332 333 334 335 336 337 338 339 340
        uuid, master_list = self.checkAnswerPrimary(conn, decode=True)
        self.assertEqual(uuid, self.app.uuid)
        self.assertEqual(len(master_list), 2)
        self.assertEqual(master_list[0], (self.app.server, self.app.uuid))
        master_node = self.app.nm.getMasterList()[0]
        master_node = (master_node.getAddress(), master_node.getUUID())
        self.assertEqual(master_list[1], master_node)

    def test_askPrimary2(self):
        """ Ask the primary to a secondary that known who's te primary """
        node, conn = self.identifyToMasterNode()
        self.app.primary = False
        # it will answer ourself as primary
        self.app.primary_master_node = node
341
        self.election.askPrimary(conn)
342 343 344 345 346 347 348 349 350 351 352 353
        uuid, master_list = self.checkAnswerPrimary(conn, decode=True)
        self.assertEqual(uuid, node.getUUID())
        self.assertEqual(len(master_list), 2)
        self.assertEqual(master_list[0], (self.app.server, self.app.uuid))
        master_node = (node.getAddress(), node.getUUID())
        self.assertEqual(master_list[1], master_node)

    def test_askPrimary3(self):
        """ Ask the primary to a master that don't known who's the primary """
        node, conn = self.identifyToMasterNode()
        self.app.primary = False
        self.app.primary_master_node = None
354
        self.election.askPrimary(conn)
355 356 357 358 359 360 361 362 363 364
        uuid, master_list = self.checkAnswerPrimary(conn, decode=True)
        self.assertEqual(uuid, None)
        self.assertEqual(len(master_list), 2)
        self.assertEqual(master_list[0], (self.app.server, self.app.uuid))
        master_node = self.app.nm.getMasterList()[0]
        master_node = (node.getAddress(), node.getUUID())
        self.assertEqual(master_list[1], master_node)

    def test_reelectPrimary(self):
        node, conn = self.identifyToMasterNode()
365
        self.assertRaises(ElectionFailure, self.election.reelectPrimary, conn)
366 367


368 369 370
if __name__ == '__main__':
    unittest.main()