Reimplement election (of the primary master)

The election is not a separate process anymore.
It happens during the RECOVERING phase, and there's no use of timeouts anymore.

Each master node keeps a timestamp of when it started to play the primary role,
and the node with the smallest timestamp is elected. The election stops when
the cluster is started: as long as it is operational, the primary master can't
be deposed.

An election must happen whenever the cluster is not operational anymore, to
handle the case of a network cut between a primary master and all other nodes:
then another master node (secondary) takes over and when the initial primary
master is back, it loses against the new primary master if the cluster is
already started.

TODO

@@ -24,8 +24,6 @@
       This is mainly the case for :
         - Client rejected before the cluster is operational
         - Empty storages rejected during recovery process
-      Masters implies in the election process should still reject any connection
-      as the primary master is still unknown.
     - Implement transaction garbage collection API (FEATURE)
       NEO packing implementation does not update transaction metadata when
       deleting object revisions. This inconsistency must be made possible to

neo/client/app.py

@@ -194,17 +194,17 @@ class Application(ThreadedApplication):
                 self.nm.reset()
                 if self.primary_master_node is not None:
                     # If I know a primary master node, pinpoint it.
-                    self.trying_master_node = self.primary_master_node
+                    node = self.primary_master_node
                     self.primary_master_node = None
                 else:
                     # Otherwise, check one by one.
                     master_list = self.nm.getMasterList()
                     index = (index + 1) % len(master_list)
-                    self.trying_master_node = master_list[index]
+                    node = master_list[index]
                 # Connect to master
                 conn = MTClientConnection(self,
                         self.notifications_handler,
-                        node=self.trying_master_node,
+                        node=node,
                         dispatcher=self.dispatcher)
                 p = Packets.RequestIdentification(
                     NodeTypes.CLIENT, self.uuid, None, self.name, None)
@@ -212,10 +212,8 @@ class Application(ThreadedApplication):
                     ask(conn, p, handler=handler)
                 except ConnectionClosed:
                     fail_count += 1
-                    continue
-                # If we reached the primary master node, mark as connected
-                if self.primary_master_node is not None and \
-                   self.primary_master_node is self.trying_master_node:
+                else:
+                    self.primary_master_node = node
                     break
             else:
                 raise NEOPrimaryMasterLost(

neo/client/handlers/master.py

@@ -15,10 +15,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from neo.lib import logging
+from neo.lib.exception import PrimaryElected
 from neo.lib.handler import MTEventHandler
 from neo.lib.pt import MTPartitionTable as PartitionTable
-from neo.lib.protocol import NodeStates, ProtocolError
-from neo.lib.util import dump
+from neo.lib.protocol import NodeStates
 from . import AnswerBaseHandler
 from ..exception import NEOStorageError
 
@@ -26,10 +26,6 @@ from ..exception import NEOStorageError
 class PrimaryBootstrapHandler(AnswerBaseHandler):
     """ Bootstrap handler used when looking for the primary master """
 
-    def notReady(self, conn, message):
-        self.app.trying_master_node = None
-        conn.close()
-
     def answerPartitionTable(self, conn, ptid, row_list):
         assert row_list
         self.app.pt.load(ptid, row_list, self.app.nm)
@@ -40,57 +36,14 @@ class PrimaryBootstrapHandler(AnswerBaseHandler):
 class PrimaryNotificationsHandler(MTEventHandler):
     """ Handler that process the notifications from the primary master """
 
-    def _acceptIdentification(self, node, uuid, num_partitions,
-            num_replicas, your_uuid, primary, known_master_list):
-        app = self.app
-
-        # Register new master nodes.
-        found = False
-        conn_address = node.getAddress()
-        for node_address, node_uuid in known_master_list:
-            if node_address == conn_address:
-                assert uuid == node_uuid, (dump(uuid), dump(node_uuid))
-                found = True
-            n = app.nm.getByAddress(node_address)
-            if n is None:
-                n = app.nm.createMaster(address=node_address)
-            if node_uuid is not None and n.getUUID() != node_uuid:
-                n.setUUID(node_uuid)
-        assert found, (node, dump(uuid), known_master_list)
-
-        conn = node.getConnection()
-        if primary is not None:
-            primary_node = app.nm.getByAddress(primary)
-            if primary_node is None:
-                # I don't know such a node. Probably this information
-                # is old. So ignore it.
-                logging.warning('Unknown primary master: %s. Ignoring.',
-                                primary)
-                return
-            else:
-                if app.trying_master_node is not primary_node:
-                    app.trying_master_node = None
-                    conn.close()
-                app.primary_master_node = primary_node
-        else:
-            if app.primary_master_node is not None:
-                # The primary master node is not a primary master node
-                # any longer.
-                app.primary_master_node = None
-
-            app.trying_master_node = None
-            conn.close()
-            return
-
-        # the master must give an UUID
-        if your_uuid is None:
-            raise ProtocolError('No UUID supplied')
-        app.uuid = your_uuid
-        logging.info('Got an UUID: %s', dump(app.uuid))
-        app.id_timestamp = None
+    def notPrimaryMaster(self, *args):
+        try:
+            super(PrimaryNotificationsHandler, self).notPrimaryMaster(*args)
+        except PrimaryElected, e:
+            app.primary_master_node, = e.args
 
-        # Always create partition table
-        app.pt = PartitionTable(num_partitions, num_replicas)
+    def _acceptIdentification(self, node, num_partitions, num_replicas):
+        self.app.pt = PartitionTable(num_partitions, num_replicas)
 
     def answerLastTransaction(self, conn, ltid):
         app = self.app

neo/client/handlers/storage.py

@@ -42,13 +42,8 @@ class StorageEventHandler(MTEventHandler):
         self.app.cp.removeConnection(node)
         super(StorageEventHandler, self).connectionFailed(conn)
 
-    def _acceptIdentification(self, node,
-           uuid, num_partitions, num_replicas, your_uuid, primary,
-           master_list):
-        assert self.app.master_conn is None or \
-          primary == self.app.master_conn.getAddress(), (
-            primary, self.app.master_conn)
-        assert uuid == node.getUUID(), (uuid, node.getUUID())
+    def _acceptIdentification(*args):
+        pass
 
 class StorageBootstrapHandler(AnswerBaseHandler):
     """ Handler used when connecting to a storage node """

neo/lib/bootstrap.py

@@ -15,8 +15,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from . import logging
+from .exception import PrimaryElected
 from .handler import EventHandler
-from .protocol import uuid_str, Packets
+from .protocol import Packets
 from .connection import ClientConnection
 
 
@@ -24,7 +25,6 @@ class BootstrapManager(EventHandler):
     """
     Manage the bootstrap stage, lookup for the primary master then connect to it
     """
-    accepted = False
 
     def __init__(self, app, node_type, server=None):
         """
@@ -32,85 +32,30 @@ class BootstrapManager(EventHandler):
         primary master node, connect to it then returns when the master node
         is ready.
         """
-        self.primary = None
         self.server = server
         self.node_type = node_type
         self.num_replicas = None
         self.num_partitions = None
-        self.current = None
         app.nm.reset()
 
     uuid = property(lambda self: self.app.uuid)
 
-    def announcePrimary(self, conn):
-        # We found the primary master early enough to be notified of election
-        # end. Lucky. Anyway, we must carry on with identification request, so
-        # nothing to do here.
-        pass
-
     def connectionCompleted(self, conn):
-        """
-        Triggered when the network connection is successful.
-        Now ask who's the primary.
-        """
         EventHandler.connectionCompleted(self, conn)
-        self.current.setRunning()
         conn.ask(Packets.RequestIdentification(self.node_type, self.uuid,
             self.server, self.app.name, None))
 
     def connectionFailed(self, conn):
-        """
-        Triggered when the network connection failed.
-        Restart bootstrap.
-        """
         EventHandler.connectionFailed(self, conn)
         self.current = None
 
     def connectionLost(self, conn, new_state):
-        """
-        Triggered when an established network connection is lost.
-        Restart bootstrap.
-        """
-        self.current.setTemporarilyDown()
         self.current = None
 
-    def notReady(self, conn, message):
-        """
-        The primary master send this message when it is still not ready to
-        handle the client node.
-        Close connection and restart.
-        """
-        conn.close()
-
-    def _acceptIdentification(self, node, uuid, num_partitions,
-            num_replicas, your_uuid, primary, known_master_list):
-        nm  = self.app.nm
-
-        # Register new master nodes.
-        for address, uuid in known_master_list:
-            master_node = nm.getByAddress(address)
-            if master_node is None:
-                master_node = nm.createMaster(address=address)
-            master_node.setUUID(uuid)
-
-        self.primary = nm.getByAddress(primary)
-        if self.primary is None or self.current is not self.primary:
-            # three cases here:
-            # - something goes wrong (unknown UUID)
-            # - this master doesn't know who's the primary
-            # - got the primary's uuid, so cut here
-            node.getConnection().close()
-            return
-
-        logging.info('connected to a primary master node')
+    def _acceptIdentification(self, node, num_partitions, num_replicas):
+        assert self.current is node, (self.current, node)
         self.num_partitions = num_partitions
         self.num_replicas = num_replicas
-        if self.uuid != your_uuid:
-            # got an uuid from the primary master
-            self.app.uuid = your_uuid
-            logging.info('Got a new UUID: %s', uuid_str(self.uuid))
-        self.app.id_timestamp = None
-        self.accepted = True
 
     def getPrimaryConnection(self):
         """
@@ -122,25 +67,26 @@ class BootstrapManager(EventHandler):
         poll = app.em.poll
         index = 0
         self.current = None
-        conn = None
         # retry until identified to the primary
-        while not self.accepted:
-            if self.current is None:
-                # conn closed
-                conn = None
+        while True:
+            try:
+                while self.current:
+                    if self.current.isIdentified():
+                        return (self.current, self.current.getConnection(),
+                            self.num_partitions, self.num_replicas)
+                    poll(1)
+            except PrimaryElected, e:
+                if self.current:
+                    self.current.getConnection().close()
+                self.current, = e.args
+                index = app.nm.getMasterList().index(self.current)
+            else:
                 # select a master
                 master_list = app.nm.getMasterList()
                 index = (index + 1) % len(master_list)
                 self.current = master_list[index]
-            if conn is None:
-                # open the connection
-                conn = ClientConnection(app, self, self.current)
-                # Yes, the connection may be already closed. This happens when
-                # the kernel reacts so quickly to a closed port that 'connect'
-                # fails on the first call. In such case, poll(1) would deadlock
-                # if there's no other connection to timeout.
-                if conn.isClosed():
-                    continue
-            # still processing
-            poll(1)
-        return self.current, conn, self.num_partitions, self.num_replicas
+            ClientConnection(app, self, self.current)
+            # Note that the connection may be already closed. This happens when
+            # the kernel reacts so quickly to a closed port that 'connect'
+            # fails on the first call. In such case, poll(1) would deadlock
+            # if there's no other connection to timeout.

neo/lib/connection.py

@@ -259,10 +259,12 @@ class BaseConnection(object):
         )
 
     def setHandler(self, handler):
-        if self._handlers.setHandler(handler):
-            logging.debug('Set handler %r on %r', handler, self)
+        changed = self._handlers.setHandler(handler)
+        if changed:
+            logging.debug('Handler changed on %r', self)
         else:
             logging.debug('Delay handler %r on %r', handler, self)
+        return changed
 
     def getUUID(self):
         return None

neo/lib/exception.py

@@ -17,7 +17,7 @@
 class NeoException(Exception):
     pass
 
-class ElectionFailure(NeoException):
+class PrimaryElected(NeoException):
     pass
 
 class PrimaryFailure(NeoException):

neo/lib/handler.py

@@ -19,9 +19,10 @@ from collections import deque
 from operator import itemgetter
 from . import logging
 from .connection import ConnectionClosed
-from .protocol import (
-    NodeStates, Packets, Errors, BackendNotImplemented, NonReadableCell,
-    NotReadyError, PacketMalformedError, ProtocolError, UnexpectedPacketError)
+from .exception import PrimaryElected
+from .protocol import (NodeStates, NodeTypes, Packets, uuid_str,
+    Errors, BackendNotImplemented, NonReadableCell, NotReadyError,
+    PacketMalformedError, ProtocolError, UnexpectedPacketError)
 from .util import cached_property
 
 
@@ -147,16 +148,41 @@ class EventHandler(object):
 
     # Packet handlers.
 
-    def acceptIdentification(self, conn, node_type, *args):
-        try:
-            acceptIdentification = self._acceptIdentification
-        except AttributeError:
-            raise UnexpectedPacketError('no handler found')
-        node = self.app.nm.getByAddress(conn.getAddress())
+    def notPrimaryMaster(self, conn, primary, known_master_list):
+        nm = self.app.nm
+        for address in known_master_list:
+            nm.createMaster(address=address)
+        if primary is not None:
+            primary = known_master_list[primary]
+            assert primary != self.app.server
+            raise PrimaryElected(nm.getByAddress(primary))
+
+    def _acceptIdentification(*args):
+        pass
+
+    def acceptIdentification(self, conn, node_type, uuid,
+                             num_partitions, num_replicas, your_uuid):
+        app = self.app
+        node = app.nm.getByAddress(conn.getAddress())
         assert node.getConnection() is conn, (node.getConnection(), conn)
         if node.getType() == node_type:
+            if node_type == NodeTypes.MASTER:
+                other = app.nm.getByUUID(uuid)
+                if other is not None:
+                    other.setUUID(None)
+                node.setUUID(uuid)
+                node.setRunning()
+                if your_uuid is None:
+                    raise ProtocolError('No UUID supplied')
+                logging.info('connected to a primary master node')
+                if app.uuid != your_uuid:
+                    app.uuid = your_uuid
+                    logging.info('Got a new UUID: %s', uuid_str(your_uuid))
+                app.id_timestamp = None
+            elif node.getUUID() != uuid or app.uuid != your_uuid != None:
+                raise ProtocolError('invalid uuids')
             node.setIdentified()
-            acceptIdentification(node, *args)
+            self._acceptIdentification(node, num_partitions, num_replicas)
             return
         conn.close()
 

neo/lib/node.py

@@ -423,7 +423,7 @@ class NodeManager(EventQueue):
             # lookup in current table
             node_by_uuid = self.getByUUID(uuid)
             node_by_addr = self.getByAddress(addr)
-            node = node_by_uuid or node_by_addr
+            node = node_by_addr or node_by_uuid
 
             log_args = node_type, uuid_str(uuid), addr, state, id_timestamp
             if node is None:
@@ -434,10 +434,11 @@ class NodeManager(EventQueue):
             else:
                 assert isinstance(node, klass), 'node %r is not ' \
                     'of expected type: %r' % (node, klass)
-                assert None in (node_by_uuid, node_by_addr) or \
-                    node_by_uuid is node_by_addr, \
+                if None is not node_by_uuid is not node_by_addr is not None:
+                    assert added_list is not None, \
                     'Discrepancy between node_by_uuid (%r) and ' \
                     'node_by_addr (%r)' % (node_by_uuid, node_by_addr)
+                    node_by_uuid.setUUID(None)
                 if state == NodeStates.DOWN:
                     logging.debug('dropping node %r (%r), found with %s '
                         '%s %s %s %s', node, node.isConnected(), *log_args)

neo/lib/protocol.py

@@ -578,12 +578,14 @@ class PChecksum(PItem):
     def _decode(self, reader):
         return reader(20)
 
-class PUUID(PStructItemOrNone):
+class PSignedNull(PStructItemOrNone):
+    _fmt = '!l'
+    _None = Struct(_fmt).pack(0)
+
+class PUUID(PSignedNull):
     """
         An UUID (node identifier, 4-bytes signed integer)
     """
-    _fmt = '!l'
-    _None = Struct(_fmt).pack(0)
 
 class PTID(PItem):
     """
@@ -715,13 +717,6 @@ class RequestIdentification(Packet):
         PNumber('num_partitions'),
         PNumber('num_replicas'),
         PUUID('your_uuid'),
-        PAddress('primary'),
-        PList('known_master_list',
-            PStruct('master',
-                PAddress('address'),
-                PUUID('uuid'),
-            ),
-        ),
     )
 
     def __init__(self, *args, **kw):
@@ -742,15 +737,16 @@ class PrimaryMaster(Packet):
         PUUID('primary_uuid'),
     )
 
-class AnnouncePrimary(Packet):
-    """
-    Announce a primary master node election. PM -> SM.
-    """
-
-class ReelectPrimary(Packet):
+class NotPrimaryMaster(Packet):
     """
-    Force a re-election of a primary master node. M -> M.
+    Send list of known master nodes. SM -> Any.
     """
+    _fmt = PStruct('not_primary_master',
+        PSignedNull('primary'),
+        PList('known_master_list',
+            PAddress('address'),
+        ),
+    )
 
 class Recovery(Packet):
     """
@@ -1687,10 +1683,8 @@ class Packets(dict):
                     Notify)
     AskPrimary, AnswerPrimary = register(
                     PrimaryMaster)
-    AnnouncePrimary = register(
-                    AnnouncePrimary)
-    ReelectPrimary = register(
-                    ReelectPrimary)
+    NotPrimaryMaster = register(
+                    NotPrimaryMaster)
     NotifyNodeInformation = register(
                     NotifyNodeInformation)
     AskRecovery, AnswerRecovery = register(

neo/master/handlers/__init__.py

@@ -29,34 +29,9 @@ class MasterHandler(EventHandler):
         if new is None:
             super(MasterHandler, self).connectionCompleted(conn)
 
-    def requestIdentification(self, conn, node_type, uuid, address, name, _):
-        self.checkClusterName(name)
-        app = self.app
-        node = app.nm.getByUUID(uuid)
-        if node:
-            if node_type is NodeTypes.MASTER and not (
-               None != address == node.getAddress()):
-                raise ProtocolError
-        peer_uuid = self._setupNode(conn, node_type, uuid, address, node)
-        if app.primary:
-            primary_address = app.server
-        elif app.primary_master_node is not None:
-            primary_address = app.primary_master_node.getAddress()
-        else:
-            primary_address = None
-
-        known_master_list = []
-        for n in app.nm.getMasterList():
-            known_master_list.append((n.getAddress(), n.getUUID()))
-        conn.answer(Packets.AcceptIdentification(
-            NodeTypes.MASTER,
-            app.uuid,
-            app.pt.getPartitions(),
-            app.pt.getReplicas(),
-            peer_uuid,
-            primary_address,
-            known_master_list),
-        )
+    def connectionLost(self, conn, new_state=None):
+        if self.app.listening_conn: # if running
+            self._connectionLost(conn)
 
     def askClusterState(self, conn):
         state = self.app.getClusterState()

neo/master/handlers/client.py

@@ -22,16 +22,15 @@ from . import MasterHandler
 class ClientServiceHandler(MasterHandler):
     """ Handler dedicated to client during service state """
 
-    def connectionLost(self, conn, new_state):
+    def _connectionLost(self, conn):
         # cancel its transactions and forgot the node
         app = self.app
-        if app.listening_conn: # if running
-            node = app.nm.getByUUID(conn.getUUID())
-            assert node is not None
-            app.tm.clientLost(node)
-            node.setState(NodeStates.DOWN)
-            app.broadcastNodesInformation([node])
-            app.nm.remove(node)
+        node = app.nm.getByUUID(conn.getUUID())
+        assert node is not None, conn
+        app.tm.clientLost(node)
+        node.setState(NodeStates.DOWN)
+        app.broadcastNodesInformation([node])
+        app.nm.remove(node)
 
     def askBeginTransaction(self, conn, tid):
         """

neo/master/handlers/election.py

-#
-# Copyright (C) 2006-2017  Nexedi SA
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-from neo.lib import logging
-from neo.lib.protocol import uuid_str, NodeTypes, Packets
-from neo.lib.protocol import NotReadyError
-from neo.lib.exception import ElectionFailure
-from neo.lib.handler import EventHandler
-from . import MasterHandler
-
-class BaseElectionHandler(EventHandler):
-
-    def _notifyNodeInformation(self, conn):
-        pass
-
-    def reelectPrimary(self, conn):
-        raise ElectionFailure, 'reelection requested'
-
-    def announcePrimary(self, conn):
-        app = self.app
-        if app.primary:
-            # I am also the primary... So restart the election.
-            raise ElectionFailure, 'another primary arises'
-        try:
-            address = app.master_address_dict[conn]
-            assert conn.isServer()
-        except KeyError:
-            address = conn.getAddress()
-            assert conn.isClient()
-        app.primary = False
-        app.primary_master_node = node = app.nm.getByAddress(address)
-        app.negotiating_master_node_set.clear()
-        logging.info('%s is the primary', node)
-
-    def elect(self, conn, peer_address):
-        app = self.app
-        if app.server < peer_address:
-            app.primary = False
-            if conn is not None:
-                app.master_address_dict[conn] = peer_address
-        app.negotiating_master_node_set.discard(peer_address)
-
-
-class ClientElectionHandler(BaseElectionHandler):
-
-    def notifyNodeInformation(self, conn, timestamp, node_list):
-        # XXX: For the moment, do nothing because
-        # we'll close this connection and reconnect.
-        pass
-
-    def connectionFailed(self, conn):
-        addr = conn.getAddress()
-        node = self.app.nm.getByAddress(addr)
-        assert node is not None, (uuid_str(self.app.uuid), addr)
-        # node may still be in unknown state
-        self.app.negotiating_master_node_set.discard(addr)
-        super(ClientElectionHandler, self).connectionFailed(conn)
-
-    def connectionCompleted(self, conn):
-        app = self.app
-        conn.ask(Packets.RequestIdentification(
-            NodeTypes.MASTER,
-            app.uuid,
-            app.server,
-            app.name,
-            None,
-        ))
-        super(ClientElectionHandler, self).connectionCompleted(conn)
-
-    def connectionLost(self, conn, new_state):
-        # Retry connection. Either the node just died (and we will end up in
-        # connectionFailed) or it just got elected (and we must not ignore
-        # that node).
-        addr = conn.getAddress()
-        self.app.unconnected_master_node_set.add(addr)
-        self.app.negotiating_master_node_set.discard(addr)
-
-    def _acceptIdentification(self, node, peer_uuid, num_partitions,
-            num_replicas, your_uuid, primary, known_master_list):
-        app = self.app
-
-        # Register new master nodes.
-        for address, uuid in known_master_list:
-            if app.server == address:
-                # This is self.
-                assert node.getAddress() != primary or uuid == your_uuid, (
-                    uuid_str(uuid), uuid_str(your_uuid))
-                continue
-            n = app.nm.getByAddress(address)
-            if n is None:
-                n = app.nm.createMaster(address=address)
-
-        if primary is not None:
-            # The primary master is defined.
-            if app.primary_master_node is not None \
-                    and app.primary_master_node.getAddress() != primary:
-                # There are multiple primary master nodes. This is
-                # dangerous.
-                raise ElectionFailure, 'multiple primary master nodes'
-            primary_node = app.nm.getByAddress(primary)
-            if primary_node is None:
-                # I don't know such a node. Probably this information
-                # is old. So ignore it.
-                logging.warning('received an unknown primary node')
-            else:
-                # Whatever the situation is, I trust this master.
-                app.primary = False
-                app.primary_master_node = primary_node
-                # Stop waiting for connections than primary master's to
-                # complete to exit election phase ASAP.
-                app.negotiating_master_node_set.clear()
-                return
-
-        self.elect(None, node.getAddress())
-
-
-class ServerElectionHandler(BaseElectionHandler, MasterHandler):
-
-    def _setupNode(self, conn, node_type, uuid, address, node):
-        app = self.app
-        if node_type != NodeTypes.MASTER:
-            logging.info('reject a connection from a non-master')
-            raise NotReadyError
-
-        if node is None is app.nm.getByAddress(address):
-            app.nm.createMaster(address=address)
-
-        self.elect(conn, address)
-        return uuid
-

neo/master/handlers/identification.py

@@ -15,27 +15,25 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from neo.lib import logging
+from neo.lib.exception import PrimaryElected
+from neo.lib.handler import EventHandler
 from neo.lib.protocol import ClusterStates, NodeStates, NodeTypes, \
-    NotReadyError, ProtocolError, uuid_str
+    NotReadyError, Packets, ProtocolError, uuid_str
 from ..app import monotonic_time
-from . import MasterHandler
 
-class IdentificationHandler(MasterHandler):
+class IdentificationHandler(EventHandler):
 
-    def requestIdentification(self, conn, *args, **kw):
-        super(IdentificationHandler, self).requestIdentification(conn, *args,
-            **kw)
-        handler = conn.getHandler()
-        assert not isinstance(handler, IdentificationHandler), handler
-        handler._notifyNodeInformation(conn)
-        handler.connectionCompleted(conn, True)
-
-    def _setupNode(self, conn, node_type, uuid, address, node):
+    def requestIdentification(self, conn, node_type, uuid,
+                              address, name, id_timestamp):
         app = self.app
+        self.checkClusterName(name)
+        if address == app.server:
+            raise ProtocolError('address conflict')
+        node = app.nm.getByUUID(uuid)
         by_addr = address and app.nm.getByAddress(address)
         while 1:
             if by_addr:
-                if not by_addr.isConnected():
+                if not by_addr.isIdentified():
                     if node is by_addr:
                         break
                     if not node or uuid < 0:
@@ -44,7 +42,7 @@ class IdentificationHandler(MasterHandler):
                         node = by_addr
                         break
             elif node:
-                if node.isConnected():
+                if node.isIdentified():
                     if uuid < 0:
                         # The peer wants a temporary id that's already assigned.
                         # Let's give it another one.
@@ -78,7 +76,14 @@ class IdentificationHandler(MasterHandler):
                 uuid is not None and node is not None)
             human_readable_node_type = ' storage (%s) ' % (state, )
         elif node_type == NodeTypes.MASTER:
-            handler = app.secondary_master_handler
+            if app.election:
+                if id_timestamp and \
+                  (id_timestamp, address) < (app.election, app.server):
+                    raise PrimaryElected(by_addr or
+                        app.nm.createMaster(address=address))
+                handler = app.election_handler
+            else:
+                handler = app.secondary_handler
             human_readable_node_type = ' master '
         elif node_type == NodeTypes.ADMIN:
             handler = app.administration_handler
@@ -95,22 +100,43 @@ class IdentificationHandler(MasterHandler):
             node.setUUID(uuid)
         node.id_timestamp = monotonic_time()
         node.setState(state)
-        node.setConnection(conn)
         conn.setHandler(handler)
+        node.setConnection(conn, not node.isIdentified())
         app.broadcastNodesInformation([node], node)
-        return uuid
 
-class SecondaryIdentificationHandler(MasterHandler):
+        conn.answer(Packets.AcceptIdentification(
+            NodeTypes.MASTER,
+            app.uuid,
+            app.pt.getPartitions(),
+            app.pt.getReplicas(),
+            uuid))
+        handler._notifyNodeInformation(conn)
+        handler.connectionCompleted(conn, True)
 
-    def announcePrimary(self, conn):
-        # If we received AnnouncePrimary on a client connection, we might have
-        # set this handler on server connection, and might receive
-        # AnnouncePrimary there too. As we cannot reach this without already
-        # handling a first AnnouncePrimary, we can safely ignore this one.
-        pass
 
-    def _setupNode(self, conn, node_type, uuid, address, node):
-        # Nothing to do, storage will disconnect when it receives our answer.
-        # Primary will do the checks.
-        return uuid
+class SecondaryIdentificationHandler(EventHandler):
 
+    def requestIdentification(self, conn, node_type, uuid,
+                              address, name, id_timestamp):
+        app = self.app
+        self.checkClusterName(name)
+        if address == app.server:
+            raise ProtocolError('address conflict')
+        primary = app.primary_master.getAddress()
+        if primary == address:
+            primary = None
+        elif not app.primary_master.isIdentified():
+            if node_type == NodeTypes.MASTER:
+                node = app.nm.createMaster(address=address)
+                if id_timestamp:
+                    conn.close()
+                    raise PrimaryElected(node)
+            primary = None
+        # For some cases, we rely on the fact that the remote will not retry
+        # immediately (see SocketConnector.CONNECT_LIMIT).
+        known_master_list = [node.getAddress()
+            for node in app.nm.getMasterList()]
+        conn.send(Packets.NotPrimaryMaster(
+            primary and known_master_list.index(primary),
+            known_master_list))
+        conn.abort()

neo/master/handlers/secondary.py → neo/master/handlers/master.py

@@ -15,60 +15,76 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import sys
-from ..app import monotonic_time
 from . import MasterHandler
-from neo.lib.handler import EventHandler
-from neo.lib.exception import ElectionFailure, PrimaryFailure
-from neo.lib.protocol import NodeStates, NodeTypes, Packets, uuid_str
-from neo.lib import logging
+from neo.lib.exception import PrimaryElected, PrimaryFailure
+from neo.lib.protocol import ClusterStates, NodeStates, NodeTypes, Packets
 
-class SecondaryMasterHandler(MasterHandler):
-    """ Handler used by primary to handle secondary masters"""
 
-    def connectionLost(self, conn, new_state):
-        app = self.app
-        if app.listening_conn: # if running
-            node = app.nm.getByUUID(conn.getUUID())
-            node.setDown()
-            app.broadcastNodesInformation([node])
+class SecondaryHandler(MasterHandler):
+    """Handler used by primary to handle secondary masters"""
 
-    def announcePrimary(self, conn):
-        raise ElectionFailure, 'another primary arises'
+    def _connectionLost(self, conn):
+        app = self.app
+        node = app.nm.getByUUID(conn.getUUID())
+        node.setTemporarilyDown()
+        app.broadcastNodesInformation([node])
 
-    def reelectPrimary(self, conn):
-        raise ElectionFailure, 'reelection requested'
 
-class PrimaryHandler(EventHandler):
-    """ Handler used by secondaries to handle primary master"""
+class ElectionHandler(MasterHandler):
+    """Handler used by primary to handle secondary masters during election"""
 
-    def connectionLost(self, conn, new_state):
-        self.connectionFailed(conn)
+    def connectionCompleted(self, conn, new=None):
+        if new is None:
+            super(ElectionHandler, self).connectionCompleted(conn)
+            app = self.app
+            conn.ask(Packets.RequestIdentification(NodeTypes.MASTER,
+                app.uuid, app.server, app.name, app.election))
 
     def connectionFailed(self, conn):
-        self.app.primary_master_node.setDown()
-        if self.app.listening_conn: # if running
+        super(ElectionHandler, self).connectionFailed(conn)
+        self.connectionLost(conn)
+
+    def _acceptIdentification(self, node, *args):
+        raise PrimaryElected(node)
+
+    def _connectionLost(self, *args):
+        if self.app.primary: # not switching to secondary role
+            self.app._current_manager.try_secondary = True
+
+    def notPrimaryMaster(self, *args):
+        try:
+            super(ElectionHandler, self).notPrimaryMaster(*args)
+        except PrimaryElected, e:
+            # We keep playing the primary role when the peer does not
+            # know yet that we won election against the returned node.
+            if not e.args[0].isIdentified():
+                raise
+        # There may be new master nodes. Connect to them.
+        self.app._current_manager.try_secondary = True
+
+
+class PrimaryHandler(ElectionHandler):
+    """Handler used by secondaries to handle primary master"""
+
+    def _acceptIdentification(self, node, num_partitions, num_replicas):
+        assert self.app.primary_master is node, (self.app.primary_master, node)
+
+    def _connectionLost(self, conn):
+        node = self.app.primary_master
+        # node is None when switching to primary role
+        if node and not node.isConnected(True):
             raise PrimaryFailure('primary master is dead')
 
-    def connectionCompleted(self, conn):
-        app = self.app
-        addr = conn.getAddress()
-        node = app.nm.getByAddress(addr)
-        # connection successful, set it as running
-        node.setRunning()
-        conn.ask(Packets.RequestIdentification(
-            NodeTypes.MASTER,
-            app.uuid,
-            app.server,
-            app.name,
-            None,
-        ))
-        super(PrimaryHandler, self).connectionCompleted(conn)
-
-    def reelectPrimary(self, conn):
-        raise ElectionFailure, 'reelection requested'
+    def notPrimaryMaster(self, *args):
+        try:
+            super(ElectionHandler, self).notPrimaryMaster(*args)
+        except PrimaryElected, e:
+            if e.args[0] is not self.app.primary_master:
+                raise
 
     def notifyClusterInformation(self, conn, state):
-        self.app.cluster_state = state
+        if state == ClusterStates.STOPPING:
+            sys.exit()
 
     def notifyNodeInformation(self, conn, timestamp, node_list):
         super(PrimaryHandler, self).notifyNodeInformation(
@@ -77,17 +93,3 @@ class PrimaryHandler(EventHandler):
             assert node_type == NodeTypes.MASTER, node_type
             if uuid == self.app.uuid and state == NodeStates.UNKNOWN:
                 sys.exit()
-
-    def _acceptIdentification(self, node, uuid, num_partitions,
-            num_replicas, your_uuid, primary, known_master_list):
-        app = self.app
-        if primary != app.primary_master_node.getAddress():
-            raise PrimaryFailure('unexpected primary uuid')
-
-        if your_uuid != app.uuid:
-            app.uuid = your_uuid
-            logging.info('My UUID: ' + uuid_str(your_uuid))
-
-        node.setUUID(uuid)
-        app.id_timestamp = None
-

neo/master/recovery.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from neo.lib import logging
+from neo.lib.connection import ClientConnection
 from neo.lib.protocol import Packets, ProtocolError, ClusterStates, NodeStates
 from .app import monotonic_time
 from .handlers import MasterHandler
@@ -47,6 +48,7 @@ class RecoveryManager(MasterHandler):
         TID, and the last Partition Table ID from storage nodes, then get
         back the latest partition table or make a new table from scratch,
         if this is the first time.
+        A new primary master may also arise during this phase.
         """
         logging.info('begin the recovery of the status')
         app = self.app
@@ -54,9 +56,30 @@ class RecoveryManager(MasterHandler):
         app.changeClusterState(ClusterStates.RECOVERING)
         pt.clear()
 
+        self.try_secondary = True
+
         # collect the last partition table available
         poll = app.em.poll
         while 1:
+            if self.try_secondary:
+                # Keep trying to connect to all other known masters,
+                # to make sure there is a challege between each pair
+                # of masters in the cluster. If we win, all connections
+                # opened here will be closed.
+                self.try_secondary = False
+                node_list = []
+                for node in app.nm.getMasterList():
+                    if not (node is app._node or node.isConnected(True)):
+                        # During recovery, master nodes are not put back in
+                        # TEMPORARILY_DOWN state by handlers. This is done
+                        # entirely in this method (here and after this poll
+                        # loop), to minimize the notification packets.
+                        if not node.isTemporarilyDown():
+                            node.setTemporarilyDown()
+                            node_list.append(node)
+                        ClientConnection(app, app.election_handler, node)
+                if node_list:
+                    app.broadcastNodesInformation(node_list)
             poll(1)
             if pt.filled():
                 # A partition table exists, we are starting an existing
@@ -100,6 +123,17 @@ class RecoveryManager(MasterHandler):
         for node in node_list:
             assert node.isPending(), node
             node.setRunning()
+
+        for node in app.nm.getMasterList():
+            if not (node is app._node or node.isIdentified()):
+                if node.isConnected(True):
+                    node.getConnection().close()
+                    assert node.isTemporarilyDown(), node
+                elif not node.isTemporarilyDown():
+                    assert self.try_secondary, node
+                    node.setTemporarilyDown()
+                    node_list.append(node)
+
         app.broadcastNodesInformation(node_list)
 
         if pt.getID() is None:

neo/scripts/runner.py

@@ -42,7 +42,6 @@ from neo.tests.benchmark import BenchmarkRunner
 # each of them have to import its TestCase classes
 UNIT_TEST_MODULES = [
     # generic parts
-    'neo.tests.testBootstrap',
     'neo.tests.testConnection',
     'neo.tests.testHandler',
     'neo.tests.testNodes',
@@ -50,7 +49,6 @@ UNIT_TEST_MODULES = [
     'neo.tests.testPT',
     # master application
     'neo.tests.master.testClientHandler',
-    'neo.tests.master.testElectionHandler',
     'neo.tests.master.testMasterApp',
     'neo.tests.master.testMasterPT',
     'neo.tests.master.testRecovery',

neo/storage/handlers/identification.py

@@ -65,6 +65,5 @@ class IdentificationHandler(EventHandler):
             node.setConnection(conn, app.uuid < uuid)
         # accept the identification and trigger an event
         conn.answer(Packets.AcceptIdentification(NodeTypes.STORAGE, uuid and
-            app.uuid, app.pt.getPartitions(), app.pt.getReplicas(), uuid,
-            app.master_node.getAddress(), ()))
+            app.uuid, app.pt.getPartitions(), app.pt.getReplicas(), uuid))
         handler.connectionCompleted(conn)

neo/tests/__init__.py

@@ -309,10 +309,6 @@ class NeoUnitTestBase(NeoTestBase):
         """ Check if the ProtocolError exception was raised """
         self.assertRaises(protocol.ProtocolError, method, *args, **kwargs)
 
-    def checkNotReadyErrorRaised(self, method, *args, **kwargs):
-        """ Check if the NotReadyError exception was raised """
-        self.assertRaises(protocol.NotReadyError, method, *args, **kwargs)
-
     def checkAborted(self, conn):
         """ Ensure the connection was aborted """
         self.assertEqual(len(conn.mockGetNamedCalls('abort')), 1)
@@ -330,16 +326,6 @@ class NeoUnitTestBase(NeoTestBase):
         self._checkNoPacketSend(conn, 'answer')
         self._checkNoPacketSend(conn, 'ask')
 
-    def checkUUIDSet(self, conn, uuid=None, check_intermediate=True):
-        """ ensure UUID was set on the connection """
-        calls = conn.mockGetNamedCalls('setUUID')
-        found_uuid = calls.pop().getParam(0)
-        if check_intermediate:
-            for call in calls:
-                self.assertEqual(found_uuid, call.getParam(0))
-        if uuid is not None:
-            self.assertEqual(found_uuid, uuid)
-
     # in check(Ask|Answer|Notify)Packet we return the packet so it can be used
     # in tests if more accurate checks are required
 

neo/tests/client/testClientApp.py

@@ -21,7 +21,6 @@ from .. import NeoUnitTestBase, buildUrlFromString
 from neo.client.app import Application
 from neo.client.cache import test as testCache
 from neo.client.exception import NEOStorageError
-from neo.lib.protocol import NodeTypes, UUID_NAMESPACES
 
 class ClientApplicationTests(NeoUnitTestBase):
 
@@ -97,63 +96,6 @@ class ClientApplicationTests(NeoUnitTestBase):
         # no packet sent
         self.checkNoPacketSent(app.master_conn)
 
-    def test_connectToPrimaryNode(self):
-        # here we have three master nodes :
-        # the connection to the first will fail
-        # the second will have changed
-        # the third will not be ready
-        # after the third, the partition table will be operational
-        # (as if it was connected to the primary master node)
-        # will raise IndexError at the third iteration
-        app = self.getApp('127.0.0.1:10010 127.0.0.1:10011')
-        # TODO: test more connection failure cases
-        # askLastTransaction
-        def _ask8(_):
-            pass
-        # Sixth packet : askPartitionTable succeeded
-        def _ask7(_):
-            app.pt = Mock({'operational': True})
-        # fifth packet : request node identification succeeded
-        def _ask6(conn):
-            app.master_conn = conn
-            app.uuid = 1 + (UUID_NAMESPACES[NodeTypes.CLIENT] << 24)
-            app.trying_master_node = app.primary_master_node = Mock({
-                'getAddress': ('127.0.0.1', 10011),
-                '__str__': 'Fake master node',
-            })
-        # third iteration : node not ready
-        def _ask4(_):
-            app.trying_master_node = None
-        # second iteration : master node changed
-        def _ask3(_):
-            app.primary_master_node = Mock({
-                'getAddress': ('127.0.0.1', 10010),
-                '__str__': 'Fake master node',
-            })
-        # first iteration : connection failed
-        def _ask2(_):
-            app.trying_master_node = None
-        # do nothing for the first call
-        # Case of an unknown primary_uuid (XXX: handler should probably raise,
-        # it's not normal for a node to inform of a primary uuid without
-        # telling us what its address is.)
-        def _ask1(_):
-            pass
-        ask_func_list = [_ask1, _ask2, _ask3, _ask4, _ask6, _ask7, _ask8]
-        def _ask_base(conn, _, handler=None):
-            ask_func_list.pop(0)(conn)
-            app.nm.getByAddress(conn.getAddress())._connection = None
-        app._ask = _ask_base
-        # fake environment
-        app.em.close()
-        app.em = Mock({'getConnectionList': []})
-        app.pt = Mock({ 'operational': False})
-        app.start = lambda: None
-        app.master_conn = app._connectToPrimaryNode()
-        self.assertFalse(ask_func_list)
-        self.assertTrue(app.master_conn is not None)
-        self.assertTrue(app.pt.operational())
-
 if __name__ == '__main__':
     unittest.main()
 

neo/tests/functional/testMaster.py

@@ -77,12 +77,13 @@ class MasterTests(NEOFunctionalTest):
         killed_uuid_list = self.neo.killSecondaryMaster()
         # Test sanity checks.
         self.assertEqual(len(killed_uuid_list), 1)
-        self.neo.expectMasterState(killed_uuid_list[0], None)
-        self.assertEqual(len(self.neo.getMasterList()), 2)
+        self.neo.expectMasterState(killed_uuid_list[0],
+                                   NodeStates.TEMPORARILY_DOWN)
+        self.assertEqual(len(self.neo.getMasterList()), MASTER_NODE_COUNT)
 
         uuid, = self.neo.killPrimary()
         # Check the state of the primary we just killed
-        self.neo.expectMasterState(uuid, (None, NodeStates.UNKNOWN))
+        self.neo.expectMasterState(uuid, NodeStates.TEMPORARILY_DOWN)
         # Check that a primary master arose.
         self.neo.expectPrimary(timeout=10)
         # Check that the uuid really changed.

neo/tests/testBootstrap.py

-#
-# Copyright (C) 2009-2017  Nexedi SA
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import unittest
-from . import NeoUnitTestBase
-from neo.storage.app import Application
-from neo.lib.bootstrap import BootstrapManager
-from neo.lib.protocol import NodeTypes, Packets
-
-class BootstrapManagerTests(NeoUnitTestBase):
-
-    def setUp(self):
-        NeoUnitTestBase.setUp(self)
-        self.prepareDatabase(number=1)
-        # create an application object
-        config = self.getStorageConfiguration()
-        self.app = Application(config)
-        self.bootstrap = BootstrapManager(self.app,  NodeTypes.STORAGE)
-        # define some variable to simulate client and storage node
-        self.master_port = 10010
-        self.storage_port = 10020
-        self.num_partitions = 1009
-        self.num_replicas = 2
-
-    def _tearDown(self, success):
-        self.app.close()
-        del self.app
-        super(BootstrapManagerTests, self)._tearDown(success)
-
-    # Tests
-    def testConnectionCompleted(self):
-        address = ("127.0.0.1", self.master_port)
-        conn = self.getFakeConnection(address=address)
-        self.bootstrap.current = self.app.nm.createMaster(address=address)
-        self.bootstrap.connectionCompleted(conn)
-        self.checkAskPacket(conn, Packets.RequestIdentification)
-
-    def testHandleNotReady(self):
-        # the primary is not ready
-        address = ("127.0.0.1", self.master_port)
-        conn = self.getFakeConnection(address=address)
-        self.bootstrap.current = self.app.nm.createMaster(address=address)
-        self.bootstrap.notReady(conn, '')
-        self.checkClosed(conn)
-        self.checkNoPacketSent(conn)
-
-
-if __name__ == "__main__":
-    unittest.main()
-

neo/tests/threaded/__init__.py

@@ -35,7 +35,7 @@ from neo.lib.connection import BaseConnection, \
 from neo.lib.connector import SocketConnector, ConnectorException
 from neo.lib.handler import EventHandler
 from neo.lib.locking import SimpleQueue
-from neo.lib.protocol import ClusterStates, NodeStates, NodeTypes, Packets
+from neo.lib.protocol import ClusterStates, Enum, NodeStates, NodeTypes, Packets
 from neo.lib.util import cached_property, parseMasterList, p64
 from .. import NeoTestBase, Patch, getTempDirectory, setupMySQLdb, \
     ADDRESS_TYPE, IP_VERSION_FORMAT_DICT, DB_PREFIX, DB_SOCKET, DB_USER
@@ -745,27 +745,39 @@ class NEOCluster(object):
     def __exit__(self, t, v, tb):
         self.stop(None)
 
-    def start(self, storage_list=None, fast_startup=False):
+    def start(self, storage_list=None, master_list=None, recovering=False):
         self.started = True
         self._patch()
         self.neoctl = NeoCTL(self.admin.getVirtualAddress(), ssl=self.SSL)
-        for node_type in 'master', 'admin':
-            for node in getattr(self, node_type + '_list'):
-                node.start()
+        for node in self.master_list if master_list is None else master_list:
+            node.start()
+        for node in self.admin_list:
+            node.start()
         Serialized.tic()
-        if fast_startup:
-            self.startCluster()
         if storage_list is None:
             storage_list = self.storage_list
         for node in storage_list:
             node.start()
         Serialized.tic()
-        if not fast_startup:
+        if recovering:
+            expected_state = ClusterStates.RECOVERING
+        else:
             self.startCluster()
             Serialized.tic()
+            expected_state = ClusterStates.RUNNING, ClusterStates.BACKINGUP
+        self.checkStarted(expected_state, storage_list)
+
+    def checkStarted(self, expected_state, storage_list=None):
+        if isinstance(expected_state, Enum.Item):
+            expected_state = expected_state,
         state = self.neoctl.getClusterState()
-        assert state in (ClusterStates.RUNNING, ClusterStates.BACKINGUP), state
-        self.enableStorageList(storage_list)
+        assert state in expected_state, state
+        expected_state = (NodeStates.PENDING
+            if state == ClusterStates.RECOVERING
+            else NodeStates.RUNNING)
+        for node in self.storage_list if storage_list is None else storage_list:
+            state = self.getNodeState(node)
+            assert state == expected_state, (node, state)
 
     def stop(self, clear_database=False, __print_exc=traceback.print_exc, **kw):
         if self.started:

neo/tests/threaded/test.py

@@ -34,8 +34,8 @@ from neo.lib.connection import ConnectionClosed, \
 from neo.lib.exception import DatabaseFailure, StoppedOperation
 from neo.lib.handler import DelayEvent
 from neo.lib import logging
-from neo.lib.protocol import CellStates, ClusterStates, NodeStates, Packets, \
-    Packet, uuid_str, ZERO_OID, ZERO_TID
+from neo.lib.protocol import (CellStates, ClusterStates, NodeStates, NodeTypes,
+    Packets, Packet, uuid_str, ZERO_OID, ZERO_TID)
 from .. import expectedFailure, Patch, TransactionalResource
 from . import ClientApplication, ConnectionFilter, LockLock, NEOThreadedTest, \
     RandomConflictDict, ThreadId, with_cluster
@@ -837,12 +837,6 @@ class Test(NEOThreadedTest):
 
     @with_cluster(master_count=3, partitions=10, replicas=1, storage_count=3)
     def testShutdown(self, cluster):
-        # BUG: Due to bugs in election, master nodes sometimes crash, or they
-        #      declare themselves primary too quickly, but issues seem to be
-        #      only reproducible with SSL enabled.
-        self._testShutdown(cluster)
-
-    def _testShutdown(self, cluster):
         def before_finish(_):
             # tell admin to shutdown the cluster
             cluster.neoctl.setClusterState(ClusterStates.STOPPING)
@@ -1225,12 +1219,10 @@ class Test(NEOThreadedTest):
 
     @with_cluster(start_cluster=0, storage_count=3, autostart=3)
     def testAutostart(self, cluster):
-        def startCluster(orig):
-            getClusterState = cluster.neoctl.getClusterState
-            self.assertEqual(ClusterStates.RECOVERING, getClusterState())
-            cluster.storage_list[2].start()
-        with Patch(cluster, startCluster=startCluster):
-            cluster.start(cluster.storage_list[:2])
+        cluster.start(cluster.storage_list[:2], recovering=True)
+        cluster.storage_list[2].start()
+        self.tic()
+        cluster.checkStarted(ClusterStates.RUNNING)
 
     @with_cluster(storage_count=2, partitions=2)
     def testAbortVotedTransaction(self, cluster):
@@ -2219,6 +2211,60 @@ class Test(NEOThreadedTest):
     def testConflictAfterDeadlockWithSlowReplica2(self):
         self.testConflictAfterDeadlockWithSlowReplica1(True)
 
+    @with_cluster(start_cluster=0, master_count=3)
+    def testElection(self, cluster):
+        m0, m1, m2 = cluster.master_list
+        cluster.start(master_list=(m0,), recovering=True)
+        getClusterState = cluster.neoctl.getClusterState
+        m0.em.removeReader(m0.listening_conn)
+        m1.start()
+        self.tic()
+        m2.start()
+        self.tic()
+        self.assertTrue(m0.primary)
+        self.assertTrue(m1.primary)
+        self.assertFalse(m2.primary)
+        m0.em.addReader(m0.listening_conn)
+        with ConnectionFilter() as f:
+            f.delayAcceptIdentification()
+            self.tic()
+        self.tic()
+        self.assertTrue(m0.primary)
+        self.assertFalse(m1.primary)
+        self.assertFalse(m2.primary)
+        self.assertEqual(getClusterState(), ClusterStates.RECOVERING)
+        cluster.startCluster()
+        def stop(node):
+            node.stop()
+            cluster.join((node,))
+            node.resetNode()
+        stop(m1)
+        self.tic()
+        self.assertEqual(getClusterState(), ClusterStates.RUNNING)
+        self.assertTrue(m0.primary)
+        self.assertFalse(m2.primary)
+        stop(m0)
+        self.tic()
+        self.assertEqual(getClusterState(), ClusterStates.RUNNING)
+        self.assertTrue(m2.primary)
+        # Check for proper update of node ids on first NotifyNodeInformation.
+        stop(m2)
+        m0.start()
+        def update(orig, app, timestamp, node_list):
+            orig(app, timestamp, sorted(node_list, reverse=1))
+        with Patch(cluster.storage.nm, update=update):
+            with ConnectionFilter() as f:
+                f.add(lambda conn, packet:
+                    isinstance(packet, Packets.RequestIdentification)
+                    and packet.decode()[0] == NodeTypes.STORAGE)
+                self.tic()
+                m2.start()
+                self.tic()
+            self.tic()
+        self.assertEqual(getClusterState(), ClusterStates.RUNNING)
+        self.assertTrue(m0.primary)
+        self.assertFalse(m2.primary)
+
 
 if __name__ == "__main__":
     unittest.main()

neo/tests/threaded/testSSL.py

@@ -17,7 +17,7 @@
 import unittest
 from neo.lib.protocol import Packets
 from .. import SSL
-from . import NEOCluster, with_cluster, test, testReplication
+from . import NEOCluster, test, testReplication
 
 
 class SSLMixin:
@@ -36,14 +36,6 @@ class SSLTests(SSLMixin, test.Test):
     testDeadlockAvoidance = None
     testUndoConflict = testUndoConflictDuringStore = None
 
-    if 1:
-        testShutdownWithSeveralMasterNodes = unittest.skip("fails randomly")(
-            test.Test.testShutdown.__func__)
-
-        @with_cluster(partitions=10, replicas=1, storage_count=3)
-        def testShutdown(self, cluster):
-            self._testShutdown(cluster)
-
     def testAbortConnection(self, after_handshake=1):
         with self.getLoopbackConnection() as conn:
             conn.ask(Packets.Ping())