Commit e45dfae8 authored by Julien Muchembled's avatar Julien Muchembled

Fix handling of incoming non-empty storage nodes just after startup is allowed

git-svn-id: https://svn.erp5.org/repos/neo/trunk@2835 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent 0f8f681c
...@@ -157,11 +157,6 @@ class MasterEventHandler(EventHandler): ...@@ -157,11 +157,6 @@ class MasterEventHandler(EventHandler):
def notifyNodeInformation(self, conn, node_list): def notifyNodeInformation(self, conn, node_list):
app = self.app app = self.app
app.nm.update(node_list) app.nm.update(node_list)
if not app.pt.filled():
# Re-ask partition table, in case node change filled it.
# XXX: we should only ask it if received states indicates it is
# possible (ignore TEMPORARILY_DOWN for example)
conn.ask(Packets.AskPartitionTable())
class MasterRequestEventHandler(EventHandler): class MasterRequestEventHandler(EventHandler):
""" This class handle all answer from primary master node""" """ This class handle all answer from primary master node"""
......
...@@ -23,7 +23,6 @@ from neo.lib.protocol import Packets, ProtocolError, ClusterStates, NodeStates ...@@ -23,7 +23,6 @@ from neo.lib.protocol import Packets, ProtocolError, ClusterStates, NodeStates
from neo.lib.protocol import NotReadyError, ZERO_OID, ZERO_TID from neo.lib.protocol import NotReadyError, ZERO_OID, ZERO_TID
from neo.master.handlers import MasterHandler from neo.master.handlers import MasterHandler
REQUIRED_NODE_NUMBER = 1
class RecoveryManager(MasterHandler): class RecoveryManager(MasterHandler):
""" """
...@@ -42,14 +41,7 @@ class RecoveryManager(MasterHandler): ...@@ -42,14 +41,7 @@ class RecoveryManager(MasterHandler):
""" """
Returns the handler for storage nodes Returns the handler for storage nodes
""" """
# XXX: Looking at 'uuid' is not a good criteria to know if the storage return uuid, NodeStates.PENDING, self
# is empty. Empty node should be accepted here.
# This is also the first step to fix handling of incoming
# non-empty storage nodes, whereas startup was already allowed.
if uuid is None and not self.app._startup_allowed:
neo.lib.logging.info('reject empty storage node')
raise NotReadyError
return (uuid, NodeStates.RUNNING, self)
def run(self): def run(self):
""" """
...@@ -67,22 +59,37 @@ class RecoveryManager(MasterHandler): ...@@ -67,22 +59,37 @@ class RecoveryManager(MasterHandler):
self.app.pt.setID(None) self.app.pt.setID(None)
# collect the last partition table available # collect the last partition table available
while not self.app._startup_allowed: while 1:
em.poll(1) em.poll(1)
if self.app._startup_allowed:
allowed_node_set = set()
for node in self.app.nm.getStorageList():
if node.isPending():
break # waiting for an answer
if node.isRunning():
allowed_node_set.add(node)
else:
if allowed_node_set:
break # no ready storage node
neo.lib.logging.info('startup allowed') neo.lib.logging.info('startup allowed')
# build a new partition table
if self.app.pt.getID() is None: if self.app.pt.getID() is None:
self.buildFromScratch() neo.lib.logging.info('creating a new partition table')
# reset IDs generators & build new partition with running nodes
self.app.tm.setLastOID(ZERO_OID)
self.app.pt.make(allowed_node_set)
self._broadcastPartitionTable(self.app.pt.getID(),
self.app.pt.getRowList())
# collect node that are connected but not in the selected partition # collect node that are connected but not in the selected partition
# table and set them in pending state # table and set them in pending state
allowed_node_set = set(self.app.pt.getNodeList()) refused_node_set = allowed_node_set.difference(
refused_node_set = set(self.app.nm.getStorageList()) - allowed_node_set self.app.pt.getNodeList())
for node in refused_node_set: if refused_node_set:
node.setPending() for node in refused_node_set:
self.app.broadcastNodesInformation(refused_node_set) node.setPending()
self.app.broadcastNodesInformation(refused_node_set)
self.app.setLastTransaction(self.app.tm.getLastTID()) self.app.setLastTransaction(self.app.tm.getLastTID())
neo.lib.logging.debug( neo.lib.logging.debug(
...@@ -90,23 +97,6 @@ class RecoveryManager(MasterHandler): ...@@ -90,23 +97,6 @@ class RecoveryManager(MasterHandler):
'table :', dump(self.app.tm.getLastOID())) 'table :', dump(self.app.tm.getLastOID()))
self.app.pt.log() self.app.pt.log()
def buildFromScratch(self):
nm, em, pt = self.app.nm, self.app.em, self.app.pt
neo.lib.logging.debug('creating a new partition table, wait for a ' \
'storage node')
# wait for some empty storage nodes, their are accepted
while len(nm.getStorageList()) < REQUIRED_NODE_NUMBER:
em.poll(1)
# take the first node available
node_list = nm.getStorageList()[:REQUIRED_NODE_NUMBER]
for node in node_list:
node.setRunning()
self.app.broadcastNodesInformation(node_list)
# resert IDs generators
self.app.tm.setLastOID(ZERO_OID)
# build the partition with this node
pt.make(node_list)
def connectionLost(self, conn, new_state): def connectionLost(self, conn, new_state):
node = self.app.nm.getByUUID(conn.getUUID()) node = self.app.nm.getByUUID(conn.getUUID())
assert node is not None assert node is not None
...@@ -117,10 +107,8 @@ class RecoveryManager(MasterHandler): ...@@ -117,10 +107,8 @@ class RecoveryManager(MasterHandler):
self.app.broadcastNodesInformation([node]) self.app.broadcastNodesInformation([node])
def connectionCompleted(self, conn): def connectionCompleted(self, conn):
# XXX: handler split review needed to remove this hack # ask the last IDs to perform the recovery
if not self.app._startup_allowed: conn.ask(Packets.AskLastIDs())
# ask the last IDs to perform the recovery
conn.ask(Packets.AskLastIDs())
def answerLastIDs(self, conn, loid, ltid, lptid): def answerLastIDs(self, conn, loid, ltid, lptid):
# Get max values. # Get max values.
...@@ -132,13 +120,25 @@ class RecoveryManager(MasterHandler): ...@@ -132,13 +120,25 @@ class RecoveryManager(MasterHandler):
# something newer # something newer
self.target_ptid = lptid self.target_ptid = lptid
conn.ask(Packets.AskPartitionTable()) conn.ask(Packets.AskPartitionTable())
else:
node = self.app.nm.getByUUID(conn.getUUID())
assert node.isPending()
node.setRunning()
self.app.broadcastNodesInformation([node])
def answerPartitionTable(self, conn, ptid, row_list): def answerPartitionTable(self, conn, ptid, row_list):
node = self.app.nm.getByUUID(conn.getUUID())
assert node.isPending()
node.setRunning()
if ptid != self.target_ptid: if ptid != self.target_ptid:
# If this is not from a target node, ignore it. # If this is not from a target node, ignore it.
neo.lib.logging.warn('Got %s while waiting %s', dump(ptid), neo.lib.logging.warn('Got %s while waiting %s', dump(ptid),
dump(self.target_ptid)) dump(self.target_ptid))
return else:
self._broadcastPartitionTable(ptid, row_list)
self.app.broadcastNodesInformation([node])
def _broadcastPartitionTable(self, ptid, row_list):
try: try:
new_nodes = self.app.pt.load(ptid, row_list, self.app.nm) new_nodes = self.app.pt.load(ptid, row_list, self.app.nm)
except IndexError: except IndexError:
...@@ -152,4 +152,3 @@ class RecoveryManager(MasterHandler): ...@@ -152,4 +152,3 @@ class RecoveryManager(MasterHandler):
for node in self.app.nm.getAdminList(only_identified=True): for node in self.app.nm.getAdminList(only_identified=True):
node.notify(notification) node.notify(notification)
node.notify(partition_table) node.notify(partition_table)
...@@ -97,12 +97,14 @@ class MasterRecoveryTests(NeoUnitTestBase): ...@@ -97,12 +97,14 @@ class MasterRecoveryTests(NeoUnitTestBase):
# not from target node, ignore # not from target node, ignore
uuid = self.identifyToMasterNode(NodeTypes.STORAGE, port=self.storage_port) uuid = self.identifyToMasterNode(NodeTypes.STORAGE, port=self.storage_port)
conn = self.getFakeConnection(uuid, self.storage_port) conn = self.getFakeConnection(uuid, self.storage_port)
node = self.app.nm.getByUUID(conn.getUUID())
offset = 1 offset = 1
cell_list = [(offset, uuid, CellStates.UP_TO_DATE)] cell_list = [(offset, uuid, CellStates.UP_TO_DATE)]
cells = self.app.pt.getRow(offset) cells = self.app.pt.getRow(offset)
for cell, state in cells: for cell, state in cells:
self.assertEqual(state, CellStates.OUT_OF_DATE) self.assertEqual(state, CellStates.OUT_OF_DATE)
recovery.target_ptid = 2 recovery.target_ptid = 2
node.setPending()
recovery.answerPartitionTable(conn, 1, cell_list) recovery.answerPartitionTable(conn, 1, cell_list)
cells = self.app.pt.getRow(offset) cells = self.app.pt.getRow(offset)
for cell, state in cells: for cell, state in cells:
...@@ -114,6 +116,7 @@ class MasterRecoveryTests(NeoUnitTestBase): ...@@ -114,6 +116,7 @@ class MasterRecoveryTests(NeoUnitTestBase):
cells = self.app.pt.getRow(offset) cells = self.app.pt.getRow(offset)
for cell, state in cells: for cell, state in cells:
self.assertEqual(state, CellStates.OUT_OF_DATE) self.assertEqual(state, CellStates.OUT_OF_DATE)
node.setPending()
recovery.answerPartitionTable(conn, None, cell_list) recovery.answerPartitionTable(conn, None, cell_list)
cells = self.app.pt.getRow(offset) cells = self.app.pt.getRow(offset)
for cell, state in cells: for cell, state in cells:
...@@ -124,6 +127,7 @@ class MasterRecoveryTests(NeoUnitTestBase): ...@@ -124,6 +127,7 @@ class MasterRecoveryTests(NeoUnitTestBase):
offset = 1000000 offset = 1000000
self.assertFalse(self.app.pt.hasOffset(offset)) self.assertFalse(self.app.pt.hasOffset(offset))
cell_list = [(offset, ((uuid, NodeStates.DOWN,),),)] cell_list = [(offset, ((uuid, NodeStates.DOWN,),),)]
node.setPending()
self.checkProtocolErrorRaised(recovery.answerPartitionTable, conn, self.checkProtocolErrorRaised(recovery.answerPartitionTable, conn,
2, cell_list) 2, cell_list)
......
...@@ -109,8 +109,7 @@ class Test(NEOThreadedTest): ...@@ -109,8 +109,7 @@ class Test(NEOThreadedTest):
cluster.reset() cluster.reset()
try: try:
cluster.start(storage_list=(s1,), fast_startup=fast_startup) cluster.start(storage_list=(s1,), fast_startup=fast_startup)
self.assertEqual((NodeStates.UNKNOWN, None)[fast_startup], self.assertEqual(NodeStates.UNKNOWN, cluster.getNodeState(s2))
cluster.getNodeState(s2))
finally: finally:
cluster.stop() cluster.stop()
...@@ -140,10 +139,4 @@ class Test(NEOThreadedTest): ...@@ -140,10 +139,4 @@ class Test(NEOThreadedTest):
cluster.stop() cluster.stop()
def testVerificationCommitUnfinishedTransactionsFastStartup(self): def testVerificationCommitUnfinishedTransactionsFastStartup(self):
# XXX: This test fails because if the admin starts the cluster without
# any storage node, the master (which is still in recovery stage)
# does not handle properly incoming non-empty storage nodes.
# In particular, it does not ask the last ids to the storage,
# and the client will ask objects at tid 0.
# See also RecoveryManager.identifyStorageNode
self.testVerificationCommitUnfinishedTransactions(True) self.testVerificationCommitUnfinishedTransactions(True)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment