Skip to content

N
neo
Commit aded81bc authored by Grégory Wisniewski's avatar Grégory Wisniewski
Browse files
Options

Use decorators and UnexpectedPacketError exception instead of calls 

handleUnexpectedPacket() in storage handlers.


git-svn-id: https://svn.erp5.org/repos/neo/branches/prototype3@505 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent 863a3207
Hide whitespace changes
Inline Side-by-side
Showing with 433 additions and 486 deletions
neo/storage/bootstrap.py
View file @ aded81bc
......@@ -23,10 +23,12 @@ from neo.protocol import INVALID_UUID, RUNNING_STATE, BROKEN_STATE, \
MASTER_NODE_TYPE, STORAGE_NODE_TYPE, CLIENT_NODE_TYPE
from neo.node import MasterNode, StorageNode, ClientNode
from neo.connection import ClientConnection
from neo.protocol import Packet
from neo.protocol import Packet, UnexpectedPacketError
from neo.pt import PartitionTable
from neo.storage.verification import VerificationEventHandler
from neo.util import dump
from neo.handler import identification_required, restrict_node_types, \
server_connection_required, client_connection_required
class BootstrapEventHandler(StorageEventHandler):
"""This class deals with events for a bootstrap phase."""
......@@ -105,144 +107,138 @@ class BootstrapEventHandler(StorageEventHandler):
conn.close()
@server_connection_required
def handleRequestNodeIdentification(self, conn, packet, node_type,
uuid, ip_address, port, name):
if not conn.isServerConnection():
self.handleUnexpectedPacket(conn, packet)
else:
app = self.app
if node_type != MASTER_NODE_TYPE:
logging.info('reject a connection from a non-master')
conn.answer(protocol.notReady('retry later'), packet)
conn.abort()
return
if name != app.name:
logging.error('reject an alien cluster')
conn.answer(protocol.protocolError('invalid cluster name'), packet)
conn.abort()
return
addr = (ip_address, port)
node = app.nm.getNodeByServer(addr)
if node is None:
node = MasterNode(server = addr, uuid = uuid)
app.nm.add(node)
else:
# If this node is broken, reject it.
if node.getUUID() == uuid:
if node.getState() == BROKEN_STATE:
p = protocol.brokenNodeDisallowedError('go away')
conn.answer(p, packet)
conn.abort()
return
# Trust the UUID sent by the peer.
node.setUUID(uuid)
conn.setUUID(uuid)
p = protocol.acceptNodeIdentification(STORAGE_NODE_TYPE, app.uuid,
app.server[0], app.server[1], 0, 0, uuid)
conn.answer(p, packet)
# Now the master node should know that I am not the right one.
app = self.app
if node_type != MASTER_NODE_TYPE:
logging.info('reject a connection from a non-master')
conn.answer(protocol.notReady('retry later'), packet)
conn.abort()
return
if name != app.name:
logging.error('reject an alien cluster')
conn.answer(protocol.protocolError('invalid cluster name'), packet)
conn.abort()
return
addr = (ip_address, port)
node = app.nm.getNodeByServer(addr)
if node is None:
node = MasterNode(server = addr, uuid = uuid)
app.nm.add(node)
else:
# If this node is broken, reject it.
if node.getUUID() == uuid:
if node.getState() == BROKEN_STATE:
p = protocol.brokenNodeDisallowedError('go away')
conn.answer(p, packet)
conn.abort()
return
# Trust the UUID sent by the peer.
node.setUUID(uuid)
conn.setUUID(uuid)
p = protocol.acceptNodeIdentification(STORAGE_NODE_TYPE, app.uuid,
app.server[0], app.server[1], 0, 0, uuid)
conn.answer(p, packet)
# Now the master node should know that I am not the right one.
conn.abort()
@client_connection_required
def handleAcceptNodeIdentification(self, conn, packet, node_type,
uuid, ip_address, port,
num_partitions, num_replicas, your_uuid):
if conn.isServerConnection():
self.handleUnexpectedPacket(conn, packet)
else:
app = self.app
node = app.nm.getNodeByServer(conn.getAddress())
if node_type != MASTER_NODE_TYPE:
# The peer is not a master node!
logging.error('%s:%d is not a master node', ip_address, port)
app.nm.remove(node)
conn.close()
return
if conn.getAddress() != (ip_address, port):
# The server address is different! Then why was
# the connection successful?
logging.error('%s:%d is waiting for %s:%d',
conn.getAddress()[0], conn.getAddress()[1],
ip_address, port)
app.nm.remove(node)
conn.close()
return
if app.num_partitions is None or app.num_replicas is None or \
app.num_replicas != num_replicas:
# changing number of replicas is not an issue
app.num_partitions = num_partitions
app.dm.setNumPartitions(app.num_partitions)
app.num_replicas = num_replicas
app.dm.setNumReplicas(app.num_replicas)
app.pt = PartitionTable(num_partitions, num_replicas)
app.loadPartitionTable()
app.ptid = app.dm.getPTID()
elif app.num_partitions != num_partitions:
raise RuntimeError('the number of partitions is inconsistent')
if your_uuid != INVALID_UUID and app.uuid != your_uuid:
# got an uuid from the primary master
app.uuid = your_uuid
app.dm.setUUID(app.uuid)
logging.info('Got a new UUID from master : %s' % dump(app.uuid))
conn.setUUID(uuid)
node.setUUID(uuid)
# Ask a primary master.
conn.ask(protocol.askPrimaryMaster())
app = self.app
node = app.nm.getNodeByServer(conn.getAddress())
if node_type != MASTER_NODE_TYPE:
# The peer is not a master node!
logging.error('%s:%d is not a master node', ip_address, port)
app.nm.remove(node)
conn.close()
return
if conn.getAddress() != (ip_address, port):
# The server address is different! Then why was
# the connection successful?
logging.error('%s:%d is waiting for %s:%d',
conn.getAddress()[0], conn.getAddress()[1],
ip_address, port)
app.nm.remove(node)
conn.close()
return
if app.num_partitions is None or app.num_replicas is None or \
app.num_replicas != num_replicas:
# changing number of replicas is not an issue
app.num_partitions = num_partitions
app.dm.setNumPartitions(app.num_partitions)
app.num_replicas = num_replicas
app.dm.setNumReplicas(app.num_replicas)
app.pt = PartitionTable(num_partitions, num_replicas)
app.loadPartitionTable()
app.ptid = app.dm.getPTID()
elif app.num_partitions != num_partitions:
raise RuntimeError('the number of partitions is inconsistent')
if your_uuid != INVALID_UUID and app.uuid != your_uuid:
# got an uuid from the primary master
app.uuid = your_uuid
app.dm.setUUID(app.uuid)
logging.info('Got a new UUID from master : %s' % dump(app.uuid))
conn.setUUID(uuid)
node.setUUID(uuid)
# Ask a primary master.
conn.ask(protocol.askPrimaryMaster())
@client_connection_required
def handleAnswerPrimaryMaster(self, conn, packet, primary_uuid,
known_master_list):
if conn.isServerConnection():
self.handleUnexpectedPacket(conn, packet)
else:
app = self.app
# Register new master nodes.
for ip_address, port, uuid in known_master_list:
addr = (ip_address, port)
n = app.nm.getNodeByServer(addr)
if n is None:
n = MasterNode(server = addr)
app.nm.add(n)
if uuid != INVALID_UUID:
# If I don't know the UUID yet, believe what the peer
# told me at the moment.
if n.getUUID() is None or n.getUUID() != uuid:
n.setUUID(uuid)
if primary_uuid != INVALID_UUID:
primary_node = app.nm.getNodeByUUID(primary_uuid)
if primary_node is None:
# I don't know such a node. Probably this information
# is old. So ignore it.
pass
else:
app.primary_master_node = primary_node
if app.trying_master_node is primary_node:
# I am connected to the right one.
logging.info('connected to a primary master node')
# This is a workaround to prevent handling of
# packets for the verification phase.
handler = VerificationEventHandler(app)
conn.setHandler(handler)
else:
app.trying_master_node = None
conn.close()
app = self.app
# Register new master nodes.
for ip_address, port, uuid in known_master_list:
addr = (ip_address, port)
n = app.nm.getNodeByServer(addr)
if n is None:
n = MasterNode(server = addr)
app.nm.add(n)
if uuid != INVALID_UUID:
# If I don't know the UUID yet, believe what the peer
# told me at the moment.
if n.getUUID() is None or n.getUUID() != uuid:
n.setUUID(uuid)
if primary_uuid != INVALID_UUID:
primary_node = app.nm.getNodeByUUID(primary_uuid)
if primary_node is None:
# I don't know such a node. Probably this information
# is old. So ignore it.
pass
else:
if app.primary_master_node is not None:
# The primary master node is not a primary master node
# any longer.
app.primary_master_node = None
app.primary_master_node = primary_node
if app.trying_master_node is primary_node:
# I am connected to the right one.
logging.info('connected to a primary master node')
# This is a workaround to prevent handling of
# packets for the verification phase.
handler = VerificationEventHandler(app)
conn.setHandler(handler)
else:
app.trying_master_node = None
conn.close()
else:
if app.primary_master_node is not None:
# The primary master node is not a primary master node
# any longer.
app.primary_master_node = None
app.trying_master_node = None
conn.close()
app.trying_master_node = None
conn.close()
def handleAskLastIDs(self, conn, packet):
logging.warning('/!\ handleAskLastIDs')
......
neo/storage/handler.py
View file @ aded81bc
......@@ -18,13 +18,14 @@
import logging
from neo.handler import EventHandler
from neo.protocol import Packet, \
from neo.protocol import Packet, UnexpectedPacketError, \
INVALID_UUID, RUNNING_STATE, BROKEN_STATE, \
MASTER_NODE_TYPE, STORAGE_NODE_TYPE, CLIENT_NODE_TYPE
from neo.util import dump
from neo.node import MasterNode, StorageNode, ClientNode
from neo.connection import ClientConnection
from neo.exception import PrimaryFailure
from neo.handler import identification_required, restrict_node_types
class StorageEventHandler(EventHandler):
"""This class implements a generic part of the event handlers."""
......@@ -69,24 +70,17 @@ class StorageEventHandler(EventHandler):
known_master_list):
raise NotImplementedError('this method must be overridden')
@identification_required
@restrict_node_types(MASTER_NODE_TYPE)
def handleAnnouncePrimaryMaster(self, conn, packet):
"""Theoretically speaking, I should not get this message,
because the primary master election must happen when I am
not connected to any master node."""
uuid = conn.getUUID()
if uuid is None:
self.handleUnexpectedPacket(conn, packet)
return
app = self.app
node = app.nm.getNodeByUUID(uuid)
if node is None:
raise RuntimeError('I do not know the uuid %r' % dump(uuid))
if node.getNodeType() != MASTER_NODE_TYPE:
self.handleUnexpectedPacket(conn, packet)
return
if app.primary_master_node is None:
# Hmm... I am somehow connected to the primary master already.
app.primary_master_node = node
......@@ -106,19 +100,16 @@ class StorageEventHandler(EventHandler):
def handleReelectPrimaryMaster(self, conn, packet):
raise PrimaryFailure('re-election occurs')
@identification_required
@restrict_node_types(MASTER_NODE_TYPE)
def handleNotifyNodeInformation(self, conn, packet, node_list):
"""Store information on nodes, only if this is sent by a primary
master node."""
# XXX it might be better to implement this callback in each handler.
uuid = conn.getUUID()
if uuid is None:
self.handleUnexpectedPacket(conn, packet)
return
app = self.app
node = app.nm.getNodeByUUID(uuid)
if node.getNodeType() != MASTER_NODE_TYPE \
or app.primary_master_node is None \
if app.primary_master_node is None \
or app.primary_master_node.getUUID() != uuid:
return
......@@ -209,21 +200,21 @@ class StorageEventHandler(EventHandler):
raise NotImplementedError('this method must be overridden')
def handleAskObject(self, conn, packet, oid, serial, tid):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
def handleAskTIDs(self, conn, packet, first, last, partition):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
def handleAskObjectHistory(self, conn, packet, oid, first, last):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
def handleAskStoreTransaction(self, conn, packet, tid, user, desc,
ext, oid_list):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
def handleAskStoreObject(self, conn, packet, oid, serial,
compression, checksum, data, tid):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
def handleAbortTransaction(self, conn, packet, tid):
logging.info('ignoring abort transaction')
......
neo/storage/operation.py
View file @ aded81bc
......@@ -19,16 +19,18 @@ import logging
from neo import protocol
from neo.storage.handler import StorageEventHandler
from neo.protocol import INVALID_UUID, INVALID_SERIAL, INVALID_TID, \
from neo.protocol import INVALID_SERIAL, INVALID_TID, \
INVALID_PARTITION, \
RUNNING_STATE, BROKEN_STATE, TEMPORARILY_DOWN_STATE, \
BROKEN_STATE, TEMPORARILY_DOWN_STATE, \
MASTER_NODE_TYPE, STORAGE_NODE_TYPE, CLIENT_NODE_TYPE, \
DISCARDED_STATE, OUT_OF_DATE_STATE
from neo.util import dump
from neo.node import MasterNode, StorageNode, ClientNode
from neo.connection import ClientConnection
from neo.protocol import Packet
from neo.protocol import Packet, UnexpectedPacketError
from neo.exception import PrimaryFailure, OperationFailure
from neo.handler import identification_required, restrict_node_types, \
server_connection_required, client_connection_required
class TransactionInformation(object):
"""This class represents information on a transaction."""
......@@ -131,123 +133,115 @@ class OperationEventHandler(StorageEventHandler):
StorageEventHandler.peerBroken(self, conn)
@server_connection_required
def handleRequestNodeIdentification(self, conn, packet, node_type,
uuid, ip_address, port, name):
if not conn.isServerConnection():
self.handleUnexpectedPacket(conn, packet)
else:
app = self.app
if name != app.name:
logging.error('reject an alien cluster')
p = protocol.protocolError('invalid cluster name')
conn.answer(p, packet)
app = self.app
if name != app.name:
logging.error('reject an alien cluster')
p = protocol.protocolError('invalid cluster name')
conn.answer(p, packet)
conn.abort()
return
addr = (ip_address, port)
node = app.nm.getNodeByUUID(uuid)
if node is None:
if node_type == MASTER_NODE_TYPE:
node = app.nm.getNodeByServer(addr)
if node is None:
node = MasterNode(server = addr, uuid = uuid)
app.nm.add(node)
else:
# If I do not know such a node, and it is not even a master
# node, simply reject it.
logging.error('reject an unknown node %s', dump(uuid))
conn.answer(protocol.notReady('unknown node'), packet)
conn.abort()
return
addr = (ip_address, port)
node = app.nm.getNodeByUUID(uuid)
if node is None:
if node_type == MASTER_NODE_TYPE:
node = app.nm.getNodeByServer(addr)
if node is None:
node = MasterNode(server = addr, uuid = uuid)
app.nm.add(node)
else:
# If I do not know such a node, and it is not even a master
# node, simply reject it.
logging.error('reject an unknown node %s', dump(uuid))
conn.answer(protocol.notReady('unknown node'), packet)
else:
# If this node is broken, reject it.
if node.getUUID() == uuid:
if node.getState() == BROKEN_STATE:
p = protocol.brokenNodeDisallowedError('go away')
conn.answer(p, packet)
conn.abort()
return
else:
# If this node is broken, reject it.
if node.getUUID() == uuid:
if node.getState() == BROKEN_STATE:
p = protocol.brokenNodeDisallowedError('go away')
conn.answer(p, packet)
conn.abort()
return
# Trust the UUID sent by the peer.
node.setUUID(uuid)
conn.setUUID(uuid)
p = protocol.acceptNodeIdentification(STORAGE_NODE_TYPE, app.uuid,
app.server[0], app.server[1], app.num_partitions,
app.num_replicas, uuid)
conn.answer(p, packet)
if node_type == MASTER_NODE_TYPE:
conn.abort()
# Trust the UUID sent by the peer.
node.setUUID(uuid)
conn.setUUID(uuid)
p = protocol.acceptNodeIdentification(STORAGE_NODE_TYPE, app.uuid,
app.server[0], app.server[1], app.num_partitions,
app.num_replicas, uuid)
conn.answer(p, packet)
if node_type == MASTER_NODE_TYPE:
conn.abort()
@client_connection_required
def handleAcceptNodeIdentification(self, conn, packet, node_type,
uuid, ip_address, port,
num_partitions, num_replicas, your_uuid):
if not conn.isServerConnection():
raise NotImplementedError
else:
self.handleUnexpectedPacket(conn, packet)
raise NotImplementedError
def handleAnswerPrimaryMaster(self, conn, packet, primary_uuid,
known_master_list):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
def handleAskLastIDs(self, conn, packet):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
def handleAskPartitionTable(self, conn, packet, offset_list):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
def handleSendPartitionTable(self, conn, packet, ptid, row_list):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
@client_connection_required
def handleNotifyPartitionChanges(self, conn, packet, ptid, cell_list):
"""This is very similar to Send Partition Table, except that
the information is only about changes from the previous."""
if not conn.isServerConnection():
app = self.app
nm = app.nm
pt = app.pt
if app.ptid >= ptid:
# Ignore this packet.
logging.info('ignoring older partition changes')
return
app = self.app
nm = app.nm
pt = app.pt
if app.ptid >= ptid:
# Ignore this packet.
logging.info('ignoring older partition changes')
return
# First, change the table on memory.
app.ptid = ptid
for offset, uuid, state in cell_list:
node = nm.getNodeByUUID(uuid)
if node is None:
node = StorageNode(uuid = uuid)
if uuid != app.uuid:
node.setState(TEMPORARILY_DOWN_STATE)
nm.add(node)
pt.setCell(offset, node, state)
if uuid == app.uuid:
# If this is for myself, this can affect replications.
if state == DISCARDED_STATE:
app.replicator.removePartition(offset)
elif state == OUT_OF_DATE_STATE:
app.replicator.addPartition(offset)
# Then, the database.
app.dm.changePartitionTable(ptid, cell_list)
else:
self.handleUnexpectedPacket(conn, packet)
# First, change the table on memory.
app.ptid = ptid
for offset, uuid, state in cell_list:
node = nm.getNodeByUUID(uuid)
if node is None:
node = StorageNode(uuid = uuid)
if uuid != app.uuid:
node.setState(TEMPORARILY_DOWN_STATE)
nm.add(node)
pt.setCell(offset, node, state)
if uuid == app.uuid:
# If this is for myself, this can affect replications.
if state == DISCARDED_STATE:
app.replicator.removePartition(offset)
elif state == OUT_OF_DATE_STATE:
app.replicator.addPartition(offset)
# Then, the database.
app.dm.changePartitionTable(ptid, cell_list)
def handleStartOperation(self, conn, packet):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
@client_connection_required
def handleStopOperation(self, conn, packet):
if not conn.isServerConnection():
raise OperationFailure('operation stopped')
else:
self.handleUnexpectedPacket(conn, packet)
raise OperationFailure('operation stopped')
def handleAskUnfinishedTransactions(self, conn, packet):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
def handleAskTransactionInformation(self, conn, packet, tid):
app = self.app
......@@ -260,51 +254,46 @@ class OperationEventHandler(StorageEventHandler):
conn.answer(p, packet)
def handleAskObjectPresent(self, conn, packet, oid, tid):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
def handleDeleteTransaction(self, conn, packet, tid):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
def handleCommitTransaction(self, conn, packet, tid):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
@client_connection_required
def handleLockInformation(self, conn, packet, tid):
if not conn.isServerConnection():
app = self.app
try:
t = app.transaction_dict[tid]
object_list = t.getObjectList()
for o in object_list:
app.load_lock_dict[o[0]] = tid
app.dm.storeTransaction(tid, object_list, t.getTransaction())
except KeyError:
pass
app = self.app
try:
t = app.transaction_dict[tid]
object_list = t.getObjectList()
for o in object_list:
app.load_lock_dict[o[0]] = tid
conn.answer(protocol.notifyInformationLocked(tid), packet)
else:
self.handleUnexpectedPacket(conn, packet)
app.dm.storeTransaction(tid, object_list, t.getTransaction())
except KeyError:
pass
conn.answer(protocol.notifyInformationLocked(tid), packet)
@client_connection_required
def handleUnlockInformation(self, conn, packet, tid):
if not conn.isServerConnection():
app = self.app
try:
t = app.transaction_dict[tid]
object_list = t.getObjectList()
for o in object_list:
oid = o[0]
del app.load_lock_dict[oid]
del app.store_lock_dict[oid]
app = self.app
try:
t = app.transaction_dict[tid]
object_list = t.getObjectList()
for o in object_list:
oid = o[0]
del app.load_lock_dict[oid]
del app.store_lock_dict[oid]
app.dm.finishTransaction(tid)
del app.transaction_dict[tid]
app.dm.finishTransaction(tid)
del app.transaction_dict[tid]
# Now it may be possible to execute some events.
app.executeQueuedEvents()
except KeyError:
pass
else:
self.handleUnexpectedPacket(conn, packet)
# Now it may be possible to execute some events.
app.executeQueuedEvents()
except KeyError:
pass
def handleAskObject(self, conn, packet, oid, serial, tid):
app = self.app
......@@ -369,25 +358,19 @@ class OperationEventHandler(StorageEventHandler):
p = protocol.answerObjectHistory(oid, history_list)
conn.answer(p, packet)
@identification_required
def handleAskStoreTransaction(self, conn, packet, tid, user, desc,
ext, oid_list):
uuid = conn.getUUID()
if uuid is None:
self.handleUnexpectedPacket(conn, packet)
return
app = self.app
t = app.transaction_dict.setdefault(tid, TransactionInformation(uuid))
t.addTransaction(oid_list, user, desc, ext)
conn.answer(protocol.answerStoreTransaction(tid), packet)
@identification_required
def handleAskStoreObject(self, conn, packet, oid, serial,
compression, checksum, data, tid):
uuid = conn.getUUID()
if uuid is None:
self.handleUnexpectedPacket(conn, packet)
return
# First, check for the locking state.
app = self.app
locking_tid = app.store_lock_dict.get(oid)
......@@ -421,12 +404,9 @@ class OperationEventHandler(StorageEventHandler):
conn.answer(p, packet)
app.store_lock_dict[oid] = tid
@identification_required
def handleAbortTransaction(self, conn, packet, tid):
uuid = conn.getUUID()
if uuid is None:
self.handleUnexpectedPacket(conn, packet)
return
app = self.app
try:
t = app.transaction_dict[tid]
......@@ -446,17 +426,13 @@ class OperationEventHandler(StorageEventHandler):
except KeyError:
pass
@client_connection_required
def handleAnswerLastIDs(self, conn, packet, loid, ltid, lptid):
if not conn.isServerConnection():
self.app.replicator.setCriticalTID(packet, ltid)
else:
self.handleUnexpectedPacket(conn, packet)
self.app.replicator.setCriticalTID(packet, ltid)
@client_connection_required
def handleAnswerUnfinishedTransactions(self, conn, packet, tid_list):
if not conn.isServerConnection():
self.app.replicator.setUnfinishedTIDList(tid_list)
else:
self.handleUnexpectedPacket(conn, packet)
self.app.replicator.setUnfinishedTIDList(tid_list)
def handleAskOIDs(self, conn, packet, first, last, partition):
# This method is complicated, because I must return OIDs only
......
neo/storage/tests/testStorageBootstrapHandler.py
View file @ aded81bc
......@@ -109,6 +109,14 @@ server: 127.0.0.1:10020
def getLastUUID(self):
return self.uuid
def checkUnexpectedPacketRaised(self, method, *args, **kwargs):
""" Check if the UnexpectedPacketError exception wxas raised """
self.assertRaises(UnexpectedPacketError, method, *args, **kwargs)
def checkIdenficationRequired(self, method, *args, **kwargs):
""" Check is the identification_required decorator is applied """
self.checkUnexpectedPacketRaised(method, *args, **kwargs)
# Method to test the kind of packet returned in answer
def checkCalledRequestNodeIdentification(self, conn, packet_number=0):
""" Check Request Node Identification has been send"""
......
neo/storage/tests/testStorageOperation.py
View file @ aded81bc
......@@ -50,6 +50,14 @@ class StorageOperationTests(unittest.TestCase):
return min(ptids), max(ptids)
ptid = min(ptids)
def checkUnexpectedPacketRaised(self, method, *args, **kwargs):
""" Check if the UnexpectedPacketError exception wxas raised """
self.assertRaises(UnexpectedPacketError, method, *args, **kwargs)
def checkIdenficationRequired(self, method, *args, **kwargs):
""" Check is the identification_required decorator is applied """
self.checkUnexpectedPacketRaised(method, *args, **kwargs)
def checkCalledAbort(self, conn, packet_number=0):
"""Check the abort method has been called and an error packet has been sent"""
# sometimes we answer an error, sometimes we just send it
......@@ -81,9 +89,7 @@ class StorageOperationTests(unittest.TestCase):
packet = Packet(msg_type=_msg_type)
# hook
self.operation.peerBroken = lambda c: c.peerBrokendCalled()
_call(conn=conn, packet=packet, **kwargs)
self.checkCalledAbort(conn)
self.assertEquals(len(conn.mockGetNamedCalls("peerBrokendCalled")), 1)
self.checkUnexpectedPacketRaised(_call, conn=conn, packet=packet, **kwargs)
def checkNoPacketSent(self, conn):
# no packet should be sent
......
neo/storage/tests/testStorageVerificationHandler.py
View file @ aded81bc
......@@ -36,7 +36,7 @@ from neo.protocol import ACCEPT_NODE_IDENTIFICATION, REQUEST_NODE_IDENTIFICATION
UNLOCK_INFORMATION, TID_NOT_FOUND_CODE, ASK_TRANSACTION_INFORMATION, ANSWER_TRANSACTION_INFORMATION, \
ANSWER_PARTITION_TABLE,SEND_PARTITION_TABLE, COMMIT_TRANSACTION
from neo.protocol import ERROR, BROKEN_NODE_DISALLOWED_CODE, ASK_PRIMARY_MASTER
from neo.protocol import ANSWER_PRIMARY_MASTER
from neo.protocol import ANSWER_PRIMARY_MASTER, UnexpectedPacketError
from neo.exception import PrimaryFailure, OperationFailure
from neo.storage.mysqldb import MySQLDatabaseManager, p64, u64
......@@ -127,6 +127,14 @@ server: 127.0.0.1:10020
return min(ptids), max(ptids)
ptid = min(ptids)
def checkUnexpectedPacketRaised(self, method, *args, **kwargs):
""" Check if the UnexpectedPacketError exception wxas raised """
self.assertRaises(UnexpectedPacketError, method, *args, **kwargs)
def checkIdenficationRequired(self, method, *args, **kwargs):
""" Check is the identification_required decorator is applied """
self.checkUnexpectedPacketRaised(method, *args, **kwargs)
def checkCalledAbort(self, conn, packet_number=0):
"""Check the abort method has been called and an error packet has been sent"""
# sometimes we answer an error, sometimes we just notify it
......@@ -306,9 +314,8 @@ server: 127.0.0.1:10020
"getAddress" : ("127.0.0.1", self.client_port),
"isServerConnection" : True})
p = Packet(msg_type=ACCEPT_NODE_IDENTIFICATION)
self.verification.handleAcceptNodeIdentification(conn, p, CLIENT_NODE_TYPE,
self.getNewUUID(),"127.0.0.1", self.client_port, 1009, 2, uuid)
self.checkCalledAbort(conn)
self.checkUnexpectedPacketRaised(self.verification.handleAcceptNodeIdentification,
conn, p, CLIENT_NODE_TYPE, self.getNewUUID(),"127.0.0.1", self.client_port, 1009, 2, uuid)
def test_07_handleAnswerPrimaryMaster(self):
# reject server connection
......@@ -317,8 +324,7 @@ server: 127.0.0.1:10020
conn = Mock({"getUUID" : uuid,
"getAddress" : ("127.0.0.1", self.client_port),
"isServerConnection" : True})
self.verification.handleAnswerPrimaryMaster(conn, packet,self.getNewUUID(), ())
self.checkCalledAbort(conn)
self.checkUnexpectedPacketRaised(self.verification.handleAnswerPrimaryMaster, conn, packet,self.getNewUUID(), ())
# raise id uuid is different
conn = Mock({"getUUID" : uuid,
......@@ -343,8 +349,7 @@ server: 127.0.0.1:10020
conn = Mock({"getUUID" : uuid,
"getAddress" : ("127.0.0.1", self.client_port),
"isServerConnection" : True})
self.verification.handleAskLastIDs(conn, packet)
self.checkCalledAbort(conn)
self.checkUnexpectedPacketRaised(self.verification.handleAskLastIDs, conn, packet)
# return invalid if db store nothing
conn = Mock({"getUUID" : uuid,
......@@ -402,8 +407,7 @@ server: 127.0.0.1:10020
conn = Mock({"getUUID" : uuid,
"getAddress" : ("127.0.0.1", self.client_port),
"isServerConnection" : True})
self.verification.handleAskPartitionTable(conn, packet, [1,])
self.checkCalledAbort(conn)
self.checkUnexpectedPacketRaised(self.verification.handleAskPartitionTable, conn, packet, [1,])
# try to get unknown offset
self.assertEqual(len(self.app.pt.getNodeList()), 0)
......@@ -449,9 +453,8 @@ server: 127.0.0.1:10020
"getAddress" : ("127.0.0.1", self.client_port),
"isServerConnection" : True})
self.app.ptid = 1
self.verification.handleSendPartitionTable(conn, packet, 0, ())
self.checkUnexpectedPacketRaised(self.verification.handleSendPartitionTable, conn, packet, 0, ())
self.assertEquals(self.app.ptid, 1)
self.checkCalledAbort(conn)
# send a table
conn = Mock({"getUUID" : uuid,
......@@ -496,9 +499,8 @@ server: 127.0.0.1:10020
"getAddress" : ("127.0.0.1", self.client_port),
"isServerConnection" : True})
self.app.ptid = 1
self.verification.handleNotifyPartitionChanges(conn, packet, 0, ())
self.checkUnexpectedPacketRaised(self.verification.handleNotifyPartitionChanges, conn, packet, 0, ())
self.assertEquals(self.app.ptid, 1)
self.checkCalledAbort(conn)
# old partition change
conn = Mock({
......@@ -534,8 +536,7 @@ server: 127.0.0.1:10020
conn = Mock({ "getAddress" : ("127.0.0.1", self.master_port),
'isServerConnection': True })
packet = Packet(msg_type=STOP_OPERATION)
self.verification.handleStartOperation(conn, packet)
self.checkCalledAbort(conn)
self.checkUnexpectedPacketRaised(self.verification.handleStartOperation, conn, packet)
conn = Mock({ "getAddress" : ("127.0.0.1", self.master_port),
'isServerConnection': False })
self.assertFalse(self.app.operational)
......@@ -547,8 +548,7 @@ server: 127.0.0.1:10020
conn = Mock({ "getAddress" : ("127.0.0.1", self.master_port),
'isServerConnection': True })
packet = Packet(msg_type=STOP_OPERATION)
self.verification.handleStopOperation(conn, packet)
self.checkCalledAbort(conn)
self.checkUnexpectedPacketRaised(self.verification.handleStopOperation, conn, packet)
conn = Mock({ "getAddress" : ("127.0.0.1", self.master_port),
'isServerConnection': False })
packet = Packet(msg_type=STOP_OPERATION)
......@@ -559,8 +559,7 @@ server: 127.0.0.1:10020
conn = Mock({ "getAddress" : ("127.0.0.1", self.master_port),
'isServerConnection': True })
packet = Packet(msg_type=ASK_UNFINISHED_TRANSACTIONS)
self.verification.handleAskUnfinishedTransactions(conn, packet)
self.checkCalledAbort(conn)
self.checkUnexpectedPacketRaised(self.verification.handleAskUnfinishedTransactions, conn, packet)
# client connection with no data
conn = Mock({ "getAddress" : ("127.0.0.1", self.master_port),
'isServerConnection': False})
......@@ -688,8 +687,7 @@ server: 127.0.0.1:10020
conn = Mock({ "getAddress" : ("127.0.0.1", self.master_port),
'isServerConnection': True })
packet = Packet(msg_type=ASK_OBJECT_PRESENT)
self.verification.handleAskObjectPresent(conn, packet, p64(1), p64(2))
self.checkCalledAbort(conn)
self.checkUnexpectedPacketRaised(self.verification.handleAskObjectPresent, conn, packet, p64(1), p64(2))
# client connection with no data
conn = Mock({ "getAddress" : ("127.0.0.1", self.master_port),
'isServerConnection': False})
......@@ -724,8 +722,7 @@ server: 127.0.0.1:10020
conn = Mock({ "getAddress" : ("127.0.0.1", self.master_port),
'isServerConnection': True })
packet = Packet(msg_type=ASK_OBJECT_PRESENT)
self.verification.handleDeleteTransaction(conn, packet, p64(1))
self.checkCalledAbort(conn)
self.checkUnexpectedPacketRaised(self.verification.handleDeleteTransaction, conn, packet, p64(1))
# client connection with no data
conn = Mock({ "getAddress" : ("127.0.0.1", self.master_port),
'isServerConnection': False})
......@@ -747,8 +744,7 @@ server: 127.0.0.1:10020
dm = Mock()
self.app.dm = dm
packet = Packet(msg_type=COMMIT_TRANSACTION)
self.verification.handleCommitTransaction(conn, packet, p64(1))
self.checkCalledAbort(conn)
self.checkUnexpectedPacketRaised(self.verification.handleCommitTransaction, conn, packet, p64(1))
self.assertEqual(len(dm.mockGetNamedCalls("finishTransaction")), 0)
# commit a transaction
conn = Mock({ "getAddress" : ("127.0.0.1", self.master_port),
......
neo/storage/verification.py
View file @ aded81bc
......@@ -20,13 +20,15 @@ import logging
from neo import protocol
from neo.storage.handler import StorageEventHandler
from neo.protocol import INVALID_OID, INVALID_TID, \
RUNNING_STATE, BROKEN_STATE, TEMPORARILY_DOWN_STATE, \
MASTER_NODE_TYPE, STORAGE_NODE_TYPE, CLIENT_NODE_TYPE, \
Packet
BROKEN_STATE, TEMPORARILY_DOWN_STATE, \
MASTER_NODE_TYPE, STORAGE_NODE_TYPE, \
Packet, UnexpectedPacketError
from neo.util import dump
from neo.node import MasterNode, StorageNode, ClientNode
from neo.connection import ClientConnection
from neo.exception import PrimaryFailure, OperationFailure
from neo.handler import identification_required, restrict_node_types, \
server_connection_required, client_connection_required
class VerificationEventHandler(StorageEventHandler):
"""This class deals with events for a verification phase."""
......@@ -61,149 +63,105 @@ class VerificationEventHandler(StorageEventHandler):
StorageEventHandler.peerBroken(self, conn)
@server_connection_required
def handleRequestNodeIdentification(self, conn, packet, node_type,
uuid, ip_address, port, name):
if not conn.isServerConnection():
self.handleUnexpectedPacket(conn, packet)
else:
app = self.app
if node_type != MASTER_NODE_TYPE:
logging.info('reject a connection from a non-master')
conn.answer(protocol.notReady('retry later'), packet)
conn.abort()
return
if name != app.name:
logging.error('reject an alien cluster')
conn.answer(protocol.protocolError(
'invalid cluster name'), packet)
conn.abort()
return
addr = (ip_address, port)
node = app.nm.getNodeByServer(addr)
if node is None:
node = MasterNode(server = addr, uuid = uuid)
app.nm.add(node)
else:
# If this node is broken, reject it.
if node.getUUID() == uuid:
if node.getState() == BROKEN_STATE:
p = protocol.brokenNodeDisallowedError('go away')
conn.answer(p, packet)
conn.abort()
return
# Trust the UUID sent by the peer.
node.setUUID(uuid)
conn.setUUID(uuid)
p = protocol.acceptNodeIdentification(STORAGE_NODE_TYPE, app.uuid,
app.server[0], app.server[1], app.num_partitions,
app.num_replicas, uuid)
conn.answer(p, packet)
# Now the master node should know that I am not the right one.
app = self.app
if node_type != MASTER_NODE_TYPE:
logging.info('reject a connection from a non-master')
conn.answer(protocol.notReady('retry later'), packet)
conn.abort()
return
if name != app.name:
logging.error('reject an alien cluster')
conn.answer(protocol.protocolError(
'invalid cluster name'), packet)
conn.abort()
return
addr = (ip_address, port)
node = app.nm.getNodeByServer(addr)
if node is None:
node = MasterNode(server = addr, uuid = uuid)
app.nm.add(node)
else:
# If this node is broken, reject it.
if node.getUUID() == uuid:
if node.getState() == BROKEN_STATE:
p = protocol.brokenNodeDisallowedError('go away')
conn.answer(p, packet)
conn.abort()
return
# Trust the UUID sent by the peer.
node.setUUID(uuid)
conn.setUUID(uuid)
p = protocol.acceptNodeIdentification(STORAGE_NODE_TYPE, app.uuid,
app.server[0], app.server[1], app.num_partitions,
app.num_replicas, uuid)
conn.answer(p, packet)
# Now the master node should know that I am not the right one.
conn.abort()
def handleAcceptNodeIdentification(self, conn, packet, node_type,
uuid, ip_address, port,
num_partitions, num_replicas, your_uuid):
self.handleUnexpectedPacket(conn, packet)
raise UnexpectedPacketError
@client_connection_required
def handleAnswerPrimaryMaster(self, conn, packet, primary_uuid,
known_master_list):
if not conn.isServerConnection():
app = self.app
if app.primary_master_node.getUUID() != primary_uuid:
raise PrimaryFailure('the primary master node seems to have changed')
# XXX is it better to deal with known_master_list here?
# But a primary master node is supposed not to send any info
# with this packet, so it would be useless.
else:
self.handleUnexpectedPacket(conn, packet)
app = self.app
if app.primary_master_node.getUUID() != primary_uuid:
raise PrimaryFailure('the primary master node seems to have changed')
# XXX is it better to deal with known_master_list here?
# But a primary master node is supposed not to send any info
# with this packet, so it would be useless.
@client_connection_required
def handleAskLastIDs(self, conn, packet):
if not conn.isServerConnection():
app = self.app
oid = app.dm.getLastOID() or INVALID_OID
tid = app.dm.getLastTID() or INVALID_TID
p = protocol.answerLastIDs(oid, tid, app.ptid)
conn.answer(p, packet)
else:
self.handleUnexpectedPacket(conn, packet)
app = self.app
oid = app.dm.getLastOID() or INVALID_OID
tid = app.dm.getLastTID() or INVALID_TID
p = protocol.answerLastIDs(oid, tid, app.ptid)
conn.answer(p, packet)
@client_connection_required
def handleAskPartitionTable(self, conn, packet, offset_list):
if not conn.isServerConnection():
app = self.app
row_list = []
try:
for offset in offset_list:
row = []
try:
for cell in app.pt.getCellList(offset):
row.append((cell.getUUID(), cell.getState()))
except TypeError:
pass
row_list.append((offset, row))
except IndexError:
p = protocol.protocolError( 'invalid partition table offset')
conn.answer(p, packer)
return
p = protocol.answerPartitionTable(app.ptid, row_list)
conn.answer(p, packet)
else:
self.handleUnexpectedPacket(conn, packet)
app = self.app
row_list = []
try:
for offset in offset_list:
row = []
try:
for cell in app.pt.getCellList(offset):
row.append((cell.getUUID(), cell.getState()))
except TypeError:
pass
row_list.append((offset, row))
except IndexError:
p = protocol.protocolError( 'invalid partition table offset')
conn.answer(p, packer)
return
p = protocol.answerPartitionTable(app.ptid, row_list)
conn.answer(p, packet)
@client_connection_required
def handleSendPartitionTable(self, conn, packet, ptid, row_list):
"""A primary master node sends this packet to synchronize a partition
table. Note that the message can be split into multiple packets."""
if not conn.isServerConnection():
app = self.app
nm = app.nm
pt = app.pt
if app.ptid != ptid:
app.ptid = ptid
pt.clear()
for offset, row in row_list:
for uuid, state in row:
node = nm.getNodeByUUID(uuid)
if node is None:
node = StorageNode(uuid = uuid)
if uuid != app.uuid:
node.setState(TEMPORARILY_DOWN_STATE)
nm.add(node)
pt.setCell(offset, node, state)
if pt.filled():
# If the table is filled, I assume that the table is ready
# to use. Thus install it into the database for persistency.
cell_list = []
for offset in xrange(app.num_partitions):
for cell in pt.getCellList(offset):
cell_list.append((offset, cell.getUUID(),
cell.getState()))
app.dm.setPartitionTable(ptid, cell_list)
else:
self.handleUnexpectedPacket(conn, packet)
def handleNotifyPartitionChanges(self, conn, packet, ptid, cell_list):
"""This is very similar to Send Partition Table, except that
the information is only about changes from the previous."""
if not conn.isServerConnection():
app = self.app
nm = app.nm
pt = app.pt
if app.ptid >= ptid:
# Ignore this packet.
logging.info('ignoring older partition changes')
return
# First, change the table on memory.
app = self.app
nm = app.nm
pt = app.pt
if app.ptid != ptid:
app.ptid = ptid
for offset, uuid, state in cell_list:
pt.clear()
for offset, row in row_list:
for uuid, state in row:
node = nm.getNodeByUUID(uuid)
if node is None:
node = StorageNode(uuid = uuid)
......@@ -213,31 +171,56 @@ class VerificationEventHandler(StorageEventHandler):
pt.setCell(offset, node, state)
# Then, the database.
app.dm.changePartitionTable(ptid, cell_list)
else:
self.handleUnexpectedPacket(conn, packet)
if pt.filled():
# If the table is filled, I assume that the table is ready
# to use. Thus install it into the database for persistency.
cell_list = []
for offset in xrange(app.num_partitions):
for cell in pt.getCellList(offset):
cell_list.append((offset, cell.getUUID(),
cell.getState()))
app.dm.setPartitionTable(ptid, cell_list)
@client_connection_required
def handleNotifyPartitionChanges(self, conn, packet, ptid, cell_list):
"""This is very similar to Send Partition Table, except that
the information is only about changes from the previous."""
app = self.app
nm = app.nm
pt = app.pt
if app.ptid >= ptid:
# Ignore this packet.
logging.info('ignoring older partition changes')
return
# First, change the table on memory.
app.ptid = ptid
for offset, uuid, state in cell_list:
node = nm.getNodeByUUID(uuid)
if node is None:
node = StorageNode(uuid = uuid)
if uuid != app.uuid:
node.setState(TEMPORARILY_DOWN_STATE)
nm.add(node)
pt.setCell(offset, node, state)
# Then, the database.
app.dm.changePartitionTable(ptid, cell_list)
@client_connection_required
def handleStartOperation(self, conn, packet):
if not conn.isServerConnection():
self.app.operational = True
else:
self.handleUnexpectedPacket(conn, packet)
self.app.operational = True
@client_connection_required
def handleStopOperation(self, conn, packet):
if not conn.isServerConnection():
raise OperationFailure('operation stopped')
else:
self.handleUnexpectedPacket(conn, packet)
raise OperationFailure('operation stopped')
@client_connection_required
def handleAskUnfinishedTransactions(self, conn, packet):
if not conn.isServerConnection():
app = self.app
tid_list = app.dm.getUnfinishedTIDList()
p = protocol.answerUnfinishedTransactions(tid_list)
conn.answer(p, packet)
else:
self.handleUnexpectedPacket(conn, packet)
tid_list = self.app.dm.getUnfinishedTIDList()
p = protocol.answerUnfinishedTransactions(tid_list)
conn.answer(p, packet)
def handleAskTransactionInformation(self, conn, packet, tid):
app = self.app
......@@ -255,31 +238,22 @@ class VerificationEventHandler(StorageEventHandler):
p = protocol.answerTransactionInformation(tid, t[1], t[2], t[3], t[0])
conn.answer(p, packet)
@client_connection_required
def handleAskObjectPresent(self, conn, packet, oid, tid):
if not conn.isServerConnection():
app = self.app
if app.dm.objectPresent(oid, tid):
p = protocol.answerObjectPresent(oid, tid)
else:
p = protocol.oidNotFound(
'%s:%s do not exist' % (dump(oid), dump(tid)))
conn.answer(p, packet)
if self.app.dm.objectPresent(oid, tid):
p = protocol.answerObjectPresent(oid, tid)
else:
self.handleUnexpectedPacket(conn, packet)
p = protocol.oidNotFound(
'%s:%s do not exist' % (dump(oid), dump(tid)))
conn.answer(p, packet)
@client_connection_required
def handleDeleteTransaction(self, conn, packet, tid):
if not conn.isServerConnection():
app = self.app
app.dm.deleteTransaction(tid, all = True)
else:
self.handleUnexpectedPacket(conn, packet)
self.app.dm.deleteTransaction(tid, all = True)
@client_connection_required
def handleCommitTransaction(self, conn, packet, tid):
if not conn.isServerConnection():
app = self.app
app.dm.finishTransaction(tid)
else:
self.handleUnexpectedPacket(conn, packet)
self.app.dm.finishTransaction(tid)
def handleLockInformation(self, conn, packet, tid):
pass
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7