Commit faf121b5 authored by Grégory Wisniewski's avatar Grégory Wisniewski

Allow reconnect to a storage node when it was found not ready.

This commit fix random issues found with functionnal tests where the client
was refuse by the storage, because the latter was not fully initialized,
but never tried to reconnect to it if no other storages were available.

The main change introoduced is the availability of 'iterateForObject'
method on ConnectionPool. It allow iterate over potential node connections
for a given object id with the ability of waiting for the node to be ready
if not. It includes the common pattern that retreive the cell list,
randomize then sort them and never returns a None value, which suppose that
the outer loop must check if at least one iteration happens, for example.

Also included:
- getPartitionTable is now private because the connection needs it
- Deletion of _getCellListFor*
- Fixed tests
- New tests for ConnectionPool.iterateForObject

git-svn-id: https://svn.erp5.org/repos/neo/trunk@2578 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent 79568a61
This diff is collapsed.
......@@ -15,13 +15,16 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
import time
from random import shuffle
import neo
from neo.util import dump
from neo.locking import RLock
from neo.protocol import NodeTypes, Packets
from neo.connection import MTClientConnection, ConnectionClosed
from neo.client.exception import NEOStorageError
from neo.profiling import profiler_decorator
import time
# How long before we might retry a connection to a node to which connection
# failed in the past.
......@@ -35,6 +38,8 @@ CELL_GOOD = 0
# Storage node hosting cell failed recently, low priority
CELL_FAILED = 1
NOT_READY = object()
class ConnectionPool(object):
"""This class manages a pool of connections to storage nodes."""
......@@ -92,7 +97,7 @@ class ConnectionPool(object):
else:
neo.logging.info('%r not ready', node)
self.notifyFailure(node)
return None
return NOT_READY
@profiler_decorator
def _dropConnections(self):
......@@ -135,11 +140,26 @@ class ConnectionPool(object):
return result
@profiler_decorator
def getConnForCell(self, cell):
return self.getConnForNode(cell.getNode())
def getConnForCell(self, cell, wait_ready=False):
return self.getConnForNode(cell.getNode(), wait_ready=wait_ready)
def iterateForObject(self, object_id, readable=False, writable=False,
wait_ready=False):
""" Iterate over nodes responsible of a object by it's ID """
pt = self.app.getPartitionTable()
cell_list = pt.getCellListForOID(object_id, readable, writable)
if cell_list:
shuffle(cell_list)
cell_list.sort(key=self.getCellSortKey)
getConnForNode = self.getConnForNode
for cell in cell_list:
node = cell.getNode()
conn = getConnForNode(node, wait_ready=wait_ready)
if conn is not None:
yield (node, conn)
@profiler_decorator
def getConnForNode(self, node):
def getConnForNode(self, node, wait_ready=True):
"""Return a locked connection object to a given node
If no connection exists, create a new one"""
if not node.isRunning():
......@@ -155,10 +175,16 @@ class ConnectionPool(object):
# must drop some unused connections
self._dropConnections()
# Create new connection to node
while True:
conn = self._initNodeConnection(node)
if conn is not None:
if conn is NOT_READY and wait_ready:
time.sleep(1)
continue
if conn not in (None, NOT_READY):
self.connection_dict[uuid] = conn
return conn
else:
return None
finally:
self.connection_lock_release()
......
This diff is collapsed.
......@@ -68,6 +68,38 @@ class ConnectionPoolTests(NeoUnitTestBase):
self.assertEqual(getCellSortKey(node_uuid_2, 10), getCellSortKey(
node_uuid_3, 10))
def test_iterateForObject_noStorageAvailable(self):
# no node available
oid = self.getOID(1)
pt = Mock({'getCellListForOID': []})
app = Mock({'getPartitionTable': pt})
pool = ConnectionPool(app)
self.assertRaises(StopIteration, pool.iterateForObject(oid).next)
def test_iterateForObject_connectionRefused(self):
# connection refused
oid = self.getOID(1)
node = Mock({'__repr__': 'node'})
cell = Mock({'__repr__': 'cell', 'getNode': node})
conn = Mock({'__repr__': 'conn'})
pt = Mock({'getCellListForOID': [cell]})
app = Mock({'getPartitionTable': pt})
pool = ConnectionPool(app)
pool.getConnForNode = Mock({'__call__': None})
self.assertRaises(StopIteration, pool.iterateForObject(oid).next)
def test_iterateForObject_connectionRefused(self):
# connection refused
oid = self.getOID(1)
node = Mock({'__repr__': 'node'})
cell = Mock({'__repr__': 'cell', 'getNode': node})
conn = Mock({'__repr__': 'conn'})
pt = Mock({'getCellListForOID': [cell]})
app = Mock({'getPartitionTable': pt})
pool = ConnectionPool(app)
pool.getConnForNode = Mock({'__call__': conn})
self.assertEqual(list(pool.iterateForObject(oid)), [(node, conn)])
if __name__ == '__main__':
unittest.main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment