Commit 0cdbf0ea authored by Olivier Cros's avatar Olivier Cros

Implementing ipv6 on neo

In order to synchronise neo with slapos, it has to work perfectly with ipv4
and ipv6. This allows to integrate neo in erp5 and to prepare different buildout
installations of neo.
The protocol and connectors are no more generic but can now support IPv4 and
IPv6 connections. We adopted a specific way of development which allow to
easily add new protocols in the future.

git-svn-id: https://svn.erp5.org/repos/neo/trunk@2654 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent c4d7ac19
...@@ -56,18 +56,15 @@ class Application(object): ...@@ -56,18 +56,15 @@ class Application(object):
"""The storage node application.""" """The storage node application."""
def __init__(self, config): def __init__(self, config):
# always use default connector for now
self.connector_handler = getConnectorHandler()
# Internal attributes. # Internal attributes.
self.em = EventManager() self.em = EventManager()
self.nm = NodeManager() self.nm = NodeManager()
self.name = config.getCluster() self.name = config.getCluster()
self.server = config.getBind() self.server = config.getBind()
self.master_addresses = config.getMasters()
self.master_addresses, connector_name = config.getMasters()
self.connector_handler = getConnectorHandler(connector_name)
neo.lib.logging.debug('IP address is %s, port is %d', *(self.server)) neo.lib.logging.debug('IP address is %s, port is %d', *(self.server))
# The partition table is initialized after getting the number of # The partition table is initialized after getting the number of
......
...@@ -77,7 +77,8 @@ class Application(object): ...@@ -77,7 +77,8 @@ class Application(object):
# Internal Attributes common to all thread # Internal Attributes common to all thread
self._db = None self._db = None
self.name = name self.name = name
self.connector_handler = getConnectorHandler(connector) master_addresses, connector_name = parseMasterList(master_nodes)
self.connector_handler = getConnectorHandler(connector_name)
self.dispatcher = Dispatcher(self.poll_thread) self.dispatcher = Dispatcher(self.poll_thread)
self.nm = NodeManager() self.nm = NodeManager()
self.cp = ConnectionPool(self) self.cp = ConnectionPool(self)
...@@ -87,7 +88,7 @@ class Application(object): ...@@ -87,7 +88,7 @@ class Application(object):
self.trying_master_node = None self.trying_master_node = None
# load master node list # load master node list
for address in parseMasterList(master_nodes): for address in master_addresses:
self.nm.createMaster(address=address) self.nm.createMaster(address=address)
# no self-assigned UUID, primary master will supply us one # no self-assigned UUID, primary master will supply us one
......
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
from ConfigParser import SafeConfigParser from ConfigParser import SafeConfigParser
from neo.lib import util from neo.lib import util
from neo.lib.util import parseNodeAddress
class ConfigurationManager(object): class ConfigurationManager(object):
""" """
...@@ -48,20 +48,13 @@ class ConfigurationManager(object): ...@@ -48,20 +48,13 @@ class ConfigurationManager(object):
def getMasters(self): def getMasters(self):
""" Get the master node list except itself """ """ Get the master node list except itself """
masters = self.__get('masters') masters = self.__get('masters')
if not masters:
return []
# lod master node list except itself # lod master node list except itself
return util.parseMasterList(masters, except_node=self.getBind()) return util.parseMasterList(masters, except_node=self.getBind())
def getBind(self): def getBind(self):
""" Get the address to bind to """ """ Get the address to bind to """
bind = self.__get('bind') bind = self.__get('bind')
if ':' in bind: return parseNodeAddress(bind, 0)
(ip, port) = bind.split(':')
else:
(ip, port) = (bind, 0)
ip = util.resolve(ip)
return (ip, int(port))
def getDatabase(self): def getDatabase(self):
return self.__get('database') return self.__get('database')
......
...@@ -22,7 +22,7 @@ import errno ...@@ -22,7 +22,7 @@ import errno
# Fill by calling registerConnectorHandler. # Fill by calling registerConnectorHandler.
# Read by calling getConnectorHandler. # Read by calling getConnectorHandler.
connector_registry = {} connector_registry = {}
DEFAULT_CONNECTOR = 'SocketConnector' DEFAULT_CONNECTOR = 'SocketConnectorIPv4'
def registerConnectorHandler(connector_handler): def registerConnectorHandler(connector_handler):
connector_registry[connector_handler.__name__] = connector_handler connector_registry[connector_handler.__name__] = connector_handler
...@@ -52,7 +52,7 @@ class SocketConnector: ...@@ -52,7 +52,7 @@ class SocketConnector:
self.is_listening = False self.is_listening = False
self.is_closed = False self.is_closed = False
if s is None: if s is None:
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket = socket.socket(self.af_type, socket.SOCK_STREAM)
else: else:
self.socket = s self.socket = s
self.socket_fd = self.socket.fileno() self.socket_fd = self.socket.fileno()
...@@ -90,7 +90,7 @@ class SocketConnector: ...@@ -90,7 +90,7 @@ class SocketConnector:
return self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) return self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
def getAddress(self): def getAddress(self):
return self.socket.getsockname() raise NotImplementedError
def getDescriptor(self): def getDescriptor(self):
# this descriptor must only be used by the event manager, where it # this descriptor must only be used by the event manager, where it
...@@ -100,9 +100,9 @@ class SocketConnector: ...@@ -100,9 +100,9 @@ class SocketConnector:
def getNewConnection(self): def getNewConnection(self):
try: try:
new_s, addr = self.socket.accept() (new_s, addr) = self._accept()
new_s = SocketConnector(new_s, accepted_from=addr) new_s = self.__class__(new_s, accepted_from=addr)
return new_s, addr return (new_s, addr)
except socket.error, (err, errmsg): except socket.error, (err, errmsg):
if err == errno.EAGAIN: if err == errno.EAGAIN:
raise ConnectorTryAgainException raise ConnectorTryAgainException
...@@ -166,7 +166,35 @@ class SocketConnector: ...@@ -166,7 +166,35 @@ class SocketConnector:
result += ' %s' % (self.remote_addr, ) result += ' %s' % (self.remote_addr, )
return result + '>' return result + '>'
registerConnectorHandler(SocketConnector) def _accept(self):
raise NotImplementedError
class SocketConnectorIPv4(SocketConnector):
" Wrapper for IPv4 sockets"
af_type = socket.AF_INET
def _accept(self):
return self.socket.accept()
def getAddress(self):
return self.socket.getsockname()
class SocketConnectorIPv6(SocketConnector):
" Wrapper for IPv6 sockets"
af_type = socket.AF_INET6
def _accept(self):
new_s, addr = self.socket.accept()
addr = (addr[0], addr[1])
return (new_s, addr)
def getAddress(self):
addr = self.socket.getsockname()
addr = (addr[0], addr[1])
return addr
registerConnectorHandler(SocketConnectorIPv4)
registerConnectorHandler(SocketConnectorIPv6)
class ConnectorException(Exception): class ConnectorException(Exception):
pass pass
......
...@@ -15,13 +15,14 @@ ...@@ -15,13 +15,14 @@
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
import socket
import sys import sys
import traceback import traceback
from types import ClassType from types import ClassType
from socket import inet_ntoa, inet_aton from socket import inet_ntoa, inet_aton
from cStringIO import StringIO from cStringIO import StringIO
from neo.lib.util import Enum, Struct from neo.lib.util import Enum, Struct, getAddressType
# The protocol version (major, minor). # The protocol version (major, minor).
PROTOCOL_VERSION = (4, 1) PROTOCOL_VERSION = (4, 1)
...@@ -391,25 +392,60 @@ class PEnum(PStructItem): ...@@ -391,25 +392,60 @@ class PEnum(PStructItem):
enum = self._enum.__class__.__name__ enum = self._enum.__class__.__name__
raise ValueError, 'Invalid code for %s enum: %r' % (enum, code) raise ValueError, 'Invalid code for %s enum: %r' % (enum, code)
class PAddressIPGeneric(PStructItem):
def __init__(self, name, format):
PStructItem.__init__(self, name, format)
def encode(self, writer, address):
host, port = address
host = socket.inet_pton(self.af_type, host)
writer(self.pack(host, port))
def decode(self, reader):
data = reader(self.size)
address = self.unpack(data)
host, port = address
host = socket.inet_ntop(self.af_type, host)
return (host, port)
class PAddressIPv4(PAddressIPGeneric):
af_type = socket.AF_INET
def __init__(self, name):
PAddressIPGeneric.__init__(self, name, '!4sH')
class PAddressIPv6(PAddressIPGeneric):
af_type = socket.AF_INET6
def __init__(self, name):
PAddressIPGeneric.__init__(self, name, '!16sH')
class PAddress(PStructItem): class PAddress(PStructItem):
""" """
An host address (IPv4 for now) An host address (IPv4/IPv6)
""" """
address_format_dict = {
socket.AF_INET: PAddressIPv4('ipv4'),
socket.AF_INET6: PAddressIPv6('ipv6'),
}
def __init__(self, name): def __init__(self, name):
PStructItem.__init__(self, name, '!4sH') PStructItem.__init__(self, name, '!L')
def _encode(self, writer, address): def _encode(self, writer, address):
if address is None: if address is None:
address = INVALID_ADDRESS address = INVALID_ADDRESS
assert len(address) == 2, address af_type = getAddressType(address)
host, port = address writer(self.pack(af_type))
host = inet_aton(host) encoder = self.address_format_dict[af_type]
writer(self.pack(host, port)) encoder.encode(writer, address)
def _decode(self, reader): def _decode(self, reader):
data = reader(self.size) af_type = self.unpack(reader(self.size))[0]
host, port = self.unpack(data) decoder = self.address_format_dict[af_type]
host = inet_ntoa(host) host, port = decoder.decode(reader)
if (host, port) == INVALID_ADDRESS: if (host, port) == INVALID_ADDRESS:
return None return None
return (host, port) return (host, port)
......
...@@ -22,6 +22,11 @@ from zlib import adler32 ...@@ -22,6 +22,11 @@ from zlib import adler32
from Queue import deque from Queue import deque
from struct import pack, unpack from struct import pack, unpack
SOCKET_CONNECTORS_DICT = {
socket.AF_INET : 'SocketConnectorIPv4',
socket.AF_INET6: 'SocketConnectorIPv6',
}
try: try:
from struct import Struct from struct import Struct
except ImportError: except ImportError:
...@@ -89,22 +94,65 @@ def resolve(hostname): ...@@ -89,22 +94,65 @@ def resolve(hostname):
return None return None
return address_list[0] return address_list[0]
def getAddressType(address):
"Return the type (IPv4 or IPv6) of an ip"
(host, port) = address
for af_type in SOCKET_CONNECTORS_DICT.keys():
try :
socket.inet_pton(af_type, host)
except:
continue
else:
break
else:
raise ValueError("Unknown type of host", host)
return af_type
def getConnectorFromAddress(address):
address_type = getAddressType(address)
return SOCKET_CONNECTORS_DICT[address_type]
def parseNodeAddress(address, port_opt=None):
if ']' in address:
(ip, port) = address.split(']')
ip = ip.lstrip('[')
port = port.lstrip(':')
if port == '':
port = port_opt
elif ':' in address:
(ip, port) = address.split(':')
ip = resolve(ip)
else:
ip = address
port = port_opt
if port is None:
raise ValueError
return (ip, int(port))
def parseMasterList(masters, except_node=None): def parseMasterList(masters, except_node=None):
if not masters: assert masters, 'At least one master must be defined'
return [] socket_connector = ''
# load master node list # load master node list
master_node_list = [] master_node_list = []
# XXX: support '/' and ' ' as separator # XXX: support '/' and ' ' as separator
masters = masters.replace('/', ' ') masters = masters.replace('/', ' ')
for node in masters.split(' '): for node in masters.split(' '):
ip_address, port = node.split(':') address = parseNodeAddress(node)
ip_address = resolve(ip_address)
address = (ip_address, int(port))
if (address != except_node): if (address != except_node):
master_node_list.append(address) master_node_list.append(address)
return tuple(master_node_list)
socket_connector_temp = getConnectorFromAddress(address)
if socket_connector == '':
socket_connector = socket_connector_temp
elif socket_connector == socket_connector_temp:
pass
else:
return TypeError, (" Wrong connector type : you're trying to use ipv6 and ipv4 simultaneously")
return tuple(master_node_list), socket_connector
class Enum(dict): class Enum(dict):
""" """
......
...@@ -45,9 +45,6 @@ class Application(object): ...@@ -45,9 +45,6 @@ class Application(object):
last_transaction = ZERO_TID last_transaction = ZERO_TID
def __init__(self, config): def __init__(self, config):
# always use default connector for now
self.connector_handler = getConnectorHandler()
# Internal attributes. # Internal attributes.
self.em = EventManager() self.em = EventManager()
self.nm = NodeManager() self.nm = NodeManager()
...@@ -57,10 +54,11 @@ class Application(object): ...@@ -57,10 +54,11 @@ class Application(object):
self.server = config.getBind() self.server = config.getBind()
self.storage_readiness = set() self.storage_readiness = set()
master_addresses, connector_name = config.getMasters()
for address in config.getMasters(): self.connector_handler = getConnectorHandler(connector_name)
self.nm.createMaster(address=address) for master_address in master_addresses :
self.nm.createMaster(address=master_address)
neo.lib.logging.debug('IP address is %s, port is %d', *(self.server)) neo.lib.logging.debug('IP address is %s, port is %d', *(self.server))
# Partition table # Partition table
......
...@@ -36,8 +36,8 @@ action_dict = { ...@@ -36,8 +36,8 @@ action_dict = {
} }
class TerminalNeoCTL(object): class TerminalNeoCTL(object):
def __init__(self, ip, port, handler): def __init__(self, address):
self.neoctl = NeoCTL(ip, port, handler) self.neoctl = NeoCTL(address)
# Utility methods (could be functions) # Utility methods (could be functions)
def asNodeState(self, value): def asNodeState(self, value):
...@@ -187,8 +187,8 @@ class TerminalNeoCTL(object): ...@@ -187,8 +187,8 @@ class TerminalNeoCTL(object):
class Application(object): class Application(object):
"""The storage node application.""" """The storage node application."""
def __init__(self, ip, port, handler): def __init__(self, address):
self.neoctl = TerminalNeoCTL(ip, port, handler) self.neoctl = TerminalNeoCTL(address)
def execute(self, args): def execute(self, args):
"""Execute the command given.""" """Execute the command given."""
......
...@@ -20,6 +20,7 @@ from neo.lib.connection import ClientConnection ...@@ -20,6 +20,7 @@ from neo.lib.connection import ClientConnection
from neo.lib.event import EventManager from neo.lib.event import EventManager
from neo.neoctl.handler import CommandEventHandler from neo.neoctl.handler import CommandEventHandler
from neo.lib.protocol import ClusterStates, NodeStates, ErrorCodes, Packets from neo.lib.protocol import ClusterStates, NodeStates, ErrorCodes, Packets
from neo.lib.util import getConnectorFromAddress
class NotReadyException(Exception): class NotReadyException(Exception):
pass pass
...@@ -29,9 +30,10 @@ class NeoCTL(object): ...@@ -29,9 +30,10 @@ class NeoCTL(object):
connection = None connection = None
connected = False connected = False
def __init__(self, ip, port, handler): def __init__(self, address):
self.connector_handler = getConnectorHandler(handler) connector_name = getConnectorFromAddress(address)
self.server = (ip, port) self.connector_handler = getConnectorHandler(connector_name)
self.server = address
self.em = EventManager() self.em = EventManager()
self.handler = CommandEventHandler(self) self.handler = CommandEventHandler(self)
self.response_queue = [] self.response_queue = []
......
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
import sys import sys
from optparse import OptionParser from optparse import OptionParser
from neo.lib import setupLog from neo.lib import setupLog
from neo.lib.util import parseNodeAddress
parser = OptionParser() parser = OptionParser()
parser.add_option('-v', '--verbose', action = 'store_true', parser.add_option('-v', '--verbose', action = 'store_true',
...@@ -29,16 +30,13 @@ parser.add_option('--handler', help = 'specify the connection handler') ...@@ -29,16 +30,13 @@ parser.add_option('--handler', help = 'specify the connection handler')
def main(args=None): def main(args=None):
(options, args) = parser.parse_args(args=args) (options, args) = parser.parse_args(args=args)
address = options.address if options.address is not None:
if ':' in address: address = parseNodeAddress(options.address, 9999)
address, port = address.split(':', 1)
port = int(port)
else: else:
port = 9999 address = ('127.0.0.1', 9999)
handler = options.handler or "SocketConnector"
setupLog('NEOCTL', options.verbose) setupLog('NEOCTL', options.verbose)
from neo.neoctl.app import Application from neo.neoctl.app import Application
print Application(address, port, handler).execute(args) print Application(address).execute(args)
...@@ -41,9 +41,6 @@ class Application(object): ...@@ -41,9 +41,6 @@ class Application(object):
"""The storage node application.""" """The storage node application."""
def __init__(self, config): def __init__(self, config):
# always use default connector for now
self.connector_handler = getConnectorHandler()
# set the cluster name # set the cluster name
self.name = config.getCluster() self.name = config.getCluster()
...@@ -54,9 +51,11 @@ class Application(object): ...@@ -54,9 +51,11 @@ class Application(object):
self.dm = buildDatabaseManager(config.getAdapter(), config.getDatabase()) self.dm = buildDatabaseManager(config.getAdapter(), config.getDatabase())
# load master nodes # load master nodes
for address in config.getMasters(): master_addresses, connector_name = config.getMasters()
self.nm.createMaster(address=address) self.connector_handler = getConnectorHandler(connector_name)
for master_address in master_addresses :
self.nm.createMaster(address=master_address)
# set the bind address # set the bind address
self.server = config.getBind() self.server = config.getBind()
neo.lib.logging.debug('IP address is %s, port is %d', *(self.server)) neo.lib.logging.debug('IP address is %s, port is %d', *(self.server))
......
...@@ -21,10 +21,13 @@ import random ...@@ -21,10 +21,13 @@ import random
import unittest import unittest
import tempfile import tempfile
import MySQLdb import MySQLdb
import socket
import neo import neo
from mock import Mock from mock import Mock
from neo.lib import protocol from neo.lib import protocol
from neo.lib.protocol import Packets from neo.lib.protocol import Packets
from neo.lib.util import getAddressType
from time import time, gmtime from time import time, gmtime
from struct import pack, unpack from struct import pack, unpack
...@@ -33,6 +36,22 @@ DB_ADMIN = os.getenv('NEO_DB_ADMIN', 'root') ...@@ -33,6 +36,22 @@ DB_ADMIN = os.getenv('NEO_DB_ADMIN', 'root')
DB_PASSWD = os.getenv('NEO_DB_PASSWD', None) DB_PASSWD = os.getenv('NEO_DB_PASSWD', None)
DB_USER = os.getenv('NEO_DB_USER', 'test') DB_USER = os.getenv('NEO_DB_USER', 'test')
IP_VERSION_FORMAT_DICT = {
socket.AF_INET: '127.0.0.1',
socket.AF_INET6: '::1',
}
ADDRESS_TYPE = socket.AF_INET
def buildUrlFromString(address):
try:
socket.inet_pton(socket.AF_INET6, address)
address = '[%s]' % address
except:
pass
return address
class NeoTestBase(unittest.TestCase): class NeoTestBase(unittest.TestCase):
def setUp(self): def setUp(self):
sys.stdout.write(' * %s ' % (self.id(), )) sys.stdout.write(' * %s ' % (self.id(), ))
...@@ -47,8 +66,10 @@ class NeoTestBase(unittest.TestCase): ...@@ -47,8 +66,10 @@ class NeoTestBase(unittest.TestCase):
class NeoUnitTestBase(NeoTestBase): class NeoUnitTestBase(NeoTestBase):
""" Base class for neo tests, implements common checks """ """ Base class for neo tests, implements common checks """
local_ip = IP_VERSION_FORMAT_DICT[ADDRESS_TYPE]
def prepareDatabase(self, number, admin=DB_ADMIN, password=DB_PASSWD, def prepareDatabase(self, number, admin=DB_ADMIN, password=DB_PASSWD,
user=DB_USER, prefix=DB_PREFIX): user=DB_USER, prefix=DB_PREFIX, address_type = ADDRESS_TYPE):
""" create empties databases """ """ create empties databases """
# SQL connection # SQL connection
connect_arg_dict = {'user': admin} connect_arg_dict = {'user': admin}
...@@ -69,11 +90,13 @@ class NeoUnitTestBase(NeoTestBase): ...@@ -69,11 +90,13 @@ class NeoUnitTestBase(NeoTestBase):
def getMasterConfiguration(self, cluster='main', master_number=2, def getMasterConfiguration(self, cluster='main', master_number=2,
replicas=2, partitions=1009, uuid=None): replicas=2, partitions=1009, uuid=None):
assert master_number >= 1 and master_number <= 10 assert master_number >= 1 and master_number <= 10
masters = [('127.0.0.1', 10010 + i) for i in xrange(master_number)] masters = ([(self.local_ip, 10010 + i)
for i in xrange(master_number)])
return Mock({ return Mock({
'getCluster': cluster, 'getCluster': cluster,
'getBind': masters[0], 'getBind': masters[0],
'getMasters': masters, 'getMasters': (masters, getAddressType((
self.local_ip, 0))),
'getReplicas': replicas, 'getReplicas': replicas,
'getPartitions': partitions, 'getPartitions': partitions,
'getUUID': uuid, 'getUUID': uuid,
...@@ -83,13 +106,15 @@ class NeoUnitTestBase(NeoTestBase): ...@@ -83,13 +106,15 @@ class NeoUnitTestBase(NeoTestBase):
index=0, prefix=DB_PREFIX, uuid=None): index=0, prefix=DB_PREFIX, uuid=None):
assert master_number >= 1 and master_number <= 10 assert master_number >= 1 and master_number <= 10
assert index >= 0 and index <= 9 assert index >= 0 and index <= 9
masters = [('127.0.0.1', 10010 + i) for i in xrange(master_number)] masters = [(buildUrlFromString(self.local_ip),
10010 + i) for i in xrange(master_number)]
database = '%s@%s%s' % (DB_USER, prefix, index) database = '%s@%s%s' % (DB_USER, prefix, index)
return Mock({ return Mock({
'getCluster': cluster, 'getCluster': cluster,
'getName': 'storage', 'getName': 'storage',
'getBind': ('127.0.0.1', 10020 + index), 'getBind': (masters[0], 10020 + index),
'getMasters': masters, 'getMasters': (masters, getAddressType((
self.local_ip, 0))),
'getDatabase': database, 'getDatabase': database,
'getUUID': uuid, 'getUUID': uuid,
'getReset': False, 'getReset': False,
......
...@@ -19,13 +19,13 @@ import unittest ...@@ -19,13 +19,13 @@ import unittest
from cPickle import dumps from cPickle import dumps
from mock import Mock, ReturnValues from mock import Mock, ReturnValues
from ZODB.POSException import StorageTransactionError, UndoError, ConflictError from ZODB.POSException import StorageTransactionError, UndoError, ConflictError
from neo.tests import NeoUnitTestBase from neo.tests import NeoUnitTestBase, buildUrlFromString, ADDRESS_TYPE
from neo.client.app import Application, RevisionIndex from neo.client.app import Application, RevisionIndex
from neo.client.exception import NEOStorageError, NEOStorageNotFoundError from neo.client.exception import NEOStorageError, NEOStorageNotFoundError
from neo.client.exception import NEOStorageDoesNotExistError from neo.client.exception import NEOStorageDoesNotExistError
from neo.lib.protocol import Packet, Packets, Errors, INVALID_TID, \ from neo.lib.protocol import Packet, Packets, Errors, INVALID_TID, \
INVALID_PARTITION INVALID_PARTITION
from neo.lib.util import makeChecksum from neo.lib.util import makeChecksum, SOCKET_CONNECTORS_DICT
import time import time
def _getMasterConnection(self): def _getMasterConnection(self):
...@@ -103,8 +103,10 @@ class ClientApplicationTests(NeoUnitTestBase): ...@@ -103,8 +103,10 @@ class ClientApplicationTests(NeoUnitTestBase):
return packet.decode() return packet.decode()
return packet return packet
def getApp(self, master_nodes='127.0.0.1:10010', name='test', def getApp(self, master_nodes=None, name='test', **kw):
connector='SocketConnector', **kw): connector = SOCKET_CONNECTORS_DICT[ADDRESS_TYPE]
if master_nodes is None:
master_nodes = '%s:10010' % buildUrlFromString(self.local_ip)
app = Application(master_nodes, name, connector, **kw) app = Application(master_nodes, name, connector, **kw)
self._to_stop_list.append(app) self._to_stop_list.append(app)
app.dispatcher = Mock({ }) app.dispatcher = Mock({ })
...@@ -999,7 +1001,7 @@ class ClientApplicationTests(NeoUnitTestBase): ...@@ -999,7 +1001,7 @@ class ClientApplicationTests(NeoUnitTestBase):
def test_askPrimary(self): def test_askPrimary(self):
""" _askPrimary is private but test it anyway """ """ _askPrimary is private but test it anyway """
app = self.getApp('') app = self.getApp()
conn = Mock() conn = Mock()
app.master_conn = conn app.master_conn = conn
app.primary_handler = Mock() app.primary_handler = Mock()
......
...@@ -30,8 +30,9 @@ import threading ...@@ -30,8 +30,9 @@ import threading
from neo.neoctl.neoctl import NeoCTL, NotReadyException from neo.neoctl.neoctl import NeoCTL, NotReadyException
from neo.lib.protocol import ClusterStates, NodeTypes, CellStates, NodeStates from neo.lib.protocol import ClusterStates, NodeTypes, CellStates, NodeStates
from neo.lib.util import dump from neo.lib.util import dump, SOCKET_CONNECTORS_DICT
from neo.tests import DB_ADMIN, DB_PASSWD, NeoTestBase from neo.tests import DB_ADMIN, DB_PASSWD, NeoTestBase, buildUrlFromString, \
ADDRESS_TYPE, IP_VERSION_FORMAT_DICT
from neo.client.Storage import Storage from neo.client.Storage import Storage
NEO_MASTER = 'neomaster' NEO_MASTER = 'neomaster'
...@@ -171,7 +172,9 @@ class NEOCluster(object): ...@@ -171,7 +172,9 @@ class NEOCluster(object):
db_super_user=DB_ADMIN, db_super_password=DB_PASSWD, db_super_user=DB_ADMIN, db_super_password=DB_PASSWD,
cleanup_on_delete=False, temp_dir=None, cleanup_on_delete=False, temp_dir=None,
clear_databases=True, adapter='MySQL', clear_databases=True, adapter='MySQL',
verbose=True): verbose=True,
address_type=ADDRESS_TYPE,
):
self.zodb_storage_list = [] self.zodb_storage_list = []
self.cleanup_on_delete = cleanup_on_delete self.cleanup_on_delete = cleanup_on_delete
self.verbose = verbose self.verbose = verbose
...@@ -181,6 +184,8 @@ class NEOCluster(object): ...@@ -181,6 +184,8 @@ class NEOCluster(object):
self.db_user = db_user self.db_user = db_user
self.db_password = db_password self.db_password = db_password
self.db_list = db_list self.db_list = db_list
self.address_type = address_type
self.local_ip = IP_VERSION_FORMAT_DICT[self.address_type]
if clear_databases: if clear_databases:
self.setupDB() self.setupDB()
self.process_dict = {} self.process_dict = {}
...@@ -192,12 +197,16 @@ class NEOCluster(object): ...@@ -192,12 +197,16 @@ class NEOCluster(object):
admin_port = self.__allocatePort() admin_port = self.__allocatePort()
self.cluster_name = 'neo_%s' % (random.randint(0, 100), ) self.cluster_name = 'neo_%s' % (random.randint(0, 100), )
master_node_list = [self.__allocatePort() for i in xrange(master_node_count)] master_node_list = [self.__allocatePort() for i in xrange(master_node_count)]
self.master_nodes = '/'.join('127.0.0.1:%s' % (x, ) for x in master_node_list) self.master_nodes = '/'.join('%s:%s' % (
buildUrlFromString(self.local_ip), x, )
for x in master_node_list)
# create admin node # create admin node
self.__newProcess(NEO_ADMIN, { self.__newProcess(NEO_ADMIN, {
'--cluster': self.cluster_name, '--cluster': self.cluster_name,
'--name': 'admin', '--name': 'admin',
'--bind': '127.0.0.1:%d' % (admin_port, ), '--bind': '%s:%d' % (buildUrlFromString(
self.local_ip), admin_port, ),
'--masters': self.master_nodes, '--masters': self.master_nodes,
}) })
# create master nodes # create master nodes
...@@ -205,7 +214,8 @@ class NEOCluster(object): ...@@ -205,7 +214,8 @@ class NEOCluster(object):
self.__newProcess(NEO_MASTER, { self.__newProcess(NEO_MASTER, {
'--cluster': self.cluster_name, '--cluster': self.cluster_name,
'--name': 'master_%d' % index, '--name': 'master_%d' % index,
'--bind': '127.0.0.1:%d' % (port, ), '--bind': '%s:%d' % (buildUrlFromString(
self.local_ip), port, ),
'--masters': self.master_nodes, '--masters': self.master_nodes,
'--replicas': replicas, '--replicas': replicas,
'--partitions': partitions, '--partitions': partitions,
...@@ -215,14 +225,16 @@ class NEOCluster(object): ...@@ -215,14 +225,16 @@ class NEOCluster(object):
self.__newProcess(NEO_STORAGE, { self.__newProcess(NEO_STORAGE, {
'--cluster': self.cluster_name, '--cluster': self.cluster_name,
'--name': 'storage_%d' % index, '--name': 'storage_%d' % index,
'--bind': '%s:%d' % (buildUrlFromString(
self.local_ip),
0 ),
'--masters': self.master_nodes, '--masters': self.master_nodes,
'--database': '%s:%s@%s' % (db_user, db_password, db), '--database': '%s:%s@%s' % (db_user, db_password, db),
'--adapter': adapter, '--adapter': adapter,
}) })
# create neoctl # create neoctl
self.neoctl = NeoCTL('127.0.0.1', admin_port,
'SocketConnector') self.neoctl = NeoCTL((self.local_ip, admin_port))
def __newProcess(self, command, arguments): def __newProcess(self, command, arguments):
uuid = self.__allocateUUID() uuid = self.__allocateUUID()
arguments['--uuid'] = uuid arguments['--uuid'] = uuid
...@@ -236,10 +248,10 @@ class NEOCluster(object): ...@@ -236,10 +248,10 @@ class NEOCluster(object):
def __allocatePort(self): def __allocatePort(self):
port_set = self.port_set port_set = self.port_set
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s = socket.socket(self.address_type, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
while True: while True:
s.bind(('127.0.0.1', 0)) s.bind((self.local_ip, 0))
port = s.getsockname()[1] port = s.getsockname()[1]
if port not in port_set: if port not in port_set:
break break
...@@ -343,12 +355,12 @@ class NEOCluster(object): ...@@ -343,12 +355,12 @@ class NEOCluster(object):
def getNEOCTL(self): def getNEOCTL(self):
return self.neoctl return self.neoctl
def getZODBStorage(self, **kw): def getZODBStorage(self,connector = SOCKET_CONNECTORS_DICT[ADDRESS_TYPE], **kw):
master_nodes = self.master_nodes.replace('/', ' ') master_nodes = self.master_nodes.replace('/', ' ')
result = Storage( result = Storage(
master_nodes=master_nodes, master_nodes=master_nodes,
name=self.cluster_name, name=self.cluster_name,
connector='SocketConnector', connector=connector,
logfile=os.path.join(self.temp_dir, 'client.log'), logfile=os.path.join(self.temp_dir, 'client.log'),
verbose=self.verbose, verbose=self.verbose,
**kw **kw
......
...@@ -19,13 +19,17 @@ import os ...@@ -19,13 +19,17 @@ import os
import unittest import unittest
import transaction import transaction
import ZODB import ZODB
import socket
from struct import pack, unpack from struct import pack, unpack
from neo.neoctl.neoctl import NeoCTL
from ZODB.FileStorage import FileStorage from ZODB.FileStorage import FileStorage
from ZODB.POSException import ConflictError from ZODB.POSException import ConflictError
from ZODB.tests.StorageTestBase import zodb_pickle from ZODB.tests.StorageTestBase import zodb_pickle
from persistent import Persistent from persistent import Persistent
from neo.lib.util import SOCKET_CONNECTORS_DICT
from neo.tests.functional import NEOCluster, NEOFunctionalTest from neo.tests.functional import NEOCluster, NEOFunctionalTest
from neo.tests import IP_VERSION_FORMAT_DICT
TREE_SIZE = 6 TREE_SIZE = 6
...@@ -263,6 +267,23 @@ class ClientTests(NEOFunctionalTest): ...@@ -263,6 +267,23 @@ class ClientTests(NEOFunctionalTest):
self.assertRaises(ConflictError, st2.tpc_vote, t2) self.assertRaises(ConflictError, st2.tpc_vote, t2)
self.runWithTimeout(40, test) self.runWithTimeout(40, test)
def testIPv6Client(self):
""" Test the connectivity of an IPv6 connection for neo client """
def test():
"""
Implement the IPv6Client test
"""
self.neo = NEOCluster(['test_neo1'], replicas=0,
temp_dir = self.getTempDirectory(),
address_type = socket.AF_INET6
)
neoctl = NeoCTL(('::1', 0))
self.neo.start()
db1, conn1 = self.neo.getZODBConnection()
db2, conn2 = self.neo.getZODBConnection()
self.runWithTimeout(40, test)
def testDelayedLocksCancelled(self): def testDelayedLocksCancelled(self):
""" """
Hold a lock on an object, try to get another lock on the same Hold a lock on an object, try to get another lock on the same
......
...@@ -20,7 +20,8 @@ from mock import Mock ...@@ -20,7 +20,8 @@ from mock import Mock
from neo.lib import protocol from neo.lib import protocol
from neo.tests import NeoUnitTestBase from neo.tests import NeoUnitTestBase
from neo.lib.protocol import Packet, NodeTypes, NodeStates from neo.lib.protocol import Packet, NodeTypes, NodeStates
from neo.master.handlers.election import ClientElectionHandler, ServerElectionHandler from neo.master.handlers.election import ClientElectionHandler, \
ServerElectionHandler
from neo.master.app import Application from neo.master.app import Application
from neo.lib.exception import ElectionFailure from neo.lib.exception import ElectionFailure
from neo.lib.connection import ClientConnection from neo.lib.connection import ClientConnection
...@@ -41,7 +42,7 @@ class MasterClientElectionTests(NeoUnitTestBase): ...@@ -41,7 +42,7 @@ class MasterClientElectionTests(NeoUnitTestBase):
self.app.pt.clear() self.app.pt.clear()
self.app.em = Mock() self.app.em = Mock()
self.app.uuid = self._makeUUID('M') self.app.uuid = self._makeUUID('M')
self.app.server = ('127.0.0.1', 10000) self.app.server = (self.local_ip, 10000)
self.app.name = 'NEOCLUSTER' self.app.name = 'NEOCLUSTER'
self.election = ClientElectionHandler(self.app) self.election = ClientElectionHandler(self.app)
self.app.unconnected_master_node_set = set() self.app.unconnected_master_node_set = set()
...@@ -207,9 +208,9 @@ class MasterServerElectionTests(NeoUnitTestBase): ...@@ -207,9 +208,9 @@ class MasterServerElectionTests(NeoUnitTestBase):
self.app.unconnected_master_node_set.add(node.getAddress()) self.app.unconnected_master_node_set.add(node.getAddress())
node.setState(NodeStates.RUNNING) node.setState(NodeStates.RUNNING)
# define some variable to simulate client and storage node # define some variable to simulate client and storage node
self.client_address = ('127.0.0.1', 1000) self.client_address = (self.local_ip, 1000)
self.storage_address = ('127.0.0.1', 2000) self.storage_address = (self.local_ip, 2000)
self.master_address = ('127.0.0.1', 3000) self.master_address = (self.local_ip, 3000)
# apply monkey patches # apply monkey patches
self._addPacket = ClientConnection._addPacket self._addPacket = ClientConnection._addPacket
ClientConnection._addPacket = _addPacket ClientConnection._addPacket = _addPacket
......
...@@ -16,9 +16,10 @@ ...@@ -16,9 +16,10 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
import unittest import unittest
import socket
from neo.lib.protocol import NodeTypes, NodeStates, CellStates, ClusterStates from neo.lib.protocol import NodeTypes, NodeStates, CellStates, ClusterStates
from neo.lib.protocol import ErrorCodes, Packets, Errors, LockState from neo.lib.protocol import ErrorCodes, Packets, Errors, LockState
from neo.tests import NeoUnitTestBase from neo.tests import NeoUnitTestBase, IP_VERSION_FORMAT_DICT
class ProtocolTests(NeoUnitTestBase): class ProtocolTests(NeoUnitTestBase):
...@@ -71,14 +72,26 @@ class ProtocolTests(NeoUnitTestBase): ...@@ -71,14 +72,26 @@ class ProtocolTests(NeoUnitTestBase):
def test_11_RequestIdentification(self): def test_11_RequestIdentification(self):
uuid = self.getNewUUID() uuid = self.getNewUUID()
p = Packets.RequestIdentification(NodeTypes.CLIENT, p = Packets.RequestIdentification(NodeTypes.CLIENT,
uuid, ("127.0.0.1", 9080), "unittest") uuid, (self.local_ip, 9080), "unittest")
(plow, phigh), node, p_uuid, (ip, port), name = p.decode() (plow, phigh), node, p_uuid, (ip, port), name = p.decode()
self.assertEqual(node, NodeTypes.CLIENT) self.assertEqual(node, NodeTypes.CLIENT)
self.assertEqual(p_uuid, uuid) self.assertEqual(p_uuid, uuid)
self.assertEqual(ip, "127.0.0.1") self.assertEqual(ip, self.local_ip)
self.assertEqual(port, 9080) self.assertEqual(port, 9080)
self.assertEqual(name, "unittest") self.assertEqual(name, "unittest")
def test_11_bis_RequestIdentification_IPv6(self):
uuid = self.getNewUUID()
self.local_ip = IP_VERSION_FORMAT_DICT[socket.AF_INET6]
p = Packets.RequestIdentification(NodeTypes.CLIENT,
uuid, (self.local_ip, 9080), "unittest")
(plow, phigh), node, p_uuid, (ip, port), name = p.decode()
self.assertEqual(node, NodeTypes.CLIENT)
self.assertEqual(p_uuid, uuid)
self.assertEqual(ip, self.local_ip)
self.assertEqual(port, 9080)
self.assertEqual(name, "unittest")
def test_12_AcceptIdentification(self): def test_12_AcceptIdentification(self):
uuid1, uuid2 = self.getNewUUID(), self.getNewUUID() uuid1, uuid2 = self.getNewUUID(), self.getNewUUID()
p = Packets.AcceptIdentification(NodeTypes.CLIENT, uuid1, p = Packets.AcceptIdentification(NodeTypes.CLIENT, uuid1,
...@@ -107,6 +120,21 @@ class ProtocolTests(NeoUnitTestBase): ...@@ -107,6 +120,21 @@ class ProtocolTests(NeoUnitTestBase):
self.assertEqual(primary_uuid, uuid) self.assertEqual(primary_uuid, uuid)
self.assertEqual(master_list, p_master_list) self.assertEqual(master_list, p_master_list)
def test_14_bis_answerPrimaryIPv6(self):
""" Try to get primary master through IPv6 """
self.address_type = socket.AF_INET6
uuid = self.getNewUUID()
uuid1 = self.getNewUUID()
uuid2 = self.getNewUUID()
uuid3 = self.getNewUUID()
master_list = [(("::1", 1), uuid1),
(("::2", 2), uuid2),
(("::3", 3), uuid3)]
p = Packets.AnswerPrimary(uuid, master_list)
primary_uuid, p_master_list = p.decode()
self.assertEqual(primary_uuid, uuid)
self.assertEqual(master_list, p_master_list)
def test_15_announcePrimary(self): def test_15_announcePrimary(self):
p = Packets.AnnouncePrimary() p = Packets.AnnouncePrimary()
self.assertEqual(p.decode(), ()) self.assertEqual(p.decode(), ())
...@@ -119,9 +147,11 @@ class ProtocolTests(NeoUnitTestBase): ...@@ -119,9 +147,11 @@ class ProtocolTests(NeoUnitTestBase):
uuid1 = self.getNewUUID() uuid1 = self.getNewUUID()
uuid2 = self.getNewUUID() uuid2 = self.getNewUUID()
uuid3 = self.getNewUUID() uuid3 = self.getNewUUID()
node_list = [(NodeTypes.CLIENT, ("127.0.0.1", 1), uuid1, NodeStates.RUNNING), node_list = \
(NodeTypes.CLIENT, ("127.0.0.2", 2), uuid2, NodeStates.DOWN), [(NodeTypes.CLIENT, ("127.0.0.1", 1), uuid1, NodeStates.RUNNING),
(NodeTypes.CLIENT, ("127.0.0.3", 3), uuid3, NodeStates.BROKEN)] (NodeTypes.CLIENT, ("127.0.0.2", 2), uuid2, NodeStates.DOWN),
(NodeTypes.CLIENT, ("127.0.0.3", 3), uuid3, NodeStates.BROKEN
)]
p = Packets.NotifyNodeInformation(node_list) p = Packets.NotifyNodeInformation(node_list)
p_node_list = p.decode()[0] p_node_list = p.decode()[0]
self.assertEqual(node_list, p_node_list) self.assertEqual(node_list, p_node_list)
...@@ -549,13 +579,22 @@ class ProtocolTests(NeoUnitTestBase): ...@@ -549,13 +579,22 @@ class ProtocolTests(NeoUnitTestBase):
self.assertEqual(p.decode(), (node_type, )) self.assertEqual(p.decode(), (node_type, ))
def test_AnswerNodeList(self): def test_AnswerNodeList(self):
node1 = (NodeTypes.CLIENT, ('127.0.0.1', 1000), node1 = (NodeTypes.CLIENT, (self.local_ip, 1000),
self.getNewUUID(), NodeStates.DOWN) self.getNewUUID(), NodeStates.DOWN)
node2 = (NodeTypes.MASTER, ('127.0.0.1', 2000), node2 = (NodeTypes.MASTER, (self.local_ip, 2000),
self.getNewUUID(), NodeStates.RUNNING) self.getNewUUID(), NodeStates.RUNNING)
p = Packets.AnswerNodeList((node1, node2)) p = Packets.AnswerNodeList((node1, node2))
self.assertEqual(p.decode(), ([node1, node2], )) self.assertEqual(p.decode(), ([node1, node2], ))
def test_AnswerNodeListIPv6(self):
self.address_type = socket.AF_INET6
node1 = (NodeTypes.CLIENT, (self.local_ip, 1000),
self.getNewUUID(), NodeStates.DOWN)
node2 = (NodeTypes.MASTER, (self.local_ip, 2000),
self.getNewUUID(), NodeStates.RUNNING)
p = Packets.AnswerNodeList((node1, node2))
self.assertEqual(p.decode(), ([node1, node2], ))
def test_AskPartitionList(self): def test_AskPartitionList(self):
min_offset = 10 min_offset = 10
max_offset = 20 max_offset = 20
......
...@@ -17,11 +17,51 @@ ...@@ -17,11 +17,51 @@
import unittest import unittest
from neo.tests import NeoUnitTestBase import socket
from neo.lib.util import ReadBuffer from neo.tests import NeoUnitTestBase, IP_VERSION_FORMAT_DICT
from neo.lib.util import ReadBuffer, getAddressType, parseNodeAddress, \
getConnectorFromAddress, SOCKET_CONNECTORS_DICT
class UtilTests(NeoUnitTestBase): class UtilTests(NeoUnitTestBase):
def test_getConnectorFromAddress(self):
""" Connector name must correspond to address type """
connector = getConnectorFromAddress((
IP_VERSION_FORMAT_DICT[socket.AF_INET], 0))
self.assertEqual(connector, SOCKET_CONNECTORS_DICT[socket.AF_INET])
connector = getConnectorFromAddress((
IP_VERSION_FORMAT_DICT[socket.AF_INET6], 0))
self.assertEqual(connector, SOCKET_CONNECTORS_DICT[socket.AF_INET6])
self.assertRaises(ValueError, getConnectorFromAddress, ('', 0))
self.assertRaises(ValueError, getConnectorFromAddress, ('test', 0))
def test_getAddressType(self):
""" Get the type on an IP Address """
self.assertRaises(ValueError, parseNodeAddress, '', 0)
address_type = getAddressType(('::1', 0))
self.assertEqual(address_type, socket.AF_INET6)
address_type = getAddressType(('0.0.0.0', 0))
self.assertEqual(address_type, socket.AF_INET)
address_type = getAddressType(('127.0.0.1', 0))
self.assertEqual(address_type, socket.AF_INET)
def test_parseNodeAddress(self):
""" Parsing of addesses """
ip_address = parseNodeAddress('127.0.0.1:0')
self.assertEqual(('127.0.0.1', 0), ip_address)
ip_address = parseNodeAddress('127.0.0.1:0', 100)
self.assertEqual(('127.0.0.1', 0), ip_address)
ip_address = parseNodeAddress('127.0.0.1', 500)
self.assertEqual(('127.0.0.1', 500), ip_address)
self.assertRaises(ValueError, parseNodeAddress, '127.0.0.1')
ip_address = parseNodeAddress('[::1]:0')
self.assertEqual(('::1', 0), ip_address)
ip_address = parseNodeAddress('[::1]:0', 100)
self.assertEqual(('::1', 0), ip_address)
ip_address = parseNodeAddress('[::1]', 500)
self.assertEqual(('::1', 500), ip_address)
self.assertRaises(ValueError, parseNodeAddress, ('[::1]'))
def testReadBufferRead(self): def testReadBufferRead(self):
""" Append some chunk then consume the data """ """ Append some chunk then consume the data """
buf = ReadBuffer() buf = ReadBuffer()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment