Commit 2b321c02 authored by Julien Muchembled's avatar Julien Muchembled

WIP: Make admin node a web-app

The goal is to get rid off the neoctl command-line tool, and to manage the
cluster via a web browser, or tools like 'wget'. Then, it will be possible to
provide an web user interface to connect to the underlying DB of any storage
node, usually a SQL client.

The design of admin app is finished:
- it's threaded like clients
- it's a WSGI app

I also hacked a HTTP API as quickly as possible to make all tests pass.

- define a better HTTP API
- there's no UI at all yet
- remove all unused packets from the protocol (those that were only used
  between neoctl and admin node)

There's currently no UI implemented.

There are a few dead files, not deleted yet, in case that they contain a few
pieces of useful code:
parent 9bd14bf2
This diff is collapsed.
......@@ -14,73 +14,18 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <>.
from neo.lib import logging, protocol
from neo.lib.handler import EventHandler
from neo.lib.protocol import uuid_str, Packets
from bottle import HTTPError
from neo.lib import logging
from neo.lib.handler import AnswerBaseHandler, EventHandler, MTEventHandler
from neo.lib.exception import PrimaryFailure
def check_primary_master(func):
def wrapper(self, *args, **kw):
return func(self, *args, **kw)
raise protocol.NotReadyError('Not connected to a primary master.')
return wrapper
def forward_ask(klass):
return check_primary_master(lambda self, conn, *args, **kw:*args, **kw),
conn=conn, msg_id=conn.getPeerId()))
class AdminEventHandler(EventHandler):
"""This class deals with events for administrating cluster."""
def askPartitionList(self, conn, min_offset, max_offset, uuid):"ask partition list from %s to %s for %s",
min_offset, max_offset, uuid_str(uuid)), min_offset, max_offset, uuid)
def askNodeList(self, conn, node_type):
if node_type is None:
node_type = 'all'
node_filter = None
node_filter = lambda n: n.getType() is node_type"ask list of %s nodes", node_type)
node_list =
node_information_list = [node.asTuple() for node in node_list ]
p = Packets.AnswerNodeList(node_information_list)
def askClusterState(self, conn):
def askPrimary(self, conn):
master_node =
askLastIDs = forward_ask(Packets.AskLastIDs)
askLastTransaction = forward_ask(Packets.AskLastTransaction)
addPendingNodes = forward_ask(Packets.AddPendingNodes)
tweakPartitionTable = forward_ask(Packets.TweakPartitionTable)
setClusterState = forward_ask(Packets.SetClusterState)
setNodeState = forward_ask(Packets.SetNodeState)
checkReplicas = forward_ask(Packets.CheckReplicas)
class MasterEventHandler(EventHandler):
""" This class is just used to dispacth message to right handler"""
def _connectionLost(self, conn):
app =
if app.listening_conn: # if running
assert app.master_conn in (conn, None)
conn.cancelRequests("connection to master lost")
app.uuid = None
conn.cancelRequests("connection to master lost")
if is not None:
assert is conn
raise PrimaryFailure
def connectionFailed(self, conn):
......@@ -89,18 +34,6 @@ class MasterEventHandler(EventHandler):
def connectionClosed(self, conn):
def dispatch(self, conn, packet, kw={}):
if 'conn' in kw:
# expected answer
if packet.isResponse():
else:, packet, kw)
# unexpected answers and notifications
super(MasterEventHandler, self).dispatch(conn, packet, kw)
def answerClusterState(self, conn, state): = state
......@@ -109,23 +42,22 @@ class MasterEventHandler(EventHandler):
# implemented for factorize code (as done for bootstrap)
def notifyPartitionChanges(self, conn, ptid, cell_list):, cell_list,
def notifyNodeInformation(self, conn, node_list):
def answerPartitionTable(self, conn, ptid, row_list):, row_list, = True
def sendPartitionTable(self, conn, ptid, row_list):
if, row_list,
class MasterNotificationsHandler(MasterEventHandler, MTEventHandler):
def notifyClusterInformation(self, conn, cluster_state): = cluster_state
notifyClusterInformation = MasterEventHandler.answerClusterState.im_func
sendPartitionTable = MasterEventHandler.answerPartitionTable.im_func
def notifyNodeInformation(self, conn, node_list):
def notifyPartitionChanges(self, conn, ptid, cell_list):, cell_list,
class MasterRequestEventHandler(EventHandler):
class PrimaryAnswersHandler(AnswerBaseHandler):
""" This class handle all answer from primary master node"""
# XXX: to be deleted ?
def protocolError(self, conn, message):
raise HTTPError(400, message)
......@@ -592,6 +592,11 @@ class ClientConnection(Connection):
def convertToMT(self, dispatcher):
assert self.__class__ is ClientConnection, self
self.__class__ = MTClientConnection
def _connect(self):
connected = self.connector.makeClientConnection()
......@@ -688,11 +693,14 @@ class MTClientConnection(ClientConnection):
return wrapper
def __init__(self, *args, **kwargs):
self.lock = lock = RLock()
self.dispatcher = kwargs.pop('dispatcher')
with lock:
with self.lock:
super(MTClientConnection, self).__init__(*args, **kwargs)
def _initMT(self, dispatcher):
self.lock = RLock()
self.dispatcher = dispatcher
def ask(self, packet, timeout=CRITICAL_TIMEOUT, on_timeout=None,
queue=None, **kw):
with self.lock:
......@@ -135,10 +135,16 @@ class ThreadedApplication(BaseApplication):
handler.dispatch(conn, packet, kw)
def _ask(self, conn, packet, handler=None, **kw):
queue = self._thread_container.queue
msg_id = conn.ask(packet, queue=queue, **kw)
get = queue.get
# The following line is more than optimization. If an admin node sends
# a packet that causes the master to disconnect (e.g. stop a cluster),
# we want at least to return the answer for this request, even if the
# polling thread already exited and cleared self.__dict__: returning
# the result of getHandlerData() would raise an AttributeError.
# This is tested by testShutdown (neo.tests.threaded.test.Test).
thread_container = self._thread_container
thread_container.answer = None
msg_id = conn.ask(packet, queue=thread_container.queue, **kw)
get = thread_container.queue.get
_handlePacket = self._handlePacket
while True:
qconn, qpacket, kw = get(True)
......@@ -152,7 +158,6 @@ class ThreadedApplication(BaseApplication):
raise ValueError, 'ForgottenPacket for an ' \
'explicitely expected packet.'
_handlePacket(qconn, qpacket, kw, handler)
return thread_container.answer # see above comment
if not is_forgotten and qpacket is not None:
_handlePacket(qconn, qpacket, kw)
return self.getHandlerData()
......@@ -14,157 +14,100 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <>.
from import BaseApplication
from neo.lib.connection import ClientConnection
from neo.lib.protocol import ClusterStates, NodeStates, ErrorCodes, Packets
from .handler import CommandEventHandler
import json, socket
from urllib import URLopener, urlencode
from neo.lib.protocol import CellStates, ClusterStates, NodeTypes, NodeStates, \
from neo.lib.util import u64
class NotReadyException(Exception):
class NeoCTL(BaseApplication):
connection = None
connected = False
def __init__(self, address, **kw):
super(NeoCTL, self).__init__(**kw)
self.server = self.nm.createAdmin(address=address)
self.handler = CommandEventHandler(self)
self.response_queue = []
def __getConnection(self):
if not self.connected:
self.connection = ClientConnection(self, self.handler, self.server)
# Never delay reconnection to master. This speeds up unit tests
# and it should not change anything for normal use.
while not self.connected:
if self.connection is None:
raise NotReadyException('not connected')
return self.connection
def __ask(self, packet):
# TODO: make thread-safe
connection = self.__getConnection()
response_queue = self.response_queue
assert len(response_queue) == 0
while self.connected:
if response_queue:
raise NotReadyException, 'Connection closed'
response = response_queue.pop()
if response[0] == Packets.Error and \
response[1] == ErrorCodes.NOT_READY:
raise NotReadyException(response[2])
return response
class NeoCTL(object):
def __init__(self, address):
host, port = address
if ":" in host:
host = "[%s]" % host
self.base_url = "http://%s:%s/" % (host, port)
self._open = URLopener().open
def _ask(self, path, **kw):
if kw:
path += "?" + urlencode(sorted(x for x in kw.iteritems()
if '' is not x[1] is not None))
return self._open(self.base_url + path).read()
except IOError, e:
e0 = e[0]
if e0 == 'socket error' or e0 == 'http error' and e[1] == 503:
raise NotReadyException
def enableStorageList(self, uuid_list):
Put all given storage nodes in "running" state.
packet = Packets.AddPendingNodes(uuid_list)
response = self.__ask(packet)
if response[0] != Packets.Error or response[1] != ErrorCodes.ACK:
raise RuntimeError(response)
return response[2]
self._ask('enableStorageList', node_list=','.join(map(str, uuid_list)))
def tweakPartitionTable(self, uuid_list=()):
response = self.__ask(Packets.TweakPartitionTable(uuid_list))
if response[0] != Packets.Error or response[1] != ErrorCodes.ACK:
raise RuntimeError(response)
return response[2]
self._ask('tweakPartitionTable', node_list=','.join(map(str, uuid_list)))
def setClusterState(self, state):
Set cluster state.
packet = Packets.SetClusterState(state)
response = self.__ask(packet)
if response[0] != Packets.Error or response[1] != ErrorCodes.ACK:
raise RuntimeError(response)
return response[2]
def _setNodeState(self, node, state):
Kill node, or remove it permanently
response = self.__ask(Packets.SetNodeState(node, state))
if response[0] != Packets.Error or response[1] != ErrorCodes.ACK:
raise RuntimeError(response)
return response[2]
self._ask('setClusterState', state=state)
def getClusterState(self):
Get cluster state.
packet = Packets.AskClusterState()
response = self.__ask(packet)
if response[0] != Packets.AnswerClusterState:
raise RuntimeError(response)
return response[1]
def getLastIds(self):
response = self.__ask(Packets.AskLastIDs())
if response[0] != Packets.AnswerLastIDs:
raise RuntimeError(response)
return response[1:]
def getLastTransaction(self):
response = self.__ask(Packets.AskLastTransaction())
if response[0] != Packets.AnswerLastTransaction:
raise RuntimeError(response)
return response[1]
state = self._ask('getClusterState')
if state:
return getattr(ClusterStates, state)
def getNodeList(self, node_type=None):
Get a list of nodes, filtering with given type.
packet = Packets.AskNodeList(node_type)
response = self.__ask(packet)
if response[0] != Packets.AnswerNodeList:
raise RuntimeError(response)
return response[1] # node_list
node_list = json.loads(self._ask('getNodeList', node_type=node_type))
return ((getattr(NodeTypes, node_type), address and tuple(address),
uuid, getattr(NodeStates, state))
for node_type, address, uuid, state in node_list)
def getPartitionRowList(self, min_offset=0, max_offset=0, node=None):
Get a list of partition rows, bounded by min & max and involving
given node.
packet = Packets.AskPartitionList(min_offset, max_offset, node)
response = self.__ask(packet)
if response[0] != Packets.AnswerPartitionList:
raise RuntimeError(response)
return response[1:3] # ptid, row_list
ptid, row_list = json.loads(self._ask('getPartitionRowList',
min_offset=min_offset, max_offset=max_offset, node=node))
return ptid, [(offset, [(node, getattr(CellStates, state))
for node, state in row])
for offset, row in row_list]
def startCluster(self):
Set cluster into "verifying" state.
return self.setClusterState(ClusterStates.VERIFYING)
def killNode(self, node):
return self._setNodeState(node, NodeStates.UNKNOWN)
self._ask('killNode', node=node)
def dropNode(self, node):
return self._setNodeState(node, NodeStates.DOWN)
self._ask('dropNode', node=node)
def getPrimary(self):
Return the primary master UUID.
packet = Packets.AskPrimary()
response = self.__ask(packet)
if response[0] != Packets.AnswerPrimary:
raise RuntimeError(response)
return response[1]
def checkReplicas(self, *args):
response = self.__ask(Packets.CheckReplicas(*args))
if response[0] != Packets.Error or response[1] != ErrorCodes.ACK:
raise RuntimeError(response)
return response[2]
return int(self._ask('getPrimary'))
def checkReplicas(self, partition_dict, min_tid=ZERO_TID, max_tid=None):
kw = {'pt': ','.join('%s:%s' % (k, '' if v is None else v)
for k, v in partition_dict.iteritems())}
if max_tid is not None:
kw['max_tid'] = u64(max_tid)
self._ask('checkReplicas', min_tid=u64(min_tid), **kw)
......@@ -29,6 +29,7 @@ defaults = dict(
masters = '',
def main(args=None):
# build configuration dict from command line options
(options, args) = parser.parse_args(args=args)
......@@ -39,6 +40,5 @@ def main(args=None):
# and then, load and run the application
from import Application
app = Application(config)
host, port = config.getBind()
Application(config).serve(host=host, port=port)
......@@ -363,7 +363,7 @@ class NEOCluster(object):
pending_count += 1
if pending_count == target[0]:
except (NotReadyException, RuntimeError):
except (NotReadyException, IOError):
if not pdb.wait(test, MAX_START_TIME):
raise AssertionError('Timeout when starting cluster')
......@@ -47,7 +47,7 @@ class MasterTests(NEOFunctionalTest):
self.assertRaises(RuntimeError, neoctl.killNode, primary_uuid)
self.assertRaises(IOError, neoctl.killNode, primary_uuid)
def testStoppingPrimaryWithTwoSecondaries(self):
# Wait for masters to stabilize
......@@ -173,7 +173,7 @@ class StorageTests(NEOFunctionalTest):
self.assertRaises(RuntimeError, self.neo.neoctl.killNode,
self.assertRaises(IOError, self.neo.neoctl.killNode,
# Cluster not operational anymore. Only cells of second storage that
......@@ -324,7 +324,7 @@ class StorageTests(NEOFunctionalTest):
self.neo.expectAssignedCells(started[0], 0)
self.neo.expectAssignedCells(started[1], 10)
self.assertRaises(RuntimeError, self.neo.neoctl.dropNode,
self.assertRaises(IOError, self.neo.neoctl.dropNode,
This diff is collapsed.
......@@ -534,9 +534,8 @@ class Test(NEOThreadedTest):
# tell admin to shutdown the cluster
# all nodes except clients should exit
+ cluster.storage_list
+ cluster.admin_list)
cluster.join(cluster.master_list + cluster.storage_list,
cluster.reset() # reopen DB to check partition tables
......@@ -26,7 +26,7 @@ if not os.path.exists(''):
zodb_require = ['ZODB3>=3.10', 'ZODB3<3.11dev']
extras_require = {
'admin': [],
'admin': ['bottle'],
'client': zodb_require,
'ctl': [],
'master': [],
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment