WIP: Make admin node a web-app

The goal is to get rid off the neoctl command-line tool, and to manage the
cluster via a web browser, or tools like 'wget'. Then, it will be possible to
provide an web user interface to connect to the underlying DB of any storage
node, usually a SQL client.

The design of admin app is finished:
- it's threaded like clients
- it's a WSGI app

I also hacked a HTTP API as quickly as possible to make all tests pass.

TODO:
- define a better HTTP API
- there's no UI at all yet
- remove all unused packets from the protocol (those that were only used
  between neoctl and admin node)

There's currently no UI implemented.

There are a few dead files, not deleted yet, in case that they contain a few
pieces of useful code:
 neo/neoctl/app.py
 neo/neoctl/handler.py
 neo/scripts/neoctl.py

neo/admin/handler.py

@@ -14,73 +14,18 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from neo.lib import logging, protocol
-from neo.lib.handler import EventHandler
-from neo.lib.protocol import uuid_str, Packets
+from bottle import HTTPError
+from neo.lib import logging
+from neo.lib.handler import AnswerBaseHandler, EventHandler, MTEventHandler
 from neo.lib.exception import PrimaryFailure
 
-def check_primary_master(func):
-    def wrapper(self, *args, **kw):
-        if self.app.bootstrapped:
-            return func(self, *args, **kw)
-        raise protocol.NotReadyError('Not connected to a primary master.')
-    return wrapper
-
-def forward_ask(klass):
-    return check_primary_master(lambda self, conn, *args, **kw:
-        self.app.master_conn.ask(klass(*args, **kw),
-                                 conn=conn, msg_id=conn.getPeerId()))
-
-class AdminEventHandler(EventHandler):
-    """This class deals with events for administrating cluster."""
-
-    @check_primary_master
-    def askPartitionList(self, conn, min_offset, max_offset, uuid):
-        logging.info("ask partition list from %s to %s for %s",
-                     min_offset, max_offset, uuid_str(uuid))
-        self.app.sendPartitionTable(conn, min_offset, max_offset, uuid)
-
-    @check_primary_master
-    def askNodeList(self, conn, node_type):
-        if node_type is None:
-            node_type = 'all'
-            node_filter = None
-        else:
-            node_filter = lambda n: n.getType() is node_type
-        logging.info("ask list of %s nodes", node_type)
-        node_list = self.app.nm.getList(node_filter)
-        node_information_list = [node.asTuple() for node in node_list ]
-        p = Packets.AnswerNodeList(node_information_list)
-        conn.answer(p)
-
-    @check_primary_master
-    def askClusterState(self, conn):
-        conn.answer(Packets.AnswerClusterState(self.app.cluster_state))
-
-    @check_primary_master
-    def askPrimary(self, conn):
-        master_node = self.app.master_node
-        conn.answer(Packets.AnswerPrimary(master_node.getUUID()))
-
-    askLastIDs = forward_ask(Packets.AskLastIDs)
-    askLastTransaction = forward_ask(Packets.AskLastTransaction)
-    addPendingNodes = forward_ask(Packets.AddPendingNodes)
-    tweakPartitionTable = forward_ask(Packets.TweakPartitionTable)
-    setClusterState = forward_ask(Packets.SetClusterState)
-    setNodeState = forward_ask(Packets.SetNodeState)
-    checkReplicas = forward_ask(Packets.CheckReplicas)
-
-
 class MasterEventHandler(EventHandler):
-    """ This class is just used to dispacth message to right handler"""
 
     def _connectionLost(self, conn):
-        app = self.app
-        if app.listening_conn: # if running
-            assert app.master_conn in (conn, None)
-            conn.cancelRequests("connection to master lost")
-            app.reset()
-            app.uuid = None
+        conn.cancelRequests("connection to master lost")
+        self.app.nm.getByUUID(conn.getUUID()).setUnknown()
+        if self.app.master_conn is not None:
+            assert self.app.master_conn is conn
             raise PrimaryFailure
 
     def connectionFailed(self, conn):
@@ -89,18 +34,6 @@ class MasterEventHandler(EventHandler):
     def connectionClosed(self, conn):
         self._connectionLost(conn)
 
-    def dispatch(self, conn, packet, kw={}):
-        if 'conn' in kw:
-            # expected answer
-            if packet.isResponse():
-                packet.setId(kw['msg_id'])
-                kw['conn'].answer(packet)
-            else:
-                self.app.request_handler.dispatch(conn, packet, kw)
-        else:
-            # unexpected answers and notifications
-            super(MasterEventHandler, self).dispatch(conn, packet, kw)
-
     def answerClusterState(self, conn, state):
         self.app.cluster_state = state
 
@@ -109,23 +42,22 @@ class MasterEventHandler(EventHandler):
         # implemented for factorize code (as done for bootstrap)
         logging.debug("answerNodeInformation")
 
-    def notifyPartitionChanges(self, conn, ptid, cell_list):
-        self.app.pt.update(ptid, cell_list, self.app.nm)
+    def notifyNodeInformation(self, conn, node_list):
+        self.app.nm.update(node_list)
 
     def answerPartitionTable(self, conn, ptid, row_list):
         self.app.pt.load(ptid, row_list, self.app.nm)
-        self.app.bootstrapped = True
 
-    def sendPartitionTable(self, conn, ptid, row_list):
-        if self.app.bootstrapped:
-            self.app.pt.load(ptid, row_list, self.app.nm)
+class MasterNotificationsHandler(MasterEventHandler, MTEventHandler):
 
-    def notifyClusterInformation(self, conn, cluster_state):
-        self.app.cluster_state = cluster_state
+    notifyClusterInformation = MasterEventHandler.answerClusterState.im_func
+    sendPartitionTable = MasterEventHandler.answerPartitionTable.im_func
 
-    def notifyNodeInformation(self, conn, node_list):
-        self.app.nm.update(node_list)
+    def notifyPartitionChanges(self, conn, ptid, cell_list):
+        self.app.pt.update(ptid, cell_list, self.app.nm)
 
-class MasterRequestEventHandler(EventHandler):
+class PrimaryAnswersHandler(AnswerBaseHandler):
     """ This class handle all answer from primary master node"""
-    # XXX: to be deleted ?
+
+    def protocolError(self, conn, message):
+        raise HTTPError(400, message)

neo/lib/connection.py

@@ -592,6 +592,11 @@ class ClientConnection(Connection):
         handler.connectionStarted(self)
         self._connect()
 
+    def convertToMT(self, dispatcher):
+        assert self.__class__ is ClientConnection, self
+        self.__class__ = MTClientConnection
+        self._initMT(dispatcher)
+
     def _connect(self):
         try:
             connected = self.connector.makeClientConnection()
@@ -688,11 +693,14 @@ class MTClientConnection(ClientConnection):
         return wrapper
 
     def __init__(self, *args, **kwargs):
-        self.lock = lock = RLock()
-        self.dispatcher = kwargs.pop('dispatcher')
-        with lock:
+        self._initMT(kwargs.pop('dispatcher'))
+        with self.lock:
             super(MTClientConnection, self).__init__(*args, **kwargs)
 
+    def _initMT(self, dispatcher):
+        self.lock = RLock()
+        self.dispatcher = dispatcher
+
     def ask(self, packet, timeout=CRITICAL_TIMEOUT, on_timeout=None,
             queue=None, **kw):
         with self.lock:

neo/lib/threaded_app.py

@@ -135,10 +135,16 @@ class ThreadedApplication(BaseApplication):
             handler.dispatch(conn, packet, kw)
 
     def _ask(self, conn, packet, handler=None, **kw):
-        self.setHandlerData(None)
-        queue = self._thread_container.queue
-        msg_id = conn.ask(packet, queue=queue, **kw)
-        get = queue.get
+        # The following line is more than optimization. If an admin node sends
+        # a packet that causes the master to disconnect (e.g. stop a cluster),
+        # we want at least to return the answer for this request, even if the
+        # polling thread already exited and cleared self.__dict__: returning
+        # the result of getHandlerData() would raise an AttributeError.
+        # This is tested by testShutdown (neo.tests.threaded.test.Test).
+        thread_container = self._thread_container
+        thread_container.answer = None
+        msg_id = conn.ask(packet, queue=thread_container.queue, **kw)
+        get = thread_container.queue.get
         _handlePacket = self._handlePacket
         while True:
             qconn, qpacket, kw = get(True)
@@ -152,7 +158,6 @@ class ThreadedApplication(BaseApplication):
                         raise ValueError, 'ForgottenPacket for an ' \
                             'explicitely expected packet.'
                     _handlePacket(qconn, qpacket, kw, handler)
-                    break
+                    return thread_container.answer # see above comment
             if not is_forgotten and qpacket is not None:
                 _handlePacket(qconn, qpacket, kw)
-        return self.getHandlerData()

neo/neoctl/neoctl.py

@@ -14,157 +14,100 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from neo.lib.app import BaseApplication
-from neo.lib.connection import ClientConnection
-from neo.lib.protocol import ClusterStates, NodeStates, ErrorCodes, Packets
-from .handler import CommandEventHandler
+import json, socket
+from urllib import URLopener, urlencode
+from neo.lib.protocol import CellStates, ClusterStates, NodeTypes, NodeStates, \
+    ZERO_TID
+from neo.lib.util import u64
 
 class NotReadyException(Exception):
     pass
 
-class NeoCTL(BaseApplication):
-
-    connection = None
-    connected = False
-
-    def __init__(self, address, **kw):
-        super(NeoCTL, self).__init__(**kw)
-        self.server = self.nm.createAdmin(address=address)
-        self.handler = CommandEventHandler(self)
-        self.response_queue = []
-
-    def __getConnection(self):
-        if not self.connected:
-            self.connection = ClientConnection(self, self.handler, self.server)
-            # Never delay reconnection to master. This speeds up unit tests
-            # and it should not change anything for normal use.
-            self.connection.setReconnectionNoDelay()
-            while not self.connected:
-                self.em.poll(1)
-                if self.connection is None:
-                    raise NotReadyException('not connected')
-        return self.connection
-
-    def __ask(self, packet):
-        # TODO: make thread-safe
-        connection = self.__getConnection()
-        connection.ask(packet)
-        response_queue = self.response_queue
-        assert len(response_queue) == 0
-        while self.connected:
-            self.em.poll(1)
-            if response_queue:
-                break
-        else:
-            raise NotReadyException, 'Connection closed'
-        response = response_queue.pop()
-        if response[0] == Packets.Error and \
-           response[1] == ErrorCodes.NOT_READY:
-            raise NotReadyException(response[2])
-        return response
+class NeoCTL(object):
+
+    def __init__(self, address):
+        host, port = address
+        if ":" in host:
+            host = "[%s]" % host
+        self.base_url = "http://%s:%s/" % (host, port)
+        self._open = URLopener().open
+
+    def _ask(self, path, **kw):
+        if kw:
+            path += "?" + urlencode(sorted(x for x in kw.iteritems()
+                                             if '' is not x[1] is not None))
+        try:
+            return self._open(self.base_url + path).read()
+        except IOError, e:
+            e0 = e[0]
+            if e0 == 'socket error' or e0 == 'http error' and e[1] == 503:
+                raise NotReadyException
+            raise
 
     def enableStorageList(self, uuid_list):
         """
           Put all given storage nodes in "running" state.
         """
-        packet = Packets.AddPendingNodes(uuid_list)
-        response = self.__ask(packet)
-        if response[0] != Packets.Error or response[1] != ErrorCodes.ACK:
-            raise RuntimeError(response)
-        return response[2]
+        self._ask('enableStorageList', node_list=','.join(map(str, uuid_list)))
 
     def tweakPartitionTable(self, uuid_list=()):
-        response = self.__ask(Packets.TweakPartitionTable(uuid_list))
-        if response[0] != Packets.Error or response[1] != ErrorCodes.ACK:
-            raise RuntimeError(response)
-        return response[2]
+        self._ask('tweakPartitionTable', node_list=','.join(map(str, uuid_list)))
 
     def setClusterState(self, state):
         """
           Set cluster state.
         """
-        packet = Packets.SetClusterState(state)
-        response = self.__ask(packet)
-        if response[0] != Packets.Error or response[1] != ErrorCodes.ACK:
-            raise RuntimeError(response)
-        return response[2]
-
-    def _setNodeState(self, node, state):
-        """
-          Kill node, or remove it permanently
-        """
-        response = self.__ask(Packets.SetNodeState(node, state))
-        if response[0] != Packets.Error or response[1] != ErrorCodes.ACK:
-            raise RuntimeError(response)
-        return response[2]
+        self._ask('setClusterState', state=state)
 
     def getClusterState(self):
         """
           Get cluster state.
         """
-        packet = Packets.AskClusterState()
-        response = self.__ask(packet)
-        if response[0] != Packets.AnswerClusterState:
-            raise RuntimeError(response)
-        return response[1]
-
-    def getLastIds(self):
-        response = self.__ask(Packets.AskLastIDs())
-        if response[0] != Packets.AnswerLastIDs:
-            raise RuntimeError(response)
-        return response[1:]
-
-    def getLastTransaction(self):
-        response = self.__ask(Packets.AskLastTransaction())
-        if response[0] != Packets.AnswerLastTransaction:
-            raise RuntimeError(response)
-        return response[1]
+        state = self._ask('getClusterState')
+        if state:
+            return getattr(ClusterStates, state)
 
     def getNodeList(self, node_type=None):
         """
           Get a list of nodes, filtering with given type.
         """
-        packet = Packets.AskNodeList(node_type)
-        response = self.__ask(packet)
-        if response[0] != Packets.AnswerNodeList:
-            raise RuntimeError(response)
-        return response[1] # node_list
+        node_list = json.loads(self._ask('getNodeList', node_type=node_type))
+        return ((getattr(NodeTypes, node_type), address and tuple(address),
+                 uuid, getattr(NodeStates, state))
+            for node_type, address, uuid, state in node_list)
 
     def getPartitionRowList(self, min_offset=0, max_offset=0, node=None):
         """
           Get a list of partition rows, bounded by min & max and involving
           given node.
         """
-        packet = Packets.AskPartitionList(min_offset, max_offset, node)
-        response = self.__ask(packet)
-        if response[0] != Packets.AnswerPartitionList:
-            raise RuntimeError(response)
-        return response[1:3] # ptid, row_list
+        ptid, row_list = json.loads(self._ask('getPartitionRowList',
+            min_offset=min_offset, max_offset=max_offset, node=node))
+        return ptid, [(offset, [(node, getattr(CellStates, state))
+                                for node, state in row])
+                      for offset, row in row_list]
 
     def startCluster(self):
         """
           Set cluster into "verifying" state.
         """
-        return self.setClusterState(ClusterStates.VERIFYING)
+        self._ask('startCluster')
 
     def killNode(self, node):
-        return self._setNodeState(node, NodeStates.UNKNOWN)
+        self._ask('killNode', node=node)
 
     def dropNode(self, node):
-        return self._setNodeState(node, NodeStates.DOWN)
+        self._ask('dropNode', node=node)
 
     def getPrimary(self):
         """
           Return the primary master UUID.
         """
-        packet = Packets.AskPrimary()
-        response = self.__ask(packet)
-        if response[0] != Packets.AnswerPrimary:
-            raise RuntimeError(response)
-        return response[1]
-
-    def checkReplicas(self, *args):
-        response = self.__ask(Packets.CheckReplicas(*args))
-        if response[0] != Packets.Error or response[1] != ErrorCodes.ACK:
-            raise RuntimeError(response)
-        return response[2]
+        return int(self._ask('getPrimary'))
+
+    def checkReplicas(self, partition_dict, min_tid=ZERO_TID, max_tid=None):
+        kw = {'pt': ','.join('%s:%s' % (k, '' if v is None else v)
+                             for k, v in partition_dict.iteritems())}
+        if max_tid is not None:
+            kw['max_tid'] = u64(max_tid)
+        self._ask('checkReplicas', min_tid=u64(min_tid), **kw)

neo/scripts/neoadmin.py

@@ -29,6 +29,7 @@ defaults = dict(
     masters = '127.0.0.1:10000',
 )
 
+
 def main(args=None):
     # build configuration dict from command line options
     (options, args) = parser.parse_args(args=args)
@@ -39,6 +40,5 @@ def main(args=None):
 
     # and then, load and run the application
     from neo.admin.app import Application
-    app = Application(config)
-    app.run()
-
+    host, port = config.getBind()
+    Application(config).serve(host=host, port=port)

neo/tests/functional/__init__.py

@@ -363,7 +363,7 @@ class NEOCluster(object):
                         pending_count += 1
                     if pending_count == target[0]:
                         neoctl.startCluster()
-            except (NotReadyException, RuntimeError):
+            except (NotReadyException, IOError):
                 pass
         if not pdb.wait(test, MAX_START_TIME):
             raise AssertionError('Timeout when starting cluster')

neo/tests/functional/testMaster.py

@@ -47,7 +47,7 @@ class MasterTests(NEOFunctionalTest):
                 break
         neoctl.killNode(uuid)
         self.neo.expectDead(master)
-        self.assertRaises(RuntimeError, neoctl.killNode, primary_uuid)
+        self.assertRaises(IOError, neoctl.killNode, primary_uuid)
 
     def testStoppingPrimaryWithTwoSecondaries(self):
         # Wait for masters to stabilize

neo/tests/functional/testStorage.py

@@ -173,7 +173,7 @@ class StorageTests(NEOFunctionalTest):
         self.neo.expectOudatedCells(2)
         self.neo.expectClusterRunning()
 
-        self.assertRaises(RuntimeError, self.neo.neoctl.killNode,
+        self.assertRaises(IOError, self.neo.neoctl.killNode,
             started[1].getUUID())
         started[1].stop()
         # Cluster not operational anymore. Only cells of second storage that
@@ -324,7 +324,7 @@ class StorageTests(NEOFunctionalTest):
         self.neo.expectStorageNotKnown(started[0])
         self.neo.expectAssignedCells(started[0], 0)
         self.neo.expectAssignedCells(started[1], 10)
-        self.assertRaises(RuntimeError, self.neo.neoctl.dropNode,
+        self.assertRaises(IOError, self.neo.neoctl.dropNode,
                           started[1].getUUID())
         self.neo.expectClusterRunning()
 

neo/tests/threaded/test.py

@@ -534,9 +534,8 @@ class Test(NEOThreadedTest):
             # tell admin to shutdown the cluster
             cluster.neoctl.setClusterState(ClusterStates.STOPPING)
             # all nodes except clients should exit
-            cluster.join(cluster.master_list
-                       + cluster.storage_list
-                       + cluster.admin_list)
+            cluster.join(cluster.master_list + cluster.storage_list,
+                         (cluster.admin,))
         finally:
             cluster.stop()
         cluster.reset() # reopen DB to check partition tables

setup.py

@@ -26,7 +26,7 @@ if not os.path.exists('mock.py'):
 zodb_require = ['ZODB3>=3.10', 'ZODB3<3.11dev']
 
 extras_require = {
-    'admin': [],
+    'admin': ['bottle'],
     'client': zodb_require,
     'ctl': [],
     'master': [],