Commit a4846b10 by Julien Muchembled

WIP: Make admin node a web-app

The goal is to get rid off the neoctl command-line tool, and to manage the
cluster via a web browser, or tools like 'wget'. Then, it will be possible to
provide an web user interface to connect to the underlying DB of any storage
node, usually a SQL client.

The design of admin app is finished:
- it's threaded like clients
- it's a WSGI app

I also hacked a HTTP API as quickly as possible to make all tests pass.

TODO:
- SSL
- define a better HTTP API
- there's no UI at all yet
- remove all unused packets from the protocol (those that were only used
  between neoctl and admin node)

There are a few dead files, not deleted yet, in case that they contain a few
pieces of useful code:
 neo/neoctl/app.py
 neo/neoctl/handler.py
 neo/scripts/neoctl.py
1 parent 0b34a051
......@@ -14,102 +14,25 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from neo.lib import logging, protocol
from neo.lib.handler import EventHandler
from neo.lib.protocol import uuid_str, Packets
from bottle import HTTPError
from neo.lib.handler import AnswerBaseHandler, EventHandler, MTEventHandler
from neo.lib.pt import PartitionTable
from neo.lib.exception import PrimaryFailure
def check_primary_master(func):
def wrapper(self, *args, **kw):
if self.app.master_conn is not None:
return func(self, *args, **kw)
raise protocol.NotReadyError('Not connected to a primary master.')
return wrapper
def forward_ask(klass):
return check_primary_master(lambda self, conn, *args, **kw:
self.app.master_conn.ask(klass(*args, **kw),
conn=conn, msg_id=conn.getPeerId()))
class AdminEventHandler(EventHandler):
"""This class deals with events for administrating cluster."""
@check_primary_master
def askPartitionList(self, conn, min_offset, max_offset, uuid):
logging.info("ask partition list from %s to %s for %s",
min_offset, max_offset, uuid_str(uuid))
self.app.sendPartitionTable(conn, min_offset, max_offset, uuid)
@check_primary_master
def askNodeList(self, conn, node_type):
if node_type is None:
node_type = 'all'
node_filter = None
else:
node_filter = lambda n: n.getType() is node_type
logging.info("ask list of %s nodes", node_type)
node_list = self.app.nm.getList(node_filter)
node_information_list = [node.asTuple() for node in node_list ]
p = Packets.AnswerNodeList(node_information_list)
conn.answer(p)
@check_primary_master
def askClusterState(self, conn):
conn.answer(Packets.AnswerClusterState(self.app.cluster_state))
@check_primary_master
def askPrimary(self, conn):
master_node = self.app.master_node
conn.answer(Packets.AnswerPrimary(master_node.getUUID()))
@check_primary_master
def flushLog(self, conn):
self.app.master_conn.send(Packets.FlushLog())
super(AdminEventHandler, self).flushLog(conn)
askLastIDs = forward_ask(Packets.AskLastIDs)
askLastTransaction = forward_ask(Packets.AskLastTransaction)
addPendingNodes = forward_ask(Packets.AddPendingNodes)
askRecovery = forward_ask(Packets.AskRecovery)
tweakPartitionTable = forward_ask(Packets.TweakPartitionTable)
setClusterState = forward_ask(Packets.SetClusterState)
setNodeState = forward_ask(Packets.SetNodeState)
setNumReplicas = forward_ask(Packets.SetNumReplicas)
checkReplicas = forward_ask(Packets.CheckReplicas)
truncate = forward_ask(Packets.Truncate)
repair = forward_ask(Packets.Repair)
class MasterEventHandler(EventHandler):
""" This class is just used to dispatch message to right handler"""
def _connectionLost(self, conn):
app = self.app
if app.listening_conn: # if running
assert app.master_conn in (conn, None)
conn.cancelRequests("connection to master lost")
app.reset()
app.uuid = None
raise PrimaryFailure
class MasterBootstrapHandler(EventHandler):
def connectionFailed(self, conn):
self._connectionLost(conn)
raise AssertionError
def connectionClosed(self, conn):
self._connectionLost(conn)
def dispatch(self, conn, packet, kw={}):
if 'conn' in kw:
# expected answer
if packet.isResponse():
packet.setId(kw['msg_id'])
kw['conn'].answer(packet)
else:
self.app.request_handler.dispatch(conn, packet, kw)
else:
# unexpected answers and notifications
super(MasterEventHandler, self).dispatch(conn, packet, kw)
app = self.app
try:
app.__dict__.pop('pt').clear()
except KeyError:
pass
if app.master_conn is not None:
assert app.master_conn is conn
raise PrimaryFailure
def answerClusterState(self, conn, state):
self.app.cluster_state = state
......@@ -124,7 +47,26 @@ class MasterEventHandler(EventHandler):
def notifyClusterInformation(self, conn, cluster_state):
self.app.cluster_state = cluster_state
class MasterEventHandler(MasterBootstrapHandler, MTEventHandler):
pass
class MasterRequestEventHandler(EventHandler):
class PrimaryAnswersHandler(AnswerBaseHandler):
""" This class handle all answer from primary master node"""
# XXX: to be deleted ?
def ack(self, conn, message):
super(PrimaryAnswersHandler, self).ack(conn, message)
self.app.setHandlerData(message)
def denied(self, conn, message):
raise HTTPError(405, message)
def protocolError(self, conn, message):
raise HTTPError(500, message)
def answerClusterState(self, conn, state):
self.app.cluster_state = state
answerRecovery = \
answerTweakPartitionTable = \
lambda self, conn, *args: self.app.setHandlerData(args)
......@@ -607,6 +607,14 @@ class ClientConnection(Connection):
handler.connectionStarted(self)
self._connect()
def convertToMT(self, dispatcher):
# XXX: The bootstrap code of the client should at least been moved to
# threaded_app so that the admin can reuse it. Ideally, we'd like
# to also merge with BootstrapManager.
assert self.__class__ is ClientConnection, self
self.__class__ = MTClientConnection
self._initMT(dispatcher)
def _connect(self):
try:
connected = self.connector.makeClientConnection()
......@@ -701,11 +709,14 @@ class MTClientConnection(ClientConnection):
return func(*args, **kw)
return wrapper
def __init__(self, *args, **kwargs):
self.lock = lock = RLock()
self.dispatcher = kwargs.pop('dispatcher')
with lock:
super(MTClientConnection, self).__init__(*args, **kwargs)
def __init__(self, *args, **kw):
self._initMT(kw.pop('dispatcher'))
with self.lock:
super(MTClientConnection, self).__init__(*args, **kw)
def _initMT(self, dispatcher):
self.lock = RLock()
self.dispatcher = dispatcher
# Alias without lock (cheaper than super())
_ask = ClientConnection.ask.__func__
......
......@@ -72,7 +72,9 @@ class ThreadedApplication(BaseApplication):
logging.debug('Stopping %s', self.poll_thread)
self.em.wakeup(thread.exit)
else:
super(ThreadedApplication, self).close()
self._close()
_close = BaseApplication.close.__func__
def start(self):
self.poll_thread.is_alive() or self.poll_thread.start()
......@@ -82,7 +84,7 @@ class ThreadedApplication(BaseApplication):
try:
self._run()
finally:
super(ThreadedApplication, self).close()
self._close()
logging.debug("Poll thread stopped")
def _run(self):
......@@ -135,8 +137,15 @@ class ThreadedApplication(BaseApplication):
handler.dispatch(conn, packet, kw)
def _ask(self, conn, packet, handler=None, **kw):
self.setHandlerData(None)
queue = self._thread_container.queue
# The following line is more than optimization. If an admin node sends
# a packet that causes the master to disconnect (e.g. stop a cluster),
# we want at least to return the answer for this request, even if the
# polling thread already exited and cleared self.__dict__: returning
# the result of getHandlerData() would raise an AttributeError.
# This is tested by testShutdown (neo.tests.threaded.test.Test).
thread_container = self._thread_container
thread_container.answer = None
queue = thread_container.queue
msg_id = conn.ask(packet, queue=queue, **kw)
get = queue.get
_handlePacket = self._handlePacket
......@@ -144,6 +153,5 @@ class ThreadedApplication(BaseApplication):
qconn, qpacket, kw = get(True)
if conn is qconn and msg_id == qpacket.getId():
_handlePacket(qconn, qpacket, kw, handler)
break
return thread_container.answer # see above comment
_handlePacket(qconn, qpacket, kw)
return self.getHandlerData()
......@@ -19,6 +19,7 @@
from neo.lib import logging
def main(args=None):
from neo.admin.app import Application
config = Application.option_parser.parse(args)
......@@ -28,5 +29,5 @@ def main(args=None):
# and then, load and run the application
app = Application(config)
app.run()
app.serve()
......@@ -702,14 +702,16 @@ class NEOCluster(object):
def expectStorageUnknown(self, process, *args, **kw):
process_uuid = process.getUUID()
def expected_storage_not_known(last_try):
for storage in self.getStorageList():
if storage[2] == process_uuid:
return False, storage
try:
for storage in self.getStorageList():
if storage[2] == process_uuid:
return False, storage
except NotReadyException:
return False, None
return True, None
self.expectCondition(expected_storage_not_known, *args, **kw)
def __del__(self):
self.neoctl.close()
if self.cleanup_on_delete:
os.removedirs(self.temp_dir)
......
......@@ -852,9 +852,8 @@ class Test(NEOThreadedTest):
t1.commit()
self.assertRaises(ConnectionClosed, t2.join)
# all nodes except clients should exit
cluster.join(cluster.master_list
+ cluster.storage_list
+ cluster.admin_list)
cluster.join(cluster.master_list + cluster.storage_list,
(cluster.admin,))
cluster.stop() # stop and reopen DB to check partition tables
cluster.start()
pt = cluster.admin.pt
......@@ -2753,8 +2752,8 @@ class Test(NEOThreadedTest):
@with_cluster(start_cluster=0, master_count=2)
def testIdentifyUnknownMaster(self, cluster):
m0, m1 = cluster.master_list
cluster.master_nodes = ()
m0.resetNode()
with Patch(cluster, master_nodes=()):
m0.resetNode()
cluster.start(master_list=(m0,))
m1.start()
self.tic()
......
......@@ -47,7 +47,7 @@ get3rdParty(x, '3rdparty/' + x, 'https://lab.nexedi.com/nexedi/erp5'
zodb_require = ['ZODB3>=3.10dev']
extras_require = {
'admin': [],
'admin': ['bottle'],
'client': zodb_require,
'ctl': [],
'master': [],
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!