Commit 9f1e4eef authored by Julien Muchembled's avatar Julien Muchembled

Use argparse instead of optparse

Besides the use of another module for option parsing, the main change is that
there's no more Config class that mixes configuration for different components.
Application classes now takes a simple 'dict' with parsed values.

The changes in 'neoctl' are somewhat ugly, because command-line options are not
defined on the command-line class, but this component is likely to disappear
in the future.

It remains possible to pass options via a configuration file. The code is a bit
complex but isolated in neo.lib.config

For SSL, the code may be simpler if we change for a single --ssl option that
takes 3 paths. Not done to not break compatibility. Hence, the hack with
an extra OptionList class in neo.lib.app

A new functional test tests the 'neomigrate' script, instead of just the
internal API to migrate data.
parent 56d0b764
......@@ -15,7 +15,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from neo.lib import logging
from neo.lib.app import BaseApplication
from neo.lib.app import BaseApplication, buildOptionParser
from neo.lib.connection import ListeningConnection
from neo.lib.exception import PrimaryFailure
from .handler import AdminEventHandler, MasterEventHandler, \
......@@ -25,24 +25,35 @@ from neo.lib.pt import PartitionTable
from neo.lib.protocol import ClusterStates, Errors, NodeTypes, Packets
from neo.lib.debug import register as registerLiveDebugger
@buildOptionParser
class Application(BaseApplication):
"""The storage node application."""
@classmethod
def _buildOptionParser(cls):
_ = cls.option_parser
_.description = "NEO Admin node"
cls.addCommonServerOptions('admin', '127.0.0.1:9999')
_ = _.group('admin')
_.int('u', 'uuid',
help="specify an UUID to use for this process (testing purpose)")
def __init__(self, config):
super(Application, self).__init__(
config.getSSL(), config.getDynamicMasterList())
for address in config.getMasters():
config.get('ssl'), config.get('dynamic_master_list'))
for address in config['masters']:
self.nm.createMaster(address=address)
self.name = config.getCluster()
self.server = config.getBind()
self.name = config['cluster']
self.server = config['bind']
logging.debug('IP address is %s, port is %d', *self.server)
# The partition table is initialized after getting the number of
# partitions.
self.pt = None
self.uuid = config.getUUID()
self.uuid = config.get('uuid')
self.request_handler = MasterRequestEventHandler(self)
self.master_event_handler = MasterEventHandler(self)
self.cluster_state = None
......
......@@ -14,16 +14,57 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from . import logging
from . import logging, util
from .config import OptionList
from .event import EventManager
from .node import NodeManager
def buildOptionParser(cls):
parser = cls.option_parser = cls.OptionList()
_ = parser.path
_('l', 'logfile',
help="log debugging information to specified SQLite DB")
_('ca', help="(SSL) certificate authority in PEM format")
_('cert', help="(SSL) certificate in PEM format")
_('key', help="(SSL) private key in PEM format")
cls._buildOptionParser()
return cls
class BaseApplication(object):
class OptionList(OptionList):
def parse(self, argv=None):
config = OptionList.parse(self, argv)
ssl = (
config.pop('ca', None),
config.pop('cert', None),
config.pop('key', None),
)
if any(ssl):
config['ssl'] = ssl
return config
server = None
ssl = None
@classmethod
def addCommonServerOptions(cls, section, bind, masters='127.0.0.1:10000'):
_ = cls.option_parser.group('server node')
_.path('f', 'file', help='specify a configuration file')
_('s', 'section', default=section,
help='specify a configuration section')
_('c', 'cluster', required=True, help='the cluster name')
_('m', 'masters', default=masters, parse=util.parseMasterList,
help='master node list')
_('b', 'bind', default=bind,
parse=lambda x: util.parseNodeAddress(x, 0),
help='the local address to bind to')
_.path('D', 'dynamic-master-list',
help='path of the file containing dynamic master node list')
def __init__(self, ssl=None, dynamic_master_list=None):
if ssl:
if not all(ssl):
......
......@@ -14,139 +14,187 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
from optparse import OptionParser
from ConfigParser import SafeConfigParser, NoOptionError
from . import util
from .util import parseNodeAddress
def getOptionParser():
parser = OptionParser()
parser.add_option('-l', '--logfile',
help='log debugging information to specified SQLite DB')
parser.add_option('--ca', help='certificate authority in PEM format')
parser.add_option('--cert', help='certificate in PEM format')
parser.add_option('--key', help='private key in PEM format')
return parser
def getServerOptionParser():
parser = getOptionParser()
parser.add_option('-f', '--file', help='specify a configuration file')
parser.add_option('-s', '--section', help='specify a configuration section')
parser.add_option('-c', '--cluster', help='the cluster name')
parser.add_option('-m', '--masters', help='master node list')
parser.add_option('-b', '--bind', help='the local address to bind to')
parser.add_option('-D', '--dynamic-master-list',
help='path of the file containing dynamic master node list')
return parser
class ConfigurationManager(object):
"""
Configuration manager that load options from a configuration file and
command line arguments
"""
def __init__(self, defaults, options, section):
self.argument_list = options = {k: v
for k, v in options.__dict__.iteritems()
if v is not None}
self.defaults = defaults
config_file = options.pop('file', None)
if config_file:
self.parser = SafeConfigParser(defaults)
self.parser.read(config_file)
else:
self.parser = None
self.section = options.pop('section', section)
def __get(self, key, optional=False):
value = self.argument_list.get(key)
if value is None:
if self.parser is None:
value = self.defaults.get(key)
else:
try:
value = self.parser.get(self.section, key)
except NoOptionError:
pass
if value is None and not optional:
raise RuntimeError("Option '%s' is undefined'" % (key, ))
return value
def __getPath(self, *args, **kw):
path = self.__get(*args, **kw)
if path:
return os.path.expanduser(path)
def getLogfile(self):
return self.__getPath('logfile', True)
def getSSL(self):
r = [self.__getPath(key, True) for key in ('ca', 'cert', 'key')]
if any(r):
return r
def getMasters(self):
""" Get the master node list except itself """
return util.parseMasterList(self.__get('masters'))
def getBind(self):
""" Get the address to bind to """
bind = self.__get('bind')
return parseNodeAddress(bind, 0)
def getDisableDropPartitions(self):
return self.__get('disable_drop_partitions', True)
import argparse, os, sys
from functools import wraps
from ConfigParser import SafeConfigParser
def getDatabase(self):
return self.__get('database')
def getEngine(self):
return self.__get('engine', True)
class _Required(object):
def getWait(self):
# XXX: see also DatabaseManager.__init__
return self.__get('wait')
def __init__(self, *args):
self._option_list, self._name = args
def getDynamicMasterList(self):
return self.__getPath('dynamic_master_list', optional=True)
def __nonzero__(self):
with_required = self._option_list._with_required
return with_required is not None and self._name not in with_required
def getAdapter(self):
return self.__get('adapter')
class _Option(object):
def getCluster(self):
cluster = self.__get('cluster')
assert cluster != '', "Cluster name must be non-empty"
return cluster
def getReplicas(self):
return int(self.__get('replicas'))
def getPartitions(self):
return int(self.__get('partitions'))
def getReset(self):
# only from command line
return self.argument_list.get('reset', False)
def getUUID(self):
# only from command line
uuid = self.argument_list.get('uuid', None)
if uuid:
return int(uuid)
def __init__(self, *args, **kw):
if len(args) > 1:
self.short, self.name = args
else:
self.name, = args
self.__dict__.update(kw)
def _asArgparse(self, parser, option_list):
kw = self._argument_kw()
args = ['--' + self.name]
try:
args.insert(0, '-' + self.short)
except AttributeError:
pass
kw['help'] = self.help
action = parser.add_argument(*args, **kw)
if action.required:
assert not hasattr(self, 'default')
action.required = _Required(option_list, self.name)
def fromConfigFile(self, cfg, section):
return self(cfg.get(section, self.name.replace('-', '_')))
@staticmethod
def parse(value):
return value
def getUpstreamCluster(self):
return self.__get('upstream_cluster', True)
class BoolOption(_Option):
def getUpstreamMasters(self):
return util.parseMasterList(self.__get('upstream_masters'))
def _argument_kw(self):
return {'action': 'store_true'}
def getAutostart(self):
n = self.__get('autostart', True)
if n:
return int(n)
def __call__(self, value):
return value
def getDedup(self):
return self.__get('dedup', True)
def fromConfigFile(self, cfg, section):
return cfg.getboolean(section, self.name)
class Option(_Option):
@property
def __name__(self):
return self.type.__name__
def _argument_kw(self):
kw = {'type': self}
for x in 'default', 'metavar', 'required', 'choices':
try:
kw[x] = getattr(self, x)
except AttributeError:
pass
return kw
@staticmethod
def type(value):
if value:
return value
raise argparse.ArgumentTypeError('value is empty')
def __call__(self, value):
return self.type(value)
class OptionGroup(object):
def __init__(self, description=None):
self.description = description
self._options = []
def _asArgparse(self, parser, option_list):
g = parser.add_argument_group(self.description)
for option in self._options:
option._asArgparse(g, option_list)
def set_defaults(self, **kw):
option_dict = self.getOptionDict()
for k, v in kw.iteritems():
option_dict[k].default = v
def getOptionDict(self):
option_dict = {}
for option in self._options:
if isinstance(option, OptionGroup):
option_dict.update(option.getOptionDict())
else:
option_dict[option.name.replace('-', '_')] = option
return option_dict
def __call__(self, *args, **kw):
self._options.append(Option(*args, **kw))
def __option_type(t):
return wraps(t)(lambda self, *args, **kw: self(type=t, *args, **kw))
float = __option_type(float)
int = __option_type(int)
path = __option_type(os.path.expanduser)
def bool(self, *args, **kw):
self._options.append(BoolOption(*args, **kw))
class Argument(Option):
def __init__(self, name, **kw):
super(Argument, self).__init__(name, **kw)
def _asArgparse(self, parser, option_list):
kw = {'help': self.help, 'type': self}
for x in 'default', 'metavar', 'nargs', 'choices':
try:
kw[x] = getattr(self, x)
except AttributeError:
pass
parser.add_argument(self.name, **kw)
class OptionList(OptionGroup):
_with_required = None
def argument(self, *args, **kw):
self._options.append(Argument(*args, **kw))
def group(self, description):
group = OptionGroup(description)
self._options.append(group)
return group
def parse(self, argv=None):
parser = argparse.ArgumentParser(description=self.description,
argument_default=argparse.SUPPRESS,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
for option in self._options:
option._asArgparse(parser, self)
_format_help = parser.format_help
def format_help():
self._with_required = ()
try:
return _format_help()
finally:
del self._with_required
parser.format_help = format_help
if argv is None:
argv = sys.argv[1:]
args = parser.parse_args(argv)
option_dict = self.getOptionDict()
try:
config_file = args.file
except AttributeError:
d = ()
else:
cfg = SafeConfigParser()
cfg.read(config_file)
section = args.section
d = {}
for name in cfg.options(section):
try:
option = option_dict[name]
except KeyError:
continue
d[name] = option.fromConfigFile(cfg, section)
parser.set_defaults(**d)
self._with_required = d
try:
args = parser.parse_args(argv)
finally:
del self._with_required
return {name: option.parse(getattr(args, name))
for name, option in option_dict.iteritems()
if hasattr(args, name)}
......@@ -18,8 +18,8 @@ import sys
from collections import defaultdict
from time import time
from neo.lib import logging
from neo.lib.app import BaseApplication
from neo.lib import logging, util
from neo.lib.app import BaseApplication, buildOptionParser
from neo.lib.debug import register as registerLiveDebugger
from neo.lib.protocol import uuid_str, UUID_NAMESPACES, ZERO_TID
from neo.lib.protocol import ClusterStates, NodeStates, NodeTypes, Packets
......@@ -47,6 +47,7 @@ from .transactions import TransactionManager
from .verification import VerificationManager
@buildOptionParser
class Application(BaseApplication):
"""The master node application."""
packing = None
......@@ -72,26 +73,47 @@ class Application(BaseApplication):
if self.primary and self.cluster_state == ClusterStates.RECOVERING:
return self.primary
@classmethod
def _buildOptionParser(cls):
_ = cls.option_parser
_.description = "NEO Master node"
cls.addCommonServerOptions('master', '127.0.0.1:10000', '')
_ = _.group('master')
_.int('r', 'replicas', default=0, help="replicas number")
_.int('p', 'partitions', default=100, help="partitions number")
_.int('A', 'autostart',
help="minimum number of pending storage nodes to automatically"
" start new cluster (to avoid unwanted recreation of the"
" cluster, this should be the total number of storage nodes)")
_('C', 'upstream-cluster',
help='the name of cluster to backup')
_('M', 'upstream-masters', parse=util.parseMasterList,
help='list of master nodes in the cluster to backup')
_.int('u', 'uuid',
help="specify an UUID to use for this process (testing purpose)")
def __init__(self, config):
super(Application, self).__init__(
config.getSSL(), config.getDynamicMasterList())
config.get('ssl'), config.get('dynamic_master_list'))
self.tm = TransactionManager(self.onTransactionCommitted)
self.name = config.getCluster()
self.server = config.getBind()
self.autostart = config.getAutostart()
self.name = config['cluster']
self.server = config['bind']
self.autostart = config.get('autostart')
self.storage_ready_dict = {}
self.storage_starting_set = set()
for master_address in config.getMasters():
for master_address in config['masters']:
self.nm.createMaster(address=master_address)
self._node = self.nm.createMaster(address=self.server,
uuid=config.getUUID())
uuid=config.get('uuid'))
logging.debug('IP address is %s, port is %d', *self.server)
# Partition table
replicas, partitions = config.getReplicas(), config.getPartitions()
replicas = config['replicas']
partitions = config['partitions']
if replicas < 0:
raise RuntimeError, 'replicas must be a positive integer'
if partitions <= 0:
......@@ -107,13 +129,13 @@ class Application(BaseApplication):
self._current_manager = None
# backup
upstream_cluster = config.getUpstreamCluster()
upstream_cluster = config.get('upstream_cluster')
if upstream_cluster:
if upstream_cluster == self.name:
raise ValueError("upstream cluster name must be"
" different from cluster name")
self.backup_app = BackupApplication(self, upstream_cluster,
config.getUpstreamMasters())
config['upstream_masters'])
self.administration_handler = administration.AdministrationHandler(
self)
......
......@@ -14,6 +14,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
from .neoctl import NeoCTL, NotReadyException
from neo.lib.util import p64, u64, tidFromTime, timeStringFromTID
from neo.lib.protocol import uuid_str, formatNodeList, \
......@@ -266,18 +267,18 @@ class Application(object):
# state (RUNNING, DOWN...) and modify the partition if asked
# set cluster name [shutdown|operational] : either shutdown the
# cluster or mark it as operational
if not args:
return self.usage()
current_action = action_dict
level = 0
while current_action is not None and \
level < len(args) and \
isinstance(current_action, dict):
current_action = current_action.get(args[level])
level += 1
action = None
if isinstance(current_action, basestring):
action = getattr(self.neoctl, current_action, None)
if action is None:
return self.usage('unknown command')
try:
while level < len(args) and \
isinstance(current_action, dict):
current_action = current_action[args[level]]
level += 1
except KeyError:
sys.exit('invalid command: ' + ' '.join(args))
action = getattr(self.neoctl, current_action)
try:
return action(args[level:])
except NotReadyException, message:
......@@ -312,8 +313,8 @@ class Application(object):
for x in docstring_line_list])
return '\n'.join(result)
def usage(self, message):
output_list = (message, 'Available commands:', self._usage(action_dict),
def usage(self):
output_list = ('Available commands:', self._usage(action_dict),
"TID arguments can be either integers or timestamps as floats,"
" e.g. '257684787499560686', '0x3937af2eeeeeeee' or '1325421296.'"
" for 2012-01-01 12:34:56 UTC")
......
......@@ -14,7 +14,9 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from neo.lib.app import BaseApplication
import argparse
from neo.lib import util
from neo.lib.app import BaseApplication, buildOptionParser
from neo.lib.connection import ClientConnection, ConnectionClosed
from neo.lib.protocol import ClusterStates, NodeStates, ErrorCodes, Packets
from .handler import CommandEventHandler
......@@ -22,11 +24,24 @@ from .handler import CommandEventHandler
class NotReadyException(Exception):
pass
@buildOptionParser
class NeoCTL(BaseApplication):
connection = None
connected = False
@classmethod
def _buildOptionParser(cls):
# XXX: Use argparse sub-commands.
parser = cls.option_parser
parser.description = "NEO Control node"
parser('a', 'address', default='127.0.0.1:9999',
parse=lambda x: util.parseNodeAddress(x, 9999),
help="address of an admin node")
parser.argument('cmd', nargs=argparse.REMAINDER,
help="command to execute; if not supplied,"
" the list of available commands is displayed")
def __init__(self, address, **kw):
super(NeoCTL, self).__init__(**kw)
self.server = self.nm.createAdmin(address=address)
......
......@@ -18,27 +18,15 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from neo.lib import logging
from neo.lib.config import getServerOptionParser, ConfigurationManager
parser = getServerOptionParser()
parser.add_option('-u', '--uuid', help='specify an UUID to use for this ' \
'process')
defaults = dict(
bind = '127.0.0.1:9999',
masters = '127.0.0.1:10000',
)
def main(args=None):
# build configuration dict from command line options
(options, args) = parser.parse_args(args=args)
config = ConfigurationManager(defaults, options, 'admin')
from neo.admin.app import Application
config = Application.option_parser.parse(args)
# setup custom logging
logging.setup(config.getLogfile())
logging.setup(config.get('logfile'))
# and then, load and run the application
from neo.admin.app import Application
app = Application(config)
app.run()
......@@ -18,30 +18,22 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from neo.lib import logging
from neo.lib.config import getOptionParser
from neo.lib.util import parseNodeAddress
parser = getOptionParser()
parser.add_option('-a', '--address', help = 'specify the address (ip:port) ' \
'of an admin node', default = '127.0.0.1:9999')
def main(args=None):
(options, args) = parser.parse_args(args=args)
if options.address is not None:
address = parseNodeAddress(options.address, 9999)
else:
address = ('127.0.0.1', 9999)
from neo.neoctl.neoctl import NeoCTL
config = NeoCTL.option_parser.parse(args)
if options.logfile:
logfile = config.get('logfile')
if logfile:
# Contrary to daemons, we log everything to disk automatically
# because a user using -l option here:
# - is certainly debugging an issue and wants everything,
# - would not have to time to send SIGRTMIN before neoctl exits.
logging.backlog(None)
logging.setup(options.logfile)
from neo.neoctl.app import Application
logging.setup(logfile)
ssl = options.ca, options.cert, options.key
r = Application(address, ssl=ssl if any(ssl) else None).execute(args)
from neo.neoctl.app import Application
app = Application(config['address'], ssl=config.get('ssl'))
r = app.execute(config['cmd'])
if r is not None:
print r
......@@ -18,38 +18,14 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from neo.lib import logging
from neo.lib.config import getServerOptionParser, ConfigurationManager
parser = getServerOptionParser()
parser.add_option('-u', '--uuid', help='the node UUID (testing purpose)')
parser.add_option('-r', '--replicas', help = 'replicas number')
parser.add_option('-p', '--partitions', help = 'partitions number')
parser.add_option('-A', '--autostart',
help='minimum number of pending storage nodes to automatically start'
' new cluster (to avoid unwanted recreation of the cluster,'
' this should be the total number of storage nodes)')
parser.add_option('-C', '--upstream-cluster',
help='the name of cluster to backup')
parser.add_option('-M', '--upstream-masters',
help='list of master nodes in cluster to backup')
defaults = dict(
bind = '127.0.0.1:10000',
masters = '',
replicas = 0,
partitions = 100,
)
def main(args=None):
# build configuration dict from command line options
(options, args) = parser.parse_args(args=args)
config = ConfigurationManager(defaults, options, 'master')
from neo.master.app import Application
config = Application.option_parser.parse(args)
# setup custom logging
logging.setup(config.getLogfile())
logging.setup(config.get('logfile'))
# and then, load and run the application
from neo.master.app import Application
app = Application(config)
app.run()
......@@ -17,51 +17,62 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from neo.lib.config import getOptionParser
from __future__ import print_function
import time
import os
from neo.lib.app import buildOptionParser