Commit 296c10f3 authored by Tomáš Peterka's avatar Tomáš Peterka

Production-ready modifications

- configurable manager list via slapos.cfg
- XML serialization now ommits all private fields
parent b8105d89
......@@ -63,6 +63,10 @@ from slapos import version
logger = logging.getLogger("slapos.format")
# dict[str: ManagerClass] used in configuration and XML dump of computer
# this dictionary is intended to be filled after each definition of a Manager
available_managers = {}
def prettify_xml(xml):
root = lxml.etree.fromstring(xml)
return lxml.etree.tostring(root, pretty_print=True)
......@@ -233,20 +237,31 @@ def _getDict(obj):
key: _getDict(value) \
for key, value in dikt.iteritems() \
# do not attempt to serialize logger: it is both useless and recursive.
if not isinstance(value, logging.Logger)
# do not serialize attributes starting with "_", let the classes have some privacy
if not key.startswith("_")
}
class CGroupManager(object):
class Manager(object):
short_name = None
def format(self):
raise UnimplementedError("Implement function to format underlaying OS")
def update(self):
raise UnimplementedError("Implement function to format underlaying OS")
Please register or sign in to reply
class CGroupManager(Manager):
"""Manage cgroups in terms on initializing and runtime operations.
This class takes advantage of slapformat being run periodically thus
it can act as a "daemon" performing runtime tasks.
"""
short_name = 'cgroup'
cpu_exclusive_file = ".slapos-cpu-exclusive"
cpuset_path = "/sys/fs/cgroup/cpuset/"
task_write_mode = "wt"
task_write_mode = "at"
def __init__(self, computer):
"""Extract necessary information from the ``computer``.
......@@ -255,8 +270,13 @@ class CGroupManager(object):
"""
self.instance_root = computer.instance_root
self.software_gid = computer.software_gid
logger.info("Allowing " + self.__class__.__name__)
def allowed(self):
def __str__(self):
"""Manager representation when dumped to string."""
return self.short_name
def is_allowed(self):
return os.path.exists("/sys/fs/cgroup/cpuset/cpuset.cpus")
def format(self):
......@@ -270,6 +290,8 @@ class CGroupManager(object):
def update(self):
"""Control runtime state of the computer."""
if os.path.exists(os.path.join(self.cpuset_path, "cpu0")):
# proceed only whe CPUSETs were formatted by this manager
self.prepare_cpu_space()
self.ensure_exlusive_cpu()
......@@ -327,11 +349,21 @@ class CGroupManager(object):
def prepare_cpu_space(self):
"""Move all PIDs from the pool of all CPUs into the first exclusive CPU."""
with open(os.path.join(self.cpuset_path, "tasks"), "rt") as fi:
running_set = set(map(int, fi.read().split()))
running_list = sorted(list(map(int, fi.read().split())), reverse=True)
first_cpu = self._cpu_list()[0]
for pid in running_set:
success_set = set()
refused_set = set()
for pid in running_list:
try:
self._move_task(pid, first_cpu)
success_set.add(pid)
time.sleep(0.01)
except IOError as e:
refused_set.add(pid)
logger.debug("Refused to move {:d} PIDs: {!s}\n"
"Suceeded in moving {:d} PIDs {!s}\n".format(
len(refused_set), refused_set, len(success_set), success_set)
)
def _cpu_list(self):
"""Extract IDs of available CPUs and return them as a list.
......@@ -386,6 +418,9 @@ class CGroupManager(object):
pass # touche
return folder
# mark manager available
available_managers[CGroupManager.short_name] = CGroupManager
class Computer(object):
"""Object representing the computer"""
......@@ -394,7 +429,7 @@ class Computer(object):
ipv6_interface=None, software_user='slapsoft',
tap_gateway_interface=None,
instance_root=None, software_root=None, instance_storage_home=None,
partition_list=None, manager_list=None):
partition_list=None, managers=None):
"""
Attributes:
reference: str, the reference of the computer.
......@@ -423,8 +458,11 @@ class Computer(object):
self.slapos_version = None
# HASA relation to managers (could turn into plugins with `format` and `update` methods)
self.manager_list = [manager(self) for manager in manager_list if manager(self).allowed()] \
if manager_list else tuple()
self.managers = managers # for serialization
# hide list[Manager] from serializer by prepending "_"
self._manager_list = tuple(filter(lambda manager: manager.is_allowed(),
(available_managers[manager_str](self) for manager_str in managers))) \
if managers else tuple()
def __getinitargs__(self):
return (self.reference, self.interface)
......@@ -463,7 +501,7 @@ class Computer(object):
def update(self):
"""Update computer runtime info and state."""
for manager in self.manager_list:
for manager in self._manager_list:
manager.update()
# Collect environmental hardware/network information.
......@@ -553,7 +591,7 @@ class Computer(object):
@classmethod
def load(cls, path_to_xml, reference, ipv6_interface, tap_gateway_interface,
instance_root=None, software_root=None):
instance_root=None, software_root=None, managers=None):
"""
Create a computer object from a valid xml file.
......@@ -564,8 +602,8 @@ class Computer(object):
Return:
A Computer object.
"""
dumped_dict = xml_marshaller.xml_marshaller.loads(open(path_to_xml).read())
with open(path_to_xml, "rb") as fi:
dumped_dict = xml_marshaller.xml_marshaller.load(fi)
# Reconstructing the computer object from the xml
computer = Computer(
......@@ -577,6 +615,7 @@ class Computer(object):
tap_gateway_interface=tap_gateway_interface,
software_root=dumped_dict.get('software_root', software_root),
instance_root=dumped_dict.get('instance_root', instance_root),
managers=dumped_dict.get('managers', managers),
)
for i, partition_dict in enumerate(dumped_dict['partition_list']):
......@@ -698,7 +737,8 @@ class Computer(object):
os.chmod(self.software_root, 0o755)
# Iterate over all managers and let them `format` the computer too
for manager in self.manager_list:
for manager in self._manager_list:
logger.info("Formatting with " + manager.__class__.__name__)
manager.format()
# get list of instance external storage if exist
......@@ -1134,7 +1174,7 @@ class Interface(object):
name: String, the name of the interface
"""
self.logger = logger
self._logger = logger
self.name = str(name)
self.ipv4_local_network = ipv4_local_network
self.ipv6_interface = ipv6_interface
......@@ -1318,7 +1358,7 @@ class Interface(object):
if self._addSystemAddress(addr, netmask, False):
return dict(addr=addr, netmask=netmask)
else:
self.logger.warning('Impossible to add old local IPv4 %s. Generating '
self._logger.warning('Impossible to add old local IPv4 %s. Generating '
'new IPv4 address.' % addr)
return self._generateRandomIPv4Address(netmask)
else:
......@@ -1376,7 +1416,7 @@ class Interface(object):
# succeed, return it
return dict(addr=addr, netmask=netmask)
else:
self.logger.warning('Impossible to add old public IPv6 %s. '
self._logger.warning('Impossible to add old public IPv6 %s. '
'Generating new IPv6 address.' % addr)
# Try 10 times to add address, raise in case if not possible
......@@ -1456,7 +1496,10 @@ def parse_computer_xml(conf, xml_path):
computer = Computer.load(xml_path,
reference=conf.computer_id,
ipv6_interface=conf.ipv6_interface,
tap_gateway_interface=conf.tap_gateway_interface)
tap_gateway_interface=conf.tap_gateway_interface,
software_root=conf.software_root,
instance_root=conf.instance_root,
managers=conf.managers)
# Connect to the interface defined by the configuration
computer.interface = interface
else:
......@@ -1464,12 +1507,15 @@ def parse_computer_xml(conf, xml_path):
conf.logger.warning('Creating new computer data with id %r', conf.computer_id)
computer = Computer(
reference=conf.computer_id,
software_root=conf.software_root,
instance_root=conf.software_root,
interface=interface,
addr=None,
netmask=None,
ipv6_interface=conf.ipv6_interface,
software_user=conf.software_user,
tap_gateway_interface=conf.tap_gateway_interface,
managers=conf.managers,
)
partition_amount = int(conf.partition_amount)
......@@ -1540,8 +1586,6 @@ def do_format(conf):
# no definition file, figure out computer
computer = parse_computer_xml(conf, conf.computer_xml)
computer.instance_root = conf.instance_root
computer.software_root = conf.software_root
computer.instance_storage_home = conf.instance_storage_home
conf.logger.info('Updating computer')
address = computer.getAddress(conf.create_tap)
......@@ -1586,6 +1630,7 @@ class FormatConfig(object):
tap_gateway_interface = None
use_unique_local_address_block = None
instance_storage_home = None
managers = None
def __init__(self, logger):
self.logger = logger
......@@ -1673,6 +1718,16 @@ class FormatConfig(object):
self.logger.error(message)
raise UsageError(message)
# Split str into list[str] and check availability of every manager
# Config value is expected to be strings separated by spaces or commas
managers = []
for manager in self.managers.replace(",", " ").split():
if manager not in available_managers:
raise ValueError("Unknown manager \"{}\"! Known are: {!s}".format(
manager, list(available_managers.keys())))
managers.append(manager)
self.managers = managers # replace original str with list[str] of known managers
if not self.dry_run:
if self.alter_user:
self.checkRequiredBinary(['groupadd', 'useradd', 'usermod', ['passwd', '-h']])
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment