Commit 296c10f3 authored by Tomáš Peterka's avatar Tomáš Peterka

Production-ready modifications

- configurable manager list via slapos.cfg
- XML serialization now ommits all private fields
parent b8105d89
...@@ -63,6 +63,10 @@ from slapos import version ...@@ -63,6 +63,10 @@ from slapos import version
logger = logging.getLogger("slapos.format") logger = logging.getLogger("slapos.format")
# dict[str: ManagerClass] used in configuration and XML dump of computer
# this dictionary is intended to be filled after each definition of a Manager
available_managers = {}
def prettify_xml(xml): def prettify_xml(xml):
root = lxml.etree.fromstring(xml) root = lxml.etree.fromstring(xml)
return lxml.etree.tostring(root, pretty_print=True) return lxml.etree.tostring(root, pretty_print=True)
...@@ -233,20 +237,31 @@ def _getDict(obj): ...@@ -233,20 +237,31 @@ def _getDict(obj):
key: _getDict(value) \ key: _getDict(value) \
for key, value in dikt.iteritems() \ for key, value in dikt.iteritems() \
# do not attempt to serialize logger: it is both useless and recursive. # do not attempt to serialize logger: it is both useless and recursive.
if not isinstance(value, logging.Logger) # do not serialize attributes starting with "_", let the classes have some privacy
if not key.startswith("_")
} }
class CGroupManager(object): class Manager(object):
short_name = None
def format(self):
raise UnimplementedError("Implement function to format underlaying OS")
def update(self):
raise UnimplementedError("Implement function to format underlaying OS")
Please register or sign in to reply
class CGroupManager(Manager):
"""Manage cgroups in terms on initializing and runtime operations. """Manage cgroups in terms on initializing and runtime operations.
This class takes advantage of slapformat being run periodically thus This class takes advantage of slapformat being run periodically thus
it can act as a "daemon" performing runtime tasks. it can act as a "daemon" performing runtime tasks.
""" """
short_name = 'cgroup'
cpu_exclusive_file = ".slapos-cpu-exclusive" cpu_exclusive_file = ".slapos-cpu-exclusive"
cpuset_path = "/sys/fs/cgroup/cpuset/" cpuset_path = "/sys/fs/cgroup/cpuset/"
task_write_mode = "wt" task_write_mode = "at"
def __init__(self, computer): def __init__(self, computer):
"""Extract necessary information from the ``computer``. """Extract necessary information from the ``computer``.
...@@ -255,8 +270,13 @@ class CGroupManager(object): ...@@ -255,8 +270,13 @@ class CGroupManager(object):
""" """
self.instance_root = computer.instance_root self.instance_root = computer.instance_root
self.software_gid = computer.software_gid self.software_gid = computer.software_gid
logger.info("Allowing " + self.__class__.__name__)
def allowed(self): def __str__(self):
"""Manager representation when dumped to string."""
return self.short_name
def is_allowed(self):
return os.path.exists("/sys/fs/cgroup/cpuset/cpuset.cpus") return os.path.exists("/sys/fs/cgroup/cpuset/cpuset.cpus")
def format(self): def format(self):
...@@ -270,6 +290,8 @@ class CGroupManager(object): ...@@ -270,6 +290,8 @@ class CGroupManager(object):
def update(self): def update(self):
"""Control runtime state of the computer.""" """Control runtime state of the computer."""
if os.path.exists(os.path.join(self.cpuset_path, "cpu0")):
# proceed only whe CPUSETs were formatted by this manager
self.prepare_cpu_space() self.prepare_cpu_space()
self.ensure_exlusive_cpu() self.ensure_exlusive_cpu()
...@@ -327,11 +349,21 @@ class CGroupManager(object): ...@@ -327,11 +349,21 @@ class CGroupManager(object):
def prepare_cpu_space(self): def prepare_cpu_space(self):
"""Move all PIDs from the pool of all CPUs into the first exclusive CPU.""" """Move all PIDs from the pool of all CPUs into the first exclusive CPU."""
with open(os.path.join(self.cpuset_path, "tasks"), "rt") as fi: with open(os.path.join(self.cpuset_path, "tasks"), "rt") as fi:
running_set = set(map(int, fi.read().split())) running_list = sorted(list(map(int, fi.read().split())), reverse=True)
first_cpu = self._cpu_list()[0] first_cpu = self._cpu_list()[0]
for pid in running_set: success_set = set()
refused_set = set()
for pid in running_list:
try:
self._move_task(pid, first_cpu) self._move_task(pid, first_cpu)
success_set.add(pid)
time.sleep(0.01) time.sleep(0.01)
except IOError as e:
refused_set.add(pid)
logger.debug("Refused to move {:d} PIDs: {!s}\n"
"Suceeded in moving {:d} PIDs {!s}\n".format(
len(refused_set), refused_set, len(success_set), success_set)
)
def _cpu_list(self): def _cpu_list(self):
"""Extract IDs of available CPUs and return them as a list. """Extract IDs of available CPUs and return them as a list.
...@@ -386,6 +418,9 @@ class CGroupManager(object): ...@@ -386,6 +418,9 @@ class CGroupManager(object):
pass # touche pass # touche
return folder return folder
# mark manager available
available_managers[CGroupManager.short_name] = CGroupManager
class Computer(object): class Computer(object):
"""Object representing the computer""" """Object representing the computer"""
...@@ -394,7 +429,7 @@ class Computer(object): ...@@ -394,7 +429,7 @@ class Computer(object):
ipv6_interface=None, software_user='slapsoft', ipv6_interface=None, software_user='slapsoft',
tap_gateway_interface=None, tap_gateway_interface=None,
instance_root=None, software_root=None, instance_storage_home=None, instance_root=None, software_root=None, instance_storage_home=None,
partition_list=None, manager_list=None): partition_list=None, managers=None):
""" """
Attributes: Attributes:
reference: str, the reference of the computer. reference: str, the reference of the computer.
...@@ -423,8 +458,11 @@ class Computer(object): ...@@ -423,8 +458,11 @@ class Computer(object):
self.slapos_version = None self.slapos_version = None
# HASA relation to managers (could turn into plugins with `format` and `update` methods) # HASA relation to managers (could turn into plugins with `format` and `update` methods)
self.manager_list = [manager(self) for manager in manager_list if manager(self).allowed()] \ self.managers = managers # for serialization
if manager_list else tuple() # hide list[Manager] from serializer by prepending "_"
self._manager_list = tuple(filter(lambda manager: manager.is_allowed(),
(available_managers[manager_str](self) for manager_str in managers))) \
if managers else tuple()
def __getinitargs__(self): def __getinitargs__(self):
return (self.reference, self.interface) return (self.reference, self.interface)
...@@ -463,7 +501,7 @@ class Computer(object): ...@@ -463,7 +501,7 @@ class Computer(object):
def update(self): def update(self):
"""Update computer runtime info and state.""" """Update computer runtime info and state."""
for manager in self.manager_list: for manager in self._manager_list:
manager.update() manager.update()
# Collect environmental hardware/network information. # Collect environmental hardware/network information.
...@@ -553,7 +591,7 @@ class Computer(object): ...@@ -553,7 +591,7 @@ class Computer(object):
@classmethod @classmethod
def load(cls, path_to_xml, reference, ipv6_interface, tap_gateway_interface, def load(cls, path_to_xml, reference, ipv6_interface, tap_gateway_interface,
instance_root=None, software_root=None): instance_root=None, software_root=None, managers=None):
""" """
Create a computer object from a valid xml file. Create a computer object from a valid xml file.
...@@ -564,8 +602,8 @@ class Computer(object): ...@@ -564,8 +602,8 @@ class Computer(object):
Return: Return:
A Computer object. A Computer object.
""" """
with open(path_to_xml, "rb") as fi:
dumped_dict = xml_marshaller.xml_marshaller.loads(open(path_to_xml).read()) dumped_dict = xml_marshaller.xml_marshaller.load(fi)
# Reconstructing the computer object from the xml # Reconstructing the computer object from the xml
computer = Computer( computer = Computer(
...@@ -577,6 +615,7 @@ class Computer(object): ...@@ -577,6 +615,7 @@ class Computer(object):
tap_gateway_interface=tap_gateway_interface, tap_gateway_interface=tap_gateway_interface,
software_root=dumped_dict.get('software_root', software_root), software_root=dumped_dict.get('software_root', software_root),
instance_root=dumped_dict.get('instance_root', instance_root), instance_root=dumped_dict.get('instance_root', instance_root),
managers=dumped_dict.get('managers', managers),
) )
for i, partition_dict in enumerate(dumped_dict['partition_list']): for i, partition_dict in enumerate(dumped_dict['partition_list']):
...@@ -698,7 +737,8 @@ class Computer(object): ...@@ -698,7 +737,8 @@ class Computer(object):
os.chmod(self.software_root, 0o755) os.chmod(self.software_root, 0o755)
# Iterate over all managers and let them `format` the computer too # Iterate over all managers and let them `format` the computer too
for manager in self.manager_list: for manager in self._manager_list:
logger.info("Formatting with " + manager.__class__.__name__)
manager.format() manager.format()
# get list of instance external storage if exist # get list of instance external storage if exist
...@@ -1134,7 +1174,7 @@ class Interface(object): ...@@ -1134,7 +1174,7 @@ class Interface(object):
name: String, the name of the interface name: String, the name of the interface
""" """
self.logger = logger self._logger = logger
self.name = str(name) self.name = str(name)
self.ipv4_local_network = ipv4_local_network self.ipv4_local_network = ipv4_local_network
self.ipv6_interface = ipv6_interface self.ipv6_interface = ipv6_interface
...@@ -1318,7 +1358,7 @@ class Interface(object): ...@@ -1318,7 +1358,7 @@ class Interface(object):
if self._addSystemAddress(addr, netmask, False): if self._addSystemAddress(addr, netmask, False):
return dict(addr=addr, netmask=netmask) return dict(addr=addr, netmask=netmask)
else: else:
self.logger.warning('Impossible to add old local IPv4 %s. Generating ' self._logger.warning('Impossible to add old local IPv4 %s. Generating '
'new IPv4 address.' % addr) 'new IPv4 address.' % addr)
return self._generateRandomIPv4Address(netmask) return self._generateRandomIPv4Address(netmask)
else: else:
...@@ -1376,7 +1416,7 @@ class Interface(object): ...@@ -1376,7 +1416,7 @@ class Interface(object):
# succeed, return it # succeed, return it
return dict(addr=addr, netmask=netmask) return dict(addr=addr, netmask=netmask)
else: else:
self.logger.warning('Impossible to add old public IPv6 %s. ' self._logger.warning('Impossible to add old public IPv6 %s. '
'Generating new IPv6 address.' % addr) 'Generating new IPv6 address.' % addr)
# Try 10 times to add address, raise in case if not possible # Try 10 times to add address, raise in case if not possible
...@@ -1456,7 +1496,10 @@ def parse_computer_xml(conf, xml_path): ...@@ -1456,7 +1496,10 @@ def parse_computer_xml(conf, xml_path):
computer = Computer.load(xml_path, computer = Computer.load(xml_path,
reference=conf.computer_id, reference=conf.computer_id,
ipv6_interface=conf.ipv6_interface, ipv6_interface=conf.ipv6_interface,
tap_gateway_interface=conf.tap_gateway_interface) tap_gateway_interface=conf.tap_gateway_interface,
software_root=conf.software_root,
instance_root=conf.instance_root,
managers=conf.managers)
# Connect to the interface defined by the configuration # Connect to the interface defined by the configuration
computer.interface = interface computer.interface = interface
else: else:
...@@ -1464,12 +1507,15 @@ def parse_computer_xml(conf, xml_path): ...@@ -1464,12 +1507,15 @@ def parse_computer_xml(conf, xml_path):
conf.logger.warning('Creating new computer data with id %r', conf.computer_id) conf.logger.warning('Creating new computer data with id %r', conf.computer_id)
computer = Computer( computer = Computer(
reference=conf.computer_id, reference=conf.computer_id,
software_root=conf.software_root,
instance_root=conf.software_root,
interface=interface, interface=interface,
addr=None, addr=None,
netmask=None, netmask=None,
ipv6_interface=conf.ipv6_interface, ipv6_interface=conf.ipv6_interface,
software_user=conf.software_user, software_user=conf.software_user,
tap_gateway_interface=conf.tap_gateway_interface, tap_gateway_interface=conf.tap_gateway_interface,
managers=conf.managers,
) )
partition_amount = int(conf.partition_amount) partition_amount = int(conf.partition_amount)
...@@ -1540,8 +1586,6 @@ def do_format(conf): ...@@ -1540,8 +1586,6 @@ def do_format(conf):
# no definition file, figure out computer # no definition file, figure out computer
computer = parse_computer_xml(conf, conf.computer_xml) computer = parse_computer_xml(conf, conf.computer_xml)
computer.instance_root = conf.instance_root
computer.software_root = conf.software_root
computer.instance_storage_home = conf.instance_storage_home computer.instance_storage_home = conf.instance_storage_home
conf.logger.info('Updating computer') conf.logger.info('Updating computer')
address = computer.getAddress(conf.create_tap) address = computer.getAddress(conf.create_tap)
...@@ -1586,6 +1630,7 @@ class FormatConfig(object): ...@@ -1586,6 +1630,7 @@ class FormatConfig(object):
tap_gateway_interface = None tap_gateway_interface = None
use_unique_local_address_block = None use_unique_local_address_block = None
instance_storage_home = None instance_storage_home = None
managers = None
def __init__(self, logger): def __init__(self, logger):
self.logger = logger self.logger = logger
...@@ -1673,6 +1718,16 @@ class FormatConfig(object): ...@@ -1673,6 +1718,16 @@ class FormatConfig(object):
self.logger.error(message) self.logger.error(message)
raise UsageError(message) raise UsageError(message)
# Split str into list[str] and check availability of every manager
# Config value is expected to be strings separated by spaces or commas
managers = []
for manager in self.managers.replace(",", " ").split():
if manager not in available_managers:
raise ValueError("Unknown manager \"{}\"! Known are: {!s}".format(
manager, list(available_managers.keys())))
managers.append(manager)
self.managers = managers # replace original str with list[str] of known managers
if not self.dry_run: if not self.dry_run:
if self.alter_user: if self.alter_user:
self.checkRequiredBinary(['groupadd', 'useradd', 'usermod', ['passwd', '-h']]) self.checkRequiredBinary(['groupadd', 'useradd', 'usermod', ['passwd', '-h']])
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment