slapformat: WIP

slapos/format.py

@@ -32,6 +32,8 @@ import ipaddress
 import logging
 import netifaces
 import os
+import pwd
+import grp
 import subprocess
 
 from collections import defaultdict
@@ -39,6 +41,7 @@ from netifaces import AF_INET, AF_INET6
 
 
 def do_format(conf):
+  with WrappedSystem(conf):
     # load configuration
     computer = Computer(conf)
     # sanity checks
@@ -109,6 +112,8 @@ class FormatConfig(Parameters, Options):
   CHECK_FILES = ['key_file', 'cert_file', 'input_definition_file']
   NORMALIZE_PATHS = ['instance_root', 'software_root']
 
+  logger : logging.Logger
+
   def __init__(self, logger):
     self.logger = logger
 
@@ -119,7 +124,7 @@ class FormatConfig(Parameters, Options):
 
   def parse(self, name, value, t):
     if not isinstance(value, str):
-      if type(value).__name__ != t:
+      if type(value).__name__ != t and value is not None:
         self.error("Option %s takes type %s, not %r", name, t, value)
       return value
     if t == 'int':
@@ -147,6 +152,7 @@ class FormatConfig(Parameters, Options):
     self.__dict__.update(args.__dict__)
 
   def setConfig(self):
+    # Deprecated options
     for option in self.DEPRECATED:
       if option in self.__dict__:
         if option == 'computer_xml':
@@ -158,20 +164,20 @@ class FormatConfig(Parameters, Options):
           )
         else:
           self.error("Option %r is no longer supported" % option)
-
+    # Mandatory parameters
     for option, t in Parameters.__annotations__.items():
       setattr(self, option, self.parse(option, self.get(option), t))
-
-    for option, t in self.__annotations__.items():
+    # Optional inputs
+    for option, t in Options.__annotations__.items():
       setattr(self, option, self.parse(option, getattr(self, option), t))
-
+    # Existence checks for files
     for option in self.CHECK_FILES:
       path = getattr(self, option)
       if path:
         if not os.path.exists(path):
           self.error("File %r does not exist or is not readable", path)
         setattr(self, option, os.path.abspath(path))
-
+    # Paths normalization
     for option in self.NORMALIZE_PATHS:
       path = getattr(self, option)
       if path:
@@ -191,24 +197,30 @@ class Definition(defaultdict):
 
 
 class Computer(object):
+  conf : FormatConfig
   reference : str
   interface : Interface
   partitions : list[Partition]
-  address : ipaddress.IPv4Interface or ipaddress.IPv6Interface
+  address : ipaddress.IPv4Interface | ipaddress.IPv6Interface
   user : User
-  conf : FormatConfig
 
   def __init__(self, conf):
+    definition = Definition(conf.input_definition_file)
+    section = definition['computer']
+    # Basic configuration
     self.conf = conf
     self.reference = conf.computer_id
+    # Interface
     self.interface = Interface(conf)
-    definition = Definition(conf.input_definition_file)
-    computer = definition['computer']
-    addr = computer['address']
+    # Address
+    addr = section['address']
     address = ipaddress.ip_interface(addr) if addr else None
     self.address = address or self.interface.getComputerIPv6Addr()
-    username = computer['software_user'] or conf.software_user
-    self.user = User(username, conf.software_root)
+    # User
+    username = section['software_user'] or conf.software_user
+    path = conf.software_root
+    self.user = User(username, path)
+    # Partitions
     amount = conf.partition_amount
     self.partitions = [Partition(i, self, definition) for i in range(amount)]
 
@@ -216,7 +228,12 @@ class Computer(object):
     pass
 
   def format(self):
-    pass
+    # Non local bind for IPv6 ranges
+    if self.conf.ipv6_range or any(p.ipv6_range for p in self.partitions):
+      self.interface.enableIPv6NonLocalBind()
+    # Format each partitions
+    for p in self.partitions:
+      p.format()
 
   def update(self):
     pass
@@ -283,8 +300,17 @@ class Interface(object):
     addr = network[(1 << (bits - 2)) + (index << (128 - prefixlen))]
     return ipaddress.IPv6Network((addr, prefixlen))
 
+  def enableIPv6NonLocalBind(self):
+    call(['sysctl', 'net.ipv6.ip_nonlocal_bind=1'])
+    network = str(self.ipv6_network)
+    _, result = call(['ip', '-6', 'route', 'show', 'table', 'local', network])
+    if not 'dev lo' in result:
+      call(['ip', '-6', 'route', 'add', 'local', network, 'dev', 'lo'])
+
+
 
 class Partition(object):
+  interface : Interface
   reference : str
   index: int
   path : str
@@ -298,9 +324,10 @@ class Partition(object):
   def __init__(self, index, computer, definition=None):
     i = str(index)
     conf = computer.conf
-    interface = computer.interface
     section = definition['partition_' + i]
     options = defaultdict(type(None), section, **definition['default'])
+    # Interface
+    self.interface = interface = computer.interface
     # Reference, path & user
     self.reference = options['pathname'] or conf.partition_base_name + i
     self.path = os.path.join(conf.instance_root, self.reference)
@@ -324,7 +351,7 @@ class Partition(object):
     # XXX
 
   def format(self):
-    pass
+    self.createPath()
 
   def createPath(self):
     if not os.path.exists(self.path):
@@ -335,9 +362,9 @@ class Partition(object):
 
 
 class User(object):
-  name: str
-  path: str
-  groups: list[str]
+  name : str
+  path : str
+  groups : list[str]
 
   SHELL = '/bin/sh'
 
@@ -349,19 +376,19 @@ class User(object):
   def create(self):
     grpname = 'grp_' + self.name if sys.platform == 'cygwin' else self.name
     if not self.isGroupAvailable(grpname):
-      callAndRead(['groupadd', grpname])
+      call(['groupadd', grpname])
     user_parameter_list = ['-d', self.path, '-g', self.name, '-s', self.SHELL]
     if self.groups:
       user_parameter_list.extend(['-G', ','.join(self.groups), '-a'])
     user_parameter_list.append(self.name)
     if self.isUserAvailable(self.name):
       # if the user is already created and used we should not fail
-      callAndRead(['usermod'] + user_parameter_list, raise_on_error=False)
+      call(['usermod'] + user_parameter_list, raise_on_error=False)
     else:
       user_parameter_list.append('-r')
-      callAndRead(['useradd'] + user_parameter_list)
+      call(['useradd'] + user_parameter_list)
     # lock the password of user
-    callAndRead(['passwd', '-l', self.name])
+    call(['passwd', '-l', self.name])
 
   @classmethod
   def isGroupAvailable(cls, name):
@@ -381,84 +408,78 @@ class User(object):
 
 # Utilities
 
-def callAndRead(argument_list, raise_on_error=True):
+def call(args, raise_on_error=True):
   popen = subprocess.Popen(
-    argument_list,
+    args,
     stdout=subprocess.PIPE,
     stderr=subprocess.STDOUT,
     universal_newlines=True,
   )
   result = popen.communicate()[0]
   if raise_on_error and popen.returncode != 0:
-    raise ValueError('Issue while invoking %r, result was:\n%s' % (
-                     argument_list, result))
+    raise ValueError(
+      "Issue while invoking %r, result was:\n%s" % (args, result)
+    )
   return popen.returncode, result
 
 
-# Tracing
-
+# Wrapping, Tracing & Dry-running
 
-class OS(object):
-  """Wrap parts of the 'os' module to provide logging of performed actions."""
-
-  _os = os
-
-  def __init__(self, conf):
-    self._dry_run = conf.dry_run
-    self._logger = conf.logger
+class WrappedModule(object):
+  def __init__(self, conf, module, *methods):
+    self._conf = conf
+    self._module = module
     add = self._addWrapper
-    add('chown')
-    add('chmod')
-    add('makedirs')
-    add('mkdir')
+    for method in methods:
+      add(method)
+      add(method)
+      add(method)
+      add(method)
 
   def _addWrapper(self, name):
+    logger = self._conf.logger
+    if self._conf.dry_run:
       def wrapper(*args, **kw):
-      arg_list = [repr(x) for x in args] + [
-          '%s=%r' % (x, y) for x, y in six.iteritems(kw)
-      ]
-      self._logger.debug('%s(%s)' % (name, ', '.join(arg_list)))
-      if not self._dry_run:
-        getattr(self._os, name)(*args, **kw)
+        l = [repr(x) for x in args] + ['%s=%r' % item for item in kw.items()]
+        logger.debug('%s(%s)' % (name, ', '.join(l)))
+    else:
+      def wrapper(*args, **kw):
+        l = [repr(x) for x in args] + ['%s=%r' % item for item in kw.items()]
+        logger.debug('%s(%s)' % (name, ', '.join(l)))
+        return getattr(self._module, name)(*args, **kw)
     setattr(self, name, wrapper)
 
   def __getattr__(self, name):
-    return getattr(self._os, name)
-
-tracing_monkeypatch_mark = []
-def tracing_monkeypatch(conf):
-  """Substitute os module and callAndRead function with tracing wrappers."""
-
-  # This function is called again if "slapos node boot" failed.
-  # Don't wrap the logging method again, otherwise the output becomes double.
-  if tracing_monkeypatch_mark:
-    return
+    return getattr(self._module, name)
 
-  global os
-  global callAndRead
 
-  original_callAndRead = callAndRead
-
-  os = OS(conf)
+class WrappedSystem(object):
+  def __init__(self, conf):
+    self.conf = conf
+    self.call = call
+    self.os = os
+    self.pwd = pwd
+
+  def __enter__(self):
+    global call, os, pwd
+    conf = self.conf
+    os = WrappedModule(conf, os, 'chmod', 'chown', 'makedirs', 'mkdir')
+    pwd = WrappedModule(conf, pwd, 'getpwnam') # XXX return value of fake
     if conf.dry_run:
-    def dry_callAndRead(argument_list, raise_on_error=True):
+      def dry_call(args, raise_on_error=True):
+        conf.logger.debug(' '.join(args))
         return 0, ''
-    applied_callAndRead = dry_callAndRead
-
-    def fake_getpwnam(user):
-      class result(object):
-        pw_uid = 12345
-        pw_gid = 54321
-      return result
-    pwd.getpwnam = fake_getpwnam
-
+      call = dry_call
     else:
-    applied_callAndRead = original_callAndRead
-
-  def logging_callAndRead(argument_list, raise_on_error=True):
-    conf.logger.debug(' '.join(argument_list))
-    return applied_callAndRead(argument_list, raise_on_error)
-  callAndRead = logging_callAndRead
+      def tracing_call(args, raise_on_error=True):
+        conf.logger.debug(' '.join(args))
+        return self.call(args, raise_on_error)
+      call = tracing_call
+
+  def __exit__(self, *exc):
+    global call, os, pwd
+    call = self.call
+    os = self.os
+    pwd = self.pwd
+    return False
     
\ No newline at end of file
-  # Put a mark. This function was called once.
-  tracing_monkeypatch_mark.append(None)