testing: test API on top of slapos standalone

slapos/testing/__init__.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################

slapos/testing/testcase.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import os
+import logging
+
+try:
+  import subprocess32 as subprocess
+except ImportError:
+  import subprocess
+  subprocess  # pyflakes
+
+from .utils import getPortFromPath
+
+from ..slap.standalone import StandaloneSlapOS
+from ..slap.standalone import SlapOSNodeCommandError
+from ..slap.standalone import PathTooDeepError
+
+try:
+  from typing import Iterable, Tuple, Callable, Type
+except ImportError:
+  pass
+
+
+def makeModuleSetUpAndTestCaseClass(
+    software_url,
+    base_directory=None,
+    ipv4_address=os.environ['SLAPOS_TEST_IPV4'],
+    ipv6_address=os.environ['SLAPOS_TEST_IPV6'],
+    debug=bool(int(os.environ.get('SLAPOS_TEST_DEBUG', 0))),
+    verbose=bool(int(os.environ.get('SLAPOS_TEST_VERBOSE', 0))),
+):
+  # type: (str, str, str, str, bool, bool) -> Tuple[Callable[[], None], Type[SlapOSInstanceTestCase]]
+  """Create a setup module function and a testcase for testing `software_url`.
+
+  This function returns a tuple of two arguments:
+   * a function to install the software, to be used as `unittest`'s
+     `setUpModule`
+   * a base class for test cases.
+
+  The SlapOS instance will be using ip addresses defined by
+  environment variables `SLAPOS_TEST_IPV4` and `SLAPOS_TEST_IPV6`, or by the
+  explicits `ipv4_address` and `ipv6_address` arguments.
+
+  To ease development and troubleshooting, two switches are available:
+   * `verbose` (also controlled by `SLAPOS_TEST_VERBOSE` environment variable)
+     to tell the test framework to log information describing the actions taken.
+   * `debug` (also controlled by `SLAPOS_TEST_DEBUG` environment variable) to
+     enable debugging mode which will drop in a debugger session when errors
+     occurs.
+
+  The base_directory directory is by default .slapos in current directory,
+  or a path from `SLAPOS_TEST_WORKING_DIR` environment variable.
+
+  A note about paths:
+    SlapOS itself and some services running in SlapOS uses unix sockets and
+    (sometimes very) deep paths, which does not play very well together.
+    To workaround this, users can set `SLAPOS_TEST_WORKING_DIR` environment
+    variable to the path of a short enough directory and local slapos will
+    use this directory.
+    The partitions references will be named after the unittest class name,
+    which can also lead to long paths. For this, unit test classes can define
+    a `__partition_reference__` attribute which will be used as partition
+    reference. If the class names are long, the trick is then to use a shorter
+    `__partition_reference__`.
+    See https://lab.nexedi.com/kirr/slapns for a solution to this problem.
+  """
+  if base_directory is None:
+    base_directory = os.path.realpath(
+        os.environ.get(
+            'SLAPOS_TEST_WORKING_DIR', os.path.join(os.getcwd(), '.slapos')))
+  # TODO: fail if already running ?
+  try:
+    slap = StandaloneSlapOS(
+        base_directory=base_directory,
+        server_ip=ipv4_address,
+        server_port=getPortFromPath(base_directory))
+  except PathTooDeepError:
+    raise RuntimeError(
+        'base directory ( {} ) is too deep, try setting '
+        'SLAPOS_TEST_WORKING_DIR to a shallow enough directory'.format(
+            base_directory))
+
+  cls = type(
+      'SlapOSInstanceTestCase for {}'.format(software_url),
+      (SlapOSInstanceTestCase,), {
+          'slap': slap,
+          'getSoftwareURL': classmethod(lambda _cls: software_url),
+          '_debug': debug,
+          '_verbose': verbose,
+          '_ipv4_address': ipv4_address,
+          '_ipv6_address': ipv6_address
+      })
+
+  class SlapOSInstanceTestCase_(cls, SlapOSInstanceTestCase):
+    # useless intermediate class so that editors provide completion anyway.
+    pass
+
+  def setUpModule():
+    # type: () -> None
+    if debug:
+      unittest.installHandler()
+    if verbose or debug:
+      logging.basicConfig(level=logging.DEBUG)
+    installSoftwareUrlList(cls, [software_url], debug=debug)
+
+  return setUpModule, SlapOSInstanceTestCase_
+
+
+def installSoftwareUrlList(cls, software_url_list, max_retry=2, debug=False):
+  # type: (Type[SlapOSInstanceTestCase], Iterable[str], int, bool) -> None
+  """Install softwares on the current testing slapos, for use in `setUpModule`
+  """
+  try:
+    for software_url in software_url_list:
+      cls.logger.debug("Supplying %s", software_url)
+      cls.slap.supply(software_url)
+    cls.logger.debug("Waiting for slapos node software to build")
+    cls.slap.waitForSoftware(max_retry=max_retry, debug=debug)
+  except BaseException as e:
+    if not debug:
+      cls.logger.exception("Error building software, removing")
+      try:
+        for software_url in software_url_list:
+          cls.logger.debug("Removing %s", software_url)
+          cls.slap.supply(software_url, state="destroyed")
+        cls.logger.debug("Waiting for slapos node software to remove")
+        cls.slap.waitForSoftware(max_retry=max_retry, debug=debug)
+      except BaseException:
+        cls.logger.exception("Error removing software")
+        pass
+    cls._cleanup()
+    raise e
+
+
+class SlapOSInstanceTestCase(unittest.TestCase):
+  """Install one slapos instance.
+
+  This test case install software(s) and request one instance
+  during `setUpClass` and destroy that instance during `tearDownClass`.
+
+  Software Release URL, Instance Software Type and Instance Parameters
+  can be defined on the class.
+
+  All tests from the test class will run with the same instance.
+
+  The following class attributes are available:
+
+    * `computer_partition`:  the `slapos.slap.slap.ComputerPartition`
+      computer partition instance.
+
+    * `computer_partition_root_path`: the path of the instance root
+      directory.
+
+  This class is not supposed to be imported directly, but needs to be setup by
+  calling makeModuleSetUpAndTestCaseClass.
+  """
+  # can set this to true to enable debugging utilities
+  _debug = False
+  # can set this to true to enable more verbose output
+  _verbose = False
+  # maximum retries for `slapos node instance`
+  instance_max_retry = 10
+  # maximum retries for `slapos node report`
+  report_max_retry = 0
+  # number of partitions needed for this instance
+  partition_count = 10
+  # reference of the default requested partition
+  default_partition_reference = 'testing partition 0'
+
+  # a logger for messages of the testing framework
+  logger = logging.getLogger(__name__)
+
+  # Dynamic members
+  slap = None # type: StandaloneSlapOS
+  _ipv4_address = ""
+  _ipv6_address = ""
+
+  # Methods to be defined by subclasses.
+  @classmethod
+  def getSoftwareURL(cls):
+    """Return URL of software release to request instance.
+
+    This method will be defined when initialising the class
+    with makeModuleSetUpAndTestCaseClass.
+    """
+    raise NotImplementedError()
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    """Return instance parameters.
+
+    To be defined by subclasses if they need to request instance
+    with specific parameters.
+    """
+    return {}
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    """Return software type for instance, default "".
+
+    To be defined by subclasses if they need to request instance with specific
+    software type.
+    """
+    return ""
+
+  # Unittest methods
+  @classmethod
+  def setUpClass(cls):
+    """Request an instance.
+    """
+    cls._instance_parameter_dict = cls.getInstanceParameterDict()
+
+    try:
+      cls.logger.debug("Starting")
+      cls.slap.start()
+      cls.logger.debug(
+          "Formatting to remove old partitions XXX should not be needed because we delete ..."
+      )
+      cls.slap.format(0, cls._ipv4_address, cls._ipv6_address)
+      cls.logger.debug("Formatting with %s partitions", cls.partition_count)
+      cls.slap.format(
+          cls.partition_count, cls._ipv4_address, cls._ipv6_address,
+          getattr(cls, '__partition_reference__', '{}-'.format(cls.__name__)))
+
+      # request
+      cls.requestDefaultInstance()
+
+      # slapos node instance
+      cls.logger.debug("Waiting for instance")
+      # waitForInstance does not tolerate any error but with instances,
+      # promises sometimes fail on first run, because services did not
+      # have time to start.
+      # To make debug usable, we tolerate instance_max_retry-1 errors and
+      # only debug the last.
+      if cls._debug and cls.instance_max_retry:
+        try:
+          cls.slap.waitForInstance(max_retry=cls.instance_max_retry - 1)
+        except SlapOSNodeCommandError:
+          cls.slap.waitForInstance(debug=True)
+      else:
+        cls.slap.waitForInstance(
+            max_retry=cls.instance_max_retry, debug=cls._debug)
+
+      # expose some class attributes so that tests can use them:
+      # the main ComputerPartition instance, to use getInstanceParameterDict
+      cls.computer_partition = cls.requestDefaultInstance()
+
+      # the path of the instance on the filesystem, for low level inspection
+      cls.computer_partition_root_path = os.path.join(
+          cls.slap._instance_root, cls.computer_partition.getId())
+      cls.logger.debug("setUpClass done")
+
+    except BaseException:
+      cls.logger.exception("Error during setUpClass")
+      cls._cleanup()
+      cls.setUp = lambda self: self.fail('Setup Class failed.')
+      raise
+
+  @classmethod
+  def tearDownClass(cls):
+    """Tear down class, stop the processes and destroy instance.
+    """
+    cls._cleanup()
+
+  # implementation methods
+  @classmethod
+  def _cleanup(cls):
+    """Destroy all instances and stop subsystem.
+    Catches and log all exceptions.
+    """
+    try:
+      cls.requestDefaultInstance(state='destroyed')
+    except:
+      cls.logger.exception("Error during request destruction")
+    try:
+      cls.slap.waitForReport(max_retry=cls.report_max_retry, debug=cls._debug)
+    except:
+      cls.logger.exception("Error during actual destruction")
+    leaked_partitions = [
+        cp for cp in cls.slap.computer.getComputerPartitionList()
+        if cp.getState() != 'destroyed'
+    ]
+    # XXX how about shared instances ???
+    cls.logger.critical(
+        "The following partitions were not cleaned up: %s",
+        [cp.getId() for cp in leaked_partitions])
+    for cp in leaked_partitions:
+      try:
+        cls.slap.request(
+            software_release=cp.getSoftwareRelease().getURI(),
+            # software_type=cp.getType(), # TODO
+            # XXX is this really the reference ?
+            partition_reference=cp.getInstanceParameterDict()['instance_title'],
+            state="destroyed")
+      except:
+        cls.logger.exception(
+            "Error during request destruction of leaked partition")
+    try:
+      cls.slap.waitForReport(max_retry=cls.report_max_retry, debug=cls._debug)
+    except:
+      cls.logger.exception(
+          "Error during leaked partitions actual destruction")
+    try:
+      cls.slap.stop()
+    except:
+      cls.logger.exception("Error during stop")
+
+  @classmethod
+  def requestDefaultInstance(cls, state='started'):
+    software_url = cls.getSoftwareURL()
+    software_type = cls.getInstanceSoftwareType()
+    cls.logger.debug(
+        'requesting "%s" software:%s type:%s state:%s parameters:%s',
+        cls.default_partition_reference, software_url, software_type, state,
+        cls._instance_parameter_dict)
+    return cls.slap.request(
+        software_release=software_url,
+        software_type=software_type,
+        partition_reference=cls.default_partition_reference,
+        partition_parameter_kw=cls._instance_parameter_dict,
+        state=state)

slapos/testing/utils.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import socket
+import hashlib
+from contextlib import closing
+
+
+# Utility functions
+def findFreeTCPPort(ip=''):
+  # type: (str) -> int
+  """Find a free TCP port to listen to.
+  """
+  s = socket.socket(
+      socket.AF_INET6 if ':' in ip else socket.AF_INET, socket.SOCK_STREAM)
+  with closing(s):
+    s.bind((ip, 0))
+    return s.getsockname()[1]
+
+
+def getPortFromPath(path):
+  # type: (str) -> int
+  """A stable port using a hash from path.
+  """
+  return 1024 + int(
+      hashlib.md5(path.encode('utf-8', 'backslashreplace')).hexdigest(),
+      16) % (65535 - 1024)