slapos/testing: introduce managed resources

In SlapOS software release tests we often have to start a process or store
temporary files in a directory for the lifetime of the test. Cleaning up the
resources is a bit error prone and source of code duplication.

This introduce a registry of resources that are automatically created and freed
by the test framework, to simplify tests.

slapos/testing/testcase.py

@@ -46,6 +46,7 @@ except ImportError:
   subprocess  # pyflakes
 
 from .utils import getPortFromPath
+from .utils import ManagedResource
 
 from ..slap.standalone import StandaloneSlapOS
 from ..slap.standalone import SlapOSNodeCommandError
@@ -54,7 +55,8 @@ from ..grid.utils import md5digest
 from ..util import mkdir_p
 
 try:
-  from typing import Iterable, Tuple, Callable, Type, Dict, List, Optional
+  from typing import Iterable, Tuple, Callable, Type, Dict, List, Optional, TypeVar
+  ManagedResourceType = TypeVar("ManagedResourceType", bound=ManagedResource)
 except ImportError:
   pass
 
@@ -484,6 +486,11 @@ class SlapOSInstanceTestCase(unittest.TestCase):
   _ipv4_address = ""
   _ipv6_address = ""
 
+  _resources = {} # type: Dict[str, ManagedResource]
+  _instance_parameter_dict = None # type: Dict
+  computer_partition = None # type: ComputerPartition
+  computer_partition_root_path = None # type: str
+
   # a short name of that software URL.
   # eg. helloworld instead of
   # https://lab.nexedi.com/nexedi/slapos/raw/software/helloworld/software.cfg
@@ -503,6 +510,29 @@ class SlapOSInstanceTestCase(unittest.TestCase):
       'etc/',
   )
 
+  @classmethod
+  def getManagedResource(cls, resource_name, resource_class):
+    # type: (str, Type[ManagedResourceType]) -> ManagedResourceType
+    """Get the managed resource for this name.
+
+    If resouce was not created yet, it is created and `open`. The
+    resource will automatically be `close` at the end of the test
+    class.
+    """
+    try:
+      existing_resource = cls._resources[resource_name]
+    except KeyError:
+      resource = resource_class(cls, resource_name)
+      cls._resources[resource_name] = resource
+      resource.open()
+      return resource
+    else:
+      if not isinstance(existing_resource, resource_class):
+        raise ValueError(
+            "Resource %s is of unexpected class %s" %
+            (resource_name, existing_resource), )
+      return existing_resource
+
   # Methods to be defined by subclasses.
   @classmethod
   def getSoftwareURL(cls):
@@ -692,6 +722,12 @@ class SlapOSInstanceTestCase(unittest.TestCase):
     """Destroy all instances and stop subsystem.
     Catches and log all exceptions and take snapshot named `snapshot_name` + the failing step.
     """
+    for resource_name in list(cls._resources):
+      cls.logger.debug("closing resource %s", resource_name)
+      try:
+        cls._resources.pop(resource_name).close()
+      except:
+        cls.logger.exception("Error closing resource %s", resource_name)
     try:
       if hasattr(cls, '_instance_parameter_dict'):
         cls.requestDefaultInstance(state='destroyed')

slapos/testing/utils.py

@@ -30,15 +30,21 @@ import socket
 import hashlib
 import unittest
 import os
+import multiprocessing
+import shutil
 import subprocess
+import tempfile
 import sys
 import json
 from contextlib import closing
+from six.moves import BaseHTTPServer
+from six.moves import urllib_parse
 
 try:
   import typing
   if typing.TYPE_CHECKING:
     from PIL import Image # pylint:disable=unused-import
+    from .testcase import SlapOSInstanceTestCase
 except ImportError:
   pass
 
@@ -84,6 +90,108 @@ print(json.dumps(extra_config_dict))
   return json.loads(extra_config_dict_json)
 
 
+class ManagedResource(object):
+  """A resource that will be available for the lifetime of test.
+  """
+  def __init__(self, slapos_instance_testcase_class, resource_name):
+    # type: (typing.Type[SlapOSInstanceTestCase], str) -> None
+    self._cls = slapos_instance_testcase_class
+    self._name = resource_name
+  def open(self):
+    NotImplemented
+  def close(self):
+    NotImplemented
+
+
+class ManagedHTTPServer(ManagedResource):
+  """Simple HTTP Server for testing.
+  """
+  # Request Handler, needs to be defined by subclasses.
+  RequestHandler = None # type: typing.Type[BaseHTTPServer.BaseHTTPRequestHandler]
+
+  proto = 'http'
+  # hostname to listen to, default to ipv4 address of the current test
+  hostname = None # type: str
+  # port to listen to, default
+  port = None # type: int
+
+  @property
+  def url(self):
+    # type: () -> str
+    return '{self.proto}://{self.hostname}:{self.port}'.format(self=self)
+
+  @property
+  def netloc(self):
+    # type: () -> str
+    return urllib_parse.urlparse(self.url).netloc
+
+  def _makeServer(self):
+    # type: () -> BaseHTTPServer.HTTPServer
+    """build the server class.
+
+    This is a method to make it possible to subclass to add https support.
+    """
+    logger = self._cls.logger
+
+    class ErrorLoggingHTTPServer(BaseHTTPServer.HTTPServer):
+      def handle_error(self, request , client_addr):
+        # redirect errors to log
+        logger.info("Error processing request from %s", client_addr, exc_info=True)
+
+    logger.debug(
+        "starting %s (%s) on %s:%s",
+        self.__class__.__name__,
+        self._name,
+        self.hostname,
+        self.port,
+    )
+    server = ErrorLoggingHTTPServer(
+        (self.hostname, self.port),
+        self.RequestHandler,
+    )
+
+    return server
+
+  def open(self):
+    # type: () -> None
+    if not self.hostname:
+      self.hostname = self._cls._ipv4_address
+    if not self.port:
+      self.port = findFreeTCPPort(self.hostname)
+
+    server = self._makeServer()
+    self._process = multiprocessing.Process(
+        target=server.serve_forever,
+        name=self._name,
+    )
+    self._process.start()
+
+  def close(self):
+    # type: () -> None
+    self._process.terminate()
+    self._process.join()
+
+
+class ManagedHTTPSServer(ManagedHTTPServer):
+  """An HTTPS Server
+  """
+  proto = 'https'
+  certificate_file = None # type: str
+
+  def _makeServer(self):
+    # type: () -> BaseHTTPServer.HTTPServer
+    server = super(ManagedHTTPSServer, self)._makeServer()
+    raise NotImplementedError("TODO")
+    return server
+
+class ManagedTemporaryDirectory(ManagedResource):
+  path = None # type: str
+  def open(self):
+    self.path = tempfile.mkdtemp()
+  def close(self):
+    shutil.rmtree(self.path)
+
+
 class CrontabMixin(object):
   computer_partition_root_path = None # type: str
   def _getCrontabCommand(self, crontab_name):