Update Release Candidate

slapos/recipe/simplehttpserver/__init__.py

@@ -29,46 +29,37 @@ import string, random
 import os
 from six.moves import range
 
-class Recipe(GenericBaseRecipe):
+from zc.buildout import UserError
+from zc.buildout.buildout import bool_option
 
-  def __init__(self, buildout, name, options):
 
-    base_path = options['base-path']
-    if options.get('use-hash-url', 'True') in ['true', 'True']:
-      pool = string.ascii_letters + string.digits
-      hash_string = ''.join(random.choice(pool) for i in range(64))
-      path = os.path.join(base_path, hash_string)
-  
-      if os.path.exists(base_path):
-        path_list = os.listdir(base_path)
-        if len(path_list) == 1:
-          hash_string = path_list[0]
-          path = os.path.join(base_path, hash_string)
-        elif len(path_list) > 1:
-          raise ValueError("Folder %s should contain 0 or 1 element." % base_path)
-  
-      options['root-dir'] = path
-      options['path'] = hash_string
-    else:
-      options['root-dir'] = base_path
-      options['path'] = ''
-    return GenericBaseRecipe.__init__(self, buildout, name, options)
+def issubpathof(subpath, path):
+  subpath = os.path.abspath(subpath)
+  path = os.path.abspath(path)
+  relpath = os.path.relpath(subpath, start=path)
+  return not relpath.startswith(os.pardir)
 
 
-  def install(self):
+class Recipe(GenericBaseRecipe):
+  def __init__(self, buildout, name, options):
+    host, port, socketpath, abstract = (
+      options.get(k) for k in ('host', 'port', 'socketpath', 'abstract'))
+    oneof = host, socketpath, abstract
+    if sum(bool(v) for v in oneof) != 1 or bool(host) != bool(port):
+      raise UserError("Specify one of (host, port) | socketpath | abstract")
+    address = (host, int(port)) if host else socketpath or '\0' + abstract
+    options['address'] = address
+    return GenericBaseRecipe.__init__(self, buildout, name, options)
 
-    if not os.path.exists(self.options['root-dir']):
-      os.mkdir( self.options['root-dir'] )
+  def install(self):
     parameters = {
-      'host': self.options['host'],
-      'port': int(self.options['port']),
+      'address': self.options['address'],
       'cwd': self.options['base-path'],
       'log-file': self.options['log-file'],
-      'cert-file': self.options.get('cert-file', ''),
-      'key-file': self.options.get('key-file', ''),
-      'root-dir': self.options['root-dir']
+      'cert-file': self.options.get('cert-file'),
+      'key-file': self.options.get('key-file'),
+      'allow-write': bool_option(self.options, 'allow-write', 'false')
     }
-
     return self.createPythonScript(
         self.options['wrapper'].strip(),
         __name__ + '.simplehttpserver.run',

slapos/recipe/simplehttpserver/simplehttpserver.py

 # -*- coding: utf-8 -*-
 from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
-from six.moves.BaseHTTPServer import HTTPServer
-import ssl
-import os
+from six.moves.socketserver import TCPServer
+
+import cgi
+import contextlib
+import errno
 import logging
-from netaddr import valid_ipv4, valid_ipv6
+import os
+import ssl
 import socket
-import cgi, errno
 
 from slapos.util import str2bytes
 
+from . import issubpathof
+
 
 class ServerHandler(SimpleHTTPRequestHandler):
+  base_path = None # set by run
+  restrict_write = True # set by run
+  _additional_logs = None
+
+  @contextlib.contextmanager
+  def _log_extra(self, msg):
+    self._additional_logs = msg
+    try:
+      yield
+    finally:
+      self._additional_logs = None
+
+  def _log(self, level, msg, *args):
+    if self._additional_logs:
+      msg += self._additional_logs
+    logging.log(level, '%s - - ' + msg, self.client_address[0], *args)
+
+  def log_message(self, msg, *args):
+    self._log(logging.INFO, msg, *args)
+
+  def log_error(self, msg, *args):
+    self._log(logging.ERROR, msg, *args)
 
-  document_path = ''
-  restrict_root_folder = True
+  def log_request(self, *args):
+    with self._log_extra('\n' + str(self.headers)):
+      SimpleHTTPRequestHandler.log_request(self, *args)
 
   def respond(self, code=200, type='text/html'):
     self.send_response(code)
     self.send_header("Content-type", type)
     self.end_headers()
 
-  def restrictedRootAccess(self):
-    if self.restrict_root_folder and self.path and self.path == '/':
-      # no access to root path
+  def restrictedWriteAccess(self):
+    if self.restrict_write and self.command not in ('GET', 'HEAD'):
+      # no write access
       self.respond(403)
       self.wfile.write(b"Forbidden")
       return True
     return False
 
-  def do_GET(self):
-    logging.info('%s - GET: %s \n%s' % (self.client_address[0], self.path, self.headers))
-    if self.restrictedRootAccess():
-      return
-    SimpleHTTPRequestHandler.do_GET(self)
-
   def do_POST(self):
     """Write to a file on the server.
 
@@ -45,8 +66,7 @@ class ServerHandler(SimpleHTTPRequestHandler):
 
     request can be encoded as application/x-www-form-urlencoded or multipart/form-data
     """
-    logging.info('%s - POST: %s \n%s' % (self.client_address[0], self.path, self.headers))
-    if self.restrictedRootAccess():
+    if self.restrictedWriteAccess():
       return
 
     form = cgi.FieldStorage(
@@ -67,64 +87,76 @@ class ServerHandler(SimpleHTTPRequestHandler):
     file_open_mode = 'wb' if ('clear' in form and form['clear'].value in ('1', b'1')) else 'ab'
 
     self.writeFile(file_path, file_content, file_open_mode)
-    self.respond(200, type=self.headers['Content-Type'])
-    self.wfile.write(b"Content written to %s" % str2bytes(file_path))
 
   def writeFile(self, filename, content, method='ab'):
-    file_path = os.path.abspath(os.path.join(self.document_path, filename))
-    if not file_path.startswith(self.document_path):
+    file_path = os.path.abspath(os.path.join(self.base_path, filename))
+    # Check writing there is allowed
+    if not issubpathof(file_path, self.base_path):
       self.respond(403, 'text/plain')
       self.wfile.write(b"Forbidden")
-
+      return
+    # Create missing directories if needed
     try:
       os.makedirs(os.path.dirname(file_path))
     except OSError as exception:
       if exception.errno != errno.EEXIST:
-        logging.error('Failed to create file in %s. The error is \n%s' % (
-          file_path, str(exception)))
-
-    logging.info('Writing recieved content to file %s' % file_path)
+        self.log_error('Failed to create file in %s. The error is \n%s',
+          file_path, exception)
+    # Write content to file
+    self.log_message('Writing received content to file %s', file_path)
     try:
       with open(file_path, method) as myfile:
         myfile.write(content)
-        logging.info('Done.')
+        self.log_message('Done.')
     except IOError as e:
-      logging.error('Something happened while processing \'writeFile\'. The message is %s' %
-                    str(e))
-
-class HTTPServerV6(HTTPServer):
-  address_family = socket.AF_INET6
+      self.log_error(
+        'Something happened while processing \'writeFile\'. The message is %s',
+        e)
+    self.respond(200, type=self.headers['Content-Type'])
+    self.wfile.write(b"Content written to %s" % str2bytes(filename))
 
 
 def run(args):
-  
-  # minimal web server.  serves files relative to the
-  # current directory.
-  logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-                            filename=args['log-file'] ,level=logging.INFO)
-  
-  port = args['port']
-  host = args['host']
-  os.chdir(args['cwd'])
-  
+  # minimal web server. serves files relative to the current directory.
+  logging.basicConfig(
+      format="%(asctime)s %(levelname)s - %(message)s",
+      filename=args['log-file'],
+      level=logging.INFO)
+
+  address = args['address']
+  cwd = args['cwd']
+
+  os.chdir(cwd)
+
   Handler = ServerHandler
-  Handler.document_path = args['root-dir']
-  Handler.restrict_root_folder = (args['root-dir'] != args['cwd'])
-  
-  if valid_ipv6(host):
-    server = HTTPServerV6
-  else:
-    server = HTTPServer
-  
-  httpd = server((host, port), Handler)
-  scheme = 'http'
-  if 'cert-file' in args and 'key-file' in args and \
-      os.path.exists(args['cert-file']) and os.path.exists(args['key-file']):
-    scheme = 'https'
-    httpd.socket = ssl.wrap_socket (httpd.socket, 
-                                     server_side=True,
-                                     certfile=args['cert-file'],
-                                     keyfile=args['key-file'])
-
-  logging.info("Starting simple http server at %s://%s:%s" % (scheme, host, port))
+  Handler.base_path = cwd
+  Handler.restrict_write = not args['allow-write']
+
+  try:
+    host, port = address
+    family, _, _, _, _ = socket.getaddrinfo(host, port)[0]
+  except ValueError:
+    family = socket.AF_UNIX
+
+  class Server(TCPServer):
+    allow_reuse_address = 1 # for tests, HTTPServer in stdlib sets it too
+    address_family = family
+
+  httpd = Server(address, Handler)
+
+  certfile = args['cert-file']
+  if certfile: # keyfile == None signifies key is in certfile
+    PROTOCOL_TLS_SERVER = getattr(ssl, 'PROTOCOL_TLS_SERVER', None)
+    if PROTOCOL_TLS_SERVER:
+      sslcontext = ssl.SSLContext(PROTOCOL_TLS_SERVER)
+      sslcontext.load_cert_chain(certfile, args['key-file'])
+      httpd.socket = sslcontext.wrap_socket(httpd.socket, server_side=True)
+    else: # BBB Py2, Py<3.6
+      httpd.socket = ssl.wrap_socket(
+          httpd.socket,
+          server_side=True,
+          certfile=certfile,
+          keyfile=args['key-file'])
+
+  logging.info("Starting simple http server at %s", address)
   httpd.serve_forever()

slapos/test/recipe/test_simplehttpserver.py

+import errno
 import os
 import shutil
 import tempfile
 import unittest
+import socket
 import subprocess
 import time
+import warnings
 
 from six.moves.urllib import parse as urlparse
 import requests
@@ -11,6 +14,58 @@ import requests
 from slapos.recipe import simplehttpserver
 from slapos.test.utils import makeRecipe
 
+CERTIFICATE_FOR_TEST = """
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC8/zt/ndbvsCXb
+2Kf5CaYlSsngykwfeeekDSoYHqWrl/WltFbdz/yw1ggRZUXo0l1ueJrDWqQzZIAT
+9YjoNkX3G21nEIzg9/aKqq1vqHKBH+JaaAt+m84GnErFDztnkiMUKWFKyFmseg0O
+QtkYGw179bXfcXX2x18gz8aBmCkjBjKjfiQtYWs9sPU0grBl9rE+h1maRh2uQXnF
+BTMKHJ6wNGyFgg0ATqrBiLRv+wxCnuCdGzJzkZ3ytKuhqkwEcEIsVHSSwAx+hdBR
+3AUBl1jfwUukj8a4rf23RR3pvYIZiMEdalsuiLBKyjzCqSPo5VZzSWSiK5CTPspM
+4bz9OXPHAgMBAAECggEAMQcg/y0J+em/GHXutSrsn9Xz4s13y96K2cLUfadNoOLt
+xYuv0SDIU3NiamjUJt6TgDnnI/Bakj5q/0J9vod9xOmnisn/Ucjhev1luoZ/FcIY
+rQ06liCC5LIcr1wRM//z+6H0bDrnEFglFOMAgEFcUSDfilRbnqX/pnpf63R2j2/0
+ttsSI3/plJAJbGha01S9jLTRKqHWy0vV0XJUXWkg0BETJci0w4fJ1kdMmnELaq4L
+kU8IZoHwbRq/RBudQoN4ceZjUnMFcVSQCFa+5coYEJvrYvcrLzA8E01435AGyHyv
+DzkiYwIrAzfQYhNVKLXgXrMGclNk8k9SMISSpVq92QKBgQDtJZjWrKxz5iWheIe8
+uGM2rQ7ZgtAO9pYhDNXwKvlxEEXQrtWrVT2KA02/rbyOGoB4D7hlJXlopSLenV3o
+5d3lRXhIXHSz2Qzy5/opPK0rt6TBXKWZ3+AxV7lpzJReltgRSn6mg1bgB2D14GYa
+1gfH1W2fVJ2B5LrB3dPOCJOC4wKBgQDMBbEBsUh1HSAN9tR9icZcgYD2/1aYVHVJ
+bnGUR1xs1cQRHKZZn6y/BBy021YAGIbgbFb8YhJC5lCMmeLADho3W1XxYhe6ssiE
+E4sbK4y+fD2MFvAe7Y//IB0KRmAzTG3tPyOjBMftAMwrGoXIo990BAFtrO8tTIeb
+9XcUnd0MzQKBgA8jz1YlP/1GPDDK2R+bRfo/oisQxuetpngFscLbe4FUYKCqCMof
+bwZYn6YVGWyZFIqVtlf+xHmB0XAU6+HqivgQL1WvUWQJ/2Ginb30OboIx2Pw3kGs
+oUuFJjky7mX7i1/POba3u9whnHcWFG6yK1z+qzj41fVs/N9ToioNMh2xAoGAIAY4
+rYpVVES5FlgLLJVmtHiDdMHJpumC637RhzPYVyEKwKDdn63HoMgVdXIEQsmWyj1X
+PhBqy2N5e0hgZkMQbGYCzHvYO676eHjU2fPxCKlZw9aJ5GDnvGUfCdDYItU5YAcM
+IfeLJjF82rs0CrVmSsCiNMPzWwnrM1jJU0wgOXUCgYEAzAu7kDTITpMBvtUxWapQ
+c1YoE4CqCG6Kq+l65SDe+c9VCckEmVPEhHbPmTv28iUh9n5MipCi7tehok2YX/Cw
+o8M12F9A9sOWTqrNylCgIjU0GCTBkA5LvYV786TYJgWPZ5Mwdkmq5Ifbf3Ti/uGk
+z6Cids97LVTVrV4iAZ+alY0=
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIDXzCCAkegAwIBAgIUAXy1ly1SQ41kXIKV2orz+SghlrUwDQYJKoZIhvcNAQEL
+BQAwPjELMAkGA1UEBhMCRVUxDzANBgNVBAoMBk5leGVkaTEeMBwGA1UEAwwVdGVz
+dF9zaW1wbGVodHRwc2VydmVyMCAXDTI0MTIwMjE0MTkzNVoYDzIxMDcwMTIyMTQx
+OTM1WjA+MQswCQYDVQQGEwJFVTEPMA0GA1UECgwGTmV4ZWRpMR4wHAYDVQQDDBV0
+ZXN0X3NpbXBsZWh0dHBzZXJ2ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQC8/zt/ndbvsCXb2Kf5CaYlSsngykwfeeekDSoYHqWrl/WltFbdz/yw1ggR
+ZUXo0l1ueJrDWqQzZIAT9YjoNkX3G21nEIzg9/aKqq1vqHKBH+JaaAt+m84GnErF
+DztnkiMUKWFKyFmseg0OQtkYGw179bXfcXX2x18gz8aBmCkjBjKjfiQtYWs9sPU0
+grBl9rE+h1maRh2uQXnFBTMKHJ6wNGyFgg0ATqrBiLRv+wxCnuCdGzJzkZ3ytKuh
+qkwEcEIsVHSSwAx+hdBR3AUBl1jfwUukj8a4rf23RR3pvYIZiMEdalsuiLBKyjzC
+qSPo5VZzSWSiK5CTPspM4bz9OXPHAgMBAAGjUzBRMB0GA1UdDgQWBBQc1p1Qudnk
+WcxOnVt+4zw+MpmOITAfBgNVHSMEGDAWgBQc1p1QudnkWcxOnVt+4zw+MpmOITAP
+BgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCjZfuToaybR8JqTQ1l
+4MZ8BEzFlq6Ebn8x4shiWc3wiX5cd1RSF4iilpDv2yp9MNiTHXkMxNEnx9NMdK/+
+0bNlBn6tcv5MLZynXQnT+keJ73iYFB0Og298NauEQPI9x5/gf3zVGKBJ7d/aOumR
+VRUugFQLzwWj27Muh1rbdayh73gpuNm1ZF+HgwWy8vYc5XoLS3gZ+tlGX3Im0Agg
+ug2Kng5JY+f1aC8ZWBtTTFa2k2QALD1dD+vzGsoKitUEarg1CMHO/f6VsAFTfJT3
+NDI4ky4bVMpkq17t65YXf1QVgquOEPfAnkzn51/vPzvezOMzPYQbsQqMbc4jehZT
+oxpd
+-----END CERTIFICATE-----
+""".strip()
+
 
 class SimpleHTTPServerTest(unittest.TestCase):
   process = None
@@ -21,55 +76,124 @@ class SimpleHTTPServerTest(unittest.TestCase):
     self.install_dir = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, self.install_dir)
     self.wrapper = os.path.join(self.install_dir, 'server')
-    host, port = os.environ['SLAPOS_TEST_IPV4'], 9999
-    self.server_url = 'http://{host}:{port}'.format(host=host, port=port)
+    self.logfile = self.wrapper + '.log'
+    self.process = None
+
+  def setUpRecipe(self, opt=None):
+    opt = opt or {}
+    self.certfile = opt.get('cert-file')
+    if not 'socketpath' in opt and not 'abstract' in opt:
+      opt['host'] = host = os.environ['SLAPOS_TEST_IPV4']
+      opt['port'] = port = 9999
+      scheme = 'https' if self.certfile else 'http'
+      self.server_url = scheme + '://{}:{}'.format(host, port)
+    else:
+      self.server_url = None
+    options = {
+        'base-path': self.base_path,
+        'log-file': self.logfile,
+        'wrapper': self.wrapper,
+    }
+    options.update(opt)
     self.recipe = makeRecipe(
         simplehttpserver.Recipe,
-        options={
-            'base-path': self.base_path,
-            'host': host,
-            'port': port,
-            'log-file': os.path.join(self.install_dir, 'simplehttpserver.log'),
-            'wrapper': self.wrapper,
-        },
+        options=options,
         name='simplehttpserver',
     )
 
-  def tearDown(self):
-    if self.process:
-      self.process.terminate()
-      self.process.wait()
-
-  def test_options(self):
-    self.assertNotEqual(self.recipe.options['path'], '')
-    self.assertEqual(
-        self.recipe.options['root-dir'],
-        os.path.join(
-            self.base_path,
-            self.recipe.options['path'],
-        ))
-
-  def test_install(self):
+  def startServer(self):
     self.assertEqual(self.recipe.install(), self.wrapper)
     self.process = subprocess.Popen(
         self.wrapper,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
+        universal_newlines=True, # BBB Py2, use text= in Py3
     )
-    server_base_url = urlparse.urljoin(
-        self.server_url,
-        self.recipe.options['path'],
-    )
-    for i in range(16):
-      try:
-        resp = requests.get(server_base_url)
-        break
-      except requests.exceptions.ConnectionError:
-        time.sleep(i * .1)
+    address = self.recipe.options['address']
+    if self.server_url:
+      kwargs = {'verify': False} if self.certfile else {}
+      def check_connection():
+        resp = requests.get(self.server_url, **kwargs)
+        self.assertIn('Directory listing for /', resp.text)
+      ConnectionError = requests.exceptions.ConnectionError
+      cleanup = None
     else:
-      self.fail(
-          'server did not start.\nout: %s error: %s' % self.process.communicate())
-    self.assertIn('Directory listing for /', resp.text)
+      s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+      def check_connection():
+        s.connect(address)
+      ConnectionError = socket.error
+      cleanup = lambda: s.close()
+    try:
+      for i in range(16):
+        try:
+          check_connection()
+          break
+        except ConnectionError:
+          time.sleep(i * .1)
+      else:
+        # Kill process in case it did not crash
+        # otherwise .communicate() may hang forever.
+        self.process.terminate()
+        self.process.wait()
+        with open(self.logfile) as f:
+          log = f.read()
+        self.fail(
+          "Server did not start\n"
+          "out: %s\n"
+          "err: %s\n"
+          "log: %s"
+          % (self.process.communicate() + (log,)))
+    finally:
+      if cleanup:
+        cleanup()
+    with open(self.logfile) as f:
+      self.assertIn("Starting simple http server at %s" % (address,), f.read())
+    return self.server_url
+
+  def tearDown(self):
+    if self.process:
+      self.process.terminate()
+      self.process.wait()
+      self.process.communicate() # close pipes
+      self.process = None
+
+  def write_should_fail(self, url, hack_path, hack_content):
+    # post with multipart/form-data encoding
+    resp = requests.post(
+        url,
+        files={
+            'path': hack_path,
+            'content': hack_content,
+        },
+    )
+    # First check for actual access to forbidden files
+    try:
+      with open(hack_path) as f:
+        content = f.read()
+      if content == hack_content:
+        self.fail(content)
+      self.fail("%s should not have been created" % hack_path)
+    except IOError as e:
+      if e.errno != errno.ENOENT:
+        raise
+    # Now check for proper response
+    self.assertEqual(resp.status_code, requests.codes.forbidden)
+    self.assertEqual(resp.text, 'Forbidden')
+
+  def test_write_outside_base_path_should_fail(self):
+    self.setUpRecipe({'allow-write': 'true'})
+    server_base_url = self.startServer()
+
+    # A file outside the server's root directory
+    hack_path = os.path.join(self.install_dir, 'forbidden', 'hack.txt')
+    hack_content = "You should not be able to write to hack.txt"
+
+    self.write_should_fail(server_base_url, hack_path, hack_content)
+    self.assertFalse(os.path.exists(os.path.dirname(hack_path)))
+
+  def test_write(self):
+    self.setUpRecipe({'allow-write': 'true'})
+    server_base_url = self.startServer()
 
     # post with multipart/form-data encoding
     resp = requests.post(
@@ -81,15 +205,21 @@ class SimpleHTTPServerTest(unittest.TestCase):
     )
     self.assertEqual(resp.status_code, requests.codes.ok)
     self.assertEqual(resp.text, 'Content written to hello-form-data.txt')
-    with open(
-        os.path.join(self.base_path, self.recipe.options['path'],
-                     'hello-form-data.txt')) as f:
+    hello_form_file = os.path.join(self.base_path, 'hello-form-data.txt')
+    with open(hello_form_file) as f:
       self.assertEqual(f.read(), 'hello-form-data')
 
     self.assertIn('hello-form-data.txt', requests.get(server_base_url).text)
     self.assertEqual(
         requests.get(server_base_url + '/hello-form-data.txt').text, 'hello-form-data')
 
+    # check GET and POST are logged
+    with open(self.logfile) as f:
+      log = f.read()
+    self.assertIn('Writing received content to file ' + hello_form_file, log)
+    self.assertIn('"POST / HTTP/1.1" 200 -', log)
+    self.assertIn('"GET /hello-form-data.txt HTTP/1.1" 200 -', log)
+
     # post as application/x-www-form-urlencoded
     resp = requests.post(
         server_base_url,
@@ -100,8 +230,7 @@ class SimpleHTTPServerTest(unittest.TestCase):
     )
     self.assertEqual(resp.status_code, requests.codes.ok)
     with open(
-        os.path.join(self.base_path, self.recipe.options['path'],
-                     'hello-form-urlencoded.txt')) as f:
+        os.path.join(self.base_path, 'hello-form-urlencoded.txt')) as f:
       self.assertEqual(f.read(), 'hello-form-urlencoded')
 
     self.assertIn('hello-form-urlencoded.txt', requests.get(server_base_url).text)
@@ -119,3 +248,58 @@ class SimpleHTTPServerTest(unittest.TestCase):
           },
       )
       self.assertEqual(resp.status_code, requests.codes.forbidden)
+
+  def test_readonly(self):
+    self.setUpRecipe()
+
+    indexpath = os.path.join(self.base_path, 'index.txt')
+    indexcontent = "This file is served statically and readonly"
+    with open(indexpath, 'w') as f:
+      f.write(indexcontent)
+
+    server_base_url = self.startServer()
+    indexurl = os.path.join(server_base_url, 'index.txt')
+
+    resp = requests.get(indexurl)
+    self.assertEqual(resp.status_code, requests.codes.ok)
+    self.assertEqual(resp.text, indexcontent)
+
+    resp = requests.post(
+        server_base_url,
+        files={
+            'path': 'index.txt',
+            'content': 'Not readonly after all',
+        },
+    )
+    self.assertEqual(resp.status_code, requests.codes.forbidden)
+    with open(indexpath) as f:
+      self.assertEqual(f.read(), indexcontent)
+
+  def test_socketpath(self):
+    socketpath = os.path.join(self.install_dir, 'http.sock')
+    self.setUpRecipe({'socketpath': socketpath})
+    self.assertEqual(socketpath, self.recipe.options['address'])
+    self.startServer()
+
+  def test_abstract(self):
+    abstract = os.path.join(self.install_dir, 'abstract.http')
+    self.setUpRecipe({'abstract': abstract})
+    self.assertEqual('\0' + abstract, self.recipe.options['address'])
+    self.startServer()
+
+  def test_tls_self_signed(self):
+    certfile = os.path.join(self.install_dir, 'cert.pem')
+    with open(certfile, 'w') as f:
+      f.write(CERTIFICATE_FOR_TEST)
+
+    self.setUpRecipe({'cert-file': certfile})
+    with warnings.catch_warnings():
+      warnings.simplefilter("ignore") # suppress verify=False warning
+      server_base_url = self.startServer()
+
+    # Check self-signed certificate is not accepted without verify=False
+    self.assertRaises(
+      requests.exceptions.ConnectionError,
+      requests.get,
+      server_base_url
+    )

software/kvm/boot-image-input-schema.json

+{
+  "type": "object",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Input Parameters",
+  "$defs": {
+    "instance-parameters": {
+      "type": "object",
+      "properties": {
+        "boot-image-url-select": {
+          "title": "Boot image",
+          "description": "Selectable list of provided ISO images.",
+          "type": "string",
+          "default": "Debian Bookworm 12 netinst x86_64",
+          "enum": [
+            "Debian Bookworm 12 netinst x86_64",
+            "Debian Bullseye 11 netinst x86_64",
+            "Centos 8.2004 Minimal x86_64",
+            "Ubuntu Noble 24.04 Live Server x86_64",
+            "Ubuntu Jammy 22.04 Live Server x86_64",
+            "Ubuntu Focal 20.04 Live Server x86_64",
+            "openSUSE Leap 15 NET x86_64",
+            "Arch Linux 2020.09.01 x86_64",
+            "Fedora Server 32 netinst x86_64",
+            "FreeBSD 12.1 RELEASE bootonly x86_64",
+            "SUSE Linux Enterprise Server 15 SP6 x86_64"
+          ]
+        },
+        "boot-image-url-list": {
+          "title": "[EXPERT] Boot image list",
+          "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. Maximum images: 4. Maximum image size: 20GB. Download tries: 4. Maximum ownload time: 4h.",
+          "type": "string",
+          "textarea": true
+        }
+      }
+    }
+  },
+  "unevaluatedProperties": false,
+  "allOf": [
+    {
+      "$ref": "#/$defs/instance-parameters"
+    }
+  ]
+}

software/kvm/buildout.hash.cfg

@@ -43,7 +43,7 @@ md5sum = a02f0694dcb944c18d99f7f79afa2384
 
 [template-kvm-export-script]
 filename = template/kvm-export.sh.jinja2
-md5sum = a1da7809d547b4c61e7c6337bf9f8b8a
+md5sum = ba1e0359178925788792a1c6cc29ba59
 
 [template-nginx]
 filename = template/nginx_conf.in

software/kvm/instance-kvm-cluster-input-schema.json

@@ -453,7 +453,7 @@
             },
             "boot-image-url-list": {
               "title": "[EXPERT] Boot image list",
-              "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. Maximum images: 4. Maximum image size: 20GB. Download tires: 4. Maximum download time: 4h.",
+              "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. Maximum images: 4. Maximum image size: 20GB. Download tries: 4. Maximum download time: 4h.",
               "type": "string",
               "textarea": true
             },

software/kvm/instance-kvm-input-schema.json

@@ -285,31 +285,6 @@
           "format": "uri",
           "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
         },
-        "boot-image-url-select": {
-          "title": "Boot image",
-          "description": "Selectable list of provided ISO images.",
-          "type": "string",
-          "default": "Debian Bookworm 12 netinst x86_64",
-          "enum": [
-            "Debian Bookworm 12 netinst x86_64",
-            "Debian Bullseye 11 netinst x86_64",
-            "Centos 8.2004 Minimal x86_64",
-            "Ubuntu Noble 24.04 Live Server x86_64",
-            "Ubuntu Jammy 22.04 Live Server x86_64",
-            "Ubuntu Focal 20.04 Live Server x86_64",
-            "openSUSE Leap 15 NET x86_64",
-            "Arch Linux 2020.09.01 x86_64",
-            "Fedora Server 32 netinst x86_64",
-            "FreeBSD 12.1 RELEASE bootonly x86_64",
-            "SUSE Linux Enterprise Server 15 SP6 x86_64"
-          ]
-        },
-        "boot-image-url-list": {
-          "title": "[EXPERT] Boot image list",
-          "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. Maximum images: 4. Maximum image size: 20GB. Download tires: 4. Maximum ownload time: 4h.",
-          "type": "string",
-          "textarea": true
-        },
         "whitelist-domains": {
           "title": "Whitelist domains",
           "description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
@@ -373,6 +348,9 @@
   },
   "unevaluatedProperties": false,
   "allOf": [
+    {
+      "$ref": "./boot-image-input-schema.json#/$defs/instance-parameters"
+    },
     {
       "$ref": "#/$defs/instance-parameters"
     }

software/kvm/software.cfg

@@ -36,13 +36,15 @@ parts = ${:common-parts}
 recipe = zc.recipe.egg
 eggs =
   qemu.qmp
+  colorlog
   qmpbackup
 find-links +=
-  https://github.com/abbbi/qmpbackup/releases/download/v0.37/qmpbackup-0.37.tar.gz
+  https://github.com/abbbi/qmpbackup/releases/download/v0.43/qmpbackup-0.43.tar.gz
 
 [versions]
 qemu.qmp = 0.0.3:whl
-qmpbackup = 0.37
+colorlog = 6.9.0:whl
+qmpbackup = 0.43
 
 [python-with-eggs]
 recipe = zc.recipe.egg

software/kvm/template/kvm-export.sh.jinja2

@@ -6,6 +6,9 @@ set -e
 LC_ALL=C
 export LC_ALL
 BACKUP_DIR={{ directory['backup'] }}
+# decolorize colorlog in qmpbackup
+NO_COLOR=true
+export NO_COLOR
 
 log=$(mktemp --tmpdir={{ directory['tmp'] }})
 trap "rm -f $log" EXIT TERM INT
@@ -25,6 +28,9 @@ if [ $RESULT -ne 0 ] ; then
     find $BACKUP_DIR/{{ disk['device'] }} -name '*.qcow2' -delete
     $qmpbackup --level full || exit $?
     echo "Post take-over cleanup"
+  elif egrep -q 'No full backup found for device.*{{ disk['device']}}.*in.*{{ disk['device']}}.*: Execute full backup first.' $log ; then
+    $qmpbackup --level full || exit $?
+    echo "Recovered from empty backup"
   else
     exit $RESULT
   fi

software/kvm/test/test.py

@@ -941,7 +941,11 @@ class TestInstanceResilientBackupImporter(
     # the real assertions comes from re-stabilizing the instance tree
     self.slap.waitForInstance(max_retry=10)
     # check that all stabilizes after backup after takeover
-    self.call_exporter()
+    status_text = self.call_exporter()
+    self.assertIn(
+      'Post take-over cleanup',
+      status_text
+    )
     self.slap.waitForInstance(max_retry=10)
 
 
@@ -951,10 +955,9 @@ class TestInstanceResilientBackupImporterIde(
   disk_type = 'ide'
 
 
-@skipUnlessKvm
-class TestInstanceResilientBackupExporter(
-  TestInstanceResilientBackupMixin, KVMTestCase):
-  def test(self):
+class TestInstanceResilientBackupExporterMixin(
+  TestInstanceResilientBackupMixin):
+  def initialBackup(self):
     status_text = self.call_exporter()
     self.assertEqual(
       len(glob.glob(self.getBackupPartitionPath('FULL-*.qcow2'))),
@@ -966,6 +969,44 @@ class TestInstanceResilientBackupExporter(
       'Recovered from partial backup by removing partial',
       status_text
     )
+    self.assertNotIn(
+      'Recovered from empty backup',
+      status_text
+    )
+    self.assertNotIn(
+      'Post take-over cleanup',
+      status_text
+    )
+
+
+@skipUnlessKvm
+class TestInstanceResilientBackupExporter(
+  TestInstanceResilientBackupExporterMixin, KVMTestCase):
+  def test(self):
+    self.initialBackup()
+
+
+@skipUnlessKvm
+class TestInstanceResilientBackupExporterMigrateOld(
+  TestInstanceResilientBackupExporterMixin, KVMTestCase):
+  def test(self):
+    backup_partition = self.getPartitionPath(
+      'kvm-export', 'srv', 'backup', 'kvm')
+    backup_file_list = ['virtual.qcow2', 'virtual.qcow2.gz']
+    for backup_file in backup_file_list:
+      with open(os.path.join(backup_partition, backup_file), 'w') as fh:
+        fh.write('')
+    self.initialBackup()
+    post_backup_file_list = os.listdir(backup_partition)
+    for backup_file in backup_file_list:
+      self.assertNotIn(backup_file, post_backup_file_list)
+
+
+@skipUnlessKvm
+class TestInstanceResilientBackupExporterPartialRecovery(
+  TestInstanceResilientBackupExporterMixin, KVMTestCase):
+  def test(self):
+    self.initialBackup()
     # cover .partial file in the backup directory with fallback to full
     current_backup = glob.glob(self.getBackupPartitionPath('FULL-*'))[0]
     with open(current_backup + '.partial', 'w') as fh:
@@ -986,12 +1027,56 @@ class TestInstanceResilientBackupExporter(
         'kvm-export', 'etc', 'plugin', 'check-backup-directory.py'))))
 
 
+@skipUnlessKvm
+class TestInstanceResilientBackupExporterEmptyRecovery(
+  TestInstanceResilientBackupExporterMixin, KVMTestCase):
+  def test(self):
+    self.initialBackup()
+    # cover empty backup recovery
+    current_backup_list = glob.glob(self.getBackupPartitionPath('*.qcow2'))
+    self.assertEqual(
+      1,
+      len(current_backup_list)
+    )
+    for file in current_backup_list:
+      os.unlink(file)
+    status_text = self.call_exporter()
+    self.assertEqual(
+      len(glob.glob(self.getBackupPartitionPath('FULL-*.qcow2'))),
+      1)
+    self.assertEqual(
+      len(glob.glob(self.getBackupPartitionPath('INC-*.qcow2'))),
+      0)
+    self.assertIn(
+      'Recovered from empty backup',
+      status_text
+    )
+
+
 @skipUnlessKvm
 class TestInstanceResilientBackupExporterIde(
   TestInstanceResilientBackupExporter):
   disk_type = 'ide'
 
 
+@skipUnlessKvm
+class TestInstanceResilientBackupExporterMigrateOldIde(
+  TestInstanceResilientBackupExporterMigrateOld):
+  disk_type = 'ide'
+
+
+@skipUnlessKvm
+class TestInstanceResilientBackupExporterPartialRecoveryIde(
+  TestInstanceResilientBackupExporterPartialRecovery):
+  disk_type = 'ide'
+
+
+@skipUnlessKvm
+class TestInstanceResilientBackupExporterEmptyRecoveryIde(
+  TestInstanceResilientBackupExporterEmptyRecovery):
+  disk_type = 'ide'
+
+
 @skipUnlessKvm
 class TestInstanceResilient(KVMTestCase, KvmMixin):
   __partition_reference__ = 'ir'

software/neoppod/instance-neo-input-schema.json

@@ -67,7 +67,7 @@
               "description": "Only track references.",
               "default": false,
               "type": "boolean"
-            },
+            }
           },
           "type": "object"
         },

software/rapidspace-vps/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template]
+filename = instance.cfg.in
+md5sum = 2a578c1dfea2b7ebe83bbacb052127c0
+
+[software.json]
+filename = software.cfg.json
+md5sum = d40a0e467955be469645c517a3ca4881
+
+[instance.json]
+filename = ../kvm/boot-image-input-schema.json
+md5sum = 83f45b6fd98dc988c548ab5f14dcdbe6

software/rapidspace-vps/instance.cfg.in

+[buildout]
+parts =
+  switch_softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+[switch_softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+default = dynamic-template-kvm:output
+
+[slap-configuration]
+# we usejsonschema recipe in order to force some values for VPS (see all the const in the JSON schema)
+<= slap-connection
+recipe = slapos.cookbook:slapconfiguration.jsonschema
+jsonschema = ${software.json:target}
+set-default = main
+validate-parameters = main
+
+[slap-configuration-vps]
+# this section will force all constant values for VPS
+recipe  = slapos.recipe.build
+depends = $${slap-configuration:configuration}
+init =
+  conf = self.buildout['slap-configuration']['configuration']
+  # we know for sure that there is only the boot-image parameters in conf
+  # so only set what is custom compared to default values
+  conf['ram-size'] = 245760
+  conf['ram-max-size'] = 246272
+  conf['auto-ballooning'] = False
+  conf['cpu-count'] = 40
+  conf['cpu-max-count'] = 41
+  conf['wipe-disk-ondestroy'] = True
+  conf['use-tap'] = True
+  conf['frontend-software-type'] = "default"
+  conf['frontend-software-url'] = "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
+  conf['frontend-additional-software-type'] = "default"
+  conf['frontend-additional-software-url'] = "chinary-frontend-sr"
+  conf['disk-device-path'] = "/dev/sdb"
+  options['configuration'] = conf
+  # XXX we should make sure this configuration matches KVM json schema...
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+output = $${buildout:directory}/$${:filename}
+extensions = jinja2.ext.do
+extra-context =
+context =
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+  key ipv4 slap-configuration:ipv4
+  key ipv6 slap-configuration:ipv6
+  key global_ipv4_prefix network-information:global-ipv4-network
+  key storage_dict slap-configuration:storage-dict
+  key slapparameter_dict slap-configuration-vps:configuration
+  key computer_id slap-configuration:computer-id
+  raw openssl_executable_location ${openssl:location}/bin/openssl
+  $${:extra-context}
+
+[dynamic-template-kvm]
+<= jinja2-template-base
+url = ${template-kvm:location}/instance-kvm.cfg.jinja2
+filename = template-kvm.cfg
+extra-context =
+  section slap_configuration slap-configuration
+  raw ansible_promise_tpl ${template-ansible-promise:target}
+  raw curl_executable_location ${curl:location}/bin/curl
+  raw dash_executable_location ${dash:location}/bin/dash
+  raw dnsresolver_executable ${buildout:bin-directory}/dnsresolver
+  raw dcron_executable_location ${dcron:location}/sbin/crond
+  raw boot_image_select_source_config ${boot-image-select-source-config:target}
+  raw whitelist_domains_default ${whitelist-domains-default:target}
+  raw whitelist_firewall_download_controller ${whitelist-firewall-download-controller:output}
+  raw image_download_controller ${image-download-controller:output}
+  raw image_download_config_creator ${image-download-config-creator:output}
+  raw logrotate_cfg ${template-logrotate-base:output}
+  raw novnc_location ${noVNC:location}
+  raw netcat_bin ${netcat:location}/bin/netcat
+  raw nginx_executable ${nginx-output:nginx}
+  raw nginx_mime ${nginx-output:mime}
+  raw python_executable ${buildout:executable}
+  raw python_eggs_executable ${buildout:bin-directory}/${python-with-eggs:interpreter}
+  raw qemu_executable_location ${qemu:location}/bin/qemu-system-x86_64
+  raw qemu_img_executable_location ${qemu:location}/bin/qemu-img
+  raw qemu_start_promise_tpl ${template-qemu-ready:target}
+  raw sixtunnel_executable_location ${6tunnel:location}/bin/6tunnel
+  raw template_httpd_cfg ${template-httpd:output}
+  raw template_content ${template-content:target}
+  raw template_kvm_controller_run ${template-kvm-controller:target}
+  raw template_kvm_run ${template-kvm-run:target}
+  raw template_monitor ${monitor2-template:output}
+  raw template_nginx ${template-nginx:target}
+  raw websockify_executable_location ${buildout:directory}/bin/websockify
+  raw wipe_disk_wrapper ${buildout:directory}/bin/securedelete
+template-parts-destination = ${template-parts:target}
+template-replicated-destination = ${template-replicated:target}
+import-list = file parts :template-parts-destination
+              file replicated :template-replicated-destination
+

software/rapidspace-vps/software.cfg

+[buildout]
+extends =
+  ../kvm/software.cfg
+  buildout.hash.cfg
+
+parts += instance.json
+
+[download-vps-base]
+# we cannot use "download-base" section because:
+# 1. we need _profile_base_location to point to current directory for our own files
+# 2. we need json to be in specific directory
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+
+[directory]
+recipe = slapos.recipe.build:mkdirectory
+json-vps = ${buildout:parts-directory}/json-schema/vps
+json-kvm = ${buildout:parts-directory}/json-schema/kvm
+
+[software.json]
+<= download-vps-base
+destination = ${directory:json-vps}/${:filename}
+
+[instance.json]
+<= download-vps-base
+destination = ${directory:json-kvm}/${:filename}
+
+[template]
+<= template-base
+# we need to overwrite _profile_base_location to current directory
+url = ${:_profile_base_location_}/${:filename}

software/rapidspace-vps/software.cfg.json

+{
+  "name": "VPS",
+  "description": "VPS",
+  "serialisation": "json-in-xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "Default VPS",
+      "request": "../kvm/boot-image-input-schema.json",
+      "response": "../kvm/instance-kvm-output-schema.json",
+      "index": 0
+    }
+  }
+}

stack/slapos.cfg

@@ -360,12 +360,12 @@ sgmllib3k = 1.0.0
 simplegeneric = 0.8.1
 singledispatch = 3.4.0.3
 six = 1.16.0
-slapos.cookbook = 1.0.373
+slapos.cookbook = 1.0.386
 slapos.core = 1.14.3
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.25
 slapos.rebootstrap = 4.7
-slapos.recipe.build = 0.57
+slapos.recipe.build = 0.58
 slapos.recipe.cmmi = 0.22
 slapos.recipe.template = 5.1
 slapos.toolbox = 0.146