Merge remote-tracking branch 'origin/feat/cli_request_file' into fix/support-request-rss-id

setup.py

@@ -74,6 +74,8 @@ setup(name=name,
           'six',
           'cachecontrol',
           'lockfile',
+          'jsonschema',
+          'PyYAML',
           'uritemplate', # used by hateoas navigator
           'subprocess32; python_version<"3"',
           'ipaddress; python_version<"3"',  # used by whitelistfirewall

slapos/cli/request.py

@@ -27,15 +27,49 @@
 #
 ##############################################################################
 
+import argparse
+import json
+import os.path
 import pprint
 
+import lxml.etree
+import six
+import yaml
+
 from slapos.cli.config import ClientConfigCommand
-from slapos.client import init, ClientConfig, _getSoftwareReleaseFromSoftwareString
+from slapos.client import (ClientConfig, _getSoftwareReleaseFromSoftwareString,
+                           init)
 from slapos.slap import ResourceNotReady
-
+from slapos.util import SoftwareReleaseSchema, SoftwareReleaseSerialisation
+
+try:
+    from typing import IO, Dict
+except ImportError:
+    pass
+
+
+def getParametersFromFile(file, serialisation):
+    # type: (IO[str], SoftwareReleaseSerialisation) -> Dict
+    extension = os.path.splitext(file.name)[1]
+    if extension in ('.yaml', '.yml'):
+        params = yaml.safe_load(file)
+    elif extension == '.xml':
+        tree = lxml.etree.parse(file)
+        params = {e.attrib['id']: e.text for e in tree.findall('/parameter')}
+        # because the use case of xml files is to copy paste existing XML parameters
+        # as found on slapos web interface, we aren't clever regarding the
+        # serialisation and assume they are already correct.
+        serialisation = None
+    else:
+        params = json.load(file)
+
+    if serialisation == SoftwareReleaseSerialisation.JsonInXml and list(params.keys()) != ['_']:
+        params = {'_': json.dumps(params)}
+    return params
 
 
 def parse_option_dict(options):
+    # type: (str) -> Dict
     """
     Parse a list of option strings like foo=bar baz=qux and return a dictionary.
     Will raise if keys are repeated.
@@ -80,8 +114,12 @@ class RequestCommand(ClientConfigCommand):
 
         ap.add_argument('--parameters',
                         nargs='+',
-                        help="Give your configuration 'option1=value1 option2=value2'")
+                        help="Instance parameters, in the form 'option1=value1 option2=value2'.")
 
+        ap.add_argument('--parameters-file',
+                        type=argparse.FileType('r'),
+                        help="Instance parameters, in a file.\n"
+                        "The file will be interpreted as json, yaml or xml depending on the file extension.")
         return ap
 
     def take_action(self, args):
@@ -95,6 +133,16 @@ class RequestCommand(ClientConfigCommand):
         do_request(self.app.log, conf, local)
 
 
+# BBB on python3 we can use pprint.pformat
+class StrPrettyPrinter(pprint.PrettyPrinter):
+  """A PrettyPrinter which produces consistent output on python 2 and 3
+  """
+  def format(self, object, context, maxlevels, level):
+    if six.PY2 and isinstance(object, six.text_type):
+      object = object.encode('utf-8')
+    return pprint.PrettyPrinter.format(self, object, context, maxlevels, level)
+
+
 def do_request(logger, conf, local):
     logger.info('Requesting %s as instance of %s...',
                 conf.reference, conf.software_url)
@@ -104,11 +152,17 @@ def do_request(logger, conf, local):
 
     if conf.software_url in local:
         conf.software_url = local[conf.software_url]
+
+    software_schema = SoftwareReleaseSchema(conf.software_url, conf.type)
+    software_schema_serialisation = software_schema.getSerialisation()
+    parameters = conf.parameters
+    if conf.parameters_file:
+        parameters = getParametersFromFile(conf.parameters_file, software_schema_serialisation)
     try:
         partition = local['slap'].registerOpenOrder().request(
             software_release=conf.software_url,
             partition_reference=conf.reference,
-            partition_parameter_kw=conf.parameters,
+            partition_parameter_kw=parameters,
             software_type=conf.type,
             filter_kw=conf.node,
             state=conf.state,
@@ -116,7 +170,11 @@ def do_request(logger, conf, local):
         )
         logger.info('Instance requested.\nState is : %s.', partition.getState())
         logger.info('Connection parameters of instance are:')
-        logger.info(pprint.pformat(partition.getConnectionParameterDict()))
+        connection_parameter_dict = partition.getConnectionParameterDict()
+        if software_schema_serialisation == SoftwareReleaseSerialisation.JsonInXml:
+            if '_' in connection_parameter_dict:
+                connection_parameter_dict = json.loads(connection_parameter_dict['_'])
+        logger.info(StrPrettyPrinter().pformat(connection_parameter_dict))
         logger.info('You can rerun the command to get up-to-date information.')
     except ResourceNotReady:
         logger.warning('Instance requested. Master is provisioning it. Please rerun in a '

slapos/slap/slap.py

@@ -40,14 +40,17 @@ import os
 import logging
 import re
 from functools import wraps
+import warnings
 
 import json
+import jsonschema
 import six
 
 from .exception import ResourceNotReady, ServerError, NotFoundError, \
           ConnectionError
 from .hateoas import SlapHateoasNavigator, ConnectionHelper
-from slapos.util import loads, dumps, bytes2str, unicode2str, xml2dict, dict2xml, calculate_dict_hash
+from slapos.util import (SoftwareReleaseSchema, bytes2str, calculate_dict_hash,
+                         dict2xml, dumps, loads, unicode2str, xml2dict)
 
 from xml.sax import saxutils
 from zope.interface import implementer
@@ -87,6 +90,27 @@ class SlapRequester(SlapDocument):
   """
 
   def _requestComputerPartition(self, request_dict):
+
+    try:
+      SoftwareReleaseSchema(
+          request_dict['software_release'],
+          request_dict['software_type']
+      ).validateInstanceParameterDict(
+          loads(request_dict['partition_parameter_xml']))
+    except jsonschema.ValidationError as e:
+      warnings.warn(
+        "Request parameters do not validate against schema definition:\n{e}".format(e=e),
+        UserWarning,
+      )
+    except Exception as e:
+      # note that we intentionally catch wide exceptions, so that if anything
+      # is wrong with fetching the schema or the schema itself this does not
+      # prevent users from requesting instances.
+      warnings.warn(
+        "Error validating request parameters against schema definition:\n{e.__class__.__name__} {e}".format(e=e),
+        UserWarning,
+      )
+
     try:
       xml = self._connection_helper.POST('requestComputerPartition', data=request_dict)
     except ResourceNotReady:

slapos/testing/utils.py

@@ -119,8 +119,10 @@ class ManagedHTTPServer(ManagedResource):
   proto = 'http'
   # hostname to listen to, default to ipv4 address of the current test
   hostname = None # type: str
-  # port to listen to, default
+  # port to listen to, default to a free port selected with `findFreeTCPPort`
   port = None # type: int
+  # current working directory of the server process
+  working_directory = None # type: str
 
   @property
   def url(self):
@@ -166,10 +168,19 @@ class ManagedHTTPServer(ManagedResource):
     if not self.port:
       self.port = findFreeTCPPort(self.hostname)
 
+    def serve_forever(server, cwd):
+      if cwd:
+        os.chdir(cwd)
+      server.serve_forever()
+
     server = self._makeServer()
     self._process = multiprocessing.Process(
-        target=server.serve_forever,
+        target=serve_forever,
         name=self._name,
+        kwargs={
+            'server': server,
+            'cwd': self.working_directory,
+        }
     )
     self._process.start()
     # from now on, socket is used by server subprocess, we can close it

slapos/tests/test_cli.py

@@ -25,6 +25,7 @@
 #
 ##############################################################################
 
+import json
 import logging
 import pprint
 import unittest
@@ -40,6 +41,7 @@ from contextlib import contextmanager
 from mock import patch, create_autospec
 import mock
 from slapos.util import sqlite_connect, bytes2str
+from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE
 
 import slapos.cli.console
 import slapos.cli.entry
@@ -49,10 +51,12 @@ import slapos.cli.computer_info
 import slapos.cli.computer_list
 import slapos.cli.computer_token
 import slapos.cli.supervisorctl
+import slapos.cli.request
 from slapos.cli.proxy_show import do_show, StringIO
 from slapos.cli.cache import do_lookup as cache_do_lookup
 from slapos.cli.cache_source import do_lookup as cache_source_do_lookup
 from slapos.client import ClientConfig
+from slapos.slap import SoftwareProductCollection
 import slapos.grid.svcbackend
 import slapos.proxy
 import slapos.slap
@@ -65,8 +69,8 @@ def raiseNotFoundError(*args, **kwargs):
 class CliMixin(unittest.TestCase):
   def setUp(self):
     slap = slapos.slap.slap()
-    self.local = {'slap': slap}
     self.logger = create_autospec(logging.Logger)
+    self.local = {'slap': slap, 'product': SoftwareProductCollection(self.logger, slap)}
     self.conf = create_autospec(ClientConfig)
 
 class TestCliCache(CliMixin):
@@ -641,3 +645,227 @@ class TestCliComplete(CliMixin):
     with patch.object(sys, 'stdout', StringIO()) as app_stdout:
       self.assertEqual(slapos.cli.entry.SlapOSApp().run(['complete', '--shell=fish']), 0)
     self.assertIn('__fish_seen_subcommand_from', app_stdout.getvalue())
+
+
+class TestCliRequest(CliMixin):
+  def test_parse_option_dict(self):
+    parse_option_dict = slapos.cli.request.parse_option_dict
+
+    self.assertEqual(parse_option_dict(['foo=bar', 'a=b']), {'foo': 'bar', 'a': 'b'})
+    # malformed option = assignment
+    self.assertRaises(ValueError, parse_option_dict, ['a'])
+    # duplicated key
+    self.assertRaises(ValueError, parse_option_dict, ['a=b', 'a=c'])
+    # corner cases
+    self.assertEqual(parse_option_dict(['a=a=b']), {'a': 'a=b'})
+    self.assertEqual(parse_option_dict(['a=a\nb']), {'a': 'a\nb'})
+    self.assertEqual(parse_option_dict([]), {})
+
+  def test_request(self):
+    self.conf.reference = 'instance reference'
+    self.conf.software_url = 'software URL'
+    self.conf.parameters = {'key': 'value'}
+    self.conf.parameters_file = None
+    self.conf.node = {'computer_guid': 'COMP-1234'}
+    self.conf.type = None
+    self.conf.state = None
+    self.conf.slave = False
+
+    with patch.object(
+        slapos.slap.slap,
+        'registerOpenOrder',
+        return_value=mock.create_autospec(slapos.slap.OpenOrder)) as registerOpenOrder:
+      slapos.cli.request.do_request(self.logger, self.conf, self.local)
+
+    registerOpenOrder().request.assert_called_once_with(
+        software_release='software URL',
+        partition_reference='instance reference',
+        partition_parameter_kw={'key': 'value'},
+        software_type=None,
+        filter_kw={'computer_guid': 'COMP-1234'},
+        state=None,
+        shared=False,
+    )
+
+    self.logger.info.assert_any_call(
+        'Requesting %s as instance of %s...',
+        'instance reference',
+        'software URL',
+    )
+
+  def test_request_json_in_xml_published_parameters(self):
+    tmpdir = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, tmpdir)
+    with open(os.path.join(tmpdir, 'software.cfg.json'), 'w') as f:
+      json.dump(
+          {
+              "name": "Test Software",
+              "description": "Dummy software for Test",
+              "serialisation": "json-in-xml",
+              "software-type": {
+                  DEFAULT_SOFTWARE_TYPE: {
+                      "title": "Default",
+                      "description": "Default type",
+                      "request": "instance-default-input-schema.json",
+                      "response": "instance-default-output-schema.json",
+                      "index": 0
+                  },
+              }
+          }, f)
+
+    self.conf.reference = 'instance reference'
+    self.conf.software_url = os.path.join(tmpdir, 'software.cfg')
+    self.conf.parameters = {'key': 'value'}
+    self.conf.parameters_file = None
+    self.conf.node = {'computer_guid': 'COMP-1234'}
+    self.conf.type = None
+    self.conf.state = None
+    self.conf.slave = False
+
+    cp = slapos.slap.ComputerPartition(
+        'computer_%s' % self.id(),
+        'partition_%s' % self.id())
+    cp._requested_state = 'started'
+    cp._connection_dict = {'_': json.dumps({'foo': 'bar'})}
+
+    with patch.object(
+        slapos.slap.slap,
+        'registerOpenOrder',
+        return_value=mock.create_autospec(slapos.slap.OpenOrder)) as registerOpenOrder:
+      registerOpenOrder().request.return_value = cp
+      slapos.cli.request.do_request(self.logger, self.conf, self.local)
+
+    registerOpenOrder().request.assert_called_once()
+
+    self.assertEqual(self.logger.info.mock_calls, [
+        mock.call('Requesting %s as instance of %s...', self.conf.reference,
+                  self.conf.software_url),
+        mock.call('Instance requested.\nState is : %s.', 'started'),
+        mock.call('Connection parameters of instance are:'),
+        mock.call("{'foo': 'bar'}"),
+        mock.call('You can rerun the command to get up-to-date information.'),
+    ])
+
+
+class TestCliRequestParametersFileJson(CliMixin):
+  """Request with --parameter-file, with a .json file.
+  """
+  expected_partition_parameter_kw = {'foo': ['bar']}
+
+  def _makeParameterFile(self):
+    f = tempfile.NamedTemporaryFile(suffix='.json', mode='w', delete=False)
+    self.addCleanup(os.unlink, f.name)
+    f.write(textwrap.dedent('''\
+    {
+      "foo": ["bar"]
+    }
+    '''))
+    f.flush()
+    return f.name
+
+  def test_request_parameters_file(self):
+    self.conf.reference = 'instance reference'
+    self.conf.software_url = 'software URL'
+    self.conf.parameters =  None
+    f = open(self._makeParameterFile())
+    self.addCleanup(f.close)
+    self.conf.parameters_file = f
+    self.conf.node = {'computer_guid': 'COMP-1234'}
+    self.conf.type = None
+    self.conf.state = None
+    self.conf.slave = False
+
+    with patch.object(
+        slapos.slap.slap,
+        'registerOpenOrder',
+        return_value=mock.create_autospec(slapos.slap.OpenOrder)) as registerOpenOrder:
+      slapos.cli.request.do_request(self.logger, self.conf, self.local)
+
+    registerOpenOrder().request.assert_called_once_with(
+        software_release='software URL',
+        partition_reference='instance reference',
+        partition_parameter_kw=self.expected_partition_parameter_kw,
+        software_type=None,
+        filter_kw={'computer_guid': 'COMP-1234'},
+        state=None,
+        shared=False,
+    )
+    self.logger.info.assert_any_call(
+        'Requesting %s as instance of %s...',
+        'instance reference',
+        'software URL',
+    )
+
+
+class TestCliRequestParametersFileJsonJsonInXMLSerialisation(
+    TestCliRequestParametersFileJson):
+  """Request with --parameter-file, with a .json file and a software using
+  json-in-xml for serialisation. In that case, the parameters are automatically
+  serialised with {'_': json.dumps(params)}
+  """
+  expected_partition_parameter_kw = {"_": "{\"foo\": [\"bar\"]}"}
+
+  def test_request_parameters_file(self):
+    with mock.patch(
+        'slapos.cli.request.SoftwareReleaseSchema.getSerialisation',
+        return_value='json-in-xml'):
+      super(TestCliRequestParametersFileJsonJsonInXMLSerialisation,
+            self).test_request_parameters_file()
+
+
+class TestCliRequestParametersFileJsonJsonInXMLSerialisationAlreadySerialised(
+    TestCliRequestParametersFileJson):
+  """Request with --parameter-file, with a .json file and a software using
+  json-in-xml for serialisation and parameters already serialised with
+  {'_': json.dumps(params)}. In that case, parameters are not serialized one
+  more time.
+  """
+  expected_partition_parameter_kw = {"_": "{\"foo\": [\"bar\"]}"}
+
+  def _makeParameterFile(self):
+    f = tempfile.NamedTemporaryFile(suffix='.json', mode='w', delete=False)
+    self.addCleanup(os.unlink, f.name)
+    f.write(textwrap.dedent(r'''
+      {"_": "{\"foo\": [\"bar\"]}"}
+    '''))
+    f.flush()
+    return f.name
+
+  def test_request_parameters_file(self):
+    with mock.patch(
+        'slapos.cli.request.SoftwareReleaseSchema.getSerialisation',
+        return_value='json-in-xml'):
+      super(
+          TestCliRequestParametersFileJsonJsonInXMLSerialisationAlreadySerialised,
+          self).test_request_parameters_file()
+
+
+class TestCliRequestParametersFileYaml(TestCliRequestParametersFileJson):
+  """Request with --parameter-file, with a .yaml file. This behaves like json.
+  """
+  def _makeParameterFile(self):
+    f = tempfile.NamedTemporaryFile(suffix='.yaml', mode='w', delete=False)
+    self.addCleanup(os.unlink, f.name)
+    f.write(textwrap.dedent('''\
+      foo:
+      - bar
+    '''))
+    f.flush()
+    return f.name
+
+
+class TestCliRequestParametersFileXml(TestCliRequestParametersFileJson):
+  """Request with --parameter-file, with a .xml file
+  """
+  expected_partition_parameter_kw = {'foo': 'bar'}
+  def _makeParameterFile(self):
+    f = tempfile.NamedTemporaryFile(suffix='.xml', mode='w', delete=False)
+    f.write(textwrap.dedent('''\
+      <?xml version="1.0" encoding="utf-8"?>
+      <instance>
+          <parameter id="foo">bar</parameter>
+      </instance>
+    '''))
+    f.flush()
+    self.addCleanup(os.unlink, f.name)
+    return f.name

slapos/tests/test_slap.py

@@ -32,6 +32,7 @@ from six.moves.urllib import parse
 from six import PY3
 import tempfile
 import logging
+import warnings
 
 from collections import OrderedDict
 import httmock
@@ -872,6 +873,188 @@ class TestComputerPartition(SlapMixin):
         content_list = f.read().splitlines()
         self.assertEqual(sorted(content_list), ['myref', 'mysecondref'])
 
+  def test_request_validate_request_parameter(self):
+
+    def handler(url, req):
+      if url.path.endswith('/software.cfg.json'):
+        return json.dumps(
+          {
+              "name": "Test Software",
+              "description": "Dummy software for Test",
+              "serialisation": "json-in-xml",
+              "software-type": {
+                  'default': {
+                      "title": "Default",
+                      "description": "Default type",
+                      "request": "instance-default-input-schema.json",
+                      "response": "instance-default-output-schema.json",
+                      "index": 0
+                  },
+              }
+          })
+      if url.path.endswith('/instance-default-input-schema.json'):
+        return json.dumps(
+            {
+                "$schema": "http://json-schema.org/draft-07/schema",
+                "description": "Simple instance parameters schema for tests",
+                "required": ["foo"],
+                "properties": {
+                    "foo": {
+                        "$ref": "./schemas-definitions.json#/foo"
+                    }
+                },
+                "type": "object"
+            })
+      if url.path.endswith('/schemas-definitions.json'):
+        return json.dumps({"foo": {"type": "string", "const": "bar"}})
+      raise ValueError(404)
+
+    with httmock.HTTMock(handler):
+      with mock.patch.object(warnings, 'warn') as warn:
+        cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
+        cp._connection_helper = mock.Mock()
+        cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady
+        cp.request(
+            'https://example.com/software.cfg', 'default', 'reference',
+            partition_parameter_kw={'foo': 'bar'})
+      warn.assert_not_called()
+
+    with httmock.HTTMock(handler):
+      with mock.patch.object(warnings, 'warn') as warn:
+        cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
+        cp._connection_helper = mock.Mock()
+        cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady
+        cp.request(
+            'https://example.com/software.cfg', 'default', 'reference',
+            partition_parameter_kw={'foo': 'baz'})
+      if PY3:
+        warn.assert_called_with(
+          "Request parameters do not validate against schema definition:\n"
+          "'bar' was expected\n\n"
+          "Failed validating 'const' in schema['properties']['foo']:\n"
+          "    {'const': 'bar', 'type': 'string'}\n\n"
+          "On instance['foo']:\n    'baz'", UserWarning
+        )
+      else: # BBB
+        warn.assert_called_with(
+          "Request parameters do not validate against schema definition:\n"
+          "u'bar' was expected\n\n"
+          "Failed validating u'const' in schema[u'properties'][u'foo']:\n"
+          "    {u'const': u'bar', u'type': u'string'}\n\n"
+          "On instance[u'foo']:\n    'baz'", UserWarning
+        )
+
+  def test_request_validate_request_parameter_broken_software_release_schema(self):
+    """Corner case tests for incorrect software release schema, these should
+    not prevent the request (mostly for backward compatibility)
+    """
+    def wrong_software_cfg_schema(url, req):
+      if url.path.endswith('/software.cfg.json'):
+        return "wrong"
+      raise ValueError(404)
+
+    def wrong_instance_parameter_schema(url, req):
+      if url.path.endswith('/software.cfg.json'):
+        return json.dumps(
+          {
+              "name": "Test Software",
+              "description": "Dummy software for Test",
+              "serialisation": "json-in-xml",
+              "software-type": {
+                  'default': {
+                      "title": "Default",
+                      "description": "Default type",
+                      "request": "instance-default-input-schema.json",
+                      "response": "instance-default-output-schema.json",
+                      "index": 0
+                  },
+              }
+          })
+      if url.path.endswith('/instance-default-input-schema.json'):
+        return "wrong"
+      raise ValueError(404)
+
+    def invalid_instance_parameter_schema(url, req):
+      if url.path.endswith('/software.cfg.json'):
+        return json.dumps(
+          {
+              "name": "Test Software",
+              "description": "Dummy software for Test",
+              "serialisation": "json-in-xml",
+              "software-type": {
+                  'default': {
+                      "title": "Default",
+                      "description": "Default type",
+                      "request": "instance-default-input-schema.json",
+                      "response": "instance-default-output-schema.json",
+                      "index": 0
+                  },
+              }
+          })
+      if url.path.endswith('/instance-default-input-schema.json'):
+        return json.dumps(
+            {
+                "$schema": "http://json-schema.org/draft-07/schema",
+                "description": "Invalid json schema",
+                "required": {"wrong": True},
+                "properties": {
+                    ["wrong schema"]
+                },
+                "type": "object"
+            })
+      raise ValueError(404)
+
+    def broken_reference(url, req):
+      if url.path.endswith('/software.cfg.json'):
+        return json.dumps(
+          {
+              "name": "Test Software",
+              "description": "Dummy software for Test",
+              "serialisation": "json-in-xml",
+              "software-type": {
+                  'default': {
+                      "title": "Default",
+                      "description": "Default type",
+                      "request": "instance-default-input-schema.json",
+                      "response": "instance-default-output-schema.json",
+                      "index": 0
+                  },
+              }
+          })
+      if url.path.endswith('/instance-default-input-schema.json'):
+        return json.dumps(
+            {
+                "$schema": "http://json-schema.org/draft-07/schema",
+                "description": "Simple instance parameters schema for tests",
+                "required": ["foo"],
+                "properties": {
+                    "foo": {
+                        "$ref": "broken"
+                    }
+                },
+                "type": "object"
+            })
+      raise ValueError(404)
+
+    for handler, warning_expected in (
+        (broken_reference, True),
+        (wrong_software_cfg_schema, False),
+        (wrong_instance_parameter_schema, True),
+        (invalid_instance_parameter_schema, True),
+    ):
+      with httmock.HTTMock(handler):
+        with mock.patch.object(warnings, 'warn') as warn:
+          cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
+          cp._connection_helper = mock.Mock()
+          cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady
+          cp.request(
+              'https://example.com/software.cfg', 'default', 'reference',
+              partition_parameter_kw={'foo': 'bar'})
+        if warning_expected:
+          warn.assert_called()
+        else:
+          warn.assert_not_called()
+
   def _test_new_computer_partition_state(self, state):
     """
     Helper method to automate assertions of failing states on new Computer

slapos/tests/test_util.py

@@ -24,14 +24,26 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+import functools
+import json
+import logging
 import os
-import slapos.util
-from slapos.util import string_to_boolean, unicode2str
+import shutil
 import tempfile
+import textwrap
 import unittest
-import shutil
 from pwd import getpwnam
 
+from six.moves import SimpleHTTPServer
+import jsonschema
+
+import slapos.util
+from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE
+from slapos.testing.utils import ManagedHTTPServer
+from slapos.util import (SoftwareReleaseSchema, SoftwareReleaseSerialisation,
+                         string_to_boolean, unicode2str)
+
+
 class TestUtil(unittest.TestCase):
   """
   Tests methods available in the slapos.util module.
@@ -232,5 +244,221 @@ class TestUtil(unittest.TestCase):
     self.assertRaises(Exception, slapos.util.dumps, Nasty())
 
 
+class SoftwareReleaseSchemaTestXmlSerialisationMixin:
+  serialisation = SoftwareReleaseSerialisation.Xml
+
+
+class SoftwareReleaseSchemaTestJsonInXmlSerialisationMixin:
+  serialisation = SoftwareReleaseSerialisation.JsonInXml
+
+
+class SoftwareReleaseSchemaTestMixin(object):
+  """Mixin with test methods
+  """
+  software_url = None # type: str
+  serialisation = None # type: SoftwareReleaseSerialisation
+
+
+  def test_software_schema(self):
+    schema = SoftwareReleaseSchema(self.software_url, None)
+    software_schema = schema.getSoftwareSchema()
+    self.assertEqual(software_schema['name'], 'Test Software')
+    self.assertEqual(len(software_schema['software-type']), 2)
+
+  def test_serialisation(self):
+    schema = SoftwareReleaseSchema(self.software_url, None)
+    self.assertEqual(schema.getSerialisation(), self.serialisation)
+
+  def test_instance_request_parameter_schema_default_software_type(self):
+    schema = SoftwareReleaseSchema(self.software_url, None)
+    self.assertTrue(schema.getInstanceRequestParameterSchemaURL())
+    instance_parameter_schema = schema.getInstanceRequestParameterSchema()
+    self.assertEqual(instance_parameter_schema['description'],
+                     "Simple instance parameters schema for tests")
+
+  def test_connection_parameter_schema(self):
+    schema = SoftwareReleaseSchema(self.software_url, None)
+    self.assertTrue(schema.getInstanceConnectionParameterSchemaURL())
+    instance_parameter_schema = schema.getInstanceConnectionParameterSchema()
+    self.assertEqual(instance_parameter_schema['description'],
+                     "Simple connection parameters schema for tests")
+
+  def test_instance_request_parameter_validate_default_software_type(self):
+    schema = SoftwareReleaseSchema(self.software_url, None)
+    self.assertTrue(schema.getInstanceRequestParameterSchemaURL())
+    instance_ok = {'key': 'value', 'type': 'default'}
+    schema.validateInstanceParameterDict(instance_ok)
+
+    if self.serialisation == SoftwareReleaseSerialisation.JsonInXml:
+      # already serialized values are also tolerated
+      schema.validateInstanceParameterDict({'_': json.dumps(instance_ok)})
+
+    with self.assertRaises(jsonschema.ValidationError):
+      schema.validateInstanceParameterDict({"wrong": True})
+    instance_ok['key'] = False # wrong type
+    with self.assertRaises(jsonschema.ValidationError):
+      schema.validateInstanceParameterDict(instance_ok)
+    with self.assertRaises(jsonschema.ValidationError):
+      schema.validateInstanceParameterDict({'_': json.dumps(instance_ok)})
+
+  def test_instance_request_parameter_validate_alternate_software_type(self):
+    schema = SoftwareReleaseSchema(self.software_url, 'alternate')
+    self.assertTrue(schema.getInstanceRequestParameterSchemaURL())
+    instance_ok = {'key': 'value', 'type': 'alternate'}
+    schema.validateInstanceParameterDict(instance_ok)
+
+    if self.serialisation == SoftwareReleaseSerialisation.JsonInXml:
+      # already serialized values are also tolerated
+      schema.validateInstanceParameterDict({'_': json.dumps(instance_ok)})
+
+    with self.assertRaises(jsonschema.ValidationError):
+      schema.validateInstanceParameterDict({"wrong": True})
+    instance_ok['type'] = 'wrong'
+    with self.assertRaises(jsonschema.ValidationError):
+      schema.validateInstanceParameterDict(instance_ok)
+    with self.assertRaises(jsonschema.ValidationError):
+      schema.validateInstanceParameterDict({'_': json.dumps(instance_ok)})
+
+  def test_instance_request_parameter_schema_alternate_software_type(self):
+    schema = SoftwareReleaseSchema(self.software_url, 'alternate')
+    self.assertTrue(schema.getInstanceRequestParameterSchemaURL())
+    instance_parameter_schema = schema.getInstanceRequestParameterSchema()
+    self.assertEqual(instance_parameter_schema['description'],
+                     "Simple instance parameters schema for tests")
+
+
+class SoftwareReleaseSchemaTestFileSoftwareReleaseMixin(SoftwareReleaseSchemaTestMixin):
+  """Mixin with tests and software release profiles and schema in a
+  temporary directory.
+  """
+  def setUp(self):
+    self.tmpdir = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, self.tmpdir)
+    tmpfile = functools.partial(os.path.join, self.tmpdir)
+    with open(tmpfile('software.cfg'), 'w') as f:
+      f.write(
+        textwrap.dedent("""\
+          [buildout]
+          """))
+    with open(tmpfile('software.cfg.json'), 'w') as f:
+      json.dump(
+          {
+              "name": "Test Software",
+              "description": "Dummy software for Test",
+              "serialisation": self.serialisation,
+              "software-type": {
+                  DEFAULT_SOFTWARE_TYPE: {
+                      "title": "Default",
+                      "description": "Default type",
+                      "request": "instance-default-input-schema.json",
+                      "response": "instance-default-output-schema.json",
+                      "index": 0
+                  },
+                  "alternate": {
+                      "title": "Alternate",
+                      "description": "Alternate type",
+                      "request": "instance-alternate-input-schema.json",
+                      "response": "instance-alternate-output-schema.json",
+                      "index": 0
+                  },
+              }
+          }, f)
+
+    for software_type in ('default', 'alternate'):
+      with open(
+          tmpfile('instance-{software_type}-input-schema.json'.format(
+              software_type=software_type)), 'w') as f:
+        json.dump(
+            {
+                "$schema": "http://json-schema.org/draft-07/schema",
+                "description": "Simple instance parameters schema for tests",
+                "required": ["key", "type"],
+                "properties": {
+                    "key": {
+                        "$ref": "./schemas-definitions.json#/key"
+                    },
+                    "type": {
+                        "type": "string",
+                        "const": software_type
+                    }
+                },
+                "type": "object"
+            }, f)
+      with open(
+          tmpfile('instance-{software_type}-output-schema.json'.format(
+              software_type=software_type)), 'w') as f:
+        json.dump(
+            {
+                "$schema": "http://json-schema.org/draft-07/schema",
+                "description": "Simple connection parameters schema for tests",
+            }, f)
+
+    with open(tmpfile('schemas-definitions.json'), 'w') as f:
+      json.dump({"key": {"type": "string"}}, f)
+
+    self.software_url = tmpfile('software.cfg')
+
+
+class SoftwareReleaseSchemaTestHTTPSoftwareReleaseMixin(SoftwareReleaseSchemaTestFileSoftwareReleaseMixin):
+  """Mixin serving software release files over http.
+  """
+  def setUp(self):
+    super(SoftwareReleaseSchemaTestHTTPSoftwareReleaseMixin, self).setUp()
+
+    class ProfileHTTPServer(ManagedHTTPServer):
+      hostname = os.environ['SLAPOS_TEST_IPV4']
+      working_directory = self.tmpdir
+      RequestHandler = SimpleHTTPServer.SimpleHTTPRequestHandler
+
+    self.logger = logging.getLogger(self.id())
+    self.logger.propagate = False
+
+    server = ProfileHTTPServer(self, 'server')
+    server.open()
+    self.addCleanup(server.close)
+
+    self.software_url = server.url + '/software.cfg'
+
+
+class TestSoftwareReleaseSchemaFileSoftwareReleaseXmlSerialisation(
+    SoftwareReleaseSchemaTestXmlSerialisationMixin,
+    SoftwareReleaseSchemaTestFileSoftwareReleaseMixin,
+    unittest.TestCase):
+  pass
+
+
+class TestSoftwareReleaseSchemaFileSoftwareReleaseJsonInXmlSerialisation(
+    SoftwareReleaseSchemaTestJsonInXmlSerialisationMixin,
+    SoftwareReleaseSchemaTestFileSoftwareReleaseMixin,
+    unittest.TestCase):
+  pass
+
+
+class TestSoftwareReleaseSchemaHTTPSoftwareReleaseXmlSerialisation(
+    SoftwareReleaseSchemaTestXmlSerialisationMixin,
+    SoftwareReleaseSchemaTestHTTPSoftwareReleaseMixin,
+    unittest.TestCase):
+  pass
+
+
+class TestSoftwareReleaseSchemaHTTPSoftwareReleaseJsonInXmlSerialisation(
+    SoftwareReleaseSchemaTestJsonInXmlSerialisationMixin,
+    SoftwareReleaseSchemaTestHTTPSoftwareReleaseMixin,
+    unittest.TestCase):
+  pass
+
+
+
+class TestSoftwareReleaseSchemaEdgeCases(unittest.TestCase):
+
+  def test_software_schema_file_not_exist(self):
+    schema = SoftwareReleaseSchema('/file/not/exist', None)
+    self.assertIsNone(schema.getSoftwareSchema())
+
+  def test_software_schema_wrong_URL(self):
+    schema = SoftwareReleaseSchema('http://slapos.invalid/software.cfg', None)
+    self.assertIsNone(schema.getSoftwareSchema())
+
+
 if __name__ == '__main__':
   unittest.main()

slapos/util.py

@@ -27,19 +27,32 @@
 #
 ##############################################################################
 
+import enum
 import errno
+import hashlib
+import json
 import os
+import shutil
 import socket
+import sqlite3
 import struct
 import subprocess
-import sqlite3
-from xml_marshaller.xml_marshaller import Marshaller, Unmarshaller
-from lxml import etree
+import warnings
+
+import jsonschema
+import netaddr
+import requests
 import six
+from lxml import etree
 from six.moves.urllib import parse
-import hashlib
-import netaddr
-import shutil
+from six.moves.urllib_parse import urljoin
+from xml_marshaller.xml_marshaller import Marshaller, Unmarshaller
+
+try:
+  from typing import Dict, Optional, IO
+except ImportError:
+  pass
+
 
 
 try:
@@ -307,3 +320,148 @@ def rmtree(path):
     raise e  # XXX make pylint happy
 
   shutil.rmtree(path, onerror=chmod_retry)
+
+
+
+def _readAsJson(url):
+  # type: (str) -> Optional[Dict]
+  """Reads and parse the json file located at `url`.
+
+  `url` can also be the path of a local file.
+  """
+  if url.startswith('file://'):
+    url = url[len('file://'):]
+  path = url if os.path.exists(url) else None
+
+  if path:
+    with open(path) as f:
+      try:
+        return json.load(f)
+      except ValueError:
+        return None
+  if url.startswith('http://') or url.startswith('https://'):
+    try:
+      r = requests.get(url)
+      r.raise_for_status()
+      return r.json()
+    except (requests.exceptions.RequestException, ValueError):
+      return None
+  return None
+
+
+class SoftwareReleaseSerialisation(str, enum.Enum):
+  Xml = 'xml'
+  JsonInXml = 'json-in-xml'
+
+
+class SoftwareReleaseSchema(object):
+  def __init__(self, software_url, software_type):
+    # type: (str, Optional[str]) ->  None
+    self.software_url = software_url
+    self.software_type = software_type
+
+  def getSoftwareSchema(self):
+    # type: () -> Optional[Dict]
+    """Returns the schema for this software.
+    """
+    return _readAsJson(self.software_url + '.json')
+
+  def getSoftwareTypeSchema(self):
+    # type: () -> Optional[Dict]
+    """Returns schema for this software type.
+    """
+    software_schema = self.getSoftwareSchema()
+    if software_schema is None:
+      return None
+
+    software_type = self.software_type
+    from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE # TODO
+    if software_type is None:
+      software_type = DEFAULT_SOFTWARE_TYPE
+    # XXX some software are using "default" for default software type
+    if software_type == DEFAULT_SOFTWARE_TYPE \
+        and software_type not in software_schema['software-type'] \
+        and 'default' in software_schema['software-type']:
+      warnings.warn(
+          "Software release {} does not have schema for DEFAULT_SOFTWARE_TYPE but has one for 'default'."
+          " Using 'default' instead.".format(self.software_url),
+          UserWarning,
+      )
+      software_type = 'default'
+    return software_schema['software-type'].get(software_type)
+
+  def getSerialisation(self):
+    # type: () -> Optional[SoftwareReleaseSerialisation]
+    """Returns the serialisation method used for parameters.
+    """
+    software_schema = self.getSoftwareSchema()
+    if software_schema is None:
+      return None
+    return SoftwareReleaseSerialisation(software_schema['serialisation'])
+
+  def getInstanceRequestParameterSchemaURL(self):
+    # type: () -> Optional[str]
+    """Returns the URL of the schema defining instance parameters.
+    """
+    software_type_schema = self.getSoftwareTypeSchema()
+    if software_type_schema is None:
+      return None
+    software_url = self.software_url
+    if os.path.exists(software_url):
+      software_url = 'file://' + software_url
+    return urljoin(software_url, software_type_schema['request'])
+
+  def getInstanceRequestParameterSchema(self):
+    # type: () -> Optional[Dict]
+    """Returns the schema defining instance parameters.
+    """
+    instance_parameter_schema_url = self.getInstanceRequestParameterSchemaURL()
+    if instance_parameter_schema_url is None:
+      return None
+    schema = _readAsJson(instance_parameter_schema_url)
+    if schema:
+      # so that jsonschema knows how to resolve references
+      schema.setdefault('$id', instance_parameter_schema_url)
+    return schema
+
+  def getInstanceConnectionParameterSchemaURL(self):
+    # type: () -> Optional[str]
+    """Returns the URL of the schema defining connection parameters published by the instance.
+    """
+    software_type_schema = self.getSoftwareTypeSchema()
+    if software_type_schema is None:
+      return None
+    return urljoin(self.software_url, software_type_schema['response'])
+
+  def getInstanceConnectionParameterSchema(self):
+    # type: () -> Optional[Dict]
+    """Returns the schema defining connection parameters published by the instance.
+    """
+    instance_parameter_schema_url = self.getInstanceConnectionParameterSchemaURL()
+    if instance_parameter_schema_url is None:
+      return None
+    schema = _readAsJson(instance_parameter_schema_url)
+    if schema:
+      # so that jsonschema knows how to resolve references
+      schema.setdefault('$id', instance_parameter_schema_url)
+    return schema
+
+  def validateInstanceParameterDict(self, parameter_dict):
+    # type: (Dict) -> None
+    """Validate instance parameters against the software schema.
+
+    Raise jsonschema.ValidationError if parameters does not validate.
+    """
+    schema_url = self.getInstanceRequestParameterSchemaURL()
+    if schema_url:
+      instance = parameter_dict
+      if self.getSerialisation() == SoftwareReleaseSerialisation.JsonInXml:
+        try:
+          instance = json.loads(parameter_dict['_'])
+        except KeyError:
+          instance = parameter_dict
+      instance.pop('$schema', None)
+      jsonschema.validate(
+          instance=instance,
+          schema=self.getInstanceRequestParameterSchema(),
+      )