slapproxy: Add minimal hateoas support for cli

Add the minimum hateoas support required for the cli commands:
- `slapos service list`
- `slapos service info <reference>`
- `slapos computer list`
- `slapos computer info <reference>`

To enable hateoas, complete the slapos.cfg configuration with:
```
[slapos]
master_rest_url = <proxyaddr>/hateoas
```

Also: Fix some bugs and python3 incompatibilities in slap/hateoas.py.

See merge request nexedi/slapos.core!290

slapos/proxy/views.py

@@ -31,6 +31,7 @@
 import random
 import string
 import time
+import re
 from datetime import datetime
 from slapos.slap.slap import Computer, ComputerPartition, \
     SoftwareRelease, SoftwareInstance, NotFoundError
@@ -39,12 +40,12 @@ import slapos.slap
 from slapos.util import bytes2str, unicode2str, sqlite_connect, \
     xml2dict, dict2xml
 
-from flask import g, Flask, request, abort
+from flask import g, Flask, request, abort, redirect, url_for
 from slapos.util import loads, dumps
 
 import six
 from six.moves import range
-from six.moves.urllib.parse import urlparse
+from six.moves.urllib.parse import urlparse, unquote, urljoin
 
 app = Flask(__name__)
 
@@ -930,3 +931,186 @@ def getSoftwareReleaseListFromSoftwareProduct():
       software_release_url_list = []
     return dumps(software_release_url_list)
 
+
+# hateoas routing
+# ---------------
+
+# We only need to handle the hateoas requests made by
+#   slapos service list
+#   slapos service info <reference>
+#   slapos computer list
+#   slapos computer info <reference>
+
+
+def unquoted_url_for(method, **kwargs):
+  return unquote(url_for(method, **kwargs))
+
+def busy_root_partitions_list(title=None):
+  partitions = []
+  query = 'SELECT * FROM %s WHERE slap_state<>"free" AND requested_by IS NULL'
+  args = []
+  if title:
+    query += ' AND partition_reference=?'
+    args.append(title)
+  for row in execute_db('partition', query, args):
+    p = dict(row)
+    p['url_string'] = p['software_release']
+    p['title'] = p['partition_reference']
+    p['relative_url'] = url_for('hateoas_partitions', partition_reference=p['partition_reference'])
+    partitions.append(p)
+  return partitions
+
+def computers_list(reference=None):
+  computers = []
+  query = 'SELECT * FROM %s'
+  args = []
+  if reference:
+    query += ' WHERE reference=?'
+    args.append(reference)
+  for row in execute_db('computer', query, args):
+    c = dict(row)
+    c['title'] = c['reference']
+    c['relative_url'] = url_for('hateoas_computers', computer_reference=c['reference'])
+    computers.append(c)
+  return computers
+
+r_string = re.compile('"((\\.|[^\\"])*)"')
+
+def is_valid(name):
+  match = r_string.match(name)
+  if match.group(0) == name:
+    return True
+  return False
+
+p_service_list = 'portal_type:"Hosting Subscription" AND validation_state:validated'
+p_service_info = p_service_list + ' AND title:='
+p_computer_list = 'portal_type:"Computer" AND validation_state:validated'
+p_computer_info = p_computer_list + ' AND reference:='
+
+def parse_query(query):
+  if query == p_service_list:
+    return busy_root_partitions_list()
+  elif query.startswith(p_service_info):
+    title = query[len(p_service_info):]
+    if is_valid(title):
+      return busy_root_partitions_list(title.strip('"'))
+  elif query == p_computer_list:
+    return computers_list()
+  elif query.startswith(p_computer_info):
+    reference = query[len(p_computer_info):]
+    if is_valid(reference):
+      return computers_list(reference.strip('"'))
+  return None
+
+@app.route('/hateoas/partitions/<partition_reference>', methods=['GET'])
+def hateoas_partitions(partition_reference):
+  partition = execute_db('partition', 'SELECT * FROM %s WHERE partition_reference=?', [partition_reference], one=True)
+  if partition is None:
+    abort(404)
+  return {
+    '_embedded': {
+      '_view': {
+        'form_id': {
+          'type': 'StringField',
+          'key': 'partition',
+          'default': partition['reference'],
+        },
+        'my_reference': {
+          'type': 'StringField',
+          'key': 'partition_reference',
+          'default': partition['partition_reference'],
+        },
+        'my_slap_state': {
+          'type': 'StringField',
+          'key': 'slap_state',
+          'default': partition['slap_state'],
+        },
+        'my_text_content': {
+          'type': 'StringField',
+          'key': 'xml',
+          'default': partition['xml'],
+        },
+        'my_connection_parameter_list': {
+          'type': 'StringField',
+          'key': 'connection_xml',
+          'default': partition['connection_xml'],
+        },
+        'my_url_string': {
+          'type': 'StringField',
+          'key': 'software_release',
+          'default': partition['software_release'],
+        },
+      },
+    },
+    '_links': {
+      'type': {
+        'name': 'Hosting Subscription',
+      },
+    },
+  }
+
+@app.route('/hateoas/computers/<computer_reference>', methods=['GET'])
+def hateoas_computers(computer_reference):
+  computer = execute_db('computer', 'SELECT * FROM %s WHERE reference=?', [computer_reference], one=True)
+  if computer is None:
+    abort(404)
+  return {
+    '_embedded': {
+      '_view': {
+        'form_id': {
+          'type': 'StringField',
+          'key': 'computer',
+          'default': computer['reference'],
+        },
+        'my_reference': {
+          'type': 'StringField',
+          'key': 'reference',
+          'default': computer['reference'],
+        },
+        'my_title': {
+          'type': 'StringField',
+          'key': 'reference',
+          'default': computer['reference'],
+        },
+      },
+    },
+    '_links': {
+      'type': {
+        'name': 'Computer',
+      },
+    },
+  }
+
+def hateoas_traverse():
+  return redirect(request.args.get('relative_url'))
+
+def hateoas_search():
+  contents = parse_query(request.args.get("query"))
+  if contents is None:
+    abort(400)
+  return { '_embedded': {'contents': contents} }
+
+def hateoas_root():
+  return {
+    '_links': {
+      'raw_search': {
+      'href': urljoin(request.host_url, unquoted_url_for('hateoas', mode='search', query='{query}', select_list='{select_list}'))
+    },
+      'traverse': {
+        'href': urljoin(request.host_url, unquoted_url_for('hateoas', mode='traverse', relative_url='{relative_url}', view='{view}'))
+      },
+    }
+  }
+
+mode_handlers = {
+  None: hateoas_root,
+  'search': hateoas_search,
+  'traverse': hateoas_traverse,
+}
+
+@app.route('/hateoas', methods=['GET'])
+def hateoas():
+  mode = request.args.get('mode')
+  handler = mode_handlers.get(mode, lambda: abort(400))
+  resp = handler()
+  return resp

slapos/slap/hateoas.py

@@ -383,7 +383,7 @@ class SlapHateoasNavigator(HateoasNavigator):
     hosting_subscription_dict = {}
     for hosting_subscription in hosting_subscription_list:
       software_instance = TempDocument()
-      for key, value in hosting_subscription.iteritems():
+      for key, value in six.iteritems(hosting_subscription):
         if key in ['_links', 'url_string']:
           continue
         setattr(software_instance, '_%s' % key, value)
@@ -397,7 +397,7 @@ class SlapHateoasNavigator(HateoasNavigator):
     computer_dict = {}
     for computer_json  in computer_list:
       computer = TempDocument()
-      for key, value in computer_json.iteritems():
+      for key, value in six.iteritems(computer_json):
         if key in ['_links']:
           continue
         setattr(computer, '_%s' % key, value)
@@ -420,6 +420,7 @@ class SlapHateoasNavigator(HateoasNavigator):
     assert len(hosting_subscription_list) <= 1, \
       "There are more them one Hosting Subscription for this reference"
 
+    hosting_subscription_jio_key= None
     for hosting_subscription_candidate in hosting_subscription_list:
       if hosting_subscription_candidate.get('title') == reference:
         hosting_subscription_jio_key = hosting_subscription_candidate['relative_url']
@@ -437,6 +438,7 @@ class SlapHateoasNavigator(HateoasNavigator):
     assert len(computer_list) <= 1, \
       "There are more them one Computer for this reference"
 
+    computer_jio_key = None
     for computer_candidate in computer_list:
       if computer_candidate.get("reference") == reference:
         computer_jio_key = computer_candidate['relative_url']

slapos/tests/test_slapproxy.py

@@ -42,8 +42,12 @@ import sys
 import tempfile
 import time
 import unittest
+import json
 import mock
 
+import requests
+from six.moves.urllib.parse import urljoin
+
 import slapos.proxy
 import slapos.proxy.views as views
 import slapos.slap
@@ -1163,6 +1167,185 @@ class TestSlaveRequest(MasterMixin):
     self.assertEqual(slave._partition_id, partition._partition_id)
 
 
+class TestAppSession(requests.Session):
+  """
+  A request session that exposes the necessary interface to seamlessly
+  replace the object returned by views.app.test_client().
+  """
+  def __init__(self, prefix_url, *args, **kwargs):
+    super(TestAppSession, self).__init__(*args, **kwargs)
+    self.prefix_url = prefix_url
+
+  def request(self, method, url, *args, **kwargs):
+    url = urljoin(self.prefix_url, url)
+    resp = super(TestAppSession, self).request(method, url, *args, **kwargs)
+    setattr(resp, '_status_code', resp.status_code)
+    setattr(resp, 'data', resp.content)
+    return resp
+
+class CliMasterMixin(MasterMixin):
+  """
+  Start a real proxy via the cli so that it will anwser to cli requests.
+  """
+  def createSlapOSConfigurationFile(self):
+    host = os.environ['SLAPOS_TEST_IPV4']
+    self.proxyaddr = 'http://%s:8080' % host
+    with open(self.slapos_cfg, 'w') as f:
+      f.write("""[slapos]
+software_root = %(tempdir)s/opt/slapgrid
+instance_root = %(tempdir)s/srv/slapgrid
+master_url = %(proxyaddr)s
+master_rest_url = %(proxyaddr)s/hateoas
+computer_id = computer
+[slapproxy]
+host = %(host)s
+port = 8080
+database_uri = %(tempdir)s/lib/proxy.db
+""" % {'tempdir': self._tempdir, 'proxyaddr': self.proxyaddr, 'host': host})
+
+  def cliDoSlapos(self, command, method=subprocess.check_output, **kwargs):
+    return method(
+      (sys.executable, '-m', 'slapos.cli.entry') + command + ('--cfg', self.slapos_cfg),
+      env={"PYTHONPATH": ':'.join(sys.path)},
+      cwd=os.chdir(os.path.join(os.path.dirname(slapos.proxy.__file__), os.pardir, os.pardir)),
+      universal_newlines=True,
+      **kwargs
+    )
+
+  def startProxy(self):
+    self.proxy_process = self.cliDoSlapos(
+      ('proxy', 'start'),
+      method=subprocess.Popen,
+      stdout=subprocess.DEVNULL,
+      stderr=subprocess.DEVNULL,
+    )
+    self.app = TestAppSession(self.proxyaddr)
+    # Wait a bit for proxy to be started
+    for attempts in range(1, 20):
+      try:
+        self.app.get('/')
+      except requests.ConnectionError:
+        time.sleep(0.1 * attempts)
+      else:
+        break
+    else:
+      self.fail('Could not start proxy.')
+
+  def tearDown(self):
+    self.proxy_process.kill()
+    self.proxy_process.wait()
+    super(CliMasterMixin, self).tearDown()
+
+
+class TestCliInformation(CliMasterMixin):
+  """
+  Test minimal hateoas support for cli.
+  """
+  def test_computer_list(self):
+    self.format_for_number_of_partitions(1)
+    self.format_for_number_of_partitions(1, 'COMP-1')
+    self.format_for_number_of_partitions(1, 'COMP-2')
+    output = self.cliDoSlapos(('computer', 'list'), stderr=subprocess.DEVNULL).splitlines()
+    self.assertEqual(len(output), 4)
+    self.assertEqual(output[0], 'List of Computers:')
+    self.assertEqual(
+      sorted(output[1:]),
+      ['COMP-1 COMP-1', 'COMP-2 COMP-2', 'computer computer'],
+    )
+
+  def test_computer_info(self):
+    self.format_for_number_of_partitions(1)
+    self.format_for_number_of_partitions(1, 'COMP-1')
+    self.format_for_number_of_partitions(1, 'COMP-2')
+    output0 = self.cliDoSlapos(('computer', 'info', 'computer'), stderr=subprocess.DEVNULL)
+    self.assertEqual(
+      output0.splitlines(),
+      ['Computer Reference: computer', 'Computer Title    : computer'],
+    )
+    output1 = self.cliDoSlapos(('computer', 'info', 'COMP-1'), stderr=subprocess.DEVNULL)
+    self.assertEqual(
+      output1.splitlines(),
+      ['Computer Reference: COMP-1', 'Computer Title    : COMP-1'],
+    )
+    output2 = self.cliDoSlapos(('computer', 'info', 'COMP-2'), stderr=subprocess.DEVNULL)
+    self.assertEqual(
+      output2.splitlines(),
+      ['Computer Reference: COMP-2', 'Computer Title    : COMP-2'],
+    )
+
+  def test_service_list(self):
+    self.format_for_number_of_partitions(4)
+    self.request('http://sr0//', None, 'MyInstance0', None)
+    self.request('http://sr1//', None, 'MyInstance1', None)
+    self.request('http://sr2//', None, 'MyInstance2', None)
+    self.request('http://sr3//', None, 'MyInstance3', 'slappart0')
+    output = self.cliDoSlapos(('service', 'list'), stderr=subprocess.DEVNULL).splitlines()
+    self.assertEqual(len(output), 4)
+    self.assertEqual(output[0], 'List of services:')
+    self.assertEqual(
+      sorted(output[1:]),
+      ['MyInstance0 http://sr0//', 'MyInstance1 http://sr1//', 'MyInstance2 http://sr2//'],
+    )
+
+  def test_service_info(self):
+    self.format_for_number_of_partitions(3)
+    self.request('http://sr0//', None, 'MyInstance0', None)
+    self.request('http://sr1//', None, 'MyInstance1', None, partition_parameter_kw={'couscous': 'hello'})
+    self.request('http://sr2//', None, 'MyInstance2', 'slappart0')
+    output0 = self.cliDoSlapos(('service', 'info', 'MyInstance0'), stderr=subprocess.DEVNULL)
+    self.assertEqual(
+      output0.splitlines(),
+      [
+        'Software Release URL: http://sr0//',
+        'Instance state: busy',
+        'Instance parameters:',
+        '{}',
+        'Connection parameters:',
+        'None'
+      ],
+    )
+    output1 = self.cliDoSlapos(('service', 'info', 'MyInstance1'), stderr=subprocess.DEVNULL)
+    self.assertEqual(
+      output1.splitlines(),
+      [
+        'Software Release URL: http://sr1//',
+        'Instance state: busy',
+        'Instance parameters:',
+        "{'couscous': 'hello'}",
+        'Connection parameters:',
+        'None'
+      ],
+    )
+    try:
+      output2 = self.cliDoSlapos(('service', 'info', 'MyInstance2'), stderr=subprocess.STDOUT)
+      self.fail()
+    except subprocess.CalledProcessError as e:
+      self.assertIn('Instance MyInstance2 does not exist.', e.output)
+
+  def test_invalid_service_names(self):
+    invalid_names = ('"MyInstance0', 'MyInstance1"', 'My"Instance2', 'title:="MyInstance3"', 'reference:="MyInstance4"')
+    self.format_for_number_of_partitions(len(invalid_names))
+    for i, name in enumerate(invalid_names):
+      self.request('http://sr%d//' % i, None, name)
+    for i, name in enumerate(invalid_names):
+      try:
+        self.cliDoSlapos(('service', 'info', name), method=subprocess.check_output, stderr=subprocess.STDOUT)
+        self.fail()
+      except subprocess.CalledProcessError as e:
+        self.assertIn("HTTPError: 400 Client Error: BAD REQUEST", e.output)
+
+  def test_invalid_computer_names(self):
+    invalid_names = ('"COMP-0', 'COMP-1"', 'COMP"-2', 'title:="COMP-3"', 'reference:="COMP-4"')
+    for name in invalid_names:
+      self.format_for_number_of_partitions(1, name)
+    for name in invalid_names:
+      try:
+        self.cliDoSlapos(('computer', 'info', name), method=subprocess.check_output, stderr=subprocess.STDOUT)
+        self.fail()
+      except subprocess.CalledProcessError as e:
+        self.assertIn("HTTPError: 400 Client Error: BAD REQUEST", e.output)
+
+
 class TestMultiNodeSupport(MasterMixin):
   def test_multi_node_support_different_software_release_list(self):
     """