kvm: Switch default and kvm-cluster to json-in-xml

default and kvm-cluster software types are now json-in-xml, so they accept
and return json-in-xml.

Update tests to show that xml and json-in-xml serialization works.

Note: There was invective to publish with pure XML serialisation, but it is
wrong idea because:

 1) serialisation in the software.cfg.json is provided for both ways
 2) requesting instances with given serialisation by profiles expect same
    serialisation

Request resilient instances with serialised, as they reply with json-in-xml.

software/kvm/buildout.hash.cfg

@@ -19,15 +19,15 @@ md5sum = 7e90da1f6dac4233e1aa3248f48e357c
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = fb8f9a962cc1c0d986305c6a6ce59082
+md5sum = 6db48369e93d740db3cf283f021a9652
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = 8562882ac4a888d18db50e2a22e3ca86
+md5sum = 406bf455afe1c71cb0e8cc04dd877f1d
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2
-md5sum = a0fd4911401cbbda74323e8d1c7b18ad
+md5sum = 839fc16c112d3b87e2dbd2e382e326de
 
 [template-kvm-import]
 filename = instance-kvm-import.cfg.jinja2.in

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -32,7 +32,6 @@ global-ipv6 = {{ ipv6 }}
 {% endfor -%}
 
 [request-common]
-recipe = slapos.cookbook:request
 software-url = ${slap-connection:software-release-url}
 server-url = ${slap-connection:server-url}
 key-file = ${slap-connection:key-file}
@@ -47,6 +46,7 @@ config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 {% set use_nat = kvm_parameter_dict.get('use-nat', True) -%}
 [{{ section }}]
 <= request-common
+recipe = slapos.cookbook:request.serialised
 software-type = kvm
 name = {{ instance_name }}
 {% if kvm_parameter_dict.get('sticky-computer', '') -%}
@@ -319,7 +319,7 @@ private-path-list +=
 [publish-connection-information]
 <= monitor-publish
 -extends = publish-early
-recipe = slapos.cookbook:publish
+recipe = slapos.cookbook:publish.serialised
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor %}

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -61,6 +61,7 @@ etc = ${buildout:directory}/etc
 
 # Bubble down the parameters of the requested instance to the user
 [request-kvm]
+recipe = slapos.cookbook:request.serialised
 # Note: += doesn't work.
 return =
 # Resilient related parameters

software/kvm/instance-kvm.cfg.jinja2

@@ -819,7 +819,9 @@ blank-line =
 
 [publish-connection-information]
 <= monitor-publish
-recipe = slapos.cookbook:publish
+# even if serialisation is json-in-xml, still use xml for publish, as
+# SlapOS Master implementation does not support it
+recipe = slapos.cookbook:publish.serialised
 ipv6 = ${slap-network-information:global-ipv6}
 backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
 url = ${request-slave-frontend:connection-secure_access}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}

software/kvm/software.cfg.json

@@ -5,6 +5,7 @@
   "software-type": {
     "default": {
       "title": "Default",
+      "serialisation": "json-in-xml",
       "description": "Standalone KVM",
       "request": "instance-kvm-input-schema.json",
       "response": "instance-kvm-output-schema.json",

software/kvm/test/test.py

@@ -119,6 +119,10 @@ bootstrap_machine_param_dict = {
 
 
 class KvmMixin(object):
+  def getConnectionParameterDictJson(self):
+    return json.loads(
+      self.computer_partition.getConnectionParameterDict()['_'])
+
   def getProcessInfo(self):
     hash_value = generateHashFromFiles([
       os.path.join(self.computer_partition_root_path, hash_file)
@@ -161,13 +165,25 @@ class KvmMixin(object):
         state=state)
 
 
+class KvmMixinJson(object):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      '_': json.dumps(super(KvmMixinJson, cls).getInstanceParameterDict())}
+
+  def rerequestInstance(self, parameter_dict, *args, **kwargs):
+    return super(KvmMixinJson, self).rerequestInstance(
+      parameter_dict={'_': json.dumps(parameter_dict)},
+      *args, **kwargs
+    )
+
+
 @skipUnlessKvm
 class TestInstance(InstanceTestCase, KvmMixin):
   __partition_reference__ = 'i'
 
   def test(self):
-    connection_parameter_dict = self\
-      .computer_partition.getConnectionParameterDict()
+    connection_parameter_dict = self.getConnectionParameterDictJson()
     present_key_list = []
     assert_key_list = [
      'backend-url', 'url', 'monitor-setup-url', 'ipv6-network-info',
@@ -308,7 +324,12 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
     )
 
 
-class MonitorAccessMixin(object):
+@skipUnlessKvm
+class TestMemoryManagementJson(KvmMixinJson, TestMemoryManagement):
+  pass
+
+
+class MonitorAccessMixin(KvmMixin):
   def sqlite3_connect(self):
     sqlitedb_file = os.path.join(
       os.path.abspath(
@@ -335,8 +356,7 @@ class MonitorAccessMixin(object):
       db.close()
 
   def test_access_monitor(self):
-    connection_parameter_dict = self.computer_partition\
-      .getConnectionParameterDict()
+    connection_parameter_dict = self.getConnectionParameterDictJson()
     monitor_setup_url = connection_parameter_dict['monitor-setup-url']
     monitor_url_with_auth = 'https' + monitor_setup_url.split('https')[2]
 
@@ -349,8 +369,8 @@ class MonitorAccessMixin(object):
       connection_xml = partition_information.get('connection_xml')
       if not connection_xml:
         continue
-      connection_dict = slapos.util.xml2dict(
-        connection_xml if six.PY3 else connection_xml.encode('utf-8'))
+      connection_dict = json.loads(slapos.util.xml2dict(
+        connection_xml if six.PY3 else connection_xml.encode('utf-8'))['_'])
       monitor_base_url = connection_dict.get('monitor-base-url')
       if not monitor_base_url:
         continue
@@ -377,8 +397,7 @@ class TestAccessDefault(MonitorAccessMixin, InstanceTestCase):
   expected_partition_with_monitor_base_url_count = 1
 
   def test(self):
-    connection_parameter_dict = self.computer_partition\
-      .getConnectionParameterDict()
+    connection_parameter_dict = self.getConnectionParameterDictJson()
     result = requests.get(connection_parameter_dict['url'], verify=False)
     self.assertEqual(
       httplib.OK,
@@ -388,6 +407,11 @@ class TestAccessDefault(MonitorAccessMixin, InstanceTestCase):
     self.assertNotIn('url-additional', connection_parameter_dict)
 
 
+@skipUnlessKvm
+class TestAccessDefaultJson(KvmMixinJson, TestAccessDefault):
+  pass
+
+
 @skipUnlessKvm
 class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
   __partition_reference__ = 'ada'
@@ -400,8 +424,7 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
     }
 
   def test(self):
-    connection_parameter_dict = self.computer_partition\
-      .getConnectionParameterDict()
+    connection_parameter_dict = self.getConnectionParameterDictJson()
 
     result = requests.get(connection_parameter_dict['url'], verify=False)
     self.assertEqual(
@@ -419,6 +442,12 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
     self.assertIn('<title>noVNC</title>', result.text)
 
 
+@skipUnlessKvm
+class TestAccessDefaultAdditionalJson(
+  KvmMixinJson, TestAccessDefaultAdditional):
+  pass
+
+
 @skipUnlessKvm
 class TestAccessDefaultBootstrap(MonitorAccessMixin, InstanceTestCase):
   __partition_reference__ = 'adb'
@@ -456,8 +485,7 @@ class TestAccessDefaultBootstrap(MonitorAccessMixin, InstanceTestCase):
     self.slap.waitForInstance(max_retry=10)
     # END: mock .slapos-resource with tap.ipv4_addr
 
-    connection_parameter_dict = self.computer_partition\
-      .getConnectionParameterDict()
+    connection_parameter_dict = self.getConnectionParameterDictJson()
 
     result = requests.get(connection_parameter_dict['url'], verify=False)
     self.assertEqual(
@@ -493,8 +521,7 @@ class TestAccessKvmCluster(MonitorAccessMixin, InstanceTestCase):
     })}
 
   def test(self):
-    connection_parameter_dict = self.computer_partition\
-      .getConnectionParameterDict()
+    connection_parameter_dict = self.getConnectionParameterDictJson()
     result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
     self.assertEqual(
       httplib.OK,
@@ -527,8 +554,7 @@ class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase):
     })}
 
   def test(self):
-    connection_parameter_dict = self.computer_partition\
-      .getConnectionParameterDict()
+    connection_parameter_dict = self.getConnectionParameterDictJson()
     result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
     self.assertEqual(
       httplib.OK,
@@ -571,8 +597,7 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
     }))}
 
   def test(self):
-    connection_parameter_dict = self.computer_partition\
-      .getConnectionParameterDict()
+    connection_parameter_dict = self.getConnectionParameterDictJson()
     result = requests.get(
       connection_parameter_dict['test-machine1-url'], verify=False)
     self.assertEqual(
@@ -697,6 +722,12 @@ ir3:sshd-on-watch RUNNING""",
     )
 
 
+@skipUnlessKvm
+class TestInstanceResilientJson(
+  KvmMixinJson, TestInstanceResilient):
+  pass
+
+
 @skipUnlessKvm
 class TestInstanceResilientDiskTypeIde(InstanceTestCase, KvmMixin):
   @classmethod
@@ -706,6 +737,12 @@ class TestInstanceResilientDiskTypeIde(InstanceTestCase, KvmMixin):
     }
 
 
+@skipUnlessKvm
+class TestInstanceResilientDiskTypeIdeJson(
+  KvmMixinJson, TestInstanceResilientDiskTypeIde):
+  pass
+
+
 @skipUnlessKvm
 class TestAccessResilientAdditional(InstanceTestCase):
   __partition_reference__ = 'ara'
@@ -741,6 +778,12 @@ class TestAccessResilientAdditional(InstanceTestCase):
     self.assertIn('<title>noVNC</title>', result.text)
 
 
+@skipUnlessKvm
+class TestAccessResilientAdditionalJson(
+  KvmMixinJson, TestAccessResilientAdditional):
+  pass
+
+
 class TestInstanceNbdServer(InstanceTestCase):
   __partition_reference__ = 'ins'
   instance_max_retry = 5
@@ -769,6 +812,12 @@ class TestInstanceNbdServer(InstanceTestCase):
     self.assertIn("WARNING", connection_parameter_dict['status_message'])
 
 
+@skipUnlessKvm
+class TestInstanceNbdServerJson(
+  KvmMixinJson, TestInstanceNbdServer):
+  pass
+
+
 class FakeImageHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
   def log_message(self, *args):
     if os.environ.get('SLAPOS_TEST_DEBUG'):
@@ -1047,6 +1096,12 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
     self.assertPromiseFails(self.config_state_promise)
 
 
+@skipUnlessKvm
+class TestBootImageUrlListJson(
+  KvmMixinJson, TestBootImageUrlList):
+  pass
+
+
 @skipUnlessKvm
 class TestBootImageUrlListResilient(TestBootImageUrlList):
   kvm_instance_partition_reference = 'biul2'
@@ -1056,6 +1111,12 @@ class TestBootImageUrlListResilient(TestBootImageUrlList):
     return 'kvm-resilient'
 
 
+@skipUnlessKvm
+class TestBootImageUrlListResilientJson(
+  KvmMixinJson, TestBootImageUrlListResilient):
+  pass
+
+
 @skipUnlessKvm
 class TestBootImageUrlSelect(TestBootImageUrlList):
   __partition_reference__ = 'bius'
@@ -1158,6 +1219,12 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     )
 
 
+@skipUnlessKvm
+class TestBootImageUrlSelectJson(
+  KvmMixinJson, TestBootImageUrlSelect):
+  pass
+
+
 @skipUnlessKvm
 class TestBootImageUrlSelectResilient(TestBootImageUrlSelect):
   kvm_instance_partition_reference = 'bius2'
@@ -1167,6 +1234,12 @@ class TestBootImageUrlSelectResilient(TestBootImageUrlSelect):
     return 'kvm-resilient'
 
 
+@skipUnlessKvm
+class TestBootImageUrlSelectResilientJson(
+  KvmMixinJson, TestBootImageUrlSelectResilient):
+  pass
+
+
 @skipUnlessKvm
 class TestBootImageUrlListKvmCluster(InstanceTestCase, FakeImageServerMixin):
   __partition_reference__ = 'biulkc'
@@ -1246,7 +1319,7 @@ class TestBootImageUrlSelectKvmCluster(TestBootImageUrlListKvmCluster):
 
 
 @skipUnlessKvm
-class TestNatRules(InstanceTestCase):
+class TestNatRules(KvmMixin, InstanceTestCase):
   __partition_reference__ = 'nr'
 
   @classmethod
@@ -1256,8 +1329,7 @@ class TestNatRules(InstanceTestCase):
     }
 
   def test(self):
-    connection_parameter_dict = self.computer_partition\
-      .getConnectionParameterDict()
+    connection_parameter_dict = self.getConnectionParameterDictJson()
 
     self.assertIn('nat-rule-port-tcp-100', connection_parameter_dict)
     self.assertIn('nat-rule-port-tcp-200', connection_parameter_dict)
@@ -1272,6 +1344,12 @@ class TestNatRules(InstanceTestCase):
     )
 
 
+@skipUnlessKvm
+class TestNatRulesJson(
+  KvmMixinJson, TestNatRules):
+  pass
+
+
 @skipUnlessKvm
 class TestNatRulesKvmCluster(InstanceTestCase):
   __partition_reference__ = 'nrkc'
@@ -1351,6 +1429,12 @@ class TestWhitelistFirewall(InstanceTestCase):
     self.assertGreater(len(self.content_json), len(resolv_conf_ip_list))
 
 
+@skipUnlessKvm
+class TestWhitelistFirewallJson(
+  KvmMixinJson, TestWhitelistFirewall):
+  pass
+
+
 @skipUnlessKvm
 class TestWhitelistFirewallRequest(TestWhitelistFirewall):
   whitelist_domains = '2.2.2.2 3.3.3.3\n4.4.4.4'
@@ -1368,6 +1452,12 @@ class TestWhitelistFirewallRequest(TestWhitelistFirewall):
     self.assertIn('4.4.4.4', self.content_json)
 
 
+@skipUnlessKvm
+class TestWhitelistFirewallRequestJson(
+  KvmMixinJson, TestWhitelistFirewallRequest):
+  pass
+
+
 @skipUnlessKvm
 class TestWhitelistFirewallResilient(TestWhitelistFirewall):
   kvm_instance_partition_reference = 'wf2'
@@ -1377,6 +1467,12 @@ class TestWhitelistFirewallResilient(TestWhitelistFirewall):
     return 'kvm-resilient'
 
 
+@skipUnlessKvm
+class TestWhitelistFirewallResilientJson(
+  KvmMixinJson, TestWhitelistFirewallResilient):
+  pass
+
+
 @skipUnlessKvm
 class TestWhitelistFirewallRequestResilient(TestWhitelistFirewallRequest):
   kvm_instance_partition_reference = 'wf2'
@@ -1386,6 +1482,12 @@ class TestWhitelistFirewallRequestResilient(TestWhitelistFirewallRequest):
     return 'kvm-resilient'
 
 
+@skipUnlessKvm
+class TestWhitelistFirewallRequestResilientJson(
+  KvmMixinJson, TestWhitelistFirewallRequestResilient):
+  pass
+
+
 @skipUnlessKvm
 class TestWhitelistFirewallCluster(TestWhitelistFirewall):
   kvm_instance_partition_reference = 'wf1'
@@ -1454,6 +1556,12 @@ dd if=/dev/zero of=/dev/virt1 bs=4096 count=500k"""
     self.assertTrue(os.access(slapos_wipe_device_disk, os.X_OK))
 
 
+@skipUnlessKvm
+class TestDiskDevicePathWipeDiskOndestroyJson(
+  KvmMixinJson, TestDiskDevicePathWipeDiskOndestroy):
+  pass
+
+
 @skipUnlessKvm
 class TestImageDownloadController(InstanceTestCase, FakeImageServerMixin):
   __partition_reference__ = 'idc'
@@ -1716,6 +1824,12 @@ class TestParameterDefault(InstanceTestCase, KvmMixin):
     self._test({'ram-size': 2048}, "ram_max_size = '2560'")
 
 
+@skipUnlessKvm
+class TestParameterDefaultJson(
+  KvmMixinJson, TestParameterDefault):
+  pass
+
+
 @skipUnlessKvm
 class TestParameterResilient(TestParameterDefault):
   __partition_reference__ = 'pr'