kvm: Minimise restart reaction of the KVM wrapper

Thanks to minimising amount of changing parameters of the wrapper, by setting
RAM and CPU to "init" values, it's possible to not restart the wrapper in case
of device hotplugging in for changes of those parameters.

Note that this shows regressiong to enable-device-hotplug which happened
after upgrade to qemu 5.2.0 in 8dc22418.

slapos.toolbox is added in order to use qemuqmpclient.

software/kvm/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 56e28a231bb869afce23ffbf966e8e41
+md5sum = d0f96be4e80b96e6ac33f6d474767b13
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = be750fb62f7057c97dd6c6887b2149cc
+md5sum = a502782244d1be536b732ebb40725f47
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/instance-kvm.cfg.jinja2

@@ -437,13 +437,17 @@ disk-path = ${directory:srv}/virtual.${slap-parameter:disk-format}
 pid-file-path = ${kvm-controller-parameter-dict:pid-file}
 socket-path = ${kvm-controller-parameter-dict:socket-path}
 
-enable-device-hotplug = ${kvm-controller-parameter-dict:enable-device-hotplug}
-smp-count = ${kvm-controller-parameter-dict:cpu-count}
+{%- set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() == 'true' %}
 smp-max-count = {{ cpu_max_count }}
-
-ram-size = ${kvm-controller-parameter-dict:ram-size}
 ram-max-size = {{ ram_max_size }}
+{%- if enable_device_hotplug %}
 init-ram-size = 1024
+init-smp-count = 1
+{%- else %}
+init-ram-size = ${kvm-controller-parameter-dict:ram-size}
+init-smp-count = ${kvm-controller-parameter-dict:cpu-count}
+{%- endif %}
+
 mac-address = ${create-mac:mac-address}
 tap-mac-address = ${create-tap-mac:mac-address}
 

software/kvm/template/template-kvm-run.in

@@ -37,9 +37,9 @@ mac_address = '{{ parameter_dict.get("mac-address") }}'
 tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
 tap_ipv6_addr = '{{ parameter_dict.get("tap-ipv6-addr") }}'
 numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
-ram_size = {{ parameter_dict.get("ram-size") }}
 ram_max_size = '{{ parameter_dict.get("ram-max-size") }}'
 init_ram_size = {{ parameter_dict.get("init-ram-size") }}
+init_smp_count = {{ parameter_dict.get("init-smp-count") }}
 pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
 external_disk_number = {{ parameter_dict.get("external-disk-number") }}
 external_disk_size = {{ parameter_dict.get("external-disk-size") }}
@@ -78,7 +78,6 @@ if not disk_info_list:
 {%- endfor %}
   })
 
-smp_count = {{ parameter_dict.get("smp-count") }}
 smp_max_count = {{ parameter_dict.get("smp-max-count") }}
 machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip()
 cpu_model = '{{ parameter_dict.get("cpu-model") }}'.strip()
@@ -267,12 +266,8 @@ if use_tap == 'true':
             tap_interface, vhost),
           '-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
 
-if enable_device_hotplug != 'true':
-  smp = '%s,maxcpus=%s' % (smp_count, smp_max_count)
-  ram = '%sM,slots=128,maxmem=%sM' % (ram_size, ram_max_size)
-else:
-  smp = '1,maxcpus=%s' % smp_max_count
-  ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
+smp = '%s,maxcpus=%s' % (init_smp_count, smp_max_count)
+ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
 
 kvm_argument_list = [qemu_path,
   '-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',

software/kvm/test/setup.py

@@ -43,6 +43,7 @@ setup(name=name,
       install_requires=[
           'slapos.core',
           'slapos.cookbook',
+          'slapos.toolbox',
           'slapos.libnetworkcache',
           'erp5.util',
           'supervisor',

software/kvm/test/test.py

@@ -46,6 +46,7 @@ import time
 import shutil
 import sys
 
+from slapos.qemuqmpclient import QemuQMPWrapper
 from slapos.proxy.db_version import DB_VERSION
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
@@ -203,6 +204,108 @@ i0:whitelist-firewall-{hash} RUNNING""",
     )
 
 
+@skipUnlessKvm
+class TestMemoryManagement(InstanceTestCase, KvmMixin):
+  __partition_reference__ = 'i'
+
+  def getKvmProcessInfo(self, switch_list):
+    return_list = []
+    with self.slap.instance_supervisor_rpc as instance_supervisor:
+      kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
+                 if 'kvm-' in q['name']][0]['pid']
+      kvm_process = psutil.Process(kvm_pid)
+      get_next = False
+      for entry in kvm_process.cmdline():
+        if get_next:
+          return_list.append(entry)
+          get_next = False
+        elif entry in switch_list:
+          get_next = True
+    return kvm_pid, return_list
+
+  def test(self):
+    kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
+    self.assertEqual(
+      ['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
+      info_list
+    )
+    self.rerequestInstance({
+      'ram-size': '1536',
+      'cpu-count': '2',
+    })
+    self.slap.waitForInstance(max_retry=10)
+    kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
+    self.assertEqual(
+      ['2,maxcpus=3', '1536M,slots=128,maxmem=2048M'],
+      info_list
+    )
+
+    # assert that process was restarted
+    self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
+
+  def tearDown(self):
+    self.rerequestInstance({})
+    self.slap.waitForInstance(max_retry=10)
+
+  def test_enable_device_hotplug(self):
+    def getHotpluggedCpuRamValue():
+      qemu_wrapper = QemuQMPWrapper(os.path.join(
+        self.computer_partition_root_path, 'var', 'qmp_socket'))
+      ram_mb = sum(
+        [q['size']
+         for q in qemu_wrapper.getMemoryInfo()['hotplugged']]) / 1024 / 1024
+      cpu_count = len(
+        [q['CPU'] for q in qemu_wrapper.getCPUInfo()['hotplugged']])
+      return {'cpu_count': cpu_count, 'ram_mb': ram_mb}
+
+    kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
+    self.assertEqual(
+      ['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
+      info_list
+    )
+    self.assertEqual(
+      getHotpluggedCpuRamValue(),
+      {'cpu_count': 0, 'ram_mb': 0}
+    )
+
+    parameter_dict = {
+      'enable-device-hotplug': 'true',
+      # to avoid restarts the max RAM and CPU has to be static
+      'ram-max-size': '2048',
+      'cpu-max-count': '4',
+    }
+    self.rerequestInstance(parameter_dict)
+    self.slap.waitForInstance(max_retry=2)
+    kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
+
+    self.assertEqual(
+      ['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
+      info_list
+    )
+    self.assertEqual(
+      getHotpluggedCpuRamValue(),
+      {'cpu_count': 0, 'ram_mb': 0}
+    )
+    self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
+    parameter_dict.update(**{
+      'ram-size': '1536',
+      'cpu-count': '2'
+    })
+    self.rerequestInstance(parameter_dict)
+    self.slap.waitForInstance(max_retry=10)
+    kvm_pid_3, info_list = self.getKvmProcessInfo(['-smp', '-m'])
+
+    self.assertEqual(
+      ['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
+      info_list
+    )
+    self.assertEqual(kvm_pid_2, kvm_pid_3, "Unexpected: KVM restarted")
+    self.assertEqual(
+      getHotpluggedCpuRamValue(),
+      {'cpu_count': 1, 'ram_mb': 512}
+    )
+
+
 class MonitorAccessMixin(object):
   def sqlite3_connect(self):
     sqlitedb_file = os.path.join(