kvm: add support for RAM and CPU hotplug

Allow to Add CPU or RAM in hotplug mode, without need of restart qemu process

software/kvm/common.cfg

@@ -41,6 +41,7 @@ parts = ${:common-parts}
 
 [eggs]
 recipe = zc.recipe.egg
+interpreter = python.eggs
 eggs =
   ${python-cffi:egg}
   ${python-cryptography:egg}
@@ -89,7 +90,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = bf5ef731c0d8da0267a4939882b4eeee 
+md5sum = 5a17fc127190bbc19361c5ffb10711b3 
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -98,7 +99,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = e2b8f86bdc12c86e7d959b55c6d54f6d
+md5sum = 152b052f4b418ab3138a3e109a3ec01e
 download-only = true
 on-update = true
 
@@ -107,7 +108,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = 05b6004e8c7a94de14f247affcef4971
+md5sum = b3ff4f80e4d607e2a977349416c66e39
 download-only = true
 on-update = true
 
@@ -185,7 +186,7 @@ ignore-existing = true
 url = ${:_profile_base_location_}/template/template-kvm-run.in
 mode = 644
 filename = template-kvm-run.in
-md5sum = 178a24cdad77cb6c2e519ac629dd0e74
+md5sum = 7f34fbb05f0cfdb6753e008e47cacf30
 download-only = true
 on-update = true
 
@@ -195,7 +196,7 @@ ignore-existing = true
 url = ${:_profile_base_location_}/template/kvm-controller-run.in
 mode = 644
 filename = kvm-controller-run.in
-md5sum = 71afd2d13f6e56993ae413a168e012d7
+md5sum = 9ba27ec934e0e86ef88493356a40bed5
 download-only = true
 on-update = true
 
@@ -225,7 +226,7 @@ ignore-existing = true
 url = ${:_profile_base_location_}/template/qemu-is-ready.in
 mode = 644
 filename = qemu-is-ready.in
-md5sum = 0066fa0f5f3dd47bded5e5924df2550d
+md5sum = b304eec8e2cb71f10ea83cac22f6db12
 download-only = true
 on-update = true
 

software/kvm/instance-kvm-cluster-input-schema.json

@@ -197,6 +197,29 @@
               "minimum": 128,
               "multipleOf": 128
             },
+            "hotplug-slot-size": {
+              "title": "Size of Hotpluggable RAM slot, in MB",
+              "description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
+              "type": "integer",
+              "default": 512,
+              "minimum": 128,
+              "multipleOf": 128
+            },
+            "max-ram-hotplug-size": {
+              "title": "Maximum hotpluggable RAM size, in MB",
+              "description": "Define the maximum size of hotpluggable memory. The size is in MB and should be a multiple of the defined hotpluggable slot size.",
+              "type": "integer",
+              "default": 0,
+              "minimum": 128,
+              "multipleOf": 128
+            },
+            "hotplugged-slot-amount": {
+              "title": "Hotplugged RAM slot amount",
+              "description": "Define the number of Memory slot to hotplug. Ex: if slot-hotplug-size=512 MB, then to hotplug 1G of RAM 2 slots will be used.",
+              "type": "integer",
+              "default": 0,
+              "minimum": 0
+            },
             "auto-ballooning": {
               "title": "Enable qemu auto ballooning.",
               "description": "Enable virtio balloon device to allows KVM guests to reduce/re-increase their memory size.",
@@ -245,10 +268,17 @@
               "type": "integer",
               "minimum": 1
             },
-            "cpu-options": {
-              "title": "CPU Additional options: cores, threads, sockets, maxcpus.",
-              "description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.",
-              "type": "string"
+            "max-cpu-hotplug-count": {
+              "title": "Maximum hotpluggable CPU amount",
+              "description": "Specifies the maximum number of hotpluggable CPUs.",
+              "type": "integer",
+              "default": 1
+            },
+            "cpu-hotplug-amount": {
+              "title": "hotpluggable CPU amount",
+              "description": "Specifies the number of CPUs to hotplug",
+              "type": "integer",
+              "default": 0
             },
             "numa": {
               "title": "Simulate a multi node NUMA system.",

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -51,11 +51,15 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi
 config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
 config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
 config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
+config-max-ram-hotplug-size = {{ dumps(kvm_parameter_dict.get('max-ram-hotplug-size', 0)) }}
+config-hotplug-slot-size = {{ dumps(kvm_parameter_dict.get('hotplug-slot-size', 512)) }}
+config-hotplugged-slot-amount = {{ dumps(kvm_parameter_dict.get('hotplugged-slot-amount', 0)) }}
 config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
 config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
 config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
+config-max-cpu-hotplug-count = {{ dumps(kvm_parameter_dict.get('max-cpu-hotplug-count', 1)) }}
+config-cpu-hotplug-amount = {{ dumps(kvm_parameter_dict.get('cpu-hotplug-amount', 0)) }}
 {{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
-{{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
 {{ setconfig('cpu-options', kvm_parameter_dict.get('cpu-options', '')) }}
 {{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}
 {{ setconfig('host2', kvm_parameter_dict.get('host2', '')) }}

software/kvm/instance-kvm-input-schema.json

@@ -12,6 +12,29 @@
       "minimum": 128,
       "multipleOf": 128
     },
+    "hotplug-slot-size": {
+      "title": "Size of Hotpluggable RAM slot, in MB",
+      "description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
+      "type": "integer",
+      "default": 512,
+      "minimum": 128,
+      "multipleOf": 128
+    },
+    "max-ram-hotplug-size": {
+      "title": "Maximum hotpluggable RAM size, in MB",
+      "description": "Define the maximum size of hotpluggable memory. The size is in MB and should be a multiple of the defined hotpluggable slot size.",
+      "type": "integer",
+      "default": 0,
+      "minimum": 128,
+      "multipleOf": 128
+    },
+    "hotplugged-slot-amount": {
+      "title": "Hotplugged RAM slot amount",
+      "description": "Define the number of Memory slot to hotplug. Ex: if slot-hotplug-size=512 MB, then to hotplug 1G of RAM 2 slots will be used.",
+      "type": "integer",
+      "default": 0,
+      "minimum": 0
+    },
     "auto-ballooning": {
       "title": "Enable qemu auto ballooning.",
       "description": "Enable virtio balloon device to allows KVM guests to reduce/re-increase their memory size.",
@@ -54,10 +77,17 @@
       "type": "integer",
       "minimum": 1
     },
-    "cpu-options": {
-      "title": "CPU Additional options: cores, threads, sockets, maxcpus.",
-      "description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.",
-      "type": "string"
+    "max-cpu-hotplug-count": {
+      "title": "Maximum hotpluggable CPU amount",
+      "description": "Specifies the maximum number of hotpluggable CPUs.",
+      "type": "integer",
+      "default": 1
+    },
+    "cpu-hotplug-amount": {
+      "title": "CPU hotplug amount",
+      "description": "Specifies the number of CPUs to hotplug",
+      "type": "integer",
+      "default": 0
     },
     "numa": {
       "title": "Simulate a multi node NUMA system.",

software/kvm/instance-kvm.cfg.jinja2

@@ -62,10 +62,14 @@ storage-path = ${directory:srv}/passwd
 bytes = 8
 
 [kvm-controller-parameter-dict]
-python-path = {{ python_executable }}
+python-path = {{ python_eggs_executable }}
 vnc-passwd = ${gen-passwd:passwd}
 socket-path = ${directory:var}/qmp_socket
 kvm-status-path = ${directory:var}/qemu-vm-is-ready
+cpu-hotplug-amount = ${slap-parameter:cpu-hotplug-amount}
+hotplugged-slot-amount = ${slap-parameter:hotplugged-slot-amount}
+hotplug-slot-size = ${slap-parameter:hotplug-slot-size}
+max-ram-hotplug-size = ${slap-parameter:max-ram-hotplug-size}
 
 [kvm-parameter-dict]
 python-path = {{ python_executable }}
@@ -91,9 +95,10 @@ pid-file-path = ${directory:run}/pid_file
 socket-path = ${kvm-controller-parameter-dict:socket-path}
 
 smp-count = ${slap-parameter:cpu-count}
-smp-options = ${slap-parameter:cpu-options}
+max-smp-hotplug-count = ${slap-parameter:max-cpu-hotplug-count}
 ram-size = ${slap-parameter:ram-size}
-numa = ${slap-parameter:numa}
+max-ram-hotplug-size = ${slap-parameter:max-ram-hotplug-size}
+hotplug-slot-size = ${slap-parameter:hotplug-slot-size}
 mac-address = ${create-mac:mac-address}
 tap-mac-address = ${create-tap-mac:mac-address}
 
@@ -143,7 +148,7 @@ disk-cache = ${slap-parameter:disk-cache}
 disk-aio = ${slap-parameter:disk-aio}
 auto-ballooning = ${slap-parameter:auto-ballooning}
 machine-options = ${slap-parameter:machine-options}
-cpu-options = ${slap-parameter:cpu-model}
+cpu-hotplug-slot-size = ${slap-parameter:cpu-model}
 
 log-file = ${directory:log}/qemu.log
 
@@ -551,14 +556,15 @@ nbd2-port = 1024
 nbd2-host =
 
 ram-size = 1024
+hotplug-slot-size = 512
+max-ram-hotplug-size = 0
+hotplugged-slot-amount = 0
 disk-size = 10
 disk-type = virtio
 
 cpu-count = 1
-# cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]
-cpu-options = 
-# list of numa options separate by space: node,nodeid=1,cpus=9-15 node,nodeid=2,cpus=1,3,7
-numa = 
+max-cpu-hotplug-count = 1
+cpu-hotplug-amount = 0
 disk-cache = writeback
 disk-aio = native
 auto-ballooning = True

software/kvm/instance.cfg.in

@@ -93,6 +93,7 @@ context =
     raw netcat_bin ${netcat:location}/bin/netcat
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw python_executable ${buildout:executable}
+    raw python_eggs_executable ${buildout:bin-directory}/${eggs:interpreter}
     raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
     raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
     raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}

software/kvm/software.cfg

@@ -45,4 +45,4 @@ paramiko = 2.0.2
 
 # Required by:
 # slapos.toolbox==0.71
-passlib = 1.6.5
\ No newline at end of file
+passlib = 1.6.5

software/kvm/template/kvm-controller-run.in

@@ -6,40 +6,47 @@
 import socket
 import time
 import os
+from slapos.qemuqmpclient import QemuQMPWrapper
 
 # XXX: to be factored with slapos.toolbox qemu qmp wrapper.
 
 socket_path = '{{ parameter_dict.get("socket-path") }}'
 vnc_password = '{{ parameter_dict.get("vnc-passwd") }}'
 status_path = '{{ parameter_dict.get("kvm-status-path") }}'
+cpu_hotplug_amount = {{ parameter_dict.get("cpu-hotplug-amount", 0) }}
+cpu_model = '{{ parameter_dict.get("cpu-model", '') }}'
+max_ram_hotplug_size = {{ parameter_dict.get("max-ram-hotplug-size", 0) }}
+ram_hotplug_amount = {{ parameter_dict.get("hotplugged-slot-amount", 0) }}
+slot_hotplug_size = {{ parameter_dict.get("hotplug-slot-size", 512) }}
 
-if os.path.exists(status_path):
-  os.unlink(status_path)
+def update():
+  if os.path.exists(status_path):
+    os.unlink(status_path)
 
-# Connect to KVM qmp socket
-so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-connected = False
-while not connected:
+  qemu_wrapper = QemuQMPWrapper(socket_path)
   try:
-    so.connect(socket_path)
-  except socket.error:
-    time.sleep(1)
-  else:
-    connected = True
-data = so.recv(1024)
-
-# Enable qmp
-so.send('{ "execute": "qmp_capabilities" }')
-data = so.recv(1024)
-
-# Set VNC password
-so.send('{ "execute": "change", ' \
-        '"arguments": { "device": "vnc", "target": "password", ' \
-        '               "arg": "' + vnc_password + '" } }')
-data = so.recv(1024)
-
-# Finish
-so.close()
-
-with open(status_path, 'w') as status_file:
-  status_file.write("OK")
+    qemu_wrapper.setVNCPassword(vnc_password)
+    cpu_option_dict = {
+      'device': 'cpu',
+      'amount': cpu_hotplug_amount
+    }
+    if cpu_model:
+      cpu_option_dict['model'] = cpu_model
+    qemu_wrapper.updateDevice(cpu_option_dict)
+    if max_ram_hotplug_size > 0:
+      qemu_wrapper.updateDevice({
+        'device': 'memory',
+        'mem': ram_hotplug_amount * slot_hotplug_size,
+        'slot': slot_hotplug_size,
+        'nslot': max_ram_hotplug_size / slot_hotplug_size
+      })
+  except ValueError, e:
+    with open(status_path, 'w') as status_file:
+      status_file.write(str(e))
+    raise
+
+  with open(status_path, 'w') as status_file:
+    status_file.write("")
+
+if __name__ == "__main__":
+  update()

software/kvm/template/qemu-is-ready.in

 #!{{ dash }}
 
-if [ -f "{{ qemu_ready_path }}" ]; then
-  echo "VM correctly started."
+FILE="{{ qemu_ready_path }}"
+# don't start checks too fast
+sleep 2
+if [ -f "$FILE" ]; then
+  if [ "$(cat $FILE)" = "" ]; then
+    echo "VM correctly started."
+  else
+    >&2 echo "Qemu Controller failed"
+    >&2 cat $FILE
+    exit 1
+  fi
 else
   log_file="{{ qemu_service_log_file }}"
   >&2 echo "Qemu process is not correctly started."

software/kvm/template/template-kvm-run.in

@@ -39,13 +39,15 @@ tap_interface = '{{ parameter_dict.get("tap-interface") }}'
 listen_ip = '{{ parameter_dict.get("ipv4") }}'
 mac_address = '{{ parameter_dict.get("mac-address") }}'
 tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
-smp_count = '{{ parameter_dict.get("smp-count") }}'
-smp_options = '{{ parameter_dict.get("smp-options") }}'.strip()
-numa_list = '{{ parameter_dict.get("numa") }}'.split()
-ram_size = '{{ parameter_dict.get("ram-size") }}'
+smp_count = {{ parameter_dict.get("smp-count") }}
+max_smp_hotplug_count = {{ parameter_dict.get("max-smp-hotplug-count") }}
+numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
+ram_size = {{ parameter_dict.get("ram-size") }}
+max_ram_hotplug_size = {{ parameter_dict.get("max-ram-hotplug-size") }}
+slot_hotplug_size = {{ parameter_dict.get("hotplug-slot-size") }}
 pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
 external_disk_number = {{ parameter_dict.get("external-disk-number") }}
-external_disk_size = '{{ parameter_dict.get("external-disk-size") }}'
+external_disk_size = {{ parameter_dict.get("external-disk-size") }}
 external_disk_format = '{{ parameter_dict.get("external-disk-format") }}'
 disk_storage_dict = {}
 disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
@@ -144,6 +146,12 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
       lf.write('%s' % external_disk_number)
   return id_list, external_disk_number
 
+def getMemHotplugDict(max_mem_size, slot_size):
+  if max_mem_size == 0:
+    return None
+  number = max_mem_size / slot_size
+  return {'maxmem': max_mem_size + ram_size, 'slots': number}
+
 # Download existing hard drive if needed at first boot
 if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
   print('Downloading virtual hard drive...')
@@ -255,15 +263,18 @@ if use_tap == 'true':
             tap_interface, vhost),
           '-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
 
-smp = smp_count
-if smp_options:
-  for option in smp_options.split(','):
-    key, val = option.split('=')
-    if key in ('cores', 'threads', 'sockets', 'maxcpus') and val.isdigit():
-      smp += ',%s=%s' % (key, val)
+if not max_smp_hotplug_count >= 0:
+  raise ValueError("Max CPU hotpluggable count is not a valid interger")
+smp ='%s,maxcpus=%s' % (smp_count, smp_count + max_smp_hotplug_count)
+
+mem_dict = getMemHotplugDict(max_ram_hotplug_size, slot_hotplug_size)
+if mem_dict is not None:
+ ram_string = '%sM,slots=%s,maxmem=%sM' % (ram_size, mem_dict['slots'], mem_dict['maxmem'])
+else:
+  ram_string = '%s' % ram_size
 kvm_argument_list = [qemu_path,
   '-enable-kvm', '-smp', smp, '-name', vm_name,
-  '-m', ram_size, '-vga', 'std',
+  '-m', ram_string, '-vga', 'std',
   '-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio),
   '-vnc', '%s:1,ipv4,password' % listen_ip,
   '-boot', 'order=cd,menu=on',

[Jérome Perrin]

@alain.takoudjou do you remember if this is needed ? This is what @rafael and @romain discussed. Without knowing this I feel that the we don't need to sleep and the promise will fail if it's not ready and succeed on next slapos node instance run

[Alain Takoudjou]

@jerome It's not absolutely needed, it can be removed. If I remember well, it was because it was to wait a bit for qemu to start (so the promise won't fail at first run). But as you mentionned, without sleep the promise will succeed on next run.

[Jérome Perrin]

Thanks @alain.takoudjou let's try !1194 (merged)