diff --git a/software/kvm/buildout.hash.cfg b/software/kvm/buildout.hash.cfg index d0f124f009b89f2655283d56ad09959ba8475200..c09de87111b561f6ec9cc761980e8bcbc8600714 100644 --- a/software/kvm/buildout.hash.cfg +++ b/software/kvm/buildout.hash.cfg @@ -19,7 +19,7 @@ md5sum = 087f226ba90928dcc5a722d7008c867a [template-kvm] filename = instance-kvm.cfg.jinja2 -md5sum = baa3ee5b653731124bfc2ac2fa835787 +md5sum = 2ff55931eab48f7992e8e1cb16b44b95 [template-kvm-cluster] filename = instance-kvm-cluster.cfg.jinja2.in @@ -55,7 +55,7 @@ md5sum = a8cf453d20f01c707f02c4b4014580d8 [template-kvm-run] filename = template/template-kvm-run.in -md5sum = 875261817970d0f83335824373288b9d +md5sum = 395ee373ccda3382d257fde1ff4222b0 [template-kvm-controller] filename = template/kvm-controller-run.in @@ -79,11 +79,11 @@ md5sum = d57764bb7135037b4d21543b2f56ce1d [image-download-controller] filename = template/image-download-controller.py -md5sum = 9c67058edcc4edae0b57956c0932a9fc +md5sum = 4d48b3da5bc611fc6533335b5953c840 [image-download-config-creator] filename = template/image-download-config-creator.py -md5sum = 54261e418ab9860efe73efd514c4d47f +md5sum = 8fbe05c4175a7f31b6bffced9ad4e91d [whitelist-firewall-download-controller] filename = template/whitelist-firewall-download-controller.py diff --git a/software/kvm/instance-kvm.cfg.jinja2 b/software/kvm/instance-kvm.cfg.jinja2 index 26134064b6789df9bfd023d3520f02cf69f9278f..03ad33628e01d738858f46eed75637b33a4f9c82 100644 --- a/software/kvm/instance-kvm.cfg.jinja2 +++ b/software/kvm/instance-kvm.cfg.jinja2 @@ -162,7 +162,8 @@ config-filename = ${boot-image-url-select-json-config:error-state-file} # wrapper to execute boot-image-url-select-download on each run recipe = slapos.cookbook:wrapper wrapper-path = ${directory:scripts}/boot-image-url-select-updater -command-line = {{ python_executable }} {{ image_download_controller }} ${boot-image-url-select-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${boot-image-url-select-processed-config:processed-md5sum} +command-line = {{ python_executable }} {{ image_download_controller }} ${:config} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${boot-image-url-select-processed-config:processed-md5sum} +config = ${boot-image-url-select-json-config:rendered} md5sum-state-filename = boot-image-url-select-download-controller-md5sum-fail.json md5sum-state-file = ${directory:boot-image-url-select-expose}/${:md5sum-state-filename} error-state-filename = boot-image-url-select-download-controller-error.text @@ -258,7 +259,8 @@ config-filename = ${boot-image-url-list-json-config:error-state-file} # wrapper to execute boot-image-url-list-download on each run recipe = slapos.cookbook:wrapper wrapper-path = ${directory:scripts}/boot-image-url-list-updater -command-line = {{ python_executable }} {{ image_download_controller }} ${boot-image-url-list-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${boot-image-url-list-processed-config:processed-md5sum} +command-line = {{ python_executable }} {{ image_download_controller }} ${:config} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${boot-image-url-list-processed-config:processed-md5sum} +config = ${boot-image-url-list-json-config:rendered} md5sum-state-filename = boot-image-url-list-download-controller-md5sum-fail.json md5sum-state-file = ${directory:boot-image-url-list-expose}/${:md5sum-state-filename} error-state-filename = boot-image-url-list-download-controller-error.text @@ -355,7 +357,8 @@ config-filename = ${virtual-hard-drive-url-json-config:error-state-file} # wrapper to execute virtual-hard-drive-url-download on each run recipe = slapos.cookbook:wrapper wrapper-path = ${directory:scripts}/virtual-hard-drive-url-updater -command-line = {{ python_executable }} {{ image_download_controller }} ${virtual-hard-drive-url-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${virtual-hard-drive-url-processed-config:processed-md5sum} +command-line = {{ python_executable }} {{ image_download_controller }} ${:config} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${virtual-hard-drive-url-processed-config:processed-md5sum} +config = ${virtual-hard-drive-url-json-config:rendered} md5sum-state-filename = virtual-hard-drive-url-download-controller-md5sum-fail.json md5sum-state-file = ${directory:virtual-hard-drive-url-expose}/${:md5sum-state-filename} error-state-filename = virtual-hard-drive-url-download-controller-error.text @@ -547,13 +550,13 @@ command = [ ! -f {{ '${' + key + '}' }} ] && touch {{ '${' + key + '}' }} {%- endmacro %} {#- Create depending sections, as state files appear late, so it's better to have empty file which will impact the hash anyway #} {%- if boot_image_url_list_enabled %} -{{ generate_depend_section('boot-image-url-list-depend', 'boot-image-url-list-download-wrapper:md5sum-state-file') }} +{{ generate_depend_section('boot-image-url-list-depend', 'boot-image-url-list-download-wrapper:config') }} {%- endif %} {%- if boot_image_url_select_enabled %} -{{ generate_depend_section('boot-image-url-select-depend', 'boot-image-url-select-download-wrapper:md5sum-state-file') }} +{{ generate_depend_section('boot-image-url-select-depend', 'boot-image-url-select-download-wrapper:config') }} {%- endif %} {%- if virtual_hard_drive_url_enabled %} -{{ generate_depend_section('virtual-hard-drive-url-depend', 'virtual-hard-drive-url-download-wrapper:md5sum-state-file') }} +{{ generate_depend_section('virtual-hard-drive-url-depend', 'virtual-hard-drive-url-download-wrapper:config') }} {%- endif %} [kvm-instance] diff --git a/software/kvm/template/image-download-config-creator.py b/software/kvm/template/image-download-config-creator.py index cfaba51cb4a8a30164f9fdafa379d28b58f212b9..7c740d65adc137ea6314fdbb4c11d22e60924ed2 100644 --- a/software/kvm/template/image-download-config-creator.py +++ b/software/kvm/template/image-download-config-creator.py @@ -51,7 +51,7 @@ if __name__ == "__main__": 'url': url, 'destination': md5sum, 'destination-tmp': md5sum + '_tmp', - 'link': 'image_%03i' % (image_number,), + 'image-number': '%03i' % (image_number,), }) else: print('INF: checksum %s repeated, used url %s' % (url, )) diff --git a/software/kvm/template/image-download-controller.py b/software/kvm/template/image-download-controller.py index 028d19a6c86641120d2e4a6c84fa450598f2f246..e3c31918d3423aaa412d37cac08a8adc5ae98aa2 100644 --- a/software/kvm/template/image-download-controller.py +++ b/software/kvm/template/image-download-controller.py @@ -41,11 +41,13 @@ if __name__ == "__main__": print('ERR: There are problems with configuration') print('INF: Storing errors in %s' % (error_state_file,)) + # switch to error state during image download + with open(error_state_file, 'w') as fh: + fh.write('\n'.join(['INF Download in progress'])) # clean the destination directory file_to_keep_list = [] for image in config['image-list']: file_to_keep_list.append(image['destination']) - file_to_keep_list.append(image['link']) for fname in os.listdir(config['destination-directory']): if fname not in file_to_keep_list: print('INF: Removing obsolete %s' % (fname,)) @@ -118,20 +120,6 @@ if __name__ == "__main__": os.rename(destination_tmp, destination) print('INF: %s : Stored with checksum %s' % ( image['url'], image['md5sum'])) - for image in config['image-list']: - destination = os.path.join( - config['destination-directory'], image['destination']) - link = os.path.join(config['destination-directory'], image['link']) - if os.path.exists(destination): - if os.path.lexists(link): - if not os.path.islink(link): - os.remove(link) - if os.path.islink(link) and os.readlink(link) != destination: - os.remove(link) - if not os.path.lexists(link): - print('INF: %s : Symlinking %s -> %s' % ( - image['url'], link, destination)) - os.symlink(destination, link) with open(md5sum_fail_file, 'w') as fh: if new_md5sum_state_dict != {}: json.dump(new_md5sum_state_dict, fh, indent=2) diff --git a/software/kvm/template/template-kvm-run.in b/software/kvm/template/template-kvm-run.in index bf63449abe78a0fa92b52085d81f060531cb5c4b..e5727490e8c18d2afe2501b3f89367aaaf78a10d 100644 --- a/software/kvm/template/template-kvm-run.in +++ b/software/kvm/template/template-kvm-run.in @@ -329,6 +329,22 @@ if cpu_model: if rgx.match(cpu_model): kvm_argument_list.extend(['-cpu', cpu_model]) +def handle_image(config, name): + with open(config) as fh: + image_config = json.load(fh) + if image_config['error-amount'] == 0: + for image in sorted(image_config['image-list'], key=lambda k: k['image-number']): + destination = os.path.join(image_config['destination-directory'], image['destination']) + if os.path.exists(destination): + kvm_argument_list.extend([ + '-drive', + 'file=%s,media=cdrom' % (destination,) + ]) + else: + raise ValueError('%s not ready yet' % (name,)) + else: + raise ValueError('%s not ready yet' % (name,)) + # Try to connect to NBD server (and second nbd if defined). # If not available, don't even specify it in qemu command line parameters. # Reason: if qemu starts with unavailable NBD drive, it will just crash. @@ -350,33 +366,10 @@ else: # Debian installation CDs, rendering it uninstallable if boot_image_url_select_json_config: # Support boot-image-url-select - with open(boot_image_url_select_json_config) as fh: - image_config = json.load(fh) - if image_config['error-amount'] == 0: - for image in sorted(image_config['image-list'], key=lambda k: k['link']): - link = os.path.join(image_config['destination-directory'], image['link']) - if os.path.exists(link) and os.path.islink(link): - kvm_argument_list.extend([ - '-drive', - 'file=%s,media=cdrom' % (link,) - ]) - else: - raise ValueError('boot-image-url-select not ready yet') - + handle_image(boot_image_url_select_json_config, 'boot-image-url-select') if boot_image_url_list_json_config: # Support boot-image-url-list - with open(boot_image_url_list_json_config) as fh: - image_config = json.load(fh) - if image_config['error-amount'] == 0: - for image in sorted(image_config['image-list'], key=lambda k: k['link']): - link = os.path.join(image_config['destination-directory'], image['link']) - if os.path.exists(link) and os.path.islink(link): - kvm_argument_list.extend([ - '-drive', - 'file=%s,media=cdrom' % (link,) - ]) - else: - raise ValueError('boot-image-url-list not ready yet') + handle_image(boot_image_url_list_json_config, 'boot-image-url-list') # Always add by default the default image kvm_argument_list.extend([ '-drive', 'file=%s,media=cdrom' % default_cdrom_iso