{% macro replicate(namebase, nbbackup, typeexport, typeimport, heriteLeader='', heriteBackup='', slapparameter_dict={}, monitor_parameter_dict={}) %} {% set sla_parameter_dict = {} -%} {% set monitor_dict = monitor_parameter_dict.get('parameter', {}) -%} {% set monitor_return = ' '.join(monitor_parameter_dict.get('return', [])) -%} {% set monitor_url_list = [] -%} # prepare sla-parameters {% if slapparameter_dict is defined -%} {% for key in slapparameter_dict.keys() -%} {% if key.startswith('-sla-') -%} {% do sla_parameter_dict.__setitem__(key, slapparameter_dict.pop(key)) -%} {% endif -%} {% endfor -%} {% endif -%} [resilient-directory] recipe = slapos.cookbook:mkdirectory home = ${buildout:directory} etc = ${:home}/etc bin = ${:home}/bin ## Tells the Backupable recipe that we want a backup [resilient] recipe = slapos.cookbook:request config-namebase = {{namebase}} software-url = ${slap-connection:software-release-url} [request-{{namebase}}] <= resilient slap-connection {{heriteLeader}} software-type = {{typeexport}} name = {{namebase}}0 return = ssh-public-key resilient-ssh-url notification-id ip {{ monitor_return }} config-number = 0 config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %} config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %} config-name = {{namebase}}0 # Bubble up all the instance parameters to the requested export instance. {% if slapparameter_dict is defined %} {% for parameter_name, parameter_value in six.iteritems(slapparameter_dict) %} {% if parameter_value is string %} config-{{parameter_name}} = {{ parameter_value.split('\n') | join('\n ') }} {% else %} config-{{parameter_name}} = {{ parameter_value }} {% endif %} {% endfor %} {% endif %} {% for key, value in six.iteritems(monitor_dict) -%} config-{{ key }} = {{ value }} {% endfor -%} {% if sla_parameter_dict == {} -%} sla-mode = unique_by_network {% else %} {% set sla_key_main = "-sla-%s%s-" % (namebase, 0) -%} {% set sla_key_secondary = "-sla-%s-" % (0) -%} {% set sla_key_main_length = sla_key_main | length -%} {% set sla_key_secondary_length = sla_key_secondary | length -%} {% set sla_dict = {} -%} {% for key in sla_parameter_dict.keys() -%} {% if key.startswith(sla_key_main) -%} {% do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%} {% elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%} {% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%} {% endif -%} {% endfor -%} {% for key, value in six.iteritems(sla_dict) -%} sla-{{ key }} = {{ value }} {% endfor -%} {% endif -%} {% if 'monitor-base-url' in monitor_return and not monitor_parameter_dict.get('set-monitor-url', False) -%} {% do monitor_url_list.append('${request-' ~ namebase ~ ':connection-monitor-base-url}') -%} {% endif -%} {% for id in range(1,nbbackup|int) %} [request-{{namebase}}-pseudo-replicating-{{id}}] <= slap-connection resilient {{heriteBackup}} recipe = slapos.cookbook:request name = {{namebase}}{{id}} software-url = ${slap-connection:software-release-url} software-type = {{typeimport}} return = ssh-public-key resilient-ssh-url notification-url ip takeover-url takeover-password {{ monitor_return }} pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-push {% for extra_parameter_for_pseudo_replicating_instance in ["software-root", "buildout-shared-folder"] %} {% if slapparameter_dict.get(extra_parameter_for_pseudo_replicating_instance) %} config-{{ extra_parameter_for_pseudo_replicating_instance }} = {{ slapparameter_dict.get(extra_parameter_for_pseudo_replicating_instance) }} {% endif %} {% endfor %} config-number = {{id}} config-name = {{namebase}}{{id}} config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key} config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id} {% for key, value in six.iteritems(monitor_dict) -%} config-{{ key }} = {{ value }} {% endfor -%} {% if sla_parameter_dict == {} -%} sla-mode = unique_by_network {% else %} {% set sla_key_main = "-sla-%s%s-" % (namebase, id) -%} {% set sla_key_secondary = "-sla-%s-" % (id) -%} {% set sla_key_main_length = sla_key_main | length -%} {% set sla_key_secondary_length = sla_key_secondary | length -%} {% set sla_dict = {} -%} {% for key in sla_parameter_dict.keys() -%} {% if key.startswith(sla_key_main) -%} {% do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%} {% elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%} {% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%} {% endif -%} {% endfor -%} {% for key, value in six.iteritems(sla_dict) -%} sla-{{ key }} = {{ value }} {% endfor -%} {% endif %} [publish-connection-information] feed-url-{{namebase}}-{{id}}-push = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id} takeover-{{namebase}}-{{id}}-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-takeover-url} takeover-{{namebase}}-{{id}}-password = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-takeover-password} {% if 'monitor-base-url' in monitor_return -%} {% do monitor_url_list.append('${request-' ~ namebase ~ '-pseudo-replicating-' ~ id ~ ':connection-monitor-base-url}') -%} {% endif -%} {% endfor -%} [resilient-request-{{namebase}}-public-key-promise-bin] # Check that public-key-value parameter exists and is not empty # XXX: maybe we should consider empty values to be non-nexistent. recipe = collective.recipe.template # XXX: don't use system executable input = inline:#!/bin/bash PUBLIC_KEY_CONTENT="${request-{{namebase}}:connection-ssh-public-key}" if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then exit 1 fi output = ${resilient-directory:bin}/resilient-request-{{namebase}}-public-key mode = 700 [resilient-request-{{namebase}}-public-key-promise] <= monitor-promise-base module = check_command_execute name = resilient-request-{{namebase}}-public-key.py config-command = ${resilient-request-{{namebase}}-public-key-promise-bin:output} {% for id in range(1,nbbackup|int) %} [resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise-bin] # Check that public-key-value parameter exists and is not empty # XXX: maybe we should consider empty values to be non-nexistent. recipe = collective.recipe.template # XXX: don't use system executable input = inline:#!/bin/bash PUBLIC_KEY_CONTENT="${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key}" if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then exit 1 fi output = ${resilient-directory:bin}/resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key mode = 700 [resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise] <= monitor-promise-base module = check_command_execute name = resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key.py config-command = ${resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise-bin:output} {% endfor %} ## The PBS and their push / pull slaves ## Adding a PBS provides resiliency ## Adding a backup server provides availability ## Having 3 backups pulling from the same PBS provides ##only availability, not resiliency [request-pbs-common] <= slap-connection recipe = slapos.cookbook:request software-url = ${slap-connection:software-release-url} software-type = pull-backup {% for id in range(1,nbbackup|int) %} [request-pbs-{{namebase}}-{{id}}] <= request-pbs-common name = PBS ({{namebase}} / {{id}}) config-ignore-known-hosts-file = ${slap-parameter:ignore-known-hosts-file} config-monitor-title = PBS ${slap-connection:computer-id}-{{namebase}}-{{id}} {% for key, value in six.iteritems(monitor_dict) -%} config-{{ key }} = {{ value }} {% endfor -%} return = ssh-key notification-url feeds-url {{ monitor_return }} slave = false {% if sla_parameter_dict == {} -%} sla-mode = unique_by_network {% else %} {% set sla_key_main = "-sla-%s%s-" % ("pbs", id) -%} {% set sla_key_secondary = "-sla-%s-" % (id) -%} {% set sla_key_main_length = sla_key_main | length -%} {% set sla_key_secondary_length = sla_key_secondary | length -%} {% set sla_dict = {} -%} {% for key in sla_parameter_dict.keys() -%} {% if key.startswith(sla_key_main) -%} {% do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%} {% elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%} {% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%} {% endif -%} {% endfor -%} {% for key, value in six.iteritems(sla_dict) -%} sla-{{ key }} = {{ value }} {% endfor %} {% endif %} [resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise-bin] # Check that public-key-value parameter exists and is not empty # XXX: maybe we should consider empty values to be non-nexistent. recipe = collective.recipe.template # XXX: don't use system executable input = inline:#!/bin/bash PUBLIC_KEY_CONTENT="${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}" if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then exit 1 fi output = ${resilient-directory:bin}/resilient-request-pbs-{{namebase}}-{{id}}-public-key mode = 700 [resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise] <= monitor-promise-base module = check_command_execute name = resilient-request-pbs-{{namebase}}-{{id}}-public-key.py config-command = ${resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise-bin:output} [request-pull-backup-server-{{namebase}}-{{id}}] <= request-pbs-common name = PBS {{id}} pulling from ${request-{{namebase}}:name} config-url = ${request-{{namebase}}:connection-resilient-ssh-url} config-type = pull config-server-key = ${request-{{namebase}}:connection-ssh-public-key} config-on-notification = ${request-{{namebase}}:connection-notification-id} config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url} config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}} config-title = Pulling from {{namebase}} config-remove-backup-older-than = {{ slapparameter_dict.get('remove-backup-older-than', '2W') }} slave = true sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid} [publish-connection-information] feed-url-{{namebase}}-{{id}}-pull = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-pull-backup-server-{{namebase}}-{{id}}:config-notification-id} {% if 'monitor-base-url' in monitor_return -%} {% do monitor_url_list.append('${request-pbs-' ~ namebase ~ '-' ~ id ~ ':connection-monitor-base-url}') -%} {% endif -%} {% if monitor_parameter_dict.get('set-monitor-url', False) -%} [request-{{namebase}}] config-monitor-url-list = {{ monitor_url_list | join(' ') }} {% else -%} [monitor-conf-parameters] monitor-url-list = {{ monitor_url_list | join(' ') }} {% endif -%} [request-pull-backup-server-{{namebase}}-backup-{{id}}] <= request-pbs-common name = PBS pushing on ${request-{{namebase}}-pseudo-replicating-{{id}}:name} config-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-resilient-ssh-url} config-type = push config-server-key = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key} config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-pull-backup-server-{{namebase}}-{{id}}:config-notification-id} config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-notification-url} config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id} config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}} config-title = Pushing to {{namebase}} backup {{id}} slave = true sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid} {% endfor %} [slap-parameter] # Default parameters for distributed deployment # I.e state "backup1 of maria should go there, ..." {% for id in range(1,nbbackup|int) %} {{namebase}}{{id}}-computer-guid = pbs-{{namebase}}{{id}}-computer-guid = {% endfor %} ignore-known-hosts-file = false {% endmacro %}