instance-apache-replicate.cfg.in 37.1 KB
Newer Older
1
{% if instance_parameter_dict['slap-software-type'] in software_type %}
2
{% set aibcc_enabled = True %}
3
{% import "caucase" as caucase with context %}
4 5
{#- SERVER_POLLUTED_KEY_LIST is a list of keys which comes from various SlapOS Master implementations, which mix request and publish keys on each slave information -#}
{%- set SERVER_POLLUTED_KEY_LIST = ['connection-parameter-hash', 'timestamp', 'slave_title', 'slap_software_type'] -%}
6
{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
7
{%- set GOOD_CIPHER_LIST = ['ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-GCM-SHA384', 'ECDHE-ECDSA-AES128-GCM-SHA256', 'ECDHE-RSA-AES128-GCM-SHA256', 'ECDHE-ECDSA-WITH-CHACHA20-POLY1305', 'ECDHE-RSA-WITH-CHACHA20-POLY1305', 'ECDHE-RSA-AES256-CBC-SHA', 'ECDHE-RSA-AES128-CBC-SHA', 'ECDHE-ECDSA-AES256-CBC-SHA', 'ECDHE-ECDSA-AES128-CBC-SHA', 'RSA-AES256-CBC-SHA', 'RSA-AES128-CBC-SHA', 'ECDHE-RSA-3DES-EDE-CBC-SHA', 'RSA-3DES-EDE-CBC-SHA'] %}
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
{#- Allow to pass only some parameters to frontend nodes #}
{%- set FRONTEND_NODE_PASSED_KEY_LIST = [
        'plain_http_port',
        'port',
        'apache-certificate',
        'apache-key',
        'domain',
        'enable-http2-by-default',
        'global-disable-http2',
        'mpm-graceful-shutdown-timeout',
        'public-ipv4',
        're6st-verification-url',
        'backend-connect-timeout',
        'backend-connect-retries',
        'ciphers',
        'request-timeout',
        'authenticate-to-backend',
    ]
%}
27
{% set aikc_enabled = slapparameter_dict.get('automatic-internal-kedifa-caucase-csr', 'true').lower() in TRUE_VALUES %}
28
{% set aibcc_enabled = slapparameter_dict.get('automatic-internal-backend-client-caucase-csr', 'true').lower() in TRUE_VALUES %}
29 30 31 32
{# Ports 8401, 8402 and 8410+1..N are reserved for monitor ports on various partitions #}
{% set master_partition_monitor_monitor_httpd_port = 8401 %}
{% set kedifa_partition_monitor_httpd_port = 8402 %}
{% set frontend_monitor_httpd_base_port = 8410 %}
33 34
{% set caucase_host = '[' ~ instance_parameter_dict['ipv6-random'] ~ ']' %}
{% set caucase_netloc = caucase_host ~ ':' ~ instance_parameter_dict['configuration.caucase_backend_client_port'] %}
35
{% set caucase_url = 'http://' ~ caucase_netloc %}
36 37 38 39 40 41
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extra-context =
context =
    import json_module json
42
    raw profile_common {{ software_parameter_dict['profile_common'] }}
43 44
    ${:extra-context}

45
{% set popen = functools_module.partial(subprocess_module.Popen, stdout=subprocess_module.PIPE, stderr=subprocess_module.STDOUT, stdin=subprocess_module.PIPE) %}
46
{% set part_list = [] %}
47
{% set single_type_key = 'single-' %}
48
{% if instance_parameter_dict['slap-software-type'] == "replicate" %}
49
{%   set frontend_type = slapparameter_dict.pop('-frontend-type', 'single-default') %}
50
{% elif instance_parameter_dict['slap-software-type'] in ['default', 'RootSoftwareInstance'] %}
51 52
{%   set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %}
{% else %}
53
{%   set frontend_type = "%s%s" % (single_type_key, instance_parameter_dict['slap-software-type']) %}
54 55 56
{% endif %}
{% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '1') | int %}
{% set slave_list_name = 'extra_slave_instance_list' %}
57 58 59
{% set frontend_list = [] %}
{% set frontend_section_list = [] %}
{% set request_dict = {} %}
60
{% set namebase = 'caddy-frontend' %}
61
# XXX Dirty hack, not possible to define default value before
62 63 64 65
{% set sla_computer_caddy_1_key = '-sla-1-computer_guid' %}
{% if not sla_computer_caddy_1_key in slapparameter_dict %}
{%   do slapparameter_dict.__setitem__(sla_computer_caddy_1_key, '${slap-connection:computer-id}') %}
{% endif %}
66 67 68 69
{% set sla_computer_kedifa_key = '-sla-kedifa-computer_guid' %}
{% if not sla_computer_kedifa_key in slapparameter_dict %}
{%   do slapparameter_dict.__setitem__(sla_computer_kedifa_key, '${slap-connection:computer-id}') %}
{% endif %}
70

71
# Here we request individually each frontend.
72
# The presence of sla parameters is checked and added if found
73 74 75 76
{% for i in range(1, frontend_quantity + 1) %}
{%   set frontend_name = "%s-%s" % (namebase, i) %}
{%   set request_section_title = 'request-%s' % frontend_name %}
{%   set sla_key = "-sla-%s-" % i %}
77 78 79 80 81 82 83 84 85
{%   set sla_key_length = sla_key | length %}
{%   set sla_dict = {} %}
{%   set config_key = "-frontend-config-%s-" % i %}
{%   set config_key_length = config_key | length %}
{%   set config_dict = {} %}
{%   for key in slapparameter_dict.keys() %}
{%     if key.startswith(sla_key) %}
{%       do sla_dict.__setitem__(key[sla_key_length:], slapparameter_dict.pop(key)) %}
# We check for specific configuration regarding the frontend
86
{%     elif key.startswith(config_key) %}
87
{%       do config_dict.__setitem__(key[config_key_length:], slapparameter_dict.pop(key)) %}
88 89
{%     endif %}
{%   endfor %}
90
{%   do config_dict.__setitem__('monitor-httpd-port', frontend_monitor_httpd_base_port + i) %}
91
{%   do config_dict.__setitem__('backend-client-caucase-url', caucase_url) %}
92 93 94 95 96 97
{%   set state_key = "-frontend-%s-state" % i %}
{%   set frontend_state = slapparameter_dict.pop(state_key, None) %}
{%   if frontend_state != 'destroyed' %}
{%     do frontend_list.append(frontend_name) %}
{%     do frontend_section_list.append(request_section_title) %}
{%   endif %}
98
{%   do part_list.append(request_section_title) %}
99
# Filling request dict for slave
100
{%   set request_content_dict = {
101 102 103
                                  'config': config_dict,
                                  'name': frontend_name,
                                  'sla': sla_dict,
104
                                  'state': frontend_state
105 106 107 108
                                  } %}
{%   set frontend_software_url_key = "-frontend-%s-software-release-url" % i %}
{%   do request_content_dict.__setitem__('software-url', slapparameter_dict.get(frontend_software_url_key) or '${slap-connection:software-release-url}') %}
{%   do request_dict.__setitem__(request_section_title, request_content_dict) %}
109
{% endfor %}
110

111
{% set authorized_slave_string_list = [] %}
112
{% set authorized_slave_list = [] %}
113
{% set rejected_slave_dict = {} %}
114
{% set warning_slave_dict = {} %}
115
{% set used_host_list = [] %}
116
{% for slave in sorted(instance_parameter_dict['slave-instance-list']) %}
117
{%   set slave_error_list = [] %}
118
{%   set slave_warning_list = [] %}
119
{%   set slave_server_alias_unclashed = [] %}
120 121 122
{%   set slave_type = slave.get('type') %}
{%   if slave_type == 'eventsource' %}
{%     do slave_error_list.append('type:eventsource is not implemented') %}
123
{%   elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
124 125
{%     do slave_error_list.append('type:%s is not supported' % (slave_type,)) %}
{%   endif %}
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
{#   Check backend-active-check-* #}
{%   set backend_active_check = (str(slave.get('backend-active-check', False)) or 'false').lower() %}
{%   if backend_active_check in TRUE_VALUES %}
{%     set backend_active_check_http_method = slave.get('backend-active-check-http-method') or 'GET' %}
{%     if backend_active_check_http_method not in ['GET', 'OPTIONS', 'CONNECT', 'POST'] %}
{%       do slave_error_list.append('Wrong backend-active-check-http-method %s' % (backend_active_check_http_method,)) %}
{%     endif %}
{%     set backend_active_check_http_path = slave.get('backend-active-check-http-path') or '/' %}
{%     set backend_active_check_http_version = slave.get('backend-active-check-http-version') or 'HTTP/1.1' %}
{%     if backend_active_check_http_version not in ['HTTP/1.1', 'HTTP/1.0'] %}
{%       do slave_error_list.append('Wrong backend-active-check-http-version %s' % (backend_active_check_http_version,)) %}
{%     endif %}
{%     set backend_active_check_timeout = (slave.get('backend-active-check-timeout') or '2') | int(False) %}
{%     if backend_active_check_timeout in [False] or backend_active_check_timeout <= 0 %}
{%       do slave_error_list.append('Wrong backend-active-check-timeout %s' % (slave.get('backend-active-check-timeout'),)) %}
{%     endif %}
{%     set backend_active_check_interval = (slave.get('backend-active-check-interval') or '5') | int(False) %}
{%     if backend_active_check_interval in [False] or backend_active_check_interval <= 0 %}
{%       do slave_error_list.append('Wrong backend-active-check-interval %s' % (slave.get('backend-active-check-interval'),)) %}
{%     endif %}
{%     set backend_active_check_rise = (slave.get('backend-active-check-rise') or '1') | int(False) %}
{%     if backend_active_check_rise in [False] or backend_active_check_rise <= 0 %}
{%       do slave_error_list.append('Wrong backend-active-check-rise %s' % (slave.get('backend-active-check-rise'),)) %}
{%     endif %}
{%     set backend_active_check_fall = (slave.get('backend-active-check-fall') or '1') | int(False) %}
{%     if backend_active_check_fall in [False] or backend_active_check_fall <= 0 %}
{%       do slave_error_list.append('Wrong backend-active-check-fall %s' % (slave.get('backend-active-check-fall'),)) %}
{%     endif %}
{%   endif %}
155 156 157 158 159 160 161 162 163
{#   Check ciphers #}
{%   set slave_cipher_list = slave.get('ciphers', '').strip().split() %}
{%   if slave_cipher_list %}
{%     for cipher in slave_cipher_list %}
{%       if cipher not in GOOD_CIPHER_LIST  %}
{%         do slave_error_list.append('Cipher %r is not supported.' % (cipher,)) %}
{%       endif %}
{%     endfor %}
{%   endif %}
164 165
{%   set custom_domain = slave.get('custom_domain') %}
{%   if custom_domain and custom_domain in used_host_list %}
166
{%      do slave_error_list.append('custom_domain %r clashes' % (custom_domain,)) %}
167 168 169 170
{%   else %}
{%      do used_host_list.append(custom_domain) %}
{%   endif %}
{%   if slave.get('server-alias') %}
171
{%     for slave_alias in ('' ~ slave['server-alias']).split() %}
172 173 174 175 176 177
{%       if slave_alias.startswith('*.') %}
{%         set clean_slave_alias = slave_alias[2:] %}
{%       else %}
{%         set clean_slave_alias = slave_alias %}
{%       endif %}
{%       if not validators.domain(clean_slave_alias) %}
178
{%         do slave_error_list.append('server-alias \'%s\' not valid' % (slave_alias,)) %}
179
{%       else %}
180 181 182
{%         if slave_alias in slave_server_alias_unclashed or slave_alias == custom_domain %}
{#           optionally do something about reporting back that server-alias has been unclashed #}
{%         elif slave_alias in used_host_list %}
183
{%           do slave_error_list.append('server-alias \'%s\' clashes' % (slave_alias,)) %}
184
{%         else %}
185
{%           do slave_server_alias_unclashed.append(slave_alias) %}
186 187 188 189
{%           do used_host_list.append(slave_alias) %}
{%         endif %}
{%       endif %}
{%     endfor %}
190
{%     do slave.__setitem__('server-alias', ' '.join(slave_server_alias_unclashed)) %}
191
{%   endif %}
192
{%   for url_key in ['url', 'https-url'] %}
193
{%     if url_key in slave %}
194
{%       set url = (slave[url_key] or '').strip() %}
195
{%       if not validators.url(url) %}
196
{%         do slave_error_list.append('slave %s %r invalid' % (url_key, url)) %}
197 198
{%       elif url != slave[url_key] %}
{%         do slave_warning_list.append('slave %s %r has been converted to %r' % (url_key, slave[url_key], url)) %}
199
{%       endif %}
200
{%     endif %}
201
{%   endfor %}
202 203
{%   if 'ssl_proxy_ca_crt' in slave %}
{%     set ssl_proxy_ca_crt = slave.get('ssl_proxy_ca_crt', '') %}
204
{%     set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %}
205 206 207 208 209
{%     do check_popen.communicate(ssl_proxy_ca_crt) %}
{%     if check_popen.returncode != 0 %}
{%       do slave_error_list.append('ssl_proxy_ca_crt is invalid') %}
{%     endif %}
{%   endif %}
210 211 212 213 214 215 216 217 218 219
{# BBB: SlapOS Master non-zero knowledge BEGIN #}
{%   for key in ['ssl_key', 'ssl_crt', 'ssl_ca_crt'] %}
{%     if key in slave %}
{%       do slave_warning_list.append('%s is obsolete, please use key-upload-url' % (key,)) %}
{%     endif %}
{%   endfor %}
{%   if slave.get('ssl_ca_crt') and not (slave.get('ssl_crt') and slave.get('ssl_key')) %}
{%     do slave_error_list.append('ssl_ca_crt is present, so ssl_crt and ssl_key are required')  %}
{%   endif %}
{%   if slave.get('ssl_key') and slave.get('ssl_crt') %}
220 221
{%     set key_popen = popen([software_parameter_dict['openssl'], 'rsa', '-noout', '-modulus']) %}
{%     set crt_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout', '-modulus']) %}
222 223 224 225 226 227 228
{%     set key_modulus = key_popen.communicate(slave['ssl_key'])[0] | trim %}
{%     set crt_modulus = crt_popen.communicate(slave['ssl_crt'])[0] | trim %}
{%     if not key_modulus or key_modulus != crt_modulus  %}
{%       do slave_error_list.append('slave ssl_key and ssl_crt does not match')  %}
{%     endif %}
{%   endif %}
{# BBB: SlapOS Master non-zero knowledge END #}
229
{%   if slave.get('custom_domain') %}
230 231 232
{%     set slave_custom_domain = '' ~ slave['custom_domain'] %}
{%     if slave_custom_domain.startswith('*.') %}
{%       set clean_custom_domain = slave_custom_domain[2:] %}
233
{%     else %}
234
{%       set clean_custom_domain = slave_custom_domain %}
235 236
{%     endif %}
{%     if not validators.domain(clean_custom_domain) %}
237
{%       do slave_error_list.append('custom_domain %r invalid' % (slave['custom_domain'],)) %}
238
{%     endif %}
239
{%   endif %}
240
{%   if len(slave_error_list) == 0 %}
241 242 243 244 245 246
{#   Cleanup slave from not needed keys which come from implementation of SlapOS Master #}
{%     set authorized_slave = slave.copy() %}
{%     for key in SERVER_POLLUTED_KEY_LIST %}
{%       do authorized_slave.pop(key, None) %}
{%     endfor %}
{%     do authorized_slave_list.append(authorized_slave) %}
247
{%   else %}
248
{%     do rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_error_list)) %}
249
{%   endif %}
250
{%   if len(slave_warning_list) > 0 %}
251
{%     do warning_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_warning_list)) %}
252
{%   endif %}
253
{% endfor %}
254
{% do authorized_slave_list.sort() %}
255

256
[monitor-instance-parameter]
257
monitor-httpd-port = {{ master_partition_monitor_monitor_httpd_port }}
258

259 260
[replicate]
<= slap-connection
261
recipe = slapos.cookbook:requestoptional.serialised
262 263 264 265 266
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}

software-type = {{frontend_type}}
267
return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url backend-client-csr_id-url csr_id-url csr_id-certificate backend-haproxy-statistic-url
268

269 270 271 272 273 274 275
{#- Send only needed parameters to frontend nodes #}
{%- set base_node_configuration_dict = {} %}
{%- for key in FRONTEND_NODE_PASSED_KEY_LIST %}
{%-   if key in slapparameter_dict %}
{%-     do base_node_configuration_dict.__setitem__(key,  slapparameter_dict[key]) %}
{%-   endif %}
{%- endfor %}
276
{% for section, frontend_request in request_dict.iteritems() %}
277
{%   set state = frontend_request.get('state', '') %}
278 279 280
[{{section}}]
<= replicate
name = {{ frontend_request.get('name') }}
281
software-url = {{ frontend_request['software-url'] }}
282 283 284
{%   if state %}
state = {{ state }}
{%   endif %}
285 286
config-slave-kedifa-information = ${request-kedifa:connection-slave-kedifa-information}
config-kedifa-caucase-url = ${request-kedifa:connection-caucase-url}
287
config-backend-client-caucase-url = {{ caucase_url }}
288
config-master-key-download-url = ${request-kedifa:connection-master-key-download-url}
289
config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }}
290 291
{#   Do not send additional parameters for destroyed nodes #}
{%   if state != 'destroyed' %}
292 293
{%     set node_configuration_dict = {} %}
{%     do node_configuration_dict.update(frontend_request.get('config')) %}
294
{# sort_keys are important in order to avoid shuffling parameters on each run #}
295 296 297 298 299 300
{%     do node_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list, sort_keys=True)) %}
{%     do node_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
{%-     for config_key, config_value in node_configuration_dict.iteritems() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{%     endfor -%}
{%-     for config_key, config_value in base_node_configuration_dict.iteritems() %}
301
config-{{ config_key }} = {{ dumps(config_value) }}
302 303
{%     endfor -%}
{%   endif %}
304
{%   if frontend_request.get('sla') %}
305
{%     for parameter, value in frontend_request.get('sla').iteritems() %}
306
sla-{{ parameter }} = {{ value }}
307 308 309
{%     endfor %}
{%   endif %}
{% endfor %}
310

311 312 313 314 315 316
{%  set warning_list = [] %}
{%  for key in ['apache-certificate', 'apache-key'] %}
{%    if key in slapparameter_dict %}
{%      do warning_list.append('%s is obsolete, please use master-key-upload-url' % (key, )) %}
{%    endif %}
{%  endfor %}
317 318 319 320 321

[publish-information]
<= monitor-publish
recipe = slapos.cookbook:publish
domain = {{ slapparameter_dict.get('domain') }}
322
slave-amount = {{ instance_parameter_dict['slave-instance-list'] | length }}
323
accepted-slave-amount = {{ authorized_slave_list | length }}
324
rejected-slave-amount = {{ rejected_slave_dict | length }}
325
backend-client-caucase-url = {{ caucase_url }}
326
{# sort_keys are important in order to avoid shuffling parameters on each run #}
327
rejected-slave-dict = {{ dumps(json_module.dumps(rejected_slave_dict, sort_keys=True)) }}
328
rejected-slave-promise-url = ${rejected-slave-promise:config-url}
329 330
master-key-upload-url = ${request-kedifa:connection-master-key-upload-url}
master-key-generate-auth-url = ${request-kedifa:connection-master-key-generate-auth-url}
331
kedifa-caucase-url = ${request-kedifa:connection-caucase-url}
332
{% if len(warning_list) > 0 %}
333 334
{# sort_keys are important in order to avoid shuffling parameters on each run #}
warning-list = {{ dumps(json_module.dumps(warning_list, sort_keys=True)) }}
335 336
{% endif %}
{% if len(warning_slave_dict) > 0 %}
337 338
{# sort_keys are important in order to avoid shuffling parameters on each run #}
warning-slave-dict = {{ dumps(json_module.dumps(warning_slave_dict, sort_keys=True)) }}
339
{% endif %}
340 341 342 343 344 345
{% if not aikc_enabled or not aibcc_enabled %}
{% for frontend in frontend_list %}
  {% set section_part = '${request-' + frontend %}
{{ frontend }}-csr_id-certificate = {{ section_part }}:connection-csr_id-certificate}
{% endfor %}
{% endif %}
346
{% if not aikc_enabled %}
347 348 349 350 351
kedifa-csr_id-url = ${request-kedifa:connection-csr_id-url}
kedifa-csr_id-certificate = ${request-kedifa:connection-csr_id-certificate}
{% for frontend in frontend_list %}
  {% set section_part = '${request-' + frontend %}
{{ frontend }}-csr_id-url = {{ section_part }}:connection-csr_id-url}
352 353
{% endfor %}
{% endif %}
354 355 356 357
{% for frontend in frontend_list %}
  {% set section_part = '${request-' + frontend %}
{{ frontend }}-backend-haproxy-statistic-url = {{ section_part }}:connection-backend-haproxy-statistic-url}
{% endfor %}
358 359 360 361
{% if not aibcc_enabled %}
{% for frontend in frontend_list %}
  {% set section_part = '${request-' + frontend %}
{{ frontend }}-backend-client-csr_id-url = {{ section_part }}:connection-backend-client-csr_id-url}
362
{% endfor %}
363
{% endif %}
364

365 366 367 368 369 370 371 372 373 374 375 376 377
# Generate promises for requested nodes
{% for frontend in frontend_list %}
{%   set part_name = 'promise-backend-haproxy-statistic-url-' + frontend %}
{%   do part_list.append(part_name) %}
{%   set section_part = '${request-' + frontend %}
[{{ part_name }}]
<= monitor-promise-base
module = check_url_available
name = check-backend-haproxy-statistic-url-{{ frontend }}.py
config-url =
  {{ section_part }}:connection-backend-haproxy-statistic-url}
{% endfor %}

378 379 380 381 382 383 384 385 386 387 388
#----------------------------
#--
#-- Publish slave information
[publish-slave-information]
recipe = slapos.cookbook:softwaretype
default = ${dynamic-publish-slave-information:rendered}
RootSoftwareInstance = ${dynamic-publish-slave-information:rendered}
replicate = ${dynamic-publish-slave-information:rendered}
custom-personal = ${dynamic-publish-slave-information:rendered}
custom-group = ${dynamic-publish-slave-information:rendered}

389 390 391 392 393 394
[request-kedifa]
<= slap-connection
recipe = slapos.cookbook:requestoptional.serialised
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
395
config-monitor-httpd-port = {{ kedifa_partition_monitor_httpd_port }}
396 397 398 399 400
{% for key in ['kedifa_port', 'caucase_port'] -%}
{%-   if key in slapparameter_dict %}
config-{{ key }} = {{ dumps(slapparameter_dict[key]) }}
{%-   endif %}
{%- endfor %}
401
config-slave-list = {{ dumps(authorized_slave_list) }}
402
config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }}
403

404 405 406
{% set software_url_key = "-kedifa-software-release-url" %}
{% if slapparameter_dict.has_key(software_url_key) %}
software-url = {{ slapparameter_dict.pop(software_url_key) }}
407 408 409 410 411
{% else %}
software-url = ${slap-connection:software-release-url}
{% endif %}
software-type = kedifa
name = kedifa
412
return = slave-kedifa-information master-key-generate-auth-url master-key-upload-url master-key-download-url caucase-url csr_id-url csr_id-certificate  monitor-base-url
413 414 415 416 417 418 419 420
{% set sla_kedifa_key = "-sla-kedifa-" %}
{% set sla_kedifa_key_length = sla_kedifa_key | length %}
{% for key in slapparameter_dict.keys() %}
{%   if key.startswith(sla_kedifa_key) %}
sla-{{ key[sla_kedifa_key_length:] }} = {{ slapparameter_dict.pop(key) }}
{%   endif %}
{% endfor %}

421
[rejected-slave-information]
422
rejected-slave-dict = {{ dumps(rejected_slave_dict) }}
423

424
[warning-slave-information]
425
warning-slave-dict = {{ dumps(warning_slave_dict) }}
426

427
[slave-information]
428
{% for frontend_section in frontend_section_list %}
429
{{ frontend_section }} = {{ "${%s:connection-slave-instance-information-list}" % frontend_section }}
430
{% endfor %}
431

432
[active-slave-instance]
433
{% set active_slave_instance_list = [] %}
434
{% for slave_instance in instance_parameter_dict['slave-instance-list'] %}
435 436
{# Provide a list of slave titles send by master, in order to filter out already destroyed slaves #}
{# Note: This functionality is not yet covered by tests, please modify with care #}
437
{%   do active_slave_instance_list.append(slave_instance['slave_reference']) %}
438
{% endfor %}
439 440
{# sort_keys are important in order to avoid shuffling parameters on each run #}
active-slave-instance-list = {{ json_module.dumps(active_slave_instance_list, sort_keys=True) }}
441

442 443
[dynamic-publish-slave-information]
< = jinja2-template-base
444
template = {{ software_parameter_dict['profile_replicate_publish_slave_information'] }}
445 446 447 448
filename = dynamic-publish-slave-information.cfg
extensions = jinja2.ext.do
extra-context =
    section slave_information slave-information
449
    section rejected_slave_information rejected-slave-information
450
    section active_slave_instance_dict active-slave-instance
451
    section warning_slave_information warning-slave-information
452
    key slave_kedifa_information request-kedifa:connection-slave-kedifa-information
453

454 455
[monitor-base-url-dict]
kedifa = ${request-kedifa:connection-monitor-base-url}
456
{% for frontend in frontend_section_list %}
457
{{ frontend }} = {{ '${' + frontend + ':connection-monitor-base-url}' }}
458
{% endfor %}
459

460 461 462 463
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin/
srv = ${buildout:directory}/srv/
464 465 466 467
backup = ${:srv}/backup
# CAUCASE directories
caucased = ${:srv}/caucased
backup-caucased = ${:backup}/caucased
468 469
# NGINX
rejected-var = ${:var}/rejected-nginx
470 471 472

{% if aikc_enabled %}
[directory]
473 474 475 476 477 478 479 480 481 482 483 484 485
aikc = ${:srv}/aikc

[aikc-config]
caucase-url = ${request-kedifa:connection-caucase-url}

csr = ${directory:aikc}/csr.pem
key = ${directory:aikc}/key.pem
ca-certificate = ${directory:aikc}/cas-ca-certificate.pem
crl = ${directory:aikc}/crl.pem
user-ca-certificate = ${directory:aikc}/user-ca-certificate.pem
user-crl = ${directory:aikc}/user-crl.pem
user-created = ${directory:aikc}/user-created
csr_id = ${directory:aikc}/csr_id
486
data_dir = ${directory:aikc}/caucase-updater
487 488 489

[aikc-user-csr]
recipe = plone.recipe.command
490
organization = {{ instance_parameter_dict['root-instance-title'] }}
491 492 493
organizational_unit = Automatic Internal Kedifa Caucase CSR
command =
  if [ ! -f ${:csr} ] && [ ! -f ${:key} ]  ; then
494
    {{ software_parameter_dict['openssl'] }} req -new -sha256 \
495 496 497 498 499 500 501
      -newkey rsa:2048 -nodes -keyout ${:key} \
      -subj "/O=${:organization}/OU=${:organizational_unit}" \
      -out ${:csr}
  fi
update-command = ${:command}
csr = ${aikc-config:csr}
key = ${aikc-config:key}
502
{#- Can be stopped on error, as does not rely on self provided service #}
503 504 505 506 507 508 509 510
stop-on-error = True


[aikc-caucase-wrapper]
{# jinja2 instead of wrapper is used with context to remove py'u' #}
recipe = slapos.recipe.template:jinja2
context =
  key caucase_url aikc-config:caucase-url
511 512
template = inline:#!{{ software_parameter_dict['dash'] }}/bin/dash
  exec {{ software_parameter_dict['bin_directory'] }}/caucase \
513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528
{# raw block to use context #}
{% raw %}
  --ca-url {{ caucase_url }} \
{% endraw %}
  --ca-crt ${aikc-config:ca-certificate} \
  --user-ca-crt ${aikc-config:user-ca-certificate} \
  --user-crl ${aikc-config:user-crl} \
  --crl ${aikc-config:crl} \
  "$@"

rendered = ${directory:bin}/aikc-caucase-wrapper
mode = 0700

{% do part_list.append('aikc-create-user') %}
[aikc-create-user]
recipe = plone.recipe.command
529 530
{#- The called command is smart enough to survive errors and retry #}
stop-on-error = False
531 532 533 534 535 536 537 538 539 540 541
update-command = ${:command}
command =
  if ! [ -f ${aikc-config:user-created} ] ; then
    ${aikc-caucase-wrapper:rendered} --mode user --send-csr ${aikc-user-csr:csr} > ${aikc-config:csr_id} || exit 1
    cut -d ' ' -f 1 ${aikc-config:csr_id} || exit 1
    csr_id=`cut -d ' ' -f 1 ${aikc-config:csr_id}`
    sleep 1
    ${aikc-caucase-wrapper:rendered} --mode user --get-crt $csr_id ${aikc-config:key} || exit 1
    touch ${aikc-config:user-created}
  fi

542
{% do part_list.append('aikc-user-caucase-updater') %}
543
{% do part_list.append('aikc-user-caucase-updater-promise') %}
544 545
{{ caucase.updater(
     prefix='aikc-user-caucase-updater',
546
     buildout_bin_directory=software_parameter_dict['bin_directory'],
547 548
     updater_path='${directory:service}/aikc-user-caucase-updater',
     url='${aikc-config:caucase-url}',
549
     data_dir='${aikc-config:data_dir}',
550 551 552 553 554 555 556 557
     crt_path='${aikc-config:key}',
     ca_path='${aikc-config:user-ca-certificate}',
     crl_path='${aikc-config:user-crl}',
     key_path='${aikc-config:key}',
     mode='user',
)}}


558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
[aikc-check-certificate]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/aikc-check-certificate
template = inline:
  import sys
  import ssl
  import urlparse
  certificate = sys.argv[2]
  parsed = urlparse.urlparse(sys.argv[1])
  got_certificate = ssl.get_server_certificate((parsed.hostname, parsed.port))
  sys.exit(0) if certificate.strip() == got_certificate.strip() else sys.exit(1)

{% for csr in frontend_list + ['kedifa'] %}
[aikc-{{ csr }}-wrapper]
{# jinja2 instead of wrapper is used with context to remove py'u' #}
recipe = slapos.recipe.template:jinja2
context =
  key csr_id_url request-{{ csr }}:connection-csr_id-url
  key csr_id_certificate request-{{ csr }}:connection-csr_id-certificate
577
template = inline:#!{{ software_parameter_dict['dash'] }}/bin/dash
578 579 580 581 582 583 584 585
  test -f ${directory:aikc}/{{ csr }}-done && exit 0
  ${buildout:executable} ${aikc-check-certificate:rendered} \
{# raw block to use context #}
{% raw %}
  {{ csr_id_url }} \
  """{{ csr_id_certificate }}"""
{% endraw %}
  if [ $? = 0 ]; then
586
    csr_id=`{{ software_parameter_dict['curl'] }}/bin/curl -s -k -g \
587 588 589 590 591 592 593 594 595 596 597 598
{% raw %}
  {{ csr_id_url }} \
{% endraw %}
  ` || exit 1
    ${aikc-caucase-wrapper:rendered} --user-key ${aikc-config:key} --sign-csr $csr_id && touch ${directory:aikc}/{{ csr }}-done
  fi
rendered = ${directory:bin}/aikc-{{ csr }}-wrapper
mode = 0700

{% do part_list.append('aikc-%s' % (csr,)) %}
[aikc-{{ csr }}]
recipe = plone.recipe.command
599 600
{#- The called command is smart enough to survive errors and retry #}
stop-on-error = False
601 602 603 604
command =
  ${aikc-{{ csr }}-wrapper:rendered}
update-command = ${:command}
{% endfor %}
605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
{% endif %} {# if aikc_enabled #}

{% if aibcc_enabled %}
[directory]
aibcc = ${:srv}/aibcc

[aibcc-config]
caucase-url = {{ caucase_url }}

csr = ${directory:aibcc}/csr.pem
key = ${directory:aibcc}/key.pem
ca-certificate = ${directory:aibcc}/cas-ca-certificate.pem
crl = ${directory:aibcc}/crl.pem
user-ca-certificate = ${directory:aibcc}/user-ca-certificate.pem
user-crl = ${directory:aibcc}/user-crl.pem
user-created = ${directory:aibcc}/user-created
csr_id = ${directory:aibcc}/csr_id
622
data_dir = ${directory:aibcc}/caucase-updater
623 624 625

[aibcc-user-csr]
recipe = plone.recipe.command
626
organization = {{ instance_parameter_dict['root-instance-title'] }}
627 628 629
organizational_unit = Automatic Sign Backend Client Caucase CSR
command =
  if [ ! -f ${:csr} ] && [ ! -f ${:key} ]  ; then
630
    {{ software_parameter_dict['openssl'] }} req -new -sha256 \
631 632 633 634 635 636 637
      -newkey rsa:2048 -nodes -keyout ${:key} \
      -subj "/O=${:organization}/OU=${:organizational_unit}" \
      -out ${:csr}
  fi
update-command = ${:command}
csr = ${aibcc-config:csr}
key = ${aibcc-config:key}
638
{#- Can be stopped on error, as does not rely on self provided service #}
639 640 641 642 643 644 645 646
stop-on-error = True


[aibcc-caucase-wrapper]
{# jinja2 instead of wrapper is used with context to remove py'u' #}
recipe = slapos.recipe.template:jinja2
context =
  key caucase_url aibcc-config:caucase-url
647 648
template = inline:#!{{ software_parameter_dict['dash'] }}/bin/dash
  exec {{ software_parameter_dict['bin_directory'] }}/caucase \
649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666
{# raw block to use context #}
{% raw %}
  --ca-url {{ caucase_url }} \
{% endraw %}
  --ca-crt ${aibcc-config:ca-certificate} \
  --user-ca-crt ${aibcc-config:user-ca-certificate} \
  --user-crl ${aibcc-config:user-crl} \
  --crl ${aibcc-config:crl} \
  "$@"

rendered = ${directory:bin}/aibcc-caucase-wrapper
mode = 0700

{% do part_list.append('aibcc-create-user') %}
[aibcc-create-user]
recipe = plone.recipe.command
# the caucase for this part is provided in this profile, so we can't fail
# as otherwise caucase will never be started...
667
{#- XXX: Create promise #}
668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
stop-on-error = False
update-command = ${:command}
command =
  if ! [ -f ${aibcc-config:user-created} ] ; then
    ${aibcc-caucase-wrapper:rendered} --mode user --send-csr ${aibcc-user-csr:csr} > ${aibcc-config:csr_id} || exit 1
    cut -d ' ' -f 1 ${aibcc-config:csr_id} || exit 1
    csr_id=`cut -d ' ' -f 1 ${aibcc-config:csr_id}`
    sleep 1
    ${aibcc-caucase-wrapper:rendered} --mode user --get-crt $csr_id ${aibcc-config:key} || exit 1
    touch ${aibcc-config:user-created}
  fi

{% do part_list.append('aibcc-user-caucase-updater') %}
{% do part_list.append('aibcc-user-caucase-updater-promise') %}
{{ caucase.updater(
     prefix='aibcc-user-caucase-updater',
684
     buildout_bin_directory=software_parameter_dict['bin_directory'],
685 686
     updater_path='${directory:service}/aibcc-user-caucase-updater',
     url='${aibcc-config:caucase-url}',
687
     data_dir='${aibcc-config:data_dir}',
688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713
     crt_path='${aibcc-config:key}',
     ca_path='${aibcc-config:user-ca-certificate}',
     crl_path='${aibcc-config:user-crl}',
     key_path='${aibcc-config:key}',
     mode='user',
)}}

[aibcc-check-certificate]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/aibcc-check-certificate
template = inline:
  import sys
  import ssl
  import urlparse
  certificate = sys.argv[2]
  parsed = urlparse.urlparse(sys.argv[1])
  got_certificate = ssl.get_server_certificate((parsed.hostname, parsed.port))
  sys.exit(0) if certificate.strip() == got_certificate.strip() else sys.exit(1)

{% for csr in frontend_list %}
[aibcc-{{ csr }}-wrapper]
{# jinja2 instead of wrapper is used with context to remove py'u' #}
recipe = slapos.recipe.template:jinja2
context =
  key csr_id_url request-{{ csr }}:connection-backend-client-csr_id-url
  key csr_id_certificate request-{{ csr }}:connection-csr_id-certificate
714
template = inline:#!{{ software_parameter_dict['dash'] }}/bin/dash
715 716 717 718 719 720 721 722
  test -f ${directory:aibcc}/{{ csr }}-done && exit 0
  ${buildout:executable} ${aibcc-check-certificate:rendered} \
{# raw block to use context #}
{% raw %}
  {{ csr_id_url }} \
  """{{ csr_id_certificate }}"""
{% endraw %}
  if [ $? = 0 ]; then
723
    csr_id=`{{ software_parameter_dict['curl'] }}/bin/curl -s -k -g \
724 725 726 727 728 729 730 731 732 733 734 735
{% raw %}
  {{ csr_id_url }} \
{% endraw %}
  ` || exit 1
    ${aibcc-caucase-wrapper:rendered} --user-key ${aibcc-config:key} --sign-csr $csr_id && touch ${directory:aibcc}/{{ csr }}-done
  fi
rendered = ${directory:bin}/aibcc-{{ csr }}-wrapper
mode = 0700

{% do part_list.append('aibcc-%s' % (csr,)) %}
[aibcc-{{ csr }}]
recipe = plone.recipe.command
736 737
{#- The called command is smart enough to survive errors and retry #}
stop-on-error = False
738 739 740 741 742
command =
  ${aibcc-{{ csr }}-wrapper:rendered}
update-command = ${:command}
{% endfor %}
{% endif %} {# if aibcc_enabled #}
743

744 745 746 747 748
[rejected-slave-json]
recipe = slapos.recipe.template:jinja2
filename = rejected-slave.json
directory = ${directory:promise-output}
rendered = ${:directory}/${:filename}
749
template = {{ software_parameter_dict['template_empty'] }}
750
{% if rejected_slave_dict %}
751
{# sort_keys are important in order to avoid shuffling parameters on each run #}
752
content = {{ dumps(json_module.dumps(rejected_slave_dict, indent=2, sort_keys=True)) }}
753 754 755 756 757 758 759 760 761 762 763
{% else %}
content =
{% endif %}
context =
  key content :content

[directory]
service = ${:etc}/service
promise-output = ${:srv}/promise-output

[rejected-slave-publish-configuration]
764
ip = {{ instance_parameter_dict['ipv6-random'] }}
765 766 767 768 769 770
port = 14455

[rejected-slave-publish]
directory = ${rejected-slave-json:directory}
url = https://${rejected-slave-password:user}:${rejected-slave-password:passwd}@[${rejected-slave-publish-configuration:ip}]:${rejected-slave-publish-configuration:port}/${rejected-slave-json:filename}
recipe = slapos.cookbook:wrapper
771 772
command-line = {{ software_parameter_dict['nginx'] }}
  -c ${rejected-slave-template:rendered}
773 774

wrapper-path = ${directory:service}/rejected-slave-publish
775
hash-existing-files =
776
  ${buildout:directory}/software_release/buildout.cfg
777
hash-files =
778 779 780 781 782 783 784 785
  ${rejected-slave-template:rendered}
  ${rejected-slave-certificate:certificate}

[rejected-slave-certificate]
recipe = plone.recipe.command
certificate = ${directory:etc}/rejected-slave.pem
key = ${:certificate}

786
{#- Can be stopped on error, as does not rely on self provided service #}
787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802
stop-on-error = True
update-command = ${:command}
command =
  [ -f ${:certificate} ] && find ${:certificate} -type f -mtime +3 -delete
  if ! [ -f ${:certificate} ] ; then
    openssl req -new -newkey rsa:2048 -sha256 -subj \
      "/CN=${rejected-slave-publish-configuration:ip}" \
      -days 5 -nodes -x509 -keyout ${:certificate} -out ${:certificate}
  fi

[rejected-slave-password]
recipe = slapos.cookbook:generate.password
storage-path = ${directory:etc}/.rejected-slave.passwd
bytes = 8
user = admin

803 804
[rejected-slave-htpasswd]
recipe = plone.recipe.command
805
{#- Can be stopped on error, as does not rely on self provided service #}
806 807
stop-on-error = True
file = ${directory:var}/nginx-rejected.htpasswd
808 809 810
{#- update-command is not needed, as if the ${:password} would change, the whole part will be recalculated #}
password = ${rejected-slave-password:passwd}
command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} ${rejected-slave-password:user} ${:password}
811

812 813
[rejected-slave-template]
recipe = slapos.recipe.template:jinja2
814 815
var = ${directory:rejected-var}
pid = ${directory:var}/nginx-rejected.pid
816
template = inline:
817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
  daemon off;
  pid ${:pid};
  error_log stderr;
  events {
  }
  http {
    include {{ software_parameter_dict['nginx_mime'] }};
    server {
      server_name_in_redirect off;
      port_in_redirect off;
      error_log stderr;
      access_log /dev/null;
      listen [${rejected-slave-publish-configuration:ip}]:${rejected-slave-publish-configuration:port} ssl;
      ssl_certificate ${rejected-slave-certificate:certificate};
      ssl_certificate_key ${rejected-slave-certificate:certificate};
      default_type application/octet-stream;
      client_body_temp_path ${:var} 1 2;
      proxy_temp_path ${:var} 1 2;
      fastcgi_temp_path ${:var} 1 2;
      uwsgi_temp_path ${:var} 1 2;
      scgi_temp_path ${:var} 1 2;

      location / {
        alias ${rejected-slave-json:directory}/;
        autoindex off;
        sendfile on;
        sendfile_max_chunk 1m;
        auth_basic "Rejected slave template";
        auth_basic_user_file ${rejected-slave-htpasswd:file};
      }
    }
848 849
  }

850
rendered = ${directory:etc}/nginx-rejected-slave.conf
851 852

[promise-rejected-slave-publish-ip-port]
853
<= monitor-promise-base
854 855 856 857 858 859
module = check_port_listening
name = rejected-slave-publish-ip-port-listening.py
config-hostname = ${rejected-slave-publish-configuration:ip}
config-port = ${rejected-slave-publish-configuration:port}

[rejected-slave-promise]
860
<= monitor-promise-base
861 862 863 864 865 866 867
module = check_port_listening
module = check_file_state
name = rejected-slave.py
config-filename = ${rejected-slave-json:rendered}
config-state = empty
config-url = ${rejected-slave-publish:url}

868 869 870 871
[caucased-backend-client]
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{{  caucase.caucased(
      prefix='caucased-backend-client',
872
      buildout_bin_directory=software_parameter_dict['bin_directory'],
873 874 875 876 877 878 879 880 881
      caucased_path='${directory:service}/caucased-backend-client',
      backup_dir='${directory:backup-caucased}',
      data_dir='${directory:caucased}',
      netloc=caucase_netloc,
      service_auto_approve_count=0,
      user_auto_approve_count=1,
      key_len=2048,
)}}

882
[buildout]
883
extends =
884 885
  {{ software_parameter_dict['profile_common'] }}
  {{ software_parameter_dict['profile_monitor2'] }}
886 887 888 889
parts =
  monitor-base
  publish-slave-information
  publish-information
890
  request-kedifa
891 892
  rejected-slave-promise
  promise-rejected-slave-publish-ip-port
893 894
  caucased-backend-client
  caucased-backend-client-promise
895
{% for part in part_list %}
896
{{ '  %s' % part }}
897
{% endfor %}
898
#      publish-information
899
{% endif %}