diff --git a/software/caddy-frontend/buildout.hash.cfg b/software/caddy-frontend/buildout.hash.cfg index 97e036f1c57b6e1b80a4a7acbc724ea25e454f2d..0035969993290da979323dd962404303f3c368a2 100644 --- a/software/caddy-frontend/buildout.hash.cfg +++ b/software/caddy-frontend/buildout.hash.cfg @@ -14,7 +14,7 @@ # not need these here). [template] filename = instance.cfg.in -md5sum = d43a1631bcd0f4307507268a06f0fac2 +md5sum = 6d5d97b5f253637c4311eb6a079b3f2d [template-common] filename = instance-common.cfg.in @@ -22,15 +22,19 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b [template-apache-frontend] filename = instance-apache-frontend.cfg.in -md5sum = 9533b13b800ac12b7921e661221a68aa +md5sum = 0c616e6b0369bee0d6e3d25c30244300 + +[template-kedifa] +filename = instance-kedifa.cfg.in +md5sum = a6015ef1571f3ca7bf291a81119653f8 [template-apache-replicate] filename = instance-apache-replicate.cfg.in -md5sum = 1576859772052bcb85ff2b5a7b786410 +md5sum = cd86347c687b99e4fc97c9fcf3ef45dd [template-slave-list] filename = templates/apache-custom-slave-list.cfg.in -md5sum = a244336f20094c632076d30e4ace2254 +md5sum = 0de8bd3f1d1e74033005ffaa48f15481 [template-slave-configuration] filename = templates/custom-virtualhost.conf.in @@ -38,15 +42,15 @@ md5sum = 54ae95597a126ae552c3a913ddf29e5e [template-replicate-publish-slave-information] filename = templates/replicate-publish-slave-information.cfg.in -md5sum = 01efde8febafcff6dde2ebb43e75a9e4 +md5sum = 2d4277abf798905d4fb87be07674b31a [template-caddy-frontend-configuration] filename = templates/Caddyfile.in -md5sum = 7c987ad75fcce6f5b925c7696ff41971 +md5sum = 5f49673807325810e69018d54299f7c8 [template-custom-slave-list] filename = templates/apache-custom-slave-list.cfg.in -md5sum = a244336f20094c632076d30e4ace2254 +md5sum = 0de8bd3f1d1e74033005ffaa48f15481 [caddy-backend-url-validator] filename = templates/caddy-backend-url-validator.in @@ -62,7 +66,7 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b [template-default-slave-virtualhost] filename = templates/default-virtualhost.conf.in -md5sum = 9e00b6d981b9f93a486ef06a47345ebd +md5sum = f07ec2ab9ca4b3656170735f704c8db9 [template-cached-slave-virtualhost] filename = templates/cached-virtualhost.conf.in @@ -70,7 +74,7 @@ md5sum = 7cbcadc295860821ac9d3aaa3cca72c5 [template-log-access] filename = templates/template-log-access.conf.in -md5sum = f2a74f88c7248f199011fa9ec6182f73 +md5sum = 87c55a8b4d6bda7ad4877a52ac2ea758 [template-empty] filename = templates/empty.in @@ -90,7 +94,7 @@ md5sum = 117238225b3fc3c5b5be381815f44c67 [template-nginx-configuration] filename = templates/nginx.cfg.in -md5sum = fadb2fcaf0f2b4fe735617fac222f7ed +md5sum = 5346c66771dec99084eb929d965fff8f [template-nginx-eventsource-slave-virtualhost] filename = templates/nginx-eventsource-slave.conf.in @@ -98,7 +102,7 @@ md5sum = 176cbca2070734a185a7ae5a4d1181c5 [template-nginx-notebook-slave-virtualhost] filename = templates/nginx-notebook-slave.conf.in -md5sum = e018935e2cec2368991f743cab725741 +md5sum = aa6b0860455dc04252c8c8cd29d7cd22 [template-apache-lazy-script-call] filename = templates/apache-lazy-script-call.sh.in @@ -106,7 +110,7 @@ md5sum = ebe5d3d19923eb812a40019cb11276d8 [template-caddy-graceful-script] filename = templates/caddy-graceful-script.sh.in -md5sum = 455f8765a3afd39fb78562fb9e326c42 +md5sum = 41ac81c7939e6dd65f589d3edf5607b1 [caddyprofiledeps-setup] filename = setup.py diff --git a/software/caddy-frontend/common.cfg b/software/caddy-frontend/common.cfg index 0067b749f7e3bf1f8b14fa8a943949b754c57981..8cf2e80b29c3ca06c634bb194596b9818d9b0509 100644 --- a/software/caddy-frontend/common.cfg +++ b/software/caddy-frontend/common.cfg @@ -30,6 +30,24 @@ parts += http-proxy caddyprofiledeps + kedifa-develop + kedifa + +[kedifa-repository] +recipe = slapos.recipe.build:gitclone +repository = https://lab.nexedi.com/luke/kedifa.git +branch = initial-implementation +git-executable = ${git:location}/bin/git + +[kedifa-develop] +recipe = zc.recipe.egg:develop +setup = ${kedifa-repository:location} + +[kedifa] +recipe = zc.recipe.egg +eggs = + kedifa + [caddyprofiledeps-setup] recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/setup.py @@ -90,6 +108,8 @@ gzip = ${gzip:location} logrotate = ${logrotate:location} openssl = ${openssl:location} trafficserver = ${trafficserver:location} +kedifa = ${:bin_directory}/kedifa +kedifa-getter = ${:bin_directory}/kedifa-getter monitor_template = ${monitor-template:output} template_cached_slave_virtualhost = ${template-cached-slave-virtualhost:target} @@ -123,6 +143,7 @@ context = key monitor2_template monitor2-template:rendered key template_caddy_frontend template-caddy-frontend:target key template_caddy_replicate template-caddy-replicate:target + key template_kedifa template-kedifa:target key template_replicate_publish_slave_information template-replicate-publish-slave-information:target key caddy_backend_url_validator caddy-backend-url-validator:output key caddy_custom_http_validator caddy-custom-http-validator:output @@ -153,6 +174,11 @@ recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in mode = 0644 +[template-kedifa] +recipe = slapos.recipe.build:download +url = ${:_profile_base_location_}/instance-kedifa.cfg.in +mode = 0644 + [download-template] recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/templates/${:filename} diff --git a/software/caddy-frontend/instance-apache-frontend.cfg.in b/software/caddy-frontend/instance-apache-frontend.cfg.in index 6682a45844ce93b1b07cd7a3a8d0dacdae80f1f8..dd584176f5eaa4b90a7532134a95bb676f6c3653 100644 --- a/software/caddy-frontend/instance-apache-frontend.cfg.in +++ b/software/caddy-frontend/instance-apache-frontend.cfg.in @@ -1,5 +1,6 @@ {%- if slap_software_type == software_type -%} {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%} +{%- set master_reserved_reference = slapparameter_dict['master-reserved-reference'] -%} [buildout] extends = {{ parameter_dict['common_profile'] }} @@ -126,17 +127,22 @@ filename = custom-personal-instance-slave-list.cfg extensions = jinja2.ext.do slave_instance_list = {{ dumps(instance_parameter['slave-instance-list']) }} extra_slave_instance_list = {{ dumps(instance_parameter.get('configuration.extra_slave_instance_list')) }} +master_key_download_url = {{ dumps(slapparameter_dict['master-key-download-url']) }} +master_reserved_reference = {{ dumps(master_reserved_reference) }} +slave_kedifa_information = {{ dumps(slapparameter_dict['slave-kedifa-information']) }} local_ipv4 = {{ dumps(instance_parameter['ipv4-random']) }} local_ipv6 = {{ dumps(instance_parameter['ipv6-random']) }} software_type = single-custom-personal bin_directory = {{ parameter_dict['bin_directory'] }} sixtunnel_executable = {{ parameter_dict['sixtunnel'] }}/bin/6tunnel +kedifa-getter = {{ parameter_dict['kedifa-getter'] }} service_directory = ${directory:service} extra-context = key caddy_configuration_directory caddy-directory:slave-configuration key nginx_configuration_directory caddy-directory:nginx-slave-configuration key caddy_cached_configuration_directory caddy-directory:slave-with-cache-configuration key slave_with_cache_configuration_directory caddy-directory:slave-with-cache-configuration + key kedifa_getter :kedifa-getter key http_port configuration:plain_http_port key https_port configuration:port key nginx_http_port configuration:plain_nginx_port @@ -144,7 +150,11 @@ extra-context = key public_ipv4 configuration:public-ipv4 key slave_instance_list :slave_instance_list key extra_slave_instance_list :extra_slave_instance_list - key custom_ssl_directory caddy-directory:vh-ssl + key master_key_download_url :master_key_download_url + key master_reserved_reference :master_reserved_reference + key slave_kedifa_information :slave_kedifa_information + key autocert caddy-directory:autocert + key master_autocert_dir caddy-directory:master-autocert-dir key caddy_log_directory caddy-directory:slave-log key local_ipv4 :local_ipv4 key local_ipv6 :local_ipv6 @@ -174,6 +184,7 @@ extra-context = key error_log caddy-configuration:error-log key sixtunnel_executable :sixtunnel_executable key service_directory directory:service + key run_directory directory:etc-run key not_found_file caddy-configuration:not-found-file [dynamic-virtualhost-template-slave] @@ -200,8 +211,7 @@ extra-context = key httpd_home software-release-path:caddy-location key httpd_mod_ssl_cache_directory caddy-directory:mod-ssl key instance_home buildout:directory - key login_certificate ca-frontend:cert-file - key login_key ca-frontend:key-file + key master_autocert_dir caddy-directory:master-autocert-dir key login_ca_crt ca-custom-frontend:rendered key ca_dir certificate-authority:ca-dir key ca_crl certificate-authority:ca-crl @@ -255,9 +265,10 @@ slave-configuration = ${directory:etc}/caddy-slave-conf.d/ slave-with-cache-configuration = ${directory:etc}/caddy-slave-with-cache-conf.d/ cache = ${directory:var}/cache mod-ssl = ${:cache}/httpd_mod_ssl -vh-ssl = ${:slave-configuration}/ssl slave-log = ${directory:log}/httpd nginx-slave-configuration = ${directory:etc}/nginx-slave-conf.d/ +autocert = ${directory:srv}/autocert +master-autocert-dir = ${:autocert}/{{ master_reserved_reference.replace('-','.') }} [caddy-configuration] frontend-configuration = ${directory:etc}/Caddyfile @@ -502,6 +513,7 @@ extra-context = key directory_run directory:run key directory_etc directory:etc key directory_bin directory:bin + key directory_autocert caddy-directory:autocert key caddy_graceful_reload_command caddy-configuration:frontend-graceful-command [frontend-caddy-lazy-graceful] @@ -661,6 +673,7 @@ extra-context = key error_log nginx-configuration:error_log key access_log nginx-configuration:access_log key not_found_file caddy-configuration:not-found-file + key master_autocert_dir caddy-directory:master-autocert-dir [nginx-configuration] access_log = ${directory:log}/nginx-access.log diff --git a/software/caddy-frontend/instance-apache-replicate.cfg.in b/software/caddy-frontend/instance-apache-replicate.cfg.in index 9ea54ae9b0d439d7ad95aa9c763d96ed6059748b..8af66ad2b5383b7dd066c86daac086ecc9b62e82 100644 --- a/software/caddy-frontend/instance-apache-replicate.cfg.in +++ b/software/caddy-frontend/instance-apache-replicate.cfg.in @@ -152,6 +152,7 @@ software-url = ${slap-connection:software-release-url} software-type = {{frontend_type}} return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url +{%- set master_reserved_reference = 'DEFAULT_FRONTEND_KEY' -%} {% for section, frontend_request in request_dict.iteritems() %} [{{section}}] <= replicate @@ -159,6 +160,9 @@ name = {{ frontend_request.get('name') }} {% if frontend_request.get('state') %} state = {{ frontend_request.get('state') }} {% endif%} +config-slave-kedifa-information = ${request-kedifa:connection-slave-kedifa-information} +config-master-key-download-url = ${request-kedifa:connection-master-key-download-url} +config-master-reserved-reference = {{ master_reserved_reference }} {% set slave_configuration_dict = slapparameter_dict %} {% do slave_configuration_dict.update(frontend_request.get('config')) %} {% do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %} @@ -182,6 +186,7 @@ slave-amount = {{ slave_instance_list | length }} accepted-slave-amount = {{ authorized_slave_list | length }} rejected-slave-amount = {{ rejected_slave_dict | length }} rejected-slave-dict = {{ dumps(json_module.dumps(rejected_slave_dict)) }} +master-key-upload-url = ${request-kedifa:connection-master-key-upload-url} #---------------------------- #-- @@ -194,6 +199,26 @@ replicate = ${dynamic-publish-slave-information:rendered} custom-personal = ${dynamic-publish-slave-information:rendered} custom-group = ${dynamic-publish-slave-information:rendered} +[request-kedifa] +<= slap-connection +recipe = slapos.cookbook:requestoptional.serialised +config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }} +config-monitor-username = ${monitor-instance-parameter:username} +config-monitor-password = ${monitor-htpasswd:passwd} + +config-slave-list = {{ dumps(slave_instance_list) }} +config-master-reserved-reference = {{ master_reserved_reference }} + +{% set frontend_software_url_key = "-frontend-software-release-url" %} +{% if slapparameter_dict.has_key(frontend_software_url_key) %} +software-url = {{ slapparameter_dict.pop(frontend_software_url_key) }} +{% else %} +software-url = ${slap-connection:software-release-url} +{% endif %} +software-type = kedifa +name = kedifa +return = slave-kedifa-information master-key-upload-url master-key-download-url + [rejected-slave-information] {% for slave_id, rejected_list in rejected_slave_dict.iteritems() %} {{ slave_id }} = {{ dumps(json_module.dumps(rejected_list)) }} @@ -212,6 +237,7 @@ extensions = jinja2.ext.do extra-context = section slave_information slave-information section rejected_slave_information rejected-slave-information + key slave_kedifa_information request-kedifa:connection-slave-kedifa-information [monitor-conf-parameters] monitor-url-list += @@ -227,6 +253,7 @@ parts = monitor-base publish-slave-information publish-information + request-kedifa {% for part in part_list %} {{ ' %s' % part }} {% endfor %} diff --git a/software/caddy-frontend/instance-caddy-input-schema.json b/software/caddy-frontend/instance-caddy-input-schema.json index 5cc5015b7a262750acf2c5ee9dfc1b5687f06321..bb25a5b70da7338d8cab4e3335ead421da185830 100644 --- a/software/caddy-frontend/instance-caddy-input-schema.json +++ b/software/caddy-frontend/instance-caddy-input-schema.json @@ -17,18 +17,6 @@ "title": "[NOT IMPLEMENTED] SSL CA Certificate", "type": "string" }, - "apache-certificate": { - "description": "SSL Certificate used by the server.", - "textarea": true, - "title": "SSL Certificate", - "type": "string" - }, - "apache-key": { - "description": "SSL Key used by the server.", - "textarea": true, - "title": "SSL Key", - "type": "string" - }, "domain": { "description": "Base Domain for create subdomains (ie.: example.com).", "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$", diff --git a/software/caddy-frontend/instance-kedifa.cfg.in b/software/caddy-frontend/instance-kedifa.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..5c14050542f1f81af6186a93d5aeaaf255e8c0dc --- /dev/null +++ b/software/caddy-frontend/instance-kedifa.cfg.in @@ -0,0 +1,84 @@ +{%- if slap_software_type == software_type -%} +# KeDiFa instance profile +[buildout] +extends = + {{ parameter_dict['common_profile'] }} + {{ parameter_dict['monitor_template'] }} + +parts = + directory + kedifa + slave-kedifa-information + +# Create all needed directories +[directory] +recipe = slapos.cookbook:mkdirectory + +bin = ${buildout:directory}/bin/ +etc = ${buildout:directory}/etc/ +srv = ${buildout:directory}/srv/ +var = ${buildout:directory}/var/ + +backup = ${:srv}/backup +log = ${:var}/log +run = ${:var}/run +service = ${:etc}/service +etc-run = ${:etc}/run +promise = ${:etc}/promise + +logrotate-backup = ${:backup}/logrotate +logrotate-entries = ${:etc}/logrotate.d + +cron-entries = ${:etc}/cron.d +crontabs = ${:etc}/crontabs +cronstamps = ${:etc}/cronstamps + +# KeDiFa directories +kedifa-store = ${:srv}/kedifa + +[jinja2-template-base] +recipe = slapos.recipe.template:jinja2 +rendered = ${buildout:directory}/${:filename} +extra-context = +slapparameter_dict = {{ dumps(instance_parameter['configuration']) }} +slap_software_type = {{ dumps(instance_parameter['slap-software-type']) }} +context = + import json_module json + raw common_profile {{ parameter_dict['common_profile'] }} + key slap_software_type :slap_software_type + key slapparameter_dict :slapparameter_dict + section directory directory + ${:extra-context} + +[kedifa-config] +ip = {{ instance_parameter['ipv4-random'] }} +port = 8080 +store = ${directory:kedifa-store} + +[kedifa] +recipe = slapos.cookbook:wrapper +command-line = {{ parameter_dict['kedifa'] }} + ${kedifa-config:ip} + ${kedifa-config:port} + ${kedifa-config:store} + +wrapper-path = ${directory:service}/kedifa + +# Publish KeDiFa configuration for upload and download for each slave +{%- set slave_kedifa_information = {} -%} +{%- set master_reserved_reference = slapparameter_dict['master-reserved-reference'] -%} +{%- for slave in slapparameter_dict['slave-list'] -%} +{%- set slave_reference = slave['slave_reference'] -%} +{%- if slave_reference != master_reserved_reference -%} +{%- set slave_dict = {} -%} +{%- do slave_dict.__setitem__('key-upload-url', 'http://${kedifa-config:ip}:${kedifa-config:port}/%s' % (slave_reference,)) -%} +{%- do slave_dict.__setitem__('key-download-url', 'http://${kedifa-config:ip}:${kedifa-config:port}/%s' % (slave_reference,)) -%} +{%- do slave_kedifa_information.__setitem__(slave_reference, slave_dict) -%} +{%- endif -%} +{% endfor %} +[slave-kedifa-information] +recipe = slapos.cookbook:publish.serialised +slave-kedifa-information = {{ json_module.dumps(slave_kedifa_information) }} +master-key-upload-url = {{ 'http://${kedifa-config:ip}:${kedifa-config:port}/%s' % (master_reserved_reference,) }} +master-key-download-url = {{ 'http://${kedifa-config:ip}:${kedifa-config:port}/%s' % (master_reserved_reference,) }} +{%- endif -%} {# if slap_software_type in software_type #} diff --git a/software/caddy-frontend/instance-output-schema.json b/software/caddy-frontend/instance-output-schema.json index d8c1746597f7636d1e35823f4a99d56d51852cbd..1b7df844889a256837bfe0138afd92f90b6aac78 100644 --- a/software/caddy-frontend/instance-output-schema.json +++ b/software/caddy-frontend/instance-output-schema.json @@ -10,6 +10,10 @@ "description": "Base domain used by the instance", "type": "string" }, + "master-key-upload-url": { + "description": "URL to PUT PEM bundle of main certificate and key", + "type": "string" + }, "monitor-base-url": { "description": "Base url for monitor", "type": "string" diff --git a/software/caddy-frontend/instance-slave-caddy-input-schema.json b/software/caddy-frontend/instance-slave-caddy-input-schema.json index 92df19e912538c0492061ec53b6a3989048aacf1..9ad678a9bc5be6fa71ebe5033344a8094164c218 100644 --- a/software/caddy-frontend/instance-slave-caddy-input-schema.json +++ b/software/caddy-frontend/instance-slave-caddy-input-schema.json @@ -146,20 +146,6 @@ "title": "[NOT Implemented] SSL Certificate Authority's Certificate", "type": "string" }, - "ssl_crt": { - "default": "", - "description": "Content of the SSL Certificate file", - "textarea": true, - "title": "SSL Certificate", - "type": "string" - }, - "ssl_key": { - "default": "", - "description": "Content of the SSL Key file", - "textarea": true, - "title": "SSL Key", - "type": "string" - }, "ssl_proxy_ca_crt": { "default": "", "description": "[NOT Implemented] Content of the SSL Certificate Authority file of the backend (to be used with ssl-proxy-verify)", diff --git a/software/caddy-frontend/instance-slave-output-schema.json b/software/caddy-frontend/instance-slave-output-schema.json index d77c3f3a895932afe92fcc7114b540999b40e3ab..1cf4b2e0f9a3e469af4caa09eaa136a40041a65a 100644 --- a/software/caddy-frontend/instance-slave-output-schema.json +++ b/software/caddy-frontend/instance-slave-output-schema.json @@ -6,6 +6,10 @@ "description": "Base domain used by the instance", "type": "string" }, + "key-upload-url": { + "description": "URL to PUT PEM bundle of certificate and key", + "type": "array" + }, "log-access-url": { "description": "List of URLs to access logs", "type": "array" diff --git a/software/caddy-frontend/instance.cfg.in b/software/caddy-frontend/instance.cfg.in index ef5b9e881db70c1c6ad6fa29d4be79f917581782..0de5946d9bc4997010589b22238e645efddf9fcb 100644 --- a/software/caddy-frontend/instance.cfg.in +++ b/software/caddy-frontend/instance.cfg.in @@ -28,6 +28,7 @@ custom-personal = ${dynamic-template-caddy-replicate:rendered} single-default = ${dynamic-template-caddy-frontend:rendered} single-custom-personal = ${dynamic-template-caddy-frontend:rendered} replicate = ${dynamic-template-caddy-replicate:rendered} +kedifa = ${dynamic-template-kedifa:rendered} [dynamic-template-caddy-frontend-parameters] {% for key,value in template_frontend_parameter_dict.iteritems() %} @@ -63,6 +64,15 @@ extra-context = raw template_monitor {{ monitor2_template }} raw common_profile {{ common_profile }} +[dynamic-template-kedifa] +< = jinja2-template-base +template = {{ template_kedifa }} +filename = instance-kedifa.cfg +extensions = jinja2.ext.do +extra-context = + section parameter_dict dynamic-template-caddy-frontend-parameters + raw software_type kedifa + [instance-parameter] # Fetches parameters defined in SlapOS Master for this instance. # Always the same. diff --git a/software/caddy-frontend/templates/Caddyfile.in b/software/caddy-frontend/templates/Caddyfile.in index c4ce0e268ee7d91c4c6b21a93e6b648424296d85..e035d688247b14b3061277d37b6b3fd2db36cb2d 100644 --- a/software/caddy-frontend/templates/Caddyfile.in +++ b/software/caddy-frontend/templates/Caddyfile.in @@ -6,7 +6,7 @@ import {{ slave_with_cache_configuration_directory }}/*.conf # Catch-all and 404 for not configured instances :{{ https_port }} { - tls {{ login_certificate }} {{ login_key }} + tls { load {{ master_autocert_dir }} } bind {{ local_ipv4 }} # Compress the output gzip @@ -30,7 +30,7 @@ import {{ slave_with_cache_configuration_directory }}/*.conf # Access to server-status Caddy-style https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv4 }}:{{ https_port }}/server-status { - tls {{ login_certificate }} {{ login_key }} + tls { load {{ master_autocert_dir }} } # Compress the output gzip bind {{ local_ipv4 }} diff --git a/software/caddy-frontend/templates/apache-custom-slave-list.cfg.in b/software/caddy-frontend/templates/apache-custom-slave-list.cfg.in index 30091c545472beaba1fc565205d84fb12a86c6ab..9ff84040447f724a0ea71c63e64b68b4d74942a6 100644 --- a/software/caddy-frontend/templates/apache-custom-slave-list.cfg.in +++ b/software/caddy-frontend/templates/apache-custom-slave-list.cfg.in @@ -36,17 +36,30 @@ sharedscripts = true notifempty = true create = true -[cadirectory] -recipe = slapos.cookbook:mkdirectory -requests = {{ custom_ssl_directory }}/requests/ -private = {{ custom_ssl_directory }}/private/ -certs = {{ custom_ssl_directory }}/certs/ -newcerts = {{ custom_ssl_directory }}/newcerts/ -crl = {{ custom_ssl_directory }}/crl/ +{% if master_key_download_url %} +{% do part_list.append(master_reserved_reference) %} +[{{ master_reserved_reference}}] +recipe = slapos.cookbook:wrapper +destination = {{ master_autocert_dir }}/master.pem +command-line = {{ kedifa_getter }} {{ master_key_download_url }} ${:destination} +filename = {{ master_reserved_reference }}-download +wrapper-path = {{ run_directory }}/${:filename} +{% endif %} +{% if slave_kedifa_information %} +{% set slave_kedifa_information = json_module.loads(slave_kedifa_information) %} +{% else %} +{% set slave_kedifa_information = {} %} +{% endif %} {# Loop thought slave list to set up slaves #} {% for slave_instance in slave_instance_list %} {% set slave_reference = slave_instance.get('slave_reference') %} +{% set slave_kedifa = slave_kedifa_information.get(slave_reference) %} +{% if slave_kedifa %} +{% set key_download_url = slave_kedifa.get('key-download-url') %} +{% else %} +{% set key_download_url = '' %} +{% endif %} {% set slave_type = slave_instance.get('type', '') %} {% set slave_section_title = 'dynamic-template-slave-instance-%s' % slave_reference %} {% set slave_parameter_dict = generic_instance_parameter_dict.copy() %} @@ -137,66 +150,34 @@ bytes = 8 {# ################################################## #} {# Set Slave Certificates if needed #} +{% set cert_dirname = slave_reference.replace('-','.') %} +{% set autocert_dir = '/'.join([autocert, cert_dirname]) %} +{% do slave_parameter_dict.__setitem__('autocert_dir', autocert_dir) %} +[{{ slave_reference }}-path] +recipe = slapos.cookbook:mkdirectory +cert = {{ autocert_dir }} -{# Set ssl certificates for each slave #} -{% for cert_name in ('ssl_ca_crt', 'ssl_csr', 'ssl_proxy_ca_crt')%} -{% if cert_name in slave_instance %} -{% set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) %} -{% set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) %} -{% do part_list.append(cert_title) %} -{% do slave_parameter_dict.__setitem__(cert_name, cert_file) %} -{% do slave_instance.__setitem__('path_to_' + cert_name, cert_file) %} -{# Store certificates on fs #} -[{{ cert_title }}] -< = jinja2-template-base -template = {{ empty_template }} -rendered = {{ cert_file }} -extra-context = - key content {{ cert_title + '-config:value' }} -# Store certificate in config -[{{ cert_title + '-config' }}] -value = {{ dumps(slave_instance.get(cert_name)) }} -{% endif %} -{% endfor %} - -{#- Set Up Certs #} -{% do slave_instance.__setitem__('login_certificate', login_certificate) %} -{% do slave_instance.__setitem__('login_key', login_key) %} -{% do slave_instance.__setitem__('login_ca_crt', login_ca_crt) %} -{% do slave_parameter_dict.__setitem__('ssl_crt', login_certificate) %} -{% do slave_parameter_dict.__setitem__('ssl_key', login_key) %} -{% if 'ssl_key' in slave_instance and 'ssl_crt' in slave_instance %} -{% set cert_title = '%s-crt' % (slave_reference) %} -{% set key_title = '%s-key' % (slave_reference) %} -{% set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) %} -{% set key_file = '/'.join([custom_ssl_directory, key_title.replace('-','.')]) %} -{% do part_list.append(cert_title) %} -{% do part_list.append(key_title) %} -{% do slave_parameter_dict.__setitem__("ssl_crt", cert_file) %} -{% do slave_parameter_dict.__setitem__("ssl_key", key_file) %} -{% do slave_instance.__setitem__('path_to_ssl_crt', cert_file) %} -{% do slave_instance.__setitem__('path_to_ssl_key', key_file) %} - -[{{key_title}}] -< = jinja2-template-base -template = {{ empty_template }} -rendered = {{ key_file }} -key-content = {{ dumps(slave_instance.get('ssl_key')) }} -extra-context = - key content :key-content - -[{{cert_title}}] -< = jinja2-template-base -template = {{ empty_template }} -rendered = {{ cert_file }} -cert-content = {{ dumps(slave_instance.get('ssl_crt')) }} -extra-context = - key content :cert-content -{% endif %} +{% do part_list.append(slave_reference) %} +[{{ slave_reference }}] +recipe = slapos.recipe.template:jinja2 +destination = {{ '${' + slave_reference + '-path:cert}/slave.pem' }} +destination-master = {{ '${' + slave_reference + '-path:cert}/master.pem' }} +source-master = {{ '${' + master_reserved_reference + ':destination}'}} +template = + inline:#!/bin/sh + {{ kedifa_getter }} {{ key_download_url }} ${:destination} + rm -f ${:destination-master} + if [ ! -f ${:destination} ] && [ -f ${:source-master} ] ; then + ln -sf ${:source-master} ${:destination-master} + fi +filename = {{ slave_reference }}-download +rendered = {{ run_directory }}/${:filename} +mode = 700 {# ########################################## #} {# Set Slave Configuration #} [{{ slave_configuration_section_name }}] +autocert_dir = {{ autocert_dir }} https_port = {{ dumps(https_port) }} http_port = {{ dumps(http_port) }} local_ipv4 = {{ dumps(local_ipv4) }} @@ -384,6 +365,7 @@ login_key = {{ dumps(login_key) }} access_log = {{ dumps(access_log) }} error_log = {{ dumps(error_log) }} not_found_file = {{ dumps(not_found_file) }} +master_autocert_dir = {{ master_autocert_dir }} [caddy-log-access] < = jinja2-template-base diff --git a/software/caddy-frontend/templates/caddy-graceful-script.sh.in b/software/caddy-frontend/templates/caddy-graceful-script.sh.in index fb686da00cf27080c315bedea7bc29c511377fc8..9a700e05d7aee853514feab6abebe81324a9714e 100644 --- a/software/caddy-frontend/templates/caddy-graceful-script.sh.in +++ b/software/caddy-frontend/templates/caddy-graceful-script.sh.in @@ -3,12 +3,13 @@ RUN_DIR={{ directory_run }} ETC_DIR={{ directory_etc }} BIN_DIR={{ directory_bin }} +AUTOCERT_DIR={{ directory_autocert }} CADDY_SIGNATURE_FILE=$RUN_DIR/caddy_configuration.signature NCADDY_SIGNATURE_FILE=$RUN_DIR/ncaddy_configuration.signature touch $CADDY_SIGNATURE_FILE -sha256sum $ETC_DIR/Caddyfile $ETC_DIR/log-access.conf $ETC_DIR/caddy-*.d/*.conf $ETC_DIR/caddy-*.d/ssl/*.*key $ETC_DIR/caddy-*.d/ssl/*.*crt* | sort -k 66 > $NCADDY_SIGNATURE_FILE +sha256sum $ETC_DIR/Caddyfile $ETC_DIR/log-access.conf $ETC_DIR/caddy-*.d/*.conf $AUTOCERT_DIR/*/* | sort -k 66 > $NCADDY_SIGNATURE_FILE # If no diff, no restart for now if diff "$CADDY_SIGNATURE_FILE" "$NCADDY_SIGNATURE_FILE"; then diff --git a/software/caddy-frontend/templates/default-virtualhost.conf.in b/software/caddy-frontend/templates/default-virtualhost.conf.in index 3eae50a20cedd68caf7ba3a4b22bd5a8b079cc3e..e03ff496e4a16bbb6aa76b7ed8130fa08cdcdd3b 100644 --- a/software/caddy-frontend/templates/default-virtualhost.conf.in +++ b/software/caddy-frontend/templates/default-virtualhost.conf.in @@ -35,7 +35,8 @@ {%- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %} status 501 / {%- endif %} {#- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter #} - tls {{ slave_parameter.get('path_to_ssl_crt', slave_parameter.get('login_certificate')) }} {{ slave_parameter.get('path_to_ssl_key', slave_parameter.get('login_key')) }} { + tls { + load {{ slave_parameter['autocert_dir'] }} {%- if slave_parameter.get('path_to_ssl_ca_crt') %} # Configuration of accepted clients clients {{ slave_parameter.get('path_to_ssl_ca_crt') }} diff --git a/software/caddy-frontend/templates/nginx-notebook-slave.conf.in b/software/caddy-frontend/templates/nginx-notebook-slave.conf.in index 2067dd8d3f61b6f09724ddc79822520dda9d15cd..18b5defa51494734bf2a3c2d0f911f80ec5db2c6 100644 --- a/software/caddy-frontend/templates/nginx-notebook-slave.conf.in +++ b/software/caddy-frontend/templates/nginx-notebook-slave.conf.in @@ -12,7 +12,8 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_htt log / {{ slave_parameter.get('access_log') }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" errors {{ slave_parameter.get('error_log') }} - tls {{ slave_parameter.get('path_to_ssl_crt', slave_parameter.get('login_certificate')) }} {{ slave_parameter.get('path_to_ssl_key', slave_parameter.get('login_key')) }} { + tls { + load {{ slave_parameter['autocert_dir'] }} {%- if slave_parameter.get('path_to_ssl_ca_crt') %} clients {{ slave_parameter.get('path_to_ssl_ca_crt') }} {%- endif %} diff --git a/software/caddy-frontend/templates/nginx.cfg.in b/software/caddy-frontend/templates/nginx.cfg.in index 4c00360cdc67935d66e5a9a056be5e9f305e6dd6..b566ac80275d454b90b127e051d40189a6176361 100644 --- a/software/caddy-frontend/templates/nginx.cfg.in +++ b/software/caddy-frontend/templates/nginx.cfg.in @@ -59,7 +59,7 @@ import {{ slave_configuration_directory }}/*.conf # Catch-all and 404 for not configured instances :{{ port }} { - tls {{ ssl_certificate }} {{ ssl_key }} + tls { load {{ master_autocert_dir }} } bind {{ local_ip }} # Serve an error 204 (No Content) for favicon.ico status 204 /favicon.ico diff --git a/software/caddy-frontend/templates/replicate-publish-slave-information.cfg.in b/software/caddy-frontend/templates/replicate-publish-slave-information.cfg.in index 02fe3fd4c7f9b1d07b993007ce34309cb040fca4..37dfe6df6c9e47a18200fa05d62150e9bcea2362 100644 --- a/software/caddy-frontend/templates/replicate-publish-slave-information.cfg.in +++ b/software/caddy-frontend/templates/replicate-publish-slave-information.cfg.in @@ -34,6 +34,13 @@ {% do slave_information_dict[slave_reference].__setitem__('request-error-list', rejected_info_list) %} {% endfor %} +{% for slave_reference, kedifa_dict in json_module.loads(slave_kedifa_information).iteritems() %} +{% if slave_reference not in slave_information_dict %} +{% do slave_information_dict.__setitem__(slave_reference, {}) %} +{% endif %} +{% do slave_information_dict[slave_reference].__setitem__('key-upload-url', kedifa_dict['key-upload-url']) %} +{% endfor %} + # Publish information for each slave {% for slave_reference, slave_information in slave_information_dict.iteritems() %} {% set publish_section_title = 'publish-%s' % slave_reference %} diff --git a/software/caddy-frontend/templates/template-log-access.conf.in b/software/caddy-frontend/templates/template-log-access.conf.in index 18864d48e77c2b4a3be8194dd73636a2fb303519..c91d45421c0e19e1418ff5a16f7bebcbe1174eb5 100644 --- a/software/caddy-frontend/templates/template-log-access.conf.in +++ b/software/caddy-frontend/templates/template-log-access.conf.in @@ -3,7 +3,7 @@ https://[{{ parameter_dict['global_ipv6'] }}]:{{ parameter_dict['https_port'] }} bind {{ parameter_dict['local_ipv4'] }} root {{ directory }}/ browse - tls {{ parameter_dict['login_certificate'] }} {{ parameter_dict['login_key'] }} + tls { load {{ parameter_dict['master_autocert_dir'] }} } basicauth "{{ slave }}" {{ slave_password[slave] | trim }} { "Log Access {{ slave }}" / diff --git a/software/caddy-frontend/test/test.py b/software/caddy-frontend/test/test.py index d77980a12c687fbd2bee63929d2463524e868959..09f2b203986bf6b69e2f27ef788e78021b86ace3 100644 --- a/software/caddy-frontend/test/test.py +++ b/software/caddy-frontend/test/test.py @@ -63,6 +63,9 @@ MONITOR_HTTPD_PORT = '13000' MONITOR_F1_HTTPD_PORT = '13001' MONITOR_F2_HTTPD_PORT = '13002' +MASTER_KEY = open('wildcard.example.com.crt').read() + \ + open('wildcard.example.com.key').read() + # for development: debugging logs and install Ctrl+C handler if os.environ.get('DEBUG'): @@ -347,6 +350,16 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase): cls.server_process.terminate() cls.server_https_process.terminate() + @classmethod + def setUpMaster(cls): + parameter_dict = cls.computer_partition.getConnectionParameterDict() + master_key_upload_url = parameter_dict['master-key-upload-url'] + result = requests.put(master_key_upload_url, data=MASTER_KEY) + assert result.status_code == 201 + + # run partitions to update information about the key + cls.runComputerPartition() + @classmethod def setUpSlaves(cls): cls.slave_connection_parameter_dict_dict = {} @@ -376,6 +389,7 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase): try: cls.startServerProcess() super(SlaveHttpFrontendTestCase, cls).setUpClass() + cls.setUpMaster() cls.setUpSlaves() except Exception: cls.tearDownClass() @@ -467,7 +481,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): caddy_custom_https = '''# caddy_custom_https_filled_in_accepted https://caddycustomhttpsaccepted.example.com:%%(https_port)s { bind %%(local_ipv4)s - tls %%(ssl_crt)s %%(ssl_key)s + tls { load %%(autocert_dir)s } log / %%(access_log)s {combined} errors %%(error_log)s @@ -497,7 +511,7 @@ http://caddycustomhttpsaccepted.example.com:%%(http_port)s { apache_custom_https = '''# apache_custom_https_filled_in_accepted https://apachecustomhttpsaccepted.example.com:%%(https_port)s { bind %%(local_ipv4)s - tls %%(ssl_crt)s %%(ssl_key)s + tls { load %%(autocert_dir)s } log / %%(access_log)s {combined} errors %%(error_log)s @@ -530,8 +544,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s { 'domain': 'example.com', 'nginx-domain': 'nginx.example.com', 'public-ipv4': LOCAL_IPV4, - 'apache-certificate': open('wildcard.example.com.crt').read(), - 'apache-key': open('wildcard.example.com.key').read(), '-frontend-authorized-slave-string': '_apache_custom_http_s-accepted _caddy_custom_http_s-accepted', 'port': HTTPS_PORT, @@ -860,15 +872,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s { 'url'].copy() self.assertLogAccessUrlWithPop(parameter_dict, 'url') self.assertEqual( - parameter_dict, { 'domain': 'url.example.com', + 'key-upload-url': 'http://%s:8080/_url' % (LOCAL_IPV4,), 'replication_number': '1', 'url': 'http://url.example.com', 'site_url': 'http://url.example.com', 'secure_access': 'https://url.example.com', 'public-ipv4': LOCAL_IPV4, - } + }, + parameter_dict ) result = self.fakeHTTPSResult( @@ -2259,8 +2272,6 @@ class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin): 'domain': 'example.com', 'nginx-domain': 'nginx.example.com', 'public-ipv4': LOCAL_IPV4, - 'apache-certificate': open('wildcard.example.com.crt').read(), - 'apache-key': open('wildcard.example.com.key').read(), '-frontend-quantity': 2, '-sla-2-computer_guid': 'slapos.test', '-frontend-2-state': 'stopped', @@ -2331,8 +2342,6 @@ class TestEnableHttp2ByDefaultFalseSlave(SlaveHttpFrontendTestCase, 'domain': 'example.com', 'nginx-domain': 'nginx.example.com', 'public-ipv4': LOCAL_IPV4, - 'apache-certificate': open('wildcard.example.com.crt').read(), - 'apache-key': open('wildcard.example.com.key').read(), 'enable-http2-by-default': 'false', 'port': HTTPS_PORT, 'plain_http_port': HTTP_PORT, @@ -2424,8 +2433,6 @@ class TestEnableHttp2ByDefaultDefaultSlave(SlaveHttpFrontendTestCase, 'domain': 'example.com', 'nginx-domain': 'nginx.example.com', 'public-ipv4': LOCAL_IPV4, - 'apache-certificate': open('wildcard.example.com.crt').read(), - 'apache-key': open('wildcard.example.com.key').read(), 'port': HTTPS_PORT, 'plain_http_port': HTTP_PORT, 'nginx_port': NGINX_HTTPS_PORT, @@ -2615,8 +2622,6 @@ class TestMalformedBackenUrlSlave(SlaveHttpFrontendTestCase, 'domain': 'example.com', 'nginx-domain': 'nginx.example.com', 'public-ipv4': LOCAL_IPV4, - 'apache-certificate': open('wildcard.example.com.crt').read(), - 'apache-key': open('wildcard.example.com.key').read(), 'port': HTTPS_PORT, 'plain_http_port': HTTP_PORT, 'nginx_port': NGINX_HTTPS_PORT, @@ -2754,8 +2759,6 @@ class TestQuicEnabled(SlaveHttpFrontendTestCase, TestDataMixin): 'nginx-domain': 'nginx.example.com', 'public-ipv4': LOCAL_IPV4, 'enable-quic': 'true', - 'apache-certificate': open('wildcard.example.com.crt').read(), - 'apache-key': open('wildcard.example.com.key').read(), '-frontend-authorized-slave-string': '_apache_custom_http_s-accepted _caddy_custom_http_s-accepted', 'port': HTTPS_PORT, @@ -2860,8 +2863,6 @@ class TestSlaveBadParameters(SlaveHttpFrontendTestCase, TestDataMixin): 'domain': 'example.com', 'nginx-domain': 'nginx.example.com', 'public-ipv4': LOCAL_IPV4, - 'apache-certificate': open('wildcard.example.com.crt').read(), - 'apache-key': open('wildcard.example.com.key').read(), '-frontend-authorized-slave-string': '_caddy_custom_http_s-reject', 'port': HTTPS_PORT, 'plain_http_port': HTTP_PORT, @@ -3286,8 +3287,6 @@ class TestDuplicateSiteKeyProtection(SlaveHttpFrontendTestCase, TestDataMixin): 'domain': 'example.com', 'nginx-domain': 'nginx.example.com', 'public-ipv4': LOCAL_IPV4, - 'apache-certificate': open('wildcard.example.com.crt').read(), - 'apache-key': open('wildcard.example.com.key').read(), '-frontend-authorized-slave-string': '_caddy_custom_http_s-reject', 'port': HTTPS_PORT, 'plain_http_port': HTTP_PORT,