Commit 2481bc39 authored by Łukasz Nowak's avatar Łukasz Nowak

caddy-frontend: Convert to python3

Profiles and required scripts are converted to be python3 only compatible.
parent 1dd5d303
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg.in
md5sum = 051ae51b86f9aba169a6777fa2239901
md5sum = f1f04e7f27bc6e40a655dd4badb2a8af
[profile-common]
filename = instance-common.cfg.in
......@@ -22,19 +22,19 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
[profile-caddy-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = 1e912fb970401a4b7670b25ba8284a5b
md5sum = 874133120f3a4eda1d0505b8608b280f
[profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 57388e76c7e61b3d7213df8aac0b407d
md5sum = 02a10d92d2b0e270454998cf865b6895
[profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = 964a7f673f441f3a3e90c88ab03e3351
md5sum = 268a945e5c7a52c8766b54a817215c4c
[profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
md5sum = be54431846fe7f3cee65260eefc83d62
md5sum = b3422f3624054f57b78d0e50a0de399a
[profile-caddy-frontend-configuration]
_update_hash_filename_ = templates/Caddyfile.in
......@@ -98,11 +98,11 @@ md5sum = f6f72d03af7d9dc29fb4d4fef1062e73
[caddyprofiledeps-dummy]
filename = caddyprofiledummy.py
md5sum = b41b8de115ad815d0b0db306ad650365
md5sum = 1c866272ec0ea0c161f0c0d80cb6e584
[profile-kedifa]
filename = instance-kedifa.cfg.in
md5sum = b5426129668f39ace55f14012c4a2fd2
md5sum = 2f1c9cc9a3d2f4c6ac59eba5a99d4983
[template-backend-haproxy-rsyslogd-conf]
_update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
......@@ -110,7 +110,7 @@ md5sum = 3336d554661b138dcef97b1d1866803c
[template-slave-introspection-httpd-nginx]
_update_hash_filename_ = templates/slave-introspection-httpd-nginx.conf.in
md5sum = 3067e6ba6c6901821d57d2109517d39c
md5sum = b79addf01b6fb93c2f3d018e83eff766
[template-expose-csr-nginx-conf]
_update_hash_filename_ = templates/expose-csr-nginx.conf.in
......
from __future__ import print_function
import caucase.client
import caucase.utils
import os
import ssl
import sys
import urllib
import urlparse
import urllib.request, urllib.parse, urllib.error
import urllib.parse
from cryptography import x509
from cryptography.hazmat.primitives import serialization
......@@ -24,7 +24,7 @@ class Recipe(object):
def validate_netloc(netloc):
# a bit crazy way to validate that the passed parameter is haproxy
# compatible server netloc
parsed = urlparse.urlparse('scheme://'+netloc)
parsed = urllib.parse.urlparse('scheme://'+netloc)
if ':' in parsed.hostname:
hostname = '[%s]' % parsed.hostname
else:
......@@ -33,7 +33,7 @@ def validate_netloc(netloc):
def _check_certificate(url, certificate):
parsed = urlparse.urlparse(url)
parsed = urllib.parse.urlparse(url)
got_certificate = ssl.get_server_certificate((parsed.hostname, parsed.port))
if certificate.strip() != got_certificate.strip():
raise ValueError('Certificate for %s does not match expected one' % (url,))
......@@ -44,7 +44,7 @@ def _get_exposed_csr(url, certificate):
self_signed = ssl.create_default_context()
self_signed.check_hostname = False
self_signed.verify_mode = ssl.CERT_NONE
return urllib.urlopen(url, context=self_signed).read()
return urllib.request.urlopen(url, context=self_signed).read().decode()
def _get_caucase_client(ca_url, ca_crt, user_key):
......@@ -72,7 +72,7 @@ def _csr_match(*csr_list):
number_list = set([])
for csr in csr_list:
number_list.add(
x509.load_pem_x509_csr(str(csr)).public_key().public_numbers())
x509.load_pem_x509_csr(csr.encode()).public_key().public_numbers())
return len(number_list) == 1
......
......@@ -99,7 +99,7 @@ hash-salt = ${frontend-node-private-salt:value}
init =
import hashlib
import base64
options['value'] = base64.urlsafe_b64encode(hashlib.md5(''.join([options['software-release-url'].strip(), options['hash-salt']])).digest())
options['value'] = base64.urlsafe_b64encode(hashlib.md5(''.join([options['software-release-url'].strip(), options['hash-salt']]).encode()).digest()).decode()
[frontend-node-information]
recipe = slapos.recipe.build
......@@ -359,9 +359,9 @@ partition_ipv6 = ${slap-configuration:ipv6-random}
extra-context =
key caddy_configuration_directory caddy-directory:slave-configuration
key backend_client_caucase_url :backend-client-caucase-url
import urlparse_module urlparse
import furl_module furl
import urllib_module urllib
import operator_module operator
key master_key_download_url :master_key_download_url
key autocert caddy-directory:autocert
key caddy_log_directory caddy-directory:slave-log
......@@ -475,9 +475,14 @@ slave-introspection-graceful-command = ${slave-introspection-validate:output} &&
# BBB: SlapOS Master non-zero knowledge BEGIN
[get-self-signed-fallback-access]
recipe = collective.recipe.shelloutput
commands =
certificate = cat ${self-signed-fallback-access:certificate}
recipe = slapos.recipe.build
certificate-file = ${self-signed-fallback-access:certificate}
init =
import os
options['certificate'] = ''
if os.path.exists(options['certificate-file']):
with open(options['certificate-file'], 'r') as fh:
options['certificate'] = fh.read()
[apache-certificate]
recipe = slapos.recipe.template:jinja2
......@@ -1066,7 +1071,7 @@ config-command =
${logrotate:wrapper-path} -d
[configuration]
{%- for key, value in instance_parameter_dict.iteritems() -%}
{%- for key, value in instance_parameter_dict.items() -%}
{%- if key.startswith('configuration.') %}
{{ key.replace('configuration.', '') }} = {{ dumps(value) }}
{%- endif -%}
......@@ -1076,13 +1081,13 @@ config-command =
{#- There are dangerous keys like recipe, etc #}
{#- XXX: Some other approach would be useful #}
{%- set DROP_KEY_LIST = ['recipe', '__buildout_signature__', 'computer', 'partition', 'url', 'key', 'cert'] %}
{%- for key, value in instance_parameter_dict.iteritems() -%}
{%- for key, value in instance_parameter_dict.items() -%}
{%- if not key.startswith('configuration.') and key not in DROP_KEY_LIST %}
{{ key }} = {{ dumps(value) }}
{%- endif -%}
{%- endfor %}
[software-parameter-section]
{%- for key, value in software_parameter_dict.iteritems() %}
{%- for key, value in software_parameter_dict.items() %}
{{ key }} = {{ dumps(value) }}
{%- endfor %}
......@@ -129,7 +129,7 @@ context =
{% set config_key = "-frontend-config-%s-" % i %}
{% set config_key_length = config_key | length %}
{% set config_dict = {} %}
{% for key in slapparameter_dict.keys() %}
{% for key in list(slapparameter_dict.keys()) %}
{% if key.startswith(sla_key) %}
{% do sla_dict.__setitem__(key[sla_key_length:], slapparameter_dict.pop(key)) %}
# We check for specific configuration regarding the frontend
......@@ -164,7 +164,7 @@ context =
{% set critical_rejected_slave_dict = {} %}
{% set warning_slave_dict = {} %}
{% set used_host_list = [] %}
{% for slave in sorted(instance_parameter_dict['slave-instance-list']) %}
{% for slave in sorted(instance_parameter_dict['slave-instance-list'], key=operator_module.itemgetter('slave_reference')) %}
{% set slave_error_list = [] %}
{% set slave_critical_error_list = [] %}
{% set slave_warning_list = [] %}
......@@ -278,7 +278,7 @@ context =
{% if k in slave %}
{% set crt = slave.get(k, '') %}
{% set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %}
{% do check_popen.communicate(crt) %}
{% do check_popen.communicate(crt.encode()) %}
{% if check_popen.returncode != 0 %}
{% do slave_error_list.append('%s is invalid' % (k,)) %}
{% endif %}
......@@ -296,8 +296,8 @@ context =
{% if slave.get('ssl_key') and slave.get('ssl_crt') %}
{% set key_popen = popen([software_parameter_dict['openssl'], 'rsa', '-noout', '-modulus']) %}
{% set crt_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout', '-modulus']) %}
{% set key_modulus = key_popen.communicate(slave['ssl_key'])[0] | trim %}
{% set crt_modulus = crt_popen.communicate(slave['ssl_crt'])[0] | trim %}
{% set key_modulus = key_popen.communicate(slave['ssl_key'].encode())[0] | trim %}
{% set crt_modulus = crt_popen.communicate(slave['ssl_crt'].encode())[0] | trim %}
{% if not key_modulus or key_modulus != crt_modulus %}
{% do slave_error_list.append('slave ssl_key and ssl_crt does not match') %}
{% endif %}
......@@ -334,7 +334,7 @@ context =
{% do warning_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_warning_list)) %}
{% endif %}
{% endfor %}
{% do authorized_slave_list.sort() %}
{% do authorized_slave_list.sort(key=operator_module.itemgetter('slave_reference')) %}
[monitor-instance-parameter]
monitor-httpd-port = {{ master_partition_monitor_monitor_httpd_port }}
......@@ -356,7 +356,7 @@ return = slave-instance-information-list monitor-base-url backend-client-csr-url
{%- do base_node_configuration_dict.__setitem__(key, slapparameter_dict[key]) %}
{%- endif %}
{%- endfor %}
{% for section, frontend_request in request_dict.iteritems() %}
{% for section, frontend_request in request_dict.items() %}
{% set state = frontend_request.get('state', '') %}
[{{section}}]
<= replicate
......@@ -377,14 +377,14 @@ config-cluster-identification = {{ instance_parameter_dict['root-instance-title'
{# sort_keys are important in order to avoid shuffling parameters on each run #}
{% do node_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list, sort_keys=True)) %}
{% do node_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
{%- for config_key, config_value in node_configuration_dict.iteritems() %}
{%- for config_key, config_value in node_configuration_dict.items() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{%- for config_key, config_value in base_node_configuration_dict.iteritems() %}
{%- for config_key, config_value in base_node_configuration_dict.items() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{% if frontend_request.get('sla') %}
{% for parameter, value in frontend_request.get('sla').iteritems() %}
{% for parameter, value in frontend_request.get('sla').items() %}
sla-{{ parameter }} = {{ value }}
{% endfor %}
{% endif %}
......@@ -489,7 +489,7 @@ config-slave-list = {{ dumps(authorized_slave_list) }}
config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }}
{% set software_url_key = "-kedifa-software-release-url" %}
{% if slapparameter_dict.has_key(software_url_key) %}
{% if software_url_key in slapparameter_dict %}
software-url = {{ slapparameter_dict.pop(software_url_key) }}
{% else %}
software-url = ${slap-connection:software-release-url}
......@@ -499,7 +499,7 @@ name = kedifa
return = slave-kedifa-information master-key-generate-auth-url master-key-upload-url master-key-download-url caucase-url kedifa-csr-url csr-certificate monitor-base-url
{% set sla_kedifa_key = "-sla-kedifa-" %}
{% set sla_kedifa_key_length = sla_kedifa_key | length %}
{% for key in slapparameter_dict.keys() %}
{% for key in list(slapparameter_dict.keys()) %}
{% if key.startswith(sla_kedifa_key) %}
sla-{{ key[sla_kedifa_key_length:] }} = {{ slapparameter_dict.pop(key) }}
{% endif %}
......
......@@ -171,9 +171,14 @@ wrapper-path = ${directory:service}/expose-csr
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[expose-csr-certificate-get]
recipe = collective.recipe.shelloutput
commands =
certificate = cat ${expose-csr-certificate:certificate}
recipe = slapos.recipe.build
certificate-file = ${expose-csr-certificate:certificate}
init =
import os
options['certificate'] = ''
if os.path.exists(options['certificate-file']):
with open(options['certificate-file'], 'r') as fh:
options['certificate'] = fh.read()
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
......@@ -259,10 +264,8 @@ command =
update-command = ${:command}
[{{ slave_reference }}-auth-random]
recipe = collective.recipe.shelloutput
<= auth-random
file = {{ '${' + slave_reference }}-auth-random-generate:file}
commands =
passwd = cat ${:file} 2>/dev/null || echo "NotReadyYet"
{% endfor %}
......@@ -273,11 +276,18 @@ command =
[ ! -f ${:file} ] && {{ software_parameter_dict['curl'] }}/bin/curl -s -g -X POST https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id --cert ${kedifa-config:certificate} --cacert ${kedifa-config:ca-certificate} > ${:file}.tmp && mv ${:file}.tmp ${:file}
update-command = ${:command}
[auth-random]
recipe = slapos.recipe.build
init =
import os
options['passwd'] = 'NotReadyYet'
if os.path.exists(options['file']):
with open(options['file'], 'r') as fh:
options['passwd'] = fh.read()
[master-auth-random]
recipe = collective.recipe.shelloutput
<= auth-random
file = ${master-auth-random-generate:file}
commands =
passwd = cat ${:file} 2>/dev/null || echo "NotReadyYet"
[slave-kedifa-information]
recipe = slapos.cookbook:publish.serialised
......
......@@ -34,7 +34,7 @@ replicate = dynamic-profile-caddy-replicate:output
kedifa = dynamic-profile-kedifa:output
[software-parameter-section]
{% for key,value in software_parameter_dict.iteritems() %}
{% for key,value in software_parameter_dict.items() %}
{{ key }} = {{ dumps(value) }}
{% endfor -%}
......@@ -54,6 +54,7 @@ filename = instance-caddy-replicate.cfg
extra-context =
import subprocess_module subprocess
import functools_module functools
import operator_module operator
import validators validators
import caddyprofiledummy caddyprofiledummy
# Must match the key id in [switch-softwaretype] which uses this section.
......
......@@ -60,7 +60,6 @@ recipe = zc.recipe.egg
eggs =
caddyprofiledeps
websockify
collective.recipe.shelloutput
[profile-common]
recipe = slapos.recipe.template:jinja2
......
......@@ -52,13 +52,13 @@ context =
{#- * setup defaults to simplify other profiles #}
{#- * stabilise values for backend #}
{%- for key, prefix in [('url', 'http_backend'), ('https-url', 'https_backend')] %}
{%- set parsed = urlparse_module.urlparse(slave_instance.get(key, '').strip()) %}
{%- set parsed = urllib_module.parse.urlparse(slave_instance.get(key, '').strip()) %}
{%- set info_dict = {'scheme': parsed.scheme, 'hostname': parsed.hostname, 'port': parsed.port or DEFAULT_PORT[parsed.scheme], 'path': parsed.path, 'fragment': parsed.fragment, 'query': parsed.query, 'netloc-list': slave_instance.get(key + '-netloc-list', '').split() } %}
{%- do slave_instance.__setitem__(prefix, info_dict) %}
{%- endfor %}
{%- do slave_instance.__setitem__('ssl_proxy_verify', ('' ~ slave_instance.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
{%- for key, prefix in [('health-check-failover-url', 'http_backend'), ('health-check-failover-https-url', 'https_backend')] %}
{%- set parsed = urlparse_module.urlparse(slave_instance.get(key, '').strip()) %}
{%- set parsed = urllib_module.parse.urlparse(slave_instance.get(key, '').strip()) %}
{%- set info_dict = slave_instance[prefix] %}
{%- do info_dict.__setitem__('health-check-failover-scheme', parsed.scheme) %}
{%- do info_dict.__setitem__('health-check-failover-hostname', parsed.hostname) %}
......@@ -189,7 +189,7 @@ context =
{%- do furled.set(password = '${'+ slave_password_section +':passwd}') %}
{%- do furled.set(path = slave_reference + '/') %}
{#- We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #}
{%- set slave_log_access_url = urlparse_module.unquote(furled.tostr()) %}
{%- set slave_log_access_url = urllib_module.parse.unquote(furled.tostr()) %}
{%- do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %}
{%- do slave_publish_dict.__setitem__('slave-reference', slave_reference) %}
{%- do slave_publish_dict.__setitem__('backend-client-caucase-url', backend_client_caucase_url) %}
......@@ -212,7 +212,7 @@ context =
{%- for websocket_path in slave_instance.get('websocket-path-list', '').split() %}
{%- set websocket_path = websocket_path.strip('/') %}
{#- Unquote the path, so %20 and similar can be represented correctly #}
{%- set websocket_path = urllib_module.unquote(websocket_path.strip()) %}
{%- set websocket_path = urllib_module.parse.unquote(websocket_path.strip()) %}
{%- if websocket_path %}
{%- do websocket_path_list.append(websocket_path) %}
{%- endif %}
......@@ -332,7 +332,7 @@ http_port = {{ dumps('' ~ configuration['plain_http_port']) }}
local_ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }}
version-hash = {{ version_hash }}
node-id = {{ node_id }}
{%- for key, value in slave_instance.iteritems() %}
{%- for key, value in slave_instance.items() %}
{%- if value is not none %}
{{ key }} = {{ dumps(value) }}
{%- endif %}
......@@ -383,7 +383,7 @@ config-frequency = 720
{%- do part_list.append(publish_section_title) %}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
{%- for key, value in slave_publish_dict.iteritems() %}
{%- for key, value in slave_publish_dict.items() %}
{{ key }} = {{ value }}
{%- endfor %}
{%- else %}
......@@ -463,7 +463,7 @@ csr-certificate = ${expose-csr-certificate-get:certificate}
{%- do furled.set(password = backend_haproxy_configuration['statistic-password']) %}
{%- do furled.set(path = '/') %}
{#- We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #}
{%- set statistic_url = urlparse_module.unquote(furled.tostr()) %}
{%- set statistic_url = urllib_module.parse.unquote(furled.tostr()) %}
backend-haproxy-statistic-url = {{ statistic_url }}
{#- sort_keys are important in order to avoid shuffling parameters on each run #}
node-information-json = {{ json_module.dumps(node_information, sort_keys=True) }}
......@@ -503,7 +503,7 @@ output = ${:file}
< = jinja2-template-base
url = {{ template_backend_haproxy_configuration }}
output = ${backend-haproxy-config:file}
backend_slave_list = {{ dumps(sorted(backend_slave_list)) }}
backend_slave_list = {{ dumps(sorted(backend_slave_list, key=operator_module.itemgetter('slave_reference'))) }}
extra-context =
key backend_slave_list :backend_slave_list
section configuration backend-haproxy-config
......@@ -611,9 +611,14 @@ wrapper-path = {{ directory['service'] }}/expose-csr
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[expose-csr-certificate-get]
recipe = collective.recipe.shelloutput
commands =
certificate = cat ${expose-csr-certificate:certificate}
recipe = slapos.recipe.build
certificate-file = ${expose-csr-certificate:certificate}
init =
import os
options['certificate'] = ''
if os.path.exists(options['certificate-file']):
with open(options['certificate-file'], 'r') as fh:
options['certificate'] = fh.read()
[promise-logrotate-setup]
<= monitor-promise-base
......
......@@ -2,7 +2,7 @@
{% set slave_information_dict = {} %}
# regroup slave information from all frontends
{% for frontend, slave_list_raw in slave_information.iteritems() %}
{% for frontend, slave_list_raw in slave_information.items() %}
{% if slave_list_raw %}
{% set slave_list = json_module.loads(slave_list_raw) %}
{% else %}
......@@ -27,21 +27,21 @@
{% endfor %}
{% endfor %}
{% for slave_reference, rejected_info_list in rejected_slave_information['rejected-slave-dict'].iteritems() %}
{% for slave_reference, rejected_info_list in rejected_slave_information['rejected-slave-dict'].items() %}
{% if slave_reference not in slave_information_dict %}
{% do slave_information_dict.__setitem__(slave_reference, {}) %}
{% endif %}
{% do slave_information_dict[slave_reference].__setitem__('request-error-list', json_module.dumps(rejected_info_list)) %}
{% endfor %}
{% for slave_reference, warning_info_list in warning_slave_information['warning-slave-dict'].iteritems() %}
{% for slave_reference, warning_info_list in warning_slave_information['warning-slave-dict'].items() %}
{% if slave_reference not in slave_information_dict %}
{% do slave_information_dict.__setitem__(slave_reference, {}) %}
{% endif %}
{% do slave_information_dict[slave_reference].__setitem__('warning-list', json_module.dumps(warning_info_list)) %}
{% endfor %}
{% for slave_reference, kedifa_dict in json_module.loads(slave_kedifa_information).iteritems() %}
{% for slave_reference, kedifa_dict in json_module.loads(slave_kedifa_information).items() %}
{% if slave_reference not in rejected_slave_information['rejected-slave-dict'] %}
{% if slave_reference not in slave_information_dict %}
{% do slave_information_dict.__setitem__(slave_reference, {}) %}
......@@ -54,7 +54,7 @@
# Publish information for each slave
{% set active_slave_instance_list = json_module.loads(active_slave_instance_dict['active-slave-instance-list']) %}
{% for slave_reference, slave_information in slave_information_dict.iteritems() %}
{% for slave_reference, slave_information in slave_information_dict.items() %}
{# Filter out destroyed, so not existing anymore, slaves #}
{# Note: This functionality is not yet covered by tests, please modify with care #}
{% if slave_reference in active_slave_instance_list %}
......@@ -68,11 +68,11 @@ recipe = slapos.cookbook:publish
{# sort_keys are important in order to avoid shuffling parameters on each run #}
log-access-url = {{ dumps(json_module.dumps(log_access_url, sort_keys=True)) }}
{% endif %}
{% for key, value in slave_information.iteritems() %}
{% for key, value in slave_information.items() %}
{{ key }} = {{ dumps(value) }}
{% endfor %}
{% endif %}
{% for frontend_key, frontend_value in frontend_information.iteritems() %}
{% for frontend_key, frontend_value in frontend_information.items() %}
{{ frontend_key }} = {{ frontend_value }}
{% endfor %}
{% endfor %}
......
......@@ -23,7 +23,7 @@ http {
fastcgi_temp_path {{ parameter_dict['var'] }} 1 2;
uwsgi_temp_path {{ parameter_dict['var'] }} 1 2;
scgi_temp_path {{ parameter_dict['var'] }} 1 2;
{% for slave, directory in slave_log_directory.iteritems() %}
{% for slave, directory in slave_log_directory.items() %}
location /{{ slave }} {
alias {{ directory }};
autoindex on;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment