Skip to content

slapos
slapos
Commit fe38d320 authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

XXX rapid-cdn: Implement simple and working frontend support 

Checkpoint: front-haproxy serves some backend, but it still speaks directly to the backend, and shall use back-haproxy for backend intelligence.
parent a1804173
Show whitespace changes
Inline Side-by-side
Showing with 965 additions and 2 deletions
software/rapid-cdn/buildout.hash.cfg 0 → 100644
View file @ fe38d320
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template]
_update_hash_filename_ = instance.cfg.jinja2
md5sum = 32e0289fa8e4b8b7baf6b0454fe9b7e2
[template-common]
_update_hash_filename_ = instance-common.cfg.jinja2
md5sum = c3a54ac864508ecb693c16cdec06671e
[template-rapid-cdn-human]
_update_hash_filename_ = instance-human.cfg.jinja2
md5sum = 6a57e7727c8d258f5ae81bdbb86328cb
[template-rapid-cdn-pop]
_update_hash_filename_ = instance-pop.cfg.jinja2
md5sum = 151e07e7ced18c631c11344bf4d7cf51
[template-rapid-cdn-kedifa]
_update_hash_filename_ = instance-kedifa.cfg.jinja2
md5sum = 13c92f006a0c06789b691ca6b21a8195
[template-rapid-cdn-log-aggregator]
_update_hash_filename_ = instance-log-aggregator.cfg.jinja2
md5sum = 6a57e7727c8d258f5ae81bdbb86328cb
[template-rapid-cdn-frontend]
_update_hash_filename_ = instance-frontend.cfg.jinja2
md5sum = 281e99430d9ca523ae8a015550edffb3
[template-front-haproxy]
_update_hash_filename_ = template/front-haproxy.cfg.jinja2
md5sum = 9a7529e2633b59a132f30395b6755ac2
[template-front-rsyslogd]
_update_hash_filename_ = template/front-rsyslogd.conf.jinja2
md5sum = e975b5cb71915eb9743928a5fee5acaf
software/rapid-cdn/instance-common.cfg.jinja2 0 → 100644
View file @ fe38d320
[buildout]
eggs-directory = {{ software_parameter_dict['eggs-directory'] }}
develop-eggs-directory = {{ software_parameter_dict['develop-eggs-directory'] }}
offline = true
[kedifa-config]
# shared kedifa configuration parameters for the whole cluster
marker = __NotReadyYet__
[csr]
recipe = plone.recipe.command
organization = ${slap-configuration:root-instance-title}
command =
if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ] ; then
/bin/bash -c '{{ software_parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
-subj "/O=${:organization}/OU=${:organizational_unit}" \
-reqexts SAN \
-config <(cat {{ software_parameter_dict['openssl-cnf'] }} \
<(printf "\n[SAN]\nsubjectAltName=IP:${:ip}")) \
-out ${:template-csr}'
fi
update-command = ${:command}
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin/
etc = ${buildout:directory}/etc/
srv = ${buildout:directory}/srv/
var = ${buildout:directory}/var/
tmp = ${buildout:directory}/tmp/
backup = ${:srv}/backup
log = ${:var}/log
run = ${:var}/run
service = ${:etc}/service
etc-run = ${:etc}/run
[slap_connection]
# Kept for backward compatibility
computer_id = ${slap-connection:computer-id}
partition_id = ${slap-connection:partition-id}
server_url = ${slap-connection:server-url}
software_release_url = ${slap-connection:software-release-url}
key_file = ${slap-connection:key-file}
cert_file = ${slap-connection:cert-file}
software/rapid-cdn/instance-frontend.cfg.jinja2 0 → 100644
View file @ fe38d320
[buildout]
extends =
{{ software_parameter_dict['template-common'] }}
{{ software_parameter_dict['template-monitor2'] }}
{#- Manage frontends #}
{%- set DEFAULT_SCHEME_PORT_MAPPING = {'http': 80, 'https': 443} %}
{%- set FRONTEND_PARAMETER_KEY_LIST = ['url', 'domain', 'domain-icp-validation'] %} {#- matches schema-rapid-cdn-frontend-shared.input.json #}
{%- set BACKWARD_COMPATIBLE_FRONTEND_KEY_LIST = ['url'] %} {#- based on old JSON and also found important keys #}
{%- set FRONTEND_TRANSMIT_LIST = [] %} {#- List of frontend parameters to be passed to kedifa and exposed nodes #}
{%- set FRONTEND_LIST = [] %} {#- List of frontend parameters used internally for publishing and processing #}
{%- for slave in instance_parameter_dict['slave-instance-list'] %}
{%- set frontend = {} %}
{%- set frontend_transmit = {} %}
{%- do frontend.__setitem__('title', slave['slave_title']) %}
{%- do frontend.__setitem__('reference', slave['slave_reference']) %}
{%- do frontend.__setitem__('hostname', '%s.%s' % (frontend['reference'].replace("-", "").replace("_", "").lower(), slapparameter_dict['autogeneration-domain'])) %}
{%- do frontend_transmit.__setitem__('title', slave['slave_title']) %}
{%- do frontend_transmit.__setitem__('reference', slave['slave_reference']) %}
{%- do frontend_transmit.__setitem__('hostname', '%s.%s' % (frontend['reference'].replace("-", "").replace("_", "").lower(), slapparameter_dict['autogeneration-domain'])) %}
{%- do frontend.__setitem__('rapid-https','https://%s/' % (frontend['hostname'],)) %}
{%- do frontend.__setitem__('rapid-http','http://%s/' % (frontend['hostname'],)) %}
{%- do frontend.__setitem__('section-title', 'publish-frontend-' ~ hashlib.md5(frontend['title'].encode('utf-8')).hexdigest()) %}
{%- set frontend_dict = {} %}
{%- if '_' in slave %}
{%- do frontend_dict.update(json.loads(slave['_'])) %}
{%- endif %}
{%- do frontend.__setitem__('__style__', 'modern') %}
{%- for key in BACKWARD_COMPATIBLE_FRONTEND_KEY_LIST %}
{%- if key in slave %}
{%- do frontend.__setitem__('__style__', 'obsolete') %}
{%- do frontend_dict.__setitem__(key, slave[key]) %}
{%- endif %}
{%- endfor %}
{%- do frontend.__setitem__('_', frontend_dict) %}
{%- for key in FRONTEND_PARAMETER_KEY_LIST %}
{%- if key in frontend %}
{%- do frontend_transmit.__setitem__(key, frontend[key]) %}
{%- endif %}
{%- endfor %}
{%- set url_parsed = urllib_parse.urlparse(frontend['_']['url']) %}
{%- do frontend_transmit.__setitem__('backend-url', {
'scheme': url_parsed.scheme,
'hostname': url_parsed.hostname,
'port': url_parsed.port or DEFAULT_SCHEME_PORT_MAPPING[url_parsed.scheme],
'path': url_parsed.path,
'fragment': url_parsed.fragment,
'query': url_parsed.query}) %}
{%- do FRONTEND_TRANSMIT_LIST.append(frontend_transmit) %}
{%- do FRONTEND_LIST.append(frontend) %}
{%- endfor %}
{%- set PART_LIST = [] %}
{%- for frontend in FRONTEND_LIST | sort(attribute='title') %}
{%- do PART_LIST.append(frontend['section-title']) %}
[{{ frontend['section-title'] }}]
-slave-reference = {{ frontend['reference'] }}
{%- if frontend.pop('__style__') == 'modern' %}
recipe = slapos.cookbook:publish.serialised
rapid-url = {{ frontend['rapid-https'] }}
{%- else %}
recipe = slapos.cookbook:publish
01-ATTENTION = Backward compatbility and obsolete frontend definition detected.
02-ATTENTION = The instance parameters are hidden in the UI, you see everything empty.
03-ATTENTION = Do not update parameters in the UI without checking the previous ones, as otherwise they will be lost.
04-ATTENTION = You can access your parameters by switching to XML view with "Show Parameter XML".
05-ATTENTION = Then you can wisely migrate them to the new way.
url = {{ frontend['rapid-http'] }}
secure_access = {{ frontend['rapid-https'] }}
site_url = {{ frontend['rapid-http'] }}
domain = {{ frontend['hostname'] }}
log-access-url = XXX-TODO
backend-client-caucase-url = XXX-TODO
key-generate-auth-url = XXX-TODO
replication_number = XXX-TODO
{%- endif %}
{%- endfor %}
{#- Request the tree #}
{%- macro sla(sla_key) %}
{%- set sla_key_length = sla_key | length %}
{%- for key in slapparameter_dict.keys() %}
{%- if key.startswith(sla_key) %}
sla-{{ key[sla_key_length:] }} = {{ slapparameter_dict[key] }}
{%- endif %}
{%- endfor %}
{%- endmacro %}
{%- for name in ['kedifa', 'human', 'log-aggregator'] %}
{%- set sla_key = '-sla-%s-computer_guid' % (name,) %}
{%- set software_type = 'rapid-cdn-' ~ name %}
{%- if not sla_key in slapparameter_dict %}
{%- do slapparameter_dict.__setitem__(sla_key, '${slap-connection:computer-id}') %}
{%- endif %}
{%- set software_key = '-%s-software-release-url' % (name,) %}
{%- if not software_key in slapparameter_dict %}
{%- do slapparameter_dict.__setitem__(software_key, '${slap-connection:software-release-url}') %}
{%- endif %}
[request-{{ name }}]
<= slap-connection
recipe = slapos.cookbook:requestoptional.serialised
name = {{ name }}
software-type = {{ software_type }}
software-url = {{ slapparameter_dict['-%s-software-release-url' % (name,)] }}
{{ sla('-sla-%s-' % (name,)) }}
{%- endfor %}
{%- set NODE_PART_LIST = [] %}
{%- for node, node_dict in slapparameter_dict.get('cluster-dict', {}).items() %}
{%- set node_name = 'node-' ~ node %}
{%- set part_name = 'request-' ~ node %}
{%- do NODE_PART_LIST.append(part_name) %}
{%- set sla_dict = node_dict.get('sla', {'computer_guid': '${slap-connection:computer-id}'}) %}
{%- set software_release_url = node_dict.pop('-software-release-url', '${slap-connection:software-release-url}') %}
[{{part_name }}]
<= slap-connection
recipe = slapos.cookbook:requestoptional.serialised
name = {{ node_name }}
software-type = rapid-cdn-pop
software-url = {{ software_release_url }}
config-node-name = {{ node_name }}
config-frontend-list = {{ dumps(FRONTEND_TRANSMIT_LIST | sort(attribute='title')) }}
config-kedifa-caucase-url = ${kedifa-caucase-promise:config-url}
{%- for sla_key, sla_value in sla_dict.items() %}
sla-{{ sla_key }} = {{ sla_value }}
{%- endfor %}
{%- endfor %}
[request-kedifa]
config-kedifa-port = {{ dumps(slapparameter_dict.get('-kedifa-port', 7890)) }}
config-caucase-port = {{ dumps(slapparameter_dict.get('-kedifa-caucase-port', 8090)) }}
config-frontend-list = {{ dumps(FRONTEND_TRANSMIT_LIST | sort(attribute='title')) }}
return =
caucase-url
master-key-generate-auth-url
master-key-upload-url
[kedifa-caucase-promise]
<= monitor-promise-base
promise = check_url_available
name = ${:_buildout_section_name_}.py
config-http-code = 200
config-url = ${request-kedifa:connection-caucase-url}
[directory]
url-done = ${:var}/url-done
[url-ready]
recipe = slapos.recipe.build
depot = ${directory:url-done}
file = ${directory:url-done}/${:_buildout_section_name_}
marker = ${kedifa-config:marker}
init =
import os
if options['marker'] in options['url']:
if os.path.exists(options['file']):
os.unlink(options['file'])
else:
try:
with open(options['file'], 'w') as fh:
fh.write('')
except FileNotFoundError:
# directories are created during install, but promise runs during init
# so do nothing in such case
pass
[kedifa-master-key-generate-auth-url]
<= url-ready
url = ${request-kedifa:connection-master-key-generate-auth-url}
[kedifa-master-key-upload-url]
<= url-ready
url = ${request-kedifa:connection-master-key-upload-url}
[kedifa-master-key-promise]
<= monitor-promise-base
promise = check_file_state
name = ${:_buildout_section_name_}.py
config-state = present
[kedifa-master-key-generate-auth-url-promise]
<= kedifa-master-key-promise
config-filename = ${kedifa-master-key-generate-auth-url:file}
url = ${kedifa-master-key-generate-auth-url:url}
[kedifa-master-key-upload-url-promise]
<= kedifa-master-key-promise
config-filename = ${kedifa-master-key-upload-url:file}
url = ${kedifa-master-key-upload-url:url}
[publish-information]
<= monitor-publish
recipe = slapos.cookbook:publish.serialised
xxx-replace-with-information-fetch-depends =
${request-human:name}
${request-log-aggregator:name}
{%- for node_part in NODE_PART_LIST %}
{{ '${' ~ node_part}}:name}
{%- endfor %}
kedifa-caucase-url = ${kedifa-caucase-promise:config-url}
key-generate-auth-url = ${kedifa-master-key-generate-auth-url-promise:url}
key-upload-url = ${kedifa-master-key-upload-url-promise:url}
[buildout]
parts +=
publish-information
{% for part in PART_LIST %}
{{ ' %s' % part }}
{% endfor %}
software/rapid-cdn/instance-human.cfg.jinja2 0 → 100644
View file @ fe38d320
[buildout]
extends =
{{ software_parameter_dict['template-common'] }}
{{ software_parameter_dict['template-monitor2'] }}
software/rapid-cdn/instance-kedifa.cfg.jinja2 0 → 100644
View file @ fe38d320
{%- import "caucase" as caucase with context %}
[buildout]
extends =
{{ software_parameter_dict['template-common'] }}
{{ software_parameter_dict['template-monitor2'] }}
parts +=
kedifa
caucased
caucase-updater
publish-information
[caucased]
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{% set caucase_host = '[' ~ instance_parameter_dict['ipv6-random'] ~ ']' %}
{%- set caucase_netloc = caucase_host ~ ':' ~ slapparameter_dict['caucase-port'] -%}
{%- set caucase_url = 'http://' ~ caucase_netloc -%}
{{- caucase.caucased(
prefix='caucased',
buildout_bin_directory=software_parameter_dict['bin-directory'],
caucased_path='${directory:service}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:caucased}',
netloc=caucase_netloc,
tmp='${directory:tmp}',
service_auto_approve_count=0,
user_auto_approve_count=1,
key_len=2048
)}}
[caucase-updater]
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=software_parameter_dict['bin-directory'],
updater_path='${directory:service}/caucase-updater',
url=caucase_url,
data_dir='${directory:srv}/caucase-updater',
crt_path='${kedifa-config:certificate}',
ca_path='${kedifa-config:ca-certificate}',
crl_path='${kedifa-config:crl}',
key_path='${kedifa-csr:key}',
on_renew='${kedifa-reloader:rendered}',
template_csr='${kedifa-csr:template-csr}'
)}}
[kedifa-csr]
<= csr
organizational_unit = Kedifa Partition
template-csr = ${kedifa-config:template-csr}
key = ${kedifa-config:key}
ip = ${kedifa-config:ip}
[kedifa-reloader]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:etc-run}/kedifa-reloader
mode = 0700
template = inline:#!{{ software_parameter_dict['dash'] }}
{%- raw %}
{{ content }}
{%- endraw %}
command =
kill -HUP `cat ${kedifa-config:pidfile}`
context =
key content :command
[directory]
# KeDiFa directories
srv-kedifa = ${:srv}/kedifa
# CAUCASE directories
caucased = ${:srv}/caucased
backup-caucased = ${:backup}/caucased
# reservation
reservation = ${:srv}/reservation
[kedifa-config]
ip = {{ instance_parameter_dict['ipv6-random'] }}
port = {{ slapparameter_dict['kedifa-port'] }}
db = ${directory:srv-kedifa}/kedifa.sqlite
certificate = ${directory:srv-kedifa}/certificate.pem
key = ${:certificate}
ca-certificate = ${directory:srv-kedifa}/ca-certificate.pem
crl = ${directory:srv-kedifa}/crl.pem
template-csr = ${directory:srv-kedifa}/template-csr.pem
pidfile = ${directory:run}/kedifa.pid
logfile = ${directory:log}/kedifa.log
[kedifa]
recipe = slapos.cookbook:wrapper
command-line = {{ software_parameter_dict['kedifa'] }}
--ip ${kedifa-config:ip}
--port ${kedifa-config:port}
--db ${kedifa-config:db}
--certificate ${kedifa-config:certificate}
--ca-certificate ${kedifa-config:ca-certificate}
--crl ${kedifa-config:crl}
--pidfile ${kedifa-config:pidfile}
--logfile ${kedifa-config:logfile}
wrapper-path = ${directory:service}/kedifa
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[master-auth-random-config]
file = ${directory:reservation}/${:_buildout_section_name_}
[master-auth-random-promise]
<= monitor-promise-base
promise = check_file_state
name = ${:_buildout_section_name_}.py
config-filename = ${master-auth-random-config:file}
config-state = present
[master-auth-random]
recipe = slapos.recipe.build
kedifa-reserve = https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id
cert = ${kedifa-config:certificate}
cacert = ${kedifa-config:ca-certificate}
result = ${kedifa-config:marker}
file = ${master-auth-random-promise:config-filename}
# The reservation has to happen during init, this slapos.recipe.build is used
# with python's requests instead of curl
init =
import errno
import os
import requests
try:
with open(options['file']) as fh:
options['result'] = fh.read()
except FileNotFoundError:
try:
result = requests.post(
options['kedifa-reserve'],
cert=options['cert'],
verify=options['cacert']
)
status_code = result.status_code
except Exception:
status_code = 500
if status_code == 201:
with open(options['file'], 'w') as fh:
fh.write(result.text)
options['result'] = result.text
[promise-kedifa-http-reply]
<= monitor-promise-base
promise = check_url_available
name = kedifa-http-reply.py
# Kedifa replies 400 on /, so use it to be sure that Kedifa replied
config-http-code = 400
config-url = https://[${kedifa-config:ip}]:${kedifa-config:port}
config-ca-cert-file = ${kedifa-config:ca-certificate}
[publish-information]
recipe = slapos.cookbook:publish.serialised
caucase-url = {{ caucase_url }}
master-key-generate-auth-url = ${promise-kedifa-http-reply:config-url}/${master-auth-random:result}/generateauth
master-key-upload-url = ${promise-kedifa-http-reply:config-url}/${master-auth-random:result}?auth=
master-key-download-url = ${promise-kedifa-http-reply:config-url}/${master-auth-random:result}
software/rapid-cdn/instance-log-aggregator.cfg.jinja2 0 → 100644
View file @ fe38d320
[buildout]
extends =
{{ software_parameter_dict['template-common'] }}
{{ software_parameter_dict['template-monitor2'] }}
software/rapid-cdn/instance-pop.cfg.jinja2 0 → 100644
View file @ fe38d320
{%- import "caucase" as caucase with context %}
[buildout]
extends =
{{ software_parameter_dict['template-common'] }}
{{ software_parameter_dict['template-monitor2'] }}
parts +=
kedifa-caucase-updater
front-haproxy
[directory]
kedifa-ssl = ${:srv}/kedifa-ssl
kedifa-caucase-updater = ${:srv}/kedifa-caucase-updater
[kedifa-config]
certificate = ${directory:kedifa-ssl}/certificate.pem
key = ${:certificate}
ca-certificate = ${directory:kedifa-ssl}/ca-certificate.pem
crl = ${directory:kedifa-ssl}/crl.pem
template-csr = ${directory:kedifa-ssl}/template-csr.pem
[kedifa-csr]
<= csr
organizational_unit = {{ slapparameter_dict['node-name'] }}
template-csr = ${kedifa-config:template-csr}
key = ${kedifa-config:key}
ip = {{ instance_parameter_dict['ipv6-random'] }}
[kedifa-caucase-updater]
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{{ caucase.updater(
prefix='kedifa-caucase-updater',
buildout_bin_directory=software_parameter_dict['bin-directory'],
updater_path='${directory:service}/kedifa-caucase-updater',
url=slapparameter_dict['kedifa-caucase-url'],
data_dir='${directory:kedifa-caucase-updater}',
crt_path='${kedifa-config:certificate}',
ca_path='${kedifa-config:ca-certificate}',
crl_path='${kedifa-config:crl}',
key_path='${kedifa-csr:key}',
template_csr='${kedifa-csr:template-csr}'
)}}
[directory]
backend-haproxy-rsyslogd-spool = ${:run}/backend-haproxy-rsyslogd-spool
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
extra-context =
context =
${:extra-context}
[front-rsyslogd-configuration-section]
log-socket = ${directory:run}/frlog.sck
log-file = ${directory:log}/front-haproxy.log
pid-file = ${directory:run}/front-rsyslogd.pid
spool-directory = ${directory:backend-haproxy-rsyslogd-spool}
[front-rsyslogd-configuration]
<= jinja2-template-base
template = {{ software_parameter_dict['template-front-rsyslogd'] }}
rendered = ${directory:etc}/front-rsyslogd.conf
extra-context =
section configuration front-rsyslogd-configuration-section
[front-rsyslogd]
recipe = slapos.cookbook:wrapper
command-line = {{ software_parameter_dict['rsyslogd'] }} -n -i ${front-rsyslogd-configuration-section:pid-file} -f ${front-rsyslogd-configuration:rendered}
wrapper-path = ${directory:service}/${:_buildout_section_name_}
log-socket = ${front-rsyslogd-configuration-section:log-socket}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[front-configuration-section]
log-socket = ${front-rsyslogd:log-socket}
frontend-list = {{ dumps(slapparameter_dict['frontend-list']) }}
pid-file = ${directory:run}/front-haproxy.pid
# XXX!
request-timeout = 100
backend-connect-timeout = 100
backend-connect-retries = 3
http-port = 8080
https-port = 4443
ipv4 = {{ dumps(instance_parameter_dict['ipv4-random']) }}
ipv6 = {{ dumps(instance_parameter_dict['ipv6-random']) }}
statistic-port = 9696
statistic-certificate = /dev/null
statistic-identification = oink
statistic-username = statu
statistic-password = statp
certificate = /dev/null
[front-configuration]
<= jinja2-template-base
binary = {{ software_parameter_dict['haproxy'] }}
template = {{ software_parameter_dict['template-front-haproxy'] }}
rendered = ${directory:etc}/front-haproxy.cfg
extra-context =
section configuration front-configuration-section
[front-haproxy]
recipe = slapos.cookbook:wrapper
command-line = ${front-configuration:binary} -f ${front-configuration:rendered}
wrapper-path = ${directory:service}/${:_buildout_section_name_}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
software/rapid-cdn/instance.cfg.jinja2 0 → 100644
View file @ fe38d320
[buildout]
extends = {{ software_parameter_dict['template-common'] }}
parts =
switch-softwaretype
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
{#- Cluster itself #}
rapid-cdn-frontend = template-rapid-cdn-frontend:rendered
rapid-cdn-pop = template-rapid-cdn-pop:rendered
rapid-cdn-human = template-rapid-cdn-human:rendered
rapid-cdn-kedifa = template-rapid-cdn-kedifa:rendered
rapid-cdn-log-aggregator = template-rapid-cdn-log-aggregator:rendered
{#- Default #}
default = ${:rapid-cdn-frontend}
{#- Fallback #}
RootSoftwareInstance = ${:rapid-cdn-frontend}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extensions = jinja2.ext.do
extra-context =
context =
import json json
import hashlib hashlib
key slapparameter_dict slap-configuration:configuration
section instance_parameter_dict slap-configuration
section software_parameter_dict software-parameter-section
${:extra-context}
import-list =
file caucase software-parameter-section:caucase-jinja2-library
[template-base]
<= jinja2-template-base
filename = ${:_buildout_section_name_}.cfg
{%- for template in software_parameter_dict.keys() %}
{%- if template.startswith('template-rapid-cdn-') %}
[{{ template }}]
<= template-base
template = {{ software_parameter_dict[template] }}
{%- endif %}
{%- endfor %}
[template-rapid-cdn-frontend]
extra-context =
import urllib_parse urllib.parse
[software-parameter-section]
{%- for key, value in software_parameter_dict.items() %}
{{ key }} = {{ dumps(value) }}
{% endfor -%}
[slap-configuration]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
software/rapid-cdn/software.cfg
View file @ fe38d320
[buildout]
extends =
buildout.hash.cfg
../../stack/caucase/buildout.cfg
../../stack/slapos.cfg
../../component/haproxy/buildout.cfg
../../component/nginx/buildout.cfg
../../component/python3/buildout.cfg
../../component/rsyslogd/buildout.cfg
../../component/trafficserver/buildout.cfg
parts +=
template
[python]
part = python3
[template-render-base]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:_update_hash_filename_}
rendered = ${buildout:directory}/${:_buildout_section_name_}.cfg
mode = 0644
context =
section software_parameter_dict software-parameter-section
${:extra-context}
extra-context =
[template]
<= template-render-base
[template-common]
<= template-render-base
[template-download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[template-rapid-cdn-frontend]
<= template-download-base
[template-rapid-cdn-human]
<= template-download-base
[template-rapid-cdn-pop]
<= template-download-base
[template-rapid-cdn-kedifa]
<= template-download-base
[template-rapid-cdn-log-aggregator]
<= template-download-base
[template-front-haproxy]
<= template-download-base
[template-front-rsyslogd]
<= template-download-base
[kedifa]
recipe = zc.recipe.egg
eggs =
${python-cryptography:egg}
kedifa
[software-parameter-section]
# This section passes binaries and other software provided outputs to all
# profiles. In the same time allows to gather all parameters in one place.
# Buildout parameters
bin-directory = ${buildout:bin-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
eggs-directory = ${buildout:eggs-directory}
# Software
caucase = ${caucase-eggs:bin-directory}/caucase
caucase-probe = ${caucase-eggs:bin-directory}/caucase-probe
caucase-updater = ${caucase-eggs:bin-directory}/caucase-updater
caucased = ${caucase-eggs:bin-directory}/caucased
dash = ${dash-output:dash}
haproxy = ${haproxy:location}/sbin/haproxy
kedifa = ${kedifa:bin-directory}/kedifa
kedifa-updater = ${kedifa:bin-directory}/kedifa-updater
nginx = ${nginx-output:nginx}
nginx_mime = ${nginx-output:mime}
openssl = ${openssl:location}/bin/openssl
openssl-cnf = ${openssl:location}/etc/ssl/openssl.cnf
rsyslogd = ${rsyslogd:location}/sbin/rsyslogd
trafficserver = ${trafficserver:location} {#- XXX: Fix to expose exact binaries #}
# Libraries
caucase-jinja2-library = ${caucase-jinja2-library:target}
# Templates
template-common = ${template-common:rendered}
template-front-haproxy = ${template-front-haproxy:target}
template-front-rsyslogd = ${template-front-rsyslogd:target}
template-monitor2 = ${monitor2-template:rendered}
template-rapid-cdn-frontend = ${template-rapid-cdn-frontend:target}
template-rapid-cdn-human = ${template-rapid-cdn-human:target}
template-rapid-cdn-kedifa = ${template-rapid-cdn-kedifa:target}
template-rapid-cdn-log-aggregator = ${template-rapid-cdn-log-aggregator:target}
template-rapid-cdn-pop = ${template-rapid-cdn-pop:target}
[versions]
kedifa = 0.0.6
# modernish zc.lockfile is required by kedifa
zc.lockfile = 1.4
software/rapid-cdn/template/front-haproxy.cfg.jinja2 0 → 100644
View file @ fe38d320
{%- macro apache_log_format() %}
capture request header Referer len 512
capture request header User-Agent len 512
log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Tt"
{%- endmacro %}
global
pidfile {{ configuration['pid-file'] }}
# master-worker is compatible with foreground with process management
master-worker
log {{ configuration['log-socket'] }} local0
defaults
log global
mode http
option httplog
timeout queue 60s
timeout server {{ configuration['request-timeout'] }}s
timeout client {{ configuration['request-timeout'] }}s
timeout connect {{ configuration['backend-connect-timeout'] }}s
retries {{ configuration['backend-connect-retries'] }}
{#- Allow to start with not resolved yet servers #}
default-server init-addr last,libc,none
# statistic
frontend statistic
bind {{ configuration['ipv6']}}:{{ configuration['statistic-port'] }} {#- ssl crt-list /proper-file #}
stats enable
stats uri /
stats show-desc {{ configuration['statistic-identification'] }}
stats auth {{ configuration['statistic-username'] }}:{{ configuration['statistic-password'] }}
stats realm {{ configuration['statistic-identification'] }}
{{- apache_log_format() }}
frontend http
bind {{ configuration['ipv4'] }}:{{ configuration['http-port'] }}
bind {{ configuration['ipv6'] }}:{{ configuration['http-port'] }}
{{- apache_log_format() }}
{%- for frontend in configuration['frontend-list'] %}
acl is_{{ frontend['reference'] }}_http hdr_reg(host) -i ^{{ frontend['hostname'] }}($|:.*)
use_backend {{ frontend['reference'] }}-http if is_{{ frontend['reference'] }}_http
{%- endfor %}
default_backend INSTANCE_NOT_READY_BACKEND
frontend https
bind {{ configuration['ipv4'] }}:{{ configuration['https-port'] }}
bind {{ configuration['ipv6'] }}:{{ configuration['https-port'] }}
{{- apache_log_format() }}
default_backend INSTANCE_NOT_READY_BACKEND
{%- for frontend in configuration['frontend-list'] %}
{%- set ssl_list = [] %}
{%- if frontend['backend-url']['scheme'] == 'https' %}
{%- do ssl_list.append('ssl verify none') %}
{%- endif %}
backend {{ frontend['reference'] }}-http
http-request set-path {{ frontend['backend-url']['path'] }}%[path]
server {{ frontend['reference'] }}-backend-http {{ frontend['backend-url']['hostname'] }}:{{ frontend['backend-url']['port'] }} {{ ' '.join(ssl_list) }}
{%- endfor %}
backend INSTANCE_NOT_READY_BACKEND
server INSTANCE_NOT_READY_BACKEND 2001:67c:1254:53:b362:dd91:0:1:8080
http-request set-path STUB_FOR_NOT_READY_BACKEND%[path]
{#- keep the empty line below #}
software/rapid-cdn/template/front-rsyslogd.conf.jinja2 0 → 100644
View file @ fe38d320
module(
load="imuxsock"
SysSock.Name="{{ configuration['log-socket'] }}")
# Just simply output the raw line without any additional information, as
# haproxy emits enough information by itself
# Also cut out first empty space in msg, which is related to rsyslogd
# internal and end up cutting on 8k, as it's default of $MaxMessageSize
template(name="rawoutput" type="string" string="%msg:2:8192%\n")
$ActionFileDefaultTemplate rawoutput
$FileCreateMode 0600
$DirCreateMode 0700
$Umask 0022
$WorkDirectory {{ configuration['spool-directory'] }}
## Setup logging per slave, by extracting the slave name from the log stream
{%- raw %}
#{%- set regex = ".*-backend (.*)-http.{0,1}(|-failover)/" %}
#template(name="extract_slave_name" type="string" string="%msg:R,ERE,1,FIELD:{{ regex }}--end%")
#set $!slave_name = exec_template("extract_slave_name");
#template(name="slave_output" type="string" string="{{ configuration['caddy-log-directory'] }}/%$!slave_name%_backend_log")
#if (re_match($msg, "{{ regex }}")) then {
# action(type="omfile" dynaFile="slave_output")
# stop
#}
{%- endraw %}
{#- emit all not catched messages to full log file #}
*.* {{ configuration['log-file'] }}
software/rapid-cdn/test/setup.py
View file @ fe38d320
......@@ -43,6 +43,8 @@ setup(
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
'caucase',
],
zip_safe=True,
test_suite='test',
......
software/rapid-cdn/test/test.py
View file @ fe38d320
......@@ -25,7 +25,12 @@
#
##############################################################################
import json
import os
import shutil
import subprocess
import tempfile
import slapos.slap.standalone
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -34,6 +39,109 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class CaucaseBinaryClient(object):
# Class to mimic user, which is using caucase binaries and some directories
# to be a client, thus real binaries are used with executable call examples
def __init__(self, path, url):
self.path = path
os.mkdir(self.path)
self.url = url
def makepath(p):
return os.path.join(self.path, p)
self.client_csr = makepath('client.csr.pem')
self.client = makepath('client.pem')
self.ca_crt = makepath('ca-crt.pem')
self.user_ca_crt = makepath('user-ca-crt.pem')
self.crl = makepath('crl.pem')
self.user_crl = makepath('user-crl.pem')
subprocess.check_call([
"openssl", "req", "-out", self.client_csr, "-new", "-newkey", "rsa:2048",
"-nodes", "-keyout", self.client, "-subj", "/CN=user"])
self.cau_list = [
"caucase", "--ca-url", self.url, "--ca-crt", self.ca_crt,
"--user-ca-crt", self.user_ca_crt, "--crl", self.crl, "--user-crl",
self.user_crl]
self.user_cau_list = self.cau_list + ["--mode", "user"]
self.service_cau_list = self.cau_list + [
"--user-key", self.client, "--mode", "service"]
output = subprocess.check_output(
self.user_cau_list + ["--send-csr", self.client_csr])
subprocess.check_call(
self.user_cau_list + ["--get-crt", output.split()[0], self.client])
def signServiceCsr(self, ou):
for line in subprocess.check_output(
self.service_cau_list + ["--list-csr"]).splitlines():
splitted = line.decode().split('|')
if len(splitted) == 2:
if 'OU=%s)' % (ou,) in splitted[1]:
csr_id = splitted[0].strip()
subprocess.check_call(self.service_cau_list + ["--sign-csr", csr_id])
break
class Test(SlapOSInstanceTestCase):
def test(self):
self.fail('TODO')
# full diffs are very informative for those tests
maxDiff = None
# as instantiation is split in chunks with cluster administrator operations,
# let it run a bit rarely
instance_max_retry = 5
@classmethod
def _setUpClass(cls):
cls.work_dir = tempfile.mkdtemp()
try:
super()._setUpClass()
except slapos.slap.standalone.SlapOSNodeInstanceError:
pass
# Cluster is initially setup, now it's required to allow joining nodes,
# which is usually cluster administrator responsibility
connection_parameter_dict = json.loads(
cls.requestDefaultInstance().getConnectionParameterDict()['_'])
cls.kedifa_caucase_client = CaucaseBinaryClient(
os.path.join(cls.work_dir, 'kedifa_caucase'),
connection_parameter_dict['kedifa-caucase-url'])
cls.kedifa_caucase_client.signServiceCsr("Kedifa Partition")
# Time travel to the future: restart all caucase updaters, so they will
# pick up just signed service CSR; note that in reality the cluster
# administrator can wait for the caucase updater to kick in
with cls.slap.instance_supervisor_rpc as instance_supervisor_rpc:
for process in instance_supervisor_rpc.getAllProcessInfo():
if 'caucase-updater' in process['name']:
process_id = '%(group)s:%(name)s' % process
instance_supervisor_rpc.stopProcess(process_id)
instance_supervisor_rpc.startProcess(process_id)
cls.waitForInstance()
@classmethod
def tearDownClass(cls):
super().tearDownClass()
shutil.rmtree(cls.work_dir)
def assertMonitorSetupUrl(self, value):
# TODO: check that monitor can correctly acces whole cluster
pass
def test_connection_parameter(self):
connection_parameter_dict = self.requestDefaultInstance(
).getConnectionParameterDict()
self.assertIn('_', connection_parameter_dict)
json_parmeter_dict = json.loads(connection_parameter_dict.pop('_'))
self.assertEqual({}, connection_parameter_dict)
self.assertNotIn(
'__NotReadyYet__', json_parmeter_dict.pop('key-generate-auth-url'))
self.assertNotIn(
'__NotReadyYet__', json_parmeter_dict.pop('key-upload-url'))
self.assertMonitorSetupUrl(json_parmeter_dict.pop('monitor-setup-url'))
self.assertEqual(
{
'kedifa-caucase-url': 'http://[%s]:8090' % (self._ipv6_address,),
'monitor-base-url': 'https://[%s]:8196' % (self._ipv6_address,),
'xxx-replace-with-information-fetch-depends': 'human\nlog-aggregator'
},
json_parmeter_dict)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7