Commit 09649287 authored by Xavier Thompson's avatar Xavier Thompson

software/theia: Add resilient framework

parent e28fde2c
...@@ -15,11 +15,23 @@ ...@@ -15,11 +15,23 @@
[instance-theia] [instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in _update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = c0b209121901c80f9c5ea5d2cb71f39e md5sum = 7475b6e11f573169f993068372087150
[instance] [instance]
_update_hash_filename_ = instance.cfg.in _update_hash_filename_ = instance.cfg.in
md5sum = 063d3e19da9d3d4bfb77e8e638aa3a77 md5sum = a7d78b4002266c69ece05a476df82791
[instance-import]
_update_hash_filename_ = instance-import.cfg.jinja.in
md5sum = d1181f788461a31026e36677ab56398d
[instance-export]
_update_hash_filename_ = instance-export.cfg.jinja.in
md5sum = 4f52e4cff8e72369b38e47df297df7e2
[instance-resilient]
_update_hash_filename_ = instance-resilient.cfg.jinja
md5sum = d78a9f885bdebf6720197209e0c21aa0
[yarn.lock] [yarn.lock]
_update_hash_filename_ = yarn.lock _update_hash_filename_ = yarn.lock
......
{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) -%}
[buildout]
extends = {{ theia_instance_cfg }}
{{ pbsready_export_cfg }}
parts +=
monitor-base
$${:theia-parts}
$${:theia-environment-parts}
resilient-publish-connection-parameter
# The resilient stack makes the 'resilient' instance
# request the 'export' instance with a 'namebase' parameter.
# The export template then expects to receive it in
# slap-parameter:namebase
[slap-parameter]
namebase = {{ parameter_dict['namebase'] }}
# The resilient export stack periodically calls exporter:wrapper
# and then notifies the pull-backup instance that data is ready
# to be pulled from the export instance.
# All it expects is that a script be available in exporter:wrapper.
[exporter]
recipe = slapos.cookbook:wrapper
command-line = echo "Export Not Implemented Yet"
wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-exporter
wrapper = $${:wrapper-path}
# Extend resilient parameters with normal theia connection parameters
[resilient-publish-connection-parameter]
<= publish-connection-parameter
{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) -%}
{%- set additional_frontend = parameter_dict['additional-frontend-guid'] -%}
[buildout]
extends = {{ theia_instance_cfg }}
{{ pbsready_import_cfg }}
parts +=
monitor-base
$${:theia-parts}
$${:theia-environment-parts}
# The resilient stack makes the 'resilient' instance
# request the 'import' instance with a 'namebase' parameter.
# The import template then expects to receive it in
# slap-parameter:namebase
[slap-parameter]
namebase = {{ parameter_dict['namebase'] }}
# Change frontend name to avoid conflicts
[remote-frontend]
name = Import {{ parameter_dict['frontend-name'] }}
{% if additional_frontend -%}
[remote-additional-frontend]
name = Import {{ parameter_dict['additional-frontend-name'] }}
{%- endif %}
# Change port ranges to avoid race conditions on port allocation
[frontend-instance-port]
minimum = 3200
maximum = 3300
[theia-service-port]
minimum = 3700
maximum = 3800
[slapos-standalone-port]
minimum = 4200
maximum = 4300
# Always disable autoprocessing in the import instance
[slapos-autorun]
autorun = stopped
# Change the gravatar favicon seed
[favicon.ico]
seed = Import {{ root_title }}
# The resilient stack calls post-notification-run:output followed by
# importer:wrapper when the instance is notified that the backup files
# have just been pushed to it. All it expects is the path of a script
# in post-notification-run:output and in importer:wrapper.
[post-notification-run]
recipe = slapos.cookbook:wrapper
command-line = echo "Import Post Notification Run Not Implemented Yet"
wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-post-run
wrapper = $${:wrapper-path}
[importer]
recipe = slapos.cookbook:wrapper
command-line = echo "Import Not Implemented Yet"
wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-importer
wrapper = $${:wrapper-path}
# Resilient connection parameters of import instance are published
# through the resilient stack.
# Extend resilient parameters with normal theia connection parameters
[resilient-publish-connection-parameter]
<= publish-connection-parameter
{% import 'parts' as parts %}
{% import 'replicated' as replicated with context %}
{% set number_of_instances = slapparameter_dict.get('resilient-clone-number', 1)|int %}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
extends =
{{ monitor_template }}
parts +=
# Generate the parts to request theia-export, pull-backup and theia-import
# See stack/resilient/template-parts.cfg.in and stack/resilient/template-replicated.cfg.in
# See below for the generation of the sections corresponding to the parts generated here
{{ parts.replicate("theia", number_of_instances + 1) }}
# Also publish some connection parameters
publish-connection-parameter
[ArgLeader]
[ArgBackup]
# Generate sections to request theia-export, pull-backup and theia-import
# See stack/resilient/template-replicated.cfg.in
# In particular:
#
# [request-theia]
# <= ArgLeader
# software-type = export
# ...
#
# [request-theia-pseudo-replicating-1]
# <= ArgBackup
# software-type = import
# ...
#
# [request-pbs-theia-1]
# software-type = pull-backup
# ...
#
{{ replicated.replicate("theia", number_of_instances + 1,
"export", "import",
"ArgLeader", "ArgBackup",
slapparameter_dict=slapparameter_dict) }}
# Extend the list of return parameters for the export request
# The monitor parameters are only there to assert they are
# actually published by the export instance
[request-theia]
return += url username password backend-url monitor-base-url monitor-setup-url
# Extend the list of return parameters for the import request
# with the monitor parameters to assert they are actually published
[request-theia-pseudo-replicating-1]
return += monitor-base-url monitor-setup-url
# Publish some parameters from the export instance
[publish-connection-parameter]
recipe = slapos.cookbook:publish
url = ${request-theia:connection-url}
username = ${request-theia:connection-username}
password = ${request-theia:connection-password}
backend-url = ${request-theia:connection-backend-url}
# Publish resiliency parameters fetched by the resilient stack
[publish-connection-parameter]
<= publish-connection-information
...@@ -280,13 +280,14 @@ wait-for-files = $${frontend-instance:pidfile} ...@@ -280,13 +280,14 @@ wait-for-files = $${frontend-instance:pidfile}
[favicon.ico] [favicon.ico]
# generate a pseudo random favicon, different for each instance name. # generate a pseudo random favicon, different for each instance name.
recipe = slapos.recipe.build recipe = slapos.recipe.build
seed = {{ root_title }}
install = install =
import hashlib, shutil import hashlib, shutil
buildout_offline = self.buildout['buildout']['offline'] buildout_offline = self.buildout['buildout']['offline']
self.buildout['buildout']['offline'] = 'false' self.buildout['buildout']['offline'] = 'false'
try: try:
gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5( gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5(
b'''{{ root_title }}''' b'''$${:seed}'''
).hexdigest() + "?s=256&d=retro" ).hexdigest() + "?s=256&d=retro"
shutil.copy(self.download(gravatar_url), '''$${:location}''') shutil.copy(self.download(gravatar_url), '''$${:location}''')
except Exception: except Exception:
......
...@@ -2,6 +2,9 @@ ...@@ -2,6 +2,9 @@
parts = parts =
switch-softwaretype switch-softwaretype
extends =
${template-resilient-templates:output}
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
...@@ -18,6 +21,11 @@ recipe = slapos.cookbook:switch-softwaretype ...@@ -18,6 +21,11 @@ recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:default} RootSoftwareInstance = $${:default}
default = $${:theia} default = $${:theia}
theia = theia:rendered theia = theia:rendered
export = export:rendered
import = import:rendered
resilient = resilient:rendered
frozen = instance-frozen:rendered
pull-backup = template-pull-backup:rendered
[theia] [theia]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -46,3 +54,43 @@ default-parameters = ...@@ -46,3 +54,43 @@ default-parameters =
"additional-frontend-guid": null "additional-frontend-guid": null
} }
frontend-sr = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg frontend-sr = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
[import]
recipe = slapos.recipe.template:jinja2
template = ${instance-import:output}
rendered = $${buildout:directory}/instance-import.cfg
mode = 0644
context =
jsonkey default_parameter_dict theia:default-parameters
key parameter_dict slap-configuration:configuration
key theia_instance_cfg theia:rendered
key pbsready_import_cfg template-pbsready-import:rendered
key root_title slap-configuration:root-instance-title
[export]
recipe = slapos.recipe.template:jinja2
template = ${instance-export:output}
rendered = $${buildout:directory}/instance-export.cfg
mode = 0644
context =
jsonkey default_parameter_dict theia:default-parameters
key parameter_dict slap-configuration:configuration
key theia_instance_cfg theia:rendered
key pbsready_export_cfg template-pbsready-export:rendered
[resilient]
recipe = slapos.recipe.template:jinja2
template = ${instance-resilient:output}
rendered = $${buildout:directory}/instance-resilient.cfg
mode = 0644
extensions = jinja2.ext.do
context =
key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
raw monitor_template ${monitor-template:rendered}
template-parts-destination = ${template-parts:target}
template-replicated-destination = ${template-replicated:target}
import-list = file parts :template-parts-destination
file replicated :template-replicated-destination
...@@ -15,6 +15,7 @@ extends = ...@@ -15,6 +15,7 @@ extends =
../../stack/nodejs.cfg ../../stack/nodejs.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
../../stack/resilient/buildout.cfg
../../component/defaults.cfg ../../component/defaults.cfg
./download-plugins.cfg ./download-plugins.cfg
./buildout.hash.cfg ./buildout.hash.cfg
...@@ -24,6 +25,9 @@ parts = ...@@ -24,6 +25,9 @@ parts =
slapos-cookbook slapos-cookbook
instance-theia instance-theia
instance instance
instance-import
instance-export
instance-resilient
# default for slapos-standalone # default for slapos-standalone
shared-part-list = shared-part-list =
...@@ -350,3 +354,14 @@ output = ${buildout:directory}/instance-theia.cfg.jinja ...@@ -350,3 +354,14 @@ output = ${buildout:directory}/instance-theia.cfg.jinja
[instance] [instance]
<= template-base <= template-base
output = ${buildout:directory}/instance.cfg output = ${buildout:directory}/instance.cfg
[instance-import]
<= template-base
output = ${buildout:directory}/instance-import.cfg.jinja
[instance-export]
<= template-base
output = ${buildout:directory}/instance-export.cfg.jinja
[instance-resilient]
<= download-base
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment