software/theia: Add resilient framework

software/theia/buildout.hash.cfg

@@ -15,11 +15,23 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = c0b209121901c80f9c5ea5d2cb71f39e
+md5sum = 7475b6e11f573169f993068372087150
 
 [instance]
 _update_hash_filename_ = instance.cfg.in
-md5sum = 063d3e19da9d3d4bfb77e8e638aa3a77
+md5sum = a7d78b4002266c69ece05a476df82791
+
+[instance-import]
+_update_hash_filename_ = instance-import.cfg.jinja.in
+md5sum = d1181f788461a31026e36677ab56398d
+
+[instance-export]
+_update_hash_filename_ = instance-export.cfg.jinja.in
+md5sum = 4f52e4cff8e72369b38e47df297df7e2
+
+[instance-resilient]
+_update_hash_filename_ = instance-resilient.cfg.jinja
+md5sum = d78a9f885bdebf6720197209e0c21aa0
 
 [yarn.lock]
 _update_hash_filename_ = yarn.lock

software/theia/instance-export.cfg.jinja.in

+{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) -%}
+
+[buildout]
+extends = {{ theia_instance_cfg }}
+          {{ pbsready_export_cfg }}
+
+parts +=
+  monitor-base
+  $${:theia-parts}
+  $${:theia-environment-parts}
+  resilient-publish-connection-parameter
+
+
+# The resilient stack makes the 'resilient' instance
+# request the 'export' instance with a 'namebase' parameter.
+# The export template then expects to receive it in
+# slap-parameter:namebase
+[slap-parameter]
+namebase = {{ parameter_dict['namebase'] }}
+
+
+# The resilient export stack periodically calls exporter:wrapper
+# and then notifies the pull-backup instance that data is ready
+# to be pulled from the export instance.
+# All it expects is that a script be available in exporter:wrapper.
+[exporter]
+recipe = slapos.cookbook:wrapper
+command-line = echo "Export Not Implemented Yet"
+wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-exporter
+wrapper = $${:wrapper-path}
+
+# Extend resilient parameters with normal theia connection parameters
+[resilient-publish-connection-parameter]
+<= publish-connection-parameter

software/theia/instance-import.cfg.jinja.in

+{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) -%}
+{%- set additional_frontend = parameter_dict['additional-frontend-guid'] -%}
+
+[buildout]
+extends = {{ theia_instance_cfg }}
+          {{ pbsready_import_cfg }}
+
+parts +=
+  monitor-base
+  $${:theia-parts}
+  $${:theia-environment-parts}
+
+
+# The resilient stack makes the 'resilient' instance
+# request the 'import' instance with a 'namebase' parameter.
+# The import template then expects to receive it in
+# slap-parameter:namebase
+[slap-parameter]
+namebase = {{ parameter_dict['namebase'] }}
+
+
+# Change frontend name to avoid conflicts
+[remote-frontend]
+name = Import {{ parameter_dict['frontend-name'] }}
+
+{% if additional_frontend -%}
+[remote-additional-frontend]
+name = Import {{ parameter_dict['additional-frontend-name'] }}
+{%- endif %}
+
+
+# Change port ranges to avoid race conditions on port allocation
+[frontend-instance-port]
+minimum = 3200
+maximum = 3300
+
+[theia-service-port]
+minimum = 3700
+maximum = 3800
+
+[slapos-standalone-port]
+minimum = 4200
+maximum = 4300
+
+
+# Always disable autoprocessing in the import instance
+[slapos-autorun]
+autorun = stopped
+
+
+# Change the gravatar favicon seed
+[favicon.ico]
+seed = Import {{ root_title }}
+
+
+# The resilient stack calls post-notification-run:output followed by
+# importer:wrapper when the instance is notified that the backup files
+# have just been pushed to it. All it expects is the path of a script
+# in post-notification-run:output and in importer:wrapper.
+
+[post-notification-run]
+recipe = slapos.cookbook:wrapper
+command-line = echo "Import Post Notification Run Not Implemented Yet"
+wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-post-run
+wrapper = $${:wrapper-path}
+
+[importer]
+recipe = slapos.cookbook:wrapper
+command-line = echo "Import Not Implemented Yet"
+wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-importer
+wrapper = $${:wrapper-path}
+
+
+# Resilient connection parameters of import instance are published
+# through the resilient stack.
+# Extend resilient parameters with normal theia connection parameters
+[resilient-publish-connection-parameter]
+<= publish-connection-parameter

software/theia/instance-resilient.cfg.jinja

+{% import 'parts' as parts %}
+{% import 'replicated' as replicated with context %}
+
+{% set number_of_instances = slapparameter_dict.get('resilient-clone-number', 1)|int %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+
+extends =
+  {{ monitor_template }}
+
+parts +=
+# Generate the parts to request theia-export, pull-backup and theia-import
+# See stack/resilient/template-parts.cfg.in and stack/resilient/template-replicated.cfg.in
+# See below for the generation of the sections corresponding to the parts generated here
+  {{ parts.replicate("theia", number_of_instances + 1) }}
+# Also publish some connection parameters
+  publish-connection-parameter
+
+[ArgLeader]
+
+[ArgBackup]
+
+# Generate sections to request theia-export, pull-backup and theia-import
+# See stack/resilient/template-replicated.cfg.in
+# In particular:
+#
+# [request-theia]
+# <= ArgLeader
+# software-type = export
+# ...
+#
+# [request-theia-pseudo-replicating-1]
+# <= ArgBackup
+# software-type = import
+# ...
+#
+# [request-pbs-theia-1]
+# software-type = pull-backup
+# ...
+#
+{{ replicated.replicate("theia", number_of_instances + 1,
+                        "export", "import",
+                        "ArgLeader", "ArgBackup",
+                        slapparameter_dict=slapparameter_dict) }}
+
+# Extend the list of return parameters for the export request
+# The monitor parameters are only there to assert they are
+# actually published by the export instance
+[request-theia]
+return += url username password backend-url monitor-base-url monitor-setup-url
+
+# Extend the list of return parameters for the import request
+# with the monitor parameters to assert they are actually published
+[request-theia-pseudo-replicating-1]
+return += monitor-base-url monitor-setup-url
+
+# Publish some parameters from the export instance
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+url = ${request-theia:connection-url}
+username = ${request-theia:connection-username}
+password = ${request-theia:connection-password}
+backend-url = ${request-theia:connection-backend-url}
+
+# Publish resiliency parameters fetched by the resilient stack
+[publish-connection-parameter]
+<= publish-connection-information

software/theia/instance-theia.cfg.jinja.in

@@ -280,13 +280,14 @@ wait-for-files = $${frontend-instance:pidfile}
 [favicon.ico]
 # generate a pseudo random favicon, different for each instance name.
 recipe = slapos.recipe.build
+seed = {{ root_title }}
 install =
   import hashlib, shutil
   buildout_offline = self.buildout['buildout']['offline']
   self.buildout['buildout']['offline'] = 'false'
   try:
     gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5(
-      b'''{{ root_title }}'''
+      b'''$${:seed}'''
     ).hexdigest() + "?s=256&d=retro"
     shutil.copy(self.download(gravatar_url), '''$${:location}''')
   except Exception:

software/theia/instance.cfg.in

@@ -2,6 +2,9 @@
 parts =
   switch-softwaretype
 
+extends =
+  ${template-resilient-templates:output}
+
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 
@@ -18,6 +21,11 @@ recipe = slapos.cookbook:switch-softwaretype
 RootSoftwareInstance = $${:default}
 default = $${:theia}
 theia = theia:rendered
+export = export:rendered
+import = import:rendered
+resilient = resilient:rendered
+frozen = instance-frozen:rendered
+pull-backup = template-pull-backup:rendered
 
 [theia]
 recipe = slapos.recipe.template:jinja2
@@ -46,3 +54,43 @@ default-parameters =
     "additional-frontend-guid": null
   }
 frontend-sr = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+
+[import]
+recipe = slapos.recipe.template:jinja2
+template = ${instance-import:output}
+rendered = $${buildout:directory}/instance-import.cfg
+mode = 0644
+context =
+  jsonkey default_parameter_dict theia:default-parameters
+  key parameter_dict slap-configuration:configuration
+  key theia_instance_cfg theia:rendered
+  key pbsready_import_cfg template-pbsready-import:rendered
+  key root_title slap-configuration:root-instance-title
+
+[export]
+recipe = slapos.recipe.template:jinja2
+template = ${instance-export:output}
+rendered = $${buildout:directory}/instance-export.cfg
+mode = 0644
+context =
+  jsonkey default_parameter_dict theia:default-parameters
+  key parameter_dict slap-configuration:configuration
+  key theia_instance_cfg theia:rendered
+  key pbsready_export_cfg template-pbsready-export:rendered
+
+[resilient]
+recipe = slapos.recipe.template:jinja2
+template = ${instance-resilient:output}
+rendered = $${buildout:directory}/instance-resilient.cfg
+mode = 0644
+extensions = jinja2.ext.do
+context =
+  key buildout buildout:bin-directory
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+  key slapparameter_dict slap-configuration:configuration
+  raw monitor_template ${monitor-template:rendered}
+template-parts-destination = ${template-parts:target}
+template-replicated-destination = ${template-replicated:target}
+import-list = file parts :template-parts-destination
+              file replicated :template-replicated-destination

software/theia/software.cfg

@@ -15,6 +15,7 @@ extends =
    ../../stack/nodejs.cfg
    ../../stack/slapos.cfg
    ../../stack/monitor/buildout.cfg
+   ../../stack/resilient/buildout.cfg
    ../../component/defaults.cfg
    ./download-plugins.cfg
    ./buildout.hash.cfg
@@ -24,6 +25,9 @@ parts =
   slapos-cookbook
   instance-theia
   instance
+  instance-import
+  instance-export
+  instance-resilient
 
 # default for slapos-standalone
 shared-part-list =
@@ -350,3 +354,14 @@ output = ${buildout:directory}/instance-theia.cfg.jinja
 [instance]
 <= template-base
 output = ${buildout:directory}/instance.cfg
+
+[instance-import]
+<= template-base
+output = ${buildout:directory}/instance-import.cfg.jinja
+
+[instance-export]
+<= template-base
+output = ${buildout:directory}/instance-export.cfg.jinja
+
+[instance-resilient]
+<= download-base