stack/erp5: Enable requesting mariadb replication

Allow requesting a mariadb set-up to replicate another mariadb:
- bootstrap from a statically served backup file
- replicate from a primary mariadb

software/erp5/instance-mariadb-schema.json

@@ -13,6 +13,36 @@
         }
       ]
     },
+    "replication": {
+      "description": "Parameters for mariadb replication",
+      "type": "object",
+      "default": {},
+      "oneOf": {
+        "properties": {
+          "primary-url": {
+            "description": "URL of the primary mariadb to replicate from",
+            "type": "string"
+          },
+          "bootstrap-url": {
+            "description": "URL of a recent mariadb backup to bootstrap replication from",
+            "type": "string"
+          },
+          "seconds-behind-master-threshold": {
+            "description": "Monitoring threshold for Seconds_Behind_Master in the replica",
+            "type": "integer",
+            "default": 0
+          },
+          "oneOf": {
+            {
+              "required": ["primary-url", "bootstrap-url"]
+            },
+            {
+              "enum": [{}]
+            }
+          }
+        }
+      }
+    },
     "database-list": {
       "description": "Databases to create and respective user credentials getting all privileges on it",
       "default": [

software/neoppod/software-common.cfg

@@ -120,6 +120,9 @@ inline =
       --basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
       --datadir="$datadir"
     rm "$marker"
+  {%- if initialisation is defined and initialisation %}
+    {{ initialisation | indent(2) }}
+  {%- endif %}
   }
   {%- if environ is defined %}
   {%-   for variable in environ.splitlines() %}

stack/erp5/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 336353523d6528c3ffe06d00f5799552
+md5sum = f54720de0030bd2c3c72b6ad5f98e71b
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -70,7 +70,7 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
 
 [template]
 filename = instance.cfg.in
-md5sum = 88529a1574d4aae74ef3183b9fa9b066
+md5sum = 3fc69d972ed1caa3ec98ecfbda389cdf
 
 [template-erp5]
 filename = instance-erp5.cfg.in

stack/erp5/instance-mariadb.cfg.in

@@ -18,7 +18,9 @@
 {%   set ip = (ipv4_set | list)[0] -%}
 {%   set ip_as_host = ip -%}
 {% endif -%}
-{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
+{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' -%}
+{% set replication = slapparameter_dict.get('replication', {}) -%}
+
 
 [{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised
@@ -197,10 +199,6 @@ mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
 recipe = slapos.recipe.template:jinja2
 output = ${directory:services}/mariadb
 url = {{ parameter_dict['template-mysqld-wrapper'] }}
-context =
-  key defaults_file my-cnf:output
-  key datadir my-cnf-parameters:data-directory
-  key environ :environ
 environ =
   GRN_PLUGINS_PATH='${my-cnf-parameters:groonga-plugins-path}'
   ODBCSYSINI='${my-cnf-parameters:etc-directory}'
@@ -208,6 +206,128 @@ environ =
   {%- for variable in slapparameter_dict.get('environment-variables', ()) %}
   {{ variable }}
   {%- endfor %}
+context =
+  key defaults_file my-cnf:output
+  key datadir my-cnf-parameters:data-directory
+  key environ :environ
+{%- if replication %}
+  key initialisation mariadb-setup-replication:output
+{%- endif %}
+
+{%- if replication %}
+{% set bootstrap_url = replication['bootstrap-url'] -%}
+{% set primary = urllib_parse.urlsplit(replication['primary-url']) -%}
+[{{ section('mariadb-setup-replication') }}]
+recipe = slapos.recipe.template
+output = ${directory:bin}/${:_buildout_section_name_}
+mysqld = {{ parameter_dict['mariadb-location'] }}/bin/mysqld
+zcat = {{ parameter_dict['gzip-location'] }}/bin/zcat
+wget = {{ parameter_dict['wget-location'] }}/bin/wget
+head = {{ parameter_dict['coreutils_location'] }}/bin/head
+grep = ${binary-link:target-directory}/grep
+sed = ${binary-link:target-directory}/sed
+bootstrapfile = ${directory:mariadb-backup-full}/bootstrap.sql.gz
+inline =
+  #! {{ dash }}
+  set -eu
+  echo -n "Starting mariadb for backup restoration"
+  ${:mysqld} --defaults-file='${my-cnf:output}' --innodb-flush-method=nosync --skip-innodb-doublewrite --innodb-flush-log-at-trx-commit=0 --sync-frm=0 --slow-query-log=0 --skip-log-bin &
+  PID=$!
+  trap "kill $PID; wait; exit 1" EXIT
+  while true; do
+    if [ ! -e "/proc/$PID" ]; then
+      trap EXIT
+      echo "Service exited, check logs"
+      wait
+      exit 1
+    fi
+    test -e ${my-cnf-parameters:socket} && break
+    echo -n .
+    sleep 0.5
+  done
+
+  echo -n "Updating mariadb"
+  ${update-mysql:output}
+
+  echo "Fetching ${:bootstrapfile}"
+  ${:wget} -O ${:bootstrapfile} {{ bootstrap_url }}
+
+  echo "Importing ${:bootstrapfile}"
+  ${:zcat} ${:bootstrapfile} | ${binary-wrap-mysql:wrapper-path}
+
+  echo "Extracting GTID from backup"
+  SQL_SET_GTID=$(${:zcat} ${:bootstrapfile} | ${:head} -n 100 | ${:grep} "^--\s*SET GLOBAL gtid_slave_pos=" | ${:sed} "s/^--\s*//")
+  if [ -z "$SQL_SET_GTID" ]; then
+    echo "GTID not found in backup; aborting"
+    exit 1
+  fi
+  echo "Found GTID: $SQL_SET_GTID"
+
+  echo "Configuring server as replica"
+  ${binary-wrap-mysql:wrapper-path} -e "$SQL_SET_GTID"
+  ${binary-wrap-mysql:wrapper-path} -e "
+    CHANGE MASTER TO
+    MASTER_HOST='{{ primary.hostname }}',
+    MASTER_USER='{{ primary.username }}',
+    MASTER_PASSWORD='{{ primary.password }}',
+    MASTER_PORT={{ primary.port }},
+    MASTER_USE_GTID="slave_pos";
+  "
+  ${binary-wrap-mysql:wrapper-path} -e "START SLAVE;"
+
+  echo "Stopping mariadb..."
+  trap EXIT
+  kill $PID
+  wait
+
+  echo "Done. Starting mariadb normally."
+
+{% set threshold = replication.get('seconds-behind-master-threshold', 0) -%}
+{% set primary = urllib_parse.urlsplit(replication['primary-url']) -%}
+[mariadb-replication-test]
+recipe = slapos.recipe.template
+output = ${directory:bin}/${:_buildout_section_name_}
+inline =
+  from contextlib import closing
+
+  from zope.interface import implementer
+  from slapos.grid.promise import interface
+  from slapos.grid.promise.generic import GenericPromise
+
+  import pymysql.cursors
+
+  max_delay = {{ int(replication.get('seconds-behind-master-threshold', 0)) }}
+
+  @implementer(interface.IPromise)
+  class RunPromise(GenericPromise):
+    def sense(self):
+      conn = pymysql.connect(
+        read_default_file='${my-cnf:output}',
+        cursorclass=pymysql.cursors.DictCursor)
+      with closing(conn):
+        with closing(conn.cursor()) as cursor:
+          cursor.execute("SHOW SLAVE STATUS")
+          data = cursor.fetchone()
+      data = {k.encode('utf-8'): s for k, s in data.items()}
+      seconds_behind = data.get('Seconds_Behind_Master')
+      if seconds_behind is None:
+        from pprint import pformat
+        self.logger.error("Replication is in bad state:\n%s", pformat(data))
+        return
+      log = self.logger.info
+      msg, args = "%d seconds behind", (seconds_behind,)
+      if seconds_behind > max_delay:
+        log = self.logger.error
+      log(msg, *args)
+
+[{{ section('mariadb-replication-promise') }}]
+recipe = slapos.cookbook:promise.plugin
+eggs =
+  slapos.core
+  PyMySQL
+file = ${mariadb-replication-test:output}
+output = ${directory:plugins}/mariadb_replication.py
+{%- endif %}
 
 
 [{{ section('mariadb-backup-static-server-promise') }}]

stack/erp5/instance.cfg.in

@@ -189,6 +189,7 @@ unixodbc-location = {{ unixodbc_location }}
 mroonga-mariadb-install-sql = {{ mroonga_mariadb_install_sql }}
 mroonga-mariadb-plugin-dir = {{ mroonga_mariadb_plugin_dir }}
 groonga-plugins-path = {{ groonga_plugin_dir }}:{{ groonga_mysql_normalizer_plugin_dir }}
+wget-location = {{ wget_location }}
 coreutils_location = {{ coreutils_location }}
 
 [dynamic-template-mariadb]
@@ -197,6 +198,7 @@ url = {{ template_mariadb }}
 filename = instance-mariadb.cfg
 extra-context =
     section parameter_dict dynamic-template-mariadb-parameters
+    import urllib_parse six.moves.urllib.parse
 
 # Keep a section for backward compatibility for removed types
 # Once the section is removed, ghost instances will keep failing until