stack/erp5: Enable requesting mariadb replication

Allow requesting a mariadb set-up to replicate another mariadb:
- bootstrap-url: bootstrap from a statically served backup file
- primary-url: replicate from a primary mariadb

This happens in mariadb first initialization, when no data exists yet.
That way existing data in a non-replicating mariadb cannot be deleted
by setting the replication parameters after the fact.

A promise checks that the state (replica / primary, replication source)
of the running mariadb matches the requested state; but if it doesn't,
the mariadb will not automatically converge without human intervention
if ~/srv/mariadb data directory already exists, to avoid deleting data.

software/erp5/instance-mariadb-schema.json

@@ -13,6 +13,37 @@
         }
       ]
     },
+    "replication": {
+      "description": "Parameters for mariadb replication",
+      "type": "object",
+      "default": {},
+      "oneOf": {
+        "properties": {
+          "primary-url": {
+            "description": "URL of the primary mariadb to replicate from",
+            "type": "string"
+          },
+          "bootstrap-url": {
+            "description": "URL of a recent mariadb backup to bootstrap replication from",
+            "type": "string"
+          },
+          "seconds-behind-master-threshold": {
+            "description": "Monitoring threshold for Seconds_Behind_Master in the replica, in seconds, -1 meaning no threshold",
+            "type": "integer",
+            "minimum": -1,
+            "default": -1
+          },
+          "oneOf": {
+            {
+              "required": ["primary-url", "bootstrap-url"]
+            },
+            {
+              "enum": [{}]
+            }
+          }
+        }
+      }
+    },
     "database-list": {
       "description": "Databases to create and respective user credentials getting all privileges on it",
       "default": [

software/neoppod/software-common.cfg

@@ -118,6 +118,9 @@ inline =
       --auth-root-authentication-method=normal \
       --basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
       --datadir="$datadir"
+  {%- if initialisation is defined and initialisation %}
+    {{ initialisation | indent(2) }}
+  {%- endif %}
     rm "$marker"
   }
   {%- if environ is defined %}

stack/erp5/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = f0ee8909d5b10a919bd8f14ec5c74550
+md5sum = 78dd18b8342eea855c2c84b6a453c8c9
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -70,7 +70,7 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
 
 [template]
 filename = instance.cfg.in
-md5sum = ca0cb83950dd9079cc289891cce08e76
+md5sum = 2820cc6073263a0e1290b76a485e9c54
 
 [template-erp5]
 filename = instance-erp5.cfg.in

stack/erp5/instance-mariadb.cfg.in

@@ -18,7 +18,10 @@
 {%   set ip = (ipv4_set | list)[0] -%}
 {%   set ip_as_host = ip -%}
 {% endif -%}
-{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
+{% set bash = parameter_dict['bash'] ~ '/bin/bash' -%}
+{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' -%}
+{% set replication = slapparameter_dict.get('replication', {}) -%}
+
 
 [{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised
@@ -200,10 +203,6 @@ mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
 recipe = slapos.recipe.template:jinja2
 output = ${directory:services}/mariadb
 url = {{ parameter_dict['template-mysqld-wrapper'] }}
-context =
-  key defaults_file my-cnf:output
-  key datadir my-cnf-parameters:data-directory
-  key environ :environ
 environ =
   GRN_PLUGINS_PATH='${my-cnf-parameters:groonga-plugins-path}'
   ODBCSYSINI='${my-cnf-parameters:etc-directory}'
@@ -211,7 +210,178 @@ environ =
   {%- for variable in slapparameter_dict.get('environment-variables', ()) %}
   {{ variable }}
   {%- endfor %}
+context =
+  key defaults_file my-cnf:output
+  key datadir my-cnf-parameters:data-directory
+  key environ :environ
+{%- if replication %}
+  key initialisation mariadb-setup-replication:output
+{%- endif %}
+
+{% if replication -%}
+{% set bootstrap_url = replication['bootstrap-url'] -%}
+{% set primary_url = replication['primary-url'] -%}
+{% set primary = urllib_parse.urlsplit(primary_url) -%}
+[{{ section('mariadb-setup-replication') }}]
+recipe = slapos.recipe.template
+output = ${directory:bin}/${:_buildout_section_name_}
+inline =
+  #! {{ bash }}
+  set -euo pipefail
+
+  mariadb="{{ parameter_dict['mariadb-location'] }}/bin/mysqld"
+  mycnf="${my-cnf:output}"
+  mycnf_socket="${my-cnf-parameters:socket}"
+  update_mysql="${update-mysql:output}"
+  bootstrap_url={{ six_moves.shlex_quote(bootstrap_url) }}
+  bootstrap_file="${directory:mariadb-backup-full}/bootstrap.sql.gz"
+  curl="{{ parameter_dict['curl-location'] }}/bin/curl"
+  zcat="{{ parameter_dict['gzip-location'] }}/bin/zcat"
+  mysql="${binary-wrap-mysql:wrapper-path}"
+
+  date
+
+  echo "Starting mariadb for replication initialization"
+  echo -n "  \_"
+  $mariadb --defaults-file=$mycnf --innodb-flush-method=nosync --skip-innodb-doublewrite --innodb-flush-log-at-trx-commit=0 --sync-frm=0 --slow-query-log=0 --skip-log-bin &
+  PID=$!
+  trap "kill $PID; wait; exit 1" EXIT
+  while true; do
+    if [ ! -e "/proc/$PID" ]; then
+      trap EXIT
+      echo "Service exited; ABORT. Check the logs"
+      wait
+      exit 1
+    fi
+    test -S $mycnf_socket && break
+    echo -n .
+    sleep 0.5
+  done
+  echo " OK"
+
+  echo "Updating mariadb"
+  $update_mysql
+  echo "  \_ OK"
+
+  echo "Fetching $bootstrap_url"
+  $curl -o $bootstrap_file $bootstrap_url
+  echo "  \_ OK"
+
+  echo "Importing $bootstrap_file"
+  $zcat $bootstrap_file | $mysql
+  echo "  \_ OK"
+
+  echo "Extracting GTID from backup"
+  set +o pipefail
+  SQL_SET_GTID=$($zcat $bootstrap_file 2>/dev/null | sed '100q; /^--\s*SET GLOBAL gtid_slave_pos=/!d; s/^--\s*//; q')
+  set -o pipefail
+  if [ -z "$SQL_SET_GTID" ]; then
+    echo "  \_ GTID not found in backup; ABORT"
+    exit 1
+  fi
+  echo "  \_ OK, found $SQL_SET_GTID"
+
+  echo "Configuring server as replica"
+  echo -n "  \_"
+  $mysql -e "$SQL_SET_GTID"
+  echo -n .
+  $mysql -e "
+    CHANGE MASTER TO
+    MASTER_HOST='{{ primary.hostname }}',
+    MASTER_USER='{{ primary.username }}',
+    MASTER_PASSWORD='{{ primary.password }}',
+    MASTER_PORT={{ primary.port }},
+    MASTER_USE_GTID="slave_pos";
+  "
+  echo -n .
+  $mysql -e "START SLAVE;"
+  echo -n .
+  echo " OK"
+
+  echo "Stopping mariadb"
+  trap EXIT
+  kill $PID
+  wait
+  echo "  \_ OK"
+
+  echo "Starting mariadb normally"
+{%- endif %}
 
+{% if replication -%}
+{# cast to assert types -#}
+{% set threshold = int(replication.get('seconds-behind-master-threshold', -1)) -%}
+{% set primary_url = str(primary_url) -%}
+{% else -%}
+{% set primary_url = None -%}
+{% set primary = {'hostname': None, 'port': None, 'username': None} -%}
+{% set threshold = None -%}
+{% endif -%}
+[mariadb-replication-sense]
+recipe = slapos.recipe.template
+output = ${directory:bin}/${:_buildout_section_name_}
+inline =
+  from contextlib import closing
+
+  from zope.interface import implementer
+  from slapos.grid.promise import interface
+  from slapos.grid.promise.generic import GenericPromise
+
+  import pymysql.cursors
+
+  expected_url = {{ repr(primary_url) }}
+  expected = {{ repr((primary.hostname, primary.port, primary.username)) }}
+  max_delay = {{ threshold }}
+
+  @implementer(interface.IPromise)
+  class RunPromise(GenericPromise):
+    def sense(self):
+      conn = pymysql.connect(
+        read_default_file='${my-cnf:output}',
+        cursorclass=pymysql.cursors.DictCursor)
+      with closing(conn):
+        with closing(conn.cursor()) as cursor:
+          cursor.execute("SHOW SLAVE STATUS")
+          data = cursor.fetchone()
+      if data is None:
+        if expected_url is None:
+          self.logger.info("Mariadb is in primary mode")
+        else:
+          self.logger.error(
+            "Mariadb is not in replica mode\n"
+            "If this is expected, please unset 'replication' parameter"
+          )
+        return
+      if expected is None:
+        self.logger.error(
+          "Mariadb is in replica mode\n"
+          "If this is expected, please set 'replication' parameter"
+        )
+        return
+      primary = tuple(data.get('Master_' + k) for k in ('Host', 'Port', 'User'))
+      seconds_behind = data.get('Seconds_Behind_Master')
+      if None in primary or seconds_behind is None:
+        from pprint import pformat
+        self.logger.error("Replication is in bad state:\n%s", pformat(data))
+        return
+      if primary != expected:
+        self.logger.error(
+          "Replica is not following given primary %s\n"
+          "Instead it's following %r\n"
+          "If this is expected, please correct 'replication' parameter'",
+          expected_primary_url,
+          primary,
+        )
+        return
+      (self.logger.error if seconds_behind > max_delay >= 0 else
+       self.logger.info)("Replica is %d seconds behind", seconds_behind)
+
+[{{ section('mariadb-replication-promise') }}]
+recipe = slapos.cookbook:promise.plugin
+eggs =
+  slapos.core
+  PyMySQL
+file = ${mariadb-replication-sense:output}
+output = ${directory:plugins}/mariadb_replication.py
 
 [{{ section('mariadb-backup-static-server-promise') }}]
 <= monitor-promise-base

stack/erp5/instance.cfg.in

@@ -189,6 +189,7 @@ unixodbc-location = {{ unixodbc_location }}
 mroonga-mariadb-install-sql = {{ mroonga_mariadb_install_sql }}
 mroonga-mariadb-plugin-dir = {{ mroonga_mariadb_plugin_dir }}
 groonga-plugins-path = {{ groonga_plugin_dir }}:{{ groonga_mysql_normalizer_plugin_dir }}
+curl-location = {{ curl_location }}
 
 [dynamic-template-mariadb]
 <= jinja2-template-base
@@ -196,6 +197,8 @@ url = {{ template_mariadb }}
 filename = instance-mariadb.cfg
 extra-context =
     section parameter_dict dynamic-template-mariadb-parameters
+    import urllib_parse six.moves.urllib.parse
+    import six_moves six.moves
 
 # Keep a section for backward compatibility for removed types
 # Once the section is removed, ghost instances will keep failing until