Revert a few commits pushed by mistake.

software/rapid-cdn/test/test.py

@@ -90,27 +90,6 @@ SOURCE_IPV6 = '::1'
 # IP on which test run, in order to mimic HTTP[s] access
 TEST_IP = os.environ['SLAPOS_TEST_IPV4']
 
-# "--resolve" inspired from https://stackoverflow.com/a/44378047/9256748
-DNS_CACHE = {}
-
-MONTH_DICT = {
-  'Jan': 1,
-  'Feb': 2,
-  'Mar': 3,
-  'Apr': 4,
-  'May': 5,
-  'Jun': 6,
-  'Jul': 7,
-  'Aug': 8,
-  'Sep': 9,
-  'Oct': 10,
-  'Nov': 11,
-  'Dec': 12,
-}
-match_nginx_datetime = re.compile(
-  '(?P<dow>...), (?P<day>..) (?P<monthname>...) (?P<year>....) '
-  '(?P<hour>..):(?P<minute>..):(?P<second>..)',
-).match
 
 def unicode_escape(s):
   return s.encode('unicode_escape').decode()
@@ -1072,45 +1051,9 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
       'application/json',
       result.headers['Content-Type']
     )
-    result_json = result.json()
-    # expected content for each file entry:
-    # - 'type' with files having the value 'file'
-    # - 'name' with the file's basename as value
-    # - 'mtime' with the modification time (though this specific part
-    #   probably cannot be reliably tested) in some specific format (which
-    #   happens to be the one nginx produces)
-    # - 'size' with the file's size (again, not much to actually test for
-    #   exctitude I guess)
-    self.assertEqual(
-      sorted([q['name'] for q in result_json]),
-      ['access.log', 'backend.log', 'frontend.log'])
-    self.assertIn(
-      'file',
-      {q['type'] for q in result_json},
-    )
-    # The following loop must not raise to pass the test, but nothing specific
-    # is done with extracted values.
-    for entry in result_json:
-      self.assertIn('size', entry)
-      parsed_remote_mtime = match_nginx_datetime(
-        entry['mtime'],
-      ).groupdict()
-      datetime.datetime(
-        int(parsed_remote_mtime['year'], 10),
-        MONTH_DICT[parsed_remote_mtime['monthname']],
-        int(parsed_remote_mtime['day'], 10),
-        int(parsed_remote_mtime['hour'], 10),
-        int(parsed_remote_mtime['minute'], 10),
-        int(parsed_remote_mtime['second'], 10),
-      )
     self.assertEqual(
-      http.client.OK,
-      requests.get(url + 'access.log', verify=False).status_code
-    )
-    self.assertEqual(
-      http.client.OK,
-      requests.get(url + 'error.log', verify=False).status_code
-    )
+      sorted([q['name'] for q in result.json()]),
+      ['access.log', 'backend.log', 'frontend.log'])
     # assert only for few tests, as logs are available for sure only
     # for few of them
     for test_name in [

stack/erp5/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [mariadb-resiliency-after-import-script]
 filename = instance-mariadb-resiliency-after-import-script.sh.in
-md5sum = f97ad39b3c9f539e80ac86cf0c0e145b
+md5sum = 85ce1e2f3d251aa435fef8118dca8a63
 
 [mariadb-slow-query-report-script]
 filename = mysql-querydigest.sh.in
@@ -22,11 +22,11 @@ md5sum = 6457ab192d709aa2c9014e9a3e91ca20
 
 [mariadb-start-clone-from-backup]
 filename = instance-mariadb-start-clone-from-backup.sh.in
-md5sum = 710b812d41c46d5a44b8b487a4f54f8f
+md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 43acff9f4df025318a4cab0dd2bd4e32
+md5sum = 93b2277185e4949a3d17be79d3710d2d
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -90,7 +90,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = e7776de36718bd7e27a4defe7d2e5b29
+md5sum = 9c580be982d8c63ec06fc273ef3cb971
 
 [template-balancer]
 filename = instance-balancer.cfg.in
@@ -98,7 +98,7 @@ md5sum = 409a7505548576ebf0e4d5cc218e0753
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in
-md5sum = bdb624dcc3571e4280d2cb2cc7cd7a15
+md5sum = 2cd76971b64b0bf7771978ad07bfc2e5
 
 [template-rsyslogd-cfg]
 filename = rsyslogd.cfg.in

stack/erp5/haproxy.cfg.in

@@ -261,7 +261,7 @@ frontend frontend_{{ group_name }}
 
 {%   for name, url in sorted(backend_dict.items()) %}
 backend backend_{{ group_name }}_{{ name }}
-  http-request replace-path ^/{{ name }}(.*) /VirtualHostBase/https/%[req.hdr(Host)]/VirtualHostRoot/_vh_{{ name }}\1
+  http-request replace-path ^/{{ name }}(.*) /VirtualHostBase/https/{{ ip }}:{{ port }}/VirtualHostRoot/_vh_{{ name }}\1
   timeout server 8h
   server {{ name }} {{ urllib_parse.urlparse(url).netloc }}
 {%-  endfor %}

stack/erp5/instance-mariadb-resiliency-after-import-script.sh.in

 #!{{ dash }}
-set -eu
 
-if [ $# -ne 1 ]; then
-  echo "Restaure a mariadb instance from available backup data."
-  echo "  $0 <BACKUP FILE>"
-  exit 1
-fi
+# DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
+# OR MYSQL DATA WILL BE ERASED.
+
+# This script will import the dump of the mysql database to the real
+# database. It is launched by the clone (importer) instance of webrunner
+# in the end of the import script.
+
+# Depending on the output, it will create a file containing
+# the status of the restoration (success or failure)
+
+set -e
 
-dump=$1
 mysql_executable='{{ mysql_executable }}'
-mariadb_data_directory='{{ data_directory }}'
+mariadb_data_directory='{{ mariadb_data_directory }}'
+mariadb_backup_directory='{{ mariadb_backup_directory }}'
 pid_file='{{ pid_file }}'
 binlog_path='{{ binlog_path }}'
 server_executable='{{ server_executable }}'
-server_socket='{{ socket }}'
 
 # Make sure mariadb is not already running
 if [ -e "$pid_file" ]; then
@@ -27,16 +31,7 @@ if [ -e "$pid_file" ]; then
   fi
 fi
 
-echo "EXISTING DATABASE CONTENT WILL BE DESTROYED"
-echo "You have 5 seconds to interrupt this script..."
-if sleep 5; then
-  echo "Expired, proceeding"
-else
-  echo "Interrupted, aborting"
-  exit 1
-fi
-
-echo "Emptying data directory..."
+echo "Deleting existing database..."
 find "$mariadb_data_directory" -mindepth 1 -delete
 
 # $binlog_path can be empty if incremental_backup_retention_days <= -1
@@ -50,35 +45,24 @@ if [ -n "$binlog_path" ]; then
   fi
 fi
 
-echo -n "Starting mariadb for backup restoration"
+echo "Starting mariadb..."
 "$server_executable" --innodb-flush-method=nosync --skip-innodb-doublewrite --innodb-flush-log-at-trx-commit=0 --sync-frm=0 --slow-query-log=0 --skip-log-bin &
 mysqld_pid=$!
-trap "kill $mysqld_pid; wait; exit 1" EXIT TERM INT
-while true; do
-  if [ ! -e "/proc/$mysqld_pid" ]; then
-    trap EXIT TERM INT
-    echo "Service exited, check logs"
-    wait
-    exit 1
-  fi
-  if [ -e "$server_socket" ]; then
-    echo "started."
-    break
-  fi
-  echo -n .
-  sleep 0.5
-done
+trap "kill $mysqld_pid" EXIT TERM INT
+sleep 30
+# If mysql has stopped, abort
+if ! [ -d /proc/$mysql_pid ]; then
+  echo "mysqld exited, aborting."
+  exit 1
+fi
 
-echo "Importing $dump ..."
-zcat "$dump" | "$mysql_executable" || {
+echo "Importing data..."
+# Use latest dump XXX can contain funny characters
+dump=$(ls -r "$mariadb_backup_directory" | head -1)
+zcat "$mariadb_backup_directory/$dump" | $mysql_executable || {
   RESTORE_EXIT_CODE=$?
   echo 'Backup restoration failed.'
   exit $RESTORE_EXIT_CODE
 }
 
-echo "Stopping mariadb..."
-trap EXIT TERM INT
-kill $mysqld_pid
-wait
-
-echo "Done. Start mariadb normally."
+echo 'Backup restoration successfully completed.'

stack/erp5/instance-mariadb-start-clone-from-backup.sh.in

@@ -7,48 +7,50 @@ if [ $# -ne 7 ]; then
   exit 1
 fi
 
-dump=$1
-master_host=$2
-master_port=$3
-master_user=$4
-master_ssl_ca=$5
-master_ssl_cert=$6
-master_ssl_key=$7
+BACKUP=$1
+MASTER_HOST=$2
+MASTER_PORT=$3
+MASTER_USER=$4
+MASTER_SSL_CA=$5
+MASTER_SSL_CERT=$6
+MASTER_SSL_KEY=$7
 
-mysql_executable='{{ mysql_executable }}'
-mariadb_data_directory='{{ data_directory }}'
-pid_file='{{ pid_file }}'
-server_executable='{{ server_executable }}'
-server_socket='{{ socket }}'
+CLIENT='{{ client }}'
+DATA_DIRECTORY='{{ data_directory }}'
+PID_FILE='{{ pid_file }}'
+SERVER='{{ server }}'
+UPDATE='{{ update }}'
+SOCKET='{{ socket }}'
 
 # Make sure mariadb is not already running
-if [ -e "$pid_file" ]; then
-  if ! pid=$(cat "$pid_file"); then
+if [ -e "$PID_FILE" ]; then
+  PID=$(cat "$PID_FILE")
+  if [ $? -ne 0 ]; then
     echo "Cannot read Mariadb pidfile, assuming running. Aborting."
     exit 1
   fi
-  if kill -0 "$pid"; then
-    echo "Mariadb is already running with pid $pid. Aborting."
+  if kill -0 "$PID"; then
+    echo "Mariadb is already running with pid $PID. Aborting."
     exit 1
   fi
 fi
 
-backup_head="$(zcat "$dump" | head -n 100)"
+BACKUP_HEAD="$(zcat "$BACKUP" | head -n 100)"
 
-sql_change_master=$(printf "%s" "$backup_head" | grep "^--\s*CHANGE MASTER TO " | sed "s/^--\s*//")
-if [ -z "$sql_change_master" ]; then
+SQL_CHANGE_MASTER=$(echo "$BACKUP_HEAD" | grep "^--\s*CHANGE MASTER TO " | sed "s/^--\s*//")
+if [ -z "$SQL_CHANGE_MASTER" ]; then
   echo "'CHANGE MASTER TO' statement not found in given backup file."
   echo "Is replication enabled on future master ?"
   exit 1
 fi
 
-sql_set_gtid="$(printf "%s" "$backup_head" | grep "^--\s*SET GLOBAL gtid_slave_pos=" | sed "s/^--\s*//")"
-if [ -z "$sql_set_gtid" ]; then
+SQL_SET_GTID="$(echo "$BACKUP_HEAD" | grep "^--\s*SET GLOBAL gtid_slave_pos=" | sed "s/^--\s*//")"
+if [ -z "$SQL_SET_GTID" ]; then
   echo "Info: GTID not found in backup, it will not be enabled."
-  master_use_gtid=0
+  MASTER_USE_GTID=0
 else
   echo "Info: GTID found in backup, it will be enabled."
-  master_use_gtid=1
+  MASTER_USE_GTID=1
 fi
 
 echo "EXISTING DATABASE CONTENT WILL BE DESTROYED"
@@ -61,66 +63,61 @@ else
 fi
 
 echo "Emptying data directory..."
-find "$mariadb_data_directory" -mindepth 1 -delete
+find "$DATA_DIRECTORY" -mindepth 1 -delete
 
 echo -n "Starting mariadb for backup restoration"
-"$server_executable" --innodb-flush-method=nosync --skip-innodb-doublewrite --innodb-flush-log-at-trx-commit=0 --sync-frm=0 --slow-query-log=0 --skip-log-bin &
-mysqld_pid=$!
-trap "kill $mysqld_pid; wait; exit 1" EXIT TERM INT
+"$SERVER" --innodb-flush-method=nosync --skip-innodb-doublewrite --innodb-flush-log-at-trx-commit=0 --sync-frm=0 --slow-query-log=0 --skip-log-bin &
+PID=$!
+trap "kill $PID; wait; exit 1" EXIT
 while true; do
-  if [ ! -e "/proc/$mysqld_pid" ]; then
-    trap EXIT TERM INT
+  if [ ! -e "/proc/$PID" ]; then
+    trap EXIT
     echo "Service exited, check logs"
     wait
     exit 1
   fi
-  if [ -e "$server_socket" ]; then
-    echo "started."
-    break
-  fi
+  test -e "$SOCKET" && break
   echo -n .
   sleep 0.5
 done
 
-echo "Importing $dump ..."
-zcat "$dump" | "$mysql_executable" || {
-  RESTORE_EXIT_CODE=$?
-  echo 'Backup restoration failed.'
-  exit $RESTORE_EXIT_CODE
-}
+"$UPDATE"
+
+echo "Importing $BACKUP ..."
+zcat "$BACKUP" | "$CLIENT"
 
 echo "Configuring server as slave..."
-if [ "$master_use_gtid" -eq 1 ]; then
-  "$mysql_executable" -e "$sql_set_gtid"
-  maser_use_gtid_sql="slave_pos"
+if [ "$MASTER_USE_GTID" -eq 1 ]; then
+  "$CLIENT" -e "$SQL_SET_GTID"
+  MASTER_USE_GTID_SQL="slave_pos"
 else
-  maser_use_gtid_sql="NO"
+  MASTER_USE_GTID_SQL="NO"
 fi
-"$mysql_executable" -e "
+"$CLIENT" -e "
   CHANGE MASTER TO
-  MASTER_HOST='$master_host',
-  MASTER_USER='$master_user',
-  MASTER_PORT=$master_port,
+  MASTER_HOST='$MASTER_HOST',
+  MASTER_USER='$MASTER_USER',
+  MASTER_PORT=$MASTER_PORT,
   MASTER_SSL=1,
-  MASTER_SSL_CA='$master_ssl_ca',
-  MASTER_SSL_CERT='$master_ssl_cert',
-  MASTER_SSL_KEY='$master_ssl_key',
+  MASTER_SSL_CA='$MASTER_SSL_CA',
+  MASTER_SSL_CERT='$MASTER_SSL_CERT',
+  MASTER_SSL_KEY='$MASTER_SSL_KEY',
   MASTER_SSL_VERIFY_SERVER_CERT=1,
-  MASTER_USE_GTID=$maser_use_gtid_sql;
+  MASTER_USE_GTID=$MASTER_USE_GTID_SQL;
 "
-if [ "$master_use_gtid" -eq 0 ]; then
+if [ "$MASTER_USE_GTID" -eq 0 ]; then
   # No GTID, use binlog name & offset as provided by backup file.
   # Example: CHANGE MASTER TO MASTER_LOG_FILE='binlog.003447', MASTER_LOG_POS=360;
   # Notes:
-  # - Must happen after setting master_host & master_port.
-  # - Implicitly sets master_use_gtid=NO if it was set before.
-  "$mysql_executable" -e "$sql_change_master"
+  # - Must happen after setting MASTER_HOST & MASTER_PORT.
+  # - Implicitly sets MASTER_USE_GTID=NO if it was set before.
+  "$CLIENT" -e "$SQL_CHANGE_MASTER"
 fi
-"$mysql_executable" -e "START SLAVE;"
+"$CLIENT" -e "START SLAVE;"
 
 echo "Stopping mariadb..."
-trap EXIT TERM INT
-kill $mysqld_pid
+trap EXIT
+kill $PID
 wait
 
 echo "Done. Start mariadb normally. You may use 'show slave status' SQL command to monitor progress."

stack/erp5/instance-mariadb.cfg.in

@@ -275,11 +275,11 @@ url = {{ parameter_dict['mariadb-start-clone-from-backup'] }}
 output = ${directory:bin}/start-clone-from-backup
 context =
   key dash dash:dash
-  key mysql_executable binary-wrap-mysql:wrapper-path
+  key client binary-wrap-mysql:wrapper-path
   key data_directory my-cnf-parameters:data-directory
   key pid_file my-cnf-parameters:pid-file
-  key server_executable mysqld:output
-  key update_executable update-mysql:output
+  key server mysqld:output
+  key update update-mysql:output
   key socket my-cnf-parameters:socket
 
 [{{ section('resiliency-after-import-script') }}]
@@ -290,13 +290,11 @@ output = ${directory:bin}/restore-from-backup
 context =
   key dash dash:dash
   key mysql_executable binary-wrap-mysql:wrapper-path
-  key data_directory my-cnf-parameters:data-directory
-  key backup_directory directory:mariadb-backup-full
+  key mariadb_data_directory my-cnf-parameters:data-directory
+  key mariadb_backup_directory directory:mariadb-backup-full
   key pid_file my-cnf-parameters:pid-file
   key binlog_path my-cnf-parameters:binlog-path
   key server_executable mysqld:output
-  key update_executable update-mysql:output
-  key socket my-cnf-parameters:socket
 
 [{{ section('monitor-generate-mariadb-slow-query-report') }}]
 recipe = slapos.cookbook:cron.d

stack/erp5/instance-zope.cfg.in

@@ -500,7 +500,7 @@ wrapper-path = ${buildout:bin-directory}/runTestSuite
 {%     do connection_string_list.append(
          '%s@%s:%s %s %s' % (
            parsed_url.path.lstrip('/'),
-           'erp5-catalog-0',
+           parsed_url.hostname,
            parsed_url.port,
            parsed_url.username,
            parsed_url.password,