repman: fix mariadb data location and fix reseed backup in dbjobs

software/repman/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = 4bcd62ac0b5c9d2c584533f5ba08d922
+md5sum = af2fc4a7a0f782fed2cb1112ef3cb397
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
@@ -22,7 +22,7 @@ md5sum = b905ac6dad9b6dddef426b69fea7286b
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
-md5sum = cc15b607435106beaaab9200f4cd525a
+md5sum = 28ab31bbf073f777221f90902e9d435e
 
 [config-cluster-toml.in]
 _update_hash_filename_ = templates/cluster-config.toml.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = 73326b36dca4717f2b0d1dbb7e76b596
+md5sum = c2777aef6c51300f632fa6cac3ba6783
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in
@@ -42,7 +42,7 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
 
 [template-mariadb-initial-setup]
 _update_hash_filename_ = templates/mariadb_initial_setup.sql.in
-md5sum = 5a070090fc5b266bf866855c9c54dadf
+md5sum = 28fdc24f85b82242751b6c3f40267b39
 
 [template-publish-slave-information]
 _update_hash_filename_ = publish-database-slave-parameters.cfg.in
@@ -62,7 +62,7 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
 
 [dbjobs-in]
 _update_hash_filename_ = templates/dbjobs.in
-md5sum = da15968c75e7c856662099edaea5ef46
+md5sum = df15bd12bbe34d206ddf76d01e683241
 
 [mysqld-need-start.sh.in]
 _update_hash_filename_ = templates/mysqld-need-start.sh.in

software/repman/instance-mariadb.cfg.jinja2.in

@@ -107,7 +107,7 @@ commands =
 [my-cnf-parameters]
 socket = ${directory:run}/mariadb.sock
 ip = {{ ip }}
-data-directory = ${directory:srv}/mariadb
+data-directory = ${directory:mysql}
 pid-file = ${directory:run}/mariadb.pid
 plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
 groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
@@ -202,6 +202,7 @@ recipe = slapos.cookbook:wrapper
 command-line = ${mysqld:rendered}
 wrapper-path = ${directory:controller}/mariadb
 wait-for-files =
+  ${directory:etc}/mysql/my.cnf
   ${directory:ssl}/mariadb-cert.crt
 
 {% import "supervisord_lib" as supervisord_lib with context %}

software/repman/instance.cfg.in

@@ -50,8 +50,8 @@ context =
     raw bash_bin                {{ bash_location }}/bin/bash
     raw jq_bin                  {{ jq_location }}/bin/jq
     raw curl_bin                {{ curl_location }}/bin/curl
-    raw openssl_bin            {{ openssl_location }}/bin/openssl
-    raw caucase_bin_client     {{ caucase_bin_client }}
+    raw openssl_bin             {{ openssl_location }}/bin/openssl
+    raw caucase_bin_client      {{ caucase_bin_client }}
 
     ${:extra-context}
 extra-context =

software/repman/software.cfg

@@ -42,6 +42,7 @@ part = python2.7
 eggs +=
   collective.recipe.shelloutput
 
+
 [template-mysqld-wrapper]
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
@@ -51,8 +52,9 @@ template =
   basedir='${mariadb:location}'
   datadir='{{datadir}}'
   marker=$datadir/.slapos_initializing
+  system=$datadir/.system
   [ -d "$datadir/mysql" ] && [ ! -f "$marker" ] || {
-    find "$datadir/" -mindepth 1 ! -path $marker -delete || mkdir "$datadir"
+    find "$datadir/" -mindepth 1 ! \( -path $marker -o -path $system -o -path "*$system/*" \) -delete || mkdir "$datadir"
     touch "$marker"
     "$basedir/scripts/mysql_install_db" \
       --defaults-file='{{defaults_file}}' \

software/repman/templates/config.toml.in

@@ -48,6 +48,7 @@ backup-restic-password = "{{ parameter_dict['password'] }}"
 backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
 backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
 backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
+backup-mysqldump-options = "--hex-blob --single-transaction --verbose --all-databases --add-drop-database"
 
 # Mail configuration
 # Alert email sender (default "mrm@localhost")

software/repman/templates/dbjobs.in

 #!/bin/bash
 
+
 DBUSER={{ parameter_dict['db-user'] }}
 PASSWORD={{ parameter_dict['db-password'] }}
 ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
@@ -43,7 +44,7 @@ partialRestore()
   done
   for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
-    mysql --defaults-file=/etc/mysql/dbjob.cnf  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
   done
   for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
@@ -84,7 +85,7 @@ do
       flashbackmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
        $MYSQL_CLIENT  -e 'start slave;'
       ;;
       reseedmariabackup)
@@ -117,7 +118,7 @@ do
        cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
       optimize)
-       mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       $MYSQL_CLIENT -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
       restart)
         {{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out

software/repman/templates/mariadb_initial_setup.sql.in

@@ -10,7 +10,7 @@ DROP FUNCTION IF EXISTS sphinx_snippets;
 
 CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
 
-{% macro database(name, user, password, charset="") -%}
+{% macro database(name, user, password, ssl, charset="") -%}
 {% set charset_collate = "" -%}
 {% set charset_dict = {'utf8_general_ci': 'utf8',
                        'utf8_unicode_ci': 'utf8',
@@ -29,12 +29,12 @@ CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
 {% endif -%}
 CREATE DATABASE IF NOT EXISTS `{{ name }}` {{ charset_collate }};
 {% if user -%}
-GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
+GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}'{% if ssl %} REQUIRE SSL{% endif %};
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
 {%- endif %}
 {% endmacro -%}
 
 {% for entry in parameter_dict['database-list'] -%}
-{{   database(entry['name'], entry.get('user'), entry.get('password'), entry.get('charset')) }}
+{{   database(entry['name'], entry.get('user'), entry.get('password'), entry.get('require_ssl', True), entry.get('charset')) }}
 {% endfor -%}