repman: fix conflict with socat port and inform repman which port to uses

software/repman/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = da50540b1c0fc69ffbf8f6e345a3baad
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 482c9f2cad5a35fd0e556dd353a0d39c
+md5sum = 2b5552d587e7735b4ff6daec24fcb8c1
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
@@ -26,7 +26,7 @@ md5sum = 5cfa75ca5a0048a050c0041dfe541f3d
 
 [config-cluster-toml.in]
 _update_hash_filename_ = templates/cluster-config.toml.in
-md5sum = 45f4628a5660b663dfdd9037e803d884
+md5sum = 9be37f86d0fad81cc13e64c5a0557c59
 
 [nginx.conf.in]
 _update_hash_filename_ = templates/nginx.conf.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = 7049767810d8b8a6a16668512672f5a3
+md5sum = 63af2f4fc184cfc9d09f8b72dee5165a
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in
@@ -58,7 +58,7 @@ md5sum = 99324b56192003254081ef336dcee94c
 
 [dbjobs-in]
 _update_hash_filename_ = templates/dbjobs.in
-md5sum = d623a4c684578602b9d8ee49034aebfa
+md5sum = 9731eff8918fa3e68198de6b1d45ecfa
 
 [mysqld-need-start.sh.in]
 _update_hash_filename_ = templates/mysqld-need-start.sh.in

software/repman/instance-mariadb.cfg.jinja2.in

@@ -21,6 +21,7 @@ recipe = slapos.cookbook:publish.serialised
 database-host = {{ host }}:{{ port }}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 partition-path = ${buildout:directory}
+receiver-port = ${dbjob-parameter:socat-port}
 
 [publish-early]
 recipe = slapos.cookbook:publish-early
@@ -250,12 +251,15 @@ dbjob-cnf = ${directory:etc}/mysql/my.cnf
 log-dir = ${directory:mysql}/.system/logs
 tmp-dir = ${directory:tmp}
 mysqld-socket  = ${my-cnf-parameters:socket}
-socat-port = {{ port + 1 }}
+socat-port = {{ int(port) + 9 }}
 restart-script = ${mysqld-restart-script:rendered}
 socat-location = {{ parameter_dict['socat-location'] }}
 mysql-location = {{ parameter_dict['mariadb-location'] }}
+gzip-location = {{ parameter_dict['gzip-location'] }}
 ip = {{ ip }}
+host = {{ host }}
 port = {{ port }}
+use-ipv6 = {{ dumps(use_ipv6) }}
 
 [dbjobs-executable]
 < = jinja2-template-executable

software/repman/instance-repman.cfg.jinja2.in

@@ -3,6 +3,7 @@
 {% set monitor_base_url_dict = {} -%}
 {% set mariadb_dict = {} -%}
 {% set mariadb_server_list = [] -%}
+{% set receiver_port_list = [] -%}
 {% set mariadb_path_list = [] -%}
 {% set ip = (ipv6_set | list)[0] -%}
 {% set ipv4 = (ipv4_set | list)[0] -%}
@@ -95,11 +96,13 @@ config-cluster = {{ name }}
 config-name = {{ dbname }}
 return = 
   database-host
+  receiver-port
   monitor-base-url
   partition-path
 
 {% do part_list.append(section) -%}
 {% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
+{% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
 {% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
 {% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
 
@@ -138,6 +141,7 @@ db-prefered-master = {{ mariadb_server_list[0] }}
 proxysql-servers = {{ ipv4 }}
 password = ${repman-parameter:password}
 proxysql-partition = ${buildout:directory}
+receiver-port-list = {{ receiver_port_list | join(',') }}
 proxy-tags = {{ parameter_dict.get("proxy-tags", "pkg,masterslave,linux,noreadwritesplit") }}
 logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
 physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
@@ -476,7 +480,7 @@ config-check-secure = 1
 [repman-backend-promise]
 <= monitor-promise-base
 module = check_url_available
-name = check_repman_frontend.py
+name = check_repman_backend.py
 config-url = ${nginx-parameter:backend-ssl-url}
 config-check-secure = 1
 

software/repman/templates/cluster-config.toml.in

@@ -36,7 +36,7 @@ scheduler-db-servers-physical-backup = true
 scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backup-cron'] }}"
 backup-physical-type = "mariabackup"
 backup-logical-type = "mysqldump"
-# scheduler-db-servers-receiver-ports="4444,4445"
+scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
 
 prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
 prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}

software/repman/templates/dbjobs.in

@@ -6,7 +6,12 @@ ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
 SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
 BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
 DATADIR={{ parameter_dict['mysql-dir'] }}/
-export PATH={{ parameter_dict['socat-location'] }}/bin:{{ parameter_dict['mysql-location'] }}/bin:$PATH
+{% if parameter_dict['use-ipv6'] == True -%}
+{% set listen = "TCP6-LISTEN" -%}
+{% else -%}
+{% set listen = "TCP-LISTEN" -%}
+{% endif -%}
+export PATH={{ parameter_dict['socat-location'] }}/bin:{{ parameter_dict['mysql-location'] }}/bin:{{ parameter_dict['gzip-location'] }}/bin:$PATH
 JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "reseedxtrabackup" "reseedmariabackup" "reseedmysqldump" "flashbackxtrabackup" "flashbackmariadbackup" "flashbackmysqldump" "stop" "start")
 
 doneJob()
@@ -52,6 +57,9 @@ partialRestore()
  mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}   -e"flush privileges;start slave;"
 }
 
+kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
+kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP6:LISTEN)
+
 for job in "${JOBS[@]}"
 do
 
@@ -70,13 +78,13 @@ do
       reseedmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u TCP-LISTEN:{{ parameter_dict['socat-port'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
         mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e 'start slave;'
       ;;
       flashbackmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u TCP-LISTEN:{{ parameter_dict['socat-port'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
        mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e 'start slave;'
       ;;
       reseedmariabackup)
@@ -84,7 +92,7 @@ do
        mkdir $BACKUPDIR
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u TCP-LISTEN:{{ parameter_dict['socat-port'] }},reuseaddr STDOUT | mbstream -x -C $BACKUPDIR
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | mbstream -x -C $BACKUPDIR
        # mbstream -p, --parallel
        mariabackup --prepare --export --target-dir=$BACKUPDIR
        partialRestore
@@ -94,7 +102,7 @@ do
        mkdir $BACKUPDIR
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u TCP-LISTEN:{{ parameter_dict['socat-port'] }},reuseaddr STDOUT | xbstream -x -C $BACKUPDIR
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | xbstream -x -C $BACKUPDIR
        mariabackup --prepare --export --target-dir=$BACKUPDIR
        partialRestore
       ;;