Commit a3dc73a8 authored by Alain Takoudjou's avatar Alain Takoudjou

repman: fix configurations for new replication manager release

parent 0e4c73a9
...@@ -18,11 +18,11 @@ md5sum = 8a08be95a04f1a47098c4fdef80bdfed ...@@ -18,11 +18,11 @@ md5sum = 8a08be95a04f1a47098c4fdef80bdfed
[instance-repman.cfg] [instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in _update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 1d6eba3984b3e2009682f6ce49b8ac4d md5sum = 0155bd363569f5e10e8d90162e4adf48
[config-toml.in] [config-toml.in]
_update_hash_filename_ = templates/config.toml.in _update_hash_filename_ = templates/config.toml.in
md5sum = 2ee2c4bc5f985c11c2167b819d261256 md5sum = cc15b607435106beaaab9200f4cd525a
[config-cluster-toml.in] [config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in _update_hash_filename_ = templates/cluster-config.toml.in
...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30 ...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg] [template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = 21a29a41768b2370d671d3086b3ef2bb md5sum = a8b5bdf4959bb2a24235567e8dd16bf9
[template-my-cnf] [template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in _update_hash_filename_ = templates/my.cnf.in
...@@ -58,11 +58,11 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9 ...@@ -58,11 +58,11 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
[dbjobs-in] [dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in _update_hash_filename_ = templates/dbjobs.in
md5sum = d2ebd2ec55bf8489789a52c808729925 md5sum = bf3a3d91e3152d5dfd807f35890d89c6
[mysqld-need-start.sh.in] [mysqld-need-start.sh.in]
_update_hash_filename_ = templates/mysqld-need-start.sh.in _update_hash_filename_ = templates/mysqld-need-start.sh.in
md5sum = e9bcee5dc1318fe3acda2663472214f5 md5sum = 8f14054b3761b167cf8a86f4f084a574
[proxy-need-start-stop.sh.in] [proxy-need-start-stop.sh.in]
_update_hash_filename_ = templates/proxy-need-start-stop.sh.in _update_hash_filename_ = templates/proxy-need-start-stop.sh.in
......
...@@ -242,10 +242,11 @@ bash-bin = {{ bash_bin }} ...@@ -242,10 +242,11 @@ bash-bin = {{ bash_bin }}
db-user = ${init-script-parameters:root-user} db-user = ${init-script-parameters:root-user}
db-password = ${init-script-parameters:password} db-password = ${init-script-parameters:password}
mysql-dir = ${directory:mysql} mysql-dir = ${directory:mysql}
dbjob-cnf = ${directory:etc}/mysql/my.cnf dbjob-cnf = ${directory:etc}/mysql/dbjob.cnf
log-dir = ${directory:mysql}/.system/logs log-dir = ${directory:mysql}/.system/logs
tmp-dir = ${directory:tmp} tmp-dir = ${directory:tmp}
mysqld-socket = ${my-cnf-parameters:socket} mysqld-socket = ${my-cnf-parameters:socket}
mysql-client = ${binary-wrap-mysql:wrapper-path}
socat-port = {{ int(port) + 9 }} socat-port = {{ int(port) + 9 }}
restart-script = ${mysqld-restart-script:rendered} restart-script = ${mysqld-restart-script:rendered}
socat-location = {{ parameter_dict['socat-location'] }} socat-location = {{ parameter_dict['socat-location'] }}
...@@ -313,7 +314,7 @@ template = inline:#!{{ bash_bin }} ...@@ -313,7 +314,7 @@ template = inline:#!{{ bash_bin }}
rm -rf ${directory:etc}/mysql rm -rf ${directory:etc}/mysql
cp -r etc/mysql ${directory:etc} cp -r etc/mysql ${directory:etc}
ln -sf ${directory:mysql}/.system ${directory:var}/system ln -sf ${directory:mysql}/.system ${directory:var}/system
ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom/01_mariadb.cnf ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom.d/01_mariadb.cnf
[{{ section('install-mysql-config') }}] [{{ section('install-mysql-config') }}]
recipe = plone.recipe.command recipe = plone.recipe.command
......
...@@ -54,7 +54,7 @@ template = inline:#!{{ bash_bin }} ...@@ -54,7 +54,7 @@ template = inline:#!{{ bash_bin }}
cd ${repman:config-tmp}/proxies cd ${repman:config-tmp}/proxies
{{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config {{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
tar -xzf proxies-$NAME.tar.gz tar -xzf proxies-$NAME.tar.gz
cp conf/proxysql.cnf $CONFIG cp etc/proxysql.cnf $CONFIG
rendered = ${directory:bin}/update-proxysql-config rendered = ${directory:bin}/update-proxysql-config
mode = 755 mode = 755
...@@ -209,6 +209,7 @@ command-line = ...@@ -209,6 +209,7 @@ command-line =
wrapper-path = ${directory:controller}/proxysql-{{ name }} wrapper-path = ${directory:controller}/proxysql-{{ name }}
wait-for-files = wait-for-files =
${repman:bootstrap}/{{ name }}_bootstrapped ${repman:bootstrap}/{{ name }}_bootstrapped
${config-proxysql-{{ name }}:config}
depends = depends =
{{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }} {{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }}
{{ '${proxysql-' ~ name ~ '-promise:recipe}' }} {{ '${proxysql-' ~ name ~ '-promise:recipe}' }}
......
...@@ -31,7 +31,7 @@ parts = ...@@ -31,7 +31,7 @@ parts =
gowork gowork
[python] [python]
part = python3 part = python2.7
[instance.cfg] [instance.cfg]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
......
...@@ -21,7 +21,7 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}" ...@@ -21,7 +21,7 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
#db-servers-ignored-hosts = #db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
monitoring-address = "{{ parameter_dict['ipv4'] }}" monitoring-address = "{{ parameter_dict['ipv4'] }}"
monitoring-wait-retry = 40 monitoring-wait-retry = 1200
#haproxy = true #haproxy = true
#haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}" #haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
......
...@@ -6,6 +6,7 @@ ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log ...@@ -6,6 +6,7 @@ ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
DATADIR={{ parameter_dict['mysql-dir'] }}/ DATADIR={{ parameter_dict['mysql-dir'] }}/
MYSQL_CLIENT={{ parameter_dict['mysql-client'] }}
{% if parameter_dict['use-ipv6'] == True -%} {% if parameter_dict['use-ipv6'] == True -%}
{% set listen = "TCP6-LISTEN" -%} {% set listen = "TCP6-LISTEN" -%}
{% else -%} {% else -%}
...@@ -16,29 +17,29 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" " ...@@ -16,29 +17,29 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "
doneJob() doneJob()
{ {
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" & $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
} }
pauseJob() pauseJob()
{ {
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" & $MYSQL_CLIENT -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
} }
partialRestore() partialRestore()
{ {
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'" $MYSQL_CLIENT -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;" $MYSQL_CLIENT -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;" $MYSQL_CLIENT -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE" $MYSQL_CLIENT -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
done done
for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/ mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
...@@ -46,15 +47,15 @@ partialRestore() ...@@ -46,15 +47,15 @@ partialRestore()
done done
for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/ mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE $dir.$file" $MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done done
done done
for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/ mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE mysql.$file" $MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
done done
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e"flush privileges;start slave;" $MYSQL_CLIENT -e "flush privileges;start slave;"
} }
kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN) kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
...@@ -67,7 +68,7 @@ do ...@@ -67,7 +68,7 @@ do
ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }')) ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
ID=($(echo $TASK | awk -F@ '{ print $1 }')) ID=($(echo $TASK | awk -F@ '{ print $1 }'))
#purge de past #purge de past
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';" $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
if [ "$ADDRESS" == "" ]; then if [ "$ADDRESS" == "" ]; then
echo "No $job needed" echo "No $job needed"
...@@ -78,13 +79,13 @@ do ...@@ -78,13 +79,13 @@ do
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1 socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;' $MYSQL_CLIENT -e 'start slave;'
;; ;;
flashbackmysqldump) flashbackmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1 socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;' $MYSQL_CLIENT -e 'start slave;'
;; ;;
reseedmariabackup) reseedmariabackup)
rm -rf $BACKUPDIR rm -rf $BACKUPDIR
......
...@@ -4,13 +4,11 @@ curl () { ...@@ -4,13 +4,11 @@ curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@" {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
} }
# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
# Checking if mariadb start is needed # Checking if mariadb start is needed
#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start) CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
if [ $CODE -eq 200 ]; then if [ $CODE -eq 200 ]; then
sleep 5
echo "$CODE: Updating mysql configuration..." echo "$CODE: Updating mysql configuration..."
# update mysql configuration # update mysql configuration
{{ update_config }} {{ update_config }}
...@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then ...@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then
sleep 5 sleep 5
# check again if the service is still up... # check again if the service is still up...
{{ mariadb_controller }} status mariadb {{ mariadb_controller }} status mariadb
else
echo "CODE: $CODE. No need start!"
fi fi
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment