Commit 73001c02 authored by Alain Takoudjou's avatar Alain Takoudjou

repman: switch to mariadb 10.4 and fix database backup failure

parent 3121bdee
......@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = 6af0fb36ebdc5f1730ed3a9319eafdad
md5sum = 6d26b96a25d927452f2d41b3844e7be0
[template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in
......@@ -42,7 +42,7 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
[template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = 96c151c376600a90fe2a44825f2de550
md5sum = 5a070090fc5b266bf866855c9c54dadf
[template-publish-slave-information]
_update_hash_filename_ = publish-database-slave-parameters.cfg.in
......@@ -54,7 +54,7 @@ md5sum = d8a0a7d72b02f911dafda655941b805d
[init-root-wrapper-in]
_update_hash_filename_ = templates/init_root_wrapper.in
md5sum = b3dffa3c8b9b2cc86b536d411cbde9d4
md5sum = 6d5e15e586810520f7d4a873ec9d3cc3
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
......@@ -62,7 +62,7 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
md5sum = bf3a3d91e3152d5dfd807f35890d89c6
md5sum = da15968c75e7c856662099edaea5ef46
[mysqld-need-start.sh.in]
_update_hash_filename_ = templates/mysqld-need-start.sh.in
......@@ -74,4 +74,4 @@ md5sum = 455aaf369bf5141758dc57f2c0e67b08
[slave-db-manage.in]
_update_hash_filename_ = templates/slave-db-manage.in
md5sum = 6af818d4513561eea7def0f4f8d42164
md5sum = b9220b1867974d4164abcb48ed01872f
......@@ -131,8 +131,9 @@ template = {{ parameter_dict['template-init-root-wrapper'] }}
[mysql-slave-db-cleanup]
< = jinja2-template-script-base
rendered = ${directory:bin}/manage_slave_db
db-name = {{ slapparameter_dict['database-name'] }}
extra-context =
raw database_name {{ slapparameter_dict['database-name'] }}
key database_name :db-name
template = {{ parameter_dict['template-manage-db'] }}
[mysqld]
......@@ -193,13 +194,18 @@ target-directory = ${directory:bin}
link-binary = {{ dumps(parameter_dict['link-binary']) }}
[binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it.
command-line =
"{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
--defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args}
wrapper-path = ${directory:bin}/${:command}
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/${:command}
mode = 755
template =
inline:#!/bin/sh
exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
--defaults-file="${directory:etc}/mysql/my.cnf" \
--protocol=socket --user=$USER \
${:extra-args} "$@"
wrapper-path = ${:rendered}
extra-args =
[binary-wrap-mysql]
......@@ -223,6 +229,11 @@ command = mysqladmin
<= binary-wrap-base
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
command = mysql_tzinfo_to_sql
template =
inline:#!/bin/sh
exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
--defaults-file="${directory:etc}/mysql/my.cnf" \
--skip-write-binlog "$@"
[binary-wrap-pt-digest]
<= binary-wrap-base
......
......@@ -20,7 +20,6 @@ extends =
../../component/jq/buildout.cfg
../../stack/supervisord/buildout.cfg
../../stack/monitor/buildout.cfg
../neoppod/software-common.cfg
parts =
slapos-cookbook
......@@ -30,9 +29,44 @@ parts =
template-mysqld-wrapper
gowork
[mariadb]
location = ${mariadb-10.4:location}
[python]
part = python2.7
[template-mysqld-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
mode = 644
template =
inline:{% raw %}#!/bin/sh -e
basedir='${mariadb:location}'
datadir='{{datadir}}'
marker=$datadir/.slapos_initializing
[ -d "$datadir/mysql" ] && [ ! -f "$marker" ] || {
find "$datadir/" -mindepth 1 ! -path $marker -delete || mkdir "$datadir"
touch "$marker"
"$basedir/scripts/mysql_install_db" \
--defaults-file='{{defaults_file}}' \
--skip-name-resolve \
--auth-root-authentication-method=socket \
--cross-bootstrap \
--auth-root-socket-user=$USER \
--user=$USER \
--basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
--datadir="$datadir"
rm "$marker"
}
{%- if environ is defined %}
{%- for variable in environ.splitlines() %}
{{ variable }} \
{%- endfor %}
{%- endif %}
exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
{% endraw %}
[instance.cfg]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/instance.cfg
......
#!/bin/bash
USER={{ parameter_dict['db-user'] }}
DBUSER={{ parameter_dict['db-user'] }}
PASSWORD={{ parameter_dict['db-password'] }}
ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
......@@ -54,7 +54,7 @@ partialRestore()
mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
$MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
done
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER
$MYSQL_CLIENT -e "flush privileges;start slave;"
}
......@@ -63,7 +63,7 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
for job in "${JOBS[@]}"
do
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N))
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER -N))
ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
ID=($(echo $TASK | awk -F@ '{ print $1 }'))
......@@ -78,13 +78,13 @@ do
reseedmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
$MYSQL_CLIENT -e 'start slave;'
;;
flashbackmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
$MYSQL_CLIENT -e 'start slave;'
;;
reseedmariabackup)
......@@ -108,7 +108,7 @@ do
;;
mariabackup)
cd {{ parameter_dict['tmp-dir'] }}
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=$USER --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
error)
cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
......
......@@ -2,7 +2,7 @@
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
--protocol=socket -uroot -hlocalhost "$@"
--protocol=socket -u$USER -hlocalhost "$@"
}
if [ ! -f "{{ init_password_done }}" ]; then
......
......@@ -3,7 +3,7 @@ SET @@SESSION.SQL_LOG_BIN=0;
USE mysql;
{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
{% if mroonga %}
SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
#SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
......@@ -22,9 +22,9 @@ CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
'ascii_general_ci': 'ascii',
'ascii_bin': 'ascii'} -%}
{% if charset %}
{% set charset_collate = "CHARACTER SET %s" % charset -%}
{% set charset_collate = "DEFAULT CHARACTER SET %s" % charset -%}
{% if charset_dict.get(charset) -%}
{% set charset_collate = "CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
{% set charset_collate = "DEFAULT CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
{% endif -%}
{% endif -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}` {{ charset_collate }};
......
......@@ -10,7 +10,7 @@ get_token () {
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
--protocol=socket -uroot -hlocalhost "$@"
--protocol=socket -u$USER -hlocalhost "$@"
}
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
......@@ -23,10 +23,10 @@ if [ "$DATADIR" = "{{ partition_dir }}" ]; then
use repman_slave_definition;
CREATE TABLE IF NOT EXISTS \`slave\` (
\`name\` varchar(20) NOT NULL,
\`name\` varchar(80) NOT NULL,
\`state\` tinyint(1) DEFAULT NULL,
PRIMARY KEY (\`name\`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
UPDATE \`slave\` set \`state\`=false;
{% for name in database_name.split(' ') -%}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment