Skip to content

slapos
slapos
Commit 3d49e677 authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

repman: switch to mariadb 10.4 and fix database backup failure 

Also solve when database initial user created with authentication plugin socket on
master database doesn't exist on slave database mariadb
because there are running diffrent slapuser.

OPENED ERR00056 : Master user 'slapuser4'@'localhost' is not defined on
replication candidate [2001:67c:1254:e:5:513b:0:5f35]:2199
parent c9bc3dcb
Show whitespace changes
Inline Side-by-side
Showing with 82 additions and 38 deletions
software/repman/buildout.hash.cfg
View file @ 3d49e677
......@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = f1a03197b3a208cf6167769157e2df47
md5sum = 284cf030eeba4bdcc884908b2d730e47
[template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in
......@@ -42,7 +42,7 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
[template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = 96c151c376600a90fe2a44825f2de550
md5sum = 5a070090fc5b266bf866855c9c54dadf
[template-publish-slave-information]
_update_hash_filename_ = publish-database-slave-parameters.cfg.in
......@@ -54,7 +54,7 @@ md5sum = d8a0a7d72b02f911dafda655941b805d
[init-root-wrapper-in]
_update_hash_filename_ = templates/init_root_wrapper.in
md5sum = b3dffa3c8b9b2cc86b536d411cbde9d4
md5sum = c203f40a58386310a433b58fd345a341
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
......@@ -62,7 +62,7 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
md5sum = bf3a3d91e3152d5dfd807f35890d89c6
md5sum = 44ef0b7c2c03715a3765400b71d240c8
[mysqld-need-start.sh.in]
_update_hash_filename_ = templates/mysqld-need-start.sh.in
......@@ -74,4 +74,4 @@ md5sum = 455aaf369bf5141758dc57f2c0e67b08
[slave-db-manage.in]
_update_hash_filename_ = templates/slave-db-manage.in
md5sum = 6af818d4513561eea7def0f4f8d42164
md5sum = b45313ae5fb06972cc7fc945e34e434a
software/repman/instance-mariadb.cfg.jinja2.in
View file @ 3d49e677
......@@ -128,11 +128,12 @@ extra-context =
key init_database_sql init-script:rendered
template = {{ parameter_dict['template-init-root-wrapper'] }}
[mysql-slave-db-cleanup]
[{{ section('mysql-slave-db-cleanup') }}]
< = jinja2-template-script-base
rendered = ${directory:bin}/manage_slave_db
rendered = ${directory:scripts}/manage-slave-db
db-name = {{ slapparameter_dict['database-name'] }}
extra-context =
raw database_name {{ slapparameter_dict['database-name'] }}
key database_name :db-name
template = {{ parameter_dict['template-manage-db'] }}
[mysqld]
......@@ -193,13 +194,18 @@ target-directory = ${directory:bin}
link-binary = {{ dumps(parameter_dict['link-binary']) }}
[binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it.
command-line =
"{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
--defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args}
wrapper-path = ${directory:bin}/${:command}
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/${:command}
mode = 755
template =
inline:#!/bin/sh
exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
--defaults-file="${directory:etc}/mysql/my.cnf" \
--protocol=socket \
${:extra-args} "$@"
wrapper-path = ${:rendered}
extra-args =
[binary-wrap-mysql]
......@@ -223,6 +229,11 @@ command = mysqladmin
<= binary-wrap-base
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
command = mysql_tzinfo_to_sql
template =
inline:#!/bin/sh
exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
--defaults-file="${directory:etc}/mysql/my.cnf" \
--skip-write-binlog "$@"
[binary-wrap-pt-digest]
<= binary-wrap-base
......@@ -300,13 +311,6 @@ name = mariadb-need-start
frequency = * * * * *
command = ${template-mysqld-need-start:rendered}
[{{ section('db-remove-cron-entry') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = slave-db-remove
frequency = */5 * * * *
command = ${mysql-slave-db-cleanup:rendered}
[template-mysqld-need-start]
< = jinja2-template-executable
rendered = ${directory:bin}/mysqld_need_start
......
software/repman/software.cfg
View file @ 3d49e677
......@@ -20,7 +20,6 @@ extends =
../../component/jq/buildout.cfg
../../stack/supervisord/buildout.cfg
../../stack/monitor/buildout.cfg
../neoppod/software-common.cfg
parts =
slapos-cookbook
......@@ -30,6 +29,9 @@ parts =
template-mysqld-wrapper
gowork
[mariadb]
location = ${mariadb-10.4:location}
[python]
part = python3
......@@ -37,6 +39,35 @@ part = python3
# replication-manager does not build on golang 1.17
golang = ${golang1.16:location}
[template-mysqld-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
mode = 644
template =
inline:{% raw %}#!/bin/sh -e
basedir='${mariadb:location}'
datadir='{{datadir}}'
marker=$datadir/.slapos_initializing
[ -d "$datadir/mysql" ] && [ ! -f "$marker" ] || {
find "$datadir/" -mindepth 1 ! -path $marker -delete || mkdir "$datadir"
touch "$marker"
"$basedir/scripts/mysql_install_db" \
--defaults-file='{{defaults_file}}' \
--skip-name-resolve \
--auth-root-authentication-method=normal \
--basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
--datadir="$datadir"
rm "$marker"
}
{%- if environ is defined %}
{%- for variable in environ.splitlines() %}
{{ variable }} \
{%- endfor %}
{%- endif %}
exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
{% endraw %}
[instance.cfg]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/instance.cfg
......
software/repman/templates/dbjobs.in
View file @ 3d49e677
#!/bin/bash
USER={{ parameter_dict['db-user'] }}
DBUSER={{ parameter_dict['db-user'] }}
PASSWORD={{ parameter_dict['db-password'] }}
ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
......@@ -54,7 +54,7 @@ partialRestore()
mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
$MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
done
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER
$MYSQL_CLIENT -e "flush privileges;start slave;"
}
......@@ -63,7 +63,7 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
for job in "${JOBS[@]}"
do
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N))
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER -N))
ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
ID=($(echo $TASK | awk -F@ '{ print $1 }'))
......@@ -78,13 +78,13 @@ do
reseedmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
$MYSQL_CLIENT -e 'start slave;'
;;
flashbackmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
$MYSQL_CLIENT -e 'start slave;'
;;
reseedmariabackup)
......@@ -108,7 +108,7 @@ do
;;
mariabackup)
cd {{ parameter_dict['tmp-dir'] }}
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=root --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
error)
cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
......
software/repman/templates/init_root_wrapper.in
View file @ 3d49e677
......@@ -2,7 +2,7 @@
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
--protocol=socket -uroot -hlocalhost "$@"
--protocol=socket -hlocalhost "$@"
}
if [ ! -f "{{ init_password_done }}" ]; then
......@@ -19,6 +19,9 @@ if [ ! -f "{{ init_password_done }}" ]; then
fi
echo "Setting mariabdb root password...";
run_mysql < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
echo "remove unwanted users...";
run_mysql -e "DROP USER IF EXISTS 'root'@'$HOSTNAME';"
run_mysql -e "DROP USER IF EXISTS ''@'$HOSTNAME';"
echo "done"
fi
......
software/repman/templates/mariadb_initial_setup.sql.in
View file @ 3d49e677
......@@ -3,7 +3,7 @@ SET @@SESSION.SQL_LOG_BIN=0;
USE mysql;
{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
{% if mroonga %}
SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
#SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
......@@ -22,9 +22,9 @@ CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
'ascii_general_ci': 'ascii',
'ascii_bin': 'ascii'} -%}
{% if charset %}
{% set charset_collate = "CHARACTER SET %s" % charset -%}
{% set charset_collate = "DEFAULT CHARACTER SET %s" % charset -%}
{% if charset_dict.get(charset) -%}
{% set charset_collate = "CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
{% set charset_collate = "DEFAULT CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
{% endif -%}
{% endif -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}` {{ charset_collate }};
......
software/repman/templates/slave-db-manage.in
View file @ 3d49e677
#!/bin/bash
set -e
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
......@@ -9,8 +11,7 @@ get_token () {
}
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
--protocol=socket -uroot -hlocalhost "$@"
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" --protocol=socket "$@"
}
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
......@@ -23,14 +24,16 @@ if [ "$DATADIR" = "{{ partition_dir }}" ]; then
use repman_slave_definition;
CREATE TABLE IF NOT EXISTS \`slave\` (
\`name\` varchar(20) NOT NULL,
\`name\` varchar(80) NOT NULL,
\`state\` tinyint(1) DEFAULT NULL,
PRIMARY KEY (\`name\`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
UPDATE \`slave\` set \`state\`=false;
{% for name in database_name.split(' ') -%}
{% if name -%}
REPLACE INTO \`slave\` VALUES ('{{ name }}', true);
{% endif -%}
{% endfor -%}
EOF
......@@ -39,16 +42,19 @@ EOF
run_mysql < {{ tmp_dir }}/.script.sql
rm -f {{ tmp_dir }}/.script.sql
DBNAME=$(run_mysql -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false");
DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false");
RET=$?
if [ ! "$RET" = "0" ]; then
echo "Mysql command failed: $DBNAME"
exit $RET
fi
if [ -z "$DBNAME" ]; then
echo "No database for slave to remove.";
fi
for NAME in $DBNAME; do
if [ ! "$NAME" = "name" ]; then
if [ ! -z "$NAME" ]; then
echo "Deleting database $NAME..."
run_mysql -e "DROP DATABASE IF EXISTS $NAME";
run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7