Commit e76ad90d authored by Alain Takoudjou's avatar Alain Takoudjou

repman: configure first cluster with mariadb instances, add proxysql and sysbench

parent 554162a3
...@@ -14,16 +14,48 @@ ...@@ -14,16 +14,48 @@
# not need these here). # not need these here).
[instance.cfg] [instance.cfg]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 4373aa04326e85ae6c3136d12367646e md5sum = 158e4c884b04442aff9726704b3f4d60
[instance-repman.cfg] [instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in _update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = d5b3bbd9317e8ce60acdf03730fca1e5 md5sum = 0a17bad0a5e7f472641c8a2984ae9400
[config-toml.in] [config-toml.in]
_update_hash_filename_ = templates/config.toml.in _update_hash_filename_ = templates/config.toml.in
md5sum = aa7f9fc85cef6061b87b080882d3d0d7 md5sum = 5114d1bb43025cdc5357572634871da7
[config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in
md5sum = d883337a75140c08853960cd6a8fdc5f
[nginx.conf.in] [nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in _update_hash_filename_ = templates/nginx.conf.in
md5sum = 6d555e5e296654dc3e28d2ad5fa1bb8d md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
\ No newline at end of file
[template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = 57ffd0fc1a7bca46b07cdc3121bbf36f
[template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in
md5sum = 04a12d3271857fcf29cb010fd7e38a5c
[template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = 6c88f4a98668d7c5c0aec14ae662686e
[mariadb-init-root-sql]
_update_hash_filename_ = templates/mariadb_init_root.sql.in
md5sum = bd39fb1e20212c820e5c6b74102b72e8
[init-root-wrapper-in]
_update_hash_filename_ = templates/init_root_wrapper.in
md5sum = 3ba02b99c3fcbc24b3659917980b310f
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
md5sum = 8ad6565e7b26e57b41bda4023a9d29ed
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
md5sum = d623a4c684578602b9d8ee49034aebfa
This diff is collapsed.
# GitLab "switch-softwaretype" instance
[buildout] [buildout]
parts = switch-softwaretype parts = switch-softwaretype
# std stuff for slapos instance eggs-directory = {{ buildout_egg_directory }}
eggs-directory = ${buildout:eggs-directory} develop-eggs-directory = {{ buildout_develop_directory }}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true offline = true
[switch-softwaretype] [switch-softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:switch-softwaretype
repman = $${template-instance-repman.cfg:rendered} default = template-instance-repman.cfg:rendered
default = $${:repman} RootSoftwareInstance = ${:default}
mariadb = template-instance-mariadb.cfg:rendered
# macro: render instance-*.cfg from instance-*.cfg.in [slap-configuration]
[instance-cfg] recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
mode = 0644 mode = 0644
rendered= $${buildout:directory}/$${:_buildout_section_name_} extensions = jinja2.ext.do
rendered= ${buildout:directory}/${:_buildout_section_name_}
context = context =
key slapparameter_dict slap-configuration:configuration
key bin_directory buildout:bin-directory key computer_id slap-configuration:computer
key eggs_directory buildout:eggs-directory key ipv4_set slap-configuration:ipv4
key develop_eggs_directory buildout:develop-eggs-directory key ipv6_set slap-configuration:ipv6
raw bin_directory {{ buildout_directory }}
raw eggs_directory {{ buildout_egg_directory }}
raw develop_eggs_directory {{ buildout_develop_directory }}
raw mariadb_location {{ mariadb_location }}
raw template_monitor {{ template_monitor_cfg }}
# program binaries # program binaries
raw bash_bin ${bash:location}/bin/bash raw bash_bin {{ bash_location }}/bin/bash
raw gowork_bin ${gowork:bin} raw jq_bin {{ jq_location }}/jq
raw nginx_bin ${nginx:location}/sbin/nginx raw curl_bin {{ curl_location }}/bin/curl
raw repman_src_location ${git.signal18.io_signal18_repman:location}
raw template_monitor_cfg ${monitor2-template:rendered}
# config files ${:extra-context}
raw config_toml_in ${config-toml.in:target}
raw nginx_conf_in ${nginx.conf.in:target}
$${:extra-context}
extra-context = extra-context =
[template-instance-repman.cfg] [template-instance-repman.cfg]
<= instance-cfg <= jinja2-template-base
template= ${instance-repman.cfg:target} template= {{ template_repman_cfg }}
extra-context =
raw gowork_bin {{ gowork_bin }}
raw haproxy_location {{ haproxy_location }}
raw nginx_bin {{ nginx_location }}/sbin/nginx
raw repman_src_location {{ repman_src_location }}
# config files
raw config_toml_in {{ config_toml_in }}
raw config_cluster_toml_in {{ config_cluster_toml_in }}
raw nginx_conf_in {{ nginx_conf_in }}
raw rsync_location {{ rsync_location }}
raw sysbench_location {{ sysbench_location }}
raw proxysql_location {{ proxysql_location }}
raw template_repman_manager_sh {{ template_repman_manager_sh }}
[template-mariadb-parameters]
bash = {{ bash_location }}
dash-location = {{ dash_location }}
gzip-location = {{ gzip_location }}
mariadb-location = {{ mariadb_location }}
template-my-cnf = {{ template_my_cnf }}
template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
template-mariadb-init-root = {{ template_init_root_sql }}
template-init-root-wrapper = {{ template_init_root_wrapper }}
template-mysqld-wrapper = {{ template_mysqld_wrapper }}
link-binary = {{ dumps(mariadb_link_binary) }}
check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
bin-directory = {{ bin_directory }}
percona-tools-location = {{ percona_toolkit_location }}
unixodbc-location = {{ unixodbc_location }}
curl-location = {{ curl_location }}
dbjobs-template = {{ dbjobs_in }}
socat-location = {{ socat_location }}
[template-instance-mariadb.cfg]
<= jinja2-template-base
template = {{ template_mariadb }}
filename = instance-mariadb.cfg
extra-context = extra-context =
section parameter_dict template-mariadb-parameters
...@@ -2,24 +2,88 @@ ...@@ -2,24 +2,88 @@
extends = extends =
buildout.hash.cfg buildout.hash.cfg
../neoppod/software-common.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
../../component/replication-manager/buildout.cfg ../../component/replication-manager/buildout.cfg
../../component/mariadb/buildout.cfg ../../component/mariadb/buildout.cfg
../../component/nginx/buildout.cfg ../../component/nginx/buildout.cfg
../../stack/monitor/buildout.cfg ../../component/haproxy/buildout.cfg
../../component/logrotate/buildout.cfg ../../component/logrotate/buildout.cfg
../../component/percona-toolkit/buildout.cfg
../../component/gzip/buildout.cfg
../../component/sed/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/grep/buildout.cfg
../../component/sysbench/buildout.cfg
../../component/proxysql/buildout.cfg
../../component/socat/buildout.cfg
../../component/rsync/buildout.cfg
../../stack/monitor/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
parts = parts =
slapos-cookbook slapos-cookbook
mroonga-mariadb
instance.cfg instance.cfg
mariadb template-mariadb.cfg
template-mysqld-wrapper
[mariadb]
# Compile dir is for plugins, there's no plugin yet
keep-compile-dir = false
[instance.cfg] [instance.cfg]
recipe = slapos.recipe.template recipe = slapos.recipe.template:jinja2
url = ${:_profile_base_location_}/${:filename} rendered = ${buildout:directory}/instance.cfg
output = ${buildout:directory}/instance.cfg template = ${:_profile_base_location_}/${:filename}
mode = 0644
context =
key bash_location bash:location
key bin_directory buildout:bin-directory
key config_toml_in config-toml.in:target
key config_cluster_toml_in config-cluster-toml.in:target
key coreutils_location coreutils:location
key curl_location curl:location
key buildout_egg_directory buildout:eggs-directory
key buildout_develop_directory buildout:develop-eggs-directory
key buildout_directory buildout:directory
key dbjobs_in dbjobs-in:target
key dash_location dash:location
key jq_location jq-binary:location
key logrotate_cfg template-logrotate-base:rendered
key gowork_bin gowork:bin
key gzip_location gzip:location
key haproxy_location haproxy:location
key template_monitor monitor2-template:rendered
key mariadb_link_binary template-mariadb.cfg:link-binary
key mariadb_location mariadb:location
key nginx_conf_in nginx.conf.in:target
key nginx_location nginx:location
key percona_toolkit_location percona-toolkit:location
key repman_src_location git.signal18.io_signal18_repman:location
key rsync_location rsync:location
key socat_location socat:location
key template_repman_manager_sh repman-manager-sh.in:target
key template_mariadb template-mariadb.cfg:target
key template_mariadb_initial_setup template-mariadb-initial-setup:target
key template_monitor_cfg monitor2-template:rendered
key template_my_cnf template-my-cnf:target
key template_mysqld_wrapper template-mysqld-wrapper:rendered
key template_init_root_sql mariadb-init-root-sql:target
key template_init_root_wrapper init-root-wrapper-in:target
key template_repman_cfg instance-repman.cfg:target
key unixodbc_location unixodbc:location
key sysbench_location sysbench:location
key proxysql_location proxysql:location
[jq-binary]
recipe = hexagonit.recipe.download
url = https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
md5sum = 1fffde9f3c7944f063265e9a5e67ae4f
filename = jq
mode = 0755
download-only = true
[download-file] [download-file]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
...@@ -29,13 +93,48 @@ destination = ${buildout:directory}/${:_buildout_section_name_} ...@@ -29,13 +93,48 @@ destination = ${buildout:directory}/${:_buildout_section_name_}
[config-toml.in] [config-toml.in]
<= download-file <= download-file
[config-cluster-toml.in]
<= download-file
[instance-repman.cfg] [instance-repman.cfg]
<= download-file <= download-file
[repman-manager-sh.in]
<= download-file
[template-mariadb.cfg]
<= download-file
link-binary =
${coreutils:location}/bin/basename
${coreutils:location}/bin/cat
${coreutils:location}/bin/cp
${coreutils:location}/bin/ls
${coreutils:location}/bin/tr
${coreutils:location}/bin/uname
${gettext:location}/lib/gettext/hostname
${grep:location}/bin/grep
${sed:location}/bin/sed
${mariadb:location}/bin/mysqlbinlog
[template-mariadb-initial-setup]
<= download-file
[template-my-cnf]
<= download-file
[mariadb-init-root-sql]
<= download-file
[init-root-wrapper-in]
<= download-file
[nginx.conf.in] [nginx.conf.in]
<= download-file <= download-file
[dbjobs-in]
<= download-file
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions] [versions]
slapos.recipe.template = 4.3 slapos.recipe.template = 4.3
rubygemsrecipe = 0.2.2+slapos001 rubygemsrecipe = 0.2.2+slapos001
\ No newline at end of file
[{{ parameter_dict['name'] }}]
title = "{{ parameter_dict['name'] }}"
db-servers-hosts = "{{ parameter_dict['db-list'] }}"
db-servers-prefered-master = "{{ parameter_dict['db-prefered-master'] }}"
db-servers-credential = "{{ parameter_dict['db-credential'] }}"
replication-credential = "{{ parameter_dict['db-credential'] }}"
db-servers-connect-timeout = 1
slapos-db-partitions = "{{ parameter_dict['partition-list'] }}"
slapos-proxysql-partitions = "{{ parameter_dict['proxysql-partition'] }}"
proxysql = true
proxysql-port = {{ parameter_dict['proxy-port'] }}
proxysql-servers = "{{ parameter_dict['proxysql-servers'] }}"
proxysql-user = "{{ parameter_dict['proxysql-user'] }}"
proxysql-bootstrap = true
proxysql-admin-port = {{ parameter_dict['proxy-admin-port'] }}
proxysql-password = "{{ parameter_dict['password'] }}"
prov-proxy-tags = "pkg,masterslave,linux,noreadwritesplit"
monitoring-scheduler = true
scheduler-db-servers-logical-backup = true
scheduler-db-servers-logical-backup-cron= "0 0 1 * * 6"
scheduler-db-servers-logs = true
scheduler-db-servers-logs-cron = "0 0 * * * *"
#scheduler-db-servers-logs-table-keep = 12
scheduler-db-servers-logs-table-keep = 4
scheduler-db-servers-logs-table-rotate = true
#scheduler-db-servers-logs-table-rotate-cron = "0 0 0/6 * * *"
scheduler-db-servers-logs-table-rotate-cron = "0 0 23 * * *"
scheduler-db-servers-optimize = true
scheduler-db-servers-optimize-cron = "0 0 3 1 * 5"
scheduler-db-servers-physical-backup = true
scheduler-db-servers-physical-backup-cron = "0 0 0 * * *"
backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump"
# scheduler-db-servers-receiver-ports="4444,4445"
\ No newline at end of file
[Default] [Default]
api-bind = "[{{ parameter_dict['ipv4'] }}]" api-bind = "{{ parameter_dict['ipv4'] }}"
http-bind-address = "[{{ parameter_dict['ipv4'] }}]" http-bind-address = "{{ parameter_dict['ipv4'] }}"
monitoring-save-config = true monitoring-save-config = false
api-https-bind = true api-https-bind = true
api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}" api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
include = "{{ parameter_dict['cluster-d'] }}"
autorejoin = true
db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
# Database list of hosts to ignore in election
#db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
# XXX - check if mandatory (as present in cluster)
#db-servers-hosts =
#haproxy = true
haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
# HaProxy input bind address for read (default "0.0.0.0")
#haproxy-ip-read-bind =
# HaProxy input bind address for write (default "0.0.0.0")
#haproxy-ip-write-bind =
# HaProxy load balance read port to all nodes (default 3307)
#haproxy-read-port =
# HaProxy hosts (default "127.0.0.1")
#haproxy-servers =
# HaProxy statistics port (default 1988)
#haproxy-stat-port =
#HaProxy read-write port to leader (default 3306)
#haproxy-write-port =
# Mail configuration
# Alert email sender (default "mrm@localhost")
mail-from = "{{ parameter_dict['mail-from'] }}"
# Alert email SMTP server address, in host:[port] format (default "localhost:25")
mail-smtp-addr = "{{ parameter_dict['mail-smtp-addr'] }}"
mail-smtp-password = "{{ parameter_dict['mail-smtp-password'] }}"
mail-smtp-user = "{{ parameter_dict['mail-smtp-user'] }}"
# Alert email recipients, separated by commas
mail-to = "{{ parameter_dict['mail-to'] }}"
mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
prov-orchestrator = "slapos"
prov-db-tags="gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog"
sysbench-binary-path = "{{ parameter_dict['sysbench-bin'] }}"
# Number of threads to run benchmark (default 4)
sysbench-threads = 4
# Time to run benchmark (default 100)
sysbench-time = 100
sysbench-v1 = true
[cluster1]
title = "cluster1"
#db-servers-hosts = "[2408:5180:0:43::d2a5]:3306,[2409:5180:0:43::d2a5]:3306"
#db-servers-prefered-master = "[2408:5180:0:43::d2a5]:3306"
db-servers-hosts = ""
db-servers-credential = "root:test"
#!{{ parameter_dict['bash-bin'] }}
USER={{ parameter_dict['db-user'] }}
PASSWORD={{ parameter_dict['db-password'] }}
ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
DATADIR={{ parameter_dict['mysql-dir'] }}/
export PATH={{ parameter_dict['socat-location'] }}/bin:{{ parameter_dict['mysql-location'] }}/bin:$PATH
JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "reseedxtrabackup" "reseedmariabackup" "reseedmysqldump" "flashbackxtrabackup" "flashbackmariadbackup" "flashbackmysqldump" "stop" "start")
doneJob()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
}
pauseJob()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
}
partialRestore()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
done
for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file=/etc/mysql/dbjob.cnf -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done
for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done
done
for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
done
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e"flush privileges;start slave;"
}
for job in "${JOBS[@]}"
do
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N))
ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
ID=($(echo $TASK | awk -F@ '{ print $1 }'))
#purge de past
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
if [ "$ADDRESS" == "" ]; then
echo "No $job needed"
else
echo "Processing $job"
case "$job" in
reseedmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u TCP-LISTEN:{{ parameter_dict['socat-port'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
;;
flashbackmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u TCP-LISTEN:{{ parameter_dict['socat-port'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
;;
reseedmariabackup)
rm -rf $BACKUPDIR
mkdir $BACKUPDIR
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u TCP-LISTEN:{{ parameter_dict['socat-port'] }},reuseaddr STDOUT | mbstream -x -C $BACKUPDIR
# mbstream -p, --parallel
mariabackup --prepare --export --target-dir=$BACKUPDIR
partialRestore
;;
flashbackmariadbackup)
rm -rf $BACKUPDIR
mkdir $BACKUPDIR
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u TCP-LISTEN:{{ parameter_dict['socat-port'] }},reuseaddr STDOUT | xbstream -x -C $BACKUPDIR
mariabackup --prepare --export --target-dir=$BACKUPDIR
partialRestore
;;
mariabackup)
cd {{ parameter_dict['tmp-dir'] }}
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
error)
cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
> $ERROLOG
;;
slowquery)
cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
> $SLOWLOG
;;
optimize)
mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
restart)
{{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out
;;
esac
doneJob
fi
done
#!{{ dash_bin }}
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
--protocol=socket -uroot -hlocalhost $@
}
if [ ! -f "{{ init_password_done }}" ]; then
for i in {30..0}; do
if echo 'SELECT 1' | run_mysql &> /dev/null; then
break
fi
echo 'MySQL init process in progress...'
sleep 1
done
if [ "$i" = 0 ]; then
echo >&2 'MySQL init process failed.'
exit 1
fi
echo "Setting mariabdb root password...";
run_mysql < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
echo "done"
fi
# Run mariadb_upgrade when replication is bootstrapped will break replication topology.
# skip when already upgraded until we have a good solution.
if [ ! -f "{{ upgrade_done }}" ]; then
{{ mysql_update }}
if [ $? -eq 0 ]; then
touch {{ upgrade_done }};
fi
fi
\ No newline at end of file
-- What's done in this file shouldn't be replicated
-- or products like mysql-fabric won't work
SET @@SESSION.SQL_LOG_BIN=0;
CREATE USER 'repman'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO 'repman'@'localhost' WITH GRANT OPTION ;
CREATE USER 'repman'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO 'repman'@'%' WITH GRANT OPTION ;
DROP DATABASE IF EXISTS test ;
FLUSH PRIVILEGES ;
\ No newline at end of file
USE mysql;
DROP FUNCTION IF EXISTS last_insert_grn_id;
DROP FUNCTION IF EXISTS mroonga_snippet;
DROP FUNCTION IF EXISTS mroonga_command;
{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
{% if mroonga %}CREATE FUNCTION last_insert_grn_id RETURNS INTEGER SONAME '{{ mroonga }}';
CREATE FUNCTION mroonga_snippet RETURNS STRING SONAME '{{ mroonga }}';
CREATE FUNCTION mroonga_command RETURNS STRING SONAME '{{ mroonga }}';{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
{% macro database(name, user, password) -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}`;
{% if user -%}
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
{%- endif %}
{% endmacro -%}
{% for entry in parameter_dict['database-list'] -%}
{{ database(entry['name'], entry.get('user'), entry.get('password')) }}
{% endfor -%}
\ No newline at end of file
{% set socket = parameter_dict['socket'] -%}
# ERP5 buildout my.cnf template based on my-huge.cnf shipped with mysql
# The MySQL server
[mysqld]
# ERP5 by default requires InnoDB storage. MySQL by default fallbacks to using
# different engine, like MyISAM. Such behaviour generates problems only, when
# tables requested as InnoDB are silently created with MyISAM engine.
#
# Loud fail is really required in such case.
# Already present in REPMAN
#sql_mode="NO_ENGINE_SUBSTITUTION"
socket = {{ socket }}
bind_address = {{ parameter_dict['ip'] }}
datadir = {{ parameter_dict['data-directory'] }}
pid_file = {{ parameter_dict['pid-file'] }}
default_time_zone = '+00:00'
plugin_load = ha_mroonga
max_connections = {{ parameter_dict['max-connection-count'] }}
{% set innodb_buffer_pool_size = parameter_dict['innodb-buffer-pool-size'] -%}
{% if innodb_buffer_pool_size %}innodb_buffer_pool_size = {{ innodb_buffer_pool_size }}{% endif %}
{% set innodb_buffer_pool_instances = parameter_dict['innodb-buffer-pool-instances'] -%}
{% if innodb_buffer_pool_instances %}innodb_buffer_pool_instances = {{ innodb_buffer_pool_instances }}{% endif %}
{% set innodb_log_file_size = parameter_dict['innodb-log-file-size'] -%}
{% if innodb_log_file_size %} innodb_log_file_size = {{ innodb_log_file_size }}{% endif %}
{% set innodb_log_buffer_size = parameter_dict['innodb-log-buffer-size'] -%}
{% if innodb_log_buffer_size %} innodb_log_buffer_size = {{ innodb_log_buffer_size }}{% endif %}
# very important to allow parallel indexing
# Note: this is compatible with binlog-based incremental backups, because ERP5
# doesn't use "insert ... select" (in any number of queries) pattern.
innodb_locks_unsafe_for_binlog = 1
{% if 'ssl-key' in parameter_dict -%}
ssl
ssl-cert = {{ parameter_dict['ssl-crt'] }}
ssl-key = {{ parameter_dict['ssl-key'] }}
{% if 'ssl-ca-crt' in parameter_dict -%}
ssl-ca = {{ parameter_dict['ssl-ca-crt'] }}
{%- endif %}
{% if 'ssl-crl' in parameter_dict -%}
ssl-crl = {{ parameter_dict['ssl-crl'] }}
{%- endif %}
{% if 'ssl-cipher' in parameter_dict -%}
ssl-cipher = {{ parameter_dict['ssl-cipher'] }}
{%- endif %}
{%- endif %}
#server_id = {{ parameter_dict['server-id'] }}
# Some dangerous settings you may want to uncomment temporarily
# if you only want performance or less disk access.
{% set x = '' if parameter_dict['relaxed-writes'] else '#' -%}
{{x}}innodb_flush_log_at_trx_commit = 0
{{x}}innodb_flush_method = nosync
{{x}}innodb_doublewrite = 0
{{x}}sync_frm = 0
# Force utf8 usage
character_set_server = utf8mb4
collation_server = utf8mb4_general_ci
skip_character_set_client_handshake
[client]
socket = {{ socket }}
user = root
[mysql]
no_auto_rehash
[mysqlhotcopy]
interactive_timeout
[mysqldump]
max_allowed_packet = 128M
...@@ -18,7 +18,7 @@ http { ...@@ -18,7 +18,7 @@ http {
} }
server { server {
listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['port'] }} ssl; listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['ssl-port'] }} ssl;
server_name _; server_name _;
ssl_certificate {{ parameter_dict['ssl-certificate'] }}; ssl_certificate {{ parameter_dict['ssl-certificate'] }};
ssl_certificate_key {{ parameter_dict['ssl-key'] }}; ssl_certificate_key {{ parameter_dict['ssl-key'] }};
...@@ -27,6 +27,21 @@ http { ...@@ -27,6 +27,21 @@ http {
ssl_prefer_server_ciphers on; ssl_prefer_server_ciphers on;
keepalive_timeout 90s; keepalive_timeout 90s;
location / {
proxy_redirect off;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_connect_timeout 200;
proxy_pass {{ parameter_dict['repman-secure-url'] }};
}
}
server {
listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['port'] }};
server_name _;
location / { location / {
proxy_redirect off; proxy_redirect off;
proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Proto $scheme;
......
#!{{ bash_bin }}
#set -e
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
wait_database () {
NAME=$1
for retry in {1..5}; do
echo ">> Wait until $NAME databases are ready...";
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/$NAME/actions/waitdatabases);
if [ $CODE -eq 504 ]; then
# We have a timeout try again
echo ">> [$retry] Timeout on {{ secure_url }}/api/clusters/$NAME/actions/waitdatabases, trying again...";
else
if [ $CODE -eq 200 ]; then
break;
else:
echo ">> [$retry] waitdatabases returned code $CODE...";
fi
fi
sleep 1
done
}
TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
# Always reload cluster configuration to apply recent changes
{% for name in cluster_name_list -%}
# reload {{ name }} settings
echo "Reloading settings for {{ name }}..."
curl -H "Authorization: Bearer ${TOKEN}" \
{{ secure_url }}/api/clusters/{{ name }}/settings/actions/reload
# Start Replication on {{ name }}
if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
wait_database {{ name }}
echo "Bootstrap replication on {{ name }}..."
TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
curl -H "Authorization: Bearer ${TOKEN}" \
{{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave)
if [ $CODE -eq 200 ]; then
# Mark boostrap done!
echo "Cluster {{ name }} replication bootstrapped"
echo "DO NOT REMOVE THIS FILE" > {{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped
else
echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE"
fi
fi
{% endfor %}
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment