Commit d0c397f9 authored by Alain Takoudjou's avatar Alain Takoudjou

repman: include restic for backup, add more parameter for instance and json parameter schema

parent 59ae57d8
......@@ -14,19 +14,19 @@
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = 158e4c884b04442aff9726704b3f4d60
md5sum = b38fb4c5cdba2223c84d62ef60513e49
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 0a17bad0a5e7f472641c8a2984ae9400
md5sum = e50536613b175a2e25a041c61adeb59f
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
md5sum = 5114d1bb43025cdc5357572634871da7
md5sum = 1efa65af63b14ca2dd1b8c692b2af80e
[config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in
md5sum = d883337a75140c08853960cd6a8fdc5f
md5sum = 39fb83a9b542b9809948cab98aefa2e7
[nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in
......@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = 57ffd0fc1a7bca46b07cdc3121bbf36f
md5sum = d32a50db737a9131de7f3eb27f704232
[template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in
......@@ -42,11 +42,11 @@ md5sum = 04a12d3271857fcf29cb010fd7e38a5c
[template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = 6c88f4a98668d7c5c0aec14ae662686e
md5sum = c31618ef115ecd72b30ae59d0440a3ec
[mariadb-init-root-sql]
_update_hash_filename_ = templates/mariadb_init_root.sql.in
md5sum = bd39fb1e20212c820e5c6b74102b72e8
md5sum = 864d419ea506c9ac543409ba7abb22b0
[init-root-wrapper-in]
_update_hash_filename_ = templates/init_root_wrapper.in
......@@ -54,7 +54,7 @@ md5sum = 3ba02b99c3fcbc24b3659917980b310f
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
md5sum = 8ad6565e7b26e57b41bda4023a9d29ed
md5sum = 99324b56192003254081ef336dcee94c
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
......
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', True) -%}
{% set port = slapparameter_dict['tcp-port'] %}
{% set host = (ipv4_set | list)[0] -%}
{% if use_ipv6 -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set host = '[' ~ (ipv4_set | list)[0] ~ ']' -%}
{% set host = '[' ~ ip ~ ']' -%}
{% else -%}
{% set ip = (ipv4_set | list)[0] -%}
{% endif -%}
{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
{% set database_list = slapparameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
# XXX- TODO: add cron with check db need restart then restart
# API check restard needed: http://repman/api/clusters/{clusterName}/servers/{serverName}/{serverPort}/need-restart
[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
-extends = publish-early
......@@ -274,7 +277,7 @@ cluster = cluster1
config = ${directory:etc}/mysql/my.cnf
command =
cd ${directory:config-tmp} &&
{{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ ip }}/{{ port }}/config &&
{{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config &&
tar -xzf config.tar.gz &&
cp -r data/.system ${directory:mysql} &&
rm -rf ${directory:etc}/mysql &&
......
This diff is collapsed.
{
"name": "Output Parameters",
"properties": {
"backend-url": {
"title": "Backend URL",
"description": "URL used to connect directly to backend without frontend. Requires IPv6.",
"type": "string",
"format": "uri",
"require": true
},
"url": {
"title": "URL",
"description": "URL used to connect to the service.",
"type": "string",
"format": "uri",
"require": false
},
"repman-password": {
"title": "Repman password",
"description": "Password for Replication Manager service.",
"type": "string",
"require": false
}
}
}
......@@ -60,6 +60,7 @@ extra-context =
raw config_cluster_toml_in {{ config_cluster_toml_in }}
raw nginx_conf_in {{ nginx_conf_in }}
raw rsync_location {{ rsync_location }}
raw restic_bin_location {{ restic_bin_location }}
raw sysbench_location {{ sysbench_location }}
raw proxysql_location {{ proxysql_location }}
raw template_repman_manager_sh {{ template_repman_manager_sh }}
......
......@@ -4,6 +4,7 @@ extends =
buildout.hash.cfg
../neoppod/software-common.cfg
../../stack/monitor/buildout.cfg
../../component/restic/buildout.cfg
../../component/replication-manager/buildout.cfg
../../component/mariadb/buildout.cfg
../../component/nginx/buildout.cfg
......@@ -27,6 +28,8 @@ parts =
instance.cfg
template-mariadb.cfg
template-mysqld-wrapper
gowork
restic
[mariadb]
# Compile dir is for plugins, there's no plugin yet
......@@ -62,6 +65,7 @@ context =
key percona_toolkit_location percona-toolkit:location
key repman_src_location git.signal18.io_signal18_repman:location
key rsync_location rsync:location
key restic_bin_location restic:location
key socat_location socat:location
key template_repman_manager_sh repman-manager-sh.in:target
key template_mariadb template-mariadb.cfg:target
......
{
"name": "Replication Manager",
"description": "Replication Manager",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"description": "Replication Manager",
"request": "instance-repman-input-schema.json",
"response": "instance-repman-output-schema.json",
"index": 0
}
}
}
{% macro setbool(value) -%}
{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
{% endmacro -%}
[{{ parameter_dict['name'] }}]
title = "{{ parameter_dict['name'] }}"
monitoring-save-config = true
db-servers-hosts = "{{ parameter_dict['db-list'] }}"
db-servers-prefered-master = "{{ parameter_dict['db-prefered-master'] }}"
db-servers-credential = "{{ parameter_dict['db-credential'] }}"
......@@ -16,22 +20,48 @@ proxysql-bootstrap = true
proxysql-admin-port = {{ parameter_dict['proxy-admin-port'] }}
proxysql-password = "{{ parameter_dict['password'] }}"
prov-proxy-tags = "pkg,masterslave,linux,noreadwritesplit"
prov-proxy-tags = "{{ parameter_dict['proxy-tags'] }}"
monitoring-scheduler = true
scheduler-db-servers-logical-backup = true
scheduler-db-servers-logical-backup-cron= "0 0 1 * * 6"
scheduler-db-servers-logical-backup-cron = "0 {{ parameter_dict['logical-backup-cron'] }}"
scheduler-db-servers-logs = true
scheduler-db-servers-logs-cron = "0 0 * * * *"
#scheduler-db-servers-logs-table-keep = 12
scheduler-db-servers-logs-table-keep = 4
scheduler-db-servers-logs-table-rotate = true
#scheduler-db-servers-logs-table-rotate-cron = "0 0 0/6 * * *"
scheduler-db-servers-logs-table-rotate-cron = "0 0 23 * * *"
scheduler-db-servers-optimize = true
scheduler-db-servers-optimize-cron = "0 0 3 1 * 5"
scheduler-db-servers-physical-backup = true
scheduler-db-servers-physical-backup-cron = "0 0 0 * * *"
scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backup-cron'] }}"
backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump"
# scheduler-db-servers-receiver-ports="4444,4445"
prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
prov-db-cpu-cores = {{ parameter_dict['db-cpu-cores'] }}
prov-db-disk-iops = {{ parameter_dict['db-disk-iops'] }}
prov-db-memory = {{ parameter_dict['db-memory'] }}
prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
test-inject-traffic = true
# failover
failover-mode = "{{ parameter_dict['failover-mode'] }}"
failover-limit = {{ parameter_dict['failover-limit'] }}
failover-falsepositive-heartbeat = {{ setbool(parameter_dict['failover-falsepositive-heartbeat']) }}
failover-falsepositive-heartbeat-timeout = {{ parameter_dict['failover-falsepositive-heartbeat-timeout'] }}
failover-falsepositive-ping-counter = {{ parameter_dict['failover-falsepositive-ping-counter'] }}
failover-max-slave-delay = {{ parameter_dict['failover-max-slave-delay'] }}
failover-readonly-state = {{ setbool(parameter_dict['failover-readonly-state']) }}
failover-restart-unsafe = {{ setbool(parameter_dict['failover-restart-unsafe']) }}
failover-time-limit = {{ parameter_dict['failover-time-limit'] }}
#switchover
switchover-at-equal-gtid = {{ setbool(parameter_dict['switchover-at-equal-gtid']) }}
switchover-slave-wait-catch = {{ setbool(parameter_dict['switchover-slave-wait-catch']) }}
switchover-wait-kill = {{ parameter_dict['switchover-wait-kill'] }}
switchover-wait-trx = {{ parameter_dict['switchover-wait-trx'] }}
switchover-wait-write-query = {{ parameter_dict['switchover-wait-write-query'] }}
[Default]
api-bind = "{{ parameter_dict['ipv4'] }}"
http-bind-address = "{{ parameter_dict['ipv4'] }}"
http-server = true
http-session-lifetime = {{ parameter_dict['http-session-lifetime'] }}
http-refresh-interval = {{ int(parameter_dict['http-refresh-interval'])*1000 }}
monitoring-save-config = false
api-https-bind = true
api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
......@@ -10,11 +13,10 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
# Database list of hosts to ignore in election
#db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
# XXX - check if mandatory (as present in cluster)
#db-servers-hosts =
monitoring-address = "{{ parameter_dict['ipv4'] }}"
#haproxy = true
haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
#haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
# HaProxy input bind address for read (default "0.0.0.0")
#haproxy-ip-read-bind =
# HaProxy input bind address for write (default "0.0.0.0")
......@@ -28,6 +30,17 @@ haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
#HaProxy read-write port to leader (default 3306)
#haproxy-write-port =
# Use restic to archive and restore backups
backup = true
backup-restic = true
backup-restic-binary-path = "{{ parameter_dict['restic-bin'] }}"
backup-restic-aws = false
backup-restic-password = "{{ parameter_dict['password'] }}"
backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
# Mail configuration
# Alert email sender (default "mrm@localhost")
mail-from = "{{ parameter_dict['mail-from'] }}"
......@@ -38,12 +51,8 @@ mail-smtp-user = "{{ parameter_dict['mail-smtp-user'] }}"
# Alert email recipients, separated by commas
mail-to = "{{ parameter_dict['mail-to'] }}"
mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
prov-orchestrator = "slapos"
prov-db-tags="gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog"
prov-db-tags = "{{ parameter_dict['enabled-tags'] }}"
sysbench-binary-path = "{{ parameter_dict['sysbench-bin'] }}"
# Number of threads to run benchmark (default 4)
......
......@@ -6,5 +6,7 @@ CREATE USER 'repman'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}
GRANT ALL ON *.* TO 'repman'@'localhost' WITH GRANT OPTION ;
CREATE USER 'repman'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO 'repman'@'%' WITH GRANT OPTION ;
CREATE USER 'repman'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO 'repman'@'::' WITH GRANT OPTION ;
DROP DATABASE IF EXISTS test ;
FLUSH PRIVILEGES ;
\ No newline at end of file
......@@ -14,6 +14,7 @@ CREATE DATABASE IF NOT EXISTS `{{ name }}`;
{% if user -%}
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
{%- endif %}
{% endmacro -%}
......
......@@ -17,14 +17,26 @@ wait_database () {
else
if [ $CODE -eq 200 ]; then
break;
else:
echo ">> [$retry] waitdatabases returned code $CODE...";
fi
echo ">> [$retry] waitdatabases returned code $CODE...";
fi
sleep 1
done
}
activate_proxy () {
NAME=$1
URL="{{ secure_url }}/api/clusters/$NAME/settings/actions/switch/database-hearbeat"
echo ">> Calling $URL...";
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" $URL)
if [ $CODE -eq 200 ]; then
return 0;
else
echo ">> ERROR: failed to activate proxy: $URL returned code $CODE"
return 1;
fi
}
TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
# Always reload cluster configuration to apply recent changes
......@@ -45,10 +57,14 @@ if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
curl -H "Authorization: Bearer ${TOKEN}" \
{{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave)
SUCCESS=0
if [ $CODE -eq 200 ]; then
activate_proxy {{ name }}
if [ $? -eq 0 ]; then
# Mark boostrap done!
echo "Cluster {{ name }} replication bootstrapped"
echo "DO NOT REMOVE THIS FILE" > {{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped
fi
else
echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE"
fi
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment