Skip to content

slapos
slapos
Commit d0c397f9 authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

repman: include restic for backup, add more parameter for instance and json parameter schema

parent 59ae57d8
Show whitespace changes
Inline Side-by-side
Showing with 554 additions and 48 deletions
software/repman/buildout.hash.cfg
View file @ d0c397f9
......@@ -14,19 +14,19 @@
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = 158e4c884b04442aff9726704b3f4d60
md5sum = b38fb4c5cdba2223c84d62ef60513e49
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 0a17bad0a5e7f472641c8a2984ae9400
md5sum = e50536613b175a2e25a041c61adeb59f
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
md5sum = 5114d1bb43025cdc5357572634871da7
md5sum = 1efa65af63b14ca2dd1b8c692b2af80e
[config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in
md5sum = d883337a75140c08853960cd6a8fdc5f
md5sum = 39fb83a9b542b9809948cab98aefa2e7
[nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in
......@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = 57ffd0fc1a7bca46b07cdc3121bbf36f
md5sum = d32a50db737a9131de7f3eb27f704232
[template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in
......@@ -42,11 +42,11 @@ md5sum = 04a12d3271857fcf29cb010fd7e38a5c
[template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = 6c88f4a98668d7c5c0aec14ae662686e
md5sum = c31618ef115ecd72b30ae59d0440a3ec
[mariadb-init-root-sql]
_update_hash_filename_ = templates/mariadb_init_root.sql.in
md5sum = bd39fb1e20212c820e5c6b74102b72e8
md5sum = 864d419ea506c9ac543409ba7abb22b0
[init-root-wrapper-in]
_update_hash_filename_ = templates/init_root_wrapper.in
......@@ -54,7 +54,7 @@ md5sum = 3ba02b99c3fcbc24b3659917980b310f
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
md5sum = 8ad6565e7b26e57b41bda4023a9d29ed
md5sum = 99324b56192003254081ef336dcee94c
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
......
software/repman/instance-mariadb.cfg.jinja2.in
View file @ d0c397f9
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', True) -%}
{% set port = slapparameter_dict['tcp-port'] %}
{% set host = (ipv4_set | list)[0] -%}
{% if use_ipv6 -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set host = '[' ~ (ipv4_set | list)[0] ~ ']' -%}
{% set host = '[' ~ ip ~ ']' -%}
{% else -%}
{% set ip = (ipv4_set | list)[0] -%}
{% endif -%}
{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
{% set database_list = slapparameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
# XXX- TODO: add cron with check db need restart then restart
# API check restard needed: http://repman/api/clusters/{clusterName}/servers/{serverName}/{serverPort}/need-restart
[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
-extends = publish-early
......@@ -274,7 +277,7 @@ cluster = cluster1
config = ${directory:etc}/mysql/my.cnf
command =
cd ${directory:config-tmp} &&
{{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ ip }}/{{ port }}/config &&
{{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config &&
tar -xzf config.tar.gz &&
cp -r data/.system ${directory:mysql} &&
rm -rf ${directory:etc}/mysql &&
......
software/repman/instance-repman-input-schema.json 0 → 100644
View file @ d0c397f9
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"slave-frontend": {
"title": "Web frontend",
"description": "Front end used to provide web access for internal services at the kvm.",
"properties": {
"slave-domain": {
"title": "Slave frontend domain",
"description": "Unique domain name for this slave frontend.",
"type": "string",
"default": ""
},
"instance-guid": {
"title": "Main Frontend Instance ID",
"description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",
"type": "string",
"default": ""
},
"frontend-software-type": {
"title": "Frontend Software Type",
"description": "Type of the frontend instance, like \"frontend\".",
"type": "string"
},
"frontend-software-url": {
"title": "Frontend Software URL",
"description": "Software Release URL of the frontend instance, like \"http://example.com/path/to/software.cfg\".",
"type": "string",
"format": "uri",
"default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
}
},
"type": "object"
},
"computer-memory-percent-threshold": {
"title": "Computer memory percent threshold.",
"description": "Computer memory percent threshold.",
"type": "int",
"default": 80
},
"monitor-interface-url": {
"title": "Monitor Web Interface URL",
"description": "Give Url of HTML web interface that will be used to render this monitor instance.",
"type": "string",
"format": "uri",
"default": "https://monitor.app.officejs.com"
},
"monitor-cors-domains": {
"title": "Monitor CORS domains",
"description": "List of cors domains separated with space. Needed for ajax query on this monitor instance from a different domain.",
"type": "string",
"default": "monitor.app.officejs.com"
},
"mail-from": {
"title": "Mail From",
"description": "Mail From address",
"type": "string",
"default": ""
},
"mail-smtp-addr": {
"title": "Mail SMTP address",
"description": "Mail SMTP address. Default: localhost:25",
"type": "string",
"default": "localhost:25"
},
"mail-smtp-password": {
"title": "Mail SMTP password",
"description": "Mail SMTP password",
"type": "string",
"default": ""
},
"mail-smtp-user": {
"title": "Mail SMTP User",
"description": "Mail SMTP User",
"type": "string",
"default": ""
},
"mail-to": {
"title": "Mail To",
"description": "",
"type": "string",
"default": ""
},
"tags": {
"title": "Provisioning db tags",
"description": "Provisioning db tags",
"type": "string",
"default": "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog"
},
"http-session-lifetime": {
"title": "Web Session life time in s",
"description": "Web interface Session life time in seconds. Default 86400",
"type": "integer",
"default": 86400
},
"http-refresh-interval": {
"title": "Web refresh interval in s",
"description": "Web interface refresh interval in ms",
"type": "integer",
"default": 4
},
"repman-cluster-dict": {
"title": "Replication Manager clusters definition",
"description": "Replication Manager clusters definition",
"patternProperties": {
".*": {
"properties": {
"name": {
"title": "Name of the cluster",
"description": "Name of the cluster: Should not contains spaces or any special characters.",
"type": "string",
"default": ""
},
"database-amount": {
"title": "Amount of databases for cluster",
"description": "Database amount to deploy with this cluster. Minimal amount is 2 required to enable replication.",
"type": "integer",
"default": 2,
"minimum": 2
},
"-sla-0-computer_guid": {
"title": "Prefered Master Computer Guid",
"description": "Computer Guid for prefered Master database.",
"type": "string",
"default": ""
},
"-sla-1-computer_guid": {
"title": "Slave 1 Computer Guid",
"description": "Computer Guid for the first slave database.",
"type": "string",
"default": ""
},
"-sla-2-computer_guid": {
"title": "Slave 2 Computer Guid",
"description": "Computer Guid for the second slave database.",
"type": "string",
"default": ""
},
"-sla-3-computer_guid": {
"title": "Slave 3 Computer Guid",
"description": "Computer Guid for the third slave database.",
"type": "string",
"default": ""
},
"-sla-4-computer_guid": {
"title": "Slave 4 Computer Guid",
"description": "Computer Guid for the fourth slave database.",
"type": "string",
"default": ""
},
"proxysql-user": {
"title": "Proxysql username",
"description": "Proxysql external user, default is 'external'.",
"type": "string",
"default": "external"
},
"proxy-tags": {
"title": "Proxy tags",
"description": "playbook configuration tags. Default: pkg,masterslave,linux,noreadwritesplit",
"type": "string",
"default": "pkg,masterslave,linux,noreadwritesplit"
},
"logical-backup-cron": {
"title": "Mysqldump backup cron definition.",
"description": "Logical backup cron expression represents a set of times, using cron format.",
"type": "string",
"default": "0 1 * * 6"
},
"physical-backup-cron": {
"title": "Mariabackup cron definition",
"description": "Physical backup cron expression represents a set of times, using cron format.",
"type": "string",
"default": "0 0 * * *"
},
"proxy-cpu-cores": {
"title": "Proxy Cpu cores",
"description": "Proxy Cpu cores. Default: 2",
"type": "integer",
"default": 2,
"minimum": 1
},
"proxy-memory": {
"title": "Proxy Memory usage in giga bytes",
"description": "Proxy Memory usage in giga bytes. Default: 1G",
"type": "integer",
"default": 1,
"minimum": 1
},
"db-cpu-cores": {
"title": "Database Cpu cores",
"description": "Database Cpu cores. Default: 2",
"type": "integer",
"default": 2,
"minimum": 1
},
"db-disk-iops": {
"title": "Database Rnd IO/s",
"description": "Rnd IO/s in seconds for micro service VM (default 300).",
"type": "integer",
"default": 300
},
"db-memory": {
"title": "Database memory in M",
"description": "Memory in M for micro service VM (default 256)",
"type": "integer",
"default": 256,
"minimum": 256,
"multipleOf": 256
},
"db-memory-shared-pct": {
"title": "Percent memory shared per buffer",
"description": "Percent memory shared per buffer (default \"threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0\")",
"type": "string",
"default": "threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0"
},
"db-memory-threaded-pct": {
"title": "Percent memory allocted per threads",
"description": "Percent memory allocted per threads. (default \"tmp:70,join:20,sort:10\")",
"type": "string",
"default": "tmp:70,join:20,sort:10"
},
"innodb-file-per-table": {
"title": "enable Innodb file per table",
"description": "enable Innodb file per table. Possible value: 0=disabled, 1=enabled",
"type": "integer",
"default": 1,
"minimum": 0,
"maximum": 1
},
"use-ipv6": {
"title": "Mariadb server listen on IPv6",
"description": "Listen on IPv6 instead of IPv4.",
"type": "boolean",
"default": true
},
"failover-mode": {
"title": "Failover mode",
"description": "Failover is manual or automatic (default \"automatic\").",
"type": "string",
"default": "automatic",
"enum": [
"manual",
"automatic"
]
},
"failover-limit": {
"title": "Failover amount limit",
"description": "Failover is canceld if already failover this number of time (0: unlimited) (default 5).",
"type": "integer",
"default": 5
},
"failover-falsepositive-heartbeat": {
"title": "Failover check slaves do not receive heartbeat",
"description": "Failover checks that slaves do not receive heartbeat (default true).",
"type": "boolean",
"default": true
},
"failover-falsepositive-heartbeat-timeout": {
"title": "Failover check slaves do not receive heartbeat timeout",
"description": "Failover checks that slaves do not receive heartbeat detection timeout (default 3).",
"type": "integer",
"default": 3
},
"failover-falsepositive-ping-counter": {
"title": "Failover amount of ping failures",
"description": "Failover after this number of ping failures (interval 1s) (default 5).",
"type": "integer",
"default": 5
},
"failover-max-slave-delay": {
"title": "Failover election ignore slave with replication delay",
"description": "Election ignore slave with replication delay over this time in sec (default 30).",
"type": "integer",
"default": 30
},
"failover-readonly-state": {
"title": "Failover Switchover set slaves as read-only",
"description": "Failover Switchover set slaves as read-only (default true).",
"type": "boolean",
"default": true
},
"failover-restart-unsafe": {
"title": "Failover when cluster down if a slave is start first",
"description": "Failover when cluster down if a slave is start first.",
"type": "boolean",
"default": false
},
"failover-time-limit": {
"title": "Cancel failover if time in sec not passed after previous failover",
"description": "Failover is canceled if timer in sec is not passed with previous failover (0: do not wait).",
"type": "integer",
"default": 0
},
"switchover-at-equal-gtid": {
"title": "Switchover only when slaves are fully in sync",
"description": "Switchover only when slaves are fully in sync.",
"type": "boolean",
"default": false
},
"switchover-slave-wait-catch": {
"title": "Switchover wait for slave to catch with replication",
"description": "Switchover wait for slave to catch with replication, not needed in GTID mode but enable to detect possible issues like witing on old master (default true).",
"type": "boolean",
"default": true
},
"switchover-wait-kill": {
"title": "Switchover wait ms before killing threads on demoted master",
"description": "Switchover wait this many milliseconds before killing threads on demoted master (default 5000).",
"type": "integer",
"default": 5000
},
"switchover-wait-trx": {
"title": "Cancel switchover after timeout seconds if can't aquire FTWRL",
"description": "Switchover is cancel after this timeout in second if can't aquire FTWRL (default 10).",
"type": "integer",
"default": 10
},
"switchover-wait-write-query": {
"title": "Cancel switchover after timeout if a write query is running",
"description": "Switchover is canceled if a write query is running for this time (default 10).",
"type": "integer",
"default": 10
}
},
"type": "object"
}
},
"type": "object"
}
}
}
\ No newline at end of file
software/repman/instance-repman-output-schema.json 0 → 100644
View file @ d0c397f9
{
"name": "Output Parameters",
"properties": {
"backend-url": {
"title": "Backend URL",
"description": "URL used to connect directly to backend without frontend. Requires IPv6.",
"type": "string",
"format": "uri",
"require": true
},
"url": {
"title": "URL",
"description": "URL used to connect to the service.",
"type": "string",
"format": "uri",
"require": false
},
"repman-password": {
"title": "Repman password",
"description": "Password for Replication Manager service.",
"type": "string",
"require": false
}
}
}
software/repman/instance-repman.cfg.jinja2.in
View file @ d0c397f9
......@@ -6,6 +6,8 @@
{% set mariadb_path_list = [] -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog" -%}
{% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -31,11 +33,35 @@ cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
[proxy-admin-port]
recipe = slapos.cookbook:free_port
ip = {{ ipv4 }}
minimum = 6032
maximum = 6132
[proxy-port]
recipe = slapos.cookbook:free_port
ip = {{ ipv4 }}
minimum = 7032
maximum = 7132
{% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
{% do mariadb_dict.__setitem__('innodb-file-per-table', slapparameter_dict.get('innodb-file-per-table', 1)) -%}
{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
"database-amount": 2, "proxysql-user": "external", "proxy-tags": "pkg,masterslave,linux,noreadwritesplit",
"logical-backup-cron": "0 1 * * 6", "physical-backup-cron": "0 0 * * *"}} -%}
{% for name, parameter_dict in slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).items() -%}
{% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
{% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
# Request mariadb instances
{% for i in range(0, slapparameter_dict.get('database-amount', 2)) -%}
{% set db_amount = parameter_dict.get('database-amount', 2) -%}
{% if db_amount < 2 -%}
{% set db_amount = 2 -%}
{% endif -%}
{% for i in range(0, db_amount) -%}
{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
{% set section = 'request-mariadb-' ~ i -%}
{% set name = 'Mariadb-' ~ i -%}
......@@ -44,12 +70,12 @@ partition-id = ${slap-connection:partition-id}
<= request-common
software-type = mariadb
name = {{ name }}
sla-computer_guid = {{ dumps(slapparameter_dict.get('mariadb-computer-guid', '')) }}
sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
{% for key, value in mariadb_dict.items() -%}
config-{{ key }} = {{ dumps(value) }}
{% endfor -%}
config-monitor-passwd = ${publish-early:monitor-password}
config-root-password = ${publish-early:root-password}
config-root-password = ${publish-early:db-root-password}
config-repman-url = ${nginx-parameter:backend-url}
config-repman-secure-url = ${nginx-parameter:backend-url}
config-name = {{ name }}
......@@ -66,14 +92,15 @@ return =
{% endfor -%}
# Manage Replication Manager clusters
{% set cluster_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "", "proxy-admin-port": 6032, "proxy-port": 6033, "proxysql-user": "external"}} -%}
{% for name, parameter_dict in cluster_parameter_dict.items() -%}
{% set prefered_master = parameter_dict.pop("db-prefered-master") -%}
{% set prefered_master = parameter_dict.get("db-prefered-master") -%}
[{{ name ~ '-cluster-parameter' }}]
{% for key, value in parameter_dict.items() -%}
{{ key }} = {{ dumps(value) }}
{% endfor -%}
db-credential = repman:${publish-early:root-password}
proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
proxy-port = ${proxy-port:port}
proxy-admin-port = ${proxy-admin-port:port}
db-credential = repman:${publish-early:db-root-password}
db-list = {{ mariadb_server_list | join(',') }}
partition-list = {{ mariadb_path_list | join(',') }}
{% if prefered_master -%}
......@@ -85,6 +112,32 @@ db-prefered-master = {{ mariadb_server_list[0] }}
proxysql-servers = {{ ipv4 }}
password = ${repman-parameter:password}
proxysql-partition = ${buildout:directory}
proxy-tags = {{ parameter_dict.get("proxy-tags", "pkg,masterslave,linux,noreadwritesplit") }}
logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 1 * * 6") }}
physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }}
proxy-memory = {{ parameter_dict.get("proxy-memory", 1) }}
db-cpu-cores = {{ parameter_dict.get("db-cpu-cores", 2) }}
db-disk-iops = {{ parameter_dict.get("db-disk-iops", 300) }}
db-memory = {{ parameter_dict.get("db-memory", 256) }}
db-memory-shared-pct = {{ parameter_dict.get("db-memory-shared-pct", "threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0") }}
db-memory-threaded-pct = {{ parameter_dict.get("db-memory-threaded-pct", "tmp:70,join:20,sort:10") }}
# failover
failover-mode = {{ parameter_dict.get('failover-mode', 'automatic') }}
failover-limit = {{ parameter_dict.get('failover-limit', 5) }}
failover-falsepositive-heartbeat = {{ parameter_dict.get('failover-falsepositive-heartbeat', True) }}
failover-falsepositive-heartbeat-timeout = {{ parameter_dict.get('failover-falsepositive-heartbeat-timeout', 3) }}
failover-falsepositive-ping-counter = {{ parameter_dict.get('failover-falsepositive-ping-counter', 5) }}
failover-max-slave-delay = {{ parameter_dict.get('failover-max-slave-delay', 30) }}
failover-readonly-state = {{ parameter_dict.get('failover-readonly-state', True) }}
failover-restart-unsafe = {{ parameter_dict.get('failover-restart-unsafe', False) }}
failover-time-limit = {{ parameter_dict.get('failover-time-limit', 0) }}
#switchover
switchover-at-equal-gtid = {{ parameter_dict.get('switchover-at-equal-gtid', False) }}
switchover-slave-wait-catch = {{ parameter_dict.get('switchover-slave-wait-catch', True) }}
switchover-wait-kill = {{ parameter_dict.get('switchover-wait-kill', 5000) }}
switchover-wait-trx = {{ parameter_dict.get('switchover-wait-trx', 10) }}
switchover-wait-write-query = {{ parameter_dict.get('switchover-wait-write-query', 10) }}
[{{ 'config-' ~ name }}]
recipe = slapos.recipe.template:jinja2
......@@ -105,7 +158,7 @@ command =
mkdir -p ${repman:config-tmp}/proxies &&
mkdir -p ${:data} &&
cd ${repman:config-tmp}/proxies &&
{{ curl_bin }} -o proxies.tar.gz ${nginx-parameter:repman-url}/api/clusters/{{ name }}/servers/{{ ipv4 }}/{{ parameter_dict['proxy-admin-port'] }}/config &&
{{ curl_bin }} -o proxies.tar.gz ${nginx-parameter:repman-url}/api/clusters/{{ name }}/servers/{{ ipv4 }}/${proxy-admin-port:port}/config &&
tar -xzf proxies.tar.gz &&
cp conf/proxysql.cnf ${:config}
update-command = ${:command}
......@@ -185,7 +238,7 @@ secure-port = ${instance-parameter:repman-secure-port}
port = ${instance-parameter:repman-port}
ipv4 = ${instance-parameter:ipv4-random}
username = admin
password = ${repman-password:passwd}
password = ${publish-early:repman-password}
cluster-d = ${repman:clusters}
mysql-bin-dir = {{ mariadb_location }}/bin
mysqlbinlog-path = {{ mariadb_location }}/bin/mysqlbinlog
......@@ -193,12 +246,17 @@ mysqlclient-path = {{ mariadb_location }}/bin/mysql
mysqldump-path = {{ mariadb_location }}/bin/mysqldump
haproxy-bin = {{ haproxy_location }}/sbin/haproxy
sysbench-bin = {{ sysbench_location }}/bin/sysbench
restic-bin = {{ restic_bin_location }}
mail-from = {{ slapparameter_dict.get("mail-from", "mrm@localhost") }}
mail-smtp-addr = {{ slapparameter_dict.get("mail-smtp-addr", "localhost:25") }}
mail-smtp-password = {{ slapparameter_dict.get("mail-smtp-password", "") }}
mail-smtp-user = {{ slapparameter_dict.get("mail-smtp-user", "") }}
mail-to = {{ slapparameter_dict.get("mail-to", "") }}
enabled-tags = {{ slapparameter_dict.get("tags", tags) }}
http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }}
http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }}
[repman-config-folder]
recipe = plone.recipe.command
......@@ -208,13 +266,12 @@ command =
{{ rsync_location }}/bin/rsync -av share ${repman:root-dir}/
{{ rsync_location }}/bin/rsync -av dashboard ${repman:root-dir}/
# XXX - this need to be checked/optimized in case of upgrade.
update-command = ${:command}
[replication-manager-reload]
recipe = slapos.recipe.template:jinja2
template = {{ template_repman_manager_sh }}
cluster-list = {{ dumps(cluster_parameter_dict.keys() ) }}
cluster-list = {{ dumps(slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).keys() ) }}
context =
section parameter_dict repman
key username repman-parameter:username
......@@ -245,6 +302,8 @@ wrapper-path = ${directory:service}/replication-manager
depends =
${repman-config-folder:recipe}
${replication-manager-reload:recipe}
${repman-listen-promise:recipe}
${repman-listen-ssl-promise:recipe}
[repman-config.toml]
recipe = slapos.recipe.template:jinja2
......@@ -261,6 +320,13 @@ name = repman_service_listen.py
config-hostname = ${repman-parameter:ipv4}
config-port = ${repman-parameter:port}
[repman-listen-ssl-promise]
<= monitor-promise-base
module = check_port_listening
name = repman_service_ssl_listen.py
config-hostname = ${repman-parameter:ipv4}
config-port = ${repman-parameter:secure-port}
[nginx-conf]
recipe = slapos.recipe.template:jinja2
template = {{ nginx_conf_in }}
......@@ -312,15 +378,16 @@ post = kill -USR1 $(cat ${nginx-parameter:pid-file})
recipe = slapos.cookbook:publish-early
-init =
monitor-password monitor-htpasswd:passwd
root-password gen-root-password:passwd
db-root-password gen-root-password:passwd
repman-password repman-password:passwd
[publish-connection-parameter]
<= monitor-publish
-extends = publish-early
recipe = slapos.cookbook:publish
backend-url = ${nginx-parameter:backend-ssl-url}
url = ${repman-frontend:connection-secure_access}
username = ${repman-parameter:username}
password = ${repman-parameter:password}
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
{% endfor %}
......@@ -342,10 +409,11 @@ password = ${publish-early:monitor-password}
recipe = slapos.cookbook:requestoptional
name = Replication Manager Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
software-url = {{ frontend_parameter_dict.get('frontend-software-url', '') }}
software-type ={{ frontend_parameter_dict.get('frontend-software-type', '') }}
slave = true
config-url = ${nginx-parameter:backend-ssl-url}
config-domain =
config-domain = {{ frontend_parameter_dict.get('slave-domain', '') }}
return = domain secure_access
[repman-frontend-promise]
......@@ -373,7 +441,6 @@ extends =
{{ template_monitor }}
parts =
replication-manager
repman-listen-promise
monitor-base
logrotate-entry-nginx
ca-nginx-service
......
software/repman/instance.cfg.in
View file @ d0c397f9
......@@ -60,6 +60,7 @@ extra-context =
raw config_cluster_toml_in {{ config_cluster_toml_in }}
raw nginx_conf_in {{ nginx_conf_in }}
raw rsync_location {{ rsync_location }}
raw restic_bin_location {{ restic_bin_location }}
raw sysbench_location {{ sysbench_location }}
raw proxysql_location {{ proxysql_location }}
raw template_repman_manager_sh {{ template_repman_manager_sh }}
......
software/repman/software.cfg
View file @ d0c397f9
......@@ -4,6 +4,7 @@ extends =
buildout.hash.cfg
../neoppod/software-common.cfg
../../stack/monitor/buildout.cfg
../../component/restic/buildout.cfg
../../component/replication-manager/buildout.cfg
../../component/mariadb/buildout.cfg
../../component/nginx/buildout.cfg
......@@ -27,6 +28,8 @@ parts =
instance.cfg
template-mariadb.cfg
template-mysqld-wrapper
gowork
restic
[mariadb]
# Compile dir is for plugins, there's no plugin yet
......@@ -62,6 +65,7 @@ context =
key percona_toolkit_location percona-toolkit:location
key repman_src_location git.signal18.io_signal18_repman:location
key rsync_location rsync:location
key restic_bin_location restic:location
key socat_location socat:location
key template_repman_manager_sh repman-manager-sh.in:target
key template_mariadb template-mariadb.cfg:target
......
software/repman/software.cfg.json 0 → 100644
View file @ d0c397f9
{
"name": "Replication Manager",
"description": "Replication Manager",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"description": "Replication Manager",
"request": "instance-repman-input-schema.json",
"response": "instance-repman-output-schema.json",
"index": 0
}
}
}
software/repman/templates/cluster-config.toml.in
View file @ d0c397f9
{% macro setbool(value) -%}
{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
{% endmacro -%}
[{{ parameter_dict['name'] }}]
title = "{{ parameter_dict['name'] }}"
monitoring-save-config = true
db-servers-hosts = "{{ parameter_dict['db-list'] }}"
db-servers-prefered-master = "{{ parameter_dict['db-prefered-master'] }}"
db-servers-credential = "{{ parameter_dict['db-credential'] }}"
......@@ -16,22 +20,48 @@ proxysql-bootstrap = true
proxysql-admin-port = {{ parameter_dict['proxy-admin-port'] }}
proxysql-password = "{{ parameter_dict['password'] }}"
prov-proxy-tags = "pkg,masterslave,linux,noreadwritesplit"
prov-proxy-tags = "{{ parameter_dict['proxy-tags'] }}"
monitoring-scheduler = true
scheduler-db-servers-logical-backup = true
scheduler-db-servers-logical-backup-cron= "0 0 1 * * 6"
scheduler-db-servers-logical-backup-cron = "0 {{ parameter_dict['logical-backup-cron'] }}"
scheduler-db-servers-logs = true
scheduler-db-servers-logs-cron = "0 0 * * * *"
#scheduler-db-servers-logs-table-keep = 12
scheduler-db-servers-logs-table-keep = 4
scheduler-db-servers-logs-table-rotate = true
#scheduler-db-servers-logs-table-rotate-cron = "0 0 0/6 * * *"
scheduler-db-servers-logs-table-rotate-cron = "0 0 23 * * *"
scheduler-db-servers-optimize = true
scheduler-db-servers-optimize-cron = "0 0 3 1 * 5"
scheduler-db-servers-physical-backup = true
scheduler-db-servers-physical-backup-cron = "0 0 0 * * *"
scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backup-cron'] }}"
backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump"
# scheduler-db-servers-receiver-ports="4444,4445"
prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
prov-db-cpu-cores = {{ parameter_dict['db-cpu-cores'] }}
prov-db-disk-iops = {{ parameter_dict['db-disk-iops'] }}
prov-db-memory = {{ parameter_dict['db-memory'] }}
prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
test-inject-traffic = true
# failover
failover-mode = "{{ parameter_dict['failover-mode'] }}"
failover-limit = {{ parameter_dict['failover-limit'] }}
failover-falsepositive-heartbeat = {{ setbool(parameter_dict['failover-falsepositive-heartbeat']) }}
failover-falsepositive-heartbeat-timeout = {{ parameter_dict['failover-falsepositive-heartbeat-timeout'] }}
failover-falsepositive-ping-counter = {{ parameter_dict['failover-falsepositive-ping-counter'] }}
failover-max-slave-delay = {{ parameter_dict['failover-max-slave-delay'] }}
failover-readonly-state = {{ setbool(parameter_dict['failover-readonly-state']) }}
failover-restart-unsafe = {{ setbool(parameter_dict['failover-restart-unsafe']) }}
failover-time-limit = {{ parameter_dict['failover-time-limit'] }}
#switchover
switchover-at-equal-gtid = {{ setbool(parameter_dict['switchover-at-equal-gtid']) }}
switchover-slave-wait-catch = {{ setbool(parameter_dict['switchover-slave-wait-catch']) }}
switchover-wait-kill = {{ parameter_dict['switchover-wait-kill'] }}
switchover-wait-trx = {{ parameter_dict['switchover-wait-trx'] }}
switchover-wait-write-query = {{ parameter_dict['switchover-wait-write-query'] }}
software/repman/templates/config.toml.in
View file @ d0c397f9
[Default]
api-bind = "{{ parameter_dict['ipv4'] }}"
http-bind-address = "{{ parameter_dict['ipv4'] }}"
http-server = true
http-session-lifetime = {{ parameter_dict['http-session-lifetime'] }}
http-refresh-interval = {{ int(parameter_dict['http-refresh-interval'])*1000 }}
monitoring-save-config = false
api-https-bind = true
api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
......@@ -10,11 +13,10 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
# Database list of hosts to ignore in election
#db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
# XXX - check if mandatory (as present in cluster)
#db-servers-hosts =
monitoring-address = "{{ parameter_dict['ipv4'] }}"
#haproxy = true
haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
#haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
# HaProxy input bind address for read (default "0.0.0.0")
#haproxy-ip-read-bind =
# HaProxy input bind address for write (default "0.0.0.0")
......@@ -28,6 +30,17 @@ haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
#HaProxy read-write port to leader (default 3306)
#haproxy-write-port =
# Use restic to archive and restore backups
backup = true
backup-restic = true
backup-restic-binary-path = "{{ parameter_dict['restic-bin'] }}"
backup-restic-aws = false
backup-restic-password = "{{ parameter_dict['password'] }}"
backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
# Mail configuration
# Alert email sender (default "mrm@localhost")
mail-from = "{{ parameter_dict['mail-from'] }}"
......@@ -38,12 +51,8 @@ mail-smtp-user = "{{ parameter_dict['mail-smtp-user'] }}"
# Alert email recipients, separated by commas
mail-to = "{{ parameter_dict['mail-to'] }}"
mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
prov-orchestrator = "slapos"
prov-db-tags="gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog"
prov-db-tags = "{{ parameter_dict['enabled-tags'] }}"
sysbench-binary-path = "{{ parameter_dict['sysbench-bin'] }}"
# Number of threads to run benchmark (default 4)
......
software/repman/templates/mariadb_init_root.sql.in
View file @ d0c397f9
......@@ -6,5 +6,7 @@ CREATE USER 'repman'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}
GRANT ALL ON *.* TO 'repman'@'localhost' WITH GRANT OPTION ;
CREATE USER 'repman'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO 'repman'@'%' WITH GRANT OPTION ;
CREATE USER 'repman'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO 'repman'@'::' WITH GRANT OPTION ;
DROP DATABASE IF EXISTS test ;
FLUSH PRIVILEGES ;
\ No newline at end of file
software/repman/templates/mariadb_initial_setup.sql.in
View file @ d0c397f9
......@@ -14,6 +14,7 @@ CREATE DATABASE IF NOT EXISTS `{{ name }}`;
{% if user -%}
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
{%- endif %}
{% endmacro -%}
......
software/repman/templates/repman-manager.sh.in
View file @ d0c397f9
......@@ -17,14 +17,26 @@ wait_database () {
else
if [ $CODE -eq 200 ]; then
break;
else:
echo ">> [$retry] waitdatabases returned code $CODE...";
fi
echo ">> [$retry] waitdatabases returned code $CODE...";
fi
sleep 1
done
}
activate_proxy () {
NAME=$1
URL="{{ secure_url }}/api/clusters/$NAME/settings/actions/switch/database-hearbeat"
echo ">> Calling $URL...";
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" $URL)
if [ $CODE -eq 200 ]; then
return 0;
else
echo ">> ERROR: failed to activate proxy: $URL returned code $CODE"
return 1;
fi
}
TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login | {{ jq_bin }} -r '.token')
# Always reload cluster configuration to apply recent changes
......@@ -45,10 +57,14 @@ if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
curl -H "Authorization: Bearer ${TOKEN}" \
{{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave)
SUCCESS=0
if [ $CODE -eq 200 ]; then
activate_proxy {{ name }}
if [ $? -eq 0 ]; then
# Mark boostrap done!
echo "Cluster {{ name }} replication bootstrapped"
echo "DO NOT REMOVE THIS FILE" > {{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped
fi
else
echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE"
fi
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7