Commit b91c7cd8 authored by Alain Takoudjou's avatar Alain Takoudjou

repman: check start and stop needed for all proxysql and mariadb services

parent fca591cd
...@@ -14,15 +14,15 @@ ...@@ -14,15 +14,15 @@
# not need these here). # not need these here).
[instance.cfg] [instance.cfg]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 8f3e5d9c0de16006cb36f0b8121fc11a md5sum = da50540b1c0fc69ffbf8f6e345a3baad
[instance-repman.cfg] [instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in _update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 2688b8aa946d4878d3f1c4fa08375cfe md5sum = 3b858324e8940a08379956bf76fb88bd
[config-toml.in] [config-toml.in]
_update_hash_filename_ = templates/config.toml.in _update_hash_filename_ = templates/config.toml.in
md5sum = 1efa65af63b14ca2dd1b8c692b2af80e md5sum = 3adc842de5c0e8c94e90ac5f41ecab6c
[config-cluster-toml.in] [config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in _update_hash_filename_ = templates/cluster-config.toml.in
...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30 ...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg] [template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = 10dea9ec48ce889315202d112a66f121 md5sum = 0c6e9a3220cf6e8b096e523d0d1f407a
[template-my-cnf] [template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in _update_hash_filename_ = templates/my.cnf.in
...@@ -59,3 +59,11 @@ md5sum = 99324b56192003254081ef336dcee94c ...@@ -59,3 +59,11 @@ md5sum = 99324b56192003254081ef336dcee94c
[dbjobs-in] [dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in _update_hash_filename_ = templates/dbjobs.in
md5sum = d623a4c684578602b9d8ee49034aebfa md5sum = d623a4c684578602b9d8ee49034aebfa
[mysqld-need-start.sh.in]
_update_hash_filename_ = templates/mysqld-need-start.sh.in
md5sum = 76b984ef5e37248bc3a8010a74b24de5
[proxy-need-start-stop.sh.in]
_update_hash_filename_ = templates/proxy-need-start-stop.sh.in
md5sum = 1ee3d69f0866605b33eac7ea596b21c3
...@@ -275,16 +275,38 @@ command = ${dbjobs-executable:rendered} ...@@ -275,16 +275,38 @@ command = ${dbjobs-executable:rendered}
< = jinja2-template-executable < = jinja2-template-executable
rendered = ${directory:bin}/mysqld_restart rendered = ${directory:bin}/mysqld_restart
template = inline:#!/bin/sh template = inline:#!/bin/sh
${mariadb-controller-bin:wrapper-path} status
${mariadb-controller-bin:wrapper-path} restart mariadb ${mariadb-controller-bin:wrapper-path} restart mariadb
sleep 1 sleep 1
${mariadb-controller-bin:wrapper-path} status ${mariadb-controller-bin:wrapper-path} status
[{{ section('mariadb-need-start') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = mariadb-need-start
frequency = * * * * *
command = ${template-mysqld-need-start:rendered}
[template-mysqld-need-start]
< = jinja2-template-executable
rendered = ${directory:bin}/mysqld_need_start
template = {{ parameter_dict['template-mysqld-need-start'] }}
context =
key mariadb_controller mariadb-controller-bin:wrapper-path
raw username {{ slapparameter_dict['repman-user'] }}
raw repman_url {{ slapparameter_dict['repman-url'] }}
raw jq_bin {{ jq_bin }}
raw cluster {{ slapparameter_dict['cluster'] }}
raw db_host {{ host }}
raw db_port {{ port }}
raw bash_bin {{ bash_bin }}
raw curl_bin {{ curl_bin }}
# Donwnload mariadb configuration from repman # Donwnload mariadb configuration from repman
[{{ section('install-mysql-config') }}] [{{ section('install-mysql-config') }}]
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
# XXX - for now, we make only one cluster cluster = {{ slapparameter_dict['cluster'] }}
cluster = cluster1
config = ${directory:etc}/mysql/my.cnf config = ${directory:etc}/mysql/my.cnf
command = command =
cd ${directory:config-tmp} && cd ${directory:config-tmp} &&
......
...@@ -237,9 +237,9 @@ ...@@ -237,9 +237,9 @@
}, },
"failover-mode": { "failover-mode": {
"title": "Failover mode", "title": "Failover mode",
"description": "Failover is manual or automatic (default \"automatic\").", "description": "Failover is manual or automatic (default \"manual\").",
"type": "string", "type": "string",
"default": "automatic", "default": "manual",
"enum": [ "enum": [
"manual", "manual",
"automatic" "automatic"
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
{% set mariadb_path_list = [] -%} {% set mariadb_path_list = [] -%}
{% set ip = (ipv6_set | list)[0] -%} {% set ip = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%} {% set ipv4 = (ipv4_set | list)[0] -%}
{% set cluster_list = [] -%}
{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog" -%} {% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog" -%}
{% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%} {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
...@@ -17,12 +18,17 @@ var = ${:home}/var ...@@ -17,12 +18,17 @@ var = ${:home}/var
run = ${:var}/run run = ${:var}/run
scripts = ${:etc}/run scripts = ${:etc}/run
service = ${:etc}/service service = ${:etc}/service
controller = ${:etc}/controller
promise = ${:etc}/promise promise = ${:etc}/promise
log = ${:var}/log log = ${:var}/log
data = ${:var}/lib data = ${:var}/lib
nginx-prefix = ${:var}/nginx nginx-prefix = ${:var}/nginx
tmp = ${:home}/tmp tmp = ${:home}/tmp
{% import "supervisord_lib" as supervisord_lib with context %}
{% set proxysql_controller = "proxysql-controller" -%}
{{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
{% do part_list.append("proxysql-controller-service") -%}
[request-common] [request-common]
recipe = slapos.cookbook:request.serialised recipe = slapos.cookbook:request.serialised
...@@ -33,18 +39,6 @@ cert-file = ${slap-connection:cert-file} ...@@ -33,18 +39,6 @@ cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id} computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id} partition-id = ${slap-connection:partition-id}
[proxy-admin-port]
recipe = slapos.cookbook:free_port
ip = {{ ipv4 }}
minimum = 6032
maximum = 6132
[proxy-port]
recipe = slapos.cookbook:free_port
ip = {{ ipv4 }}
minimum = 7032
maximum = 7132
{% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%} {% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "", {% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
...@@ -64,21 +58,24 @@ maximum = 7132 ...@@ -64,21 +58,24 @@ maximum = 7132
{% for i in range(0, db_amount) -%} {% for i in range(0, db_amount) -%}
{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%} {% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
{% set section = 'request-mariadb-' ~ i -%} {% set section = 'request-mariadb-' ~ i -%}
{% set name = 'Mariadb-' ~ i -%} {% set dbname = 'Mariadb-' ~ i -%}
[{{ section }}] [{{ section }}]
<= request-common <= request-common
software-type = mariadb software-type = mariadb
name = {{ name }} name = {{ dbname }}
sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }} sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
{% for key, value in mariadb_dict.items() -%} {% for key, value in mariadb_dict.items() -%}
config-{{ key }} = {{ dumps(value) }} config-{{ key }} = {{ dumps(value) }}
{% endfor -%} {% endfor -%}
config-monitor-passwd = ${publish-early:monitor-password} config-monitor-passwd = ${publish-early:monitor-password}
config-root-password = ${publish-early:db-root-password} config-root-password = ${publish-early:db-root-password}
config-repman-user = ${repman-parameter:username}
#config-repman-passwd = ${repman-parameter:password}
config-repman-url = ${nginx-parameter:backend-url} config-repman-url = ${nginx-parameter:backend-url}
config-repman-secure-url = ${nginx-parameter:backend-url} config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
config-name = {{ name }} config-cluster = {{ name }}
config-name = {{ dbname }}
return = return =
database-host database-host
monitor-base-url monitor-base-url
...@@ -92,14 +89,26 @@ return = ...@@ -92,14 +89,26 @@ return =
{% endfor -%} {% endfor -%}
# Manage Replication Manager clusters # Manage Replication Manager clusters
[{{name}}-admin-port]
recipe = slapos.cookbook:free_port
ip = {{ ipv4 }}
minimum = 6032
maximum = 6132
[{{name}}-port]
recipe = slapos.cookbook:free_port
ip = {{ ipv4 }}
minimum = 7032
maximum = 7132
{% set prefered_master = parameter_dict.get("db-prefered-master") -%} {% set prefered_master = parameter_dict.get("db-prefered-master") -%}
[{{ name ~ '-cluster-parameter' }}] [{{ name ~ '-cluster-parameter' }}]
{% for key, value in parameter_dict.items() -%} {% for key, value in parameter_dict.items() -%}
{{ key }} = {{ dumps(value) }} {{ key }} = {{ dumps(value) }}
{% endfor -%} {% endfor -%}
proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }} proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
proxy-port = ${proxy-port:port} proxy-port = {{ '${' ~ name ~ '-port:port}' }}
proxy-admin-port = ${proxy-admin-port:port} proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
db-credential = repman:${publish-early:db-root-password} db-credential = repman:${publish-early:db-root-password}
db-list = {{ mariadb_server_list | join(',') }} db-list = {{ mariadb_server_list | join(',') }}
partition-list = {{ mariadb_path_list | join(',') }} partition-list = {{ mariadb_path_list | join(',') }}
...@@ -123,7 +132,7 @@ db-memory = {{ parameter_dict.get("db-memory", 256) }} ...@@ -123,7 +132,7 @@ db-memory = {{ parameter_dict.get("db-memory", 256) }}
db-memory-shared-pct = {{ parameter_dict.get("db-memory-shared-pct", "threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0") }} db-memory-shared-pct = {{ parameter_dict.get("db-memory-shared-pct", "threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0") }}
db-memory-threaded-pct = {{ parameter_dict.get("db-memory-threaded-pct", "tmp:70,join:20,sort:10") }} db-memory-threaded-pct = {{ parameter_dict.get("db-memory-threaded-pct", "tmp:70,join:20,sort:10") }}
# failover # failover
failover-mode = {{ parameter_dict.get('failover-mode', 'automatic') }} failover-mode = {{ parameter_dict.get('failover-mode', 'manual') }}
failover-limit = {{ parameter_dict.get('failover-limit', 5) }} failover-limit = {{ parameter_dict.get('failover-limit', 5) }}
failover-falsepositive-heartbeat = {{ parameter_dict.get('failover-falsepositive-heartbeat', True) }} failover-falsepositive-heartbeat = {{ parameter_dict.get('failover-falsepositive-heartbeat', True) }}
failover-falsepositive-heartbeat-timeout = {{ parameter_dict.get('failover-falsepositive-heartbeat-timeout', 3) }} failover-falsepositive-heartbeat-timeout = {{ parameter_dict.get('failover-falsepositive-heartbeat-timeout', 3) }}
...@@ -158,12 +167,12 @@ command = ...@@ -158,12 +167,12 @@ command =
mkdir -p ${repman:config-tmp}/proxies && mkdir -p ${repman:config-tmp}/proxies &&
mkdir -p ${:data} && mkdir -p ${:data} &&
cd ${repman:config-tmp}/proxies && cd ${repman:config-tmp}/proxies &&
{{ curl_bin }} -o proxies.tar.gz ${nginx-parameter:repman-url}/api/clusters/{{ name }}/servers/{{ ipv4 }}/${proxy-admin-port:port}/config && {{ curl_bin }} -o proxies.tar.gz ${nginx-parameter:repman-url}/api/clusters/{{ name }}/servers/{{ ipv4 }}/{{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }}/config &&
tar -xzf proxies.tar.gz && tar -xzf proxies.tar.gz &&
cp conf/proxysql.cnf ${:config} cp conf/proxysql.cnf ${:config}
update-command = ${:command} update-command = ${:command}
[service-proxysql-{{ name }}] [proxysql-{{ name }}-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = command-line =
{{ proxysql_location }}/bin/proxysql -f {{ proxysql_location }}/bin/proxysql -f
...@@ -171,12 +180,19 @@ command-line = ...@@ -171,12 +180,19 @@ command-line =
-D ${config-proxysql-{{ name }}:data} -D ${config-proxysql-{{ name }}:data}
--reload --reload
# -S /tmp/proxysql_admin.sock # -S /tmp/proxysql_admin.sock
wrapper-path = ${directory:service}/proxysql-{{ name }} wrapper-path = ${directory:controller}/proxysql-{{ name }}
wait-for-files = wait-for-files =
${repman:bootstrap}/{{ name }}_bootstrapped ${repman:bootstrap}/{{ name }}_bootstrapped
{% set service_name = "proxysql-" ~ name -%}
{% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
"stopwaitsecs": 60, "environment": []} %}
{{ supervisord_lib.supervisord_program(proxysql_controller, service_name, proxysql_dict) }}
{% do part_list.append(proxysql_controller ~ "-" ~ service_name) %}
{% do part_list.append('config-' ~ name) -%} {% do part_list.append('config-' ~ name) -%}
{% do part_list.append('service-proxysql-' ~ name) -%} {% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
{% endfor -%} {% endfor -%}
[slap-configuration] [slap-configuration]
...@@ -430,6 +446,26 @@ name = check_repman_frontend.py ...@@ -430,6 +446,26 @@ name = check_repman_frontend.py
config-url = ${nginx-parameter:backend-ssl-url} config-url = ${nginx-parameter:backend-ssl-url}
config-check-secure = 1 config-check-secure = 1
[template-proxysql-need-stop-start]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/proxysql_check_stop_start
template = {{ template_proxy_need_stopstart }}
mode = 755
cluster-list = !py![{{ cluster_list | join(', ') }}]
context =
key proxysql_controller {{proxysql_controller}}-bin:wrapper-path
key repman_url nginx-parameter:backend-url
key cluster_list :cluster-list
raw jq_bin {{ jq_bin }}
raw bash_bin {{ bash_bin }}
raw curl_bin {{ curl_bin }}
[proxy-need-stop-start]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = proxysql-need-stop-start
frequency = * * * * *
command = ${template-proxysql-need-stop-start:rendered}
############################# #############################
# #
...@@ -447,6 +483,7 @@ parts = ...@@ -447,6 +483,7 @@ parts =
publish-connection-parameter publish-connection-parameter
repman-frontend-promise repman-frontend-promise
repman-backend-promise repman-backend-promise
proxy-need-stop-start
# Complete parts with sections # Complete parts with sections
{{ part_list | join('\n ') }} {{ part_list | join('\n ') }}
......
...@@ -70,6 +70,7 @@ extra-context = ...@@ -70,6 +70,7 @@ extra-context =
raw sysbench_location {{ sysbench_location }} raw sysbench_location {{ sysbench_location }}
raw proxysql_location {{ proxysql_location }} raw proxysql_location {{ proxysql_location }}
raw template_repman_manager_sh {{ template_repman_manager_sh }} raw template_repman_manager_sh {{ template_repman_manager_sh }}
raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
[template-mariadb-parameters] [template-mariadb-parameters]
bash = {{ bash_location }} bash = {{ bash_location }}
...@@ -81,6 +82,7 @@ template-mariadb-initial-setup = {{ template_mariadb_initial_setup }} ...@@ -81,6 +82,7 @@ template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
template-mariadb-init-root = {{ template_init_root_sql }} template-mariadb-init-root = {{ template_init_root_sql }}
template-init-root-wrapper = {{ template_init_root_wrapper }} template-init-root-wrapper = {{ template_init_root_wrapper }}
template-mysqld-wrapper = {{ template_mysqld_wrapper }} template-mysqld-wrapper = {{ template_mysqld_wrapper }}
template-mysqld-need-start = {{ mysqld_start_template }}
link-binary = {{ dumps(mariadb_link_binary) }} link-binary = {{ dumps(mariadb_link_binary) }}
check-computer-memory-binary = {{ bin_directory }}/check-computer-memory check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
bin-directory = {{ bin_directory }} bin-directory = {{ bin_directory }}
......
...@@ -62,9 +62,11 @@ context = ...@@ -62,9 +62,11 @@ context =
key template_monitor monitor2-template:rendered key template_monitor monitor2-template:rendered
key mariadb_link_binary template-mariadb.cfg:link-binary key mariadb_link_binary template-mariadb.cfg:link-binary
key mariadb_location mariadb:location key mariadb_location mariadb:location
key mysqld_start_template mysqld-need-start.sh.in:target
key nginx_conf_in nginx.conf.in:target key nginx_conf_in nginx.conf.in:target
key nginx_location nginx:location key nginx_location nginx:location
key percona_toolkit_location percona-toolkit:location key percona_toolkit_location percona-toolkit:location
key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
key repman_src_location git.signal18.io_signal18_repman:location key repman_src_location git.signal18.io_signal18_repman:location
key rsync_location rsync:location key rsync_location rsync:location
key restic_bin_location restic:location key restic_bin_location restic:location
...@@ -142,6 +144,12 @@ link-binary = ...@@ -142,6 +144,12 @@ link-binary =
[dbjobs-in] [dbjobs-in]
<= download-file <= download-file
[mysqld-need-start.sh.in]
<= download-file
[proxy-need-start-stop.sh.in]
<= download-file
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions] [versions]
slapos.recipe.template = 4.3 slapos.recipe.template = 4.3
......
...@@ -14,6 +14,7 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}" ...@@ -14,6 +14,7 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
#db-servers-ignored-hosts = #db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
monitoring-address = "{{ parameter_dict['ipv4'] }}" monitoring-address = "{{ parameter_dict['ipv4'] }}"
monitoring-wait-retry = 40
#haproxy = true #haproxy = true
#haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}" #haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
......
#!{{ bash_bin }}
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
# Checking if mariadb start is needed
#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
if [ $CODE -eq 200 ]; then
echo "$CODE: Starting mariadb service..."
# print current status, can be useful for debug...
{{ mariadb_controller }} status mariadb
{{ mariadb_controller }} start mariadb
sleep 5
# check again if the service is still up...
{{ mariadb_controller }} status mariadb
fi
#!{{ bash_bin }}
curl () {
{{ curl_bin }} -k --silent "$@"
}
check_start_cluster () {
NAME=$1
HOST=$2
PORT=$3
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/$NAME/servers/$HOST/$PORT/need-start)
if [ $CODE -eq 200 ]; then
echo "$CODE: Starting proxysql $HOST:$PORT..."
{{ proxysql_controller }} start proxysql-$NAME
sleep 1
# check again if the service is still up...
{{ proxysql_controller }} status proxysql-$NAME
fi
}
check_stop_cluster () {
NAME=$1
HOST=$2
PORT=$3
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/$NAME/servers/$HOST/$PORT/need-stop)
if [ $CODE -eq 200 ]; then
echo "$CODE: Stoping proxysql $HOST:$PORT..."
{{ proxysql_controller }} stop proxysql-$NAME
sleep 1
# check again if the service is stopped...
{{ proxysql_controller }} status proxysql-$NAME
fi
}
{% for cluster_dict in cluster_list -%}
check_start_cluster {{ cluster_dict['name'] }} {{ cluster_dict['host'] }} {{ cluster_dict['port'] }}
check_stop_cluster {{ cluster_dict['name'] }} {{ cluster_dict['host'] }} {{ cluster_dict['port'] }}
{% endfor -%}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment