Commit 6b705beb authored by Alain Takoudjou's avatar Alain Takoudjou

repman: update configs to use more tags by default, fix publish and parameter json schema

parent 9bc09581
......@@ -14,11 +14,11 @@
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = da50540b1c0fc69ffbf8f6e345a3baad
md5sum = b41f521b5f7980c64260ed0e5c494450
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 2b5552d587e7735b4ff6daec24fcb8c1
md5sum = 4ddb88852c1bb89d64ecca2f2a7ada0a
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
......@@ -34,15 +34,15 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = 63af2f4fc184cfc9d09f8b72dee5165a
md5sum = 44c4b22c37adc6b7add3581aee7bbadf
[template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in
md5sum = 04a12d3271857fcf29cb010fd7e38a5c
md5sum = f3661b788099bb31d71ba6e7d36836d9
[template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = c31618ef115ecd72b30ae59d0440a3ec
md5sum = 9be53e2e92333b93e92556b8a01d9c42
[mariadb-init-root-sql]
_update_hash_filename_ = templates/mariadb_init_root.sql.in
......@@ -54,11 +54,11 @@ md5sum = 3ba02b99c3fcbc24b3659917980b310f
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
md5sum = 99324b56192003254081ef336dcee94c
md5sum = 96fcd571ebf5546957a9a8677a87c129
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
md5sum = 9731eff8918fa3e68198de6b1d45ecfa
md5sum = f273ed3cb2173f36ff8230b04424d4e3
[mysqld-need-start.sh.in]
_update_hash_filename_ = templates/mysqld-need-start.sh.in
......
......@@ -10,7 +10,7 @@
{% set ip = (ipv4_set | list)[0] -%}
{% endif -%}
{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
{% set database_list = slapparameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
{% set database_list = slapparameter_dict['database-list'] -%}
# XXX- TODO: add cron with check db need restart then restart
# API check restard needed: http://repman/api/clusters/{clusterName}/servers/{serverName}/{serverPort}/need-restart
......@@ -26,17 +26,8 @@ receiver-port = ${dbjob-parameter:socat-port}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
server-id gen-server-id:value
{%- set server_id = slapparameter_dict.get('server-id') %}
{%- if server_id %}
server-id = {{ dumps(server_id) }}
{%- endif %}
database-list = {{ dumps(database_list) }}
[gen-server-id]
recipe = slapos.cookbook:random.integer
minimum = {{ dumps(1) }}
maximum = {{ dumps(2**32 - 1) }}
database-list = {{ dumps(database_list) }}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
......@@ -67,14 +58,14 @@ socket = ${directory:run}/mariadb.sock
ip = {{ ip }}
data-directory = ${directory:srv}/mariadb
pid-file = ${directory:run}/mariadb.pid
max-connection-count = {{ dumps(slapparameter_dict.get('max-connection-count', 1000)) }}
plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }}
innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }}
innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }}
innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
server-id = ${publish-early:server-id}
ssl-crt = ${directory:mariadb-ssl}/crt.pem
ssl-key = ${directory:mariadb-ssl}/key.pem
ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
......@@ -95,6 +86,7 @@ mode = 600
[init-script-parameters]
password = {{ slapparameter_dict['root-password'] }}
database-list = {{ dumps(database_list) }}
mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
root-user = repman
[init-script]
......@@ -136,6 +128,7 @@ context =
key datadir my-cnf-parameters:data-directory
key environ :environ
environ =
GRN_PLUGINS_PATH='${my-cnf-parameters:groonga-plugins-path}'
ODBCSYSINI='${directory:etc}'
LD_LIBRARY_PATH=$${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}'{{ parameter_dict['unixodbc-location'] }}/lib'
{%- for variable in slapparameter_dict.get('environment-variables', ()) %}
......@@ -170,12 +163,11 @@ template = inline:{% raw -%}
{%- endraw %}
context = section parameter_dict odbc-ini-text
# XXX - probably not needed anymore, to check
#[logrotate-entry-mariadb]
#< = logrotate-entry-base
#name = mariadb
#log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
#post = "${binary-wrap-mysql:wrapper-path}" -B -e "FLUSH LOGS"
[{{ section('logrotate-entry-mariadb') }}]
< = logrotate-entry-base
name = mariadb
log = ${dbjob-parameter:log-dir}/errors.log ${dbjob-parameter:log-dir}/sql-errors
post = "${binary-wrap-mysql:wrapper-path}" -B -e "FLUSH LOGS"
[{{ section('binary-link') }}]
recipe = slapos.cookbook:symbolic.link
......
......@@ -87,7 +87,7 @@
"title": "Provisioning db tags",
"description": "Provisioning db tags",
"type": "string",
"default": "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog"
"default": "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime,readcommitted,nohandshake"
},
"http-session-lifetime": {
"title": "Web Session life time in s",
......@@ -97,7 +97,7 @@
},
"http-refresh-interval": {
"title": "Web refresh interval in s",
"description": "Web interface refresh interval in ms",
"description": "Web interface refresh interval in s. Default 4s.",
"type": "integer",
"default": 4
},
......
......@@ -5,21 +5,18 @@
"title": "Backend URL",
"description": "URL used to connect directly to backend without frontend. Requires IPv6.",
"type": "string",
"format": "uri",
"require": true
"format": "uri"
},
"url": {
"title": "URL",
"description": "URL used to connect to the service.",
"type": "string",
"format": "uri",
"require": false
"format": "uri"
},
"repman-password": {
"title": "Repman password",
"description": "Password for Replication Manager service.",
"type": "string",
"require": false
"type": "string"
}
}
}
......@@ -8,7 +8,7 @@
{% set ip = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set cluster_list = [] -%}
{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog" -%}
{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime,readcommitted,nohandshake" -%}
{% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
[directory]
......@@ -67,6 +67,7 @@ mode = 755
{% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
{% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
{% set database_list = parameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
# Request mariadb instances
{% set db_amount = parameter_dict.get('database-amount', 2) -%}
{% if db_amount < 2 -%}
......@@ -94,6 +95,7 @@ config-repman-url = ${nginx-parameter:backend-url}
config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
config-cluster = {{ name }}
config-name = {{ dbname }}
config-database-list = {{ dumps(database_list) }}
return =
database-host
receiver-port
......@@ -220,13 +222,26 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
{% set service_name = "proxysql-" ~ name -%}
{% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
"stopwaitsecs": 60, "environment": []} %}
"stopwaitsecs": 60, "environment": [],
"stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
"stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
{{ supervisord_lib.supervisord_program(proxysql_controller, service_name, proxysql_dict) }}
{% do part_list.append(proxysql_controller ~ "-" ~ service_name) %}
{% do part_list.append('config-' ~ name) -%}
{% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
{% set publish_database_list = [] -%}
{% for database in database_list -%}
{% if database.get('user') -%}
{% do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% else -%}
{% do publish_database_list.append("mysql://" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% endif -%}
{% endfor -%}
{% do publish_dict.__setitem__(name ~ '-database-list', "!py!['" ~ publish_database_list | join("', '") ~ "']") -%}
{% endfor -%}
[slap-configuration]
......@@ -256,6 +271,7 @@ proxies = ${:etc}/proxy
proxy-data = ${:data-dir}/proxy
config-tmp = ${directory:tmp}/config
bootstrap = ${:etc}/bootstrap
proxies-log = ${directory:log}/proxy
[nginx-parameter]
ipv6 = ${instance-parameter:ipv6-random}
......
......@@ -91,6 +91,9 @@ unixodbc-location = {{ unixodbc_location }}
curl-location = {{ curl_location }}
dbjobs-template = {{ dbjobs_in }}
socat-location = {{ socat_location }}
mroonga-mariadb-install-sql = {{ mroonga_mariadb_install_sql }}
mroonga-mariadb-plugin-dir = {{ mroonga_mariadb_plugin_dir }}
groonga-plugins-path = {{ groonga_plugin_dir }}:{{ groonga_mysql_normalizer_plugin_dir }}
[template-instance-mariadb.cfg]
<= jinja2-template-base
......
......@@ -2,8 +2,6 @@
extends =
buildout.hash.cfg
../neoppod/software-common.cfg
../../stack/monitor/buildout.cfg
../../component/restic/buildout.cfg
../../component/replication-manager/buildout.cfg
../../component/mariadb/buildout.cfg
......@@ -21,7 +19,7 @@ extends =
../../component/rsync/buildout.cfg
../../stack/supervisord/buildout.cfg
../../stack/monitor/buildout.cfg
../../stack/slapos.cfg
../neoppod/software-common.cfg
parts =
slapos-cookbook
......@@ -31,11 +29,6 @@ parts =
template-mysqld-wrapper
gowork
[mariadb]
# Compile dir is for plugins, there's no plugin yet
keep-compile-dir = false
[instance.cfg]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/instance.cfg
......@@ -63,6 +56,10 @@ context =
key mariadb_link_binary template-mariadb.cfg:link-binary
key mariadb_location mariadb:location
key mysqld_start_template mysqld-need-start.sh.in:target
key mroonga_mariadb_install_sql mroonga-mariadb:install-sql
key mroonga_mariadb_plugin_dir mroonga-mariadb:plugin-dir
key groonga_plugin_dir groonga:groonga-plugin-dir
key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
key nginx_conf_in nginx.conf.in:target
key nginx_location nginx:location
key percona_toolkit_location percona-toolkit:location
......@@ -152,5 +149,5 @@ link-binary =
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.3
slapos.recipe.template = 4.4
rubygemsrecipe = 0.2.2+slapos001
......@@ -5,6 +5,7 @@
"software-type": {
"default": {
"title": "Default",
"serialisation": "json-in-xml",
"description": "Replication Manager",
"request": "instance-repman-input-schema.json",
"response": "instance-repman-output-schema.json",
......
......@@ -58,7 +58,6 @@ partialRestore()
}
kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP6:LISTEN)
for job in "${JOBS[@]}"
do
......@@ -78,7 +77,7 @@ do
reseedmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
;;
flashbackmysqldump)
......@@ -112,11 +111,9 @@ do
;;
error)
cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
> $ERROLOG
;;
slowquery)
cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
> $SLOWLOG
;;
optimize)
mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
......
SET @@SESSION.SQL_LOG_BIN=0;
USE mysql;
DROP FUNCTION IF EXISTS last_insert_grn_id;
DROP FUNCTION IF EXISTS mroonga_snippet;
DROP FUNCTION IF EXISTS mroonga_command;
{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
{% if mroonga %}CREATE FUNCTION last_insert_grn_id RETURNS INTEGER SONAME '{{ mroonga }}';
CREATE FUNCTION mroonga_snippet RETURNS STRING SONAME '{{ mroonga }}';
CREATE FUNCTION mroonga_command RETURNS STRING SONAME '{{ mroonga }}';{% endif %}
{% if mroonga %}
SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
......
......@@ -11,16 +11,9 @@
#sql_mode="NO_ENGINE_SUBSTITUTION"
socket = {{ socket }}
bind_address = {{ parameter_dict['ip'] }}
datadir = {{ parameter_dict['data-directory'] }}
pid_file = {{ parameter_dict['pid-file'] }}
default_time_zone = '+00:00'
plugin_load = ha_mroonga
max_connections = {{ parameter_dict['max-connection-count'] }}
{% set innodb_buffer_pool_size = parameter_dict['innodb-buffer-pool-size'] -%}
{% if innodb_buffer_pool_size %}innodb_buffer_pool_size = {{ innodb_buffer_pool_size }}{% endif %}
{% set innodb_buffer_pool_instances = parameter_dict['innodb-buffer-pool-instances'] -%}
......@@ -33,8 +26,10 @@ max_connections = {{ parameter_dict['max-connection-count'] }}
# very important to allow parallel indexing
# Note: this is compatible with binlog-based incremental backups, because ERP5
# doesn't use "insert ... select" (in any number of queries) pattern.
innodb_locks_unsafe_for_binlog = 1
# innodb_locks_unsafe_for_binlog = 1
#plugin_load = ha_mroonga
plugin-dir = {{ parameter_dict['plugin-directory'] }}
{% if 'ssl-key' in parameter_dict -%}
ssl
......@@ -51,8 +46,6 @@ ssl-cipher = {{ parameter_dict['ssl-cipher'] }}
{%- endif %}
{%- endif %}
#server_id = {{ parameter_dict['server-id'] }}
# Some dangerous settings you may want to uncomment temporarily
# if you only want performance or less disk access.
......@@ -62,10 +55,7 @@ ssl-cipher = {{ parameter_dict['ssl-cipher'] }}
{{x}}innodb_doublewrite = 0
{{x}}sync_frm = 0
# Force utf8 usage
character_set_server = utf8mb4
collation_server = utf8mb4_general_ci
skip_character_set_client_handshake
# skip_character_set_client_handshake
[client]
......
......@@ -8,7 +8,7 @@ curl () {
wait_database () {
NAME=$1
for retry in {1..5}; do
for retry in {1..10}; do
echo ">> Wait until $NAME databases are ready...";
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/$NAME/actions/waitdatabases);
if [ $CODE -eq 504 ]; then
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment