Skip to content

slapos
slapos
Commit 5e03b689 authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

Repman: Replication Manager for SlapOS 

Create a Software Release for Replication Manager 2.1

https://github.com/signal18/replication-manager

See merge request nexedi/slapos!719
parents 7a82aadc 494eb5e1
Show whitespace changes
Inline Side-by-side
Showing with 2786 additions and 2 deletions
component/cmake/buildout.cfg
View file @ 5e03b689
[buildout]
extends =
../ncurses/buildout.cfg
../openssl/buildout.cfg
parts =
cmake
......@@ -11,5 +12,5 @@ shared = true
url = https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz
md5sum = 79bd7e65cd81ea3aa2619484ad6ff25a
environment =
CMAKE_INCLUDE_PATH=${ncurses:location}/include
CMAKE_LIBRARY_PATH=${ncurses:location}/lib
CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib
component/golang/buildout.cfg
View file @ 5e03b689
......@@ -68,6 +68,14 @@ md5sum = 6132109d4050da349eadc9f7b0304ef4
environment-extra =
GOROOT_BOOTSTRAP=${golang14:location}
[golang1.13]
<= golang-common
url = https://dl.google.com/go/go1.13.9.src.tar.gz
md5sum = 4ad8b04f962be93a32f3021e6f35b3b9
# go1.13 needs go1.4 to bootstrap
environment-extra =
GOROOT_BOOTSTRAP=${golang14:location}
# ---- infrastructure to build Go workspaces / projects ----
......
component/libmicrohttpd/buildout.cfg 0 → 100644
View file @ 5e03b689
[buildout]
extends =
../libiconv/buildout.cfg
../gnutls/buildout.cfg
../curl/buildout.cfg
parts =
libmicrohttpd
[libmicrohttpd]
recipe = slapos.recipe.cmmi
url = https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.70.tar.gz
md5sum = dcd6045ecb4ea18c120afedccbd1da74
configure-options =
--with-libcurl=${curl:location}
--with-gnutls=${gnutls:location}
--with-libiconv-prefix=${libiconv:location}
--without-libintl-prefix
component/proxysql/buildout.cfg 0 → 100644
View file @ 5e03b689
[buildout]
extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../libtool/buildout.cfg
../cmake/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../git/buildout.cfg
../openssl/buildout.cfg
../bzip2/buildout.cfg
../perl/buildout.cfg
../gnutls/buildout.cfg
../curl/buildout.cfg
../gnutls/buildout.cfg
../libzip/buildout.cfg
../m4/buildout.cfg
../pcre/buildout.cfg
../jemalloc/buildout.cfg
../libmicrohttpd/buildout.cfg
parts =
proxysql
[proxysql]
recipe = slapos.recipe.cmmi
version = v2.0.12
url = https://github.com/sysown/proxysql/archive/${:version}.tar.gz
md5sum = 70ec17fe73703a25730fdd44b6bc3ef5
prefix = @@LOCATION@@
# Patch installation path for SlapOS
pre-configure =
mkdir -p ${:prefix}/bin ${:prefix}/etc/init.d ${:prefix}/lib/systemd/system
sed -ri "
s#(\s)/usr/bin#\1${:prefix}/bin#g
s#(\s)/etc#\1${:prefix}/etc#g
s#(\s)/usr/lib#\1${:prefix}/lib#
s#(\s)/var/lib#\1${:prefix}/lib#g
s#(\s)useradd#\1echo useradd#g
s#(\s)systemctl#\1echo systemctl#g
s#(\s)chkconfig#\1echo chkconfig#g
s#(\s)update-rc.d#\1echo update-rc.d#g" Makefile
configure-command = true
environment =
GIT_VERSION=${:version}
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${libgcrypt:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
PATH=${m4:location}/bin:${libtool:location}/bin:${libgcrypt:location}/bin:${curl:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${bzip2:location}/bin:${autoconf:location}/bin:${git:location}/bin:${automake:location}/bin:${patch:location}/bin:${cmake:location}/bin:%(PATH)s
CXXFLAGS=-I${openssl:location}/include -I${gnutls:location}/include -I${zlib:location}/include
CFLAGS=-I${gnutls:location}/include
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath=${curl:location}/lib -L${libtool:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${curl:location}/lib -L${pcre:location}/lib -L${jemalloc:location}/lib -L${libmicrohttpd:location}/lib
CMAKE_INCLUDE_PATH=${openssl:location}/include:${gnutls:location}/include:${curl:location}/include:${pcre:location}/include:${jemalloc:location}/include:${libmicrohttpd:location}/include
CMAKE_LIBRARY_PATH=${openssl:location}/lib:${gnutls:location}/lib:${curl:location}/lib:${pcre:location}/lib:${jemalloc:location}/lib:${libmicrohttpd:location}/lib
LIBTOOL=libtool
ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${libtool:location}/share/aclocal
component/replication-manager/buildout.cfg 0 → 100644
View file @ 5e03b689
[buildout]
extends =
../../component/golang/buildout.cfg
parts =
gowork
[gowork]
golang = ${golang1.13:location}
install =
buildflags = -v --tags server --ldflags "-extldflags 'static' -w -s -X main.GoOS=linux -X main.GoArch=amd64 -X main.Version=2.1 -X main.FullVersion=$FULLVERSION -X main.Build=$(date +%FT%T%z) -X main.WithProvisioning=ON -X main.WithOpenSVC=OFF -X main.WithHaproxy=ON -X main.WithMaxscale=ON -X main.WithMariadbshardproxy=ON -X main.WithProxysql=ON -X main.WithSphinx=ON -X main.WithArbitration=OFF -X main.WithArbitrationClient=ON -X main.WithMonitoring=ON -X main.WithHttp=ON -X main.WithBackup=ON -X main.WithMail=ON -X main.WithEnforce=ON -X main.WithDeprecate=ON"
[gowork.goinstall]
depends_gitfetch =
${git.signal18.io_signal18_repman:recipe}
command = set -e
. ${gowork:env.sh}
cd ${git.signal18.io_signal18_repman:location}
export GO111MODULE=on
export FULLVERSION=$(git describe --tags)
go build ${gowork:buildflags} -o ${gowork:bin}/replication-manager
chmod -R u+w .
# Remove binary files provided with replication manager else testnode will complain with 'libXXX => not found'.
rm -f ${git.signal18.io_signal18_repman:location}/share/amd64/darwin/*
rm -f ${git.signal18.io_signal18_repman:location}/share/amd64/linux/*
[git.signal18.io_signal18_repman]
<= go-git-package
go.importpath = github.com/signal18/replication-manager
repository = https://github.com/signal18/replication-manager
branch = 2.1
revision = 9167a82c81af8f7be41cf51bc9be8a37dc3d8c03
\ No newline at end of file
component/restic/buildout.cfg 0 → 100644
View file @ 5e03b689
[buildout]
extends =
../../component/golang/buildout.cfg
parts =
restic
[gowork]
golang = ${golang1.13:location}
[restic]
recipe = plone.recipe.command
update-command = ${:command}
stop-on-error = True
# GO111MODULE=on enables go modules support
# the chmod is needed as modules are fetched with u-w
command =
. ${gowork:env.sh} &&
cd ${git.github.com_restic_restic:location} &&
export GO111MODULE=on &&
go run build.go -o ${:output} &&
chmod -R u+w .
output = ${gowork:bin}/restic
location = ${:output}
[git.github.com_restic_restic]
<= go-git-package
go.importpath = github.com/restic/restic
repository = https://github.com/restic/restic
revision = v0.9.6
\ No newline at end of file
component/socat/buildout.cfg
View file @ 5e03b689
......@@ -7,3 +7,5 @@ recipe = slapos.recipe.cmmi
url = http://www.dest-unreach.org/socat/download/socat-${:version}.tar.gz
version = 1.7.3.2
md5sum = aec3154f7854580cfab0c2d81e910519
environment =
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
\ No newline at end of file
component/sysbench/buildout.cfg 0 → 100644
View file @ 5e03b689
[buildout]
extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../libtool/buildout.cfg
../gettext/buildout.cfg
../m4/buildout.cfg
../mariadb/buildout.cfg
parts =
sysbench
[sysbench]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/akopytov/sysbench/archive/1.0.19.tar.gz
md5sum = 2912bfe7238cac7351459019a84e2557
pre-configure =
aclocal -I${pkgconfig:location}/share/aclocal -I${libtool:location}/share/aclocal -I${gettext:location}/share/aclocal
./autogen.sh
configure-options =
--disable-static
--with-mysql-includes=${mariadb:location}/include/mysql
--with-mysql-libs=${mariadb:location}/lib
environment =
PATH=${m4:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${pkgconfig:location}/bin:${libtool:location}/bin:%(PATH)s
CPPFLAGS=-I${gettext:location}/include -I${mariadb:location}/include
LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${mariadb:location}/lib -Wl,-rpath=${mariadb:location}/lib
ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${gettext:location}/share/aclocal:${libtool:location}/share/aclocal
\ No newline at end of file
software/repman/buildout.hash.cfg 0 → 100644
View file @ 5e03b689
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = b41f521b5f7980c64260ed0e5c494450
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 7dbaace0d7db0e26d582ad17f36ac9cd
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
md5sum = 5cfa75ca5a0048a050c0041dfe541f3d
[config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in
md5sum = d2e79a9435082d9420281b4f59a5d464
[nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in
md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = 189ccee60d0fb53e29431a45e0816bc1
[template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in
md5sum = f3661b788099bb31d71ba6e7d36836d9
[template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = 9be53e2e92333b93e92556b8a01d9c42
[mariadb-init-root-sql]
_update_hash_filename_ = templates/mariadb_init_root.sql.in
md5sum = d927b5d36410bb02717d5ca125525785
[init-root-wrapper-in]
_update_hash_filename_ = templates/init_root_wrapper.in
md5sum = 83ef59b5afaf4454d368823c33aef9cb
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
md5sum = 50503bec392e31126328f51eadc11634
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
md5sum = d2ebd2ec55bf8489789a52c808729925
[mysqld-need-start.sh.in]
_update_hash_filename_ = templates/mysqld-need-start.sh.in
md5sum = e9bcee5dc1318fe3acda2663472214f5
[proxy-need-start-stop.sh.in]
_update_hash_filename_ = templates/proxy-need-start-stop.sh.in
md5sum = 455aaf369bf5141758dc57f2c0e67b08
software/repman/instance-mariadb.cfg.jinja2.in 0 → 100644
View file @ 5e03b689
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', True) -%}
{% set port = slapparameter_dict['tcp-port'] %}
{% set host = (ipv4_set | list)[0] -%}
{% if use_ipv6 -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set host = '[' ~ ip ~ ']' -%}
{% else -%}
{% set ip = (ipv4_set | list)[0] -%}
{% endif -%}
{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
{% set database_list = slapparameter_dict['database-list'] -%}
# XXX- TODO: add cron with check db need restart then restart
# API check restard needed: http://repman/api/clusters/{clusterName}/servers/{serverName}/{serverPort}/need-restart
[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
-extends = publish-early
database-host = {{ host }}:{{ port }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
partition-path = ${buildout:directory}
receiver-port = ${dbjob-parameter:socat-port}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
database-list = {{ dumps(database_list) }}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
mode = 644
[jinja2-template-executable]
< = jinja2-template-base
mode = 755
[simplefile]
< = jinja2-template-base
template = inline:{{ '{{ content }}' }}
{% macro simplefile(section_name, file_path, content, mode='') -%}
{% set content_section_name = section_name ~ '-content' -%}
[{{ content_section_name }}]
content = {{ dumps(content) }}
[{{ section(section_name) }}]
< = simplefile
rendered = {{ file_path }}
context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
[my-cnf-parameters]
socket = ${directory:run}/mariadb.sock
ip = {{ ip }}
data-directory = ${directory:srv}/mariadb
pid-file = ${directory:run}/mariadb.pid
plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }}
innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }}
innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }}
innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
ssl-crt = ${directory:mariadb-ssl}/crt.pem
ssl-key = ${directory:mariadb-ssl}/key.pem
ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
[my-cnf]
< = jinja2-template-base
rendered = ${directory:etc}/mariadb.cnf
template = {{ parameter_dict['template-my-cnf'] }}
context = section parameter_dict my-cnf-parameters
[init-root-sql]
< = jinja2-template-base
rendered = ${directory:etc}/.init-root.sql
template = {{ parameter_dict['template-mariadb-init-root'] }}
context = section parameter_dict init-script-parameters
mode = 600
[init-script-parameters]
password = {{ slapparameter_dict['root-password'] }}
database-list = {{ dumps(database_list) }}
mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
root-user = repman
heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }}
[init-script]
< = jinja2-template-executable
# XXX: is there a better location ?
rendered = ${directory:etc}/mariadb_initial_setup.sql
template = {{ parameter_dict['template-mariadb-initial-setup'] }}
context = section parameter_dict init-script-parameters
[update-mysql]
recipe = slapos.cookbook:generic.mysql.wrap_update_mysql
output = ${directory:bin}/mariadb_update
binary = ${binary-wrap-mysql_upgrade:wrapper-path}
mysql = ${binary-wrap-mysql:wrapper-path}
init-script = ${init-script:rendered}
mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
[{{ section('update-mysql-script') }}]
< = jinja2-template-executable
rendered = ${directory:scripts}/mariadb_update
init-password = ${directory:etc}/.init-passwd.done
upgrade-done = ${directory:lib}/mariadb-update-done
context =
key init_password_done :init-password
key upgrade_done :upgrade-done
key init_root_sql init-root-sql:rendered
key mysql_update update-mysql:output
raw mysql_conf ${directory:etc}/mysql/my.cnf
raw dash_bin {{ dash }}
raw mysql_bin {{ parameter_dict['mariadb-location'] }}/bin/mysql
template = {{ parameter_dict['template-init-root-wrapper'] }}
[mysqld]
< = jinja2-template-executable
rendered = ${directory:bin}/mysqld
template = {{ parameter_dict['template-mysqld-wrapper'] }}
context =
key defaults_file install-mysql-config:config
key datadir my-cnf-parameters:data-directory
key environ :environ
environ =
GRN_PLUGINS_PATH='${my-cnf-parameters:groonga-plugins-path}'
ODBCSYSINI='${directory:etc}'
LD_LIBRARY_PATH=$${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}'{{ parameter_dict['unixodbc-location'] }}/lib'
{%- for variable in slapparameter_dict.get('environment-variables', ()) %}
{{ variable }}
{%- endfor %}
[ca-mysqld]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
key-file = ${my-cnf-parameters:ssl-key}
cert-file = ${my-cnf-parameters:ssl-crt}
executable = ${mysqld:rendered}
wrapper = ${directory:controller}/mariadb
{% import "supervisord_lib" as supervisord_lib with context %}
{{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
{% do part_list.append("supervisord-mariadb-ctl") -%}
{% set maradb_program_dict = {"name": "mariadb", "command": "${ca-mysqld:wrapper}",
"stopwaitsecs": 86400, "environment": [],
"stdout_logfile": "${directory:log}/mariadb_stdout.log",
"stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
{{ supervisord_lib.supervisord_program("mariadb", maradb_program_dict) }}
{% do part_list.append("supervisord-mariadb") %}
[odbc-ini-text]
text = {{ dumps(slapparameter_dict.get('odbc-ini', '').encode('base64')) }}
[{{ section('odbc-ini') }}]
< = jinja2-template-base
rendered = ${directory:etc}/odbc.ini
template = inline:{% raw -%}
{{ parameter_dict['text'].decode('base64') }}
{%- endraw %}
context = section parameter_dict odbc-ini-text
[{{ section('logrotate-entry-mariadb') }}]
< = logrotate-entry-base
name = mariadb
log = ${dbjob-parameter:log-dir}/errors.log ${dbjob-parameter:log-dir}/sql-errors
post = "${binary-wrap-mysql:wrapper-path}" -B -e "FLUSH LOGS"
[{{ section('binary-link') }}]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:bin}
link-binary = {{ dumps(parameter_dict['link-binary']) }}
[binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it.
command-line =
"{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
--defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args}
wrapper-path = ${directory:bin}/${:command}
extra-args =
[binary-wrap-mysql]
<= binary-wrap-base
command = mysql
[binary-wrap-mysqldump]
<= binary-wrap-base
command = mysqldump
[binary-wrap-mysql_upgrade]
<= binary-wrap-base
command = mysql_upgrade
extra-args = --skip-write-binlog
[binary-wrap-mysqladmin]
<= binary-wrap-base
command = mysqladmin
[binary-wrap-mysql_tzinfo_to_sql]
<= binary-wrap-base
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
command = mysql_tzinfo_to_sql
[binary-wrap-pt-digest]
<= binary-wrap-base
command-line = "{{ parameter_dict['percona-tools-location'] }}/bin/${:command}"
command = pt-query-digest
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
scripts = ${:etc}/run
services = ${:etc}/service
controller = ${:etc}/controller
plugin = ${:etc}/plugin
srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp
backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental
mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var
lib = ${:var}/lib
mysql = ${:lib}/mysql
log = ${:var}/log
run = ${:var}/run
config-tmp = ${:tmp}/config
custom = ${directory:etc}/mysql/custom
[dbjob-parameter]
bash-bin = {{ bash_bin }}
db-user = ${init-script-parameters:root-user}
db-password = ${init-script-parameters:password}
mysql-dir = ${directory:mysql}
dbjob-cnf = ${directory:etc}/mysql/my.cnf
log-dir = ${directory:mysql}/.system/logs
tmp-dir = ${directory:tmp}
mysqld-socket = ${my-cnf-parameters:socket}
socat-port = {{ int(port) + 9 }}
restart-script = ${mysqld-restart-script:rendered}
socat-location = {{ parameter_dict['socat-location'] }}
mysql-location = {{ parameter_dict['mariadb-location'] }}
gzip-location = {{ parameter_dict['gzip-location'] }}
ip = {{ ip }}
host = {{ host }}
port = {{ port }}
use-ipv6 = {{ dumps(use_ipv6) }}
[dbjobs-executable]
< = jinja2-template-executable
rendered = ${directory:bin}/dbjobs
context =
section parameter_dict dbjob-parameter
template = {{ parameter_dict['dbjobs-template'] }}
[{{ section('dbjobs-cron-entry') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = dbjobs
frequency = * * * * *
command = ${dbjobs-executable:rendered}
[mysqld-restart-script]
< = jinja2-template-executable
rendered = ${directory:bin}/mysqld_restart
template = inline:#!/bin/sh
# This script is disabled on SlapOS
echo "RESTART FROM dbjbos WAS DISABLED"
[{{ section('mariadb-need-start') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = mariadb-need-start
frequency = * * * * *
command = ${template-mysqld-need-start:rendered}
[template-mysqld-need-start]
< = jinja2-template-executable
rendered = ${directory:bin}/mysqld_need_start
template = {{ parameter_dict['template-mysqld-need-start'] }}
context =
key mariadb_controller mariadb-ctl-bin:wrapper-path
key update_config mysql-get-config:rendered
raw username {{ slapparameter_dict['repman-user'] }}
raw repman_url {{ slapparameter_dict['repman-url'] }}
raw jq_bin {{ jq_bin }}
raw cluster {{ slapparameter_dict['cluster'] }}
raw db_host {{ host }}
raw db_port {{ port }}
raw bash_bin {{ bash_bin }}
raw curl_bin {{ curl_bin }}
# Donwnload mariadb configuration from repman
[mysql-get-config]
< = jinja2-template-executable
rendered = ${directory:bin}/mysqld-update-config
cluster = {{ slapparameter_dict['cluster'] }}
template = inline:#!{{ bash_bin }}
cd ${directory:config-tmp} &&
{{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config
tar -xzf config.tar.gz
cp -r data/.system ${directory:mysql}
rm -rf ${directory:etc}/mysql
cp -r etc/mysql ${directory:etc}
ln -sf ${directory:mysql}/.system ${directory:var}/system
ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom/01_mariadb.cnf
[{{ section('install-mysql-config') }}]
recipe = plone.recipe.command
stop-on-error = true
config = ${directory:etc}/mysql/my.cnf
command = ${mysql-get-config:rendered}
update-command = ${:command}
[dash]
dash = {{ dumps(dash) }}
[{{ section('promise-check-computer-memory') }}]
<= monitor-promise-base
module = check_command_execute
name = check-computer-memory.py
config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
[{{ section('promise') }}]
<= monitor-promise-base
module = check_command_execute
name = mariadb.py
config-command = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "{{ ip }}" "{{ port }}"
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ port + 1 }}
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
[buildout]
extends =
{{ template_monitor }}
parts +=
{{ part_list | join('\n ') }}
software/repman/instance-repman-input-schema.json 0 → 100644
View file @ 5e03b689
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"slave-frontend": {
"title": "Web frontend",
"description": "Front end used to provide web access for internal services at the kvm.",
"properties": {
"slave-domain": {
"title": "Slave frontend domain",
"description": "Unique domain name for this slave frontend.",
"type": "string",
"default": ""
},
"instance-guid": {
"title": "Main Frontend Instance ID",
"description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",
"type": "string",
"default": ""
},
"frontend-software-type": {
"title": "Frontend Software Type",
"description": "Type of the frontend instance, like \"frontend\".",
"type": "string"
},
"frontend-software-url": {
"title": "Frontend Software URL",
"description": "Software Release URL of the frontend instance, like \"http://example.com/path/to/software.cfg\".",
"type": "string",
"format": "uri",
"default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
}
},
"type": "object"
},
"computer-memory-percent-threshold": {
"title": "Computer memory percent threshold.",
"description": "Computer memory percent threshold.",
"type": "int",
"default": 80
},
"monitor-interface-url": {
"title": "Monitor Web Interface URL",
"description": "Give Url of HTML web interface that will be used to render this monitor instance.",
"type": "string",
"format": "uri",
"default": "https://monitor.app.officejs.com"
},
"monitor-cors-domains": {
"title": "Monitor CORS domains",
"description": "List of cors domains separated with space. Needed for ajax query on this monitor instance from a different domain.",
"type": "string",
"default": "monitor.app.officejs.com"
},
"mail-from": {
"title": "Mail From",
"description": "Mail From address",
"type": "string",
"default": ""
},
"mail-smtp-addr": {
"title": "Mail SMTP address",
"description": "Mail SMTP address. Default: localhost:25",
"type": "string",
"default": "localhost:25"
},
"mail-smtp-password": {
"title": "Mail SMTP password",
"description": "Mail SMTP password",
"type": "string",
"default": ""
},
"mail-smtp-user": {
"title": "Mail SMTP User",
"description": "Mail SMTP User",
"type": "string",
"default": ""
},
"mail-to": {
"title": "Mail To",
"description": "",
"type": "string",
"default": ""
},
"tags": {
"title": "Provisioning db tags",
"description": "Provisioning db tags",
"type": "string",
"default": "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime,readcommitted,nohandshake"
},
"http-session-lifetime": {
"title": "Web Session life time in s",
"description": "Web interface Session life time in seconds. Default 86400",
"type": "integer",
"default": 86400
},
"http-refresh-interval": {
"title": "Web refresh interval in s",
"description": "Web interface refresh interval in s. Default 4s.",
"type": "integer",
"default": 4
},
"autorejoin": {
"title": "Automatic rejoin a failed master",
"description": "Automatic rejoin a failed master (default true)",
"type": "boolean",
"default": true
},
"autoseed": {
"title": "Automatic join a standalone node",
"description": "Automatic join a standalone node",
"type": "boolean",
"default": true
},
"repman-cluster-dict": {
"title": "Replication Manager clusters definition",
"description": "Replication Manager clusters definition",
"patternProperties": {
".*": {
"properties": {
"name": {
"title": "Name of the cluster",
"description": "Name of the cluster: Should not contains spaces or any special characters.",
"type": "string",
"default": ""
},
"database-amount": {
"title": "Amount of databases for cluster",
"description": "Database amount to deploy with this cluster. Minimal amount is 2 required to enable replication.",
"type": "integer",
"default": 2,
"minimum": 2
},
"-sla-0-computer_guid": {
"title": "Prefered Master Computer Guid",
"description": "Computer Guid for prefered Master database.",
"type": "string",
"default": ""
},
"-sla-1-computer_guid": {
"title": "Slave 1 Computer Guid",
"description": "Computer Guid for the first slave database.",
"type": "string",
"default": ""
},
"-sla-2-computer_guid": {
"title": "Slave 2 Computer Guid",
"description": "Computer Guid for the second slave database.",
"type": "string",
"default": ""
},
"-sla-3-computer_guid": {
"title": "Slave 3 Computer Guid",
"description": "Computer Guid for the third slave database.",
"type": "string",
"default": ""
},
"-sla-4-computer_guid": {
"title": "Slave 4 Computer Guid",
"description": "Computer Guid for the fourth slave database.",
"type": "string",
"default": ""
},
"proxysql-user": {
"title": "Proxysql username",
"description": "Proxysql external user, default is 'external'.",
"type": "string",
"default": "external"
},
"proxy-tags": {
"title": "Proxy tags",
"description": "playbook configuration tags. Default: pkg,masterslave,linux,noreadwritesplit",
"type": "string",
"default": "pkg,masterslave,linux,noreadwritesplit"
},
"logical-backup-cron": {
"title": "Mysqldump backup cron definition.",
"description": "Logical backup cron expression represents a set of times, using cron format.",
"type": "string",
"default": "0 21 * * *"
},
"physical-backup-cron": {
"title": "Mariabackup cron definition",
"description": "Physical backup cron expression represents a set of times, using cron format.",
"type": "string",
"default": "0 1 * * *"
},
"proxy-cpu-cores": {
"title": "Proxy Cpu cores",
"description": "Proxy Cpu cores. Default: 2",
"type": "integer",
"default": 2,
"minimum": 1
},
"proxy-memory": {
"title": "Proxy Memory usage in giga bytes",
"description": "Proxy Memory usage in giga bytes. Default: 1G",
"type": "integer",
"default": 1,
"minimum": 1
},
"db-cpu-cores": {
"title": "Database Cpu cores",
"description": "Database Cpu cores. Default: 2",
"type": "integer",
"default": 2,
"minimum": 1
},
"db-disk-iops": {
"title": "Database Rnd IO/s",
"description": "Rnd IO/s in seconds for micro service VM (default 300).",
"type": "integer",
"default": 300
},
"db-memory": {
"title": "Database memory in M",
"description": "Memory in M for micro service VM (default 256)",
"type": "integer",
"default": 256,
"minimum": 256,
"multipleOf": 256
},
"db-memory-shared-pct": {
"title": "Percent memory shared per buffer",
"description": "Percent memory shared per buffer (default \"threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0\")",
"type": "string",
"default": "threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0"
},
"db-memory-threaded-pct": {
"title": "Percent memory allocted per threads",
"description": "Percent memory allocted per threads. (default \"tmp:70,join:20,sort:10\")",
"type": "string",
"default": "tmp:70,join:20,sort:10"
},
"innodb-file-per-table": {
"title": "enable Innodb file per table",
"description": "enable Innodb file per table. Possible value: 0=disabled, 1=enabled",
"type": "integer",
"default": 1,
"minimum": 0,
"maximum": 1
},
"use-ipv6": {
"title": "Mariadb server listen on IPv6",
"description": "Listen on IPv6 instead of IPv4.",
"type": "boolean",
"default": true
},
"failover-mode": {
"title": "Failover mode",
"description": "Failover is manual or automatic (default \"manual\").",
"type": "string",
"default": "manual",
"enum": [
"manual",
"automatic"
]
},
"failover-limit": {
"title": "Failover amount limit",
"description": "Failover is canceld if already failover this number of time (0: unlimited) (default 5).",
"type": "integer",
"default": 5
},
"failover-falsepositive-heartbeat": {
"title": "Failover check slaves do not receive heartbeat",
"description": "Failover checks that slaves do not receive heartbeat (default true).",
"type": "boolean",
"default": true
},
"failover-falsepositive-heartbeat-timeout": {
"title": "Failover check slaves do not receive heartbeat timeout",
"description": "Failover checks that slaves do not receive heartbeat detection timeout (default 3).",
"type": "integer",
"default": 3
},
"failover-falsepositive-ping-counter": {
"title": "Failover amount of ping failures",
"description": "Failover after this number of ping failures (interval 1s) (default 5).",
"type": "integer",
"default": 5
},
"failover-max-slave-delay": {
"title": "Failover election ignore slave with replication delay",
"description": "Election ignore slave with replication delay over this time in sec (default 30).",
"type": "integer",
"default": 30
},
"failover-readonly-state": {
"title": "Failover Switchover set slaves as read-only",
"description": "Failover Switchover set slaves as read-only (default true).",
"type": "boolean",
"default": true
},
"failover-restart-unsafe": {
"title": "Failover when cluster down if a slave is start first",
"description": "Failover when cluster down if a slave is start first.",
"type": "boolean",
"default": false
},
"failover-time-limit": {
"title": "Cancel failover if time in sec not passed after previous failover",
"description": "Failover is canceled if timer in sec is not passed with previous failover (0: do not wait).",
"type": "integer",
"default": 0
},
"switchover-at-equal-gtid": {
"title": "Switchover only when slaves are fully in sync",
"description": "Switchover only when slaves are fully in sync.",
"type": "boolean",
"default": false
},
"switchover-slave-wait-catch": {
"title": "Switchover wait for slave to catch with replication",
"description": "Switchover wait for slave to catch with replication, not needed in GTID mode but enable to detect possible issues like witing on old master (default true).",
"type": "boolean",
"default": true
},
"switchover-wait-kill": {
"title": "Switchover wait ms before killing threads on demoted master",
"description": "Switchover wait this many milliseconds before killing threads on demoted master (default 5000).",
"type": "integer",
"default": 5000
},
"switchover-wait-trx": {
"title": "Cancel switchover after timeout seconds if can't aquire FTWRL",
"description": "Switchover is cancel after this timeout in second if can't aquire FTWRL (default 10).",
"type": "integer",
"default": 10
},
"switchover-wait-write-query": {
"title": "Cancel switchover after timeout if a write query is running",
"description": "Switchover is canceled if a write query is running for this time (default 10).",
"type": "integer",
"default": 10
}
},
"type": "object"
}
},
"type": "object"
}
}
}
\ No newline at end of file
software/repman/instance-repman-output-schema.json 0 → 100644
View file @ 5e03b689
{
"name": "Output Parameters",
"properties": {
"backend-url": {
"title": "Backend URL",
"description": "URL used to connect directly to backend without frontend. Requires IPv6.",
"type": "string",
"format": "uri"
},
"url": {
"title": "URL",
"description": "URL used to connect to the service.",
"type": "string",
"format": "uri"
},
"repman-password": {
"title": "Repman password",
"description": "Password for Replication Manager service.",
"type": "string"
}
}
}
software/repman/instance-repman.cfg.jinja2.in 0 → 100644
View file @ 5e03b689
{% set publish_dict = {} -%}
{% set part_list = [] -%}
{% set monitor_base_url_dict = {} -%}
{% set mariadb_dict = {} -%}
{% set mariadb_server_list = [] -%}
{% set receiver_port_list = [] -%}
{% set mariadb_path_list = [] -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set cluster_list = [] -%}
{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime,readcommitted,nohandshake" -%}
{% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
run = ${:var}/run
scripts = ${:etc}/run
service = ${:etc}/service
controller = ${:etc}/controller
promise = ${:etc}/promise
log = ${:var}/log
data = ${:var}/lib
nginx-prefix = ${:var}/nginx
tmp = ${:home}/tmp
{% import "supervisord_lib" as supervisord_lib with context %}
{% set proxysql_controller = "proxysql-ctl" -%}
{{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
{% do part_list.append("supervisord-proxysql-ctl") -%}
[request-common]
recipe = slapos.cookbook:request.serialised
software-url = ${slap-connection:software-release-url}
server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
[download-proxy-config]
recipe = slapos.recipe.template:jinja2
template = inline:#!{{ bash_bin }}
NAME=$1
HOST=$2
PORT=$3
CONFIG=$4
if [ -z "$CONFIG" ]; then
CONFIG="${repman:proxies}/proxysql-$NAME.cnf"
fi
mkdir -p ${repman:config-tmp}/proxies
cd ${repman:config-tmp}/proxies
{{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
tar -xzf proxies-$NAME.tar.gz
cp conf/proxysql.cnf $CONFIG
rendered = ${directory:bin}/update-proxysql-config
mode = 755
{% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
"database-amount": 2, "proxysql-user": "external", "proxy-tags": "pkg,masterslave,linux,noreadwritesplit",
"logical-backup-cron": "0 21 * * *", "physical-backup-cron": "0 1 * * *"}} -%}
{% for name, parameter_dict in slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).items() -%}
{% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
{% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
{% set database_list = parameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
# Request mariadb instances
{% set db_amount = parameter_dict.get('database-amount', 2) -%}
{% if db_amount < 2 -%}
{% set db_amount = 2 -%}
{% endif -%}
{% for i in range(0, db_amount) -%}
{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
{% set section = 'request-mariadb-' ~ i -%}
{% set dbname = 'Mariadb-' ~ i -%}
[{{ section }}]
<= request-common
software-type = mariadb
name = {{ dbname }}
sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
{% for key, value in mariadb_dict.items() -%}
config-{{ key }} = {{ dumps(value) }}
{% endfor -%}
config-monitor-passwd = ${publish-early:monitor-password}
config-root-password = ${publish-early:db-root-password}
config-repman-user = ${repman-parameter:username}
config-heartbeat-user = ${repman-parameter:heartbeat-user}
#config-repman-passwd = ${repman-parameter:password}
config-repman-url = ${nginx-parameter:backend-url}
config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
config-cluster = {{ name }}
config-name = {{ dbname }}
config-database-list = {{ dumps(database_list) }}
return =
database-host
receiver-port
monitor-base-url
partition-path
{% do part_list.append(section) -%}
{% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
{% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
{% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
{% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{% endfor -%}
# Manage Replication Manager clusters
[{{name}}-admin-port]
recipe = slapos.cookbook:free_port
ip = {{ ipv4 }}
minimum = 6032
maximum = 6132
[{{name}}-port]
recipe = slapos.cookbook:free_port
ip = {{ ipv4 }}
minimum = 7032
maximum = 7132
{% set prefered_master = parameter_dict.get("db-prefered-master") -%}
[{{ name ~ '-cluster-parameter' }}]
{% for key, value in parameter_dict.items() -%}
{{ key }} = {{ value }}
{% endfor -%}
proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
proxy-port = {{ '${' ~ name ~ '-port:port}' }}
proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
db-user = repman
db-password = ${publish-early:db-root-password}
db-list = {{ mariadb_server_list | join(',') }}
heartbeat-user = ${repman-parameter:heartbeat-user}
heartbeat-password = ${publish-early:db-root-password}
partition-list = {{ mariadb_path_list | join(',') }}
{% if prefered_master -%}
db-prefered-master = {{ prefered_master }}
{% else -%}
# First database is the prefered master
db-prefered-master = {{ mariadb_server_list[0] }}
{% endif -%}
proxysql-servers = {{ ipv4 }}
proxysql-servers-ipv6 = [{{ ip }}]
password = ${repman-parameter:password}
proxysql-partition = ${buildout:directory}
receiver-port-list = {{ receiver_port_list | join(',') }}
proxy-tags = {{ parameter_dict.get("proxy-tags", "pkg,masterslave,linux,noreadwritesplit") }}
logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }}
proxy-memory = {{ parameter_dict.get("proxy-memory", 1) }}
db-cpu-cores = {{ parameter_dict.get("db-cpu-cores", 2) }}
db-disk-iops = {{ parameter_dict.get("db-disk-iops", 300) }}
db-memory = {{ parameter_dict.get("db-memory", 256) }}
db-memory-shared-pct = {{ parameter_dict.get("db-memory-shared-pct", "threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0") }}
db-memory-threaded-pct = {{ parameter_dict.get("db-memory-threaded-pct", "tmp:70,join:20,sort:10") }}
# failover
failover-mode = {{ parameter_dict.get('failover-mode', 'manual') }}
failover-limit = {{ parameter_dict.get('failover-limit', 5) }}
failover-falsepositive-heartbeat = {{ parameter_dict.get('failover-falsepositive-heartbeat', True) }}
failover-falsepositive-heartbeat-timeout = {{ parameter_dict.get('failover-falsepositive-heartbeat-timeout', 3) }}
failover-falsepositive-ping-counter = {{ parameter_dict.get('failover-falsepositive-ping-counter', 5) }}
failover-max-slave-delay = {{ parameter_dict.get('failover-max-slave-delay', 30) }}
failover-readonly-state = {{ parameter_dict.get('failover-readonly-state', True) }}
failover-restart-unsafe = {{ parameter_dict.get('failover-restart-unsafe', False) }}
failover-time-limit = {{ parameter_dict.get('failover-time-limit', 0) }}
#switchover
switchover-at-equal-gtid = {{ parameter_dict.get('switchover-at-equal-gtid', False) }}
switchover-slave-wait-catch = {{ parameter_dict.get('switchover-slave-wait-catch', True) }}
switchover-wait-kill = {{ parameter_dict.get('switchover-wait-kill', 5000) }}
switchover-wait-trx = {{ parameter_dict.get('switchover-wait-trx', 10) }}
switchover-wait-write-query = {{ parameter_dict.get('switchover-wait-write-query', 10) }}
[{{ 'config-' ~ name }}]
recipe = slapos.recipe.template:jinja2
template = {{ config_cluster_toml_in }}
rendered = ${repman:clusters}/config-{{ name }}.toml
extra-context =
context =
section parameter_dict {{ name ~ '-cluster-parameter' }}
# Donwnload mariadb configuration from repman
[config-proxysql-{{ name }}]
recipe = plone.recipe.command
# if Repman is not started, cannot download config from server
stop-on-error = false
config = ${repman:proxies}/proxysql-{{ name }}.cnf
data = ${repman:proxy-data}/{{ name }}
command =
mkdir -p ${:data} &&
${download-proxy-config:rendered} {{ name }} {{ ipv4 }} {{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }} ${:config}
update-command = ${:command}
[proxysql-{{ name }}-wrapper]
recipe = slapos.cookbook:wrapper
command-line =
{{ proxysql_location }}/bin/proxysql -f
-c ${config-proxysql-{{ name }}:config}
-D ${config-proxysql-{{ name }}:data}
--reload
# -S /tmp/proxysql_admin.sock
wrapper-path = ${directory:controller}/proxysql-{{ name }}
wait-for-files =
${repman:bootstrap}/{{ name }}_bootstrapped
depends =
{{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }}
{{ '${proxysql-' ~ name ~ '-promise:recipe}' }}
{{ '${proxysql-' ~ name ~ '-ipv6-promise:recipe}' }}
[proxysql-{{ name }}-admin-promise]
<= monitor-promise-base
module = check_port_listening
name = proxysql-{{ name }}-admin-port-listening.py
config-hostname= {{ ipv4 }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }}
[proxysql-{{ name }}-promise]
<= monitor-promise-base
module = check_port_listening
name = proxysql-{{ name }}-port-listening.py
config-hostname= {{ ipv4 }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
[proxysql-{{ name }}-ipv6-promise]
<= monitor-promise-base
module = check_port_listening
name = proxysql-{{ name }}-ipv6-port-listening.py
config-hostname= {{ ip }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
{% set service_name = "proxysql-" ~ name -%}
{% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
"stopwaitsecs": 60, "environment": [],
"stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
"stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
{{ supervisord_lib.supervisord_program(service_name, proxysql_dict) }}
{% do part_list.append("supervisord-" ~ service_name) %}
{% do part_list.append('config-' ~ name) -%}
{% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
{% set publish_database_list = [] -%}
{% set publish_database_v6_list = [] -%}
{% for database in database_list -%}
{% if database.get('user') -%}
{% do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% do publish_database_v6_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% else -%}
{% do publish_database_list.append("mysql://" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% do publish_database_v6_list.append("mysql://[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% endif -%}
{% endfor -%}
{% do publish_dict.__setitem__(name ~ '-database-list', "!py!['" ~ publish_database_list | join("', '") ~ "']") -%}
{% do publish_dict.__setitem__(name ~ '-database-list-v6', "!py!['" ~ publish_database_v6_list | join("', '") ~ "']") -%}
{% endfor -%}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[instance-parameter]
<= slap-configuration
# repman monitor seems to use a fixed port
repman-secure-port = 10005
repman-port = 10001
nginx-ssl-port = 10006
nginx-port = 10007
[repman]
recipe = slapos.cookbook:mkdirectory
etc = ${directory:etc}/repman
data-dir = ${directory:var}/lib
root-dir = ${directory:srv}/repman
clusters = ${:etc}/cluster.d
proxies = ${:etc}/proxy
proxy-data = ${:data-dir}/proxy
config-tmp = ${directory:tmp}/config
bootstrap = ${:etc}/bootstrap
proxies-log = ${directory:log}/proxy
[nginx-parameter]
ipv6 = ${instance-parameter:ipv6-random}
port = ${instance-parameter:nginx-port}
ssl-port = ${instance-parameter:nginx-ssl-port}
ssl-certificate = ${ca-nginx:cert-file}
ssl-key = ${ca-nginx:key-file}
pid-file = ${directory:run}/nginx.pid
access-log = ${directory:log}/nginx_access.log
error-log = ${directory:log}/nginx_error.log
repman-secure-url = https://${repman-parameter:ipv4}:${repman-parameter:secure-port}
repman-url = http://${repman-parameter:ipv4}:${repman-parameter:port}
config-file = ${directory:etc}/nginx.conf
backend-ssl-url = https://[${:ipv6}]:${:ssl-port}
backend-url = http://[${:ipv6}]:${:port}
[repman-password]
recipe = slapos.cookbook:generate.password
bytes = 12
[gen-root-password]
recipe = slapos.cookbook:generate.password
bytes = 12
[repman-parameter]
log = ${directory:log}/repman.log
http-root = ${repman:root-dir}/dashboard
share-dir = ${repman:root-dir}/share
secure-port = ${instance-parameter:repman-secure-port}
port = ${instance-parameter:repman-port}
ipv4 = ${instance-parameter:ipv4-random}
username = admin
heartbeat-user = heartbeat
password = ${publish-early:repman-password}
cluster-d = ${repman:clusters}
autorejoin = {{ slapparameter_dict.get("autorejoin", True) }}
autoseed = {{ slapparameter_dict.get("autoseed", True) }}
mysql-bin-dir = {{ mariadb_location }}/bin
mysqlbinlog-path = {{ mariadb_location }}/bin/mysqlbinlog
mysqlclient-path = {{ mariadb_location }}/bin/mysql
mysqldump-path = {{ mariadb_location }}/bin/mysqldump
haproxy-bin = {{ haproxy_location }}/sbin/haproxy
sysbench-bin = {{ sysbench_location }}/bin/sysbench
restic-bin = {{ restic_bin_location }}
mail-from = {{ slapparameter_dict.get("mail-from", "mrm@localhost") }}
mail-smtp-addr = {{ slapparameter_dict.get("mail-smtp-addr", "localhost:25") }}
mail-smtp-password = {{ slapparameter_dict.get("mail-smtp-password", "") }}
mail-smtp-user = {{ slapparameter_dict.get("mail-smtp-user", "") }}
mail-to = {{ slapparameter_dict.get("mail-to", "") }}
enabled-tags = {{ slapparameter_dict.get("tags", tags) }}
http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }}
http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }}
[repman-config-folder]
recipe = plone.recipe.command
repman-location = {{ repman_src_location }}
command =
cd ${:repman-location}
{{ rsync_location }}/bin/rsync -av share ${repman:root-dir}/
{{ rsync_location }}/bin/rsync -av dashboard ${repman:root-dir}/
update-command = ${:command}
[replication-manager-reload]
recipe = slapos.recipe.template:jinja2
template = {{ template_repman_manager_sh }}
cluster-list = {{ dumps(slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).keys() ) }}
context =
section parameter_dict repman
key username repman-parameter:username
key password repman-parameter:password
key secure_url nginx-parameter:backend-ssl-url
key cluster_name_list :cluster-list
raw jq_bin {{ jq_bin }}
raw curl_bin {{ curl_bin }}
raw bash_bin {{ bash_bin }}
rendered = ${directory:scripts}/repman-reload
mode = 755
[replication-manager]
recipe = slapos.cookbook:wrapper
command-line =
{{ gowork_bin }}/replication-manager
--monitoring-basedir=${repman:root-dir}
--monitoring-sharedir=${repman-parameter:share-dir}
--http-root=${repman-parameter:http-root}
--monitoring-datadir=${repman:data-dir}
--config=${repman-config.toml:rendered}
--log-file=${repman-parameter:log}
--memprofile=${directory:tmp}/repmgr.mprof
monitor
wrapper-path = ${directory:service}/replication-manager
# setup repman instance folder
depends =
${repman-config-folder:recipe}
${replication-manager-reload:recipe}
${repman-listen-promise:recipe}
${repman-listen-ssl-promise:recipe}
[repman-config.toml]
recipe = slapos.recipe.template:jinja2
template = {{ config_toml_in }}
rendered = ${repman:etc}/config.toml
extra-context =
context =
section parameter_dict repman-parameter
[repman-listen-promise]
<= monitor-promise-base
module = check_port_listening
name = repman_service_listen.py
config-hostname = ${repman-parameter:ipv4}
config-port = ${repman-parameter:port}
[repman-listen-ssl-promise]
<= monitor-promise-base
module = check_port_listening
name = repman_service_ssl_listen.py
config-hostname = ${repman-parameter:ipv4}
config-port = ${repman-parameter:secure-port}
[nginx-conf]
recipe = slapos.recipe.template:jinja2
template = {{ nginx_conf_in }}
rendered = ${nginx-parameter:config-file}
context =
section parameter_dict nginx-parameter
[nginx-launcher]
recipe = slapos.cookbook:wrapper
command-line =
{{ nginx_bin }}
-p ${directory:nginx-prefix}
-c ${nginx-conf:rendered}
wrapper-path = ${directory:bin}/nginx-start
wait-for-files =
${ca-directory:certs}/nginx.key
${ca-directory:certs}/nginx.crt
${nginx-graceful-wrapper:rendered}
[nginx-graceful-wrapper]
recipe = slapos.recipe.template:jinja2
template = inline:#!{{ bash_bin }}
kill -USR1 "$(cat ${nginx-parameter:pid-file})"
rendered = ${directory:scripts}/nginx-graceful
context =
mode = 755
[ca-nginx]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
cert-file = ${ca-directory:certs}/nginx.crt
key-file = ${ca-directory:certs}/nginx.key
executable = ${nginx-launcher:wrapper-path}
wrapper = ${directory:bin}/ca-nginx
[ca-nginx-service]
recipe = slapos.cookbook:wrapper
command-line = ${ca-nginx:wrapper}
wrapper-path = ${directory:services}/nginx
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
log = ${nginx-parameter:access-log} ${nginx-parameter:error-log}
post = kill -USR1 $(cat ${nginx-parameter:pid-file})
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
monitor-password monitor-htpasswd:passwd
db-root-password gen-root-password:passwd
repman-password repman-password:passwd
[publish-connection-parameter]
<= monitor-publish
-extends = publish-early
recipe = slapos.cookbook:publish
backend-url = ${nginx-parameter:backend-ssl-url}
url = ${repman-frontend:connection-secure_access}
username = ${repman-parameter:username}
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
{% endfor %}
[monitor-instance-parameter]
monitor-httpd-port = 8060
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
username = admin
password = ${publish-early:monitor-password}
[monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }}
{% endfor %}
[repman-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Replication Manager Frontend
# XXX We have hardcoded SR URL here.
software-url = {{ frontend_parameter_dict.get('frontend-software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg') }}
{% if frontend_parameter_dict.get('frontend-software-type', '') -%}
software-type ={{ frontend_parameter_dict['frontend-software-type'] }}
{% endif -%}
slave = true
config-url = ${nginx-parameter:backend-ssl-url}
config-domain = {{ frontend_parameter_dict.get('slave-domain', '') }}
return = domain secure_access
[repman-frontend-promise]
<= monitor-promise-base
module = check_url_available
name = check_repman_frontend.py
config-url = https://${repman-frontend:connection-domain}
config-check-secure = 1
[repman-backend-promise]
<= monitor-promise-base
module = check_url_available
name = check_repman_backend.py
config-url = ${nginx-parameter:backend-ssl-url}
config-check-secure = 1
[template-proxysql-need-stop-start]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/proxysql_check_stop_start
template = {{ template_proxy_need_stopstart }}
mode = 755
cluster-list = !py![{{ cluster_list | join(', ') }}]
context =
key proxysql_controller {{proxysql_controller}}-bin:wrapper-path
key repman_url nginx-parameter:backend-url
key get_proxy_config download-proxy-config:rendered
key cluster_list :cluster-list
raw jq_bin {{ jq_bin }}
raw bash_bin {{ bash_bin }}
raw curl_bin {{ curl_bin }}
[proxy-need-stop-start]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = proxysql-need-stop-start
frequency = * * * * *
command = ${template-proxysql-need-stop-start:rendered}
#############################
#
# Deploy replication-manager instance
#
#############################
[buildout]
extends =
{{ template_monitor }}
parts =
replication-manager
monitor-base
logrotate-entry-nginx
ca-nginx-service
publish-connection-parameter
repman-frontend-promise
repman-backend-promise
proxy-need-stop-start
# Complete parts with sections
{{ part_list | join('\n ') }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
software/repman/instance.cfg.in 0 → 100644
View file @ 5e03b689
[buildout]
parts = switch-softwaretype
eggs-directory = {{ buildout_egg_directory }}
develop-eggs-directory = {{ buildout_develop_directory }}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
default = template-instance-repman.cfg:rendered
RootSoftwareInstance = ${:default}
mariadb = template-instance-mariadb.cfg:rendered
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
mode = 0644
extensions = jinja2.ext.do
rendered= ${buildout:directory}/${:_buildout_section_name_}
supervisord-lib = {{ supervisord_lib }}
import-list =
file supervisord_lib :supervisord-lib
context =
key slapparameter_dict slap-configuration:configuration
key computer_id slap-configuration:computer
key ipv4_set slap-configuration:ipv4
key ipv6_set slap-configuration:ipv6
raw buildout_directory {{ buildout_directory }}
raw buildout_bin_directory {{ buildout_bin_directory }}
raw eggs_directory {{ buildout_egg_directory }}
raw develop_eggs_directory {{ buildout_develop_directory }}
raw mariadb_location {{ mariadb_location }}
raw supervisord_lib {{ supervisord_lib }}
raw supervisord_conf {{ supervisord_conf }}
raw template_monitor {{ template_monitor_cfg }}
# program binaries
raw bash_bin {{ bash_location }}/bin/bash
raw jq_bin {{ jq_location }}/jq
raw curl_bin {{ curl_location }}/bin/curl
${:extra-context}
extra-context =
[template-instance-repman.cfg]
<= jinja2-template-base
template= {{ template_repman_cfg }}
extra-context =
raw gowork_bin {{ gowork_bin }}
raw haproxy_location {{ haproxy_location }}
raw nginx_bin {{ nginx_location }}/sbin/nginx
raw repman_src_location {{ repman_src_location }}
# config files
raw config_toml_in {{ config_toml_in }}
raw config_cluster_toml_in {{ config_cluster_toml_in }}
raw nginx_conf_in {{ nginx_conf_in }}
raw rsync_location {{ rsync_location }}
raw restic_bin_location {{ restic_bin_location }}
raw sysbench_location {{ sysbench_location }}
raw proxysql_location {{ proxysql_location }}
raw template_repman_manager_sh {{ template_repman_manager_sh }}
raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
[template-mariadb-parameters]
bash = {{ bash_location }}
dash-location = {{ dash_location }}
gzip-location = {{ gzip_location }}
mariadb-location = {{ mariadb_location }}
template-my-cnf = {{ template_my_cnf }}
template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
template-mariadb-init-root = {{ template_init_root_sql }}
template-init-root-wrapper = {{ template_init_root_wrapper }}
template-mysqld-wrapper = {{ template_mysqld_wrapper }}
template-mysqld-need-start = {{ mysqld_start_template }}
link-binary = {{ dumps(mariadb_link_binary) }}
check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
bin-directory = {{ bin_directory }}
percona-tools-location = {{ percona_toolkit_location }}
unixodbc-location = {{ unixodbc_location }}
curl-location = {{ curl_location }}
dbjobs-template = {{ dbjobs_in }}
socat-location = {{ socat_location }}
mroonga-mariadb-install-sql = {{ mroonga_mariadb_install_sql }}
mroonga-mariadb-plugin-dir = {{ mroonga_mariadb_plugin_dir }}
groonga-plugins-path = {{ groonga_plugin_dir }}:{{ groonga_mysql_normalizer_plugin_dir }}
[template-instance-mariadb.cfg]
<= jinja2-template-base
template = {{ template_mariadb }}
filename = instance-mariadb.cfg
extra-context =
section parameter_dict template-mariadb-parameters
software/repman/software.cfg 0 → 100644
View file @ 5e03b689
[buildout]
extends =
buildout.hash.cfg
../../component/restic/buildout.cfg
../../component/replication-manager/buildout.cfg
../../component/mariadb/buildout.cfg
../../component/nginx/buildout.cfg
../../component/haproxy/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/percona-toolkit/buildout.cfg
../../component/gzip/buildout.cfg
../../component/sed/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/grep/buildout.cfg
../../component/sysbench/buildout.cfg
../../component/proxysql/buildout.cfg
../../component/socat/buildout.cfg
../../component/rsync/buildout.cfg
../../stack/supervisord/buildout.cfg
../../stack/monitor/buildout.cfg
../neoppod/software-common.cfg
parts =
slapos-cookbook
mroonga-mariadb
instance.cfg
template-mariadb.cfg
template-mysqld-wrapper
gowork
[instance.cfg]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/instance.cfg
template = ${:_profile_base_location_}/${:filename}
mode = 0644
context =
key bash_location bash:location
key bin_directory buildout:bin-directory
key config_toml_in config-toml.in:target
key config_cluster_toml_in config-cluster-toml.in:target
key coreutils_location coreutils:location
key curl_location curl:location
key buildout_egg_directory buildout:eggs-directory
key buildout_develop_directory buildout:develop-eggs-directory
key buildout_directory buildout:directory
key buildout_bin_directory buildout:bin-directory
key dbjobs_in dbjobs-in:target
key dash_location dash:location
key jq_location jq-binary:location
key logrotate_cfg template-logrotate-base:rendered
key gowork_bin gowork:bin
key gzip_location gzip:location
key haproxy_location haproxy:location
key template_monitor monitor2-template:rendered
key mariadb_link_binary template-mariadb.cfg:link-binary
key mariadb_location mariadb:location
key mysqld_start_template mysqld-need-start.sh.in:target
key mroonga_mariadb_install_sql mroonga-mariadb:install-sql
key mroonga_mariadb_plugin_dir mroonga-mariadb:plugin-dir
key groonga_plugin_dir groonga:groonga-plugin-dir
key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
key nginx_conf_in nginx.conf.in:target
key nginx_location nginx:location
key percona_toolkit_location percona-toolkit:location
key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
key repman_src_location git.signal18.io_signal18_repman:location
key rsync_location rsync:location
key restic_bin_location restic:location
key socat_location socat:location
key supervisord_lib supervisord-library:target
key supervisord_conf supervisord-conf:target
key template_repman_manager_sh repman-manager-sh.in:target
key template_mariadb template-mariadb.cfg:target
key template_mariadb_initial_setup template-mariadb-initial-setup:target
key template_monitor_cfg monitor2-template:rendered
key template_my_cnf template-my-cnf:target
key template_mysqld_wrapper template-mysqld-wrapper:rendered
key template_init_root_sql mariadb-init-root-sql:target
key template_init_root_wrapper init-root-wrapper-in:target
key template_repman_cfg instance-repman.cfg:target
key unixodbc_location unixodbc:location
key sysbench_location sysbench:location
key proxysql_location proxysql:location
[jq-binary]
recipe = hexagonit.recipe.download
url = https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
md5sum = 1fffde9f3c7944f063265e9a5e67ae4f
filename = jq
mode = 0755
download-only = true
[download-file]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/${:_buildout_section_name_}
[config-toml.in]
<= download-file
[config-cluster-toml.in]
<= download-file
[instance-repman.cfg]
<= download-file
[repman-manager-sh.in]
<= download-file
[template-mariadb.cfg]
<= download-file
link-binary =
${coreutils:location}/bin/basename
${coreutils:location}/bin/cat
${coreutils:location}/bin/cp
${coreutils:location}/bin/ls
${coreutils:location}/bin/tr
${coreutils:location}/bin/uname
${gettext:location}/lib/gettext/hostname
${grep:location}/bin/grep
${sed:location}/bin/sed
${mariadb:location}/bin/mysqlbinlog
[template-mariadb-initial-setup]
<= download-file
[template-my-cnf]
<= download-file
[mariadb-init-root-sql]
<= download-file
[init-root-wrapper-in]
<= download-file
[nginx.conf.in]
<= download-file
[dbjobs-in]
<= download-file
[mysqld-need-start.sh.in]
<= download-file
[proxy-need-start-stop.sh.in]
<= download-file
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.4
rubygemsrecipe = 0.2.2+slapos001
software/repman/software.cfg.json 0 → 100644
View file @ 5e03b689
{
"name": "Replication Manager",
"description": "Replication Manager",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"serialisation": "json-in-xml",
"description": "Replication Manager",
"request": "instance-repman-input-schema.json",
"response": "instance-repman-output-schema.json",
"index": 0
}
}
}
software/repman/templates/cluster-config.toml.in 0 → 100644
View file @ 5e03b689
{% macro setbool(value) -%}
{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
{% endmacro -%}
[{{ parameter_dict['name'] }}]
title = "{{ parameter_dict['name'] }}"
monitoring-save-config = true
db-servers-hosts = "{{ parameter_dict['db-list'] }}"
db-servers-prefered-master = "{{ parameter_dict['db-prefered-master'] }}"
db-servers-credential = "{{ parameter_dict['db-user'] }}:{{ parameter_dict['db-password'] }}"
replication-credential = "{{ parameter_dict['db-user'] }}:{{ parameter_dict['db-password'] }}"
monitoring-write-heartbeat-credential="{{ parameter_dict['heartbeat-user'] }}:{{ parameter_dict['heartbeat-password'] }}"
db-servers-connect-timeout = 1
slapos-db-partitions = "{{ parameter_dict['partition-list'] }}"
slapos-proxysql-partitions = "{{ parameter_dict['proxysql-partition'] }}"
proxysql = true
proxysql-port = {{ parameter_dict['proxy-port'] }}
proxysql-servers = "{{ parameter_dict['proxysql-servers'] }}"
proxysql-servers-ipv6 = "{{ parameter_dict['proxysql-servers-ipv6'] }}"
proxysql-user = "{{ parameter_dict['proxysql-user'] }}"
proxysql-bootstrap = true
proxysql-admin-port = {{ parameter_dict['proxy-admin-port'] }}
proxysql-password = "{{ parameter_dict['password'] }}"
prov-proxy-tags = "{{ parameter_dict['proxy-tags'] }}"
monitoring-scheduler = true
scheduler-db-servers-logical-backup = true
scheduler-db-servers-logical-backup-cron = "0 {{ parameter_dict['logical-backup-cron'] }}"
scheduler-db-servers-logs = true
scheduler-db-servers-logs-cron = "0 0 23 * * *"
scheduler-db-servers-logs-table-keep = 4
scheduler-db-servers-logs-table-rotate = true
scheduler-db-servers-logs-table-rotate-cron = "0 0 23 * * *"
scheduler-db-servers-optimize = true
scheduler-db-servers-optimize-cron = "0 0 3 1 * 5"
scheduler-db-servers-physical-backup = true
scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backup-cron'] }}"
backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump"
scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
prov-db-cpu-cores = {{ parameter_dict['db-cpu-cores'] }}
prov-db-disk-iops = {{ parameter_dict['db-disk-iops'] }}
prov-db-memory = {{ parameter_dict['db-memory'] }}
prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
test-inject-traffic = true
# failover
failover-mode = "{{ parameter_dict['failover-mode'] }}"
failover-limit = {{ parameter_dict['failover-limit'] }}
failover-falsepositive-heartbeat = {{ setbool(parameter_dict['failover-falsepositive-heartbeat']) }}
failover-falsepositive-heartbeat-timeout = {{ parameter_dict['failover-falsepositive-heartbeat-timeout'] }}
failover-falsepositive-ping-counter = {{ parameter_dict['failover-falsepositive-ping-counter'] }}
failover-max-slave-delay = {{ parameter_dict['failover-max-slave-delay'] }}
failover-readonly-state = {{ setbool(parameter_dict['failover-readonly-state']) }}
failover-restart-unsafe = {{ setbool(parameter_dict['failover-restart-unsafe']) }}
failover-time-limit = {{ parameter_dict['failover-time-limit'] }}
#switchover
switchover-at-equal-gtid = {{ setbool(parameter_dict['switchover-at-equal-gtid']) }}
switchover-slave-wait-catch = {{ setbool(parameter_dict['switchover-slave-wait-catch']) }}
switchover-wait-kill = {{ parameter_dict['switchover-wait-kill'] }}
switchover-wait-trx = {{ parameter_dict['switchover-wait-trx'] }}
switchover-wait-write-query = {{ parameter_dict['switchover-wait-write-query'] }}
software/repman/templates/config.toml.in 0 → 100644
View file @ 5e03b689
{% macro setbool(value) -%}
{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
{% endmacro -%}
[Default]
api-bind = "{{ parameter_dict['ipv4'] }}"
http-bind-address = "{{ parameter_dict['ipv4'] }}"
http-server = true
http-session-lifetime = {{ parameter_dict['http-session-lifetime'] }}
http-refresh-interval = {{ int(parameter_dict['http-refresh-interval'])*1000 }}
monitoring-save-config = false
api-https-bind = true
api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
include = "{{ parameter_dict['cluster-d'] }}"
autorejoin = {{ setbool(parameter_dict['autorejoin']) }}
autoseed = {{ setbool(parameter_dict['autoseed']) }}
{% if parameter_dict['autoseed'] in ['true', 'True'] -%}
autorejoin-logical-backup = true
{% endif -%}
db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
# Database list of hosts to ignore in election
#db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
monitoring-address = "{{ parameter_dict['ipv4'] }}"
monitoring-wait-retry = 40
#haproxy = true
#haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
# HaProxy input bind address for read (default "0.0.0.0")
#haproxy-ip-read-bind =
# HaProxy input bind address for write (default "0.0.0.0")
#haproxy-ip-write-bind =
# HaProxy load balance read port to all nodes (default 3307)
#haproxy-read-port =
# HaProxy hosts (default "127.0.0.1")
#haproxy-servers =
# HaProxy statistics port (default 1988)
#haproxy-stat-port =
#HaProxy read-write port to leader (default 3306)
#haproxy-write-port =
# Use restic to archive and restore backups
backup = true
backup-restic = true
backup-restic-binary-path = "{{ parameter_dict['restic-bin'] }}"
backup-restic-aws = false
backup-restic-password = "{{ parameter_dict['password'] }}"
backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
# Mail configuration
# Alert email sender (default "mrm@localhost")
mail-from = "{{ parameter_dict['mail-from'] }}"
# Alert email SMTP server address, in host:[port] format (default "localhost:25")
mail-smtp-addr = "{{ parameter_dict['mail-smtp-addr'] }}"
mail-smtp-password = "{{ parameter_dict['mail-smtp-password'] }}"
mail-smtp-user = "{{ parameter_dict['mail-smtp-user'] }}"
# Alert email recipients, separated by commas
mail-to = "{{ parameter_dict['mail-to'] }}"
prov-orchestrator = "slapos"
prov-db-tags = "{{ parameter_dict['enabled-tags'] }}"
sysbench-binary-path = "{{ parameter_dict['sysbench-bin'] }}"
# Number of threads to run benchmark (default 4)
sysbench-threads = 4
# Time to run benchmark (default 100)
sysbench-time = 100
sysbench-v1 = true
software/repman/templates/dbjobs.in 0 → 100644
View file @ 5e03b689
#!/bin/bash
USER={{ parameter_dict['db-user'] }}
PASSWORD={{ parameter_dict['db-password'] }}
ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
DATADIR={{ parameter_dict['mysql-dir'] }}/
{% if parameter_dict['use-ipv6'] == True -%}
{% set listen = "TCP6-LISTEN" -%}
{% else -%}
{% set listen = "TCP-LISTEN" -%}
{% endif -%}
export PATH={{ parameter_dict['socat-location'] }}/bin:{{ parameter_dict['mysql-location'] }}/bin:{{ parameter_dict['gzip-location'] }}/bin:$PATH
JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "reseedxtrabackup" "reseedmariabackup" "reseedmysqldump" "flashbackxtrabackup" "flashbackmariadbackup" "flashbackmysqldump" "stop" "start")
doneJob()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
}
pauseJob()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
}
partialRestore()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
done
for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file=/etc/mysql/dbjob.cnf -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done
for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done
done
for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
done
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e"flush privileges;start slave;"
}
kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
for job in "${JOBS[@]}"
do
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N))
ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
ID=($(echo $TASK | awk -F@ '{ print $1 }'))
#purge de past
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
if [ "$ADDRESS" == "" ]; then
echo "No $job needed"
else
echo "Processing $job"
case "$job" in
reseedmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
;;
flashbackmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
;;
reseedmariabackup)
rm -rf $BACKUPDIR
mkdir $BACKUPDIR
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | mbstream -x -C $BACKUPDIR
# mbstream -p, --parallel
mariabackup --prepare --export --target-dir=$BACKUPDIR
partialRestore
;;
flashbackmariadbackup)
rm -rf $BACKUPDIR
mkdir $BACKUPDIR
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | xbstream -x -C $BACKUPDIR
mariabackup --prepare --export --target-dir=$BACKUPDIR
partialRestore
;;
mariabackup)
cd {{ parameter_dict['tmp-dir'] }}
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
error)
cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
slowquery)
cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
optimize)
mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
restart)
{{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out
;;
esac
doneJob
fi
done
software/repman/templates/init_root_wrapper.in 0 → 100644
View file @ 5e03b689
#!/bin/bash
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
--protocol=socket -uroot -hlocalhost $@
}
if [ ! -f "{{ init_password_done }}" ]; then
for i in {30..0}; do
if echo 'SELECT 1' | run_mysql &> /dev/null; then
break
fi
echo 'MySQL init process in progress...'
sleep 1
done
if [ "$i" = 0 ]; then
echo >&2 'MySQL init process failed.'
exit 1
fi
echo "Setting mariabdb root password...";
run_mysql < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
echo "done"
fi
# Run mariadb_upgrade when replication is bootstrapped will break replication topology.
# skip when already upgraded until we have a good solution.
if [ ! -f "{{ upgrade_done }}" ]; then
{{ mysql_update }}
if [ $? -eq 0 ]; then
touch {{ upgrade_done }};
fi
fi
\ No newline at end of file
software/repman/templates/mariadb_init_root.sql.in 0 → 100644
View file @ 5e03b689
-- What's done in this file shouldn't be replicated
-- or products like mysql-fabric won't work
SET @@SESSION.SQL_LOG_BIN=0;
CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ;
DROP DATABASE IF EXISTS test ;
FLUSH PRIVILEGES ;
\ No newline at end of file
software/repman/templates/mariadb_initial_setup.sql.in 0 → 100644
View file @ 5e03b689
SET @@SESSION.SQL_LOG_BIN=0;
USE mysql;
{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
{% if mroonga %}
SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
{% macro database(name, user, password) -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}`;
{% if user -%}
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
{%- endif %}
{% endmacro -%}
{% for entry in parameter_dict['database-list'] -%}
{{ database(entry['name'], entry.get('user'), entry.get('password')) }}
{% endfor -%}
\ No newline at end of file
software/repman/templates/my.cnf.in 0 → 100644
View file @ 5e03b689
{% set socket = parameter_dict['socket'] -%}
# ERP5 buildout my.cnf template based on my-huge.cnf shipped with mysql
# The MySQL server
[mysqld]
# ERP5 by default requires InnoDB storage. MySQL by default fallbacks to using
# different engine, like MyISAM. Such behaviour generates problems only, when
# tables requested as InnoDB are silently created with MyISAM engine.
#
# Loud fail is really required in such case.
# Already present in REPMAN
#sql_mode="NO_ENGINE_SUBSTITUTION"
socket = {{ socket }}
datadir = {{ parameter_dict['data-directory'] }}
pid_file = {{ parameter_dict['pid-file'] }}
{% set innodb_buffer_pool_size = parameter_dict['innodb-buffer-pool-size'] -%}
{% if innodb_buffer_pool_size %}innodb_buffer_pool_size = {{ innodb_buffer_pool_size }}{% endif %}
{% set innodb_buffer_pool_instances = parameter_dict['innodb-buffer-pool-instances'] -%}
{% if innodb_buffer_pool_instances %}innodb_buffer_pool_instances = {{ innodb_buffer_pool_instances }}{% endif %}
{% set innodb_log_file_size = parameter_dict['innodb-log-file-size'] -%}
{% if innodb_log_file_size %} innodb_log_file_size = {{ innodb_log_file_size }}{% endif %}
{% set innodb_log_buffer_size = parameter_dict['innodb-log-buffer-size'] -%}
{% if innodb_log_buffer_size %} innodb_log_buffer_size = {{ innodb_log_buffer_size }}{% endif %}
# very important to allow parallel indexing
# Note: this is compatible with binlog-based incremental backups, because ERP5
# doesn't use "insert ... select" (in any number of queries) pattern.
# innodb_locks_unsafe_for_binlog = 1
#plugin_load = ha_mroonga
plugin-dir = {{ parameter_dict['plugin-directory'] }}
{% if 'ssl-key' in parameter_dict -%}
ssl
ssl-cert = {{ parameter_dict['ssl-crt'] }}
ssl-key = {{ parameter_dict['ssl-key'] }}
{% if 'ssl-ca-crt' in parameter_dict -%}
ssl-ca = {{ parameter_dict['ssl-ca-crt'] }}
{%- endif %}
{% if 'ssl-crl' in parameter_dict -%}
ssl-crl = {{ parameter_dict['ssl-crl'] }}
{%- endif %}
{% if 'ssl-cipher' in parameter_dict -%}
ssl-cipher = {{ parameter_dict['ssl-cipher'] }}
{%- endif %}
{%- endif %}
# Some dangerous settings you may want to uncomment temporarily
# if you only want performance or less disk access.
{% set x = '' if parameter_dict['relaxed-writes'] else '#' -%}
{{x}}innodb_flush_log_at_trx_commit = 0
{{x}}innodb_flush_method = nosync
{{x}}innodb_doublewrite = 0
{{x}}sync_frm = 0
# skip_character_set_client_handshake
[client]
socket = {{ socket }}
user = root
[mysql]
no_auto_rehash
[mysqlhotcopy]
interactive_timeout
[mysqldump]
max_allowed_packet = 128M
software/repman/templates/mysqld-need-start.sh.in 0 → 100644
View file @ 5e03b689
#!{{ bash_bin }}
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
# Checking if mariadb start is needed
#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
if [ $CODE -eq 200 ]; then
echo "$CODE: Updating mysql configuration..."
# update mysql configuration
{{ update_config }}
echo "$CODE: Starting mariadb service..."
# print current status, can be useful for debug...
{{ mariadb_controller }} status mariadb
{{ mariadb_controller }} start mariadb
sleep 5
# check again if the service is still up...
{{ mariadb_controller }} status mariadb
fi
software/repman/templates/nginx.conf.in 0 → 100644
View file @ 5e03b689
pid {{ parameter_dict['pid-file'] }};
error_log {{ parameter_dict['error-log'] }};
daemon off;
events {
worker_connections 1024;
accept_mutex off;
}
http {
default_type application/octet-stream;
access_log {{ parameter_dict['access-log'] }} combined;
client_max_body_size 10M;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['ssl-port'] }} ssl;
server_name _;
ssl_certificate {{ parameter_dict['ssl-certificate'] }};
ssl_certificate_key {{ parameter_dict['ssl-key'] }};
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
keepalive_timeout 90s;
location / {
proxy_redirect off;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_connect_timeout 200;
proxy_pass {{ parameter_dict['repman-secure-url'] }};
}
}
server {
listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['port'] }};
server_name _;
location / {
proxy_redirect off;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_connect_timeout 200;
proxy_pass {{ parameter_dict['repman-url'] }};
}
}
}
software/repman/templates/proxy-need-start-stop.sh.in 0 → 100644
View file @ 5e03b689
#!{{ bash_bin }}
curl () {
{{ curl_bin }} -k --silent "$@"
}
check_start_cluster () {
NAME=$1
HOST=$2
PORT=$3
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/$NAME/servers/$HOST/$PORT/need-start)
if [ $CODE -eq 200 ]; then
echo "$CODE: Starting proxysql $HOST:$PORT..."
{{ proxysql_controller }} start proxysql-$NAME
sleep 1
# check again if the service is still up...
{{ proxysql_controller }} status proxysql-$NAME
fi
}
check_stop_cluster () {
NAME=$1
HOST=$2
PORT=$3
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/$NAME/servers/$HOST/$PORT/need-stop)
if [ $CODE -eq 200 ]; then
echo "$CODE: updating proxysql config $HOST:$PORT..."
{{ get_proxy_config }} $NAME $HOST $PORT
echo "$CODE: Stoping proxysql $HOST:$PORT..."
{{ proxysql_controller }} stop proxysql-$NAME
sleep 1
# check again if the service is stopped...
{{ proxysql_controller }} status proxysql-$NAME
fi
}
{% for cluster_dict in cluster_list -%}
check_start_cluster {{ cluster_dict['name'] }} {{ cluster_dict['host'] }} {{ cluster_dict['port'] }}
check_stop_cluster {{ cluster_dict['name'] }} {{ cluster_dict['host'] }} {{ cluster_dict['port'] }}
{% endfor -%}
software/repman/templates/repman-manager.sh.in 0 → 100644
View file @ 5e03b689
#!{{ bash_bin }}
#set -e
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
get_token () {
curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
}
wait_database () {
NAME=$1
for retry in {1..50}; do
echo ">> Wait until $NAME databases are ready...";
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/$NAME/actions/waitdatabases);
if [ $CODE -eq 504 ]; then
# We have a timeout try again
echo ">> [$retry] Timeout on {{ secure_url }}/api/clusters/$NAME/actions/waitdatabases, trying again...";
else
if [ $CODE -eq 200 ]; then
break;
else
if [ $CODE -eq 401 ]; then
# try again with new token
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
fi
fi
echo ">> [$retry] waitdatabases returned code $CODE...";
fi
sleep 30
done
}
activate_proxy () {
NAME=$1
URL="{{ secure_url }}/api/clusters/$NAME/settings/actions/switch/database-hearbeat"
echo ">> Calling $URL...";
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" $URL)
if [ $CODE -eq 200 ]; then
return 0;
else
echo ">> ERROR: failed to activate proxy: $URL returned code $CODE"
return 1;
fi
}
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
# Always reload cluster configuration to apply recent changes
{% for name in cluster_name_list -%}
# reload {{ name }} settings
echo "Reloading settings for {{ name }}..."
curl -H "Authorization: Bearer ${TOKEN}" \
{{ secure_url }}/api/clusters/{{ name }}/settings/actions/reload
# Start Replication on {{ name }}
if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
wait_database {{ name }}
echo "Bootstrap replication on {{ name }}..."
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
curl -H "Authorization: Bearer ${TOKEN}" \
{{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave)
SUCCESS=0
if [ $CODE -eq 200 ]; then
activate_proxy {{ name }}
if [ $? -eq 0 ]; then
# Mark boostrap done!
echo "Cluster {{ name }} replication bootstrapped"
echo "DO NOT REMOVE THIS FILE" > {{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped
fi
else
echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE"
fi
fi
{% endfor %}
\ No newline at end of file
software/repman/test/README.md 0 → 100644
View file @ 5e03b689
Tests for Replication Manager software release
software/repman/test/setup.py 0 → 100644
View file @ 5e03b689
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.repman'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' Replication Manager",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'supervisor',
'pexpect',
'requests',
],
zip_safe=True,
test_suite='test',
)
software/repman/test/test.py 0 → 100644
View file @ 5e03b689
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import unicode_literals
import os
import textwrap
import logging
import tempfile
import time
from six.moves.urllib.parse import urlparse, urljoin
import pexpect
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestRepman(SlapOSInstanceTestCase):
__partition_reference__ = 'R' # solve path too long for postgresql and unicorn
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
def setUp(self):
self.backend_url = self.computer_partition.getConnectionParameterDict(
)['backend-url']
def test_http_get(self):
resp = requests.get(self.backend_url, verify=False)
self.assertTrue(
resp.status_code in [requests.codes.ok, requests.codes.found])
software/slapos-sr-testing/software.cfg
View file @ 5e03b689
......@@ -148,6 +148,11 @@ setup = ${slapos-repository:location}/software/cloudooo/test/
egg = slapos.test.dream
setup = ${slapos-repository:location}/software/dream/test/
[slapos.test.repman-setup]
<= setup-develop-egg
egg = slapos.test.repman
setup = ${slapos-repository:location}/software/repman/test/
[slapos.core-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.core.git
......@@ -193,6 +198,7 @@ eggs =
${slapos.test.cloudooo-setup:egg}
${slapos.test.dream-setup:egg}
${slapos.test.metabase-setup:egg}
${slapos.test.repman-setup:egg}
${backports.lzma:egg}
entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite
......@@ -259,6 +265,7 @@ extra =
${slapos.test.gitlab-setup:setup}
${slapos.test.cloudooo-setup:setup}
${slapos.test.dream-setup:setup}
${slapos.test.repman-setup:setup}
[versions]
# slapos.core is used from the clone always
......
stack/supervisord/README.rst 0 → 100644
View file @ 5e03b689
Supervisord process manager
How to use
==========
Supervisord stack provides a library which can be called in your instance slapos. This stack can be used to run sub services in a partition.
To use:
* extend ``stack/supervisord/buildout.cfg`` in your software.cfg file.
* provide ``supervisord-library:target`` and ``supervisord-conf:target`` to your instance template which require to use supervisord controller.
* add ``{% import "supervisord" as supervisord with context %}`` to instance template which call supervisord library. See example below:
**software.cfg**
::
[template-instance]
recipe = slapos.recipe.template:jinja2
context =
key buildout_bin_directory buildout:bin-directory
key supervisord supervisord-library:target
key supervisord_conf supervisord-conf:target
**instance.cfg.in**
::
[template-custom-instance.cfg]
recipe = slapos.recipe.template:jinja2
supervisord-lib = {{ supervisord }}
import-list =
file supervisord :supervisord-lib
context =
raw buildout_bin_directory {{ buildout_bin_directory }}
raw supervisord_conf {{ supervisord_conf }}
**custom-instance.cfg**
::
{% import "supervisord" as supervisord with context %}
{{ supervisord.supervisord("custom-controller", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
# add program to service controller
{% set program_dict = {"name": "mariadb", "command": "${mariadb-service:wrapper}",
"stopwaitsecs": 300, "environment": []} %}
{{ supervisord.supervisord_program("mariadb", program_dict) }}
...
[buildout]
parts =
...
supervisord-custom-controller
supervisord-mariadb
Supervisord inside partition
============================
Check supervisord controlled services status:
::
$ instance/slappartXX/bin/custom-controller status
mariadb RUNNING pid 5511, uptime 6:04:54
`supervisord_program` parameters and defaults:
.. code-block:: python
program_dict = {
"name": "NAME",
"command": "WRAPPER_PATH",
"stopwaitsecs": 60,
"environment": ['PATH="/usr/bin/:/partition/bin/:$PATH"', 'MAKEFLAGS="-j2"'],
"autostart": True,
"autorestart": False,
"startsecs": 0,
"startretries": 0,
"stopsignal": "TERM",
"stdout_logfile": "NONE",
"stderr_logfile": "NONE"
}
stack/supervisord/buildout.cfg 0 → 100644
View file @ 5e03b689
[buildout]
extends =
../slapos.cfg
buildout.hash.cfg
parts =
supervisord-conf
supervisord-library
[supervisord-download-base]
recipe = slapos.recipe.build:download
mode = 0644
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[supervisord-eggs]
recipe = zc.recipe.egg
eggs =
${slapos-cookbook:eggs}
supervisor
scripts =
supervisord
supervisorctl
[supervisord-library]
<= supervisord-download-base
filename = supervisord.jinja2.in
depends = ${supervisord-eggs:recipe}
[supervisord-conf]
<= supervisord-download-base
filename = supervisord.conf.in
stack/supervisord/buildout.hash.cfg 0 → 100644
View file @ 5e03b689
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[supervisord-library]
_update_hash_filename_ = supervisord.jinja2.in
md5sum = 163c9f60e4ad3842162cbb11d771b7b8
[supervisord-conf]
_update_hash_filename_ = supervisord.conf.in
md5sum = d624f65151233493c6dbdafa83ae8cbd
stack/supervisord/supervisord.conf.in 0 → 100644
View file @ 5e03b689
[unix_http_server]
file = {{ parameter_dict['socket-path'] }}
chmod=0700
[include]
files = {{ parameter_dict['include-dir'] }}/*.conf
[supervisorctl]
serverurl = unix://{{ parameter_dict['socket-path'] }}
[supervisord]
loglevel = {{ parameter_dict['log-level'] }}
logfile_maxbytes = 2MB
nodaemon = false
logfile-backups = 3
logfile = {{ parameter_dict['log-file'] }}
pidfile = {{ parameter_dict['pid-file'] }}
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
stack/supervisord/supervisord.jinja2.in 0 → 100644
View file @ 5e03b689
{% macro supervisord(
name,
buildout_bin_directory,
supervisord_conf,
use_service_hash=False
) -%}
[controller-directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
supervisord = ${:etc}/supervisord-{{ name }}.conf.d
[controller-parameters]
socket-path = ${controller-directory:run}/{{ name }}.socket
include-dir = ${controller-directory:supervisord}
log-file = ${controller-directory:log}/supervisord-{{ name }}.log
log-level = info
pid-file = ${controller-directory:run}/supervisord-{{ name }}.pid
[supervisord-controller-conf]
recipe = slapos.recipe.template:jinja2
template = {{ supervisord_conf }}
context =
section parameter_dict controller-parameters
rendered = ${controller-directory:etc}/supervisord-{{ name }}.conf
[supervisord-{{ name }}]
recipe = slapos.cookbook:wrapper
command-line = {{ buildout_bin_directory }}/supervisord -c ${supervisord-controller-conf:rendered} --nodaemon
wrapper-path = ${directory:services}/supervisord-{{ name }}
{% if use_service_hash -%}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{% endif -%}
depends =
{{ '${' ~ name ~ '-bin:recipe}' }}
[{{ name }}-bin]
recipe = slapos.cookbook:wrapper
command-line = {{ buildout_bin_directory }}/supervisorctl -c ${supervisord-controller-conf:rendered}
wrapper-path = ${directory:bin}/{{ name }}
{%- endmacro %}
{% macro supervisord_program(
name,
parameter_dict
) -%}
[supervisord-{{ name }}]
recipe = slapos.recipe.template:jinja2
template = inline:[program:{{ parameter_dict['name'] }}]
directory = ${buildout:directory}
command = {{ parameter_dict['command'] }}
process_name = {{ parameter_dict['name'] }}
autostart = {{ parameter_dict.get('autostart', True) }}
autorestart = {{ parameter_dict.get('autorestart', False) }}
startsecs = {{ parameter_dict.get('startsecs', 0) }}
startretries = {{ parameter_dict.get('startretries', 0) }}
exitcodes = {{ parameter_dict.get('exitcodes', 0) }}
stopsignal = {{ parameter_dict.get('stopsignal', 'TERM') }}
stopwaitsecs = {{ parameter_dict.get('stopwaitsecs', 60) }}
serverurl=AUTO
redirect_stderr=true
stdout_logfile = {{ parameter_dict.get('stdout_logfile', 'NONE') }}
stdout_logfile_maxbytes = 1000KB
stdout_logfile_backups = 1
stderr_logfile = {{ parameter_dict.get('stderr_logfile', 'NONE') }}
stderr_logfile_maxbytes = 1000KB
stderr_logfile_backups = 1
environment = {{ parameter_dict['environment'] | join(',') }}
rendered = ${controller-directory:supervisord}/{{ name }}.conf
{%- endmacro %}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7