Commit 77eac59d authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents 3ff99309 90551275
...@@ -109,7 +109,6 @@ template = inline: ...@@ -109,7 +109,6 @@ template = inline:
[versions] [versions]
mock = 3.0.5
random2 = 1.0.1 random2 = 1.0.1
zope.testing = 4.7 zope.testing = 4.7
zope.testrunner = 5.2 zope.testrunner = 5.2
......
...@@ -14,8 +14,3 @@ md5sum = c0feb5855604f68b09bdb3acb623619e ...@@ -14,8 +14,3 @@ md5sum = c0feb5855604f68b09bdb3acb623619e
environment = environment =
CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib
[cmake-3.18]
<= cmake
url = https://cmake.org/files/v3.18/cmake-3.18.4.tar.gz
md5sum = 0380beaee1c39a22455db02651abe7be
...@@ -26,8 +26,7 @@ configure-command = : ...@@ -26,8 +26,7 @@ configure-command = :
make-binary = : make-binary = :
post-install = post-install =
mv * %(location)s mv * %(location)s
for file in %(location)s/bin/* %(location)s/lib/*.so ; do cd %(location)s
echo appending rpath to $file set -x
${patchelf:location}/bin/patchelf --set-rpath %(rpath)s $file ${patchelf:location}/bin/patchelf --set-rpath %(rpath)s bin/* lib/*.so
done
rpath = ${zlib:location}/lib:${alsa:location}/lib:${freetype:location}/lib:${libpng:location}/lib:${libXrender:location}/lib:${libXtst:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXext:location}/lib:${libXdmcp:location}/lib:${libXi:location}/lib:${libxcb:location}/lib:@@LOCATION@@/lib:@@LOCATION@@/lib/server rpath = ${zlib:location}/lib:${alsa:location}/lib:${freetype:location}/lib:${libpng:location}/lib:${libXrender:location}/lib:${libXtst:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXext:location}/lib:${libXdmcp:location}/lib:${libXi:location}/lib:${libxcb:location}/lib:@@LOCATION@@/lib:@@LOCATION@@/lib/server
...@@ -10,5 +10,6 @@ patch-options = -p1 ...@@ -10,5 +10,6 @@ patch-options = -p1
patches = patches =
${:_profile_base_location_}/libiconv.gets.patch#8a20d8afe0617fce56f77537d2b84621 ${:_profile_base_location_}/libiconv.gets.patch#8a20d8afe0617fce56f77537d2b84621
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.14.tar.gz url = http://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.14.tar.gz
md5sum = e34509b1623cec449dfeb73d7ce9c6c6 md5sum = e34509b1623cec449dfeb73d7ce9c6c6
...@@ -10,6 +10,7 @@ parts = ...@@ -10,6 +10,7 @@ parts =
[libmicrohttpd] [libmicrohttpd]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
shared = true
url = https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.70.tar.gz url = https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.70.tar.gz
md5sum = dcd6045ecb4ea18c120afedccbd1da74 md5sum = dcd6045ecb4ea18c120afedccbd1da74
configure-options = configure-options =
......
[buildout] [buildout]
extends = extends =
../xz-utils/buildout.cfg ../xz-utils/buildout.cfg
../patch/buildout.cfg
parts = parts =
m4 m4
[m4] [m4]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
shared = true shared = true
url = http://ftp.gnu.org/gnu/m4/m4-1.4.18.tar.xz url = http://ftp.gnu.org/gnu/m4/m4-1.4.19.tar.xz
md5sum = 730bb15d96fffe47e148d1e09235af82 md5sum = 0d90823e1426f1da2fd872df0311298d
environment = environment =
PATH=${xz-utils:location}/bin:%(PATH)s PATH=${xz-utils:location}/bin:%(PATH)s
patch-binary = ${patch:location}/bin/patch
patch-options = -p1
patches =
https://sources.debian.org/data/main/m/m4/1.4.18-5/debian/patches/01-fix-ftbfs-with-glibc-2.28.patch#058a786425e507f911649205b61ffcac
[buildout] [buildout]
extends =
../defaults.cfg
parts = parts =
ninja ninja
...@@ -8,8 +10,8 @@ shared = true ...@@ -8,8 +10,8 @@ shared = true
version = 1.10.2 version = 1.10.2
url = https://github.com/ninja-build/ninja/archive/refs/tags/v${:version}.tar.gz url = https://github.com/ninja-build/ninja/archive/refs/tags/v${:version}.tar.gz
md5sum = 639f75bc2e3b19ab893eaf2c810d4eb4 md5sum = 639f75bc2e3b19ab893eaf2c810d4eb4
configure-command = ./configure.py configure-command = ${python:executable} configure.py
configure-options = --bootstrap configure-options = --bootstrap --verbose
make-binary = true make-binary = true
post-install = post-install =
set %(location)s/bin set %(location)s/bin
......
...@@ -9,15 +9,11 @@ extends = ...@@ -9,15 +9,11 @@ extends =
# nxdtest is bin/ program to run nxdtest. # nxdtest is bin/ program to run nxdtest.
# use ${nxdtest:exe} to run it. # use ${nxdtest:exe} to run it.
[nxdtest] [nxdtest]
recipe = slapos.recipe.template:jinja2 <= pyprog
exe = ${buildout:bin-directory}/nxdtest exe = ${buildout:bin-directory}/nxdtest
rendered= ${:exe} entry = nxdtest:main
mode = 755 eggs = ${nxdtest-egg:egg}
eggs = ${.nxdtest.pyexe:eggs} initialization =
template=
inline:
#!${.nxdtest.pyexe:exe}
# $PATH for unshare and mount # $PATH for unshare and mount
import os import os
path = os.environ.get('PATH', '') path = os.environ.get('PATH', '')
...@@ -25,19 +21,6 @@ template= ...@@ -25,19 +21,6 @@ template=
path = ':' + path path = ':' + path
os.environ['PATH'] = '${util-linux:location}/bin' + path os.environ['PATH'] = '${util-linux:location}/bin' + path
from nxdtest import main; main()
# .nxdtest.pyexe is python interpreter used by nxdtest.
# the interpreter is located at ${.nxdtest.pyexe:exe}.
[.nxdtest.pyexe]
<= python-interpreter
eggs +=
${pygolang:egg}
${nxdtest-egg:egg}
interpreter = ${:_buildout_section_name_}
exe = ${buildout:bin-directory}/${:interpreter}
[nxdtest-egg] [nxdtest-egg]
recipe = zc.recipe.egg:develop recipe = zc.recipe.egg:develop
...@@ -48,6 +31,6 @@ depends = ${python-prctl:egg} ...@@ -48,6 +31,6 @@ depends = ${python-prctl:egg}
[nxdtest-repository] [nxdtest-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nxdtest.git repository = https://lab.nexedi.com/nexedi/nxdtest.git
revision = 9f413221 revision = 6f75fa90
location = ${buildout:parts-directory}/nxdtest location = ${buildout:parts-directory}/nxdtest
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
...@@ -2,54 +2,49 @@ ...@@ -2,54 +2,49 @@
extends = extends =
../autoconf/buildout.cfg ../autoconf/buildout.cfg
../automake/buildout.cfg ../automake/buildout.cfg
../libtool/buildout.cfg
../cmake/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../git/buildout.cfg
../openssl/buildout.cfg
../bzip2/buildout.cfg ../bzip2/buildout.cfg
../perl/buildout.cfg ../cmake/buildout.cfg
../gnutls/buildout.cfg
../curl/buildout.cfg ../curl/buildout.cfg
../git/buildout.cfg
../gnutls/buildout.cfg ../gnutls/buildout.cfg
../libzip/buildout.cfg
../m4/buildout.cfg
../pcre/buildout.cfg
../jemalloc/buildout.cfg ../jemalloc/buildout.cfg
../libmicrohttpd/buildout.cfg ../libmicrohttpd/buildout.cfg
../libtool/buildout.cfg
../m4/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../pcre/buildout.cfg
../perl/buildout.cfg
# https://github.com/libinjection/libinjection/issues/4
../python-2.7/buildout.cfg
parts = parts =
proxysql proxysql
[proxysql] [proxysql]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
version = v2.0.12 shared = true
version = v2.3.2
url = https://github.com/sysown/proxysql/archive/${:version}.tar.gz url = https://github.com/sysown/proxysql/archive/${:version}.tar.gz
md5sum = 70ec17fe73703a25730fdd44b6bc3ef5 md5sum = 969129ac43c9f64641509891a116e0e1
prefix = @@LOCATION@@
# Patch installation path for SlapOS
pre-configure =
mkdir -p ${:prefix}/bin ${:prefix}/etc/init.d ${:prefix}/lib/systemd/system
sed -ri "
s#(\s)/usr/bin#\1${:prefix}/bin#g
s#(\s)/etc#\1${:prefix}/etc#g
s#(\s)/usr/lib#\1${:prefix}/lib#
s#(\s)/var/lib#\1${:prefix}/lib#g
s#(\s)useradd#\1echo useradd#g
s#(\s)systemctl#\1echo systemctl#g
s#(\s)chkconfig#\1echo chkconfig#g
s#(\s)update-rc.d#\1echo update-rc.d#g" Makefile
configure-command = true configure-command = true
make-options = GIT_VERSION=${:version}
# The install target does not honor DESTDIR:
# let's forget it and only install what we need.
make-targets = src/proxysql
post-install =
install -Dt %(location)s/bin %(make-targets)s
environment = environment =
GIT_VERSION=${:version}
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${libgcrypt:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${libgcrypt:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
PATH=${m4:location}/bin:${libtool:location}/bin:${libgcrypt:location}/bin:${curl:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${bzip2:location}/bin:${autoconf:location}/bin:${git:location}/bin:${automake:location}/bin:${patch:location}/bin:${cmake-3.18:location}/bin:%(PATH)s PATH=${m4:location}/bin:${libtool:location}/bin:${libgcrypt:location}/bin:${curl:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${bzip2:location}/bin:${autoconf:location}/bin:${git:location}/bin:${automake:location}/bin:${patch:location}/bin:${cmake:location}/bin:%(PATH)s:${python2.7:location}/bin
CXXFLAGS=-I${openssl:location}/include -I${gnutls:location}/include -I${zlib:location}/include CXXFLAGS=-I${openssl:location}/include -I${gnutls:location}/include -I${zlib:location}/include
CFLAGS=-I${gnutls:location}/include CFLAGS=-I${gnutls:location}/include
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath=${curl:location}/lib -L${libtool:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${curl:location}/lib -L${pcre:location}/lib -L${jemalloc:location}/lib -L${libmicrohttpd:location}/lib LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath=${curl:location}/lib -L${libtool:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${curl:location}/lib -L${pcre:location}/lib -L${jemalloc:location}/lib -L${libmicrohttpd:location}/lib
CMAKE_INCLUDE_PATH=${openssl:location}/include:${gnutls:location}/include:${curl:location}/include:${pcre:location}/include:${jemalloc:location}/include:${libmicrohttpd:location}/include CMAKE_INCLUDE_PATH=${openssl:location}/include:${gnutls:location}/include:${curl:location}/include:${pcre:location}/include:${jemalloc:location}/include:${libmicrohttpd:location}/include:${zlib:location}/include
CMAKE_LIBRARY_PATH=${openssl:location}/lib:${gnutls:location}/lib:${curl:location}/lib:${pcre:location}/lib:${jemalloc:location}/lib:${libmicrohttpd:location}/lib CMAKE_LIBRARY_PATH=${openssl:location}/lib:${gnutls:location}/lib:${curl:location}/lib:${pcre:location}/lib:${jemalloc:location}/lib:${libmicrohttpd:location}/lib:${zlib:location}/lib
LIBTOOL=libtool LIBTOOL=libtool
ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${libtool:location}/share/aclocal ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${libtool:location}/share/aclocal
patch-options = -p1
patches =
# PR #3402 (mariadb_client: backport patch to fix syntax error in cmake 3.20)
https://github.com/sysown/proxysql/commit/a3cfa56d257219f7610cd5711045bb5d84485a91.patch
...@@ -49,5 +49,64 @@ initialization = ...@@ -49,5 +49,64 @@ initialization =
scripts = ${:interpreter} scripts = ${:interpreter}
# pyprog provides macro recipe to build python programs.
#
# Contrary to zc.recipe.egg:scripts it generates scripts that are run with
# sys.executable being correctly set. In particular it is valid to spawn
# sys.executable from inside the program and assume that all specified eggs are
# still accessible and can be imported.
#
# Usage example:
#
# [myprog]
# <= pyprog
# exe = ${buildout:bin-directory}/myprog
# entry = my.py.mod:main
# eggs = ...
[pyprog]
recipe = slapos.recipe.build
initialization =
depends =
_name = ${:_buildout_section_name_}
init =
name = options['_name'] # options['_buildout_section_name_'] does not work
exe = options['exe']
entry = options['entry']
eggs = options['eggs']
pyinit = options['initialization']
options['depends'] += '$${.%s.pyprog:recipe}' % name
# mod:func -> 'from mod import func; sys.exit(func())'
mod, func = entry.split(':')
entry_run = 'from %s import %s; sys.exit(%s())' % (mod, func, func)
# indent pyinit with ' '
__pyinit = '\n'.join([' '+_ for _ in pyinit.splitlines()])
self.buildout.parse("""
# .X.pyprog is python program to start and run entry
# it uses .X.pyexe as underlying python interpreter
[.%(name)s.pyprog]
recipe = slapos.recipe.template:jinja2
exe = %(exe)s
rendered= $${:exe}
mode = 755
template=
inline:
#!$${.%(name)s.pyexe:exe}
import sys
%(__pyinit)s
%(entry_run)s
# .X.pyexe is python interpreter used by .X.pyprog
[.%(name)s.pyexe]
<= python-interpreter
eggs += %(eggs)s
interpreter = $${:_buildout_section_name_}
exe = $${buildout:bin-directory}/$${:interpreter}
""" % locals())
[versions] [versions]
pygolang = 0.0.9 pygolang = 0.0.9
...@@ -16,3 +16,4 @@ eggs = ${pytest:eggs} ...@@ -16,3 +16,4 @@ eggs = ${pytest:eggs}
[versions] [versions]
pytest = 4.6.11:whl pytest = 4.6.11:whl
pytest-timeout = 1.4.2 pytest-timeout = 1.4.2
pytest-mock = 2.0.0:whl
...@@ -28,5 +28,6 @@ command = set -e ...@@ -28,5 +28,6 @@ command = set -e
<= go-git-package <= go-git-package
go.importpath = github.com/signal18/replication-manager go.importpath = github.com/signal18/replication-manager
repository = https://github.com/signal18/replication-manager repository = https://github.com/signal18/replication-manager
branch = 2.1 #branch = 2.1
revision = 9167a82c81af8f7be41cf51bc9be8a37dc3d8c03 branch = develop
revision = 838ffeb205ea4477f1c1bda607940fa07d8bcf51
\ No newline at end of file
[buildout] [buildout]
# You need to define rootdir and destdir on an upper level # You need to define rootdir and destdir on an upper level
#
# rootdir is where the parts will be installed by the debian package (e.g. /opt/slapos)
# destdir is where we can put the files during compilation (e.g. /tmp/build-slapos)
# buildout MUST BE launched in $destdir/$rootdir (this is enforced by an assert)
#
# in this file, we teach many packages to use prefix = $rootdir/parts/package and we
# set DESTDIR = $destdir so in reality the files land in $destdir/$rootdir/parts/package
# which is EXACTLY what cmmi recipe uses by default (buildout run dir/parts/package)
# that's why everything is working
#
# please be careful when touching this file
#rootdir = %TARGET_DIRECTORY% #rootdir = %TARGET_DIRECTORY%
#destdir = %BUILD_ROOT_DIRECTORY% #destdir = %BUILD_ROOT_DIRECTORY%
#builddir = %BUILD_DIRECTORY%
extends = extends =
buildout.cfg buildout.cfg
...@@ -13,34 +25,24 @@ extensions = ...@@ -13,34 +25,24 @@ extensions =
extends-cache = extends-cache extends-cache = extends-cache
download-cache = download-cache download-cache = download-cache
# Uguu, upstream buildout.cfg must be patched as it works the other way [python]
# around from a packager point of view at least, thus at the end static init +=
# path, such as Python HOME directory, are wrong... buildout = self.buildout['buildout']
# assert buildout['directory'] == buildout['destdir'] + buildout['rootdir'], (
# Currently: "Buildout MUST BE launched in destdir/rootdir (currently launched in %s but should be launched in %s)",
# ./configure --prefix=BUILD_DIRECTORY && make install buildout['directory'], buildout['destdir'] + buildout['rootdir'])
# Instead of:
# ./configure --prefix=INSTALL_DIRECTORY && make install DESTDIR=BUILD_DIRECTORY
[python2.7] [python2.7]
configure-options += configure-options +=
--prefix=${buildout:rootdir}/parts/${:_buildout_section_name_} --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
environment += environment +=
DESTDIR=${buildout:destdir} DESTDIR=${buildout:destdir}
[gettext]
# Add gettext library path to RPATH as its binaries are used to build
# glib for example
environment =
PATH=${perl:location}/bin:${lunzip:location}/bin:%(PATH)s
CPPFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${ncurses:location}/include
LDFLAGS=-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${buildout:builddir}/parts/${:_buildout_section_name_}/lib
[bison] [bison]
configure-options += configure-options +=
--prefix=${buildout:rootdir}/parts/${:_buildout_section_name_} --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
make-options += make-options +=
DESTDIR=${buildout:destdir} DESTDIR=${buildout:destdir}
environment += environment +=
PERL5LIB=${perl:location}/lib/5.28.1/ PERL5LIB=${perl:location}/lib/5.28.1/
...@@ -62,7 +64,6 @@ environment += ...@@ -62,7 +64,6 @@ environment +=
[dbus] [dbus]
location = ${buildout:parts-directory}/${:_buildout_section_name_}
configure-options += configure-options +=
--prefix=${buildout:rootdir}/parts/${:_buildout_section_name_} --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
make-options += make-options +=
...@@ -74,7 +75,6 @@ post-install = ...@@ -74,7 +75,6 @@ post-install =
mkdir -p ${buildout:destdir}/parts/${:_buildout_section_name_}/var/run/dbus mkdir -p ${buildout:destdir}/parts/${:_buildout_section_name_}/var/run/dbus
[dbus-glib] [dbus-glib]
location = ${buildout:parts-directory}/${:_buildout_section_name_}
environment += environment +=
LD_LIBRARY_PATH=${dbus:location}/lib LD_LIBRARY_PATH=${dbus:location}/lib
DBUS_CFLAGS=-I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include DBUS_CFLAGS=-I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include
...@@ -92,21 +92,13 @@ post-install = ...@@ -92,21 +92,13 @@ post-install =
sed -i 's#${dbus:location}/lib/libdbus-1.la#/opt/slapos/parts/dbus/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la sed -i 's#${dbus:location}/lib/libdbus-1.la#/opt/slapos/parts/dbus/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
[openssl] [openssl]
shared = false
prefix = ${buildout:rootdir}/parts/${:_buildout_section_name_} prefix = ${buildout:rootdir}/parts/${:_buildout_section_name_}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
make-options += make-options +=
INSTALL_PREFIX=${buildout:destdir} INSTALL_PREFIX=${buildout:destdir}
DESTDIR=${buildout:destdir} DESTDIR=${buildout:destdir}
environment +=
environment =
PERL5LIB=${perl:location}/lib/5.28.1/ PERL5LIB=${perl:location}/lib/5.28.1/
PERL=${perl:location}/bin/perl
[bison-go]
<= bison
configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
make-options =
[gobject-introspection] [gobject-introspection]
pre-configure = pre-configure =
...@@ -115,10 +107,8 @@ configure-options += ...@@ -115,10 +107,8 @@ configure-options +=
--enable-shared --enable-shared
environment += environment +=
PERL5LIB=${perl:location}/lib/5.28.1/ PERL5LIB=${perl:location}/lib/5.28.1/
post-install =
post-install =
sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
rm -rf ${bison-go:location}
[pygobject3] [pygobject3]
pre-configure += pre-configure +=
...@@ -126,18 +116,6 @@ pre-configure += ...@@ -126,18 +116,6 @@ pre-configure +=
post-install = post-install =
sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
[ncurses]
configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--with-shared
--without-ada
--without-manpages
--without-tests
--without-normal
--without-debug
--without-gpm
--enable-rpath
[flex] [flex]
environment += environment +=
BISON_PKGDATADIR=${bison:location}/share/bison/ BISON_PKGDATADIR=${bison:location}/share/bison/
...@@ -154,7 +132,6 @@ post-install = ...@@ -154,7 +132,6 @@ post-install =
sed -i "s#'${buildout:destdir}/parts/site_perl#'/opt/slapos/parts/site_perl#" ${perl:location}/lib/5.28.1/*-linux-thread-multi/Config.pm sed -i "s#'${buildout:destdir}/parts/site_perl#'/opt/slapos/parts/site_perl#" ${perl:location}/lib/5.28.1/*-linux-thread-multi/Config.pm
[perl] [perl]
location = ${buildout:destdir}/parts/perl
configure-command = configure-command =
sh Configure -des \ sh Configure -des \
-Dprefix=${buildout:rootdir}/parts/${:_buildout_section_name_} \ -Dprefix=${buildout:rootdir}/parts/${:_buildout_section_name_} \
......
...@@ -42,5 +42,4 @@ template = inline: ...@@ -42,5 +42,4 @@ template = inline:
[versions] [versions]
mock = 3.0.5
freezegun = 1.0.0 freezegun = 1.0.0
...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages ...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob import glob
import os import os
version = '1.0.220' version = '1.0.226'
name = 'slapos.cookbook' name = 'slapos.cookbook'
long_description = open("README.rst").read() long_description = open("README.rst").read()
......
from __future__ import print_function
import os import os
import subprocess import subprocess
import time import time
...@@ -16,35 +18,39 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file): ...@@ -16,35 +18,39 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
while True: while True:
while True: while True:
mysql_upgrade = subprocess.Popen(mysql_upgrade_binary, mysql_upgrade = subprocess.Popen(mysql_upgrade_binary,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True)
result = mysql_upgrade.communicate()[0] result = mysql_upgrade.communicate()[0]
if mysql_upgrade.returncode: if mysql_upgrade.returncode:
print "Command %r failed with result:\n%s" % (mysql_upgrade_binary, result) print("Command %r failed with result:\n%s" % (mysql_upgrade_binary, result))
break break
print "MySQL database upgraded with result:\n%s" % result print("MySQL database upgraded with result:\n%s" % result)
mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE, mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True)
result = mysql.communicate(mysql_script)[0] result = mysql.communicate(mysql_script)[0]
if mysql.returncode: if mysql.returncode:
print 'Command %r failed with:\n%s' % (mysql_list, result) print('Command %r failed with:\n%s' % (mysql_list, result))
break break
# import timezone database # import timezone database
mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE, mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True)
timezone_sql = mysql_tzinfo_to_sql.communicate()[0] timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
if mysql_tzinfo_to_sql.returncode != 0: if mysql_tzinfo_to_sql.returncode != 0:
print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result) print('Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result))
break break
mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE, mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True)
result = mysql.communicate(timezone_sql)[0] result = mysql.communicate(timezone_sql)[0]
if mysql.returncode: if mysql.returncode:
print 'Command %r failed with:\n%s' % (mysql_list, result) print('Command %r failed with:\n%s' % (mysql_list, result))
break break
print 'SlapOS initialisation script succesfully applied on database.' print('SlapOS initialisation script succesfully applied on database.')
return return
sleep = min(sleep+1, 30) sleep = min(sleep+1, 30)
print 'Sleeping for %ss and retrying' % sleep print('Sleeping for %ss and retrying' % sleep)
sys.stdout.flush() sys.stdout.flush()
sys.stderr.flush() sys.stderr.flush()
time.sleep(sleep) time.sleep(sleep)
...@@ -217,7 +217,7 @@ ...@@ -217,7 +217,7 @@
}, },
"timerserver-interval": { "timerserver-interval": {
"description": "Timerserver tick period, in seconds, or 0 to disable", "description": "Timerserver tick period, in seconds, or 0 to disable",
"default": 5, "default": 1,
"type": "number" "type": "number"
}, },
"private-dev-shm": { "private-dev-shm": {
......
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
"software-type": { "software-type": {
"default": { "default": {
"title": "Default", "title": "Default",
"software-type": "default",
"description": "No automated database modification (ERP5Site is not automatically created).", "description": "No automated database modification (ERP5Site is not automatically created).",
"request": "instance-erp5-input-schema.json", "request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json", "response": "instance-erp5-output-schema.json",
......
...@@ -5,9 +5,11 @@ ...@@ -5,9 +5,11 @@
"software-type": { "software-type": {
"default": { "default": {
"title": "Default", "title": "Default",
"software-type": "default",
"description": "Default setup for ERP5TestNode Instance.", "description": "Default setup for ERP5TestNode Instance.",
"request": "instance-erp5testnode-input-schema.json", "request": "instance-erp5testnode-input-schema.json",
"response": "instance-output-schema.json" "response": "instance-output-schema.json",
"index": 0
} }
} }
} }
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[deploy-script-controller-script] [deploy-script-controller-script]
filename = deploy-script-controller filename = deploy-script-controller
md5sum = 8288e59eb442c662544daffbf446a033 md5sum = 8357771b70efd0740561b1cb46f6955e
[template-deploy-test] [template-deploy-test]
filename = instance.cfg.in filename = instance.cfg.in
......
...@@ -83,8 +83,6 @@ function upload () ...@@ -83,8 +83,6 @@ function upload ()
add_checks $LOG_FILE add_checks $LOG_FILE
t=`date '+%Y%m%d%H%S'` t=`date '+%Y%m%d%H%S'`
mv $LOG_FILE ${LOG_FILE}.$t mv $LOG_FILE ${LOG_FILE}.$t
# just to be sure flush all disk operations before uploading
flush
curl -q -X POST --data-urlencode "path=test-script-result/log-file.log.$t" --data-urlencode "content@${LOG_FILE}.$t" http://10.0.2.100/ curl -q -X POST --data-urlencode "path=test-script-result/log-file.log.$t" --data-urlencode "content@${LOG_FILE}.$t" http://10.0.2.100/
} }
......
...@@ -125,7 +125,6 @@ template = ...@@ -125,7 +125,6 @@ template =
[versions] [versions]
coverage = 4.5.1 coverage = 4.5.1
mock = 3.0.5
ecdsa = 0.13 ecdsa = 0.13
mysqlclient = 1.3.12 mysqlclient = 1.3.12
pycrypto = 2.6.1 pycrypto = 2.6.1
......
...@@ -14,19 +14,19 @@ ...@@ -14,19 +14,19 @@
# not need these here). # not need these here).
[instance.cfg] [instance.cfg]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 8a08be95a04f1a47098c4fdef80bdfed md5sum = af2fc4a7a0f782fed2cb1112ef3cb397
[instance-repman.cfg] [instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in _update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 697a1b546c883da45c14dbcd2d73b2b9 md5sum = c6e41783b8e3e844b6319953a8d665ec
[config-toml.in] [config-toml.in]
_update_hash_filename_ = templates/config.toml.in _update_hash_filename_ = templates/config.toml.in
md5sum = 2ee2c4bc5f985c11c2167b819d261256 md5sum = dc9fdcf6c3c775da6b0b72f83c4a4105
[config-cluster-toml.in] [config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in _update_hash_filename_ = templates/cluster-config.toml.in
md5sum = c78c6be9537d3dffbb526b0e819bacb5 md5sum = ff44b81dc1177e32019ddbf4f1c6b616
[nginx.conf.in] [nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in _update_hash_filename_ = templates/nginx.conf.in
...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30 ...@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg] [template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = a5c204cac552754520aee0570d379723 md5sum = 938f1e8087395757f5ae861d20500658
[template-my-cnf] [template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in _update_hash_filename_ = templates/my.cnf.in
...@@ -42,15 +42,19 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9 ...@@ -42,15 +42,19 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
[template-mariadb-initial-setup] [template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in _update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = df44c2d6fb1971df582345daedad280c md5sum = 47b76144e1b116580c8acf08274af976
[template-publish-slave-information]
_update_hash_filename_ = publish-database-slave-parameters.cfg.in
md5sum = 9616477ff993c55c2b43bf8797db6994
[mariadb-init-root-sql] [mariadb-init-root-sql]
_update_hash_filename_ = templates/mariadb_init_root.sql.in _update_hash_filename_ = templates/mariadb_init_root.sql.in
md5sum = d8a0a7d72b02f911dafda655941b805d md5sum = 6e5c02542568f250aacd84694249efc0
[init-root-wrapper-in] [init-root-wrapper-in]
_update_hash_filename_ = templates/init_root_wrapper.in _update_hash_filename_ = templates/init_root_wrapper.in
md5sum = 7661e3b2f66982b9b17bd2ab73e2e8ef md5sum = c203f40a58386310a433b58fd345a341
[repman-manager-sh.in] [repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in _update_hash_filename_ = templates/repman-manager.sh.in
...@@ -58,12 +62,16 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9 ...@@ -58,12 +62,16 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
[dbjobs-in] [dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in _update_hash_filename_ = templates/dbjobs.in
md5sum = d2ebd2ec55bf8489789a52c808729925 md5sum = d5c19d866c5a76d3bc2f434a91b1553a
[mysqld-need-start.sh.in] [mysqld-need-start.sh.in]
_update_hash_filename_ = templates/mysqld-need-start.sh.in _update_hash_filename_ = templates/mysqld-need-start.sh.in
md5sum = e9bcee5dc1318fe3acda2663472214f5 md5sum = 8f14054b3761b167cf8a86f4f084a574
[proxy-need-start-stop.sh.in] [proxy-need-start-stop.sh.in]
_update_hash_filename_ = templates/proxy-need-start-stop.sh.in _update_hash_filename_ = templates/proxy-need-start-stop.sh.in
md5sum = 455aaf369bf5141758dc57f2c0e67b08 md5sum = 455aaf369bf5141758dc57f2c0e67b08
[slave-db-manage.in]
_update_hash_filename_ = templates/slave-db-manage.in
md5sum = b45313ae5fb06972cc7fc945e34e434a
...@@ -17,17 +17,11 @@ ...@@ -17,17 +17,11 @@
[{{ section('publish') }}] [{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised recipe = slapos.cookbook:publish.serialised
-extends = publish-early
database-host = {{ host }}:{{ port }} database-host = {{ host }}:{{ port }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
partition-path = ${buildout:directory} partition-path = ${buildout:directory}
receiver-port = ${dbjob-parameter:socat-port} receiver-port = ${dbjob-parameter:socat-port}
csr-id = ${caucase-csr-id:csr-id}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
database-list = {{ dumps(database_list) }}
[jinja2-template-base] [jinja2-template-base]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -37,6 +31,22 @@ mode = 644 ...@@ -37,6 +31,22 @@ mode = 644
< = jinja2-template-base < = jinja2-template-base
mode = 755 mode = 755
[jinja2-template-script-base]
< = jinja2-template-executable
context =
key tmp_dir directory:tmp
key partition_dir buildout:directory
raw username {{ slapparameter_dict['repman-user'] }}
raw password {{ slapparameter_dict['repman-passwd'] }}
raw cluster_name {{ slapparameter_dict['cluster'] }}
raw secure_url {{ slapparameter_dict['repman-secure-url'] }}
raw mysql_conf ${directory:etc}/mysql/my.cnf
raw dash_bin {{ dash }}
raw jq_bin {{ jq_bin }}
raw curl_bin {{ curl_bin }}
raw mysql_bin {{ parameter_dict['mariadb-location'] }}/bin/mysql
${:extra-context}
[simplefile] [simplefile]
< = jinja2-template-base < = jinja2-template-base
template = inline:{{ '{{ content }}' }} template = inline:{{ '{{ content }}' }}
...@@ -53,10 +63,54 @@ context = key content {{content_section_name}}:content ...@@ -53,10 +63,54 @@ context = key content {{content_section_name}}:content
mode = {{ mode }} mode = {{ mode }}
{%- endmacro %} {%- endmacro %}
{% import "caucase" as caucase with context %}
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-updater',
url=slapparameter_dict['caucase-url'],
data_dir='${mysql-directory:ssl}',
crt_path='${mysql-directory:ssl}/server-cert.pem',
ca_path='${mysql-directory:ssl}/ca-cert.pem',
crl_path='${mysql-directory:ssl}/crl.pem',
key_path='${mysql-directory:ssl}/server-key.pem',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
)}}
{% do part_list.append('caucase-updater') -%}
{% do part_list.append('caucase-updater-promise') -%}
[get-csr-id]
recipe = plone.recipe.command
output = ${directory:tmp}/csr_id
command =
if [ -s "${mysql-directory:ssl}/server-cert.pem" ]; then
RESULT="None";
else
if [ -f "${caucase-updater-csr:csr}" ]; then
RESULT=$({{ caucase_bin_client }} --ca-url {{ slapparameter_dict['caucase-url'] }} --send-csr ${caucase-updater-csr:csr} | cut -d ' ' -f1)
if [ ! $? -eq 0 ]; then
RESULT="None";
fi
fi
fi
cat <<EOF > ${:output}
[caucase]
csr-id = $(echo $RESULT)
EOF
update-command = ${:command}
[caucase-csr-id]
recipe = slapos.cookbook:zero-knowledge.read
file-path = ${get-csr-id:output}
csr-id =
[my-cnf-parameters] [my-cnf-parameters]
socket = ${directory:run}/mariadb.sock socket = ${directory:run}/mariadb.sock
ip = {{ ip }} ip = {{ ip }}
data-directory = ${directory:srv}/mariadb data-directory = ${mysql-directory:data}
pid-file = ${directory:run}/mariadb.pid pid-file = ${directory:run}/mariadb.pid
plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }} plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }} groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
...@@ -66,9 +120,6 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0 ...@@ -66,9 +120,6 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0
innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }} innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }} innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }} relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
ssl-crt = ${directory:mariadb-ssl}/crt.pem
ssl-key = ${directory:mariadb-ssl}/key.pem
ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
[my-cnf] [my-cnf]
< = jinja2-template-base < = jinja2-template-base
...@@ -89,9 +140,11 @@ database-list = {{ dumps(database_list) }} ...@@ -89,9 +140,11 @@ database-list = {{ dumps(database_list) }}
mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }} mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
root-user = repman root-user = repman
heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }} heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }}
require-ssl = {{ dumps(slapparameter_dict['require-ssl']) }}
[init-script] [init-script]
< = jinja2-template-executable < = jinja2-template-executable
extensions = jinja2.ext.do
# XXX: is there a better location ? # XXX: is there a better location ?
rendered = ${directory:etc}/mariadb_initial_setup.sql rendered = ${directory:etc}/mariadb_initial_setup.sql
template = {{ parameter_dict['template-mariadb-initial-setup'] }} template = {{ parameter_dict['template-mariadb-initial-setup'] }}
...@@ -106,20 +159,26 @@ init-script = ${init-script:rendered} ...@@ -106,20 +159,26 @@ init-script = ${init-script:rendered}
mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path} mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
[{{ section('update-mysql-script') }}] [{{ section('update-mysql-script') }}]
< = jinja2-template-executable < = jinja2-template-script-base
rendered = ${directory:scripts}/mariadb_update rendered = ${directory:scripts}/mariadb_update
init-password = ${directory:etc}/.init-passwd.done init-password = ${directory:etc}/.init-passwd.done
upgrade-done = ${directory:lib}/mariadb-update-done upgrade-done = ${directory:lib}/mariadb-update-done
context = extra-context =
key init_password_done :init-password key init_password_done :init-password
key upgrade_done :upgrade-done key upgrade_done :upgrade-done
key init_root_sql init-root-sql:rendered key init_root_sql init-root-sql:rendered
key mysql_update update-mysql:output key mysql_update update-mysql:output
raw mysql_conf ${directory:etc}/mysql/my.cnf key init_database_sql init-script:rendered
raw dash_bin {{ dash }}
raw mysql_bin {{ parameter_dict['mariadb-location'] }}/bin/mysql
template = {{ parameter_dict['template-init-root-wrapper'] }} template = {{ parameter_dict['template-init-root-wrapper'] }}
[{{ section('mysql-slave-db-cleanup') }}]
< = jinja2-template-script-base
rendered = ${directory:scripts}/manage-slave-db
db-name = {{ slapparameter_dict['database-name'] }}
extra-context =
key database_name :db-name
template = {{ parameter_dict['template-manage-db'] }}
[mysqld] [mysqld]
< = jinja2-template-executable < = jinja2-template-executable
rendered = ${directory:bin}/mysqld rendered = ${directory:bin}/mysqld
...@@ -136,19 +195,19 @@ environ = ...@@ -136,19 +195,19 @@ environ =
{{ variable }} {{ variable }}
{%- endfor %} {%- endfor %}
[ca-mysqld] [mysqld-launcher]
<= certificate-authority recipe = slapos.cookbook:wrapper
recipe = slapos.cookbook:certificate_authority.request command-line = ${mysqld:rendered}
key-file = ${my-cnf-parameters:ssl-key} wrapper-path = ${directory:controller}/mariadb
cert-file = ${my-cnf-parameters:ssl-crt} wait-for-files =
executable = ${mysqld:rendered} ${mysql-directory:mysql}/my.cnf
wrapper = ${directory:controller}/mariadb ${mysql-directory:ssl}/server-cert.pem
{% import "supervisord_lib" as supervisord_lib with context %} {% import "supervisord_lib" as supervisord_lib with context %}
{{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }} {{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
{% do part_list.append("supervisord-mariadb-ctl") -%} {% do part_list.append("supervisord-mariadb-ctl") -%}
{% set maradb_program_dict = {"name": "mariadb", "command": "${ca-mysqld:wrapper}", {% set maradb_program_dict = {"name": "mariadb", "command": "${mysqld-launcher:wrapper-path}",
"stopwaitsecs": 86400, "environment": [], "stopwaitsecs": 86400, "startretries": 10, "autorestart": True, "environment": [],
"stdout_logfile": "${directory:log}/mariadb_stdout.log", "stdout_logfile": "${directory:log}/mariadb_stdout.log",
"stderr_logfile": "${directory:log}/mariadb_stdout.log" } %} "stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
...@@ -178,13 +237,18 @@ target-directory = ${directory:bin} ...@@ -178,13 +237,18 @@ target-directory = ${directory:bin}
link-binary = {{ dumps(parameter_dict['link-binary']) }} link-binary = {{ dumps(parameter_dict['link-binary']) }}
[binary-wrap-base] [binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary # Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it. # will reject it.
command-line = recipe = slapos.recipe.template:jinja2
"{{ parameter_dict['mariadb-location'] }}/bin/${:command}" rendered = ${directory:bin}/${:command}
--defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args} mode = 755
wrapper-path = ${directory:bin}/${:command} template =
inline:#!/bin/sh
exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
--defaults-file="${directory:etc}/mysql/my.cnf" \
--protocol=socket \
${:extra-args} "$@"
wrapper-path = ${:rendered}
extra-args = extra-args =
[binary-wrap-mysql] [binary-wrap-mysql]
...@@ -208,6 +272,11 @@ command = mysqladmin ...@@ -208,6 +272,11 @@ command = mysqladmin
<= binary-wrap-base <= binary-wrap-base
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
command = mysql_tzinfo_to_sql command = mysql_tzinfo_to_sql
template =
inline:#!/bin/sh
exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
--defaults-file="${directory:etc}/mysql/my.cnf" \
--skip-write-binlog "$@"
[binary-wrap-pt-digest] [binary-wrap-pt-digest]
<= binary-wrap-base <= binary-wrap-base
...@@ -226,26 +295,29 @@ plugin = ${:etc}/plugin ...@@ -226,26 +295,29 @@ plugin = ${:etc}/plugin
srv = ${buildout:directory}/srv srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp tmp = ${buildout:directory}/tmp
backup = ${:srv}/backup backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental
mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var var = ${buildout:directory}/var
lib = ${:var}/lib lib = ${:var}/lib
mysql = ${:lib}/mysql
log = ${:var}/log log = ${:var}/log
run = ${:var}/run run = ${:var}/run
config-tmp = ${:tmp}/config config-tmp = ${:tmp}/config
custom = ${directory:etc}/mysql/custom
[mysql-directory]
recipe = slapos.cookbook:mkdirectory
mysql = ${directory:etc}/mysql
ssl = ${:mysql}/ssl
data = ${directory:lib}/mysql
custom = ${:mysql}/custom.d
[dbjob-parameter] [dbjob-parameter]
bash-bin = {{ bash_bin }} bash-bin = {{ bash_bin }}
db-user = ${init-script-parameters:root-user} db-user = ${init-script-parameters:root-user}
db-password = ${init-script-parameters:password} db-password = ${init-script-parameters:password}
mysql-dir = ${directory:mysql} mysql-dir = ${mysql-directory:data}
dbjob-cnf = ${directory:etc}/mysql/my.cnf dbjob-cnf = ${mysql-directory:mysql}/dbjob.cnf
log-dir = ${directory:mysql}/.system/logs log-dir = ${mysql-directory:data}/.system/logs
tmp-dir = ${directory:tmp} tmp-dir = ${directory:tmp}
mysqld-socket = ${my-cnf-parameters:socket} mysqld-socket = ${my-cnf-parameters:socket}
mysql-client = ${binary-wrap-mysql:wrapper-path}
socat-port = {{ int(port) + 9 }} socat-port = {{ int(port) + 9 }}
restart-script = ${mysqld-restart-script:rendered} restart-script = ${mysqld-restart-script:rendered}
socat-location = {{ parameter_dict['socat-location'] }} socat-location = {{ parameter_dict['socat-location'] }}
...@@ -309,11 +381,11 @@ template = inline:#!{{ bash_bin }} ...@@ -309,11 +381,11 @@ template = inline:#!{{ bash_bin }}
cd ${directory:config-tmp} && cd ${directory:config-tmp} &&
{{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config {{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config
tar -xzf config.tar.gz tar -xzf config.tar.gz
cp -r data/.system ${directory:mysql} rm -r etc/mysql/ssl # we don't need generated ssl files
rm -rf ${directory:etc}/mysql cp -r data/.system ${mysql-directory:data}
cp -r etc/mysql ${directory:etc} cp -r etc/mysql/* ${mysql-directory:mysql}
ln -sf ${directory:mysql}/.system ${directory:var}/system ln -sf ${mysql-directory:data}/.system ${directory:var}/system
ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom/01_mariadb.cnf ln -sf ${my-cnf:rendered} ${mysql-directory:custom}/01_mariadb.cnf
[{{ section('install-mysql-config') }}] [{{ section('install-mysql-config') }}]
recipe = plone.recipe.command recipe = plone.recipe.command
......
...@@ -113,7 +113,8 @@ ...@@ -113,7 +113,8 @@
"mroonga", "mroonga",
"utctime", "utctime",
"readcommitted", "readcommitted",
"nohandshake" "nohandshake",
"ssl"
] ]
}, },
"http-session-lifetime": { "http-session-lifetime": {
...@@ -140,6 +141,12 @@ ...@@ -140,6 +141,12 @@
"type": "boolean", "type": "boolean",
"default": true "default": true
}, },
"require-ssl": {
"title": "Enable REQUIRE SSL for repman database users",
"description": "Enable secure connection between repman and mariadb databases. This option is an initial value, used only when creating database users.",
"type": "boolean",
"default": false
},
"repman-cluster-dict": { "repman-cluster-dict": {
"title": "Replication Manager clusters definition", "title": "Replication Manager clusters definition",
"description": "Replication Manager clusters definition", "description": "Replication Manager clusters definition",
...@@ -197,7 +204,7 @@ ...@@ -197,7 +204,7 @@
}, },
"proxy-tags": { "proxy-tags": {
"title": "Proxy tag list", "title": "Proxy tag list",
"description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\"]", "description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\", \"ssl\"]",
"type": "array", "type": "array",
"items": { "items": {
"type": "string" "type": "string"
...@@ -206,7 +213,8 @@ ...@@ -206,7 +213,8 @@
"pkg", "pkg",
"masterslave", "masterslave",
"linux", "linux",
"noreadwritesplit" "noreadwritesplit",
"ssl"
] ]
}, },
"logical-backup-cron": { "logical-backup-cron": {
......
{
"$schema": "http://json-schema.org/draft-04/schema",
"properties": {
"db_user": {
"description": "Database User, default is 'user'. This parameter is set only if database is not created yet.",
"title": "Database User",
"type": "string",
"default": "user"
},
"db_password": {
"description": "Database password. If no password set, a password will be generated. This parameter is set only if database is not created yet.",
"title": "Initial database password",
"type": "string",
"default": ""
},
"db_charset": {
"description": "Database Character set information. This parameter is set only if database is not created yet.",
"title": "Database Character set information",
"type": "string",
"enum": [
"latin1",
"latin2",
"latin5",
"latin7",
"ascii",
"latin5",
"utf8",
"utf8mb4",
"utf16",
"utf16le",
"utf32",
"binary",
"utf8_general_ci",
"utf8_unicode_ci",
"utf8_unicode_ci",
"utf8mb4_bin",
"utf8mb4_general_ci",
"utf8mb4_unicode_ci",
"latin1_general_ci",
"latin1_bin",
"ascii_general_ci",
"ascii_bin"
],
"default": ""
}
},
"title": "Input Parameters",
"type": "object"
}
...@@ -8,8 +8,23 @@ ...@@ -8,8 +8,23 @@
{% set ip = (ipv6_set | list)[0] -%} {% set ip = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%} {% set ipv4 = (ipv4_set | list)[0] -%}
{% set cluster_list = [] -%} {% set cluster_list = [] -%}
{% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow", "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci", "mroonga", "utctime", "readcommitted", "nohandshake"] -%} {% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow",
"pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci",
"mroonga", "utctime", "readcommitted", "nohandshake", "ssl"] -%}
{% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%} {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
{% set database_slave_list = [] -%}
{% set db_name_list = [] -%}
{% set count = namespace(value=2) %}
{% set caucase_bind = '[' ~ ip ~ ']:8890' -%}
{% set caucase_url = 'http://' ~ caucase_bind -%}
{% set csrid_list = [] -%}
{% macro password(name) -%}
[{{ name }}-password]
recipe = slapos.cookbook:generate.password
bytes = 12
{% endmacro -%}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
...@@ -25,11 +40,25 @@ log = ${:var}/log ...@@ -25,11 +40,25 @@ log = ${:var}/log
data = ${:var}/lib data = ${:var}/lib
nginx-prefix = ${:var}/nginx nginx-prefix = ${:var}/nginx
tmp = ${:home}/tmp tmp = ${:home}/tmp
backup-caucased = ${:srv}/backup-caucased
caucased = ${:srv}/caucased
ssl = ${:etc}/ssl
[proxysql-directory]
recipe = slapos.cookbook:mkdirectory
config = ${directory:etc}/proxysql
ssl = ${:config}/ssl
{% import "supervisord_lib" as supervisord_lib with context %} {% import "supervisord_lib" as supervisord_lib with context %}
{% set proxysql_controller = "proxysql-ctl" -%} {% set proxysql_controller = "proxysql-ctl" -%}
{{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }} {{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
[proxysql-controller-update]
recipe = slapos.cookbook:wrapper
command-line = ${proxysql-ctl-bin:wrapper-path} update
wrapper-path = ${directory:scripts}/proxysql-ctl-update
{% do part_list.append("supervisord-proxysql-ctl") -%} {% do part_list.append("supervisord-proxysql-ctl") -%}
{% do part_list.append("proxysql-controller-update") -%}
[request-common] [request-common]
recipe = slapos.cookbook:request.serialised recipe = slapos.cookbook:request.serialised
...@@ -39,6 +68,7 @@ key-file = ${slap-connection:key-file} ...@@ -39,6 +68,7 @@ key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file} cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id} computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id} partition-id = ${slap-connection:partition-id}
config-caucase-url = {{ caucase_url }}
[download-proxy-config] [download-proxy-config]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -48,16 +78,35 @@ template = inline:#!{{ bash_bin }} ...@@ -48,16 +78,35 @@ template = inline:#!{{ bash_bin }}
PORT=$3 PORT=$3
CONFIG=$4 CONFIG=$4
if [ -z "$CONFIG" ]; then if [ -z "$CONFIG" ]; then
CONFIG="${repman:proxies}/proxysql-$NAME.cnf" CONFIG="${proxysql-directory:config}/proxysql.cnf"
fi fi
mkdir -p ${repman:config-tmp}/proxies mkdir -p ${repman:config-tmp}/proxies
cd ${repman:config-tmp}/proxies cd ${repman:config-tmp}/proxies
{{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config {{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
tar -xzf proxies-$NAME.tar.gz tar -xzf proxies-$NAME.tar.gz
cp conf/proxysql.cnf $CONFIG cp etc/proxysql/proxysql.cnf $CONFIG
rendered = ${directory:bin}/update-proxysql-config rendered = ${directory:bin}/update-proxysql-config
mode = 755 mode = 755
{% for instance_dict in slave_instance_list -%}
{% set slave_dict = {
'name': 'db_%s' % instance_dict['slave_reference'].replace('-', '_').lower(),
'user': instance_dict.get('db_user', 'user'),
'password': instance_dict.get('db_password', '${' ~ instance_dict['slave_reference'] ~ '-password:passwd}'),
'slave_reference': instance_dict['slave_reference'],
'charset': instance_dict.get('db_charset', ''),
'require_ssl': True
} -%}
{% do database_slave_list.append(slave_dict) -%}
{% do db_name_list.append(slave_dict['name']) -%}
{{ password(instance_dict['slave_reference']) }}
{% endfor %}
[database-slave-information]
{% for slave_dict in database_slave_list -%}
{{ slave_dict['name'] }} = !py!{{ slave_dict }}
{% endfor %}
{% set db_list = db_name_list | join(' ') -%}
{% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%} {% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "", {% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
"database-amount": 2, "proxysql-user": "external", "database-amount": 2, "proxysql-user": "external",
...@@ -67,22 +116,23 @@ mode = 755 ...@@ -67,22 +116,23 @@ mode = 755
{% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%} {% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
{% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%} {% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
{% set database_list = parameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
# Request mariadb instances # Request mariadb instances
{% set db_amount = parameter_dict.get('database-amount', 2) -%} {% set db_amount = parameter_dict.get('database-amount', 2) -%}
{% if db_amount < 2 -%} {% if db_amount < 2 -%}
{% set db_amount = 2 -%} {% set db_amount = 2 -%}
{% endif -%} {% endif -%}
{% set count.value = count.value + db_amount %}
{% for i in range(0, db_amount) -%} {% for i in range(0, db_amount) -%}
{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%} {% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
{% set section = 'request-mariadb-' ~ i -%} {% set section = 'request-mariadb-' ~ i -%}
{% set dbname = 'Mariadb-' ~ i -%} {% set dbname = 'Mariadb-' ~ i %}
[{{ section }}] [{{ section }}]
<= request-common <= request-common
software-type = mariadb software-type = mariadb
name = {{ dbname }} name = {{ dbname ~ '-' ~ name}}
sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }} sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
{% for key, value in mariadb_dict.items() -%} {% for key, value in mariadb_dict.items() -%}
config-{{ key }} = {{ dumps(value) }} config-{{ key }} = {{ dumps(value) }}
...@@ -91,23 +141,27 @@ config-monitor-passwd = ${publish-early:monitor-password} ...@@ -91,23 +141,27 @@ config-monitor-passwd = ${publish-early:monitor-password}
config-root-password = ${publish-early:db-root-password} config-root-password = ${publish-early:db-root-password}
config-repman-user = ${repman-parameter:username} config-repman-user = ${repman-parameter:username}
config-heartbeat-user = ${repman-parameter:heartbeat-user} config-heartbeat-user = ${repman-parameter:heartbeat-user}
#config-repman-passwd = ${repman-parameter:password} config-repman-passwd = ${repman-parameter:password}
config-repman-url = ${nginx-parameter:backend-url} config-repman-url = ${nginx-parameter:backend-url}
config-repman-secure-url = ${nginx-parameter:backend-ssl-url} config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
config-cluster = {{ name }} config-cluster = {{ name }}
config-name = {{ dbname }} config-name = {{ dbname }}
config-database-list = {{ dumps(database_list) }} config-database-list = !py!{{ database_slave_list }}
return = config-database-name = {{ dumps(db_list) }}
config-require-ssl = {{ dumps(slapparameter_dict.get('require-ssl', False)) }}
return =
database-host database-host
receiver-port receiver-port
monitor-base-url monitor-base-url
partition-path partition-path
csr-id
{% do part_list.append(section) -%} {% do part_list.append(section) -%}
{% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%} {% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
{% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%} {% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
{% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%} {% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
{% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%} {% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do csrid_list.append('${' ~ section ~ ':connection-csr-id}') -%}
{% endfor -%} {% endfor -%}
...@@ -129,6 +183,7 @@ maximum = 7132 ...@@ -129,6 +183,7 @@ maximum = 7132
{% for key, value in parameter_dict.items() -%} {% for key, value in parameter_dict.items() -%}
{{ key }} = {{ value }} {{ key }} = {{ value }}
{% endfor -%} {% endfor -%}
name = {{ name }}
proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }} proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
proxy-port = {{ '${' ~ name ~ '-port:port}' }} proxy-port = {{ '${' ~ name ~ '-port:port}' }}
proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }} proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
...@@ -151,7 +206,7 @@ password = ${repman-parameter:password} ...@@ -151,7 +206,7 @@ password = ${repman-parameter:password}
proxysql-partition = ${buildout:directory} proxysql-partition = ${buildout:directory}
receiver-port-list = {{ receiver_port_list | join(',') }} receiver-port-list = {{ receiver_port_list | join(',') }}
enabled-tags = {{ slapparameter_dict.get("tag-list", tag_list) | join(',') }} enabled-tags = {{ slapparameter_dict.get("tag-list", tag_list) | join(',') }}
proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit"]) | join(',') }} proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit", "ssl"]) | join(',') }}
logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }} logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }} physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }} proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }}
...@@ -191,7 +246,7 @@ context = ...@@ -191,7 +246,7 @@ context =
recipe = plone.recipe.command recipe = plone.recipe.command
# if Repman is not started, cannot download config from server # if Repman is not started, cannot download config from server
stop-on-error = false stop-on-error = false
config = ${repman:proxies}/proxysql-{{ name }}.cnf config = ${proxysql-directory:config}/proxysql.cnf
data = ${repman:proxy-data}/{{ name }} data = ${repman:proxy-data}/{{ name }}
command = command =
mkdir -p ${:data} && mkdir -p ${:data} &&
...@@ -209,6 +264,7 @@ command-line = ...@@ -209,6 +264,7 @@ command-line =
wrapper-path = ${directory:controller}/proxysql-{{ name }} wrapper-path = ${directory:controller}/proxysql-{{ name }}
wait-for-files = wait-for-files =
${repman:bootstrap}/{{ name }}_bootstrapped ${repman:bootstrap}/{{ name }}_bootstrapped
${config-proxysql-{{ name }}:config}
depends = depends =
{{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }} {{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }}
{{ '${proxysql-' ~ name ~ '-promise:recipe}' }} {{ '${proxysql-' ~ name ~ '-promise:recipe}' }}
...@@ -237,7 +293,7 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }} ...@@ -237,7 +293,7 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
{% set service_name = "proxysql-" ~ name -%} {% set service_name = "proxysql-" ~ name -%}
{% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}", {% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
"stopwaitsecs": 60, "environment": [], "stopwaitsecs": 60, "startretries": 10, "autorestart": True, "environment": [],
"stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log", "stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
"stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %} "stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
...@@ -247,22 +303,126 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }} ...@@ -247,22 +303,126 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
{% do part_list.append('config-' ~ name) -%} {% do part_list.append('config-' ~ name) -%}
{% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%} {% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
{% set publish_database_list = [] -%}
{% set publish_database_v6_list = [] -%}
{% for database in database_list -%}
{% if database.get('user') -%}
{% do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% do publish_database_v6_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% else -%}
{% do publish_database_list.append("mysql://" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% do publish_database_v6_list.append("mysql://[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% endif -%}
{% endfor -%}
{% do publish_dict.__setitem__(name ~ '-database-list', "!py!['" ~ publish_database_list | join("', '") ~ "']") -%} [{{ name}}-publish-slave-information]
{% do publish_dict.__setitem__(name ~ '-database-list-v6', "!py!['" ~ publish_database_v6_list | join("', '") ~ "']") -%} recipe = slapos.cookbook:switch-softwaretype
default = {{ 'dynamic-' ~name ~ '-publish-slave-information:rendered' }}
RootSoftwareInstance = ${:default}
[dynamic-{{ name}}-publish-slave-information]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/database-publish-slave-information.cfg
extensions = jinja2.ext.do
template = {{ template_publish_slave_information }}
context =
import json_module json
key proxy_port {{ name }}-cluster-parameter:proxy-port
raw eggs_directory {{ eggs_directory }}
raw develop_eggs_directory {{ develop_eggs_directory }}
raw ipv6 {{ ip }}
raw ipv4 {{ ipv4 }}
section slave_dict database-slave-information
{% do part_list.append(name ~ '-publish-slave-information') -%}
{% endfor -%} {% endfor -%}
# deploy caucase
{% import "caucase" as caucase with context %}
{{ caucase.caucased(
prefix='caucased',
buildout_bin_directory=buildout_bin_directory,
caucased_path='${directory:service}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:caucased}',
netloc=caucase_bind,
tmp='${directory:tmp}',
service_auto_approve_count=count.value,
user_auto_approve_count=1,
key_len=2048,
)}}
{% do part_list.append('caucased') -%}
{% do part_list.append('caucased-promise') -%}
{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-updater',
url=caucase_url,
data_dir='${directory:srv}/caucase-updater',
crt_path='${directory:ssl}/repman-cert.crt',
ca_path='${directory:srv}/caucase-updater/ca.crt',
crl_path='${directory:srv}/caucase-updater/crl.pem',
key_path='${directory:ssl}/repman-cert.key',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
)}}
{% do part_list.append('caucase-updater') -%}
{% do part_list.append('caucase-updater-promise') -%}
#caucase user certificate
{{ caucase.updater(
prefix='caucase-user-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-user-updater',
url=caucase_url,
data_dir='${directory:srv}/caucase-user-updater',
crt_path='${directory:ssl}/caucase.user.crt',
ca_path='${directory:srv}/caucase-user-updater/ca.crt',
crl_path='${directory:srv}/caucase-user-updater/crl.pem',
key_path='${directory:ssl}/caucase.user.key',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
mode='user',
)}}
{% do part_list.append('caucase-user-updater') -%}
{% do part_list.append('caucase-user-updater-promise') -%}
#caucase proxysql certificate
{{ caucase.updater(
prefix='caucase-proxysql-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-proxysql-updater',
url=caucase_url,
data_dir='${proxysql-directory:ssl}',
crt_path='${proxysql-directory:ssl}/client-cert.pem',
ca_path='${proxysql-directory:ssl}/ca-cert.pem',
crl_path='${proxysql-directory:ssl}/crl.pem',
key_path='${proxysql-directory:ssl}/client-key.pem',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
)}}
{% do part_list.append('caucase-proxysql-updater') -%}
{% do part_list.append('caucase-proxysql-updater-promise') -%}
[caucase-sign-csr]
recipe = slapos.recipe.template:jinja2
mode = 755
rendered = ${directory:bin}/caucase-sign
template =
inline:#!/bin/sh
cp ${directory:ssl}/caucase.user.key ${directory:ssl}/caucase-full.key
cat ${directory:ssl}/caucase.user.crt >> ${directory:ssl}/caucase-full.key
for csr_id in {{ csrid_list | join(' ') }}; do
if [ "$csr_id" = "None" ] || [ -z "$csr_id"]; then
continue
fi
{{ buildout_bin_directory }}/caucase --ca-url {{ caucase_url }} --user-key ${directory:ssl}/caucase-full.key --sign-csr $csr_id
done
[cron-caucase-sign-csr]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = caucase-sign-csr
frequency = * * * * *
command = ${caucase-sign-csr:rendered}
{% do part_list.append('cron-caucase-sign-csr') -%}
[slap-configuration] [slap-configuration]
recipe = slapos.cookbook:slapconfiguration recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id} computer = ${slap-connection:computer-id}
...@@ -286,7 +446,6 @@ etc = ${directory:etc}/repman ...@@ -286,7 +446,6 @@ etc = ${directory:etc}/repman
data-dir = ${directory:var}/lib data-dir = ${directory:var}/lib
root-dir = ${directory:srv}/repman root-dir = ${directory:srv}/repman
clusters = ${:etc}/cluster.d clusters = ${:etc}/cluster.d
proxies = ${:etc}/proxy
proxy-data = ${:data-dir}/proxy proxy-data = ${:data-dir}/proxy
config-tmp = ${directory:tmp}/config config-tmp = ${directory:tmp}/config
bootstrap = ${:etc}/bootstrap bootstrap = ${:etc}/bootstrap
...@@ -296,8 +455,9 @@ proxies-log = ${directory:log}/proxy ...@@ -296,8 +455,9 @@ proxies-log = ${directory:log}/proxy
ipv6 = ${instance-parameter:ipv6-random} ipv6 = ${instance-parameter:ipv6-random}
port = ${instance-parameter:nginx-port} port = ${instance-parameter:nginx-port}
ssl-port = ${instance-parameter:nginx-ssl-port} ssl-port = ${instance-parameter:nginx-ssl-port}
ssl-certificate = ${ca-nginx:cert-file} ssl-certificate = ${directory:ssl}/repman-cert.crt
ssl-key = ${ca-nginx:key-file} ca-certificate = ${directory:srv}/caucase-updater/ca.crt
ssl-key = ${directory:ssl}/repman-cert.key
pid-file = ${directory:run}/nginx.pid pid-file = ${directory:run}/nginx.pid
access-log = ${directory:log}/nginx_access.log access-log = ${directory:log}/nginx_access.log
error-log = ${directory:log}/nginx_error.log error-log = ${directory:log}/nginx_error.log
...@@ -345,6 +505,8 @@ mail-to = {{ slapparameter_dict.get("mail-to", "") }} ...@@ -345,6 +505,8 @@ mail-to = {{ slapparameter_dict.get("mail-to", "") }}
http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }} http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }}
http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }} http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }}
ssl-cert = ${directory:ssl}/repman-cert.crt
ssl-key = ${directory:ssl}/repman-cert.key
[repman-config-folder] [repman-config-folder]
recipe = plone.recipe.command recipe = plone.recipe.command
...@@ -392,6 +554,9 @@ depends = ...@@ -392,6 +554,9 @@ depends =
${replication-manager-reload:recipe} ${replication-manager-reload:recipe}
${repman-listen-promise:recipe} ${repman-listen-promise:recipe}
${repman-listen-ssl-promise:recipe} ${repman-listen-ssl-promise:recipe}
wait-for-files =
${directory:ssl}/repman-cert.crt
${directory:ssl}/repman-cert.key
[repman-config.toml] [repman-config.toml]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -428,11 +593,12 @@ command-line = ...@@ -428,11 +593,12 @@ command-line =
{{ nginx_bin }} {{ nginx_bin }}
-p ${directory:nginx-prefix} -p ${directory:nginx-prefix}
-c ${nginx-conf:rendered} -c ${nginx-conf:rendered}
wrapper-path = ${directory:bin}/nginx-start wrapper-path = ${directory:services}/nginx
wait-for-files = wait-for-files =
${ca-directory:certs}/nginx.key ${directory:ssl}/repman-cert.crt
${ca-directory:certs}/nginx.crt ${directory:srv}/caucase-updater/ca.crt
${nginx-graceful-wrapper:rendered} ${directory:ssl}/repman-cert.key
[nginx-graceful-wrapper] [nginx-graceful-wrapper]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -442,20 +608,6 @@ rendered = ${directory:scripts}/nginx-graceful ...@@ -442,20 +608,6 @@ rendered = ${directory:scripts}/nginx-graceful
context = context =
mode = 755 mode = 755
[ca-nginx]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
cert-file = ${ca-directory:certs}/nginx.crt
key-file = ${ca-directory:certs}/nginx.key
executable = ${nginx-launcher:wrapper-path}
wrapper = ${directory:bin}/ca-nginx
[ca-nginx-service]
recipe = slapos.cookbook:wrapper
command-line = ${ca-nginx:wrapper}
wrapper-path = ${directory:services}/nginx
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[logrotate-entry-nginx] [logrotate-entry-nginx]
<= logrotate-entry-base <= logrotate-entry-base
name = nginx name = nginx
...@@ -468,6 +620,7 @@ recipe = slapos.cookbook:publish-early ...@@ -468,6 +620,7 @@ recipe = slapos.cookbook:publish-early
monitor-password monitor-htpasswd:passwd monitor-password monitor-htpasswd:passwd
db-root-password gen-root-password:passwd db-root-password gen-root-password:passwd
repman-password repman-password:passwd repman-password repman-password:passwd
database-list = {{ db_list }}
[publish-connection-parameter] [publish-connection-parameter]
<= monitor-publish <= monitor-publish
...@@ -476,6 +629,7 @@ recipe = slapos.cookbook:publish ...@@ -476,6 +629,7 @@ recipe = slapos.cookbook:publish
backend-url = ${nginx-parameter:backend-ssl-url} backend-url = ${nginx-parameter:backend-ssl-url}
url = ${repman-frontend:connection-secure_access} url = ${repman-frontend:connection-secure_access}
username = ${repman-parameter:username} username = ${repman-parameter:username}
database-list = {{ db_list }}
{% for name, value in publish_dict.items() -%} {% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }} {{ name }} = {{ value }}
{% endfor %} {% endfor %}
...@@ -545,6 +699,15 @@ command = ${template-proxysql-need-stop-start:rendered} ...@@ -545,6 +699,15 @@ command = ${template-proxysql-need-stop-start:rendered}
# Deploy replication-manager instance # Deploy replication-manager instance
# #
############################# #############################
[slap_connection]
# Kept for backward compatibility
computer_id = ${slap-connection:computer-id}
partition_id = ${slap-connection:partition-id}
server_url = ${slap-connection:server-url}
software_release_url = ${slap-connection:software-release-url}
key_file = ${slap-connection:key-file}
cert_file = ${slap-connection:cert-file}
[buildout] [buildout]
extends = extends =
{{ template_monitor }} {{ template_monitor }}
...@@ -552,7 +715,7 @@ parts = ...@@ -552,7 +715,7 @@ parts =
replication-manager replication-manager
monitor-base monitor-base
logrotate-entry-nginx logrotate-entry-nginx
ca-nginx-service nginx-launcher
publish-connection-parameter publish-connection-parameter
repman-frontend-promise repman-frontend-promise
repman-backend-promise repman-backend-promise
......
...@@ -28,8 +28,10 @@ mode = 0644 ...@@ -28,8 +28,10 @@ mode = 0644
extensions = jinja2.ext.do extensions = jinja2.ext.do
rendered= ${buildout:directory}/${:_buildout_section_name_} rendered= ${buildout:directory}/${:_buildout_section_name_}
supervisord-lib = {{ supervisord_lib }} supervisord-lib = {{ supervisord_lib }}
causace-lib = {{ caucase_library }}
import-list = import-list =
file supervisord_lib :supervisord-lib file supervisord_lib :supervisord-lib
file caucase :causace-lib
context = context =
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
key computer_id slap-configuration:computer key computer_id slap-configuration:computer
...@@ -48,6 +50,8 @@ context = ...@@ -48,6 +50,8 @@ context =
raw bash_bin {{ bash_location }}/bin/bash raw bash_bin {{ bash_location }}/bin/bash
raw jq_bin {{ jq_location }}/bin/jq raw jq_bin {{ jq_location }}/bin/jq
raw curl_bin {{ curl_location }}/bin/curl raw curl_bin {{ curl_location }}/bin/curl
raw openssl_bin {{ openssl_location }}/bin/openssl
raw caucase_bin_client {{ caucase_bin_client }}
${:extra-context} ${:extra-context}
extra-context = extra-context =
...@@ -56,6 +60,7 @@ extra-context = ...@@ -56,6 +60,7 @@ extra-context =
<= jinja2-template-base <= jinja2-template-base
template= {{ template_repman_cfg }} template= {{ template_repman_cfg }}
extra-context = extra-context =
key slave_instance_list slap-configuration:slave-instance-list
raw gowork_bin {{ gowork_bin }} raw gowork_bin {{ gowork_bin }}
raw haproxy_location {{ haproxy_location }} raw haproxy_location {{ haproxy_location }}
raw nginx_bin {{ nginx_location }}/sbin/nginx raw nginx_bin {{ nginx_location }}/sbin/nginx
...@@ -71,6 +76,7 @@ extra-context = ...@@ -71,6 +76,7 @@ extra-context =
raw proxysql_location {{ proxysql_location }} raw proxysql_location {{ proxysql_location }}
raw template_repman_manager_sh {{ template_repman_manager_sh }} raw template_repman_manager_sh {{ template_repman_manager_sh }}
raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }} raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
raw template_publish_slave_information {{ template_publish_slave }}
[template-mariadb-parameters] [template-mariadb-parameters]
bash = {{ bash_location }} bash = {{ bash_location }}
...@@ -83,6 +89,7 @@ template-mariadb-init-root = {{ template_init_root_sql }} ...@@ -83,6 +89,7 @@ template-mariadb-init-root = {{ template_init_root_sql }}
template-init-root-wrapper = {{ template_init_root_wrapper }} template-init-root-wrapper = {{ template_init_root_wrapper }}
template-mysqld-wrapper = {{ template_mysqld_wrapper }} template-mysqld-wrapper = {{ template_mysqld_wrapper }}
template-mysqld-need-start = {{ mysqld_start_template }} template-mysqld-need-start = {{ mysqld_start_template }}
template-manage-db = {{ template_slave_db_manage }}
link-binary = {{ dumps(mariadb_link_binary) }} link-binary = {{ dumps(mariadb_link_binary) }}
check-computer-memory-binary = {{ bin_directory }}/check-computer-memory check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
bin-directory = {{ bin_directory }} bin-directory = {{ bin_directory }}
......
{% set part_list = [] -%}
[slap_connection]
# Kept for backward compatibility
computer_id = ${slap-connection:computer-id}
partition_id = ${slap-connection:partition-id}
server_url = ${slap-connection:server-url}
software_release_url = ${slap-connection:software-release-url}
key_file = ${slap-connection:key-file}
cert_file = ${slap-connection:cert-file}
{% for _, database in slave_dict.items() -%}
{% set slave_reference = database['slave_reference'] -%}
{% do part_list.append('publish-slave-' ~ slave_reference) -%}
[publish-slave-{{ slave_reference }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_reference }}
{% if database.get('user') -%}
url_v4 = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":" ~ proxy_port ~ '/' ~ database['name'] }}
url = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ipv6 ~ "]:" ~ proxy_port ~ '/' ~ database['name'] }}
{% else -%}
url = {{ "mysql://" ~ ipv4 ~ ":" ~ proxy_port ~ database['name'] }}
url_v6 = {{ "mysql://[" ~ ip ~ "]:" ~ proxy_port ~ database['name'] }}
{% endif -%}
{% endfor %}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
parts =
{% for part in part_list -%}
{{ ' %s' % part }}
{% endfor -%}
...@@ -18,9 +18,10 @@ extends = ...@@ -18,9 +18,10 @@ extends =
../../component/socat/buildout.cfg ../../component/socat/buildout.cfg
../../component/rsync/buildout.cfg ../../component/rsync/buildout.cfg
../../component/jq/buildout.cfg ../../component/jq/buildout.cfg
../../component/openssl/buildout.cfg
../../stack/supervisord/buildout.cfg ../../stack/supervisord/buildout.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
../neoppod/software-common.cfg ../../stack/caucase/buildout.cfg
parts = parts =
slapos-cookbook slapos-cookbook
...@@ -28,14 +29,43 @@ parts = ...@@ -28,14 +29,43 @@ parts =
instance.cfg instance.cfg
template-mariadb.cfg template-mariadb.cfg
template-mysqld-wrapper template-mysqld-wrapper
caucase-eggs
gowork gowork
[mariadb]
location = ${mariadb-10.4:location}
[python] [python]
part = python3 part = python3
[gowork] [template-mysqld-wrapper]
# replication-manager does not build on golang 1.17 recipe = slapos.recipe.template:jinja2
golang = ${golang1.16:location} rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
mode = 644
template =
inline:{% raw %}#!/bin/sh -e
basedir='${mariadb:location}'
datadir='{{datadir}}'
marker=$datadir/.slapos_initializing
system=$datadir/.system
[ -d "$datadir/mysql" ] && [ ! -f "$marker" ] || {
find "$datadir/" -mindepth 1 ! \( -path $marker -o -path $system -o -path "*$system/*" \) -delete || mkdir "$datadir"
touch "$marker"
"$basedir/scripts/mysql_install_db" \
--defaults-file='{{defaults_file}}' \
--skip-name-resolve \
--auth-root-authentication-method=normal \
--basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
--datadir="$datadir"
rm "$marker"
}
{%- if environ is defined %}
{%- for variable in environ.splitlines() %}
{{ variable }} \
{%- endfor %}
{%- endif %}
exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
{% endraw %}
[instance.cfg] [instance.cfg]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -45,6 +75,7 @@ mode = 0644 ...@@ -45,6 +75,7 @@ mode = 0644
context = context =
key bash_location bash:location key bash_location bash:location
key bin_directory buildout:bin-directory key bin_directory buildout:bin-directory
key caucase_library caucase-jinja2-library:target
key config_toml_in config-toml.in:target key config_toml_in config-toml.in:target
key config_cluster_toml_in config-cluster-toml.in:target key config_cluster_toml_in config-cluster-toml.in:target
key coreutils_location coreutils:location key coreutils_location coreutils:location
...@@ -70,6 +101,7 @@ context = ...@@ -70,6 +101,7 @@ context =
key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
key nginx_conf_in nginx.conf.in:target key nginx_conf_in nginx.conf.in:target
key nginx_location nginx:location key nginx_location nginx:location
key openssl_location openssl:location
key percona_toolkit_location percona-toolkit:location key percona_toolkit_location percona-toolkit:location
key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
key repman_src_location git.signal18.io_signal18_repman:location key repman_src_location git.signal18.io_signal18_repman:location
...@@ -87,10 +119,12 @@ context = ...@@ -87,10 +119,12 @@ context =
key template_init_root_sql mariadb-init-root-sql:target key template_init_root_sql mariadb-init-root-sql:target
key template_init_root_wrapper init-root-wrapper-in:target key template_init_root_wrapper init-root-wrapper-in:target
key template_repman_cfg instance-repman.cfg:target key template_repman_cfg instance-repman.cfg:target
key template_publish_slave template-publish-slave-information:target
key template_slave_db_manage slave-db-manage.in:target
key unixodbc_location unixodbc:location key unixodbc_location unixodbc:location
key sysbench_location sysbench:location key sysbench_location sysbench:location
key proxysql_location proxysql:location key proxysql_location proxysql:location
raw caucase_bin_client ${buildout:bin-directory}/caucase
[download-file] [download-file]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
...@@ -126,6 +160,9 @@ link-binary = ...@@ -126,6 +160,9 @@ link-binary =
[template-mariadb-initial-setup] [template-mariadb-initial-setup]
<= download-file <= download-file
[template-publish-slave-information]
<= download-file
[template-my-cnf] [template-my-cnf]
<= download-file <= download-file
...@@ -146,3 +183,6 @@ link-binary = ...@@ -146,3 +183,6 @@ link-binary =
[proxy-need-start-stop.sh.in] [proxy-need-start-stop.sh.in]
<= download-file <= download-file
[slave-db-manage.in]
<= download-file
...@@ -10,6 +10,15 @@ ...@@ -10,6 +10,15 @@
"request": "instance-repman-input-schema.json", "request": "instance-repman-input-schema.json",
"response": "instance-repman-output-schema.json", "response": "instance-repman-output-schema.json",
"index": 0 "index": 0
},
"default-slave": {
"title": "Replication Manager Database",
"description": "Replication Manager Database",
"software-type": "default",
"request": "instance-repman-slave-input-schema.json",
"response": "instance-repman-output-schema.json",
"shared": true,
"index": 1
} }
} }
} }
...@@ -40,6 +40,7 @@ scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backu ...@@ -40,6 +40,7 @@ scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backu
backup-physical-type = "mariabackup" backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump" backup-logical-type = "mysqldump"
scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}" scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
#scheduler-db-servers-receiver-use-ssl ## set to true when ssl enabled on socat
prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }} prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
prov-proxy-memory = {{ parameter_dict['proxy-memory'] }} prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
...@@ -49,6 +50,7 @@ prov-db-memory = {{ parameter_dict['db-memory'] }} ...@@ -49,6 +50,7 @@ prov-db-memory = {{ parameter_dict['db-memory'] }}
prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}" prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}" prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
replication-use-ssl = true
test-inject-traffic = true test-inject-traffic = true
# failover # failover
failover-mode = "{{ parameter_dict['failover-mode'] }}" failover-mode = "{{ parameter_dict['failover-mode'] }}"
......
...@@ -21,7 +21,9 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}" ...@@ -21,7 +21,9 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
#db-servers-ignored-hosts = #db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
monitoring-address = "{{ parameter_dict['ipv4'] }}" monitoring-address = "{{ parameter_dict['ipv4'] }}"
monitoring-wait-retry = 40 monitoring-wait-retry = 1200
monitoring-ssl-cert = "{{ parameter_dict['ssl-cert'] }}"
monitoring-ssl-key = "{{ parameter_dict['ssl-key'] }}"
#haproxy = true #haproxy = true
#haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}" #haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
...@@ -48,6 +50,7 @@ backup-restic-password = "{{ parameter_dict['password'] }}" ...@@ -48,6 +50,7 @@ backup-restic-password = "{{ parameter_dict['password'] }}"
backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}" backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}" backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}" backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
backup-mysqldump-options = "--hex-blob --single-transaction --verbose --all-databases --add-drop-database"
# Mail configuration # Mail configuration
# Alert email sender (default "mrm@localhost") # Alert email sender (default "mrm@localhost")
......
#!/bin/bash #!/bin/bash
USER={{ parameter_dict['db-user'] }}
DBUSER={{ parameter_dict['db-user'] }}
PASSWORD={{ parameter_dict['db-password'] }} PASSWORD={{ parameter_dict['db-password'] }}
ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
DATADIR={{ parameter_dict['mysql-dir'] }}/ DATADIR={{ parameter_dict['mysql-dir'] }}/
MYSQL_CLIENT={{ parameter_dict['mysql-client'] }}
{% if parameter_dict['use-ipv6'] == True -%} {% if parameter_dict['use-ipv6'] == True -%}
{% set listen = "TCP6-LISTEN" -%} {% set listen = "TCP6-LISTEN" -%}
{% else -%} {% else -%}
...@@ -16,45 +18,45 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" " ...@@ -16,45 +18,45 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "
doneJob() doneJob()
{ {
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" & $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
} }
pauseJob() pauseJob()
{ {
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" & $MYSQL_CLIENT -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
} }
partialRestore() partialRestore()
{ {
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'" $MYSQL_CLIENT -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;" $MYSQL_CLIENT -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;" $MYSQL_CLIENT -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE" $MYSQL_CLIENT -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
done done
for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/ mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file=/etc/mysql/dbjob.cnf -e "set sql_log_bin=0;FLUSH TABLE $dir.$file" $MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done done
for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/ mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE $dir.$file" $MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done done
done done
for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/ mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE mysql.$file" $MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
done done
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e"flush privileges;start slave;" $MYSQL_CLIENT -e "flush privileges;start slave;"
} }
kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN) kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
...@@ -62,12 +64,12 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN) ...@@ -62,12 +64,12 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
for job in "${JOBS[@]}" for job in "${JOBS[@]}"
do do
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N)) TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER -N))
ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }')) ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
ID=($(echo $TASK | awk -F@ '{ print $1 }')) ID=($(echo $TASK | awk -F@ '{ print $1 }'))
#purge de past #purge de past
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';" $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
if [ "$ADDRESS" == "" ]; then if [ "$ADDRESS" == "" ]; then
echo "No $job needed" echo "No $job needed"
...@@ -77,14 +79,14 @@ do ...@@ -77,14 +79,14 @@ do
reseedmysqldump) reseedmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1 socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="reset master;set sql_log_bin=0" &>{{ parameter_dict['log-dir'] }}/dbjob.out
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;' $MYSQL_CLIENT -e 'start slave;'
;; ;;
flashbackmysqldump) flashbackmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1 socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER &>{{ parameter_dict['log-dir'] }}/dbjob.out
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;' $MYSQL_CLIENT -e 'start slave;'
;; ;;
reseedmariabackup) reseedmariabackup)
rm -rf $BACKUPDIR rm -rf $BACKUPDIR
...@@ -107,7 +109,7 @@ do ...@@ -107,7 +109,7 @@ do
;; ;;
mariabackup) mariabackup)
cd {{ parameter_dict['tmp-dir'] }} cd {{ parameter_dict['tmp-dir'] }}
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=root --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;; ;;
error) error)
cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
...@@ -116,10 +118,10 @@ do ...@@ -116,10 +118,10 @@ do
cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;; ;;
optimize) optimize)
mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out $MYSQL_CLIENT -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
;; ;;
restart) restart)
{{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out {{ parameter_dict['restart-script'] }} &>{{ parameter_dict['log-dir'] }}/dbjob.out
;; ;;
esac esac
doneJob doneJob
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
run_mysql () { run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \ {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
--protocol=socket -uroot -hlocalhost $@ --protocol=socket -hlocalhost "$@"
} }
if [ ! -f "{{ init_password_done }}" ]; then if [ ! -f "{{ init_password_done }}" ]; then
...@@ -19,6 +19,9 @@ if [ ! -f "{{ init_password_done }}" ]; then ...@@ -19,6 +19,9 @@ if [ ! -f "{{ init_password_done }}" ]; then
fi fi
echo "Setting mariabdb root password..."; echo "Setting mariabdb root password...";
run_mysql < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1; run_mysql < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
echo "remove unwanted users...";
run_mysql -e "DROP USER IF EXISTS 'root'@'$HOSTNAME';"
run_mysql -e "DROP USER IF EXISTS ''@'$HOSTNAME';"
echo "done" echo "done"
fi fi
...@@ -29,4 +32,7 @@ if [ ! -f "{{ upgrade_done }}" ]; then ...@@ -29,4 +32,7 @@ if [ ! -f "{{ upgrade_done }}" ]; then
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
touch {{ upgrade_done }}; touch {{ upgrade_done }};
fi fi
else
# only create databases now
run_mysql < {{ init_database_sql }} || exit 1;
fi fi
...@@ -2,16 +2,18 @@ ...@@ -2,16 +2,18 @@
-- or products like mysql-fabric won't work -- or products like mysql-fabric won't work
SET @@SESSION.SQL_LOG_BIN=0; SET @@SESSION.SQL_LOG_BIN=0;
# {{ parameter_dict["require-ssl"] }}
{% set ssl = parameter_dict["require-ssl"] -%}
CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ; CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ; GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ;
DROP DATABASE IF EXISTS test ; DROP DATABASE IF EXISTS test ;
......
SET @@SESSION.SQL_LOG_BIN=0; SET @@SESSION.SQL_LOG_BIN=0;
USE mysql; USE mysql;
{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
{% if mroonga %}
SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets; DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so'; #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
{% macro database(name, user, password) -%} CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
CREATE DATABASE IF NOT EXISTS `{{ name }}`;
{% macro database(name, user, password, ssl, charset="") -%}
{% set charset_collate = "" -%}
{% set charset_dict = {'utf8_general_ci': 'utf8',
'utf8_unicode_ci': 'utf8',
'utf8mb4_bin': 'utf8mb4',
'utf8mb4_general_ci': 'utf8mb4',
'utf8mb4_unicode_ci': 'utf8mb4',
'latin1_general_ci': 'latin1',
'latin1_bin': 'latin1',
'ascii_general_ci': 'ascii',
'ascii_bin': 'ascii'} -%}
{% if charset %}
{% set charset_collate = "DEFAULT CHARACTER SET %s" % charset -%}
{% if charset_dict.get(charset) -%}
{% set charset_collate = "DEFAULT CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
{% endif -%}
{% endif -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}` {{ charset_collate }};
{% if user -%} {% if user -%}
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}'; GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}'{% if ssl %} REQUIRE SSL{% endif %};
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}'; GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}'; GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
{%- endif %} {%- endif %}
{% endmacro -%} {% endmacro -%}
{% for entry in parameter_dict['database-list'] -%} {% for entry in parameter_dict['database-list'] -%}
{{ database(entry['name'], entry.get('user'), entry.get('password')) }} {{ database(entry['name'], entry.get('user'), entry.get('password'), entry.get('require_ssl', True) ~ '', entry.get('charset')) }}
{% endfor -%} {% endfor -%}
...@@ -4,13 +4,11 @@ curl () { ...@@ -4,13 +4,11 @@ curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@" {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
} }
# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
# Checking if mariadb start is needed # Checking if mariadb start is needed
#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start) CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
if [ $CODE -eq 200 ]; then if [ $CODE -eq 200 ]; then
sleep 5
echo "$CODE: Updating mysql configuration..." echo "$CODE: Updating mysql configuration..."
# update mysql configuration # update mysql configuration
{{ update_config }} {{ update_config }}
...@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then ...@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then
sleep 5 sleep 5
# check again if the service is still up... # check again if the service is still up...
{{ mariadb_controller }} status mariadb {{ mariadb_controller }} status mariadb
else
echo "CODE: $CODE. No need start!"
fi fi
#!/bin/bash
set -e
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
get_token () {
curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
}
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" --protocol=socket "$@"
}
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
DATADIR=$(curl -H "Authorization: Bearer ${TOKEN}" \ {{ secure_url }}/api/clusters/{{ cluster_name }}/topology/master | {{ jq_bin }} -r '.slaposDatadir')
# Only write or delete on master database else, we break replication.
if [ "$DATADIR" = "{{ partition_dir }}" ]; then
cat << EOF > {{ tmp_dir }}/.script.sql
use repman_slave_definition;
CREATE TABLE IF NOT EXISTS \`slave\` (
\`name\` varchar(80) NOT NULL,
\`state\` tinyint(1) DEFAULT NULL,
PRIMARY KEY (\`name\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
UPDATE \`slave\` set \`state\`=false;
{% for name in database_name.split(' ') -%}
{% if name -%}
REPLACE INTO \`slave\` VALUES ('{{ name }}', true);
{% endif -%}
{% endfor -%}
EOF
# Update requested slaves database
run_mysql < {{ tmp_dir }}/.script.sql
rm -f {{ tmp_dir }}/.script.sql
DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false");
RET=$?
if [ ! "$RET" = "0" ]; then
echo "Mysql command failed: $DBNAME"
exit $RET
fi
if [ -z "$DBNAME" ]; then
echo "No database for slave to remove.";
fi
for NAME in $DBNAME; do
if [ ! -z "$NAME" ]; then
echo "Deleting database $NAME..."
run_mysql -e "DROP DATABASE IF EXISTS $NAME";
run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
echo "Done."
fi
done
fi
...@@ -62,12 +62,6 @@ class TestRepman(SlapOSInstanceTestCase): ...@@ -62,12 +62,6 @@ class TestRepman(SlapOSInstanceTestCase):
token = resp.json()['token'] token = resp.json()['token']
headers = {"authorization": "Bearer " + token} headers = {"authorization": "Bearer " + token}
resp = requests.get(
urljoin(self.url, '/api/monitor'),
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
for i in range(20): for i in range(20):
resp = requests.get( resp = requests.get(
...@@ -87,3 +81,11 @@ class TestRepman(SlapOSInstanceTestCase): ...@@ -87,3 +81,11 @@ class TestRepman(SlapOSInstanceTestCase):
self.assertTrue(cluster['isProvision']) self.assertTrue(cluster['isProvision'])
self.assertTrue(cluster['isFailable']) self.assertTrue(cluster['isFailable'])
self.assertFalse(cluster['isDown']) self.assertFalse(cluster['isDown'])
resp = requests.get(
urljoin(self.url, '/api/clusters/cluster1/status'),
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
self.assertEqual(resp.json(), {"alive": "running"})
...@@ -60,7 +60,7 @@ def setUpModule(): ...@@ -60,7 +60,7 @@ def setUpModule():
class ERP5Mixin(object): class ERP5Mixin(object):
_test_software_url = erp5_software_release_url _test_software_url = erp5_software_release_url
_connexion_parameters_regex = re.compile(r"{\s*'_'\s*:\s*'(.*)'\s*}") _connexion_parameters_regex = re.compile(r"{.*}", re.DOTALL)
def _getERP5ConnexionParameters(self, software_type='export'): def _getERP5ConnexionParameters(self, software_type='export'):
slapos = self._getSlapos(software_type) slapos = self._getSlapos(software_type)
...@@ -69,7 +69,7 @@ class ERP5Mixin(object): ...@@ -69,7 +69,7 @@ class ERP5Mixin(object):
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
) )
print(out) print(out)
return json.loads(self._connexion_parameters_regex.search(out).group(1)) return json.loads(self._connexion_parameters_regex.search(out).group(0).replace("'", '"'))
def _getERP5Url(self, connexion_parameters, path=''): def _getERP5Url(self, connexion_parameters, path=''):
return urljoin(connexion_parameters['family-default-v6'], path) return urljoin(connexion_parameters['family-default-v6'], path)
......
...@@ -74,7 +74,7 @@ md5sum = bbef65b4edeb342f08309604ca3717d5 ...@@ -74,7 +74,7 @@ md5sum = bbef65b4edeb342f08309604ca3717d5
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = c10634353841bb09a847168b4add8d2f md5sum = cb41e15a1585973896a7645dc2ae3a5c
[template-zeo] [template-zeo]
filename = instance-zeo.cfg.in filename = instance-zeo.cfg.in
......
...@@ -297,7 +297,7 @@ config-name = {{ dumps(custom_name) }} ...@@ -297,7 +297,7 @@ config-name = {{ dumps(custom_name) }}
config-instance-count = {{ dumps(zope_parameter_dict['instance-count']) }} config-instance-count = {{ dumps(zope_parameter_dict['instance-count']) }}
config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }} config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
config-thread-amount = {{ dumps(zope_parameter_dict['thread-amount']) }} config-thread-amount = {{ dumps(zope_parameter_dict['thread-amount']) }}
config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }} config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 1)) }}
config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }} config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }} config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }} config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
......
...@@ -169,6 +169,7 @@ itsdangerous = 0.24 ...@@ -169,6 +169,7 @@ itsdangerous = 0.24
lock-file = 2.0 lock-file = 2.0
lxml = 4.6.3 lxml = 4.6.3
meld3 = 1.0.2 meld3 = 1.0.2
mock = 3.0.5
more-itertools = 5.0.0 more-itertools = 5.0.0
netaddr = 0.7.19 netaddr = 0.7.19
pathlib2 = 2.3.5 pathlib2 = 2.3.5
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment