Skip to content

slapos
slapos
Commit a8f35d7e authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

Merge branch 'master+ZODB4-wc2' into y/wc2-next 

* master+ZODB4-wc2: (56 commits)
  golang: v↑ go1.17  (1.17.3 -> 1.17.6)
  golang: v↑ go1.16  (1.16.10 -> 1.16.13)
  stack/erp5: Shorten haproxy socket path
  software/erp5/test: fix flaky test_activity_processing
  component/apache: version up 2.4.52, including security fixes.
  component/trafficserver: version up 9.1.1.
  stack/slapos: version up lxml 4.6.5, including security fixes.
  ninja: simplify configure step
  consul, packer, phantomjs: enable shared
  version up: zc.buildout 2.7.1+slapos015, slapos.recipe.build 0.53, slapos.recipe.cmmi 0.19
  version up: CMake 3.22.1
  component/userhosts: use github.com/figiel/hosts for compatibility with recent libc
  software/erp5/upgrade_test: backport new fixes in old software
  software/erp5/upgrade_test: skip software checks
  component/coturn: build with slapos libnsl
  component/apache-php: build with slapos libnsl
  repman: improve parameters used for cluster
  component/theia: version up 1.21.0
  software/theia: update python packages in python-language-server-requirements
  component/theia: update plugins with current versions
  ...
parents 1f35a79f 7005a464
Show whitespace changes
Inline Side-by-side
Showing with 3257 additions and 1787 deletions
component/apache-php/buildout.cfg
View file @ a8f35d7e
......@@ -23,6 +23,7 @@ extends =
../imagemagick/buildout.cfg
../icu/buildout.cfg
../openssl/buildout.cfg
../libnsl/buildout.cfg
[php-redis]
recipe = slapos.recipe.cmmi
......@@ -112,7 +113,7 @@ environment =
PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${libzip:location}/lib/pkgconfig
PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s
CPPFLAGS=-I${libzip:location}/include
LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -Wl,-rpath -Wl,${curl:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid -L${libzip:location}/lib -Wl,-rpath -Wl,${libzip:location}/lib -L${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${zstd:location}/lib
LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -Wl,-rpath -Wl,${curl:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid -L${libzip:location}/lib -Wl,-rpath -Wl,${libzip:location}/lib -L${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${zstd:location}/lib -L${libnsl:location}/lib -Wl,-rpath -Wl,${libnsl:location}/lib
TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
HOME=${apache:location}
......
component/apache/buildout.cfg
View file @ a8f35d7e
......@@ -39,9 +39,9 @@ configure-options =
[apache]
recipe = slapos.recipe.cmmi
shared = true
version = 2.4.51
version = 2.4.52
url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = d2793fc1c8cb8ba355cee877d1f2d46d
md5sum = a94ae42b84309d5ef6e613ae825b92fa
configure-options = --disable-static
--enable-authn-alias
--enable-bucketeer
......
component/chromedriver/buildout.cfg
View file @ a8f35d7e
......@@ -15,37 +15,29 @@ parts =
[chromedriver-wrapper-91]
<= chromedriver-wrapper
wrapper-name = chromedriver-91
part = ${chromedriver-91:location}
part = chromedriver-91
[chromedriver-wrapper-2.41]
<= chromedriver-wrapper
wrapper-name = chromedriver-2.41
part = ${chromedriver-2.41:location}
part = chromedriver-2.41
[chromedriver-wrapper]
# generate a wrapper named ${:wrapper-name} setting $LD_LIBRARY_PATH
# to run chromedriver installed from ${:part}
wrapper-name = chromedriver
part = ${chromedriver:location}
part = chromedriver
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:wrapper-name}
location = ${buildout:bin-directory}/${:part}
init =
self.buildout[options['part']]
install =
import os
chromedriver = options['part']
part = self.buildout[os.path.basename(chromedriver)]
part = self.buildout[options['part']]
with open(location, 'w') as f:
f.write("""#!/bin/sh -e
cd {}
export LD_LIBRARY_PATH=$PWD:{}
export PATH={}:$PATH
exec ./chromedriver "$@"
""".format(
chromedriver,
':'.join(part['library'].split()),
':'.join(part['path'].split()),
))
export LD_LIBRARY_PATH=%s
exec %s/chromedriver "$@"
""" % (':'.join(part['library'].split()), part['location']))
os.fchmod(f.fileno(), 0o755)
......@@ -67,18 +59,13 @@ md5sum-x86_64 = cc43ba0babbfff7f22b48165ec8e8c81
[chromedriver-download]
# Installs chromedriver ${version}.
# This chromedriver is not usable directly, it needs a wrapper.
version =
md5sum-x86_64 =
recipe = slapos.recipe.build
x86_64-linux-gnu = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_linux64.zip ${:md5sum-x86_64}
recipe = slapos.recipe.build:download-unpacked
url = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_${:_url}.zip
library =
${nss:location}/lib
${nspr:location}/lib
${libX11:location}/lib
path =
install =
import os, shutil
extract_dir = self.extract(self.download(*options[multiarch()].split()))
os.mkdir(location)
shutil.copy(os.path.join(extract_dir, 'chromedriver'), location)
[chromedriver-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux64
md5sum = ${:md5sum-x86_64}
component/chromium/buildout.cfg
View file @ a8f35d7e
......@@ -11,7 +11,6 @@
# in the system.
extends =
../alsa/buildout.cfg
../coreutils/buildout.cfg
../cups/buildout.cfg
../dbus/buildout.cfg
../findutils/buildout.cfg
......@@ -37,30 +36,30 @@ parts =
[chromium-wrapper]
# Install a chromium wrapper named ${:wrapper-name} in buildout's bin-directory,
# wrapping chromium installed in ${:part}
# Install a chromium wrapper named ${:part} in buildout's bin-directory,
# wrapping chromium installed in [${:part}]
# This [chromium-wrapper] installs the default chromium with name `chromium` and
# can also be used as a macro to install under a different name.
wrapper-name = chromium
part = ${chromium:location}
part = chromium
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:wrapper-name}
location = ${buildout:bin-directory}/${:part}
init =
self.buildout[options['part']]
install =
import os
chromium = options['part']
part = self.buildout[os.path.basename(chromium)]
part = self.buildout[options['part']]
with open(location, 'w') as f:
f.write("""#!/bin/sh -e
cd {}
d=%s
# this also needs a $FONTCONFIG_FILE set, otherwise system fonts will be used and if
# no system fonts are available, chrome will refuse to start with this error:
# FATAL:platform_font_linux.cc(83)] Check failed: InitDefaultFont(). Could not find the default font
export LD_LIBRARY_PATH=$PWD:{}
export PATH={}:$PATH
exec ./chrome --disable-setuid-sandbox --no-sandbox --disable-gpu $@
""".format(
chromium,
export LD_LIBRARY_PATH=$d:%s
export PATH=%s:$PATH
exec $d/chrome --disable-setuid-sandbox --no-sandbox --disable-gpu "$@"
""" % (
part['location'],
':'.join(part['library'].split()),
':'.join(part['path'].split()),
))
......@@ -68,13 +67,11 @@ install =
[chromium-wrapper-91]
<= chromium-wrapper
wrapper-name = chromium-91
part = ${chromium-91:location}
part = chromium-91
[chromium-wrapper-69]
<= chromium-wrapper
wrapper-name = chromium-69
part = ${chromium-69:location}
part = chromium-69
[chromium]
......@@ -101,24 +98,7 @@ generation-x86_64 = 1532051976706023
[chromium-download]
# macro to download a binary build of chromium and generate a
# wrapper as chrome-slapos in the part directory
# the full version tag
version =
# needs the following URL parts:
# revision is from the "base position"
revision_x86-64 =
# generation is in the final download URL
generation-x86_64 =
# this is the md5sum of the downloaded archive
md5sum-x86_64 =
recipe = slapos.recipe.build
x86_64-linux-gnu = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media ${:md5sum-x86_64}
recipe = slapos.recipe.build:download-unpacked
library =
${atk:location}/lib
${at-spi2-atk:location}/lib
......@@ -166,12 +146,7 @@ library =
${zlib:location}/lib
path =
${fontconfig:location}/bin
install =
url, md5sum = self.options[multiarch()].split()
extract_dir = self.extract(self.download(url, md5sum))
self.copyTree(guessworkdir(extract_dir), location)
# XXX adjust some permissions
import os
os.system("'${findutils:location}/bin/find' '%s' -type d -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
os.system("'${findutils:location}/bin/find' '%s' -type f -executable -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
os.system("'${findutils:location}/bin/find' '%s' -type f -exec '${coreutils:location}/bin/chmod' a+r {} \\;" % location)
[chromium-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
url = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media
md5sum = ${:md5sum-x86_64}
component/cmake/buildout.cfg
View file @ a8f35d7e
......@@ -9,8 +9,8 @@ parts =
[cmake]
recipe = slapos.recipe.cmmi
shared = true
url = https://cmake.org/files/v3.21/cmake-3.21.3.tar.gz
md5sum = c0feb5855604f68b09bdb3acb623619e
url = https://cmake.org/files/v3.22/cmake-3.22.1.tar.gz
md5sum = 83802be08c114bb34729300206488321
environment =
CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib
component/consul/buildout.cfg
View file @ a8f35d7e
......@@ -4,7 +4,14 @@ parts =
[consul]
recipe = slapos.recipe.build:download-unpacked
shared = true
version = 0.8.3
base = https://releases.hashicorp.com/consul/${:version}/consul_${:version}
i386-linux-gnu = ${:base}_linux_386.zip dfdc0eedd79baab7e6bc56c1582fd02e
x86_64-linux-gnu = ${:base}_linux_amd64.zip d6bc0898ea37ae2198370a9e1978d1bb
url = https://releases.hashicorp.com/consul/${:version}/consul_${:version}_${:_url}.zip
[consul:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url = linux_386
md5sum = dfdc0eedd79baab7e6bc56c1582fd02e
[consul:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux_amd64
md5sum = d6bc0898ea37ae2198370a9e1978d1bb
component/coturn/buildout.cfg
View file @ a8f35d7e
......@@ -3,6 +3,7 @@ extends =
../sqlite3/buildout.cfg
../openssl/buildout.cfg
../libevent/buildout.cfg
../libnsl/buildout.cfg
parts = coturn
[coturn]
......@@ -13,4 +14,4 @@ environment =
PATH=${sqlite3:location}/bin:%(PATH)s
CPPFLAGS=-I${openssl:location}/include -I${sqlite3:location}/include -I${libevent2:location}/include
CFLAGS=-I${libevent2:location}/include
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath -Wl,${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath -Wl,${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib -L${libnsl:location}/lib -Wl,-rpath -Wl,${libnsl:location}/lib
component/cpio/963304-remove-superfluous-declaration-of-program_name 0 → 100644
View file @ a8f35d7e
author Sergey Poznyakoff <gray@gnu.org>
https://git.savannah.gnu.org/cgit/cpio.git/commit/?id=641d3f489cf6238bb916368d4ba0d9325a235afb
* src/global.c: Remove superfluous declaration of program_name
diff --git a/src/global.c b/src/global.c
index fb3abe9..acf92bc 100644
--- a/src/global.c
+++ b/src/global.c
@@ -184,9 +184,6 @@ unsigned int warn_option = 0;
/* Extract to standard output? */
bool to_stdout_option = false;
-/* The name this program was run with. */
-char *program_name;
-
/* A pointer to either lstat or stat, depending on whether
dereferencing of symlinks is done for input files. */
int (*xstat) ();
component/cpio/buildout.cfg
View file @ a8f35d7e
# GNU cpio copies files into or out of a cpio or tar archive. The archive can be another file on the disk, a magnetic tape, or a pipe.
[buildout]
extends =
../patch/buildout.cfg
parts = cpio
[cpio]
recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnu.org/gnu/cpio/cpio-2.12.tar.bz2
md5sum = 93eea9f07c0058c097891c73e4955456
url = https://ftp.gnu.org/gnu/cpio/cpio-2.13.tar.bz2
md5sum = f3438e672e3fa273a7dc26339dd1eed6
patch-options = -p1
patches =
${:_profile_base_location_}/963304-remove-superfluous-declaration-of-program_name#1aadb39448eedaf49d03cf45f9d78ea4
environment =
PATH=${patch:location}/bin:%(PATH)s
component/firefox/buildout.cfg
View file @ a8f35d7e
......@@ -3,6 +3,7 @@ extends =
../alsa/buildout.cfg
../dbus/buildout.cfg
../fontconfig/buildout.cfg
../fonts/buildout.cfg
../gtk-2/buildout.cfg
../gtk-3/buildout.cfg
../libffi/buildout.cfg
......@@ -22,59 +23,52 @@ parts =
[firefox-wrapper]
# Install a firefox wrapper names ${:wrapper-name} in buildout's bin-directory,
# wrapping firefox installed in ${:part}
# Install a firefox wrapper named ${:part} in buildout's bin-directory,
# wrapping firefox installed in [${:part}]
# This [firefox-wrapper] installs the default firefox with name `firefox` and
# can also be used as a macro to install under a different name.
wrapper-name = firefox
part = ${firefox:location}
part = firefox
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:wrapper-name}
location = ${buildout:bin-directory}/${:part}
fonts-conf = ${firefox-default-fonts-conf:rendered}
init =
self.buildout[options['part']]
install =
import os
firefox = options['part']
part = self.buildout[os.path.basename(firefox)]
part = self.buildout[options['part']]
with open(location, 'w') as f:
f.write("""#!/bin/sh -e
cd {}
export LD_LIBRARY_PATH=$PWD:{}
export PATH={}:$PATH
d=%s
export LD_LIBRARY_PATH=$d:%s
export PATH=%s:$PATH
# BBB use a default fonts.conf for compatibility, but it's software instance
# responsability to build a fonts.conf with the fonts they want.
[ -z $FONTCONFIG_FILE ] && export FONTCONFIG_FILE=${firefox-default-fonts-conf:rendered}
exec ./firefox "$@"
""".format(
firefox,
[ "$FONTCONFIG_FILE" ] || export FONTCONFIG_FILE=%s
exec $d/firefox "$@"
""" % (
part['location'],
':'.join(part['library'].split()),
':'.join(part['path'].split()),
options['fonts-conf'],
))
os.fchmod(f.fileno(), 0o755)
[firefox-wrapper-78]
<= firefox-wrapper
wrapper-name = firefox-78
part = ${firefox-78:location}
part = firefox-78
[firefox-wrapper-68]
<= firefox-wrapper
wrapper-name = firefox-68
part = ${firefox-68:location}
part = firefox-68
[firefox-wrapper-60]
<= firefox-wrapper
wrapper-name = firefox-60
part = ${firefox-60:location}
part = firefox-60
[firefox-wrapper-52]
<= firefox-wrapper
wrapper-name = firefox-52
part = ${firefox-52:location}
[firefox-wrapper-51]
<= firefox-wrapper
wrapper-name = firefox-51
part = ${firefox-51:location}
part = firefox-52
[firefox-default-fonts-conf]
recipe = slapos.recipe.template:jinja2
......@@ -124,20 +118,12 @@ version = 52.9.0esr
i686-md5sum = 9aa18888b7812670208490609d75c9bc
x86_64-md5sum = 9336d70f45070c743d08e5473b783a7a
[firefox-51]
<= firefox-download
version = 51.0.1
i686-md5sum = 9a5b67e9d759a1e4df004294a24b2b43
x86_64-md5sum = bd93f2652d1d90d59ae462439a93c85f
[firefox-download]
recipe = slapos.recipe.build
slapos_promise =
file:firefox
i386-linux-gnu = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 ${:i686-md5sum}
x86_64-linux-gnu = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 ${:x86_64-md5sum}
url = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/${:_url}/en-US/firefox-${:version}.tar.bz2
library =
${alsa:location}/lib
......@@ -185,12 +171,19 @@ path =
${fontconfig:location}/bin
install =
url, md5sum = options[multiarch()].split()
extract_dir = self.extract(self.download(url, md5sum))
self.copyTree(guessworkdir(extract_dir), location)
self.copyTree(guessworkdir(self.extract(self.download())), location)
${:post-install}
post-install =
[firefox-download:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url = linux-i686
md5sum = ${:i686-md5sum}
[firefox-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux-x86_64
md5sum = ${:x86_64-md5sum}
[geckodriver]
# Current geckodriver installed as ${buildout:bin-directory}/geckodriver
<= geckodriver-0.24.0
......@@ -207,47 +200,26 @@ version = 0.22.0
i686-md5sum = 6de7544753fda56fbaa8382dcac99aaa
x86_64-md5sum = 81746200ce5841e00cabf3b8ea7db542
[geckodriver-0.21.0]
<= geckodriver-base
version = 0.21.0
i686-md5sum = 9fc1657dd1b94272d0cdb3b29ca80f79
x86_64-md5sum = 6bc36d4fd4975e296cdb3fa3c5e26a41
[geckodriver-0.19.0]
<= geckodriver-base
version = 0.19.0
i686-md5sum = 07cd383c8aef8ea5ef194a506141afd6
x86_64-md5sum = ca6935a72fd0527d15a78a17a35e56e8
[geckodriver-0.17.0]
<= geckodriver-base
version = 0.17.0
i686-md5sum = 79b1a158f96d29942a111c0905f1c807
x86_64-md5sum = be18faeea6e7db9db6990d8667e2298f
[geckodriver-0.16.1]
<= geckodriver-base
version = 0.16.1
i686-md5sum = not not on github
x86_64-md5sum = 57dfd55d4759d9878eb75b4c0123d00c
[geckodriver-0.14.0]
<= geckodriver-base
version = 0.14.0
i686-md5sum = b5836f5a944fe9f3ed1a67c7b342c6a7
x86_64-md5sum = 4a185d3179862a35104603b9274452e7
[geckodriver-base]
# Installs geckodriver ${version}
recipe = slapos.recipe.build
url = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-${:_url}.tar.gz
location = ${buildout:bin-directory}/${:_buildout_section_name_}
install =
import os, shutil
extract_dir = self.extract(self.download(options['url'], options['md5sum']))
shutil.copy(os.path.join(extract_dir, 'geckodriver'), location)
i386-linux-gnu = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux32.tar.gz ${:i686-md5sum}
x86_64-linux-gnu = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux64.tar.gz ${:x86_64-md5sum}
[geckodriver-base:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url = linux32
md5sum = ${:i686-md5sum}
install =
import shutil
url, md5sum = options[multiarch()].split()
extract_dir = self.extract(self.download(url, md5sum))
shutil.copy(extract_dir + '/geckodriver', location)
[geckodriver-base:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux64
md5sum = ${:x86_64-md5sum}
component/golang/buildout.cfg
View file @ a8f35d7e
......@@ -59,8 +59,8 @@ environment-extra =
[golang1.16]
<= golang-common
url = https://golang.org/dl/go1.16.10.src.tar.gz
md5sum = 49f0a54f0bdcba297bac194d8dafe431
url = https://golang.org/dl/go1.16.13.src.tar.gz
md5sum = 1c076f952d9af57590a36fa7d36f695a
# go1.16 needs go1.4 to bootstrap
environment-extra =
......@@ -68,8 +68,8 @@ environment-extra =
[golang1.17]
<= golang-common
url = https://golang.org/dl/go1.17.3.src.tar.gz
md5sum = 3ea82e5966275f405f0db4f52511bb6e
url = https://golang.org/dl/go1.17.6.src.tar.gz
md5sum = dc57f93f323e9f8189e5ffc1f223e346
# go1.17 needs go1.4 to bootstrap
environment-extra =
......
component/headless-chromium/buildout.cfg
View file @ a8f35d7e
......@@ -45,9 +45,9 @@ gclient-location = ${buildout:parts-directory}/${:_buildout_section_name_}
# called "src".
name = src
# There's nothing special about version 92.0.4515.107. It just happened
# to be the current Chromium stable version at the time of writing.
version = 92.0.4515.107
# 96.0.4664.129 version is the latest stable version in December 2021.
# Note that we need a version compiling without python2
version = 96.0.4664.129
[headless-chromium]
......
component/libnsl/buildout.cfg 0 → 100644
View file @ a8f35d7e
[buildout]
extends =
../xz-utils/buildout.cfg
../libtirpc/buildout.cfg
[libnsl]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/thkukuk/libnsl/releases/download/v1.3.0/libnsl-1.3.0.tar.xz
md5sum = 9214f674bd0c2bcfdd6c1da0cadb061f
make-options =
CFLAGS=-I${libtirpc:location}/include/tirpc
LDFLAGS='-L${libtirpc:location}/lib -Wl,-rpath=${libtirpc:location}/lib -ltirpc'
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
component/libreoffice-bin/buildout.cfg
View file @ a8f35d7e
......@@ -8,26 +8,15 @@ parts =
[libreoffice-bin]
recipe = slapos.recipe.build
# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
version = 5.2.4.2
url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/%s/LibreOffice_${:version}_Linux_%s_rpm.tar.gz
# supported architectures md5sums
md5sum_x86 = 7a0b33a2d18f06143258c428c32de213
md5sum_x86-64 = cbea6cd17063b5bcbe0cb32f7819f0cf
url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/${:_url1}/LibreOffice_${:version}_${:_url2}_rpm.tar.gz
# where office code can be found?
officedir = libreoffice5.2
install =
import os
import sys
arch = {
'i386-linux-gnu': ('x86', 'x86'),
'x86_64-linux-gnu': ('x86_64', 'x86-64'),
}[multiarch()]
url = options['url'] % arch
md5sum = options['md5sum_' + arch[1]]
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
workdir = guessworkdir(self.extract(self.download()))
storagedir = os.path.join(workdir, 'storage')
os.mkdir(storagedir)
rpmsdir = os.path.join(workdir, 'RPMS')
......@@ -48,3 +37,13 @@ install =
# helper binaries
cpio = ${cpio:location}/bin/cpio
rpm2cpio = ${rpm2cpio:target}
[libreoffice-bin:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url1 = x86
_url2 = Linux_x86
md5sum = 7a0b33a2d18f06143258c428c32de213
[libreoffice-bin:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url1 = x86_64
_url2 = Linux_x86-64
md5sum = cbea6cd17063b5bcbe0cb32f7819f0cf
component/libtirpc/buildout.cfg 0 → 100644
View file @ a8f35d7e
[buildout]
parts = libtirpc
[libtirpc]
recipe = slapos.recipe.cmmi
shared = true
url = https://downloads.sourceforge.net/libtirpc/libtirpc-1.3.2.tar.bz2
md5sum = cf4ca51f3fc401bea61c702c69171ab0
configure-options = --disable-gssapi
component/nginx/buildout.cfg
View file @ a8f35d7e
......@@ -61,8 +61,8 @@ mime = ${nginx:location}/conf/mime.types
[nginx-push-stream-module]
recipe = slapos.recipe.build:download-unpacked
shared = true
url = https://github.com/wandenberg/nginx-push-stream-module/archive/0.4.0.tar.gz
md5sum = d9cba621b8739e13bdb5e02b9425f205
url = https://github.com/wandenberg/nginx-push-stream-module/archive/0.5.5.tar.gz
md5sum = 9bb5237a93c957130aaf7033ed3c0989
[nginx-push-stream]
<= nginx-common
......
component/ninja/buildout.cfg
View file @ a8f35d7e
[buildout]
extends =
../defaults.cfg
parts =
ninja
......@@ -10,7 +8,7 @@ shared = true
version = 1.10.2
url = https://github.com/ninja-build/ninja/archive/refs/tags/v${:version}.tar.gz
md5sum = 639f75bc2e3b19ab893eaf2c810d4eb4
configure-command = ${python:executable} configure.py
configure-command = ${buildout:executable} configure.py
configure-options = --bootstrap --verbose
make-binary = true
post-install =
......
component/p7zip/buildout.cfg
View file @ a8f35d7e
[buildout]
extends =
../patch/buildout.cfg
parts =
p7zip
[p7zip]
recipe = slapos.recipe.cmmi
shared = true
url = http://downloads.sf.net/project/p7zip/p7zip/${:version}/p7zip_${:version}_src_all.tar.bz2
version = 9.38.1
md5sum = 6cba8402ccab2370d3b70c5e28b3d651
url = https://downloads.sourceforge.net/project/p7zip/p7zip/${:version}/p7zip_${:version}_src_all.tar.bz2
version = 16.02
md5sum = a0128d661cfe7cc8c121e73519c54fbf
patch-binary = ${patch:location}/bin/patch
patch-options = -p1
patch-base = https://sources.debian.org/data/main/p/p7zip/16.02%2Bdfsg-8/debian/patches/
patches =
${:patch-base}/12-CVE-2016-9296.patch#f0f5dc36d97d9a71a6da3adba717fbf2
${:patch-base}/13-CVE-2017-17969.patch#8399bdc913cf8cf8194ecdb5487f48d9
${:patch-base}/14-Fix-g%2B%2B-warning.patch#9db8fbfbc8d624669c095240621460bd
${:patch-base}/15-Fix-FTBFS-gcc10.patch#50ebfd79c715d8b5d649e57a1da2ce34
configure-command = rm -r DOC
make-binary =
make-options =
make-targets = make -j1 7z install
DEST_HOME=@@LOCATION@@
make-targets = 7z
post-install = make install DEST_HOME=%(location)s
component/packer/buildout.cfg
View file @ a8f35d7e
......@@ -4,7 +4,14 @@ parts =
[packer]
recipe = slapos.recipe.build:download-unpacked
shared = true
version = 0.7.5
base = https://releases.hashicorp.com/packer/${:version}/packer_${:version}
i386-linux-gnu = ${:base}_linux_386.zip a545108a0ccfde7c1e74de6c4e6fdded
x86_64-linux-gnu = ${:base}_linux_amd64.zip f343d709b84db494e8d6ec38259aa4a6
url = https://releases.hashicorp.com/packer/${:version}/packer_${:version}_${:_url}.zip
[packer:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url = linux_386
md5sum = a545108a0ccfde7c1e74de6c4e6fdded
[packer:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux_amd64
md5sum = f343d709b84db494e8d6ec38259aa4a6
component/phantomjs/buildout.cfg
View file @ a8f35d7e
......@@ -7,23 +7,36 @@ parts =
[phantomjs]
recipe = slapos.recipe.build
shared = true
slapos_promise =
file:phantomjs-slapos
base = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux
i386-linux-gnu = ${:base}-i686.tar.bz2 9c1426eef5b04679d65198b1bdd6ef88
x86_64-linux-gnu = ${:base}-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
url = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-${:_url}.tar.bz2
install =
import os
extract_dir = self.extract(self.download(*options[multiarch()].split()))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, location)
self.copyTree(guessworkdir(self.extract(self.download())), location)
with open(os.path.join(location, "phantomjs-slapos"), 'w') as wrapper:
wrapper.write("""#!/bin/sh -e
cd %(location)s
export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/bin/phantomjs "$@"
""" % options)
export LD_LIBRARY_PATH=%s
export PATH=%s:$PATH
exec %s/bin/phantomjs "$@"
""" % (
':'.join(options['library'].split()),
':'.join(options['path'].split()),
location,
))
os.fchmod(wrapper.fileno(), 0o755)
library =
${freetype:location}/lib
${fontconfig:location}/lib
${libexpat:location}/lib
path =
${fontconfig:location}/bin
[phantomjs:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url = linux-i686
md5sum = 9c1426eef5b04679d65198b1bdd6ef88
[phantomjs:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux-x86_64
md5sum = f278996c3edd0e8d8ec4893807f27d71
component/postfix/buildout.cfg
View file @ a8f35d7e
......@@ -8,6 +8,7 @@ extends =
../patch/buildout.cfg
../pcre/buildout.cfg
../cyrus-sasl/buildout.cfg
../libnsl/buildout.cfg
../m4/buildout.cfg
[postfix]
......@@ -20,7 +21,9 @@ patch-options = -p1
patches =
${:_profile_base_location_}/noroot.patch#05fc6333e05576ea8e5a49f27a6ef951
configure-command = make
configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
configure-options = makefiles CCARGS=${:configure-options-CCARGS} AUXLIBS=${:configure-options-AUXLIBS}
configure-options-CCARGS = '-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl -I${libnsl:location}/include'
configure-options-AUXLIBS = '-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -L${libtirpc:location}/lib -L${libnsl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib -Wl,-rpath=${libnsl:location}/lib'
make-targets = non-interactive-package install_root=${:location}
environment =
PATH=${patch:location}/bin:${m4:location}/bin:%(PATH)s
component/python-2.7/buildout.cfg
View file @ a8f35d7e
......@@ -42,6 +42,7 @@ patches =
${:_profile_base_location_}/pytracemalloc_pep445.patch#9f3145817afa2b7fad801fde8447e396
${:_profile_base_location_}/disabled_module_list.patch#e038a8016475574c810cbaaf0e42f4ac
${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
${:_profile_base_location_}/py27-subproc-closefds-fast.patch#e495e44491694a8972da11739206f2e6
url =
http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
configure-options =
......
component/python-2.7/py27-subproc-closefds-fast.patch 0 → 100644
View file @ a8f35d7e
From 9a79fa64c6298998d04a5ae6d699222a93cd0cb8 Mon Sep 17 00:00:00 2001
From: Kirill Smelkov <kirr@nexedi.com>
Date: Fri, 24 Dec 2021 13:55:08 +0300
Subject: [PATCH] [py27] subprocess: Use /proc/self/fd/ to know opened file
descriptors
... to avoid O(MAXFD) slowdown on closefds=True.
This is semantic backport of what Python3 does.
---
Lib/subprocess.py | 23 ++++++++++++++++++++++-
1 file changed, 22 insertions(+), 1 deletion(-)
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 1f2da0ffbe8..300557220ac 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -869,7 +869,28 @@ def pipe_cloexec(self):
def _close_fds(self, but):
- if hasattr(os, 'closerange'):
+ # try to retrieve list of opened file descriptors to avoid being slow on large MAXFD
+ # https://www.erp5.com/project_section/vifib/forum/python2--subprocess.Popen-is-slow-when-close_fds-True-and-nofile-ulimit-is-high-PO01sREVXn
+ # https://bugs.python.org/issue8052
+ # https://github.com/python/cpython/commit/8facece99a59
+ fdv = None
+ try:
+ fdv = os.listdir('/proc/self/fd')
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ if fdv is not None:
+ for fd_ in fdv:
+ fd = int(fd_)
+ if (0 <= fd <= 2) or fd == but:
+ continue
+ try:
+ os.close(fd)
+ except Exception: # e.g. not KeyboardInterrupt nor SystemExit
+ pass
+
+ # fallback to closing all file descriptors in 3..MAXFD range
+ elif hasattr(os, 'closerange'):
os.closerange(3, but)
os.closerange(but + 1, MAXFD)
else:
--
2.30.2
component/replication-manager/buildout.cfg
View file @ a8f35d7e
......@@ -28,5 +28,6 @@ command = set -e
<= go-git-package
go.importpath = github.com/signal18/replication-manager
repository = https://github.com/signal18/replication-manager
branch = 2.1
revision = 9167a82c81af8f7be41cf51bc9be8a37dc3d8c03
#branch = 2.1
branch = develop
revision = 838ffeb205ea4477f1c1bda607940fa07d8bcf51
\ No newline at end of file
component/theia/buildout.hash.cfg
View file @ a8f35d7e
......@@ -19,4 +19,4 @@ md5sum = 8157c22134200bd862a07c6521ebf799
[yarn.lock]
_update_hash_filename_ = yarn.lock
md5sum = 067d2db611b21f77885f3adfd7f81453
md5sum = e0341b7a715cc757a671aef902e6767d
component/theia/download-plugins.cfg
View file @ a8f35d7e
# This file is automatically generated from generate_download_plugins_cfg.py
# Do not edit directly.
[theia-download-plugins]
urls = vscode-bat https://open-vsx.org/api/vscode/bat/1.54.3/file/vscode.bat-1.54.3.vsix 77de5b930e0b9baa90059abb1711bd22
vscode-clojure https://open-vsx.org/api/vscode/clojure/1.54.3/file/vscode.clojure-1.54.3.vsix f89e90ad2e808e58bc1b4d24fbbf680f
vscode-coffeescript https://open-vsx.org/api/vscode/coffeescript/1.54.3/file/vscode.coffeescript-1.54.3.vsix 0800e4a04e9a0cfa4810c0461f26ab00
vscode-configuration-editing https://open-vsx.org/api/vscode/configuration-editing/1.54.3/file/vscode.configuration-editing-1.54.3.vsix 44c5f0439733f93813482138656001a0
vscode-cpp https://open-vsx.org/api/vscode/cpp/1.54.3/file/vscode.cpp-1.54.3.vsix 851461a32d8a69b817d85f82bf9b4035
vscode-csharp https://open-vsx.org/api/vscode/csharp/1.54.3/file/vscode.csharp-1.54.3.vsix e9f69d28870a3ba1fa8cf30d6ceaabcf
vscode-css https://open-vsx.org/api/vscode/css/1.54.3/file/vscode.css-1.54.3.vsix 8539c80c26897a95e8a55d1640c205dc
vscode-css-language-features https://open-vsx.org/api/vscode/css-language-features/1.54.3/file/vscode.css-language-features-1.54.3.vsix 530806dfef61b6936bce7dccd79a9383
vscode-debug-auto-launch https://open-vsx.org/api/vscode/debug-auto-launch/1.54.3/file/vscode.debug-auto-launch-1.54.3.vsix 5840891614f42fd22a7adcb812a2a26b
vscode-docker https://open-vsx.org/api/vscode/docker/1.54.3/file/vscode.docker-1.54.3.vsix a4e3cdd03833fe6bf3f5045fc5be4433
vscode-emmet https://open-vsx.org/api/vscode/emmet/1.54.3/file/vscode.emmet-1.54.3.vsix 5786c95921794ed3188083a7f804fa29
vscode-fsharp https://open-vsx.org/api/vscode/fsharp/1.54.3/file/vscode.fsharp-1.54.3.vsix 028f4ec9593533ce38603032b3f2821e
vscode-git https://open-vsx.org/api/vscode/git/1.54.3/file/vscode.git-1.54.3.vsix da2f2c0626b4a254a660ae20b0c97611
vscode-go https://open-vsx.org/api/vscode/go/1.54.3/file/vscode.go-1.54.3.vsix 9f095a75e4137079351150722d091256
vscode-groovy https://open-vsx.org/api/vscode/groovy/1.54.3/file/vscode.groovy-1.54.3.vsix 7ae14c40786311fda29059178f30c310
vscode-grunt https://open-vsx.org/api/vscode/grunt/1.54.3/file/vscode.grunt-1.54.3.vsix f7894259f1fa60e939ff2d9b49138a02
vscode-gulp https://open-vsx.org/api/vscode/gulp/1.54.3/file/vscode.gulp-1.54.3.vsix cc5c570dda3b69ba3e1e1ffbe6f3a0c8
vscode-handlebars https://open-vsx.org/api/vscode/handlebars/1.54.3/file/vscode.handlebars-1.54.3.vsix ea433237fb8940ba2ff075253f58b9a6
vscode-hlsl https://open-vsx.org/api/vscode/hlsl/1.54.3/file/vscode.hlsl-1.54.3.vsix d111abc12ad0e9f6d15e3b3c522e3072
vscode-html https://open-vsx.org/api/vscode/html/1.54.3/file/vscode.html-1.54.3.vsix f8fa8f2e54ca8f3d3d64c60cb617bfde
urls = vscode-bat https://open-vsx.org/api/vscode/bat/1.62.3/file/vscode.bat-1.62.3.vsix 61e32be47fbd5a88907bf7b2e8498621
vscode-clojure https://open-vsx.org/api/vscode/clojure/1.62.3/file/vscode.clojure-1.62.3.vsix 9b44f96739dcac6d5117859fd72fbdba
vscode-coffeescript https://open-vsx.org/api/vscode/coffeescript/1.62.3/file/vscode.coffeescript-1.62.3.vsix 54fa3459253d7817535c4151f704c84c
vscode-configuration-editing https://open-vsx.org/api/vscode/configuration-editing/1.62.3/file/vscode.configuration-editing-1.62.3.vsix 941b13b5f15a4b4de122f68987b79e0c
vscode-cpp https://open-vsx.org/api/vscode/cpp/1.62.3/file/vscode.cpp-1.62.3.vsix aefd964c2bddc59c87f14e76aee959fa
vscode-csharp https://open-vsx.org/api/vscode/csharp/1.62.3/file/vscode.csharp-1.62.3.vsix 38cde5827cf2f88c8a2988f559a5a02f
vscode-css https://open-vsx.org/api/vscode/css/1.62.3/file/vscode.css-1.62.3.vsix 56ba10d5d7d63140b7b68c47be090c72
vscode-css-language-features https://open-vsx.org/api/vscode/css-language-features/1.62.3/file/vscode.css-language-features-1.62.3.vsix 9bc4e515a491a9e4749084aa1753ddac
vscode-debug-auto-launch https://open-vsx.org/api/vscode/debug-auto-launch/1.62.3/file/vscode.debug-auto-launch-1.62.3.vsix 9236132f864d689e839b51bb52d28def
vscode-docker https://open-vsx.org/api/vscode/docker/1.62.3/file/vscode.docker-1.62.3.vsix f27713e5ca6d1d08cb4b58668d59db66
vscode-emmet https://open-vsx.org/api/vscode/emmet/1.62.3/file/vscode.emmet-1.62.3.vsix 5ad6c366cd49cb8786eec28908fabf1c
vscode-fsharp https://open-vsx.org/api/vscode/fsharp/1.62.3/file/vscode.fsharp-1.62.3.vsix 9450d03e78d9dc671b8068c9a73e5570
vscode-git https://open-vsx.org/api/vscode/git/1.62.3/file/vscode.git-1.62.3.vsix 76390cc667080a62bdebc7302134be08
vscode-go https://open-vsx.org/api/vscode/go/1.62.3/file/vscode.go-1.62.3.vsix d7b620cd630dfd126100a1c7366a665b
vscode-groovy https://open-vsx.org/api/vscode/groovy/1.62.3/file/vscode.groovy-1.62.3.vsix ed8b4d51bc41d68cfbe2ac72f6884d2d
vscode-grunt https://open-vsx.org/api/vscode/grunt/1.62.3/file/vscode.grunt-1.62.3.vsix 96800d71f8a3842d0413684550bc5e0f
vscode-gulp https://open-vsx.org/api/vscode/gulp/1.62.3/file/vscode.gulp-1.62.3.vsix e50eb3f086fa2053249c0d817a208f57
vscode-handlebars https://open-vsx.org/api/vscode/handlebars/1.62.3/file/vscode.handlebars-1.62.3.vsix b762f43fff8bd5629dd9dc6454c3045b
vscode-hlsl https://open-vsx.org/api/vscode/hlsl/1.62.3/file/vscode.hlsl-1.62.3.vsix 899e646896216df4ed2d46e8f17812c9
vscode-html https://open-vsx.org/api/vscode/html/1.62.3/file/vscode.html-1.62.3.vsix a3448adf65e4158a250b37d470ce396c
vscode-html-language-features https://open-vsx.org/api/vscode/html-language-features/1.52.1/file/vscode.html-language-features-1.52.1.vsix a83619f2924ec58713691e76f17c7fb6
vscode-ini https://open-vsx.org/api/vscode/ini/1.54.3/file/vscode.ini-1.54.3.vsix 3f758e07aad99c4ba15fbc59c0807e2d
vscode-jake https://open-vsx.org/api/vscode/jake/1.54.3/file/vscode.jake-1.54.3.vsix 60b90d9620e0b85d8effe0a5265079f0
vscode-java https://open-vsx.org/api/vscode/java/1.54.3/file/vscode.java-1.54.3.vsix 9b933657f3e5eedfe0247006a268ac92
vscode-javascript https://open-vsx.org/api/vscode/javascript/1.54.3/file/vscode.javascript-1.54.3.vsix 32860b0b1bea5b2fc15b4d7eb5eaf630
ms-vscode-js-debug https://open-vsx.org/api/ms-vscode/js-debug/1.52.2/file/ms-vscode.js-debug-1.52.2.vsix f0f0c11c9c1855b6d3c87b230790c949
vscode-json https://open-vsx.org/api/vscode/json/1.54.3/file/vscode.json-1.54.3.vsix 7dda28a6cdcf703d0af938ffd2a60a56
vscode-ini https://open-vsx.org/api/vscode/ini/1.62.3/file/vscode.ini-1.62.3.vsix ee71130f95861b61d92d644c3cd8ded7
vscode-jake https://open-vsx.org/api/vscode/jake/1.62.3/file/vscode.jake-1.62.3.vsix fb7d81d41eaa04d53183bd6163e83ed3
vscode-java https://open-vsx.org/api/vscode/java/1.62.3/file/vscode.java-1.62.3.vsix d8239ae11719762e8b8152010bca50c9
vscode-javascript https://open-vsx.org/api/vscode/javascript/1.62.3/file/vscode.javascript-1.62.3.vsix 6f5babbbd2f437f2e34df2feb97ce595
ms-vscode-js-debug https://open-vsx.org/api/ms-vscode/js-debug/1.61.0/file/ms-vscode.js-debug-1.61.0.vsix 6457840c5c8d43e5b7a39fabf72b7367
vscode-json https://open-vsx.org/api/vscode/json/1.62.3/file/vscode.json-1.62.3.vsix 9e0e7b25a2d4c5df88ea3a75cf60372b
vscode-json-language-features https://open-vsx.org/api/vscode/json-language-features/1.45.1/file/vscode.json-language-features-1.45.1.vsix b7aa9d96d67792dedf9c2558880f38c0
vscode-less https://open-vsx.org/api/vscode/less/1.54.3/file/vscode.less-1.54.3.vsix 1edc8fca796c520d7e1aa89c069070a4
vscode-log https://open-vsx.org/api/vscode/log/1.54.3/file/vscode.log-1.54.3.vsix 9552f796a8f2ca38950d0f25df21f9cb
vscode-lua https://open-vsx.org/api/vscode/lua/1.54.3/file/vscode.lua-1.54.3.vsix ed6a4a09c8505e0f1437d2530c69bfe5
vscode-make https://open-vsx.org/api/vscode/make/1.54.3/file/vscode.make-1.54.3.vsix 996b8d1a2292318d177ae4951b67c990
vscode-markdown https://open-vsx.org/api/vscode/markdown/1.54.3/file/vscode.markdown-1.54.3.vsix 4188ca7c2bda0037e88f24cc443a24f8
vscode-markdown-language-features https://open-vsx.org/api/vscode/markdown-language-features/1.39.2/file/vscode.markdown-language-features-1.39.2.vsix 9ab6e372796b3bacede80fcbcfd35640
vscode-merge-conflict https://open-vsx.org/api/vscode/merge-conflict/1.54.3/file/vscode.merge-conflict-1.54.3.vsix 554eeb664788a7b5b80ce1da4795f0b1
vscode-npm https://open-vsx.org/api/vscode/npm/1.54.3/file/vscode.npm-1.54.3.vsix e6eeeedebf3b6986f1d7043c94807d01
ms-vscode-node-debug https://open-vsx.org/api/ms-vscode/node-debug/1.44.8/file/ms-vscode.node-debug-1.44.8.vsix 367bac016859084e63acf792ed4a1aeb
ms-vscode-node-debug2 https://open-vsx.org/api/ms-vscode/node-debug2/1.42.5/file/ms-vscode.node-debug2-1.42.5.vsix 7882d7d90d18820be17aa5f3e12e71e1
vscode-objective-c https://open-vsx.org/api/vscode/objective-c/1.54.3/file/vscode.objective-c-1.54.3.vsix 6b3e02fc4ef1ddefa7e3eabef5bf5906
vscode-perl https://open-vsx.org/api/vscode/perl/1.54.3/file/vscode.perl-1.54.3.vsix 6f3bc3e5b42e34e23fa287db2548b7c8
vscode-powershell https://open-vsx.org/api/vscode/powershell/1.54.3/file/vscode.powershell-1.54.3.vsix da887b8a031620f77676f529a7403580
vscode-pug https://open-vsx.org/api/vscode/pug/1.54.3/file/vscode.pug-1.54.3.vsix 8144700e4a33e99fa9b669d1254f8836
vscode-python https://open-vsx.org/api/vscode/python/1.54.3/file/vscode.python-1.54.3.vsix cea58111aad00d216f3643d9a180e1e5
vscode-r https://open-vsx.org/api/vscode/r/1.54.3/file/vscode.r-1.54.3.vsix fa4452b7777db74cb80a037b84a594a4
vscode-razor https://open-vsx.org/api/vscode/razor/1.54.3/file/vscode.razor-1.54.3.vsix da9e67b2f3530658778ac3180725e483
vscode-ruby https://open-vsx.org/api/vscode/ruby/1.54.3/file/vscode.ruby-1.54.3.vsix 9784e2d9ee737ea31192f60c97659832
vscode-rust https://open-vsx.org/api/vscode/rust/1.54.3/file/vscode.rust-1.54.3.vsix 8763f9e271841772e096a9fbbe4e342d
vscode-scss https://open-vsx.org/api/vscode/scss/1.54.3/file/vscode.scss-1.54.3.vsix 89d19984246b7c708738534648bb9fd0
vscode-shaderlab https://open-vsx.org/api/vscode/shaderlab/1.54.3/file/vscode.shaderlab-1.54.3.vsix 6cda4de8ff8c5d84b53dd8a234ae4d4f
vscode-shellscript https://open-vsx.org/api/vscode/shellscript/1.54.3/file/vscode.shellscript-1.54.3.vsix 32ec524b436a5c8a9d9be7c452112552
vscode-sql https://open-vsx.org/api/vscode/sql/1.54.3/file/vscode.sql-1.54.3.vsix bd448cac364ea09b4593db7efca625d3
vscode-swift https://open-vsx.org/api/vscode/swift/1.54.3/file/vscode.swift-1.54.3.vsix 32b12e0e1b284c273e5c120c394a75bf
vscode-theme-abyss https://open-vsx.org/api/vscode/theme-abyss/1.54.3/file/vscode.theme-abyss-1.54.3.vsix 9c580498861d19e4aff9ab2bf46020e0
vscode-theme-defaults https://open-vsx.org/api/vscode/theme-defaults/1.54.3/file/vscode.theme-defaults-1.54.3.vsix 80ff64379434c5773be739e89a8f2f34
vscode-theme-kimbie-dark https://open-vsx.org/api/vscode/theme-kimbie-dark/1.54.3/file/vscode.theme-kimbie-dark-1.54.3.vsix 52aa8d28f1f1299400d72d9023e65a01
vscode-theme-monokai https://open-vsx.org/api/vscode/theme-monokai/1.54.3/file/vscode.theme-monokai-1.54.3.vsix 6d65eb1998a7a7eb06ac56e555659999
vscode-theme-monokai-dimmed https://open-vsx.org/api/vscode/theme-monokai-dimmed/1.54.3/file/vscode.theme-monokai-dimmed-1.54.3.vsix f7ccf3d56fbb811db36e075b604cbe25
vscode-theme-quietlight https://open-vsx.org/api/vscode/theme-quietlight/1.54.3/file/vscode.theme-quietlight-1.54.3.vsix d66199d4c0104df14a0365933b3afe60
vscode-theme-red https://open-vsx.org/api/vscode/theme-red/1.54.3/file/vscode.theme-red-1.54.3.vsix 2d996393ab212a781682e0a8ee641713
vscode-theme-solarized-dark https://open-vsx.org/api/vscode/theme-solarized-dark/1.54.3/file/vscode.theme-solarized-dark-1.54.3.vsix e63d031121dd1c19bca3e88df5745851
vscode-theme-tomorrow-night-blue https://open-vsx.org/api/vscode/theme-tomorrow-night-blue/1.54.3/file/vscode.theme-tomorrow-night-blue-1.54.3.vsix 955cb9ac6f759e424182152552927d52
vscode-typescript https://open-vsx.org/api/vscode/typescript/1.54.3/file/vscode.typescript-1.54.3.vsix c2488834b57ba3413e7d43db2af0ed77
vscode-less https://open-vsx.org/api/vscode/less/1.62.3/file/vscode.less-1.62.3.vsix 9eb5e65d5d089abae8fc7cb2ebcdd53a
vscode-log https://open-vsx.org/api/vscode/log/1.62.3/file/vscode.log-1.62.3.vsix 454bea32383e2f93a035c8c8c9d680f1
vscode-lua https://open-vsx.org/api/vscode/lua/1.62.3/file/vscode.lua-1.62.3.vsix 89bf9895c9562b15ed5ae11c035df577
vscode-make https://open-vsx.org/api/vscode/make/1.62.3/file/vscode.make-1.62.3.vsix 19189fe2785ff3c0386a8877c9f9e132
vscode-markdown https://open-vsx.org/api/vscode/markdown/1.62.3/file/vscode.markdown-1.62.3.vsix b84496cb28d9032fe1dbf5738139c315
vscode-markdown-language-features https://open-vsx.org/api/vscode/markdown-language-features/1.62.3/file/vscode.markdown-language-features-1.62.3.vsix 4acc850aef4282c9adf773489eecedc5
vscode-merge-conflict https://open-vsx.org/api/vscode/merge-conflict/1.62.3/file/vscode.merge-conflict-1.62.3.vsix db6c52dafd6eeb65837262c7740c2b7a
vscode-npm https://open-vsx.org/api/vscode/npm/1.62.3/file/vscode.npm-1.62.3.vsix 16b3da4eac4e66ac3db90d3bb3e24479
ms-vscode-node-debug https://open-vsx.org/api/ms-vscode/node-debug/1.45.0/file/ms-vscode.node-debug-1.45.0.vsix 676769e9901f5f51ed6a21d7c6a831fb
ms-vscode-node-debug2 https://open-vsx.org/api/ms-vscode/node-debug2/1.43.0/file/ms-vscode.node-debug2-1.43.0.vsix 4c78ac1f3c4d753d005d72a68c13f7c7
vscode-objective-c https://open-vsx.org/api/vscode/objective-c/1.62.3/file/vscode.objective-c-1.62.3.vsix ea4419075d1a17cb57f984ec3ea783d0
vscode-perl https://open-vsx.org/api/vscode/perl/1.62.3/file/vscode.perl-1.62.3.vsix e1f9ebd43c5b3d4092418806bd066783
vscode-powershell https://open-vsx.org/api/vscode/powershell/1.62.3/file/vscode.powershell-1.62.3.vsix baf348ed743d3ba4a5b0b0702f6fa1c4
vscode-pug https://open-vsx.org/api/vscode/pug/1.62.3/file/vscode.pug-1.62.3.vsix 0d6fa06e57f2f9dbc5841e6996ba45d8
vscode-python https://open-vsx.org/api/vscode/python/1.62.3/file/vscode.python-1.62.3.vsix 98799c0d59f986378bc26d3a5809a76d
vscode-r https://open-vsx.org/api/vscode/r/1.62.3/file/vscode.r-1.62.3.vsix 9240efffdd93d5bcfa9e3833937df156
vscode-razor https://open-vsx.org/api/vscode/razor/1.62.3/file/vscode.razor-1.62.3.vsix 209d13c511ef5a6bdb8c23324a5df022
vscode-ruby https://open-vsx.org/api/vscode/ruby/1.62.3/file/vscode.ruby-1.62.3.vsix 52befb51090436c73273696b4e54baa4
vscode-rust https://open-vsx.org/api/vscode/rust/1.62.3/file/vscode.rust-1.62.3.vsix 0b6c07f593b7a06aa1acc21201b14b44
vscode-scss https://open-vsx.org/api/vscode/scss/1.62.3/file/vscode.scss-1.62.3.vsix e3ee9c810fb8c871f5df5dade2cc7479
vscode-shaderlab https://open-vsx.org/api/vscode/shaderlab/1.62.3/file/vscode.shaderlab-1.62.3.vsix d5f55aeedf313b3fe14d3c326ada2f2b
vscode-shellscript https://open-vsx.org/api/vscode/shellscript/1.62.3/file/vscode.shellscript-1.62.3.vsix 4abb4cad91156b25efd15cd029e02737
vscode-sql https://open-vsx.org/api/vscode/sql/1.62.3/file/vscode.sql-1.62.3.vsix 77392ebc2f3a1869823733099ca9ed87
vscode-swift https://open-vsx.org/api/vscode/swift/1.62.3/file/vscode.swift-1.62.3.vsix 29a1ea0de4f36baebee5f117336a92cc
vscode-theme-abyss https://open-vsx.org/api/vscode/theme-abyss/1.62.3/file/vscode.theme-abyss-1.62.3.vsix 04a5dcd017e04daa4269b7785d99def0
vscode-theme-defaults https://open-vsx.org/api/vscode/theme-defaults/1.62.3/file/vscode.theme-defaults-1.62.3.vsix 7fa0789b9ee1cb55a49698b018ac59ad
vscode-theme-kimbie-dark https://open-vsx.org/api/vscode/theme-kimbie-dark/1.62.3/file/vscode.theme-kimbie-dark-1.62.3.vsix ae60051044aab7708f33c13678f03f0b
vscode-theme-monokai https://open-vsx.org/api/vscode/theme-monokai/1.62.3/file/vscode.theme-monokai-1.62.3.vsix b2235d3b2fcb07858fbd2e9e3b9b9c77
vscode-theme-monokai-dimmed https://open-vsx.org/api/vscode/theme-monokai-dimmed/1.62.3/file/vscode.theme-monokai-dimmed-1.62.3.vsix 78aae18c5946da43bd756755a6df069a
vscode-theme-quietlight https://open-vsx.org/api/vscode/theme-quietlight/1.62.3/file/vscode.theme-quietlight-1.62.3.vsix 740e79e24fb7edeb277b6349a930c190
vscode-theme-red https://open-vsx.org/api/vscode/theme-red/1.62.3/file/vscode.theme-red-1.62.3.vsix d6ed6650920a1253a9c522300442f6f6
vscode-theme-solarized-dark https://open-vsx.org/api/vscode/theme-solarized-dark/1.62.3/file/vscode.theme-solarized-dark-1.62.3.vsix 6216c9d67d5d9443ca27e3afdb816e46
vscode-theme-tomorrow-night-blue https://open-vsx.org/api/vscode/theme-tomorrow-night-blue/1.62.3/file/vscode.theme-tomorrow-night-blue-1.62.3.vsix 04ca5f2984a9b5fed4102e040e4e4fda
vscode-typescript https://open-vsx.org/api/vscode/typescript/1.62.3/file/vscode.typescript-1.62.3.vsix 4f4b4b40506dacce128306facf3ed092
vscode-typescript-language-features https://open-vsx.org/api/vscode/typescript-language-features/1.54.3/file/vscode.typescript-language-features-1.54.3.vsix 968364ffa226c88d685f55cb21359954
vscode-vb https://open-vsx.org/api/vscode/vb/1.54.3/file/vscode.vb-1.54.3.vsix ac0b74fb681eaa2c51c6698623c3d43f
vscode-vscode-theme-seti https://open-vsx.org/api/vscode/vscode-theme-seti/1.54.3/file/vscode.vscode-theme-seti-1.54.3.vsix 4d0c2e66b8aef747221900367adba946
vscode-xml https://open-vsx.org/api/vscode/xml/1.54.3/file/vscode.xml-1.54.3.vsix f7aca865a3403a6e1b9e11e8019928b1
vscode-yaml https://open-vsx.org/api/vscode/yaml/1.54.3/file/vscode.yaml-1.54.3.vsix 0b2f74133505286388c043a52494168d
EditorConfig-EditorConfig https://open-vsx.org/api/EditorConfig/EditorConfig/0.16.4/file/EditorConfig.EditorConfig-0.16.4.vsix 0f8e4278d7a3638b9f9df84a07a56021
dbaeumer-vscode-eslint https://open-vsx.org/api/dbaeumer/vscode-eslint/2.1.8/file/dbaeumer.vscode-eslint-2.1.8.vsix d3668785b2744386867631d79537552a
ms-vscode-references-view https://open-vsx.org/api/ms-vscode/references-view/0.0.77/file/ms-vscode.references-view-0.0.77.vsix 0d0be6e6fd0ccfdd2a230b077ff8b4d7
vscode-vb https://open-vsx.org/api/vscode/vb/1.62.3/file/vscode.vb-1.62.3.vsix cd17e947324bb9cd05e799546d7d4638
vscode-vscode-theme-seti https://open-vsx.org/api/vscode/vscode-theme-seti/1.62.3/file/vscode.vscode-theme-seti-1.62.3.vsix b27e54ecb1c38f270f8c671a6574dcbc
vscode-xml https://open-vsx.org/api/vscode/xml/1.62.3/file/vscode.xml-1.62.3.vsix e3ba682c8ecb22516a1fd02abcdff841
vscode-yaml https://open-vsx.org/api/vscode/yaml/1.62.3/file/vscode.yaml-1.62.3.vsix f9719aaae4fd9671fd2fa15d41343e1d
EditorConfig-EditorConfig https://open-vsx.org/api/EditorConfig/EditorConfig/0.16.6/file/EditorConfig.EditorConfig-0.16.6.vsix e787245e6c68617178ae995ad97c3ccb
dbaeumer-vscode-eslint https://open-vsx.org/api/dbaeumer/vscode-eslint/2.1.20/file/dbaeumer.vscode-eslint-2.1.20.vsix 1cb024ac02ebeb5ce6b0dfed6e51cdd2
ms-vscode-references-view https://open-vsx.org/api/ms-vscode/references-view/0.0.82/file/ms-vscode.references-view-0.0.82.vsix d23827c6600ae821c829fca1629968c7
vscjava-vscode-java-debug https://open-vsx.org/api/vscjava/vscode-java-debug/0.29.0/file/vscjava.vscode-java-debug-0.29.0.vsix 1eb95110f84ff8dcabbe3c672066b86d
redhat-java https://open-vsx.org/api/redhat/java/0.61.0/file/redhat.java-0.61.0.vsix 72e548e2845e1ff655f28111558d6942
vscjava-vscode-java-test https://open-vsx.org/api/vscjava/vscode-java-test/0.26.0/file/vscjava.vscode-java-test-0.26.0.vsix fd63da5537a4bee1d3ceaae0fa6bf419
ms-python-python https://open-vsx.org/api/ms-python/python/2020.9.112786/file/ms-python.python-2020.9.112786.vsix c64b79fa822418e07b6d0f57b8838b44
perrinjerome-vscode-zc-buildout https://open-vsx.org/api/perrinjerome/vscode-zc-buildout/0.5.0/file/perrinjerome.vscode-zc-buildout-0.5.0.vsix 739889de7ccaf8232fb6c6fe8571a1fa
perrinjerome-vscode-zc-buildout https://open-vsx.org/api/perrinjerome/vscode-zc-buildout/0.6.2/file/perrinjerome.vscode-zc-buildout-0.6.2.vsix 76c1420f238a0754a505459563220973
jebbs-plantuml https://open-vsx.org/api/jebbs/plantuml/2.14.0/file/jebbs.plantuml-2.14.0.vsix 13fa7cbd14a30ecca166c41a307c7a73
rafaelmaiolla-diff https://open-vsx.org/api/rafaelmaiolla/diff/0.0.1/file/rafaelmaiolla.diff-0.0.1.vsix 1d8f868bc19b7d703c1be2bf99c4c7f9
perrinjerome-git-commit-syntax https://open-vsx.org/api/perrinjerome/git-commit-syntax/0.0.1/file/perrinjerome.git-commit-syntax-0.0.1.vsix 46625f2f05e244911c2cb9cc5032c0ef
......
component/theia/generate_download_plugins_cfg.py
View file @ a8f35d7e
......@@ -41,8 +41,7 @@ for plugin_and_version in '''\
vscode/lua/latest
vscode/make/latest
vscode/markdown/latest
# https://github.com/eclipse-theia/theia/issues/7780
vscode/markdown-language-features/1.39.2
vscode/markdown-language-features/latest
vscode/merge-conflict/latest
vscode/npm/latest
ms-vscode/node-debug/latest
......@@ -71,13 +70,17 @@ for plugin_and_version in '''\
vscode/theme-solarized-dark/latest
vscode/theme-tomorrow-night-blue/latest
vscode/typescript/latest
vscode/typescript-language-features/latest
# latest (1.62.3) does not activate because it uses a proposed API not in theia:
# Activating extension 'TypeScript and JavaScript Language Features (built-in)' failed: r.languages.createLanguageStatusItem is not a function
vscode/typescript-language-features/1.54.3
vscode/vb/latest
vscode/vscode-theme-seti/latest
vscode/xml/latest
vscode/yaml/latest
EditorConfig/EditorConfig/latest
dbaeumer/vscode-eslint/latest
# latest (2.2.2) does not activate:
# Activating extension 'ESLint' failed: Class extends value undefined is not a constructor or null
dbaeumer/vscode-eslint/2.1.20
ms-vscode/references-view/latest
# golang.Go removed because it overwrites the PATH in theia shell
# golang/Go/0.16.2
......
component/theia/yarn.lock
View file @ a8f35d7e
This source diff could not be displayed because it is too large. You can view the blob instead.
component/trafficserver/buildout.cfg
View file @ a8f35d7e
......@@ -24,8 +24,8 @@ min_version = 8
[trafficserver]
recipe = slapos.recipe.cmmi
url = http://apache.claz.org/trafficserver/trafficserver-9.1.0.tar.bz2
md5sum = 994b0aa879cbd95054048f34bf8ed954
url = https://dlcdn.apache.org/trafficserver/trafficserver-9.1.1.tar.bz2
md5sum = 666945652942cce8c3ff0ae2757cc3c6
shared = true
patch-options = -p1
configure-options =
......
component/userhosts/buildout.cfg
View file @ a8f35d7e
[userhosts]
recipe = slapos.recipe.cmmi
shared = true
url = https://lab.nexedi.com/nexedi/userhosts/repository/${:revision}/archive.tar.gz
revision = a05fe5a3a5cb7005351ef4ec41460089f3ce4d0a
md5sum = 5a80b4d962d975f290a60cf790c3334d
url = https://github.com/figiel/hosts/archive/${:revision}.tar.gz
revision = fdb45fe219593d63453f4be55cfc2a1199c18f59
md5sum = 9ff5407132a2a67200ef1036a02f0ee2
configure-command = true
make-options = PREFIX=@@LOCATION@@
make-targets = check install
make-targets = test install
component/wkhtmltopdf/buildout.cfg
View file @ a8f35d7e
......@@ -4,7 +4,6 @@ extends =
../freetype/buildout.cfg
../fontconfig/buildout.cfg
../libpng/buildout.cfg
../tar/buildout.cfg
../xorg/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
......@@ -14,27 +13,39 @@ parts =
[wkhtmltopdf]
recipe = slapos.recipe.build
# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
base = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_linux-generic
i386-linux-gnu = ${:base}-i386.tar.xz ce1a2c0b2cf786ccc5d5828c42c99ddd
x86_64-linux-gnu = ${:base}-amd64.tar.xz 96b7306cebb9e65355f69f7ab63df68b
url = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_${:_url}.tar.xz
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
install =
import os,shutil, sys, tempfile
path = self.download(*options[multiarch()].split())
extract_dir = tempfile.mkdtemp(self.name)
self.cleanup_dir_list.append(extract_dir)
self.logger.debug('Created working directory %s', extract_dir)
env = os.environ.copy()
env["PATH"] = "${tar:location}/bin:${xz-utils:location}/bin" + (":" + env["PATH"] if env.get("PATH") else "")
env["LD_LIBRARY_PATH"] = "${xz-utils:location}/lib" + (":" + env["LD_LIBRARY_PATH"] if env.get("LD_LIBRARY_PATH") else "")
call(["tar", "xJf", path, "-C", extract_dir], env=env)
shutil.move(os.path.join(extract_dir, "wkhtmltox"), location)
import os
self.copyTree(guessworkdir(self.extract(self.download())), location)
with open(os.path.join(location, "wkhtmltopdf"), 'w') as wrapper:
wrapper.write("""#!/bin/sh
export LD_LIBRARY_PATH=%(location)s/lib:${fontconfig:location}/lib:${freetype:location}/lib:${libX11:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libpng12:location}/lib:${zlib:location}/lib
export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/bin/wkhtmltopdf "$@"
""" % options)
wrapper.write("""#!/bin/sh -e
d=%s
export LD_LIBRARY_PATH=$d/lib:%s
export PATH=%s:$PATH
exec $d/bin/wkhtmltopdf "$@"
""" % (
location,
':'.join(options['library'].split()),
':'.join(options['path'].split()),
))
os.fchmod(wrapper.fileno(), 0o755)
library =
${fontconfig:location}/lib
${freetype:location}/lib
${libX11:location}/lib
${libXext:location}/lib
${libXrender:location}/lib
${libpng12:location}/lib
${zlib:location}/lib
path =
${fontconfig:location}/bin
[wkhtmltopdf:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url = linux-generic-i386
md5sum = ce1a2c0b2cf786ccc5d5828c42c99ddd
[wkhtmltopdf:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux-generic-amd64
md5sum = 96b7306cebb9e65355f69f7ab63df68b
setup.py
View file @ a8f35d7e
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '1.0.220'
version = '1.0.226'
name = 'slapos.cookbook'
long_description = open("README.rst").read()
......
slapos/recipe/generic_mysql/mysql.py
View file @ a8f35d7e
from __future__ import print_function
import os
import subprocess
import time
......@@ -16,35 +18,39 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
while True:
while True:
mysql_upgrade = subprocess.Popen(mysql_upgrade_binary,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True)
result = mysql_upgrade.communicate()[0]
if mysql_upgrade.returncode:
print "Command %r failed with result:\n%s" % (mysql_upgrade_binary, result)
print("Command %r failed with result:\n%s" % (mysql_upgrade_binary, result))
break
print "MySQL database upgraded with result:\n%s" % result
print("MySQL database upgraded with result:\n%s" % result)
mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True)
result = mysql.communicate(mysql_script)[0]
if mysql.returncode:
print 'Command %r failed with:\n%s' % (mysql_list, result)
print('Command %r failed with:\n%s' % (mysql_list, result))
break
# import timezone database
mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True)
timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
if mysql_tzinfo_to_sql.returncode != 0:
print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result)
print('Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result))
break
mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True)
result = mysql.communicate(timezone_sql)[0]
if mysql.returncode:
print 'Command %r failed with:\n%s' % (mysql_list, result)
print('Command %r failed with:\n%s' % (mysql_list, result))
break
print 'SlapOS initialisation script succesfully applied on database.'
print('SlapOS initialisation script succesfully applied on database.')
return
sleep = min(sleep+1, 30)
print 'Sleeping for %ss and retrying' % sleep
print('Sleeping for %ss and retrying' % sleep)
sys.stdout.flush()
sys.stderr.flush()
time.sleep(sleep)
software/erp5/test/test/__init__.py
View file @ a8f35d7e
......@@ -49,17 +49,20 @@ def setUpModule():
class ERP5InstanceTestCase(SlapOSInstanceTestCase):
"""ERP5 base test case
"""
def getRootPartitionConnectionParameterDict(self):
@classmethod
def getRootPartitionConnectionParameterDict(cls):
"""Return the output paramters from the root partition"""
return json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
cls.computer_partition.getConnectionParameterDict()['_'])
def getComputerPartition(self, partition_reference):
for computer_partition in self.slap.computer.getComputerPartitionList():
@classmethod
def getComputerPartition(cls, partition_reference):
for computer_partition in cls.slap.computer.getComputerPartitionList():
if partition_reference == computer_partition.getInstanceParameter(
'instance_title'):
return computer_partition
def getComputerPartitionPath(self, partition_reference):
partition_id = self.getComputerPartition(partition_reference).getId()
return os.path.join(self.slap._instance_root, partition_id)
@classmethod
def getComputerPartitionPath(cls, partition_reference):
partition_id = cls.getComputerPartition(partition_reference).getId()
return os.path.join(cls.slap._instance_root, partition_id)
software/erp5/test/test/test_erp5.py
View file @ a8f35d7e
......@@ -26,21 +26,30 @@
##############################################################################
from __future__ import absolute_import
import os
import json
import contextlib
import glob
import six.moves.urllib.parse
import json
import os
import shutil
import socket
import ssl
import subprocess
import sys
import tempfile
import time
import unittest
import psutil
import requests
from . import ERP5InstanceTestCase
from . import setUpModule
import six
from six.moves import map
from six.moves import range
import six.moves.urllib.parse
import six.moves.xmlrpc_client
import urllib3
from slapos.testing.utils import CrontabMixin
from . import ERP5InstanceTestCase, setUpModule
setUpModule # pyflakes
......@@ -68,8 +77,8 @@ class TestPublishedURLIsReachableMixin(object):
session.mount(
base_url,
requests.adapters.HTTPAdapter(
max_retries=requests.packages.urllib3.util.retry.Retry(
total=60,
max_retries=urllib3.util.retry.Retry(
total=20,
backoff_factor=.5,
status_forcelist=(404, 500, 503))))
......@@ -389,3 +398,455 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
}, {
"cache-size": None,
})
class TestWatchActivities(ERP5InstanceTestCase):
"""Tests for bin/watch_activities scripts in zope partitions.
"""
__partition_reference__ = 'wa'
def test(self):
# "watch_activites" scripts use watch command. We'll fake a watch command
# that executes the actual command only once to check the output.
tmpdir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmpdir)
with open(os.path.join(tmpdir, 'watch'), 'w') as f:
f.write("""#!/bin/sh
if [ "$1" != "-n" ] || [ "$2" != "5" ]
then
echo unexpected arguments: "$1" "$2"
exit 1
fi
shift
shift
exec bash -c "$@"
""")
os.fchmod(f.fileno(), 0o700)
try:
output = subprocess.check_output(
[
os.path.join(
self.getComputerPartitionPath('zope-1'),
'bin',
'watch_activities',
)
],
env=dict(os.environ,
PATH=os.pathsep.join([tmpdir, os.environ['PATH']])),
stderr=subprocess.STDOUT,
universal_newlines=True,
)
except subprocess.CalledProcessError as e:
self.fail(e.output)
self.assertIn(' dict ', output)
class ZopeTestMixin(CrontabMixin):
"""Mixin class for zope features.
"""
wsgi = NotImplemented # type: bool
__partition_reference__ = 'z'
@classmethod
def getInstanceParameterDict(cls):
return {
'_':
json.dumps({
"zope-partition-dict": {
"default": {
"longrequest-logger-interval": 1,
"longrequest-logger-timeout": 1,
},
},
"wsgi": cls.wsgi,
})
}
@classmethod
def _setUpClass(cls):
super(ZopeTestMixin, cls)._setUpClass()
param_dict = cls.getRootPartitionConnectionParameterDict()
# rebuild an url with user and password
parsed = six.moves.urllib.parse.urlparse(param_dict['family-default'])
cls.zope_base_url = parsed._replace(
netloc='{}:{}@{}:{}'.format(
param_dict['inituser-login'],
param_dict['inituser-password'],
parsed.hostname,
parsed.port,
),
path=param_dict['site-id'] + '/',
).geturl()
cls.zope_deadlock_debugger_url = six.moves.urllib_parse.urljoin(
cls.zope_base_url,
'/manage_debug_threads?{deadlock-debugger-password}'.format(
**param_dict),
)
@contextlib.contextmanager
def getXMLRPCClient():
# don't verify certificate
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
erp5_xmlrpc_client = six.moves.xmlrpc_client.ServerProxy(
cls.zope_base_url,
context=ssl_context,
)
# BBB use as a context manager only on python3
if sys.version_info < (3, ):
yield erp5_xmlrpc_client
else:
with erp5_xmlrpc_client:
yield erp5_xmlrpc_client
with getXMLRPCClient() as erp5_xmlrpc_client:
# wait for ERP5 to be ready (TODO: this should probably be a promise)
for _ in range(120):
time.sleep(1)
try:
erp5_xmlrpc_client.getTitle()
except (six.moves.xmlrpc_client.ProtocolError,
six.moves.xmlrpc_client.Fault):
pass
else:
break
def addPythonScript(script_id, params, body):
with getXMLRPCClient() as erp5_xmlrpc_client:
custom = erp5_xmlrpc_client.portal_skins.custom
try:
custom.manage_addProduct.PythonScripts.manage_addPythonScript(
script_id)
except six.moves.xmlrpc_client.ProtocolError as e:
if e.errcode != 302:
raise
getattr(custom, script_id).ZPythonScriptHTML_editAction(
'',
'',
params,
body,
)
# a python script to verify activity processing
addPythonScript(
script_id='ERP5Site_verifyActivityProcessing',
params='mode',
body='''if 1:
import json
portal = context.getPortalObject()
if mode == "count":
return json.dumps(dict(count=len(portal.portal_activities.getMessageList())))
if mode == "activate":
for _ in range(10):
portal.portal_templates.activate(activity="SQLQueue").getTitle()
return "activated"
raise ValueError("Unknown mode: %s" % mode)
''',
)
cls.zope_verify_activity_processing_url = six.moves.urllib_parse.urljoin(
cls.zope_base_url,
'ERP5Site_verifyActivityProcessing',
)
# a python script logging to event log
addPythonScript(
script_id='ERP5Site_logMessage',
params='name',
body='''if 1:
from erp5.component.module.Log import log
return log("hello %s" % name)
''',
)
cls.zope_log_message_url = six.moves.urllib_parse.urljoin(
cls.zope_base_url,
'ERP5Site_logMessage',
)
# a python script issuing a long request
addPythonScript(
script_id='ERP5Site_executeLongRequest',
params='',
body='''if 1:
import time
for _ in range(5):
time.sleep(1)
return "done"
''',
)
cls.zope_long_request_url = six.moves.urllib_parse.urljoin(
cls.zope_base_url,
'ERP5Site_executeLongRequest',
)
def setUp(self):
super(ZopeTestMixin, self).setUp()
# run logrotate a first time so that it create state files
self._executeCrontabAtDate('logrotate', '2000-01-01')
def tearDown(self):
super(ZopeTestMixin, self).tearDown()
# reset logrotate status
logrotate_status = os.path.join(
self.getComputerPartitionPath('zope-default'),
'srv',
'logrotate.status',
)
if os.path.exists(logrotate_status):
os.unlink(logrotate_status)
for logfile in glob.glob(
os.path.join(
self.getComputerPartitionPath('zope-default'),
'srv',
'backup',
'logrotate',
'*',
)):
os.unlink(logfile)
for logfile in glob.glob(
os.path.join(
self.getComputerPartitionPath('zope-default'),
'srv',
'monitor',
'private',
'documents',
'*',
)):
os.unlink(logfile)
def _getCrontabCommand(self, crontab_name):
# type: (str) -> str
"""Read a crontab and return the command that is executed.
overloaded to use crontab from zope partition
"""
with open(
os.path.join(
self.getComputerPartitionPath('zope-default'),
'etc',
'cron.d',
crontab_name,
)) as f:
crontab_spec, = f.readlines()
self.assertNotEqual(crontab_spec[0], '@', crontab_spec)
return crontab_spec.split(None, 5)[-1]
def test_event_log_rotation(self):
requests.get(
self.zope_log_message_url,
params={
"name": "world"
},
verify=False,
).raise_for_status()
zope_event_log_path = os.path.join(
self.getComputerPartitionPath('zope-default'),
'var',
'log',
'zope-0-event.log',
)
with open(zope_event_log_path) as f:
self.assertIn('hello world', f.read())
self._executeCrontabAtDate('logrotate', '2050-01-01')
# this logrotate leaves the log for the day as non compressed
rotated_log_file = os.path.join(
self.getComputerPartitionPath('zope-default'),
'srv',
'backup',
'logrotate',
'zope-0-event.log-20500101',
)
with open(rotated_log_file) as f:
self.assertIn('hello world', f.read())
requests.get(
self.zope_log_message_url,
params={
"name": "le monde"
},
verify=False,
).raise_for_status()
with open(zope_event_log_path) as f:
self.assertNotIn('hello world', f.read())
with open(zope_event_log_path) as f:
self.assertIn('hello le monde', f.read())
# on next day execution of logrotate, log files are compressed
self._executeCrontabAtDate('logrotate', '2050-01-02')
self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
self.assertFalse(os.path.exists(rotated_log_file))
def test_access_log_rotation(self):
requests.get(
self.zope_base_url,
verify=False,
headers={
'User-Agent': 'before rotation'
},
).raise_for_status()
zope_access_log_path = os.path.join(
self.getComputerPartitionPath('zope-default'),
'var',
'log',
'zope-0-Z2.log',
)
with open(zope_access_log_path) as f:
self.assertIn('before rotation', f.read())
self._executeCrontabAtDate('logrotate', '2050-01-01')
# this logrotate leaves the log for the day as non compressed
rotated_log_file = os.path.join(
self.getComputerPartitionPath('zope-default'),
'srv',
'backup',
'logrotate',
'zope-0-Z2.log-20500101',
)
with open(rotated_log_file) as f:
self.assertIn('before rotation', f.read())
requests.get(
self.zope_base_url,
verify=False,
headers={
'User-Agent': 'after rotation'
},
).raise_for_status()
with open(zope_access_log_path) as f:
self.assertNotIn('before rotation', f.read())
with open(zope_access_log_path) as f:
self.assertIn('after rotation', f.read())
# on next day execution of logrotate, log files are compressed
self._executeCrontabAtDate('logrotate', '2050-01-02')
self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
self.assertFalse(os.path.exists(rotated_log_file))
def test_long_request_log_rotation(self):
requests.get(self.zope_long_request_url,
verify=False,
params={
'when': 'before rotation'
}).raise_for_status()
zope_long_request_log_path = os.path.join(
self.getComputerPartitionPath('zope-default'),
'var',
'log',
'longrequest_logger_zope-0.log',
)
with open(zope_long_request_log_path) as f:
self.assertIn('before rotation', f.read())
self._executeCrontabAtDate('logrotate', '2050-01-01')
# this logrotate leaves the log for the day as non compressed
rotated_log_file = os.path.join(
self.getComputerPartitionPath('zope-default'),
'srv',
'backup',
'logrotate',
'longrequest_logger_zope-0.log-20500101',
)
with open(rotated_log_file) as f:
self.assertIn('before rotation', f.read())
requests.get(
self.zope_long_request_url,
verify=False,
params={
'when': 'after rotation'
},
).raise_for_status()
with open(zope_long_request_log_path) as f:
self.assertNotIn('before rotation', f.read())
with open(zope_long_request_log_path) as f:
self.assertIn('after rotation', f.read())
# on next day execution of logrotate, log files are compressed
self._executeCrontabAtDate('logrotate', '2050-01-02')
self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
self.assertFalse(os.path.exists(rotated_log_file))
def test_basic_authentication_user_in_access_log(self):
param_dict = self.getRootPartitionConnectionParameterDict()
requests.get(self.zope_base_url,
verify=False,
auth=requests.auth.HTTPBasicAuth(
param_dict['inituser-login'],
param_dict['inituser-password'],
)).raise_for_status()
zope_access_log_path = os.path.join(
self.getComputerPartitionPath('zope-default'),
'var',
'log',
'zope-0-Z2.log',
)
with open(zope_access_log_path) as f:
self.assertIn(param_dict['inituser-login'], f.read())
def test_deadlock_debugger(self):
dump_response = requests.get(
self.zope_deadlock_debugger_url,
verify=False,
)
dump_response.raise_for_status()
self.assertIn('Thread ', dump_response.text)
def test_activity_processing(self):
def wait_for_activities(max_retries):
for retry in range(max_retries):
time.sleep(10)
resp = requests.get(
self.zope_verify_activity_processing_url,
params={
'mode': 'count',
'retry': retry,
},
verify=False,
)
if not resp.ok:
# XXX we start by flushing existing activities from site creation
# and inital upgrader run. During this time it may happen that
# ERP5 replies with site errors, we tolerate these errors and only
# check the final state.
continue
count = resp.json()['count']
if not count:
break
else:
self.assertEqual(count, 0)
wait_for_activities(60)
requests.get(
self.zope_verify_activity_processing_url,
params={
'mode': 'activate'
},
verify=False,
).raise_for_status()
wait_for_activities(10)
class TestZopeMedusa(ZopeTestMixin, ERP5InstanceTestCase):
wsgi = False
class TestZopeWSGI(ZopeTestMixin, ERP5InstanceTestCase):
wsgi = True
@unittest.expectedFailure
def test_long_request_log_rotation(self):
super(TestZopeWSGI, self).test_long_request_log_rotation(self)
@unittest.expectedFailure
def test_basic_authentication_user_in_access_log(self):
super(TestZopeWSGI, self).test_basic_authentication_user_in_access_log(self)
software/erp5/upgrade_test/test.py
View file @ a8f35d7e
......@@ -39,13 +39,15 @@ from slapos.testing.testcase import installSoftwareUrlList
from slapos.testing.testcase import SlapOSNodeCommandError
from slapos.grid.utils import md5digest
old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.4/software/erp5/software.cfg'
old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.5/software/erp5/software.cfg'
new_software_release_url = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
_, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
old_software_release_url, software_id="upgrade_erp5")
old_software_release_url,
software_id="upgrade_erp5",
skip_software_check=True,
)
def setUpModule():
installSoftwareUrlList(
......
software/fluentd/test/test.py
View file @ a8f35d7e
......@@ -35,6 +35,7 @@ import subprocess
import tempfile
import time
import six
import sys
from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
from six.moves.socketserver import StreamRequestHandler, TCPServer
......@@ -171,7 +172,7 @@ class SensorConfTestCase(WendelinTutorialTestCase):
<source>
@type exec
tag tag.name
command python %s
command %s %s
run_interval %ss
<parse>
keys pressure, humidity, temperature
......@@ -186,7 +187,7 @@ class SensorConfTestCase(WendelinTutorialTestCase):
<buffer>
flush_mode immediate
</buffer>
</match>''' % (script_path, FLUSH_INTERVAL, cls._ipv6_address)
</match>''' % (sys.executable, script_path, FLUSH_INTERVAL, cls._ipv6_address)
@classmethod
def sensor_script(cls, measurementList):
......
software/kvm/buildout.hash.cfg
View file @ a8f35d7e
......@@ -19,7 +19,7 @@ md5sum = 087f226ba90928dcc5a722d7008c867a
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = b260fce887535fc69a259d1fd31af1b2
md5sum = d949e6100fd4e01819cc2867a65c6e65
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
......
software/kvm/instance-kvm.cfg.jinja2
View file @ a8f35d7e
......@@ -1029,15 +1029,12 @@ config-command = ${ansible-vm-bin:rendered}
{% if bootstrap_script_url -%}
[download-bootstrap-script]
recipe = slapos.recipe.build
location = ${directory:public}/vm-bootstrap
install =
from zc.buildout.download import check_md5sum, urlretrieve, ChecksumError
url, md5sum = {{ repr(bootstrap_script_url) }}.split('#')
urlretrieve(url, location)
if not check_md5sum(location, md5sum):
os.remove(location)
raise ChecksumError
recipe = slapos.recipe.build:download
offline = false
destination = ${directory:public}/vm-bootstrap
{% set bootstrap_script_url = bootstrap_script_url.split('#') -%}
url = {{ bootstrap_script_url[0] }}
md5sum = {{ bootstrap_script_url[1] }}
{% endif -%}
[logrotate-vm-bootstrap]
......
software/ors-amarisoft/buildout.hash.cfg
View file @ a8f35d7e
......@@ -16,27 +16,27 @@
[template]
filename = instance.cfg
md5sum = 96a76300b2f714b6c47157920fe79a53
md5sum = 6ea4fa210a91c15278c847a809de5991
[template-lte-enb-epc]
_update_hash_filename_ = instance-enb-epc.jinja2.cfg
md5sum = 762d55291e75e8b61e35f9f28d29915a
md5sum = 25fae79db2b30be0f365ad967b278c3c
[template-lte-enb]
_update_hash_filename_ = instance-enb.jinja2.cfg
md5sum = 83c37b43d7b70584bf71aed9289ea13c
md5sum = fe249168a3f50b0efe6aeae39afb03ae
[template-lte-gnb-epc]
_update_hash_filename_ = instance-gnb-epc.jinja2.cfg
md5sum = e43a726dd3023a4bbaa474bb2d7a6ebe
md5sum = 5ec4508b02f6e7fbdbe1f37c65b9d897
[template-lte-gnb]
_update_hash_filename_ = instance-gnb.jinja2.cfg
md5sum = fc59d15a7f7f942951f9e38d7a1cca2c
md5sum = 6deb38b3de44f6e0a43b49fd13f0f072
[template-lte-epc]
_update_hash_filename_ = instance-epc.jinja2.cfg
md5sum = 0585b4f8fd42538595e6abb61d07fd93
md5sum = 089f62e736cdc620eafc2c47b050fe13
[ue_db.jinja2.cfg]
filename = config/ue_db.jinja2.cfg
......@@ -44,11 +44,11 @@ md5sum = d33163012d6c98efc59161974c649557
[enb.jinja2.cfg]
filename = config/enb.jinja2.cfg
md5sum = d841debc51d9f12555a47d1556a6a3c1
md5sum = 97d9cdd07704cae36c5e4234c87025e8
[gnb.jinja2.cfg]
filename = config/gnb.jinja2.cfg
md5sum = da64ea9c5003f40987a8bba3f18e8839
md5sum = b0df7f3b679b25d5296dd67e074364b4
[ltelogs.jinja2.sh]
filename = ltelogs.jinja2.sh
......@@ -61,3 +61,11 @@ md5sum = 518c71ce57204304b703b977c665a164
[ims.jinja2.cfg]
filename = config/ims.jinja2.cfg
md5sum = e561ec26a70943c61557def1781cf65f
[sdr-busy-promise]
_update_hash_filename_ = promise/check_sdr_busy.py
md5sum = 9e867282d7dd80c4255e87a82a47a19d
[interface-up-promise]
_update_hash_filename_ = promise/check_interface_up.py
md5sum = 505efcbe04e717088924f2267b10c2b9
software/ors-amarisoft/config/enb.jinja2.cfg
View file @ a8f35d7e
......@@ -56,15 +56,15 @@
{
/* address of MME for S1AP connection. Must be modified if the MME
runs on a different host. */
mme_addr: "127.0.1.100"
mme_addr: "{{ slapparameter_dict.get('mme_addr', '127.0.1.100') }}"
},
],
/* GTP bind address (=address of the ethernet interface connected to
the MME). Must be modified if the MME runs on a different host. */
gtp_addr: "{{ slapparameter_dict.get('gtp_addr', '127.0.1.1') }}",
gtp_addr: "{{ gtp_addr }}",
/* high 20 bits of SIB1.cellIdentifier */
enb_id: 0x1A2D0,
enb_id: {{ slapparameter_dict.get('enb_id', '0x1A2D0') }},
/* list of cells */
cell_list: [
......
software/ors-amarisoft/config/gnb.jinja2.cfg
View file @ a8f35d7e
......@@ -37,15 +37,15 @@
amf_list: [
{
/* address of AMF for NGAP connection. Must be modified if the AMF runs on a different host. */
amf_addr: "127.0.1.100",
amf_addr: "{{ slapparameter_dict.get('amf_addr', '127.0.1.100') }}",
},
],
/* GTP bind address (=address of the ethernet interface connected to
the AMF). Must be modified if the AMF runs on a different host. */
gtp_addr: "{{ slapparameter_dict.get('gtp_addr', '127.0.1.1') }}",
gtp_addr: "{{ gtp_addr }}",
gnb_id_bits: 28,
gnb_id: 0x12345,
gnb_id: {{ slapparameter_dict.get('gnb_id', '0x12345') }},
nr_support: true,
......
software/ors-amarisoft/instance-enb-epc.jinja2.cfg
View file @ a8f35d7e
......@@ -12,7 +12,7 @@ develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
......@@ -34,7 +34,7 @@ promise = ${:etc}/promise
log = ${:var}/log
[request-common-base]
recipe = slapos.cookbook:request.serialised
recipe = slapos.cookbook:request
software-url = {{ slap_connection['software-release-url'] }}
server-url = {{ slap_connection['server-url'] }}
computer-id = {{ slap_connection['computer-id'] }}
......@@ -42,6 +42,8 @@ partition-id = {{ slap_connection['partition-id'] }}
key-file = {{ slap_connection['key-file'] }}
cert-file = {{ slap_connection['cert-file'] }}
sla-computer_guid = {{ slap_connection['computer-id'] }}
config-monitor-password = ${monitor-htpasswd:passwd}
return = monitor-base-url
......
software/ors-amarisoft/instance-enb-input-schema.jinja2.json
View file @ a8f35d7e
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": {{ default_lte_n_rb_dl }}
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
software/ors-amarisoft/instance-enb.jinja2.cfg
View file @ a8f35d7e
......@@ -4,7 +4,7 @@ parts =
ltelogs
lte-enb-config
lte-enb-service
# Temporarily extend monitor-base until promises are added
sdr-busy-promise
monitor-base
publish-connection-information
......@@ -15,15 +15,15 @@ develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
key = {{ slap_connection['key-file'] }}
cert = {{ slap_connection['cert-file'] }}
configuration.network_name = VIFIB
configuration.domain = vifib.com
configuration.network_name = RAPIDSPACE
configuration.domain = rapid.space
configuration.mme_ws_port = 9000
configuration.enb_ws_port = 9002
configuration.ims_ws_port = 9003
......@@ -60,17 +60,27 @@ extensions = jinja2.ext.do
context =
section directory directory
[lte-enb-sh-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
template =
inline:#!/bin/sh
sudo /opt/amarisoft/rm-tmp-lte | true;
{{ enb }}/lteenb ${directory:etc}/enb.cfg >> ${directory:log}/enb-output.cfg 2>> ${directory:log}/enb-output.cfg
### eNodeB (enb)
[lte-enb-service]
recipe = slapos.cookbook:wrapper
init = ${ltelogs:rendered} ${directory:log}/enb.log; sleep 2
command-line = {{ enb }}/lteenb ${directory:etc}/enb.cfg
command-line = ${lte-enb-sh-wrapper:rendered}
wrapper-path = ${directory:service}/lte-enb
mode = 0775
reserve-cpu = True
pidfile = ${directory:run}/enb.pid
hash-files =
${lte-enb-config:rendered}
${lte-enb-sh-wrapper:rendered}
environment =
LD_LIBRARY_PATH={{ openssl_location }}/lib
AMARISOFT_PATH=/opt/amarisoft/.amarisoft
......@@ -83,6 +93,7 @@ context =
section directory directory
section slap_configuration slap-configuration
key slapparameter_dict slap-configuration:configuration
raw gtp_addr {{ local_ipv4 }}
import netaddr netaddr
[lte-enb-config]
......@@ -91,9 +102,21 @@ template = {{ enb_template }}
rendered = ${directory:etc}/enb.cfg
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
<= monitor-publish
recipe = slapos.cookbook:publish
ipv4 = {{ local_ipv4 }}
[monitor-instance-parameter]
{% if slapparameter_dict.get("name", None) %}
monitor-title = {{ slapparameter_dict['name'] | string }}
{% endif %}
{% if slapparameter_dict.get("monitor-password", None) %}
password = {{ slapparameter_dict['monitor-password'] | string }}
{% endif %}
# Add custom promise to check if /dev/sdr0 is busy
[sdr-busy-promise]
recipe = slapos.cookbook:promise.plugin
eggs = slapos.core
file = {{ sdr_busy_promise }}
output = ${directory:plugins}/check-sdr-busy.py
software/ors-amarisoft/instance-epc.jinja2.cfg
View file @ a8f35d7e
......@@ -4,7 +4,7 @@ parts =
ltelogs
lte-mme-config
lte-mme-service
# Temporarily extend monitor-base until promises are added
tun-up-promise
monitor-base
publish-connection-information
......@@ -15,15 +15,15 @@ develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
key = {{ slap_connection['key-file'] }}
cert = {{ slap_connection['cert-file'] }}
configuration.network_name = VIFIB
configuration.domain = vifib.com
configuration.network_name = RAPIDSPACE
configuration.domain = rapid.space
configuration.log_size = 50M
configuration.mme_ws_port = 9000
configuration.enb_ws_port = 9002
......@@ -62,7 +62,7 @@ context =
[lte-ims-service]
recipe = slapos.cookbook:wrapper
init = ${ltelogs:rendered} ${directory:log}/ims.log; sleep 1
command-line = rm -f ${directory:var}/lte_ue.db; {{ mme }}/lteims ${directory:etc}/ims.cfg
command-line = {{ mme }}/lteims ${directory:etc}/ims.cfg
wrapper-path = ${directory:service}/lte-ims
mode = 0775
pidfile = ${directory:run}/ims.pid
......@@ -71,17 +71,29 @@ hash-files =
{{ ue_db_path }}
environment = AMARISOFT_PATH=/opt/amarisoft/.amarisoft
[lte-mme-sh-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
template =
inline:#!/bin/sh
rm -f ${directory:var}/lte_ue.db;
{{ mme }}/ltemme ${directory:etc}/mme.cfg >> ${directory:log}/mme-output.cfg 2>> ${directory:log}/mme-output.cfg
### MME
[lte-mme-service]
recipe = slapos.cookbook:wrapper
# When the machine shutdowns abruptly, lte_ue is not cleaned up which causes
# amarisoft ltemme to fail. TODO: find a cleaner way to handle this
init = ${ltelogs:rendered} ${directory:log}/mme.log
command-line = {{ mme }}/ltemme ${directory:etc}/mme.cfg
command-line = ${lte-mme-sh-wrapper:rendered}
wrapper-path = ${directory:service}/lte-mme
mode = 0775
pidfile = ${directory:run}/mme.pid
hash-files =
${lte-mme-config:rendered}
{{ ue_db_path }}
${lte-mme-sh-wrapper:rendered}
environment =
LD_LIBRARY_PATH={{ openssl_location }}/lib:{{ nghttp2_location }}/lib
AMARISOFT_PATH=/opt/amarisoft/.amarisoft
......@@ -119,17 +131,18 @@ context =
import netaddr netaddr
key ifup_empty lte-mme-ifup-empty:wrapper-path
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
{% if slapparameter_dict.get("monitor-password", None) %}
monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
{% else %}
monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
{% endif %}
{% if slapparameter_dict.get("monitor-password", None) %}
[monitor-instance-parameter]
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif %}
[publish-connection-information]
<= monitor-publish
recipe = slapos.cookbook:publish
# Add custom promise to check if /dev/sdr0 is busy
[tun-up-promise]
recipe = slapos.cookbook:promise.plugin
eggs = slapos.core
file = {{ interface_up_promise }}
output = ${directory:plugins}/check-tun-up.py
config-ifname = ${slap-configuration:tun-name}
software/ors-amarisoft/instance-gnb-epc.jinja2.cfg
View file @ a8f35d7e
......@@ -12,7 +12,7 @@ develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
......@@ -34,7 +34,7 @@ promise = ${:etc}/promise
log = ${:var}/log
[request-common-base]
recipe = slapos.cookbook:request.serialised
recipe = slapos.cookbook:request
software-url = {{ slap_connection['software-release-url'] }}
server-url = {{ slap_connection['server-url'] }}
computer-id = {{ slap_connection['computer-id'] }}
......@@ -42,6 +42,8 @@ partition-id = {{ slap_connection['partition-id'] }}
key-file = {{ slap_connection['key-file'] }}
cert-file = {{ slap_connection['cert-file'] }}
sla-computer_guid = {{ slap_connection['computer-id'] }}
config-monitor-password = ${monitor-htpasswd:passwd}
return = monitor-base-url
......
software/ors-amarisoft/instance-gnb-input-schema.jinja2.json
View file @ a8f35d7e
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": {{ default_nr_bandwidth }}
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
software/ors-amarisoft/instance-gnb.jinja2.cfg
View file @ a8f35d7e
......@@ -4,7 +4,7 @@ parts =
ltelogs
lte-gnb-config
lte-enb-service
# Temporarily extend monitor-base until promises are added
sdr-busy-promise
monitor-base
publish-connection-information
......@@ -15,15 +15,15 @@ develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
key = {{ slap_connection['key-file'] }}
cert = {{ slap_connection['cert-file'] }}
configuration.network_name = VIFIB
configuration.domain = vifib.com
configuration.network_name = RAPIDSPACE
configuration.domain = rapid.space
configuration.mme_ws_port = 9000
configuration.enb_ws_port = 9002
configuration.ims_ws_port = 9003
......@@ -60,17 +60,27 @@ extensions = jinja2.ext.do
context =
section directory directory
[lte-enb-sh-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
template =
inline:#!/bin/sh
sudo /opt/amarisoft/rm-tmp-lte | true;
{{ enb }}/lteenb ${directory:etc}/gnb.cfg >> ${directory:log}/gnb-output.cfg 2>> ${directory:log}/gnb-output.cfg
### eNodeB (enb)
[lte-enb-service]
recipe = slapos.cookbook:wrapper
init = ${ltelogs:rendered} ${directory:log}/enb.log; sleep 2
command-line = {{ enb }}/lteenb ${directory:etc}/gnb.cfg
command-line = ${lte-enb-sh-wrapper:rendered}
wrapper-path = ${directory:service}/lte-enb
mode = 0775
reserve-cpu = True
pidfile = ${directory:run}/enb.pid
hash-files =
${lte-gnb-config:rendered}
${lte-enb-sh-wrapper:rendered}
environment =
LD_LIBRARY_PATH={{ openssl_location }}/lib
AMARISOFT_PATH=/opt/amarisoft/.amarisoft
......@@ -83,6 +93,7 @@ context =
section directory directory
section slap_configuration slap-configuration
key slapparameter_dict slap-configuration:configuration
raw gtp_addr {{ local_ipv4 }}
import netaddr netaddr
[lte-gnb-config]
......@@ -91,9 +102,21 @@ template = {{ gnb_template }}
rendered = ${directory:etc}/gnb.cfg
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
<= monitor-publish
recipe = slapos.cookbook:publish
ipv4 = {{ local_ipv4 }}
[monitor-instance-parameter]
{% if slapparameter_dict.get("name", None) %}
monitor-title = {{ slapparameter_dict['name'] | string }}
{% endif %}
{% if slapparameter_dict.get("monitor-password", None) %}
password = {{ slapparameter_dict['monitor-password'] | string }}
{% endif %}
# Add custom promise to check if /dev/sdr0 is busy
[sdr-busy-promise]
recipe = slapos.cookbook:promise.plugin
eggs = slapos.core
file = {{ sdr_busy_promise }}
output = ${directory:plugins}/check-sdr-busy.py
software/ors-amarisoft/instance-tdd1900-enb-epc-input-schema.json
View file @ a8f35d7e
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd1900-enb-input-schema.json
View file @ a8f35d7e
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd1900-gnb-epc-input-schema.json
View file @ a8f35d7e
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd1900-gnb-input-schema.json
View file @ a8f35d7e
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd2600-enb-epc-input-schema.json
View file @ a8f35d7e
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd2600-enb-input-schema.json
View file @ a8f35d7e
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd2600-gnb-epc-input-schema.json
View file @ a8f35d7e
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd2600-gnb-input-schema.json
View file @ a8f35d7e
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd3500-enb-epc-input-schema.json
View file @ a8f35d7e
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd3500-enb-input-schema.json
View file @ a8f35d7e
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd3500-gnb-epc-input-schema.json
View file @ a8f35d7e
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd3500-gnb-input-schema.json
View file @ a8f35d7e
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd3700-enb-epc-input-schema.json
View file @ a8f35d7e
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd3700-enb-input-schema.json
View file @ a8f35d7e
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd3700-gnb-epc-input-schema.json
View file @ a8f35d7e
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance-tdd3700-gnb-input-schema.json
View file @ a8f35d7e
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
software/ors-amarisoft/instance.cfg
View file @ a8f35d7e
......@@ -7,7 +7,7 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
......@@ -36,6 +36,20 @@ gnb = dynamic-template-lte-gnb:rendered
epc = dynamic-template-lte-epc:rendered
RootSoftwareInstance = $${:enb-epc}
[local-ipv4-address]
recipe = slapos.recipe.build
init =
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
s.connect(('1.1.1.1', 1))
ip = s.getsockname()[0]
except Exception:
ip = '127.0.0.1'
finally:
s.close()
options['local-ipv4'] = ip
[dynamic-template-lte-enb-epc]
< = jinja2-template-base
template = ${template-lte-enb-epc:target}
......@@ -43,6 +57,7 @@ filename = instance-lte-enb-epc.cfg
extensions = jinja2.ext.do
extra-context =
raw monitor_template ${monitor2-template:rendered}
key local_ipv4 local-ipv4-address:local-ipv4
[dynamic-template-lte-gnb-epc]
< = jinja2-template-base
......@@ -51,6 +66,7 @@ filename = instance-lte-gnb-epc.cfg
extensions = jinja2.ext.do
extra-context =
raw monitor_template ${monitor2-template:rendered}
key local_ipv4 local-ipv4-address:local-ipv4
[dynamic-template-lte-enb]
< = jinja2-template-base
......@@ -62,6 +78,7 @@ extra-context =
raw enb ${enb:destination}
raw enb_template ${enb.jinja2.cfg:target}
raw ltelogs_template ${ltelogs.jinja2.sh:target}
raw sdr_busy_promise ${sdr-busy-promise:target}
raw openssl_location ${openssl:location}
raw default_dl_earfcn ${enb:default-dl-earfcn}
raw default_lte_dl_freq ${enb:default-lte-dl-freq}
......@@ -71,6 +88,7 @@ extra-context =
raw default_lte_rx_gain ${enb:default-lte-rx-gain}
raw min_frequency ${enb:min-frequency}
raw max_frequency ${enb:max-frequency}
key local_ipv4 local-ipv4-address:local-ipv4
[dynamic-template-lte-gnb]
< = jinja2-template-base
......@@ -82,6 +100,7 @@ extra-context =
raw enb ${enb:destination}
raw gnb_template ${gnb.jinja2.cfg:target}
raw ltelogs_template ${ltelogs.jinja2.sh:target}
raw sdr_busy_promise ${sdr-busy-promise:target}
raw openssl_location ${openssl:location}
raw default_dl_nr_arfcn ${enb:default-dl-nr-arfcn}
raw default_nr_band ${enb:default-nr-band}
......@@ -91,6 +110,7 @@ extra-context =
raw default_nr_rx_gain ${enb:default-nr-rx-gain}
raw min_frequency ${enb:min-frequency}
raw max_frequency ${enb:max-frequency}
key local_ipv4 local-ipv4-address:local-ipv4
[dynamic-template-lte-epc]
< = jinja2-template-base
......@@ -99,6 +119,7 @@ filename = instance-lte-epc.cfg
extensions = jinja2.ext.do
extra-context =
raw monitor_template ${monitor2-template:rendered}
raw interface_up_promise ${interface-up-promise:target}
raw mme ${mme:destination}
raw mme_template ${mme.jinja2.cfg:target}
raw ims_template ${ims.jinja2.cfg:target}
......
software/ors-amarisoft/promise/check_interface_up.py 0 → 100644
View file @ a8f35d7e
import socket
import errno
from zope.interface import implementer
from slapos.grid.promise import interface
from slapos.grid.promise.generic import GenericPromise
@implementer(interface.IPromise)
class RunPromise(GenericPromise):
def __init__(self, config):
"""
Called when initialising the promise before testing.
Sets the configuration and the periodicity.
"""
super(RunPromise, self).__init__(config)
self.setPeriodicity(minute=2)
def sense(self):
"""
Called every time the promise is tested.
Signals a positive or negative result.
In this case, check whether the file exists.
"""
ifname = self.getConfig('ifname')
f = open('/sys/class/net/%s/operstate' % ifname, 'r')
if f.read() == 'up\n':
self.logger.info("%s is up", ifname)
else:
self.logger.error("%s is down", ifname)
f.close()
def test(self):
"""
Called after sense() if the instance is still converging.
Returns success or failure based on sense results.
In this case, fail if the previous sensor result is negative.
"""
return self._test(result_count=1, failure_amount=1)
def anomaly(self):
"""
Called after sense() if the instance has finished converging.
Returns success or failure based on sense results.
Failure signals the instance has diverged.
In this case, fail if two out of the last three results are negative.
"""
return self._anomaly(result_count=3, failure_amount=2)
software/ors-amarisoft/promise/check_sdr_busy.py 0 → 100644
View file @ a8f35d7e
import os
import errno
from zope.interface import implementer
from slapos.grid.promise import interface
from slapos.grid.promise.generic import GenericPromise
@implementer(interface.IPromise)
class RunPromise(GenericPromise):
def __init__(self, config):
"""
Called when initialising the promise before testing.
Sets the configuration and the periodicity.
"""
super(RunPromise, self).__init__(config)
self.setPeriodicity(minute=2)
def sense(self):
"""
Called every time the promise is tested.
Signals a positive or negative result.
In this case, check whether the file exists.
"""
sdr_dev = '/dev/sdr0'
try:
open(sdr_dev, 'w').close()
self.logger.error("eNB is not using %s", sdr_dev)
except IOError as e:
if e.errno == errno.EBUSY:
self.logger.info("eNB is using %s", sdr_dev)
def test(self):
"""
Called after sense() if the instance is still converging.
Returns success or failure based on sense results.
In this case, fail if the previous sensor result is negative.
"""
return self._test(result_count=1, failure_amount=1)
def anomaly(self):
"""
Called after sense() if the instance has finished converging.
Returns success or failure based on sense results.
Failure signals the instance has diverged.
In this case, fail if two out of the last three results are negative.
"""
return self._anomaly(result_count=3, failure_amount=2)
software/ors-amarisoft/software.cfg
View file @ a8f35d7e
......@@ -49,6 +49,12 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
[template-lte-epc]
<= download-base
[sdr-busy-promise]
<= download-base
[interface-up-promise]
<= download-base
[amarisoft]
recipe = slapos.recipe.build
path = /opt/amarisoft/lte
......
software/repman/buildout.hash.cfg
View file @ a8f35d7e
......@@ -14,19 +14,19 @@
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = 8a08be95a04f1a47098c4fdef80bdfed
md5sum = af2fc4a7a0f782fed2cb1112ef3cb397
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 697a1b546c883da45c14dbcd2d73b2b9
md5sum = 7782adde357c9444fed51f2dcc230a52
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
md5sum = 2ee2c4bc5f985c11c2167b819d261256
md5sum = 0a4ce036ea524373a85fac10e81edb96
[config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in
md5sum = c78c6be9537d3dffbb526b0e819bacb5
md5sum = 8cdeb63a2acc294801266f24599f6d32
[nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in
......@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = a5c204cac552754520aee0570d379723
md5sum = 938f1e8087395757f5ae861d20500658
[template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in
......@@ -42,15 +42,19 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
[template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = df44c2d6fb1971df582345daedad280c
md5sum = 47b76144e1b116580c8acf08274af976
[template-publish-slave-information]
_update_hash_filename_ = publish-database-slave-parameters.cfg.in
md5sum = 9616477ff993c55c2b43bf8797db6994
[mariadb-init-root-sql]
_update_hash_filename_ = templates/mariadb_init_root.sql.in
md5sum = d8a0a7d72b02f911dafda655941b805d
md5sum = 6e5c02542568f250aacd84694249efc0
[init-root-wrapper-in]
_update_hash_filename_ = templates/init_root_wrapper.in
md5sum = 7661e3b2f66982b9b17bd2ab73e2e8ef
md5sum = c203f40a58386310a433b58fd345a341
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
......@@ -58,12 +62,16 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
md5sum = d2ebd2ec55bf8489789a52c808729925
md5sum = d5c19d866c5a76d3bc2f434a91b1553a
[mysqld-need-start.sh.in]
_update_hash_filename_ = templates/mysqld-need-start.sh.in
md5sum = e9bcee5dc1318fe3acda2663472214f5
md5sum = 8f14054b3761b167cf8a86f4f084a574
[proxy-need-start-stop.sh.in]
_update_hash_filename_ = templates/proxy-need-start-stop.sh.in
md5sum = 455aaf369bf5141758dc57f2c0e67b08
[slave-db-manage.in]
_update_hash_filename_ = templates/slave-db-manage.in
md5sum = b45313ae5fb06972cc7fc945e34e434a
software/repman/instance-mariadb.cfg.jinja2.in
View file @ a8f35d7e
......@@ -17,17 +17,11 @@
[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
-extends = publish-early
database-host = {{ host }}:{{ port }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
partition-path = ${buildout:directory}
receiver-port = ${dbjob-parameter:socat-port}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
database-list = {{ dumps(database_list) }}
csr-id = ${caucase-csr-id:csr-id}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
......@@ -37,6 +31,22 @@ mode = 644
< = jinja2-template-base
mode = 755
[jinja2-template-script-base]
< = jinja2-template-executable
context =
key tmp_dir directory:tmp
key partition_dir buildout:directory
raw username {{ slapparameter_dict['repman-user'] }}
raw password {{ slapparameter_dict['repman-passwd'] }}
raw cluster_name {{ slapparameter_dict['cluster'] }}
raw secure_url {{ slapparameter_dict['repman-secure-url'] }}
raw mysql_conf ${directory:etc}/mysql/my.cnf
raw dash_bin {{ dash }}
raw jq_bin {{ jq_bin }}
raw curl_bin {{ curl_bin }}
raw mysql_bin {{ parameter_dict['mariadb-location'] }}/bin/mysql
${:extra-context}
[simplefile]
< = jinja2-template-base
template = inline:{{ '{{ content }}' }}
......@@ -53,10 +63,54 @@ context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
{% import "caucase" as caucase with context %}
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-updater',
url=slapparameter_dict['caucase-url'],
data_dir='${mysql-directory:ssl}',
crt_path='${mysql-directory:ssl}/server-cert.pem',
ca_path='${mysql-directory:ssl}/ca-cert.pem',
crl_path='${mysql-directory:ssl}/crl.pem',
key_path='${mysql-directory:ssl}/server-key.pem',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
)}}
{% do part_list.append('caucase-updater') -%}
{% do part_list.append('caucase-updater-promise') -%}
[get-csr-id]
recipe = plone.recipe.command
output = ${directory:tmp}/csr_id
command =
if [ -s "${mysql-directory:ssl}/server-cert.pem" ]; then
RESULT="None";
else
if [ -f "${caucase-updater-csr:csr}" ]; then
RESULT=$({{ caucase_bin_client }} --ca-url {{ slapparameter_dict['caucase-url'] }} --send-csr ${caucase-updater-csr:csr} | cut -d ' ' -f1)
if [ ! $? -eq 0 ]; then
RESULT="None";
fi
fi
fi
cat <<EOF > ${:output}
[caucase]
csr-id = $(echo $RESULT)
EOF
update-command = ${:command}
[caucase-csr-id]
recipe = slapos.cookbook:zero-knowledge.read
file-path = ${get-csr-id:output}
csr-id =
[my-cnf-parameters]
socket = ${directory:run}/mariadb.sock
ip = {{ ip }}
data-directory = ${directory:srv}/mariadb
data-directory = ${mysql-directory:data}
pid-file = ${directory:run}/mariadb.pid
plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
......@@ -66,9 +120,6 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0
innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
ssl-crt = ${directory:mariadb-ssl}/crt.pem
ssl-key = ${directory:mariadb-ssl}/key.pem
ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
[my-cnf]
< = jinja2-template-base
......@@ -89,9 +140,11 @@ database-list = {{ dumps(database_list) }}
mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
root-user = repman
heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }}
require-ssl = {{ dumps(slapparameter_dict['require-ssl']) }}
[init-script]
< = jinja2-template-executable
extensions = jinja2.ext.do
# XXX: is there a better location ?
rendered = ${directory:etc}/mariadb_initial_setup.sql
template = {{ parameter_dict['template-mariadb-initial-setup'] }}
......@@ -106,20 +159,26 @@ init-script = ${init-script:rendered}
mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
[{{ section('update-mysql-script') }}]
< = jinja2-template-executable
< = jinja2-template-script-base
rendered = ${directory:scripts}/mariadb_update
init-password = ${directory:etc}/.init-passwd.done
upgrade-done = ${directory:lib}/mariadb-update-done
context =
extra-context =
key init_password_done :init-password
key upgrade_done :upgrade-done
key init_root_sql init-root-sql:rendered
key mysql_update update-mysql:output
raw mysql_conf ${directory:etc}/mysql/my.cnf
raw dash_bin {{ dash }}
raw mysql_bin {{ parameter_dict['mariadb-location'] }}/bin/mysql
key init_database_sql init-script:rendered
template = {{ parameter_dict['template-init-root-wrapper'] }}
[{{ section('mysql-slave-db-cleanup') }}]
< = jinja2-template-script-base
rendered = ${directory:scripts}/manage-slave-db
db-name = {{ slapparameter_dict['database-name'] }}
extra-context =
key database_name :db-name
template = {{ parameter_dict['template-manage-db'] }}
[mysqld]
< = jinja2-template-executable
rendered = ${directory:bin}/mysqld
......@@ -136,19 +195,19 @@ environ =
{{ variable }}
{%- endfor %}
[ca-mysqld]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
key-file = ${my-cnf-parameters:ssl-key}
cert-file = ${my-cnf-parameters:ssl-crt}
executable = ${mysqld:rendered}
wrapper = ${directory:controller}/mariadb
[mysqld-launcher]
recipe = slapos.cookbook:wrapper
command-line = ${mysqld:rendered}
wrapper-path = ${directory:controller}/mariadb
wait-for-files =
${mysql-directory:mysql}/my.cnf
${mysql-directory:ssl}/server-cert.pem
{% import "supervisord_lib" as supervisord_lib with context %}
{{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
{% do part_list.append("supervisord-mariadb-ctl") -%}
{% set maradb_program_dict = {"name": "mariadb", "command": "${ca-mysqld:wrapper}",
"stopwaitsecs": 86400, "environment": [],
{% set maradb_program_dict = {"name": "mariadb", "command": "${mysqld-launcher:wrapper-path}",
"stopwaitsecs": 86400, "startretries": 10, "autorestart": True, "environment": [],
"stdout_logfile": "${directory:log}/mariadb_stdout.log",
"stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
......@@ -178,13 +237,18 @@ target-directory = ${directory:bin}
link-binary = {{ dumps(parameter_dict['link-binary']) }}
[binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it.
command-line =
"{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
--defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args}
wrapper-path = ${directory:bin}/${:command}
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/${:command}
mode = 755
template =
inline:#!/bin/sh
exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
--defaults-file="${directory:etc}/mysql/my.cnf" \
--protocol=socket \
${:extra-args} "$@"
wrapper-path = ${:rendered}
extra-args =
[binary-wrap-mysql]
......@@ -208,6 +272,11 @@ command = mysqladmin
<= binary-wrap-base
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
command = mysql_tzinfo_to_sql
template =
inline:#!/bin/sh
exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
--defaults-file="${directory:etc}/mysql/my.cnf" \
--skip-write-binlog "$@"
[binary-wrap-pt-digest]
<= binary-wrap-base
......@@ -226,26 +295,29 @@ plugin = ${:etc}/plugin
srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp
backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental
mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var
lib = ${:var}/lib
mysql = ${:lib}/mysql
log = ${:var}/log
run = ${:var}/run
config-tmp = ${:tmp}/config
custom = ${directory:etc}/mysql/custom
[mysql-directory]
recipe = slapos.cookbook:mkdirectory
mysql = ${directory:etc}/mysql
ssl = ${:mysql}/ssl
data = ${directory:lib}/mysql
custom = ${:mysql}/custom.d
[dbjob-parameter]
bash-bin = {{ bash_bin }}
db-user = ${init-script-parameters:root-user}
db-password = ${init-script-parameters:password}
mysql-dir = ${directory:mysql}
dbjob-cnf = ${directory:etc}/mysql/my.cnf
log-dir = ${directory:mysql}/.system/logs
mysql-dir = ${mysql-directory:data}
dbjob-cnf = ${mysql-directory:mysql}/dbjob.cnf
log-dir = ${mysql-directory:data}/.system/logs
tmp-dir = ${directory:tmp}
mysqld-socket = ${my-cnf-parameters:socket}
mysql-client = ${binary-wrap-mysql:wrapper-path}
socat-port = {{ int(port) + 9 }}
restart-script = ${mysqld-restart-script:rendered}
socat-location = {{ parameter_dict['socat-location'] }}
......@@ -309,11 +381,11 @@ template = inline:#!{{ bash_bin }}
cd ${directory:config-tmp} &&
{{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config
tar -xzf config.tar.gz
cp -r data/.system ${directory:mysql}
rm -rf ${directory:etc}/mysql
cp -r etc/mysql ${directory:etc}
ln -sf ${directory:mysql}/.system ${directory:var}/system
ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom/01_mariadb.cnf
rm -r etc/mysql/ssl # we don't need generated ssl files
cp -r data/.system ${mysql-directory:data}
cp -r etc/mysql/* ${mysql-directory:mysql}
ln -sf ${mysql-directory:data}/.system ${directory:var}/system
ln -sf ${my-cnf:rendered} ${mysql-directory:custom}/01_mariadb.cnf
[{{ section('install-mysql-config') }}]
recipe = plone.recipe.command
......
software/repman/instance-repman-input-schema.json
View file @ a8f35d7e
......@@ -89,33 +89,6 @@
"type": "string",
"default": ""
},
"tags": {
"title": "Provisioning db tag list",
"description": "Provisioning db tags. Set one tag per line.",
"type": "array",
"items": {
"type": "string"
},
"default": [
"gtidstrict",
"bind",
"pkg",
"innodb",
"noquerycache",
"slow",
"pfs",
"linux",
"readonly",
"diskmonitor",
"sqlerror",
"compressbinlog",
"bm4ci",
"mroonga",
"utctime",
"readcommitted",
"nohandshake"
]
},
"http-session-lifetime": {
"title": "Web Session life time in s",
"description": "Web interface Session life time in seconds. Default 86400",
......@@ -128,17 +101,11 @@
"type": "integer",
"default": 4
},
"autorejoin": {
"title": "Automatic rejoin a failed master",
"description": "Automatic rejoin a failed master (default true)",
"require-ssl": {
"title": "Enable REQUIRE SSL for repman database users",
"description": "Enable secure connection between repman and mariadb databases. This option is an initial value, used only when creating database users.",
"type": "boolean",
"default": true
},
"autoseed": {
"title": "Automatic join a standalone node",
"description": "Automatic join a standalone node",
"type": "boolean",
"default": true
"default": false
},
"repman-cluster-dict": {
"title": "Replication Manager clusters definition",
......@@ -189,6 +156,34 @@
"type": "string",
"default": ""
},
"tags": {
"title": "Provisioning db tag list",
"description": "Provisioning db tags. Set one tag per line.",
"type": "array",
"items": {
"type": "string"
},
"default": [
"gtidstrict",
"bind",
"pkg",
"innodb",
"noquerycache",
"slow",
"pfs",
"linux",
"readonly",
"diskmonitor",
"sqlerror",
"compressbinlog",
"bm4ci",
"mroonga",
"utctime",
"readcommitted",
"nohandshake",
"ssl"
]
},
"proxysql-user": {
"title": "Proxysql username",
"description": "Proxysql external user, default is 'external'.",
......@@ -197,7 +192,7 @@
},
"proxy-tags": {
"title": "Proxy tag list",
"description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\"]",
"description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\", \"ssl\"]",
"type": "array",
"items": {
"type": "string"
......@@ -206,7 +201,8 @@
"pkg",
"masterslave",
"linux",
"noreadwritesplit"
"noreadwritesplit",
"ssl"
]
},
"logical-backup-cron": {
......@@ -302,6 +298,18 @@
"type": "boolean",
"default": true
},
"autorejoin": {
"title": "Automatic rejoin a failed master db",
"description": "Automatic rejoin a failed master (default: true).",
"type": "boolean",
"default": true
},
"autoseed": {
"title": "Automatic join a standalone node",
"description": "Automatic join a standalone mariadb node to the cluster.",
"type": "boolean",
"default": true
},
"failover-mode": {
"title": "Failover mode",
"description": "Failover is manual or automatic (default \"manual\").",
......
software/repman/instance-repman-slave-input-schema.json 0 → 100644
View file @ a8f35d7e
{
"$schema": "http://json-schema.org/draft-04/schema",
"properties": {
"db_user": {
"description": "Database User, default is 'user'. This parameter is set only if database is not created yet.",
"title": "Database User",
"type": "string",
"default": "user"
},
"db_password": {
"description": "Database password. If no password set, a password will be generated. This parameter is set only if database is not created yet.",
"title": "Initial database password",
"type": "string",
"default": ""
},
"db_charset": {
"description": "Database Character set information. This parameter is set only if database is not created yet.",
"title": "Database Character set information",
"type": "string",
"enum": [
"latin1",
"latin2",
"latin5",
"latin7",
"ascii",
"latin5",
"utf8",
"utf8mb4",
"utf16",
"utf16le",
"utf32",
"binary",
"utf8_general_ci",
"utf8_unicode_ci",
"utf8_unicode_ci",
"utf8mb4_bin",
"utf8mb4_general_ci",
"utf8mb4_unicode_ci",
"latin1_general_ci",
"latin1_bin",
"ascii_general_ci",
"ascii_bin"
],
"default": ""
}
},
"title": "Input Parameters",
"type": "object"
}
software/repman/instance-repman.cfg.jinja2.in
View file @ a8f35d7e
......@@ -8,8 +8,23 @@
{% set ip = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set cluster_list = [] -%}
{% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow", "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci", "mroonga", "utctime", "readcommitted", "nohandshake"] -%}
{% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow",
"pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci",
"mroonga", "utctime", "readcommitted", "nohandshake", "ssl"] -%}
{% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
{% set database_slave_list = [] -%}
{% set db_name_list = [] -%}
{% set count = namespace(value=2) %}
{% set caucase_bind = '[' ~ ip ~ ']:8890' -%}
{% set caucase_url = 'http://' ~ caucase_bind -%}
{% set csrid_list = [] -%}
{% macro password(name) -%}
[{{ name }}-password]
recipe = slapos.cookbook:generate.password
bytes = 12
{% endmacro -%}
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -25,11 +40,25 @@ log = ${:var}/log
data = ${:var}/lib
nginx-prefix = ${:var}/nginx
tmp = ${:home}/tmp
backup-caucased = ${:srv}/backup-caucased
caucased = ${:srv}/caucased
ssl = ${:etc}/ssl
[proxysql-directory]
recipe = slapos.cookbook:mkdirectory
config = ${directory:etc}/proxysql
ssl = ${:config}/ssl
{% import "supervisord_lib" as supervisord_lib with context %}
{% set proxysql_controller = "proxysql-ctl" -%}
{{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
[proxysql-controller-update]
recipe = slapos.cookbook:wrapper
command-line = ${proxysql-ctl-bin:wrapper-path} update
wrapper-path = ${directory:scripts}/proxysql-ctl-update
{% do part_list.append("supervisord-proxysql-ctl") -%}
{% do part_list.append("proxysql-controller-update") -%}
[request-common]
recipe = slapos.cookbook:request.serialised
......@@ -39,6 +68,7 @@ key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
config-caucase-url = {{ caucase_url }}
[download-proxy-config]
recipe = slapos.recipe.template:jinja2
......@@ -48,16 +78,35 @@ template = inline:#!{{ bash_bin }}
PORT=$3
CONFIG=$4
if [ -z "$CONFIG" ]; then
CONFIG="${repman:proxies}/proxysql-$NAME.cnf"
CONFIG="${proxysql-directory:config}/proxysql.cnf"
fi
mkdir -p ${repman:config-tmp}/proxies
cd ${repman:config-tmp}/proxies
{{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
tar -xzf proxies-$NAME.tar.gz
cp conf/proxysql.cnf $CONFIG
cp etc/proxysql/proxysql.cnf $CONFIG
rendered = ${directory:bin}/update-proxysql-config
mode = 755
{% for instance_dict in slave_instance_list -%}
{% set slave_dict = {
'name': 'db_%s' % instance_dict['slave_reference'].replace('-', '_').lower(),
'user': instance_dict.get('db_user', 'user'),
'password': instance_dict.get('db_password', '${' ~ instance_dict['slave_reference'] ~ '-password:passwd}'),
'slave_reference': instance_dict['slave_reference'],
'charset': instance_dict.get('db_charset', ''),
'require_ssl': True
} -%}
{% do database_slave_list.append(slave_dict) -%}
{% do db_name_list.append(slave_dict['name']) -%}
{{ password(instance_dict['slave_reference']) }}
{% endfor %}
[database-slave-information]
{% for slave_dict in database_slave_list -%}
{{ slave_dict['name'] }} = !py!{{ slave_dict }}
{% endfor %}
{% set db_list = db_name_list | join(' ') -%}
{% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
"database-amount": 2, "proxysql-user": "external",
......@@ -67,22 +116,23 @@ mode = 755
{% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
{% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
{% set database_list = parameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
# Request mariadb instances
{% set db_amount = parameter_dict.get('database-amount', 2) -%}
{% if db_amount < 2 -%}
{% set db_amount = 2 -%}
{% endif -%}
{% set count.value = count.value + db_amount %}
{% for i in range(0, db_amount) -%}
{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
{% set section = 'request-mariadb-' ~ i -%}
{% set dbname = 'Mariadb-' ~ i -%}
{% set dbname = 'Mariadb-' ~ i %}
[{{ section }}]
<= request-common
software-type = mariadb
name = {{ dbname }}
name = {{ dbname ~ '-' ~ name}}
sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
{% for key, value in mariadb_dict.items() -%}
config-{{ key }} = {{ dumps(value) }}
......@@ -91,23 +141,27 @@ config-monitor-passwd = ${publish-early:monitor-password}
config-root-password = ${publish-early:db-root-password}
config-repman-user = ${repman-parameter:username}
config-heartbeat-user = ${repman-parameter:heartbeat-user}
#config-repman-passwd = ${repman-parameter:password}
config-repman-passwd = ${repman-parameter:password}
config-repman-url = ${nginx-parameter:backend-url}
config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
config-cluster = {{ name }}
config-name = {{ dbname }}
config-database-list = {{ dumps(database_list) }}
config-database-list = !py!{{ database_slave_list }}
config-database-name = {{ dumps(db_list) }}
config-require-ssl = {{ dumps(slapparameter_dict.get('require-ssl', False)) }}
return =
database-host
receiver-port
monitor-base-url
partition-path
csr-id
{% do part_list.append(section) -%}
{% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
{% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
{% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
{% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do csrid_list.append('${' ~ section ~ ':connection-csr-id}') -%}
{% endfor -%}
......@@ -129,6 +183,7 @@ maximum = 7132
{% for key, value in parameter_dict.items() -%}
{{ key }} = {{ value }}
{% endfor -%}
name = {{ name }}
proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
proxy-port = {{ '${' ~ name ~ '-port:port}' }}
proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
......@@ -136,6 +191,8 @@ db-user = repman
db-password = ${publish-early:db-root-password}
db-list = {{ mariadb_server_list | join(',') }}
autorejoin = {{ parameter_dict.get("autorejoin", True) }}
autoseed = {{ parameter_dict.get("autoseed", True) }}
heartbeat-user = ${repman-parameter:heartbeat-user}
heartbeat-password = ${publish-early:db-root-password}
partition-list = {{ mariadb_path_list | join(',') }}
......@@ -150,8 +207,8 @@ proxysql-servers-ipv6 = [{{ ip }}]
password = ${repman-parameter:password}
proxysql-partition = ${buildout:directory}
receiver-port-list = {{ receiver_port_list | join(',') }}
enabled-tags = {{ slapparameter_dict.get("tag-list", tag_list) | join(',') }}
proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit"]) | join(',') }}
enabled-tags = {{ parameter_dict.get("tag-list", tag_list) | join(',') }}
proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit", "ssl"]) | join(',') }}
logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }}
......@@ -191,7 +248,7 @@ context =
recipe = plone.recipe.command
# if Repman is not started, cannot download config from server
stop-on-error = false
config = ${repman:proxies}/proxysql-{{ name }}.cnf
config = ${proxysql-directory:config}/proxysql.cnf
data = ${repman:proxy-data}/{{ name }}
command =
mkdir -p ${:data} &&
......@@ -209,6 +266,7 @@ command-line =
wrapper-path = ${directory:controller}/proxysql-{{ name }}
wait-for-files =
${repman:bootstrap}/{{ name }}_bootstrapped
${config-proxysql-{{ name }}:config}
depends =
{{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }}
{{ '${proxysql-' ~ name ~ '-promise:recipe}' }}
......@@ -237,7 +295,7 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
{% set service_name = "proxysql-" ~ name -%}
{% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
"stopwaitsecs": 60, "environment": [],
"stopwaitsecs": 60, "startretries": 10, "autorestart": True, "environment": [],
"stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
"stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
......@@ -247,22 +305,126 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
{% do part_list.append('config-' ~ name) -%}
{% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
{% set publish_database_list = [] -%}
{% set publish_database_v6_list = [] -%}
{% for database in database_list -%}
{% if database.get('user') -%}
{% do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% do publish_database_v6_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% else -%}
{% do publish_database_list.append("mysql://" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% do publish_database_v6_list.append("mysql://[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% endif -%}
{% endfor -%}
{% do publish_dict.__setitem__(name ~ '-database-list', "!py!['" ~ publish_database_list | join("', '") ~ "']") -%}
{% do publish_dict.__setitem__(name ~ '-database-list-v6', "!py!['" ~ publish_database_v6_list | join("', '") ~ "']") -%}
[{{ name}}-publish-slave-information]
recipe = slapos.cookbook:switch-softwaretype
default = {{ 'dynamic-' ~name ~ '-publish-slave-information:rendered' }}
RootSoftwareInstance = ${:default}
[dynamic-{{ name}}-publish-slave-information]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/database-publish-slave-information.cfg
extensions = jinja2.ext.do
template = {{ template_publish_slave_information }}
context =
import json_module json
key proxy_port {{ name }}-cluster-parameter:proxy-port
raw eggs_directory {{ eggs_directory }}
raw develop_eggs_directory {{ develop_eggs_directory }}
raw ipv6 {{ ip }}
raw ipv4 {{ ipv4 }}
section slave_dict database-slave-information
{% do part_list.append(name ~ '-publish-slave-information') -%}
{% endfor -%}
# deploy caucase
{% import "caucase" as caucase with context %}
{{ caucase.caucased(
prefix='caucased',
buildout_bin_directory=buildout_bin_directory,
caucased_path='${directory:service}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:caucased}',
netloc=caucase_bind,
tmp='${directory:tmp}',
service_auto_approve_count=count.value,
user_auto_approve_count=1,
key_len=2048,
)}}
{% do part_list.append('caucased') -%}
{% do part_list.append('caucased-promise') -%}
{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-updater',
url=caucase_url,
data_dir='${directory:srv}/caucase-updater',
crt_path='${directory:ssl}/repman-cert.crt',
ca_path='${directory:srv}/caucase-updater/ca.crt',
crl_path='${directory:srv}/caucase-updater/crl.pem',
key_path='${directory:ssl}/repman-cert.key',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
)}}
{% do part_list.append('caucase-updater') -%}
{% do part_list.append('caucase-updater-promise') -%}
#caucase user certificate
{{ caucase.updater(
prefix='caucase-user-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-user-updater',
url=caucase_url,
data_dir='${directory:srv}/caucase-user-updater',
crt_path='${directory:ssl}/caucase.user.crt',
ca_path='${directory:srv}/caucase-user-updater/ca.crt',
crl_path='${directory:srv}/caucase-user-updater/crl.pem',
key_path='${directory:ssl}/caucase.user.key',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
mode='user',
)}}
{% do part_list.append('caucase-user-updater') -%}
{% do part_list.append('caucase-user-updater-promise') -%}
#caucase proxysql certificate
{{ caucase.updater(
prefix='caucase-proxysql-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-proxysql-updater',
url=caucase_url,
data_dir='${proxysql-directory:ssl}',
crt_path='${proxysql-directory:ssl}/client-cert.pem',
ca_path='${proxysql-directory:ssl}/ca-cert.pem',
crl_path='${proxysql-directory:ssl}/crl.pem',
key_path='${proxysql-directory:ssl}/client-key.pem',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
)}}
{% do part_list.append('caucase-proxysql-updater') -%}
{% do part_list.append('caucase-proxysql-updater-promise') -%}
[caucase-sign-csr]
recipe = slapos.recipe.template:jinja2
mode = 755
rendered = ${directory:bin}/caucase-sign
template =
inline:#!/bin/sh
cp ${directory:ssl}/caucase.user.key ${directory:ssl}/caucase-full.key
cat ${directory:ssl}/caucase.user.crt >> ${directory:ssl}/caucase-full.key
for csr_id in {{ csrid_list | join(' ') }}; do
if [ "$csr_id" = "None" ] || [ -z "$csr_id"]; then
continue
fi
{{ buildout_bin_directory }}/caucase --ca-url {{ caucase_url }} --user-key ${directory:ssl}/caucase-full.key --sign-csr $csr_id
done
[cron-caucase-sign-csr]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = caucase-sign-csr
frequency = * * * * *
command = ${caucase-sign-csr:rendered}
{% do part_list.append('cron-caucase-sign-csr') -%}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
......@@ -286,7 +448,6 @@ etc = ${directory:etc}/repman
data-dir = ${directory:var}/lib
root-dir = ${directory:srv}/repman
clusters = ${:etc}/cluster.d
proxies = ${:etc}/proxy
proxy-data = ${:data-dir}/proxy
config-tmp = ${directory:tmp}/config
bootstrap = ${:etc}/bootstrap
......@@ -296,8 +457,9 @@ proxies-log = ${directory:log}/proxy
ipv6 = ${instance-parameter:ipv6-random}
port = ${instance-parameter:nginx-port}
ssl-port = ${instance-parameter:nginx-ssl-port}
ssl-certificate = ${ca-nginx:cert-file}
ssl-key = ${ca-nginx:key-file}
ssl-certificate = ${directory:ssl}/repman-cert.crt
ca-certificate = ${directory:srv}/caucase-updater/ca.crt
ssl-key = ${directory:ssl}/repman-cert.key
pid-file = ${directory:run}/nginx.pid
access-log = ${directory:log}/nginx_access.log
error-log = ${directory:log}/nginx_error.log
......@@ -326,8 +488,6 @@ username = admin
heartbeat-user = heartbeat
password = ${publish-early:repman-password}
cluster-d = ${repman:clusters}
autorejoin = {{ slapparameter_dict.get("autorejoin", True) }}
autoseed = {{ slapparameter_dict.get("autoseed", True) }}
mysql-bin-dir = {{ mariadb_location }}/bin
mysqlbinlog-path = {{ mariadb_location }}/bin/mysqlbinlog
mysqlclient-path = {{ mariadb_location }}/bin/mysql
......@@ -345,6 +505,8 @@ mail-to = {{ slapparameter_dict.get("mail-to", "") }}
http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }}
http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }}
ssl-cert = ${directory:ssl}/repman-cert.crt
ssl-key = ${directory:ssl}/repman-cert.key
[repman-config-folder]
recipe = plone.recipe.command
......@@ -392,6 +554,9 @@ depends =
${replication-manager-reload:recipe}
${repman-listen-promise:recipe}
${repman-listen-ssl-promise:recipe}
wait-for-files =
${directory:ssl}/repman-cert.crt
${directory:ssl}/repman-cert.key
[repman-config.toml]
recipe = slapos.recipe.template:jinja2
......@@ -428,11 +593,12 @@ command-line =
{{ nginx_bin }}
-p ${directory:nginx-prefix}
-c ${nginx-conf:rendered}
wrapper-path = ${directory:bin}/nginx-start
wrapper-path = ${directory:services}/nginx
wait-for-files =
${ca-directory:certs}/nginx.key
${ca-directory:certs}/nginx.crt
${nginx-graceful-wrapper:rendered}
${directory:ssl}/repman-cert.crt
${directory:srv}/caucase-updater/ca.crt
${directory:ssl}/repman-cert.key
[nginx-graceful-wrapper]
recipe = slapos.recipe.template:jinja2
......@@ -442,20 +608,6 @@ rendered = ${directory:scripts}/nginx-graceful
context =
mode = 755
[ca-nginx]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
cert-file = ${ca-directory:certs}/nginx.crt
key-file = ${ca-directory:certs}/nginx.key
executable = ${nginx-launcher:wrapper-path}
wrapper = ${directory:bin}/ca-nginx
[ca-nginx-service]
recipe = slapos.cookbook:wrapper
command-line = ${ca-nginx:wrapper}
wrapper-path = ${directory:services}/nginx
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
......@@ -468,6 +620,7 @@ recipe = slapos.cookbook:publish-early
monitor-password monitor-htpasswd:passwd
db-root-password gen-root-password:passwd
repman-password repman-password:passwd
database-list = {{ db_list }}
[publish-connection-parameter]
<= monitor-publish
......@@ -476,6 +629,7 @@ recipe = slapos.cookbook:publish
backend-url = ${nginx-parameter:backend-ssl-url}
url = ${repman-frontend:connection-secure_access}
username = ${repman-parameter:username}
database-list = {{ db_list }}
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
{% endfor %}
......@@ -545,6 +699,15 @@ command = ${template-proxysql-need-stop-start:rendered}
# Deploy replication-manager instance
#
#############################
[slap_connection]
# Kept for backward compatibility
computer_id = ${slap-connection:computer-id}
partition_id = ${slap-connection:partition-id}
server_url = ${slap-connection:server-url}
software_release_url = ${slap-connection:software-release-url}
key_file = ${slap-connection:key-file}
cert_file = ${slap-connection:cert-file}
[buildout]
extends =
{{ template_monitor }}
......@@ -552,7 +715,7 @@ parts =
replication-manager
monitor-base
logrotate-entry-nginx
ca-nginx-service
nginx-launcher
publish-connection-parameter
repman-frontend-promise
repman-backend-promise
......
software/repman/instance.cfg.in
View file @ a8f35d7e
......@@ -28,8 +28,10 @@ mode = 0644
extensions = jinja2.ext.do
rendered= ${buildout:directory}/${:_buildout_section_name_}
supervisord-lib = {{ supervisord_lib }}
causace-lib = {{ caucase_library }}
import-list =
file supervisord_lib :supervisord-lib
file caucase :causace-lib
context =
key slapparameter_dict slap-configuration:configuration
key computer_id slap-configuration:computer
......@@ -48,6 +50,8 @@ context =
raw bash_bin {{ bash_location }}/bin/bash
raw jq_bin {{ jq_location }}/bin/jq
raw curl_bin {{ curl_location }}/bin/curl
raw openssl_bin {{ openssl_location }}/bin/openssl
raw caucase_bin_client {{ caucase_bin_client }}
${:extra-context}
extra-context =
......@@ -56,6 +60,7 @@ extra-context =
<= jinja2-template-base
template= {{ template_repman_cfg }}
extra-context =
key slave_instance_list slap-configuration:slave-instance-list
raw gowork_bin {{ gowork_bin }}
raw haproxy_location {{ haproxy_location }}
raw nginx_bin {{ nginx_location }}/sbin/nginx
......@@ -71,6 +76,7 @@ extra-context =
raw proxysql_location {{ proxysql_location }}
raw template_repman_manager_sh {{ template_repman_manager_sh }}
raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
raw template_publish_slave_information {{ template_publish_slave }}
[template-mariadb-parameters]
bash = {{ bash_location }}
......@@ -83,6 +89,7 @@ template-mariadb-init-root = {{ template_init_root_sql }}
template-init-root-wrapper = {{ template_init_root_wrapper }}
template-mysqld-wrapper = {{ template_mysqld_wrapper }}
template-mysqld-need-start = {{ mysqld_start_template }}
template-manage-db = {{ template_slave_db_manage }}
link-binary = {{ dumps(mariadb_link_binary) }}
check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
bin-directory = {{ bin_directory }}
......
software/repman/publish-database-slave-parameters.cfg.in 0 → 100644
View file @ a8f35d7e
{% set part_list = [] -%}
[slap_connection]
# Kept for backward compatibility
computer_id = ${slap-connection:computer-id}
partition_id = ${slap-connection:partition-id}
server_url = ${slap-connection:server-url}
software_release_url = ${slap-connection:software-release-url}
key_file = ${slap-connection:key-file}
cert_file = ${slap-connection:cert-file}
{% for _, database in slave_dict.items() -%}
{% set slave_reference = database['slave_reference'] -%}
{% do part_list.append('publish-slave-' ~ slave_reference) -%}
[publish-slave-{{ slave_reference }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_reference }}
{% if database.get('user') -%}
url_v4 = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":" ~ proxy_port ~ '/' ~ database['name'] }}
url = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ipv6 ~ "]:" ~ proxy_port ~ '/' ~ database['name'] }}
{% else -%}
url = {{ "mysql://" ~ ipv4 ~ ":" ~ proxy_port ~ database['name'] }}
url_v6 = {{ "mysql://[" ~ ip ~ "]:" ~ proxy_port ~ database['name'] }}
{% endif -%}
{% endfor %}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
parts =
{% for part in part_list -%}
{{ ' %s' % part }}
{% endfor -%}
software/repman/software.cfg
View file @ a8f35d7e
......@@ -18,9 +18,10 @@ extends =
../../component/socat/buildout.cfg
../../component/rsync/buildout.cfg
../../component/jq/buildout.cfg
../../component/openssl/buildout.cfg
../../stack/supervisord/buildout.cfg
../../stack/monitor/buildout.cfg
../neoppod/software-common.cfg
../../stack/caucase/buildout.cfg
parts =
slapos-cookbook
......@@ -28,14 +29,43 @@ parts =
instance.cfg
template-mariadb.cfg
template-mysqld-wrapper
caucase-eggs
gowork
[mariadb]
location = ${mariadb-10.4:location}
[python]
part = python3
[gowork]
# replication-manager does not build on golang 1.17
golang = ${golang1.16:location}
[template-mysqld-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
mode = 644
template =
inline:{% raw %}#!/bin/sh -e
basedir='${mariadb:location}'
datadir='{{datadir}}'
marker=$datadir/.slapos_initializing
system=$datadir/.system
[ -d "$datadir/mysql" ] && [ ! -f "$marker" ] || {
find "$datadir/" -mindepth 1 ! \( -path $marker -o -path $system -o -path "*$system/*" \) -delete || mkdir "$datadir"
touch "$marker"
"$basedir/scripts/mysql_install_db" \
--defaults-file='{{defaults_file}}' \
--skip-name-resolve \
--auth-root-authentication-method=normal \
--basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
--datadir="$datadir"
rm "$marker"
}
{%- if environ is defined %}
{%- for variable in environ.splitlines() %}
{{ variable }} \
{%- endfor %}
{%- endif %}
exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
{% endraw %}
[instance.cfg]
recipe = slapos.recipe.template:jinja2
......@@ -45,6 +75,7 @@ mode = 0644
context =
key bash_location bash:location
key bin_directory buildout:bin-directory
key caucase_library caucase-jinja2-library:target
key config_toml_in config-toml.in:target
key config_cluster_toml_in config-cluster-toml.in:target
key coreutils_location coreutils:location
......@@ -70,6 +101,7 @@ context =
key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
key nginx_conf_in nginx.conf.in:target
key nginx_location nginx:location
key openssl_location openssl:location
key percona_toolkit_location percona-toolkit:location
key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
key repman_src_location git.signal18.io_signal18_repman:location
......@@ -87,10 +119,12 @@ context =
key template_init_root_sql mariadb-init-root-sql:target
key template_init_root_wrapper init-root-wrapper-in:target
key template_repman_cfg instance-repman.cfg:target
key template_publish_slave template-publish-slave-information:target
key template_slave_db_manage slave-db-manage.in:target
key unixodbc_location unixodbc:location
key sysbench_location sysbench:location
key proxysql_location proxysql:location
raw caucase_bin_client ${buildout:bin-directory}/caucase
[download-file]
recipe = slapos.recipe.build:download
......@@ -126,6 +160,9 @@ link-binary =
[template-mariadb-initial-setup]
<= download-file
[template-publish-slave-information]
<= download-file
[template-my-cnf]
<= download-file
......@@ -146,3 +183,6 @@ link-binary =
[proxy-need-start-stop.sh.in]
<= download-file
[slave-db-manage.in]
<= download-file
software/repman/software.cfg.json
View file @ a8f35d7e
......@@ -10,6 +10,15 @@
"request": "instance-repman-input-schema.json",
"response": "instance-repman-output-schema.json",
"index": 0
},
"default-slave": {
"title": "Replication Manager Database",
"description": "Replication Manager Database",
"software-type": "default",
"request": "instance-repman-slave-input-schema.json",
"response": "instance-repman-output-schema.json",
"shared": true,
"index": 1
}
}
}
software/repman/templates/cluster-config.toml.in
View file @ a8f35d7e
......@@ -12,6 +12,11 @@ monitoring-write-heartbeat-credential="{{ parameter_dict['heartbeat-user'] }}:{{
db-servers-connect-timeout = 1
slapos-db-partitions = "{{ parameter_dict['partition-list'] }}"
slapos-proxysql-partitions = "{{ parameter_dict['proxysql-partition'] }}"
autorejoin = {{ setbool(parameter_dict['autorejoin']) }}
autoseed = {{ setbool(parameter_dict['autoseed']) }}
{% if parameter_dict['autoseed'] in ['true', 'True'] -%}
autorejoin-logical-backup = true
{% endif -%}
proxysql = true
proxysql-port = {{ parameter_dict['proxy-port'] }}
......@@ -40,6 +45,7 @@ scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backu
backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump"
scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
#scheduler-db-servers-receiver-use-ssl ## set to true when ssl enabled on socat
prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
......@@ -49,6 +55,7 @@ prov-db-memory = {{ parameter_dict['db-memory'] }}
prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
replication-use-ssl = true
test-inject-traffic = true
# failover
failover-mode = "{{ parameter_dict['failover-mode'] }}"
......
software/repman/templates/config.toml.in
View file @ a8f35d7e
{% macro setbool(value) -%}
{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
{% endmacro -%}
[Default]
api-bind = "{{ parameter_dict['ipv4'] }}"
http-bind-address = "{{ parameter_dict['ipv4'] }}"
......@@ -11,17 +8,14 @@ monitoring-save-config = false
api-https-bind = true
api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
include = "{{ parameter_dict['cluster-d'] }}"
autorejoin = {{ setbool(parameter_dict['autorejoin']) }}
autoseed = {{ setbool(parameter_dict['autoseed']) }}
{% if parameter_dict['autoseed'] in ['true', 'True'] -%}
autorejoin-logical-backup = true
{% endif -%}
db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
# Database list of hosts to ignore in election
#db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
monitoring-address = "{{ parameter_dict['ipv4'] }}"
monitoring-wait-retry = 40
monitoring-wait-retry = 1200
monitoring-ssl-cert = "{{ parameter_dict['ssl-cert'] }}"
monitoring-ssl-key = "{{ parameter_dict['ssl-key'] }}"
#haproxy = true
#haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
......@@ -48,6 +42,7 @@ backup-restic-password = "{{ parameter_dict['password'] }}"
backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
backup-mysqldump-options = "--hex-blob --single-transaction --verbose --all-databases --add-drop-database"
# Mail configuration
# Alert email sender (default "mrm@localhost")
......
software/repman/templates/dbjobs.in
View file @ a8f35d7e
#!/bin/bash
USER={{ parameter_dict['db-user'] }}
DBUSER={{ parameter_dict['db-user'] }}
PASSWORD={{ parameter_dict['db-password'] }}
ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
DATADIR={{ parameter_dict['mysql-dir'] }}/
MYSQL_CLIENT={{ parameter_dict['mysql-client'] }}
{% if parameter_dict['use-ipv6'] == True -%}
{% set listen = "TCP6-LISTEN" -%}
{% else -%}
......@@ -16,45 +18,45 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "
doneJob()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
$MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
}
pauseJob()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
$MYSQL_CLIENT -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
}
partialRestore()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
$MYSQL_CLIENT -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
$MYSQL_CLIENT -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
$MYSQL_CLIENT -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
$MYSQL_CLIENT -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
done
for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file=/etc/mysql/dbjob.cnf -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
$MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done
for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
$MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done
done
for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
$MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
done
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e"flush privileges;start slave;"
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER
$MYSQL_CLIENT -e "flush privileges;start slave;"
}
kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
......@@ -62,12 +64,12 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
for job in "${JOBS[@]}"
do
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N))
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER -N))
ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
ID=($(echo $TASK | awk -F@ '{ print $1 }'))
#purge de past
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
$MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
if [ "$ADDRESS" == "" ]; then
echo "No $job needed"
......@@ -77,14 +79,14 @@ do
reseedmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="reset master;set sql_log_bin=0" &>{{ parameter_dict['log-dir'] }}/dbjob.out
$MYSQL_CLIENT -e 'start slave;'
;;
flashbackmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER &>{{ parameter_dict['log-dir'] }}/dbjob.out
$MYSQL_CLIENT -e 'start slave;'
;;
reseedmariabackup)
rm -rf $BACKUPDIR
......@@ -107,7 +109,7 @@ do
;;
mariabackup)
cd {{ parameter_dict['tmp-dir'] }}
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=root --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
error)
cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
......@@ -116,10 +118,10 @@ do
cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
optimize)
mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
$MYSQL_CLIENT -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
restart)
{{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out
{{ parameter_dict['restart-script'] }} &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
esac
doneJob
......
software/repman/templates/init_root_wrapper.in
View file @ a8f35d7e
......@@ -2,7 +2,7 @@
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
--protocol=socket -uroot -hlocalhost $@
--protocol=socket -hlocalhost "$@"
}
if [ ! -f "{{ init_password_done }}" ]; then
......@@ -19,6 +19,9 @@ if [ ! -f "{{ init_password_done }}" ]; then
fi
echo "Setting mariabdb root password...";
run_mysql < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
echo "remove unwanted users...";
run_mysql -e "DROP USER IF EXISTS 'root'@'$HOSTNAME';"
run_mysql -e "DROP USER IF EXISTS ''@'$HOSTNAME';"
echo "done"
fi
......@@ -29,4 +32,7 @@ if [ ! -f "{{ upgrade_done }}" ]; then
if [ $? -eq 0 ]; then
touch {{ upgrade_done }};
fi
else
# only create databases now
run_mysql < {{ init_database_sql }} || exit 1;
fi
software/repman/templates/mariadb_init_root.sql.in
View file @ a8f35d7e
......@@ -2,16 +2,18 @@
-- or products like mysql-fabric won't work
SET @@SESSION.SQL_LOG_BIN=0;
# {{ parameter_dict["require-ssl"] }}
{% set ssl = parameter_dict["require-ssl"] -%}
CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ;
DROP DATABASE IF EXISTS test ;
......
software/repman/templates/mariadb_initial_setup.sql.in
View file @ a8f35d7e
SET @@SESSION.SQL_LOG_BIN=0;
USE mysql;
{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
{% if mroonga %}
SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
{% macro database(name, user, password) -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}`;
CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
{% macro database(name, user, password, ssl, charset="") -%}
{% set charset_collate = "" -%}
{% set charset_dict = {'utf8_general_ci': 'utf8',
'utf8_unicode_ci': 'utf8',
'utf8mb4_bin': 'utf8mb4',
'utf8mb4_general_ci': 'utf8mb4',
'utf8mb4_unicode_ci': 'utf8mb4',
'latin1_general_ci': 'latin1',
'latin1_bin': 'latin1',
'ascii_general_ci': 'ascii',
'ascii_bin': 'ascii'} -%}
{% if charset %}
{% set charset_collate = "DEFAULT CHARACTER SET %s" % charset -%}
{% if charset_dict.get(charset) -%}
{% set charset_collate = "DEFAULT CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
{% endif -%}
{% endif -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}` {{ charset_collate }};
{% if user -%}
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}'{% if ssl %} REQUIRE SSL{% endif %};
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
{%- endif %}
{% endmacro -%}
{% for entry in parameter_dict['database-list'] -%}
{{ database(entry['name'], entry.get('user'), entry.get('password')) }}
{{ database(entry['name'], entry.get('user'), entry.get('password'), entry.get('require_ssl', True) ~ '', entry.get('charset')) }}
{% endfor -%}
software/repman/templates/mysqld-need-start.sh.in
View file @ a8f35d7e
......@@ -4,13 +4,11 @@ curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
# Checking if mariadb start is needed
#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
if [ $CODE -eq 200 ]; then
sleep 5
echo "$CODE: Updating mysql configuration..."
# update mysql configuration
{{ update_config }}
......@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then
sleep 5
# check again if the service is still up...
{{ mariadb_controller }} status mariadb
else
echo "CODE: $CODE. No need start!"
fi
software/repman/templates/slave-db-manage.in 0 → 100644
View file @ a8f35d7e
#!/bin/bash
set -e
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
get_token () {
curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
}
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" --protocol=socket "$@"
}
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
DATADIR=$(curl -H "Authorization: Bearer ${TOKEN}" \ {{ secure_url }}/api/clusters/{{ cluster_name }}/topology/master | {{ jq_bin }} -r '.slaposDatadir')
# Only write or delete on master database else, we break replication.
if [ "$DATADIR" = "{{ partition_dir }}" ]; then
cat << EOF > {{ tmp_dir }}/.script.sql
use repman_slave_definition;
CREATE TABLE IF NOT EXISTS \`slave\` (
\`name\` varchar(80) NOT NULL,
\`state\` tinyint(1) DEFAULT NULL,
PRIMARY KEY (\`name\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
UPDATE \`slave\` set \`state\`=false;
{% for name in database_name.split(' ') -%}
{% if name -%}
REPLACE INTO \`slave\` VALUES ('{{ name }}', true);
{% endif -%}
{% endfor -%}
EOF
# Update requested slaves database
run_mysql < {{ tmp_dir }}/.script.sql
rm -f {{ tmp_dir }}/.script.sql
DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false");
RET=$?
if [ ! "$RET" = "0" ]; then
echo "Mysql command failed: $DBNAME"
exit $RET
fi
if [ -z "$DBNAME" ]; then
echo "No database for slave to remove.";
fi
for NAME in $DBNAME; do
if [ ! -z "$NAME" ]; then
echo "Deleting database $NAME..."
run_mysql -e "DROP DATABASE IF EXISTS $NAME";
run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
echo "Done."
fi
done
fi
software/repman/test/test.py
View file @ a8f35d7e
......@@ -62,12 +62,6 @@ class TestRepman(SlapOSInstanceTestCase):
token = resp.json()['token']
headers = {"authorization": "Bearer " + token}
resp = requests.get(
urljoin(self.url, '/api/monitor'),
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
for i in range(20):
resp = requests.get(
......@@ -87,3 +81,11 @@ class TestRepman(SlapOSInstanceTestCase):
self.assertTrue(cluster['isProvision'])
self.assertTrue(cluster['isFailable'])
self.assertFalse(cluster['isDown'])
resp = requests.get(
urljoin(self.url, '/api/clusters/cluster1/status'),
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
self.assertEqual(resp.json(), {"alive": "running"})
software/slapos-testing/buildout.hash.cfg
View file @ a8f35d7e
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum = 597c29546519aabe7259e416d0b92095
md5sum = ba6c572678687081ce111d3805be3cee
software/slapos-testing/instance.cfg
View file @ a8f35d7e
......@@ -35,10 +35,10 @@ repository = ${caucase-repository:location}
[caucase-test-runner]
recipe = slapos.recipe.template:jinja2
template = inline:#!/bin/sh
export HOSTS="$(mktemp)"
trap 'rm "$HOSTS"' EXIT
printf '%s testhost\n%s testhost\n' "$SLAPOS_TEST_IPV4" "$SLAPOS_TEST_IPV6" > "$HOSTS"
export CAUCASE_NETLOC=testhost:8000 LD_PRELOAD=${userhosts:location}/lib/userhosts.so:$LD_PRELOAD
export HOSTS_FILE="$(mktemp)"
trap 'rm "$HOSTS_FILE"' EXIT
printf '%s testhost\n%s testhost\n' "$SLAPOS_TEST_IPV4" "$SLAPOS_TEST_IPV6" > "$HOSTS_FILE"
export CAUCASE_NETLOC=testhost:8000 LD_PRELOAD=${userhosts:location}/lib/libuserhosts.so:$LD_PRELOAD
exec python -m unittest discover -v
rendered = $${caucase:location}/host_setting.sh
......
software/theia/buildout.hash.cfg
View file @ a8f35d7e
......@@ -47,7 +47,7 @@ md5sum = 1a668d6203d42b4d46d56e24c7606cb2
[python-language-server-requirements.txt]
_update_hash_filename_ = python-language-server-requirements.txt
md5sum = 9f478fd1b03b7738f3de549cb899bf54
md5sum = e16f1414f2657fa5ffc949839207f41b
[slapos.css.in]
_update_hash_filename_ = slapos.css.in
......
software/theia/python-language-server-requirements.txt
View file @ a8f35d7e
appdirs==1.4.4
astroid==2.5.2
attrs==20.3.0
astroid==2.6.2
attrs==21.2.0
black==20.8b1
certifi==2020.12.5
cachetools==4.2.4
certifi==2021.5.30
chardet==4.0.0
click==7.1.2
future==0.18.2
idna==2.10
importlib-metadata==3.10.0
isort==5.8.0
iniconfig==1.1.1
isort==5.9.1
jedi==0.18.0
lazy-object-proxy==1.4.3
lazy-object-proxy==1.6.0
mccabe==0.6.1
mypy==0.812
mypy-extensions==0.4.3
mypy==0.930
packaging==21.0
parso==0.8.2
pathspec==0.8.1
pluggy==0.13.1
pydantic==1.8.1
pydantic==1.8.2
pydocstyle==6.0.0
pyflags==0.1
pyflakes==2.1.0
pygls==0.10.2
pylint==2.7.4
pygls==0.11.1
pylint==2.9.3
pyparsing==2.4.7
regex==2021.4.4
requests==2.25.1
rope==0.11.0
six==1.12.0
six==1.16.0
snowballstemmer==1.2.1
toml==0.10.2
tomli==1.2.3
typed-ast==1.4.1
typeguard==2.12.0
typing-extensions==3.7.4.3
urllib3==1.26.4
wrapt==1.11.2
yapf==0.29.0
typeguard==2.12.1
typing-extensions==4.0.1
urllib3==1.26.6
wrapt==1.12.1
yapf==0.31.0
zc.buildout.languageserver==0.6.2
zc.buildout==2.13.4
zc.buildout.languageserver==0.5.0
zipp==3.4.1
\ No newline at end of file
stack/erp5/buildout.hash.cfg
View file @ a8f35d7e
......@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
[template-mariadb]
filename = instance-mariadb.cfg.in
md5sum = c82ea00c4514b72fb97a6fa7ac36ec52
md5sum = f553aa7d6596dcf98e7e61bfb6bd81c7
[template-kumofs]
filename = instance-kumofs.cfg.in
......@@ -46,7 +46,7 @@ md5sum = 1de449e8c0c4a85c5ce2b447785b7654
[template-mariadb-initial-setup]
filename = mariadb_initial_setup.sql.in
md5sum = 1102c3e37a5a2e8aa2d8a2607ab633c8
md5sum = f928b9dc99f7f970caadfe7dd6f95d34
[template-postfix]
filename = instance-postfix.cfg.in
......@@ -70,7 +70,7 @@ md5sum = 7a14019abf48ca100eb94d9add20f5ae
[template]
filename = instance.cfg.in
md5sum = bbef65b4edeb342f08309604ca3717d5
md5sum = 0444d646607f8643e987bb62d1cbbc1d
[template-erp5]
filename = instance-erp5.cfg.in
......@@ -86,11 +86,11 @@ md5sum = bc821f9f9696953b10a03ad7b59a1936
[template-zope]
filename = instance-zope.cfg.in
md5sum = f3121380ab4d31ba5f4984aec74d0a2f
md5sum = 09895deebca3206425fada3caf1d92cf
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = d6166515fda7b09df754672536b131be
md5sum = cbcbe2daf3a51dfab50446a18c0be76e
[template-haproxy-cfg]
filename = haproxy.cfg.in
......
stack/erp5/instance-balancer.cfg.in
View file @ a8f35d7e
......@@ -189,7 +189,7 @@ cert = ${haproxy-conf-ssl:certificate}
ca-cert = ${haproxy-conf-ssl:ca-cert}
crl = ${haproxy-conf-ssl:crl}
{% endif %}
stats-socket = ${directory:run}/haproxy.sock
stats-socket = ${directory:run}/ha.sock
path-routing-list = {{ dumps(slapparameter_dict['path-routing-list']) }}
family-path-routing-dict = {{ dumps(slapparameter_dict['family-path-routing-dict']) }}
pidfile = ${directory:run}/haproxy.pid
......
stack/erp5/instance-mariadb.cfg.in
View file @ a8f35d7e
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]) -%}
{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure', 'with-process-privilege': True}]) -%}
{% set test_database_list = [] %}
{% for database_count in range(slapparameter_dict.get('test-database-amount', 1)) -%}
{% do test_database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%}
......
stack/erp5/instance-zope.cfg.in
View file @ a8f35d7e
......@@ -185,7 +185,7 @@ context = key host_dict hosts-parameter:host-dict
recipe = slapos.recipe.template:jinja2
mode = 755
template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
export HOSTS=${hosts:rendered}
export HOSTS_FILE=${hosts:rendered}
export LD_PRELOAD={{ parameter_dict['userhosts'] }}:$LD_PRELOAD
exec "$@"
rendered = ${directory:bin}/userhosts
......
stack/erp5/instance.cfg.in
View file @ a8f35d7e
......@@ -110,7 +110,7 @@ link-binary = {{ dumps(zope_link_binary) }}
fonts = {{ dumps(zope_fonts) }}
fontconfig-includes = {{ dumps(zope_fontconfig_includes) }}
template-fonts-conf = {{ dumps(template_fonts_conf) }}
userhosts = {{ userhosts_location }}/lib/userhosts.so
userhosts = {{ userhosts_location }}/lib/libuserhosts.so
site-zcml = {{ site_zcml }}
extra-path-list = {{ dumps(extra_path_list) }}
matplotlibrc = {{ matplotlibrc_location }}
......
stack/erp5/mariadb_initial_setup.sql.in
View file @ a8f35d7e
......@@ -4,16 +4,19 @@ USE mysql;
SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
{% macro database(name, user, password) -%}
{% macro database(name, user, password, with_process_privilege) -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}`;
{% if user -%}
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
{% if with_process_privilege %}
GRANT PROCESS ON *.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
GRANT PROCESS ON *.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
{%- endif %}
{%- endif %}
{% endmacro -%}
{% for entry in parameter_dict['database-list'] -%}
{{ database(entry['name'], entry.get('user'), entry.get('password')) }}
{{ database(entry['name'], entry.get('user'), entry.get('password'), entry.get('with-process-privilege')) }}
{% endfor -%}
stack/slapos.cfg
View file @ a8f35d7e
......@@ -132,7 +132,7 @@ eggs =
[versions]
setuptools = 44.1.1
# Use SlapOS patched zc.buildout
zc.buildout = 2.7.1+slapos014
zc.buildout = 2.7.1+slapos015
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
zc.recipe.egg = 2.0.3+slapos003
......@@ -167,7 +167,7 @@ importlib-metadata = 1.7.0:whl
inotify-simple = 1.1.1
itsdangerous = 0.24
lock-file = 2.0
lxml = 4.6.3
lxml = 4.6.5
meld3 = 1.0.2
mock = 3.0.5
more-itertools = 5.0.0
......@@ -189,14 +189,14 @@ setproctitle = 1.1.10
setuptools-dso = 1.7
rubygemsrecipe = 0.4.3
six = 1.12.0
slapos.cookbook = 1.0.220
slapos.cookbook = 1.0.226
slapos.core = 1.7.2
slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.22
slapos.rebootstrap = 4.5
slapos.recipe.build = 0.52
slapos.recipe.cmmi = 0.18
slapos.recipe.build = 0.53
slapos.recipe.cmmi = 0.19
slapos.recipe.template = 4.6
slapos.toolbox = 0.126
stevedore = 1.21.0:whl
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7