Commit 3caed389 authored by Xavier Thompson's avatar Xavier Thompson

Update Release Candidate

parents ba4ea4df cb5c5b83
...@@ -65,9 +65,9 @@ egg = ZEO ...@@ -65,9 +65,9 @@ egg = ZEO
eggs = ${:egg} eggs = ${:egg}
egg-versions = egg-versions =
ZEO = 5.2.2 ZEO = 5.2.3
trollius = 2.2.post1 trollius = 2.2.1
futures = 3.2.0 futures = 3.3.0
# ZEO4-wc2 is ZEO4 version with patches for wendelin.core 2 to work correctly. # ZEO4-wc2 is ZEO4 version with patches for wendelin.core 2 to work correctly.
...@@ -76,5 +76,9 @@ egg-versions = ...@@ -76,5 +76,9 @@ egg-versions =
# See https://github.com/zopefoundation/ZEO/pull/161 for the reference. # See https://github.com/zopefoundation/ZEO/pull/161 for the reference.
[ZEO4-wc2] [ZEO4-wc2]
<= ZEO4 <= ZEO4
revisions = bf80d23d3506 setup = ${ZEO4-wc2-repository:location}
[ZEO4-wc2-repository]
<= ZEO4-repository
revision = bf80d23d3506
location = ${buildout:parts-directory}/ZEO4-wc2 location = ${buildout:parts-directory}/ZEO4-wc2
...@@ -13,8 +13,8 @@ parts = haproxy ...@@ -13,8 +13,8 @@ parts = haproxy
[haproxy] [haproxy]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
shared = true shared = true
url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.22.tar.gz url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.24.tar.gz
md5sum = 4d6d5debca0d1bcf51293fb58914f1a3 md5sum = 493011d81eea0f3700543fc6a80acc03
configure-command = true configure-command = true
# for Linux kernel 2.6.28 and above, we use "linux-glibc" as the TARGET, # for Linux kernel 2.6.28 and above, we use "linux-glibc" as the TARGET,
# otherwise use "generic". # otherwise use "generic".
......
[buildout] [buildout]
extends = extends =
../binutils/buildout.cfg # Build dependencies:
../bison/buildout.cfg
../pkgconfig/buildout.cfg
../gperf/buildout.cfg
../ninja/buildout.cfg
../freetype/buildout.cfg
../cmake/buildout.cfg
../git/buildout.cfg
../pkgconfig/buildout.cfg
../cups/buildout.cfg
../coreutils/buildout.cfg ../coreutils/buildout.cfg
../curl/buildout.cfg
../depot_tools/buildout.cfg ../depot_tools/buildout.cfg
../findutils/buildout.cfg ../git/buildout.cfg
../fontconfig/buildout.cfg ../gperf/buildout.cfg
../gettext/buildout.cfg ../pkgconfig/buildout.cfg
../glib/buildout.cfg # Runtime dependencies:
../gtk-2/buildout.cfg
../libexpat/buildout.cfg
../libffi/buildout.cfg
../libpng/buildout.cfg
../libxml2/buildout.cfg
../mesa/buildout.cfg
../nspr/buildout.cfg ../nspr/buildout.cfg
../nss/buildout.cfg ../nss/buildout.cfg
../pcre/buildout.cfg
../sqlite3/buildout.cfg
../xorg/buildout.cfg
../zlib/buildout.cfg
parts = parts =
chromium headless-chromium-wrapper
[gconf] # The Chromium project recommends using their own `fetch' tool rather
recipe = slapos.recipe.cmmi # than doing a `git clone', but this is a little more flexible and works
url = http://ftp.gnome.org/pub/gnome/sources/GConf/3.2/GConf-3.2.6.tar.xz # fine.
md5sum = 2b16996d0e4b112856ee5c59130e822c #
configure-options = --disable-orbit --disable-static # Setting depth=1000 is a middle ground between cloning no history
environment = # (causes some scripts to break) and cloning the full history (takes a
PATH=${pkgconfig:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:%(PATH)s # really long time).
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${dbus:location}/lib/pkgconfig:${dbus-glib:location}/lib/pkgconfig:$PKG_CONFIG_PATH #
# Note: we should add a `depth' option to slapos.recipe.build:gitclone
[chromedriver] # and then migrate this part to use that recipe at some point.
recipe = hexagonit.recipe.download [chromium-source]
url = https://chromedriver.storage.googleapis.com/2.28/chromedriver_linux64.zip
md5sum = a72088c0a6b018ded2c0fff616da8f65
[chromium-download]
recipe = plone.recipe.command recipe = plone.recipe.command
# This revision is 56.0.2924.122. Because the chromedriver only support some certain
# version. Which version 56 is in the middle of 55-57.
revision = faf03429d9c3dbd483700dd42316b20776cbbd3c
path = ${buildout:parts-directory}/${:_buildout_section_name_}
command = command =
set -e export PATH=$PATH:${git:location}/bin
PATH=${depot_tools:location}:${git:location}/bin:$PATH git clone ${:repository} ${:location} \
[ -d ${:path} ] && rm -r ${:path} --branch ${:version} \
mkdir -p ${:path} --depth 1000
cd ${:path} repository = https://chromium.googlesource.com/chromium/src.git
# Do never use `fetch` with the `--no-history` option unless you find an
# option to fetch directly at the wanted revision. `--no-history` could
# reduce the download size significantly but even if it retrieves enough
# commits at the time you test this section, development continues upstream
# and at some point the next command (gclient) would break.
# ...
# This command only could work in an empty dir.
fetch --nohooks chromium
gclient sync --revision ${:revision} --with_branch_heads
stop-on-error = true stop-on-error = true
[chromium] # We should place the .gclient file in the parent directory of the
# checkout/repository itself.
location = ${:gclient-location}/${:name}
gclient-location = ${buildout:parts-directory}/${:_buildout_section_name_}
# Theoretically the checkout can be put anywhere as long as you specify
# "name" appropriately in the .gclient file, but in practice it seems
# that some automated tools break. It's safest to put it in a directory
# called "src".
name = src
# There's nothing special about version 92.0.4515.107. It just happened
# to be the current Chromium stable version at the time of writing.
version = 92.0.4515.107
[headless-chromium]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
path = ${chromium-download:path}/src path = ${chromium-source:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${:path}/out/headless
# Configuration file for GN, the tool to build the actual compilation
# configuration file.
default-gclient-config =
solutions = [
{
"name": "${chromium-source:name}",
"url": "${chromium-source:repository}",
"managed": False,
"custom_deps": {},
"custom_vars": {},
},
]
# Configuration for a headless build.
build-config-options =
import("//build/args/headless.gn")
is_debug = false
symbol_level = 0
blink_symbol_level = 0
# We need to unbundle the build toolchain in order to set our own
# LDFLAGS.
custom_toolchain = "//build/toolchain/linux/unbundle:default"
host_toolchain = "//build/toolchain/linux/unbundle:default"
current_os = "linux"
current_cpu = "x64"
# Chromium bundles its own LLVM toolchain, so we might as well use it.
llvm-toolchain = ${:path}/third_party/llvm-build/Release+Asserts/bin
configure-command = configure-command =
gclient runhooks # Sync build dependencies---this is a little finnicky.
echo '${:default-gclient-config}' \
> ${chromium-source:gclient-location}/.gclient
gclient sync --no-history
# Generate build configuration files.
mkdir -p ${:location} mkdir -p ${:location}
echo 'use_udev = true echo '${:build-config-options}' > ${:location}/args.gn
is_debug = false
enable_nacl = false' > ${:location}/args.gn
gn gen ${:location} gn gen ${:location}
# Note you can run Chromium manually by: ${:location}/chrome --headless --no-sandbox --disable-gpu # You can run the headless Chromium shell using
make-binary = ninja -C ${:location} chrome # ${:binary} --remote-debugging-port=1234
make-binary =
autoninja -C ${:location} headless_shell
# By building our own version of Chromedriver, we can ensure version
# compatibility. The build is quite cheap compared to Chromium, anyway.
autoninja -C ${:location} chromedriver
environment = environment =
PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${randrproto:location}/lib/pkgconfig:$PKG_CONFIG_PATH PATH=${depot_tools:location}:${gperf:location}/bin:${pkgconfig:location}/bin:${coreutils:location}/bin:${git:location}/bin:${curl:location}/bin:%(PATH)s
PATH=${chromedriver:location}:${dbus:location}/bin:${depot_tools:location}:${pkgconfig:location}/bin:${ninja:location}/bin:${bison:location}/bin:${gperf:location}/bin:${xserver:location}/bin:%(PATH)s LDFLAGS="-Wl,-rpath=${nss:location}/lib,-rpath=${nspr:location}/lib"
CPATH=${dbus:location}/include/dbus-1.0:${dbus:location}/lib/dbus-1.0/include/:${freetype:location}/include/freetype2:${libffi:location}/include:${mpfr:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline:location}/include:${sqlite3:location}/include:${zlib:location}/include:${bzip2:location}/include:$CPATH CC="${:llvm-toolchain}/clang"
LD_LIBRARY_PATH=${alsa:location}/lib:${gconf:location}/lib:${libXScrnSaver:location}/lib:${glib:location}/lib:${atk:location}/lib:${cairo:location}/lib:${cups:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib/:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcomposite:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXi:location}/lib:${libXrender:location}/lib/:${libXtst:location}/lib:${libexpat:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libpng12:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${mesa:location}/lib:${nspr:location}/lib:${nss:location}/lib:${pango:location}/lib:${pcre:location}/lib:${pixman:location}/lib:${sqlite3:location}/lib:${xdamage:location}/lib:${xfixes:location}/lib:${zlib:location}/lib:$LD_LIBRARY_PATH CXX="${:llvm-toolchain}/clang++"
AR="${:llvm-toolchain}/llvm-ar"
NM="${:llvm-toolchain}/llvm-nm"
# Expose devtools frontend location.
devtools-frontend = ${:location}/gen/third_party/devtools-frontend/src/front_end
binary = ${:location}/headless_shell
chromedriver = ${:location}/chromedriver
promises =
${:binary}
${:chromedriver}
# At runtime, Chromium tries to dynamically load the NSS certificate
# database from "libnssckbi.so". But Chromium does this through NSPR,
# which doesn't know where SlapOS installed NSS. Since we don't want to
# modify the NSPR component from the Chromium component, we just set
# LD_LIBRARY_PATH in a wrapper script so that NSPR knows where NSS is.
#
# Alternatively, we could patch crypto/nss_util.cc in the Chromium
# source code to use an absolute path for libnssckbi.so, but this is not
# as future-proof against new versions of Chromium.
[headless-chromium-wrapper]
recipe = slapos.recipe.template:jinja2
template =
inline:#!/bin/sh
export LD_LIBRARY_PATH="{{ nss_location }}/lib:$LD_LIBRARY_PATH"
exec {{ chromium_binary }} "$@"
rendered = ${buildout:bin-directory}/headless-chromium
context =
key nss_location nss:location
key chromium_binary headless-chromium:binary
...@@ -95,63 +95,25 @@ tornado = 4.4.2 ...@@ -95,63 +95,25 @@ tornado = 4.4.2
widgetsnbextension = 2.0.0 widgetsnbextension = 2.0.0
# numpy >= 1.13.1 is required for numpy.core.multiarray # numpy >= 1.13.1 is required for numpy.core.multiarray
numpy = 1.13.1 numpy = 1.13.1
# Required by:
# tornado==4.4.2
certifi = 2020.6.20 certifi = 2020.6.20
# Required by:
# notebook==4.3.2
# nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source. # nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
nbconvert = 4.1.0 nbconvert = 4.1.0
# Required by:
# ipython==5.3.0
pathlib2 = 2.2.1 pathlib2 = 2.2.1
# Required by:
# statsmodels==0.8.0
patsy = 0.4.1 patsy = 0.4.1
# Required by:
# ipython==5.3.0
pexpect = 4.2.1 pexpect = 4.2.1
# Required by:
# ipython==5.3.0
pickleshare = 0.7.4 pickleshare = 0.7.4
# Required by:
# pathlib2==2.2.1
scandir = 1.5 scandir = 1.5
# Required by:
# statsmodels==0.8.0
scipy = 0.19.0 scipy = 0.19.0
# Required by:
# tornado==4.4.2
singledispatch = 3.4.0.3 singledispatch = 3.4.0.3
# Required by:
# prompt-toolkit==1.0.13
wcwidth = 0.1.7 wcwidth = 0.1.7
jupyter = 1.0.0 jupyter = 1.0.0
jupyter-console = 5.1.0 jupyter-console = 5.1.0
# Required by:
# jupyter==1.0.0
qtconsole = 4.3.0 qtconsole = 4.3.0
et-xmlfile = 1.0.1 et-xmlfile = 1.0.1
h5py = 2.7.1 h5py = 2.7.1
mpmath = 1.0.0 mpmath = 1.0.0
openpyxl = 2.5.2 openpyxl = 2.5.2
sympy = 1.1.1 sympy = 1.1.1
xlrd = 1.1.0 xlrd = 1.1.0
# Required by:
# openpyxl==2.5.2
jdcal = 1.4 jdcal = 1.4
[buildout]
parts = libsecret
extends =
../gnutls/buildout.cfg
../libxslt/buildout.cfg
../pkgconfig/buildout.cfg
../glib/buildout.cfg
../gettext/buildout.cfg
../libuuid/buildout.cfg
../xz-utils/buildout.cfg
[libsecret]
recipe = slapos.recipe.cmmi
url = https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz
md5sum = bf92f48afab2891f644f311e0f37683f
shared = true
configure-options =
--with-libgcrypt-prefix=${libgcrypt:location}
--with-libintl-prefix=${gettext:location}
--disable-gtk-doc-html
--disable-manpages
--disable-vala
environment =
PATH=${xz-utils:location}/bin/:${pkgconfig:location}/bin:${libxslt:location}/bin:${glib:location}/bin:${libgcrypt:location}/bin/:%(PATH)s
PKG_CONFIG_PATH=${:pkg-config-path-depends}
LDFLAGS=-Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${libgcrypt:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libuuid:location}/lib
pkg-config-path-depends = ${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libgcrypt:location}/lib/pkgconfig:${libuuid:location}/lib/pkgconfig
pkg-config-path = @@LOCATION@@/lib/pkgconfig:${:pkg-config-path-depends}
...@@ -24,6 +24,7 @@ configure-options= ...@@ -24,6 +24,7 @@ configure-options=
--with-http_v2_module --with-http_v2_module
--with-http_gzip_static_module --with-http_gzip_static_module
--with-http_realip_module --with-http_realip_module
--with-http_sub_module
--with-mail --with-mail
--with-mail_ssl_module --with-mail_ssl_module
--with-ld-opt="-L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib" --with-ld-opt="-L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
...@@ -85,7 +86,7 @@ revision = 3d3a204177d3a7ab8a2858e04e792a6d11bf133f ...@@ -85,7 +86,7 @@ revision = 3d3a204177d3a7ab8a2858e04e792a6d11bf133f
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
[nginx-push-stream] [nginx-push-stream]
<= nginx <= nginx-common
configure-options= configure-options=
--with-ipv6 --with-ipv6
--with-http_ssl_module --with-http_ssl_module
......
...@@ -20,6 +20,6 @@ egg = nxdtest ...@@ -20,6 +20,6 @@ egg = nxdtest
[nxdtest-repository] [nxdtest-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nxdtest.git repository = https://lab.nexedi.com/nexedi/nxdtest.git
revision = b5a74214 revision = 0ad45a9c
location = ${buildout:parts-directory}/nxdtest location = ${buildout:parts-directory}/nxdtest
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
...@@ -15,3 +15,4 @@ eggs = ${pytest:eggs} ...@@ -15,3 +15,4 @@ eggs = ${pytest:eggs}
[versions] [versions]
pytest = 4.6.11:whl pytest = 4.6.11:whl
pytest-timeout = 1.4.2
...@@ -91,11 +91,11 @@ md5sum = 6097fdb9cbab47c96471274b9044e983 ...@@ -91,11 +91,11 @@ md5sum = 6097fdb9cbab47c96471274b9044e983
# XXX: This is not the latest version because # XXX: This is not the latest version because
# Debian does not provide a stable URL for it. # Debian does not provide a stable URL for it.
<= debian-amd64-netinst-base <= debian-amd64-netinst-base
version = 10.8.0 version = 10.10.0
md5sum = e221f43f4fdd409250908fc4305727d4 md5sum = c7d0e562e589e853b5d00563b4311720
[debian-amd64-testing-netinst.iso] [debian-amd64-testing-netinst.iso]
<= debian-amd64-netinst-base <= debian-amd64-netinst-base
release = bullseye_di_alpha3 release = bullseye_di_rc3
version = bullseye-DI-alpha3 version = bullseye-DI-rc3
md5sum = bff147077791586fa7c102267da9f2d2 md5sum = 405917de7062c58357a3673c9901f0c4
...@@ -26,6 +26,9 @@ md5sum = 51fe2bcbff1bbce77a25d180fd247f7d ...@@ -26,6 +26,9 @@ md5sum = 51fe2bcbff1bbce77a25d180fd247f7d
pkg_config_depends = ${leptonica:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${lcms2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig pkg_config_depends = ${leptonica:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${lcms2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
pre-configure = pre-configure =
autoreconf -ivf -I${pkgconfig:location}/share/aclocal -I${libtool:location}/share/aclocal -Wno-portability autoreconf -ivf -I${pkgconfig:location}/share/aclocal -I${libtool:location}/share/aclocal -Wno-portability
# XXX workaround path on slaprunner with a double slash
# https://github.com/tesseract-ocr/tesseract/issues/3527
configure-options = --prefix=$(python -c 'print("""@@LOCATION@@""".replace("//", "/"))')
environment = environment =
PATH=${pkgconfig:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${patch:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${patch:location}/bin:%(PATH)s
......
...@@ -23,12 +23,10 @@ min_version = 8 ...@@ -23,12 +23,10 @@ min_version = 8
[trafficserver] [trafficserver]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://apache.claz.org/trafficserver/trafficserver-9.0.1.tar.bz2 url = http://apache.claz.org/trafficserver/trafficserver-9.0.2.tar.bz2
md5sum = 98bb2de25f332715339ade87530a9f5a md5sum = 4df67ada24665116bafedd71503215cb
shared = true shared = true
patch-options = -p1 patch-options = -p1
patches =
https://patch-diff.githubusercontent.com/raw/apache/trafficserver/pull/7577.patch
configure-options = configure-options =
--with-openssl=${openssl:location} --with-openssl=${openssl:location}
--with-pcre=${pcre:location} --with-pcre=${pcre:location}
......
...@@ -7,6 +7,7 @@ extends = ...@@ -7,6 +7,7 @@ extends =
# appropriate ZODB and versions of other components. # appropriate ZODB and versions of other components.
../pytest/buildout.cfg ../pytest/buildout.cfg
../scipy/buildout.cfg
../gdb/buildout.cfg ../gdb/buildout.cfg
../../stack/nxdtest.cfg ../../stack/nxdtest.cfg
buildout.cfg buildout.cfg
...@@ -15,22 +16,25 @@ parts = ...@@ -15,22 +16,25 @@ parts =
# keep neoppod first and in parts so that ZODB is built correctly # keep neoppod first and in parts so that ZODB is built correctly
neoppod neoppod
wendelin.core
# for instance # for instance
wendelin.core-python wendelin.core-python
slapos-cookbook slapos-cookbook
instance.cfg instance.cfg
# bin/python s python interpreter with wendelin.core and all other eggs. # test-dependent eggs that must come through in-tree recipes.
[wendelin.core]
depends += ${scipy:egg}
# bin/python is python interpreter with wendelin.core and all other eggs.
[wendelin.core-python] [wendelin.core-python]
<= python-interpreter <= python-interpreter
eggs = eggs =
wendelin.core[test] ${wendelin.core:egg}[test]
pygolang[pyx.build] ${pygolang:egg}[pyx.build]
neoppod[tests] neoppod[tests]
ZEO[test] ${ZEO:egg}[test]
# env.sh for that python + go to be on $PATH # env.sh for that python + go to be on $PATH
[wendelin.core-env.sh] [wendelin.core-env.sh]
......
...@@ -10,8 +10,6 @@ extends = ...@@ -10,8 +10,6 @@ extends =
buildout-dev.cfg buildout-dev.cfg
parts = parts =
zodbtools
# for instance # for instance
zodbtools-python zodbtools-python
slapos-cookbook slapos-cookbook
...@@ -21,7 +19,7 @@ parts = ...@@ -21,7 +19,7 @@ parts =
# bin/python is preinstalled with sys.path to zodbtools & friends. # bin/python is preinstalled with sys.path to zodbtools & friends.
[zodbtools-python] [zodbtools-python]
<= python-interpreter <= python-interpreter
eggs = zodbtools[test] eggs = ${zodbtools:egg}[test]
# env.sh for zodbtools's python to be on $PATH. # env.sh for zodbtools's python to be on $PATH.
[zodbtools-env.sh] [zodbtools-env.sh]
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 04015a7a552285984d091293ef573fb9 md5sum = 1dfbd20c77fb3c1f01005a8a920d2ed9
[profile-common] [profile-common]
filename = instance-common.cfg.in filename = instance-common.cfg.in
...@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68 ...@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
[profile-caddy-frontend] [profile-caddy-frontend]
filename = instance-apache-frontend.cfg.in filename = instance-apache-frontend.cfg.in
md5sum = 8507a2ace2f789b92c522cc62ca5aace md5sum = 51087ac7615bd7cc01e60eb23701f625
[profile-caddy-replicate] [profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in filename = instance-apache-replicate.cfg.in
md5sum = 8beb438d06bbb0f917d13e182fb12d17 md5sum = b6fc5a004a1235ffad3af0b4cb0e661f
[profile-slave-list] [profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = 613f777a08373088cbaf7f51fd18ea70 md5sum = 9bb51f663f69d66b5b3708bf892dd3e6
[profile-replicate-publish-slave-information] [profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
...@@ -72,10 +72,6 @@ md5sum = d022455a8610bac2dd51101edb035987 ...@@ -72,10 +72,6 @@ md5sum = d022455a8610bac2dd51101edb035987
_update_hash_filename_ = templates/trafficserver/logging.yaml.jinja2 _update_hash_filename_ = templates/trafficserver/logging.yaml.jinja2
md5sum = 368b271215a92594ca9e2fa3102d484f md5sum = 368b271215a92594ca9e2fa3102d484f
[template-nginx-eventsource-slave-virtualhost]
_update_hash_filename_ = templates/nginx-eventsource-slave.conf.in
md5sum = 217a6c801b8330b0b825f7b8b4c77184
[template-caddy-lazy-script-call] [template-caddy-lazy-script-call]
_update_hash_filename_ = templates/apache-lazy-script-call.sh.in _update_hash_filename_ = templates/apache-lazy-script-call.sh.in
md5sum = 77d60840591de67b64ab3572e46273a0 md5sum = 77d60840591de67b64ab3572e46273a0
......
{%- if instance_parameter_dict['slap-software-type'] == software_type -%}
{% import "caucase" as caucase with context %} {% import "caucase" as caucase with context %}
{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%} {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
[buildout] [buildout]
...@@ -11,6 +10,7 @@ parts = ...@@ -11,6 +10,7 @@ parts =
directory directory
logrotate-entry-caddy logrotate-entry-caddy
caddy-frontend caddy-frontend
caddyprofiledeps
switch-caddy-softwaretype switch-caddy-softwaretype
caucase-updater caucase-updater
caucase-updater-promise caucase-updater-promise
...@@ -97,9 +97,11 @@ expose-csr_id-var = ${:var}/expose-csr_id ...@@ -97,9 +97,11 @@ expose-csr_id-var = ${:var}/expose-csr_id
slave-introspection-var = ${:var}/slave-introspection slave-introspection-var = ${:var}/slave-introspection
[switch-caddy-softwaretype] [switch-caddy-softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:switch-softwaretype
single-default = ${dynamic-custom-personal-profile-slave-list:rendered} default = dynamic-custom-personal-profile-slave-list:rendered
single-custom-personal = ${dynamic-custom-personal-profile-slave-list:rendered} RootSoftwareInstance = ${:default}
single-default = dynamic-custom-personal-profile-slave-list:rendered
single-custom-personal = dynamic-custom-personal-profile-slave-list:rendered
[frontend-configuration] [frontend-configuration]
ip-access-certificate = ${self-signed-ip-access:certificate} ip-access-certificate = ${self-signed-ip-access:certificate}
...@@ -114,7 +116,7 @@ slave-introspection-domain = ${slave-introspection-frontend:connection-domain} ...@@ -114,7 +116,7 @@ slave-introspection-domain = ${slave-introspection-frontend:connection-domain}
# Self Signed certificate for HTTPS IP accesses to the frontend # Self Signed certificate for HTTPS IP accesses to the frontend
recipe = plone.recipe.command recipe = plone.recipe.command
update-command = ${:command} update-command = ${:command}
ipv6 = ${slap-network-information:global-ipv6} ipv6 = ${slap-configuration:ipv6-random}
ipv4 = {{instance_parameter_dict['ipv4-random']}} ipv4 = {{instance_parameter_dict['ipv4-random']}}
certificate = ${caddy-directory:master-autocert-dir}/ip-access-${:ipv6}-${:ipv4}.crt certificate = ${caddy-directory:master-autocert-dir}/ip-access-${:ipv6}-${:ipv4}.crt
{#- Can be stopped on error, as does not rely on self provided service #} {#- Can be stopped on error, as does not rely on self provided service #}
...@@ -138,7 +140,7 @@ command = ...@@ -138,7 +140,7 @@ command =
# Self Signed certificate for HTTPS access to the frontend with fallback certificate # Self Signed certificate for HTTPS access to the frontend with fallback certificate
recipe = plone.recipe.command recipe = plone.recipe.command
update-command = ${:command} update-command = ${:command}
ipv6 = ${slap-network-information:global-ipv6} ipv6 = ${slap-configuration:ipv6-random}
ipv4 = {{instance_parameter_dict['ipv4-random']}} ipv4 = {{instance_parameter_dict['ipv4-random']}}
certificate = ${caddy-directory:master-autocert-dir}/fallback-access.crt certificate = ${caddy-directory:master-autocert-dir}/fallback-access.crt
{#- Can be stopped on error, as does not rely on self provided service #} {#- Can be stopped on error, as does not rely on self provided service #}
...@@ -282,6 +284,7 @@ software_type = single-custom-personal ...@@ -282,6 +284,7 @@ software_type = single-custom-personal
organization = {{ slapparameter_dict['cluster-identification'] }} organization = {{ slapparameter_dict['cluster-identification'] }}
organizational-unit = {{ instance_parameter_dict['configuration.frontend-name'] }} organizational-unit = {{ instance_parameter_dict['configuration.frontend-name'] }}
backend-client-caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }} backend-client-caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }}
partition_ipv6 = ${slap-configuration:ipv6-random}
extra-context = extra-context =
key caddy_configuration_directory caddy-directory:slave-configuration key caddy_configuration_directory caddy-directory:slave-configuration
key backend_client_caucase_url :backend-client-caucase-url key backend_client_caucase_url :backend-client-caucase-url
...@@ -293,7 +296,7 @@ extra-context = ...@@ -293,7 +296,7 @@ extra-context =
key caddy_log_directory caddy-directory:slave-log key caddy_log_directory caddy-directory:slave-log
key expose_csr_id_organization :organization key expose_csr_id_organization :organization
key expose_csr_id_organizational_unit :organizational-unit key expose_csr_id_organizational_unit :organizational-unit
key global_ipv6 slap-network-information:global-ipv6 key global_ipv6 slap-configuration:ipv6-random
key empty_template software-release-path:template-empty key empty_template software-release-path:template-empty
key template_default_slave_configuration software-release-path:template-default-slave-virtualhost key template_default_slave_configuration software-release-path:template-default-slave-virtualhost
key software_type :software_type key software_type :software_type
...@@ -328,8 +331,8 @@ extra-context = ...@@ -328,8 +331,8 @@ extra-context =
section frontend_configuration frontend-configuration section frontend_configuration frontend-configuration
key http_port configuration:plain_http_port key http_port configuration:plain_http_port
key https_port configuration:port key https_port configuration:port
key global_ipv6 slap-configuration:ipv6-random
key local_ipv4 :local_ipv4 key local_ipv4 :local_ipv4
key global_ipv6 slap-network-information:global-ipv6
key error_log caddy-configuration:error-log key error_log caddy-configuration:error-log
key not_found_file caddy-configuration:not-found-file key not_found_file caddy-configuration:not-found-file
key username monitor-instance-parameter:username key username monitor-instance-parameter:username
...@@ -904,7 +907,7 @@ recipe = slapos.cookbook:requestoptional ...@@ -904,7 +907,7 @@ recipe = slapos.cookbook:requestoptional
name = Slave Introspection Frontend {{ instance_parameter_dict['configuration.frontend-name'] }} name = Slave Introspection Frontend {{ instance_parameter_dict['configuration.frontend-name'] }}
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter_dict['configuration.slave-introspection-https-port'] }}/ config-url = https://[${slap-configuration:ipv6-random}]:{{ instance_parameter_dict['configuration.slave-introspection-https-port'] }}/
config-https-only = true config-https-only = true
return = domain secure_access return = domain secure_access
...@@ -914,7 +917,7 @@ recipe = slapos.cookbook:requestoptional ...@@ -914,7 +917,7 @@ recipe = slapos.cookbook:requestoptional
name = Backend Haproxy Statistic Frontend {{ instance_parameter_dict['configuration.frontend-name'] }} name = Backend Haproxy Statistic Frontend {{ instance_parameter_dict['configuration.frontend-name'] }}
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter_dict['configuration.backend-haproxy-statistic-port'] }}/ config-url = https://[${slap-configuration:ipv6-random}]:{{ instance_parameter_dict['configuration.backend-haproxy-statistic-port'] }}/
config-https-only = true config-https-only = true
return = domain secure_access return = domain secure_access
...@@ -1022,5 +1025,3 @@ config-command = ...@@ -1022,5 +1025,3 @@ config-command =
{%- for key, value in software_parameter_dict.iteritems() %} {%- for key, value in software_parameter_dict.iteritems() %}
{{ key }} = {{ dumps(value) }} {{ key }} = {{ dumps(value) }}
{%- endfor %} {%- endfor %}
{%- endif -%} {# if instance_parameter_dict['slap-software-type'] == software_type #}
{% if instance_parameter_dict['slap-software-type'] in software_type %}
{% set aibcc_enabled = True %} {% set aibcc_enabled = True %}
{% import "caucase" as caucase with context %} {% import "caucase" as caucase with context %}
{#- SERVER_POLLUTED_KEY_LIST is a list of keys which comes from various SlapOS Master implementations, which mix request and publish keys on each slave information -#} {#- SERVER_POLLUTED_KEY_LIST is a list of keys which comes from various SlapOS Master implementations, which mix request and publish keys on each slave information -#}
...@@ -44,13 +43,7 @@ context = ...@@ -44,13 +43,7 @@ context =
{% set popen = functools_module.partial(subprocess_module.Popen, stdout=subprocess_module.PIPE, stderr=subprocess_module.STDOUT, stdin=subprocess_module.PIPE) %} {% set popen = functools_module.partial(subprocess_module.Popen, stdout=subprocess_module.PIPE, stderr=subprocess_module.STDOUT, stdin=subprocess_module.PIPE) %}
{% set part_list = [] %} {% set part_list = [] %}
{% set single_type_key = 'single-' %} {% set single_type_key = 'single-' %}
{% if instance_parameter_dict['slap-software-type'] == "replicate" %}
{% set frontend_type = slapparameter_dict.pop('-frontend-type', 'single-default') %}
{% elif instance_parameter_dict['slap-software-type'] in ['default', 'RootSoftwareInstance'] %}
{% set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %} {% set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %}
{% else %}
{% set frontend_type = "%s%s" % (single_type_key, instance_parameter_dict['slap-software-type']) %}
{% endif %}
{% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '1') | int %} {% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '1') | int %}
{% set slave_list_name = 'extra_slave_instance_list' %} {% set slave_list_name = 'extra_slave_instance_list' %}
{% set frontend_list = [] %} {% set frontend_list = [] %}
...@@ -117,9 +110,7 @@ context = ...@@ -117,9 +110,7 @@ context =
{% set slave_warning_list = [] %} {% set slave_warning_list = [] %}
{% set slave_server_alias_unclashed = [] %} {% set slave_server_alias_unclashed = [] %}
{% set slave_type = slave.get('type') %} {% set slave_type = slave.get('type') %}
{% if slave_type == 'eventsource' %} {% if slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
{% do slave_error_list.append('type:eventsource is not implemented') %}
{% elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
{% do slave_error_list.append('type:%s is not supported' % (slave_type,)) %} {% do slave_error_list.append('type:%s is not supported' % (slave_type,)) %}
{% endif %} {% endif %}
{# Check health-check-* #} {# Check health-check-* #}
...@@ -392,12 +383,12 @@ config-url = ...@@ -392,12 +383,12 @@ config-url =
#-- #--
#-- Publish slave information #-- Publish slave information
[publish-slave-information] [publish-slave-information]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:switch-softwaretype
default = ${dynamic-publish-slave-information:rendered} default = dynamic-publish-slave-information:rendered
RootSoftwareInstance = ${dynamic-publish-slave-information:rendered} RootSoftwareInstance = ${:default}
replicate = ${dynamic-publish-slave-information:rendered} replicate = dynamic-publish-slave-information:rendered
custom-personal = ${dynamic-publish-slave-information:rendered} custom-personal = dynamic-publish-slave-information:rendered
custom-group = ${dynamic-publish-slave-information:rendered} custom-group = dynamic-publish-slave-information:rendered
[request-kedifa] [request-kedifa]
<= slap-connection <= slap-connection
...@@ -910,5 +901,3 @@ parts = ...@@ -910,5 +901,3 @@ parts =
{% for part in part_list %} {% for part in part_list %}
{{ ' %s' % part }} {{ ' %s' % part }}
{% endfor %} {% endfor %}
\ No newline at end of file
# publish-information
{% endif %}
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
"$schema": "http://json-schema.org/draft-04/schema", "$schema": "http://json-schema.org/draft-04/schema",
"properties": { "properties": {
"custom_domain": { "custom_domain": {
"description": "Custom Domain to use for the website", "description": "Custom Domain to use for the website. Shall contain only letters, numbers and -, and can look like example.com, first2.example.com special-site.example.com.",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$", "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
"title": "Custom Domain", "title": "Custom Domain",
"type": "string" "type": "string"
...@@ -15,14 +15,13 @@ ...@@ -15,14 +15,13 @@
}, },
"type": { "type": {
"default": "", "default": "",
"description": "Type of slave. If redirect, the slave will redirect to the given URL. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect, notebook and websocket, not implemneted is eventsource.", "description": "Type of slave. If redirect, the slave will redirect to the given URL. If zope, the rewrite rules will be compatible with Virtual Host Monster.",
"enum": [ "enum": [
"", "",
"zope", "zope",
"redirect", "redirect",
"notebook", "notebook",
"websocket", "websocket"
"eventsource"
], ],
"title": "Backend Type", "title": "Backend Type",
"type": "string" "type": "string"
......
...@@ -9,7 +9,7 @@ ...@@ -9,7 +9,7 @@
}, },
"type": { "type": {
"default": "", "default": "",
"description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect, notebook and websocket, not implemneted is eventsource.", "description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster.",
"enum": [ "enum": [
"", "",
"zope" "zope"
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
extends = {{ software_parameter_dict['profile_common'] }} extends = {{ software_parameter_dict['profile_common'] }}
parts = parts =
caddyprofiledeps
switch-softwaretype switch-softwaretype
[caddyprofiledeps] [caddyprofiledeps]
...@@ -14,8 +15,8 @@ extensions = jinja2.ext.do ...@@ -14,8 +15,8 @@ extensions = jinja2.ext.do
extra-context = extra-context =
context = context =
import json_module json import json_module json
key slapparameter_dict instance-parameter:configuration key slapparameter_dict slap-configuration:configuration
section instance_parameter_dict instance-parameter section instance_parameter_dict slap-configuration
section software_parameter_dict software-parameter-section section software_parameter_dict software-parameter-section
${:extra-context} ${:extra-context}
caucase-jinja2-library = {{ software_parameter_dict['caucase_jinja2_library'] }} caucase-jinja2-library = {{ software_parameter_dict['caucase_jinja2_library'] }}
...@@ -23,14 +24,14 @@ import-list = ...@@ -23,14 +24,14 @@ import-list =
file caucase :caucase-jinja2-library file caucase :caucase-jinja2-library
[switch-softwaretype] [switch-softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:switch-softwaretype
default = ${dynamic-profile-caddy-replicate:rendered} default = dynamic-profile-caddy-replicate:rendered
RootSoftwareInstance = ${dynamic-profile-caddy-replicate:rendered} RootSoftwareInstance = ${:default}
custom-personal = ${dynamic-profile-caddy-replicate:rendered} custom-personal = dynamic-profile-caddy-replicate:rendered
single-default = ${dynamic-profile-caddy-frontend:rendered} single-default = dynamic-profile-caddy-frontend:rendered
single-custom-personal = ${dynamic-profile-caddy-frontend:rendered} single-custom-personal = dynamic-profile-caddy-frontend:rendered
replicate = ${dynamic-profile-caddy-replicate:rendered} replicate = dynamic-profile-caddy-replicate:rendered
kedifa = ${dynamic-profile-kedifa:rendered} kedifa = dynamic-profile-kedifa:rendered
[software-parameter-section] [software-parameter-section]
{% for key,value in software_parameter_dict.iteritems() %} {% for key,value in software_parameter_dict.iteritems() %}
...@@ -64,7 +65,7 @@ filename = instance-kedifa.cfg ...@@ -64,7 +65,7 @@ filename = instance-kedifa.cfg
extra-context = extra-context =
raw software_type kedifa raw software_type kedifa
[instance-parameter] [slap-configuration]
# Fetches parameters defined in SlapOS Master for this instance. # Fetches parameters defined in SlapOS Master for this instance.
# Always the same. # Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised recipe = slapos.cookbook:slapconfiguration.serialised
......
{%- if software_type == slap_software_type %}
{%- set kedifa_updater_mapping = [] %} {%- set kedifa_updater_mapping = [] %}
{%- set cached_server_dict = {} %} {%- set cached_server_dict = {} %}
{%- set backend_slave_list = [] %} {%- set backend_slave_list = [] %}
...@@ -187,7 +186,7 @@ context = ...@@ -187,7 +186,7 @@ context =
{%- set furled = furl_module.furl(frontend_configuration['slave-introspection-secure_access']) %} {%- set furled = furl_module.furl(frontend_configuration['slave-introspection-secure_access']) %}
{%- do furled.set(username = slave_reference.lower()) %} {%- do furled.set(username = slave_reference.lower()) %}
{%- do furled.set(password = '${'+ slave_password_section +':passwd}') %} {%- do furled.set(password = '${'+ slave_password_section +':passwd}') %}
{%- do furled.set(path = slave_reference.lower() + '/') %} {%- do furled.set(path = slave_reference + '/') %}
{#- We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #} {#- We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #}
{%- set slave_log_access_url = urlparse_module.unquote(furled.tostr()) %} {%- set slave_log_access_url = urlparse_module.unquote(furled.tostr()) %}
{%- do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %} {%- do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %}
...@@ -237,7 +236,7 @@ context = ...@@ -237,7 +236,7 @@ context =
{#- Set slave logrotate entry #} {#- Set slave logrotate entry #}
[{{slave_log_directory_section}}] [{{slave_log_directory_section}}]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
log-directory = {{ '${slave-log-directory-dict:' + slave_reference.lower() + '}' }} log-directory = {{ '${slave-log-directory-dict:' + slave_reference + '}' }}
[{{slave_logrotate_section}}] [{{slave_logrotate_section}}]
<= logrotate-entry-base <= logrotate-entry-base
...@@ -399,8 +398,8 @@ recipe = slapos.cookbook:publish ...@@ -399,8 +398,8 @@ recipe = slapos.cookbook:publish
{#- Define IPv6 to IPV4 tunneling #} {#- Define IPv6 to IPV4 tunneling #}
[tunnel-6to4-base] [tunnel-6to4-base]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
ipv4 = ${slap-network-information:local-ipv4} ipv4 = ${slap-configuration:ipv4-random}
ipv6 = ${slap-network-information:global-ipv6} ipv6 = ${slap-configuration:ipv6-random}
wrapper-path = {{ directory['service'] }}/6tunnel-${:ipv6-port} wrapper-path = {{ directory['service'] }}/6tunnel-${:ipv6-port}
command-line = {{ software_parameter_dict['sixtunnel'] }}/bin/6tunnel -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port} command-line = {{ software_parameter_dict['sixtunnel'] }}/bin/6tunnel -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
...@@ -509,7 +508,7 @@ extra-context = ...@@ -509,7 +508,7 @@ extra-context =
{{ key }} = {{ value }} {{ key }} = {{ value }}
{%- endfor %} {%- endfor %}
local-ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }} local-ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }}
global-ipv6 = ${slap-network-information:global-ipv6} global-ipv6 = ${slap-configuration:ipv6-random}
request-timeout = {{ dumps('' ~ configuration['request-timeout']) }} request-timeout = {{ dumps('' ~ configuration['request-timeout']) }}
backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }} backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }}
backend-connect-retries = {{ dumps('' ~ configuration['backend-connect-retries']) }} backend-connect-retries = {{ dumps('' ~ configuration['backend-connect-retries']) }}
...@@ -583,12 +582,12 @@ update-command = ${:command} ...@@ -583,12 +582,12 @@ update-command = ${:command}
command = command =
if ! [ -f ${:key} ] && ! [ -f ${:certificate} ] ; then if ! [ -f ${:key} ] && ! [ -f ${:certificate} ] ; then
openssl req -new -newkey rsa:2048 -sha256 -subj \ openssl req -new -newkey rsa:2048 -sha256 -subj \
"/O={{ expose_csr_id_organization }}/OU={{ expose_csr_id_organizational_unit }}/CN=${slap-network-information:global-ipv6}" \ "/O={{ expose_csr_id_organization }}/OU={{ expose_csr_id_organizational_unit }}/CN=${slap-configuration:ipv6-random}" \
-days 5 -nodes -x509 -keyout ${:key} -out ${:certificate} -days 5 -nodes -x509 -keyout ${:key} -out ${:certificate}
fi fi
[expose-csr_id-configuration] [expose-csr_id-configuration]
ip = ${slap-network-information:global-ipv6} ip = ${slap-configuration:ipv6-random}
port = 17001 port = 17001
key = ${certificate-csr_id:key} key = ${certificate-csr_id:key}
certificate = ${certificate-csr_id:certificate} certificate = ${certificate-csr_id:certificate}
...@@ -660,4 +659,3 @@ module = check_command_execute ...@@ -660,4 +659,3 @@ module = check_command_execute
name = ${:_buildout_section_name_}.py name = ${:_buildout_section_name_}.py
config-command = config-command =
${logrotate:wrapper-path} -d ${logrotate:wrapper-path} -d
\ No newline at end of file
{%- endif %} {# if software_type == slap_software_type #}
{% set url = slave_parameter.get('url') %}
{% set https_url = slave_parameter.get('https-url', url) %}
{% if url.startswith("http://") or url.startswith("https://") %}
{% set upstream = url.split("/")[2] %}
{% set https_upstream = https_url.split("/")[2] %}
{% set protocol = url.split("/")[0] %}
{% set https_protocol = https_url.split("/")[0] %}
{% set proxy_pass = '%s//%s' % (protocol, slave_parameter.get('slave_reference')) %}
{% set https_proxy_pass = '%s//https_%s' % (protocol, slave_parameter.get('slave_reference')) %}
# TODO-Caddy upstream {{ slave_parameter.get('slave_reference') }} {
# TODO-Caddy server {{ upstream }};
# TODO-Caddy
# TODO-Caddy pstream https_{{ slave_parameter.get('slave_reference') }} {
# TODO-Caddy server {{ https_upstream }};
# TODO-Caddy
# TODO-Caddy server {
# TODO-Caddy listen {{ slave_parameter['local_ipv4'] }}:{{ slave_parameter['nginx_http_port'] }};
# TODO-Caddy
# TODO-Caddy server_name {{ slave_parameter.get('custom_domain') }};
# TODO-Caddy
# TODO-Caddy error_log {{ slave_parameter.get('error_log') }} error;
# TODO-Caddy access_log {{ slave_parameter.get('access_log') }} custom;
# TODO-Caddy
# TODO-Caddy location /pub {
# TODO-Caddy push_stream_publisher;
# TODO-Caddy push_stream_channels_path $arg_id;
# TODO-Caddy # store messages in memory
# TODO-Caddy push_stream_store_messages off;
# TODO-Caddy
# TODO-Caddy # Message size limit
# TODO-Caddy # client_max_body_size MUST be equal to client_body_buffer_size or
# TODO-Caddy # you will be sorry.
# TODO-Caddy client_max_body_size 16k;
# TODO-Caddy client_body_buffer_size 16k;
# TODO-Caddy
# TODO-Caddy }
# TODO-Caddy
# TODO-Caddy location ~ /sub/(.*) {
# TODO-Caddy # activate subscriber mode for this location
# TODO-Caddy add_header "Access-Control-Allow-Origin" "*";
# TODO-Caddy add_header 'Access-Control-Allow-Credentials' 'false';
# TODO-Caddy add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
# TODO-Caddy add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
# TODO-Caddy
# TODO-Caddy push_stream_subscriber eventsource;
# TODO-Caddy # positional channel path
# TODO-Caddy push_stream_channels_path $1;
# TODO-Caddy
# TODO-Caddy # content-type
# TODO-Caddy default_type "text/event-stream; charset=utf-8";
# TODO-Caddy }
# TODO-Caddy
# TODO-Caddy
# TODO-Caddy server {
# TODO-Caddy listen {{ slave_parameter['local_ipv4'] }}:{{ slave_parameter['nginx_https_port'] }} ssl;
# TODO-Caddy
# TODO-Caddy server_name {{ slave_parameter.get('custom_domain') }};
# TODO-Caddy
# TODO-Caddy error_log {{ slave_parameter.get('error_log') }} error;
# TODO-Caddy access_log {{ slave_parameter.get('access_log') }} custom;
# TODO-Caddy
# TODO-Caddy ssl on;
# TODO-Caddy
# TODO-Caddy ssl_session_timeout 5m;
# TODO-Caddy ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
# TODO-Caddy ssl_ciphers 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5';
# TODO-Caddy ssl_prefer_server_ciphers on;
# TODO-Caddy ssl_session_cache shared:SSL:10m;
# TODO-Caddy location /pub {
# TODO-Caddy push_stream_publisher;
# TODO-Caddy push_stream_channels_path $arg_id;
# TODO-Caddy # store messages in memory
# TODO-Caddy push_stream_store_messages off;
# TODO-Caddy
# TODO-Caddy # Message size limit
# TODO-Caddy # client_max_body_size MUST be equal to client_body_buffer_size or
# TODO-Caddy # you will be sorry.
# TODO-Caddy client_max_body_size 16k;
# TODO-Caddy client_body_buffer_size 16k;
# TODO-Caddy
# TODO-Caddy }
# TODO-Caddy
# TODO-Caddy location ~ /sub/(.*) {
# TODO-Caddy # activate subscriber mode for this location
# TODO-Caddy add_header "Access-Control-Allow-Origin" "*";
# TODO-Caddy add_header 'Access-Control-Allow-Credentials' 'false';
# TODO-Caddy add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
# TODO-Caddy add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
# TODO-Caddy
# TODO-Caddy push_stream_subscriber eventsource;
# TODO-Caddy # positional channel path
# TODO-Caddy push_stream_channels_path $1;
# TODO-Caddy
# TODO-Caddy # content-type
# TODO-Caddy default_type "text/event-stream; charset=utf-8";
# TODO-Caddy }
# TODO-Caddy}
{% endif %}
...@@ -1479,10 +1479,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -1479,10 +1479,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
'websocket-path-list': '////ws//// /with%20space/', 'websocket-path-list': '////ws//// /with%20space/',
'websocket-transparent': 'false', 'websocket-transparent': 'false',
}, },
# 'type-eventsource': {
# 'url': cls.backend_url,
# 'type': 'eventsource',
# },
'type-redirect': { 'type-redirect': {
'url': cls.backend_url, 'url': cls.backend_url,
'type': 'redirect', 'type': 'redirect',
...@@ -1817,7 +1813,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -1817,7 +1813,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
def test_server_polluted_keys_removed(self): def test_server_polluted_keys_removed(self):
buildout_file = os.path.join( buildout_file = os.path.join(
self.getMasterPartitionPath(), 'buildout-switch-softwaretype.cfg') self.getMasterPartitionPath(), 'instance-caddy-replicate.cfg')
for line in [ for line in [
q for q in open(buildout_file).readlines() q for q in open(buildout_file).readlines()
if q.startswith('config-slave-list') or q.startswith( if q.startswith('config-slave-list') or q.startswith(
...@@ -3261,54 +3257,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -3261,54 +3257,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
) )
self.assertFalse('x-real-ip' in j['Incoming Headers']) self.assertFalse('x-real-ip' in j['Incoming Headers'])
@skip('Feature postponed')
def test_type_eventsource(self):
# Caddy: For event source, if I understand
# https://github.com/mholt/caddy/issues/1355 correctly, we could use
# Caddy as a proxy in front of nginx-push-stream . If we have a
# "central shared" caddy instance, can it handle keeping connections
# opens for many clients ?
parameter_dict = self.parseSlaveParameterDict('type-eventsource')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'typeeventsource.nginx.example.com',
'replication_number': '1',
'url': 'http://typeeventsource.nginx.example.com',
'site_url': 'http://typeeventsource.nginx.example.com',
'secure_access': 'https://typeeventsource.nginx.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], 'pub',
# NGINX_HTTPS_PORT
)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
'',
result.content
)
headers = result.headers.copy()
self.assertKeyWithPop('Expires', headers)
self.assertKeyWithPop('Date', headers)
self.assertEqual(
{
'X-Nginx-PushStream-Explain': 'No channel id provided.',
'Content-Length': '0',
'Cache-Control': 'no-cache, no-store, must-revalidate',
'Connection': 'keep-alive',
'Server': 'nginx'
},
headers
)
def test_type_redirect(self): def test_type_redirect(self):
parameter_dict = self.assertSlaveBase('type-redirect') parameter_dict = self.assertSlaveBase('type-redirect')
...@@ -3629,7 +3577,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -3629,7 +3577,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotEqual(via, None) self.assertNotEqual(via, None)
self.assertRegexpMatches( self.assertRegexpMatches(
via, via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$' r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
) )
def test_enable_cache_server_alias(self): def test_enable_cache_server_alias(self):
...@@ -3671,7 +3619,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -3671,7 +3619,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotEqual(via, None) self.assertNotEqual(via, None)
self.assertRegexpMatches( self.assertRegexpMatches(
via, via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$' r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
) )
result = fakeHTTPResult( result = fakeHTTPResult(
...@@ -3788,7 +3736,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -3788,7 +3736,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotEqual(via, None) self.assertNotEqual(via, None)
self.assertRegexpMatches( self.assertRegexpMatches(
via, via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$' r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
) )
# BEGIN: Check that squid.log is correctly filled in # BEGIN: Check that squid.log is correctly filled in
...@@ -3990,7 +3938,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -3990,7 +3938,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotEqual(via, None) self.assertNotEqual(via, None)
self.assertRegexpMatches( self.assertRegexpMatches(
via, via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$' r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
) )
# check stale-if-error support is really respected if not present in the # check stale-if-error support is really respected if not present in the
...@@ -4133,7 +4081,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -4133,7 +4081,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotEqual(via, None) self.assertNotEqual(via, None)
self.assertRegexpMatches( self.assertRegexpMatches(
via, via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$' r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
) )
try: try:
...@@ -4180,7 +4128,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -4180,7 +4128,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotEqual(via, None) self.assertNotEqual(via, None)
self.assertRegexpMatches( self.assertRegexpMatches(
via, via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$' r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
) )
def test_enable_http2_false(self): def test_enable_http2_false(self):
...@@ -4642,7 +4590,7 @@ class TestReplicateSlaveOtherDestroyed(SlaveHttpFrontendTestCase): ...@@ -4642,7 +4590,7 @@ class TestReplicateSlaveOtherDestroyed(SlaveHttpFrontendTestCase):
self.slap.waitForInstance(self.instance_max_retry) self.slap.waitForInstance(self.instance_max_retry)
buildout_file = os.path.join( buildout_file = os.path.join(
self.getMasterPartitionPath(), 'buildout-switch-softwaretype.cfg') self.getMasterPartitionPath(), 'instance-caddy-replicate.cfg')
with open(buildout_file) as fh: with open(buildout_file) as fh:
buildout_file_content = fh.read() buildout_file_content = fh.read()
node_1_present = re.search( node_1_present = re.search(
......
...@@ -100,10 +100,5 @@ url = ${:_profile_base_location_}/${:filename} ...@@ -100,10 +100,5 @@ url = ${:_profile_base_location_}/${:filename}
mode = 640 mode = 640
[versions] [versions]
# Required by:
# cloudooo==1.2.6.dev0
argparse = 1.4.0 argparse = 1.4.0
# Required by:
# cloudooo==1.2.6.dev0
pyPdf = 1.13 pyPdf = 1.13
...@@ -14,8 +14,8 @@ ...@@ -14,8 +14,8 @@
# not need these here). # not need these here).
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 307663d73ef3ef94b02567ecd322252e md5sum = d9c7c3acb975d44c6c66ca32c550ac58
[template-default] [template-default]
filename = instance-default.cfg filename = instance-default.cfg.jinja.in
md5sum = 24cc143b1886d443a4c29dcb8147a01c md5sum = ffb6b74b55e7ca01666254353ae1cebe
[buildout] {%- set slapparameter_dict = dict(default_parameter_dict, **slapparameter_dict) -%}
[buildout]
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
...@@ -18,6 +20,7 @@ parts = ...@@ -18,6 +20,7 @@ parts =
resiliency-exclude-file resiliency-exclude-file
shellinabox-frontend-reload shellinabox-frontend-reload
promises promises
testnode-compatibility
[monitor-publish] [monitor-publish]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
...@@ -35,20 +38,24 @@ slapos-directory = $${directory:slapos} ...@@ -35,20 +38,24 @@ slapos-directory = $${directory:slapos}
working-directory = $${directory:testnode} working-directory = $${directory:testnode}
test-suite-directory = $${directory:test-suite} test-suite-directory = $${directory:test-suite}
shared-part-list = $${directory:shared} shared-part-list = $${directory:shared}
proxy-host = $${slap-network-information:local-ipv4} proxy-host = {{ partition_ipv4 }}
proxy-port = 5000 proxy-port = 5000
log-directory = $${directory:log} log-directory = $${directory:log}
srv-directory = $${rootdirectory:srv} srv-directory = $${rootdirectory:srv}
software-directory = $${directory:software} software-directory = $${directory:software}
run-directory = $${directory:run} run-directory = $${directory:run}
test-node-title = $${slap-parameter:test-node-title} test-node-title = {{ slapparameter_dict['test-node-title'] }}
node-quantity = $${slap-parameter:node-quantity} node-quantity = {{ slapparameter_dict['node-quantity'] }}
ipv4-address = $${slap-network-information:local-ipv4} ipv4-address = {{ partition_ipv4 }}
ipv6-address = $${slap-network-information:global-ipv6} ipv6-address = {{ partition_ipv6 }}
test-suite-master-url = $${slap-parameter:test-suite-master-url} test-suite-master-url = {{ slapparameter_dict['test-suite-master-url'] }}
instance-dict = $${slap-parameter:instance-dict} instance-dict = {{ slapparameter_dict['instance-dict'] }}
software-path-list = $${slap-parameter:software-path-list} {%- if isinstance(slapparameter_dict['software-path-list'], str) %}
keep-log-days = $${slap-parameter:keep-log-days} software-path-list = {{ slapparameter_dict['software-path-list'] }}
{%- else %}
software-path-list = {{ json.dumps(slapparameter_dict['software-path-list']) }}
{%- endif %}
keep-log-days = {{ slapparameter_dict['keep-log-days'] }}
git-binary = ${git:location}/bin/git git-binary = ${git:location}/bin/git
slapos-binary = ${buildout:bin-directory}/slapos slapos-binary = ${buildout:bin-directory}/slapos
testnode = ${buildout:bin-directory}/testnode testnode = ${buildout:bin-directory}/testnode
...@@ -59,7 +66,7 @@ httpd-conf-file = $${rootdirectory:etc}/httpd.conf ...@@ -59,7 +66,7 @@ httpd-conf-file = $${rootdirectory:etc}/httpd.conf
httpd-wrapper = $${rootdirectory:bin}/httpd httpd-wrapper = $${rootdirectory:bin}/httpd
httpd-port = 9080 httpd-port = 9080
httpd-software-access-port = 9081 httpd-software-access-port = 9081
httpd-ip = $${slap-network-information:global-ipv6} httpd-ip = {{ partition_ipv6 }}
httpd-log-directory = $${basedirectory:log} httpd-log-directory = $${basedirectory:log}
httpd-software-directory = $${directory:software} httpd-software-directory = $${directory:software}
httpd-cert-file = $${rootdirectory:etc}/httpd-public.crt httpd-cert-file = $${rootdirectory:etc}/httpd-public.crt
...@@ -117,7 +124,7 @@ template = inline: ...@@ -117,7 +124,7 @@ template = inline:
/ /
} }
} }
ipv6 = $${slap-network-information:global-ipv6} ipv6 = {{ partition_ipv6 }}
hostname = [$${:ipv6}] hostname = [$${:ipv6}]
port = 8080 port = 8080
username = testnode username = testnode
...@@ -200,7 +207,7 @@ run = $${rootdirectory:var}/run ...@@ -200,7 +207,7 @@ run = $${rootdirectory:var}/run
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
slapos = $${rootdirectory:srv}/slapos slapos = $${rootdirectory:srv}/slapos
testnode = $${rootdirectory:srv}/testnode testnode = $${buildout:directory}/t
shared = $${rootdirectory:srv}/shared shared = $${rootdirectory:srv}/shared
test-suite = $${rootdirectory:srv}/test_suite test-suite = $${rootdirectory:srv}/test_suite
log = $${basedirectory:log}/testnode log = $${basedirectory:log}/testnode
...@@ -209,6 +216,16 @@ software = $${rootdirectory:srv}/software ...@@ -209,6 +216,16 @@ software = $${rootdirectory:srv}/software
shellinabox = $${rootdirectory:srv}/shellinabox shellinabox = $${rootdirectory:srv}/shellinabox
ca-dir = $${rootdirectory:srv}/ca ca-dir = $${rootdirectory:srv}/ca
[testnode-compatibility]
# Remove old ~/srv/testnode
recipe = slapos.recipe.build
update =
import os
from zc.buildout.rmtree import rmtree
old_testnode_path = self.buildout['rootdirectory']['srv'] + '/testnode'
if os.path.exists(old_testnode_path):
rmtree(old_testnode_path)
[resiliency-exclude-file] [resiliency-exclude-file]
# Generate rdiff exclude file in case of resiliency # Generate rdiff exclude file in case of resiliency
recipe = collective.recipe.template recipe = collective.recipe.template
...@@ -239,6 +256,7 @@ config-url = https://[$${testnode:httpd-ip}]:$${testnode:httpd-port} ...@@ -239,6 +256,7 @@ config-url = https://[$${testnode:httpd-ip}]:$${testnode:httpd-port}
recipe = recipe =
instance-promises = instance-promises =
$${shellinabox-frontend-listen-promise:name} $${shellinabox-frontend-listen-promise:name}
# $${shellinabox-frontend-available-promise:name}
$${testnode-log-frontend-promise:name} $${testnode-log-frontend-promise:name}
[shellinabox-frontend-listen-promise] [shellinabox-frontend-listen-promise]
...@@ -248,15 +266,18 @@ name = $${:_buildout_section_name_}.py ...@@ -248,15 +266,18 @@ name = $${:_buildout_section_name_}.py
config-host = $${shellinabox-frontend:hostname} config-host = $${shellinabox-frontend:hostname}
config-port = $${shellinabox-frontend:port} config-port = $${shellinabox-frontend:port}
## This promise fails in a test suite
## due to ports conflict with the testnode of the test suite
# [shellinabox-frontend-available-promise]
# <= monitor-promise-base
# module = check_url_available
# name = $${:_buildout_section_name_}.py
# config-url = https://$${shellinabox-frontend-config:hostname}:$${shellinabox-frontend-config:port}
# config-username = $${shellinabox-frontend-config:username}
# config-password = $${shellinabox-frontend-config:passwd}
[testnode-log-frontend-promise] [testnode-log-frontend-promise]
<= monitor-promise-base <= monitor-promise-base
module = check_url_available module = check_url_available
name = $${:_buildout_section_name_}.py name = $${:_buildout_section_name_}.py
config-url = $${testnode-log-frontend:connection-secure_access} config-url = $${testnode-log-frontend:connection-secure_access}
[slap-parameter]
node-quantity = 1
test-suite-master-url =
instance-dict =
software-path-list = ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.181/software/seleniumrunner/software.cfg"]
keep-log-days = 15
...@@ -6,6 +6,34 @@ eggs-directory = ${buildout:eggs-directory} ...@@ -6,6 +6,34 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[switch_softwaretype] [switch_softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:switch-softwaretype
default = ${template-default:output} RootSoftwareInstance = $${:default}
default = instance-default:rendered
[instance-default]
recipe = slapos.recipe.template:jinja2
template = ${template-default:output}
rendered = $${buildout:directory}/instance-default.cfg
context =
import json json
jsonkey default_parameter_dict :default-parameters
key slapparameter_dict slap-configuration:configuration
key partition_ipv6 slap-configuration:ipv6-random
key partition_ipv4 slap-configuration:ipv4-random
default-parameters =
{
"node-quantity": 1,
"test-suite-master-url": "",
"instance-dict": "",
"software-path-list": ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.181/software/seleniumrunner/software.cfg"],
"keep-log-days": 15
}
...@@ -25,20 +25,20 @@ parts = ...@@ -25,20 +25,20 @@ parts =
[eggs] [eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = eggs =
erp5.util[testnode] erp5.util[testnode,benchmark,scalability_tester]
${lxml-python:egg} ${lxml-python:egg}
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
mode = 0644 mode = 0644
[template-default] [template-default]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-default.cfg url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template-default.cfg output = ${buildout:directory}/template-default.cfg.jinja
mode = 0644 mode = 0644
[versions] [versions]
...@@ -52,23 +52,9 @@ soupsieve = 1.8 ...@@ -52,23 +52,9 @@ soupsieve = 1.8
waitress = 1.2.1 waitress = 1.2.1
z3c.etestbrowser = 2.0.1 z3c.etestbrowser = 2.0.1
zope.testbrowser = 5.3.2 zope.testbrowser = 5.3.2
# Required by:
# zope.testbrowser==5.3.2
WSGIProxy2 = 0.4.6 WSGIProxy2 = 0.4.6
# Required by:
# WebTest==2.0.33
beautifulsoup4 = 4.7.1 beautifulsoup4 = 4.7.1
# Required by:
# zope.testbrowser==5.3.2
zope.cachedescriptors = 4.3.1 zope.cachedescriptors = 4.3.1
# Required by:
# zope.schema==4.9.3
zope.event = 4.4 zope.event = 4.4
# Required by:
# zope.testbrowser==5.3.2
zope.schema = 4.9.3 zope.schema = 4.9.3
slapos.tool.nosqltester = 0.0.4.dev-r45972
Tests for erp5testnode software release
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.erp5testnode'
with open("README.md") as f:
long_description = f.read()
setup(
name=name,
version=version,
description="Test for SlapOS' erp5testnode",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'requests',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestnodeTest(SlapOSInstanceTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {"test-node-title": "hello"}
def test(self):
connexion_parameters = self.computer_partition.getConnectionParameterDict()
self.assertIn('url', connexion_parameters)
self.assertIn('frontend-url', connexion_parameters)
self.assertIn('log-frontend-url', connexion_parameters)
...@@ -23,4 +23,4 @@ context = ...@@ -23,4 +23,4 @@ context =
[fluentd] [fluentd]
gems += gems +=
fluent-plugin-wendelin==0.4 fluent-plugin-wendelin==0.4
fluent-plugin-bin fluent-plugin-bin==0.2
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[instance.cfg] [instance.cfg]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = dc3f318e8a3aa7a59f9394118543e9e3 md5sum = 47e8092404feaf3f28ee6469523689ac
[watcher] [watcher]
_update_hash_filename_ = watcher.in _update_hash_filename_ = watcher.in
...@@ -54,7 +54,7 @@ md5sum = 0f1ec4077dab586cc003ae13f689eda2 ...@@ -54,7 +54,7 @@ md5sum = 0f1ec4077dab586cc003ae13f689eda2
[instance-gitlab.cfg.in] [instance-gitlab.cfg.in]
_update_hash_filename_ = instance-gitlab.cfg.in _update_hash_filename_ = instance-gitlab.cfg.in
md5sum = 64ec65b2daa0648453022f3afcbc4da3 md5sum = 6b34d4b96ae0067977fa509046d71231
[instance-gitlab-export.cfg.in] [instance-gitlab-export.cfg.in]
_update_hash_filename_ = instance-gitlab-export.cfg.in _update_hash_filename_ = instance-gitlab-export.cfg.in
......
...@@ -5,7 +5,6 @@ ...@@ -5,7 +5,6 @@
# throughput compared to tcp over loopback). # throughput compared to tcp over loopback).
[buildout] [buildout]
extends = extends =
{{ gitlab_parameters_cfg }}
{{ monitor_template }} {{ monitor_template }}
parts = parts =
directory directory
...@@ -50,26 +49,15 @@ offline = true ...@@ -50,26 +49,15 @@ offline = true
################################## ##################################
[instance-parameter] [instance-parameter]
# std stuff to fetch slapos instance parameters {#- There are dangerous keys like recipe, etc #}
recipe = slapos.cookbook:slapconfiguration {#- XXX: Some other approach would be useful #}
computer= ${slap-connection:computer-id} {%- set DROP_KEY_LIST = ['recipe', '__buildout_signature__', 'computer', 'partition', 'url', 'key', 'cert'] %}
partition=${slap-connection:partition-id} {%- for key, value in instance_parameter_dict.iteritems() -%}
url = ${slap-connection:server-url} {%- if key not in DROP_KEY_LIST %}
key = ${slap-connection:key-file} {{ key }} = {{ value }}
cert = ${slap-connection:cert-file} {%- endif -%}
{%- endfor %}
# autogenerated gitlab instance parameters
<= gitlab-parameters
# adjust/override some default settings:
# automatically load all available CPUs
configuration.unicorn_worker_processes = {{ multiprocessing.cpu_count() + 1 }}
configuration.nginx_worker_processes = {{ multiprocessing.cpu_count() }}
# gitlab non-native parameters
configuration.icp_license =
# for convenience # for convenience
......
# GitLab "switch-softwaretype" instance # GitLab "switch-softwaretype" instance
[buildout] [buildout]
parts = switch-softwaretype extends =
${gitlab-parameters.cfg:target}
parts =
switch-softwaretype
# std stuff for slapos instance # std stuff for slapos instance
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
...@@ -9,14 +12,46 @@ offline = true ...@@ -9,14 +12,46 @@ offline = true
[switch-softwaretype] [switch-softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:switch-softwaretype
gitlab = $${instance-gitlab.cfg:rendered} gitlab = instance-gitlab.cfg:rendered
gitlab-export = $${instance-gitlab-export.cfg:rendered} gitlab-export = instance-gitlab-export.cfg:rendered
gitlab-test = $${instance-gitlab-test.cfg:rendered} gitlab-test = instance-gitlab-test.cfg:rendered
default = $${:gitlab} RootSoftwareInstance = $${:gitlab}
# TODO -import, -pull-backup # TODO -import, -pull-backup
[worker-processes]
recipe = slapos.recipe.build
init =
import multiprocessing
cpu_count = multiprocessing.cpu_count()
# automatically load all available CPUs
options['unicorn-worker-processes'] = cpu_count + 1
options['nginx-worker-processes'] = cpu_count
[slap-configuration]
# std stuff to fetch slapos instance parameters
recipe = slapos.cookbook:slapconfiguration
computer= $${slap-connection:computer-id}
partition=$${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
# autogenerated gitlab instance parameters
<= gitlab-parameters
# adjust/override some default settings:
configuration.unicorn_worker_processes = $${worker-processes:unicorn-worker-processes}
configuration.nginx_worker_processes = $${worker-processes:nginx-worker-processes}
# gitlab non-native parameters
configuration.icp_license =
# macro: render instance-*.cfg from instance-*.cfg.in # macro: render instance-*.cfg from instance-*.cfg.in
[instance-cfg] [instance-cfg]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -25,13 +60,13 @@ rendered= $${buildout:directory}/$${:_buildout_section_name_} ...@@ -25,13 +60,13 @@ rendered= $${buildout:directory}/$${:_buildout_section_name_}
context = context =
import os os import os os
import pwd pwd import pwd pwd
import multiprocessing multiprocessing
key bin_directory buildout:bin-directory key bin_directory buildout:bin-directory
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
raw gitlab_repository_location ${gitlab-repository:location} raw gitlab_repository_location ${gitlab-repository:location}
raw gitlab_shell_repository_location ${gitlab-shell-repository:location} raw gitlab_shell_repository_location ${gitlab-shell-repository:location}
section instance_parameter_dict slap-configuration
# program binaries # program binaries
raw bash_bin ${bash:location}/bin/bash raw bash_bin ${bash:location}/bin/bash
...@@ -67,7 +102,6 @@ context = ...@@ -67,7 +102,6 @@ context =
# config files # config files
raw database_yml_in ${database.yml.in:target} raw database_yml_in ${database.yml.in:target}
raw gitconfig_in ${gitconfig.in:target} raw gitconfig_in ${gitconfig.in:target}
raw gitlab_parameters_cfg ${gitlab-parameters.cfg:target}
raw monitor_template ${monitor2-template:rendered} raw monitor_template ${monitor2-template:rendered}
raw gitlab_shell_config_yml_in ${gitlab-shell-config.yml.in:target} raw gitlab_shell_config_yml_in ${gitlab-shell-config.yml.in:target}
raw gitlab_unicorn_startup_in ${gitlab-unicorn-startup.in:target} raw gitlab_unicorn_startup_in ${gitlab-unicorn-startup.in:target}
......
# Headless Chromium
This software release compiles and runs a headless Chromium shell and
exposes an interface to connect to it remotely from another browser.
After deployment, the instance is configured like this:
```
Caddy frontend
|
(HTTPS, IPv6)
|
Nginx proxy, basic authentication
|
(HTTP, IPv4)
|
Chromium shell
```
The proxy is necessary because Chromium only accepts local connections
for remote debugging.
## Parameters
The following instance parameters can be configured:
- target-url: URL for Chromium to load on startup.
- remote-debugging-port: Port for Chromium to listen on.
- nginx-proxy-port: Port for Ningx proxy to listen on.
- monitor-httpd-port: Port for monitor.
See `instance-headless-chromium-input-schema.json` for default values.
[template-cfg]
filename = instance.cfg.in
md5sum = 5dfeeb5eca125dcaa5f9e537f941dd41
[instance-headless-chromium]
_update_hash_filename_ = instance-headless-chromium.cfg.in
md5sum = fad685238b26ca20537c12ce7432e7e7
[template-nginx-conf]
_update_hash_filename_ = templates/nginx.conf.in
md5sum = c4d09d2b819f624087ef4c38551dfe2f
[template-mime-types]
_update_hash_filename_ = templates/mime-types.in
md5sum = 4ef94a7b458d885cd79ba0b930a5727e
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"target-url": {
"description": "URL for Chromium to load on startup.",
"title": "Target URL",
"type": "string",
"default": "https://www.example.com"
},
"remote-debugging-port": {
"description": "Port for Chromium to listen on.",
"title": "Remote Debugging Port",
"type": "integer",
"default": 8081
},
"nginx-proxy-port": {
"description": "Port for Nginx proxy to listen on.",
"title": "Nginx Proxy Port",
"type": "integer",
"default": 8082
},
"monitor-httpd-port": {
"description": "Port for monitor frontend.",
"title": "Monitor Httpd Port",
"type": "integer",
"default": 8083
}
}
}
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Values returned by headless Chromium instantiation",
"properties": {
"frontend-url": {
"description": "URL to access remote debugging interface",
"type": "string"
},
"username": {
"description": "Username for remote debugging interface",
"type": "string"
},
"password": {
"description": "Password for remote debugging interface",
"type": "string"
},
"monitor-base-url": {
"description": "Base URL used by monitor",
"type": "string"
},
"monitor-setup-url": {
"description": "One-click link to setup and monitor feeds",
"type": "string"
},
"proxy-url": {
"description": "Raw IPv6 address used by Nginx proxy",
"type": "string"
},
"remote-debug-url": {
"description": "Local IPv4 address used by Chromium",
"type": "string"
}
}
}
{% set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
[buildout]
parts =
chromium-launcher
generate-passwd-file
nginx-config
nginx-mime-types
nginx-launcher
logrotate-entry-nginx
publish-connection-information
frontend-ok-promise
frontend-secure-promise
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
extends = {{ parameter_list['template-monitor'] }}
# Create necessary directories.
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
tmp = ${:home}/tmp
log = ${:home}/log
etc = ${:home}/etc
ssl = ${:etc}/ssl
service = ${:etc}/service
# Options for instance configuration. See README.md for a list of
# options that can be configured when requesting an instance.
[headless-chromium]
ipv4 = {{ partition_ipv4 }}
ipv6 = {{ partition_ipv6 }}
remote-debugging-port = {{ parameter_dict['remote-debugging-port'] }}
url = {{ parameter_dict['target-url'] }}
remote-debugging-address = ${:ipv4}:${:remote-debugging-port}
devtools-frontend-root = {{ parameter_list['devtools-frontend'] }}
nginx-port = {{ parameter_dict['nginx-proxy-port'] }}
proxy-address = [${:ipv6}]:${:nginx-port}
nginx-config-target = ${directory:etc}/nginx.conf
nginx-pid-path = ${directory:log}/nginx.pid
nginx-temp-path = ${directory:tmp}
nginx-error-log = ${directory:log}/nginx-error.log
nginx-access-log = ${directory:log}/nginx-access.log
nginx-htpasswd-file = ${directory:etc}/.htpasswd
nginx-key-file = ${frontend-instance-certificate:key-file}
nginx-cert-file = ${frontend-instance-certificate:cert-file}
nginx-mime-types = ${directory:etc}/mime-types
# Create a launcher script in /etc/service for the headless shell
# executable.
[chromium-launcher]
recipe = slapos.recipe.template:jinja2
template =
inline:#!/bin/sh
export FONTCONFIG_FILE=${font-config:rendered}
exec {{ parameter_list['chromium-wrapper'] }} \
--remote-debugging-address=${headless-chromium:ipv4} \
--remote-debugging-port=${headless-chromium:remote-debugging-port} \
${headless-chromium:url}
rendered = ${directory:service}/chromium
# Configure and launch the proxy server.
[nginx-config]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_list['template-nginx-config'] }}
rendered = ${headless-chromium:nginx-config-target}
mode = 700
context =
section param_headless_chromium headless-chromium
[nginx-mime-types]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_list['template-mime-types'] }}
rendered = ${headless-chromium:nginx-mime-types}
[nginx-launcher]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_list['nginx-location'] }}/sbin/nginx -c ${headless-chromium:nginx-config-target}
wrapper-path = ${directory:service}/nginx
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
log = ${headless-chromium:nginx-error-log} ${headless-chromium:nginx-access-log}
[frontend-instance-password]
recipe = slapos.cookbook:generate.password
username = admin
bytes = 12
[generate-passwd-file]
recipe = plone.recipe.command
command =
echo -n '${frontend-instance-password:username}:' > ${headless-chromium:nginx-htpasswd-file}
openssl passwd -apr1 '${frontend-instance-password:passwd}' >> ${headless-chromium:nginx-htpasswd-file}
environment =
PATH={{ parameter_list['openssl-location'] }}/bin:%(PATH)s
# Generate a self-signed TLS certificate.
[frontend-instance-certificate]
recipe = plone.recipe.command
command =
if [ ! -e ${:key-file} ]
then
openssl req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=${:common-name}" \
-newkey rsa:1024 -keyout ${:key-file} \
-out ${:cert-file}
openssl x509 -addtrust serverAuth \
-in ${:cert-file} \
-out ${:cert-file}
fi
update-command = ${:command}
key-file = ${directory:ssl}/${:_buildout_section_name_}.key
cert-file = ${directory:ssl}/${:_buildout_section_name_}.cert
common-name = ${headless-chromium:ipv6}
environment =
PATH={{ parameter_list['openssl-location'] }}/bin:%(PATH)s
# Generate a fonts.conf file.
[font-config]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_list['template-fonts-conf'] }}
rendered = ${directory:etc}/fonts.conf
context =
key cachedir :cache-dir
key fonts :fonts
key includes :includes
cache-dir =
${directory:etc}/.fontconfig.cache
fonts =
{{ parameter_list['liberation-fonts-location'] }}
includes =
{{ parameter_list['fontconfig-location'] }}/etc/fonts/conf.d
[publish-connection-information]
recipe = slapos.cookbook:publish
<= monitor-publish
remote-debug-url = http://${headless-chromium:remote-debugging-address}
proxy-url = https://${headless-chromium:proxy-address}
frontend-url = ${remote-debugging-frontend:connection-secure_access}
username = ${frontend-instance-password:username}
password = ${frontend-instance-password:passwd}
# Request a frontend URL from the CDN for the remote debugging interface.
[remote-debugging-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Headless Chromium Remote Debugging Frontend
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = https://${headless-chromium:proxy-address}
config-https-only = true
config-type = websocket
config-websocket-path-list = /devtools
return = domain secure_access
# Monitoring: check that the Chromium process is alive and responding to
# requests through the proxy.
[monitor-instance-parameter]
monitor-httpd-port = {{ parameter_dict['monitor-httpd-port'] }}
# Promise to make sure the remote debugging frontend returns 200 when
# queried with the correct credentials.
[frontend-ok-promise]
<= monitor-promise-base
module = check_url_available
name = headless-chromium-frontend-ok.py
url = ${remote-debugging-frontend:connection-secure_access}
config-url = ${:url}
config-username = ${frontend-instance-password:username}
config-password = ${frontend-instance-password:passwd}
# Promise to make sure that the remote debugging frontend returns 401
# when queried with no credentials.
[frontend-secure-promise]
<= monitor-promise-base
module = check_url_available
name = headless-chromium-frontend-secure.py
url = ${remote-debugging-frontend:connection-secure_access}
config-url = ${:url}
config-http-code = 401
[buildout]
parts =
switch-softwaretype
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
[profile-common]
openssl-location = {{ openssl_location }}
nginx-location = {{ nginx_location }}
liberation-fonts-location = {{ liberation_fonts_location }}
fontconfig-location = {{ fontconfig_location }}
chromium-wrapper = {{ chromium_wrapper }}
devtools-frontend = {{ devtools_frontend }}
template-nginx-config = {{ template_nginx_config_target }}
template-fonts-conf = {{ template_fonts_conf_target }}
template-monitor = {{ template_monitor }}
template-mime-types = {{ template_mime_types_target }}
[instance-headless-chromium]
recipe = slapos.recipe.template:jinja2
template = {{ template_instance_headless_chromium_target }}
rendered = ${buildout:directory}/${:filename}
filename = instance-headless-chromium.cfg
context =
section buildout buildout
section parameter_list profile-common
key partition_ipv4 slap-configuration:ipv4-random
key partition_ipv6 slap-configuration:ipv6-random
key slapparameter_dict slap-configuration:configuration
jsonkey default_parameter_dict :default-parameters
default-parameters =
{
"remote-debugging-port": 8081,
"nginx-proxy-port": 8082,
"target-url": "https://www.example.com",
"monitor-httpd-port": 8083
}
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = ${:default}
default = instance-headless-chromium:rendered
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[buildout]
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
../../component/headless-chromium/buildout.cfg
../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg
../../component/fonts/buildout.cfg
../../component/fontconfig/buildout.cfg
parts =
slapos-cookbook
template-cfg
[python]
part = python3
[template-cfg]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/${:filename}
mode = 0644
context =
section buildout buildout
key openssl_location openssl:location
key nginx_location nginx:location
key liberation_fonts_location liberation-fonts:location
key fontconfig_location fontconfig:location
key chromium_wrapper headless-chromium-wrapper:rendered
key devtools_frontend headless-chromium:devtools-frontend
key template_nginx_config_target template-nginx-conf:target
key template_mime_types_target template-mime-types:target
key template_fonts_conf_target template-fonts-conf:output
key template_instance_headless_chromium_target instance-headless-chromium:target
key template_monitor monitor2-template:rendered
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[instance-headless-chromium]
<= download-base
[template-nginx-conf]
<= download-base
[template-mime-types]
<= download-base
{
"name": "Headless Chromium",
"description": "Headless (stripped-down) Chromium shell",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"description": "Standalone headless shell",
"request": "instance-headless-chromium-input-schema.json",
"response": "instance-headless-chromium-output-schema.json",
"index": 0
}
}
}
types {
text/html html htm shtml;
text/css css;
text/xml xml rss;
image/gif gif;
image/jpeg jpeg jpg;
application/x-javascript js;
application/atom+xml atom;
text/mathml mml;
text/plain txt;
text/vnd.sun.j2me.app-descriptor jad;
text/vnd.wap.wml wml;
text/x-component htc;
image/png png;
image/tiff tif tiff;
image/vnd.wap.wbmp wbmp;
image/x-icon ico;
image/x-jng jng;
image/x-ms-bmp bmp;
image/svg+xml svg svgz;
application/java-archive jar war ear;
application/mac-binhex40 hqx;
application/msword doc;
application/pdf pdf;
application/postscript ps eps ai;
application/rtf rtf;
application/vnd.ms-excel xls;
application/vnd.ms-powerpoint ppt;
application/vnd.wap.wmlc wmlc;
application/vnd.google-earth.kml+xml kml;
application/vnd.google-earth.kmz kmz;
application/x-7z-compressed 7z;
application/x-cocoa cco;
application/x-java-archive-diff jardiff;
application/x-java-jnlp-file jnlp;
application/x-makeself run;
application/x-perl pl pm;
application/x-pilot prc pdb;
application/x-rar-compressed rar;
application/x-redhat-package-manager rpm;
application/x-sea sea;
application/x-shockwave-flash swf;
application/x-stuffit sit;
application/x-tcl tcl tk;
application/x-x509-ca-cert der pem crt;
application/x-xpinstall xpi;
application/xhtml+xml xhtml;
application/zip zip;
application/octet-stream bin exe dll;
application/octet-stream deb;
application/octet-stream dmg;
application/octet-stream eot;
application/octet-stream iso img;
application/octet-stream msi msp msm;
application/ogg ogx;
audio/midi mid midi kar;
audio/mpeg mpga mpega mp2 mp3 m4a;
audio/ogg oga ogg spx;
audio/x-realaudio ra;
audio/webm weba;
video/3gpp 3gpp 3gp;
video/mp4 mp4;
video/mpeg mpeg mpg mpe;
video/ogg ogv;
video/quicktime mov;
video/webm webm;
video/x-flv flv;
video/x-mng mng;
video/x-ms-asf asx asf;
video/x-ms-wmv wmv;
video/x-msvideo avi;
}
pid {{ param_headless_chromium['nginx-pid-path'] }};
error_log {{ param_headless_chromium['nginx-error-log'] }};
events {
worker_connections 1024;
}
http {
access_log {{ param_headless_chromium['nginx-access-log'] }};
include {{ param_headless_chromium['nginx-mime-types'] }};
default_type application/octet-stream;
types {
text/html html;
text/css css;
application/javascript js;
}
server {
listen {{ param_headless_chromium['proxy-address'] }} ssl;
# Require username/password to access remote debugging port.
auth_basic "Remote Debugging";
auth_basic_user_file {{ param_headless_chromium['nginx-htpasswd-file'] }};
# Use self-signed SSL certificate.
ssl_certificate {{ param_headless_chromium['nginx-cert-file'] }};
ssl_certificate_key {{ param_headless_chromium['nginx-key-file'] }};
client_body_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
proxy_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
fastcgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
uwsgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
scgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
# All websocket connections are served from /devtools.
location /devtools {
proxy_http_version 1.1;
proxy_set_header Host {{ param_headless_chromium['remote-debugging-address'] }};
proxy_pass http://{{ param_headless_chromium['remote-debugging-address'] }};
proxy_set_header Upgrade "websocket";
proxy_set_header Connection "Upgrade";
}
# The DevTools frontend is served from /serve_file/@{version_hash}.
location ~ "^\/serve_file\/@[0-9a-f]{5,40}\/(.*)" {
alias {{ param_headless_chromium['devtools-frontend-root'] }}/$1;
}
location / {
proxy_http_version 1.1;
# The proxy must set the Host header to an IP address, since the
# headless Chromium shell refuses to run otherwise, for security
# reasons.
# See https://bugs.chromium.org/p/chromium/issues/detail?id=813540.
proxy_set_header Host {{ param_headless_chromium['remote-debugging-address'] }};
proxy_pass http://{{ param_headless_chromium['remote-debugging-address'] }};
# The browser security policy will prevent us from loading the
# Websocket connection without TLS, so we have to go through the
# frontend CDN URL. The tricky thing is that the frontend URL is
# not available yet when this file is built; what we do instead is
# use the given Host header.
sub_filter "ws={{ param_headless_chromium['remote-debugging-address'] }}" "wss=$host";
sub_filter_once on;
sub_filter_types application/json;
sub_filter "ws://{{ param_headless_chromium['remote-debugging-address'] }}" "wss://$host";
sub_filter_types application/json;
# We want to use our own DevTools frontend rather than
# https://chrome-devtools-frontend.appspot.com. There should be a
# --custom-devtools-frontend flag for Chromium, but it doesn't
# seem to work with the remote debugging port.
sub_filter "chrome-devtools-frontend.appspot.com" "$host";
sub_filter_types *;
}
}
}
Tests for headless Chromium software release
##############################################################################
#
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.headless-chromium'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS headless Chromium",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
#
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '../software.cfg')))
class TestHeadlessChromium(SlapOSInstanceTestCase):
def setUp(self):
self.connection_parameters = self.requestDefaultInstance().getConnectionParameterDict()
def test_remote_debugging_port(self):
# The headless browser should respond at /json with a nonempty list
# of available pages, each of which has a webSocketDebuggerUrl and a
# devtoolsFrontendUrl.
url = self.connection_parameters['remote-debug-url']
response = requests.get('%s/json' % url)
# Check that request was successful and the response was a nonempty
# list.
self.assertEqual(requests.codes['ok'], response.status_code)
self.assertTrue(len(response.json()) > 0)
# Check that the first page has the correct fields.
first_page = response.json()[0]
self.assertIn('webSocketDebuggerUrl', first_page)
self.assertIn('devtoolsFrontendUrl', first_page)
def test_devtools_frontend_ok(self):
# The proxy should serve the DevTools frontend from
# /serve_file/@{hash}/inspector.html, where {hash} is a 5-32 digit
# hash.
proxyURL = self.connection_parameters['proxy-url']
username = self.connection_parameters['username']
password = self.connection_parameters['password']
frontend = '/serve_file/@aaaaa/inspector.html'
response = requests.get(proxyURL + frontend, verify=False,
auth=(username, password))
self.assertEqual(requests.codes['ok'], response.status_code)
...@@ -91,78 +91,27 @@ tornado = 6.1 ...@@ -91,78 +91,27 @@ tornado = 6.1
traitlets = 5.0.5 traitlets = 5.0.5
webencodings = 0.5.1 webencodings = 0.5.1
widgetsnbextension = 2.0.0 widgetsnbextension = 2.0.0
# Required by:
# tornado==4.4.2
certifi = 2020.6.20 certifi = 2020.6.20
# Required by:
# notebook==6.1.5
Send2Trash = 1.5.0 Send2Trash = 1.5.0
# Required by:
# notebook==6.1.5
argon2-cffi = 20.1.0 argon2-cffi = 20.1.0
# Required by:
# notebook==6.1.5
nbconvert = 6.0.7 nbconvert = 6.0.7
# Required by:
# ipython==5.3.0
pathlib2 = 2.2.1 pathlib2 = 2.2.1
# Required by:
# statsmodels==0.11.1
patsy = 0.5.1 patsy = 0.5.1
# Required by:
# ipython==5.3.0
pexpect = 4.8.0 pexpect = 4.8.0
# Required by:
# ipython==5.3.0
pickleshare = 0.7.4 pickleshare = 0.7.4
# Required by:
# notebook==6.1.5
prometheus-client = 0.9.0 prometheus-client = 0.9.0
# Required by:
# pathlib2==2.2.1
scandir = 1.5 scandir = 1.5
# Required by:
# statsmodels==0.11.1
pytz = 2020.4 pytz = 2020.4
# Required by:
# statsmodels==0.11.1
scipy = 1.0.1 scipy = 1.0.1
# Required by:
# tornado==6.1
singledispatch = 3.4.0.3 singledispatch = 3.4.0.3
# Required by:
# prompt-toolkit==1.0.13
wcwidth = 0.1.7 wcwidth = 0.1.7
jupyter = 1.0.0 jupyter = 1.0.0
jupyter-console = 5.1.0 jupyter-console = 5.1.0
# Required by:
# jupyter==1.0.0
qtconsole = 4.3.0 qtconsole = 4.3.0
et-xmlfile = 1.0.1 et-xmlfile = 1.0.1
h5py = 2.7.1 h5py = 2.7.1
mpmath = 1.0.0 mpmath = 1.0.0
openpyxl = 2.5.2 openpyxl = 2.5.2
sympy = 1.1.1 sympy = 1.1.1
xlrd = 1.1.0 xlrd = 1.1.0
# Required by:
# openpyxl==2.5.2
jdcal = 1.4 jdcal = 1.4
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = c115ed9d4ff0f785d79cdcacbb0bd1ad md5sum = 2114ae8c1e92bd33ef1347f36f567c74
[template-monitor] [template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2 _update_hash_filename_ = instance-monitor.cfg.jinja2
......
...@@ -6,10 +6,11 @@ eggs-directory = ${buildout:eggs-directory} ...@@ -6,10 +6,11 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
[switch_softwaretype] [switch_softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:switch-softwaretype
default = $${instance-base-monitor:rendered} default = instance-base-monitor:rendered
edgetest = $${instance-base-edgetest:rendered} edgetest = instance-base-edgetest:rendered
edgebot = $${instance-base-edgebot:rendered} edgebot = instance-base-edgebot:rendered
RootSoftwareInstance = $${:default}
[instance-base-monitor] [instance-base-monitor]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
......
...@@ -132,11 +132,7 @@ ecdsa = 0.13 ...@@ -132,11 +132,7 @@ ecdsa = 0.13
mysqlclient = 1.3.12 mysqlclient = 1.3.12
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
setproctitle = 1.1.10
cython-zstd = 0.2 cython-zstd = 0.2
# Required by:
# mock = 3.0.5
funcsigs = 1.0.2 funcsigs = 1.0.2
......
# Nginx Push Stream
This software uses the [HTTP Push Stream](https://www.nginx.com/resources/wiki/modules/push_stream/)
module of nginx to make a [Server Sent Event](https://html.spec.whatwg.org/multipage/server-sent-events.html)
server.
Two endpoints are available, published as the following connection parameters:
- `publisher-url`, which uses`/pub{?id}` format, with `id` being the ID of the channel.
Clients can subscribe to this URL and be notified of new messages.
- `subscriber-url`, which uses `/sub{/id}` format, with `id` being the ID of the channel.
`POST` requests to this URL send new messages to this channel.
[template]
filename = instance.cfg.in
md5sum = f9b6d01e29f2edddd9d6f99591976c33
[template-nginx-configuration]
filename = template-nginx.cfg.in
md5sum = 022e4b53e1b2db16c4e518fe76f638fa
[buildout]
parts =
nginx-service
htpasswd
htpasswd-runner
publish-connection-information
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
run = $${:var}/run
log = $${:var}/log
varnginx = $${:var}/nginx
services = $${:etc}/service
cron-entries = $${:etc}/cron.d
www = $${:srv}/www
ssl = $${:etc}/ssl
#################################
# Nginx service
#################################
[nginx-service]
recipe = slapos.recipe.template
url = ${template-nginx-service:output}
output = $${directory:services}/nginx
mode = 0700
virtual-depends =
$${nginx-configuration:ip}
[nginx-configuration]
recipe = slapos.recipe.template
url = ${template-nginx-configuration:output}
output = $${directory:etc}/nginx.cfg
mode = 0600
access_log = $${directory:log}/nginx-access.log
error_log = $${directory:log}/nginx-error.log
ip = $${slap-network-information:global-ipv6}
local_ip = $${slap-network-information:local-ipv4}
port = 9443
publisher_location_prefix = /pub
publisher_push_stream_store_messages = off
publisher_client_max_body_size = 16k
publisher_client_body_buffer_size = 16k
subscriber_allow_origin = '*'
subscriber_location_prefix = /sub
# Prevent to use credential if origin is star
subscriber_allow_credential = 'false'
subscriber_allow_methods = 'GET, HEAD, OPTIONS'
subscriber_allow_headers = 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since'
[htpasswd]
recipe = slapos.cookbook:generate.password
storage-path = $${directory:etc}/.pwd
bytes = 8
[htpasswd-runner]
recipe = plone.recipe.command
stop-on-error = true
htpasswd-path = $${directory:etc}/.htpasswd
command = if [ ! -f "$${:htpasswd-path}" ]; then ${buildout:bin-directory}/htpasswd -cb $${:htpasswd-path} $${:user} $${:password}; fi
update-command = $${:command}
user = admin
password = $${htpasswd:passwd}
[publish-connection-information]
recipe = slapos.cookbook:publish
init-password = $${htpasswd:passwd}
init-user = $${htpasswd-runner:user}
publisher-url = http://$${htpasswd-runner:user}:$${htpasswd:passwd}@[$${nginx-configuration:ip}]:$${nginx-configuration:port}$${nginx-configuration:publisher_location_prefix}
subscriber-url = http://$${htpasswd-runner:user}:$${htpasswd:passwd}@[$${nginx-configuration:ip}]:$${nginx-configuration:port}$${nginx-configuration:subscriber_location_prefix}
[buildout] [buildout]
parts = parts =
switch-softwaretype nginx-service
promises
publish-connection-information
extends = ${monitor-template:rendered}
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true offline = true
[switch-softwaretype] [directory]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:mkdirectory
default = $${:nginx} etc = $${buildout:directory}/etc
nginx = $${dynamic-template-nginx:rendered} bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv
[dynamic-template-nginx] var = $${buildout:directory}/var
recipe = slapos.recipe.template:jinja2 run = $${:var}/run
template = ${template-nginx:output} log = $${:var}/log
rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename} varnginx = $${:var}/nginx
filename = instance-nginx.cfg services = $${:etc}/service
cron-entries = $${:etc}/cron.d
[slap-connection] www = $${:srv}/www
computer-id = $${slap_connection:computer_id} ssl = $${:etc}/ssl
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url} [slap-configuration]
software-release-url = $${slap_connection:software_release_url} recipe = slapos.cookbook:slapconfiguration
key-file = $${slap_connection:key_file} computer = $${slap-connection:computer-id}
cert-file = $${slap_connection:cert_file} partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
[instance-parameter] key = $${slap-connection:key-file}
# Fetches parameters defined in SlapOS Master for this instance. cert = $${slap-connection:cert-file}
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised #################################
computer = $${slap_connection:computer_id} # Nginx service
partition = $${slap_connection:partition_id} #################################
url = $${slap_connection:server_url} [nginx-service]
key = $${slap_connection:key_file} recipe = slapos.cookbook:wrapper
cert = $${slap_connection:cert_file} wrapper-path = $${directory:services}/nginx
command-line =
${nginx-push-stream-output:nginx} -c $${nginx-configuration:output}
[nginx-configuration]
recipe = slapos.recipe.template
url = ${template-nginx-configuration:output}
output = $${directory:etc}/nginx.cfg
mode = 0600
access-log = $${directory:log}/nginx-access.log
error-log = $${directory:log}/nginx-error.log
ip = $${slap-configuration:ipv6-random}
local-ip = $${slap-configuration:ipv4-random}
port = 9443
base-url = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}
# Generate a self-signed TLS certificate.
[nginx-certificate]
recipe = plone.recipe.command
command =
if [ ! -e $${:key-file} ]
then
${openssl:location}/bin/openssl req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=$${nginx-configuration:ip}" \
-newkey rsa:1024 -keyout $${:key-file} \
-out $${:cert-file}
fi
update-command = $${:command}
key-file = $${directory:ssl}/${:_buildout_section_name_}.key
cert-file = $${directory:ssl}/${:_buildout_section_name_}.cert
common-name = $${nginx-configuration:ip}
stop-on-error = true
[promises]
recipe =
promises =
$${nginx-available-promise:recipe}
[nginx-available-promise]
<= monitor-promise-base
module = check_url_available
name = $${:_buildout_section_name_}.py
config-url = $${nginx-configuration:base-url}/status
[publish-connection-information]
recipe = slapos.cookbook:publish
# publisher-url and subscriber-url are URITemplates, with an id
# parameter which is the ID of the channel.
publisher-url = $${nginx-configuration:base-url}/pub{?id}
subscriber-url = $${nginx-configuration:base-url}/sub{/id}
[buildout] [buildout]
extends = extends =
../../stack/slapos.cfg ../../stack/slapos.cfg
../../component/dash/buildout.cfg
../../component/nginx/buildout.cfg ../../component/nginx/buildout.cfg
../../stack/monitor/buildout.cfg
./buildout.hash.cfg
parts = parts =
slapos-cookbook slapos-cookbook
...@@ -10,36 +11,18 @@ parts = ...@@ -10,36 +11,18 @@ parts =
nginx-push-stream-module nginx-push-stream-module
nginx-push-stream nginx-push-stream
template template
template-nginx-service
template-nginx [python]
part = python3
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in url = ${:_profile_base_location_}/instance.cfg.in
md5sum = eb4c69df9a8dbb94fb76d0a6c11e360f
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
mode = 0644 mode = 0644
[template-nginx-service]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-nginx-service.sh.in
md5sum = 90ba5a247c489261d3304528cba56e06
output = ${buildout:directory}/template-nginx-service.sh.in
mode = 0644
[template-nginx-configuration] [template-nginx-configuration]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-nginx.cfg.in url = ${:_profile_base_location_}/${:filename}
md5sum = f5658154b82282bc1871f18ddf4529d8
output = ${buildout:directory}/template-nginx.cfg.in output = ${buildout:directory}/template-nginx.cfg.in
mode = 0644 mode = 0644
[template-nginx]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-nginx.cfg.in
md5sum = 936fea88f5548c4f14e287f1b27dc127
output = ${buildout:directory}/instance-nginx.cfg.in
mode = 0644
[versions]
inotifyx = 0.2.2
#!${dash-output:dash}
# BEWARE: This file is operated by slapos node
# BEWARE: It will be overwritten automatically
exec ${nginx-push-stream-output:nginx} \
-c $${nginx-configuration:output}
...@@ -8,7 +8,7 @@ events { ...@@ -8,7 +8,7 @@ events {
# multi_accept on; # multi_accept on;
} }
error_log $${nginx-configuration:error_log}; error_log $${nginx-configuration:error-log};
http { http {
...@@ -33,8 +33,8 @@ http { ...@@ -33,8 +33,8 @@ http {
# Logging Settings # Logging Settings
## ##
access_log $${nginx-configuration:access_log}; access_log $${nginx-configuration:access-log};
error_log $${nginx-configuration:error_log}; error_log $${nginx-configuration:error-log};
## ##
# Gzip Settings # Gzip Settings
...@@ -54,9 +54,21 @@ http { ...@@ -54,9 +54,21 @@ http {
## ##
push_stream_shared_memory_size 32m; push_stream_shared_memory_size 32m;
server { server {
listen [$${nginx-configuration:ip}]:$${nginx-configuration:port}; listen [$${nginx-configuration:ip}]:$${nginx-configuration:port} ssl http2;
listen $${nginx-configuration:local_ip}:$${nginx-configuration:port}; listen $${nginx-configuration:local-ip}:$${nginx-configuration:port} ssl http2;
# generated 2021-08-02, Mozilla Guideline v5.6, nginx 1.19.2, OpenSSL 1.1.1k, modern configuration, no HSTS, no OCSP
# https://ssl-config.mozilla.org/#server=nginx&version=1.19.2&config=modern&openssl=1.1.1k&hsts=false&ocsp=false&guideline=5.6
ssl_certificate $${nginx-certificate:cert-file};
ssl_certificate_key $${nginx-certificate:key-file};
ssl_session_timeout 1d;
ssl_session_cache shared:MozSSL:10m;
ssl_session_tickets off;
ssl_protocols TLSv1.3;
ssl_prefer_server_ciphers off;
fastcgi_temp_path $${directory:varnginx} 1 2; fastcgi_temp_path $${directory:varnginx} 1 2;
uwsgi_temp_path $${directory:varnginx} 1 2; uwsgi_temp_path $${directory:varnginx} 1 2;
...@@ -65,35 +77,35 @@ server { ...@@ -65,35 +77,35 @@ server {
client_body_temp_path $${directory:varnginx} 1 2; client_body_temp_path $${directory:varnginx} 1 2;
proxy_temp_path $${directory:varnginx} 1 2; proxy_temp_path $${directory:varnginx} 1 2;
auth_basic "Nginx Access";
auth_basic_user_file $${htpasswd-runner:htpasswd-path};
## Serve an error 204 (No Content) for favicon.ico ## Serve an error 204 (No Content) for favicon.ico
location = /favicon.ico { location = /favicon.ico {
return 204; return 204;
} }
location $${nginx-configuration:publisher_location_prefix} { location = /status {
default_type "text/plain";
return 200 'OK';
}
location /pub {
push_stream_publisher; push_stream_publisher;
push_stream_channels_path $arg_id; push_stream_channels_path $arg_id;
# store messages in memory # store messages in memory
push_stream_store_messages $${nginx-configuration:publisher_push_stream_store_messages}; push_stream_store_messages off;
# Message size limit # Message size limit
# client_max_body_size MUST be equal to client_body_buffer_size or client_max_body_size 16k;
# you will be sorry. client_body_buffer_size 16k;
client_max_body_size $${nginx-configuration:publisher_client_max_body_size};
client_body_buffer_size $${nginx-configuration:publisher_client_body_buffer_size};
} }
location ~ $${nginx-configuration:subscriber_location_prefix}/(.*) { location ~ /sub/(.*) {
# activate subscriber mode for this location # activate subscriber mode for this location
add_header "Access-Control-Allow-Origin" $${nginx-configuration:subscriber_allow_origin}; add_header "Access-Control-Allow-Origin" '*';
add_header 'Access-Control-Allow-Credentials' $${nginx-configuration:subscriber_allow_credential}; add_header 'Access-Control-Allow-Credentials' 'false';
add_header 'Access-Control-Allow-Methods' $${nginx-configuration:subscriber_allow_methods}; add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
add_header 'Access-Control-Allow-Headers' $${nginx-configuration:subscriber_allow_headers}; add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
push_stream_subscriber eventsource; push_stream_subscriber eventsource;
# positional channel path # positional channel path
......
Tests for Nginx Push Stream software release
##############################################################################
#
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.nginx_push_stream'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' Nginx Push Stream",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
'uritemplate',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
# coding: utf-8
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import multiprocessing
import uritemplate
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestNginxPushStream(SlapOSInstanceTestCase):
def setUp(self):
self.connection_parameters = \
self.computer_partition.getConnectionParameterDict()
def test_push_stream_scenario(self):
def process_messages(q):
# type:(multiprocessing.Queue[bytes]) -> None
req = requests.get(
uritemplate.URITemplate(
self.connection_parameters['subscriber-url']).expand(
id='channel_id'),
verify=False,
stream=True,
)
if not req.ok:
q.put(('error: wrong status code %s' % req.status_code).encode())
q.put(b'ready')
for _, line in zip(range(2), req.iter_lines()):
q.put(line)
q = multiprocessing.Queue() # type: multiprocessing.Queue[bytes]
subscriber = multiprocessing.Process(target=process_messages, args=(q, ))
subscriber.start()
self.assertEqual(q.get(timeout=30), b'ready')
resp = requests.post(
uritemplate.URITemplate(
self.connection_parameters['publisher-url']).expand(
id='channel_id'),
verify=False,
data='Hello',
timeout=2,
)
resp.raise_for_status()
try:
subscriber.join(timeout=30)
except multiprocessing.TimeoutError:
subscriber.terminate()
subscriber.join(timeout=30)
self.fail('Process did not terminate')
self.assertEqual(q.get_nowait(), b': ')
self.assertEqual(q.get_nowait(), b'data: Hello')
...@@ -15,4 +15,4 @@ ...@@ -15,4 +15,4 @@
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 733643122fb75dfd7374b973a95fa2ea md5sum = 2c8b202b0c366e27dd6e6f456fb18256
...@@ -44,7 +44,7 @@ tests = {{ tests | indent(2) }} ...@@ -44,7 +44,7 @@ tests = {{ tests | indent(2) }}
template = template =
inline:{% raw %} inline:{% raw %}
{%- set only_sr = slapparameter_dict.get('only-sr') %} {%- set only_sr = slapparameter_dict.get('only-sr') %}
{%- if not isinstance(only_sr, list) %} {%- if only_sr and not isinstance(only_sr, list) %}
{%- set only_sr = [only_sr] %} {%- set only_sr = [only_sr] %}
{%- endif %} {%- endif %}
{%- set unittest_args = slapparameter_dict.get('unittest-args', ['discover', '-v']) %} {%- set unittest_args = slapparameter_dict.get('unittest-args', ['discover', '-v']) %}
......
...@@ -22,3 +22,4 @@ extra = ...@@ -22,3 +22,4 @@ extra =
proftpd ${slapos.test.proftpd-setup:setup} proftpd ${slapos.test.proftpd-setup:setup}
repman ${slapos.test.repman-setup:setup} repman ${slapos.test.repman-setup:setup}
restic-rest-server ${slapos.test.restic_rest_server-setup:setup} restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
headless-chromium ${slapos.test.headless-chromium-setup:setup}
...@@ -149,6 +149,11 @@ setup = ${slapos-repository:location}/software/jupyter/test/ ...@@ -149,6 +149,11 @@ setup = ${slapos-repository:location}/software/jupyter/test/
egg = slapos.test.nextcloud egg = slapos.test.nextcloud
setup = ${slapos-repository:location}/software/nextcloud/test/ setup = ${slapos-repository:location}/software/nextcloud/test/
[slapos.test.nginx-push-stream-setup]
<= setup-develop-egg
egg = slapos.test.nginx_push_stream
setup = ${slapos-repository:location}/software/nginx-push-stream/test/
[slapos.test.turnserver-setup] [slapos.test.turnserver-setup]
<= setup-develop-egg <= setup-develop-egg
egg = slapos.test.turnserver egg = slapos.test.turnserver
...@@ -194,16 +199,26 @@ setup = ${slapos-repository:location}/software/jscrawler/test/ ...@@ -194,16 +199,26 @@ setup = ${slapos-repository:location}/software/jscrawler/test/
egg = slapos.test.galene egg = slapos.test.galene
setup = ${slapos-repository:location}/software/galene/test/ setup = ${slapos-repository:location}/software/galene/test/
[slapos.core-repository] [slapos.test.headless-chromium-setup]
<= git-clone-repository <= setup-develop-egg
repository = https://lab.nexedi.com/nexedi/slapos.core.git egg = slapos.test.headless-chromium
branch = master setup = ${slapos-repository:location}/software/headless-chromium/test/
[slapos.test.caucase-setup] [slapos.test.caucase-setup]
<= setup-develop-egg <= setup-develop-egg
egg = slapos.test.caucase egg = slapos.test.caucase
setup = ${slapos-repository:location}/software/caucase/test/ setup = ${slapos-repository:location}/software/caucase/test/
[slapos.test.erp5testnode-setup]
<= setup-develop-egg
egg = slapos.test.erp5testnode
setup = ${slapos-repository:location}/software/erp5testnode/test/
[slapos.core-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
[slapos.core-setup] [slapos.core-setup]
<= setup-develop-egg <= setup-develop-egg
egg = slapos.core egg = slapos.core
...@@ -242,6 +257,7 @@ extra-eggs = ...@@ -242,6 +257,7 @@ extra-eggs =
${slapos.test.slaprunner-setup:egg} ${slapos.test.slaprunner-setup:egg}
${slapos.test.jupyter-setup:egg} ${slapos.test.jupyter-setup:egg}
${slapos.test.nextcloud-setup:egg} ${slapos.test.nextcloud-setup:egg}
${slapos.test.nginx-push-stream-setup:egg}
${slapos.test.turnserver-setup:egg} ${slapos.test.turnserver-setup:egg}
${slapos.test.theia-setup:egg} ${slapos.test.theia-setup:egg}
${slapos.test.cloudooo-setup:egg} ${slapos.test.cloudooo-setup:egg}
...@@ -254,6 +270,8 @@ extra-eggs = ...@@ -254,6 +270,8 @@ extra-eggs =
${slapos.test.html5as-setup:egg} ${slapos.test.html5as-setup:egg}
${slapos.test.html5as-base-setup:egg} ${slapos.test.html5as-base-setup:egg}
${slapos.test.fluentd-setup:egg} ${slapos.test.fluentd-setup:egg}
${slapos.test.headless-chromium-setup:egg}
${slapos.test.erp5testnode-setup:egg}
# We don't name this interpreter `python`, so that when we run slapos node # We don't name this interpreter `python`, so that when we run slapos node
# software, installation scripts running `python` use a python without any # software, installation scripts running `python` use a python without any
...@@ -312,6 +330,7 @@ tests = ...@@ -312,6 +330,7 @@ tests =
slaprunner ${slapos.test.slaprunner-setup:setup} slaprunner ${slapos.test.slaprunner-setup:setup}
theia ${slapos.test.theia-setup:setup} theia ${slapos.test.theia-setup:setup}
metabase ${slapos.test.metabase-setup:setup} metabase ${slapos.test.metabase-setup:setup}
nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
### ###
${:extra} ${:extra}
...@@ -337,6 +356,7 @@ extra = ...@@ -337,6 +356,7 @@ extra =
jscrawler ${slapos.test.jscrawler-setup:setup} jscrawler ${slapos.test.jscrawler-setup:setup}
html5as ${slapos.test.html5as-setup:setup} html5as ${slapos.test.html5as-setup:setup}
html5as-base ${slapos.test.html5as-base-setup:setup} html5as-base ${slapos.test.html5as-base-setup:setup}
erp5testnode ${slapos.test.erp5testnode-setup:setup}
[versions] [versions]
# slapos.core is used from the clone always # slapos.core is used from the clone always
......
...@@ -15,15 +15,39 @@ ...@@ -15,15 +15,39 @@
[instance-theia] [instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in _update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = 11d347dd2bf762902341746a388673a0 md5sum = 562acf69f344fa9f6d03992d696462d0
[instance] [instance]
_update_hash_filename_ = instance.cfg.in _update_hash_filename_ = instance.cfg.in
md5sum = 063d3e19da9d3d4bfb77e8e638aa3a77 md5sum = a7d78b4002266c69ece05a476df82791
[instance-import]
_update_hash_filename_ = instance-import.cfg.jinja.in
md5sum = 861ef130f27175c2978a9b946b138dd5
[instance-export]
_update_hash_filename_ = instance-export.cfg.jinja.in
md5sum = b3cedaa1603ca8ed83fdd94ef4b35cc8
[instance-resilient]
_update_hash_filename_ = instance-resilient.cfg.jinja
md5sum = d78a9f885bdebf6720197209e0c21aa0
[theia-common]
_update_hash_filename_ = theia_common.py
md5sum = e57396473b4b6a17d26a747f0030293c
[theia-export]
_update_hash_filename_ = theia_export.py
md5sum = b5f5ac1924b27d3f2be2e5ea291c119e
[theia-import]
_update_hash_filename_ = theia_import.py
md5sum = 9e8c17a4b2d802695caf0c2c052f0d11
[yarn.lock] [yarn.lock]
_update_hash_filename_ = yarn.lock _update_hash_filename_ = yarn.lock
md5sum = 80e7ad91deea54cebcccef5a83fdb380 md5sum = 18e8302b2acff3721cad23d829e3df55
[python-language-server-requirements.txt] [python-language-server-requirements.txt]
_update_hash_filename_ = python-language-server-requirements.txt _update_hash_filename_ = python-language-server-requirements.txt
......
...@@ -13,6 +13,7 @@ urls = vscode-bat https://open-vsx.org/api/vscode/bat/1.54.3/file/vscode.bat-1.5 ...@@ -13,6 +13,7 @@ urls = vscode-bat https://open-vsx.org/api/vscode/bat/1.54.3/file/vscode.bat-1.5
vscode-docker https://open-vsx.org/api/vscode/docker/1.54.3/file/vscode.docker-1.54.3.vsix a4e3cdd03833fe6bf3f5045fc5be4433 vscode-docker https://open-vsx.org/api/vscode/docker/1.54.3/file/vscode.docker-1.54.3.vsix a4e3cdd03833fe6bf3f5045fc5be4433
vscode-emmet https://open-vsx.org/api/vscode/emmet/1.54.3/file/vscode.emmet-1.54.3.vsix 5786c95921794ed3188083a7f804fa29 vscode-emmet https://open-vsx.org/api/vscode/emmet/1.54.3/file/vscode.emmet-1.54.3.vsix 5786c95921794ed3188083a7f804fa29
vscode-fsharp https://open-vsx.org/api/vscode/fsharp/1.54.3/file/vscode.fsharp-1.54.3.vsix 028f4ec9593533ce38603032b3f2821e vscode-fsharp https://open-vsx.org/api/vscode/fsharp/1.54.3/file/vscode.fsharp-1.54.3.vsix 028f4ec9593533ce38603032b3f2821e
vscode-git https://open-vsx.org/api/vscode/git/1.54.3/file/vscode.git-1.54.3.vsix da2f2c0626b4a254a660ae20b0c97611
vscode-go https://open-vsx.org/api/vscode/go/1.54.3/file/vscode.go-1.54.3.vsix 9f095a75e4137079351150722d091256 vscode-go https://open-vsx.org/api/vscode/go/1.54.3/file/vscode.go-1.54.3.vsix 9f095a75e4137079351150722d091256
vscode-groovy https://open-vsx.org/api/vscode/groovy/1.54.3/file/vscode.groovy-1.54.3.vsix 7ae14c40786311fda29059178f30c310 vscode-groovy https://open-vsx.org/api/vscode/groovy/1.54.3/file/vscode.groovy-1.54.3.vsix 7ae14c40786311fda29059178f30c310
vscode-grunt https://open-vsx.org/api/vscode/grunt/1.54.3/file/vscode.grunt-1.54.3.vsix f7894259f1fa60e939ff2d9b49138a02 vscode-grunt https://open-vsx.org/api/vscode/grunt/1.54.3/file/vscode.grunt-1.54.3.vsix f7894259f1fa60e939ff2d9b49138a02
......
...@@ -17,6 +17,7 @@ for plugin_and_version in '''\ ...@@ -17,6 +17,7 @@ for plugin_and_version in '''\
vscode/docker/latest vscode/docker/latest
vscode/emmet/latest vscode/emmet/latest
vscode/fsharp/latest vscode/fsharp/latest
vscode/git/latest
vscode/go/latest vscode/go/latest
vscode/groovy/latest vscode/groovy/latest
vscode/grunt/latest vscode/grunt/latest
......
{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) -%}
[buildout]
extends = {{ theia_instance_cfg }}
{{ pbsready_export_cfg }}
parts +=
monitor-base
$${:theia-parts}
$${:theia-environment-parts}
resilient-publish-connection-parameter
# The resilient stack makes the 'resilient' instance
# request the 'export' instance with a 'namebase' parameter.
# The export template then expects to receive it in
# slap-parameter:namebase
[slap-parameter]
namebase = {{ parameter_dict['namebase'] }}
# The resilient export stack periodically calls exporter:wrapper
# and then notifies the pull-backup instance that data is ready
# to be pulled from the export instance.
# All it expects is that a script be available in exporter:wrapper.
[exporter]
wrapper = $${theia-export-script:rendered}
[theia-export-script]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/theia-export-script
mode = 0700
exitcode-file = $${directory:srv}/export-exitcode-file
error-file = $${directory:srv}/export-errormessage-file
context =
raw python ${software-info:python-with-eggs}
raw theia_export ${software-info:theia-export}
raw bash ${software-info:bash}
raw rsync ${software-info:rsync}
raw sqlite3 ${software-info:sqlite3}
raw root_path $${buildout:directory}
raw backup_path $${directory:backup}
raw slapos_cfg $${directory:runner}/etc/slapos.cfg
raw project_path $${directory:project}
raw public_path $${directory:frontend-static-public}
key exitfile :exitcode-file
key errorfile :error-file
{%- raw %}
template =
inline:#!{{ bash }}
{{ python }} {{ theia_export }} \
--rsync {{ rsync }} \
--sqlite3 {{ sqlite3 }} \
--root {{ root_path }} \
--backup {{ backup_path }} \
--cfg {{ slapos_cfg }} \
--dirs {{ project_path }} \
--dirs {{ public_path }} \
--exitfile {{ exitfile }} \
--errorfile {{ errorfile }}
{%- endraw %}
# Add a promise to check that the export script has run
# successfully and recently (at most 2 days ago).
[promises]
export-promises =
$${export-promise:name}
[export-promise]
<= monitor-promise-base
module = check_command_execute
name = resiliency-export-promise.py
config-command = $${export-promise-script:rendered}
[initial-export-exitcode-file]
recipe = slapos.recipe.template:jinja2
rendered = $${theia-export-script:exitcode-file}
template = inline:0
once = $${:rendered}
[export-promise-script]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/export-promise-script
exitcode-file = $${initial-export-exitcode-file:rendered}
context =
key exitcodefile :exitcode-file
key errorfile theia-export-script:error-file
{%- raw %}
template =
inline:#!/bin/sh
if [ -z $(find {{ repr(exitcodefile) }} -mtime -2) ]
then
echo "ERROR export script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 1
elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
then
echo "ERROR export script failed on " $(date -r {{ repr(exitcodefile) }})
cat {{ repr(errorfile) }}
exit 1
else
echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 0
fi
{%- endraw %}
# Extend resilient parameters with normal theia connection parameters
[resilient-publish-connection-parameter]
<= publish-connection-parameter
{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) -%}
{%- set additional_frontend = parameter_dict['additional-frontend-guid'] -%}
[buildout]
extends = {{ theia_instance_cfg }}
{{ pbsready_import_cfg }}
parts +=
monitor-base
$${:theia-parts}
$${:theia-environment-parts}
# The resilient stack makes the 'resilient' instance
# request the 'import' instance with a 'namebase' parameter.
# The import template then expects to receive it in
# slap-parameter:namebase
[slap-parameter]
namebase = {{ parameter_dict['namebase'] }}
# Change frontend name to avoid conflicts
[remote-frontend]
name = Import {{ parameter_dict['frontend-name'] }}
{% if additional_frontend -%}
[remote-additional-frontend]
name = Import {{ parameter_dict['additional-frontend-name'] }}
{%- endif %}
# Change port ranges to avoid race conditions on port allocation
[frontend-instance-port]
minimum = 3200
maximum = 3300
[theia-service-port]
minimum = 3700
maximum = 3800
[slapos-standalone-port]
minimum = 4200
maximum = 4300
# Always disable autoprocessing in the import instance
[slapos-autorun]
autorun = stopped
# Change the gravatar favicon seed
[favicon.ico]
seed = Import {{ root_title }}
# The resilient stack calls post-notification-run:output followed by
# importer:wrapper when the instance is notified that the backup files
# have just been pushed to it. All it expects is the path of a script
# in post-notification-run:output and in importer:wrapper.
[post-notification-run]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/post-notification-run-script
output = $${:rendered}
mode = 0700
template =
inline:#!${software-info:bash}
# Do nothing because the backup signature will
# be verified by the import script itself
exit 0
[importer]
wrapper = $${theia-import-script:rendered}
[theia-import-script]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/theia-import-script
mode = 0700
exitcode-file = $${directory:srv}/import-exitcode-file
error-file = $${directory:srv}/import-errormessage-file
context =
raw python ${software-info:python-with-eggs}
raw theia_import ${software-info:theia-import}
raw bash ${software-info:bash}
raw rsync ${software-info:rsync}
raw sqlite3 ${software-info:sqlite3}
raw slapos ${software-info:slapos}
raw slapos_node_software_log $${directory:runner}/var/log/slapos-node-software.log
raw slapos_node_instance_log $${directory:runner}/var/log/slapos-node-instance.log
raw supervisorctl ${software-info:supervisorctl}
raw supervisord_conf $${directory:runner}/etc/supervisord.conf
raw root_path $${buildout:directory}
raw backup_path $${directory:backup}
raw slapos_cfg $${directory:runner}/etc/slapos.cfg
raw project_path $${directory:project}
raw public_path $${directory:frontend-static-public}
key exitfile :exitcode-file
key errorfile :error-file
{%- raw %}
template =
inline:#!{{ bash }}
. $${common-environment:rendered}
. $${slapos-standalone-activate:rendered}
{{ python }} {{ theia_import }} \
--rsync {{ rsync }} \
--sqlite3 {{ sqlite3 }} \
--slapos {{ slapos }} \
--srlog {{ slapos_node_software_log }} \
--cplog {{ slapos_node_instance_log }} \
--supervisorctl {{ supervisorctl }} \
--supervisordconf {{ supervisord_conf }} \
--root {{ root_path }} \
--backup {{ backup_path }} \
--cfg {{ slapos_cfg }} \
--dirs {{ project_path }} \
--dirs {{ public_path }} \
--exitfile {{ exitfile }} \
--errorfile {{ errorfile }}
{%- endraw %}
# Add a promise to check that the import script has run
# successfully and recently (at most 2 days ago).
[promises]
import-promises =
$${import-promise:name}
[import-promise]
<= monitor-promise-base
module = check_command_execute
name = resiliency-import-promise.py
config-command = $${import-promise-script:rendered}
[initial-import-exitcode-file]
recipe = slapos.recipe.template:jinja2
rendered = $${theia-import-script:exitcode-file}
template = inline:0
once = $${:rendered}
[import-promise-script]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/import-promise-script
exitcode-file = $${initial-import-exitcode-file:rendered}
context =
key exitcodefile :exitcode-file
key errorfile theia-import-script:error-file
{%- raw %}
template =
inline:#!/bin/sh
if [ -z $(find {{ repr(exitcodefile) }} -mtime -2) ]
then
echo "ERROR import script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 1
elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
then
echo "ERROR import script failed on " $(date -r {{ repr(exitcodefile) }})
cat {{ repr(errorfile) }}
exit 1
else
echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 0
fi
{%- endraw %}
# Resilient connection parameters of import instance are published
# through the resilient stack.
# Extend resilient parameters with normal theia connection parameters
[resilient-publish-connection-parameter]
<= publish-connection-parameter
{
"$schema": "http://json-schema.org/draft-04/schema",
"type": "object",
"description": "Parameters to instantiate resilient Theia",
"allOf": [
{
"$ref": "instance-input-schema.json#/"
},
{
"properties": {
"resilient-clone-number": {
"title": "Amount of backup(s) to create",
"description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
"type": "integer",
"default": 1,
"minimum": 0,
"maximum": 2,
"optional": true
},
"-sla-theia0-computer_guid": {
"title": "Target computer for main instance",
"description": "Target computer GUID for main instance.",
"type": "string",
"optional": true
},
"-sla-theia1-computer_guid": {
"title": "Target computer for first clone",
"description": "Target computer for first clone instance.",
"type": "string",
"optional": true
},
"-sla-pbs1-computer_guid": {
"title": "Target computer for first PBS",
"description": "Target computer for first PBS instance.",
"type": "string",
"optional": true
},
"-sla-theia2-computer_guid": {
"title": "Target computer for second clone",
"description": "Target computer for second clone instance.",
"type": "string",
"optional": true
},
"-sla-pbs2-computer_guid": {
"title": "Target computer for second PBS",
"description": "Target computer for second PBS instance.",
"type": "string",
"optional": true
},
"resiliency-backup-periodicity": {
"title": "Periodicity of backup",
"description": "Periodicity of backup, in cron format.",
"type": "string",
"optional": true
},
"remove-backup-older-than": {
"title": "Remove backups older than...",
"description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
"type": "string",
"default": "2W",
"optional": true
},
"ignore-known-hosts-file": {
"title": "Ignore known_hosts file",
"description": "Set either to fill known_hosts file for ssh or not. Useful if main instance and PBS are using the same IP (slapos proxy, theia).",
"type": "boolean",
"default": false,
"optional": true
}
}
}
]
}
{% import 'parts' as parts %}
{% import 'replicated' as replicated with context %}
{% set number_of_instances = slapparameter_dict.get('resilient-clone-number', 1)|int %}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
extends =
{{ monitor_template }}
parts +=
# Generate the parts to request theia-export, pull-backup and theia-import
# See stack/resilient/template-parts.cfg.in and stack/resilient/template-replicated.cfg.in
# See below for the generation of the sections corresponding to the parts generated here
{{ parts.replicate("theia", number_of_instances + 1) }}
# Also publish some connection parameters
publish-connection-parameter
[ArgLeader]
[ArgBackup]
# Generate sections to request theia-export, pull-backup and theia-import
# See stack/resilient/template-replicated.cfg.in
# In particular:
#
# [request-theia]
# <= ArgLeader
# software-type = export
# ...
#
# [request-theia-pseudo-replicating-1]
# <= ArgBackup
# software-type = import
# ...
#
# [request-pbs-theia-1]
# software-type = pull-backup
# ...
#
{{ replicated.replicate("theia", number_of_instances + 1,
"export", "import",
"ArgLeader", "ArgBackup",
slapparameter_dict=slapparameter_dict) }}
# Extend the list of return parameters for the export request
# The monitor parameters are only there to assert they are
# actually published by the export instance
[request-theia]
return += url username password backend-url monitor-base-url monitor-setup-url
# Extend the list of return parameters for the import request
# with the monitor parameters to assert they are actually published
[request-theia-pseudo-replicating-1]
return += monitor-base-url monitor-setup-url
# Publish some parameters from the export instance
[publish-connection-parameter]
recipe = slapos.cookbook:publish
url = ${request-theia:connection-url}
username = ${request-theia:connection-username}
password = ${request-theia:connection-password}
backend-url = ${request-theia:connection-backend-url}
# Publish resiliency parameters fetched by the resilient stack
[publish-connection-parameter]
<= publish-connection-information
...@@ -41,23 +41,26 @@ backend-url = $${frontend-instance:url} ...@@ -41,23 +41,26 @@ backend-url = $${frontend-instance:url}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc home = $${buildout:directory}
var = $${buildout:directory}/var etc = $${:home}/etc
srv = $${buildout:directory}/srv var = $${:home}/var
bin = $${buildout:directory}/bin srv = $${:home}/srv
tmp = $${buildout:directory}/tmp bin = $${:home}/bin
dot-theia = $${buildout:directory}/.theia/ tmp = $${:home}/tmp
dot-theia = $${:home}/.theia/
pidfiles = $${:var}/run pidfiles = $${:var}/run
services = $${:etc}/service services = $${:etc}/service
runner = $${:srv}/runner runner = $${:srv}/runner
backup = $${:srv}/backup/theia
project = $${:srv}/project project = $${:srv}/project
frontend-static = $${:srv}/frontend-static frontend-static = $${:srv}/frontend-static
frontend-static-public = $${:frontend-static}/public frontend-static-public = $${:frontend-static}/public
frontend-static-css = $${:frontend-static}/css frontend-static-css = $${:frontend-static}/css
bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/ bash-completions = $${:home}/.local/share/bash-completion/completions/
fish-completions = $${buildout:directory}/.config/fish/completions/ fish-completions = $${:home}/.config/fish/completions/
# Promises # Promises
...@@ -68,11 +71,13 @@ recipe = ...@@ -68,11 +71,13 @@ recipe =
instance-promises = instance-promises =
$${theia-listen-promise:name} $${theia-listen-promise:name}
$${frontend-listen-promise:name} $${frontend-listen-promise:name}
$${frontend-authentification-promise:name}
$${remote-frontend-url-available-promise:name} $${remote-frontend-url-available-promise:name}
{% if additional_frontend %} {% if additional_frontend %}
$${remote-additional-frontend-url-available-promise:name} $${remote-additional-frontend-url-available-promise:name}
{% endif %} {% endif %}
$${slapos-standalone-listen-promise:name} $${slapos-standalone-listen-promise:name}
$${slapos-standalone-ready-promise:name}
$${slapos-autorun-promise:name} $${slapos-autorun-promise:name}
[theia-listen-promise] [theia-listen-promise]
...@@ -89,6 +94,16 @@ name = $${:_buildout_section_name_}.py ...@@ -89,6 +94,16 @@ name = $${:_buildout_section_name_}.py
config-host = $${frontend-instance:ip} config-host = $${frontend-instance:ip}
config-port = $${frontend-instance:port} config-port = $${frontend-instance:port}
[frontend-authentification-promise]
<= monitor-promise-base
module = check_url_available
name = $${:_buildout_section_name_}.py
username = $${frontend-instance-password:username}
password = $${frontend-instance-password:passwd}
ip = $${frontend-instance:ip}
port = $${frontend-instance:port}
config-url = https://$${:username}:$${:password}@[$${:ip}]:$${:port}
[remote-frontend-url-available-promise] [remote-frontend-url-available-promise]
<= monitor-promise-base <= monitor-promise-base
module = check_url_available module = check_url_available
...@@ -113,13 +128,19 @@ name = standalone-listen-promise.py ...@@ -113,13 +128,19 @@ name = standalone-listen-promise.py
config-host = $${slapos-standalone-instance:hostname} config-host = $${slapos-standalone-instance:hostname}
config-port = $${slapos-standalone-instance:port} config-port = $${slapos-standalone-instance:port}
[slapos-standalone-ready-promise]
<= monitor-promise-base
module = check_socket_listening
name = standalone-ready-promise.py
config-abstract = $${directory:runner}/standalone_ready
[slapos-autorun-promise] [slapos-autorun-promise]
<= monitor-promise-base <= monitor-promise-base
module = check_service_state module = check_service_state
# XXX promise plugins can not contain "slapos" in their names
name = autorun-state-promise.py name = autorun-state-promise.py
config-service = $${slapos-autorun:service-name} config-service = $${slapos-autorun:service-name}
config-expect = $${slapos-autorun:autorun} config-expect = $${slapos-autorun:autorun}
config-run-directory = $${directory:runner}/var/run
# Remote Caddy Frontend # Remote Caddy Frontend
...@@ -279,13 +300,14 @@ wait-for-files = $${frontend-instance:pidfile} ...@@ -279,13 +300,14 @@ wait-for-files = $${frontend-instance:pidfile}
[favicon.ico] [favicon.ico]
# generate a pseudo random favicon, different for each instance name. # generate a pseudo random favicon, different for each instance name.
recipe = slapos.recipe.build recipe = slapos.recipe.build
seed = {{ root_title }}
install = install =
import hashlib, shutil import hashlib, shutil
buildout_offline = self.buildout['buildout']['offline'] buildout_offline = self.buildout['buildout']['offline']
self.buildout['buildout']['offline'] = 'false' self.buildout['buildout']['offline'] = 'false'
try: try:
gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5( gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5(
b'''{{ root_title }}''' b'''$${:seed}'''
).hexdigest() + "?s=256&d=retro" ).hexdigest() + "?s=256&d=retro"
shutil.copy(self.download(gravatar_url), '''$${:location}''') shutil.copy(self.download(gravatar_url), '''$${:location}''')
except Exception: except Exception:
...@@ -310,7 +332,7 @@ mode = 0700 ...@@ -310,7 +332,7 @@ mode = 0700
template = template =
inline: inline:
#!/bin/sh #!/bin/sh
export HOME=$${buildout:directory} export HOME=$${directory:home}
export PATH=${python-language-server:location}/bin:${java-jdk:location}/bin:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH export PATH=${python-language-server:location}/bin:${java-jdk:location}/bin:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
...@@ -375,9 +397,6 @@ template = ...@@ -375,9 +397,6 @@ template =
. {{ activate }} . {{ activate }}
unset GIT_EXEC_PATH unset GIT_EXEC_PATH
set -- --rcfile {{ bashrc }} set -- --rcfile {{ bashrc }}
# otherwise, assume this shell is running task and add an artificial delay to workaround https://github.com/eclipse-theia/theia/issues/2961
else
sleep 1
fi fi
exec "$SHELL" "$@" exec "$SHELL" "$@"
{% endraw %} {% endraw %}
...@@ -419,6 +438,7 @@ ip = {{ ipv4_random }} ...@@ -419,6 +438,7 @@ ip = {{ ipv4_random }}
ipv4 = {{ ipv4_random }} ipv4 = {{ ipv4_random }}
ipv6 = {{ ipv6_random }} ipv6 = {{ ipv6_random }}
port = $${slapos-standalone-port:port} port = $${slapos-standalone-port:port}
local-software-release-root = $${directory:home}
slapos-configuration = $${directory:runner}/etc/slapos.cfg slapos-configuration = $${directory:runner}/etc/slapos.cfg
computer-id = slaprunner computer-id = slaprunner
...@@ -447,6 +467,7 @@ template = ...@@ -447,6 +467,7 @@ template =
$${slapos-standalone-config:ipv4} \ $${slapos-standalone-config:ipv4} \
$${slapos-standalone-config:ipv6} \ $${slapos-standalone-config:ipv6} \
$${slapos-standalone-config:port} \ $${slapos-standalone-config:port} \
$${slapos-standalone-config:local-software-release-root} \
$${slapos-standalone-config:computer-id} \ $${slapos-standalone-config:computer-id} \
{%- if parameter_dict.get('embedded-sr') %} {%- if parameter_dict.get('embedded-sr') %}
--sr='{{ parameter_dict['embedded-sr'] }}' \ --sr='{{ parameter_dict['embedded-sr'] }}' \
......
...@@ -2,6 +2,9 @@ ...@@ -2,6 +2,9 @@
parts = parts =
switch-softwaretype switch-softwaretype
extends =
${template-resilient-templates:output}
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
...@@ -18,6 +21,11 @@ recipe = slapos.cookbook:switch-softwaretype ...@@ -18,6 +21,11 @@ recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:default} RootSoftwareInstance = $${:default}
default = $${:theia} default = $${:theia}
theia = theia:rendered theia = theia:rendered
export = export:rendered
import = import:rendered
resilient = resilient:rendered
frozen = instance-frozen:rendered
pull-backup = template-pull-backup:rendered
[theia] [theia]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -46,3 +54,43 @@ default-parameters = ...@@ -46,3 +54,43 @@ default-parameters =
"additional-frontend-guid": null "additional-frontend-guid": null
} }
frontend-sr = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg frontend-sr = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
[import]
recipe = slapos.recipe.template:jinja2
template = ${instance-import:output}
rendered = $${buildout:directory}/instance-import.cfg
mode = 0644
context =
jsonkey default_parameter_dict theia:default-parameters
key parameter_dict slap-configuration:configuration
key theia_instance_cfg theia:rendered
key pbsready_import_cfg template-pbsready-import:rendered
key root_title slap-configuration:root-instance-title
[export]
recipe = slapos.recipe.template:jinja2
template = ${instance-export:output}
rendered = $${buildout:directory}/instance-export.cfg
mode = 0644
context =
jsonkey default_parameter_dict theia:default-parameters
key parameter_dict slap-configuration:configuration
key theia_instance_cfg theia:rendered
key pbsready_export_cfg template-pbsready-export:rendered
[resilient]
recipe = slapos.recipe.template:jinja2
template = ${instance-resilient:output}
rendered = $${buildout:directory}/instance-resilient.cfg
mode = 0644
extensions = jinja2.ext.do
context =
key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
raw monitor_template ${monitor-template:rendered}
template-parts-destination = ${template-parts:target}
template-replicated-destination = ${template-replicated:target}
import-list = file parts :template-parts-destination
file replicated :template-replicated-destination
...@@ -12,9 +12,12 @@ extends = ...@@ -12,9 +12,12 @@ extends =
../../component/coreutils/buildout.cfg ../../component/coreutils/buildout.cfg
../../component/java-jdk/buildout.cfg ../../component/java-jdk/buildout.cfg
../../component/fonts/buildout.cfg ../../component/fonts/buildout.cfg
../../component/libsecret/buildout.cfg
../../component/pkgconfig/buildout.cfg
../../stack/nodejs.cfg ../../stack/nodejs.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
../../stack/resilient/buildout.cfg
../../component/defaults.cfg ../../component/defaults.cfg
./download-plugins.cfg ./download-plugins.cfg
./buildout.hash.cfg ./buildout.hash.cfg
...@@ -22,8 +25,14 @@ extends = ...@@ -22,8 +25,14 @@ extends =
parts = parts =
theia-wrapper theia-wrapper
slapos-cookbook slapos-cookbook
python-with-eggs
instance-theia instance-theia
instance instance
instance-import
instance-export
instance-resilient
theia-common
theia-export
# default for slapos-standalone # default for slapos-standalone
shared-part-list = shared-part-list =
...@@ -54,9 +63,10 @@ initialization = ...@@ -54,9 +63,10 @@ initialization =
import argparse import argparse
import glob import glob
import json import json
import os.path import os
import sys
import signal import signal
import socket
import sys
import time import time
import slapos.slap.standalone import slapos.slap.standalone
...@@ -66,6 +76,7 @@ initialization = ...@@ -66,6 +76,7 @@ initialization =
parser.add_argument('ipv4') parser.add_argument('ipv4')
parser.add_argument('ipv6') parser.add_argument('ipv6')
parser.add_argument('server_port', type=int) parser.add_argument('server_port', type=int)
parser.add_argument('local_software_release_root')
parser.add_argument('computer_id') parser.add_argument('computer_id')
parser.add_argument('--sr') parser.add_argument('--sr')
parser.add_argument('--srtype') parser.add_argument('--srtype')
...@@ -101,8 +112,16 @@ initialization = ...@@ -101,8 +112,16 @@ initialization =
instance_root="%s/instance" % args.base_directory, instance_root="%s/instance" % args.base_directory,
partition_forward_configuration=partition_forward_configuration, partition_forward_configuration=partition_forward_configuration,
slapos_bin="${buildout:bin-directory}/slapos", slapos_bin="${buildout:bin-directory}/slapos",
local_software_release_root=args.local_software_release_root,
) )
def signal_handler(signum, frame):
print("Signal {signum} received".format(signum=signum))
sys.exit()
signal.signal(signal.SIGTERM, signal_handler)
standalone.start() standalone.start()
try:
partition_count = 20 partition_count = 20
if len(glob.glob(os.path.join(standalone.instance_directory, '*'))) < partition_count: if len(glob.glob(os.path.join(standalone.instance_directory, '*'))) < partition_count:
print("Standalone SlapOS: Formatting {partition_count} partitions".format( print("Standalone SlapOS: Formatting {partition_count} partitions".format(
...@@ -110,7 +129,7 @@ initialization = ...@@ -110,7 +129,7 @@ initialization =
standalone.format( standalone.format(
partition_count, partition_count,
args.ipv4, args.ipv4,
args.ipv6 args.ipv6,
) )
print("Standalone SlapOS for computer `{}` started".format(args.computer_id)) print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
# Run instance at least once, to start the supervisor managing instances. # Run instance at least once, to start the supervisor managing instances.
...@@ -139,20 +158,18 @@ initialization = ...@@ -139,20 +158,18 @@ initialization =
partition_parameter_kw=params, partition_parameter_kw=params,
) )
quit_requested = [] s = socket.socket(socket.AF_UNIX)
def signal_handler(signum, frame): s.bind('\0' + os.path.join(args.base_directory, 'standalone_ready'))
print("Signal {signum} received".format(signum=signum)) s.listen(5)
quit_requested.append(True)
signal.signal(signal.SIGTERM, signal_handler)
print("Standalone SlapOS ready") print("Standalone SlapOS ready")
while not quit_requested: while True:
time.sleep(.1) s.accept()[0].close()
finally:
print("Stopping standalone subsystem") print("Stopping standalone subsystem")
standalone.stop() standalone.stop()
print("Exiting") print("Exiting")
sys.exit(0)
needs-these-eggs-scripts-in-path = needs-these-eggs-scripts-in-path =
${supervisor:recipe} ${supervisor:recipe}
...@@ -191,7 +208,11 @@ stop-on-error = true ...@@ -191,7 +208,11 @@ stop-on-error = true
[theia] [theia]
recipe = plone.recipe.command recipe = plone.recipe.command
command = ${bash:location}/bin/bash -c " command = ${bash:location}/bin/bash -c "
export TMPDIR=${:location}/tmp PATH=${nodejs:location}/bin:$PATH && export \
TMPDIR=${:location}/tmp \
PATH=${nodejs:location}/bin:${pkgconfig:location}/bin:$PATH \
PKG_CONFIG_PATH=${libsecret:pkg-config-path} \
LDFLAGS='-Wl,-rpath=${libsecret:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${glib:location}/lib' && \
mkdir -p ${:location} && \ mkdir -p ${:location} && \
mkdir -p \$TMPDIR && \ mkdir -p \$TMPDIR && \
cd ${:location} && \ cd ${:location} && \
...@@ -233,6 +254,11 @@ template = ...@@ -233,6 +254,11 @@ template =
inline:{ inline:{
"private": true, "private": true,
"theia": { "theia": {
"backend": {
"config": {
"warnOnPotentiallyInsecureHostPattern": false
}
},
"frontend": { "frontend": {
"config": { "config": {
"applicationName": "Theia SlapOS", "applicationName": "Theia SlapOS",
...@@ -262,7 +288,8 @@ template = ...@@ -262,7 +288,8 @@ template =
"plantuml.render": "PlantUMLServer", "plantuml.render": "PlantUMLServer",
"gitlens.remotes": [{ "domain": "lab.nexedi.com", "type": "GitLab" }], "gitlens.remotes": [{ "domain": "lab.nexedi.com", "type": "GitLab" }],
"java.home": "${java-jdk:location}" "java.home": "${java-jdk:location}"
} },
"warnOnPotentiallyInsecureHostPattern": false
} }
}, },
"generator": { "generator": {
...@@ -272,17 +299,16 @@ template = ...@@ -272,17 +299,16 @@ template =
} }
}, },
"dependencies": { "dependencies": {
"@theia/bulk-edit": "latest",
"@theia/callhierarchy": "latest", "@theia/callhierarchy": "latest",
"@theia/console": "latest", "@theia/console": "latest",
"@theia/core": "latest", "@theia/core": "latest",
"@theia/cpp-debug": "latest",
"@theia/debug": "latest", "@theia/debug": "latest",
"@theia/editor": "latest", "@theia/editor": "latest",
"@theia/editor-preview": "latest", "@theia/editor-preview": "latest",
"@theia/file-search": "latest", "@theia/file-search": "latest",
"@theia/filesystem": "latest", "@theia/filesystem": "latest",
"@theia/getting-started": "latest", "@theia/getting-started": "latest",
"@theia/git": "latest",
"@theia/keymaps": "latest", "@theia/keymaps": "latest",
"@theia/markers": "latest", "@theia/markers": "latest",
"@theia/messages": "latest", "@theia/messages": "latest",
...@@ -292,16 +318,19 @@ template = ...@@ -292,16 +318,19 @@ template =
"@theia/navigator": "latest", "@theia/navigator": "latest",
"@theia/outline-view": "latest", "@theia/outline-view": "latest",
"@theia/output": "latest", "@theia/output": "latest",
"@theia/plugin": "latest", "@theia/plugin-dev": "latest",
"@theia/plugin-ext": "latest", "@theia/plugin-ext": "latest",
"@theia/plugin-ext-vscode": "latest", "@theia/plugin-ext-vscode": "latest",
"@theia/preferences": "latest", "@theia/preferences": "latest",
"@theia/preview": "latest", "@theia/preview": "latest",
"@theia/process": "latest", "@theia/process": "latest",
"@theia/property-view": "latest",
"@theia/scm": "latest", "@theia/scm": "latest",
"@theia/scm-extra": "latest",
"@theia/search-in-workspace": "latest", "@theia/search-in-workspace": "latest",
"@theia/task": "latest", "@theia/task": "latest",
"@theia/terminal": "latest", "@theia/terminal": "latest",
"@theia/timeline": "latest",
"@theia/typehierarchy": "latest", "@theia/typehierarchy": "latest",
"@theia/userstorage": "latest", "@theia/userstorage": "latest",
"@theia/variable-resolver": "latest", "@theia/variable-resolver": "latest",
...@@ -343,6 +372,18 @@ template = ...@@ -343,6 +372,18 @@ template =
#!/bin/sh #!/bin/sh
exec ${nodejs:location}/bin/node ${theia:location}/node_modules/.bin/theia-open "$@" exec ${nodejs:location}/bin/node ${theia:location}/node_modules/.bin/theia-open "$@"
[python-with-eggs]
recipe = zc.recipe.egg
interpreter = ${:_buildout_section_name_}
eggs =
${slapos-toolbox:eggs}
six
zc.buildout
# Only generate the interpreter script to avoid conflicts with scripts
# for eggs that are also generated by another section, like slapos.toolbox
scripts = ${:interpreter}
[instance-theia] [instance-theia]
<= template-base <= template-base
output = ${buildout:directory}/instance-theia.cfg.jinja output = ${buildout:directory}/instance-theia.cfg.jinja
...@@ -350,3 +391,37 @@ output = ${buildout:directory}/instance-theia.cfg.jinja ...@@ -350,3 +391,37 @@ output = ${buildout:directory}/instance-theia.cfg.jinja
[instance] [instance]
<= template-base <= template-base
output = ${buildout:directory}/instance.cfg output = ${buildout:directory}/instance.cfg
[instance-import]
<= template-base
output = ${buildout:directory}/instance-import.cfg.jinja
[instance-export]
<= template-base
output = ${buildout:directory}/instance-export.cfg.jinja
[instance-resilient]
<= download-base
[theia-common]
<= download-base
destination = ${buildout:directory}/theia_common.py
[theia-export]
<= download-base
destination = ${buildout:directory}/theia_export.py
[theia-import]
<= download-base
destination = ${buildout:directory}/theia_import.py
[software-info]
slapos = ${buildout:bin-directory}/slapos
python-with-eggs = ${buildout:bin-directory}/${python-with-eggs:interpreter}
python = ${python:location}/bin/python
rsync = ${rsync:location}/bin/rsync
sqlite3 = ${sqlite3:location}/bin/sqlite3
bash = ${bash:location}/bin/bash
supervisorctl = ${buildout:bin-directory}/supervisorctl
theia-export = ${theia-export:output}
theia-import = ${theia-import:output}
...@@ -9,6 +9,13 @@ ...@@ -9,6 +9,13 @@
"description": "Default", "description": "Default",
"request": "instance-input-schema.json", "request": "instance-input-schema.json",
"response": "instance-output-schema.json", "response": "instance-output-schema.json",
"index": 0
},
"resilient": {
"title": "Resilient",
"description": "Resilient Theia",
"request": "instance-resilient-input-schema.json",
"response": "instance-output-schema.json",
"index": 1 "index": 1
} }
} }
......
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import unicode_literals
import gzip
import json
import os
import re
import subprocess
import time
import unittest
import requests
from datetime import datetime, timedelta
from six.moves.urllib.parse import urljoin
from slapos.testing.testcase import installSoftwareUrlList
import test_resiliency
from test import SlapOSInstanceTestCase, theia_software_release_url
erp5_software_release_url = os.path.abspath(
os.path.join(
os.path.dirname(__file__), '..', '..', 'erp5', 'software.cfg'))
def setUpModule():
installSoftwareUrlList(
SlapOSInstanceTestCase,
[theia_software_release_url, erp5_software_release_url],
debug=bool(int(os.environ.get('SLAPOS_TEST_DEBUG', 0))),
)
class ERP5Mixin(object):
_test_software_url = erp5_software_release_url
_connexion_parameters_regex = re.compile(r"{\s*'_'\s*:\s*'(.*)'\s*}")
def _getERP5ConnexionParameters(self, software_type='export'):
slapos = self._getSlapos(software_type)
out = subprocess.check_output(
(slapos, 'request', 'test_instance', self._test_software_url),
stderr=subprocess.STDOUT,
)
print(out)
return json.loads(self._connexion_parameters_regex.search(out).group(1))
def _getERP5Url(self, connexion_parameters, path=''):
return urljoin(connexion_parameters['family-default-v6'], path)
def _getERP5User(self, connexion_parameters):
return connexion_parameters['inituser-login']
def _getERP5Password(self, connexion_parameters):
return connexion_parameters['inituser-password']
def _waitERP5connected(self, url, user, password):
for _ in range(5):
try:
resp = requests.get('%s/getId' % url, auth=(user, password), verify=False, allow_redirects=False)
except Exception:
time.sleep(20)
continue
if resp.status_code != 200:
time.sleep(20)
continue
break
else:
self.fail('Failed to connect to ERP5')
self.assertEqual(resp.text, 'erp5')
def _getERP5Partition(self, servicename):
p = subprocess.Popen(
(self._getSlapos(), 'node', 'status'),
stdout=subprocess.PIPE, universal_newlines=True)
out, _ = p.communicate()
found = set()
for line in out.splitlines():
if servicename in line:
found.add(line.split(':')[0])
if not found:
raise Exception("ERP5 %s partition not found" % servicename)
elif len(found) > 1:
raise Exception("Found several partitions for ERP5 %s" % servicename)
return found.pop()
def _getERP5PartitionPath(self, software_type, servicename, *paths):
partition = self._getERP5Partition(servicename)
return self._getPartitionPath(
software_type, 'srv', 'runner', 'instance', partition, *paths)
class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
test_instance_max_retries = 12
backup_max_tries = 480
backup_wait_interval = 60
def _prepareExport(self):
super(TestTheiaResilienceERP5, self)._prepareExport()
# Connect to erp5
info = self._getERP5ConnexionParameters()
user = self._getERP5User(info)
password = self._getERP5Password(info)
url = self._getERP5Url(info, 'erp5')
self._waitERP5connected(url, user, password)
# Change title
new_title = time.strftime("HelloTitle %a %d %b %Y %H:%M:%S", time.localtime(time.time()))
requests.get('%s/portal_types/setTitle?value=%s' % (url, new_title), auth=(user, password), verify=False)
resp = requests.get('%s/portal_types/getTitle' % url, auth=(user, password), verify=False, allow_redirects=False)
self.assertEqual(resp.text, new_title)
self._erp5_new_title = new_title
# Wait until changes have been catalogued
mariadb_partition = self._getERP5PartitionPath('export', 'mariadb')
mysql_bin = os.path.join(mariadb_partition, 'bin', 'mysql')
wait_activities_script = os.path.join(
mariadb_partition, 'software_release', 'parts', 'erp5',
'Products', 'CMFActivity', 'bin', 'wait_activities')
subprocess.check_call((wait_activities_script, 'erp5'), env={'MYSQL': mysql_bin})
# Check that changes have been catalogued
output = subprocess.check_output((mysql_bin, 'erp5', '-e', 'SELECT title FROM catalog WHERE id="portal_types"'))
self.assertIn(new_title, output)
# Compute backup date in the near future
soon = (datetime.now() + timedelta(minutes=4)).replace(second=0)
date = '*:%d:00' % soon.minute
params = '_={"zodb-zeo": {"backup-periodicity": "%s"}, "mariadb": {"backup-periodicity": "%s"} }' % (date, date)
# Update ERP5 parameters
print('Requesting ERP5 with parameters %s' % params)
slapos = self._getSlapos()
subprocess.check_call((slapos, 'request', 'test_instance', self._test_software_url, '--parameters', params))
# Process twice to propagate parameter changes
for _ in range(2):
subprocess.check_call((slapos, 'node', 'instance'))
# Restart cron (actually all) services to let them take the new date into account
# XXX this should not be required, updating ERP5 parameters should be enough
subprocess.call((slapos, 'node', 'restart', 'all'))
# Wait until after the programmed backup date, and a bit more
t = (soon - datetime.now()).total_seconds()
self.assertLess(0, t)
time.sleep(t + 120)
# Check that mariadb backup has started
mariadb_backup = os.path.join(mariadb_partition, 'srv', 'backup', 'mariadb-full')
mariadb_backup_dump, = os.listdir(mariadb_backup)
# Check that zodb backup has started
zodb_backup = self._getERP5PartitionPath('export', 'zeo', 'srv', 'backup', 'zodb', 'root')
self.assertEqual(len(os.listdir(zodb_backup)), 3)
# Check that mariadb catalog backup contains expected changes
with gzip.open(os.path.join(mariadb_backup, mariadb_backup_dump)) as f:
self.assertIn(new_title, f.read(), "Mariadb catalog backup %s is not up to date" % mariadb_backup_dump)
def _checkTakeover(self):
super(TestTheiaResilienceERP5, self)._checkTakeover()
# Connect to erp5
info = self._getERP5ConnexionParameters()
user = self._getERP5User(info)
password = self._getERP5Password(info)
url = self._getERP5Url(info, 'erp5')
self._waitERP5connected(url, user, password)
resp = requests.get('%s/portal_types/getTitle' % url, auth=(user, password), verify=False, allow_redirects=False)
self.assertEqual(resp.text, self._erp5_new_title)
# Check that the mariadb catalog is not yet restored
mariadb_partition = self._getERP5PartitionPath('export', 'mariadb')
mysql_bin = os.path.join(mariadb_partition, 'bin', 'mysql')
query = 'SELECT title FROM catalog WHERE id="portal_types"'
try:
out = subprocess.check_output((mysql_bin, 'erp5', '-e', query))
except subprocess.CalledProcessError:
out = ''
self.assertNotIn(self._erp5_new_title, out)
# Stop all services
slapos = self._getSlapos()
print("Stop all services")
subprocess.call((slapos, 'node', 'stop', 'all'))
# Manually restore mariadb from backup
mariadb_restore_script = os.path.join(mariadb_partition, 'bin', 'restore-from-backup')
print("Restore mariadb from backup")
subprocess.check_call(mariadb_restore_script)
# Check that the test instance is properly redeployed after restoring mariadb
# This restarts the services and checks the promises of the test instance
# Process twice to propagate state change
for _ in range(2):
self._processEmbeddedInstance(self.test_instance_max_retries)
# Check that the mariadb catalog was properly restored
out = subprocess.check_output((mysql_bin, 'erp5', '-e', query))
self.assertIn(self._erp5_new_title, out, 'Mariadb catalog is not properly restored')
# Resilience Dummy Software
A very simple SR to test resiliency:
- fast installation and deployment
- self-contained - no dependency outside this folder
- has a simple `exporter.exclude` and `runner-import-restore`
[buildout]
parts =
log-writer
exporter.exclude
runner-import-restore
backup-identity-script
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[directory]
recipe = plone.recipe.command
home = $${buildout:directory}
srv = $${:home}/srv
etc = $${:home}/etc
run = $${:etc}/run
command = mkdir -p $${:run} $${:srv}
[log-writer]
recipe = slapos.recipe.template:jinja2
template = inline:#!/bin/sh
echo "Hello : $(date)" >> $${directory:home}/log.log
rendered = $${directory:run}/log-writer
[exporter.exclude]
recipe = slapos.recipe.template:jinja2
template = inline:$${directory:home}/exclude
rendered = $${directory:srv}/exporter.exclude
[runner-import-restore]
recipe = slapos.recipe.template:jinja2
template = inline:#!/bin/sh
echo "Hello : $(date)" >> $${directory:home}/runner-import-restore.log
exit $TEST_RESTORE_STATUS
rendered = $${directory:srv}/runner-import-restore
[backup-identity-script]
recipe = slapos.recipe.template:jinja2
template = inline:#!/bin/sh
echo "Custom script"
for i in "$@"
do
echo $(sha256sum $i)
done
exit $TEST_BACKUP_STATUS
rendered = $${directory:srv}/.backup_identity_script
[buildout]
find-links +=
http://www.nexedi.org/static/packages/source/
http://www.nexedi.org/static/packages/source/slapos.buildout/
parts =
instance-template
plone-recipe-command
versions = versions
[plone-recipe-command]
recipe = zc.recipe.egg
eggs = plone.recipe.command
[instance-template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg
mode = 0644
[versions]
setuptools = 44.0.0
zc.buildout = 2.7.1+slapos010
zc.recipe.egg = 2.0.3+slapos003
Jinja2 = 2.11.2
MarkupSafe = 1.0
...@@ -26,31 +26,28 @@ ...@@ -26,31 +26,28 @@
############################################################################## ##############################################################################
from __future__ import unicode_literals from __future__ import unicode_literals
import os import json
import textwrap
import logging import logging
import os
import re
import subprocess import subprocess
import tempfile
import time import time
import re
import json
from six.moves.urllib.parse import urlparse, urljoin
import pexpect import pexpect
import psutil import psutil
import requests import requests
import sqlite3
import six import six
from six.moves.urllib.parse import urlparse, urljoin
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.grid.svcbackend import getSupervisorRPC from slapos.grid.svcbackend import getSupervisorRPC, _getSupervisordSocketPath
from slapos.grid.svcbackend import _getSupervisordSocketPath
software_cfg = 'software%s.cfg' % ('-py3' if six.PY3 else '') software_cfg = 'software%s.cfg' % ('-py3' if six.PY3 else '')
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass( theia_software_release_url = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', software_cfg))
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', software_cfg))) setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(theia_software_release_url)
class TheiaTestCase(SlapOSInstanceTestCase): class TheiaTestCase(SlapOSInstanceTestCase):
...@@ -67,12 +64,21 @@ class TestTheia(TheiaTestCase): ...@@ -67,12 +64,21 @@ class TestTheia(TheiaTestCase):
def setUp(self): def setUp(self):
self.connection_parameters = self.computer_partition.getConnectionParameterDict() self.connection_parameters = self.computer_partition.getConnectionParameterDict()
def get(self, url, expect_code=requests.codes.ok):
resp = requests.get(url, verify=False)
self.assertEqual(
expect_code,
resp.status_code,
'%s returned %d instead of %d' % (url, resp.status_code, expect_code),
)
return resp
def test_backend_http_get(self): def test_backend_http_get(self):
resp = requests.get(self.connection_parameters['backend-url'], verify=False) backend_url = self.connection_parameters['backend-url']
self.assertEqual(requests.codes.unauthorized, resp.status_code) self.get(backend_url, requests.codes.unauthorized)
# with login/password, this is allowed # with login/password, this is allowed
parsed_url = urlparse(self.connection_parameters['backend-url']) parsed_url = urlparse(backend_url)
authenticated_url = parsed_url._replace( authenticated_url = parsed_url._replace(
netloc='{}:{}@[{}]:{}'.format( netloc='{}:{}@[{}]:{}'.format(
self.connection_parameters['username'], self.connection_parameters['username'],
...@@ -80,12 +86,11 @@ class TestTheia(TheiaTestCase): ...@@ -80,12 +86,11 @@ class TestTheia(TheiaTestCase):
parsed_url.hostname, parsed_url.hostname,
parsed_url.port, parsed_url.port,
)).geturl() )).geturl()
resp = requests.get(authenticated_url, verify=False) self.get(authenticated_url)
self.assertEqual(requests.codes.ok, resp.status_code)
def test_http_get(self): def test_http_get(self):
resp = requests.get(self.connection_parameters['url'], verify=False) url = self.connection_parameters['url']
self.assertEqual(requests.codes.unauthorized, resp.status_code) self.get(url, requests.codes.unauthorized)
# with login/password, this is allowed # with login/password, this is allowed
parsed_url = urlparse(self.connection_parameters['url']) parsed_url = urlparse(self.connection_parameters['url'])
...@@ -96,33 +101,28 @@ class TestTheia(TheiaTestCase): ...@@ -96,33 +101,28 @@ class TestTheia(TheiaTestCase):
parsed_url.hostname, parsed_url.hostname,
parsed_url.port, parsed_url.port,
)).geturl() )).geturl()
resp = requests.get(authenticated_url, verify=False) self.get(authenticated_url)
self.assertEqual(requests.codes.ok, resp.status_code)
# there's a public folder to serve file # there's a public folder to serve file
with open('{}/srv/frontend-static/public/test_file'.format( with open('{}/srv/frontend-static/public/test_file'.format(
self.computer_partition_root_path), 'w') as f: self.computer_partition_root_path), 'w') as f:
f.write("hello") f.write("hello")
resp = requests.get(urljoin(authenticated_url, '/public/'), verify=False) resp = self.get(urljoin(authenticated_url, '/public/'))
self.assertIn('test_file', resp.text) self.assertIn('test_file', resp.text)
resp = requests.get( resp = self.get(urljoin(authenticated_url, '/public/test_file'))
urljoin(authenticated_url, '/public/test_file'), verify=False)
self.assertEqual('hello', resp.text) self.assertEqual('hello', resp.text)
# there's a (not empty) favicon # there's a (not empty) favicon
resp = requests.get( resp = self.get(urljoin(authenticated_url, '/favicon.ico'))
urljoin(authenticated_url, '/favicon.ico'), verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
self.assertTrue(resp.raw) self.assertTrue(resp.raw)
# there is a CSS referencing fonts # there is a CSS referencing fonts
css_text = requests.get(urljoin(authenticated_url, '/css/slapos.css'), verify=False).text css_text = self.get(urljoin(authenticated_url, '/css/slapos.css')).text
css_urls = re.findall(r'url\([\'"]+([^\)]+)[\'"]+\)', css_text) css_urls = re.findall(r'url\([\'"]+([^\)]+)[\'"]+\)', css_text)
self.assertTrue(css_urls) self.assertTrue(css_urls)
# and fonts are served # and fonts are served
for url in css_urls: for url in css_urls:
resp = requests.get(urljoin(authenticated_url, url), verify=False) resp = self.get(urljoin(authenticated_url, url))
self.assertEqual(requests.codes.ok, resp.status_code)
self.assertTrue(resp.raw) self.assertTrue(resp.raw)
def test_theia_slapos(self): def test_theia_slapos(self):
...@@ -281,7 +281,7 @@ class TestTheiaEnv(TheiaTestCase): ...@@ -281,7 +281,7 @@ class TestTheiaEnv(TheiaTestCase):
} }
def test_theia_env(self): def test_theia_env(self):
"""Make sure environment variables are the same wether we use shell or supervisor services. """Make sure environment variables are the same whether we use shell or supervisor services.
""" """
# The path of the env.json file expected to be generated by building the dummy software release # The path of the env.json file expected to be generated by building the dummy software release
env_json_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'software', 'env.json') env_json_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'software', 'env.json')
...@@ -303,6 +303,7 @@ class TestTheiaEnv(TheiaTestCase): ...@@ -303,6 +303,7 @@ class TestTheiaEnv(TheiaTestCase):
# Start a theia shell that inherits the environment of the theia process # Start a theia shell that inherits the environment of the theia process
# This simulates the environment of a shell launched from the browser application # This simulates the environment of a shell launched from the browser application
theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.computer_partition_root_path), env=theia_env) theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.computer_partition_root_path), env=theia_env)
try:
theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated') theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
# Launch slapos node software from theia shell # Launch slapos node software from theia shell
...@@ -318,7 +319,7 @@ class TestTheiaEnv(TheiaTestCase): ...@@ -318,7 +319,7 @@ class TestTheiaEnv(TheiaTestCase):
os.remove(env_json_path) os.remove(env_json_path)
# Launch slapos node software service from the embedded supervisord. # Launch slapos node software service from the embedded supervisord.
# Note that we have two services, slapos-not-software and slapos-not-software-all # Note that we have two services, slapos-node-software and slapos-node-software-all
# The later uses --all which is what we want to use here, because the software # The later uses --all which is what we want to use here, because the software
# is already installed and we want to install it again, this time from supervisor # is already installed and we want to install it again, this time from supervisor
embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run') embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run')
...@@ -344,6 +345,62 @@ class TestTheiaEnv(TheiaTestCase): ...@@ -344,6 +345,62 @@ class TestTheiaEnv(TheiaTestCase):
self.assertEqual(theia_shell_env['SLAPOS_CLIENT_CONFIGURATION'], supervisord_env['SLAPOS_CLIENT_CONFIGURATION']) self.assertEqual(theia_shell_env['SLAPOS_CLIENT_CONFIGURATION'], supervisord_env['SLAPOS_CLIENT_CONFIGURATION'])
self.assertEqual(theia_shell_env['HOME'], supervisord_env['HOME']) self.assertEqual(theia_shell_env['HOME'], supervisord_env['HOME'])
finally:
# Cleanup the theia shell process # Cleanup the theia shell process
theia_shell_process.terminate() theia_shell_process.terminate()
theia_shell_process.wait() theia_shell_process.wait()
class ResilientTheiaMixin(object):
@classmethod
def setUpClass(cls):
super(ResilientTheiaMixin, cls).setUpClass()
# Add resiliency files to snapshot patterns
cls._save_instance_file_pattern_list += (
'*/srv/export-exitcode-file',
'*/srv/export-errormessage-file',
'*/srv/import-exitcode-file',
'*/srv/import-errormessage-file',
)
@classmethod
def _getPartition(cls, software_type):
software_url = cls.getSoftwareURL()
for computer_partition in cls.slap.computer.getComputerPartitionList():
partition_url = computer_partition.getSoftwareRelease()._software_release
partition_type = computer_partition.getType()
if partition_url == software_url and partition_type == software_type:
return computer_partition
raise Exception("Theia %s partition not found" % software_type)
@classmethod
def _getPartitionId(cls, software_type):
return cls._getPartition(software_type).getId()
@classmethod
def _getPartitionPath(cls, software_type, *paths):
return os.path.join(cls.slap._instance_root, cls._getPartitionId(software_type), *paths)
@classmethod
def _getSlapos(cls, software_type='export'):
return cls._getPartitionPath(software_type, 'srv', 'runner', 'bin', 'slapos')
@classmethod
def getInstanceSoftwareType(cls):
return 'resilient'
class TestTheiaResilientInterface(ResilientTheiaMixin, TestTheia):
@classmethod
def setUpClass(cls):
super(TestTheiaResilientInterface, cls).setUpClass()
# Patch the computer root path to that of the export theia instance
cls.computer_partition_root_path = cls._getPartitionPath('export')
class TestTheiaResilientWithSR(ResilientTheiaMixin, TestTheiaWithSR):
@classmethod
def setUpClass(cls):
super(TestTheiaResilientWithSR, cls).setUpClass()
# Patch the computer root path to that of the export theia instance
cls.computer_partition_root_path = cls._getPartitionPath('export')
This diff is collapsed.
import contextlib
import errno
import glob
import hashlib
import os
import re
import subprocess as sp
import sqlite3
import six
import zc.buildout.configparser
from slapos.util import bytes2str, str2bytes
RSYNC_FLAGS = ('-rlptgo', '--safe-links', '--stats', '--ignore-missing-args', '--delete', '--delete-excluded')
RSYNC_REGEX = '^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
EXCLUDE_PATTERNS = ('*.sock', '*.socket', '*.pid', '.installed*.cfg')
EXCLUDE_FLAGS = ['--exclude={}'.format(x) for x in sorted(EXCLUDE_PATTERNS)]
def makedirs(path):
try:
os.makedirs(path if os.path.isdir(path) else os.path.dirname(path))
except OSError as e:
if e.errno != errno.EEXIST:
raise
def copytree(rsyncbin, src, dst, exclude=[], extrargs=[], verbosity='-v'):
# Ensure there is a trailing slash in the source directory
# to avoid creating an additional directory level at the destination
src = os.path.join(src, '')
# Compute absolute path of destination
dst = os.path.abspath(dst)
# Create destination dir if it doesn't exist
makedirs(dst)
command = [rsyncbin]
command.extend(RSYNC_FLAGS)
# Exclude destination file from sources
command.append('--filter=-/ {}'.format(dst))
command.extend(EXCLUDE_FLAGS)
command.extend(('--filter=-/ {}'.format(x) for x in sorted(exclude)))
command.extend(extrargs)
command.append(verbosity)
command.append(src)
command.append(dst)
try:
return sp.check_output(command, universal_newlines=True)
except sp.CalledProcessError as e:
# Not all rsync errors are to be considered as errors
if e.returncode != 24 or re.search(RSYNC_REGEX, e.output, re.M) is None:
raise
return e.output
def copydb(sqlite3bin, src_db, dst_db):
makedirs(dst_db)
sp.check_output((sqlite3bin, src_db, '.backup ' + dst_db))
def remove(path):
try:
os.remove(path)
except OSError:
if os.path.exists(path):
raise
def parse_installed(partition):
paths = []
custom_script = os.path.join(partition, 'srv', '.backup_identity_script')
for cfg in glob.glob(os.path.join(partition, '.installed*.cfg')):
try:
with open(cfg) as f:
installed_cfg = zc.buildout.configparser.parse(f, cfg)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in six.itervalues(installed_cfg):
for p in section.get('__buildout_installed__', '').splitlines():
p = p.strip()
if p and p != custom_script:
paths.append(p)
return paths
def sha256sum(file_path, chunk_size=1024 * 1024):
sha256 = hashlib.sha256()
with open(file_path, 'rb') as f:
chunk = f.read(chunk_size)
while chunk:
sha256.update(chunk)
chunk = f.read(chunk_size)
return sha256.hexdigest()
def hashwalk(backup_dir, mirror_partitions):
scripts = {}
for p in mirror_partitions:
script_path = os.path.join(p, 'srv', '.backup_identity_script')
if os.path.exists(script_path):
scripts[os.path.abspath(p)] = script_path
for dirpath, dirnames, filenames in os.walk(backup_dir):
filenames.sort()
for f in filenames:
filepath = os.path.join(dirpath, f)
if os.path.isfile(filepath):
displaypath = os.path.relpath(filepath, start=backup_dir)
yield '%s %s' % (sha256sum(filepath), displaypath)
remaining_dirnames = []
for subdir in dirnames:
subdirpath = os.path.abspath(os.path.join(dirpath, subdir))
custom_hashscript = scripts.get(subdirpath)
if custom_hashscript:
print('Using custom signature script %s' % custom_hashscript)
for s in hashcustom(subdirpath, backup_dir, custom_hashscript):
yield s
else:
remaining_dirnames.append(subdir)
remaining_dirnames.sort()
dirnames[:] = remaining_dirnames
@contextlib.contextmanager
def cwd(path):
old_path = os.getcwd()
try:
os.chdir(path)
yield
finally:
os.chdir(old_path)
def hashcustom(mirrordir, backup_dir, custom_hashscript):
workingdir = os.path.join(mirrordir, os.pardir, os.pardir, os.pardir)
with cwd(os.path.abspath(workingdir)):
for dirpath, _, filenames in os.walk(mirrordir):
filepaths = []
for f in filenames:
path = os.path.join(dirpath, f)
if os.path.isfile(path):
filepaths.append('./' + os.path.relpath(path, start=workingdir))
if not filepaths:
continue
hashprocess = sp.Popen(
custom_hashscript, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE)
out, err = hashprocess.communicate(str2bytes('\0'.join(filepaths)))
if hashprocess.returncode != 0:
template = "Custom signature script %s failed on inputs:\n%s"
msg = template % (custom_hashscript, '\n'.join(filepaths))
msg += "\nwith stdout:\n%s" % bytes2str(out)
msg += "\nand stderr:\n%s" % bytes2str(err)
raise Exception(msg)
signatures = bytes2str(out).strip('\n').split('\n')
signatures.sort()
displaypath = os.path.relpath(dirpath, start=backup_dir)
for s in signatures:
yield '%s %s/ (custom)' % (s, displaypath)
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
...@@ -75,8 +75,4 @@ PasteScript = 3.2.0:whl ...@@ -75,8 +75,4 @@ PasteScript = 3.2.0:whl
WSGIUtils = 0.7.2 WSGIUtils = 0.7.2
WSGIserver = 1.3 WSGIserver = 1.3
python-magic = 0.4.18 python-magic = 0.4.18
# Required by:
# PasteScript== 3.2.0
# cloudooo==1.2.6.dev0
PasteDeploy = 2.1.0 PasteDeploy = 2.1.0
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment