Skip to content

slapos
slapos
Commit 3caed389 authored by Xavier Thompson's avatar Xavier Thompson
Browse files
Options

Update Release Candidate

parents ba4ea4df cb5c5b83
Hide whitespace changes
Inline Side-by-side
Showing with 5736 additions and 7009 deletions
component/ZEO/buildout.cfg
View file @ 3caed389
......@@ -65,9 +65,9 @@ egg = ZEO
eggs = ${:egg}
egg-versions =
ZEO = 5.2.2
trollius = 2.2.post1
futures = 3.2.0
ZEO = 5.2.3
trollius = 2.2.1
futures = 3.3.0
# ZEO4-wc2 is ZEO4 version with patches for wendelin.core 2 to work correctly.
......@@ -76,5 +76,9 @@ egg-versions =
# See https://github.com/zopefoundation/ZEO/pull/161 for the reference.
[ZEO4-wc2]
<= ZEO4
revisions = bf80d23d3506
setup = ${ZEO4-wc2-repository:location}
[ZEO4-wc2-repository]
<= ZEO4-repository
revision = bf80d23d3506
location = ${buildout:parts-directory}/ZEO4-wc2
component/haproxy/buildout.cfg
View file @ 3caed389
......@@ -13,8 +13,8 @@ parts = haproxy
[haproxy]
recipe = slapos.recipe.cmmi
shared = true
url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.22.tar.gz
md5sum = 4d6d5debca0d1bcf51293fb58914f1a3
url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.24.tar.gz
md5sum = 493011d81eea0f3700543fc6a80acc03
configure-command = true
# for Linux kernel 2.6.28 and above, we use "linux-glibc" as the TARGET,
# otherwise use "generic".
......
component/headless-chromium/buildout.cfg
View file @ 3caed389
[buildout]
extends =
../binutils/buildout.cfg
../bison/buildout.cfg
../pkgconfig/buildout.cfg
../gperf/buildout.cfg
../ninja/buildout.cfg
../freetype/buildout.cfg
../cmake/buildout.cfg
../git/buildout.cfg
../pkgconfig/buildout.cfg
../cups/buildout.cfg
# Build dependencies:
../coreutils/buildout.cfg
../curl/buildout.cfg
../depot_tools/buildout.cfg
../findutils/buildout.cfg
../fontconfig/buildout.cfg
../gettext/buildout.cfg
../glib/buildout.cfg
../gtk-2/buildout.cfg
../libexpat/buildout.cfg
../libffi/buildout.cfg
../libpng/buildout.cfg
../libxml2/buildout.cfg
../mesa/buildout.cfg
../git/buildout.cfg
../gperf/buildout.cfg
../pkgconfig/buildout.cfg
# Runtime dependencies:
../nspr/buildout.cfg
../nss/buildout.cfg
../pcre/buildout.cfg
../sqlite3/buildout.cfg
../xorg/buildout.cfg
../zlib/buildout.cfg
parts =
chromium
headless-chromium-wrapper
[gconf]
recipe = slapos.recipe.cmmi
url = http://ftp.gnome.org/pub/gnome/sources/GConf/3.2/GConf-3.2.6.tar.xz
md5sum = 2b16996d0e4b112856ee5c59130e822c
configure-options = --disable-orbit --disable-static
environment =
PATH=${pkgconfig:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${dbus:location}/lib/pkgconfig:${dbus-glib:location}/lib/pkgconfig:$PKG_CONFIG_PATH
[chromedriver]
recipe = hexagonit.recipe.download
url = https://chromedriver.storage.googleapis.com/2.28/chromedriver_linux64.zip
md5sum = a72088c0a6b018ded2c0fff616da8f65
[chromium-download]
# The Chromium project recommends using their own `fetch' tool rather
# than doing a `git clone', but this is a little more flexible and works
# fine.
#
# Setting depth=1000 is a middle ground between cloning no history
# (causes some scripts to break) and cloning the full history (takes a
# really long time).
#
# Note: we should add a `depth' option to slapos.recipe.build:gitclone
# and then migrate this part to use that recipe at some point.
[chromium-source]
recipe = plone.recipe.command
# This revision is 56.0.2924.122. Because the chromedriver only support some certain
# version. Which version 56 is in the middle of 55-57.
revision = faf03429d9c3dbd483700dd42316b20776cbbd3c
path = ${buildout:parts-directory}/${:_buildout_section_name_}
command =
set -e
PATH=${depot_tools:location}:${git:location}/bin:$PATH
[ -d ${:path} ] && rm -r ${:path}
mkdir -p ${:path}
cd ${:path}
# Do never use `fetch` with the `--no-history` option unless you find an
# option to fetch directly at the wanted revision. `--no-history` could
# reduce the download size significantly but even if it retrieves enough
# commits at the time you test this section, development continues upstream
# and at some point the next command (gclient) would break.
# ...
# This command only could work in an empty dir.
fetch --nohooks chromium
gclient sync --revision ${:revision} --with_branch_heads
export PATH=$PATH:${git:location}/bin
git clone ${:repository} ${:location} \
--branch ${:version} \
--depth 1000
repository = https://chromium.googlesource.com/chromium/src.git
stop-on-error = true
[chromium]
# We should place the .gclient file in the parent directory of the
# checkout/repository itself.
location = ${:gclient-location}/${:name}
gclient-location = ${buildout:parts-directory}/${:_buildout_section_name_}
# Theoretically the checkout can be put anywhere as long as you specify
# "name" appropriately in the .gclient file, but in practice it seems
# that some automated tools break. It's safest to put it in a directory
# called "src".
name = src
# There's nothing special about version 92.0.4515.107. It just happened
# to be the current Chromium stable version at the time of writing.
version = 92.0.4515.107
[headless-chromium]
recipe = slapos.recipe.cmmi
path = ${chromium-download:path}/src
location = ${buildout:parts-directory}/${:_buildout_section_name_}
path = ${chromium-source:location}
location = ${:path}/out/headless
# Configuration file for GN, the tool to build the actual compilation
# configuration file.
default-gclient-config =
solutions = [
{
"name": "${chromium-source:name}",
"url": "${chromium-source:repository}",
"managed": False,
"custom_deps": {},
"custom_vars": {},
},
]
# Configuration for a headless build.
build-config-options =
import("//build/args/headless.gn")
is_debug = false
symbol_level = 0
blink_symbol_level = 0
# We need to unbundle the build toolchain in order to set our own
# LDFLAGS.
custom_toolchain = "//build/toolchain/linux/unbundle:default"
host_toolchain = "//build/toolchain/linux/unbundle:default"
current_os = "linux"
current_cpu = "x64"
# Chromium bundles its own LLVM toolchain, so we might as well use it.
llvm-toolchain = ${:path}/third_party/llvm-build/Release+Asserts/bin
configure-command =
gclient runhooks
# Sync build dependencies---this is a little finnicky.
echo '${:default-gclient-config}' \
> ${chromium-source:gclient-location}/.gclient
gclient sync --no-history
# Generate build configuration files.
mkdir -p ${:location}
echo 'use_udev = true
is_debug = false
enable_nacl = false' > ${:location}/args.gn
echo '${:build-config-options}' > ${:location}/args.gn
gn gen ${:location}
# Note you can run Chromium manually by: ${:location}/chrome --headless --no-sandbox --disable-gpu
make-binary = ninja -C ${:location} chrome
# You can run the headless Chromium shell using
# ${:binary} --remote-debugging-port=1234
make-binary =
autoninja -C ${:location} headless_shell
# By building our own version of Chromedriver, we can ensure version
# compatibility. The build is quite cheap compared to Chromium, anyway.
autoninja -C ${:location} chromedriver
environment =
PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${randrproto:location}/lib/pkgconfig:$PKG_CONFIG_PATH
PATH=${chromedriver:location}:${dbus:location}/bin:${depot_tools:location}:${pkgconfig:location}/bin:${ninja:location}/bin:${bison:location}/bin:${gperf:location}/bin:${xserver:location}/bin:%(PATH)s
CPATH=${dbus:location}/include/dbus-1.0:${dbus:location}/lib/dbus-1.0/include/:${freetype:location}/include/freetype2:${libffi:location}/include:${mpfr:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline:location}/include:${sqlite3:location}/include:${zlib:location}/include:${bzip2:location}/include:$CPATH
LD_LIBRARY_PATH=${alsa:location}/lib:${gconf:location}/lib:${libXScrnSaver:location}/lib:${glib:location}/lib:${atk:location}/lib:${cairo:location}/lib:${cups:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib/:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcomposite:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXi:location}/lib:${libXrender:location}/lib/:${libXtst:location}/lib:${libexpat:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libpng12:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${mesa:location}/lib:${nspr:location}/lib:${nss:location}/lib:${pango:location}/lib:${pcre:location}/lib:${pixman:location}/lib:${sqlite3:location}/lib:${xdamage:location}/lib:${xfixes:location}/lib:${zlib:location}/lib:$LD_LIBRARY_PATH
PATH=${depot_tools:location}:${gperf:location}/bin:${pkgconfig:location}/bin:${coreutils:location}/bin:${git:location}/bin:${curl:location}/bin:%(PATH)s
LDFLAGS="-Wl,-rpath=${nss:location}/lib,-rpath=${nspr:location}/lib"
CC="${:llvm-toolchain}/clang"
CXX="${:llvm-toolchain}/clang++"
AR="${:llvm-toolchain}/llvm-ar"
NM="${:llvm-toolchain}/llvm-nm"
# Expose devtools frontend location.
devtools-frontend = ${:location}/gen/third_party/devtools-frontend/src/front_end
binary = ${:location}/headless_shell
chromedriver = ${:location}/chromedriver
promises =
${:binary}
${:chromedriver}
# At runtime, Chromium tries to dynamically load the NSS certificate
# database from "libnssckbi.so". But Chromium does this through NSPR,
# which doesn't know where SlapOS installed NSS. Since we don't want to
# modify the NSPR component from the Chromium component, we just set
# LD_LIBRARY_PATH in a wrapper script so that NSPR knows where NSS is.
#
# Alternatively, we could patch crypto/nss_util.cc in the Chromium
# source code to use an absolute path for libnssckbi.so, but this is not
# as future-proof against new versions of Chromium.
[headless-chromium-wrapper]
recipe = slapos.recipe.template:jinja2
template =
inline:#!/bin/sh
export LD_LIBRARY_PATH="{{ nss_location }}/lib:$LD_LIBRARY_PATH"
exec {{ chromium_binary }} "$@"
rendered = ${buildout:bin-directory}/headless-chromium
context =
key nss_location nss:location
key chromium_binary headless-chromium:binary
component/jupyter-py2/buildout.cfg
View file @ 3caed389
......@@ -95,63 +95,25 @@ tornado = 4.4.2
widgetsnbextension = 2.0.0
# numpy >= 1.13.1 is required for numpy.core.multiarray
numpy = 1.13.1
# Required by:
# tornado==4.4.2
certifi = 2020.6.20
# Required by:
# notebook==4.3.2
# nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
nbconvert = 4.1.0
# Required by:
# ipython==5.3.0
pathlib2 = 2.2.1
# Required by:
# statsmodels==0.8.0
patsy = 0.4.1
# Required by:
# ipython==5.3.0
pexpect = 4.2.1
# Required by:
# ipython==5.3.0
pickleshare = 0.7.4
# Required by:
# pathlib2==2.2.1
scandir = 1.5
# Required by:
# statsmodels==0.8.0
scipy = 0.19.0
# Required by:
# tornado==4.4.2
singledispatch = 3.4.0.3
# Required by:
# prompt-toolkit==1.0.13
wcwidth = 0.1.7
jupyter = 1.0.0
jupyter-console = 5.1.0
# Required by:
# jupyter==1.0.0
qtconsole = 4.3.0
et-xmlfile = 1.0.1
h5py = 2.7.1
mpmath = 1.0.0
openpyxl = 2.5.2
sympy = 1.1.1
xlrd = 1.1.0
# Required by:
# openpyxl==2.5.2
jdcal = 1.4
component/libsecret/buildout.cfg 0 → 100644
View file @ 3caed389
[buildout]
parts = libsecret
extends =
../gnutls/buildout.cfg
../libxslt/buildout.cfg
../pkgconfig/buildout.cfg
../glib/buildout.cfg
../gettext/buildout.cfg
../libuuid/buildout.cfg
../xz-utils/buildout.cfg
[libsecret]
recipe = slapos.recipe.cmmi
url = https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz
md5sum = bf92f48afab2891f644f311e0f37683f
shared = true
configure-options =
--with-libgcrypt-prefix=${libgcrypt:location}
--with-libintl-prefix=${gettext:location}
--disable-gtk-doc-html
--disable-manpages
--disable-vala
environment =
PATH=${xz-utils:location}/bin/:${pkgconfig:location}/bin:${libxslt:location}/bin:${glib:location}/bin:${libgcrypt:location}/bin/:%(PATH)s
PKG_CONFIG_PATH=${:pkg-config-path-depends}
LDFLAGS=-Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${libgcrypt:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libuuid:location}/lib
pkg-config-path-depends = ${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libgcrypt:location}/lib/pkgconfig:${libuuid:location}/lib/pkgconfig
pkg-config-path = @@LOCATION@@/lib/pkgconfig:${:pkg-config-path-depends}
component/nginx/buildout.cfg
View file @ 3caed389
......@@ -24,6 +24,7 @@ configure-options=
--with-http_v2_module
--with-http_gzip_static_module
--with-http_realip_module
--with-http_sub_module
--with-mail
--with-mail_ssl_module
--with-ld-opt="-L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
......@@ -85,7 +86,7 @@ revision = 3d3a204177d3a7ab8a2858e04e792a6d11bf133f
git-executable = ${git:location}/bin/git
[nginx-push-stream]
<= nginx
<= nginx-common
configure-options=
--with-ipv6
--with-http_ssl_module
......
component/nxdtest/buildout.cfg
View file @ 3caed389
......@@ -20,6 +20,6 @@ egg = nxdtest
[nxdtest-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nxdtest.git
revision = b5a74214
revision = 0ad45a9c
location = ${buildout:parts-directory}/nxdtest
git-executable = ${git:location}/bin/git
component/pytest/buildout.cfg
View file @ 3caed389
......@@ -15,3 +15,4 @@ eggs = ${pytest:eggs}
[versions]
pytest = 4.6.11:whl
pytest-timeout = 1.4.2
component/qemu-kvm/buildout.cfg
View file @ 3caed389
......@@ -91,11 +91,11 @@ md5sum = 6097fdb9cbab47c96471274b9044e983
# XXX: This is not the latest version because
# Debian does not provide a stable URL for it.
<= debian-amd64-netinst-base
version = 10.8.0
md5sum = e221f43f4fdd409250908fc4305727d4
version = 10.10.0
md5sum = c7d0e562e589e853b5d00563b4311720
[debian-amd64-testing-netinst.iso]
<= debian-amd64-netinst-base
release = bullseye_di_alpha3
version = bullseye-DI-alpha3
md5sum = bff147077791586fa7c102267da9f2d2
release = bullseye_di_rc3
version = bullseye-DI-rc3
md5sum = 405917de7062c58357a3673c9901f0c4
component/tesseract/buildout.cfg
View file @ 3caed389
......@@ -26,6 +26,9 @@ md5sum = 51fe2bcbff1bbce77a25d180fd247f7d
pkg_config_depends = ${leptonica:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${lcms2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
pre-configure =
autoreconf -ivf -I${pkgconfig:location}/share/aclocal -I${libtool:location}/share/aclocal -Wno-portability
# XXX workaround path on slaprunner with a double slash
# https://github.com/tesseract-ocr/tesseract/issues/3527
configure-options = --prefix=$(python -c 'print("""@@LOCATION@@""".replace("//", "/"))')
environment =
PATH=${pkgconfig:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${patch:location}/bin:%(PATH)s
......
component/trafficserver/buildout.cfg
View file @ 3caed389
......@@ -23,12 +23,10 @@ min_version = 8
[trafficserver]
recipe = slapos.recipe.cmmi
url = http://apache.claz.org/trafficserver/trafficserver-9.0.1.tar.bz2
md5sum = 98bb2de25f332715339ade87530a9f5a
url = http://apache.claz.org/trafficserver/trafficserver-9.0.2.tar.bz2
md5sum = 4df67ada24665116bafedd71503215cb
shared = true
patch-options = -p1
patches =
https://patch-diff.githubusercontent.com/raw/apache/trafficserver/pull/7577.patch
configure-options =
--with-openssl=${openssl:location}
--with-pcre=${pcre:location}
......
component/wendelin.core/test-common.cfg
View file @ 3caed389
......@@ -7,6 +7,7 @@ extends =
# appropriate ZODB and versions of other components.
../pytest/buildout.cfg
../scipy/buildout.cfg
../gdb/buildout.cfg
../../stack/nxdtest.cfg
buildout.cfg
......@@ -15,22 +16,25 @@ parts =
# keep neoppod first and in parts so that ZODB is built correctly
neoppod
wendelin.core
# for instance
wendelin.core-python
slapos-cookbook
instance.cfg
# bin/python s python interpreter with wendelin.core and all other eggs.
# test-dependent eggs that must come through in-tree recipes.
[wendelin.core]
depends += ${scipy:egg}
# bin/python is python interpreter with wendelin.core and all other eggs.
[wendelin.core-python]
<= python-interpreter
eggs =
wendelin.core[test]
pygolang[pyx.build]
${wendelin.core:egg}[test]
${pygolang:egg}[pyx.build]
neoppod[tests]
ZEO[test]
${ZEO:egg}[test]
# env.sh for that python + go to be on $PATH
[wendelin.core-env.sh]
......
component/zodbtools/test-common.cfg
View file @ 3caed389
......@@ -10,8 +10,6 @@ extends =
buildout-dev.cfg
parts =
zodbtools
# for instance
zodbtools-python
slapos-cookbook
......@@ -21,7 +19,7 @@ parts =
# bin/python is preinstalled with sys.path to zodbtools & friends.
[zodbtools-python]
<= python-interpreter
eggs = zodbtools[test]
eggs = ${zodbtools:egg}[test]
# env.sh for zodbtools's python to be on $PATH.
[zodbtools-env.sh]
......
software/caddy-frontend/buildout.hash.cfg
View file @ 3caed389
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg.in
md5sum = 04015a7a552285984d091293ef573fb9
md5sum = 1dfbd20c77fb3c1f01005a8a920d2ed9
[profile-common]
filename = instance-common.cfg.in
......@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
[profile-caddy-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = 8507a2ace2f789b92c522cc62ca5aace
md5sum = 51087ac7615bd7cc01e60eb23701f625
[profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 8beb438d06bbb0f917d13e182fb12d17
md5sum = b6fc5a004a1235ffad3af0b4cb0e661f
[profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = 613f777a08373088cbaf7f51fd18ea70
md5sum = 9bb51f663f69d66b5b3708bf892dd3e6
[profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
......@@ -72,10 +72,6 @@ md5sum = d022455a8610bac2dd51101edb035987
_update_hash_filename_ = templates/trafficserver/logging.yaml.jinja2
md5sum = 368b271215a92594ca9e2fa3102d484f
[template-nginx-eventsource-slave-virtualhost]
_update_hash_filename_ = templates/nginx-eventsource-slave.conf.in
md5sum = 217a6c801b8330b0b825f7b8b4c77184
[template-caddy-lazy-script-call]
_update_hash_filename_ = templates/apache-lazy-script-call.sh.in
md5sum = 77d60840591de67b64ab3572e46273a0
......
software/caddy-frontend/instance-apache-frontend.cfg.in
View file @ 3caed389
{%- if instance_parameter_dict['slap-software-type'] == software_type -%}
{% import "caucase" as caucase with context %}
{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
[buildout]
......@@ -11,6 +10,7 @@ parts =
directory
logrotate-entry-caddy
caddy-frontend
caddyprofiledeps
switch-caddy-softwaretype
caucase-updater
caucase-updater-promise
......@@ -97,9 +97,11 @@ expose-csr_id-var = ${:var}/expose-csr_id
slave-introspection-var = ${:var}/slave-introspection
[switch-caddy-softwaretype]
recipe = slapos.cookbook:softwaretype
single-default = ${dynamic-custom-personal-profile-slave-list:rendered}
single-custom-personal = ${dynamic-custom-personal-profile-slave-list:rendered}
recipe = slapos.cookbook:switch-softwaretype
default = dynamic-custom-personal-profile-slave-list:rendered
RootSoftwareInstance = ${:default}
single-default = dynamic-custom-personal-profile-slave-list:rendered
single-custom-personal = dynamic-custom-personal-profile-slave-list:rendered
[frontend-configuration]
ip-access-certificate = ${self-signed-ip-access:certificate}
......@@ -114,7 +116,7 @@ slave-introspection-domain = ${slave-introspection-frontend:connection-domain}
# Self Signed certificate for HTTPS IP accesses to the frontend
recipe = plone.recipe.command
update-command = ${:command}
ipv6 = ${slap-network-information:global-ipv6}
ipv6 = ${slap-configuration:ipv6-random}
ipv4 = {{instance_parameter_dict['ipv4-random']}}
certificate = ${caddy-directory:master-autocert-dir}/ip-access-${:ipv6}-${:ipv4}.crt
{#- Can be stopped on error, as does not rely on self provided service #}
......@@ -138,7 +140,7 @@ command =
# Self Signed certificate for HTTPS access to the frontend with fallback certificate
recipe = plone.recipe.command
update-command = ${:command}
ipv6 = ${slap-network-information:global-ipv6}
ipv6 = ${slap-configuration:ipv6-random}
ipv4 = {{instance_parameter_dict['ipv4-random']}}
certificate = ${caddy-directory:master-autocert-dir}/fallback-access.crt
{#- Can be stopped on error, as does not rely on self provided service #}
......@@ -282,6 +284,7 @@ software_type = single-custom-personal
organization = {{ slapparameter_dict['cluster-identification'] }}
organizational-unit = {{ instance_parameter_dict['configuration.frontend-name'] }}
backend-client-caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }}
partition_ipv6 = ${slap-configuration:ipv6-random}
extra-context =
key caddy_configuration_directory caddy-directory:slave-configuration
key backend_client_caucase_url :backend-client-caucase-url
......@@ -293,7 +296,7 @@ extra-context =
key caddy_log_directory caddy-directory:slave-log
key expose_csr_id_organization :organization
key expose_csr_id_organizational_unit :organizational-unit
key global_ipv6 slap-network-information:global-ipv6
key global_ipv6 slap-configuration:ipv6-random
key empty_template software-release-path:template-empty
key template_default_slave_configuration software-release-path:template-default-slave-virtualhost
key software_type :software_type
......@@ -328,8 +331,8 @@ extra-context =
section frontend_configuration frontend-configuration
key http_port configuration:plain_http_port
key https_port configuration:port
key global_ipv6 slap-configuration:ipv6-random
key local_ipv4 :local_ipv4
key global_ipv6 slap-network-information:global-ipv6
key error_log caddy-configuration:error-log
key not_found_file caddy-configuration:not-found-file
key username monitor-instance-parameter:username
......@@ -904,7 +907,7 @@ recipe = slapos.cookbook:requestoptional
name = Slave Introspection Frontend {{ instance_parameter_dict['configuration.frontend-name'] }}
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter_dict['configuration.slave-introspection-https-port'] }}/
config-url = https://[${slap-configuration:ipv6-random}]:{{ instance_parameter_dict['configuration.slave-introspection-https-port'] }}/
config-https-only = true
return = domain secure_access
......@@ -914,7 +917,7 @@ recipe = slapos.cookbook:requestoptional
name = Backend Haproxy Statistic Frontend {{ instance_parameter_dict['configuration.frontend-name'] }}
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter_dict['configuration.backend-haproxy-statistic-port'] }}/
config-url = https://[${slap-configuration:ipv6-random}]:{{ instance_parameter_dict['configuration.backend-haproxy-statistic-port'] }}/
config-https-only = true
return = domain secure_access
......@@ -1022,5 +1025,3 @@ config-command =
{%- for key, value in software_parameter_dict.iteritems() %}
{{ key }} = {{ dumps(value) }}
{%- endfor %}
{%- endif -%} {# if instance_parameter_dict['slap-software-type'] == software_type #}
software/caddy-frontend/instance-apache-replicate.cfg.in
View file @ 3caed389
{% if instance_parameter_dict['slap-software-type'] in software_type %}
{% set aibcc_enabled = True %}
{% import "caucase" as caucase with context %}
{#- SERVER_POLLUTED_KEY_LIST is a list of keys which comes from various SlapOS Master implementations, which mix request and publish keys on each slave information -#}
......@@ -44,13 +43,7 @@ context =
{% set popen = functools_module.partial(subprocess_module.Popen, stdout=subprocess_module.PIPE, stderr=subprocess_module.STDOUT, stdin=subprocess_module.PIPE) %}
{% set part_list = [] %}
{% set single_type_key = 'single-' %}
{% if instance_parameter_dict['slap-software-type'] == "replicate" %}
{% set frontend_type = slapparameter_dict.pop('-frontend-type', 'single-default') %}
{% elif instance_parameter_dict['slap-software-type'] in ['default', 'RootSoftwareInstance'] %}
{% set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %}
{% else %}
{% set frontend_type = "%s%s" % (single_type_key, instance_parameter_dict['slap-software-type']) %}
{% endif %}
{% set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %}
{% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '1') | int %}
{% set slave_list_name = 'extra_slave_instance_list' %}
{% set frontend_list = [] %}
......@@ -117,9 +110,7 @@ context =
{% set slave_warning_list = [] %}
{% set slave_server_alias_unclashed = [] %}
{% set slave_type = slave.get('type') %}
{% if slave_type == 'eventsource' %}
{% do slave_error_list.append('type:eventsource is not implemented') %}
{% elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
{% if slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
{% do slave_error_list.append('type:%s is not supported' % (slave_type,)) %}
{% endif %}
{# Check health-check-* #}
......@@ -392,12 +383,12 @@ config-url =
#--
#-- Publish slave information
[publish-slave-information]
recipe = slapos.cookbook:softwaretype
default = ${dynamic-publish-slave-information:rendered}
RootSoftwareInstance = ${dynamic-publish-slave-information:rendered}
replicate = ${dynamic-publish-slave-information:rendered}
custom-personal = ${dynamic-publish-slave-information:rendered}
custom-group = ${dynamic-publish-slave-information:rendered}
recipe = slapos.cookbook:switch-softwaretype
default = dynamic-publish-slave-information:rendered
RootSoftwareInstance = ${:default}
replicate = dynamic-publish-slave-information:rendered
custom-personal = dynamic-publish-slave-information:rendered
custom-group = dynamic-publish-slave-information:rendered
[request-kedifa]
<= slap-connection
......@@ -909,6 +900,4 @@ parts =
caucased-backend-client-promise
{% for part in part_list %}
{{ ' %s' % part }}
{% endfor %}
# publish-information
{% endif %}
{% endfor %}
\ No newline at end of file
software/caddy-frontend/instance-slave-caddy-input-schema.json
View file @ 3caed389
......@@ -2,7 +2,7 @@
"$schema": "http://json-schema.org/draft-04/schema",
"properties": {
"custom_domain": {
"description": "Custom Domain to use for the website",
"description": "Custom Domain to use for the website. Shall contain only letters, numbers and -, and can look like example.com, first2.example.com special-site.example.com.",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
"title": "Custom Domain",
"type": "string"
......@@ -15,14 +15,13 @@
},
"type": {
"default": "",
"description": "Type of slave. If redirect, the slave will redirect to the given URL. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect, notebook and websocket, not implemneted is eventsource.",
"description": "Type of slave. If redirect, the slave will redirect to the given URL. If zope, the rewrite rules will be compatible with Virtual Host Monster.",
"enum": [
"",
"zope",
"redirect",
"notebook",
"websocket",
"eventsource"
"websocket"
],
"title": "Backend Type",
"type": "string"
......
software/caddy-frontend/instance-slave-caddy-simplified-input-schema.json
View file @ 3caed389
......@@ -9,7 +9,7 @@
},
"type": {
"default": "",
"description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect, notebook and websocket, not implemneted is eventsource.",
"description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster.",
"enum": [
"",
"zope"
......
software/caddy-frontend/instance.cfg.in
View file @ 3caed389
......@@ -2,6 +2,7 @@
extends = {{ software_parameter_dict['profile_common'] }}
parts =
caddyprofiledeps
switch-softwaretype
[caddyprofiledeps]
......@@ -14,8 +15,8 @@ extensions = jinja2.ext.do
extra-context =
context =
import json_module json
key slapparameter_dict instance-parameter:configuration
section instance_parameter_dict instance-parameter
key slapparameter_dict slap-configuration:configuration
section instance_parameter_dict slap-configuration
section software_parameter_dict software-parameter-section
${:extra-context}
caucase-jinja2-library = {{ software_parameter_dict['caucase_jinja2_library'] }}
......@@ -23,14 +24,14 @@ import-list =
file caucase :caucase-jinja2-library
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${dynamic-profile-caddy-replicate:rendered}
RootSoftwareInstance = ${dynamic-profile-caddy-replicate:rendered}
custom-personal = ${dynamic-profile-caddy-replicate:rendered}
single-default = ${dynamic-profile-caddy-frontend:rendered}
single-custom-personal = ${dynamic-profile-caddy-frontend:rendered}
replicate = ${dynamic-profile-caddy-replicate:rendered}
kedifa = ${dynamic-profile-kedifa:rendered}
recipe = slapos.cookbook:switch-softwaretype
default = dynamic-profile-caddy-replicate:rendered
RootSoftwareInstance = ${:default}
custom-personal = dynamic-profile-caddy-replicate:rendered
single-default = dynamic-profile-caddy-frontend:rendered
single-custom-personal = dynamic-profile-caddy-frontend:rendered
replicate = dynamic-profile-caddy-replicate:rendered
kedifa = dynamic-profile-kedifa:rendered
[software-parameter-section]
{% for key,value in software_parameter_dict.iteritems() %}
......@@ -64,7 +65,7 @@ filename = instance-kedifa.cfg
extra-context =
raw software_type kedifa
[instance-parameter]
[slap-configuration]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
......
software/caddy-frontend/templates/apache-custom-slave-list.cfg.in
View file @ 3caed389
{%- if software_type == slap_software_type %}
{%- set kedifa_updater_mapping = [] %}
{%- set cached_server_dict = {} %}
{%- set backend_slave_list = [] %}
......@@ -187,7 +186,7 @@ context =
{%- set furled = furl_module.furl(frontend_configuration['slave-introspection-secure_access']) %}
{%- do furled.set(username = slave_reference.lower()) %}
{%- do furled.set(password = '${'+ slave_password_section +':passwd}') %}
{%- do furled.set(path = slave_reference.lower() + '/') %}
{%- do furled.set(path = slave_reference + '/') %}
{#- We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #}
{%- set slave_log_access_url = urlparse_module.unquote(furled.tostr()) %}
{%- do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %}
......@@ -237,7 +236,7 @@ context =
{#- Set slave logrotate entry #}
[{{slave_log_directory_section}}]
recipe = slapos.cookbook:mkdirectory
log-directory = {{ '${slave-log-directory-dict:' + slave_reference.lower() + '}' }}
log-directory = {{ '${slave-log-directory-dict:' + slave_reference + '}' }}
[{{slave_logrotate_section}}]
<= logrotate-entry-base
......@@ -399,8 +398,8 @@ recipe = slapos.cookbook:publish
{#- Define IPv6 to IPV4 tunneling #}
[tunnel-6to4-base]
recipe = slapos.cookbook:wrapper
ipv4 = ${slap-network-information:local-ipv4}
ipv6 = ${slap-network-information:global-ipv6}
ipv4 = ${slap-configuration:ipv4-random}
ipv6 = ${slap-configuration:ipv6-random}
wrapper-path = {{ directory['service'] }}/6tunnel-${:ipv6-port}
command-line = {{ software_parameter_dict['sixtunnel'] }}/bin/6tunnel -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
......@@ -509,7 +508,7 @@ extra-context =
{{ key }} = {{ value }}
{%- endfor %}
local-ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }}
global-ipv6 = ${slap-network-information:global-ipv6}
global-ipv6 = ${slap-configuration:ipv6-random}
request-timeout = {{ dumps('' ~ configuration['request-timeout']) }}
backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }}
backend-connect-retries = {{ dumps('' ~ configuration['backend-connect-retries']) }}
......@@ -583,12 +582,12 @@ update-command = ${:command}
command =
if ! [ -f ${:key} ] && ! [ -f ${:certificate} ] ; then
openssl req -new -newkey rsa:2048 -sha256 -subj \
"/O={{ expose_csr_id_organization }}/OU={{ expose_csr_id_organizational_unit }}/CN=${slap-network-information:global-ipv6}" \
"/O={{ expose_csr_id_organization }}/OU={{ expose_csr_id_organizational_unit }}/CN=${slap-configuration:ipv6-random}" \
-days 5 -nodes -x509 -keyout ${:key} -out ${:certificate}
fi
[expose-csr_id-configuration]
ip = ${slap-network-information:global-ipv6}
ip = ${slap-configuration:ipv6-random}
port = 17001
key = ${certificate-csr_id:key}
certificate = ${certificate-csr_id:certificate}
......@@ -659,5 +658,4 @@ commands =
module = check_command_execute
name = ${:_buildout_section_name_}.py
config-command =
${logrotate:wrapper-path} -d
{%- endif %} {# if software_type == slap_software_type #}
${logrotate:wrapper-path} -d
\ No newline at end of file
software/caddy-frontend/templates/nginx-eventsource-slave.conf.in deleted 100644 → 0
View file @ ba4ea4df
{% set url = slave_parameter.get('url') %}
{% set https_url = slave_parameter.get('https-url', url) %}
{% if url.startswith("http://") or url.startswith("https://") %}
{% set upstream = url.split("/")[2] %}
{% set https_upstream = https_url.split("/")[2] %}
{% set protocol = url.split("/")[0] %}
{% set https_protocol = https_url.split("/")[0] %}
{% set proxy_pass = '%s//%s' % (protocol, slave_parameter.get('slave_reference')) %}
{% set https_proxy_pass = '%s//https_%s' % (protocol, slave_parameter.get('slave_reference')) %}
# TODO-Caddy upstream {{ slave_parameter.get('slave_reference') }} {
# TODO-Caddy server {{ upstream }};
# TODO-Caddy
# TODO-Caddy pstream https_{{ slave_parameter.get('slave_reference') }} {
# TODO-Caddy server {{ https_upstream }};
# TODO-Caddy
# TODO-Caddy server {
# TODO-Caddy listen {{ slave_parameter['local_ipv4'] }}:{{ slave_parameter['nginx_http_port'] }};
# TODO-Caddy
# TODO-Caddy server_name {{ slave_parameter.get('custom_domain') }};
# TODO-Caddy
# TODO-Caddy error_log {{ slave_parameter.get('error_log') }} error;
# TODO-Caddy access_log {{ slave_parameter.get('access_log') }} custom;
# TODO-Caddy
# TODO-Caddy location /pub {
# TODO-Caddy push_stream_publisher;
# TODO-Caddy push_stream_channels_path $arg_id;
# TODO-Caddy # store messages in memory
# TODO-Caddy push_stream_store_messages off;
# TODO-Caddy
# TODO-Caddy # Message size limit
# TODO-Caddy # client_max_body_size MUST be equal to client_body_buffer_size or
# TODO-Caddy # you will be sorry.
# TODO-Caddy client_max_body_size 16k;
# TODO-Caddy client_body_buffer_size 16k;
# TODO-Caddy
# TODO-Caddy }
# TODO-Caddy
# TODO-Caddy location ~ /sub/(.*) {
# TODO-Caddy # activate subscriber mode for this location
# TODO-Caddy add_header "Access-Control-Allow-Origin" "*";
# TODO-Caddy add_header 'Access-Control-Allow-Credentials' 'false';
# TODO-Caddy add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
# TODO-Caddy add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
# TODO-Caddy
# TODO-Caddy push_stream_subscriber eventsource;
# TODO-Caddy # positional channel path
# TODO-Caddy push_stream_channels_path $1;
# TODO-Caddy
# TODO-Caddy # content-type
# TODO-Caddy default_type "text/event-stream; charset=utf-8";
# TODO-Caddy }
# TODO-Caddy
# TODO-Caddy
# TODO-Caddy server {
# TODO-Caddy listen {{ slave_parameter['local_ipv4'] }}:{{ slave_parameter['nginx_https_port'] }} ssl;
# TODO-Caddy
# TODO-Caddy server_name {{ slave_parameter.get('custom_domain') }};
# TODO-Caddy
# TODO-Caddy error_log {{ slave_parameter.get('error_log') }} error;
# TODO-Caddy access_log {{ slave_parameter.get('access_log') }} custom;
# TODO-Caddy
# TODO-Caddy ssl on;
# TODO-Caddy
# TODO-Caddy ssl_session_timeout 5m;
# TODO-Caddy ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
# TODO-Caddy ssl_ciphers 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5';
# TODO-Caddy ssl_prefer_server_ciphers on;
# TODO-Caddy ssl_session_cache shared:SSL:10m;
# TODO-Caddy location /pub {
# TODO-Caddy push_stream_publisher;
# TODO-Caddy push_stream_channels_path $arg_id;
# TODO-Caddy # store messages in memory
# TODO-Caddy push_stream_store_messages off;
# TODO-Caddy
# TODO-Caddy # Message size limit
# TODO-Caddy # client_max_body_size MUST be equal to client_body_buffer_size or
# TODO-Caddy # you will be sorry.
# TODO-Caddy client_max_body_size 16k;
# TODO-Caddy client_body_buffer_size 16k;
# TODO-Caddy
# TODO-Caddy }
# TODO-Caddy
# TODO-Caddy location ~ /sub/(.*) {
# TODO-Caddy # activate subscriber mode for this location
# TODO-Caddy add_header "Access-Control-Allow-Origin" "*";
# TODO-Caddy add_header 'Access-Control-Allow-Credentials' 'false';
# TODO-Caddy add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
# TODO-Caddy add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
# TODO-Caddy
# TODO-Caddy push_stream_subscriber eventsource;
# TODO-Caddy # positional channel path
# TODO-Caddy push_stream_channels_path $1;
# TODO-Caddy
# TODO-Caddy # content-type
# TODO-Caddy default_type "text/event-stream; charset=utf-8";
# TODO-Caddy }
# TODO-Caddy}
{% endif %}
software/caddy-frontend/test/test.py
View file @ 3caed389
......@@ -1479,10 +1479,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
'websocket-path-list': '////ws//// /with%20space/',
'websocket-transparent': 'false',
},
# 'type-eventsource': {
# 'url': cls.backend_url,
# 'type': 'eventsource',
# },
'type-redirect': {
'url': cls.backend_url,
'type': 'redirect',
......@@ -1817,7 +1813,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
def test_server_polluted_keys_removed(self):
buildout_file = os.path.join(
self.getMasterPartitionPath(), 'buildout-switch-softwaretype.cfg')
self.getMasterPartitionPath(), 'instance-caddy-replicate.cfg')
for line in [
q for q in open(buildout_file).readlines()
if q.startswith('config-slave-list') or q.startswith(
......@@ -3261,54 +3257,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
self.assertFalse('x-real-ip' in j['Incoming Headers'])
@skip('Feature postponed')
def test_type_eventsource(self):
# Caddy: For event source, if I understand
# https://github.com/mholt/caddy/issues/1355 correctly, we could use
# Caddy as a proxy in front of nginx-push-stream . If we have a
# "central shared" caddy instance, can it handle keeping connections
# opens for many clients ?
parameter_dict = self.parseSlaveParameterDict('type-eventsource')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'typeeventsource.nginx.example.com',
'replication_number': '1',
'url': 'http://typeeventsource.nginx.example.com',
'site_url': 'http://typeeventsource.nginx.example.com',
'secure_access': 'https://typeeventsource.nginx.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], 'pub',
# NGINX_HTTPS_PORT
)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
'',
result.content
)
headers = result.headers.copy()
self.assertKeyWithPop('Expires', headers)
self.assertKeyWithPop('Date', headers)
self.assertEqual(
{
'X-Nginx-PushStream-Explain': 'No channel id provided.',
'Content-Length': '0',
'Cache-Control': 'no-cache, no-store, must-revalidate',
'Connection': 'keep-alive',
'Server': 'nginx'
},
headers
)
def test_type_redirect(self):
parameter_dict = self.assertSlaveBase('type-redirect')
......@@ -3629,7 +3577,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$'
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
)
def test_enable_cache_server_alias(self):
......@@ -3671,7 +3619,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$'
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
)
result = fakeHTTPResult(
......@@ -3788,7 +3736,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$'
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
)
# BEGIN: Check that squid.log is correctly filled in
......@@ -3990,7 +3938,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$'
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
)
# check stale-if-error support is really respected if not present in the
......@@ -4133,7 +4081,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$'
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
)
try:
......@@ -4180,7 +4128,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$'
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
)
def test_enable_http2_false(self):
......@@ -4642,7 +4590,7 @@ class TestReplicateSlaveOtherDestroyed(SlaveHttpFrontendTestCase):
self.slap.waitForInstance(self.instance_max_retry)
buildout_file = os.path.join(
self.getMasterPartitionPath(), 'buildout-switch-softwaretype.cfg')
self.getMasterPartitionPath(), 'instance-caddy-replicate.cfg')
with open(buildout_file) as fh:
buildout_file_content = fh.read()
node_1_present = re.search(
......
software/cloudooo/software-common.cfg
View file @ 3caed389
......@@ -100,10 +100,5 @@ url = ${:_profile_base_location_}/${:filename}
mode = 640
[versions]
# Required by:
# cloudooo==1.2.6.dev0
argparse = 1.4.0
# Required by:
# cloudooo==1.2.6.dev0
pyPdf = 1.13
software/erp5testnode/buildout.hash.cfg
View file @ 3caed389
......@@ -14,8 +14,8 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 307663d73ef3ef94b02567ecd322252e
md5sum = d9c7c3acb975d44c6c66ca32c550ac58
[template-default]
filename = instance-default.cfg
md5sum = 24cc143b1886d443a4c29dcb8147a01c
filename = instance-default.cfg.jinja.in
md5sum = ffb6b74b55e7ca01666254353ae1cebe
software/erp5testnode/instance-default.cfg → software/erp5testnode/instance-default.cfg.jinja.in
View file @ 3caed389
[buildout]
{%- set slapparameter_dict = dict(default_parameter_dict, **slapparameter_dict) -%}
[buildout]
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
......@@ -18,6 +20,7 @@ parts =
resiliency-exclude-file
shellinabox-frontend-reload
promises
testnode-compatibility
[monitor-publish]
recipe = slapos.cookbook:publish
......@@ -35,20 +38,24 @@ slapos-directory = $${directory:slapos}
working-directory = $${directory:testnode}
test-suite-directory = $${directory:test-suite}
shared-part-list = $${directory:shared}
proxy-host = $${slap-network-information:local-ipv4}
proxy-host = {{ partition_ipv4 }}
proxy-port = 5000
log-directory = $${directory:log}
srv-directory = $${rootdirectory:srv}
software-directory = $${directory:software}
run-directory = $${directory:run}
test-node-title = $${slap-parameter:test-node-title}
node-quantity = $${slap-parameter:node-quantity}
ipv4-address = $${slap-network-information:local-ipv4}
ipv6-address = $${slap-network-information:global-ipv6}
test-suite-master-url = $${slap-parameter:test-suite-master-url}
instance-dict = $${slap-parameter:instance-dict}
software-path-list = $${slap-parameter:software-path-list}
keep-log-days = $${slap-parameter:keep-log-days}
test-node-title = {{ slapparameter_dict['test-node-title'] }}
node-quantity = {{ slapparameter_dict['node-quantity'] }}
ipv4-address = {{ partition_ipv4 }}
ipv6-address = {{ partition_ipv6 }}
test-suite-master-url = {{ slapparameter_dict['test-suite-master-url'] }}
instance-dict = {{ slapparameter_dict['instance-dict'] }}
{%- if isinstance(slapparameter_dict['software-path-list'], str) %}
software-path-list = {{ slapparameter_dict['software-path-list'] }}
{%- else %}
software-path-list = {{ json.dumps(slapparameter_dict['software-path-list']) }}
{%- endif %}
keep-log-days = {{ slapparameter_dict['keep-log-days'] }}
git-binary = ${git:location}/bin/git
slapos-binary = ${buildout:bin-directory}/slapos
testnode = ${buildout:bin-directory}/testnode
......@@ -59,7 +66,7 @@ httpd-conf-file = $${rootdirectory:etc}/httpd.conf
httpd-wrapper = $${rootdirectory:bin}/httpd
httpd-port = 9080
httpd-software-access-port = 9081
httpd-ip = $${slap-network-information:global-ipv6}
httpd-ip = {{ partition_ipv6 }}
httpd-log-directory = $${basedirectory:log}
httpd-software-directory = $${directory:software}
httpd-cert-file = $${rootdirectory:etc}/httpd-public.crt
......@@ -117,7 +124,7 @@ template = inline:
/
}
}
ipv6 = $${slap-network-information:global-ipv6}
ipv6 = {{ partition_ipv6 }}
hostname = [$${:ipv6}]
port = 8080
username = testnode
......@@ -200,7 +207,7 @@ run = $${rootdirectory:var}/run
[directory]
recipe = slapos.cookbook:mkdirectory
slapos = $${rootdirectory:srv}/slapos
testnode = $${rootdirectory:srv}/testnode
testnode = $${buildout:directory}/t
shared = $${rootdirectory:srv}/shared
test-suite = $${rootdirectory:srv}/test_suite
log = $${basedirectory:log}/testnode
......@@ -209,6 +216,16 @@ software = $${rootdirectory:srv}/software
shellinabox = $${rootdirectory:srv}/shellinabox
ca-dir = $${rootdirectory:srv}/ca
[testnode-compatibility]
# Remove old ~/srv/testnode
recipe = slapos.recipe.build
update =
import os
from zc.buildout.rmtree import rmtree
old_testnode_path = self.buildout['rootdirectory']['srv'] + '/testnode'
if os.path.exists(old_testnode_path):
rmtree(old_testnode_path)
[resiliency-exclude-file]
# Generate rdiff exclude file in case of resiliency
recipe = collective.recipe.template
......@@ -239,6 +256,7 @@ config-url = https://[$${testnode:httpd-ip}]:$${testnode:httpd-port}
recipe =
instance-promises =
$${shellinabox-frontend-listen-promise:name}
# $${shellinabox-frontend-available-promise:name}
$${testnode-log-frontend-promise:name}
[shellinabox-frontend-listen-promise]
......@@ -248,15 +266,18 @@ name = $${:_buildout_section_name_}.py
config-host = $${shellinabox-frontend:hostname}
config-port = $${shellinabox-frontend:port}
## This promise fails in a test suite
## due to ports conflict with the testnode of the test suite
# [shellinabox-frontend-available-promise]
# <= monitor-promise-base
# module = check_url_available
# name = $${:_buildout_section_name_}.py
# config-url = https://$${shellinabox-frontend-config:hostname}:$${shellinabox-frontend-config:port}
# config-username = $${shellinabox-frontend-config:username}
# config-password = $${shellinabox-frontend-config:passwd}
[testnode-log-frontend-promise]
<= monitor-promise-base
module = check_url_available
name = $${:_buildout_section_name_}.py
config-url = $${testnode-log-frontend:connection-secure_access}
[slap-parameter]
node-quantity = 1
test-suite-master-url =
instance-dict =
software-path-list = ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.181/software/seleniumrunner/software.cfg"]
keep-log-days = 15
software/erp5testnode/instance.cfg
View file @ 3caed389
......@@ -6,6 +6,34 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${template-default:output}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:default}
default = instance-default:rendered
[instance-default]
recipe = slapos.recipe.template:jinja2
template = ${template-default:output}
rendered = $${buildout:directory}/instance-default.cfg
context =
import json json
jsonkey default_parameter_dict :default-parameters
key slapparameter_dict slap-configuration:configuration
key partition_ipv6 slap-configuration:ipv6-random
key partition_ipv4 slap-configuration:ipv4-random
default-parameters =
{
"node-quantity": 1,
"test-suite-master-url": "",
"instance-dict": "",
"software-path-list": ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.181/software/seleniumrunner/software.cfg"],
"keep-log-days": 15
}
software/erp5testnode/software.cfg
View file @ 3caed389
......@@ -25,20 +25,20 @@ parts =
[eggs]
recipe = zc.recipe.egg
eggs =
erp5.util[testnode]
erp5.util[testnode,benchmark,scalability_tester]
${lxml-python:egg}
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 0644
[template-default]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-default.cfg
output = ${buildout:directory}/template-default.cfg
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template-default.cfg.jinja
mode = 0644
[versions]
......@@ -52,23 +52,9 @@ soupsieve = 1.8
waitress = 1.2.1
z3c.etestbrowser = 2.0.1
zope.testbrowser = 5.3.2
# Required by:
# zope.testbrowser==5.3.2
WSGIProxy2 = 0.4.6
# Required by:
# WebTest==2.0.33
beautifulsoup4 = 4.7.1
# Required by:
# zope.testbrowser==5.3.2
zope.cachedescriptors = 4.3.1
# Required by:
# zope.schema==4.9.3
zope.event = 4.4
# Required by:
# zope.testbrowser==5.3.2
zope.schema = 4.9.3
slapos.tool.nosqltester = 0.0.4.dev-r45972
software/erp5testnode/test/README.md 0 → 100644
View file @ 3caed389
Tests for erp5testnode software release
software/erp5testnode/test/setup.py 0 → 100644
View file @ 3caed389
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.erp5testnode'
with open("README.md") as f:
long_description = f.read()
setup(
name=name,
version=version,
description="Test for SlapOS' erp5testnode",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'requests',
],
zip_safe=True,
test_suite='test',
)
software/erp5testnode/test/test.py 0 → 100644
View file @ 3caed389
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestnodeTest(SlapOSInstanceTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {"test-node-title": "hello"}
def test(self):
connexion_parameters = self.computer_partition.getConnectionParameterDict()
self.assertIn('url', connexion_parameters)
self.assertIn('frontend-url', connexion_parameters)
self.assertIn('log-frontend-url', connexion_parameters)
software/fluentd/software.cfg
View file @ 3caed389
......@@ -23,4 +23,4 @@ context =
[fluentd]
gems +=
fluent-plugin-wendelin==0.4
fluent-plugin-bin
fluent-plugin-bin==0.2
software/gitlab/buildout.hash.cfg
View file @ 3caed389
......@@ -14,7 +14,7 @@
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = dc3f318e8a3aa7a59f9394118543e9e3
md5sum = 47e8092404feaf3f28ee6469523689ac
[watcher]
_update_hash_filename_ = watcher.in
......@@ -54,7 +54,7 @@ md5sum = 0f1ec4077dab586cc003ae13f689eda2
[instance-gitlab.cfg.in]
_update_hash_filename_ = instance-gitlab.cfg.in
md5sum = 64ec65b2daa0648453022f3afcbc4da3
md5sum = 6b34d4b96ae0067977fa509046d71231
[instance-gitlab-export.cfg.in]
_update_hash_filename_ = instance-gitlab-export.cfg.in
......
software/gitlab/instance-gitlab.cfg.in
View file @ 3caed389
......@@ -5,7 +5,6 @@
# throughput compared to tcp over loopback).
[buildout]
extends =
{{ gitlab_parameters_cfg }}
{{ monitor_template }}
parts =
directory
......@@ -50,26 +49,15 @@ offline = true
##################################
[instance-parameter]
# std stuff to fetch slapos instance parameters
recipe = slapos.cookbook:slapconfiguration
computer= ${slap-connection:computer-id}
partition=${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
{#- There are dangerous keys like recipe, etc #}
{#- XXX: Some other approach would be useful #}
{%- set DROP_KEY_LIST = ['recipe', '__buildout_signature__', 'computer', 'partition', 'url', 'key', 'cert'] %}
{%- for key, value in instance_parameter_dict.iteritems() -%}
{%- if key not in DROP_KEY_LIST %}
{{ key }} = {{ value }}
{%- endif -%}
{%- endfor %}
# autogenerated gitlab instance parameters
<= gitlab-parameters
# adjust/override some default settings:
# automatically load all available CPUs
configuration.unicorn_worker_processes = {{ multiprocessing.cpu_count() + 1 }}
configuration.nginx_worker_processes = {{ multiprocessing.cpu_count() }}
# gitlab non-native parameters
configuration.icp_license =
# for convenience
......
software/gitlab/instance.cfg.in
View file @ 3caed389
# GitLab "switch-softwaretype" instance
[buildout]
parts = switch-softwaretype
extends =
${gitlab-parameters.cfg:target}
parts =
switch-softwaretype
# std stuff for slapos instance
eggs-directory = ${buildout:eggs-directory}
......@@ -9,14 +12,46 @@ offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
gitlab = $${instance-gitlab.cfg:rendered}
gitlab-export = $${instance-gitlab-export.cfg:rendered}
gitlab-test = $${instance-gitlab-test.cfg:rendered}
default = $${:gitlab}
recipe = slapos.cookbook:switch-softwaretype
gitlab = instance-gitlab.cfg:rendered
gitlab-export = instance-gitlab-export.cfg:rendered
gitlab-test = instance-gitlab-test.cfg:rendered
RootSoftwareInstance = $${:gitlab}
# TODO -import, -pull-backup
[worker-processes]
recipe = slapos.recipe.build
init =
import multiprocessing
cpu_count = multiprocessing.cpu_count()
# automatically load all available CPUs
options['unicorn-worker-processes'] = cpu_count + 1
options['nginx-worker-processes'] = cpu_count
[slap-configuration]
# std stuff to fetch slapos instance parameters
recipe = slapos.cookbook:slapconfiguration
computer= $${slap-connection:computer-id}
partition=$${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
# autogenerated gitlab instance parameters
<= gitlab-parameters
# adjust/override some default settings:
configuration.unicorn_worker_processes = $${worker-processes:unicorn-worker-processes}
configuration.nginx_worker_processes = $${worker-processes:nginx-worker-processes}
# gitlab non-native parameters
configuration.icp_license =
# macro: render instance-*.cfg from instance-*.cfg.in
[instance-cfg]
recipe = slapos.recipe.template:jinja2
......@@ -25,13 +60,13 @@ rendered= $${buildout:directory}/$${:_buildout_section_name_}
context =
import os os
import pwd pwd
import multiprocessing multiprocessing
key bin_directory buildout:bin-directory
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
raw gitlab_repository_location ${gitlab-repository:location}
raw gitlab_shell_repository_location ${gitlab-shell-repository:location}
section instance_parameter_dict slap-configuration
# program binaries
raw bash_bin ${bash:location}/bin/bash
......@@ -67,7 +102,6 @@ context =
# config files
raw database_yml_in ${database.yml.in:target}
raw gitconfig_in ${gitconfig.in:target}
raw gitlab_parameters_cfg ${gitlab-parameters.cfg:target}
raw monitor_template ${monitor2-template:rendered}
raw gitlab_shell_config_yml_in ${gitlab-shell-config.yml.in:target}
raw gitlab_unicorn_startup_in ${gitlab-unicorn-startup.in:target}
......
software/headless-chromium/README.md 0 → 100644
View file @ 3caed389
# Headless Chromium
This software release compiles and runs a headless Chromium shell and
exposes an interface to connect to it remotely from another browser.
After deployment, the instance is configured like this:
```
Caddy frontend
|
(HTTPS, IPv6)
|
Nginx proxy, basic authentication
|
(HTTP, IPv4)
|
Chromium shell
```
The proxy is necessary because Chromium only accepts local connections
for remote debugging.
## Parameters
The following instance parameters can be configured:
- target-url: URL for Chromium to load on startup.
- remote-debugging-port: Port for Chromium to listen on.
- nginx-proxy-port: Port for Ningx proxy to listen on.
- monitor-httpd-port: Port for monitor.
See `instance-headless-chromium-input-schema.json` for default values.
software/headless-chromium/buildout.hash.cfg 0 → 100644
View file @ 3caed389
[template-cfg]
filename = instance.cfg.in
md5sum = 5dfeeb5eca125dcaa5f9e537f941dd41
[instance-headless-chromium]
_update_hash_filename_ = instance-headless-chromium.cfg.in
md5sum = fad685238b26ca20537c12ce7432e7e7
[template-nginx-conf]
_update_hash_filename_ = templates/nginx.conf.in
md5sum = c4d09d2b819f624087ef4c38551dfe2f
[template-mime-types]
_update_hash_filename_ = templates/mime-types.in
md5sum = 4ef94a7b458d885cd79ba0b930a5727e
software/headless-chromium/instance-headless-chromium-input-schema.json 0 → 100644
View file @ 3caed389
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"target-url": {
"description": "URL for Chromium to load on startup.",
"title": "Target URL",
"type": "string",
"default": "https://www.example.com"
},
"remote-debugging-port": {
"description": "Port for Chromium to listen on.",
"title": "Remote Debugging Port",
"type": "integer",
"default": 8081
},
"nginx-proxy-port": {
"description": "Port for Nginx proxy to listen on.",
"title": "Nginx Proxy Port",
"type": "integer",
"default": 8082
},
"monitor-httpd-port": {
"description": "Port for monitor frontend.",
"title": "Monitor Httpd Port",
"type": "integer",
"default": 8083
}
}
}
software/headless-chromium/instance-headless-chromium-output-schema.json 0 → 100644
View file @ 3caed389
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Values returned by headless Chromium instantiation",
"properties": {
"frontend-url": {
"description": "URL to access remote debugging interface",
"type": "string"
},
"username": {
"description": "Username for remote debugging interface",
"type": "string"
},
"password": {
"description": "Password for remote debugging interface",
"type": "string"
},
"monitor-base-url": {
"description": "Base URL used by monitor",
"type": "string"
},
"monitor-setup-url": {
"description": "One-click link to setup and monitor feeds",
"type": "string"
},
"proxy-url": {
"description": "Raw IPv6 address used by Nginx proxy",
"type": "string"
},
"remote-debug-url": {
"description": "Local IPv4 address used by Chromium",
"type": "string"
}
}
}
software/headless-chromium/instance-headless-chromium.cfg.in 0 → 100644
View file @ 3caed389
{% set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
[buildout]
parts =
chromium-launcher
generate-passwd-file
nginx-config
nginx-mime-types
nginx-launcher
logrotate-entry-nginx
publish-connection-information
frontend-ok-promise
frontend-secure-promise
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
extends = {{ parameter_list['template-monitor'] }}
# Create necessary directories.
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
tmp = ${:home}/tmp
log = ${:home}/log
etc = ${:home}/etc
ssl = ${:etc}/ssl
service = ${:etc}/service
# Options for instance configuration. See README.md for a list of
# options that can be configured when requesting an instance.
[headless-chromium]
ipv4 = {{ partition_ipv4 }}
ipv6 = {{ partition_ipv6 }}
remote-debugging-port = {{ parameter_dict['remote-debugging-port'] }}
url = {{ parameter_dict['target-url'] }}
remote-debugging-address = ${:ipv4}:${:remote-debugging-port}
devtools-frontend-root = {{ parameter_list['devtools-frontend'] }}
nginx-port = {{ parameter_dict['nginx-proxy-port'] }}
proxy-address = [${:ipv6}]:${:nginx-port}
nginx-config-target = ${directory:etc}/nginx.conf
nginx-pid-path = ${directory:log}/nginx.pid
nginx-temp-path = ${directory:tmp}
nginx-error-log = ${directory:log}/nginx-error.log
nginx-access-log = ${directory:log}/nginx-access.log
nginx-htpasswd-file = ${directory:etc}/.htpasswd
nginx-key-file = ${frontend-instance-certificate:key-file}
nginx-cert-file = ${frontend-instance-certificate:cert-file}
nginx-mime-types = ${directory:etc}/mime-types
# Create a launcher script in /etc/service for the headless shell
# executable.
[chromium-launcher]
recipe = slapos.recipe.template:jinja2
template =
inline:#!/bin/sh
export FONTCONFIG_FILE=${font-config:rendered}
exec {{ parameter_list['chromium-wrapper'] }} \
--remote-debugging-address=${headless-chromium:ipv4} \
--remote-debugging-port=${headless-chromium:remote-debugging-port} \
${headless-chromium:url}
rendered = ${directory:service}/chromium
# Configure and launch the proxy server.
[nginx-config]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_list['template-nginx-config'] }}
rendered = ${headless-chromium:nginx-config-target}
mode = 700
context =
section param_headless_chromium headless-chromium
[nginx-mime-types]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_list['template-mime-types'] }}
rendered = ${headless-chromium:nginx-mime-types}
[nginx-launcher]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_list['nginx-location'] }}/sbin/nginx -c ${headless-chromium:nginx-config-target}
wrapper-path = ${directory:service}/nginx
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
log = ${headless-chromium:nginx-error-log} ${headless-chromium:nginx-access-log}
[frontend-instance-password]
recipe = slapos.cookbook:generate.password
username = admin
bytes = 12
[generate-passwd-file]
recipe = plone.recipe.command
command =
echo -n '${frontend-instance-password:username}:' > ${headless-chromium:nginx-htpasswd-file}
openssl passwd -apr1 '${frontend-instance-password:passwd}' >> ${headless-chromium:nginx-htpasswd-file}
environment =
PATH={{ parameter_list['openssl-location'] }}/bin:%(PATH)s
# Generate a self-signed TLS certificate.
[frontend-instance-certificate]
recipe = plone.recipe.command
command =
if [ ! -e ${:key-file} ]
then
openssl req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=${:common-name}" \
-newkey rsa:1024 -keyout ${:key-file} \
-out ${:cert-file}
openssl x509 -addtrust serverAuth \
-in ${:cert-file} \
-out ${:cert-file}
fi
update-command = ${:command}
key-file = ${directory:ssl}/${:_buildout_section_name_}.key
cert-file = ${directory:ssl}/${:_buildout_section_name_}.cert
common-name = ${headless-chromium:ipv6}
environment =
PATH={{ parameter_list['openssl-location'] }}/bin:%(PATH)s
# Generate a fonts.conf file.
[font-config]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_list['template-fonts-conf'] }}
rendered = ${directory:etc}/fonts.conf
context =
key cachedir :cache-dir
key fonts :fonts
key includes :includes
cache-dir =
${directory:etc}/.fontconfig.cache
fonts =
{{ parameter_list['liberation-fonts-location'] }}
includes =
{{ parameter_list['fontconfig-location'] }}/etc/fonts/conf.d
[publish-connection-information]
recipe = slapos.cookbook:publish
<= monitor-publish
remote-debug-url = http://${headless-chromium:remote-debugging-address}
proxy-url = https://${headless-chromium:proxy-address}
frontend-url = ${remote-debugging-frontend:connection-secure_access}
username = ${frontend-instance-password:username}
password = ${frontend-instance-password:passwd}
# Request a frontend URL from the CDN for the remote debugging interface.
[remote-debugging-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Headless Chromium Remote Debugging Frontend
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = https://${headless-chromium:proxy-address}
config-https-only = true
config-type = websocket
config-websocket-path-list = /devtools
return = domain secure_access
# Monitoring: check that the Chromium process is alive and responding to
# requests through the proxy.
[monitor-instance-parameter]
monitor-httpd-port = {{ parameter_dict['monitor-httpd-port'] }}
# Promise to make sure the remote debugging frontend returns 200 when
# queried with the correct credentials.
[frontend-ok-promise]
<= monitor-promise-base
module = check_url_available
name = headless-chromium-frontend-ok.py
url = ${remote-debugging-frontend:connection-secure_access}
config-url = ${:url}
config-username = ${frontend-instance-password:username}
config-password = ${frontend-instance-password:passwd}
# Promise to make sure that the remote debugging frontend returns 401
# when queried with no credentials.
[frontend-secure-promise]
<= monitor-promise-base
module = check_url_available
name = headless-chromium-frontend-secure.py
url = ${remote-debugging-frontend:connection-secure_access}
config-url = ${:url}
config-http-code = 401
software/headless-chromium/instance.cfg.in 0 → 100644
View file @ 3caed389
[buildout]
parts =
switch-softwaretype
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
[profile-common]
openssl-location = {{ openssl_location }}
nginx-location = {{ nginx_location }}
liberation-fonts-location = {{ liberation_fonts_location }}
fontconfig-location = {{ fontconfig_location }}
chromium-wrapper = {{ chromium_wrapper }}
devtools-frontend = {{ devtools_frontend }}
template-nginx-config = {{ template_nginx_config_target }}
template-fonts-conf = {{ template_fonts_conf_target }}
template-monitor = {{ template_monitor }}
template-mime-types = {{ template_mime_types_target }}
[instance-headless-chromium]
recipe = slapos.recipe.template:jinja2
template = {{ template_instance_headless_chromium_target }}
rendered = ${buildout:directory}/${:filename}
filename = instance-headless-chromium.cfg
context =
section buildout buildout
section parameter_list profile-common
key partition_ipv4 slap-configuration:ipv4-random
key partition_ipv6 slap-configuration:ipv6-random
key slapparameter_dict slap-configuration:configuration
jsonkey default_parameter_dict :default-parameters
default-parameters =
{
"remote-debugging-port": 8081,
"nginx-proxy-port": 8082,
"target-url": "https://www.example.com",
"monitor-httpd-port": 8083
}
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = ${:default}
default = instance-headless-chromium:rendered
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
software/headless-chromium/software.cfg 0 → 100644
View file @ 3caed389
[buildout]
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
../../component/headless-chromium/buildout.cfg
../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg
../../component/fonts/buildout.cfg
../../component/fontconfig/buildout.cfg
parts =
slapos-cookbook
template-cfg
[python]
part = python3
[template-cfg]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/${:filename}
mode = 0644
context =
section buildout buildout
key openssl_location openssl:location
key nginx_location nginx:location
key liberation_fonts_location liberation-fonts:location
key fontconfig_location fontconfig:location
key chromium_wrapper headless-chromium-wrapper:rendered
key devtools_frontend headless-chromium:devtools-frontend
key template_nginx_config_target template-nginx-conf:target
key template_mime_types_target template-mime-types:target
key template_fonts_conf_target template-fonts-conf:output
key template_instance_headless_chromium_target instance-headless-chromium:target
key template_monitor monitor2-template:rendered
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[instance-headless-chromium]
<= download-base
[template-nginx-conf]
<= download-base
[template-mime-types]
<= download-base
software/headless-chromium/software.cfg.json 0 → 100644
View file @ 3caed389
{
"name": "Headless Chromium",
"description": "Headless (stripped-down) Chromium shell",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"description": "Standalone headless shell",
"request": "instance-headless-chromium-input-schema.json",
"response": "instance-headless-chromium-output-schema.json",
"index": 0
}
}
}
software/headless-chromium/templates/mime-types.in 0 → 100644
View file @ 3caed389
types {
text/html html htm shtml;
text/css css;
text/xml xml rss;
image/gif gif;
image/jpeg jpeg jpg;
application/x-javascript js;
application/atom+xml atom;
text/mathml mml;
text/plain txt;
text/vnd.sun.j2me.app-descriptor jad;
text/vnd.wap.wml wml;
text/x-component htc;
image/png png;
image/tiff tif tiff;
image/vnd.wap.wbmp wbmp;
image/x-icon ico;
image/x-jng jng;
image/x-ms-bmp bmp;
image/svg+xml svg svgz;
application/java-archive jar war ear;
application/mac-binhex40 hqx;
application/msword doc;
application/pdf pdf;
application/postscript ps eps ai;
application/rtf rtf;
application/vnd.ms-excel xls;
application/vnd.ms-powerpoint ppt;
application/vnd.wap.wmlc wmlc;
application/vnd.google-earth.kml+xml kml;
application/vnd.google-earth.kmz kmz;
application/x-7z-compressed 7z;
application/x-cocoa cco;
application/x-java-archive-diff jardiff;
application/x-java-jnlp-file jnlp;
application/x-makeself run;
application/x-perl pl pm;
application/x-pilot prc pdb;
application/x-rar-compressed rar;
application/x-redhat-package-manager rpm;
application/x-sea sea;
application/x-shockwave-flash swf;
application/x-stuffit sit;
application/x-tcl tcl tk;
application/x-x509-ca-cert der pem crt;
application/x-xpinstall xpi;
application/xhtml+xml xhtml;
application/zip zip;
application/octet-stream bin exe dll;
application/octet-stream deb;
application/octet-stream dmg;
application/octet-stream eot;
application/octet-stream iso img;
application/octet-stream msi msp msm;
application/ogg ogx;
audio/midi mid midi kar;
audio/mpeg mpga mpega mp2 mp3 m4a;
audio/ogg oga ogg spx;
audio/x-realaudio ra;
audio/webm weba;
video/3gpp 3gpp 3gp;
video/mp4 mp4;
video/mpeg mpeg mpg mpe;
video/ogg ogv;
video/quicktime mov;
video/webm webm;
video/x-flv flv;
video/x-mng mng;
video/x-ms-asf asx asf;
video/x-ms-wmv wmv;
video/x-msvideo avi;
}
software/headless-chromium/templates/nginx.conf.in 0 → 100644
View file @ 3caed389
pid {{ param_headless_chromium['nginx-pid-path'] }};
error_log {{ param_headless_chromium['nginx-error-log'] }};
events {
worker_connections 1024;
}
http {
access_log {{ param_headless_chromium['nginx-access-log'] }};
include {{ param_headless_chromium['nginx-mime-types'] }};
default_type application/octet-stream;
types {
text/html html;
text/css css;
application/javascript js;
}
server {
listen {{ param_headless_chromium['proxy-address'] }} ssl;
# Require username/password to access remote debugging port.
auth_basic "Remote Debugging";
auth_basic_user_file {{ param_headless_chromium['nginx-htpasswd-file'] }};
# Use self-signed SSL certificate.
ssl_certificate {{ param_headless_chromium['nginx-cert-file'] }};
ssl_certificate_key {{ param_headless_chromium['nginx-key-file'] }};
client_body_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
proxy_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
fastcgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
uwsgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
scgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
# All websocket connections are served from /devtools.
location /devtools {
proxy_http_version 1.1;
proxy_set_header Host {{ param_headless_chromium['remote-debugging-address'] }};
proxy_pass http://{{ param_headless_chromium['remote-debugging-address'] }};
proxy_set_header Upgrade "websocket";
proxy_set_header Connection "Upgrade";
}
# The DevTools frontend is served from /serve_file/@{version_hash}.
location ~ "^\/serve_file\/@[0-9a-f]{5,40}\/(.*)" {
alias {{ param_headless_chromium['devtools-frontend-root'] }}/$1;
}
location / {
proxy_http_version 1.1;
# The proxy must set the Host header to an IP address, since the
# headless Chromium shell refuses to run otherwise, for security
# reasons.
# See https://bugs.chromium.org/p/chromium/issues/detail?id=813540.
proxy_set_header Host {{ param_headless_chromium['remote-debugging-address'] }};
proxy_pass http://{{ param_headless_chromium['remote-debugging-address'] }};
# The browser security policy will prevent us from loading the
# Websocket connection without TLS, so we have to go through the
# frontend CDN URL. The tricky thing is that the frontend URL is
# not available yet when this file is built; what we do instead is
# use the given Host header.
sub_filter "ws={{ param_headless_chromium['remote-debugging-address'] }}" "wss=$host";
sub_filter_once on;
sub_filter_types application/json;
sub_filter "ws://{{ param_headless_chromium['remote-debugging-address'] }}" "wss://$host";
sub_filter_types application/json;
# We want to use our own DevTools frontend rather than
# https://chrome-devtools-frontend.appspot.com. There should be a
# --custom-devtools-frontend flag for Chromium, but it doesn't
# seem to work with the remote debugging port.
sub_filter "chrome-devtools-frontend.appspot.com" "$host";
sub_filter_types *;
}
}
}
software/headless-chromium/test/README.md 0 → 100644
View file @ 3caed389
Tests for headless Chromium software release
software/headless-chromium/test/setup.py 0 → 100644
View file @ 3caed389
##############################################################################
#
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.headless-chromium'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS headless Chromium",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
],
zip_safe=True,
test_suite='test',
)
software/headless-chromium/test/test.py 0 → 100644
View file @ 3caed389
##############################################################################
#
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '../software.cfg')))
class TestHeadlessChromium(SlapOSInstanceTestCase):
def setUp(self):
self.connection_parameters = self.requestDefaultInstance().getConnectionParameterDict()
def test_remote_debugging_port(self):
# The headless browser should respond at /json with a nonempty list
# of available pages, each of which has a webSocketDebuggerUrl and a
# devtoolsFrontendUrl.
url = self.connection_parameters['remote-debug-url']
response = requests.get('%s/json' % url)
# Check that request was successful and the response was a nonempty
# list.
self.assertEqual(requests.codes['ok'], response.status_code)
self.assertTrue(len(response.json()) > 0)
# Check that the first page has the correct fields.
first_page = response.json()[0]
self.assertIn('webSocketDebuggerUrl', first_page)
self.assertIn('devtoolsFrontendUrl', first_page)
def test_devtools_frontend_ok(self):
# The proxy should serve the DevTools frontend from
# /serve_file/@{hash}/inspector.html, where {hash} is a 5-32 digit
# hash.
proxyURL = self.connection_parameters['proxy-url']
username = self.connection_parameters['username']
password = self.connection_parameters['password']
frontend = '/serve_file/@aaaaa/inspector.html'
response = requests.get(proxyURL + frontend, verify=False,
auth=(username, password))
self.assertEqual(requests.codes['ok'], response.status_code)
software/jupyter/software.cfg
View file @ 3caed389
......@@ -91,78 +91,27 @@ tornado = 6.1
traitlets = 5.0.5
webencodings = 0.5.1
widgetsnbextension = 2.0.0
# Required by:
# tornado==4.4.2
certifi = 2020.6.20
# Required by:
# notebook==6.1.5
Send2Trash = 1.5.0
# Required by:
# notebook==6.1.5
argon2-cffi = 20.1.0
# Required by:
# notebook==6.1.5
nbconvert = 6.0.7
# Required by:
# ipython==5.3.0
pathlib2 = 2.2.1
# Required by:
# statsmodels==0.11.1
patsy = 0.5.1
# Required by:
# ipython==5.3.0
pexpect = 4.8.0
# Required by:
# ipython==5.3.0
pickleshare = 0.7.4
# Required by:
# notebook==6.1.5
prometheus-client = 0.9.0
# Required by:
# pathlib2==2.2.1
scandir = 1.5
# Required by:
# statsmodels==0.11.1
pytz = 2020.4
# Required by:
# statsmodels==0.11.1
scipy = 1.0.1
# Required by:
# tornado==6.1
singledispatch = 3.4.0.3
# Required by:
# prompt-toolkit==1.0.13
wcwidth = 0.1.7
jupyter = 1.0.0
jupyter-console = 5.1.0
# Required by:
# jupyter==1.0.0
qtconsole = 4.3.0
et-xmlfile = 1.0.1
h5py = 2.7.1
mpmath = 1.0.0
openpyxl = 2.5.2
sympy = 1.1.1
xlrd = 1.1.0
# Required by:
# openpyxl==2.5.2
jdcal = 1.4
software/monitor/buildout.hash.cfg
View file @ 3caed389
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = c115ed9d4ff0f785d79cdcacbb0bd1ad
md5sum = 2114ae8c1e92bd33ef1347f36f567c74
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
......
software/monitor/instance.cfg
View file @ 3caed389
......@@ -6,10 +6,11 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${instance-base-monitor:rendered}
edgetest = $${instance-base-edgetest:rendered}
edgebot = $${instance-base-edgebot:rendered}
recipe = slapos.cookbook:switch-softwaretype
default = instance-base-monitor:rendered
edgetest = instance-base-edgetest:rendered
edgebot = instance-base-edgebot:rendered
RootSoftwareInstance = $${:default}
[instance-base-monitor]
recipe = slapos.recipe.template:jinja2
......
software/neoppod/software-common.cfg
View file @ 3caed389
......@@ -132,11 +132,7 @@ ecdsa = 0.13
mysqlclient = 1.3.12
pycrypto = 2.6.1
pycurl = 7.43.0
setproctitle = 1.1.10
cython-zstd = 0.2
# Required by:
# mock = 3.0.5
funcsigs = 1.0.2
......
software/nginx-push-stream/README.md 0 → 100644
View file @ 3caed389
# Nginx Push Stream
This software uses the [HTTP Push Stream](https://www.nginx.com/resources/wiki/modules/push_stream/)
module of nginx to make a [Server Sent Event](https://html.spec.whatwg.org/multipage/server-sent-events.html)
server.
Two endpoints are available, published as the following connection parameters:
- `publisher-url`, which uses`/pub{?id}` format, with `id` being the ID of the channel.
Clients can subscribe to this URL and be notified of new messages.
- `subscriber-url`, which uses `/sub{/id}` format, with `id` being the ID of the channel.
`POST` requests to this URL send new messages to this channel.
software/nginx-push-stream/buildout.hash.cfg 0 → 100644
View file @ 3caed389
[template]
filename = instance.cfg.in
md5sum = f9b6d01e29f2edddd9d6f99591976c33
[template-nginx-configuration]
filename = template-nginx.cfg.in
md5sum = 022e4b53e1b2db16c4e518fe76f638fa
software/nginx-push-stream/instance-nginx.cfg.in deleted 100644 → 0
View file @ ba4ea4df
[buildout]
parts =
nginx-service
htpasswd
htpasswd-runner
publish-connection-information
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
run = $${:var}/run
log = $${:var}/log
varnginx = $${:var}/nginx
services = $${:etc}/service
cron-entries = $${:etc}/cron.d
www = $${:srv}/www
ssl = $${:etc}/ssl
#################################
# Nginx service
#################################
[nginx-service]
recipe = slapos.recipe.template
url = ${template-nginx-service:output}
output = $${directory:services}/nginx
mode = 0700
virtual-depends =
$${nginx-configuration:ip}
[nginx-configuration]
recipe = slapos.recipe.template
url = ${template-nginx-configuration:output}
output = $${directory:etc}/nginx.cfg
mode = 0600
access_log = $${directory:log}/nginx-access.log
error_log = $${directory:log}/nginx-error.log
ip = $${slap-network-information:global-ipv6}
local_ip = $${slap-network-information:local-ipv4}
port = 9443
publisher_location_prefix = /pub
publisher_push_stream_store_messages = off
publisher_client_max_body_size = 16k
publisher_client_body_buffer_size = 16k
subscriber_allow_origin = '*'
subscriber_location_prefix = /sub
# Prevent to use credential if origin is star
subscriber_allow_credential = 'false'
subscriber_allow_methods = 'GET, HEAD, OPTIONS'
subscriber_allow_headers = 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since'
[htpasswd]
recipe = slapos.cookbook:generate.password
storage-path = $${directory:etc}/.pwd
bytes = 8
[htpasswd-runner]
recipe = plone.recipe.command
stop-on-error = true
htpasswd-path = $${directory:etc}/.htpasswd
command = if [ ! -f "$${:htpasswd-path}" ]; then ${buildout:bin-directory}/htpasswd -cb $${:htpasswd-path} $${:user} $${:password}; fi
update-command = $${:command}
user = admin
password = $${htpasswd:passwd}
[publish-connection-information]
recipe = slapos.cookbook:publish
init-password = $${htpasswd:passwd}
init-user = $${htpasswd-runner:user}
publisher-url = http://$${htpasswd-runner:user}:$${htpasswd:passwd}@[$${nginx-configuration:ip}]:$${nginx-configuration:port}$${nginx-configuration:publisher_location_prefix}
subscriber-url = http://$${htpasswd-runner:user}:$${htpasswd:passwd}@[$${nginx-configuration:ip}]:$${nginx-configuration:port}$${nginx-configuration:subscriber_location_prefix}
software/nginx-push-stream/instance.cfg.in
View file @ 3caed389
[buildout]
parts =
switch-softwaretype
nginx-service
promises
publish-connection-information
extends = ${monitor-template:rendered}
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${:nginx}
nginx = $${dynamic-template-nginx:rendered}
[dynamic-template-nginx]
recipe = slapos.recipe.template:jinja2
template = ${template-nginx:output}
rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
filename = instance-nginx.cfg
[slap-connection]
computer-id = $${slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
[instance-parameter]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap_connection:computer_id}
partition = $${slap_connection:partition_id}
url = $${slap_connection:server_url}
key = $${slap_connection:key_file}
cert = $${slap_connection:cert_file}
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
run = $${:var}/run
log = $${:var}/log
varnginx = $${:var}/nginx
services = $${:etc}/service
cron-entries = $${:etc}/cron.d
www = $${:srv}/www
ssl = $${:etc}/ssl
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
#################################
# Nginx service
#################################
[nginx-service]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/nginx
command-line =
${nginx-push-stream-output:nginx} -c $${nginx-configuration:output}
[nginx-configuration]
recipe = slapos.recipe.template
url = ${template-nginx-configuration:output}
output = $${directory:etc}/nginx.cfg
mode = 0600
access-log = $${directory:log}/nginx-access.log
error-log = $${directory:log}/nginx-error.log
ip = $${slap-configuration:ipv6-random}
local-ip = $${slap-configuration:ipv4-random}
port = 9443
base-url = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}
# Generate a self-signed TLS certificate.
[nginx-certificate]
recipe = plone.recipe.command
command =
if [ ! -e $${:key-file} ]
then
${openssl:location}/bin/openssl req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=$${nginx-configuration:ip}" \
-newkey rsa:1024 -keyout $${:key-file} \
-out $${:cert-file}
fi
update-command = $${:command}
key-file = $${directory:ssl}/${:_buildout_section_name_}.key
cert-file = $${directory:ssl}/${:_buildout_section_name_}.cert
common-name = $${nginx-configuration:ip}
stop-on-error = true
[promises]
recipe =
promises =
$${nginx-available-promise:recipe}
[nginx-available-promise]
<= monitor-promise-base
module = check_url_available
name = $${:_buildout_section_name_}.py
config-url = $${nginx-configuration:base-url}/status
[publish-connection-information]
recipe = slapos.cookbook:publish
# publisher-url and subscriber-url are URITemplates, with an id
# parameter which is the ID of the channel.
publisher-url = $${nginx-configuration:base-url}/pub{?id}
subscriber-url = $${nginx-configuration:base-url}/sub{/id}
software/nginx-push-stream/software.cfg
View file @ 3caed389
[buildout]
extends =
../../stack/slapos.cfg
../../component/dash/buildout.cfg
../../component/nginx/buildout.cfg
../../stack/monitor/buildout.cfg
./buildout.hash.cfg
parts =
slapos-cookbook
......@@ -10,36 +11,18 @@ parts =
nginx-push-stream-module
nginx-push-stream
template
template-nginx-service
template-nginx
[python]
part = python3
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = eb4c69df9a8dbb94fb76d0a6c11e360f
output = ${buildout:directory}/template.cfg
mode = 0644
[template-nginx-service]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-nginx-service.sh.in
md5sum = 90ba5a247c489261d3304528cba56e06
output = ${buildout:directory}/template-nginx-service.sh.in
mode = 0644
[template-nginx-configuration]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-nginx.cfg.in
md5sum = f5658154b82282bc1871f18ddf4529d8
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template-nginx.cfg.in
mode = 0644
[template-nginx]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-nginx.cfg.in
md5sum = 936fea88f5548c4f14e287f1b27dc127
output = ${buildout:directory}/instance-nginx.cfg.in
mode = 0644
[versions]
inotifyx = 0.2.2
software/nginx-push-stream/template-nginx-service.sh.in deleted 100644 → 0
View file @ ba4ea4df
#!${dash-output:dash}
# BEWARE: This file is operated by slapos node
# BEWARE: It will be overwritten automatically
exec ${nginx-push-stream-output:nginx} \
-c $${nginx-configuration:output}
software/nginx-push-stream/template-nginx.cfg.in
View file @ 3caed389
......@@ -8,7 +8,7 @@ events {
# multi_accept on;
}
error_log $${nginx-configuration:error_log};
error_log $${nginx-configuration:error-log};
http {
......@@ -33,8 +33,8 @@ http {
# Logging Settings
##
access_log $${nginx-configuration:access_log};
error_log $${nginx-configuration:error_log};
access_log $${nginx-configuration:access-log};
error_log $${nginx-configuration:error-log};
##
# Gzip Settings
......@@ -54,9 +54,21 @@ http {
##
push_stream_shared_memory_size 32m;
server {
listen [$${nginx-configuration:ip}]:$${nginx-configuration:port};
listen $${nginx-configuration:local_ip}:$${nginx-configuration:port};
listen [$${nginx-configuration:ip}]:$${nginx-configuration:port} ssl http2;
listen $${nginx-configuration:local-ip}:$${nginx-configuration:port} ssl http2;
# generated 2021-08-02, Mozilla Guideline v5.6, nginx 1.19.2, OpenSSL 1.1.1k, modern configuration, no HSTS, no OCSP
# https://ssl-config.mozilla.org/#server=nginx&version=1.19.2&config=modern&openssl=1.1.1k&hsts=false&ocsp=false&guideline=5.6
ssl_certificate $${nginx-certificate:cert-file};
ssl_certificate_key $${nginx-certificate:key-file};
ssl_session_timeout 1d;
ssl_session_cache shared:MozSSL:10m;
ssl_session_tickets off;
ssl_protocols TLSv1.3;
ssl_prefer_server_ciphers off;
fastcgi_temp_path $${directory:varnginx} 1 2;
uwsgi_temp_path $${directory:varnginx} 1 2;
......@@ -65,42 +77,42 @@ server {
client_body_temp_path $${directory:varnginx} 1 2;
proxy_temp_path $${directory:varnginx} 1 2;
auth_basic "Nginx Access";
auth_basic_user_file $${htpasswd-runner:htpasswd-path};
## Serve an error 204 (No Content) for favicon.ico
location = /favicon.ico {
return 204;
}
location $${nginx-configuration:publisher_location_prefix} {
push_stream_publisher;
location = /status {
default_type "text/plain";
return 200 'OK';
}
push_stream_channels_path $arg_id;
# store messages in memory
push_stream_store_messages $${nginx-configuration:publisher_push_stream_store_messages};
location /pub {
push_stream_publisher;
# Message size limit
# client_max_body_size MUST be equal to client_body_buffer_size or
# you will be sorry.
client_max_body_size $${nginx-configuration:publisher_client_max_body_size};
client_body_buffer_size $${nginx-configuration:publisher_client_body_buffer_size};
push_stream_channels_path $arg_id;
# store messages in memory
push_stream_store_messages off;
}
# Message size limit
client_max_body_size 16k;
client_body_buffer_size 16k;
}
location ~ $${nginx-configuration:subscriber_location_prefix}/(.*) {
# activate subscriber mode for this location
add_header "Access-Control-Allow-Origin" $${nginx-configuration:subscriber_allow_origin};
add_header 'Access-Control-Allow-Credentials' $${nginx-configuration:subscriber_allow_credential};
add_header 'Access-Control-Allow-Methods' $${nginx-configuration:subscriber_allow_methods};
add_header 'Access-Control-Allow-Headers' $${nginx-configuration:subscriber_allow_headers};
location ~ /sub/(.*) {
# activate subscriber mode for this location
add_header "Access-Control-Allow-Origin" '*';
add_header 'Access-Control-Allow-Credentials' 'false';
add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
push_stream_subscriber eventsource;
# positional channel path
push_stream_channels_path $1;
push_stream_subscriber eventsource;
# positional channel path
push_stream_channels_path $1;
# content-type
default_type "text/event-stream; charset=utf-8";
# content-type
default_type "text/event-stream; charset=utf-8";
}
location / {
......
software/nginx-push-stream/test/README.md 0 → 100644
View file @ 3caed389
Tests for Nginx Push Stream software release
software/nginx-push-stream/test/setup.py 0 → 100644
View file @ 3caed389
##############################################################################
#
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.nginx_push_stream'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' Nginx Push Stream",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
'uritemplate',
],
zip_safe=True,
test_suite='test',
)
software/nginx-push-stream/test/test.py 0 → 100644
View file @ 3caed389
##############################################################################
# coding: utf-8
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import multiprocessing
import uritemplate
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestNginxPushStream(SlapOSInstanceTestCase):
def setUp(self):
self.connection_parameters = \
self.computer_partition.getConnectionParameterDict()
def test_push_stream_scenario(self):
def process_messages(q):
# type:(multiprocessing.Queue[bytes]) -> None
req = requests.get(
uritemplate.URITemplate(
self.connection_parameters['subscriber-url']).expand(
id='channel_id'),
verify=False,
stream=True,
)
if not req.ok:
q.put(('error: wrong status code %s' % req.status_code).encode())
q.put(b'ready')
for _, line in zip(range(2), req.iter_lines()):
q.put(line)
q = multiprocessing.Queue() # type: multiprocessing.Queue[bytes]
subscriber = multiprocessing.Process(target=process_messages, args=(q, ))
subscriber.start()
self.assertEqual(q.get(timeout=30), b'ready')
resp = requests.post(
uritemplate.URITemplate(
self.connection_parameters['publisher-url']).expand(
id='channel_id'),
verify=False,
data='Hello',
timeout=2,
)
resp.raise_for_status()
try:
subscriber.join(timeout=30)
except multiprocessing.TimeoutError:
subscriber.terminate()
subscriber.join(timeout=30)
self.fail('Process did not terminate')
self.assertEqual(q.get_nowait(), b': ')
self.assertEqual(q.get_nowait(), b'data: Hello')
software/slapos-sr-testing/buildout.hash.cfg
View file @ 3caed389
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum = 733643122fb75dfd7374b973a95fa2ea
md5sum = 2c8b202b0c366e27dd6e6f456fb18256
software/slapos-sr-testing/instance.cfg
View file @ 3caed389
......@@ -44,7 +44,7 @@ tests = {{ tests | indent(2) }}
template =
inline:{% raw %}
{%- set only_sr = slapparameter_dict.get('only-sr') %}
{%- if not isinstance(only_sr, list) %}
{%- if only_sr and not isinstance(only_sr, list) %}
{%- set only_sr = [only_sr] %}
{%- endif %}
{%- set unittest_args = slapparameter_dict.get('unittest-args', ['discover', '-v']) %}
......
software/slapos-sr-testing/software-py3.cfg
View file @ 3caed389
......@@ -22,3 +22,4 @@ extra =
proftpd ${slapos.test.proftpd-setup:setup}
repman ${slapos.test.repman-setup:setup}
restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
headless-chromium ${slapos.test.headless-chromium-setup:setup}
software/slapos-sr-testing/software.cfg
View file @ 3caed389
......@@ -149,6 +149,11 @@ setup = ${slapos-repository:location}/software/jupyter/test/
egg = slapos.test.nextcloud
setup = ${slapos-repository:location}/software/nextcloud/test/
[slapos.test.nginx-push-stream-setup]
<= setup-develop-egg
egg = slapos.test.nginx_push_stream
setup = ${slapos-repository:location}/software/nginx-push-stream/test/
[slapos.test.turnserver-setup]
<= setup-develop-egg
egg = slapos.test.turnserver
......@@ -194,16 +199,26 @@ setup = ${slapos-repository:location}/software/jscrawler/test/
egg = slapos.test.galene
setup = ${slapos-repository:location}/software/galene/test/
[slapos.core-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
[slapos.test.headless-chromium-setup]
<= setup-develop-egg
egg = slapos.test.headless-chromium
setup = ${slapos-repository:location}/software/headless-chromium/test/
[slapos.test.caucase-setup]
<= setup-develop-egg
egg = slapos.test.caucase
setup = ${slapos-repository:location}/software/caucase/test/
[slapos.test.erp5testnode-setup]
<= setup-develop-egg
egg = slapos.test.erp5testnode
setup = ${slapos-repository:location}/software/erp5testnode/test/
[slapos.core-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
[slapos.core-setup]
<= setup-develop-egg
egg = slapos.core
......@@ -242,6 +257,7 @@ extra-eggs =
${slapos.test.slaprunner-setup:egg}
${slapos.test.jupyter-setup:egg}
${slapos.test.nextcloud-setup:egg}
${slapos.test.nginx-push-stream-setup:egg}
${slapos.test.turnserver-setup:egg}
${slapos.test.theia-setup:egg}
${slapos.test.cloudooo-setup:egg}
......@@ -254,6 +270,8 @@ extra-eggs =
${slapos.test.html5as-setup:egg}
${slapos.test.html5as-base-setup:egg}
${slapos.test.fluentd-setup:egg}
${slapos.test.headless-chromium-setup:egg}
${slapos.test.erp5testnode-setup:egg}
# We don't name this interpreter `python`, so that when we run slapos node
# software, installation scripts running `python` use a python without any
......@@ -312,6 +330,7 @@ tests =
slaprunner ${slapos.test.slaprunner-setup:setup}
theia ${slapos.test.theia-setup:setup}
metabase ${slapos.test.metabase-setup:setup}
nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
###
${:extra}
......@@ -337,6 +356,7 @@ extra =
jscrawler ${slapos.test.jscrawler-setup:setup}
html5as ${slapos.test.html5as-setup:setup}
html5as-base ${slapos.test.html5as-base-setup:setup}
erp5testnode ${slapos.test.erp5testnode-setup:setup}
[versions]
# slapos.core is used from the clone always
......
software/theia/buildout.hash.cfg
View file @ 3caed389
......@@ -15,15 +15,39 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = 11d347dd2bf762902341746a388673a0
md5sum = 562acf69f344fa9f6d03992d696462d0
[instance]
_update_hash_filename_ = instance.cfg.in
md5sum = 063d3e19da9d3d4bfb77e8e638aa3a77
md5sum = a7d78b4002266c69ece05a476df82791
[instance-import]
_update_hash_filename_ = instance-import.cfg.jinja.in
md5sum = 861ef130f27175c2978a9b946b138dd5
[instance-export]
_update_hash_filename_ = instance-export.cfg.jinja.in
md5sum = b3cedaa1603ca8ed83fdd94ef4b35cc8
[instance-resilient]
_update_hash_filename_ = instance-resilient.cfg.jinja
md5sum = d78a9f885bdebf6720197209e0c21aa0
[theia-common]
_update_hash_filename_ = theia_common.py
md5sum = e57396473b4b6a17d26a747f0030293c
[theia-export]
_update_hash_filename_ = theia_export.py
md5sum = b5f5ac1924b27d3f2be2e5ea291c119e
[theia-import]
_update_hash_filename_ = theia_import.py
md5sum = 9e8c17a4b2d802695caf0c2c052f0d11
[yarn.lock]
_update_hash_filename_ = yarn.lock
md5sum = 80e7ad91deea54cebcccef5a83fdb380
md5sum = 18e8302b2acff3721cad23d829e3df55
[python-language-server-requirements.txt]
_update_hash_filename_ = python-language-server-requirements.txt
......
software/theia/download-plugins.cfg
View file @ 3caed389
......@@ -13,6 +13,7 @@ urls = vscode-bat https://open-vsx.org/api/vscode/bat/1.54.3/file/vscode.bat-1.5
vscode-docker https://open-vsx.org/api/vscode/docker/1.54.3/file/vscode.docker-1.54.3.vsix a4e3cdd03833fe6bf3f5045fc5be4433
vscode-emmet https://open-vsx.org/api/vscode/emmet/1.54.3/file/vscode.emmet-1.54.3.vsix 5786c95921794ed3188083a7f804fa29
vscode-fsharp https://open-vsx.org/api/vscode/fsharp/1.54.3/file/vscode.fsharp-1.54.3.vsix 028f4ec9593533ce38603032b3f2821e
vscode-git https://open-vsx.org/api/vscode/git/1.54.3/file/vscode.git-1.54.3.vsix da2f2c0626b4a254a660ae20b0c97611
vscode-go https://open-vsx.org/api/vscode/go/1.54.3/file/vscode.go-1.54.3.vsix 9f095a75e4137079351150722d091256
vscode-groovy https://open-vsx.org/api/vscode/groovy/1.54.3/file/vscode.groovy-1.54.3.vsix 7ae14c40786311fda29059178f30c310
vscode-grunt https://open-vsx.org/api/vscode/grunt/1.54.3/file/vscode.grunt-1.54.3.vsix f7894259f1fa60e939ff2d9b49138a02
......
software/theia/generate_download_plugins_cfg.py
View file @ 3caed389
......@@ -17,6 +17,7 @@ for plugin_and_version in '''\
vscode/docker/latest
vscode/emmet/latest
vscode/fsharp/latest
vscode/git/latest
vscode/go/latest
vscode/groovy/latest
vscode/grunt/latest
......
software/theia/instance-export.cfg.jinja.in 0 → 100644
View file @ 3caed389
{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) -%}
[buildout]
extends = {{ theia_instance_cfg }}
{{ pbsready_export_cfg }}
parts +=
monitor-base
$${:theia-parts}
$${:theia-environment-parts}
resilient-publish-connection-parameter
# The resilient stack makes the 'resilient' instance
# request the 'export' instance with a 'namebase' parameter.
# The export template then expects to receive it in
# slap-parameter:namebase
[slap-parameter]
namebase = {{ parameter_dict['namebase'] }}
# The resilient export stack periodically calls exporter:wrapper
# and then notifies the pull-backup instance that data is ready
# to be pulled from the export instance.
# All it expects is that a script be available in exporter:wrapper.
[exporter]
wrapper = $${theia-export-script:rendered}
[theia-export-script]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/theia-export-script
mode = 0700
exitcode-file = $${directory:srv}/export-exitcode-file
error-file = $${directory:srv}/export-errormessage-file
context =
raw python ${software-info:python-with-eggs}
raw theia_export ${software-info:theia-export}
raw bash ${software-info:bash}
raw rsync ${software-info:rsync}
raw sqlite3 ${software-info:sqlite3}
raw root_path $${buildout:directory}
raw backup_path $${directory:backup}
raw slapos_cfg $${directory:runner}/etc/slapos.cfg
raw project_path $${directory:project}
raw public_path $${directory:frontend-static-public}
key exitfile :exitcode-file
key errorfile :error-file
{%- raw %}
template =
inline:#!{{ bash }}
{{ python }} {{ theia_export }} \
--rsync {{ rsync }} \
--sqlite3 {{ sqlite3 }} \
--root {{ root_path }} \
--backup {{ backup_path }} \
--cfg {{ slapos_cfg }} \
--dirs {{ project_path }} \
--dirs {{ public_path }} \
--exitfile {{ exitfile }} \
--errorfile {{ errorfile }}
{%- endraw %}
# Add a promise to check that the export script has run
# successfully and recently (at most 2 days ago).
[promises]
export-promises =
$${export-promise:name}
[export-promise]
<= monitor-promise-base
module = check_command_execute
name = resiliency-export-promise.py
config-command = $${export-promise-script:rendered}
[initial-export-exitcode-file]
recipe = slapos.recipe.template:jinja2
rendered = $${theia-export-script:exitcode-file}
template = inline:0
once = $${:rendered}
[export-promise-script]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/export-promise-script
exitcode-file = $${initial-export-exitcode-file:rendered}
context =
key exitcodefile :exitcode-file
key errorfile theia-export-script:error-file
{%- raw %}
template =
inline:#!/bin/sh
if [ -z $(find {{ repr(exitcodefile) }} -mtime -2) ]
then
echo "ERROR export script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 1
elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
then
echo "ERROR export script failed on " $(date -r {{ repr(exitcodefile) }})
cat {{ repr(errorfile) }}
exit 1
else
echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 0
fi
{%- endraw %}
# Extend resilient parameters with normal theia connection parameters
[resilient-publish-connection-parameter]
<= publish-connection-parameter
software/theia/instance-import.cfg.jinja.in 0 → 100644
View file @ 3caed389
{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) -%}
{%- set additional_frontend = parameter_dict['additional-frontend-guid'] -%}
[buildout]
extends = {{ theia_instance_cfg }}
{{ pbsready_import_cfg }}
parts +=
monitor-base
$${:theia-parts}
$${:theia-environment-parts}
# The resilient stack makes the 'resilient' instance
# request the 'import' instance with a 'namebase' parameter.
# The import template then expects to receive it in
# slap-parameter:namebase
[slap-parameter]
namebase = {{ parameter_dict['namebase'] }}
# Change frontend name to avoid conflicts
[remote-frontend]
name = Import {{ parameter_dict['frontend-name'] }}
{% if additional_frontend -%}
[remote-additional-frontend]
name = Import {{ parameter_dict['additional-frontend-name'] }}
{%- endif %}
# Change port ranges to avoid race conditions on port allocation
[frontend-instance-port]
minimum = 3200
maximum = 3300
[theia-service-port]
minimum = 3700
maximum = 3800
[slapos-standalone-port]
minimum = 4200
maximum = 4300
# Always disable autoprocessing in the import instance
[slapos-autorun]
autorun = stopped
# Change the gravatar favicon seed
[favicon.ico]
seed = Import {{ root_title }}
# The resilient stack calls post-notification-run:output followed by
# importer:wrapper when the instance is notified that the backup files
# have just been pushed to it. All it expects is the path of a script
# in post-notification-run:output and in importer:wrapper.
[post-notification-run]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/post-notification-run-script
output = $${:rendered}
mode = 0700
template =
inline:#!${software-info:bash}
# Do nothing because the backup signature will
# be verified by the import script itself
exit 0
[importer]
wrapper = $${theia-import-script:rendered}
[theia-import-script]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/theia-import-script
mode = 0700
exitcode-file = $${directory:srv}/import-exitcode-file
error-file = $${directory:srv}/import-errormessage-file
context =
raw python ${software-info:python-with-eggs}
raw theia_import ${software-info:theia-import}
raw bash ${software-info:bash}
raw rsync ${software-info:rsync}
raw sqlite3 ${software-info:sqlite3}
raw slapos ${software-info:slapos}
raw slapos_node_software_log $${directory:runner}/var/log/slapos-node-software.log
raw slapos_node_instance_log $${directory:runner}/var/log/slapos-node-instance.log
raw supervisorctl ${software-info:supervisorctl}
raw supervisord_conf $${directory:runner}/etc/supervisord.conf
raw root_path $${buildout:directory}
raw backup_path $${directory:backup}
raw slapos_cfg $${directory:runner}/etc/slapos.cfg
raw project_path $${directory:project}
raw public_path $${directory:frontend-static-public}
key exitfile :exitcode-file
key errorfile :error-file
{%- raw %}
template =
inline:#!{{ bash }}
. $${common-environment:rendered}
. $${slapos-standalone-activate:rendered}
{{ python }} {{ theia_import }} \
--rsync {{ rsync }} \
--sqlite3 {{ sqlite3 }} \
--slapos {{ slapos }} \
--srlog {{ slapos_node_software_log }} \
--cplog {{ slapos_node_instance_log }} \
--supervisorctl {{ supervisorctl }} \
--supervisordconf {{ supervisord_conf }} \
--root {{ root_path }} \
--backup {{ backup_path }} \
--cfg {{ slapos_cfg }} \
--dirs {{ project_path }} \
--dirs {{ public_path }} \
--exitfile {{ exitfile }} \
--errorfile {{ errorfile }}
{%- endraw %}
# Add a promise to check that the import script has run
# successfully and recently (at most 2 days ago).
[promises]
import-promises =
$${import-promise:name}
[import-promise]
<= monitor-promise-base
module = check_command_execute
name = resiliency-import-promise.py
config-command = $${import-promise-script:rendered}
[initial-import-exitcode-file]
recipe = slapos.recipe.template:jinja2
rendered = $${theia-import-script:exitcode-file}
template = inline:0
once = $${:rendered}
[import-promise-script]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/import-promise-script
exitcode-file = $${initial-import-exitcode-file:rendered}
context =
key exitcodefile :exitcode-file
key errorfile theia-import-script:error-file
{%- raw %}
template =
inline:#!/bin/sh
if [ -z $(find {{ repr(exitcodefile) }} -mtime -2) ]
then
echo "ERROR import script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 1
elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
then
echo "ERROR import script failed on " $(date -r {{ repr(exitcodefile) }})
cat {{ repr(errorfile) }}
exit 1
else
echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 0
fi
{%- endraw %}
# Resilient connection parameters of import instance are published
# through the resilient stack.
# Extend resilient parameters with normal theia connection parameters
[resilient-publish-connection-parameter]
<= publish-connection-parameter
software/theia/instance-resilient-input-schema.json 0 → 100644
View file @ 3caed389
{
"$schema": "http://json-schema.org/draft-04/schema",
"type": "object",
"description": "Parameters to instantiate resilient Theia",
"allOf": [
{
"$ref": "instance-input-schema.json#/"
},
{
"properties": {
"resilient-clone-number": {
"title": "Amount of backup(s) to create",
"description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
"type": "integer",
"default": 1,
"minimum": 0,
"maximum": 2,
"optional": true
},
"-sla-theia0-computer_guid": {
"title": "Target computer for main instance",
"description": "Target computer GUID for main instance.",
"type": "string",
"optional": true
},
"-sla-theia1-computer_guid": {
"title": "Target computer for first clone",
"description": "Target computer for first clone instance.",
"type": "string",
"optional": true
},
"-sla-pbs1-computer_guid": {
"title": "Target computer for first PBS",
"description": "Target computer for first PBS instance.",
"type": "string",
"optional": true
},
"-sla-theia2-computer_guid": {
"title": "Target computer for second clone",
"description": "Target computer for second clone instance.",
"type": "string",
"optional": true
},
"-sla-pbs2-computer_guid": {
"title": "Target computer for second PBS",
"description": "Target computer for second PBS instance.",
"type": "string",
"optional": true
},
"resiliency-backup-periodicity": {
"title": "Periodicity of backup",
"description": "Periodicity of backup, in cron format.",
"type": "string",
"optional": true
},
"remove-backup-older-than": {
"title": "Remove backups older than...",
"description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
"type": "string",
"default": "2W",
"optional": true
},
"ignore-known-hosts-file": {
"title": "Ignore known_hosts file",
"description": "Set either to fill known_hosts file for ssh or not. Useful if main instance and PBS are using the same IP (slapos proxy, theia).",
"type": "boolean",
"default": false,
"optional": true
}
}
}
]
}
software/theia/instance-resilient.cfg.jinja 0 → 100644
View file @ 3caed389
{% import 'parts' as parts %}
{% import 'replicated' as replicated with context %}
{% set number_of_instances = slapparameter_dict.get('resilient-clone-number', 1)|int %}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
extends =
{{ monitor_template }}
parts +=
# Generate the parts to request theia-export, pull-backup and theia-import
# See stack/resilient/template-parts.cfg.in and stack/resilient/template-replicated.cfg.in
# See below for the generation of the sections corresponding to the parts generated here
{{ parts.replicate("theia", number_of_instances + 1) }}
# Also publish some connection parameters
publish-connection-parameter
[ArgLeader]
[ArgBackup]
# Generate sections to request theia-export, pull-backup and theia-import
# See stack/resilient/template-replicated.cfg.in
# In particular:
#
# [request-theia]
# <= ArgLeader
# software-type = export
# ...
#
# [request-theia-pseudo-replicating-1]
# <= ArgBackup
# software-type = import
# ...
#
# [request-pbs-theia-1]
# software-type = pull-backup
# ...
#
{{ replicated.replicate("theia", number_of_instances + 1,
"export", "import",
"ArgLeader", "ArgBackup",
slapparameter_dict=slapparameter_dict) }}
# Extend the list of return parameters for the export request
# The monitor parameters are only there to assert they are
# actually published by the export instance
[request-theia]
return += url username password backend-url monitor-base-url monitor-setup-url
# Extend the list of return parameters for the import request
# with the monitor parameters to assert they are actually published
[request-theia-pseudo-replicating-1]
return += monitor-base-url monitor-setup-url
# Publish some parameters from the export instance
[publish-connection-parameter]
recipe = slapos.cookbook:publish
url = ${request-theia:connection-url}
username = ${request-theia:connection-username}
password = ${request-theia:connection-password}
backend-url = ${request-theia:connection-backend-url}
# Publish resiliency parameters fetched by the resilient stack
[publish-connection-parameter]
<= publish-connection-information
software/theia/instance-theia.cfg.jinja.in
View file @ 3caed389
......@@ -41,23 +41,26 @@ backend-url = $${frontend-instance:url}
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
var = $${buildout:directory}/var
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
dot-theia = $${buildout:directory}/.theia/
home = $${buildout:directory}
etc = $${:home}/etc
var = $${:home}/var
srv = $${:home}/srv
bin = $${:home}/bin
tmp = $${:home}/tmp
dot-theia = $${:home}/.theia/
pidfiles = $${:var}/run
services = $${:etc}/service
runner = $${:srv}/runner
backup = $${:srv}/backup/theia
project = $${:srv}/project
frontend-static = $${:srv}/frontend-static
frontend-static-public = $${:frontend-static}/public
frontend-static-css = $${:frontend-static}/css
bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/
fish-completions = $${buildout:directory}/.config/fish/completions/
bash-completions = $${:home}/.local/share/bash-completion/completions/
fish-completions = $${:home}/.config/fish/completions/
# Promises
......@@ -68,11 +71,13 @@ recipe =
instance-promises =
$${theia-listen-promise:name}
$${frontend-listen-promise:name}
$${frontend-authentification-promise:name}
$${remote-frontend-url-available-promise:name}
{% if additional_frontend %}
$${remote-additional-frontend-url-available-promise:name}
{% endif %}
$${slapos-standalone-listen-promise:name}
$${slapos-standalone-ready-promise:name}
$${slapos-autorun-promise:name}
[theia-listen-promise]
......@@ -89,6 +94,16 @@ name = $${:_buildout_section_name_}.py
config-host = $${frontend-instance:ip}
config-port = $${frontend-instance:port}
[frontend-authentification-promise]
<= monitor-promise-base
module = check_url_available
name = $${:_buildout_section_name_}.py
username = $${frontend-instance-password:username}
password = $${frontend-instance-password:passwd}
ip = $${frontend-instance:ip}
port = $${frontend-instance:port}
config-url = https://$${:username}:$${:password}@[$${:ip}]:$${:port}
[remote-frontend-url-available-promise]
<= monitor-promise-base
module = check_url_available
......@@ -113,13 +128,19 @@ name = standalone-listen-promise.py
config-host = $${slapos-standalone-instance:hostname}
config-port = $${slapos-standalone-instance:port}
[slapos-standalone-ready-promise]
<= monitor-promise-base
module = check_socket_listening
name = standalone-ready-promise.py
config-abstract = $${directory:runner}/standalone_ready
[slapos-autorun-promise]
<= monitor-promise-base
module = check_service_state
# XXX promise plugins can not contain "slapos" in their names
name = autorun-state-promise.py
config-service = $${slapos-autorun:service-name}
config-expect = $${slapos-autorun:autorun}
config-run-directory = $${directory:runner}/var/run
# Remote Caddy Frontend
......@@ -279,13 +300,14 @@ wait-for-files = $${frontend-instance:pidfile}
[favicon.ico]
# generate a pseudo random favicon, different for each instance name.
recipe = slapos.recipe.build
seed = {{ root_title }}
install =
import hashlib, shutil
buildout_offline = self.buildout['buildout']['offline']
self.buildout['buildout']['offline'] = 'false'
try:
gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5(
b'''{{ root_title }}'''
b'''$${:seed}'''
).hexdigest() + "?s=256&d=retro"
shutil.copy(self.download(gravatar_url), '''$${:location}''')
except Exception:
......@@ -310,7 +332,7 @@ mode = 0700
template =
inline:
#!/bin/sh
export HOME=$${buildout:directory}
export HOME=$${directory:home}
export PATH=${python-language-server:location}/bin:${java-jdk:location}/bin:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
......@@ -375,9 +397,6 @@ template =
. {{ activate }}
unset GIT_EXEC_PATH
set -- --rcfile {{ bashrc }}
# otherwise, assume this shell is running task and add an artificial delay to workaround https://github.com/eclipse-theia/theia/issues/2961
else
sleep 1
fi
exec "$SHELL" "$@"
{% endraw %}
......@@ -419,6 +438,7 @@ ip = {{ ipv4_random }}
ipv4 = {{ ipv4_random }}
ipv6 = {{ ipv6_random }}
port = $${slapos-standalone-port:port}
local-software-release-root = $${directory:home}
slapos-configuration = $${directory:runner}/etc/slapos.cfg
computer-id = slaprunner
......@@ -447,6 +467,7 @@ template =
$${slapos-standalone-config:ipv4} \
$${slapos-standalone-config:ipv6} \
$${slapos-standalone-config:port} \
$${slapos-standalone-config:local-software-release-root} \
$${slapos-standalone-config:computer-id} \
{%- if parameter_dict.get('embedded-sr') %}
--sr='{{ parameter_dict['embedded-sr'] }}' \
......
software/theia/instance.cfg.in
View file @ 3caed389
......@@ -2,6 +2,9 @@
parts =
switch-softwaretype
extends =
${template-resilient-templates:output}
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
......@@ -18,6 +21,11 @@ recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:default}
default = $${:theia}
theia = theia:rendered
export = export:rendered
import = import:rendered
resilient = resilient:rendered
frozen = instance-frozen:rendered
pull-backup = template-pull-backup:rendered
[theia]
recipe = slapos.recipe.template:jinja2
......@@ -46,3 +54,43 @@ default-parameters =
"additional-frontend-guid": null
}
frontend-sr = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
[import]
recipe = slapos.recipe.template:jinja2
template = ${instance-import:output}
rendered = $${buildout:directory}/instance-import.cfg
mode = 0644
context =
jsonkey default_parameter_dict theia:default-parameters
key parameter_dict slap-configuration:configuration
key theia_instance_cfg theia:rendered
key pbsready_import_cfg template-pbsready-import:rendered
key root_title slap-configuration:root-instance-title
[export]
recipe = slapos.recipe.template:jinja2
template = ${instance-export:output}
rendered = $${buildout:directory}/instance-export.cfg
mode = 0644
context =
jsonkey default_parameter_dict theia:default-parameters
key parameter_dict slap-configuration:configuration
key theia_instance_cfg theia:rendered
key pbsready_export_cfg template-pbsready-export:rendered
[resilient]
recipe = slapos.recipe.template:jinja2
template = ${instance-resilient:output}
rendered = $${buildout:directory}/instance-resilient.cfg
mode = 0644
extensions = jinja2.ext.do
context =
key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
raw monitor_template ${monitor-template:rendered}
template-parts-destination = ${template-parts:target}
template-replicated-destination = ${template-replicated:target}
import-list = file parts :template-parts-destination
file replicated :template-replicated-destination
software/theia/software.cfg
View file @ 3caed389
......@@ -12,9 +12,12 @@ extends =
../../component/coreutils/buildout.cfg
../../component/java-jdk/buildout.cfg
../../component/fonts/buildout.cfg
../../component/libsecret/buildout.cfg
../../component/pkgconfig/buildout.cfg
../../stack/nodejs.cfg
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
../../stack/resilient/buildout.cfg
../../component/defaults.cfg
./download-plugins.cfg
./buildout.hash.cfg
......@@ -22,8 +25,14 @@ extends =
parts =
theia-wrapper
slapos-cookbook
python-with-eggs
instance-theia
instance
instance-import
instance-export
instance-resilient
theia-common
theia-export
# default for slapos-standalone
shared-part-list =
......@@ -54,9 +63,10 @@ initialization =
import argparse
import glob
import json
import os.path
import sys
import os
import signal
import socket
import sys
import time
import slapos.slap.standalone
......@@ -66,6 +76,7 @@ initialization =
parser.add_argument('ipv4')
parser.add_argument('ipv6')
parser.add_argument('server_port', type=int)
parser.add_argument('local_software_release_root')
parser.add_argument('computer_id')
parser.add_argument('--sr')
parser.add_argument('--srtype')
......@@ -101,58 +112,64 @@ initialization =
instance_root="%s/instance" % args.base_directory,
partition_forward_configuration=partition_forward_configuration,
slapos_bin="${buildout:bin-directory}/slapos",
local_software_release_root=args.local_software_release_root,
)
standalone.start()
partition_count = 20
if len(glob.glob(os.path.join(standalone.instance_directory, '*'))) < partition_count:
print("Standalone SlapOS: Formatting {partition_count} partitions".format(
partition_count=partition_count))
standalone.format(
partition_count,
args.ipv4,
args.ipv6
)
print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
print("Error instanciating: {}".format(e))
if args.sr:
try:
with open(args.srparams) as f:
params = json.load(f)
except Exception:
params = None
if not isinstance(params, dict):
params = None
print("Supplying and Requesting Embedded Software {sr} with type {srtype}".format(
sr=args.sr, srtype=args.srtype))
print("With parameters {param_dict} parsed from '{srparams}'".format(
param_dict=params, srparams=args.srparams))
standalone.supply(args.sr)
standalone.request(
args.sr,
"Embedded Instance",
args.srtype,
partition_parameter_kw=params,
)
quit_requested = []
def signal_handler(signum, frame):
print("Signal {signum} received".format(signum=signum))
quit_requested.append(True)
sys.exit()
signal.signal(signal.SIGTERM, signal_handler)
print("Standalone SlapOS ready")
while not quit_requested:
time.sleep(.1)
standalone.start()
try:
partition_count = 20
if len(glob.glob(os.path.join(standalone.instance_directory, '*'))) < partition_count:
print("Standalone SlapOS: Formatting {partition_count} partitions".format(
partition_count=partition_count))
standalone.format(
partition_count,
args.ipv4,
args.ipv6,
)
print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
print("Error instanciating: {}".format(e))
if args.sr:
try:
with open(args.srparams) as f:
params = json.load(f)
except Exception:
params = None
if not isinstance(params, dict):
params = None
print("Supplying and Requesting Embedded Software {sr} with type {srtype}".format(
sr=args.sr, srtype=args.srtype))
print("With parameters {param_dict} parsed from '{srparams}'".format(
param_dict=params, srparams=args.srparams))
standalone.supply(args.sr)
standalone.request(
args.sr,
"Embedded Instance",
args.srtype,
partition_parameter_kw=params,
)
s = socket.socket(socket.AF_UNIX)
s.bind('\0' + os.path.join(args.base_directory, 'standalone_ready'))
s.listen(5)
print("Standalone SlapOS ready")
while True:
s.accept()[0].close()
finally:
print("Stopping standalone subsystem")
standalone.stop()
print("Exiting")
print("Stopping standalone subsystem")
standalone.stop()
print("Exiting")
sys.exit(0)
needs-these-eggs-scripts-in-path =
${supervisor:recipe}
......@@ -191,7 +208,11 @@ stop-on-error = true
[theia]
recipe = plone.recipe.command
command = ${bash:location}/bin/bash -c "
export TMPDIR=${:location}/tmp PATH=${nodejs:location}/bin:$PATH &&
export \
TMPDIR=${:location}/tmp \
PATH=${nodejs:location}/bin:${pkgconfig:location}/bin:$PATH \
PKG_CONFIG_PATH=${libsecret:pkg-config-path} \
LDFLAGS='-Wl,-rpath=${libsecret:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${glib:location}/lib' && \
mkdir -p ${:location} && \
mkdir -p \$TMPDIR && \
cd ${:location} && \
......@@ -233,6 +254,11 @@ template =
inline:{
"private": true,
"theia": {
"backend": {
"config": {
"warnOnPotentiallyInsecureHostPattern": false
}
},
"frontend": {
"config": {
"applicationName": "Theia SlapOS",
......@@ -262,7 +288,8 @@ template =
"plantuml.render": "PlantUMLServer",
"gitlens.remotes": [{ "domain": "lab.nexedi.com", "type": "GitLab" }],
"java.home": "${java-jdk:location}"
}
},
"warnOnPotentiallyInsecureHostPattern": false
}
},
"generator": {
......@@ -272,17 +299,16 @@ template =
}
},
"dependencies": {
"@theia/bulk-edit": "latest",
"@theia/callhierarchy": "latest",
"@theia/console": "latest",
"@theia/core": "latest",
"@theia/cpp-debug": "latest",
"@theia/debug": "latest",
"@theia/editor": "latest",
"@theia/editor-preview": "latest",
"@theia/file-search": "latest",
"@theia/filesystem": "latest",
"@theia/getting-started": "latest",
"@theia/git": "latest",
"@theia/keymaps": "latest",
"@theia/markers": "latest",
"@theia/messages": "latest",
......@@ -292,16 +318,19 @@ template =
"@theia/navigator": "latest",
"@theia/outline-view": "latest",
"@theia/output": "latest",
"@theia/plugin": "latest",
"@theia/plugin-dev": "latest",
"@theia/plugin-ext": "latest",
"@theia/plugin-ext-vscode": "latest",
"@theia/preferences": "latest",
"@theia/preview": "latest",
"@theia/process": "latest",
"@theia/property-view": "latest",
"@theia/scm": "latest",
"@theia/scm-extra": "latest",
"@theia/search-in-workspace": "latest",
"@theia/task": "latest",
"@theia/terminal": "latest",
"@theia/timeline": "latest",
"@theia/typehierarchy": "latest",
"@theia/userstorage": "latest",
"@theia/variable-resolver": "latest",
......@@ -343,6 +372,18 @@ template =
#!/bin/sh
exec ${nodejs:location}/bin/node ${theia:location}/node_modules/.bin/theia-open "$@"
[python-with-eggs]
recipe = zc.recipe.egg
interpreter = ${:_buildout_section_name_}
eggs =
${slapos-toolbox:eggs}
six
zc.buildout
# Only generate the interpreter script to avoid conflicts with scripts
# for eggs that are also generated by another section, like slapos.toolbox
scripts = ${:interpreter}
[instance-theia]
<= template-base
output = ${buildout:directory}/instance-theia.cfg.jinja
......@@ -350,3 +391,37 @@ output = ${buildout:directory}/instance-theia.cfg.jinja
[instance]
<= template-base
output = ${buildout:directory}/instance.cfg
[instance-import]
<= template-base
output = ${buildout:directory}/instance-import.cfg.jinja
[instance-export]
<= template-base
output = ${buildout:directory}/instance-export.cfg.jinja
[instance-resilient]
<= download-base
[theia-common]
<= download-base
destination = ${buildout:directory}/theia_common.py
[theia-export]
<= download-base
destination = ${buildout:directory}/theia_export.py
[theia-import]
<= download-base
destination = ${buildout:directory}/theia_import.py
[software-info]
slapos = ${buildout:bin-directory}/slapos
python-with-eggs = ${buildout:bin-directory}/${python-with-eggs:interpreter}
python = ${python:location}/bin/python
rsync = ${rsync:location}/bin/rsync
sqlite3 = ${sqlite3:location}/bin/sqlite3
bash = ${bash:location}/bin/bash
supervisorctl = ${buildout:bin-directory}/supervisorctl
theia-export = ${theia-export:output}
theia-import = ${theia-import:output}
software/theia/software.cfg.json
View file @ 3caed389
......@@ -9,6 +9,13 @@
"description": "Default",
"request": "instance-input-schema.json",
"response": "instance-output-schema.json",
"index": 0
},
"resilient": {
"title": "Resilient",
"description": "Resilient Theia",
"request": "instance-resilient-input-schema.json",
"response": "instance-output-schema.json",
"index": 1
}
}
......
software/theia/test/project_tests.py 0 → 100644
View file @ 3caed389
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import unicode_literals
import gzip
import json
import os
import re
import subprocess
import time
import unittest
import requests
from datetime import datetime, timedelta
from six.moves.urllib.parse import urljoin
from slapos.testing.testcase import installSoftwareUrlList
import test_resiliency
from test import SlapOSInstanceTestCase, theia_software_release_url
erp5_software_release_url = os.path.abspath(
os.path.join(
os.path.dirname(__file__), '..', '..', 'erp5', 'software.cfg'))
def setUpModule():
installSoftwareUrlList(
SlapOSInstanceTestCase,
[theia_software_release_url, erp5_software_release_url],
debug=bool(int(os.environ.get('SLAPOS_TEST_DEBUG', 0))),
)
class ERP5Mixin(object):
_test_software_url = erp5_software_release_url
_connexion_parameters_regex = re.compile(r"{\s*'_'\s*:\s*'(.*)'\s*}")
def _getERP5ConnexionParameters(self, software_type='export'):
slapos = self._getSlapos(software_type)
out = subprocess.check_output(
(slapos, 'request', 'test_instance', self._test_software_url),
stderr=subprocess.STDOUT,
)
print(out)
return json.loads(self._connexion_parameters_regex.search(out).group(1))
def _getERP5Url(self, connexion_parameters, path=''):
return urljoin(connexion_parameters['family-default-v6'], path)
def _getERP5User(self, connexion_parameters):
return connexion_parameters['inituser-login']
def _getERP5Password(self, connexion_parameters):
return connexion_parameters['inituser-password']
def _waitERP5connected(self, url, user, password):
for _ in range(5):
try:
resp = requests.get('%s/getId' % url, auth=(user, password), verify=False, allow_redirects=False)
except Exception:
time.sleep(20)
continue
if resp.status_code != 200:
time.sleep(20)
continue
break
else:
self.fail('Failed to connect to ERP5')
self.assertEqual(resp.text, 'erp5')
def _getERP5Partition(self, servicename):
p = subprocess.Popen(
(self._getSlapos(), 'node', 'status'),
stdout=subprocess.PIPE, universal_newlines=True)
out, _ = p.communicate()
found = set()
for line in out.splitlines():
if servicename in line:
found.add(line.split(':')[0])
if not found:
raise Exception("ERP5 %s partition not found" % servicename)
elif len(found) > 1:
raise Exception("Found several partitions for ERP5 %s" % servicename)
return found.pop()
def _getERP5PartitionPath(self, software_type, servicename, *paths):
partition = self._getERP5Partition(servicename)
return self._getPartitionPath(
software_type, 'srv', 'runner', 'instance', partition, *paths)
class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
test_instance_max_retries = 12
backup_max_tries = 480
backup_wait_interval = 60
def _prepareExport(self):
super(TestTheiaResilienceERP5, self)._prepareExport()
# Connect to erp5
info = self._getERP5ConnexionParameters()
user = self._getERP5User(info)
password = self._getERP5Password(info)
url = self._getERP5Url(info, 'erp5')
self._waitERP5connected(url, user, password)
# Change title
new_title = time.strftime("HelloTitle %a %d %b %Y %H:%M:%S", time.localtime(time.time()))
requests.get('%s/portal_types/setTitle?value=%s' % (url, new_title), auth=(user, password), verify=False)
resp = requests.get('%s/portal_types/getTitle' % url, auth=(user, password), verify=False, allow_redirects=False)
self.assertEqual(resp.text, new_title)
self._erp5_new_title = new_title
# Wait until changes have been catalogued
mariadb_partition = self._getERP5PartitionPath('export', 'mariadb')
mysql_bin = os.path.join(mariadb_partition, 'bin', 'mysql')
wait_activities_script = os.path.join(
mariadb_partition, 'software_release', 'parts', 'erp5',
'Products', 'CMFActivity', 'bin', 'wait_activities')
subprocess.check_call((wait_activities_script, 'erp5'), env={'MYSQL': mysql_bin})
# Check that changes have been catalogued
output = subprocess.check_output((mysql_bin, 'erp5', '-e', 'SELECT title FROM catalog WHERE id="portal_types"'))
self.assertIn(new_title, output)
# Compute backup date in the near future
soon = (datetime.now() + timedelta(minutes=4)).replace(second=0)
date = '*:%d:00' % soon.minute
params = '_={"zodb-zeo": {"backup-periodicity": "%s"}, "mariadb": {"backup-periodicity": "%s"} }' % (date, date)
# Update ERP5 parameters
print('Requesting ERP5 with parameters %s' % params)
slapos = self._getSlapos()
subprocess.check_call((slapos, 'request', 'test_instance', self._test_software_url, '--parameters', params))
# Process twice to propagate parameter changes
for _ in range(2):
subprocess.check_call((slapos, 'node', 'instance'))
# Restart cron (actually all) services to let them take the new date into account
# XXX this should not be required, updating ERP5 parameters should be enough
subprocess.call((slapos, 'node', 'restart', 'all'))
# Wait until after the programmed backup date, and a bit more
t = (soon - datetime.now()).total_seconds()
self.assertLess(0, t)
time.sleep(t + 120)
# Check that mariadb backup has started
mariadb_backup = os.path.join(mariadb_partition, 'srv', 'backup', 'mariadb-full')
mariadb_backup_dump, = os.listdir(mariadb_backup)
# Check that zodb backup has started
zodb_backup = self._getERP5PartitionPath('export', 'zeo', 'srv', 'backup', 'zodb', 'root')
self.assertEqual(len(os.listdir(zodb_backup)), 3)
# Check that mariadb catalog backup contains expected changes
with gzip.open(os.path.join(mariadb_backup, mariadb_backup_dump)) as f:
self.assertIn(new_title, f.read(), "Mariadb catalog backup %s is not up to date" % mariadb_backup_dump)
def _checkTakeover(self):
super(TestTheiaResilienceERP5, self)._checkTakeover()
# Connect to erp5
info = self._getERP5ConnexionParameters()
user = self._getERP5User(info)
password = self._getERP5Password(info)
url = self._getERP5Url(info, 'erp5')
self._waitERP5connected(url, user, password)
resp = requests.get('%s/portal_types/getTitle' % url, auth=(user, password), verify=False, allow_redirects=False)
self.assertEqual(resp.text, self._erp5_new_title)
# Check that the mariadb catalog is not yet restored
mariadb_partition = self._getERP5PartitionPath('export', 'mariadb')
mysql_bin = os.path.join(mariadb_partition, 'bin', 'mysql')
query = 'SELECT title FROM catalog WHERE id="portal_types"'
try:
out = subprocess.check_output((mysql_bin, 'erp5', '-e', query))
except subprocess.CalledProcessError:
out = ''
self.assertNotIn(self._erp5_new_title, out)
# Stop all services
slapos = self._getSlapos()
print("Stop all services")
subprocess.call((slapos, 'node', 'stop', 'all'))
# Manually restore mariadb from backup
mariadb_restore_script = os.path.join(mariadb_partition, 'bin', 'restore-from-backup')
print("Restore mariadb from backup")
subprocess.check_call(mariadb_restore_script)
# Check that the test instance is properly redeployed after restoring mariadb
# This restarts the services and checks the promises of the test instance
# Process twice to propagate state change
for _ in range(2):
self._processEmbeddedInstance(self.test_instance_max_retries)
# Check that the mariadb catalog was properly restored
out = subprocess.check_output((mysql_bin, 'erp5', '-e', query))
self.assertIn(self._erp5_new_title, out, 'Mariadb catalog is not properly restored')
software/theia/test/resilience_dummy/README.md 0 → 100644
View file @ 3caed389
# Resilience Dummy Software
A very simple SR to test resiliency:
- fast installation and deployment
- self-contained - no dependency outside this folder
- has a simple `exporter.exclude` and `runner-import-restore`
software/theia/test/resilience_dummy/instance.cfg.in 0 → 100644
View file @ 3caed389
[buildout]
parts =
log-writer
exporter.exclude
runner-import-restore
backup-identity-script
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[directory]
recipe = plone.recipe.command
home = $${buildout:directory}
srv = $${:home}/srv
etc = $${:home}/etc
run = $${:etc}/run
command = mkdir -p $${:run} $${:srv}
[log-writer]
recipe = slapos.recipe.template:jinja2
template = inline:#!/bin/sh
echo "Hello : $(date)" >> $${directory:home}/log.log
rendered = $${directory:run}/log-writer
[exporter.exclude]
recipe = slapos.recipe.template:jinja2
template = inline:$${directory:home}/exclude
rendered = $${directory:srv}/exporter.exclude
[runner-import-restore]
recipe = slapos.recipe.template:jinja2
template = inline:#!/bin/sh
echo "Hello : $(date)" >> $${directory:home}/runner-import-restore.log
exit $TEST_RESTORE_STATUS
rendered = $${directory:srv}/runner-import-restore
[backup-identity-script]
recipe = slapos.recipe.template:jinja2
template = inline:#!/bin/sh
echo "Custom script"
for i in "$@"
do
echo $(sha256sum $i)
done
exit $TEST_BACKUP_STATUS
rendered = $${directory:srv}/.backup_identity_script
software/theia/test/resilience_dummy/software.cfg 0 → 100644
View file @ 3caed389
[buildout]
find-links +=
http://www.nexedi.org/static/packages/source/
http://www.nexedi.org/static/packages/source/slapos.buildout/
parts =
instance-template
plone-recipe-command
versions = versions
[plone-recipe-command]
recipe = zc.recipe.egg
eggs = plone.recipe.command
[instance-template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg
mode = 0644
[versions]
setuptools = 44.0.0
zc.buildout = 2.7.1+slapos010
zc.recipe.egg = 2.0.3+slapos003
Jinja2 = 2.11.2
MarkupSafe = 1.0
software/theia/test/test.py
View file @ 3caed389
......@@ -26,31 +26,28 @@
##############################################################################
from __future__ import unicode_literals
import os
import textwrap
import json
import logging
import os
import re
import subprocess
import tempfile
import time
import re
import json
from six.moves.urllib.parse import urlparse, urljoin
import pexpect
import psutil
import requests
import sqlite3
import six
from six.moves.urllib.parse import urlparse, urljoin
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.grid.svcbackend import getSupervisorRPC
from slapos.grid.svcbackend import _getSupervisordSocketPath
from slapos.grid.svcbackend import getSupervisorRPC, _getSupervisordSocketPath
software_cfg = 'software%s.cfg' % ('-py3' if six.PY3 else '')
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', software_cfg)))
theia_software_release_url = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', software_cfg))
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(theia_software_release_url)
class TheiaTestCase(SlapOSInstanceTestCase):
......@@ -67,12 +64,21 @@ class TestTheia(TheiaTestCase):
def setUp(self):
self.connection_parameters = self.computer_partition.getConnectionParameterDict()
def get(self, url, expect_code=requests.codes.ok):
resp = requests.get(url, verify=False)
self.assertEqual(
expect_code,
resp.status_code,
'%s returned %d instead of %d' % (url, resp.status_code, expect_code),
)
return resp
def test_backend_http_get(self):
resp = requests.get(self.connection_parameters['backend-url'], verify=False)
self.assertEqual(requests.codes.unauthorized, resp.status_code)
backend_url = self.connection_parameters['backend-url']
self.get(backend_url, requests.codes.unauthorized)
# with login/password, this is allowed
parsed_url = urlparse(self.connection_parameters['backend-url'])
parsed_url = urlparse(backend_url)
authenticated_url = parsed_url._replace(
netloc='{}:{}@[{}]:{}'.format(
self.connection_parameters['username'],
......@@ -80,12 +86,11 @@ class TestTheia(TheiaTestCase):
parsed_url.hostname,
parsed_url.port,
)).geturl()
resp = requests.get(authenticated_url, verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
self.get(authenticated_url)
def test_http_get(self):
resp = requests.get(self.connection_parameters['url'], verify=False)
self.assertEqual(requests.codes.unauthorized, resp.status_code)
url = self.connection_parameters['url']
self.get(url, requests.codes.unauthorized)
# with login/password, this is allowed
parsed_url = urlparse(self.connection_parameters['url'])
......@@ -96,33 +101,28 @@ class TestTheia(TheiaTestCase):
parsed_url.hostname,
parsed_url.port,
)).geturl()
resp = requests.get(authenticated_url, verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
self.get(authenticated_url)
# there's a public folder to serve file
with open('{}/srv/frontend-static/public/test_file'.format(
self.computer_partition_root_path), 'w') as f:
f.write("hello")
resp = requests.get(urljoin(authenticated_url, '/public/'), verify=False)
resp = self.get(urljoin(authenticated_url, '/public/'))
self.assertIn('test_file', resp.text)
resp = requests.get(
urljoin(authenticated_url, '/public/test_file'), verify=False)
resp = self.get(urljoin(authenticated_url, '/public/test_file'))
self.assertEqual('hello', resp.text)
# there's a (not empty) favicon
resp = requests.get(
urljoin(authenticated_url, '/favicon.ico'), verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
resp = self.get(urljoin(authenticated_url, '/favicon.ico'))
self.assertTrue(resp.raw)
# there is a CSS referencing fonts
css_text = requests.get(urljoin(authenticated_url, '/css/slapos.css'), verify=False).text
css_text = self.get(urljoin(authenticated_url, '/css/slapos.css')).text
css_urls = re.findall(r'url\([\'"]+([^\)]+)[\'"]+\)', css_text)
self.assertTrue(css_urls)
# and fonts are served
for url in css_urls:
resp = requests.get(urljoin(authenticated_url, url), verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
resp = self.get(urljoin(authenticated_url, url))
self.assertTrue(resp.raw)
def test_theia_slapos(self):
......@@ -281,7 +281,7 @@ class TestTheiaEnv(TheiaTestCase):
}
def test_theia_env(self):
"""Make sure environment variables are the same wether we use shell or supervisor services.
"""Make sure environment variables are the same whether we use shell or supervisor services.
"""
# The path of the env.json file expected to be generated by building the dummy software release
env_json_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'software', 'env.json')
......@@ -303,47 +303,104 @@ class TestTheiaEnv(TheiaTestCase):
# Start a theia shell that inherits the environment of the theia process
# This simulates the environment of a shell launched from the browser application
theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.computer_partition_root_path), env=theia_env)
theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
# Launch slapos node software from theia shell
theia_shell_process.sendline('slapos node software')
theia_shell_process.expect('Installing software release %s' % self.dummy_software_path)
theia_shell_process.expect('Finished software releases.')
# Get the theia shell environment
with open(env_json_path) as f:
theia_shell_env = json.load(f)
# Remove the env.json file to later be sure that a new one has been generated
os.remove(env_json_path)
# Launch slapos node software service from the embedded supervisord.
# Note that we have two services, slapos-not-software and slapos-not-software-all
# The later uses --all which is what we want to use here, because the software
# is already installed and we want to install it again, this time from supervisor
embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run')
embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop']
embedded_supervisor.startProcess('slapos-node-software-all')
for _retries in range(20):
time.sleep(1)
if embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop'] != previous_stop_time:
break
else:
self.fail("the supervisord service 'slapos-node-software-all' takes too long to finish")
# Get the supervisord environment
with open(env_json_path) as f:
supervisord_env = json.load(f)
# Compare relevant variables from both environments
self.maxDiff = None
self.assertEqual(theia_shell_env['PATH'].split(':'), supervisord_env['PATH'].split(':'))
self.assertEqual(theia_shell_env['SLAPOS_CONFIGURATION'], supervisord_env['SLAPOS_CONFIGURATION'])
self.assertEqual(theia_shell_env['SLAPOS_CLIENT_CONFIGURATION'], supervisord_env['SLAPOS_CLIENT_CONFIGURATION'])
self.assertEqual(theia_shell_env['HOME'], supervisord_env['HOME'])
# Cleanup the theia shell process
theia_shell_process.terminate()
theia_shell_process.wait()
try:
theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
# Launch slapos node software from theia shell
theia_shell_process.sendline('slapos node software')
theia_shell_process.expect('Installing software release %s' % self.dummy_software_path)
theia_shell_process.expect('Finished software releases.')
# Get the theia shell environment
with open(env_json_path) as f:
theia_shell_env = json.load(f)
# Remove the env.json file to later be sure that a new one has been generated
os.remove(env_json_path)
# Launch slapos node software service from the embedded supervisord.
# Note that we have two services, slapos-node-software and slapos-node-software-all
# The later uses --all which is what we want to use here, because the software
# is already installed and we want to install it again, this time from supervisor
embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run')
embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop']
embedded_supervisor.startProcess('slapos-node-software-all')
for _retries in range(20):
time.sleep(1)
if embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop'] != previous_stop_time:
break
else:
self.fail("the supervisord service 'slapos-node-software-all' takes too long to finish")
# Get the supervisord environment
with open(env_json_path) as f:
supervisord_env = json.load(f)
# Compare relevant variables from both environments
self.maxDiff = None
self.assertEqual(theia_shell_env['PATH'].split(':'), supervisord_env['PATH'].split(':'))
self.assertEqual(theia_shell_env['SLAPOS_CONFIGURATION'], supervisord_env['SLAPOS_CONFIGURATION'])
self.assertEqual(theia_shell_env['SLAPOS_CLIENT_CONFIGURATION'], supervisord_env['SLAPOS_CLIENT_CONFIGURATION'])
self.assertEqual(theia_shell_env['HOME'], supervisord_env['HOME'])
finally:
# Cleanup the theia shell process
theia_shell_process.terminate()
theia_shell_process.wait()
class ResilientTheiaMixin(object):
@classmethod
def setUpClass(cls):
super(ResilientTheiaMixin, cls).setUpClass()
# Add resiliency files to snapshot patterns
cls._save_instance_file_pattern_list += (
'*/srv/export-exitcode-file',
'*/srv/export-errormessage-file',
'*/srv/import-exitcode-file',
'*/srv/import-errormessage-file',
)
@classmethod
def _getPartition(cls, software_type):
software_url = cls.getSoftwareURL()
for computer_partition in cls.slap.computer.getComputerPartitionList():
partition_url = computer_partition.getSoftwareRelease()._software_release
partition_type = computer_partition.getType()
if partition_url == software_url and partition_type == software_type:
return computer_partition
raise Exception("Theia %s partition not found" % software_type)
@classmethod
def _getPartitionId(cls, software_type):
return cls._getPartition(software_type).getId()
@classmethod
def _getPartitionPath(cls, software_type, *paths):
return os.path.join(cls.slap._instance_root, cls._getPartitionId(software_type), *paths)
@classmethod
def _getSlapos(cls, software_type='export'):
return cls._getPartitionPath(software_type, 'srv', 'runner', 'bin', 'slapos')
@classmethod
def getInstanceSoftwareType(cls):
return 'resilient'
class TestTheiaResilientInterface(ResilientTheiaMixin, TestTheia):
@classmethod
def setUpClass(cls):
super(TestTheiaResilientInterface, cls).setUpClass()
# Patch the computer root path to that of the export theia instance
cls.computer_partition_root_path = cls._getPartitionPath('export')
class TestTheiaResilientWithSR(ResilientTheiaMixin, TestTheiaWithSR):
@classmethod
def setUpClass(cls):
super(TestTheiaResilientWithSR, cls).setUpClass()
# Patch the computer root path to that of the export theia instance
cls.computer_partition_root_path = cls._getPartitionPath('export')
software/theia/test/test_resiliency.py 0 → 100644
View file @ 3caed389
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import unicode_literals
import errno
import os
import re
import shutil
import six
import subprocess
import time
import unittest
import requests
from slapos.testing.testcase import SlapOSNodeCommandError, installSoftwareUrlList
from test import TheiaTestCase, ResilientTheiaMixin, theia_software_release_url
dummy_software_url = os.path.abspath(
os.path.join('resilience_dummy', 'software.cfg'))
class WorkaroundSnapshotConflict(TheiaTestCase):
@classmethod
def _copySnapshot(cls, source_file_name, name):
# Workaround setUpModule snapshots name conflicts
if not name.startswith(cls.__module__):
name = '%s.%s' % (cls.__module__, name)
super(WorkaroundSnapshotConflict, cls)._copySnapshot(source_file_name, name)
def setUpModule():
installSoftwareUrlList(
WorkaroundSnapshotConflict,
[theia_software_release_url],
debug=bool(int(os.environ.get('SLAPOS_TEST_DEBUG', 0))),
)
class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
@classmethod
def _processEmbeddedInstance(cls, retries=0, software_type='export'):
slapos = cls._getSlapos(software_type)
for _ in range(retries):
try:
output = subprocess.check_output((slapos, 'node', 'instance'), stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
continue
print(output)
break
else:
if retries:
# Sleep a bit as an attempt to workaround monitoring boostrap not being ready
print("Wait before running slapos node instance one last time")
time.sleep(120)
subprocess.check_call((slapos, 'node', 'instance'))
@classmethod
def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, software_type='export'):
slapos = cls._getSlapos(software_type)
subprocess.check_call((slapos, 'supply', software_url, 'slaprunner'))
try:
subprocess.check_output((slapos, 'node', 'software'), stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
print(e.output)
raise
subprocess.check_call((slapos, 'request', instance_name, software_url))
cls._processEmbeddedInstance(retries, software_type)
@classmethod
def getInstanceParameterDict(cls):
return {'autorun': 'stopped'}
class ResilienceMixin(object):
def _prepareExport(self):
pass
def _doSync(self):
raise NotImplementedError
def _checkSync(self):
pass
def _doTakeover(self):
raise NotImplementedError
def _checkTakeover(self):
pass
def test(self):
# Do stuff on the main instance
# e.g. deploy an embedded software instance
self._prepareExport()
# Backup the main instance to a clone
# i.e. call export and import scripts
self._doSync()
# Check that the export-backup-import process went well
# e.g. look at logs and compare data
self._checkSync()
# Let the clone become a main instance
# i.e. start embedded services
self._doTakeover()
# Check that the takeover went well
# e.g. check services
self._checkTakeover()
class ExportAndImportMixin(object):
def getExportExitfile(self):
return self._getPartitionPath('export', 'srv', 'export-exitcode-file')
def getExportErrorfile(self):
return self._getPartitionPath('export', 'srv', 'export-errormessage-file')
def getImportExitfile(self):
return self._getPartitionPath('import', 'srv', 'import-exitcode-file')
def getImportErrorfile(self):
return self._getPartitionPath('import', 'srv', 'import-errormessage-file')
def makedirs(self, path):
try:
os.makedirs(path if os.path.isdir(path) else os.path.dirname(path))
except OSError as e:
if e.errno != errno.EEXIST:
raise
def writeFile(self, path, content, mode='w'):
self.makedirs(path)
executable = mode == 'exec'
mode = 'w' if executable else mode
with open(path, mode) as f:
if executable:
f.write('#!/bin/sh\n')
f.write(content)
if executable:
os.chmod(path, 0o700)
def assertPromiseSucess(self):
# Force promises to recompute regardless of periodicity
self.slap._force_slapos_node_instance_all = True
try:
self.slap.waitForInstance(error_lines=0)
except SlapOSNodeCommandError as e:
s = str(e)
self.assertNotIn("Promise 'resiliency-export-promise.py' failed", s)
self.assertNotIn('ERROR export script', s)
self.assertNotIn("Promise 'resiliency-import-promise.py' failed", s)
self.assertNotIn('ERROR import script', s)
else:
pass
def _doExport(self):
# Compute last modification of the export exitcode file
exitfile = self.getExportExitfile()
initial_exitdate = os.path.getmtime(exitfile)
# Call export script manually
theia_export_script = self._getPartitionPath('export', 'bin', 'theia-export-script')
subprocess.check_call((theia_export_script,), stderr=subprocess.STDOUT)
# Check that the export exitcode file was modified
self.assertGreater(os.path.getmtime(exitfile), initial_exitdate)
with open(exitfile) as f:
self.assertEqual('0', f.read())
# Check promises
self.assertPromiseSucess()
def _doTransfer(self):
# Copy <export>/srv/backup/theia to <import>/srv/backup/theia manually
export_backup_path = self._getPartitionPath('export', 'srv', 'backup', 'theia')
import_backup_path = self._getPartitionPath('import', 'srv', 'backup', 'theia')
shutil.rmtree(import_backup_path)
shutil.copytree(export_backup_path, import_backup_path)
def _doImport(self):
# Compute last modification of the import exitcode file
exitfile = self.getImportExitfile()
initial_exitdate = os.path.getmtime(exitfile)
# Call the import script manually
theia_import_script = self._getPartitionPath('import', 'bin', 'theia-import-script')
subprocess.check_call((theia_import_script,), stderr=subprocess.STDOUT)
# Check that the import exitcode file was updated
self.assertGreater(os.path.getmtime(exitfile), initial_exitdate)
with open(exitfile) as f:
self.assertEqual('0', f.read())
# Check promises
self.assertPromiseSucess()
class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestCase):
script_relpath = os.path.join(
'srv', 'runner', 'instance', 'slappart0',
'srv', '.backup_identity_script')
def assertPromiseFailure(self, *msg):
# Force promises to recompute regardless of periodicity
self.slap._force_slapos_node_instance_all = True
try:
self.slap.waitForInstance(error_lines=0)
except SlapOSNodeCommandError as e:
s = str(e).replace('\\n', '\n')
for m in msg:
self.assertIn(m, s)
else:
self.fail('No promise failed')
def assertScriptFailure(self, func, errorfile, exitfile, *msg):
self.assertRaises(
subprocess.CalledProcessError,
func,
)
if msg:
with open(errorfile) as f:
error = f.read()
for m in msg:
self.assertIn(m, error)
with open(exitfile) as f:
self.assertNotEqual('0', f.read())
def assertExportFailure(self, *msg):
self.assertScriptFailure(
self._doExport,
self.getExportErrorfile(),
self.getExportExitfile(),
*msg)
self.assertPromiseFailure('ERROR export script failed', *msg)
def assertImportFailure(self, *msg):
self.assertScriptFailure(
self._doImport,
self.getImportErrorfile(),
self.getImportExitfile(),
*msg)
self.assertPromiseFailure('ERROR import script failed', *msg)
def customScript(self, path, content=None):
if content:
self.writeFile(path, content, mode='exec')
else:
if os.path.exists(path):
os.remove(path)
def customSignatureScript(self, content=None):
custom_script = self._getPartitionPath('export', self.script_relpath)
self.customScript(custom_script, content)
def customRestoreScript(self, content=None):
restore_script = self._getPartitionPath('import', 'srv', 'runner-import-restore')
self.customScript(restore_script, content)
return restore_script
def cleanupExitfiles(self):
self.writeFile(self.getExportExitfile(), '0')
self.writeFile(self.getImportExitfile(), '0')
def setUp(self):
self.customSignatureScript(content=None)
self.customRestoreScript(content=None)
self.cleanupExitfiles()
def test_export_promise(self):
self.writeFile(self.getExportExitfile(), '1')
self.assertPromiseFailure('ERROR export script failed')
def test_import_promise(self):
self.writeFile(self.getImportExitfile(), '1')
self.assertPromiseFailure('ERROR import script failed')
def test_custom_hash_script(self):
errmsg = 'Bye bye'
self.customSignatureScript(content='>&2 echo "%s"\nexit 1' % errmsg)
backup_script = self._getPartitionPath(
'export', 'srv', 'backup', 'theia', self.script_relpath)
self.assertExportFailure('Compute backup signature\n ... ERROR !',
'Custom signature script %s failed' % os.path.abspath(backup_script),
'and stderr:\n%s' % errmsg)
def test_signature_mismatch(self):
signature_file = self._getPartitionPath('import', 'srv', 'backup', 'theia', 'backup.signature')
moved_file = self._getPartitionPath('import', 'srv', 'backup', 'backup.signature.moved')
self.writeFile(moved_file, 'Bogus Hash\n', mode='a')
os.rename(moved_file, signature_file)
self.assertImportFailure('ERROR the backup signatures do not match')
def test_restore_script_error(self):
self._doExport()
self._doTransfer()
restore_script = self.customRestoreScript('exit 1')
self.assertImportFailure('Run custom restore script %s\n ... ERROR !' % restore_script)
class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientTheiaTestCase):
def test_twice(self):
# Run two synchronisations on the same instances
# to make sure everything still works the second time
self._doSync()
def checkLog(self, log_path, initial=[], newline="Hello"):
with open(log_path) as f:
log = f.readlines()
self.assertEqual(len(log), len(initial) + int(bool(newline)))
for line, initial_line in zip(log, initial):
self.assertEqual(line, initial_line)
if newline:
self.assertTrue(log[-1].startswith(newline), log[-1])
return log
def _prepareExport(self):
# Copy ./resilience_dummy SR in export theia ~/srv/project/dummy
dummy_target_path = self._getPartitionPath('export', 'srv', 'project', 'dummy')
shutil.copytree(os.path.dirname(dummy_software_url), dummy_target_path)
self._test_software_url = os.path.join(dummy_target_path, 'software.cfg')
# Deploy dummy instance in export partition
self._deployEmbeddedSoftware(self._test_software_url, 'dummy_instance')
relpath_dummy = os.path.join('srv', 'runner', 'instance', 'slappart0')
self.export_dummy_root = dummy_root = self._getPartitionPath('export', relpath_dummy)
self.import_dummy_root = self._getPartitionPath('import', relpath_dummy)
# Check that dummy instance was properly deployed
self.initial_log = self.checkLog(os.path.join(dummy_root, 'log.log'))
# Create ~/include and ~/include/included
self.writeFile(os.path.join(dummy_root, 'include', 'included'),
'This file should be included in resilient backup')
# Create ~/exclude and ~/exclude/excluded
self.writeFile(os.path.join(dummy_root, 'exclude', 'excluded'),
'This file should be excluded from resilient backup')
# Check that ~/srv/exporter.exclude and ~/srv/runner-import-restore
# As well as ~/srv/.backup_identity_script
self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', 'exporter.exclude')))
self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', 'runner-import-restore')))
self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', '.backup_identity_script')))
def _doSync(self):
self._doExport()
self._doTransfer()
self._doImport()
def _checkSync(self):
dummy_root = self.import_dummy_root
# Check that the software url is correct
adapted_test_url = self._getPartitionPath('import', 'srv', 'project', 'dummy', 'software.cfg')
proxy_content = subprocess.check_output(
(self._getSlapos('import'), 'proxy', 'show'), universal_newlines=True)
self.assertIn(adapted_test_url, proxy_content)
self.assertNotIn(self._test_software_url, proxy_content)
# Check that ~/srv/project was exported
self.assertTrue(os.path.exists(adapted_test_url))
# Check that the dummy instance is not yet started
self.checkLog(os.path.join(dummy_root, 'log.log'), self.initial_log, newline=None)
# Check that ~/srv/.backup_identity_script was called
signature = self._getPartitionPath('import', 'srv', 'backup', 'backup.signature.proof')
with open(signature) as f:
self.assertIn('Custom script', f.read())
# Check that ~/include and ~/include/included were included
self.assertTrue(os.path.exists(os.path.join(dummy_root, 'include', 'included')))
# Check that ~/exclude was excluded
self.assertFalse(os.path.exists(os.path.join(dummy_root, 'exclude')))
# Check that ~/srv/runner-import-restore was called
self.checkLog(os.path.join(dummy_root, 'runner-import-restore.log'))
def _doTakeover(self):
# Start the dummy instance as a sort of fake takeover
subprocess.check_call((self._getSlapos('import'), 'node', 'instance'))
def _checkTakeover(self):
# Check that dummy instance was properly re-deployed
log_path = os.path.join(self.import_dummy_root, 'log.log')
self.checkLog(log_path, self.initial_log)
class TakeoverMixin(ExportAndImportMixin):
def _getTakeoverUrlAndPassword(self, scope="theia-1"):
parameter_dict = self.computer_partition.getConnectionParameterDict()
takeover_url = parameter_dict["takeover-%s-url" % scope]
takeover_password = parameter_dict["takeover-%s-password" % scope]
return takeover_url, takeover_password
def _getTakeoverPage(self, takeover_url):
resp = requests.get(takeover_url, verify=True)
self.assertEqual(requests.codes.ok, resp.status_code)
return resp.text
def _waitScriptDone(self, name, start, exitfile, errorfile, maxtries, interval):
print('Wait until %s script has run' % name.lower())
for t in range(maxtries):
if os.path.getmtime(exitfile) < start:
time.sleep(interval)
continue
with open(exitfile) as f:
if f.read() == '0':
print(name + ' script ran successfully')
return maxtries - t
print(name + ' script failed:\n')
with open(errorfile) as f:
print(f.read())
self.fail(name + ' script failed')
self.fail(name + ' script did not finish before timeout')
def _waitTakeoverReady(self, takeover_url, start, maxtries, interval):
export_exitfile = self.getExportExitfile()
export_errorfile = self.getExportErrorfile()
tries = self._waitScriptDone(
'Export', start, export_exitfile, export_errorfile, maxtries, interval)
import_exitfile = self.getImportExitfile()
import_errorfile = self.getImportErrorfile()
tries = self._waitScriptDone(
'Import', start, import_exitfile, import_errorfile, tries, interval)
for _ in range(tries):
info = self._getTakeoverPage(takeover_url)
if "No backup downloaded yet, takeover should not happen now." in info:
print('Takeover page still reports export script in progress')
elif "<b>Importer script(s) of backup in progress:</b> True" in info:
print('Takeover page still reports import script in progress')
else:
return
time.sleep(interval)
self.fail('Takeover page failed to report readiness')
def _requestTakeover(self, takeover_url, takeover_password):
resp = requests.get("%s?password=%s" % (takeover_url, takeover_password), verify=True)
self.assertEqual(requests.codes.ok, resp.status_code)
self.assertNotIn("Error", resp.text, "An Error occured: %s" % resp.text)
self.assertIn("Success", resp.text, "An Error occured: %s" % resp.text)
return resp.text
@unittest.skipIf(six.PY3, "resilient stack is not python3-compatible")
class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase):
test_instance_max_retries = 0
backup_max_tries = 70
backup_wait_interval = 10
_test_software_url = dummy_software_url
def _prepareExport(self):
# Deploy test instance
self._deployEmbeddedSoftware(self._test_software_url, 'test_instance', self.test_instance_max_retries)
# Check that there is an export and import instance and get their partition IDs
self.export_id = self._getPartitionId('export')
self.import_id = self._getPartitionId('import')
def _doSync(self):
start = time.time()
# Call exporter script instead of waiting for cron job
# XXX Accelerate cron frequency instead ?
exporter_script = self._getPartitionPath('export', 'bin', 'exporter')
transaction_id = str(int(time.time()))
subprocess.check_call((exporter_script, '--transaction-id', transaction_id))
takeover_url, _ = self._getTakeoverUrlAndPassword()
# Wait for takoever to be ready
self._waitTakeoverReady(takeover_url, start, self.backup_max_tries, self.backup_wait_interval)
def _doTakeover(self):
# Takeover
takeover_url, takeover_password = self._getTakeoverUrlAndPassword()
self._requestTakeover(takeover_url, takeover_password)
# Wait for import instance to become export instance and new import to be allocated
# This also checks that all promises of theia instances succeed
self.slap.waitForInstance(self.instance_max_retry)
self.computer_partition = self.requestDefaultInstance()
def _checkTakeover(self):
# Check that there is an export, import and frozen instance and get their new partition IDs
import_id = self.import_id
export_id = self.export_id
new_export_id = self._getPartitionId('export')
new_import_id = self._getPartitionId('import')
new_frozen_id = self._getPartitionId('frozen')
# Check that old export instance is now frozen
self.assertEqual(export_id, new_frozen_id)
# Check that old import instance is now the new export instance
self.assertEqual(import_id, new_export_id)
# Check that there is a new import instance
self.assertNotIn(new_import_id, (export_id, new_export_id))
# Check that the test instance is properly redeployed
# This checks the promises of the test instance
self._processEmbeddedInstance(self.test_instance_max_retries)
software/theia/theia_common.py 0 → 100644
View file @ 3caed389
import contextlib
import errno
import glob
import hashlib
import os
import re
import subprocess as sp
import sqlite3
import six
import zc.buildout.configparser
from slapos.util import bytes2str, str2bytes
RSYNC_FLAGS = ('-rlptgo', '--safe-links', '--stats', '--ignore-missing-args', '--delete', '--delete-excluded')
RSYNC_REGEX = '^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
EXCLUDE_PATTERNS = ('*.sock', '*.socket', '*.pid', '.installed*.cfg')
EXCLUDE_FLAGS = ['--exclude={}'.format(x) for x in sorted(EXCLUDE_PATTERNS)]
def makedirs(path):
try:
os.makedirs(path if os.path.isdir(path) else os.path.dirname(path))
except OSError as e:
if e.errno != errno.EEXIST:
raise
def copytree(rsyncbin, src, dst, exclude=[], extrargs=[], verbosity='-v'):
# Ensure there is a trailing slash in the source directory
# to avoid creating an additional directory level at the destination
src = os.path.join(src, '')
# Compute absolute path of destination
dst = os.path.abspath(dst)
# Create destination dir if it doesn't exist
makedirs(dst)
command = [rsyncbin]
command.extend(RSYNC_FLAGS)
# Exclude destination file from sources
command.append('--filter=-/ {}'.format(dst))
command.extend(EXCLUDE_FLAGS)
command.extend(('--filter=-/ {}'.format(x) for x in sorted(exclude)))
command.extend(extrargs)
command.append(verbosity)
command.append(src)
command.append(dst)
try:
return sp.check_output(command, universal_newlines=True)
except sp.CalledProcessError as e:
# Not all rsync errors are to be considered as errors
if e.returncode != 24 or re.search(RSYNC_REGEX, e.output, re.M) is None:
raise
return e.output
def copydb(sqlite3bin, src_db, dst_db):
makedirs(dst_db)
sp.check_output((sqlite3bin, src_db, '.backup ' + dst_db))
def remove(path):
try:
os.remove(path)
except OSError:
if os.path.exists(path):
raise
def parse_installed(partition):
paths = []
custom_script = os.path.join(partition, 'srv', '.backup_identity_script')
for cfg in glob.glob(os.path.join(partition, '.installed*.cfg')):
try:
with open(cfg) as f:
installed_cfg = zc.buildout.configparser.parse(f, cfg)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in six.itervalues(installed_cfg):
for p in section.get('__buildout_installed__', '').splitlines():
p = p.strip()
if p and p != custom_script:
paths.append(p)
return paths
def sha256sum(file_path, chunk_size=1024 * 1024):
sha256 = hashlib.sha256()
with open(file_path, 'rb') as f:
chunk = f.read(chunk_size)
while chunk:
sha256.update(chunk)
chunk = f.read(chunk_size)
return sha256.hexdigest()
def hashwalk(backup_dir, mirror_partitions):
scripts = {}
for p in mirror_partitions:
script_path = os.path.join(p, 'srv', '.backup_identity_script')
if os.path.exists(script_path):
scripts[os.path.abspath(p)] = script_path
for dirpath, dirnames, filenames in os.walk(backup_dir):
filenames.sort()
for f in filenames:
filepath = os.path.join(dirpath, f)
if os.path.isfile(filepath):
displaypath = os.path.relpath(filepath, start=backup_dir)
yield '%s %s' % (sha256sum(filepath), displaypath)
remaining_dirnames = []
for subdir in dirnames:
subdirpath = os.path.abspath(os.path.join(dirpath, subdir))
custom_hashscript = scripts.get(subdirpath)
if custom_hashscript:
print('Using custom signature script %s' % custom_hashscript)
for s in hashcustom(subdirpath, backup_dir, custom_hashscript):
yield s
else:
remaining_dirnames.append(subdir)
remaining_dirnames.sort()
dirnames[:] = remaining_dirnames
@contextlib.contextmanager
def cwd(path):
old_path = os.getcwd()
try:
os.chdir(path)
yield
finally:
os.chdir(old_path)
def hashcustom(mirrordir, backup_dir, custom_hashscript):
workingdir = os.path.join(mirrordir, os.pardir, os.pardir, os.pardir)
with cwd(os.path.abspath(workingdir)):
for dirpath, _, filenames in os.walk(mirrordir):
filepaths = []
for f in filenames:
path = os.path.join(dirpath, f)
if os.path.isfile(path):
filepaths.append('./' + os.path.relpath(path, start=workingdir))
if not filepaths:
continue
hashprocess = sp.Popen(
custom_hashscript, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE)
out, err = hashprocess.communicate(str2bytes('\0'.join(filepaths)))
if hashprocess.returncode != 0:
template = "Custom signature script %s failed on inputs:\n%s"
msg = template % (custom_hashscript, '\n'.join(filepaths))
msg += "\nwith stdout:\n%s" % bytes2str(out)
msg += "\nand stderr:\n%s" % bytes2str(err)
raise Exception(msg)
signatures = bytes2str(out).strip('\n').split('\n')
signatures.sort()
displaypath = os.path.relpath(dirpath, start=backup_dir)
for s in signatures:
yield '%s %s/ (custom)' % (s, displaypath)
software/theia/theia_export.py 0 → 100644
View file @ 3caed389
import argparse
import glob
import itertools
import os
import sys
import time
import traceback
import six
from six.moves import configparser
sys.path.append(os.path.dirname(__file__))
from theia_common import copytree, copydb, hashwalk, parse_installed, remove
os.environ['LC_ALL'] = 'C'
os.umask(0o77)
BACKUP_WAIT = 10
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--rsync', required=True)
parser.add_argument('--sqlite3', required=True)
parser.add_argument('--root', required=True)
parser.add_argument('--backup', required=True)
parser.add_argument('--cfg', required=True)
parser.add_argument('--dirs', action='append')
parser.add_argument('--exitfile', required=True)
parser.add_argument('--errorfile', required=True)
args = parser.parse_args()
TheiaExport(args)()
class TheiaExport(object):
def __init__(self, args):
self.rsync_bin = args.rsync
self.sqlite3_bin = args.sqlite3
self.root_dir = args.root
self.backup_dir = args.backup
self.slapos_cfg = cfg = args.cfg
self.dirs = args.dirs
self.exit_file = args.exitfile
self.error_file = args.errorfile
configp = configparser.SafeConfigParser()
configp.read(cfg)
self.proxy_db = configp.get('slapproxy', 'database_uri')
self.instance_dir = configp.get('slapos', 'instance_root')
partitions = glob.glob(os.path.join(self.instance_dir, 'slappart*'))
self.partition_dirs = [p for p in partitions if os.path.isdir(p)]
self.copytree_partitions_args = {}
self.logs = []
def mirrorpath(self, src):
return os.path.abspath(os.path.join(
self.backup_dir, os.path.relpath(src, start=self.root_dir)))
def backuptree(self, src, exclude=[], extrargs=[], verbosity='-v'):
dst = self.mirrorpath(src)
return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
def backupdb(self):
copydb(self.sqlite3_bin, self.proxy_db, self.mirrorpath(self.proxy_db))
def backuppartition(self, partition):
installed = parse_installed(partition)
rules = os.path.join(partition, 'srv', 'exporter.exclude')
extrargs = ('--filter=.-/ ' + rules,) if os.path.exists(rules) else ()
self.backuptree(partition, exclude=installed, extrargs=extrargs)
self.copytree_partitions_args[partition] = (installed, extrargs)
def sign(self, signaturefile):
remove(signaturefile)
pardir = os.path.abspath(os.path.join(self.backup_dir, os.pardir))
tmpfile = os.path.join(pardir, 'backup.signature.tmp')
mirror_partitions = [self.mirrorpath(p) for p in self.partition_dirs]
with open(tmpfile, 'w') as f:
for s in hashwalk(self.backup_dir, mirror_partitions):
f.write(s + '\n')
os.rename(tmpfile, signaturefile)
def checkpartition(self, partition, pattern='/srv/backup/'):
installed, extrargs = self.copytree_partitions_args[partition]
output = self.backuptree(
partition,
exclude=installed,
extrargs=extrargs + ('--dry-run', '--update'),
verbosity='--out-format=%n',
)
return [path for path in output.splitlines() if pattern in path]
def loginfo(self, msg):
print(msg)
self.logs.append(msg)
def __call__(self):
remove(self.error_file)
exitcode = 0
try:
self.export()
except Exception:
exitcode = 1
exc = traceback.format_exc()
with open(self.error_file, 'w') as f:
f.write('\n ... OK\n\n'.join(self.logs))
f.write('\n ... ERROR !\n\n')
f.write(exc)
print('\n\nERROR\n\n' + exc)
finally:
with open(self.exit_file, 'w') as f:
f.write(str(exitcode))
sys.exit(exitcode)
def export(self):
export_start_date = int(time.time())
etc_dir = os.path.join(self.root_dir, 'etc')
with open(os.path.join(etc_dir, '.resilient_timestamp'), 'w') as f:
f.write(str(export_start_date))
self.loginfo('Backup directory ' + etc_dir)
self.backuptree(etc_dir, extrargs=('--filter=- */', '--filter=-! .*'))
for d in self.dirs:
self.loginfo('Backup directory ' + d)
self.backuptree(d)
self.loginfo('Backup slapproxy database')
self.backupdb()
self.loginfo('Backup partitions')
for p in self.partition_dirs:
self.backuppartition(p)
self.loginfo('Compute backup signature')
self.sign(os.path.join(self.backup_dir, 'backup.signature'))
time.sleep(10)
self.loginfo('Check partitions')
modified = list(itertools.chain.from_iterable(
self.checkpartition(p) for p in self.partition_dirs))
if modified:
msg = 'Some files have been modified since the backup started'
self.loginfo(msg + ':')
self.loginfo('\n'.join(modified))
self.loginfo("Let's wait %d minutes and try again" % BACKUP_WAIT)
time.sleep(BACKUP_WAIT * 60)
raise Exception(msg)
self.loginfo('Done')
if __name__ == '__main__':
main()
software/theia/theia_import.py 0 → 100644
View file @ 3caed389
import argparse
import glob
import itertools
import os
import sys
import subprocess as sp
import traceback
import six
from six.moves import configparser
sys.path.append(os.path.dirname(__file__))
from theia_common import copytree, copydb, hashwalk, parse_installed, remove
os.environ['LC_ALL'] = 'C'
os.umask(0o77)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--rsync', required=True)
parser.add_argument('--sqlite3', required=True)
parser.add_argument('--slapos', required=True)
parser.add_argument('--srlog', required=True)
parser.add_argument('--cplog', required=True)
parser.add_argument('--supervisorctl', required=True)
parser.add_argument('--supervisordconf', required=True)
parser.add_argument('--root', required=True)
parser.add_argument('--backup', required=True)
parser.add_argument('--cfg', required=True)
parser.add_argument('--dirs', action='append')
parser.add_argument('--exitfile', required=True)
parser.add_argument('--errorfile', required=True)
args = parser.parse_args()
TheiaImport(args)()
class TheiaImport(object):
def __init__(self, args):
self.rsync_bin = args.rsync
self.sqlite3_bin = args.sqlite3
self.slapos_bin = args.slapos
self.sr_log = args.srlog
self.cp_log = args.cplog
self.supervisorctl_bin = args.supervisorctl
self.supervisord_conf = args.supervisordconf
self.root_dir = args.root
self.backup_dir = args.backup
self.slapos_cfg = cfg = args.cfg
self.dirs = args.dirs
self.exit_file = args.exitfile
self.error_file = args.errorfile
configp = configparser.SafeConfigParser()
configp.read(cfg)
self.proxy_db = configp.get('slapproxy', 'database_uri')
self.instance_dir = configp.get('slapos', 'instance_root')
mirror_dir = self.mirrorpath(self.instance_dir)
partitions = glob.glob(os.path.join(mirror_dir, 'slappart*'))
self.mirror_partition_dirs = [p for p in partitions if os.path.isdir(p)]
self.logs = []
def mirrorpath(self, dst):
return os.path.abspath(os.path.join(
self.backup_dir, os.path.relpath(dst, start=self.root_dir)))
def dstpath(self, src):
return os.path.abspath(os.path.join(
self.root_dir, os.path.relpath(src, start=self.backup_dir)))
def restoretree(self, dst, exclude=[], extrargs=[], verbosity='-v'):
src = self.mirrorpath(dst)
return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
def restoredb(self):
copydb(self.sqlite3_bin, self.mirrorpath(self.proxy_db), self.proxy_db)
def restorepartition(self, mirror_partition):
p = self.dstpath(mirror_partition)
installed = parse_installed(p) if os.path.exists(p) else []
copytree(self.rsync_bin, mirror_partition, p, exclude=installed)
def supervisorctl(self, *args):
supervisor_command = (self.supervisorctl_bin, '-c', self.supervisord_conf)
command = supervisor_command + args
print(' '.join(command))
sp.check_call(command)
def slapos(self, *args):
command = (self.slapos_bin,) + args + ('--cfg', self.slapos_cfg)
print(' '.join(command))
sp.check_call(command)
def verify(self, signaturefile):
pardir = os.path.abspath(os.path.join(self.backup_dir, os.pardir))
moved = os.path.join(pardir, 'backup.signature.moved')
proof = os.path.join(pardir, 'backup.signature.proof')
if os.path.exists(signaturefile):
os.rename(signaturefile, moved)
if not os.path.exists(moved):
msg = 'ERROR the backup signature file is missing'
print(msg)
raise Exception(msg)
with open(proof, 'w') as f:
for s in hashwalk(self.backup_dir, self.mirror_partition_dirs):
f.write(s + '\n')
diffcommand = ('diff', moved, proof)
print(' '.join(diffcommand))
try:
sp.check_output(
diffcommand, stderr=sp.STDOUT, universal_newlines=True)
except sp.CalledProcessError as e:
template = 'ERROR the backup signatures do not match\n\n%s'
msg = template % e.output
print(msg)
raise Exception(msg)
def loginfo(self, msg):
print(msg)
self.logs.append(msg)
def __call__(self):
remove(self.error_file)
exitcode = 0
try:
self.restore()
except Exception:
exitcode = 1
exc = traceback.format_exc()
with open(self.error_file, 'w') as f:
f.write('\n ... OK\n\n'.join(self.logs))
f.write('\n ... ERROR !\n\n')
f.write(exc)
print('\n\nERROR\n\n' + exc)
finally:
with open(self.exit_file, 'w') as f:
f.write(str(exitcode))
sys.exit(exitcode)
def restore(self):
self.loginfo('Verify backup signature')
self.verify(os.path.join(self.backup_dir, 'backup.signature'))
self.loginfo('Stop slapproxy')
self.supervisorctl('stop', 'slapos-proxy')
self.loginfo('Restore partitions')
for m in self.mirror_partition_dirs:
self.restorepartition(m)
for d in self.dirs:
self.loginfo('Restore directory ' + d)
self.restoretree(d)
self.loginfo('Restore slapproxy database')
self.restoredb()
etc_dir = os.path.join(self.root_dir, 'etc')
self.loginfo('Restore directory ' + etc_dir)
self.restoretree(etc_dir, extrargs=('--filter=- */', '--filter=-! .*'))
custom_script = os.path.join(self.root_dir, 'srv', 'runner-import-restore')
if os.path.exists(custom_script):
self.loginfo('Run custom restore script %s' % custom_script)
sp.check_call(custom_script)
self.loginfo('Start slapproxy again')
self.supervisorctl('start', 'slapos-proxy')
self.loginfo('Reformat partitions')
self.slapos('node', 'format', '--now')
self.loginfo('Remove old supervisord configuration files')
conf_dir = os.path.join(self.instance_dir, 'etc', 'supervisor.conf.d')
for f in glob.glob(os.path.join(conf_dir, '*')):
os.remove(f)
self.loginfo('Build Software Releases')
for i in range(3):
try:
self.slapos('node', 'software', '--all', '--logfile', self.sr_log)
except sp.CalledProcessError:
if i == 2:
raise
else:
break
self.loginfo('Remove old custom instance scripts')
partitions_glob = os.path.join(self.instance_dir, 'slappart*')
scripts = os.path.join(partitions_glob, 'srv', 'runner-import-restore')
for f in glob.glob(scripts):
remove(f)
self.loginfo('Remove partition timestamps')
timestamps = os.path.join(partitions_glob, '.timestamp')
for f in glob.glob(timestamps):
remove(f)
self.loginfo('Build Instances')
cp_log = self.cp_log
for i in range(3):
try:
self.slapos('node', 'instance', '--force-stop', '--logfile', cp_log)
except sp.CalledProcessError:
if i == 2:
raise
else:
break
for custom_script in glob.glob(scripts):
self.loginfo('Running custom instance script %s' % custom_script)
sp.check_call(custom_script)
self.loginfo('Done')
if __name__ == '__main__':
main()
software/theia/yarn.lock
View file @ 3caed389
This source diff could not be displayed because it is too large. You can view the blob instead.
software/unstable/condor/software.cfg
View file @ 3caed389
......@@ -62,31 +62,10 @@ slapos.recipe.cmmi = 0.1.1
slapos.recipe.template = 2.4.2
xml-marshaller = 0.9.7
slapos.cookbook=0.78.1
# Required by:
# slapos.core==0.35.1
Flask = 0.9
# Required by:
# slapos.cookbook==0.78.1
lock-file = 2.0
# Required by:
# slapos.core==0.35.1
netifaces = 0.10.4
# Required by:
# slapos.core==0.35.1
pyflakes = 0.7.2
# Required by:
# slapos.core==0.35.1
supervisor = 3.0b1
# Required by:
# slapos.core==0.35.1
unittest2 = 0.5.1
# Required by:
# slapos.core==0.35.1
zope.interface = 4.0.5
stack/cloudooo.cfg
View file @ 3caed389
......@@ -75,8 +75,4 @@ PasteScript = 3.2.0:whl
WSGIUtils = 0.7.2
WSGIserver = 1.3
python-magic = 0.4.18
# Required by:
# PasteScript== 3.2.0
# cloudooo==1.2.6.dev0
PasteDeploy = 2.1.0
stack/erp5/buildout.cfg
View file @ 3caed389
......@@ -720,88 +720,39 @@ cloudpickle = 0.5.3
dask = 0.18.1
toolz = 0.9.0
zope.globalrequest = 1.5
waitress = 1.3.0
waitress = 1.4.4
xlrd = 1.1.0
# Re-add for as it is required to be there for uninstallation
erp5.recipe.w3validator = 1.0.2
# Required by:
# Products.CMFCore==2.2.10
Products.ZSQLMethods = 2.13.5
# Required by:
# SOAPpy===0.12.0nxd001
fpconst = 0.7.2
# Required by:
# objgraph==3.1.0
graphviz = 0.5.2
# Required by:
# Pillow==6.2.2
olefile = 0.44
# Required by:
# munnel==0.3
python-libmilter = 1.0.3
# Required by:
# zope.app.testing==3.8.1
zope.app.debug = 3.4.1
# Required by:
# zope.app.testing==3.8.1
zope.app.dependable = 3.5.1
# Required by:
# Products.CMFCalendar==2.2.3
# five.formlib==1.0.4
zope.app.form = 4.0.2
et-xmlfile = 1.0.1
h5py = 2.7.1
mpmath = 0.19
openpyxl = 2.4.8
sympy = 1.1.1
# Required by:
# openpyxl==2.4.8
jdcal = 1.3
xmltodict = 0.11.0
deepdiff = 3.3.0
unidiff = 0.5.5
# Required by:
# deepdiff = 3.3.0
jsonpickle = 0.9.6
responses = 0.10.6
# Required by:
# responses = 0.10.6
cookies = 2.2.1
jedi = 0.15.1
parso = 0.5.1
yapf = 0.28.0
# Required by:
# erp5.util==0.4.65
z3c.etestbrowser = 3.0.1
zope.testbrowser = 5.5.1
# Required by:
# zope.testbrowser==5.5.1
WSGIProxy2 = 0.4.6
WebTest = 2.0.33
beautifulsoup4 = 4.8.2
# Required by:
# WSGIProxy2==0.4.6
WebOb = 1.8.5
soupsieve = 1.9.5
eggtestinfo = 0.3
stack/resilient/buildout.hash.cfg
View file @ 3caed389
......@@ -14,7 +14,7 @@
# not need these here).
[pbsready]
filename = pbsready.cfg.in
md5sum = 9ceceeee21fa90837c887d2d6866859e
md5sum = 005125621d157b3ae04c428ea6060e37
[pbsready-import]
filename = pbsready-import.cfg.in
......@@ -26,7 +26,7 @@ md5sum = 2b0c71b085cfe8017f28098c160b1f49
[template-pull-backup]
filename = instance-pull-backup.cfg.in
md5sum = e64e13854332bcc2595df187fcae1203
md5sum = b240dc76a663190304d8bcb9cabcda8f
[template-replicated]
filename = template-replicated.cfg.in
......
stack/resilient/instance-pull-backup.cfg.in
View file @ 3caed389
......@@ -137,12 +137,17 @@ command-line = ${buildout:bin-directory}/generatefeed --output $${:feed-path} --
feed-path = $${directory:monitor-resilient}/pbs-status-rss
wrapper-path = $${rootdirectory:bin}/resilient-genstatusrss.py
[pbs-status-feed-first-run]
recipe = plone.recipe.command
command = $${pbs-resilient-status-feed:wrapper-path}
stop-on-error = true
[cron-pbs-status-feed]
<= cron
recipe = slapos.cookbook:cron.d
name = resilient-pbs-status-feed
frequency = */5 * * * *
command = $${pbs-resilient-status-feed:wrapper-path}
command = $${pbs-status-feed-first-run:command}
[logrotate-entry-notifier]
<= logrotate-entry-base
......
stack/resilient/pbsready.cfg.in
View file @ 3caed389
......@@ -145,12 +145,17 @@ command-line = ${buildout:bin-directory}/generatefeed --output $${:feed-path} --
feed-path = $${directory:monitor-resilient}/notifier-status-rss
wrapper-path = $${rootdirectory:bin}/resilient-genstatusrss.py
[notifier-status-feed-first-run]
recipe = plone.recipe.command
command = $${notifier-resilient-status-feed:wrapper-path}
stop-on-error = true
[cron-entry-notifier-status-feed]
<= cron
recipe = slapos.cookbook:cron.d
name = resilient-notifier-status-feed
frequency = */5 * * * *
command = $${notifier-resilient-status-feed:wrapper-path}
command = $${notifier-status-feed-first-run:command}
[notifier-stalled-promise-bin]
recipe = slapos.cookbook:wrapper
......
stack/slapos.cfg
View file @ 3caed389
......@@ -186,11 +186,12 @@ pytz = 2016.10
regex = 2020.9.27
requests = 2.24.0
scandir = 1.10.0
setproctitle = 1.1.10
setuptools-dso = 1.7
rubygemsrecipe = 0.3.0
rubygemsrecipe = 0.4.2
six = 1.12.0
slapos.cookbook = 1.0.197
slapos.core = 1.6.17
slapos.core = 1.6.18
slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.20
......@@ -208,141 +209,46 @@ xml-marshaller = 1.0.2
zc.lockfile = 1.0.2
zdaemon = 4.2.0
zipp = 1.2.0:whl
zodburi = 2.4.0
zodburi = 2.5.0
zope.event = 3.5.2
paramiko = 2.1.3
CacheControl = 0.12.6:whl
msgpack = 0.6.2
# Required by:
# slapos.core==1.5.0
Flask = 1.1.2
# Required by:
# slapos.toolbox==0.94
GitPython = 2.1.11
# Required by:
# GitPython==2.1.11
gitdb2 = 2.0.5
# Required by:
# gitdb==2.0.5
smmap2 = 2.0.5
# Required by:
# slapos.toolbox==0.94
PyRSS2Gen = 1.1
# Required by:
# slapos.toolbox==0.94
apache-libcloud = 2.4.0
# Required by:
# slapos.toolbox==0.94
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.94
croniter = 0.3.25
# Required by:
# slapos.toolbox==0.94
dnspython = 1.16.0
enum34 = 1.1.10
# Required by:
# slapos.toolbox==0.94
erp5.util = 0.4.69
# Required by:
# slapos.toolbox==0.94
erp5.util = 0.4.70
feedparser = 5.2.1
# Required by:
# jsonschema==3.0.2
functools32 = 3.2.3.post2
# Required by:
# jsonschema==3.0.2
attrs = 18.2.0
# Required by:
# jsonschema==3.0.2
pyrsistent = 0.16.1
pytest-runner = 5.2.0:whl
ipaddress = 1.0.23
# Required by:
# slapos.cookbook==1.0.143
jsonschema = 3.0.2:whl
# Required by:
# slapos.toolbox==0.94
lockfile = 0.12.2:whl
# Required by:
# slapos.core==1.5.0
# XXX 'slapos node format' raises an exception with netifaces 0.10.5.
netifaces = 0.10.7
packaging = 16.8
# Required by:
# slapos.toolbox==0.94
passlib = 1.7.1
# Required by:
# slapos.toolbox==0.94
pyasn1 = 0.4.5
# Required by:
# cffi==1.9.1
pycparser = 2.20
# Required by:
# slapos.toolbox==0.94
pycurl = 7.43.0
# Required by:
# slapos.toolbox==0.94
python-dateutil = 2.7.3:whl
# Required by:
# slapos.toolbox==0.94
rpdb = 0.1.5
# Required by:
# slapos.core==1.5.0
supervisor = 4.1.0
# Required by:
# slapos.toolbox==0.94
tzlocal = 1.5.1
# Required by:
# slapos.core==1.5.0
uritemplate = 3.0.0
# Required by:
# slapos.core==1.5.0
zope.interface = 4.3.3
# Required by:
# requests==2.24.0
certifi = 2020.6.20
# Required by:
# requests==2.24.0
chardet = 3.0.4
# Required by:
# requests==2.24.0
urllib3 = 1.25.9
pkgconfig = 1.5.1
[networkcache]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7