Commit ef19bed5 authored by Jérome Perrin's avatar Jérome Perrin

Update Release Candidate

parents 021ae883 6b2b895c
......@@ -39,9 +39,9 @@ configure-options =
[apache]
recipe = slapos.recipe.cmmi
shared = true
version = 2.4.49
version = 2.4.51
url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = f294efbeabcf6027fccc7983a6daa55f
md5sum = d2793fc1c8cb8ba355cee877d1f2d46d
configure-options = --disable-static
--enable-authn-alias
--enable-bucketeer
......
......@@ -15,8 +15,8 @@ extends =
[groonga]
recipe = slapos.recipe.cmmi
shared = true
url = https://packages.groonga.org/source/groonga/groonga-11.0.2.tar.gz
md5sum = 753ba6fad77598baf93615c4b9c535b1
url = https://packages.groonga.org/source/groonga/groonga-11.0.9.tar.gz
md5sum = 9c66445d92c8b7536f1b28119ac1855b
groonga-plugin-dir = @@LOCATION@@/lib/groonga/plugins/
# temporary patch to respect more tokens in natural language mode.
patches =
......@@ -48,9 +48,8 @@ environment =
[groonga-normalizer-mysql]
recipe = slapos.recipe.cmmi
shared = true
url = https://packages.groonga.org/source/groonga-normalizer-mysql/groonga-normalizer-mysql-1.1.4.tar.gz
md5sum = effa67fb271d49810850a3b275d040f6
url = https://packages.groonga.org/source/groonga-normalizer-mysql/groonga-normalizer-mysql-1.1.5.tar.gz
md5sum = 842d02becc6dcc25a02fa7e789c2cba7
groonga-plugin-dir = @@LOCATION@@/lib/groonga/plugins/
pre-configure = mkdir -p ${:groonga-plugin-dir}
make-targets = GROONGA_PLUGINS_DIR=${:groonga-plugin-dir} install
......
......@@ -53,7 +53,8 @@ version = 92.0.4515.107
[headless-chromium]
recipe = slapos.recipe.cmmi
path = ${chromium-source:location}
location = ${:path}/out/headless
# XXX
fake-location = ${:path}/out/headless
# Configuration file for GN, the tool to build the actual compilation
# configuration file.
......@@ -91,17 +92,17 @@ configure-command =
> ${chromium-source:gclient-location}/.gclient
gclient sync --no-history
# Generate build configuration files.
mkdir -p ${:location}
echo '${:build-config-options}' > ${:location}/args.gn
gn gen ${:location}
mkdir -p ${:fake-location}
echo '${:build-config-options}' > ${:fake-location}/args.gn
gn gen ${:fake-location}
# You can run the headless Chromium shell using
# ${:binary} --remote-debugging-port=1234
make-binary =
autoninja -C ${:location} headless_shell
autoninja -C ${:fake-location} headless_shell
# By building our own version of Chromedriver, we can ensure version
# compatibility. The build is quite cheap compared to Chromium, anyway.
autoninja -C ${:location} chromedriver
autoninja -C ${:fake-location} chromedriver
environment =
PATH=${depot_tools:location}:${gperf:location}/bin:${pkgconfig:location}/bin:${coreutils:location}/bin:${git:location}/bin:${curl:location}/bin:%(PATH)s
LDFLAGS="-Wl,-rpath=${nss:location}/lib,-rpath=${nspr:location}/lib"
......@@ -111,10 +112,10 @@ environment =
NM="${:llvm-toolchain}/llvm-nm"
# Expose devtools frontend location.
devtools-frontend = ${:location}/gen/third_party/devtools-frontend/src/front_end
devtools-frontend = ${:fake-location}/gen/third_party/devtools-frontend/src/front_end
binary = ${:location}/headless_shell
chromedriver = ${:location}/chromedriver
binary = ${:fake-location}/headless_shell
chromedriver = ${:fake-location}/chromedriver
promises =
${:binary}
${:chromedriver}
......
# LuaJIT is a Just-In-Time Compiler (JIT) for the Lua programming language.
# https://luajit.org/luajit.html
[buildout]
parts = luajit
[luajit]
recipe = slapos.recipe.cmmi
shared = true
url = https://luajit.org/download/LuaJIT-2.0.5.tar.gz
md5sum = 48353202cbcacab84ee41a5a70ea0a2c
configure-command = true
# pass dummy LDCONFIG to skip needless calling of ldconfig by non-root user
make-options =
DPREFIX=@@LOCATION@@
LDCONFIG=/bin/echo
......@@ -30,16 +30,15 @@ parts =
recipe = slapos.recipe.cmmi
shared = true
url = https://archive.mariadb.org//mariadb-${:version}/source/mariadb-${:version}.tar.gz
version = 10.4.19
md5sum = bf60c7a3feac5854745cd1ad5133f09a
location = @@LOCATION@@
version = 10.4.22
md5sum = 0d5e1b9e3694322e18819811a2bf81fa
pre-configure =
set '\bSET(PLUGIN_AUTH_PAM YES CACHE BOOL "")' cmake/build_configurations/mysql_release.cmake
grep -q "$@"
sed -i "/$1/d" "$2"
configure-command = ${cmake:location}/bin/cmake
configure-options =
-DCMAKE_INSTALL_PREFIX=${:location}
-DCMAKE_INSTALL_PREFIX=@@LOCATION@@
-DBUILD_CONFIG=mysql_release
-DDEFAULT_CHARSET=utf8
-DDEFAULT_COLLATION=utf8_unicode_ci
......@@ -68,6 +67,10 @@ configure-options =
-DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
-DCMAKE_INCLUDE_PATH=${unixodbc:location}/include
-DCMAKE_LIBRARY_PATH=${unixodbc:location}/lib
# for RocksDB - see
# https://lore.kernel.org/linux-btrfs/ed3642c2-682e-08a1-f18d-2d63409b7631@nexedi.com/T/
-DWITH_FALLOCATE=NO
##
CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include -I${unixodbc:location}/include -I${lz4:location}/include -I${snappy:location}/include -I${zstd:location}/include
CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib:${unixodbc:location}/lib:${lz4:location}/lib:${snappy:location}/lib:${zstd:location}/lib
environment =
......@@ -80,13 +83,13 @@ patch-options = -p1
patches =
https://sources.debian.org/data/main/m/mariadb-10.3/1:10.3.22-0+deb10u1/debian/patches/0024-Revert-to-using-system-pcre-library.patch#1c6a0f2634f5a56122299674b77b1131
post-install =
ldd=`ldd ${:location}/lib/plugin/ha_rocksdb.so`
ldd=`ldd %(location)s/lib/plugin/ha_rocksdb.so`
for x in ${lz4:location} ${snappy:location} ${zstd:location}
do echo "$ldd" |grep -qF " $x/lib/"
done
set -- wsrep-lib/wsrep-API/*/wsrep_api.h
install -DpT $1 ${:location}/$1
cp -a wsrep-lib/include ${:location}/wsrep-lib
install -DpT $1 %(location)s/$1
cp -a wsrep-lib/include %(location)s/wsrep-lib
[mroonga-mariadb]
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
......@@ -96,8 +99,8 @@ post-install =
# as plugin-dir ( https://mariadb.com/kb/en/server-system-variables/#plugin_dir )
recipe = slapos.recipe.cmmi
shared = true
url = https://packages.groonga.org/source/mroonga/mroonga-11.02.tar.gz
md5sum = 0729c74efc92bfc404b88597488d07e9
url = https://packages.groonga.org/source/mroonga/mroonga-11.09.tar.gz
md5sum = 8b1786332edc61c41a769f225e6063b2
pre-configure =
rm -rf fake_mariadb_source
mkdir -p fake_mariadb_source
......@@ -131,15 +134,15 @@ environment =
### (we just override here for easier revert)
[mariadb-10.3]
<= mariadb-10.4
version = 10.3.29
md5sum = a5adad1c4fb1717d7fe6d608fd4d40de
version = 10.3.32
md5sum = 12341dc150c810c0072a40e55825ca57
post-install =
ldd=`ldd ${:location}/lib/plugin/ha_rocksdb.so`
ldd=`ldd %(location)s/lib/plugin/ha_rocksdb.so`
for x in ${lz4:location} ${snappy:location} ${zstd:location}
do echo "$ldd" |grep -qF " $x/lib/"
done
mkdir -p ${:location}/include/wsrep &&
cp -p wsrep/wsrep_api.h ${:location}/include/wsrep
mkdir -p %(location)s/include/wsrep &&
cp -p wsrep/wsrep_api.h %(location)s/include/wsrep
[mariadb]
location = ${mariadb-10.3:location}
......
[buildout]
extends =
../zlib/buildout.cfg
parts = tcl
[tcl]
......@@ -6,6 +8,9 @@ recipe = slapos.recipe.cmmi
url = http://prdownloads.sourceforge.net/tcl/tcl8.6.11-src.tar.gz
md5sum = 8a4c004f48984a03a7747e9ba06e4da4
shared = true
environment =
CPPFLAGS=-I${zlib:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
configure-command = ./unix/configure
configure-options =
--prefix=@@LOCATION@@
......
......@@ -3,6 +3,7 @@ extends =
../defaults.cfg
../libtool/buildout.cfg
../libxml2/buildout.cfg
../luajit/buildout.cfg
../make/buildout.cfg
../ncurses/buildout.cfg
../openssl/buildout.cfg
......@@ -32,6 +33,7 @@ configure-options =
--with-pcre=${pcre:location}
--with-ncurses=${ncurses:location}
--with-tcl=${tcl:location}/lib/
--with-luajit=${luajit:location}
--with-lzma=${xz-utils:location}
--with-zlib=${zlib:location}
--disable-curl
......@@ -40,7 +42,7 @@ configure-options =
--disable-posix-cap
environment =
PATH=${libtool:location}/bin:${make:location}/bin:${patch:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
LDFLAGS =-L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${tcl:location}/lib -Wl,-rpath=${tcl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
LDFLAGS =-L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${tcl:location}/lib -Wl,-rpath=${tcl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${luajit:location}/lib -lm
make-target =
check
......
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '1.0.214'
version = '1.0.217'
name = 'slapos.cookbook'
long_description = open("README.rst").read()
......
......@@ -107,7 +107,9 @@ class Mac(object):
pass
def generatePassword(length):
return ''.join(random.SystemRandom().sample(string.ascii_lowercase, length))
system_random = random.SystemRandom()
alphabet = string.ascii_letters + string.digits
return ''.join(system_random.choice(alphabet) for i in range(length))
class Password(object):
......@@ -119,7 +121,7 @@ class Password(object):
recipes like slapos.recipe.template:jinja2 to safely process the password.
Options:
- bytes: password length (default: 8 characters)
- bytes: password length (default: 16 characters)
- storage-path: plain-text persistent storage for password,
that can only be accessed by the user
(default: ${buildout:parts-directory}/${:_buildout_section_name_})
......@@ -149,7 +151,7 @@ class Password(object):
if e.errno != errno.ENOENT:
raise
if not passwd:
passwd = self.generatePassword(int(options.get('bytes', '8')))
passwd = self.generatePassword(int(options.get('bytes', '16')))
self.update = self.install
options['passwd'] = passwd
# Password must not go into .installed file, for 2 reasons:
......
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg.in
md5sum = 1dfbd20c77fb3c1f01005a8a920d2ed9
md5sum = fb3a20e7f555a9ce7fe1ec547d0fcdfc
[profile-common]
filename = instance-common.cfg.in
......@@ -26,11 +26,11 @@ md5sum = 0950e09ad1f03f0789308f5f7a7eb1b8
[profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 7c2e52b76c42bed95702763c344e41dd
md5sum = c5d1e235959a877b4f3157369c6f5e10
[profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = 313671d343ceccfca5af1baa642132c5
md5sum = c67e172c0c6eca955b18962404056a33
[profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
......@@ -50,7 +50,7 @@ md5sum = 37475d79f28c5f126bc1947fdb938fdb
[template-backend-haproxy-configuration]
_update_hash_filename_ = templates/backend-haproxy.cfg.in
md5sum = d2851c7ebd2c9baa2edecb3ca3485511
md5sum = ae4c9ce775ea003aa51eda5ecbbeec73
[template-empty]
_update_hash_filename_ = templates/empty.in
......@@ -98,7 +98,7 @@ md5sum = 8e1c6c06c09beb921965b3ce98c67c9e
[caddyprofiledeps-dummy]
filename = caddyprofiledummy.py
md5sum = 38792c2dceae38ab411592ec36fff6a8
md5sum = 59cb33f11272ee09eccea74981d2304a
[profile-kedifa]
filename = instance-kedifa.cfg.in
......
import urlparse
class Recipe(object):
def __init__(self, *args, **kwargs):
pass
......@@ -7,3 +9,13 @@ class Recipe(object):
def update(self):
return self.install()
def validate_netloc(netloc):
# a bit crazy way to validate that the passed parameter is haproxy
# compatible server netloc
parsed = urlparse.urlparse('scheme://'+netloc)
if ':' in parsed.hostname:
hostname = '[%s]' % parsed.hostname
else:
hostname = parsed.hostname
return netloc == '%s:%s' % (hostname, parsed.port)
......@@ -207,6 +207,15 @@ context =
{% endif %}
{% endif %}
{% endfor %}
{% for url_key in ['url-netloc-list', 'https-url-netloc-list', 'health-check-failover-url-netloc-list'] %}
{% if url_key in slave %}
{% for netloc in slave[url_key].split() %}
{% if not caddyprofiledummy.validate_netloc(netloc) %}
{% do slave_error_list.append('slave %s %r invalid' % (url_key, netloc)) %}
{% endif %}
{% endfor %}
{% endif %}
{% endfor %}
{% for k in ['ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt'] %}
{% if k in slave %}
{% set crt = slave.get(k, '') %}
......
......@@ -348,6 +348,26 @@
],
"type": "string",
"default": "false"
},
"url-netloc-list": {
"type": "string",
"title": "[EXPERT] List of netlocs for \"Backend URL\"",
"description": "Space separated list of netlocs (ip and port) of backend to connect to. They will share the scheme and path of the original URL and additional backend parameters (like \"SSL Backend Authority's Certificate\"). Each of them will be used, and at least one is enough for the connectivity to work, and the best results are with \"Health Check\" feature enabled. Port is mandatory, so hostnames shall be provided as hostname:port (eg. example.com:80), IPv4 - as ipv4:port (eg. 127.0.0.1:80), IPv6 - as ipv6:port (eg. ::1:80). Simply this parameters only overrides netloc (network location) of the original URL."
},
"https-url-netloc-list": {
"type": "string",
"title": "[EXPERT] List of netlocs for \"HTTPS Backend URL\"",
"description": "See \"[EXPERT] List of netlocs for \"Backend URL\"\" description."
},
"health-check-failover-url-netloc-list": {
"type": "string",
"title": "[EXPERT] List of netlocs for \"Failover backend URL\"",
"description": "See \"[EXPERT] List of netlocs for \"Backend URL\"\" description."
},
"health-check-failover-https-url-netloc-list": {
"type": "string",
"title": "[EXPERT] List of netlocs for \"Failover HTTPS Backend URL\"",
"description": "See \"[EXPERT] List of netlocs for \"Backend URL\"\" description."
}
},
"title": "Input Parameters",
......
......@@ -55,6 +55,7 @@ extra-context =
import subprocess_module subprocess
import functools_module functools
import validators validators
import caddyprofiledummy caddyprofiledummy
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-replicate
......
......@@ -53,7 +53,7 @@ context =
{#- * stabilise values for backend #}
{%- for key, prefix in [('url', 'http_backend'), ('https-url', 'https_backend')] %}
{%- set parsed = urlparse_module.urlparse(slave_instance.get(key, '').strip()) %}
{%- set info_dict = {'scheme': parsed.scheme, 'hostname': parsed.hostname, 'port': parsed.port or DEFAULT_PORT[parsed.scheme], 'path': parsed.path, 'fragment': parsed.fragment, 'query': parsed.query} %}
{%- set info_dict = {'scheme': parsed.scheme, 'hostname': parsed.hostname, 'port': parsed.port or DEFAULT_PORT[parsed.scheme], 'path': parsed.path, 'fragment': parsed.fragment, 'query': parsed.query, 'netloc-list': slave_instance.get(key + '-netloc-list', '').split() } %}
{%- do slave_instance.__setitem__(prefix, info_dict) %}
{%- endfor %}
{%- do slave_instance.__setitem__('ssl_proxy_verify', ('' ~ slave_instance.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
......@@ -66,6 +66,7 @@ context =
{%- do info_dict.__setitem__('health-check-failover-path', parsed.path) %}
{%- do info_dict.__setitem__('health-check-failover-query', parsed.query) %}
{%- do info_dict.__setitem__('health-check-failover-fragment', parsed.fragment) %}
{%- do info_dict.__setitem__('health-check-netloc-list', slave_instance.get('health-check-failover-url-netloc-list', '').split()) %}
{%- do slave_instance.__setitem__(prefix, info_dict) %}
{%- endfor %}
{%- do slave_instance.__setitem__('health-check-failover-ssl-proxy-verify', ('' ~ slave_instance.get('health-check-failover-ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
......
......@@ -106,7 +106,7 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
{%- do path_list.append(query) %}
{%- endif %}
{%- set path = '?'.join(path_list) %}
{%- if hostname and port %}
{%- if hostname and port or len(info_dict['netloc-list']) > 0 %}
timeout server {{ slave_instance['request-timeout'] }}s
timeout connect {{ slave_instance['backend-connect-timeout'] }}s
retries {{ slave_instance['backend-connect-retries'] }}
......@@ -122,7 +122,15 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
{%- endif %}
{%- do active_check_option_list.append('timeout check %ss' % (slave_instance['health-check-timeout'])) %}
{%- endif %}
{%- if len(info_dict['netloc-list']) > 0 %}
{%- set counter = {'count': 1} %}
{%- for netloc in info_dict['netloc-list'] %}
server {{ slave_instance['slave_reference'] }}-backend-{{ scheme }}-{{ counter['count'] }} {{ netloc }} {{ ' '.join(ssl_list) }} {{ ' ' + ' '.join(active_check_list)}}
{%- do counter.__setitem__('count', counter['count'] + 1) %}
{%- endfor %}
{%- else %}
server {{ slave_instance['slave_reference'] }}-backend-{{ scheme }} {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }} {{ ' ' + ' '.join(active_check_list)}}
{%- endif %}
{%- for active_check_option in active_check_option_list %}
{{ active_check_option }}
{%- endfor %}
......@@ -161,10 +169,18 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}-failover
{%- endif %}
{%- set path = '?'.join(path_list) %}
{%- if hostname and port %}
timeout server {{ slave_instance['request-timeout'] }}s
{%- if len(info_dict['health-check-netloc-list']) > 0 %}
{%- set counter = {'count': 1} %}
{%- for netloc in info_dict['health-check-netloc-list'] %}
server {{ slave_instance['slave_reference'] }}-backend-{{ scheme }}-{{ counter['count'] }} {{ netloc }} {{ ' '.join(ssl_list) }}
{%- do counter.__setitem__('count', counter['count'] + 1) %}
{%- endfor %}
{%- else %}
server {{ slave_instance['slave_reference'] }}-backend-{{ scheme }} {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
{%- endif %}
timeout connect {{ slave_instance['backend-connect-timeout'] }}s
timeout server {{ slave_instance['request-timeout'] }}s
retries {{ slave_instance['backend-connect-retries'] }}
server {{ slave_instance['slave_reference'] }}-backend-{{ scheme }} {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
{%- if path %}
http-request set-path {{ path }}%[path]
{%- endif %}
......
......@@ -644,16 +644,40 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
server_side=True)
cls.backend_url = 'http://%s:%s/' % server.server_address
cls.server_process = multiprocessing.Process(
server_process = multiprocessing.Process(
target=server.serve_forever, name='HTTPServer')
cls.server_process.start()
cls.logger.debug('Started process %s' % (cls.server_process,))
server_process.start()
cls.logger.debug('Started process %s' % (server_process,))
cls.backend_https_url = 'https://%s:%s/' % server_https.server_address
cls.server_https_process = multiprocessing.Process(
server_https_process = multiprocessing.Process(
target=server_https.serve_forever, name='HTTPSServer')
cls.server_https_process.start()
cls.logger.debug('Started process %s' % (cls.server_https_process,))
server_https_process.start()
cls.logger.debug('Started process %s' % (server_https_process,))
class NetlocHandler(TestHandler):
identification = 'netloc'
netloc_a_http = ThreadedHTTPServer(
(cls._ipv4_address, cls._server_netloc_a_http_port),
NetlocHandler)
netloc_a_http_process = multiprocessing.Process(
target=netloc_a_http.serve_forever, name='netloc-a-http')
netloc_a_http_process.start()
netloc_b_http = ThreadedHTTPServer(
(cls._ipv4_address, cls._server_netloc_b_http_port),
NetlocHandler)
netloc_b_http_process = multiprocessing.Process(
target=netloc_b_http.serve_forever, name='netloc-b-http')
netloc_b_http_process.start()
cls.server_process_list = [
server_process,
server_https_process,
netloc_a_http_process,
netloc_b_http_process,
]
@classmethod
def cleanUpCertificate(cls):
......@@ -662,8 +686,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
@classmethod
def stopServerProcess(cls):
for server in ['server_process', 'server_https_process']:
process = getattr(cls, server, None)
for process in cls.server_process_list:
if process is not None:
cls.logger.debug('Stopping process %s' % (process,))
process.join(10)
......@@ -1033,6 +1056,8 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
cls._server_http_port = findFreeTCPPort(cls._ipv4_address)
cls._server_https_port = findFreeTCPPort(cls._ipv4_address)
cls._server_https_auth_port = findFreeTCPPort(cls._ipv4_address)
cls._server_netloc_a_http_port = findFreeTCPPort(cls._ipv4_address)
cls._server_netloc_b_http_port = findFreeTCPPort(cls._ipv4_address)
cls.startServerProcess()
except BaseException:
cls.logger.exception("Error during setUpClass")
......@@ -1071,6 +1096,12 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
def _get_backend_haproxy_configuration(self):
backend_configuration_file = glob.glob(os.path.join(
self.instance_path, '*', 'etc', 'backend-haproxy.cfg'))[0]
with open(backend_configuration_file) as fh:
return fh.read()
@classmethod
def requestDefaultInstance(cls, state='started'):
default_instance = super(
......@@ -1326,6 +1357,13 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
# authenticating to http backend shall be no-op
'authenticate-to-backend': True,
},
'url-netloc-list': {
'url': cls.backend_url,
'url-netloc-list': '%(ip)s:%(port_a)s %(ip)s:%(port_b)s' % {
'ip': cls._ipv4_address,
'port_a': cls._server_netloc_a_http_port,
'port_b': cls._server_netloc_b_http_port},
},
'auth-to-backend': {
# in here use reserved port for the backend, which is going to be
# started later
......@@ -1353,6 +1391,14 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
'strict-transport-security-sub-domains': True,
'strict-transport-security-preload': True,
},
'https-url-netloc-list': {
'url': cls.backend_url + 'http',
'https-url': cls.backend_url + 'https',
'https-url-netloc-list': '%(ip)s:%(port_a)s %(ip)s:%(port_b)s' % {
'ip': cls._ipv4_address,
'port_a': cls._server_netloc_a_http_port,
'port_b': cls._server_netloc_b_http_port},
},
'server-alias': {
'url': cls.backend_url,
'server-alias': 'alias1.example.com alias2.example.com',
......@@ -1721,9 +1767,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'domain': 'example.com',
'accepted-slave-amount': '51',
'accepted-slave-amount': '54',
'rejected-slave-amount': '0',
'slave-amount': '51',
'slave-amount': '54',
'rejected-slave-dict': {
},
'warning-slave-dict': {
......@@ -1965,6 +2011,15 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertIn("backend _Url-http\n", content)
self.assertNotIn("backend _Url-https\n", content)
def test_url_netloc_list(self):
parameter_dict = self.assertSlaveBase('url-netloc-list')
result = fakeHTTPSResult(parameter_dict['domain'], 'path')
# assure that the request went to backend specified in the netloc
self.assertEqual(
result.headers['X-Backend-Identification'],
'netloc'
)
def test_auth_to_backend(self):
parameter_dict = self.assertSlaveBase('auth-to-backend')
......@@ -4464,6 +4519,19 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
timeout connect 10s
retries 5""" in content)
def test_https_url_netloc_list(self):
parameter_dict = self.assertSlaveBase('https-url-netloc-list')
result = fakeHTTPSResult(parameter_dict['domain'], 'path')
# assure that the request went to backend specified in the netloc
self.assertEqual(
result.headers['X-Backend-Identification'],
'netloc'
)
result = fakeHTTPResult(parameter_dict['domain'], 'path')
# assure that the request went to backend NOT specified in the netloc
self.assertNotIn('X-Backend-Identification', result.headers)
class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin):
instance_parameter_dict = {
......@@ -7168,6 +7236,23 @@ class TestSlaveHealthCheck(SlaveHttpFrontendTestCase, TestDataMixin):
'health-check-failover-https-url':
cls.backend_url + 'failover-https-url?a=b&c=',
},
'health-check-failover-url-netloc-list': {
'https-only': False, # http and https access to check
'health-check-timeout': 1, # fail fast for test
'health-check-interval': 1, # fail fast for test
'url': cls.backend_url + 'url',
'https-url': cls.backend_url + 'https-url',
'health-check': True,
'health-check-http-path': '/health-check-failover-url',
'health-check-failover-url': cls.backend_url + 'failover-url?a=b&c=',
'health-check-failover-https-url':
cls.backend_url + 'failover-https-url?a=b&c=',
'health-check-failover-url-netloc-list':
'%(ip)s:%(port_a)s %(ip)s:%(port_b)s' % {
'ip': cls._ipv4_address,
'port_a': cls._server_netloc_a_http_port,
'port_b': cls._server_netloc_b_http_port},
},
'health-check-failover-url-auth-to-backend': {
'https-only': False, # http and https access to check
'health-check-timeout': 1, # fail fast for test
......@@ -7257,12 +7342,6 @@ backend _health-check-default-http
timeout check 2s""" % (backend, )
}
def _get_backend_haproxy_configuration(self):
backend_configuration_file = glob.glob(os.path.join(
self.instance_path, '*', 'etc', 'backend-haproxy.cfg'))[0]
with open(backend_configuration_file) as fh:
return fh.read()
def _test(self, key):
parameter_dict = self.assertSlaveBase(key)
self.assertIn(
......@@ -7314,6 +7393,14 @@ backend _health-check-default-http
headers={'X-Reply-Status-Code': '502'})
self.assertEqual(result.status_code, httplib.CREATED)
def restoreBackend():
result = requests.put(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={})
self.assertEqual(result.status_code, httplib.CREATED)
self.addCleanup(restoreBackend)
time.sleep(3) # > health-check-timeout + health-check-interval
result = fakeHTTPSResult(parameter_dict['domain'], '/failoverpath')
......@@ -7348,6 +7435,38 @@ backend _health-check-default-http
r'"GET /failoverpath HTTP/1.1"'
)
def test_health_check_failover_url_netloc_list(self):
parameter_dict = self.assertSlaveBase(
'health-check-failover-url-netloc-list')
slave_parameter_dict = self.getSlaveParameterDictDict()[
'health-check-failover-url-netloc-list']
# check normal access
result = fakeHTTPSResult(parameter_dict['domain'], '/path')
self.assertNotIn('X-Backend-Identification', result.headers)
# start replying with bad status code
result = requests.put(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={'X-Reply-Status-Code': '502'})
self.assertEqual(result.status_code, httplib.CREATED)
self.assertEqual(result.status_code, httplib.CREATED)
def restoreBackend():
result = requests.put(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={})
self.assertEqual(result.status_code, httplib.CREATED)
self.addCleanup(restoreBackend)
time.sleep(3) # > health-check-timeout + health-check-interval
# check failover, uses netloc
result = fakeHTTPSResult(parameter_dict['domain'], '/path')
self.assertEqual(
result.headers['X-Backend-Identification'],
'netloc'
)
def test_health_check_failover_url_auth_to_backend(self):
parameter_dict = self.assertSlaveBase(
'health-check-failover-url-auth-to-backend')
......
......@@ -21,6 +21,9 @@ T-2/var/log/httpd/_auth-to-backend-not-configured_error_log
T-2/var/log/httpd/_auth-to-backend_access_log
T-2/var/log/httpd/_auth-to-backend_backend_log
T-2/var/log/httpd/_auth-to-backend_error_log
T-2/var/log/httpd/_bad-backend_access_log
T-2/var/log/httpd/_bad-backend_backend_log
T-2/var/log/httpd/_bad-backend_error_log
T-2/var/log/httpd/_ciphers_access_log
T-2/var/log/httpd/_ciphers_error_log
T-2/var/log/httpd/_custom_domain_access_log
......@@ -70,6 +73,9 @@ T-2/var/log/httpd/_enable_cache_server_alias_error_log
T-2/var/log/httpd/_https-only_access_log
T-2/var/log/httpd/_https-only_backend_log
T-2/var/log/httpd/_https-only_error_log
T-2/var/log/httpd/_https-url-netloc-list_access_log
T-2/var/log/httpd/_https-url-netloc-list_backend_log
T-2/var/log/httpd/_https-url-netloc-list_error_log
T-2/var/log/httpd/_monitor-ipv4-test_access_log
T-2/var/log/httpd/_monitor-ipv4-test_error_log
T-2/var/log/httpd/_monitor-ipv6-test_access_log
......@@ -153,6 +159,9 @@ T-2/var/log/httpd/_type-zope-virtualhostroot-https-port_error_log
T-2/var/log/httpd/_type-zope_access_log
T-2/var/log/httpd/_type-zope_backend_log
T-2/var/log/httpd/_type-zope_error_log
T-2/var/log/httpd/_url-netloc-list_access_log
T-2/var/log/httpd/_url-netloc-list_backend_log
T-2/var/log/httpd/_url-netloc-list_error_log
T-2/var/log/httpd/_url_https-url_access_log
T-2/var/log/httpd/_url_https-url_backend_log
T-2/var/log/httpd/_url_https-url_error_log
......
......@@ -21,6 +21,9 @@ T-2/var/log/httpd/_auth-to-backend-not-configured_error_log
T-2/var/log/httpd/_auth-to-backend_access_log
T-2/var/log/httpd/_auth-to-backend_backend_log
T-2/var/log/httpd/_auth-to-backend_error_log
T-2/var/log/httpd/_bad-backend_access_log
T-2/var/log/httpd/_bad-backend_backend_log
T-2/var/log/httpd/_bad-backend_error_log
T-2/var/log/httpd/_ciphers_access_log
T-2/var/log/httpd/_ciphers_error_log
T-2/var/log/httpd/_custom_domain_access_log
......@@ -70,6 +73,9 @@ T-2/var/log/httpd/_enable_cache_server_alias_error_log
T-2/var/log/httpd/_https-only_access_log
T-2/var/log/httpd/_https-only_backend_log
T-2/var/log/httpd/_https-only_error_log
T-2/var/log/httpd/_https-url-netloc-list_access_log
T-2/var/log/httpd/_https-url-netloc-list_backend_log
T-2/var/log/httpd/_https-url-netloc-list_error_log
T-2/var/log/httpd/_monitor-ipv4-test_access_log
T-2/var/log/httpd/_monitor-ipv4-test_error_log
T-2/var/log/httpd/_monitor-ipv6-test_access_log
......@@ -153,6 +159,9 @@ T-2/var/log/httpd/_type-zope-virtualhostroot-https-port_error_log
T-2/var/log/httpd/_type-zope_access_log
T-2/var/log/httpd/_type-zope_backend_log
T-2/var/log/httpd/_type-zope_error_log
T-2/var/log/httpd/_url-netloc-list_access_log
T-2/var/log/httpd/_url-netloc-list_backend_log
T-2/var/log/httpd/_url-netloc-list_error_log
T-2/var/log/httpd/_url_https-url_access_log
T-2/var/log/httpd/_url_https-url_backend_log
T-2/var/log/httpd/_url_https-url_error_log
......
......@@ -24,6 +24,9 @@ T-2/var/log/httpd/_health-check-disabled_error_log
T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_access_log
T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_backend_log
T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_error_log
T-2/var/log/httpd/_health-check-failover-url-netloc-list_access_log
T-2/var/log/httpd/_health-check-failover-url-netloc-list_backend_log
T-2/var/log/httpd/_health-check-failover-url-netloc-list_error_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_access_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_backend_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_error_log
......
......@@ -622,6 +622,11 @@
}
}
]
},
"random-activity-priority": {
"type": "string",
"title": "Random Activity Priority",
"description": "Control `random_activity_priority` argument of test runner. Can be set to an empty string to automatically generate a seed for each test."
}
},
"type": "object"
......
......@@ -1407,7 +1407,8 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin):
with open(slapos_wipe_device_disk) as fh:
self.assertEqual(
fh.read().strip(),
r"""dd if=/dev/zero of=/dev/virt0 bs=4096 count=500k
r"""#!/bin/sh
dd if=/dev/zero of=/dev/virt0 bs=4096 count=500k
dd if=/dev/zero of=/dev/virt1 bs=4096 count=500k"""
)
self.assertTrue(os.access(slapos_wipe_device_disk, os.X_OK))
......
......@@ -13,6 +13,10 @@ parts =
[python]
part = python3
[gcc]
# powerdns needs a compiler with C++17 features
min_version = 8
[eggs]
recipe = zc.recipe.egg
eggs =
......
......@@ -14,7 +14,7 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 84f099cc9852c4f53a075dccbb3880f0
md5sum = ce9c231ec47eb8f528345add21cb7822
[template-balancer]
filename = instance-balancer.cfg.in
......
......@@ -21,6 +21,7 @@
{% set test_runner_total_database_count = mariadb_test_database_amount %}
{% set test_runner_enabled = mariadb_test_database_amount > 0 %}
{% endif -%}
{% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%}
{% set monitor_base_url_dict = {} -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
......@@ -256,6 +257,7 @@ config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-z
config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
config-test-runner-random-activity-priority = {{ dumps(test_runner_random_activity_priority) }}
config-wcfs_enable = {{ dumps(wcfs_enable) }}
config-test-runner-configuration = {{ dumps(slapparameter_dict.get('test-runner', {})) }}
software-type = zope
......
......@@ -298,6 +298,7 @@ eggs = ${python-interpreter:eggs}
scripts =
slapos
supervisord
caucase
[git-clone-repository]
recipe = slapos.recipe.build:gitclone
......
......@@ -15,7 +15,7 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = f396d9a0780f4fb17016dbd32b56d7b8
md5sum = 8e4f43e603a5dd57752758c987465d41
[instance]
_update_hash_filename_ = instance.cfg.in
......@@ -47,7 +47,7 @@ md5sum = 9e8c17a4b2d802695caf0c2c052f0d11
[yarn.lock]
_update_hash_filename_ = yarn.lock
md5sum = 6faa52754c46e505912a478bc8ba3300
md5sum = 067d2db611b21f77885f3adfd7f81453
[python-language-server-requirements.txt]
_update_hash_filename_ = python-language-server-requirements.txt
......@@ -59,7 +59,7 @@ md5sum = 8157c22134200bd862a07c6521ebf799
[slapos.css.in]
_update_hash_filename_ = slapos.css.in
md5sum = 841141fc699b8d8918ed0669e6e61995
md5sum = d2930ec3ef973b7908f0fa896033fd64
[logo.png]
_update_hash_filename_ = logo.png
......
......@@ -71,7 +71,7 @@ recipe =
instance-promises =
$${theia-listen-promise:name}
$${frontend-listen-promise:name}
$${frontend-authentification-promise:name}
$${frontend-authentication-promise:name}
$${remote-frontend-url-available-promise:name}
{% if additional_frontend %}
$${remote-additional-frontend-url-available-promise:name}
......@@ -94,15 +94,15 @@ name = $${:_buildout_section_name_}.py
config-host = $${frontend-instance:ip}
config-port = $${frontend-instance:port}
[frontend-authentification-promise]
[frontend-authentication-promise]
<= monitor-promise-base
promise = check_url_available
name = $${:_buildout_section_name_}.py
username = $${frontend-instance-password:username}
password = $${frontend-instance-password:passwd}
ip = $${frontend-instance:ip}
port = $${frontend-instance:port}
config-url = https://$${:username}:$${:password}@[$${:ip}]:$${:port}
config-url = https://[$${:ip}]:$${:port}
config-username = $${frontend-instance-password:username}
config-password = $${frontend-instance-password:passwd}
[remote-frontend-url-available-promise]
<= monitor-promise-base
......@@ -184,6 +184,7 @@ sla-instance_guid = {{ parameter_dict['additional-frontend-guid'] }}
recipe = slapos.cookbook:generate.password
username = admin
bytes = 12
storage-path = $${buildout:parts-directory}/.$${:_buildout_section_name_}
[frontend-instance-port]
recipe = slapos.cookbook:free_port
......
/* backported fixes */
/* https://github.com/eclipse-theia/theia/commit/616c34e1c446a706f4cb02182b2d9195ef3ea854 */
.monaco-editor .monaco-list .monaco-list-row.focused,
.monaco-editor .monaco-list .monaco-list-row.focused,
.monaco-editor .monaco-list .monaco-list-row.focused .suggest-icon {
color: var(--theia-list-activeSelectionForeground) !important;
background-color: var(--theia-list-activeSelectionBackground) !important;
}
/* logo */
.theia-icon {
background-image: url('/{{ logo_image }}');
......
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -74,7 +74,7 @@ md5sum = bbef65b4edeb342f08309604ca3717d5
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = fcc8470824c448a56e2282c43b870cb5
md5sum = c10634353841bb09a847168b4add8d2f
[template-zeo]
filename = instance-zeo.cfg.in
......@@ -86,7 +86,7 @@ md5sum = bc821f9f9696953b10a03ad7b59a1936
[template-zope]
filename = instance-zope.cfg.in
md5sum = 769e81946c346530cebfce6ad7553165
md5sum = f3121380ab4d31ba5f4984aec74d0a2f
[template-balancer]
filename = instance-balancer.cfg.in
......
......@@ -21,6 +21,7 @@
{% set test_runner_total_database_count = mariadb_test_database_amount %}
{% set test_runner_enabled = mariadb_test_database_amount > 0 %}
{% endif -%}
{% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%}
{% set monitor_base_url_dict = {} -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
......@@ -259,6 +260,7 @@ config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-z
config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
config-test-runner-random-activity-priority = {{ dumps(test_runner_random_activity_priority) }}
config-wcfs_enable = {{ dumps(wcfs_enable) }}
config-test-runner-configuration = {{ dumps(slapparameter_dict.get('test-runner', {})) }}
software-type = zope
......
......@@ -13,6 +13,7 @@
{% set test_runner_address_list = [] -%}
{% set test_runner_enabled = slapparameter_dict['test-runner-enabled'] -%}
{% set test_runner_node_count = slapparameter_dict['test-runner-node-count'] -%}
{% set test_runner_random_activity_priority = slapparameter_dict['test-runner-random-activity-priority'] -%}
{% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
......@@ -503,6 +504,9 @@ command-line-extra =
--extra_sql_connection_string_list '{{ ','.join(connection_string_list[1:]) }}'
--zserver {{ test_runner_address_list[0][0] ~ ':' ~ test_runner_address_list[0][1] }}
--zserver_frontend_url {{ slapparameter_dict['test-runner-apache-url-list'][0] }}
{% if test_runner_random_activity_priority is not none %}
--random_activity_priority={{ test_runner_random_activity_priority }}
{%- endif %}
[{{ section('runTestSuite') }}]
< = run-test-common
......
......@@ -196,8 +196,8 @@ slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.20
slapos.rebootstrap = 4.5
slapos.recipe.build = 0.49
slapos.recipe.cmmi = 0.17
slapos.recipe.build = 0.50
slapos.recipe.cmmi = 0.18
slapos.recipe.template = 4.6
slapos.toolbox = 0.126
stevedore = 1.21.0:whl
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment