Commit ba4ea4df authored by Łukasz Nowak's avatar Łukasz Nowak

Update Release Candidate

parents 5c5b2c62 4ef735c6
......@@ -14,4 +14,4 @@ eggs = ${pytest:eggs}
[versions]
pytest = 4.6.11
pytest = 4.6.11:whl
......@@ -14,6 +14,10 @@ extends =
parts =
tesseract
[gcc]
# tesseract uses some C++ 17 features which needs gcc >= 7
min_version = 7
[tesseract]
recipe = slapos.recipe.cmmi
shared = true
......
......@@ -106,7 +106,7 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
[profile-kedifa]
filename = instance-kedifa.cfg.in
md5sum = 16901e9eeb0d4f87e708ad91e7756f12
md5sum = eab5ae579471ca86b40bd2da3b53fefa
[template-backend-haproxy-rsyslogd-conf]
_update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
......
......@@ -253,7 +253,7 @@ extra-context =
module = check_url_available
name = kedifa-http-reply.py
# Kedifa replies 400 on /, so use it to be sure that Kedifa replied
config-http_code = 400
config-http-code = 400
config-url = https://[${kedifa-config:ip}]:${kedifa-config:port}
config-ca-cert-file = ${kedifa-config:ca-certificate}
......
......@@ -531,12 +531,12 @@ class TestHandler(BaseHTTPRequestHandler):
timeout = int(config.pop('Timeout', '0'))
compress = int(config.pop('Compress', '0'))
drop_header_list = []
for header in config.pop('X-Drop-Header', '').split():
for header in (config.pop('X-Drop-Header') or '').split():
drop_header_list.append(header)
header_dict = config
else:
drop_header_list = []
for header in self.headers.dict.get('x-drop-header', '').split():
for header in (self.headers.dict.get('x-drop-header') or '').split():
drop_header_list.append(header)
response = None
status_code = 200
......
......@@ -18,4 +18,4 @@ md5sum = e986de01a57161b32425f1cd3ccac924
[template-cloudooo-instance]
filename = instance-cloudooo.cfg.in
md5sum = 440f2b82b119cbfa6f8c7d27652c3170
md5sum = 6e4bdb1df02aed5c96ccf7b9c3c71b89
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"extends": "./schema-definitions.json#",
"properties": {
"tcpv4-port": {
......
......@@ -110,7 +110,7 @@ name = apache.py
config-url = https://{{ ipv4 }}:{{ apache_dict.values()[0][0] }}
# XXX cloudooo replies "400 Bad Request" for GET on / but what we want to check
# is that we don't have a "503 Service Unavailable" from apache or haproxy.
config-http_code = 400
config-http-code = 400
[apache-conf-ssl]
cert = ${directory:apache-conf}/apache.crt
......
{
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "Parameters to instantiate ERP5",
"type": "object",
"additionalProperties": false,
"definitions": {
"routing-rule-list": {
......@@ -564,7 +565,8 @@
"target": {
"description": "Target system",
"type": "string",
"const": "selenium-server"
"const": "selenium-server",
"default": "selenium-server"
},
"server-url": {
"description": "URL of the selenium server",
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"required": [
"tcpv4-port"
],
"properties": {
"tcpv4-port": {
"allOf": [
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"required": [
"tcpv4-port"
],
"type": "object",
"properties": {
"tcpv4-port": {
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"description": "Parameters to instantiate Fluentd",
"additionalProperties": false,
"properties": {
......
......@@ -54,7 +54,7 @@ md5sum = 0f1ec4077dab586cc003ae13f689eda2
[instance-gitlab.cfg.in]
_update_hash_filename_ = instance-gitlab.cfg.in
md5sum = dfeff229aa696bb6b6051a2aff4301fe
md5sum = 64ec65b2daa0648453022f3afcbc4da3
[instance-gitlab-export.cfg.in]
_update_hash_filename_ = instance-gitlab-export.cfg.in
......
......@@ -444,7 +444,7 @@ tune-command =
<= monitor-promise-base
module = check_command_execute
name = ${:_buildout_section_name_}.py
config-http_code = 200
config-http-code = 200
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Parameters to instantiate Grafana",
"type": "object",
"additionalProperties": false,
"properties": {
"smtp-server": {
......
......@@ -21,7 +21,7 @@ md5sum = 9e486efe4ab1aba8cb72b04f6c6da8ad
[instance_html5as]
_update_hash_filename_ = instance_html5as.cfg.in
md5sum = 191ec2a8b967a3944971709365bdcd2d
md5sum = 283440057c659bde2ae7fcc2c4c5b781
[template_nginx_conf]
_update_hash_filename_ = templates/nginx_conf.in
......
......@@ -239,4 +239,3 @@ module = check_url_available
name = html5as-http-frontend.py
url = ${html5as-frontend:connection-secure_access}
config-url = ${:url}
config-check-secure = 1
......@@ -19,7 +19,7 @@ md5sum = 6c17361a49cfc47564063b867aab6e8c
[template-jscrawler]
filename = instance-jscrawler.cfg.jinja2.in
md5sum = fece076231740b414612da5ef3a1685a
md5sum = f61e0507717447e47c76a2b2712f17f4
[template-jscrawler-builder]
filename = template-jscrawler.builder.sh.in
......
......@@ -50,7 +50,6 @@ return = secure_access domain
module = check_url_available
name = jscrawler_frontend.py
config-url = ${request-jscrawler-frontend:connection-secure_access}
config-check-secure = 1
[logrotate-entry-httpd]
<= logrotate-entry-base
......
{
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "Parameters to instantiate JSTestNode",
"type": "object",
"additionalProperties": false,
"required": [
"test-suite",
......@@ -63,7 +64,8 @@
"target": {
"description": "Target system",
"type": "string",
"const": "selenium-server"
"const": "selenium-server",
"default": "selenium-server"
},
"server-url": {
"description": "URL of the selenium server",
......@@ -127,7 +129,8 @@
"target": {
"description": "Target system",
"const": "node",
"type": "string"
"type": "string",
"default": "node"
}
}
}
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"description": "Parameters to instantiate Jupyter",
"properties": {
"frontend-instance-guid": {
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"dummy": {
"title": "dummy",
......
{
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "Parameters to instantiate a NEO cluster. See https://lab.nexedi.com/nexedi/neoppod/blob/master/neo.conf for more information.",
"type": "object",
"definitions": {
"neo-cluster": {
"additionalProperties": false,
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Parameters to instantiate Grafana",
"description": "Parameters to instantiate NextCloud",
"type": "object",
"additionalProperties": false,
"properties": {
"domain": {
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Parameters to instantiate ProFTPd",
"type": "object",
"additionalProperties": false,
"properties": {
"port": {
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Parameters to instantiate Pure-FTPd",
"type": "object",
"additionalProperties": false,
"properties": {
"port": {
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"ipv6-prefix": {
"title": "Ipv6 prefix to use to setup the new re6st network",
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {}
}
......@@ -18,7 +18,7 @@ md5sum = 8a08be95a04f1a47098c4fdef80bdfed
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 0c173313b48d0005c46d968db1c8ab5f
md5sum = 839642d7a56447b3f08fa69729faca61
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
......
......@@ -511,14 +511,12 @@ return = domain secure_access
module = check_url_available
name = check_repman_frontend.py
config-url = https://${repman-frontend:connection-domain}
config-check-secure = 1
[repman-backend-promise]
<= monitor-promise-base
module = check_url_available
name = check_repman_backend.py
config-url = ${nginx-parameter:backend-ssl-url}
config-check-secure = 1
[template-proxysql-need-stop-start]
recipe = slapos.recipe.template:jinja2
......
......@@ -26,6 +26,7 @@
##############################################################################
import os
import time
from six.moves.urllib.parse import urljoin
import requests
......@@ -68,16 +69,21 @@ class TestRepman(SlapOSInstanceTestCase):
)
self.assertEqual(resp.status_code, requests.codes.ok)
resp = requests.get(
urljoin(self.url, '/api/clusters'),
params={
'query': '{"method":"GET","isArray":false}',
},
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
cluster, = resp.json()
for i in range(20):
resp = requests.get(
urljoin(self.url, '/api/clusters'),
params={
'query': '{"method":"GET","isArray":false}',
},
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
cluster, = resp.json()
if cluster['isProvision'] and cluster['isFailable'] and not cluster['isDown']:
break
time.sleep(i)
self.assertTrue(cluster['isProvision'])
self.assertTrue(cluster['isFailable'])
self.assertFalse(cluster['isDown'])
......@@ -15,4 +15,4 @@
[instance.cfg.in]
filename = instance.cfg.in
md5sum = fee2097f3d12fd4bcfbc2698ecf49afc
md5sum = 0bd2594778cc5f29abfee7cd015c1e7d
......@@ -152,7 +152,7 @@ stop-on-error = true
recipe = slapos.cookbook:wrapper
command-line =
{{ gowork_bin }}/rest-server \
--listen [${instance-parameter:ipv6-random}]:${:port}
--listen [${:ip}]:${:port}
--log ${directory:var-log}/${:_buildout_section_name_}-access.log
--path ${directory:rest-server-data-dir}
--tls
......
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum = 5d1320a9b1d2828f2870e3096f1c2bfb
md5sum = 733643122fb75dfd7374b973a95fa2ea
......@@ -4,7 +4,7 @@ parts +=
publish
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
......@@ -40,15 +40,34 @@ template = inline:
recipe = slapos.recipe.template:jinja2
rendered = ${:workdir}/.nxdtest
workdir = ${directory:nxdtest-working-dir}
template = inline:
{% for test in tests.splitlines() %}
TestCase(
{{ repr(test.split()[0]) }},
[ {{ repr(interpreter) }}, '-m', 'unittest', 'discover', '-v'],
cwd={{ repr(test.split()[1]) }},
summaryf=UnitTest.summary,
)
{% endfor %}
tests = {{ tests | indent(2) }}
template =
inline:{% raw %}
{%- set only_sr = slapparameter_dict.get('only-sr') %}
{%- if not isinstance(only_sr, list) %}
{%- set only_sr = [only_sr] %}
{%- endif %}
{%- set unittest_args = slapparameter_dict.get('unittest-args', ['discover', '-v']) %}
{%- if not isinstance(unittest_args, list) %}
{%- set unittest_args = [unittest_args] %}
{%- endif %}
command = {{ repr([interpreter, '-m', 'unittest'] + unittest_args) }}
{%- for test in tests.splitlines() %}
{%- set name, folder = test.split() %}
{%- if not only_sr or name in only_sr %}
TestCase(
{{ repr(name) }},
command,
cwd={{ repr(folder) }},
summaryf=UnitTest.summary,
)
{%- endif %}
{%- endfor %}
{%- endraw %}
context =
key slapparameter_dict slap-configuration:configuration
key tests :tests
raw interpreter {{ interpreter }}
[runTestSuite]
env.sh = ${slapos-test-runner-nxdtest-environment.sh:rendered}
......
......@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
[template-runner]
filename = instance-runner.cfg
md5sum = ebdecea5c1653ed752e06fbc8be7e6b2
md5sum = 2a09b11c7dbade65d50e66287bf4c7b9
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"instance-name": {
"title": "Instance Name",
......
......@@ -91,7 +91,7 @@ module = check_url_available
name = custom_frontend_promise.py
config-url = https://$${request-custom-frontend:connection-domain}
{% if slapparameter_dict.get('custom-frontend-basic-auth') -%}
config-check-secure = 1
config-http-code = 401
{% endif -%}
[custom-frontend-url-ready-promise-bin]
......@@ -440,7 +440,6 @@ module = check_url_available
name = $${:filename}.py
filename = apache-httpd-listening-on-tcp
config-url = $${apache-httpd:access-url}
config-check-secure = 1
[slaprunner-httpd-cors]
recipe = plone.recipe.command
......@@ -541,7 +540,6 @@ return = site_url domain
module = check_url_available
name = slaprunner_frontend.py
config-url = https://$${request-frontend:connection-domain}/login
config-check-secure = 1
[request-httpd-frontend]
<= slap-connection
......@@ -561,7 +559,6 @@ return = secure_access domain
module = check_url_available
name = slaprunner-apache-http-frontend.py
config-url = $${request-httpd-frontend:connection-secure_access}
config-check-secure = 1
{% endif %}
......
......@@ -15,7 +15,7 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = 557dec61b7bf58601051575234e7fd05
md5sum = 11d347dd2bf762902341746a388673a0
[instance]
_update_hash_filename_ = instance.cfg.in
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"description": "Parameters to instantiate Theia",
"additionalProperties": false,
"properties": {
......
......@@ -94,7 +94,7 @@ config-port = $${frontend-instance:port}
module = check_url_available
name = $${:_buildout_section_name_}.py
config-url = $${remote-frontend:connection-secure_access}
config-check-secure = 1
config-http-code = 401
{% if additional_frontend %}
[remote-additional-frontend-url-available-promise]
......@@ -102,7 +102,7 @@ config-check-secure = 1
module = check_url_available
name = $${:_buildout_section_name_}.py
config-url = $${remote-additional-frontend:connection-secure_access}
config-check-secure = 1
config-http-code = 401
{% endif %}
[slapos-standalone-listen-promise]
......
......@@ -281,6 +281,8 @@ class TestTheiaEnv(TheiaTestCase):
}
def test_theia_env(self):
"""Make sure environment variables are the same wether we use shell or supervisor services.
"""
# The path of the env.json file expected to be generated by building the dummy software release
env_json_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'software', 'env.json')
......@@ -315,18 +317,21 @@ class TestTheiaEnv(TheiaTestCase):
# Remove the env.json file to later be sure that a new one has been generated
os.remove(env_json_path)
# Launch slapos-node-software from the embedded supervisord
# Launch slapos node software service from the embedded supervisord.
# Note that we have two services, slapos-not-software and slapos-not-software-all
# The later uses --all which is what we want to use here, because the software
# is already installed and we want to install it again, this time from supervisor
embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run')
embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software')['stop']
embedded_supervisor.startProcess('slapos-node-software')
previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop']
embedded_supervisor.startProcess('slapos-node-software-all')
for _retries in range(20):
time.sleep(1)
if embedded_supervisor.getProcessInfo('slapos-node-software')['stop'] != previous_stop_time:
if embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop'] != previous_stop_time:
break
else:
self.fail("the supervisord service 'slapos-node-software' takes too long to finish")
self.fail("the supervisord service 'slapos-node-software-all' takes too long to finish")
# Get the supervisord environment
with open(env_json_path) as f:
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"extends": "./schema-definitions.json#",
"properties": {
"tcpv4-port": {
......
......@@ -42,7 +42,7 @@ md5sum = d32417746fcf671d4e86a70379815039
[template-my-cnf]
filename = my.cnf.in
md5sum = 7944ec58a2c6ee74a56219bacebfd145
md5sum = 1de449e8c0c4a85c5ce2b447785b7654
[template-mariadb-initial-setup]
filename = mariadb_initial_setup.sql.in
......
......@@ -56,6 +56,16 @@ log_bin = {{ log_bin }}
{% set binlog_expire_days = parameter_dict['binlog-expire-days'] -%}
{% if binlog_expire_days > 0 %}expire_logs_days = {{ binlog_expire_days }}{% endif %}
server_id = {{ parameter_dict['server-id'] }}
# Note: increases writes on replica (it writes to relay log and to database as
# usual, but then also to binlog), but with the advantage of allowing such node
# to switch role later, or to allow another replica to catch up with it if the
# primary is not available anymore. This also increases the disk usage on the
# replica, as the binlogs are stored as usual.
log_slave_updates = 1
# Note: as of at least mariadb 10.3.27, the relay log name is not based on the
# hostname, but still there is a line in mariadb_error.log on every start
# warning about this missing option. So provide it.
relay-log = mariadb-relay-bin
{% endif %}
# Some dangerous settings you may want to uncomment temporarily
......
......@@ -14,7 +14,7 @@
# not need these here).
[instance]
filename = instance.cfg.in
md5sum = 5c953c0f5d3376718eb9c4030288647a
md5sum = e4e070f93adaf917f9427ae9f35573d9
[instance-apache-php]
filename = instance-apache-php.cfg.in
......@@ -22,7 +22,7 @@ md5sum = 4afee4377fa9cbc1e4ff80647b2f279c
[instance-lamp]
filename = instance-lamp.cfg.jinja2.in
md5sum = 52b76d3c8aa23f467db33e32a2b50ed6
md5sum = 79f562260895df2665a85df5cb442193
[template-apache.conf]
filename = apache.conf.in
......
......@@ -82,7 +82,6 @@ name = lamp-http-frontend.py
url = ${request-frontend:connection-secure_access}
config-url = ${:url}
config-custom-domain = {{ slapparameter_dict.get('custom-domain', '') }}
config-check-secure = 1
{% do publish_dict.__setitem__('url', '${lamp-frontend-promise:url}') -%}
......
......@@ -8,7 +8,6 @@ develop-eggs-directory = {{ buildout_develop_directory }}
offline = true
[switch_softwaretype]
#recipe = slapos.cookbook:softwaretype
recipe = slapos.cookbook:switch-softwaretype
default = dynamic-template-lamp:rendered
RootSoftwareInstance = ${:default}
......
......@@ -14,7 +14,7 @@
# not need these here).
[monitor2-template]
filename = instance-monitor.cfg.jinja2.in
md5sum = 200bb126dbfc33e5c5c165ae17bab64b
md5sum = d4185c191e8b9df20e1f98cd8c556b1d
[monitor-httpd-conf]
_update_hash_filename_ = templates/monitor-httpd.conf.in
......
......@@ -127,9 +127,9 @@ recipe = slapos.recipe.template:jinja2
template = {{ monitor_conf_template }}
rendered = ${directory:etc}/${:filename}
filename = monitor.conf
context = section parameter_dict monitor-conf-parameters
context = section parameter_dict monitor-conf-parameters
section promise_parameter_dict monitor-promise-conf
section monitor_base_urls monitor-base-url-dict
section monitor_base_urls monitor-base-url-dict
[start-monitor]
recipe = slapos.cookbook:wrapper
......@@ -312,7 +312,7 @@ output = ${directory:plugins}/${:name}
module = check_url_available
name = monitor-httpd-listening-on-tcp.py
config-url = ${monitor-httpd-conf-parameter:url}
config-check-secure = 1
config-http-code = 401
[monitor-publish-parameters]
# XXX depends on monitor-base section
......@@ -326,7 +326,7 @@ monitor-title = ${slap-configuration:instance-title}
monitor-httpd-ipv6 = ${slap-configuration:ipv6-random}
monitor-httpd-port = 8196
# XXX - Set monitor-base-url = ${monitor-httpd-conf-parameter:url} => https://[ipv6]:port
monitor-base-url = ${monitor-frontend-promise:url}
monitor-base-url = ${monitor-frontend:connection-secure_access}
#monitor-base-url = ${monitor-httpd-conf-parameter:url}
root-instance-title = ${slap-configuration:root-instance-title}
monitor-url-list =
......@@ -356,13 +356,26 @@ config-https-only = true
#software-type = custom-personal
return = domain secure_access
# Requests to the frontend URL should succeed with the correct
# credentials.
[check-monitor-password-promise]
<= monitor-promise-base
module = check_url_available
name = check-monitor-frontend-password.py
url = ${monitor-frontend:connection-secure_access}
config-url = ${:url}
config-username = ${monitor-instance-parameter:username}
config-password = ${monitor-instance-parameter:password}
# Requests to the frontend URL should fail when no credentials are
# supplied.
[monitor-frontend-promise]
<= monitor-promise-base
module = check_url_available
name = monitor-http-frontend.py
url = ${monitor-frontend:connection-secure_access}
config-url = ${:url}
config-check-secure = 1
config-http-code = 401
[monitor-bootstrap-promise]
<= monitor-promise-base
......@@ -402,6 +415,10 @@ depends =
${start-monitor:wrapper-path}
${ca-monitor-httpd-service:wrapper-path}
${monitor-httpd-promise:name}
${monitor-frontend-promise:name}
# XXX this is not enabled yet because this conflicts with testnode's own
# monitoring.
# ${check-monitor-password-promise:name}
${monitor-bootstrap-promise:name}
${monitor-symlink:recipe}
${promise-check-slapgrid:recipe}
......
......@@ -155,7 +155,7 @@ cliff = 2.8.3:whl
cmd2 = 0.7.0
collective.recipe.shelloutput = 0.1
collective.recipe.template = 2.0
configparser = 4.0.2
configparser = 4.0.2:whl
contextlib2 = 0.6.0.post1
cryptography = 3.3.1
dateparser = 0.7.6
......@@ -165,7 +165,7 @@ gevent = 20.9.0
geventmp = 0.0.1
greenlet = 0.4.17
idna = 2.9
importlib-metadata = 1.7.0
importlib-metadata = 1.7.0:whl
inotify-simple = 1.1.1
itsdangerous = 0.24
lock-file = 2.0
......@@ -178,8 +178,8 @@ pbr = 2.0.0
plone.recipe.command = 1.1
prettytable = 0.7.2
psutil = 5.8.0
pluggy = 0.13.1
py = 1.9.0
pluggy = 0.13.1:whl
py = 1.9.0:whl
pyOpenSSL = 19.1.0
pyparsing = 2.2.0
pytz = 2016.10
......@@ -190,7 +190,7 @@ setuptools-dso = 1.7
rubygemsrecipe = 0.3.0