Commit 64f10409 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents ad5fe700 7ae4d31a
......@@ -14,7 +14,7 @@ recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/cloudooo.git
branch = master
git-executable = ${git:location}/bin/git
revision = be6c35c0156e028f31da3ccb205afc6f95728d97
revision = 0b5ff71a2ede76499e81659aed392057ae910917
[cloudooo]
recipe = zc.recipe.egg
......
......@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
[template-trafficserver-records-config]
_update_hash_filename_ = templates/trafficserver/records.config.jinja2
md5sum = b99403e02d1aff471a7d5ebd0afbdb6c
md5sum = 1696321871bb71083550c95d77487602
[template-trafficserver-storage-config]
_update_hash_filename_ = templates/trafficserver/storage.config.jinja2
......
......@@ -80,7 +80,7 @@ CONFIG proxy.config.http.uncacheable_requests_bypass_parent INT 1
CONFIG proxy.config.http.keep_alive_no_activity_timeout_in INT 120
CONFIG proxy.config.http.keep_alive_no_activity_timeout_out INT 120
CONFIG proxy.config.http.transaction_no_activity_timeout_in INT {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.transaction_no_activity_timeout_out INT 30 {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.transaction_no_activity_timeout_out INT {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.transaction_active_timeout_in INT 900
CONFIG proxy.config.http.transaction_active_timeout_out INT 0
CONFIG proxy.config.http.accept_no_activity_timeout INT 120
......
......@@ -18,4 +18,4 @@ md5sum = e986de01a57161b32425f1cd3ccac924
[template-cloudooo-instance]
filename = instance-cloudooo.cfg.in
md5sum = 9e1a66cf18d7c30c14afeb66c20afb46
md5sum = 440f2b82b119cbfa6f8c7d27652c3170
......@@ -162,6 +162,7 @@ environment =
LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['gcc'] }}/lib:{{ parameter_dict['gcc'] }}/lib64:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
FONTCONFIG_FILE = ${fontconfig-conf:rendered}
PATH = ${binary-link:target-directory}
LANG = C.UTF-8
mimetype_entry_addition =
{% for entry in mimetype_entry_addition.splitlines() -%}
{{ " " ~ entry.strip() }}
......@@ -172,7 +173,7 @@ ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program
ooo-paster = {{ bin_directory }}/cloudooo_paster
ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
{% for index in range(backend_count) -%}
{% for index in range(1, backend_count + 1) -%}
{% set name = 'cloudooo-' ~ index -%}
[{{ cloudooo(name) }}]
< = cloudooo-base
......@@ -207,8 +208,8 @@ prepend-path = ${buildout:bin-directory}
run-unit-test = ${buildout:bin-directory}/runUnitTest
run-test-suite = ${buildout:bin-directory}/runTestSuite
ooo-paster = ${cloudooo-0:ooo-paster}
configuration-file = ${cloudooo-0:configuration-file}
ooo-paster = ${cloudooo-1:ooo-paster}
configuration-file = ${cloudooo-1:configuration-file}
run-unit-test-binary = {{ bin_directory }}/runCloudoooUnitTest
run-test-suite-binary = {{ bin_directory }}/runCloudoooTestSuite
......
......@@ -44,6 +44,7 @@ setup(name=name,
'slapos.core',
'slapos.cookbook',
'slapos.libnetworkcache',
'requests',
'six',
'PyPDF2',
],
......
......@@ -26,23 +26,43 @@
#
##############################################################################
import codecs
import csv
import multiprocessing
import os
import json
import six.moves.xmlrpc_client as xmlrpclib
import six.moves.urllib.parse as urllib_parse
import ssl
import base64
import io
import requests
import PyPDF2
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, CloudOooTestCase = makeModuleSetUpAndTestCaseClass(
setUpModule, _CloudOooTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
# Cloudooo needs a lot of time before being available.
CloudOooTestCase.instance_max_retry = 30
class CloudOooTestCase(_CloudOooTestCase):
# Cloudooo needs a lot of time before being available.
instance_max_retry = 30
def setUp(self):
self.url = json.loads(
self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
# XXX ignore certificate errors
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
self.server = xmlrpclib.ServerProxy(
self.url,
context=ssl_context,
allow_none=True,
)
def normalizeFontName(font_name):
......@@ -88,19 +108,6 @@ class HTMLtoPDFConversionFontTestMixin:
"""Convert the HTML source to pdf bytes.
"""
def setUp(self):
self.url = json.loads(
self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
# XXX ignore certificate errors
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
self.server = xmlrpclib.ServerProxy(
self.url,
context=ssl_context,
allow_none=True,
)
def test(self):
actual_font_mapping_mapping = {}
for font in self.expected_font_mapping:
......@@ -237,3 +244,84 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
'html',
'pdf',
).encode())
class TestLibreOfficeTextConversion(CloudOooTestCase):
__partition_reference__ = 'txt'
def test_html_to_text(self):
self.assertEqual(
base64.decodestring(
self.server.convertFile(
base64.encodestring(
u'<html>héhé</html>'.encode('utf-8')).decode(),
'html',
'txt',
).encode()),
codecs.BOM_UTF8 + b'h\xc3\xa9h\xc3\xa9\n',
)
class TestLibreOfficeCluster(CloudOooTestCase):
__partition_reference__ = 'lc'
@classmethod
def getInstanceParameterDict(cls):
return {'backend-count': 4}
def test_multiple_conversions(self):
# make this function global so that it can be picked and used by multiprocessing
global _convert_html_to_text
def _convert_html_to_text(src_html):
return base64.decodestring(
self.server.convertFile(
base64.encodestring(src_html.encode()).decode(),
'html',
'txt',
).encode())
pool = multiprocessing.Pool(5)
# TODO py3: use with pool
converted = pool.map(_convert_html_to_text,
['<html><body>hello</body></html>'] * 100)
pool.terminate()
pool.join()
self.assertEqual(converted, [codecs.BOM_UTF8 + b'hello\n'] * 100)
# haproxy stats are exposed
res = requests.get(
urllib_parse.urljoin(self.url, '/haproxy;csv'),
verify=False,
stream=True,
)
reader = csv.DictReader(res.raw)
line_list = list(reader)
# requests have been balanced
total_hrsp_2xx = {
line['svname']: int(line['hrsp_2xx'])
for line in line_list
}
self.assertEqual(total_hrsp_2xx['FRONTEND'], 100)
self.assertEqual(total_hrsp_2xx['BACKEND'], 100)
for backend in 'cloudooo_1', 'cloudooo_2', 'cloudooo_3', 'cloudooo_4':
# ideally there should be 25% of requests on each backend, because we use
# round robin scheduling, but it can happen that some backend take longer
# to start, so we are tolerant here and just check that each backend
# process at least 15% of requests.
self.assertGreater(total_hrsp_2xx[backend], 15)
# no errors
total_eresp = {
line['svname']: int(line['eresp'] or 0)
for line in line_list
}
self.assertEqual(
total_eresp, {
'FRONTEND': 0,
'cloudooo_1': 0,
'cloudooo_2': 0,
'cloudooo_3': 0,
'cloudooo_4': 0,
'BACKEND': 0,
})
......@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = 854ec0f379a05246f5f3761fdb634c36
md5sum = bf0c01ac7493693bb57ebef00bb20fa0
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
......@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = a8afeb2f80199368e291621576a80ca1
md5sum = b8cc7c76438212e0522ebede88649393
[template-kvm-controller]
filename = template/kvm-controller-run.in
......
......@@ -359,7 +359,7 @@
},
"virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
"description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
"type": "string"
},
"virtual-hard-drive-gzipped": {
......
......@@ -160,7 +160,7 @@
},
"virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
"description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
"type": "string"
},
"virtual-hard-drive-gzipped": {
......
......@@ -17,6 +17,8 @@
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
{% set virtual_hard_drive_url_enabled = 'virtual-hard-drive-url' in slapparameter_dict %}
{% set virtual_hard_drive_url_gzipped = str(slapparameter_dict.get('virtual-hard-drive-gzipped', False)).lower() == 'true' %}
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%}
......@@ -56,6 +58,11 @@ public = ${:srv}/public/
cron-entries = ${:etc}/cron.d
crontabs = ${:etc}/crontabs
cronstamps = ${:etc}/cronstamps
{%- if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-repository = ${:srv}/virtual-hard-drive-url-repository
virtual-hard-drive-url-var = ${:var}/virtual-hard-drive-url
virtual-hard-drive-url-expose = ${monitor-directory:private}/virtual-hard-drive-url
{%- endif %}
{%- if boot_image_url_list_enabled %}
boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
boot-image-url-list-var = ${:var}/boot-image-url-list
......@@ -278,6 +285,106 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
## boot-image-url-list support END
{% endif %} {# if boot_image_url_list_enabled #}
{% if virtual_hard_drive_url_enabled %}
## virtual-hard-drive-url support BEGIN
[empty-file-state-base-virtual-promise]
<= monitor-promise-base
module = check_file_state
name = ${:_buildout_section_name_}.py
config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/virtual-hard-drive-url/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[virtual-hard-drive-url-source-config]
recipe = slapos.recipe.template:jinja2
template = inline:
{%- raw %}
{{ virtual_hard_drive_url }}
{% endraw -%}
{# Enforce md5sum on virtual-hard-drive-url #}
virtual-hard-drive-url = {{ slapparameter_dict['virtual-hard-drive-url'] }}#{{ slapparameter_dict['virtual-hard-drive-md5sum'] }}
context =
key virtual_hard_drive_url :virtual-hard-drive-url
rendered = ${directory:etc}/virtual-hard-drive-url.conf
[virtual-hard-drive-url-processed-config]
# compares if the current configuration has been used by
# the virtual-hard-drive-url-download, if not, exposes it as not empty file with
# information
recipe = slapos.recipe.build
install =
import os
import hashlib
if not os.path.exists(location):
os.mkdir(location)
with open('${:state-file}', 'w') as state_handler:
try:
with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
processed_md5sum = processed_handler.read()
if config_md5sum == processed_md5sum:
state_handler.write('')
else:
state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
except Exception as e:
state_handler.write(str(e))
update = ${:install}
config-file = ${virtual-hard-drive-url-source-config:rendered}
state-filename = virtual-hard-drive-url-processed-config.state
state-file = ${directory:virtual-hard-drive-url-expose}/${:state-filename}
processed-md5sum = ${directory:virtual-hard-drive-url-var}/update-image-processed.md5sum
[virtual-hard-drive-url-processed-config-promise]
# promise to check if the configuration provided by the user has been already
# processed by the virtual-hard-drive-url-download script, which runs asynchronously
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-processed-config:state-filename}
config-filename = ${virtual-hard-drive-url-processed-config:state-file}
[virtual-hard-drive-url-json-config]
# generates json configuration from user configuration
recipe = plone.recipe.command
command = {{ python_executable }} {{ image_download_config_creator }} ${virtual-hard-drive-url-source-config:rendered} ${:rendered} ${directory:virtual-hard-drive-url-repository} ${:error-state-file}
update-command = ${:command}
rendered = ${directory:virtual-hard-drive-url-var}/virtual-hard-drive-url.json
error-state-filename = virtual-hard-drive-url-json-config-error.txt
error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
[virtual-hard-drive-url-config-state-promise]
# promise to check if configuration has been parsed without errors
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-json-config:error-state-filename}
config-filename = ${virtual-hard-drive-url-json-config:error-state-file}
[virtual-hard-drive-url-download-wrapper]
# wrapper to execute virtual-hard-drive-url-download on each run
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/virtual-hard-drive-url-updater
command-line = {{ python_executable }} {{ image_download_controller }} ${virtual-hard-drive-url-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${virtual-hard-drive-url-processed-config:processed-md5sum}
md5sum-state-filename = virtual-hard-drive-url-download-controller-md5sum-fail.json
md5sum-state-file = ${directory:virtual-hard-drive-url-expose}/${:md5sum-state-filename}
error-state-filename = virtual-hard-drive-url-download-controller-error.text
error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[virtual-hard-drive-url-download-md5sum-promise]
# promise to report errors with problems with calculating md5sum of the
# downloaded images
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-filename}
config-filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-file}
[virtual-hard-drive-url-download-state-promise]
# promise to report errors during download
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-download-wrapper:error-state-filename}
config-filename = ${virtual-hard-drive-url-download-wrapper:error-state-file}
## virtual-hard-drive-url support END
{% endif %} {# if virtual_hard_drive_url_enabled #}
[kvm-controller-parameter-dict]
python-path = {{ python_eggs_executable }}
vnc-passwd = ${gen-passwd:passwd}
......@@ -298,6 +405,11 @@ vnc-ip = ${:ipv4}
vnc-port = 5901
default-cdrom-iso = {{ debian_amd64_netinst_location }}
{% if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-json-config = ${virtual-hard-drive-url-json-config:rendered}
{% else %}
virtual-hard-drive-url-json-config =
{% endif %}
{% if boot_image_url_list_enabled %}
boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
{% else %}
......@@ -425,14 +537,42 @@ ipv6-port = {{ external_port }}
{% endfor -%}
{% endif -%}
{%- set depend_section_list = [] %}
{%- set hash_file_list = ['${kvm-run:rendered}'] %}
{%- macro generate_depend_section(section, key) %}
{%- do depend_section_list.append('${' + section + ':command}' ) %}
{%- do hash_file_list.append('${' + key + '}') %}
[{{ section }}]
recipe = plone.recipe.command
update-command = ${:command}
command = [ ! -f {{ '${' + key + '}' }} ] && touch {{ '${' + key + '}' }}
{%- endmacro %}
{#- Create depending sections, as state files appear late, so it's better to have empty file which will impact the hash anyway #}
{%- if boot_image_url_list_enabled %}
{{ generate_depend_section('boot-image-url-list-depend', 'boot-image-url-list-download-wrapper:md5sum-state-file') }}
{%- endif %}
{%- if boot_image_url_select_enabled %}
{{ generate_depend_section('boot-image-url-select-depend', 'boot-image-url-select-download-wrapper:md5sum-state-file') }}
{%- endif %}
{%- if virtual_hard_drive_url_enabled %}
{{ generate_depend_section('virtual-hard-drive-url-depend', 'virtual-hard-drive-url-download-wrapper:md5sum-state-file') }}
{%- endif %}
[kvm-instance]
depends =
{%- for depend_section in depend_section_list %}
{{ depend_section }}
{%- endfor %}
recipe = slapos.cookbook:wrapper
socket-path = ${kvm-controller-parameter-dict:socket-path}
wrapper-path = ${directory:services}/kvm
command-line = ${kvm-run:rendered}
kvm-controller = ${kvm-controller-wrapper:wrapper-path}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
hash-files =
{%- for hash_file in hash_file_list %}
{{ hash_file }}
{%- endfor %}
[kvm-controller-wrapper]
recipe = slapos.cookbook:wrapper
......@@ -932,8 +1072,8 @@ disk-format = qcow2
disk-device-path =
cpu-count = 1
disk-cache = writeback
disk-aio = native
disk-cache =
disk-aio =
auto-ballooning = True
machine-options =
cpu-model = host
......@@ -1108,6 +1248,13 @@ parts =
cron-service
cron-entry-logrotate
frontend-promise
{% if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-download-wrapper
virtual-hard-drive-url-config-state-promise
virtual-hard-drive-url-download-md5sum-promise
virtual-hard-drive-url-download-state-promise
virtual-hard-drive-url-processed-config-promise
{% endif %}
{% if boot_image_url_list_enabled %}
boot-image-url-list-download-wrapper
boot-image-url-list-config-state-promise
......
......@@ -6,10 +6,6 @@ import hashlib
import os
import socket
import subprocess
try:
from urllib.request import FancyURLopener
except ImportError:
from urllib import FancyURLopener
import gzip
import shutil
from random import shuffle
......@@ -17,16 +13,11 @@ import glob
import re
import json
import ssl
# XXX: give all of this through parameter, don't use this as template, but as module
qemu_img_path = '{{ parameter_dict.get("qemu-img-path") }}'
qemu_path = '{{ parameter_dict.get("qemu-path") }}'
disk_size = '{{ parameter_dict.get("disk-size") }}'
disk_type = '{{ parameter_dict.get("disk-type") }}'
disk_format = '{{ parameter_dict.get("disk-format", "qcow2")}}'
disk_format = disk_format \
if disk_format in ['qcow2', 'raw', 'vdi', 'vmdk', 'cloop', 'qed'] else 'qcow2'
qemu_img_path = {{ repr(parameter_dict["qemu-img-path"]) }}
qemu_path = {{ repr(parameter_dict["qemu-path"]) }}
disk_size = {{ repr(parameter_dict["disk-size"]) }}
disk_type = {{ repr(parameter_dict["disk-type"]) }}
socket_path = '{{ parameter_dict.get("socket-path") }}'
nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
......@@ -34,11 +25,7 @@ nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
('{{ parameter_dict.get("nbd2-host") }}',
{{ parameter_dict.get("nbd2-port") }}))
default_cdrom_iso = '{{ parameter_dict.get("default-cdrom-iso") }}'
disk_path = '{{ parameter_dict.get("disk-path") }}'
virtual_hard_drive_url = '{{ parameter_dict.get("virtual-hard-drive-url") }}'.strip()
virtual_hard_drive_md5sum = '{{ parameter_dict.get("virtual-hard-drive-md5sum") }}'.strip()
virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
nat_rules = '{{ parameter_dict.get("nat-rules") }}'.strip()
use_tap = '{{ parameter_dict.get("use-tap") }}'.lower()
use_nat = '{{ parameter_dict.get("use-nat") }}'.lower()
......@@ -56,9 +43,7 @@ init_ram_size = {{ parameter_dict.get("init-ram-size") }}
pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = {{ parameter_dict.get("external-disk-size") }}
external_disk_format = '{{ parameter_dict.get("external-disk-format", "qcow2") }}'
external_disk_format = external_disk_format \
if external_disk_format in ['qcow2', 'raw', 'vdi', 'vmdk', 'cloop', 'qed'] else 'qcow2'
external_disk_format = {{ repr(parameter_dict["external-disk-format"]) }}
disk_storage_dict = {}
disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
map_storage_list = []
......@@ -70,11 +55,6 @@ cluster_doc_port = {{ parameter_dict.get("cluster-doc-port") }}
url_check_certificate = '{{ parameter_dict.get("hard-drive-url-check-certificate", "true") }}'.lower()
auto_ballooning = '{{ parameter_dict.get("auto-ballooning") }}' in ('true', 'True', '1')
vm_name = '{{ parameter_dict.get("name") }}'
disk_cache = '{{ parameter_dict.get("disk-cache", "writeback") }}'.strip()
disk_cache = disk_cache if disk_cache in ["none", "writeback", "unsafe",
"directsync", "writethrough"] else "writeback"
disk_aio = '{{ parameter_dict.get("disk-aio", "threads") }}'.strip()
disk_aio = disk_aio if disk_aio in ["threads", "native"] else "threads"
# If a device (ie.: /dev/sdb) is provided, use it instead
# the disk_path with disk_format
......@@ -84,16 +64,18 @@ for disk_device_path in '{{ parameter_dict.get("disk-device-path", "") }}'.split
disk_info_list.append({
'path': disk_device_path,
'format': "raw",
'io': "native",
'aio': "native",
'cache': "none"
})
if len(disk_info_list) == 0:
if not disk_info_list:
disk_info_list.append({
'path': disk_path,
'format': disk_format,
'io': disk_aio,
'cache': disk_cache,
{%- for k in 'path', 'format', 'aio', 'cache' %}
{%- set v = parameter_dict['disk-' + k] %}
{%- if v %}
{{ repr(k) }}: {{ repr(v) }},
{%- endif %}
{%- endfor %}
})
smp_count = {{ parameter_dict.get("smp-count") }}
......@@ -107,11 +89,8 @@ logfile = '{{ parameter_dict.get("log-file") }}'
boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
opener = FancyURLopener(context=ssl._create_unverified_context())
else:
opener = FancyURLopener({})
virtual_hard_drive_url_json_config = '{{ parameter_dict.get("virtual-hard-drive-url-json-config") }}'
virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
def md5Checksum(file_path):
with open(file_path, 'rb') as fh:
......@@ -172,39 +151,34 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
lf.write('%s' % external_disk_number)
return id_list, external_disk_number
# Download existing hard drive if needed at first boot
if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url != '':
print('Downloading virtual hard drive...')
try:
downloaded_disk = disk_info_list[0]['path']
# Use downloaded virtual-hard-drive-url
if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url_json_config != '':
print('Using virtual hard drive...')
with open(virtual_hard_drive_url_json_config) as fh:
image_config = json.load(fh)
if image_config['error-amount'] == 0:
image = image_config['image-list'][0]
downloaded_image = os.path.join(image_config['destination-directory'], image['destination'])
# previous version was using disk in place, but here it would result with
# redownload, so copy it
if virtual_hard_drive_gzipped == 'true':
downloaded_disk = '%s.gz' % disk_info_list[0]['path']
opener.retrieve(virtual_hard_drive_url, downloaded_disk)
except:
if os.path.exists(downloaded_disk):
os.remove(downloaded_disk)
raise
md5sum = virtual_hard_drive_md5sum.strip()
if md5sum:
print('Checking MD5 checksum...')
local_md5sum = md5Checksum(downloaded_disk)
if local_md5sum != md5sum:
os.remove(downloaded_disk)
raise Exception('MD5 mismatch. MD5 of local file is %s, Specified MD5 is %s.' % (
local_md5sum, md5sum))
print('MD5sum check passed.')
try:
with open(disk_info_list[0]['path'], 'wb') as d_fh:
with gzip.open(downloaded_image, 'rb') as s_fh:
shutil.copyfileobj(s_fh, d_fh)
except Exception:
if os.path.exists(disk_info_list[0]['path']):
os.unlink(disk_info_list[0]['path'])
raise
else:
try:
shutil.copyfile(downloaded_image, disk_info_list[0]['path'])
except Exception:
if os.path.exists(disk_info_list[0]['path']):
os.unlink(disk_info_list[0]['path'])
raise
else:
print('Warning: not checksum specified.')
if downloaded_disk.endswith('.gz'):
try:
with open(disk_info_list[0]['path'], 'w') as disk:
with gzip.open(downloaded_disk, 'rb') as disk_gz:
shutil.copyfileobj(disk_gz, disk)
except Exception:
if os.path.exists(disk_info_list[0]['path']):
os.remove(disk_info_list[0]['path'])
raise
os.remove(downloaded_disk)
raise ValueError('virtual-hard-drive-url not ready yet')
# Create disk if doesn't exist
# XXX: move to Buildout profile
......@@ -310,17 +284,12 @@ kvm_argument_list = [qemu_path,
'-nodefaults',
]
for disk_info in disk_info_list:
additional_disk_options = ''
if disk_info['io'] == 'native':
additional_disk_options += ',cache.direct=on'
if disk_info['format'] == "raw":
additional_disk_options += ',discard=on'
kvm_argument_list.extend([
kvm_argument_list += (
'-drive',
'file=%s,if=%s,cache=%s,aio=%s%s' % ( disk_info['path'], disk_type, disk_info['cache'], disk_info['io'], additional_disk_options)
])
'file=%s,if=%s,discard=on%s' % (
disk_info['path'], disk_type,
''.join(',%s=%s' % x for x in disk_info.items() if x[0] != 'path'))
)
rgx = re.compile('^[\w*\,][\=\d+\-\,\w]*$')
for numa in numa_list:
......@@ -392,6 +361,9 @@ else:
'-drive',
'file=%s,media=cdrom' % (link,)
])
else:
raise ValueError('boot-image-url-select not ready yet')
if boot_image_url_list_json_config:
# Support boot-image-url-list
with open(boot_image_url_list_json_config) as fh:
......@@ -404,6 +376,8 @@ else:
'-drive',
'file=%s,media=cdrom' % (link,)
])
else:
raise ValueError('boot-image-url-list not ready yet')
# Always add by default the default image
kvm_argument_list.extend([
'-drive', 'file=%s,media=cdrom' % default_cdrom_iso
......
......@@ -28,6 +28,7 @@
import six.moves.http_client as httplib
import json
import os
import glob
import hashlib
import psutil
import requests
......@@ -43,6 +44,7 @@ from six.moves import SimpleHTTPServer
import multiprocessing
import time
import shutil
import sys
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -120,12 +122,25 @@ class KvmMixin(object):
'software_release/buildout.cfg',
]
])
# find bin/kvm_raw
kvm_raw_list = glob.glob(
os.path.join(self.slap.instance_directory, '*', 'bin', 'kvm_raw'))
self.assertEqual(1, len(kvm_raw_list)) # allow to work only with one
hash_file_list = [
kvm_raw_list[0],
'software_release/buildout.cfg',
]
kvm_hash_value = generateHashFromFiles([
os.path.join(self.computer_partition_root_path, hash_file)
for hash_file in hash_file_list
])
with self.slap.instance_supervisor_rpc as supervisor:
running_process_info = '\n'.join(sorted([
'%(group)s:%(name)s %(statename)s' % q for q
in supervisor.getAllProcessInfo()
if q['name'] != 'watchdog' and q['group'] != 'watchdog']))
return running_process_info.replace(hash_value, '{hash}')
return running_process_info.replace(
hash_value, '{hash}').replace(kvm_hash_value, '{kvm-hash-value}')
def raising_waitForInstance(self, max_retry):
with self.assertRaises(SlapOSNodeCommandError):
......@@ -176,7 +191,7 @@ i0:6tunnel-10443-{hash}-on-watch RUNNING
i0:bootstrap-monitor EXITED
i0:certificate_authority-{hash}-on-watch RUNNING
i0:crond-{hash}-on-watch RUNNING
i0:kvm-{hash}-on-watch RUNNING
i0:kvm-{kvm-hash-value}-on-watch RUNNING
i0:kvm_controller EXITED
i0:monitor-httpd-{hash}-on-watch RUNNING
i0:monitor-httpd-graceful EXITED
......@@ -416,10 +431,11 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
"test-machine2": dict(bootstrap_machine_param_dict, **{
# Debian 9 image
"virtual-hard-drive-url":
"http://shacache.org/shacache/ce07873dbab7fa8501d1bf5565c2737b2"
"eed6c8b9361b4997b21daf5f5d1590972db9ac00131cc5b27d9aa353f2f940"
"71e073f9980cc61badd6d2427f592e6e8",
"virtual-hard-drive-md5sum": "2b113e3cd8276b9740189622603d6f99"
"http://shacache.org/shacache/93aeb72a556fe88d9889ce16558dfead"
"57a3c8f0a80d0e04ebdcd4a5830dfa6403e3976cc896b8332e74f202fccbd"
"a508930046a78cffea6e0e29d03345333cc",
"virtual-hard-drive-md5sum": "cdca79619ba987c40b98a8e31d281e4a",
"virtual-hard-drive-gzipped": True,
})
}
}))}
......@@ -499,7 +515,7 @@ ir2:bootstrap-monitor EXITED
ir2:certificate_authority-{hash}-on-watch RUNNING
ir2:crond-{hash}-on-watch RUNNING
ir2:equeue-on-watch RUNNING
ir2:kvm-{hash}-on-watch RUNNING
ir2:kvm-{kvm-hash-value}-on-watch RUNNING
ir2:kvm_controller EXITED
ir2:monitor-httpd-{hash}-on-watch RUNNING
ir2:monitor-httpd-graceful EXITED
......@@ -1282,7 +1298,7 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin):
'disk-device-path': '/dev/virt0 /dev/virt1',
'wipe-disk-ondestroy': True
})
self.slap.waitForInstance(max_retry=2)
self.raising_waitForInstance(3)
instance_path = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference)
......@@ -1298,3 +1314,210 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin):
dd if=/dev/zero of=/dev/virt1 bs=4096 count=500k"""
)
self.assertTrue(os.access(slapos_wipe_device_disk, os.X_OK))
@skipUnlessKvm
class TestImageDownloadController(InstanceTestCase, FakeImageServerMixin):
__partition_reference__ = 'idc'
maxDiff = None
def setUp(self):
super(TestImageDownloadController, self).setUp()
self.working_directory = tempfile.mkdtemp()
self.destination_directory = os.path.join(
self.working_directory, 'destination')
os.mkdir(self.destination_directory)
self.config_json = os.path.join(
self.working_directory, 'config.json')
self.md5sum_fail_file = os.path.join(
self.working_directory, 'md5sum_fail_file')
self.error_state_file = os.path.join(
self.working_directory, 'error_state_file')
self.processed_md5sum = os.path.join(
self.working_directory, 'processed_md5sum')
self.startImageHttpServer()
self.image_download_controller = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '0',
'software_release', 'parts', 'image-download-controller',
'image-download-controller')
def tearDown(self):
self.stopImageHttpServer()
shutil.rmtree(self.working_directory)
super(InstanceTestCase, self).tearDown()
def callImageDownloadController(self, *args):
call_list = [sys.executable, self.image_download_controller] + list(args)
try:
return (0, subprocess.check_output(
call_list, stderr=subprocess.STDOUT).decode('utf-8'))
except subprocess.CalledProcessError as e:
return (e.returncode, e.output.decode('utf-8'))
def runImageDownloadControlerWithDict(self, json_dict):
with open(self.config_json, 'w') as fh:
json.dump(json_dict, fh, indent=2)
return self.callImageDownloadController(
self.config_json,
'curl', # comes from test environemnt, considered to be recent enough
self.md5sum_fail_file,
self.error_state_file,
self.processed_md5sum
)
def assertFileContent(self, path, content):
self.assertTrue(os.path.exists, path)
with open(path, 'r') as fh:
self.assertEqual(
fh.read(),
content)
def test(self):
json_dict = {
'error-amount': 0,
'config-md5sum': 'config-md5sum',
'destination-directory': self.destination_directory,
'image-list': [
{
'destination-tmp': 'tmp',
'url': self.fake_image,
'destination': 'destination',
'link': 'image_001',
'gzipped': False,
'md5sum': self.fake_image_md5sum,
}
]
}
code, result = self.runImageDownloadControlerWithDict(
json_dict
)
self.assertEqual(
(code, result.strip()),
(0, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : Downloading
INF: %(fake_image)s : Stored with checksum %(checksum)s
INF: %(fake_image)s : Symlinking %(symlink)s -> %(destination)s
""".strip() % {
'fake_image': self.fake_image,
'checksum': self.fake_image_md5sum,
'error_state_file': self.error_state_file,
'symlink': os.path.join(self.destination_directory, 'image_001'),
'destination': os.path.join(self.destination_directory, 'destination'),
})
)
self.assertFileContent(self.md5sum_fail_file, '')
self.assertFileContent(self.error_state_file, '')
self.assertFileContent(self.processed_md5sum, 'config-md5sum')
self.assertFalse(
os.path.exists(os.path.join(self.destination_directory, 'tmp')))
self.assertFileContent(
os.path.join(self.destination_directory, 'destination'),
'fake_image_content'
)
# Nothing happens if all is downloaded
code, result = self.runImageDownloadControlerWithDict(
json_dict
)
self.assertEqual(
(code, result.strip()),
(0, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : already downloaded
""".strip() % {
'fake_image': self.fake_image,
'checksum': self.fake_image_md5sum,
'error_state_file': self.error_state_file,
'symlink': os.path.join(self.destination_directory, 'image_001'),
'destination': os.path.join(self.destination_directory, 'destination'),
})
)
def test_fail(self):
json_dict = {
'error-amount': 0,
'config-md5sum': 'config-md5sum',
'destination-directory': self.destination_directory,
'image-list': [
{
'destination-tmp': 'tmp',
'url': self.fake_image,
'destination': 'destination',
'link': 'image_001',
'gzipped': False,
'md5sum': self.fake_image_wrong_md5sum,
}
]
}
for try_num in range(1, 5):
code, result = self.runImageDownloadControlerWithDict(
json_dict
)
self.assertEqual(
(code, result.strip()),
(1, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : Downloading
""". strip() % {
'fake_image': self.fake_image,
'error_state_file': self.error_state_file,
'symlink': os.path.join(self.destination_directory, 'image_001'),
'destination': os.path.join(
self.destination_directory, 'destination'),
})
)
fake_image_url = '#'.join([
self.fake_image, self.fake_image_wrong_md5sum])
self.assertFileContent(
self.md5sum_fail_file, """{
"%s": %s
}""" % (fake_image_url, try_num))
self.assertFileContent(
self.error_state_file, """
ERR: %(fake_image)s : MD5 mismatch expected is %(wrong_checksum)s """
"""but got instead %(real_checksum)s""".strip() % {
'fake_image': self.fake_image,
'wrong_checksum': self.fake_image_wrong_md5sum,
'real_checksum': self.fake_image_md5sum,
})
self.assertFileContent(self.processed_md5sum, 'config-md5sum')
self.assertFalse(
os.path.exists(os.path.join(self.destination_directory, 'tmp')))
self.assertFalse(
os.path.exists(
os.path.join(self.destination_directory, 'destination')))
code, result = self.runImageDownloadControlerWithDict(
json_dict
)
self.assertEqual(
(code, result.strip()),
(1, """
INF: Storing errors in %(error_state_file)s
""". strip() % {
'fake_image': self.fake_image,
'error_state_file': self.error_state_file,
'symlink': os.path.join(self.destination_directory, 'image_001'),
'destination': os.path.join(
self.destination_directory, 'destination'),
})
)
fake_image_url = '#'.join([
self.fake_image, self.fake_image_wrong_md5sum])
self.assertFileContent(
self.md5sum_fail_file, """{
"%s": %s
}""" % (fake_image_url, 4))
self.assertFileContent(
self.error_state_file, """
ERR: %(fake_image)s : Checksum is incorrect after 4 tries, will not """
"""retry""".strip() % {
'fake_image': self.fake_image,
})
self.assertFileContent(self.processed_md5sum, 'config-md5sum')
self.assertFalse(
os.path.exists(os.path.join(self.destination_directory, 'tmp')))
self.assertFalse(
os.path.exists(
os.path.join(self.destination_directory, 'destination')))
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 9ddae686379e8d747410c1adf82b47d6
md5sum = c115ed9d4ff0f785d79cdcacbb0bd1ad
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
......@@ -26,11 +26,11 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
[template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = a57106ee88ff3295b9ffce84105da79b
md5sum = 3c8ab4e78f66c974eb95afc595a13514
[template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = c1885a42aadd45bab3185a53258d4ff4
md5sum = 365a6cc6831267a73fa5ebd56ad394ee
[network-bench-cfg]
filename = network_bench.cfg.in
......@@ -42,4 +42,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
[template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2
md5sum = 89545501f0e5bf11608978886429da3d
md5sum = 609c6cca763b73a80fa05ee56475eb20
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"nameserver": {
"default": "",
"title": "Nameserver",
"description": "Space separated list of name servers to use.",
"type": "string"
},
"check-status-code": {
"default": "200",
"title": "Default Check HTTP Code",
"description": "Default HTTP code to check against (default: 200).",
"type": "string"
},
"check-http-header-dict": {
"default": "{}",
"title": "HTTP header dict to check",
"description": "JSON dict of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}",
"region-dict": {
"title": "Regions",
"description": "Defines regions of the cluster",
"patternProperties": {
".*": {
"properties": {
"state": {
"title": "State",
"description": "State of the node of the region. Can be used to destroy not needed regions.",
"type": "string",
"default": "started",
"enum": [
"started",
"stopped",
"destroyed"
]
},
"sla-computer_guid": {
"title": "GUID of the computer on which this region shall be deployed",
"description": "Unique identifier of the computer, like \"COMP-1234\". By default, let Master choose a computer.",
"type": "string",
"default": ""
},
"nameserver-list": {
"default": [],
"title": "Nameservers",
"description": "List of nameservers to use.",
"type": "array"
},
"check-frontend-ip-list": {
"default": [],
"title": "Default Frontend IPs to check",
"description": "List of default frontend IPs to check, if empty no constraint is used.",
"type": "array"
}
},
"type": "object"
}
},
"type": "object"
},
"check-frontend-ip": {
"default": "",
"title": "Default space separated list of Frontend IPs to check",
"description": "Default list of Frontend IPs to check, if empty no constraint is used.",
"type": "string"
},
"check-certificate-expiration-days": {
"default": "15",
"title": "Default certificate expiration days check",
"description": "Default amount of days to consider certificate as being to-be-expired (default: 15).",
"type": "string"
},
"check-maximum-elapsed-time": {
"default": "2",
"title": "Default maximum elapsed time for a site to reply (seconds)",
"description": "Default maximum elapsed time for a site to reply to be considered good (default: 2s).",
"type": "string"
"nameserver-list": {
"default": [],
"title": "Nameservers (backward compatibility)",
"description": "List of nameservers to use. Note: This is backward compatibility, use region-dict for full configuration control.",
"type": "array"
},
"failure-amount": {
"default": "1",
"title": "Default amount of failures to consider URL as in bad state",
"description": "Default amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages (default: 1).",
"type": "string"
"check-frontend-ip-list": {
"default": [],
"title": "Default Frontend IPs to check (backward compatibility)",
"description": "List of default frontend IPs to check, if empty no constraint is used. Note: This is backward compatibility, use region-dict for full configuration control.",
"type": "array"
}
}
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"url": {
"title": "URL to check",
"description": "URL to check, like https://example.com",
"description": "URL to check, like https://example.com/",
"type": "string"
},
"check-status-code": {
"default": "Master default",
"title": "Default Check HTTP Code.",
"description": "HTTP code to check against (default: comes from master partition).",
"type": "string"
},
"check-http-header-dict": {
"default": "Master default",
"title": "HTTP header dict to check",
"description": "JSON dict of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}",
"type": "object"
"region-dict": {
"title": "Applicable Regions",
"description": "Puts the check on the defined regions. No definition will result with presence in all regions.",
"patternProperties": {
".*": {
"properties": {
"state": {
"title": "State",
"description": "State of the check of the region. Used only to make it correctly visible in the SlapOS Master UI if no other parameters are defined.",
"type": "string",
"default": "present",
"enum": [
"present"
]
},
"check-frontend-ip-list": {
"default": [],
"title": "Frontend IPs to check",
"description": "List of default frontend IPs to check, if empty no constraint is used. Defaults to region configuration.",
"type": "array"
}
},
"type": "object"
}
},
"type": "object",
"default": {}
},
"check-frontend-ip": {
"default": "Master default",
"title": "Space separated list of Frontend IPs to check",
"description": "List of Frontend IPs to check, if empty no constraint is used (default: comes from master partition).",
"type": "string"
"check-status-code": {
"title": "HTTP Code Check",
"description": "Expected response HTTP Code.",
"type": "number",
"default": 200,
"minimum": 100,
"maximum": 599
},
"check-certificate-expiration-days": {
"default": "Master default",
"title": "Certificate expiration days check",
"description": "Amount of days to consider certificate as being to-be-expired (default: comes from master partition).",
"type": "string"
"title": "Certificate Expiration Check (days)",
"description": "Amount of days to consider certificate as being to-be-expired.",
"type": "number",
"default": 15,
"minimum": 1
},
"check-maximum-elapsed-time": {
"default": "Master default",
"title": "Maximum elapsed time for a site to reply (seconds)",
"description": "Maximum elapsed time for a site to reply to be considered good.(default: comes from master partition).",
"type": "string"
"title": "Maximum Elapsed Check (seconds)",
"description": "Maximum elapsed time for a site to reply to be considered good.",
"type": "number",
"default": 2,
"minimum": 1
},
"check-http-header-dict": {
"title": "HTTP Header Check",
"description": "JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
"type": "object",
"default": {}
},
"failure-amount": {
"default": "Master default",
"title": "Amount of failures to consider URL as in bad state",
"description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages (default: comes from master partition).",
"type": "string"
"title": "Failure Amount",
"description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
"type": "number",
"default": 2,
"minimum": 1
},
"check-frontend-ip-list": {
"title": "Frontend IPs to check (backward compatibility)",
"description": "List of Frontend IPs to check, if empty no constraint is used. Defaults to region configuration. Note: Use region-dict's check-frontend-ip-list to ensure specific check on each region.",
"type": "array"
}
}
}
......@@ -9,39 +9,27 @@
{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
{%- if extra_slave_instance_list %}
{#- Create slaves to process with setting up defaults #}
{%- for slave in json_module.loads(extra_slave_instance_list) | sort(attribute='slave_title') %}
{%- if 'check-status-code' not in slave %}
{%- do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %}
{%- endif %}
{%- if 'check-http-header-dict' not in slave %}
{%- do slave.__setitem__('check-http-header-dict', CONFIGURATION['check-http-header-dict']) %}
{%- endif %}
{%- if 'check-certificate-expiration-days' not in slave %}
{%- do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %}
{%- endif %}
{%- if 'failure-amount' not in slave %}
{%- do slave.__setitem__('failure-amount', CONFIGURATION['failure-amount']) %}
{%- endif %}
{%- if 'check-maximum-elapsed-time' not in slave %}
{%- do slave.__setitem__('check-maximum-elapsed-time', CONFIGURATION['check-maximum-elapsed-time']) %}
{%- endif %}
{%- if 'check-frontend-ip' not in slave %}
{%- do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %}
{%- endif %}
{%- if 'url' in slave %}
{%- set class = slave['check-maximum-elapsed-time'] %}
{%- if class not in slave_instance_dict %}
{%- do slave_instance_dict.__setitem__(class, []) %}
{%- for slave in extra_slave_instance_list | sort(attribute='-slave-title') %}
{%- do slave.setdefault('check-status-code', 200) %}
{%- do slave.setdefault('check-http-header-dict', {}) %}
{%- do slave.setdefault('check-certificate-expiration-days', 15) %}
{%- do slave.setdefault('failure-amount', 2) %}
{%- do slave.setdefault('check-maximum-elapsed-time', 2) %}
{%- do slave.setdefault('check-frontend-ip-list', CONFIGURATION['check-frontend-ip-list']) %}
{%- if 'url' in slave %}
{%- set class = slave['check-maximum-elapsed-time'] %}
{%- if class not in slave_instance_dict %}
{%- do slave_instance_dict.__setitem__(class, []) %}
{%- endif %}
{%- do slave_instance_dict[class].append(slave) %}
{%- endif %}
{%- do slave_instance_dict[class].append(slave) %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
{%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
{%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
[{{part_id}}]
......@@ -51,11 +39,11 @@ name = {{ safe_name }}.py
config-report = http_query
config-url = {{ slave['url'] }}
config-status-code = {{ slave['check-status-code'] }}
config-http-header-dict = {{ slave['check-http-header-dict'] }}
config-http-header-dict = {{ json_module.dumps(slave['check-http-header-dict']) }}
config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
config-failure-amount = {{ slave['failure-amount'] }}
config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
config-ip-list = {{ slave['check-frontend-ip'] }}
config-ip-list = {{ ' '.join(slave['check-frontend-ip-list']) }}
config-json-file = ${surykatka-config-{{ class }}:json}
{%- endfor %}
......@@ -72,7 +60,7 @@ db = ${directory:srv}/surykatka-{{ class }}.db
rendered = ${directory:etc}/surykatka-{{ class }}.ini
template = {{ template_surykatka_ini }}
slave_instance_list = {{ dumps(slave_instance_list) }}
nameserver = {{ dumps(CONFIGURATION['nameserver']) }}
nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #}
timeout = {{ int(class) + 2 }}
......@@ -80,7 +68,7 @@ timeout = {{ int(class) + 2 }}
context =
import json_module json
key db :db
key nameserver :nameserver
key nameserver_list :nameserver_list
key slave_instance_list :slave_instance_list
key timeout :timeout
......@@ -110,9 +98,9 @@ cron-entries = ${directory:etc}/cron.d
name = surykatka-status-{{ class }}
frequency = */2 * * * *
command = ${surykatka-status-json-{{ class }}:rendered}
{%- do part_list.append('surykatka-' + class) %}
{%- do part_list.append('surykatka-bot-promise-' + class) %}
{%- do part_list.append('cron-entry-surykatka-status-' + class) %}
{%- do part_list.append('surykatka-%i'% (class,)) %}
{%- do part_list.append('surykatka-bot-promise-%i' % (class,)) %}
{%- do part_list.append('cron-entry-surykatka-status-%i' % (class,)) %}
{%- endfor %}
[buildout]
......@@ -163,4 +151,4 @@ key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-parameter]
{%- endif %}
{%- endif %} {#- if slap_software_type == software_type #}
......@@ -16,34 +16,86 @@ extensions = jinja2.ext.do
extra-context =
section slave_information slap-configuration
{% set part_list = [] -%}
# Publish information for each slave
{% set part_list = [] -%}
{%- set edgebot_software_type = 'edgebot' %}
{%- set edgebot_quantity = slapparameter_dict.pop('edgebot-quantity', '1') | int %}
{%- set edgebot_list = [] %}
{%- set edgebot_section_list = [] %}
{%- set slave_list_name = 'extra_slave_instance_list' %}
{%- set request_dict = {} %}
{%- set namebase = "edgebot" %}
{%- set authorized_slave_list = [] %}
{%- set monitor_base_url_dict = {} -%}
{%- if 'region-dict' not in slapparameter_dict %}
{#- Be nice and allow to work with default configuration #}
{%- do slapparameter_dict.__setitem__('region-dict', {
'1': {
'sla-computer_guid': slap_configuration['computer'],
'state': slap_configuration['instance-state'],
'nameserver-list': slapparameter_dict.get('nameserver-list', []),
'check-frontend-ip-list': slapparameter_dict.get('check-frontend-ip-list', []),
}
}) %}
{%- endif %}
{%- set active_region_list = [] %}
{%- for region_name in sorted(slapparameter_dict['region-dict']) %}
{%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
{%- if region_parameter_dict.get('state', 'started') == 'started' %}
{%- do active_region_list.append(region_name) %}
{%- endif %}
{%- endfor %}
{%- set authorized_slave_dict = {} %}
{%- set publish_slave_dict_dict = {} %}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- do authorized_slave_list.append(slave) %}
{%- set slave_reference = slave.pop('slave_reference') %}
{%- set publish_dict = {'assigned-region-dict': {}} %}
{%- if '_' in slave %}
{%- set base_slave_dict = json_module.loads(slave.pop('_')) %} {#- XXX: Unsafe! #}
{%- do base_slave_dict.__setitem__('-slave-title', slave['slave_title']) %}
{%- do base_slave_dict.__setitem__('-slave-reference', slave_reference) %}
{%- set slave_region_dict = base_slave_dict.pop('region-dict', {}) %}
{%- if slave_region_dict == {} %}
{%- for region in active_region_list %}
{%- do slave_region_dict.__setitem__(region, {}) %}
{%- endfor %}
{%- endif %}
{%- for region in slave_region_dict %}
{%- if region in active_region_list %}
{%- set region_info = {
'nameserver-list': slapparameter_dict['region-dict'][region].get('nameserver-list') or slapparameter_dict.get('slapparameter_dict') or [],
'check-frontend-ip-list': slave_region_dict[region].get('check-frontend-ip-list') or base_slave_dict.get('check-frontend-ip-list') or slapparameter_dict['region-dict'][region].get('check-frontend-ip-list') or slapparameter_dict.get('check-frontend-ip-list') or [],
} %}
{%- do publish_dict['assigned-region-dict'].__setitem__(region, region_info) %}
{%- set slave_dict = base_slave_dict.copy() %}
{%- do slave_dict.update(region_info) %}
{%- if region not in authorized_slave_dict %}
{%- do authorized_slave_dict.__setitem__(region, [slave_dict]) %}
{%- else %}
{%- do authorized_slave_dict[region].append(slave_dict) %}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- do publish_slave_dict_dict.__setitem__(slave_reference, publish_dict) %}
{%- endfor %}
{%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
{%- for i in range(1, edgebot_quantity + 1) %}
{%- set edgebot_name = "%s-%s" % (namebase, i) %}
{%- set request_section_title = 'request-%s' % edgebot_name %}
{%- do edgebot_list.append(edgebot_name) %}
{%- do edgebot_section_list.append(request_section_title) %}
{%- set number = {'i': 1} %}
{%- for region_name in sorted(slapparameter_dict['region-dict']) %}
{%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
{%- set edgebot_name = "%s-%s" % (namebase, region_name) %}
{%- set request_section_title = 'request-%s' % (hashlib_module.md5(edgebot_name.encode('utf-8')).hexdigest(),) %}
{%- do part_list.append(request_section_title) %}
{%- do request_dict.__setitem__(request_section_title,
{
'config': {'monitor-httpd-port': monitor_base_port + i},
'name': edgebot_name,
'sla': {},
'state': 'started',
}) %}
{#- Note: monitor-httpd-port will vary on regions being added and removed,
but this is accepted, as it's only internal trick #}
{%- do request_dict.__setitem__(
request_section_title,
{
'config': {
'monitor-httpd-port': monitor_base_port + number['i'],
'check-frontend-ip-list': region_parameter_dict.get('check-frontend-ip-list', []),
'nameserver-list': region_parameter_dict.get('nameserver-list', []),
'extra_slave_instance_list': authorized_slave_dict.get(region_name, [])
},
'name': edgebot_name,
'sla': {'computer_guid': region_parameter_dict['sla-computer_guid']},
'state': region_parameter_dict.get('state', 'started'),
}) %}
{%- do number.__setitem__('i', number['i'] + 1) %}
{%- endfor %}
[replicate]
......@@ -54,54 +106,58 @@ config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
software-url = ${slap-connection:software-release-url}
software-type = {{edgebot_software_type}}
return = monitor-base-url
{% for section, edgebot_request in request_dict.items() %}
{%- set monitor_base_url_dict = {} -%}
{% for section, edgebot_request in request_dict.items() %}
[{{section}}]
<= replicate
name = {{ edgebot_request.get('name') }}
{%- if edgebot_request.get('state') %}
state = {{ edgebot_request.get('state') }}
{%- endif%}
{%- set slave_configuration_dict = slapparameter_dict %}
{%- do slave_configuration_dict.update(edgebot_request.get('config')) %}
{%- do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
{%- for config_key, config_value in slave_configuration_dict.items() %}
name = {{ edgebot_request['name'] }}
state = {{ edgebot_request['state'] }}
{%- if edgebot_request['state'] != 'destroyed' %}
{%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') %}
return = monitor-base-url
{%- endif %}
{%- set edgebot_configuration_dict = edgebot_request['config'] %}
{%- for config_key, config_value in edgebot_configuration_dict.items() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{%- if edgebot_request.get('sla') %}
{%- for parameter, value in edgebot_request.get('sla').items() %}
{% endfor -%}
{%- for parameter, value in edgebot_request['sla'].items() %}
sla-{{ parameter }} = {{ value }}
{%- endfor %}
{%- else %}
# As no SLA was provided, by default it is requested on the same computer
sla-computer_guid = ${slap-connection:computer-id}
{% endif %}
{%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{%- endfor %}
{%- endfor %}
{%- set directory_list = [] -%}
{%- for slave_instance in slave_instance_list -%}
{%- set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference') -%}
# Publish information for each slave
{%- for slave_reference, publish_dict in publish_slave_dict_dict.items() -%}
{%- set publish_section_title = 'publish-%s' % (hashlib_module.md5(slave_reference.encode('utf-8')).hexdigest(),) -%}
{%- do part_list.append(publish_section_title) %}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_instance.get('slave_reference') }}
{% endfor %}
recipe = slapos.cookbook:publish.serialised
available-region-list = {{ dumps(list(active_region_list)) }}
-slave-reference = {{ slave_reference }}
{%- for key, value in publish_dict.items() %}
{{ key }} = {{ dumps(value) }}
{%- endfor %}
{% endfor %}
[monitor-conf-parameters]
monitor-title = Monitor
password = ${monitor-htpasswd:passwd}
[monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%}
{% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }}
{% endfor %}
{% endfor %}
[buildout]
extends = {{ instance_base_monitor }}
parts +=
slave-test-configuration
{% for part in part_list %}
{% for part in part_list %}
{{ ' %s' % part }}
{%- endfor %}
{%- endif %}
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
active-region-list = {{ dumps(list(active_region_list)) }}
sla-computer_guid = {{ dumps(slap_configuration['computer']) }}
sla-instance_guid = {{ dumps(slap_configuration['instance-guid']) }}
{%- endif %} {#- if slap_software_type == software_type #}
......@@ -32,6 +32,7 @@ template = ${template-monitor-edgetest:target}
rendered = $${buildout:directory}/template-monitor-base-edgetest.cfg
extensions = jinja2.ext.do
context = import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
......@@ -53,6 +54,7 @@ surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name}
template-surykatka-ini = ${template-surykatka-ini:target}
context = import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
section slap_configuration slap-configuration
......@@ -75,13 +77,8 @@ url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
# Defaults
configuration.check-status-code = 200
configuration.check-http-header-dict = {}
configuration.nameserver =
configuration.check-frontend-ip =
configuration.check-certificate-expiration-days = 15
configuration.check-maximum-elapsed-time = 2
configuration.failure-amount = 2
configuration.nameserver-list =
configuration.check-frontend-ip-list =
# use monitor-base-port to have monitor listening on each instance
# on different port and also on different port than other services
# it makes it possible to instantiate it correctly on signle IP, for
......
......@@ -15,14 +15,17 @@
"description": "Cluster of bots to perform a distributed monitoring ",
"request": "instance-edgetest-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 1
},
"edgetest-slave": {
"title": "Edge Test Slave",
"shared": true,
"software-type": "edgetest",
"description": "Cluster of bots to perform a distributed monitoring ",
"request": "instance-edgetest-slave-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 2
}
}
......
......@@ -2,7 +2,6 @@
INTERVAL = 120
TIMEOUT = {{ timeout }}
SQLITE = {{ db }}
{%- set nameserver_list = nameserver.split() %}
{%- if len(nameserver_list) > 0 %}
NAMESERVER =
{%- for nameserver_entry in sorted(nameserver_list) %}
......@@ -10,7 +9,7 @@ NAMESERVER =
{%- endfor %}
{% endif %}
URL =
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- if 'url' in slave %}
{{ slave['url'] }}
{%- endif -%}
......
......@@ -26,11 +26,13 @@
##############################################################################
import glob
import hashlib
import json
import os
import re
import requests
import subprocess
import unittest
import xml.etree.ElementTree as ET
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -69,8 +71,10 @@ class MonitorTestMixin:
monitor_setup_url_key = 'monitor-setup-url'
def test_monitor_setup(self):
connection_parameter_dict = self\
connection_parameter_dict_serialised = self\
.computer_partition.getConnectionParameterDict()
connection_parameter_dict = json.loads(
connection_parameter_dict_serialised['_'])
self.assertTrue(
self.monitor_setup_url_key in connection_parameter_dict,
'%s not in %s' % (self.monitor_setup_url_key, connection_parameter_dict))
......@@ -136,6 +140,7 @@ class MonitorTestMixin:
class EdgeSlaveMixin(MonitorTestMixin):
__partition_reference__ = 'edge'
instance_max_retry = 20
expected_connection_parameter_dict = {}
@classmethod
def getInstanceSoftwareType(cls):
......@@ -143,36 +148,37 @@ class EdgeSlaveMixin(MonitorTestMixin):
def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
software_url = self.getSoftwareURL()
self.slap.request(
return self.slap.request(
software_release=software_url,
software_type='edgetest',
partition_reference=partition_reference,
partition_parameter_kw=partition_parameter_kw,
partition_parameter_kw={'_': json.dumps(partition_parameter_kw)},
shared=True
)
def updateSurykatkaDict(self):
for class_ in self.surykatka_dict:
update_dict = {}
update_dict['ini-file'] = os.path.join(
self.bot_partition_path, 'etc', 'surykatka-%s.ini' % (class_,))
update_dict['json-file'] = os.path.join(
self.bot_partition_path, 'srv', 'surykatka-%s.json' % (class_,))
update_dict['status-json'] = os.path.join(
self.bot_partition_path, 'bin', 'surykatka-status-json-%s' % (class_,))
update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
update_dict['status-cron'] = os.path.join(
self.bot_partition_path, 'etc', 'cron.d', 'surykatka-status-%s' % (
class_,))
update_dict['db_file'] = os.path.join(
self.bot_partition_path, 'srv', 'surykatka-%s.db' % (class_,))
self.surykatka_dict[class_].update(update_dict)
def setUp(self):
self.bot_partition_path = os.path.join(
self.slap.instance_directory,
self.__partition_reference__ + '1')
self.updateSurykatkaDict()
for instance_reference in self.surykatka_dict:
for class_ in self.surykatka_dict[instance_reference]:
update_dict = {}
update_dict['ini-file'] = os.path.join(
self.slap.instance_directory, instance_reference, 'etc',
'surykatka-%s.ini' % (class_,))
update_dict['json-file'] = os.path.join(
self.slap.instance_directory, instance_reference, 'srv',
'surykatka-%s.json' % (class_,))
update_dict['status-json'] = os.path.join(
self.slap.instance_directory, instance_reference, 'bin',
'surykatka-status-json-%s' % (class_,))
update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
update_dict['status-cron'] = os.path.join(
self.slap.instance_directory, instance_reference, 'etc',
'cron.d', 'surykatka-status-%s' % (class_,))
update_dict['db_file'] = os.path.join(
self.slap.instance_directory, instance_reference, 'srv',
'surykatka-%s.db' % (class_,))
self.surykatka_dict[instance_reference][class_].update(update_dict)
def setUpMonitorConfigurationList(self):
self.monitor_configuration_list = [
{
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
......@@ -194,43 +200,65 @@ class EdgeSlaveMixin(MonitorTestMixin):
}
]
def setUp(self):
self.updateSurykatkaDict()
self.setUpMonitorConfigurationList()
def assertSurykatkaIni(self):
expected_init_path_list = []
for instance_reference in self.surykatka_dict:
expected_init_path_list.extend(
[q['ini-file']
for q in self.surykatka_dict[instance_reference].values()])
self.assertEqual(
set(
glob.glob(
os.path.join(self.bot_partition_path, 'etc', 'surykatka*.ini'))),
{q['ini-file'] for q in self.surykatka_dict.values()}
os.path.join(
self.slap.instance_directory, '*', 'etc', 'surykatka*.ini'
)
)
),
set(expected_init_path_list)
)
for info_dict in self.surykatka_dict.values():
self.assertEqual(
info_dict['expected_ini'].strip() % info_dict,
open(info_dict['ini-file']).read().strip()
)
def assertPromiseContent(self, name, content):
for instance_reference in self.surykatka_dict:
for info_dict in self.surykatka_dict[instance_reference].values():
self.assertEqual(
info_dict['expected_ini'].strip() % info_dict,
open(info_dict['ini-file']).read().strip()
)
def assertPromiseContent(self, instance_reference, name, content):
promise = open(
os.path.join(
self.bot_partition_path, 'etc', 'plugin', name
self.slap.instance_directory, instance_reference, 'etc', 'plugin', name
)).read().strip()
self.assertTrue(content in promise)
def assertHttpQueryPromiseContent(self, instance_reference, name, content):
hashed = 'http-query-%s-promise.py' % (
hashlib.md5(('_' + name).encode('utf-8')).hexdigest(),)
self.assertPromiseContent(instance_reference, hashed, content)
def assertSurykatkaBotPromise(self):
for info_dict in self.surykatka_dict.values():
self.assertPromiseContent(
info_dict['bot-promise'],
"'report': 'bot_status'")
self.assertPromiseContent(
info_dict['bot-promise'],
"'json-file': '%s'" % (info_dict['json-file'],)
)
for instance_reference in self.surykatka_dict:
for info_dict in self.surykatka_dict[instance_reference].values():
self.assertPromiseContent(
instance_reference,
info_dict['bot-promise'],
"'report': 'bot_status'")
self.assertPromiseContent(
instance_reference,
info_dict['bot-promise'],
"'json-file': '%s'" % (info_dict['json-file'],),)
def assertSurykatkaCron(self):
for info_dict in self.surykatka_dict.values():
self.assertEqual(
'*/2 * * * * %s' % (info_dict['status-json'],),
open(info_dict['status-cron']).read().strip()
)
for instance_reference in self.surykatka_dict:
for info_dict in self.surykatka_dict[instance_reference].values():
self.assertEqual(
'*/2 * * * * %s' % (info_dict['status-json'],),
open(info_dict['status-cron']).read().strip()
)
def initiateSurykatkaRun(self):
try:
......@@ -239,17 +267,30 @@ class EdgeSlaveMixin(MonitorTestMixin):
pass
def assertSurykatkaStatusJSON(self):
for info_dict in self.surykatka_dict.values():
if os.path.exists(info_dict['json-file']):
os.unlink(info_dict['json-file'])
try:
subprocess.check_call(info_dict['status-json'])
except subprocess.CalledProcessError as e:
self.fail('%s failed with code %s and message %s' % (
info_dict['status-json'], e.returncode, e.output))
with open(info_dict['json-file']) as fh:
status_json = json.load(fh)
self.assertIn('bot_status', status_json)
for instance_reference in self.surykatka_dict:
for info_dict in self.surykatka_dict[instance_reference].values():
if os.path.exists(info_dict['json-file']):
os.unlink(info_dict['json-file'])
try:
subprocess.check_call(info_dict['status-json'])
except subprocess.CalledProcessError as e:
self.fail('%s failed with code %s and message %s' % (
info_dict['status-json'], e.returncode, e.output))
with open(info_dict['json-file']) as fh:
status_json = json.load(fh)
self.assertIn('bot_status', status_json)
def assertConnectionParameterDict(self):
serialised = self.requestDefaultInstance().getConnectionParameterDict()
connection_parameter_dict = json.loads(serialised['_'])
# tested elsewhere
connection_parameter_dict.pop('monitor-setup-url', None)
# comes from instance-monitor.cfg.jinja2, not needed here
connection_parameter_dict.pop('server_log_url', None)
self.assertEqual(
self.expected_connection_parameter_dict,
connection_parameter_dict
)
def test(self):
# Note: Those tests do not run surykatka and do not do real checks, as
......@@ -266,83 +307,202 @@ class EdgeSlaveMixin(MonitorTestMixin):
self.assertSurykatkaBotPromise()
self.assertSurykatkaPromises()
self.assertSurykatkaCron()
self.assertConnectionParameterDict()
class TestEdge(EdgeSlaveMixin, SlapOSInstanceTestCase):
expected_connection_parameter_dict = {
'active-region-list': ['1'],
'sla-computer_guid': 'local', 'sla-instance_guid': 'local-edge0'}
surykatka_dict = {
2: {'expected_ini': """[SURYKATKA]
'edge1': {
1: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 3
SQLITE = %(db_file)s
URL =
https://www.checkmaximumelapsedtime1.org/"""},
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 4
SQLITE = %(db_file)s
URL =
https://www.erp5.com/
https://www.erp5.org/"""}
https://www.checkcertificateexpirationdays.org/
https://www.checkfrontendiplist.org/
https://www.checkhttpheaderdict.org/
https://www.checkstatuscode.org/
https://www.default.org/
https://www.failureamount.org/"""},
20: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 22
SQLITE = %(db_file)s
URL =
https://www.checkmaximumelapsedtime20.org/"""},
}
}
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'status-code': '300'")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'certificate-expiration-days': '15'")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
self.assertHttpQueryPromiseContent(
'edge1',
'checkcertificateexpirationdays',
"""extra_config_dict = { 'certificate-expiration-days': '20',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.checkcertificateexpirationdays.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'checkhttpheaderdict',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{"A": "AAA"}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.checkhttpheaderdict.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'checkmaximumelapsedtime1',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '1',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.checkmaximumelapsedtime1.org/'}""" % (
self.surykatka_dict['edge1'][1]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'checkmaximumelapsedtime20',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '20',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.checkmaximumelapsedtime20.org/'}""" % (
self.surykatka_dict['edge1'][20]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'checkstatuscode',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '300',
'url': 'https://www.checkstatuscode.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'default',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.default.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'failureamount',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '10',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.failureamount.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'checkfrontendiplist',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '128.129.130.131 131.134.135.136',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.checkfrontendiplist.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'default',
{'url': 'https://www.default.org/'},
)
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'failure-amount': '2'"
self.requestEdgetestSlave(
'checkstatuscode',
{'url': 'https://www.checkstatuscode.org/', 'check-status-code': 300},
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'certificate-expiration-days': '15'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
self.requestEdgetestSlave(
'checkhttpheaderdict',
{'url': 'https://www.checkhttpheaderdict.org/',
'check-http-header-dict': {"A": "AAA"}},
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'failure-amount': '2'"
self.requestEdgetestSlave(
'checkcertificateexpirationdays',
{'url': 'https://www.checkcertificateexpirationdays.org/',
'check-certificate-expiration-days': '20'},
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
'checkmaximumelapsedtime20',
{'url': 'https://www.checkmaximumelapsedtime20.org/',
'check-maximum-elapsed-time': 20},
)
self.requestEdgetestSlave(
'checkmaximumelapsedtime1',
{'url': 'https://www.checkmaximumelapsedtime1.org/',
'check-maximum-elapsed-time': 1},
)
self.requestEdgetestSlave(
'backend-300',
{'url': 'https://www.erp5.org/', 'check-status-code': '300'},
'failureamount',
{'url': 'https://www.failureamount.org/', 'failure-amount': '10'},
)
self.requestEdgetestSlave(
'checkfrontendiplist',
{'url': 'https://www.checkfrontendiplist.org/',
'check-frontend-ip-list': ['128.129.130.131', '131.134.135.136']},
)
class TestEdgeNameserverCheckFrontendIp(
class TestEdgeNameserverListCheckFrontendIpList(
EdgeSlaveMixin, SlapOSInstanceTestCase):
expected_connection_parameter_dict = {
'active-region-list': ['1'], 'sla-computer_guid': 'local',
'sla-instance_guid': 'local-edge0'}
surykatka_dict = {
2: {'expected_ini': """[SURYKATKA]
'edge1': {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 4
SQLITE = %(db_file)s
......@@ -352,37 +512,45 @@ NAMESERVER =
URL =
https://www.erp5.com/"""}
}
}
@classmethod
def getInstanceParameterDict(cls):
return {
'nameserver': '127.0.1.1 127.0.1.2',
'check-frontend-ip': '127.0.0.1 127.0.0.2',
}
return {'_': json.dumps({
'nameserver-list': ['127.0.1.1', '127.0.1.2'],
'check-frontend-ip-list': ['127.0.0.1', '127.0.0.2'],
})}
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-promise.py',
self.assertHttpQueryPromiseContent(
'edge1',
'backend',
"'ip-list': '127.0.0.1 127.0.0.2'")
self.assertPromiseContent(
'http-query-backend-promise.py',
self.assertHttpQueryPromiseContent(
'edge1',
'backend',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
self.assertHttpQueryPromiseContent(
'edge1',
'backend',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-promise.py',
self.assertHttpQueryPromiseContent(
'edge1',
'backend',
"'certificate-expiration-days': '15'")
self.assertPromiseContent(
'http-query-backend-promise.py',
self.assertHttpQueryPromiseContent(
'edge1',
'backend',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
self.assertHttpQueryPromiseContent(
'edge1',
'backend',
"'json-file': '%s'" % (self.surykatka_dict['edge1'][2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-promise.py',
self.assertHttpQueryPromiseContent(
'edge1',
'backend',
"'failure-amount': '2'"
)
......@@ -393,426 +561,833 @@ URL =
)
class TestEdgeCheckStatusCode(EdgeSlaveMixin, SlapOSInstanceTestCase):
class TestEdgeSlaveNotJson(
EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_dict = {
2: {'expected_ini': """[SURYKATKA]
'edge1': {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 4
SQLITE = %(db_file)s
URL =
https://www.erp5.com/
https://www.erp5.org/"""}
https://www.erp5.com/"""}
}
}
@classmethod
def getInstanceParameterDict(cls):
return {
'check-status-code': '500',
}
# non-json provided in slave '_' results with damaging the cluster
# test here is to expose real problem, which has no solution for now
@unittest.expectedFailure
def test(self):
EdgeSlaveMixin.test()
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'status-code': '501'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'certificate-expiration-days': '15'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'failure-amount': '2'"
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '500'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'certificate-expiration-days': '15'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'failure-amount': '2'"
)
self.assertHttpQueryPromiseContent(
'default',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.default.org/'}""" % (
self.surykatka_dict[2]['json-file'],))
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
'default',
{'url': 'https://www.default.org/'},
)
self.requestEdgetestSlave(
'backend-501',
{'url': 'https://www.erp5.org/', 'check-status-code': '501'},
software_url = self.getSoftwareURL()
self.slap.request(
software_release=software_url,
software_type='edgetest',
partition_reference='notajson',
partition_parameter_kw={'_': 'notajson'},
shared=True
)
class TestEdgeCheckHTTPHeaderDict(EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_dict = {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 4
SQLITE = %(db_file)s
URL =
https://www.erp5.com/
https://www.erp5.org/"""}
}
class TestEdgeRegion(EdgeSlaveMixin, SlapOSInstanceTestCase):
def setUpMonitorConfigurationList(self):
self.monitor_configuration_list = [
{
'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
'text': 'testing partition 0',
'title': 'testing partition 0',
'type': 'rss',
'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region One',
'title': 'edgebot-Region One',
'type': 'rss',
'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region Three',
'title': 'edgebot-Region Three',
'type': 'rss',
'url': 'https://[%s]:9702/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region Two',
'title': 'edgebot-Region Two',
'type': 'rss',
'url': 'https://[%s]:9703/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,)
}
]
@classmethod
def getInstanceParameterDict(cls):
return {
'check-http-header-dict':
'{"B": "BBB"}',
def setUpClassParameter(cls):
cls.instance_parameter_dict = {
'region-dict': {
'Region One': {
'sla-computer_guid': 'local',
'state': 'started',
'nameserver-list': ['127.0.1.1', '127.0.1.2'],
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
},
'Region Two': {
'sla-computer_guid': 'local',
'state': 'started',
'nameserver-list': ['127.0.2.1', '127.0.2.2'],
},
'Region Three': {
'sla-computer_guid': 'local',
'state': 'started',
'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
}
}
}
cls.expected_connection_parameter_dict = {
'active-region-list': [
'Region One', 'Region Three', 'Region Two'],
'sla-computer_guid': 'local', 'sla-instance_guid': 'local-edge0'}
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'http-header-dict': '{\"A\": \"AAA\"}'")
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'certificate-expiration-days': '15'")
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'failure-amount': '2'"
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'http-header-dict': '{\"B\": \"BBB\"}'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'certificate-expiration-days': '15'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'failure-amount': '2'"
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
)
self.requestEdgetestSlave(
'backend-http-header',
{'url': 'https://www.erp5.org/', 'check-http-header-dict': '{"A": "AAA"}'},
)
class TestEdgeCheckCertificateExpirationDays(
EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_dict = {
2: {'expected_ini': """[SURYKATKA]
@classmethod
def setUpClass(cls):
cls.setUpClassParameter()
super().setUpClass()
def setUpParameter(self):
self.surykatka_dict = {
'edge1': {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 4
SQLITE = %(db_file)s
URL =
https://www.erp5.com/
https://www.erp5.org/"""}
}
@classmethod
def getInstanceParameterDict(cls):
return {
'check-certificate-expiration-days': '10',
}
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'certificate-expiration-days': '20'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'failure-amount': '2'"
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'certificate-expiration-days': '10'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'failure-amount': '2'"
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
)
self.requestEdgetestSlave(
'backend-20',
{'url': 'https://www.erp5.org/',
'check-certificate-expiration-days': '20'},
)
NAMESERVER =
127.0.1.1
127.0.1.2
class TestEdgeCheckMaximumElapsedTime(
EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_dict = {
5: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 7
SQLITE = %(db_file)s
URL =
https://www.erp5.com/"""},
20: {'expected_ini': """[SURYKATKA]
https://www.all.org/
https://www.globalcheck.org/
https://www.onetwo.org/
https://www.specificcheck.org/
https://www.specificoverride.org/"""},
},
'edge2': {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 22
TIMEOUT = 4
SQLITE = %(db_file)s
URL =
https://www.erp5.org/"""},
1: {'expected_ini': """[SURYKATKA]
https://www.all.org/
https://www.three.org/"""},
},
'edge3': {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 3
TIMEOUT = 4
SQLITE = %(db_file)s
NAMESERVER =
127.0.2.1
127.0.2.2
URL =
https://www.erp5.net/"""}
}
https://www.all.org/
https://www.onetwo.org/
https://www.parialmiss.org/
https://www.specificoverride.org/"""},
}
}
def setUp(self):
self.setUpParameter()
super().setUp()
@classmethod
def getInstanceParameterDict(cls):
return {
'check-maximum-elapsed-time': '5',
}
return {'_': json.dumps(cls.instance_parameter_dict)}
slave_parameter_dict_dict = {
'all': {
'url': 'https://www.all.org/'
},
'onetwo': {
'url': 'https://www.onetwo.org/',
'region-dict': {'Region One': {}, 'Region Two': {}}
},
'three': {
'url': 'https://www.three.org/',
'region-dict': {'Region Three': {}}
},
'missed': {
'url': 'https://www.missed.org/',
'region-dict': {'Region Non Existing': {}}
},
'partialmiss': {
'url': 'https://www.parialmiss.org/',
'region-dict': {'Region Two': {}, 'Region Non Existing': {}}
},
'specificcheck': {
'url': 'https://www.specificcheck.org/',
'region-dict': {
'Region One': {'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2']}}
},
'globalcheck': {
'url': 'https://www.globalcheck.org/',
'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
'region-dict': {'Region One': {}}
},
'specificoverride': {
'url': 'https://www.specificoverride.org/',
'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
'region-dict': {
'Region One': {'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8']},
'Region Two': {}}
},
}
def requestEdgetestSlaves(self):
for reference, parameter_dict in self.slave_parameter_dict_dict.items():
self.requestEdgetestSlave(reference, parameter_dict)
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'maximum-elapsed-time': '20'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[20]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'failure-amount': '2'"
)
self.assertHttpQueryPromiseContent(
'edge1',
'all',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '127.0.1.3 127.0.1.4',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.all.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'specificcheck',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '99.99.99.1 99.99.99.2',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.specificcheck.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'globalcheck',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '99.99.99.3 99.99.99.4',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.globalcheck.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'specificoverride',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '99.99.99.7 99.99.99.8',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.specificoverride.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'onetwo',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '127.0.1.3 127.0.1.4',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.onetwo.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge2',
'all',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '127.0.3.1 127.0.3.2',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.all.org/'}""" % (
self.surykatka_dict['edge2'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge2',
'three',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '127.0.3.1 127.0.3.2',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.three.org/'}""" % (
self.surykatka_dict['edge2'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge3',
'all',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.all.org/'}""" % (
self.surykatka_dict['edge3'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge3',
'onetwo',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.onetwo.org/'}""" % (
self.surykatka_dict['edge3'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge3',
'partialmiss',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.parialmiss.org/'}""" % (
self.surykatka_dict['edge3'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge3',
'specificoverride',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '99.99.99.5 99.99.99.6',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.specificoverride.org/'}""" % (
self.surykatka_dict['edge3'][2]['json-file'],))
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'maximum-elapsed-time': '5'")
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[5]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'failure-amount': '2'"
)
def test(self):
super(TestEdgeRegion, self).test()
self.assertSlaveConnectionParameterDict()
maxDiff = None
expected_slave_connection_parameter_dict_dict = {
'all': {
'available-region-list': [
'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Three': {
'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
'nameserver-list': []
},
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
},
'onetwo': {
'available-region-list': [
'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
},
'specificcheck': {
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
}
},
'available-region-list': [
'Region One', 'Region Three', 'Region Two']
},
'specificoverride': {
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
},
'available-region-list': [
'Region One', 'Region Three', 'Region Two']
},
'three': {
'available-region-list': [
'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region Three': {
'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
'nameserver-list': []
}
}
},
'globalcheck': {
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
}
},
'available-region-list': [
'Region One', 'Region Three', 'Region Two']
},
'missed': {
'available-region-list': [
'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
}
},
'partialmiss': {
'available-region-list': [
'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
}
}
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'maximum-elapsed-time': '1'")
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'url': 'https://www.erp5.net/'")
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[1]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'failure-amount': '2'"
def assertSlaveConnectionParameterDict(self):
slave_connection_parameter_dict_dict = {}
for reference, parameter_dict in self.slave_parameter_dict_dict.items():
slave_connection_parameter_dict_dict[
reference] = self.requestEdgetestSlave(
reference, parameter_dict).getConnectionParameterDict()
# unload the json
slave_connection_parameter_dict_dict[
reference] = json.loads(
slave_connection_parameter_dict_dict[reference].pop('_'))
self.assertEqual(
self.expected_slave_connection_parameter_dict_dict,
slave_connection_parameter_dict_dict
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend-default',
{'url': 'https://www.erp5.com/'},
)
self.requestEdgetestSlave(
'backend-20',
{'url': 'https://www.erp5.org/',
'check-maximum-elapsed-time': '20'},
)
self.requestEdgetestSlave(
'backend-1',
{'url': 'https://www.erp5.net/',
'check-maximum-elapsed-time': '1'},
)
class TestEdgeRegionDestroyed(TestEdgeRegion):
def setUpMonitorConfigurationList(self):
# already for destroyed case, as test_monitor_setup will be called after
# test
self.monitor_configuration_list = [
{
'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
'text': 'testing partition 0',
'title': 'testing partition 0',
'type': 'rss',
'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region One',
'title': 'edgebot-Region One',
'type': 'rss',
'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region Two',
'title': 'edgebot-Region Two',
'type': 'rss',
'url': 'https://[%s]:9703/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,)
}
]
class TestEdgeFailureAmount(
EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_dict = {
2: {'expected_ini': """[SURYKATKA]
def test(self):
super(TestEdgeRegionDestroyed, self).test()
# hack around @classmethod
self.__class__.instance_parameter_dict[
'region-dict']['Region Three']['state'] = 'destroyed'
# Region was removed
self.__class__.expected_connection_parameter_dict[
'active-region-list'].remove('Region Three')
self.__class__._instance_parameter_dict = self.getInstanceParameterDict()
self.requestDefaultInstance()
# give time to stabilise the tree
self.slap.waitForInstance(max_retry=4)
self.assertConnectionParameterDict()
self.expected_slave_connection_parameter_dict_dict = {
'all': {
'available-region-list': [
'Region One', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
},
'onetwo': {
'available-region-list': [
'Region One', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
},
'specificcheck': {
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
}
},
'available-region-list': [
'Region One', 'Region Two']
},
'specificoverride': {
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
},
'available-region-list': [
'Region One', 'Region Two']
},
'three': {
'assigned-region-dict': {},
'available-region-list': [
'Region One', 'Region Two'],
},
'globalcheck': {
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
}
},
'available-region-list': [
'Region One', 'Region Two']
},
'missed': {
'available-region-list': [
'Region One', 'Region Two'],
'assigned-region-dict': {
}
},
'partialmiss': {
'available-region-list': [
'Region One', 'Region Two'],
'assigned-region-dict': {
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
}
}
self.assertSlaveConnectionParameterDict()
class TestEdgeRegionAdded(TestEdgeRegion):
def setUpMonitorConfigurationList(self):
# already for added case, as test_monitor_setup will be called after test
self.monitor_configuration_list = [
{
'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
'text': 'testing partition 0',
'title': 'testing partition 0',
'type': 'rss',
'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region Four',
'title': 'edgebot-Region Four',
'type': 'rss',
'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region One',
'title': 'edgebot-Region One',
'type': 'rss',
'url': 'https://[%s]:9702/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region Three',
'title': 'edgebot-Region Three',
'type': 'rss',
'url': 'https://[%s]:9703/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9704/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region Two',
'title': 'edgebot-Region Two',
'type': 'rss',
'url': 'https://[%s]:9704/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9704/public/feed' % (self._ipv6_address,)
}
]
def test(self):
super(TestEdgeRegionAdded, self).test()
self.__class__.instance_parameter_dict['region-dict']['Region Four'] = {
'sla-computer_guid': 'local',
'state': 'started',
'nameserver-list': ['127.0.4.1', '127.0.4.2'],
'check-frontend-ip-list': ['127.0.4.3', '127.0.4.4'],
}
# Region was added
self.__class__.expected_connection_parameter_dict[
'active-region-list'].insert(0, 'Region Four')
self.__class__._instance_parameter_dict = self.getInstanceParameterDict()
self.requestDefaultInstance()
# give time to stabilise the tree, 6 times as new node is added
self.slap.waitForInstance(max_retry=6)
# XXX: few more times, but ignoring good result from promises, as there is
# "Unknown Instance" of just added node which is not caught by any
# promise, but in the end it shall get better
for f in range(5):
try:
self.slap.waitForInstance()
except Exception:
pass
self.slap.waitForInstance()
self.assertConnectionParameterDict()
self.expected_slave_connection_parameter_dict_dict = {
'all': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region Four': {
'check-frontend-ip-list': ['127.0.4.3', '127.0.4.4'],
'nameserver-list': ['127.0.4.1', '127.0.4.2']
},
'Region One': {
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Three': {
'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
'nameserver-list': []
},
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
},
'onetwo': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
},
'specificcheck': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
}
},
},
'specificoverride': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
},
},
'three': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region Three': {
'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
'nameserver-list': []
}
}
},
'globalcheck': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
}
},
},
'missed': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
}
},
'partialmiss': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
}
}
self.assertSlaveConnectionParameterDict()
self.surykatka_dict['edge4'] = {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 4
SQLITE = %(db_file)s
URL =
https://www.erp5.com/
https://www.erp5.org/"""}
}
SQLITE = %(dbfile)
NAMESERVER =
127.0.4.1
127.0.4.2
@classmethod
def getInstanceParameterDict(cls):
return {
'failure-amount': '5'
URL =
https://www.all.org/"""},
}
self.updateSurykatkaDict()
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'failure-amount': '5'"
)
self.assertPromiseContent(
'http-query-backend-10-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-10-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-10-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-10-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-10-promise.py',
"'failure-amount': '10'"
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
)
self.requestEdgetestSlave(
'backend-10',
{'url': 'https://www.erp5.org/', 'failure-amount': '10'},
)
self.assertHttpQueryPromiseContent(
'edge4',
'all',
"""{ 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '127.0.4.3 127.0.4.4',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.all.org/'}""" % (
self.surykatka_dict['edge4'][2]['json-file'],))
......@@ -18,7 +18,7 @@ md5sum = 2bd1779425b7561682c0de5496d808ed
[root-common]
filename = root-common.cfg.in
md5sum = 8e28f599247ad604ec6e32df410412a8
md5sum = c13b4f1a5aa526a8d3f8e02bf6baf785
[instance-neo-admin]
filename = instance-neo-admin.cfg.in
......
......@@ -47,7 +47,7 @@
"type": "object"
},
"sla-dict": {
"description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...). The prefix 'node-' is mandatory and the number must start from 0. The total number of nodes here must be equal to the length of node-list.",
"description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...). The prefix 'node-' is mandatory and the number must start from 0. The total number of nodes here must be equal to the length of node-list. A node can be removed by requesting it without any admin, master and storage.",
"additionalProperties": {
"type": "array",
"items": {
......@@ -95,7 +95,7 @@
"type": "integer"
},
"storage-count": {
"description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage.",
"description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage. 0 to disable.",
"default": 1,
"type": "integer"
},
......
......@@ -89,7 +89,7 @@ software-type = {{ software_type }}
config-autostart = {{ dumps(sum(storage_count)) }}
{%- do assert(replicas < len(node_list)) %}
{%- set monitor = set() %}
{%- for i, node in enumerate(node_list) %}
{%- for node in node_list %}
{%- set port = node.get('monitor') %}
{%- if port %}
{%- do monitor.add(node.get('admin') != 0 and port) %}
......@@ -101,13 +101,14 @@ config-autostart = {{ dumps(sum(storage_count)) }}
{%- endif %}
{%- for i, node in enumerate(node_list) %}
{%- set section_id = prefix ~ i %}
{%- do section_id_list.append(section_id) %}
{%- if node.get('master') == 0 and not node.get('monitor') %}
{%- do node.setdefault('admin', 0) %}
{%- endif %}
{%- if monitor or node.get('admin') == 0 %}
{%- do node.setdefault('monitor', 0) %}
{%- endif %}
{%- for x in 'admin', 'master', 'storage-count' if node.get(x, 1) %}
{%- do section_id_list.append(section_id) %}
[{{section_id}}]
<= {{ prefix }}request-common
......@@ -136,6 +137,9 @@ config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
config-{{ k }} = {{ dumps(v) }}
{%- endfor %}
{{ sla(section_id) }}
{%- break %}
{%- endfor %}
{%- endfor %}
{%- do assert(len(monitor) == 1, monitor) %}
......
......@@ -31,3 +31,7 @@ if returncode:
subprocess.call(
('tar', '-caf', result('db.tar.xz'), 'neo_tests'),
cwd=tempfile.gettempdir())
p = subprocess.Popen(('journalctl', '-o', 'export'), stdout=subprocess.PIPE)
with open(result('journalctl-export.xz'), 'wb') as f:
subprocess.call(('xz',), stdin=p.stdout, stdout=f)
p.wait()
......@@ -15,7 +15,7 @@
[instance]
filename = instance.cfg.in
md5sum = f3aac995aa1129ca58f9a92851ad7091
md5sum = bfd488ba023f505be25d947ec830bab3
[yarn.lock]
filename = yarn.lock
......
......@@ -7,7 +7,7 @@ theia-environment-parts =
slapos-repository
runner-link
settings.json
request-script
request-script-template
theia-parts =
frontend-reload
......@@ -230,7 +230,7 @@ recipe = plone.recipe.command
filename = logo.png
full-path = $${directory:frontend-static}/$${:filename}
command =
cp -f ${logo.png:output} $${:full-path}
cp --remove-destination ${logo.png:output} $${:full-path}
stop-on-error = true
[frontend-instance-slapos.css]
......@@ -551,17 +551,19 @@ recipe = slapos.cookbook:symbolic.link
target-directory = $${directory:project}
link-binary = $${directory:runner}
[request-script]
[request-script-template]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:project}/$${:_buildout_section_name_}.sh
mode = 0700
template =
inline:#!/bin/sh
software_name=html5as-base #replace the software name
# This template is generated automatically, copy it in order to supply and request.
# Any manual change to this file may be lost.
software_name=html5as-base #replace the software name writen in ~/srv/project/slapos/software/
software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg
# slapos supply is used to add the software to the software list to be supplied to a node.
slapos supply $software_release_uri slaprunner
# slapos request the allocation of an instance to the master.
# slapos request also gets status and parameters of the instance if it has any
# slapos request the allocation of an instance to the master.
# slapos request also gets status and parameters of the instance if it has any
# (slapos request is meant to be run multiple time until you get the status).
slapos request $software_name'_1' $software_release_uri
......@@ -183,7 +183,7 @@ class TestTheia(TheiaTestCase):
self.computer_partition_root_path,
'srv',
'project',
'request-script.sh',
'request-script-template.sh',
)
self.assertTrue(os.path.exists(script_path))
......
......@@ -631,9 +631,6 @@ PasteDeploy = 1.5.2
argparse = 1.4.0
zope.dottedname = 4.1.0
# test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
Products.PluggableAuthService = 1.9.0
# modified version that works fine for buildout installation
SOAPpy = 0.12.0nxd001
......@@ -664,6 +661,7 @@ Products.GenericSetup = 1.8.6
Products.LongRequestLogger = 2.1.0
# Products.MimetypesRegistry 2.1 requires AccessControl>=3.0.0Acquisition.
Products.MimetypesRegistry = 2.0.10
Products.PluggableAuthService = 1.10.0
Products.PluginRegistry = 1.4
Products.TIDStorage = 5.5.0
pyPdf = 1.13
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment