Commit 64f10409 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents ad5fe700 7ae4d31a
...@@ -14,7 +14,7 @@ recipe = slapos.recipe.build:gitclone ...@@ -14,7 +14,7 @@ recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/cloudooo.git repository = https://lab.nexedi.com/nexedi/cloudooo.git
branch = master branch = master
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
revision = be6c35c0156e028f31da3ccb205afc6f95728d97 revision = 0b5ff71a2ede76499e81659aed392057ae910917
[cloudooo] [cloudooo]
recipe = zc.recipe.egg recipe = zc.recipe.egg
......
...@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce ...@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
[template-trafficserver-records-config] [template-trafficserver-records-config]
_update_hash_filename_ = templates/trafficserver/records.config.jinja2 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
md5sum = b99403e02d1aff471a7d5ebd0afbdb6c md5sum = 1696321871bb71083550c95d77487602
[template-trafficserver-storage-config] [template-trafficserver-storage-config]
_update_hash_filename_ = templates/trafficserver/storage.config.jinja2 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2
......
...@@ -80,7 +80,7 @@ CONFIG proxy.config.http.uncacheable_requests_bypass_parent INT 1 ...@@ -80,7 +80,7 @@ CONFIG proxy.config.http.uncacheable_requests_bypass_parent INT 1
CONFIG proxy.config.http.keep_alive_no_activity_timeout_in INT 120 CONFIG proxy.config.http.keep_alive_no_activity_timeout_in INT 120
CONFIG proxy.config.http.keep_alive_no_activity_timeout_out INT 120 CONFIG proxy.config.http.keep_alive_no_activity_timeout_out INT 120
CONFIG proxy.config.http.transaction_no_activity_timeout_in INT {{ ats_configuration['request-timeout'] }} CONFIG proxy.config.http.transaction_no_activity_timeout_in INT {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.transaction_no_activity_timeout_out INT 30 {{ ats_configuration['request-timeout'] }} CONFIG proxy.config.http.transaction_no_activity_timeout_out INT {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.transaction_active_timeout_in INT 900 CONFIG proxy.config.http.transaction_active_timeout_in INT 900
CONFIG proxy.config.http.transaction_active_timeout_out INT 0 CONFIG proxy.config.http.transaction_active_timeout_out INT 0
CONFIG proxy.config.http.accept_no_activity_timeout INT 120 CONFIG proxy.config.http.accept_no_activity_timeout INT 120
......
...@@ -18,4 +18,4 @@ md5sum = e986de01a57161b32425f1cd3ccac924 ...@@ -18,4 +18,4 @@ md5sum = e986de01a57161b32425f1cd3ccac924
[template-cloudooo-instance] [template-cloudooo-instance]
filename = instance-cloudooo.cfg.in filename = instance-cloudooo.cfg.in
md5sum = 9e1a66cf18d7c30c14afeb66c20afb46 md5sum = 440f2b82b119cbfa6f8c7d27652c3170
...@@ -162,6 +162,7 @@ environment = ...@@ -162,6 +162,7 @@ environment =
LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['gcc'] }}/lib:{{ parameter_dict['gcc'] }}/lib64:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['gcc'] }}/lib:{{ parameter_dict['gcc'] }}/lib64:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
FONTCONFIG_FILE = ${fontconfig-conf:rendered} FONTCONFIG_FILE = ${fontconfig-conf:rendered}
PATH = ${binary-link:target-directory} PATH = ${binary-link:target-directory}
LANG = C.UTF-8
mimetype_entry_addition = mimetype_entry_addition =
{% for entry in mimetype_entry_addition.splitlines() -%} {% for entry in mimetype_entry_addition.splitlines() -%}
{{ " " ~ entry.strip() }} {{ " " ~ entry.strip() }}
...@@ -172,7 +173,7 @@ ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program ...@@ -172,7 +173,7 @@ ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program
ooo-paster = {{ bin_directory }}/cloudooo_paster ooo-paster = {{ bin_directory }}/cloudooo_paster
ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
{% for index in range(backend_count) -%} {% for index in range(1, backend_count + 1) -%}
{% set name = 'cloudooo-' ~ index -%} {% set name = 'cloudooo-' ~ index -%}
[{{ cloudooo(name) }}] [{{ cloudooo(name) }}]
< = cloudooo-base < = cloudooo-base
...@@ -207,8 +208,8 @@ prepend-path = ${buildout:bin-directory} ...@@ -207,8 +208,8 @@ prepend-path = ${buildout:bin-directory}
run-unit-test = ${buildout:bin-directory}/runUnitTest run-unit-test = ${buildout:bin-directory}/runUnitTest
run-test-suite = ${buildout:bin-directory}/runTestSuite run-test-suite = ${buildout:bin-directory}/runTestSuite
ooo-paster = ${cloudooo-0:ooo-paster} ooo-paster = ${cloudooo-1:ooo-paster}
configuration-file = ${cloudooo-0:configuration-file} configuration-file = ${cloudooo-1:configuration-file}
run-unit-test-binary = {{ bin_directory }}/runCloudoooUnitTest run-unit-test-binary = {{ bin_directory }}/runCloudoooUnitTest
run-test-suite-binary = {{ bin_directory }}/runCloudoooTestSuite run-test-suite-binary = {{ bin_directory }}/runCloudoooTestSuite
......
...@@ -44,6 +44,7 @@ setup(name=name, ...@@ -44,6 +44,7 @@ setup(name=name,
'slapos.core', 'slapos.core',
'slapos.cookbook', 'slapos.cookbook',
'slapos.libnetworkcache', 'slapos.libnetworkcache',
'requests',
'six', 'six',
'PyPDF2', 'PyPDF2',
], ],
......
...@@ -26,23 +26,43 @@ ...@@ -26,23 +26,43 @@
# #
############################################################################## ##############################################################################
import codecs
import csv
import multiprocessing
import os import os
import json import json
import six.moves.xmlrpc_client as xmlrpclib import six.moves.xmlrpc_client as xmlrpclib
import six.moves.urllib.parse as urllib_parse
import ssl import ssl
import base64 import base64
import io import io
import requests
import PyPDF2 import PyPDF2
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, CloudOooTestCase = makeModuleSetUpAndTestCaseClass( setUpModule, _CloudOooTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath( os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))) os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
# Cloudooo needs a lot of time before being available.
CloudOooTestCase.instance_max_retry = 30 class CloudOooTestCase(_CloudOooTestCase):
# Cloudooo needs a lot of time before being available.
instance_max_retry = 30
def setUp(self):
self.url = json.loads(
self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
# XXX ignore certificate errors
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
self.server = xmlrpclib.ServerProxy(
self.url,
context=ssl_context,
allow_none=True,
)
def normalizeFontName(font_name): def normalizeFontName(font_name):
...@@ -88,19 +108,6 @@ class HTMLtoPDFConversionFontTestMixin: ...@@ -88,19 +108,6 @@ class HTMLtoPDFConversionFontTestMixin:
"""Convert the HTML source to pdf bytes. """Convert the HTML source to pdf bytes.
""" """
def setUp(self):
self.url = json.loads(
self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
# XXX ignore certificate errors
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
self.server = xmlrpclib.ServerProxy(
self.url,
context=ssl_context,
allow_none=True,
)
def test(self): def test(self):
actual_font_mapping_mapping = {} actual_font_mapping_mapping = {}
for font in self.expected_font_mapping: for font in self.expected_font_mapping:
...@@ -237,3 +244,84 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase): ...@@ -237,3 +244,84 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
'html', 'html',
'pdf', 'pdf',
).encode()) ).encode())
class TestLibreOfficeTextConversion(CloudOooTestCase):
__partition_reference__ = 'txt'
def test_html_to_text(self):
self.assertEqual(
base64.decodestring(
self.server.convertFile(
base64.encodestring(
u'<html>héhé</html>'.encode('utf-8')).decode(),
'html',
'txt',
).encode()),
codecs.BOM_UTF8 + b'h\xc3\xa9h\xc3\xa9\n',
)
class TestLibreOfficeCluster(CloudOooTestCase):
__partition_reference__ = 'lc'
@classmethod
def getInstanceParameterDict(cls):
return {'backend-count': 4}
def test_multiple_conversions(self):
# make this function global so that it can be picked and used by multiprocessing
global _convert_html_to_text
def _convert_html_to_text(src_html):
return base64.decodestring(
self.server.convertFile(
base64.encodestring(src_html.encode()).decode(),
'html',
'txt',
).encode())
pool = multiprocessing.Pool(5)
# TODO py3: use with pool
converted = pool.map(_convert_html_to_text,
['<html><body>hello</body></html>'] * 100)
pool.terminate()
pool.join()
self.assertEqual(converted, [codecs.BOM_UTF8 + b'hello\n'] * 100)
# haproxy stats are exposed
res = requests.get(
urllib_parse.urljoin(self.url, '/haproxy;csv'),
verify=False,
stream=True,
)
reader = csv.DictReader(res.raw)
line_list = list(reader)
# requests have been balanced
total_hrsp_2xx = {
line['svname']: int(line['hrsp_2xx'])
for line in line_list
}
self.assertEqual(total_hrsp_2xx['FRONTEND'], 100)
self.assertEqual(total_hrsp_2xx['BACKEND'], 100)
for backend in 'cloudooo_1', 'cloudooo_2', 'cloudooo_3', 'cloudooo_4':
# ideally there should be 25% of requests on each backend, because we use
# round robin scheduling, but it can happen that some backend take longer
# to start, so we are tolerant here and just check that each backend
# process at least 15% of requests.
self.assertGreater(total_hrsp_2xx[backend], 15)
# no errors
total_eresp = {
line['svname']: int(line['eresp'] or 0)
for line in line_list
}
self.assertEqual(
total_eresp, {
'FRONTEND': 0,
'cloudooo_1': 0,
'cloudooo_2': 0,
'cloudooo_3': 0,
'cloudooo_4': 0,
'BACKEND': 0,
})
...@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70 ...@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
[template-kvm] [template-kvm]
filename = instance-kvm.cfg.jinja2 filename = instance-kvm.cfg.jinja2
md5sum = 854ec0f379a05246f5f3761fdb634c36 md5sum = bf0c01ac7493693bb57ebef00bb20fa0
[template-kvm-cluster] [template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in filename = instance-kvm-cluster.cfg.jinja2.in
...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257 ...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run] [template-kvm-run]
filename = template/template-kvm-run.in filename = template/template-kvm-run.in
md5sum = a8afeb2f80199368e291621576a80ca1 md5sum = b8cc7c76438212e0522ebede88649393
[template-kvm-controller] [template-kvm-controller]
filename = template/kvm-controller-run.in filename = template/kvm-controller-run.in
......
...@@ -359,7 +359,7 @@ ...@@ -359,7 +359,7 @@
}, },
"virtual-hard-drive-md5sum": { "virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive", "title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.", "description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
"type": "string" "type": "string"
}, },
"virtual-hard-drive-gzipped": { "virtual-hard-drive-gzipped": {
......
...@@ -160,7 +160,7 @@ ...@@ -160,7 +160,7 @@
}, },
"virtual-hard-drive-md5sum": { "virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive", "title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.", "description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
"type": "string" "type": "string"
}, },
"virtual-hard-drive-gzipped": { "virtual-hard-drive-gzipped": {
......
...@@ -17,6 +17,8 @@ ...@@ -17,6 +17,8 @@
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%} {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%} {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%} {% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
{% set virtual_hard_drive_url_enabled = 'virtual-hard-drive-url' in slapparameter_dict %}
{% set virtual_hard_drive_url_gzipped = str(slapparameter_dict.get('virtual-hard-drive-gzipped', False)).lower() == 'true' %}
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %} {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %} {% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%} {% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%}
...@@ -56,6 +58,11 @@ public = ${:srv}/public/ ...@@ -56,6 +58,11 @@ public = ${:srv}/public/
cron-entries = ${:etc}/cron.d cron-entries = ${:etc}/cron.d
crontabs = ${:etc}/crontabs crontabs = ${:etc}/crontabs
cronstamps = ${:etc}/cronstamps cronstamps = ${:etc}/cronstamps
{%- if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-repository = ${:srv}/virtual-hard-drive-url-repository
virtual-hard-drive-url-var = ${:var}/virtual-hard-drive-url
virtual-hard-drive-url-expose = ${monitor-directory:private}/virtual-hard-drive-url
{%- endif %}
{%- if boot_image_url_list_enabled %} {%- if boot_image_url_list_enabled %}
boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
boot-image-url-list-var = ${:var}/boot-image-url-list boot-image-url-list-var = ${:var}/boot-image-url-list
...@@ -278,6 +285,106 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file} ...@@ -278,6 +285,106 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
## boot-image-url-list support END ## boot-image-url-list support END
{% endif %} {# if boot_image_url_list_enabled #} {% endif %} {# if boot_image_url_list_enabled #}
{% if virtual_hard_drive_url_enabled %}
## virtual-hard-drive-url support BEGIN
[empty-file-state-base-virtual-promise]
<= monitor-promise-base
module = check_file_state
name = ${:_buildout_section_name_}.py
config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/virtual-hard-drive-url/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[virtual-hard-drive-url-source-config]
recipe = slapos.recipe.template:jinja2
template = inline:
{%- raw %}
{{ virtual_hard_drive_url }}
{% endraw -%}
{# Enforce md5sum on virtual-hard-drive-url #}
virtual-hard-drive-url = {{ slapparameter_dict['virtual-hard-drive-url'] }}#{{ slapparameter_dict['virtual-hard-drive-md5sum'] }}
context =
key virtual_hard_drive_url :virtual-hard-drive-url
rendered = ${directory:etc}/virtual-hard-drive-url.conf
[virtual-hard-drive-url-processed-config]
# compares if the current configuration has been used by
# the virtual-hard-drive-url-download, if not, exposes it as not empty file with
# information
recipe = slapos.recipe.build
install =
import os
import hashlib
if not os.path.exists(location):
os.mkdir(location)
with open('${:state-file}', 'w') as state_handler:
try:
with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
processed_md5sum = processed_handler.read()
if config_md5sum == processed_md5sum:
state_handler.write('')
else:
state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
except Exception as e:
state_handler.write(str(e))
update = ${:install}
config-file = ${virtual-hard-drive-url-source-config:rendered}
state-filename = virtual-hard-drive-url-processed-config.state
state-file = ${directory:virtual-hard-drive-url-expose}/${:state-filename}
processed-md5sum = ${directory:virtual-hard-drive-url-var}/update-image-processed.md5sum
[virtual-hard-drive-url-processed-config-promise]
# promise to check if the configuration provided by the user has been already
# processed by the virtual-hard-drive-url-download script, which runs asynchronously
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-processed-config:state-filename}
config-filename = ${virtual-hard-drive-url-processed-config:state-file}
[virtual-hard-drive-url-json-config]
# generates json configuration from user configuration
recipe = plone.recipe.command
command = {{ python_executable }} {{ image_download_config_creator }} ${virtual-hard-drive-url-source-config:rendered} ${:rendered} ${directory:virtual-hard-drive-url-repository} ${:error-state-file}
update-command = ${:command}
rendered = ${directory:virtual-hard-drive-url-var}/virtual-hard-drive-url.json
error-state-filename = virtual-hard-drive-url-json-config-error.txt
error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
[virtual-hard-drive-url-config-state-promise]
# promise to check if configuration has been parsed without errors
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-json-config:error-state-filename}
config-filename = ${virtual-hard-drive-url-json-config:error-state-file}
[virtual-hard-drive-url-download-wrapper]
# wrapper to execute virtual-hard-drive-url-download on each run
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/virtual-hard-drive-url-updater
command-line = {{ python_executable }} {{ image_download_controller }} ${virtual-hard-drive-url-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${virtual-hard-drive-url-processed-config:processed-md5sum}
md5sum-state-filename = virtual-hard-drive-url-download-controller-md5sum-fail.json
md5sum-state-file = ${directory:virtual-hard-drive-url-expose}/${:md5sum-state-filename}
error-state-filename = virtual-hard-drive-url-download-controller-error.text
error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[virtual-hard-drive-url-download-md5sum-promise]
# promise to report errors with problems with calculating md5sum of the
# downloaded images
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-filename}
config-filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-file}
[virtual-hard-drive-url-download-state-promise]
# promise to report errors during download
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-download-wrapper:error-state-filename}
config-filename = ${virtual-hard-drive-url-download-wrapper:error-state-file}
## virtual-hard-drive-url support END
{% endif %} {# if virtual_hard_drive_url_enabled #}
[kvm-controller-parameter-dict] [kvm-controller-parameter-dict]
python-path = {{ python_eggs_executable }} python-path = {{ python_eggs_executable }}
vnc-passwd = ${gen-passwd:passwd} vnc-passwd = ${gen-passwd:passwd}
...@@ -298,6 +405,11 @@ vnc-ip = ${:ipv4} ...@@ -298,6 +405,11 @@ vnc-ip = ${:ipv4}
vnc-port = 5901 vnc-port = 5901
default-cdrom-iso = {{ debian_amd64_netinst_location }} default-cdrom-iso = {{ debian_amd64_netinst_location }}
{% if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-json-config = ${virtual-hard-drive-url-json-config:rendered}
{% else %}
virtual-hard-drive-url-json-config =
{% endif %}
{% if boot_image_url_list_enabled %} {% if boot_image_url_list_enabled %}
boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered} boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
{% else %} {% else %}
...@@ -425,14 +537,42 @@ ipv6-port = {{ external_port }} ...@@ -425,14 +537,42 @@ ipv6-port = {{ external_port }}
{% endfor -%} {% endfor -%}
{% endif -%} {% endif -%}
{%- set depend_section_list = [] %}
{%- set hash_file_list = ['${kvm-run:rendered}'] %}
{%- macro generate_depend_section(section, key) %}
{%- do depend_section_list.append('${' + section + ':command}' ) %}
{%- do hash_file_list.append('${' + key + '}') %}
[{{ section }}]
recipe = plone.recipe.command
update-command = ${:command}
command = [ ! -f {{ '${' + key + '}' }} ] && touch {{ '${' + key + '}' }}
{%- endmacro %}
{#- Create depending sections, as state files appear late, so it's better to have empty file which will impact the hash anyway #}
{%- if boot_image_url_list_enabled %}
{{ generate_depend_section('boot-image-url-list-depend', 'boot-image-url-list-download-wrapper:md5sum-state-file') }}
{%- endif %}
{%- if boot_image_url_select_enabled %}
{{ generate_depend_section('boot-image-url-select-depend', 'boot-image-url-select-download-wrapper:md5sum-state-file') }}
{%- endif %}
{%- if virtual_hard_drive_url_enabled %}
{{ generate_depend_section('virtual-hard-drive-url-depend', 'virtual-hard-drive-url-download-wrapper:md5sum-state-file') }}
{%- endif %}
[kvm-instance] [kvm-instance]
depends =
{%- for depend_section in depend_section_list %}
{{ depend_section }}
{%- endfor %}
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
socket-path = ${kvm-controller-parameter-dict:socket-path} socket-path = ${kvm-controller-parameter-dict:socket-path}
wrapper-path = ${directory:services}/kvm wrapper-path = ${directory:services}/kvm
command-line = ${kvm-run:rendered} command-line = ${kvm-run:rendered}
kvm-controller = ${kvm-controller-wrapper:wrapper-path} kvm-controller = ${kvm-controller-wrapper:wrapper-path}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
hash-files =
{%- for hash_file in hash_file_list %}
{{ hash_file }}
{%- endfor %}
[kvm-controller-wrapper] [kvm-controller-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
...@@ -932,8 +1072,8 @@ disk-format = qcow2 ...@@ -932,8 +1072,8 @@ disk-format = qcow2
disk-device-path = disk-device-path =
cpu-count = 1 cpu-count = 1
disk-cache = writeback disk-cache =
disk-aio = native disk-aio =
auto-ballooning = True auto-ballooning = True
machine-options = machine-options =
cpu-model = host cpu-model = host
...@@ -1108,6 +1248,13 @@ parts = ...@@ -1108,6 +1248,13 @@ parts =
cron-service cron-service
cron-entry-logrotate cron-entry-logrotate
frontend-promise frontend-promise
{% if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-download-wrapper
virtual-hard-drive-url-config-state-promise
virtual-hard-drive-url-download-md5sum-promise
virtual-hard-drive-url-download-state-promise
virtual-hard-drive-url-processed-config-promise
{% endif %}
{% if boot_image_url_list_enabled %} {% if boot_image_url_list_enabled %}
boot-image-url-list-download-wrapper boot-image-url-list-download-wrapper
boot-image-url-list-config-state-promise boot-image-url-list-config-state-promise
......
...@@ -6,10 +6,6 @@ import hashlib ...@@ -6,10 +6,6 @@ import hashlib
import os import os
import socket import socket
import subprocess import subprocess
try:
from urllib.request import FancyURLopener
except ImportError:
from urllib import FancyURLopener
import gzip import gzip
import shutil import shutil
from random import shuffle from random import shuffle
...@@ -17,16 +13,11 @@ import glob ...@@ -17,16 +13,11 @@ import glob
import re import re
import json import json
import ssl
# XXX: give all of this through parameter, don't use this as template, but as module # XXX: give all of this through parameter, don't use this as template, but as module
qemu_img_path = '{{ parameter_dict.get("qemu-img-path") }}' qemu_img_path = {{ repr(parameter_dict["qemu-img-path"]) }}
qemu_path = '{{ parameter_dict.get("qemu-path") }}' qemu_path = {{ repr(parameter_dict["qemu-path"]) }}
disk_size = '{{ parameter_dict.get("disk-size") }}' disk_size = {{ repr(parameter_dict["disk-size"]) }}
disk_type = '{{ parameter_dict.get("disk-type") }}' disk_type = {{ repr(parameter_dict["disk-type"]) }}
disk_format = '{{ parameter_dict.get("disk-format", "qcow2")}}'
disk_format = disk_format \
if disk_format in ['qcow2', 'raw', 'vdi', 'vmdk', 'cloop', 'qed'] else 'qcow2'
socket_path = '{{ parameter_dict.get("socket-path") }}' socket_path = '{{ parameter_dict.get("socket-path") }}'
nbd_list = (('{{ parameter_dict.get("nbd-host") }}', nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
...@@ -34,11 +25,7 @@ nbd_list = (('{{ parameter_dict.get("nbd-host") }}', ...@@ -34,11 +25,7 @@ nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
('{{ parameter_dict.get("nbd2-host") }}', ('{{ parameter_dict.get("nbd2-host") }}',
{{ parameter_dict.get("nbd2-port") }})) {{ parameter_dict.get("nbd2-port") }}))
default_cdrom_iso = '{{ parameter_dict.get("default-cdrom-iso") }}' default_cdrom_iso = '{{ parameter_dict.get("default-cdrom-iso") }}'
disk_path = '{{ parameter_dict.get("disk-path") }}'
virtual_hard_drive_url = '{{ parameter_dict.get("virtual-hard-drive-url") }}'.strip()
virtual_hard_drive_md5sum = '{{ parameter_dict.get("virtual-hard-drive-md5sum") }}'.strip()
virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
nat_rules = '{{ parameter_dict.get("nat-rules") }}'.strip() nat_rules = '{{ parameter_dict.get("nat-rules") }}'.strip()
use_tap = '{{ parameter_dict.get("use-tap") }}'.lower() use_tap = '{{ parameter_dict.get("use-tap") }}'.lower()
use_nat = '{{ parameter_dict.get("use-nat") }}'.lower() use_nat = '{{ parameter_dict.get("use-nat") }}'.lower()
...@@ -56,9 +43,7 @@ init_ram_size = {{ parameter_dict.get("init-ram-size") }} ...@@ -56,9 +43,7 @@ init_ram_size = {{ parameter_dict.get("init-ram-size") }}
pid_file_path = '{{ parameter_dict.get("pid-file-path") }}' pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
external_disk_number = {{ parameter_dict.get("external-disk-number") }} external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = {{ parameter_dict.get("external-disk-size") }} external_disk_size = {{ parameter_dict.get("external-disk-size") }}
external_disk_format = '{{ parameter_dict.get("external-disk-format", "qcow2") }}' external_disk_format = {{ repr(parameter_dict["external-disk-format"]) }}
external_disk_format = external_disk_format \
if external_disk_format in ['qcow2', 'raw', 'vdi', 'vmdk', 'cloop', 'qed'] else 'qcow2'
disk_storage_dict = {} disk_storage_dict = {}
disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n') disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
map_storage_list = [] map_storage_list = []
...@@ -70,11 +55,6 @@ cluster_doc_port = {{ parameter_dict.get("cluster-doc-port") }} ...@@ -70,11 +55,6 @@ cluster_doc_port = {{ parameter_dict.get("cluster-doc-port") }}
url_check_certificate = '{{ parameter_dict.get("hard-drive-url-check-certificate", "true") }}'.lower() url_check_certificate = '{{ parameter_dict.get("hard-drive-url-check-certificate", "true") }}'.lower()
auto_ballooning = '{{ parameter_dict.get("auto-ballooning") }}' in ('true', 'True', '1') auto_ballooning = '{{ parameter_dict.get("auto-ballooning") }}' in ('true', 'True', '1')
vm_name = '{{ parameter_dict.get("name") }}' vm_name = '{{ parameter_dict.get("name") }}'
disk_cache = '{{ parameter_dict.get("disk-cache", "writeback") }}'.strip()
disk_cache = disk_cache if disk_cache in ["none", "writeback", "unsafe",
"directsync", "writethrough"] else "writeback"
disk_aio = '{{ parameter_dict.get("disk-aio", "threads") }}'.strip()
disk_aio = disk_aio if disk_aio in ["threads", "native"] else "threads"
# If a device (ie.: /dev/sdb) is provided, use it instead # If a device (ie.: /dev/sdb) is provided, use it instead
# the disk_path with disk_format # the disk_path with disk_format
...@@ -84,16 +64,18 @@ for disk_device_path in '{{ parameter_dict.get("disk-device-path", "") }}'.split ...@@ -84,16 +64,18 @@ for disk_device_path in '{{ parameter_dict.get("disk-device-path", "") }}'.split
disk_info_list.append({ disk_info_list.append({
'path': disk_device_path, 'path': disk_device_path,
'format': "raw", 'format': "raw",
'io': "native", 'aio': "native",
'cache': "none" 'cache': "none"
}) })
if len(disk_info_list) == 0: if not disk_info_list:
disk_info_list.append({ disk_info_list.append({
'path': disk_path, {%- for k in 'path', 'format', 'aio', 'cache' %}
'format': disk_format, {%- set v = parameter_dict['disk-' + k] %}
'io': disk_aio, {%- if v %}
'cache': disk_cache, {{ repr(k) }}: {{ repr(v) }},
{%- endif %}
{%- endfor %}
}) })
smp_count = {{ parameter_dict.get("smp-count") }} smp_count = {{ parameter_dict.get("smp-count") }}
...@@ -107,11 +89,8 @@ logfile = '{{ parameter_dict.get("log-file") }}' ...@@ -107,11 +89,8 @@ logfile = '{{ parameter_dict.get("log-file") }}'
boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}' boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}' boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
virtual_hard_drive_url_json_config = '{{ parameter_dict.get("virtual-hard-drive-url-json-config") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false': virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
opener = FancyURLopener(context=ssl._create_unverified_context())
else:
opener = FancyURLopener({})
def md5Checksum(file_path): def md5Checksum(file_path):
with open(file_path, 'rb') as fh: with open(file_path, 'rb') as fh:
...@@ -172,39 +151,34 @@ def getMapStorageList(disk_storage_dict, external_disk_number): ...@@ -172,39 +151,34 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
lf.write('%s' % external_disk_number) lf.write('%s' % external_disk_number)
return id_list, external_disk_number return id_list, external_disk_number
# Download existing hard drive if needed at first boot # Use downloaded virtual-hard-drive-url
if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url != '': if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url_json_config != '':
print('Downloading virtual hard drive...') print('Using virtual hard drive...')
try: with open(virtual_hard_drive_url_json_config) as fh:
downloaded_disk = disk_info_list[0]['path'] image_config = json.load(fh)
if image_config['error-amount'] == 0:
image = image_config['image-list'][0]
downloaded_image = os.path.join(image_config['destination-directory'], image['destination'])
# previous version was using disk in place, but here it would result with
# redownload, so copy it
if virtual_hard_drive_gzipped == 'true': if virtual_hard_drive_gzipped == 'true':
downloaded_disk = '%s.gz' % disk_info_list[0]['path'] try:
opener.retrieve(virtual_hard_drive_url, downloaded_disk) with open(disk_info_list[0]['path'], 'wb') as d_fh:
except: with gzip.open(downloaded_image, 'rb') as s_fh:
if os.path.exists(downloaded_disk): shutil.copyfileobj(s_fh, d_fh)
os.remove(downloaded_disk) except Exception:
if os.path.exists(disk_info_list[0]['path']):
os.unlink(disk_info_list[0]['path'])
raise raise
md5sum = virtual_hard_drive_md5sum.strip()
if md5sum:
print('Checking MD5 checksum...')
local_md5sum = md5Checksum(downloaded_disk)
if local_md5sum != md5sum:
os.remove(downloaded_disk)
raise Exception('MD5 mismatch. MD5 of local file is %s, Specified MD5 is %s.' % (
local_md5sum, md5sum))
print('MD5sum check passed.')
else: else:
print('Warning: not checksum specified.')
if downloaded_disk.endswith('.gz'):
try: try:
with open(disk_info_list[0]['path'], 'w') as disk: shutil.copyfile(downloaded_image, disk_info_list[0]['path'])
with gzip.open(downloaded_disk, 'rb') as disk_gz:
shutil.copyfileobj(disk_gz, disk)
except Exception: except Exception:
if os.path.exists(disk_info_list[0]['path']): if os.path.exists(disk_info_list[0]['path']):
os.remove(disk_info_list[0]['path']) os.unlink(disk_info_list[0]['path'])
raise raise
os.remove(downloaded_disk) else:
raise ValueError('virtual-hard-drive-url not ready yet')
# Create disk if doesn't exist # Create disk if doesn't exist
# XXX: move to Buildout profile # XXX: move to Buildout profile
...@@ -310,17 +284,12 @@ kvm_argument_list = [qemu_path, ...@@ -310,17 +284,12 @@ kvm_argument_list = [qemu_path,
'-nodefaults', '-nodefaults',
] ]
for disk_info in disk_info_list: for disk_info in disk_info_list:
additional_disk_options = '' kvm_argument_list += (
if disk_info['io'] == 'native':
additional_disk_options += ',cache.direct=on'
if disk_info['format'] == "raw":
additional_disk_options += ',discard=on'
kvm_argument_list.extend([
'-drive', '-drive',
'file=%s,if=%s,cache=%s,aio=%s%s' % ( disk_info['path'], disk_type, disk_info['cache'], disk_info['io'], additional_disk_options) 'file=%s,if=%s,discard=on%s' % (
]) disk_info['path'], disk_type,
''.join(',%s=%s' % x for x in disk_info.items() if x[0] != 'path'))
)
rgx = re.compile('^[\w*\,][\=\d+\-\,\w]*$') rgx = re.compile('^[\w*\,][\=\d+\-\,\w]*$')
for numa in numa_list: for numa in numa_list:
...@@ -392,6 +361,9 @@ else: ...@@ -392,6 +361,9 @@ else:
'-drive', '-drive',
'file=%s,media=cdrom' % (link,) 'file=%s,media=cdrom' % (link,)
]) ])
else:
raise ValueError('boot-image-url-select not ready yet')
if boot_image_url_list_json_config: if boot_image_url_list_json_config:
# Support boot-image-url-list # Support boot-image-url-list
with open(boot_image_url_list_json_config) as fh: with open(boot_image_url_list_json_config) as fh:
...@@ -404,6 +376,8 @@ else: ...@@ -404,6 +376,8 @@ else:
'-drive', '-drive',
'file=%s,media=cdrom' % (link,) 'file=%s,media=cdrom' % (link,)
]) ])
else:
raise ValueError('boot-image-url-list not ready yet')
# Always add by default the default image # Always add by default the default image
kvm_argument_list.extend([ kvm_argument_list.extend([
'-drive', 'file=%s,media=cdrom' % default_cdrom_iso '-drive', 'file=%s,media=cdrom' % default_cdrom_iso
......
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
import six.moves.http_client as httplib import six.moves.http_client as httplib
import json import json
import os import os
import glob
import hashlib import hashlib
import psutil import psutil
import requests import requests
...@@ -43,6 +44,7 @@ from six.moves import SimpleHTTPServer ...@@ -43,6 +44,7 @@ from six.moves import SimpleHTTPServer
import multiprocessing import multiprocessing
import time import time
import shutil import shutil
import sys
from slapos.recipe.librecipe import generateHashFromFiles from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
...@@ -120,12 +122,25 @@ class KvmMixin(object): ...@@ -120,12 +122,25 @@ class KvmMixin(object):
'software_release/buildout.cfg', 'software_release/buildout.cfg',
] ]
]) ])
# find bin/kvm_raw
kvm_raw_list = glob.glob(
os.path.join(self.slap.instance_directory, '*', 'bin', 'kvm_raw'))
self.assertEqual(1, len(kvm_raw_list)) # allow to work only with one
hash_file_list = [
kvm_raw_list[0],
'software_release/buildout.cfg',
]
kvm_hash_value = generateHashFromFiles([
os.path.join(self.computer_partition_root_path, hash_file)
for hash_file in hash_file_list
])
with self.slap.instance_supervisor_rpc as supervisor: with self.slap.instance_supervisor_rpc as supervisor:
running_process_info = '\n'.join(sorted([ running_process_info = '\n'.join(sorted([
'%(group)s:%(name)s %(statename)s' % q for q '%(group)s:%(name)s %(statename)s' % q for q
in supervisor.getAllProcessInfo() in supervisor.getAllProcessInfo()
if q['name'] != 'watchdog' and q['group'] != 'watchdog'])) if q['name'] != 'watchdog' and q['group'] != 'watchdog']))
return running_process_info.replace(hash_value, '{hash}') return running_process_info.replace(
hash_value, '{hash}').replace(kvm_hash_value, '{kvm-hash-value}')
def raising_waitForInstance(self, max_retry): def raising_waitForInstance(self, max_retry):
with self.assertRaises(SlapOSNodeCommandError): with self.assertRaises(SlapOSNodeCommandError):
...@@ -176,7 +191,7 @@ i0:6tunnel-10443-{hash}-on-watch RUNNING ...@@ -176,7 +191,7 @@ i0:6tunnel-10443-{hash}-on-watch RUNNING
i0:bootstrap-monitor EXITED i0:bootstrap-monitor EXITED
i0:certificate_authority-{hash}-on-watch RUNNING i0:certificate_authority-{hash}-on-watch RUNNING
i0:crond-{hash}-on-watch RUNNING i0:crond-{hash}-on-watch RUNNING
i0:kvm-{hash}-on-watch RUNNING i0:kvm-{kvm-hash-value}-on-watch RUNNING
i0:kvm_controller EXITED i0:kvm_controller EXITED
i0:monitor-httpd-{hash}-on-watch RUNNING i0:monitor-httpd-{hash}-on-watch RUNNING
i0:monitor-httpd-graceful EXITED i0:monitor-httpd-graceful EXITED
...@@ -416,10 +431,11 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase): ...@@ -416,10 +431,11 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
"test-machine2": dict(bootstrap_machine_param_dict, **{ "test-machine2": dict(bootstrap_machine_param_dict, **{
# Debian 9 image # Debian 9 image
"virtual-hard-drive-url": "virtual-hard-drive-url":
"http://shacache.org/shacache/ce07873dbab7fa8501d1bf5565c2737b2" "http://shacache.org/shacache/93aeb72a556fe88d9889ce16558dfead"
"eed6c8b9361b4997b21daf5f5d1590972db9ac00131cc5b27d9aa353f2f940" "57a3c8f0a80d0e04ebdcd4a5830dfa6403e3976cc896b8332e74f202fccbd"
"71e073f9980cc61badd6d2427f592e6e8", "a508930046a78cffea6e0e29d03345333cc",
"virtual-hard-drive-md5sum": "2b113e3cd8276b9740189622603d6f99" "virtual-hard-drive-md5sum": "cdca79619ba987c40b98a8e31d281e4a",
"virtual-hard-drive-gzipped": True,
}) })
} }
}))} }))}
...@@ -499,7 +515,7 @@ ir2:bootstrap-monitor EXITED ...@@ -499,7 +515,7 @@ ir2:bootstrap-monitor EXITED
ir2:certificate_authority-{hash}-on-watch RUNNING ir2:certificate_authority-{hash}-on-watch RUNNING
ir2:crond-{hash}-on-watch RUNNING ir2:crond-{hash}-on-watch RUNNING
ir2:equeue-on-watch RUNNING ir2:equeue-on-watch RUNNING
ir2:kvm-{hash}-on-watch RUNNING ir2:kvm-{kvm-hash-value}-on-watch RUNNING
ir2:kvm_controller EXITED ir2:kvm_controller EXITED
ir2:monitor-httpd-{hash}-on-watch RUNNING ir2:monitor-httpd-{hash}-on-watch RUNNING
ir2:monitor-httpd-graceful EXITED ir2:monitor-httpd-graceful EXITED
...@@ -1282,7 +1298,7 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin): ...@@ -1282,7 +1298,7 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin):
'disk-device-path': '/dev/virt0 /dev/virt1', 'disk-device-path': '/dev/virt0 /dev/virt1',
'wipe-disk-ondestroy': True 'wipe-disk-ondestroy': True
}) })
self.slap.waitForInstance(max_retry=2) self.raising_waitForInstance(3)
instance_path = os.path.join( instance_path = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference) self.slap.instance_directory, self.kvm_instance_partition_reference)
...@@ -1298,3 +1314,210 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin): ...@@ -1298,3 +1314,210 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin):
dd if=/dev/zero of=/dev/virt1 bs=4096 count=500k""" dd if=/dev/zero of=/dev/virt1 bs=4096 count=500k"""
) )
self.assertTrue(os.access(slapos_wipe_device_disk, os.X_OK)) self.assertTrue(os.access(slapos_wipe_device_disk, os.X_OK))
@skipUnlessKvm
class TestImageDownloadController(InstanceTestCase, FakeImageServerMixin):
__partition_reference__ = 'idc'
maxDiff = None
def setUp(self):
super(TestImageDownloadController, self).setUp()
self.working_directory = tempfile.mkdtemp()
self.destination_directory = os.path.join(
self.working_directory, 'destination')
os.mkdir(self.destination_directory)
self.config_json = os.path.join(
self.working_directory, 'config.json')
self.md5sum_fail_file = os.path.join(
self.working_directory, 'md5sum_fail_file')
self.error_state_file = os.path.join(
self.working_directory, 'error_state_file')
self.processed_md5sum = os.path.join(
self.working_directory, 'processed_md5sum')
self.startImageHttpServer()
self.image_download_controller = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '0',
'software_release', 'parts', 'image-download-controller',
'image-download-controller')
def tearDown(self):
self.stopImageHttpServer()
shutil.rmtree(self.working_directory)
super(InstanceTestCase, self).tearDown()
def callImageDownloadController(self, *args):
call_list = [sys.executable, self.image_download_controller] + list(args)
try:
return (0, subprocess.check_output(
call_list, stderr=subprocess.STDOUT).decode('utf-8'))
except subprocess.CalledProcessError as e:
return (e.returncode, e.output.decode('utf-8'))
def runImageDownloadControlerWithDict(self, json_dict):
with open(self.config_json, 'w') as fh:
json.dump(json_dict, fh, indent=2)
return self.callImageDownloadController(
self.config_json,
'curl', # comes from test environemnt, considered to be recent enough
self.md5sum_fail_file,
self.error_state_file,
self.processed_md5sum
)
def assertFileContent(self, path, content):
self.assertTrue(os.path.exists, path)
with open(path, 'r') as fh:
self.assertEqual(
fh.read(),
content)
def test(self):
json_dict = {
'error-amount': 0,
'config-md5sum': 'config-md5sum',
'destination-directory': self.destination_directory,
'image-list': [
{
'destination-tmp': 'tmp',
'url': self.fake_image,
'destination': 'destination',
'link': 'image_001',
'gzipped': False,
'md5sum': self.fake_image_md5sum,
}
]
}
code, result = self.runImageDownloadControlerWithDict(
json_dict
)
self.assertEqual(
(code, result.strip()),
(0, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : Downloading
INF: %(fake_image)s : Stored with checksum %(checksum)s
INF: %(fake_image)s : Symlinking %(symlink)s -> %(destination)s
""".strip() % {
'fake_image': self.fake_image,
'checksum': self.fake_image_md5sum,
'error_state_file': self.error_state_file,
'symlink': os.path.join(self.destination_directory, 'image_001'),
'destination': os.path.join(self.destination_directory, 'destination'),
})
)
self.assertFileContent(self.md5sum_fail_file, '')
self.assertFileContent(self.error_state_file, '')
self.assertFileContent(self.processed_md5sum, 'config-md5sum')
self.assertFalse(
os.path.exists(os.path.join(self.destination_directory, 'tmp')))
self.assertFileContent(
os.path.join(self.destination_directory, 'destination'),
'fake_image_content'
)
# Nothing happens if all is downloaded
code, result = self.runImageDownloadControlerWithDict(
json_dict
)
self.assertEqual(
(code, result.strip()),
(0, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : already downloaded
""".strip() % {
'fake_image': self.fake_image,
'checksum': self.fake_image_md5sum,
'error_state_file': self.error_state_file,
'symlink': os.path.join(self.destination_directory, 'image_001'),
'destination': os.path.join(self.destination_directory, 'destination'),
})
)
def test_fail(self):
json_dict = {
'error-amount': 0,
'config-md5sum': 'config-md5sum',
'destination-directory': self.destination_directory,
'image-list': [
{
'destination-tmp': 'tmp',
'url': self.fake_image,
'destination': 'destination',
'link': 'image_001',
'gzipped': False,
'md5sum': self.fake_image_wrong_md5sum,
}
]
}
for try_num in range(1, 5):
code, result = self.runImageDownloadControlerWithDict(
json_dict
)
self.assertEqual(
(code, result.strip()),
(1, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : Downloading
""". strip() % {
'fake_image': self.fake_image,
'error_state_file': self.error_state_file,
'symlink': os.path.join(self.destination_directory, 'image_001'),
'destination': os.path.join(
self.destination_directory, 'destination'),
})
)
fake_image_url = '#'.join([
self.fake_image, self.fake_image_wrong_md5sum])
self.assertFileContent(
self.md5sum_fail_file, """{
"%s": %s
}""" % (fake_image_url, try_num))
self.assertFileContent(
self.error_state_file, """
ERR: %(fake_image)s : MD5 mismatch expected is %(wrong_checksum)s """
"""but got instead %(real_checksum)s""".strip() % {
'fake_image': self.fake_image,
'wrong_checksum': self.fake_image_wrong_md5sum,
'real_checksum': self.fake_image_md5sum,
})
self.assertFileContent(self.processed_md5sum, 'config-md5sum')
self.assertFalse(
os.path.exists(os.path.join(self.destination_directory, 'tmp')))
self.assertFalse(
os.path.exists(
os.path.join(self.destination_directory, 'destination')))
code, result = self.runImageDownloadControlerWithDict(
json_dict
)
self.assertEqual(
(code, result.strip()),
(1, """
INF: Storing errors in %(error_state_file)s
""". strip() % {
'fake_image': self.fake_image,
'error_state_file': self.error_state_file,
'symlink': os.path.join(self.destination_directory, 'image_001'),
'destination': os.path.join(
self.destination_directory, 'destination'),
})
)
fake_image_url = '#'.join([
self.fake_image, self.fake_image_wrong_md5sum])
self.assertFileContent(
self.md5sum_fail_file, """{
"%s": %s
}""" % (fake_image_url, 4))
self.assertFileContent(
self.error_state_file, """
ERR: %(fake_image)s : Checksum is incorrect after 4 tries, will not """
"""retry""".strip() % {
'fake_image': self.fake_image,
})
self.assertFileContent(self.processed_md5sum, 'config-md5sum')
self.assertFalse(
os.path.exists(os.path.join(self.destination_directory, 'tmp')))
self.assertFalse(
os.path.exists(
os.path.join(self.destination_directory, 'destination')))
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 9ddae686379e8d747410c1adf82b47d6 md5sum = c115ed9d4ff0f785d79cdcacbb0bd1ad
[template-monitor] [template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2 _update_hash_filename_ = instance-monitor.cfg.jinja2
...@@ -26,11 +26,11 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680 ...@@ -26,11 +26,11 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
[template-monitor-edgetest] [template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2 _update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = a57106ee88ff3295b9ffce84105da79b md5sum = 3c8ab4e78f66c974eb95afc595a13514
[template-monitor-edgebot] [template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = c1885a42aadd45bab3185a53258d4ff4 md5sum = 365a6cc6831267a73fa5ebd56ad394ee
[network-bench-cfg] [network-bench-cfg]
filename = network_bench.cfg.in filename = network_bench.cfg.in
...@@ -42,4 +42,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb ...@@ -42,4 +42,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
[template-surykatka-ini] [template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2 _update_hash_filename_ = surykatka.ini.jinja2
md5sum = 89545501f0e5bf11608978886429da3d md5sum = 609c6cca763b73a80fa05ee56475eb20
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": { "properties": {
"nameserver": { "region-dict": {
"default": "", "title": "Regions",
"title": "Nameserver", "description": "Defines regions of the cluster",
"description": "Space separated list of name servers to use.", "patternProperties": {
"type": "string" ".*": {
"properties": {
"state": {
"title": "State",
"description": "State of the node of the region. Can be used to destroy not needed regions.",
"type": "string",
"default": "started",
"enum": [
"started",
"stopped",
"destroyed"
]
}, },
"check-status-code": { "sla-computer_guid": {
"default": "200", "title": "GUID of the computer on which this region shall be deployed",
"title": "Default Check HTTP Code", "description": "Unique identifier of the computer, like \"COMP-1234\". By default, let Master choose a computer.",
"description": "Default HTTP code to check against (default: 200).", "type": "string",
"type": "string" "default": ""
}, },
"check-http-header-dict": { "nameserver-list": {
"default": "{}", "default": [],
"title": "HTTP header dict to check", "title": "Nameservers",
"description": "JSON dict of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}", "description": "List of nameservers to use.",
"type": "object" "type": "array"
}, },
"check-frontend-ip": { "check-frontend-ip-list": {
"default": "", "default": [],
"title": "Default space separated list of Frontend IPs to check", "title": "Default Frontend IPs to check",
"description": "Default list of Frontend IPs to check, if empty no constraint is used.", "description": "List of default frontend IPs to check, if empty no constraint is used.",
"type": "string" "type": "array"
}
}, },
"check-certificate-expiration-days": { "type": "object"
"default": "15", }
"title": "Default certificate expiration days check", },
"description": "Default amount of days to consider certificate as being to-be-expired (default: 15).", "type": "object"
"type": "string"
}, },
"check-maximum-elapsed-time": { "nameserver-list": {
"default": "2", "default": [],
"title": "Default maximum elapsed time for a site to reply (seconds)", "title": "Nameservers (backward compatibility)",
"description": "Default maximum elapsed time for a site to reply to be considered good (default: 2s).", "description": "List of nameservers to use. Note: This is backward compatibility, use region-dict for full configuration control.",
"type": "string" "type": "array"
}, },
"failure-amount": { "check-frontend-ip-list": {
"default": "1", "default": [],
"title": "Default amount of failures to consider URL as in bad state", "title": "Default Frontend IPs to check (backward compatibility)",
"description": "Default amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages (default: 1).", "description": "List of default frontend IPs to check, if empty no constraint is used. Note: This is backward compatibility, use region-dict for full configuration control.",
"type": "string" "type": "array"
} }
} }
} }
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": { "properties": {
"url": { "url": {
"title": "URL to check", "title": "URL to check",
"description": "URL to check, like https://example.com", "description": "URL to check, like https://example.com/",
"type": "string" "type": "string"
}, },
"check-status-code": { "region-dict": {
"default": "Master default", "title": "Applicable Regions",
"title": "Default Check HTTP Code.", "description": "Puts the check on the defined regions. No definition will result with presence in all regions.",
"description": "HTTP code to check against (default: comes from master partition).", "patternProperties": {
"type": "string" ".*": {
"properties": {
"state": {
"title": "State",
"description": "State of the check of the region. Used only to make it correctly visible in the SlapOS Master UI if no other parameters are defined.",
"type": "string",
"default": "present",
"enum": [
"present"
]
},
"check-frontend-ip-list": {
"default": [],
"title": "Frontend IPs to check",
"description": "List of default frontend IPs to check, if empty no constraint is used. Defaults to region configuration.",
"type": "array"
}
}, },
"check-http-header-dict": {
"default": "Master default",
"title": "HTTP header dict to check",
"description": "JSON dict of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}",
"type": "object" "type": "object"
}
}, },
"check-frontend-ip": { "type": "object",
"default": "Master default", "default": {}
"title": "Space separated list of Frontend IPs to check", },
"description": "List of Frontend IPs to check, if empty no constraint is used (default: comes from master partition).", "check-status-code": {
"type": "string" "title": "HTTP Code Check",
"description": "Expected response HTTP Code.",
"type": "number",
"default": 200,
"minimum": 100,
"maximum": 599
}, },
"check-certificate-expiration-days": { "check-certificate-expiration-days": {
"default": "Master default", "title": "Certificate Expiration Check (days)",
"title": "Certificate expiration days check", "description": "Amount of days to consider certificate as being to-be-expired.",
"description": "Amount of days to consider certificate as being to-be-expired (default: comes from master partition).", "type": "number",
"type": "string" "default": 15,
"minimum": 1
}, },
"check-maximum-elapsed-time": { "check-maximum-elapsed-time": {
"default": "Master default", "title": "Maximum Elapsed Check (seconds)",
"title": "Maximum elapsed time for a site to reply (seconds)", "description": "Maximum elapsed time for a site to reply to be considered good.",
"description": "Maximum elapsed time for a site to reply to be considered good.(default: comes from master partition).", "type": "number",
"type": "string" "default": 2,
"minimum": 1
},
"check-http-header-dict": {
"title": "HTTP Header Check",
"description": "JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
"type": "object",
"default": {}
}, },
"failure-amount": { "failure-amount": {
"default": "Master default", "title": "Failure Amount",
"title": "Amount of failures to consider URL as in bad state", "description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
"description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages (default: comes from master partition).", "type": "number",
"type": "string" "default": 2,
"minimum": 1
},
"check-frontend-ip-list": {
"title": "Frontend IPs to check (backward compatibility)",
"description": "List of Frontend IPs to check, if empty no constraint is used. Defaults to region configuration. Note: Use region-dict's check-frontend-ip-list to ensure specific check on each region.",
"type": "array"
} }
} }
} }
...@@ -9,25 +9,13 @@ ...@@ -9,25 +9,13 @@
{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %} {%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
{%- if extra_slave_instance_list %} {%- if extra_slave_instance_list %}
{#- Create slaves to process with setting up defaults #} {#- Create slaves to process with setting up defaults #}
{%- for slave in json_module.loads(extra_slave_instance_list) | sort(attribute='slave_title') %} {%- for slave in extra_slave_instance_list | sort(attribute='-slave-title') %}
{%- if 'check-status-code' not in slave %} {%- do slave.setdefault('check-status-code', 200) %}
{%- do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %} {%- do slave.setdefault('check-http-header-dict', {}) %}
{%- endif %} {%- do slave.setdefault('check-certificate-expiration-days', 15) %}
{%- if 'check-http-header-dict' not in slave %} {%- do slave.setdefault('failure-amount', 2) %}
{%- do slave.__setitem__('check-http-header-dict', CONFIGURATION['check-http-header-dict']) %} {%- do slave.setdefault('check-maximum-elapsed-time', 2) %}
{%- endif %} {%- do slave.setdefault('check-frontend-ip-list', CONFIGURATION['check-frontend-ip-list']) %}
{%- if 'check-certificate-expiration-days' not in slave %}
{%- do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %}
{%- endif %}
{%- if 'failure-amount' not in slave %}
{%- do slave.__setitem__('failure-amount', CONFIGURATION['failure-amount']) %}
{%- endif %}
{%- if 'check-maximum-elapsed-time' not in slave %}
{%- do slave.__setitem__('check-maximum-elapsed-time', CONFIGURATION['check-maximum-elapsed-time']) %}
{%- endif %}
{%- if 'check-frontend-ip' not in slave %}
{%- do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %}
{%- endif %}
{%- if 'url' in slave %} {%- if 'url' in slave %}
{%- set class = slave['check-maximum-elapsed-time'] %} {%- set class = slave['check-maximum-elapsed-time'] %}
{%- if class not in slave_instance_dict %} {%- if class not in slave_instance_dict %}
...@@ -40,8 +28,8 @@ ...@@ -40,8 +28,8 @@
{%- set part_list = [] %} {%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %} {%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #} {#- class is used to separate surykatka with different timeouts #}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %} {%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %} {%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
{%- do part_list.append(part_id) %} {%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %} {%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
[{{part_id}}] [{{part_id}}]
...@@ -51,11 +39,11 @@ name = {{ safe_name }}.py ...@@ -51,11 +39,11 @@ name = {{ safe_name }}.py
config-report = http_query config-report = http_query
config-url = {{ slave['url'] }} config-url = {{ slave['url'] }}
config-status-code = {{ slave['check-status-code'] }} config-status-code = {{ slave['check-status-code'] }}
config-http-header-dict = {{ slave['check-http-header-dict'] }} config-http-header-dict = {{ json_module.dumps(slave['check-http-header-dict']) }}
config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }} config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
config-failure-amount = {{ slave['failure-amount'] }} config-failure-amount = {{ slave['failure-amount'] }}
config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }} config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
config-ip-list = {{ slave['check-frontend-ip'] }} config-ip-list = {{ ' '.join(slave['check-frontend-ip-list']) }}
config-json-file = ${surykatka-config-{{ class }}:json} config-json-file = ${surykatka-config-{{ class }}:json}
{%- endfor %} {%- endfor %}
...@@ -72,7 +60,7 @@ db = ${directory:srv}/surykatka-{{ class }}.db ...@@ -72,7 +60,7 @@ db = ${directory:srv}/surykatka-{{ class }}.db
rendered = ${directory:etc}/surykatka-{{ class }}.ini rendered = ${directory:etc}/surykatka-{{ class }}.ini
template = {{ template_surykatka_ini }} template = {{ template_surykatka_ini }}
slave_instance_list = {{ dumps(slave_instance_list) }} slave_instance_list = {{ dumps(slave_instance_list) }}
nameserver = {{ dumps(CONFIGURATION['nameserver']) }} nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
json = ${directory:srv}/surykatka-{{ class }}.json json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #} {#- timeout is just a bit bigger than class time #}
timeout = {{ int(class) + 2 }} timeout = {{ int(class) + 2 }}
...@@ -80,7 +68,7 @@ timeout = {{ int(class) + 2 }} ...@@ -80,7 +68,7 @@ timeout = {{ int(class) + 2 }}
context = context =
import json_module json import json_module json
key db :db key db :db
key nameserver :nameserver key nameserver_list :nameserver_list
key slave_instance_list :slave_instance_list key slave_instance_list :slave_instance_list
key timeout :timeout key timeout :timeout
...@@ -110,9 +98,9 @@ cron-entries = ${directory:etc}/cron.d ...@@ -110,9 +98,9 @@ cron-entries = ${directory:etc}/cron.d
name = surykatka-status-{{ class }} name = surykatka-status-{{ class }}
frequency = */2 * * * * frequency = */2 * * * *
command = ${surykatka-status-json-{{ class }}:rendered} command = ${surykatka-status-json-{{ class }}:rendered}
{%- do part_list.append('surykatka-' + class) %} {%- do part_list.append('surykatka-%i'% (class,)) %}
{%- do part_list.append('surykatka-bot-promise-' + class) %} {%- do part_list.append('surykatka-bot-promise-%i' % (class,)) %}
{%- do part_list.append('cron-entry-surykatka-status-' + class) %} {%- do part_list.append('cron-entry-surykatka-status-%i' % (class,)) %}
{%- endfor %} {%- endfor %}
[buildout] [buildout]
...@@ -163,4 +151,4 @@ key = ${slap-connection:key-file} ...@@ -163,4 +151,4 @@ key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file} cert = ${slap-connection:cert-file}
[slap-parameter] [slap-parameter]
{%- endif %} {%- endif %} {#- if slap_software_type == software_type #}
...@@ -17,33 +17,85 @@ extra-context = ...@@ -17,33 +17,85 @@ extra-context =
section slave_information slap-configuration section slave_information slap-configuration
{% set part_list = [] -%} {% set part_list = [] -%}
# Publish information for each slave
{%- set edgebot_software_type = 'edgebot' %} {%- set edgebot_software_type = 'edgebot' %}
{%- set edgebot_quantity = slapparameter_dict.pop('edgebot-quantity', '1') | int %}
{%- set edgebot_list = [] %}
{%- set edgebot_section_list = [] %}
{%- set slave_list_name = 'extra_slave_instance_list' %} {%- set slave_list_name = 'extra_slave_instance_list' %}
{%- set request_dict = {} %} {%- set request_dict = {} %}
{%- set namebase = "edgebot" %} {%- set namebase = "edgebot" %}
{%- set authorized_slave_list = [] %} {%- if 'region-dict' not in slapparameter_dict %}
{%- set monitor_base_url_dict = {} -%} {#- Be nice and allow to work with default configuration #}
{%- do slapparameter_dict.__setitem__('region-dict', {
'1': {
'sla-computer_guid': slap_configuration['computer'],
'state': slap_configuration['instance-state'],
'nameserver-list': slapparameter_dict.get('nameserver-list', []),
'check-frontend-ip-list': slapparameter_dict.get('check-frontend-ip-list', []),
}
}) %}
{%- endif %}
{%- set active_region_list = [] %}
{%- for region_name in sorted(slapparameter_dict['region-dict']) %}
{%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
{%- if region_parameter_dict.get('state', 'started') == 'started' %}
{%- do active_region_list.append(region_name) %}
{%- endif %}
{%- endfor %}
{%- set authorized_slave_dict = {} %}
{%- set publish_slave_dict_dict = {} %}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %} {%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- do authorized_slave_list.append(slave) %} {%- set slave_reference = slave.pop('slave_reference') %}
{%- set publish_dict = {'assigned-region-dict': {}} %}
{%- if '_' in slave %}
{%- set base_slave_dict = json_module.loads(slave.pop('_')) %} {#- XXX: Unsafe! #}
{%- do base_slave_dict.__setitem__('-slave-title', slave['slave_title']) %}
{%- do base_slave_dict.__setitem__('-slave-reference', slave_reference) %}
{%- set slave_region_dict = base_slave_dict.pop('region-dict', {}) %}
{%- if slave_region_dict == {} %}
{%- for region in active_region_list %}
{%- do slave_region_dict.__setitem__(region, {}) %}
{%- endfor %}
{%- endif %}
{%- for region in slave_region_dict %}
{%- if region in active_region_list %}
{%- set region_info = {
'nameserver-list': slapparameter_dict['region-dict'][region].get('nameserver-list') or slapparameter_dict.get('slapparameter_dict') or [],
'check-frontend-ip-list': slave_region_dict[region].get('check-frontend-ip-list') or base_slave_dict.get('check-frontend-ip-list') or slapparameter_dict['region-dict'][region].get('check-frontend-ip-list') or slapparameter_dict.get('check-frontend-ip-list') or [],
} %}
{%- do publish_dict['assigned-region-dict'].__setitem__(region, region_info) %}
{%- set slave_dict = base_slave_dict.copy() %}
{%- do slave_dict.update(region_info) %}
{%- if region not in authorized_slave_dict %}
{%- do authorized_slave_dict.__setitem__(region, [slave_dict]) %}
{%- else %}
{%- do authorized_slave_dict[region].append(slave_dict) %}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- do publish_slave_dict_dict.__setitem__(slave_reference, publish_dict) %}
{%- endfor %} {%- endfor %}
{%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %} {%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
{%- for i in range(1, edgebot_quantity + 1) %} {%- set number = {'i': 1} %}
{%- set edgebot_name = "%s-%s" % (namebase, i) %} {%- for region_name in sorted(slapparameter_dict['region-dict']) %}
{%- set request_section_title = 'request-%s' % edgebot_name %} {%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
{%- do edgebot_list.append(edgebot_name) %} {%- set edgebot_name = "%s-%s" % (namebase, region_name) %}
{%- do edgebot_section_list.append(request_section_title) %} {%- set request_section_title = 'request-%s' % (hashlib_module.md5(edgebot_name.encode('utf-8')).hexdigest(),) %}
{%- do part_list.append(request_section_title) %} {%- do part_list.append(request_section_title) %}
{%- do request_dict.__setitem__(request_section_title, {#- Note: monitor-httpd-port will vary on regions being added and removed,
but this is accepted, as it's only internal trick #}
{%- do request_dict.__setitem__(
request_section_title,
{ {
'config': {'monitor-httpd-port': monitor_base_port + i}, 'config': {
'monitor-httpd-port': monitor_base_port + number['i'],
'check-frontend-ip-list': region_parameter_dict.get('check-frontend-ip-list', []),
'nameserver-list': region_parameter_dict.get('nameserver-list', []),
'extra_slave_instance_list': authorized_slave_dict.get(region_name, [])
},
'name': edgebot_name, 'name': edgebot_name,
'sla': {}, 'sla': {'computer_guid': region_parameter_dict['sla-computer_guid']},
'state': 'started', 'state': region_parameter_dict.get('state', 'started'),
}) %} }) %}
{%- do number.__setitem__('i', number['i'] + 1) %}
{%- endfor %} {%- endfor %}
[replicate] [replicate]
...@@ -54,38 +106,36 @@ config-monitor-username = ${monitor-instance-parameter:username} ...@@ -54,38 +106,36 @@ config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd} config-monitor-password = ${monitor-htpasswd:passwd}
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
software-type = {{edgebot_software_type}} software-type = {{edgebot_software_type}}
return = monitor-base-url
{%- set monitor_base_url_dict = {} -%}
{% for section, edgebot_request in request_dict.items() %} {% for section, edgebot_request in request_dict.items() %}
[{{section}}] [{{section}}]
<= replicate <= replicate
name = {{ edgebot_request.get('name') }} name = {{ edgebot_request['name'] }}
{%- if edgebot_request.get('state') %} state = {{ edgebot_request['state'] }}
state = {{ edgebot_request.get('state') }} {%- if edgebot_request['state'] != 'destroyed' %}
{%- endif%} {%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') %}
{%- set slave_configuration_dict = slapparameter_dict %} return = monitor-base-url
{%- do slave_configuration_dict.update(edgebot_request.get('config')) %} {%- endif %}
{%- do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %} {%- set edgebot_configuration_dict = edgebot_request['config'] %}
{%- for config_key, config_value in slave_configuration_dict.items() %} {%- for config_key, config_value in edgebot_configuration_dict.items() %}
config-{{ config_key }} = {{ dumps(config_value) }} config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%} {% endfor -%}
{%- if edgebot_request.get('sla') %} {%- for parameter, value in edgebot_request['sla'].items() %}
{%- for parameter, value in edgebot_request.get('sla').items() %}
sla-{{ parameter }} = {{ value }} sla-{{ parameter }} = {{ value }}
{%- endfor %} {%- endfor %}
{%- else %}
# As no SLA was provided, by default it is requested on the same computer
sla-computer_guid = ${slap-connection:computer-id}
{% endif %}
{%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{%- endfor %} {%- endfor %}
{%- set directory_list = [] -%} # Publish information for each slave
{%- for slave_instance in slave_instance_list -%} {%- for slave_reference, publish_dict in publish_slave_dict_dict.items() -%}
{%- set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference') -%} {%- set publish_section_title = 'publish-%s' % (hashlib_module.md5(slave_reference.encode('utf-8')).hexdigest(),) -%}
{%- do part_list.append(publish_section_title) %} {%- do part_list.append(publish_section_title) %}
[{{ publish_section_title }}] [{{ publish_section_title }}]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish.serialised
-slave-reference = {{ slave_instance.get('slave_reference') }} available-region-list = {{ dumps(list(active_region_list)) }}
-slave-reference = {{ slave_reference }}
{%- for key, value in publish_dict.items() %}
{{ key }} = {{ dumps(value) }}
{%- endfor %}
{% endfor %} {% endfor %}
[monitor-conf-parameters] [monitor-conf-parameters]
...@@ -104,4 +154,10 @@ parts += ...@@ -104,4 +154,10 @@ parts +=
{% for part in part_list %} {% for part in part_list %}
{{ ' %s' % part }} {{ ' %s' % part }}
{%- endfor %} {%- endfor %}
{%- endif %}
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
active-region-list = {{ dumps(list(active_region_list)) }}
sla-computer_guid = {{ dumps(slap_configuration['computer']) }}
sla-instance_guid = {{ dumps(slap_configuration['instance-guid']) }}
{%- endif %} {#- if slap_software_type == software_type #}
...@@ -32,6 +32,7 @@ template = ${template-monitor-edgetest:target} ...@@ -32,6 +32,7 @@ template = ${template-monitor-edgetest:target}
rendered = $${buildout:directory}/template-monitor-base-edgetest.cfg rendered = $${buildout:directory}/template-monitor-base-edgetest.cfg
extensions = jinja2.ext.do extensions = jinja2.ext.do
context = import json_module json context = import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
...@@ -53,6 +54,7 @@ surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name} ...@@ -53,6 +54,7 @@ surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name}
template-surykatka-ini = ${template-surykatka-ini:target} template-surykatka-ini = ${template-surykatka-ini:target}
context = import json_module json context = import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
section slap_configuration slap-configuration section slap_configuration slap-configuration
...@@ -75,13 +77,8 @@ url = $${slap-connection:server-url} ...@@ -75,13 +77,8 @@ url = $${slap-connection:server-url}
key = $${slap-connection:key-file} key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file} cert = $${slap-connection:cert-file}
# Defaults # Defaults
configuration.check-status-code = 200 configuration.nameserver-list =
configuration.check-http-header-dict = {} configuration.check-frontend-ip-list =
configuration.nameserver =
configuration.check-frontend-ip =
configuration.check-certificate-expiration-days = 15
configuration.check-maximum-elapsed-time = 2
configuration.failure-amount = 2
# use monitor-base-port to have monitor listening on each instance # use monitor-base-port to have monitor listening on each instance
# on different port and also on different port than other services # on different port and also on different port than other services
# it makes it possible to instantiate it correctly on signle IP, for # it makes it possible to instantiate it correctly on signle IP, for
......
...@@ -15,14 +15,17 @@ ...@@ -15,14 +15,17 @@
"description": "Cluster of bots to perform a distributed monitoring ", "description": "Cluster of bots to perform a distributed monitoring ",
"request": "instance-edgetest-input-schema.json", "request": "instance-edgetest-input-schema.json",
"response": "instance-default-output-schema.json", "response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 1 "index": 1
}, },
"edgetest-slave": { "edgetest-slave": {
"title": "Edge Test Slave", "title": "Edge Test Slave",
"shared": true,
"software-type": "edgetest", "software-type": "edgetest",
"description": "Cluster of bots to perform a distributed monitoring ", "description": "Cluster of bots to perform a distributed monitoring ",
"request": "instance-edgetest-slave-input-schema.json", "request": "instance-edgetest-slave-input-schema.json",
"response": "instance-default-output-schema.json", "response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 2 "index": 2
} }
} }
......
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
INTERVAL = 120 INTERVAL = 120
TIMEOUT = {{ timeout }} TIMEOUT = {{ timeout }}
SQLITE = {{ db }} SQLITE = {{ db }}
{%- set nameserver_list = nameserver.split() %}
{%- if len(nameserver_list) > 0 %} {%- if len(nameserver_list) > 0 %}
NAMESERVER = NAMESERVER =
{%- for nameserver_entry in sorted(nameserver_list) %} {%- for nameserver_entry in sorted(nameserver_list) %}
...@@ -10,7 +9,7 @@ NAMESERVER = ...@@ -10,7 +9,7 @@ NAMESERVER =
{%- endfor %} {%- endfor %}
{% endif %} {% endif %}
URL = URL =
{%- for slave in slave_instance_list | sort(attribute='slave_title') %} {%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- if 'url' in slave %} {%- if 'url' in slave %}
{{ slave['url'] }} {{ slave['url'] }}
{%- endif -%} {%- endif -%}
......
...@@ -26,11 +26,13 @@ ...@@ -26,11 +26,13 @@
############################################################################## ##############################################################################
import glob import glob
import hashlib
import json import json
import os import os
import re import re
import requests import requests
import subprocess import subprocess
import unittest
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from slapos.recipe.librecipe import generateHashFromFiles from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
...@@ -69,8 +71,10 @@ class MonitorTestMixin: ...@@ -69,8 +71,10 @@ class MonitorTestMixin:
monitor_setup_url_key = 'monitor-setup-url' monitor_setup_url_key = 'monitor-setup-url'
def test_monitor_setup(self): def test_monitor_setup(self):
connection_parameter_dict = self\ connection_parameter_dict_serialised = self\
.computer_partition.getConnectionParameterDict() .computer_partition.getConnectionParameterDict()
connection_parameter_dict = json.loads(
connection_parameter_dict_serialised['_'])
self.assertTrue( self.assertTrue(
self.monitor_setup_url_key in connection_parameter_dict, self.monitor_setup_url_key in connection_parameter_dict,
'%s not in %s' % (self.monitor_setup_url_key, connection_parameter_dict)) '%s not in %s' % (self.monitor_setup_url_key, connection_parameter_dict))
...@@ -136,6 +140,7 @@ class MonitorTestMixin: ...@@ -136,6 +140,7 @@ class MonitorTestMixin:
class EdgeSlaveMixin(MonitorTestMixin): class EdgeSlaveMixin(MonitorTestMixin):
__partition_reference__ = 'edge' __partition_reference__ = 'edge'
instance_max_retry = 20 instance_max_retry = 20
expected_connection_parameter_dict = {}
@classmethod @classmethod
def getInstanceSoftwareType(cls): def getInstanceSoftwareType(cls):
...@@ -143,36 +148,37 @@ class EdgeSlaveMixin(MonitorTestMixin): ...@@ -143,36 +148,37 @@ class EdgeSlaveMixin(MonitorTestMixin):
def requestEdgetestSlave(self, partition_reference, partition_parameter_kw): def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
software_url = self.getSoftwareURL() software_url = self.getSoftwareURL()
self.slap.request( return self.slap.request(
software_release=software_url, software_release=software_url,
software_type='edgetest', software_type='edgetest',
partition_reference=partition_reference, partition_reference=partition_reference,
partition_parameter_kw=partition_parameter_kw, partition_parameter_kw={'_': json.dumps(partition_parameter_kw)},
shared=True shared=True
) )
def updateSurykatkaDict(self): def updateSurykatkaDict(self):
for class_ in self.surykatka_dict: for instance_reference in self.surykatka_dict:
for class_ in self.surykatka_dict[instance_reference]:
update_dict = {} update_dict = {}
update_dict['ini-file'] = os.path.join( update_dict['ini-file'] = os.path.join(
self.bot_partition_path, 'etc', 'surykatka-%s.ini' % (class_,)) self.slap.instance_directory, instance_reference, 'etc',
'surykatka-%s.ini' % (class_,))
update_dict['json-file'] = os.path.join( update_dict['json-file'] = os.path.join(
self.bot_partition_path, 'srv', 'surykatka-%s.json' % (class_,)) self.slap.instance_directory, instance_reference, 'srv',
'surykatka-%s.json' % (class_,))
update_dict['status-json'] = os.path.join( update_dict['status-json'] = os.path.join(
self.bot_partition_path, 'bin', 'surykatka-status-json-%s' % (class_,)) self.slap.instance_directory, instance_reference, 'bin',
'surykatka-status-json-%s' % (class_,))
update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,) update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
update_dict['status-cron'] = os.path.join( update_dict['status-cron'] = os.path.join(
self.bot_partition_path, 'etc', 'cron.d', 'surykatka-status-%s' % ( self.slap.instance_directory, instance_reference, 'etc',
class_,)) 'cron.d', 'surykatka-status-%s' % (class_,))
update_dict['db_file'] = os.path.join( update_dict['db_file'] = os.path.join(
self.bot_partition_path, 'srv', 'surykatka-%s.db' % (class_,)) self.slap.instance_directory, instance_reference, 'srv',
self.surykatka_dict[class_].update(update_dict) 'surykatka-%s.db' % (class_,))
self.surykatka_dict[instance_reference][class_].update(update_dict)
def setUp(self): def setUpMonitorConfigurationList(self):
self.bot_partition_path = os.path.join(
self.slap.instance_directory,
self.__partition_reference__ + '1')
self.updateSurykatkaDict()
self.monitor_configuration_list = [ self.monitor_configuration_list = [
{ {
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,), 'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
...@@ -194,39 +200,61 @@ class EdgeSlaveMixin(MonitorTestMixin): ...@@ -194,39 +200,61 @@ class EdgeSlaveMixin(MonitorTestMixin):
} }
] ]
def setUp(self):
self.updateSurykatkaDict()
self.setUpMonitorConfigurationList()
def assertSurykatkaIni(self): def assertSurykatkaIni(self):
expected_init_path_list = []
for instance_reference in self.surykatka_dict:
expected_init_path_list.extend(
[q['ini-file']
for q in self.surykatka_dict[instance_reference].values()])
self.assertEqual( self.assertEqual(
set( set(
glob.glob( glob.glob(
os.path.join(self.bot_partition_path, 'etc', 'surykatka*.ini'))), os.path.join(
{q['ini-file'] for q in self.surykatka_dict.values()} self.slap.instance_directory, '*', 'etc', 'surykatka*.ini'
)
) )
for info_dict in self.surykatka_dict.values(): ),
set(expected_init_path_list)
)
for instance_reference in self.surykatka_dict:
for info_dict in self.surykatka_dict[instance_reference].values():
self.assertEqual( self.assertEqual(
info_dict['expected_ini'].strip() % info_dict, info_dict['expected_ini'].strip() % info_dict,
open(info_dict['ini-file']).read().strip() open(info_dict['ini-file']).read().strip()
) )
def assertPromiseContent(self, name, content): def assertPromiseContent(self, instance_reference, name, content):
promise = open( promise = open(
os.path.join( os.path.join(
self.bot_partition_path, 'etc', 'plugin', name self.slap.instance_directory, instance_reference, 'etc', 'plugin', name
)).read().strip() )).read().strip()
self.assertTrue(content in promise) self.assertTrue(content in promise)
def assertHttpQueryPromiseContent(self, instance_reference, name, content):
hashed = 'http-query-%s-promise.py' % (
hashlib.md5(('_' + name).encode('utf-8')).hexdigest(),)
self.assertPromiseContent(instance_reference, hashed, content)
def assertSurykatkaBotPromise(self): def assertSurykatkaBotPromise(self):
for info_dict in self.surykatka_dict.values(): for instance_reference in self.surykatka_dict:
for info_dict in self.surykatka_dict[instance_reference].values():
self.assertPromiseContent( self.assertPromiseContent(
instance_reference,
info_dict['bot-promise'], info_dict['bot-promise'],
"'report': 'bot_status'") "'report': 'bot_status'")
self.assertPromiseContent( self.assertPromiseContent(
instance_reference,
info_dict['bot-promise'], info_dict['bot-promise'],
"'json-file': '%s'" % (info_dict['json-file'],) "'json-file': '%s'" % (info_dict['json-file'],),)
)
def assertSurykatkaCron(self): def assertSurykatkaCron(self):
for info_dict in self.surykatka_dict.values(): for instance_reference in self.surykatka_dict:
for info_dict in self.surykatka_dict[instance_reference].values():
self.assertEqual( self.assertEqual(
'*/2 * * * * %s' % (info_dict['status-json'],), '*/2 * * * * %s' % (info_dict['status-json'],),
open(info_dict['status-cron']).read().strip() open(info_dict['status-cron']).read().strip()
...@@ -239,7 +267,8 @@ class EdgeSlaveMixin(MonitorTestMixin): ...@@ -239,7 +267,8 @@ class EdgeSlaveMixin(MonitorTestMixin):
pass pass
def assertSurykatkaStatusJSON(self): def assertSurykatkaStatusJSON(self):
for info_dict in self.surykatka_dict.values(): for instance_reference in self.surykatka_dict:
for info_dict in self.surykatka_dict[instance_reference].values():
if os.path.exists(info_dict['json-file']): if os.path.exists(info_dict['json-file']):
os.unlink(info_dict['json-file']) os.unlink(info_dict['json-file'])
try: try:
...@@ -251,6 +280,18 @@ class EdgeSlaveMixin(MonitorTestMixin): ...@@ -251,6 +280,18 @@ class EdgeSlaveMixin(MonitorTestMixin):
status_json = json.load(fh) status_json = json.load(fh)
self.assertIn('bot_status', status_json) self.assertIn('bot_status', status_json)
def assertConnectionParameterDict(self):
serialised = self.requestDefaultInstance().getConnectionParameterDict()
connection_parameter_dict = json.loads(serialised['_'])
# tested elsewhere
connection_parameter_dict.pop('monitor-setup-url', None)
# comes from instance-monitor.cfg.jinja2, not needed here
connection_parameter_dict.pop('server_log_url', None)
self.assertEqual(
self.expected_connection_parameter_dict,
connection_parameter_dict
)
def test(self): def test(self):
# Note: Those tests do not run surykatka and do not do real checks, as # Note: Those tests do not run surykatka and do not do real checks, as
# this depends too much on the environment and is really hard to # this depends too much on the environment and is really hard to
...@@ -266,82 +307,201 @@ class EdgeSlaveMixin(MonitorTestMixin): ...@@ -266,82 +307,201 @@ class EdgeSlaveMixin(MonitorTestMixin):
self.assertSurykatkaBotPromise() self.assertSurykatkaBotPromise()
self.assertSurykatkaPromises() self.assertSurykatkaPromises()
self.assertSurykatkaCron() self.assertSurykatkaCron()
self.assertConnectionParameterDict()
class TestEdge(EdgeSlaveMixin, SlapOSInstanceTestCase): class TestEdge(EdgeSlaveMixin, SlapOSInstanceTestCase):
expected_connection_parameter_dict = {
'active-region-list': ['1'],
'sla-computer_guid': 'local', 'sla-instance_guid': 'local-edge0'}
surykatka_dict = { surykatka_dict = {
'edge1': {
1: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 3
SQLITE = %(db_file)s
URL =
https://www.checkmaximumelapsedtime1.org/"""},
2: {'expected_ini': """[SURYKATKA] 2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120 INTERVAL = 120
TIMEOUT = 4 TIMEOUT = 4
SQLITE = %(db_file)s SQLITE = %(db_file)s
URL = URL =
https://www.erp5.com/ https://www.checkcertificateexpirationdays.org/
https://www.erp5.org/"""} https://www.checkfrontendiplist.org/
https://www.checkhttpheaderdict.org/
https://www.checkstatuscode.org/
https://www.default.org/
https://www.failureamount.org/"""},
20: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 22
SQLITE = %(db_file)s
URL =
https://www.checkmaximumelapsedtime20.org/"""},
}
} }
def assertSurykatkaPromises(self): def assertSurykatkaPromises(self):
self.assertPromiseContent( self.assertHttpQueryPromiseContent(
'http-query-backend-300-promise.py', 'edge1',
"'ip-list': ''") 'checkcertificateexpirationdays',
self.assertPromiseContent( """extra_config_dict = { 'certificate-expiration-days': '20',
'http-query-backend-300-promise.py', 'failure-amount': '2',
"'report': 'http_query'") 'http-header-dict': '{}',
self.assertPromiseContent( 'ip-list': '',
'http-query-backend-300-promise.py', 'json-file': '%s',
"'status-code': '300'") 'maximum-elapsed-time': '2',
self.assertPromiseContent( 'report': 'http_query',
'http-query-backend-300-promise.py', 'status-code': '200',
"'certificate-expiration-days': '15'") 'url': 'https://www.checkcertificateexpirationdays.org/'}""" % (
self.assertPromiseContent( self.surykatka_dict['edge1'][2]['json-file'],))
'http-query-backend-300-promise.py',
"'url': 'https://www.erp5.org/'") self.assertHttpQueryPromiseContent(
self.assertPromiseContent( 'edge1',
'http-query-backend-300-promise.py', 'checkhttpheaderdict',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],) """extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{"A": "AAA"}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.checkhttpheaderdict.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'checkmaximumelapsedtime1',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '1',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.checkmaximumelapsedtime1.org/'}""" % (
self.surykatka_dict['edge1'][1]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'checkmaximumelapsedtime20',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '20',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.checkmaximumelapsedtime20.org/'}""" % (
self.surykatka_dict['edge1'][20]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'checkstatuscode',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '300',
'url': 'https://www.checkstatuscode.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'default',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.default.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'failureamount',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '10',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.failureamount.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'checkfrontendiplist',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '128.129.130.131 131.134.135.136',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.checkfrontendiplist.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'default',
{'url': 'https://www.default.org/'},
) )
self.assertPromiseContent( self.requestEdgetestSlave(
'http-query-backend-300-promise.py', 'checkstatuscode',
"'failure-amount': '2'" {'url': 'https://www.checkstatuscode.org/', 'check-status-code': 300},
) )
self.requestEdgetestSlave(
self.assertPromiseContent( 'checkhttpheaderdict',
'http-query-backend-promise.py', {'url': 'https://www.checkhttpheaderdict.org/',
"'ip-list': ''") 'check-http-header-dict': {"A": "AAA"}},
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'certificate-expiration-days': '15'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
) )
self.assertPromiseContent( self.requestEdgetestSlave(
'http-query-backend-promise.py', 'checkcertificateexpirationdays',
"'failure-amount': '2'" {'url': 'https://www.checkcertificateexpirationdays.org/',
'check-certificate-expiration-days': '20'},
) )
def requestEdgetestSlaves(self):
self.requestEdgetestSlave( self.requestEdgetestSlave(
'backend', 'checkmaximumelapsedtime20',
{'url': 'https://www.erp5.com/'}, {'url': 'https://www.checkmaximumelapsedtime20.org/',
'check-maximum-elapsed-time': 20},
) )
self.requestEdgetestSlave( self.requestEdgetestSlave(
'backend-300', 'checkmaximumelapsedtime1',
{'url': 'https://www.erp5.org/', 'check-status-code': '300'}, {'url': 'https://www.checkmaximumelapsedtime1.org/',
'check-maximum-elapsed-time': 1},
)
self.requestEdgetestSlave(
'failureamount',
{'url': 'https://www.failureamount.org/', 'failure-amount': '10'},
)
self.requestEdgetestSlave(
'checkfrontendiplist',
{'url': 'https://www.checkfrontendiplist.org/',
'check-frontend-ip-list': ['128.129.130.131', '131.134.135.136']},
) )
class TestEdgeNameserverCheckFrontendIp( class TestEdgeNameserverListCheckFrontendIpList(
EdgeSlaveMixin, SlapOSInstanceTestCase): EdgeSlaveMixin, SlapOSInstanceTestCase):
expected_connection_parameter_dict = {
'active-region-list': ['1'], 'sla-computer_guid': 'local',
'sla-instance_guid': 'local-edge0'}
surykatka_dict = { surykatka_dict = {
'edge1': {
2: {'expected_ini': """[SURYKATKA] 2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120 INTERVAL = 120
TIMEOUT = 4 TIMEOUT = 4
...@@ -353,36 +513,44 @@ NAMESERVER = ...@@ -353,36 +513,44 @@ NAMESERVER =
URL = URL =
https://www.erp5.com/"""} https://www.erp5.com/"""}
} }
}
@classmethod @classmethod
def getInstanceParameterDict(cls): def getInstanceParameterDict(cls):
return { return {'_': json.dumps({
'nameserver': '127.0.1.1 127.0.1.2', 'nameserver-list': ['127.0.1.1', '127.0.1.2'],
'check-frontend-ip': '127.0.0.1 127.0.0.2', 'check-frontend-ip-list': ['127.0.0.1', '127.0.0.2'],
} })}
def assertSurykatkaPromises(self): def assertSurykatkaPromises(self):
self.assertPromiseContent( self.assertHttpQueryPromiseContent(
'http-query-backend-promise.py', 'edge1',
'backend',
"'ip-list': '127.0.0.1 127.0.0.2'") "'ip-list': '127.0.0.1 127.0.0.2'")
self.assertPromiseContent( self.assertHttpQueryPromiseContent(
'http-query-backend-promise.py', 'edge1',
'backend',
"'report': 'http_query'") "'report': 'http_query'")
self.assertPromiseContent( self.assertHttpQueryPromiseContent(
'http-query-backend-promise.py', 'edge1',
'backend',
"'status-code': '200'") "'status-code': '200'")
self.assertPromiseContent( self.assertHttpQueryPromiseContent(
'http-query-backend-promise.py', 'edge1',
'backend',
"'certificate-expiration-days': '15'") "'certificate-expiration-days': '15'")
self.assertPromiseContent( self.assertHttpQueryPromiseContent(
'http-query-backend-promise.py', 'edge1',
'backend',
"'url': 'https://www.erp5.com/'") "'url': 'https://www.erp5.com/'")
self.assertPromiseContent( self.assertHttpQueryPromiseContent(
'http-query-backend-promise.py', 'edge1',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],) 'backend',
"'json-file': '%s'" % (self.surykatka_dict['edge1'][2]['json-file'],)
) )
self.assertPromiseContent( self.assertHttpQueryPromiseContent(
'http-query-backend-promise.py', 'edge1',
'backend',
"'failure-amount': '2'" "'failure-amount': '2'"
) )
...@@ -393,426 +561,833 @@ URL = ...@@ -393,426 +561,833 @@ URL =
) )
class TestEdgeCheckStatusCode(EdgeSlaveMixin, SlapOSInstanceTestCase): class TestEdgeSlaveNotJson(
EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_dict = { surykatka_dict = {
'edge1': {
2: {'expected_ini': """[SURYKATKA] 2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120 INTERVAL = 120
TIMEOUT = 4 TIMEOUT = 4
SQLITE = %(db_file)s SQLITE = %(db_file)s
URL = URL =
https://www.erp5.com/ https://www.erp5.com/"""}
https://www.erp5.org/"""}
} }
@classmethod
def getInstanceParameterDict(cls):
return {
'check-status-code': '500',
} }
def assertSurykatkaPromises(self): # non-json provided in slave '_' results with damaging the cluster
self.assertPromiseContent( # test here is to expose real problem, which has no solution for now
'http-query-backend-501-promise.py', @unittest.expectedFailure
"'ip-list': ''") def test(self):
self.assertPromiseContent( EdgeSlaveMixin.test()
'http-query-backend-501-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'status-code': '501'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'certificate-expiration-days': '15'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'failure-amount': '2'"
)
self.assertPromiseContent( def assertSurykatkaPromises(self):
'http-query-backend-promise.py', self.assertHttpQueryPromiseContent(
"'ip-list': ''") 'default',
self.assertPromiseContent( """extra_config_dict = { 'certificate-expiration-days': '15',
'http-query-backend-promise.py', 'failure-amount': '2',
"'report': 'http_query'") 'http-header-dict': '{}',
self.assertPromiseContent( 'ip-list': '',
'http-query-backend-promise.py', 'json-file': '%s',
"'status-code': '500'") 'maximum-elapsed-time': '2',
self.assertPromiseContent( 'report': 'http_query',
'http-query-backend-promise.py', 'status-code': '200',
"'certificate-expiration-days': '15'") 'url': 'https://www.default.org/'}""" % (
self.assertPromiseContent( self.surykatka_dict[2]['json-file'],))
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'failure-amount': '2'"
)
def requestEdgetestSlaves(self): def requestEdgetestSlaves(self):
self.requestEdgetestSlave( self.requestEdgetestSlave(
'backend', 'default',
{'url': 'https://www.erp5.com/'}, {'url': 'https://www.default.org/'},
) )
self.requestEdgetestSlave( software_url = self.getSoftwareURL()
'backend-501', self.slap.request(
{'url': 'https://www.erp5.org/', 'check-status-code': '501'}, software_release=software_url,
software_type='edgetest',
partition_reference='notajson',
partition_parameter_kw={'_': 'notajson'},
shared=True
) )
class TestEdgeCheckHTTPHeaderDict(EdgeSlaveMixin, SlapOSInstanceTestCase): class TestEdgeRegion(EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_dict = { def setUpMonitorConfigurationList(self):
2: {'expected_ini': """[SURYKATKA] self.monitor_configuration_list = [
INTERVAL = 120 {
TIMEOUT = 4 'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
SQLITE = %(db_file)s 'text': 'testing partition 0',
URL = 'title': 'testing partition 0',
https://www.erp5.com/ 'type': 'rss',
https://www.erp5.org/"""} 'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region One',
'title': 'edgebot-Region One',
'type': 'rss',
'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region Three',
'title': 'edgebot-Region Three',
'type': 'rss',
'url': 'https://[%s]:9702/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region Two',
'title': 'edgebot-Region Two',
'type': 'rss',
'url': 'https://[%s]:9703/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,)
} }
]
@classmethod @classmethod
def getInstanceParameterDict(cls): def setUpClassParameter(cls):
return { cls.instance_parameter_dict = {
'check-http-header-dict': 'region-dict': {
'{"B": "BBB"}', 'Region One': {
'sla-computer_guid': 'local',
'state': 'started',
'nameserver-list': ['127.0.1.1', '127.0.1.2'],
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
},
'Region Two': {
'sla-computer_guid': 'local',
'state': 'started',
'nameserver-list': ['127.0.2.1', '127.0.2.2'],
},
'Region Three': {
'sla-computer_guid': 'local',
'state': 'started',
'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
} }
}
}
cls.expected_connection_parameter_dict = {
'active-region-list': [
'Region One', 'Region Three', 'Region Two'],
'sla-computer_guid': 'local', 'sla-instance_guid': 'local-edge0'}
def assertSurykatkaPromises(self): @classmethod
self.assertPromiseContent( def setUpClass(cls):
'http-query-backend-http-header-promise.py', cls.setUpClassParameter()
"'ip-list': ''") super().setUpClass()
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'http-header-dict': '{\"A\": \"AAA\"}'")
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'certificate-expiration-days': '15'")
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-http-header-promise.py',
"'failure-amount': '2'"
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'http-header-dict': '{\"B\": \"BBB\"}'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'certificate-expiration-days': '15'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'failure-amount': '2'"
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
)
self.requestEdgetestSlave(
'backend-http-header',
{'url': 'https://www.erp5.org/', 'check-http-header-dict': '{"A": "AAA"}'},
)
class TestEdgeCheckCertificateExpirationDays( def setUpParameter(self):
EdgeSlaveMixin, SlapOSInstanceTestCase): self.surykatka_dict = {
surykatka_dict = { 'edge1': {
2: {'expected_ini': """[SURYKATKA] 2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120 INTERVAL = 120
TIMEOUT = 4 TIMEOUT = 4
SQLITE = %(db_file)s SQLITE = %(db_file)s
URL = NAMESERVER =
https://www.erp5.com/ 127.0.1.1
https://www.erp5.org/"""} 127.0.1.2
}
@classmethod
def getInstanceParameterDict(cls):
return {
'check-certificate-expiration-days': '10',
}
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'certificate-expiration-days': '20'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'failure-amount': '2'"
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'certificate-expiration-days': '10'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'failure-amount': '2'"
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
)
self.requestEdgetestSlave(
'backend-20',
{'url': 'https://www.erp5.org/',
'check-certificate-expiration-days': '20'},
)
class TestEdgeCheckMaximumElapsedTime(
EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_dict = {
5: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 7
SQLITE = %(db_file)s
URL = URL =
https://www.erp5.com/"""}, https://www.all.org/
20: {'expected_ini': """[SURYKATKA] https://www.globalcheck.org/
https://www.onetwo.org/
https://www.specificcheck.org/
https://www.specificoverride.org/"""},
},
'edge2': {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120 INTERVAL = 120
TIMEOUT = 22 TIMEOUT = 4
SQLITE = %(db_file)s SQLITE = %(db_file)s
URL = URL =
https://www.erp5.org/"""}, https://www.all.org/
1: {'expected_ini': """[SURYKATKA] https://www.three.org/"""},
},
'edge3': {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120 INTERVAL = 120
TIMEOUT = 3 TIMEOUT = 4
SQLITE = %(db_file)s SQLITE = %(db_file)s
NAMESERVER =
127.0.2.1
127.0.2.2
URL = URL =
https://www.erp5.net/"""} https://www.all.org/
https://www.onetwo.org/
https://www.parialmiss.org/
https://www.specificoverride.org/"""},
} }
}
def setUp(self):
self.setUpParameter()
super().setUp()
@classmethod @classmethod
def getInstanceParameterDict(cls): def getInstanceParameterDict(cls):
return { return {'_': json.dumps(cls.instance_parameter_dict)}
'check-maximum-elapsed-time': '5',
slave_parameter_dict_dict = {
'all': {
'url': 'https://www.all.org/'
},
'onetwo': {
'url': 'https://www.onetwo.org/',
'region-dict': {'Region One': {}, 'Region Two': {}}
},
'three': {
'url': 'https://www.three.org/',
'region-dict': {'Region Three': {}}
},
'missed': {
'url': 'https://www.missed.org/',
'region-dict': {'Region Non Existing': {}}
},
'partialmiss': {
'url': 'https://www.parialmiss.org/',
'region-dict': {'Region Two': {}, 'Region Non Existing': {}}
},
'specificcheck': {
'url': 'https://www.specificcheck.org/',
'region-dict': {
'Region One': {'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2']}}
},
'globalcheck': {
'url': 'https://www.globalcheck.org/',
'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
'region-dict': {'Region One': {}}
},
'specificoverride': {
'url': 'https://www.specificoverride.org/',
'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
'region-dict': {
'Region One': {'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8']},
'Region Two': {}}
},
} }
def requestEdgetestSlaves(self):
for reference, parameter_dict in self.slave_parameter_dict_dict.items():
self.requestEdgetestSlave(reference, parameter_dict)
def assertSurykatkaPromises(self): def assertSurykatkaPromises(self):
self.assertPromiseContent( self.assertHttpQueryPromiseContent(
'http-query-backend-20-promise.py', 'edge1',
"'ip-list': ''") 'all',
self.assertPromiseContent( """extra_config_dict = { 'certificate-expiration-days': '15',
'http-query-backend-20-promise.py', 'failure-amount': '2',
"'report': 'http_query'") 'http-header-dict': '{}',
self.assertPromiseContent( 'ip-list': '127.0.1.3 127.0.1.4',
'http-query-backend-20-promise.py', 'json-file': '%s',
"'status-code': '200'") 'maximum-elapsed-time': '2',
self.assertPromiseContent( 'report': 'http_query',
'http-query-backend-20-promise.py', 'status-code': '200',
"'maximum-elapsed-time': '20'") 'url': 'https://www.all.org/'}""" % (
self.assertPromiseContent( self.surykatka_dict['edge1'][2]['json-file'],))
'http-query-backend-20-promise.py',
"'url': 'https://www.erp5.org/'") self.assertHttpQueryPromiseContent(
self.assertPromiseContent( 'edge1',
'http-query-backend-20-promise.py', 'specificcheck',
"'json-file': '%s'" % (self.surykatka_dict[20]['json-file'],) """extra_config_dict = { 'certificate-expiration-days': '15',
) 'failure-amount': '2',
self.assertPromiseContent( 'http-header-dict': '{}',
'http-query-backend-20-promise.py', 'ip-list': '99.99.99.1 99.99.99.2',
"'failure-amount': '2'" 'json-file': '%s',
) 'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.specificcheck.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'globalcheck',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '99.99.99.3 99.99.99.4',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.globalcheck.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'specificoverride',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '99.99.99.7 99.99.99.8',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.specificoverride.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge1',
'onetwo',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '127.0.1.3 127.0.1.4',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.onetwo.org/'}""" % (
self.surykatka_dict['edge1'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge2',
'all',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '127.0.3.1 127.0.3.2',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.all.org/'}""" % (
self.surykatka_dict['edge2'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge2',
'three',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '127.0.3.1 127.0.3.2',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.three.org/'}""" % (
self.surykatka_dict['edge2'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge3',
'all',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.all.org/'}""" % (
self.surykatka_dict['edge3'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge3',
'onetwo',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.onetwo.org/'}""" % (
self.surykatka_dict['edge3'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge3',
'partialmiss',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.parialmiss.org/'}""" % (
self.surykatka_dict['edge3'][2]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge3',
'specificoverride',
"""extra_config_dict = { 'certificate-expiration-days': '15',
'failure-amount': '2',
'http-header-dict': '{}',
'ip-list': '99.99.99.5 99.99.99.6',
'json-file': '%s',
'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.specificoverride.org/'}""" % (
self.surykatka_dict['edge3'][2]['json-file'],))
self.assertPromiseContent( def test(self):
'http-query-backend-default-promise.py', super(TestEdgeRegion, self).test()
"'ip-list': ''") self.assertSlaveConnectionParameterDict()
self.assertPromiseContent(
'http-query-backend-default-promise.py', maxDiff = None
"'report': 'http_query'")
self.assertPromiseContent( expected_slave_connection_parameter_dict_dict = {
'http-query-backend-default-promise.py', 'all': {
"'status-code': '200'") 'available-region-list': [
self.assertPromiseContent( 'Region One', 'Region Three', 'Region Two'],
'http-query-backend-default-promise.py', 'assigned-region-dict': {
"'maximum-elapsed-time': '5'") 'Region One': {
self.assertPromiseContent( 'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
'http-query-backend-default-promise.py', 'nameserver-list': ['127.0.1.1', '127.0.1.2']
"'url': 'https://www.erp5.com/'") },
self.assertPromiseContent( 'Region Three': {
'http-query-backend-default-promise.py', 'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
"'json-file': '%s'" % (self.surykatka_dict[5]['json-file'],) 'nameserver-list': []
) },
self.assertPromiseContent( 'Region Two': {
'http-query-backend-default-promise.py', 'check-frontend-ip-list': [],
"'failure-amount': '2'" 'nameserver-list': ['127.0.2.1', '127.0.2.2']
) }
}
},
'onetwo': {
'available-region-list': [
'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
},
'specificcheck': {
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
}
},
'available-region-list': [
'Region One', 'Region Three', 'Region Two']
},
'specificoverride': {
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
},
'available-region-list': [
'Region One', 'Region Three', 'Region Two']
},
'three': {
'available-region-list': [
'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region Three': {
'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
'nameserver-list': []
}
}
},
'globalcheck': {
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
}
},
'available-region-list': [
'Region One', 'Region Three', 'Region Two']
},
'missed': {
'available-region-list': [
'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
}
},
'partialmiss': {
'available-region-list': [
'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
}
}
self.assertPromiseContent( def assertSlaveConnectionParameterDict(self):
'http-query-backend-1-promise.py', slave_connection_parameter_dict_dict = {}
"'ip-list': ''") for reference, parameter_dict in self.slave_parameter_dict_dict.items():
self.assertPromiseContent( slave_connection_parameter_dict_dict[
'http-query-backend-1-promise.py', reference] = self.requestEdgetestSlave(
"'report': 'http_query'") reference, parameter_dict).getConnectionParameterDict()
self.assertPromiseContent( # unload the json
'http-query-backend-1-promise.py', slave_connection_parameter_dict_dict[
"'status-code': '200'") reference] = json.loads(
self.assertPromiseContent( slave_connection_parameter_dict_dict[reference].pop('_'))
'http-query-backend-1-promise.py', self.assertEqual(
"'maximum-elapsed-time': '1'") self.expected_slave_connection_parameter_dict_dict,
self.assertPromiseContent( slave_connection_parameter_dict_dict
'http-query-backend-1-promise.py',
"'url': 'https://www.erp5.net/'")
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[1]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'failure-amount': '2'"
) )
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend-default',
{'url': 'https://www.erp5.com/'},
)
self.requestEdgetestSlave(
'backend-20',
{'url': 'https://www.erp5.org/',
'check-maximum-elapsed-time': '20'},
)
self.requestEdgetestSlave(
'backend-1',
{'url': 'https://www.erp5.net/',
'check-maximum-elapsed-time': '1'},
)
class TestEdgeRegionDestroyed(TestEdgeRegion):
def setUpMonitorConfigurationList(self):
# already for destroyed case, as test_monitor_setup will be called after
# test
self.monitor_configuration_list = [
{
'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
'text': 'testing partition 0',
'title': 'testing partition 0',
'type': 'rss',
'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region One',
'title': 'edgebot-Region One',
'type': 'rss',
'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region Two',
'title': 'edgebot-Region Two',
'type': 'rss',
'url': 'https://[%s]:9703/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,)
}
]
class TestEdgeFailureAmount( def test(self):
EdgeSlaveMixin, SlapOSInstanceTestCase): super(TestEdgeRegionDestroyed, self).test()
surykatka_dict = { # hack around @classmethod
2: {'expected_ini': """[SURYKATKA] self.__class__.instance_parameter_dict[
INTERVAL = 120 'region-dict']['Region Three']['state'] = 'destroyed'
TIMEOUT = 4 # Region was removed
SQLITE = %(db_file)s self.__class__.expected_connection_parameter_dict[
URL = 'active-region-list'].remove('Region Three')
https://www.erp5.com/
https://www.erp5.org/"""} self.__class__._instance_parameter_dict = self.getInstanceParameterDict()
self.requestDefaultInstance()
# give time to stabilise the tree
self.slap.waitForInstance(max_retry=4)
self.assertConnectionParameterDict()
self.expected_slave_connection_parameter_dict_dict = {
'all': {
'available-region-list': [
'Region One', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
},
'onetwo': {
'available-region-list': [
'Region One', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
} }
}
},
'specificcheck': {
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
}
},
'available-region-list': [
'Region One', 'Region Two']
},
'specificoverride': {
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
},
'available-region-list': [
'Region One', 'Region Two']
},
'three': {
'assigned-region-dict': {},
'available-region-list': [
'Region One', 'Region Two'],
},
'globalcheck': {
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
}
},
'available-region-list': [
'Region One', 'Region Two']
},
'missed': {
'available-region-list': [
'Region One', 'Region Two'],
'assigned-region-dict': {
}
},
'partialmiss': {
'available-region-list': [
'Region One', 'Region Two'],
'assigned-region-dict': {
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
}
}
self.assertSlaveConnectionParameterDict()
@classmethod
def getInstanceParameterDict(cls): class TestEdgeRegionAdded(TestEdgeRegion):
return { def setUpMonitorConfigurationList(self):
'failure-amount': '5' # already for added case, as test_monitor_setup will be called after test
self.monitor_configuration_list = [
{
'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
'text': 'testing partition 0',
'title': 'testing partition 0',
'type': 'rss',
'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region Four',
'title': 'edgebot-Region Four',
'type': 'rss',
'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region One',
'title': 'edgebot-Region One',
'type': 'rss',
'url': 'https://[%s]:9702/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region Three',
'title': 'edgebot-Region Three',
'type': 'rss',
'url': 'https://[%s]:9703/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,)
},
{
'htmlUrl': 'https://[%s]:9704/public/feed' % (self._ipv6_address,),
'text': 'edgebot-Region Two',
'title': 'edgebot-Region Two',
'type': 'rss',
'url': 'https://[%s]:9704/share/private/' % (self._ipv6_address,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:9704/public/feed' % (self._ipv6_address,)
} }
]
def assertSurykatkaPromises(self): def test(self):
self.assertPromiseContent( super(TestEdgeRegionAdded, self).test()
'http-query-backend-promise.py', self.__class__.instance_parameter_dict['region-dict']['Region Four'] = {
"'report': 'http_query'") 'sla-computer_guid': 'local',
self.assertPromiseContent( 'state': 'started',
'http-query-backend-promise.py', 'nameserver-list': ['127.0.4.1', '127.0.4.2'],
"'status-code': '200'") 'check-frontend-ip-list': ['127.0.4.3', '127.0.4.4'],
self.assertPromiseContent( }
'http-query-backend-promise.py', # Region was added
"'url': 'https://www.erp5.com/'") self.__class__.expected_connection_parameter_dict[
self.assertPromiseContent( 'active-region-list'].insert(0, 'Region Four')
'http-query-backend-promise.py', self.__class__._instance_parameter_dict = self.getInstanceParameterDict()
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],) self.requestDefaultInstance()
) # give time to stabilise the tree, 6 times as new node is added
self.assertPromiseContent( self.slap.waitForInstance(max_retry=6)
'http-query-backend-promise.py', # XXX: few more times, but ignoring good result from promises, as there is
"'failure-amount': '5'" # "Unknown Instance" of just added node which is not caught by any
) # promise, but in the end it shall get better
for f in range(5):
try:
self.slap.waitForInstance()
except Exception:
pass
self.slap.waitForInstance()
self.assertConnectionParameterDict()
self.expected_slave_connection_parameter_dict_dict = {
'all': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region Four': {
'check-frontend-ip-list': ['127.0.4.3', '127.0.4.4'],
'nameserver-list': ['127.0.4.1', '127.0.4.2']
},
'Region One': {
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Three': {
'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
'nameserver-list': []
},
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
},
'onetwo': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
},
'specificcheck': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
}
},
},
'specificoverride': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
},
'Region Two': {
'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
},
},
'three': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region Three': {
'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
'nameserver-list': []
}
}
},
'globalcheck': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region One': {
'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
'nameserver-list': ['127.0.1.1', '127.0.1.2']
}
},
},
'missed': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
}
},
'partialmiss': {
'available-region-list': [
'Region Four', 'Region One', 'Region Three', 'Region Two'],
'assigned-region-dict': {
'Region Two': {
'check-frontend-ip-list': [],
'nameserver-list': ['127.0.2.1', '127.0.2.2']
}
}
}
}
self.assertSlaveConnectionParameterDict()
self.surykatka_dict['edge4'] = {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 4
SQLITE = %(dbfile)
NAMESERVER =
127.0.4.1
127.0.4.2
self.assertPromiseContent( URL =
'http-query-backend-10-promise.py', https://www.all.org/"""},
"'report': 'http_query'") }
self.assertPromiseContent( self.updateSurykatkaDict()
'http-query-backend-10-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-10-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-10-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-10-promise.py',
"'failure-amount': '10'"
)
def requestEdgetestSlaves(self): self.assertHttpQueryPromiseContent(
self.requestEdgetestSlave( 'edge4',
'backend', 'all',
{'url': 'https://www.erp5.com/'}, """{ 'certificate-expiration-days': '15',
) 'failure-amount': '2',
self.requestEdgetestSlave( 'http-header-dict': '{}',
'backend-10', 'ip-list': '127.0.4.3 127.0.4.4',
{'url': 'https://www.erp5.org/', 'failure-amount': '10'}, 'json-file': '%s',
) 'maximum-elapsed-time': '2',
'report': 'http_query',
'status-code': '200',
'url': 'https://www.all.org/'}""" % (
self.surykatka_dict['edge4'][2]['json-file'],))
...@@ -18,7 +18,7 @@ md5sum = 2bd1779425b7561682c0de5496d808ed ...@@ -18,7 +18,7 @@ md5sum = 2bd1779425b7561682c0de5496d808ed
[root-common] [root-common]
filename = root-common.cfg.in filename = root-common.cfg.in
md5sum = 8e28f599247ad604ec6e32df410412a8 md5sum = c13b4f1a5aa526a8d3f8e02bf6baf785
[instance-neo-admin] [instance-neo-admin]
filename = instance-neo-admin.cfg.in filename = instance-neo-admin.cfg.in
......
...@@ -47,7 +47,7 @@ ...@@ -47,7 +47,7 @@
"type": "object" "type": "object"
}, },
"sla-dict": { "sla-dict": {
"description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...). The prefix 'node-' is mandatory and the number must start from 0. The total number of nodes here must be equal to the length of node-list.", "description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...). The prefix 'node-' is mandatory and the number must start from 0. The total number of nodes here must be equal to the length of node-list. A node can be removed by requesting it without any admin, master and storage.",
"additionalProperties": { "additionalProperties": {
"type": "array", "type": "array",
"items": { "items": {
...@@ -95,7 +95,7 @@ ...@@ -95,7 +95,7 @@
"type": "integer" "type": "integer"
}, },
"storage-count": { "storage-count": {
"description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage.", "description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage. 0 to disable.",
"default": 1, "default": 1,
"type": "integer" "type": "integer"
}, },
......
...@@ -89,7 +89,7 @@ software-type = {{ software_type }} ...@@ -89,7 +89,7 @@ software-type = {{ software_type }}
config-autostart = {{ dumps(sum(storage_count)) }} config-autostart = {{ dumps(sum(storage_count)) }}
{%- do assert(replicas < len(node_list)) %} {%- do assert(replicas < len(node_list)) %}
{%- set monitor = set() %} {%- set monitor = set() %}
{%- for i, node in enumerate(node_list) %} {%- for node in node_list %}
{%- set port = node.get('monitor') %} {%- set port = node.get('monitor') %}
{%- if port %} {%- if port %}
{%- do monitor.add(node.get('admin') != 0 and port) %} {%- do monitor.add(node.get('admin') != 0 and port) %}
...@@ -101,13 +101,14 @@ config-autostart = {{ dumps(sum(storage_count)) }} ...@@ -101,13 +101,14 @@ config-autostart = {{ dumps(sum(storage_count)) }}
{%- endif %} {%- endif %}
{%- for i, node in enumerate(node_list) %} {%- for i, node in enumerate(node_list) %}
{%- set section_id = prefix ~ i %} {%- set section_id = prefix ~ i %}
{%- do section_id_list.append(section_id) %}
{%- if node.get('master') == 0 and not node.get('monitor') %} {%- if node.get('master') == 0 and not node.get('monitor') %}
{%- do node.setdefault('admin', 0) %} {%- do node.setdefault('admin', 0) %}
{%- endif %} {%- endif %}
{%- if monitor or node.get('admin') == 0 %} {%- if monitor or node.get('admin') == 0 %}
{%- do node.setdefault('monitor', 0) %} {%- do node.setdefault('monitor', 0) %}
{%- endif %} {%- endif %}
{%- for x in 'admin', 'master', 'storage-count' if node.get(x, 1) %}
{%- do section_id_list.append(section_id) %}
[{{section_id}}] [{{section_id}}]
<= {{ prefix }}request-common <= {{ prefix }}request-common
...@@ -136,6 +137,9 @@ config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }} ...@@ -136,6 +137,9 @@ config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
config-{{ k }} = {{ dumps(v) }} config-{{ k }} = {{ dumps(v) }}
{%- endfor %} {%- endfor %}
{{ sla(section_id) }} {{ sla(section_id) }}
{%- break %}
{%- endfor %}
{%- endfor %} {%- endfor %}
{%- do assert(len(monitor) == 1, monitor) %} {%- do assert(len(monitor) == 1, monitor) %}
......
...@@ -31,3 +31,7 @@ if returncode: ...@@ -31,3 +31,7 @@ if returncode:
subprocess.call( subprocess.call(
('tar', '-caf', result('db.tar.xz'), 'neo_tests'), ('tar', '-caf', result('db.tar.xz'), 'neo_tests'),
cwd=tempfile.gettempdir()) cwd=tempfile.gettempdir())
p = subprocess.Popen(('journalctl', '-o', 'export'), stdout=subprocess.PIPE)
with open(result('journalctl-export.xz'), 'wb') as f:
subprocess.call(('xz',), stdin=p.stdout, stdout=f)
p.wait()
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
[instance] [instance]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = f3aac995aa1129ca58f9a92851ad7091 md5sum = bfd488ba023f505be25d947ec830bab3
[yarn.lock] [yarn.lock]
filename = yarn.lock filename = yarn.lock
......
...@@ -7,7 +7,7 @@ theia-environment-parts = ...@@ -7,7 +7,7 @@ theia-environment-parts =
slapos-repository slapos-repository
runner-link runner-link
settings.json settings.json
request-script request-script-template
theia-parts = theia-parts =
frontend-reload frontend-reload
...@@ -230,7 +230,7 @@ recipe = plone.recipe.command ...@@ -230,7 +230,7 @@ recipe = plone.recipe.command
filename = logo.png filename = logo.png
full-path = $${directory:frontend-static}/$${:filename} full-path = $${directory:frontend-static}/$${:filename}
command = command =
cp -f ${logo.png:output} $${:full-path} cp --remove-destination ${logo.png:output} $${:full-path}
stop-on-error = true stop-on-error = true
[frontend-instance-slapos.css] [frontend-instance-slapos.css]
...@@ -551,13 +551,15 @@ recipe = slapos.cookbook:symbolic.link ...@@ -551,13 +551,15 @@ recipe = slapos.cookbook:symbolic.link
target-directory = $${directory:project} target-directory = $${directory:project}
link-binary = $${directory:runner} link-binary = $${directory:runner}
[request-script] [request-script-template]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
rendered = $${directory:project}/$${:_buildout_section_name_}.sh rendered = $${directory:project}/$${:_buildout_section_name_}.sh
mode = 0700 mode = 0700
template = template =
inline:#!/bin/sh inline:#!/bin/sh
software_name=html5as-base #replace the software name # This template is generated automatically, copy it in order to supply and request.
# Any manual change to this file may be lost.
software_name=html5as-base #replace the software name writen in ~/srv/project/slapos/software/
software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg
# slapos supply is used to add the software to the software list to be supplied to a node. # slapos supply is used to add the software to the software list to be supplied to a node.
slapos supply $software_release_uri slaprunner slapos supply $software_release_uri slaprunner
......
...@@ -183,7 +183,7 @@ class TestTheia(TheiaTestCase): ...@@ -183,7 +183,7 @@ class TestTheia(TheiaTestCase):
self.computer_partition_root_path, self.computer_partition_root_path,
'srv', 'srv',
'project', 'project',
'request-script.sh', 'request-script-template.sh',
) )
self.assertTrue(os.path.exists(script_path)) self.assertTrue(os.path.exists(script_path))
......
...@@ -631,9 +631,6 @@ PasteDeploy = 1.5.2 ...@@ -631,9 +631,6 @@ PasteDeploy = 1.5.2
argparse = 1.4.0 argparse = 1.4.0
zope.dottedname = 4.1.0 zope.dottedname = 4.1.0
# test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
Products.PluggableAuthService = 1.9.0
# modified version that works fine for buildout installation # modified version that works fine for buildout installation
SOAPpy = 0.12.0nxd001 SOAPpy = 0.12.0nxd001
...@@ -664,6 +661,7 @@ Products.GenericSetup = 1.8.6 ...@@ -664,6 +661,7 @@ Products.GenericSetup = 1.8.6
Products.LongRequestLogger = 2.1.0 Products.LongRequestLogger = 2.1.0
# Products.MimetypesRegistry 2.1 requires AccessControl>=3.0.0Acquisition. # Products.MimetypesRegistry 2.1 requires AccessControl>=3.0.0Acquisition.
Products.MimetypesRegistry = 2.0.10 Products.MimetypesRegistry = 2.0.10
Products.PluggableAuthService = 1.10.0
Products.PluginRegistry = 1.4 Products.PluginRegistry = 1.4
Products.TIDStorage = 5.5.0 Products.TIDStorage = 5.5.0
pyPdf = 1.13 pyPdf = 1.13
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment