Commit f9417613 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents c4b68ad5 873a49f6
......@@ -20,7 +20,7 @@ patch-options = -p1
patches =
${:_profile_base_location_}/noroot.patch#05fc6333e05576ea8e5a49f27a6ef951
configure-command = make
configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl-1.0:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl-1.0:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl-1.0:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
make-targets = non-interactive-package install_root=${:location}
environment =
PATH=${patch:location}/bin:${m4:location}/bin:%(PATH)s
......@@ -15,15 +15,15 @@
[template]
filename = instance.cfg.in
md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe
md5sum = 399b398a8eabfa6126d2a521dc779f9b
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = 31b17b55200ea065cb97085283ef5568
md5sum = 704b6ac6bf42837bcd8f4582c5a746c0
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = 28a00c28a972f42627849b25c2792abb
md5sum = 80b9b70ba1ccbc09deb8f9cad60f352c
[template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2
......@@ -31,7 +31,7 @@ md5sum = 7de5756f59ef7d823cd8ed33e6d15230
[template-kvm-import]
filename = instance-kvm-import.cfg.jinja2.in
md5sum = bd7e5db872b0dbe7716ec49c3907c401
md5sum = 3e7ff2ba85762ca47b5f90495d492570
[template-kvm-import-script]
filename = template/kvm-import.sh.jinja2
......@@ -39,7 +39,7 @@ md5sum = cd0008f1689dfca9b77370bc4d275b70
[template-kvm-export]
filename = instance-kvm-export.cfg.jinja2
md5sum = f12df4256eb5bd31a01c0ddc4b3897bb
md5sum = c15f7600389b0c641622fcfdc42260d8
[template-kvm-export-script]
filename = template/kvm-export.sh.jinja2
......@@ -88,3 +88,11 @@ md5sum = 9c67058edcc4edae0b57956c0932a9fc
[image-download-config-creator]
_update_hash_filename_ = template/image-download-config-creator.py
md5sum = 54261e418ab9860efe73efd514c4d47f
[whitelist-firewall-download-controller]
_update_hash_filename_ = template/whitelist-firewall-download-controller.py
md5sum = bc64e29546833817636261d1b28aa6dc
[whitelist-domains-default]
_update_hash_filename_ = template/whitelist-domains-default
md5sum = e9d40162ba77472775256637a2617d14
......@@ -556,6 +556,12 @@
"title": "FreeBSD 12.1 RELEASE bootonly x86_64"
}
]
},
"whitelist-domains": {
"title": "Whitelist domains",
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string",
"textarea": true
}
},
"type": "object"
......
......@@ -107,6 +107,12 @@
"title": "FreeBSD 12.1 RELEASE bootonly x86_64"
}
]
},
"whitelist-domains": {
"title": "Whitelist domains",
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string",
"textarea": true
}
},
"type": "object"
......
......@@ -24,7 +24,7 @@ global-ipv6 = {{ ipv6 }}
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{{ k }} = {{ dumps(v) }}
{% endfor -%}
[request-common]
......@@ -135,14 +135,13 @@ config-document-host = ${apache-conf:ip}
config-document-port = ${apache-conf:port}
config-document-path = ${hash-code:passwd}
config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }}
{%- if 'boot-image-url-list' in kvm_parameter_dict %}
{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
config-boot-image-url-list = {{ kvm_parameter_dict['boot-image-url-list'] }}
{%- endif %}
{%- if 'boot-image-url-select' in kvm_parameter_dict %}
{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
config-boot-image-url-select = {{ kvm_parameter_dict['boot-image-url-select'] }}
{%- for k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains'] %}
{#- play nice - use parameter only if present #}
{%- if k in kvm_parameter_dict %}
{#- play safe - dumps value #}
config-{{ k }} = {{ dumps(kvm_parameter_dict[k]) }}
{%- endif %}
{%- endfor %}
config-type = cluster
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url', kvm_parameter_dict.get('bootstrap-script-url', '')) -%}
......
......@@ -5,24 +5,11 @@ extends =
parts +=
cron-entry-backup
certificate-authority
certificate-authority-service
publish-connection-information
kvm-vnc-promise
kvm-disk-image-corruption-promise
websockify-sighandler
websockify-sighandler-service
novnc-promise
cron
cron-service
frontend-promise
# monitor parts
monitor-base
${instance-kvm-parts:parts}
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{{ k }} = {{ dumps(v) }}
{% endfor -%}
# Create the exporter executable, which is a simple shell script
......
......@@ -14,7 +14,7 @@ extends =
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{{ k }} = {{ dumps(v) }}
{% endfor -%}
[resilient-publish-connection-parameter]
......
......@@ -419,6 +419,12 @@
"title": "FreeBSD 12.1 RELEASE bootonly x86_64"
}
]
},
"whitelist-domains": {
"title": "Whitelist domains",
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string",
"textarea": true
}
}
}
{# Workaround empty parameter passing #}
{# In case of resilient '' is converted to 'None' string, but with slapproxy '' becomes None #}
{% for k, v in slapparameter_dict.items() %}
{% if v == 'None' or v is none %}
{% do slapparameter_dict.__setitem__(k, '') %}
{% endif %}
{% endfor %}
{% set additional_frontend = slapparameter_dict.get('frontend-additional-instance-guid') %}
{% set enable_http = str(slapparameter_dict.get('enable-http-server', False)).lower() == 'true' -%}
{% set use_tap = str(slapparameter_dict.get('use-tap', True)).lower() == 'true' -%}
......@@ -9,6 +16,7 @@
{% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
......@@ -92,22 +100,14 @@ config-state = empty
config-url = ${monitor-base:base-url}/private/boot-image-url-select/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[boot-image-url-select-source-config]
# generates configuration of the image from the user parameter
# special "magic" is used, to properly support multiline boot-image-url-select
# but in the same time correctly generate the configuration file
recipe = slapos.recipe.template:jinja2
template = inline:
{#- Do special trick to support boot-image-url-select being None, if key is present with value "" #}
{%- raw %}
{%- set boot_image_url_select = slap_parameter.get('boot-image-url-select') or '' %}
{%- if boot_image_url_select == 'None' %}
{#- That's insane protection, is it 'None' because of type = array? #}
{%- set boot_image_url_select = '' %}
{%- endif %}
{{ boot_image_url_select }}
{% endraw -%}
boot-image-url-select = {{ dumps(slapparameter_dict['boot-image-url-select']) }}
context =
section slap_parameter slap-parameter
key boot_image_url_select :boot-image-url-select
rendered = ${directory:etc}/boot-image-url-select.json
[boot-image-url-select-processed-config]
......@@ -199,17 +199,14 @@ config-state = empty
config-url = ${monitor-base:base-url}/private/boot-image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[boot-image-url-list-source-config]
# generates configuration of the image from the user parameter
# special "magic" is used, to properly support multiline boot-image-url-list
# but in the same time correctly generate the configuration file
recipe = slapos.recipe.template:jinja2
template = inline:
{#- Do special trick to support boot-image-url-list being None, if key is present with value "" #}
{%- raw %}
{{ slap_parameter.get('boot-image-url-list') or '' }}
{{ boot_image_url_list }}
{% endraw -%}
boot-image-url-list = {{ dumps(slapparameter_dict['boot-image-url-list']) }}
context =
section slap_parameter slap-parameter
key boot_image_url_list :boot-image-url-list
rendered = ${directory:etc}/boot-image-url-list.conf
[boot-image-url-list-processed-config]
......@@ -976,7 +973,7 @@ keyboard-layout-language = fr
{% set key_list = v.split('\n') -%}
{{ k }} =
{{ key_list | join('\n ') }}
{% elif k == 'boot-image-url-list' %}
{% elif k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains'] %}
{# needs to decorate possibly multiline or maybe unsafe value #}
{{ k }} = {{ dumps(v) }}
{% else -%}
......@@ -1042,7 +1039,62 @@ command-line =
{% endif -%}
[buildout]
{% do part_list.append('whitelist-firewall') -%}
[whitelist-firewall]
recipe = slapos.cookbook:wrapper
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
wrapper-path = ${directory:scripts}/${:_buildout_section_name_}
command-line =
{{ dnsresolver_executable }}
--style list
--output ${:output}
${:source}
source =
${whitelist-domains-request:rendered}
{{ whitelist_domains_default }}
${whitelist-domains-resolv.conf:output}
${whitelist-domains-download:output}
output = ${buildout:directory}/.slapos-whitelist-firewall
[whitelist-firewall-directory]
recipe = plone.recipe.command
location = ${buildout:parts-directory}/whitelist-firewall
command = mkdir -p ${:location}
update-command = ${:command}
[whitelist-domains-request]
recipe = slapos.recipe.template:jinja2
template = inline:
{%- raw %}
{%- for domain in whitelist_domains.split() %}
{{ domain }}
{%- endfor %}
{% endraw -%}
rendered = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
whitelist-domains = {{ dumps(whitelist_domains) }}
extensions = jinja2.ext.do
context =
key whitelist_domains :whitelist-domains
[whitelist-domains-resolv.conf]
recipe = plone.recipe.command
output = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
update-command = ${:command}
command =
egrep ^nameserver /etc/resolv.conf | cut -d ' ' -f 2 > ${:output}
[whitelist-domains-download]
recipe = slapos.cookbook:wrapper
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
wrapper-path = ${directory:scripts}/${:_buildout_section_name_}
output = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
interval = 3600
command-line = {{ python_executable }} {{ whitelist_firewall_download_controller }} {{ curl_executable_location }} 3600 ${:output} ${:url}
url = https://stream.nxdcdn.com/rapidspace-whitelist-domains
[instance-kvm-parts]
parts =
certificate-authority
certificate-authority-service
......@@ -1081,6 +1133,9 @@ parts =
# Complete parts with sections
{{ part_list | join('\n ') }}
[buildout]
parts = ${instance-kvm-parts:parts}
extends =
# Add extends list
{{ extends_list | join('\n ') }}
......
......@@ -80,9 +80,12 @@ extra-context =
raw ansible_promise_tpl ${template-ansible-promise:location}/${template-ansible-promise:filename}
raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash
raw dnsresolver_executable ${buildout:bin-directory}/dnsresolver
raw dcron_executable_location ${dcron:location}/sbin/crond
raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
raw file_download_script ${file-download-script:location}/${file-download-script:filename}
raw whitelist_domains_default ${whitelist-domains-default:location}/${whitelist-domains-default:filename}
raw whitelist_firewall_download_controller ${whitelist-firewall-download-controller:target}
raw image_download_controller ${image-download-controller:target}
raw image_download_config_creator ${image-download-config-creator:target}
raw logrotate_cfg ${template-logrotate-base:rendered}
......
......@@ -182,6 +182,10 @@ path = download_file.in
filename = download_file
on-update = true
[whitelist-domains-default]
<= download-template-base
filename = whitelist-domains-default
[template-httpd]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-kvm-http.cfg.in
......@@ -199,6 +203,10 @@ mode = 640
[image-download-config-creator]
<= image-download-controller
[whitelist-firewall-download-controller]
<= image-download-controller
[versions]
websockify = 0.9.0
......
# Minimal whitelisted domains needed to instantiate the instance
# Does not guarantee good usage of the guest VM itself
# The full list shall end up in whitelist-domains-download:url
# shcache.nxdcdn.com is default source for a lot of operations
shacache.nxdcdn.com
# stream.nxdcdn.com is needed by partition itself
stream.nxdcdn.com
# partition has to access default SlapOS Master
slap.vifib.com
slapos.vifib.com
# Partition needs access to SlapOS Master related resources
hnode.cdn.vifib.com
node.cdn.vifib.com
#!/usr/bin/env python
import os
import subprocess
import sys
import time
import logging
# Note: Assuring only one running instance is not done, as this script is only
# run from supervisord, which does it already
if __name__ == "__main__":
curl, sleep, output, url = sys.argv[1:]
sleep = int(sleep)
tmp_output = output + '.tmp'
logging.basicConfig(
format='%%(asctime)s [%%(levelname)s] %s : %%(message)s' % (url,),
level=logging.DEBUG)
logging.info('Redownloading each %is', sleep)
while True:
logging.info('Fetching')
try:
subprocess.check_output([
curl,
'--location', # follow redirects
'--no-progress-meter', # do not tell too much
'--max-time', '600', # 10 minutes is maximum
'--fail', # fail in case of wrong HTTP code
'--output', tmp_output, url],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
logging.error('Problem while downloading: %r', e.output.strip())
if os.path.exists(tmp_output):
logging.info('Stored output')
os.rename(tmp_output, output)
logging.info('Sleeping for %is', sleep)
time.sleep(sleep)
......@@ -37,10 +37,17 @@ import sqlite3
from six.moves.urllib.parse import parse_qs, urlparse
import unittest
import subprocess
import tempfile
import SocketServer
import SimpleHTTPServer
import multiprocessing
import time
import shutil
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.slap.standalone import SlapOSNodeCommandError
from slapos.testing.utils import findFreeTCPPort
has_kvm = os.access('/dev/kvm', os.R_OK | os.W_OK)
skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')
......@@ -105,34 +112,67 @@ bootstrap_machine_param_dict = {
"enable-monitor": True,
"keyboard-layout-language": "fr"
}
@skipUnlessKvm
class ServicesTestCase(InstanceTestCase):
def test_hashes(self):
hash_files = [
class KvmMixin(object):
def getProcessInfo(self):
hash_value = generateHashFromFiles([
os.path.join(self.computer_partition_root_path, hash_file)
for hash_file in [
'software_release/buildout.cfg',
]
expected_process_names = [
'6tunnel-10022-{hash}-on-watch',
'6tunnel-10080-{hash}-on-watch',
'6tunnel-10443-{hash}-on-watch',
'certificate_authority-{hash}-on-watch',
'crond-{hash}-on-watch',
'kvm-{hash}-on-watch',
'websockify-{hash}-on-watch',
]
])
with self.slap.instance_supervisor_rpc as supervisor:
process_names = [process['name']
for process in supervisor.getAllProcessInfo()]
running_process_info = '\n'.join(sorted([
'%(group)s:%(name)s %(statename)s' % q for q
in supervisor.getAllProcessInfo()
if q['name'] != 'watchdog' and q['group'] != 'watchdog']))
return running_process_info.replace(hash_value, '{hash}')
hash_files = [os.path.join(self.computer_partition_root_path, path)
for path in hash_files]
for name in expected_process_names:
h = generateHashFromFiles(hash_files)
expected_process_name = name.format(hash=h)
@skipUnlessKvm
class TestInstance(InstanceTestCase, KvmMixin):
__partition_reference__ = 'i'
self.assertIn(expected_process_name, process_names)
def test(self):
connection_parameter_dict = self\
.computer_partition.getConnectionParameterDict()
present_key_list = []
assert_key_list = [
'backend-url', 'url', 'monitor-setup-url', 'ipv6-network-info',
'tap-ipv4', 'tap-ipv6']
for k in assert_key_list:
if k in connection_parameter_dict:
present_key_list.append(k)
connection_parameter_dict.pop(k)
self.assertEqual(
connection_parameter_dict,
{
'ipv6': self._ipv6_address,
'maximum-extra-disk-amount': '0',
'monitor-base-url': 'https://[%s]:8026' % (self._ipv6_address,),
'nat-rule-port-tcp-22': '%s : 10022' % (self._ipv6_address,),
'nat-rule-port-tcp-443': '%s : 10443' % (self._ipv6_address,),
'nat-rule-port-tcp-80': '%s : 10080' % (self._ipv6_address,),
}
)
self.assertEqual(set(present_key_list), set(assert_key_list))
self.assertEqual(
"""i0:6tunnel-10022-{hash}-on-watch RUNNING
i0:6tunnel-10080-{hash}-on-watch RUNNING
i0:6tunnel-10443-{hash}-on-watch RUNNING
i0:bootstrap-monitor EXITED
i0:certificate_authority-{hash}-on-watch RUNNING
i0:crond-{hash}-on-watch RUNNING
i0:kvm-{hash}-on-watch RUNNING
i0:kvm_controller EXITED
i0:monitor-httpd-{hash}-on-watch RUNNING
i0:monitor-httpd-graceful EXITED
i0:websockify-{hash}-on-watch RUNNING
i0:whitelist-domains-download-{hash} RUNNING
i0:whitelist-firewall-{hash} RUNNING""",
self.getProcessInfo()
)
class MonitorAccessMixin(object):
......@@ -394,7 +434,7 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
@skipIfPython3
@skipUnlessKvm
class TestInstanceResilient(InstanceTestCase):
class TestInstanceResilient(InstanceTestCase, KvmMixin):
__partition_reference__ = 'ir'
instance_max_retry = 20
......@@ -403,22 +443,76 @@ class TestInstanceResilient(InstanceTestCase):
return 'kvm-resilient'
def test(self):
# just check that keys returned on requested partition are for resilient
self.assertSetEqual(
set(self.computer_partition.getConnectionParameterDict().keys()),
set([
'backend-url',
'feed-url-kvm-1-pull',
'feed-url-kvm-1-push',
'ipv6',
'ipv6-network-info',
'monitor-base-url',
'monitor-password',
'monitor-setup-url',
'monitor-user',
'takeover-kvm-1-password',
'takeover-kvm-1-url',
'url']))
connection_parameter_dict = self\
.computer_partition.getConnectionParameterDict()
present_key_list = []
assert_key_list = [
'monitor-password', 'takeover-kvm-1-password', 'backend-url', 'url',
'monitor-setup-url', 'ipv6-network-info']
for k in assert_key_list:
if k in connection_parameter_dict:
present_key_list.append(k)
connection_parameter_dict.pop(k)
self.assertEqual(
connection_parameter_dict,
{
'feed-url-kvm-1-pull': 'http://[%s]:8088/get/local-ir0-kvm-1-pull' % (
self._ipv6_address,),
'feed-url-kvm-1-push': 'http://[%s]:8088/get/local-ir0-kvm-1-push' % (
self._ipv6_address,),
'ipv6': self._ipv6_address,
'monitor-base-url': 'https://[%s]:8160' % (self._ipv6_address,),
'monitor-user': 'admin',
'takeover-kvm-1-url': 'http://[%s]:9263/' % (self._ipv6_address,),
}
)
self.assertEqual(set(present_key_list), set(assert_key_list))
self.assertEqual(
"""ir0:bootstrap-monitor EXITED
ir0:certificate_authority-{hash}-on-watch RUNNING
ir0:crond-{hash}-on-watch RUNNING
ir0:monitor-httpd-{hash}-on-watch RUNNING
ir0:monitor-httpd-graceful EXITED
ir1:bootstrap-monitor EXITED
ir1:certificate_authority-{hash}-on-watch RUNNING
ir1:crond-{hash}-on-watch RUNNING
ir1:equeue-on-watch RUNNING
ir1:monitor-httpd-{hash}-on-watch RUNNING
ir1:monitor-httpd-graceful EXITED
ir1:notifier-on-watch RUNNING
ir1:pbs_sshkeys_authority-on-watch RUNNING
ir2:6tunnel-10022-{hash}-on-watch RUNNING
ir2:6tunnel-10080-{hash}-on-watch RUNNING
ir2:6tunnel-10443-{hash}-on-watch RUNNING
ir2:bootstrap-monitor EXITED
ir2:certificate_authority-{hash}-on-watch RUNNING
ir2:crond-{hash}-on-watch RUNNING
ir2:equeue-on-watch RUNNING
ir2:kvm-{hash}-on-watch RUNNING
ir2:kvm_controller EXITED
ir2:monitor-httpd-{hash}-on-watch RUNNING
ir2:monitor-httpd-graceful EXITED
ir2:notifier-on-watch RUNNING
ir2:resilient_sshkeys_authority-on-watch RUNNING
ir2:sshd-graceful EXITED
ir2:sshd-on-watch RUNNING
ir2:websockify-{hash}-on-watch RUNNING
ir2:whitelist-domains-download-{hash} RUNNING
ir2:whitelist-firewall-{hash} RUNNING
ir3:bootstrap-monitor EXITED
ir3:certificate_authority-{hash}-on-watch RUNNING
ir3:crond-{hash}-on-watch RUNNING
ir3:equeue-on-watch RUNNING
ir3:monitor-httpd-{hash}-on-watch RUNNING
ir3:monitor-httpd-graceful EXITED
ir3:notifier-on-watch RUNNING
ir3:resilient-web-takeover-httpd-on-watch RUNNING
ir3:resilient_sshkeys_authority-on-watch RUNNING
ir3:sshd-graceful EXITED
ir3:sshd-on-watch RUNNING""",
self.getProcessInfo()
)
@skipIfPython3
......@@ -485,9 +579,74 @@ class TestInstanceNbdServer(InstanceTestCase):
self.assertIn("WARNING", connection_parameter_dict['status_message'])
class FakeImageHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def log_message(self, *args):
if os.environ.get('SLAPOS_TEST_DEBUG'):
return SimpleHTTPServer.SimpleHTTPRequestHandler.log_message(self, *args)
else:
return
class FakeImageServerMixin(object):
def rerequestInstance(self, parameter_dict, state='started'):
software_url = self.getSoftwareURL()
software_type = self.getInstanceSoftwareType()
return self.slap.request(
software_release=software_url,
software_type=software_type,
partition_reference=self.default_partition_reference,
partition_parameter_kw=parameter_dict,
state=state)
def startImageHttpServer(self):
self.image_source_directory = tempfile.mkdtemp()
server = SocketServer.TCPServer(
(self._ipv4_address, findFreeTCPPort(self._ipv4_address)),
FakeImageHandler)
fake_image_content = 'fake_image_content'
self.fake_image_md5sum = hashlib.md5(fake_image_content).hexdigest()
with open(os.path.join(
self.image_source_directory, self.fake_image_md5sum), 'wb') as fh:
fh.write(fake_image_content)
fake_image2_content = 'fake_image2_content'
self.fake_image2_md5sum = hashlib.md5(fake_image2_content).hexdigest()
with open(os.path.join(
self.image_source_directory, self.fake_image2_md5sum), 'wb') as fh:
fh.write(fake_image2_content)
self.fake_image_wrong_md5sum = self.fake_image2_md5sum
url = 'http://%s:%s' % server.server_address
self.fake_image = '/'.join([url, self.fake_image_md5sum])
self.fake_image2 = '/'.join([url, self.fake_image2_md5sum])
old_dir = os.path.realpath(os.curdir)
os.chdir(self.image_source_directory)
try:
self.server_process = multiprocessing.Process(
target=server.serve_forever, name='FakeImageHttpServer')
self.server_process.start()
finally:
os.chdir(old_dir)
def stopImageHttpServer(self):
self.logger.debug('Stopping process %s' % (self.server_process,))
self.server_process.join(10)
self.server_process.terminate()
time.sleep(0.1)
if self.server_process.is_alive():
self.logger.warning(
'Process %s still alive' % (self.server_process, ))
shutil.rmtree(self.image_source_directory)
@skipUnlessKvm
class TestBootImageUrlList(InstanceTestCase):
class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
__partition_reference__ = 'biul'
kvm_instance_partition_reference = 'biul0'
# variations
key = 'boot-image-url-list'
......@@ -520,6 +679,10 @@ class TestBootImageUrlList(InstanceTestCase):
# start with empty, but working configuration
return {}
def setUp(self):
super(InstanceTestCase, self).setUp()
self.startImageHttpServer()
def tearDown(self):
# clean up the instance for other tests
# 1st remove all images...
......@@ -528,28 +691,8 @@ class TestBootImageUrlList(InstanceTestCase):
# 2nd ...move instance to "default" state
self.rerequestInstance({})
self.slap.waitForInstance(max_retry=10)
def rerequestInstance(self, parameter_dict, state='started'):
software_url = self.getSoftwareURL()
software_type = self.getInstanceSoftwareType()
return self.slap.request(
software_release=software_url,
software_type=software_type,
partition_reference=self.default_partition_reference,
partition_parameter_kw=parameter_dict,
state=state)
fake_image, = (
"https://shacache.nxdcdn.com/shacache/05105cd25d1ad798b71fd46a206c9b73d"
"a2c285a078af33d0e739525a595886785725a68811578bc21f75d0a97700a66d5e75bc"
"e5b2721ca4556a0734cb13e65",)
fake_image_md5sum = "c98825aa1b6c8087914d2bfcafec3058"
fake_image2, = (
"https://shacache.nxdcdn.com/shacache/54f8a83a32bbf52602d9d211d592ee705"
"99f0c6b6aafe99e44aeadb0c8d3036a0e673aa994ffdb28d9fb0de155720123f74d814"
"2a74b7675a8d8ca20476dba6e",)
fake_image2_md5sum = "d4316a4d05f527d987b9d6e43e4c2bc6"
fake_image_wrong_md5sum = "c98825aa1b6c8087914d2bfcafec3057"
self.stopImageHttpServer()
super(InstanceTestCase, self).tearDown()
def raising_waitForInstance(self, max_retry):
with self.assertRaises(SlapOSNodeCommandError):
......@@ -564,8 +707,10 @@ class TestBootImageUrlList(InstanceTestCase):
self.rerequestInstance(partition_parameter_kw)
self.slap.waitForInstance(max_retry=10)
# check that image is correctly downloaded and linked
kvm_instance_partition = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference)
image_repository = os.path.join(
self.computer_partition_root_path, 'srv', self.image_directory)
kvm_instance_partition, 'srv', self.image_directory)
image = os.path.join(image_repository, self.fake_image_md5sum)
image_link = os.path.join(image_repository, 'image_001')
self.assertTrue(os.path.exists(image))
......@@ -597,25 +742,17 @@ class TestBootImageUrlList(InstanceTestCase):
if entry.startswith('file') and 'media=cdrom' in entry:
# do cleanups
entry = entry.replace(software_root, '')
entry = entry.replace(self.computer_partition_root_path, '')
entry = entry.replace(kvm_instance_partition, '')
running_image_list.append(entry)
return running_image_list
# check that the image is NOT YET available in kvm
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom'],
getRunningImageList()
)
# mimic the requirement: restart the instance by requesting it stopped and
# then started started, like user have to do it
self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1)
self.slap.waitForInstance(max_retry=3)
# now the image is available in the kvm, and its above default image
self.assertEqual(
[
'file=/srv/%s/image_001,media=cdrom' % (self.image_directory,),
......@@ -628,7 +765,8 @@ class TestBootImageUrlList(InstanceTestCase):
# cleanup of images works, also asserts that configuration changes are
# reflected
self.rerequestInstance({self.key: ''})
partition_parameter_kw[self.key] = ''
self.rerequestInstance(partition_parameter_kw)
self.slap.waitForInstance(max_retry=2)
self.assertEqual(
os.listdir(image_repository),
......@@ -640,7 +778,7 @@ class TestBootImageUrlList(InstanceTestCase):
self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1)
self.slap.waitForInstance(max_retry=3)
# again only default image is available in the running process
self.assertEqual(
......@@ -650,12 +788,15 @@ class TestBootImageUrlList(InstanceTestCase):
)
def assertPromiseFails(self, promise):
partition_directory = os.path.join(
self.slap.instance_directory,
self.kvm_instance_partition_reference)
monitor_run_promise = os.path.join(
self.computer_partition_root_path, 'software_release', 'bin',
partition_directory, 'software_release', 'bin',
'monitor.runpromise'
)
monitor_configuration = os.path.join(
self.computer_partition_root_path, 'etc', 'monitor.conf')
partition_directory, 'etc', 'monitor.conf')
self.assertNotEqual(
0,
......@@ -708,9 +849,19 @@ class TestBootImageUrlList(InstanceTestCase):
self.assertPromiseFails(self.config_state_promise)
@skipIfPython3
@skipUnlessKvm
class TestBootImageUrlListResilient(TestBootImageUrlList):
kvm_instance_partition_reference = 'biul2'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-resilient'
@skipUnlessKvm
class TestBootImageUrlSelect(TestBootImageUrlList):
__partition_reference__ = 'bius'
kvm_instance_partition_reference = 'bius0'
# variations
key = 'boot-image-url-select'
......@@ -754,7 +905,8 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
for image_directory in [
'boot-image-url-list-repository', 'boot-image-url-select-repository']:
image_repository = os.path.join(
self.computer_partition_root_path, 'srv', image_directory)
self.slap.instance_directory, self.kvm_instance_partition_reference,
'srv', image_directory)
image = os.path.join(image_repository, self.fake_image_md5sum)
image_link = os.path.join(image_repository, 'image_001')
self.assertTrue(os.path.exists(image))
......@@ -764,6 +916,9 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
self.assertTrue(os.path.islink(image_link))
self.assertEqual(os.readlink(image_link), image)
kvm_instance_partition = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference)
def getRunningImageList():
running_image_list = []
with self.slap.instance_supervisor_rpc as instance_supervisor:
......@@ -777,25 +932,17 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
if entry.startswith('file') and 'media=cdrom' in entry:
# do cleanups
entry = entry.replace(software_root, '')
entry = entry.replace(self.computer_partition_root_path, '')
entry = entry.replace(kvm_instance_partition, '')
running_image_list.append(entry)
return running_image_list
# check that the image is NOT YET available in kvm
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom'],
getRunningImageList()
)
# mimic the requirement: restart the instance by requesting it stopped and
# then started started, like user have to do it
self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1)
self.slap.waitForInstance(max_retry=3)
# now the image is available in the kvm, and its above default image
self.assertEqual(
[
'file=/srv/boot-image-url-select-repository/image_001,media=cdrom',
......@@ -814,7 +961,18 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
for image_directory in [
'boot-image-url-list-repository', 'boot-image-url-select-repository']:
image_repository = os.path.join(
self.computer_partition_root_path, 'srv', image_directory)
kvm_instance_partition, 'srv', image_directory)
self.assertEqual(
os.listdir(image_repository),
[]
)
# cleanup of images works, also asserts that configuration changes are
# reflected
partition_parameter_kw[self.key] = ''
partition_parameter_kw['boot-image-url-list'] = ''
self.rerequestInstance(partition_parameter_kw)
self.slap.waitForInstance(max_retry=2)
self.assertEqual(
os.listdir(image_repository),
[]
......@@ -825,7 +983,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1)
self.slap.waitForInstance(max_retry=3)
# again only default image is available in the running process
self.assertEqual(
......@@ -835,42 +993,44 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
)
@skipIfPython3
@skipUnlessKvm
class TestBootImageUrlListKvmCluster(InstanceTestCase):
class TestBootImageUrlSelectResilient(TestBootImageUrlSelect):
kvm_instance_partition_reference = 'bius2'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-resilient'
@skipUnlessKvm
class TestBootImageUrlListKvmCluster(InstanceTestCase, FakeImageServerMixin):
__partition_reference__ = 'biulkc'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-cluster'
fake_image, = (
"https://shacache.nxdcdn.com/shacache/05105cd25d1ad798b71fd46a206c9b73d"
"a2c285a078af33d0e739525a595886785725a68811578bc21f75d0a97700a66d5e75bc"
"e5b2721ca4556a0734cb13e65",)
fake_image_md5sum = "c98825aa1b6c8087914d2bfcafec3058"
fake_image2, = (
"https://shacache.nxdcdn.com/shacache/54f8a83a32bbf52602d9d211d592ee705"
"99f0c6b6aafe99e44aeadb0c8d3036a0e673aa994ffdb28d9fb0de155720123f74d814"
"2a74b7675a8d8ca20476dba6e",)
fake_image2_md5sum = "d4316a4d05f527d987b9d6e43e4c2bc6"
input_value = "%s#%s"
key = 'boot-image-url-list'
config_file_name = 'boot-image-url-list.conf'
def setUp(self):
super(InstanceTestCase, self).setUp()
self.startImageHttpServer()
def tearDown(self):
self.stopImageHttpServer()
super(InstanceTestCase, self).tearDown()
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
"kvm-partition-dict": {
"KVM0": {
"disable-ansible-promise": True,
cls.key: cls.input_value % (
cls.fake_image, cls.fake_image_md5sum)
},
"KVM1": {
"disable-ansible-promise": True,
cls.key: cls.input_value % (
cls.fake_image2, cls.fake_image2_md5sum)
}
}
})}
......@@ -878,6 +1038,21 @@ class TestBootImageUrlListKvmCluster(InstanceTestCase):
def test(self):
# Note: As there is no way to introspect nicely where partition landed
# we assume ordering of the cluster requests
self.rerequestInstance({'_': json.dumps({
"kvm-partition-dict": {
"KVM0": {
"disable-ansible-promise": True,
self.key: self.input_value % (
self.fake_image, self.fake_image_md5sum)
},
"KVM1": {
"disable-ansible-promise": True,
self.key: self.input_value % (
self.fake_image2, self.fake_image2_md5sum)
}
}
})})
self.slap.waitForInstance(max_retry=10)
KVM0_config = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '1', 'etc',
self.config_file_name)
......@@ -999,3 +1174,107 @@ class TestNatRulesKvmCluster(InstanceTestCase):
class TestNatRulesKvmClusterComplex(TestNatRulesKvmCluster):
__partition_reference__ = 'nrkcc'
nat_rules = ["100", "200 300"]
@skipUnlessKvm
class TestWhitelistFirewall(InstanceTestCase):
__partition_reference__ = 'wf'
kvm_instance_partition_reference = 'wf0'
def test(self):
slapos_whitelist_firewall = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference,
'.slapos-whitelist-firewall')
self.assertTrue(os.path.exists(slapos_whitelist_firewall))
with open(slapos_whitelist_firewall, 'rb') as fh:
content = fh.read().encode('utf-8')
try:
self.content_json = json.loads(content)
except ValueError:
self.fail('Failed to parse json of %s' % (content,))
self.assertTrue(isinstance(self.content_json, list))
# check /etc/resolv.conf
with open('/etc/resolv.conf', 'rb') as fh:
resolv_conf_ip_list = []
for line in fh.readlines():
line = line.encode('utf-8')
if line.startswith('nameserver'):
resolv_conf_ip_list.append(line.split()[1])
resolv_conf_ip_list = list(set(resolv_conf_ip_list))
self.assertFalse(len(resolv_conf_ip_list) == 0)
self.assertTrue(all([q in self.content_json for q in resolv_conf_ip_list]))
# there is something more
self.assertGreater(len(self.content_json), len(resolv_conf_ip_list))
@skipUnlessKvm
class TestWhitelistFirewallRequest(TestWhitelistFirewall):
whitelist_domains = '2.2.2.2 3.3.3.3\n4.4.4.4'
@classmethod
def getInstanceParameterDict(cls):
return {
'whitelist-domains': cls.whitelist_domains,
}
def test(self):
super(TestWhitelistFirewallRequest, self).test()
self.assertIn('2.2.2.2', self.content_json)
self.assertIn('3.3.3.3', self.content_json)
self.assertIn('4.4.4.4', self.content_json)
@skipUnlessKvm
class TestWhitelistFirewallResilient(TestWhitelistFirewall):
kvm_instance_partition_reference = 'wf2'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-resilient'
@skipUnlessKvm
class TestWhitelistFirewallRequestResilient(TestWhitelistFirewallRequest):
kvm_instance_partition_reference = 'wf2'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-resilient'
@skipUnlessKvm
class TestWhitelistFirewallCluster(TestWhitelistFirewall):
kvm_instance_partition_reference = 'wf1'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-cluster'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
"kvm-partition-dict": {
"KVM0": {
"disable-ansible-promise": True
}
}
})}
@skipUnlessKvm
class TestWhitelistFirewallRequestCluster(TestWhitelistFirewallRequest):
kvm_instance_partition_reference = 'wf1'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-cluster'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
"kvm-partition-dict": {
"KVM0": {
"whitelist-domains": cls.whitelist_domains,
"disable-ansible-promise": True
}
}
})}
......@@ -5,6 +5,7 @@ extends =
../../component/openssl/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/apache/buildout.cfg
../../component/geoip2/buildout.cfg
../../component/git/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../stack/slapos.cfg
......@@ -36,11 +37,13 @@ eggs =
${python-cffi:egg}
${python-cryptography:egg}
pyOpenSSL
re6stnet
re6stnet[geoip]
initialization =
import os
path = os.environ.get('PATH', '')
os.environ['PATH'] = '${git:location}/bin' + (path and ':' + path)
path = os.environ.get('PATH', '/bin:/usr/bin')
os.environ['PATH'] = '${openssl:location}/bin:${git:location}/bin' + (
path and ':' + path)
os.environ['GEOIP2_MMDB'] = "${geolite2-country:location}/GeoLite2-Country.mmdb"
depends = ${re6stnet-develop:recipe}
[re6stnet-backup]
......@@ -59,6 +62,7 @@ template =
h=`sqlite3 "$2" .dump |git hash-object --stdin -w`
git update-index --add --cacheinfo 0644 "$h" registry.sql
git diff --cached --quiet || GIT_WORK_TREE=$PWD GIT_DIR=$PWD git \
-c user.name="$${0##*/}" -c user.email="$USER@`hostname`" \
-c gc.auto=100 -c gc.autoDetach=false commit --allow-empty-message -qm ''
[ ! "$3" ] || git push --mirror "$3"
context =
......
......@@ -216,9 +216,9 @@ slapos.rebootstrap = 4.5
slapos.recipe.build = 0.46
slapos.recipe.cmmi = 0.17
slapos.recipe.template = 4.5
slapos.toolbox = 0.117
slapos.toolbox = 0.118
stevedore = 1.21.0
subprocess32 = 3.5.3
subprocess32 = 3.5.4
unicodecsv = 0.14.1
wcwidth = 0.2.5
wheel = 0.35.1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment