Skip to content

slapos
slapos
Commit 88f0bcd5 authored by Xavier Thompson's avatar Xavier Thompson
Browse files
Options

software/theia: Simplify and improve embedded SR 

See merge request nexedi/slapos!1139
parents c9f8bad9 1b877749
Hide whitespace changes
Inline Side-by-side
Showing with 533 additions and 342 deletions
software/theia/buildout.hash.cfg
View file @ 88f0bcd5
......@@ -15,24 +15,28 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = 7ea72f2300dd2019c2d27549e7128dec
md5sum = 7d146ca31d7e44e909386e21e0a562ec
[instance]
_update_hash_filename_ = instance.cfg.in
md5sum = af4d92a95dc25c2f64ddbd634996e46c
md5sum = e211c439571e2900f9f35482c9638d06
[instance-import]
_update_hash_filename_ = instance-import.cfg.jinja.in
md5sum = 29bc1c5b76d20fd46430dff5c72d192b
md5sum = 23c3df4a889ebfa9f0a94e873e95ad3b
[instance-export]
_update_hash_filename_ = instance-export.cfg.jinja.in
md5sum = e2630148998c3cdf6d03b5e884d7f464
md5sum = 84ceb4c9ee0f07fce8518ef7517ce1d4
[instance-resilient]
_update_hash_filename_ = instance-resilient.cfg.jinja
md5sum = ad9499e7355ded4975ad313442cecb7a
[slapos-standalone-script]
_update_hash_filename_ = slapos_standalone_script.py.jinja
md5sum = ef5b73648513caf46573f3302953790f
[theia-common]
_update_hash_filename_ = theia_common.py
md5sum = 6a25c6a7f1beb27232a3c9acd8a76500
......
software/theia/instance-export.cfg.jinja.in
View file @ 88f0bcd5
......@@ -92,14 +92,14 @@ inline =
then
echo "ERROR export script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 1
elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
then
echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 0
else
echo "ERROR export script failed on " $(date -r {{ repr(exitcodefile) }})
cat {{ repr(errorfile) }}
exit 1
else
echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 0
fi
{%- endraw %}
......
software/theia/instance-import.cfg.jinja.in
View file @ 88f0bcd5
......@@ -154,14 +154,14 @@ inline =
then
echo "ERROR import script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 1
elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
then
echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 0
else
echo "ERROR import script failed on " $(date -r {{ repr(exitcodefile) }})
cat {{ repr(errorfile) }}
exit 1
else
echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 0
fi
{%- endraw %}
......
software/theia/instance-input-schema.json
View file @ 88f0bcd5
......@@ -15,20 +15,14 @@
],
"default": "running"
},
"embedded-sr": {
"title": "Embedded Software URL",
"description": "Optional URL of a software to be embedded",
"type": "string"
},
"embedded-sr-type": {
"title": "Embedded Software Type",
"description": "Type of the optional embedded software",
"type": "string"
},
"embedded-instance-parameters": {
"title": "Embedded Instance Parameters",
"description": "Parameters for the embedded instance, as a JSON dict",
"type": "string"
"initial-embedded-instance": {
"title": "Initial Embedded Instance Configuration",
"description": "One-shot optional JSON preconfiguration for an embedded instance. Only applied once when Theia is instantiated. Changing this option afterward will have no effect.",
"type": "string",
"examples": [
"{\"software-url\": \"~/srv/project/slapos/software/html5as-base/software.cfg\"}",
"{\"software-url\": \"~/srv/project/slapos/software/html5as/software.cfg\", \"software-type\": \"replicate\", \"instance-parameters\": {\"replicate-quantity\": 3}}"
]
},
"frontend-guid": {
"title": "Frontend Instance ID",
......
software/theia/instance-theia.cfg.jinja.in
View file @ 88f0bcd5
{% set parameter_dict = dict(default_parameter_dict, **parameter_dict) %}
{% set additional_frontend = parameter_dict['additional-frontend-guid'] %}
{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) %}
{%- set additional_frontend = parameter_dict['additional-frontend-guid'] %}
{%- set embedded_instance_config = parameter_dict['initial-embedded-instance'] %}
[buildout]
extends =
......@@ -10,7 +11,6 @@ theia-environment-parts =
slapos-repository
runner-link
settings.json
request-script-template
theia-parts =
frontend-reload
......@@ -98,6 +98,9 @@ instance-promises =
$${slapos-standalone-listen-promise:name}
$${slapos-standalone-ready-promise:name}
$${slapos-autorun-promise:name}
{% if embedded_instance_config %}
$${embedded-instance-requested-promise:name}
{% endif %}
[theia-listen-promise]
<= monitor-promise-base
......@@ -161,6 +164,14 @@ config-service = $${slapos-autorun:service-name}
config-expect = $${slapos-autorun:autorun}
config-run-directory = $${directory:runner}/var/run
{% if embedded_instance_config %}
[embedded-instance-requested-promise]
<= monitor-promise-base
promise = check_command_execute
name = embedded-instance-requested-promise.py
config-command = $${embedded-instance-requested-promise-script:output}
{% endif %}
# Remote Caddy Frontend
# ---------------------
......@@ -402,7 +413,7 @@ base-url = $${theia-service:base-url}
[theia-shell]
recipe = slapos.recipe.template:jinja2
output = $${directory:bin}/$${:_buildout_section_name_}
inline
inline =
{% raw -%}
#!{{ bash }}
SHELL=$BASH
......@@ -441,35 +452,40 @@ command =
# Embedded Instance
# -----------------
{%- set embedded_sr = parameter_dict['embedded-sr'] %}
{%- set embedded_sr_type = parameter_dict['embedded-sr-type'] %}
{%- set embedded_instance_parameters = parameter_dict['embedded-instance-parameters'] %}
{%- if embedded_sr %}
{%- if embedded_sr.startswith('~/') %}
{%- set embedded_sr = os_module.path.join(partition_root_path, embedded_sr[2:]) %}
{%- set embedded_sr = os_module.path.normpath(embedded_sr) %}
{%- endif %}
[request-embedded-instance-script]
recipe = slapos.recipe.template
output = $${directory:project}/request_embedded.sh
[embedded-instance-config]
recipe = slapos.recipe.template:jinja2
output = $${directory:etc}/$${:_buildout_section_name_}.json
once = $${:output}
config = {{ dumps(embedded_instance_config) }}
context =
key config :config
inline =
#!/bin/sh
slapos supply {{ embedded_sr }} slaprunner
slapos request "embedded_instance" {{ embedded_sr }}
{%- if embedded_sr_type %} --type {{ embedded_sr_type }} {%- endif %}
{%- if embedded_instance_parameters %} --parameters-file $${embedded-instance-parameters:output}
{%- raw %}
{{ config or "{}"}}
{%- endraw %}
[embedded-instance-parameters]
recipe = slapos.recipe.template
output = $${directory:project}/$${:_buildout_section_name_}.json
inline = {{ embedded_instance_parameters | indent(2) }}
{%- endif %}
{%- endif %}
{%- set embedded_digest = str(embedded_sr) + str(embedded_sr_type) + str(embedded_instance_parameters) %}
{%- set embedded_digest_hash = hashlib_module.md5(embedded_digest.encode()).hexdigest() %}
[embedded-instance-requested-promise-script]
recipe = slapos.recipe.template:jinja2
output = $${directory:bin}/$${:_buildout_section_name_}
exitcode-file = $${slapos-standalone-script:embedded-request-exitcode-file}
context =
key exitcodefile :exitcode-file
{%- raw %}
inline =
#!/bin/sh
if ! [ -f {{ repr(exitcodefile) }} ]
then
echo "ERROR embedded_instance has not been requested"
exit 1
elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
then
echo "OK embedded_instance has been sucessfully requested"
exit 0
else
echo "ERROR request of embedded_instance failed"
exit 1
fi
{%- endraw %}
# SlapOS Standalone
......@@ -485,10 +501,14 @@ ip = {{ ipv4_random }}
ipv4 = {{ ipv4_random }}
ipv6 = {{ ipv6_random }}
port = $${slapos-standalone-port:port}
base-directory = $${directory:runner}
software-root = $${directory:runner}/software
instance-root = $${directory:runner}/instance
local-software-release-root = $${directory:home}
slapos-bin = ${buildout:bin-directory}/slapos
slapos-configuration = $${directory:runner}/etc/slapos.cfg
computer-id = slaprunner
abstract-socket-path = $${directory:home}/standalone-{{ embedded_digest_hash[:16] }}
abstract-socket-path = $${directory:home}/standalone-ready
[slapos-standalone-activate]
recipe = slapos.recipe.template
......@@ -500,98 +520,23 @@ inline =
echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
[slapos-standalone-script]
recipe = slapos.recipe.template
recipe = slapos.recipe.template:jinja2
output = $${directory:bin}/$${:_buildout_section_name_}
inline =
#!${python-for-standalone:executable}
import glob
import json
import os
import signal
import socket
import subprocess
import sys
import time
import traceback
import slapos.slap.standalone
# Include this hash, so that if it changes the standalone service will be restarted
# {{ embedded_digest_hash }}
shared_parts = """{{ '''${buildout:shared-part-list}''' | indent(2) }}"""
shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
partition_forward_configuration = (
slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
master_url="$${slap-connection:server-url}",
computer="$${slap-connection:computer-id}",
partition="$${slap-connection:partition-id}",
cert="$${slap-connection:cert-file}",
key="$${slap-connection:key-file}",
software_release_list=(
'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
),
),
)
standalone = slapos.slap.standalone.StandaloneSlapOS(
"$${directory:runner}",
"$${slapos-standalone-config:ipv4}",
$${slapos-standalone-config:port},
computer_id="$${slapos-standalone-config:computer-id}",
shared_part_list=shared_part_list,
software_root="$${directory:runner}/software",
instance_root="$${directory:runner}/instance",
partition_forward_configuration=partition_forward_configuration,
slapos_bin="${buildout:bin-directory}/slapos",
local_software_release_root="$${slapos-standalone-config:local-software-release-root}",
)
def signal_handler(signum, frame):
print("Signal {signum} received".format(signum=signum))
sys.exit()
signal.signal(signal.SIGTERM, signal_handler)
standalone.start()
try:
partition_count = 20
print("Standalone SlapOS: Formatting %d partitions" % partition_count)
standalone.format(partition_count, '$${slapos-standalone-config:ipv4}', '$${slapos-standalone-config:ipv6}')
print("Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` started")
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
print("Error instanciating: {}".format(e))
{%- if embedded_sr %}
# Compatibility layer
try:
for cp in standalone.computer.getComputerPartitionList():
if cp.getInstanceParameterDict().get("instance_title") == "Embedded Instance":
print("Renaming 'Embedded Instance' into 'embedded_instance'")
cp.rename(new_name="embedded_instance")
break
except Exception:
print("Exception in compatibility layer, printing and moving on")
traceback.print_exc()
# Run request script
print("Running SlapOS script $${request-embedded-instance-script:output}")
slapos_env = {
'PATH': os.path.dirname(standalone._slapos_bin),
'SLAPOS_CONFIGURATION': standalone._slapos_config,
'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config
}
subprocess.call(("$${request-embedded-instance-script:output}",), env=slapos_env)
{%- endif %}
s = socket.socket(socket.AF_UNIX)
s.bind("\0$${slapos-standalone-config:abstract-socket-path}")
s.listen(5)
print("Standalone SlapOS ready")
while True:
s.accept()[0].close()
finally:
print("Stopping standalone subsystem")
standalone.stop()
print("Exiting")
embedded-request-exitcode-file = $${directory:etc}/embedded-request-exitcode
shared-part-list =
{{ """${buildout:shared-part-list}""" | indent(2) }}
context =
raw python_for_standalone ${python-for-standalone:executable}
raw request_script_path $${directory:project}/request-embedded-instance.sh
raw parameters_file_path $${directory:project}/embedded-instance-parameters.json
key request_script_template request-script-example:inline
key shared_part_list :shared-part-list
key embedded_request_exitcode_file :embedded-request-exitcode-file
key embedded_instance_config embedded-instance-config:output
key home_path directory:home
section slap_connection slap-connection
section slapos_standalone_config slapos-standalone-config
url = ${slapos-standalone-script:output}
[slapos-standalone]
recipe = slapos.recipe.template
......@@ -725,18 +670,48 @@ recipe = slapos.cookbook:symbolic.link
target-directory = $${directory:project}
link-binary = $${directory:runner}
[request-script-template]
recipe = slapos.recipe.template
[request-script-example]
recipe = slapos.recipe.template:jinja2
output = $${directory:project}/$${:_buildout_section_name_}.sh
software_url = ~/srv/project/slapos/software/html5as-base/software.cfg
request_options = html5as-1 $${:software_url}
header_text =
# # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# This template is generated automatically by buildout. #
# Any changes to this file may be overwritten. #
# Copy and adapt it to create your own request script. #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # #
context =
key software_url :software_url
key request_options :request_options
key header_text :header_text
inline =
#!/bin/sh
# This template is generated automatically, copy it in order to supply and request.
# Any manual change to this file may be lost.
software_name=html5as-base #replace the software name writen in ~/srv/project/slapos/software/
software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg
# slapos supply is used to add the software to the software list to be supplied to a node.
slapos supply $software_release_uri slaprunner
# slapos request the allocation of an instance to the master.
# slapos request also gets status and parameters of the instance if it has any
# (slapos request is meant to be run multiple time until you get the status).
slapos request $software_name'_1' $software_release_uri
{%- raw %}
#!/bin/sh -e
{{ header_text }}
# slapos supply <url> <node> registers a software for installation on a node.
#
# A software is uniquely identified by its URL (the URL may be a local path).
# You may choose from softwares available in ~/srv/project/slapos/software.
#
# The one and only SlapOS Node embedded inside Theia is called 'slaprunner'.
#
# For more information, run:
# slapos help supply
#
slapos supply {{ software_url }} slaprunner
# slapos request <name> <url> registers an instance for allocation on a node.
#
# An instance is uniquely identified by its name (and its requester).
#
# It will be allocated on a node where the software URL is already supplied.
# Inside Theia that node can only be 'slaprunner'.
#
# For more information, run:
# slapos help request
#
slapos request {{ request_options }}
{% endraw %}
software/theia/instance.cfg.in
View file @ 88f0bcd5
......@@ -31,6 +31,7 @@ pull-backup = template-pull-backup:output
recipe = slapos.recipe.template:jinja2
url = ${instance-theia:output}
output = $${buildout:directory}/instance-theia.cfg
extensions = jinja2.ext.do
context =
jsonkey default_parameter_dict :default-parameters
key parameter_dict slap-configuration:configuration
......@@ -38,14 +39,12 @@ context =
key partition_root_path buildout:directory
key ipv6_random slap-configuration:ipv6-random
key ipv4_random slap-configuration:ipv4-random
import os_module os
import hashlib_module hashlib
import os os
import json json
default-parameters =
{
"autorun": "running",
"embedded-sr": null,
"embedded-sr-type": null,
"embedded-instance-parameters": null,
"initial-embedded-instance": null,
"frontend-name": "Theia Frontend",
"frontend-sr": "$${:frontend-sr}",
"frontend-sr-type": "RootSoftwareInstance",
......
software/theia/slapos_standalone_script.py.jinja 0 → 100644
View file @ 88f0bcd5
#!{{ python_for_standalone }}
import contextlib
import glob
import json
import logging
import os
import re
import signal
import socket
import subprocess
import sys
import time
import slapos.slap.standalone
from slapos.util import unicode2str
logging.basicConfig(format="[%(asctime)s] %(message)s", level=logging.DEBUG)
REQUEST_SCRIPT_RAW_TEMPLATE = """{{ request_script_template }}"""
{% raw -%}
REQUEST_SCRIPT_TEMPLATE = re.sub(
r"{{\s*(\w+)\s*}}",
r"{\1}",
REQUEST_SCRIPT_RAW_TEMPLATE,
)
{%- endraw %}
REQUEST_SCRIPT_HEADER_TEXT = """
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# This script is generated once by your Theia's SlapOS Standalone service. #
# #
# It was used to pre-request an instance inside Theia's embedded SlapOS #
# according to the preconfiguration parameters given to your Theia. #
# #
# It will not be overwritten or recreated. #
# It will not be launched automatically. #
# #
# You can edit it and run it to modify your embedded instance. #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
""".strip()
def signal_handler(signum, frame):
logging.info("Signal %d received", signum)
sys.exit()
@contextlib.contextmanager
def setupStandalone():
partition_forward_configuration = (
slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
master_url={{ repr(slap_connection['server-url']) }},
computer={{ repr(slap_connection['computer-id']) }},
partition={{ repr(slap_connection['partition-id']) }},
cert={{ repr(slap_connection['cert-file']) }},
key={{ repr(slap_connection['key-file']) }},
software_release_list=(
'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
),
),
)
shared_parts = {{ repr(shared_part_list) }}
shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
standalone = slapos.slap.standalone.StandaloneSlapOS(
{{ repr(slapos_standalone_config['base-directory']) }},
{{ repr(slapos_standalone_config['ipv4']) }},
{{ slapos_standalone_config['port'] }},
computer_id={{ repr(slapos_standalone_config['computer-id']) }},
shared_part_list=shared_part_list,
software_root={{ repr(slapos_standalone_config['software-root']) }},
instance_root={{ repr(slapos_standalone_config['instance-root']) }},
partition_forward_configuration=partition_forward_configuration,
slapos_bin={{ repr(slapos_standalone_config['slapos-bin']) }},
local_software_release_root={{ repr(slapos_standalone_config['local-software-release-root']) }},
)
standalone.start()
try:
partition_count = 20
logging.info("Standalone SlapOS: Formatting %d partitions", partition_count)
standalone.format(partition_count, {{ repr(slapos_standalone_config['ipv4']) }}, {{ repr(slapos_standalone_config['ipv6']) }})
logging.info("Standalone SlapOS for computer `%s` started", {{ repr(slapos_standalone_config['computer-id']) }})
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
logging.info("Error instanciating: {}".format(e))
yield standalone
finally:
logging.info("Stopping standalone subsystem")
standalone.stop()
logging.info("Exiting")
def parseEmbeddedInstanceConfig(config_json_file):
with open(config_json_file) as f:
try:
config = json.load(f)
except json.JSONDecodeError:
logging.error("%s is not valid JSON", config_json_file)
raise
assert(isinstance(config, dict))
if config:
software_url = unicode2str(config['software-url'])
software_type = config.get('software-type')
instance_parameters = config.get('instance-parameters')
assert(isinstance(software_url, str))
if software_type:
software_type = unicode2str(software_type)
assert(isinstance(software_type, str))
if instance_parameters:
assert(isinstance(instance_parameters, dict))
return software_url, software_type, instance_parameters
def createRequestScript(software_url, software_type, instance_parameters):
# Generate request script
request_script_path = {{ repr(request_script_path) }}
parameters_file_path = {{ repr(parameters_file_path) }}
request_options = "embedded_instance " + software_url
if software_type:
request_options += ' --type ' + software_type
if instance_parameters:
with open(parameters_file_path, 'w') as f:
json.dump(instance_parameters, f)
request_options += ' --parameters-file ' + parameters_file_path
with open(request_script_path, 'w') as f:
f.write(REQUEST_SCRIPT_TEMPLATE.format(
header_text = REQUEST_SCRIPT_HEADER_TEXT,
software_url = software_url,
request_options = request_options,
))
f.write("\n")
os.fchmod(f.fileno(), 0o755)
return request_script_path
def main():
signal.signal(signal.SIGTERM, signal_handler)
with setupStandalone() as standalone:
config_json_file = {{ repr(embedded_instance_config) }}
done_file = config_json_file + '.done'
if not os.path.exists(done_file):
with open(done_file, 'w'): pass
try:
config = parseEmbeddedInstanceConfig(config_json_file)
except Exception:
logging.info("Failed to parse embedded instance config", exc_info=True)
config = None
if config:
try:
request_script_path = createRequestScript(*config)
logging.info("Calling %s", request_script_path)
exitcode = subprocess.call((request_script_path,), env={
'HOME': {{ repr(home_path) }},
'PATH': os.path.dirname(standalone._slapos_bin),
'SLAPOS_CONFIGURATION': standalone._slapos_config,
'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config,
})
with open({{ repr(embedded_request_exitcode_file) }}, 'w') as f:
f.write(str(exitcode))
except Exception:
logging.info("Failed to request embedded instance", exc_info=True)
s = socket.socket(socket.AF_UNIX)
s.bind({{ repr('\0' + slapos_standalone_config['abstract-socket-path']) }})
s.listen(5)
logging.info("Standalone SlapOS ready")
while True:
s.accept()[0].close()
main()
software/theia/software.cfg
View file @ 88f0bcd5
......@@ -19,17 +19,9 @@ extends =
./buildout.hash.cfg
parts =
theia-wrapper
slapos-cookbook
python-for-resiliency
python-for-standalone
instance-theia
instance
instance-import
instance-export
instance-resilient
theia-common
theia-export
# default for slapos-standalone
shared-part-list =
......@@ -95,6 +87,10 @@ output = ${buildout:directory}/instance-export.cfg.jinja
[instance-resilient]
<= download-base
[slapos-standalone-script]
<= download-base
destination = ${buildout:directory}/slapos_standalone_script.py.jinja
[theia-common]
<= download-base
destination = ${buildout:directory}/theia_common.py
......@@ -102,10 +98,12 @@ destination = ${buildout:directory}/theia_common.py
[theia-export]
<= download-base
destination = ${buildout:directory}/theia_export.py
depends = ${theia-common:recipe}
[theia-import]
<= download-base
destination = ${buildout:directory}/theia_import.py
depends = ${theia-common:recipe}
# Utilities
......
software/theia/test/project_tests.py
View file @ 88f0bcd5
......@@ -62,10 +62,9 @@ class ERP5Mixin(object):
_test_software_url = erp5_software_release_url
_connexion_parameters_regex = re.compile(r"{.*}", re.DOTALL)
def _getERP5ConnexionParameters(self, software_type='export'):
slapos = self._getSlapos(software_type)
out = subprocess.check_output(
(slapos, 'request', 'test_instance', self._test_software_url),
def _getERP5ConnexionParameters(self, instance_type='export'):
out = self.captureSlapos(
'request', 'test_instance', self._test_software_url,
stderr=subprocess.STDOUT,
)
print(out)
......@@ -110,10 +109,10 @@ class ERP5Mixin(object):
raise Exception("Found several partitions for ERP5 %s" % servicename)
return found.pop()
def _getERP5PartitionPath(self, software_type, servicename, *paths):
def _getERP5PartitionPath(self, instance_type, servicename, *paths):
partition = self._getERP5Partition(servicename)
return self._getPartitionPath(
software_type, 'srv', 'runner', 'instance', partition, *paths)
return self.getPartitionPath(
instance_type, 'srv', 'runner', 'instance', partition, *paths)
class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
......@@ -161,16 +160,15 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
# Update ERP5 parameters
print('Requesting ERP5 with parameters %s' % params)
slapos = self._getSlapos()
subprocess.check_call((slapos, 'request', 'test_instance', self._test_software_url, '--parameters', params))
self.checkSlapos('request', 'test_instance', self._test_software_url, '--parameters', params)
# Process twice to propagate parameter changes
for _ in range(2):
subprocess.check_call((slapos, 'node', 'instance'))
self.checkSlapos('node', 'instance')
# Restart cron (actually all) services to let them take the new date into account
# XXX this should not be required, updating ERP5 parameters should be enough
subprocess.call((slapos, 'node', 'restart', 'all'))
self.callSlapos('node', 'restart', 'all')
# Wait until after the programmed backup date, and a bit more
t = (soon - datetime.now()).total_seconds()
......@@ -213,9 +211,8 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
self.assertNotIn(self._erp5_new_title, out)
# Stop all services
slapos = self._getSlapos()
print("Stop all services")
subprocess.call((slapos, 'node', 'stop', 'all'))
self.callSlapos('node', 'stop', 'all')
# Manually restore mariadb from backup
mariadb_restore_script = os.path.join(mariadb_partition, 'bin', 'restore-from-backup')
......
software/theia/test/test.py
View file @ 88f0bcd5
......@@ -52,11 +52,53 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(theia_soft
class TheiaTestCase(SlapOSInstanceTestCase):
__partition_reference__ = 'T' # for supervisord sockets in included slapos
@classmethod
def getPath(cls, *components):
return os.path.join(cls.computer_partition_root_path, *components)
@classmethod
def _getSlapos(cls):
partition_root = cls.computer_partition_root_path
slapos = os.path.join(partition_root, 'srv', 'runner', 'bin', 'slapos')
return slapos
try:
return cls._theia_slapos
except AttributeError:
cls._theia_slapos = slapos = cls.getPath('srv', 'runner', 'bin', 'slapos')
return slapos
@classmethod
def callSlapos(cls, *command, **kwargs):
return subprocess.call((cls._getSlapos(),) + command, **kwargs)
@classmethod
def checkSlapos(cls, *command, **kwargs):
return subprocess.check_call((cls._getSlapos(),) + command, **kwargs)
@classmethod
def captureSlapos(cls, *command, **kwargs):
kwargs.setdefault('universal_newlines', kwargs.pop('text', None))
return subprocess.check_output((cls._getSlapos(),) + command, **kwargs)
@classmethod
def requestInstance(cls, parameter_dict=None, state='started'):
cls.slap.request(
software_release=cls.getSoftwareURL(),
software_type=cls.getInstanceSoftwareType(),
partition_reference=cls.default_partition_reference,
partition_parameter_kw=parameter_dict,
state=state
)
@classmethod
def restartService(cls, service):
with cls.slap.instance_supervisor_rpc as supervisor:
for process_info in supervisor.getAllProcessInfo():
service_name = process_info['name']
if service in service_name:
service_id = '%s:%s' % (process_info['group'], service_name)
supervisor.stopProcess(service_id)
supervisor.startProcess(service_id)
break
else:
raise Exception("Service %s not found" % service)
class TestTheia(TheiaTestCase):
......@@ -104,7 +146,7 @@ class TestTheia(TheiaTestCase):
# there's a public folder to serve file
with open('{}/srv/frontend-static/public/test_file'.format(
self.computer_partition_root_path), 'w') as f:
self.getPath()), 'w') as f:
f.write("hello")
resp = self.get(urljoin(authenticated_url, '/public/'))
self.assertIn('test_file', resp.text)
......@@ -128,10 +170,9 @@ class TestTheia(TheiaTestCase):
self.assertIn('ipv6', self.connection_parameters)
def test_theia_slapos(self):
home = self.getPath()
# Make sure we can use the shell and the integrated slapos command
process = pexpect.spawnu(
'{}/bin/theia-shell'.format(self.computer_partition_root_path),
env={'HOME': self.computer_partition_root_path})
process = pexpect.spawnu(home + '/bin/theia-shell', env={'HOME': home})
# use a large enough terminal so that slapos proxy show table fit in the screen
process.setwinsize(5000, 5000)
......@@ -173,10 +214,11 @@ class TestTheia(TheiaTestCase):
process.wait()
def test_theia_shell_execute_tasks(self):
home = self.getPath()
# shell needs to understand -c "command" arguments for theia tasks feature
test_file = '{}/test file'.format(self.computer_partition_root_path)
test_file = home + '/test file'
subprocess.check_call([
'{}/bin/theia-shell'.format(self.computer_partition_root_path),
home + '/bin/theia-shell',
'-c',
'touch "{}"'.format(test_file)
])
......@@ -184,26 +226,23 @@ class TestTheia(TheiaTestCase):
def test_theia_request_script(self):
script_path = os.path.join(
self.computer_partition_root_path,
self.getPath(),
'srv',
'project',
'request-script-template.sh',
'request-script-example.sh',
)
self.assertTrue(os.path.exists(script_path))
def test_slapos_cli(self):
slapos = self._getSlapos()
proxy_show_output = subprocess.check_output((slapos, 'proxy', 'show'))
self.assertIn(b'slaprunner', proxy_show_output)
computer_list_output = subprocess.check_output((slapos, 'computer', 'list'))
self.assertIn(b'slaprunner', computer_list_output)
self.assertIn(b'slaprunner', self.captureSlapos('proxy', 'show'))
self.assertIn(b'slaprunner', self.captureSlapos('computer', 'list'))
class TestTheiaEmbeddedSlapOSShutdown(TheiaTestCase):
def test_stopping_instance_stops_embedded_slapos(self):
embedded_slapos_supervisord_socket = _getSupervisordSocketPath(
os.path.join(
self.computer_partition_root_path,
self.getPath(),
'srv',
'runner',
'instance',
......@@ -231,82 +270,70 @@ class TestTheiaEmbeddedSlapOSShutdown(TheiaTestCase):
self.assertFalse(embedded_slapos_process.is_running())
class ReRequestMixin(object):
def rerequest(self, parameter_dict=None, state='started'):
software_url = self.getSoftwareURL()
software_type = self.getInstanceSoftwareType()
name = self.default_partition_reference
self.slap.request(
software_release=software_url,
software_type=software_type,
partition_reference=name,
partition_parameter_kw=parameter_dict,
state=state)
def reinstantiate(self):
# Process at least twice to propagate parameter changes
try:
self.slap.waitForInstance()
except SlapOSNodeCommandError:
pass
self.slap.waitForInstance(self.instance_max_retry)
class TestTheiaWithEmbeddedInstance(TheiaTestCase):
sr_url = '~/bogus/sr/url.cfg'
sr_type = 'bogus-type'
sr_config = {"bogus": "yes"}
regexpr = re.compile(r"([\w/\-\.]+)\s+slaprunner\s+available")
class TestTheiaWithSR(TheiaTestCase, ReRequestMixin):
sr_url = '~/bogus/software.cfg'
sr_type = 'bogus_type'
instance_parameters = '{\n"bogus_param": "bogus_value",\n"bogus_param2": "bogus_value2"\n}'
def proxy_show(self, slapos):
return subprocess.check_output((slapos, 'proxy', 'show'), universal_newlines=True)
def test(self):
slapos = self._getSlapos()
home = self.computer_partition_root_path
# Check that no request script was generated
request_script = os.path.join(home, 'srv', 'project', 'request_embedded.sh')
self.assertFalse(os.path.exists(request_script))
# Manually request old-name 'Embedded Instance'
old_instance_name = "Embedded Instance"
subprocess.check_call((slapos, 'request', old_instance_name, 'bogus_url'))
self.assertIn(old_instance_name, self.proxy_show(slapos))
# Update Theia instance parameters
embedded_request_parameters = {
'embedded-sr': self.sr_url,
'embedded-sr-type': self.sr_type,
'embedded-instance-parameters': self.instance_parameters
@classmethod
def getInstanceParameterDict(cls, sr_url=None, sr_type=None, sr_config=None):
return {
'initial-embedded-instance': json.dumps({
'software-url': sr_url or cls.sr_url,
'software-type': sr_type or cls.sr_type,
'instance-parameters': sr_config or cls.sr_config,
}),
}
self.rerequest(embedded_request_parameters)
self.reinstantiate()
# Check that embedded instance was requested
instance_name = "embedded_instance"
info = self.proxy_show(slapos)
try:
self.assertIn(instance_name, info)
except AssertionError:
for filename in os.listdir(home):
if 'standalone' in filename and '.log' in filename:
filepath = os.path.join(home, filename)
with open(filepath) as f:
print("Contents of filepath: " + filepath)
print(f.read())
raise
def expandUrl(self, url):
if url.startswith('~/'):
url = os.path.join(self.getPath(), url[2:])
return url
def assertSupplied(self, sr_url, info=None):
info = info or self.captureSlapos('proxy', 'show', text=True)
self.assertIn(sr_url, info)
self.assertIn(sr_url, self.regexpr.findall(info))
def assertNotSupplied(self, sr_url, info=None):
info = info or self.captureSlapos('proxy', 'show', text=True)
self.assertNotIn(sr_url, info)
def assertEmbedded(self, sr_url, sr_type, config):
proxy_info = self.captureSlapos('proxy', 'show', text=True)
self.assertSupplied(sr_url, info=proxy_info)
name = 'embedded_instance'
self.assertIn(name, self.captureSlapos('service', 'list', text=True))
info = self.captureSlapos('service', 'info', name, text=True)
self.assertIn(sr_url, info)
self.assertIn(sr_type, proxy_info)
self.assertIn(repr(config).replace("u'", "'"), info)
def assertNotEmbedded(self, sr_url, sr_type, config):
sr_url = self.expandUrl(sr_url)
proxy_info = self.captureSlapos('proxy', 'show', text=True)
self.assertNotSupplied(sr_url, info=proxy_info)
self.assertNotIn(sr_type, proxy_info)
# Check that old-name instance was renamed
self.assertNotIn(old_instance_name, info)
def test(self):
# Check that embedded instance is supplied and requested
initial_sr_url = self.expandUrl(self.sr_url)
self.assertEmbedded(initial_sr_url, self.sr_type, self.sr_config)
# Check embedded instance parameters
bogus_sr = os.path.join(home, self.sr_url[2:])
# Change parameters for embedded instance
sr_url = '/bogus/sr/url-2.cfg'
sr_type = 'bogus-type-2'
sr_config = {"bogus-2": "true"}
self.requestInstance(
self.getInstanceParameterDict(sr_url, sr_type, sr_config))
self.waitForInstance()
self.assertIsNotNone(re.search(r"%s\s+slaprunner\s+available" % (bogus_sr,), info), info)
self.assertIsNotNone(re.search(r"%s\s+%s\s+%s" % (bogus_sr, self.sr_type, instance_name), info), info)
# Check that parameters have not been taken into account
self.assertNotEmbedded(sr_url, sr_type, sr_config)
service_info = subprocess.check_output((slapos, 'service', 'info', instance_name), universal_newlines=True)
self.assertIn("{'bogus_param': 'bogus_value', 'bogus_param2': 'bogus_value2'}", service_info)
# Check that previous instance has not been changed
self.assertEmbedded(initial_sr_url, self.sr_type, self.sr_config)
class TestTheiaFrontend(TheiaTestCase):
......@@ -331,7 +358,9 @@ class TestTheiaEnv(TheiaTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {
'embedded-sr': cls.dummy_software_path,
'initial-embedded-instance': json.dumps({
'software-url': cls.dummy_software_path,
}),
'autorun': 'stopped',
}
......@@ -339,7 +368,7 @@ class TestTheiaEnv(TheiaTestCase):
"""Make sure environment variables are the same whether we use shell or supervisor services.
"""
# The path of the env.json file expected to be generated by building the dummy software release
env_json_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'software', 'env.json')
env_json_path = self.getPath('srv', 'runner', 'software', 'env.json')
# Get the pid of the theia process from the test node's instance-supervisord
with self.slap.instance_supervisor_rpc as supervisor:
......@@ -357,7 +386,7 @@ class TestTheiaEnv(TheiaTestCase):
# Start a theia shell that inherits the environment of the theia process
# This simulates the environment of a shell launched from the browser application
theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.computer_partition_root_path), env=theia_env)
theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.getPath()), env=theia_env)
try:
theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
......@@ -377,7 +406,7 @@ class TestTheiaEnv(TheiaTestCase):
# Note that we have two services, slapos-node-software and slapos-node-software-all
# The later uses --all which is what we want to use here, because the software
# is already installed and we want to install it again, this time from supervisor
embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run')
embedded_run_path = self.getPath('srv', 'runner', 'var', 'run')
embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop']
......@@ -410,6 +439,8 @@ class ResilientTheiaMixin(object):
@classmethod
def setUpClass(cls):
super(ResilientTheiaMixin, cls).setUpClass()
# Patch the computer root path to that of the export theia instance
cls.computer_partition_root_path = cls.getPartitionPath('export')
# Add resiliency files to snapshot patterns
cls._save_instance_file_pattern_list += (
'*/srv/export-exitcode-file',
......@@ -419,43 +450,53 @@ class ResilientTheiaMixin(object):
)
@classmethod
def _getPartition(cls, software_type):
def getPartitionId(cls, instance_type):
software_url = cls.getSoftwareURL()
for computer_partition in cls.slap.computer.getComputerPartitionList():
partition_url = computer_partition.getSoftwareRelease()._software_release
partition_type = computer_partition.getType()
if partition_url == software_url and partition_type == software_type:
return computer_partition
raise Exception("Theia %s partition not found" % software_type)
if partition_url == software_url and partition_type == instance_type:
return computer_partition.getId()
raise Exception("Theia %s partition not found" % instance_type)
@classmethod
def getPartitionPath(cls, instance_type='export', *paths):
return os.path.join(cls.slap._instance_root, cls.getPartitionId(instance_type), *paths)
@classmethod
def _getPartitionId(cls, software_type):
return cls._getPartition(software_type).getId()
def _getSlapos(cls, instance_type='export'):
return cls.getPartitionPath(instance_type, 'srv', 'runner', 'bin', 'slapos')
@classmethod
def _getPartitionPath(cls, software_type, *paths):
return os.path.join(cls.slap._instance_root, cls._getPartitionId(software_type), *paths)
def callSlapos(cls, *command, **kwargs):
instance_type = kwargs.pop('instance_type', 'export')
return subprocess.call((cls._getSlapos(instance_type),) + command, **kwargs)
@classmethod
def _getSlapos(cls, software_type='export'):
return cls._getPartitionPath(software_type, 'srv', 'runner', 'bin', 'slapos')
def checkSlapos(cls, *command, **kwargs):
instance_type = kwargs.pop('instance_type', 'export')
return subprocess.check_call((cls._getSlapos(instance_type),) + command, **kwargs)
@classmethod
def captureSlapos(cls, *command, **kwargs):
kwargs.setdefault('universal_newlines', kwargs.pop('text', None))
instance_type = kwargs.pop('instance_type', 'export')
return subprocess.check_output((cls._getSlapos(instance_type),) + command, **kwargs)
@classmethod
def getInstanceSoftwareType(cls):
return 'resilient'
@classmethod
def waitForInstance(cls):
# process twice to propagate to all instances
for _ in range(2):
super(ResilientTheiaMixin, cls).waitForInstance()
class TestTheiaResilientInterface(ResilientTheiaMixin, TestTheia):
@classmethod
def setUpClass(cls):
super(TestTheiaResilientInterface, cls).setUpClass()
# Patch the computer root path to that of the export theia instance
cls.computer_partition_root_path = cls._getPartitionPath('export')
pass
class TestTheiaResilientWithSR(ResilientTheiaMixin, TestTheiaWithSR):
@classmethod
def setUpClass(cls):
super(TestTheiaResilientWithSR, cls).setUpClass()
# Patch the computer root path to that of the export theia instance
cls.computer_partition_root_path = cls._getPartitionPath('export')
class TestTheiaResilientWithEmbeddedInstance(ResilientTheiaMixin, TestTheiaWithEmbeddedInstance):
pass
software/theia/test/test_resiliency.py
View file @ 88f0bcd5
......@@ -63,11 +63,10 @@ def setUpModule():
class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
@classmethod
def _processEmbeddedInstance(cls, retries=0, software_type='export'):
slapos = cls._getSlapos(software_type)
def _processEmbeddedInstance(cls, retries=0, instance_type='export'):
for _ in range(retries):
try:
output = subprocess.check_output((slapos, 'node', 'instance'), stderr=subprocess.STDOUT)
output = cls.captureSlapos('node', 'instance', instance_type=instance_type, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
continue
print(output)
......@@ -77,19 +76,18 @@ class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
# Sleep a bit as an attempt to workaround monitoring boostrap not being ready
print("Wait before running slapos node instance one last time")
time.sleep(120)
subprocess.check_call((slapos, 'node', 'instance'))
cls.callSlapos('node', 'instance', instance_type=instance_type)
@classmethod
def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, software_type='export'):
slapos = cls._getSlapos(software_type)
subprocess.check_call((slapos, 'supply', software_url, 'slaprunner'))
def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, instance_type='export'):
cls.callSlapos('supply', software_url, 'slaprunner', instance_type=instance_type)
try:
subprocess.check_output((slapos, 'node', 'software'), stderr=subprocess.STDOUT)
cls.captureSlapos('node', 'software', instance_type=instance_type, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
print(e.output)
raise
subprocess.check_call((slapos, 'request', instance_name, software_url))
cls._processEmbeddedInstance(retries, software_type)
cls.callSlapos('request', instance_name, software_url, instance_type=instance_type)
cls._processEmbeddedInstance(retries, instance_type)
@classmethod
def getInstanceParameterDict(cls):
......@@ -136,16 +134,16 @@ class ResilienceMixin(object):
class ExportAndImportMixin(object):
def getExportExitfile(self):
return self._getPartitionPath('export', 'srv', 'export-exitcode-file')
return self.getPartitionPath('export', 'srv', 'export-exitcode-file')
def getExportErrorfile(self):
return self._getPartitionPath('export', 'srv', 'export-errormessage-file')
return self.getPartitionPath('export', 'srv', 'export-errormessage-file')
def getImportExitfile(self):
return self._getPartitionPath('import', 'srv', 'import-exitcode-file')
return self.getPartitionPath('import', 'srv', 'import-exitcode-file')
def getImportErrorfile(self):
return self._getPartitionPath('import', 'srv', 'import-errormessage-file')
return self.getPartitionPath('import', 'srv', 'import-errormessage-file')
def makedirs(self, path):
try:
......@@ -185,7 +183,7 @@ class ExportAndImportMixin(object):
initial_exitdate = os.path.getmtime(exitfile)
# Call export script manually
theia_export_script = self._getPartitionPath('export', 'bin', 'theia-export-script')
theia_export_script = self.getPartitionPath('export', 'bin', 'theia-export-script')
subprocess.check_call((theia_export_script,), stderr=subprocess.STDOUT)
# Check that the export exitcode file was modified
......@@ -198,8 +196,8 @@ class ExportAndImportMixin(object):
def _doTransfer(self):
# Copy <export>/srv/backup/theia to <import>/srv/backup/theia manually
export_backup_path = self._getPartitionPath('export', 'srv', 'backup', 'theia')
import_backup_path = self._getPartitionPath('import', 'srv', 'backup', 'theia')
export_backup_path = self.getPartitionPath('export', 'srv', 'backup', 'theia')
import_backup_path = self.getPartitionPath('import', 'srv', 'backup', 'theia')
shutil.rmtree(import_backup_path)
shutil.copytree(export_backup_path, import_backup_path)
......@@ -209,7 +207,7 @@ class ExportAndImportMixin(object):
initial_exitdate = os.path.getmtime(exitfile)
# Call the import script manually
theia_import_script = self._getPartitionPath('import', 'bin', 'theia-import-script')
theia_import_script = self.getPartitionPath('import', 'bin', 'theia-import-script')
subprocess.check_call((theia_import_script,), stderr=subprocess.STDOUT)
# Check that the import exitcode file was updated
......@@ -277,11 +275,11 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
os.remove(path)
def customSignatureScript(self, content=None):
custom_script = self._getPartitionPath('export', self.script_relpath)
custom_script = self.getPartitionPath('export', self.script_relpath)
self.customScript(custom_script, content)
def customRestoreScript(self, content=None):
restore_script = self._getPartitionPath('import', 'srv', 'runner-import-restore')
restore_script = self.getPartitionPath('import', 'srv', 'runner-import-restore')
self.customScript(restore_script, content)
return restore_script
......@@ -294,7 +292,7 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
self.customRestoreScript(content=None)
self.cleanupExitfiles()
try:
os.remove(self._getPartitionPath('import', self.signature_relpath))
os.remove(self.getPartitionPath('import', self.signature_relpath))
except OSError:
pass
......@@ -309,13 +307,13 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
def test_custom_hash_script(self):
errmsg = 'Bye bye'
self.customSignatureScript(content='>&2 echo "%s"\nexit 1' % errmsg)
custom_script = self._getPartitionPath('export', self.script_relpath)
custom_script = self.getPartitionPath('export', self.script_relpath)
self.assertExportFailure('Compute partitions backup signatures\n ... ERROR !',
'Custom signature script %s failed' % os.path.abspath(custom_script),
'and stderr:\n%s' % errmsg)
def test_signature_mismatch(self):
signature_file = self._getPartitionPath('import', self.signature_relpath)
signature_file = self.getPartitionPath('import', self.signature_relpath)
self.writeFile(signature_file, 'Bogus Hash\n', mode='a')
self.assertImportFailure('ERROR the backup signatures do not match')
......@@ -344,7 +342,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
def _prepareExport(self):
# Copy ./resilience_dummy SR in export theia ~/srv/project/dummy
dummy_target_path = self._getPartitionPath('export', 'srv', 'project', 'dummy')
dummy_target_path = self.getPartitionPath('export', 'srv', 'project', 'dummy')
shutil.copytree(os.path.dirname(dummy_software_url), dummy_target_path)
self._test_software_url = os.path.join(dummy_target_path, 'software.cfg')
......@@ -352,8 +350,8 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
self._deployEmbeddedSoftware(self._test_software_url, 'dummy_instance')
relpath_dummy = os.path.join('srv', 'runner', 'instance', 'slappart0')
self.export_dummy_root = dummy_root = self._getPartitionPath('export', relpath_dummy)
self.import_dummy_root = self._getPartitionPath('import', relpath_dummy)
self.export_dummy_root = dummy_root = self.getPartitionPath('export', relpath_dummy)
self.import_dummy_root = self.getPartitionPath('import', relpath_dummy)
# Check that dummy instance was properly deployed
self.initial_log = self.checkLog(os.path.join(dummy_root, 'log.log'))
......@@ -373,7 +371,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', '.backup_identity_script')))
# Remember content of ~/etc in the import theia
self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
def _doSync(self):
self._doExport()
......@@ -384,14 +382,13 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
dummy_root = self.import_dummy_root
# Check that the software url is correct
adapted_test_url = self._getPartitionPath('import', 'srv', 'project', 'dummy', 'software.cfg')
proxy_content = subprocess.check_output(
(self._getSlapos('import'), 'proxy', 'show'), universal_newlines=True)
adapted_test_url = self.getPartitionPath('import', 'srv', 'project', 'dummy', 'software.cfg')
proxy_content = self.captureSlapos('proxy', 'show', instance_type='import', text=True)
self.assertIn(adapted_test_url, proxy_content)
self.assertNotIn(self._test_software_url, proxy_content)
# Check that ~/etc still contains everything it did before
etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
self.assertTrue(set(self.etc_listdir).issubset(etc_listdir))
# Check that ~/srv/project was exported
......@@ -401,7 +398,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
self.checkLog(os.path.join(dummy_root, 'log.log'), self.initial_log, newline=None)
# Check that ~/srv/.backup_identity_script was detected and called
signature = self._getPartitionPath(
signature = self.getPartitionPath(
'import', 'srv', 'backup', 'theia', 'slappart0.backup.signature.custom')
self.assertTrue(os.path.exists(signature))
with open(signature) as f:
......@@ -418,7 +415,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
def _doTakeover(self):
# Start the dummy instance as a sort of fake takeover
subprocess.check_call((self._getSlapos('import'), 'node', 'instance'))
self.callSlapos('node', 'instance', instance_type='import')
def _checkTakeover(self):
# Check that dummy instance was properly re-deployed
......@@ -493,7 +490,7 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
# Run two synchronisations on the same instances
# to make sure everything still works the second time
# Check ~/etc in import theia again
self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
self._doSync()
self._checkSync()
......@@ -502,18 +499,18 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
self._deployEmbeddedSoftware(self._test_software_url, 'test_instance', self.test_instance_max_retries)
# Check that there is an export and import instance and get their partition IDs
self.export_id = self._getPartitionId('export')
self.import_id = self._getPartitionId('import')
self.export_id = self.getPartitionId('export')
self.import_id = self.getPartitionId('import')
# Remember content of ~/etc in the import theia
self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
def _doSync(self):
start = time.time()
# Call exporter script instead of waiting for cron job
# XXX Accelerate cron frequency instead ?
exporter_script = self._getPartitionPath('export', 'bin', 'exporter')
exporter_script = self.getPartitionPath('export', 'bin', 'exporter')
transaction_id = str(int(time.time()))
subprocess.check_call((exporter_script, '--transaction-id', transaction_id))
......@@ -524,7 +521,7 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
def _checkSync(self):
# Check that ~/etc still contains everything it did before
etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
self.assertTrue(set(self.etc_listdir).issubset(etc_listdir))
def _doTakeover(self):
......@@ -541,9 +538,9 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
# Check that there is an export, import and frozen instance and get their new partition IDs
import_id = self.import_id
export_id = self.export_id
new_export_id = self._getPartitionId('export')
new_import_id = self._getPartitionId('import')
new_frozen_id = self._getPartitionId('frozen')
new_export_id = self.getPartitionId('export')
new_import_id = self.getPartitionId('import')
new_frozen_id = self.getPartitionId('frozen')
# Check that old export instance is now frozen
self.assertEqual(export_id, new_frozen_id)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7