Commit 9c7bc2ba authored by Xavier Thompson's avatar Xavier Thompson

software/theia: Fix takoever of a theia with initial embedded instance

See merge request nexedi/slapos!1399
parents 1aa17677 c56a9556
Pipeline #29109 failed with stage
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
[instance-theia] [instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in _update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = d8fa4bed6b7d7c77b0aec53abae32426 md5sum = b406265f591f54a0d5a6aa3ff8522764
[instance] [instance]
_update_hash_filename_ = instance.cfg.in _update_hash_filename_ = instance.cfg.in
...@@ -23,11 +23,11 @@ md5sum = f322033a7670b9be29b1bf1bf9024b87 ...@@ -23,11 +23,11 @@ md5sum = f322033a7670b9be29b1bf1bf9024b87
[instance-import] [instance-import]
_update_hash_filename_ = instance-import.cfg.jinja.in _update_hash_filename_ = instance-import.cfg.jinja.in
md5sum = a343818079d4fc106594e5850cc1853a md5sum = 6520d2aa0c1c6094cbf276080594ec1a
[instance-export] [instance-export]
_update_hash_filename_ = instance-export.cfg.jinja.in _update_hash_filename_ = instance-export.cfg.jinja.in
md5sum = 84ceb4c9ee0f07fce8518ef7517ce1d4 md5sum = bb6d26c56b4bb9cf553c130fdd51000d
[instance-resilient] [instance-resilient]
_update_hash_filename_ = instance-resilient.cfg.jinja _update_hash_filename_ = instance-resilient.cfg.jinja
...@@ -35,7 +35,7 @@ md5sum = ad9499e7355ded4975ad313442cecb7a ...@@ -35,7 +35,7 @@ md5sum = ad9499e7355ded4975ad313442cecb7a
[slapos-standalone-script] [slapos-standalone-script]
_update_hash_filename_ = slapos_standalone_script.py.jinja _update_hash_filename_ = slapos_standalone_script.py.jinja
md5sum = 93c492cc5c186989f104f37de43f25c5 md5sum = 3572b3fa458505ae25ebcea9b1ed3267
[theia-common] [theia-common]
_update_hash_filename_ = theia_common.py _update_hash_filename_ = theia_common.py
......
...@@ -42,6 +42,7 @@ context = ...@@ -42,6 +42,7 @@ context =
raw slapos_cfg $${directory:runner}/etc/slapos.cfg raw slapos_cfg $${directory:runner}/etc/slapos.cfg
raw project_path $${directory:project} raw project_path $${directory:project}
raw public_path $${directory:frontend-static-public} raw public_path $${directory:frontend-static-public}
raw statefiles_path $${directory:statefiles}
key exitfile :exitcode-file key exitfile :exitcode-file
key errorfile :error-file key errorfile :error-file
{%- raw %} {%- raw %}
...@@ -55,6 +56,7 @@ inline = ...@@ -55,6 +56,7 @@ inline =
--cfg {{ slapos_cfg }} \ --cfg {{ slapos_cfg }} \
--dirs {{ project_path }} \ --dirs {{ project_path }} \
--dirs {{ public_path }} \ --dirs {{ public_path }} \
--dirs {{ statefiles_path }} \
--exitfile {{ exitfile }} \ --exitfile {{ exitfile }} \
--errorfile {{ errorfile }} --errorfile {{ errorfile }}
{%- endraw %} {%- endraw %}
......
...@@ -102,6 +102,7 @@ context = ...@@ -102,6 +102,7 @@ context =
raw slapos_cfg $${directory:runner}/etc/slapos.cfg raw slapos_cfg $${directory:runner}/etc/slapos.cfg
raw project_path $${directory:project} raw project_path $${directory:project}
raw public_path $${directory:frontend-static-public} raw public_path $${directory:frontend-static-public}
raw statefiles_path $${directory:statefiles}
key exitfile :exitcode-file key exitfile :exitcode-file
key errorfile :error-file key errorfile :error-file
{%- raw %} {%- raw %}
...@@ -122,6 +123,7 @@ inline = ...@@ -122,6 +123,7 @@ inline =
--cfg {{ slapos_cfg }} \ --cfg {{ slapos_cfg }} \
--dirs {{ project_path }} \ --dirs {{ project_path }} \
--dirs {{ public_path }} \ --dirs {{ public_path }} \
--dirs {{ statefiles_path }} \
--exitfile {{ exitfile }} \ --exitfile {{ exitfile }} \
--errorfile {{ errorfile }} --errorfile {{ errorfile }}
{%- endraw %} {%- endraw %}
......
...@@ -50,6 +50,7 @@ bin = $${:home}/bin ...@@ -50,6 +50,7 @@ bin = $${:home}/bin
tmp = $${:home}/tmp tmp = $${:home}/tmp
dot-theia = $${:home}/.theia/ dot-theia = $${:home}/.theia/
pidfiles = $${:var}/run pidfiles = $${:var}/run
statefiles = $${:var}/state
services = $${:etc}/service services = $${:etc}/service
runner = $${:srv}/runner runner = $${:srv}/runner
...@@ -589,7 +590,8 @@ inline = ...@@ -589,7 +590,8 @@ inline =
[slapos-standalone-script] [slapos-standalone-script]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
output = $${directory:bin}/$${:_buildout_section_name_} output = $${directory:bin}/$${:_buildout_section_name_}
embedded-request-exitcode-file = $${directory:etc}/embedded-request-exitcode embedded-request-exitcode-file = $${directory:statefiles}/embedded-request.exitcode
standalone-ran-before-flag = $${directory:statefiles}/standalone-ran-before.flag
shared-part-list = shared-part-list =
{{ """${buildout:shared-part-list}""" | indent(2) }} {{ """${buildout:shared-part-list}""" | indent(2) }}
context = context =
...@@ -599,6 +601,7 @@ context = ...@@ -599,6 +601,7 @@ context =
key request_script_template request-script-example:inline key request_script_template request-script-example:inline
key shared_part_list :shared-part-list key shared_part_list :shared-part-list
key embedded_request_exitcode_file :embedded-request-exitcode-file key embedded_request_exitcode_file :embedded-request-exitcode-file
key standalone_ran_before_flag :standalone-ran-before-flag
key embedded_instance_config embedded-instance-config:output key embedded_instance_config embedded-instance-config:output
key home_path directory:home key home_path directory:home
key forward_frontend_requests :forward-frontend-requests key forward_frontend_requests :forward-frontend-requests
......
...@@ -153,10 +153,28 @@ def main(): ...@@ -153,10 +153,28 @@ def main():
with setupStandalone() as standalone: with setupStandalone() as standalone:
config_json_file = {{ repr(embedded_instance_config) }} config_json_file = {{ repr(embedded_instance_config) }}
done_file = config_json_file + '.done'
if not os.path.exists(done_file): # backwards compatibility
with open(done_file, 'w'): pass old_flag_file = config_json_file + '.done'
old_exitcode_file = os.path.join(
{{ repr(home_path) }}, 'etc', 'embedded-request-exitcode')
# new state filesstandalone_ran_before_flag
standalone_ran_before_flag = {{ repr(standalone_ran_before_flag) }}
exitcode_file = {{ repr(embedded_request_exitcode_file) }}
# backwards compatibility
if os.path.exists(old_flag_file):
logging.info("Moving old %s to %s", old_flag_file, standalone_ran_before_flag)
os.rename(old_flag_file, standalone_ran_before_flag)
if os.path.exists(old_exitcode_file):
logging.info("Moving old %s to %s", old_exitcode_file, exitcode_file)
os.rename(old_exitcode_file, exitcode_file)
elif not os.path.exists(standalone_ran_before_flag):
logging.info("First run!")
logging.info("Creating flag file %s", standalone_ran_before_flag)
with open(standalone_ran_before_flag, 'x'): pass
try: try:
config = parseEmbeddedInstanceConfig(config_json_file) config = parseEmbeddedInstanceConfig(config_json_file)
except Exception: except Exception:
...@@ -172,7 +190,7 @@ def main(): ...@@ -172,7 +190,7 @@ def main():
'SLAPOS_CONFIGURATION': standalone._slapos_config, 'SLAPOS_CONFIGURATION': standalone._slapos_config,
'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config, 'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config,
}) })
with open({{ repr(embedded_request_exitcode_file) }}, 'w') as f: with open(exitcode_file, 'w') as f:
f.write(str(exitcode)) f.write(str(exitcode))
except Exception: except Exception:
logging.info("Failed to request embedded instance", exc_info=True) logging.info("Failed to request embedded instance", exc_info=True)
......
...@@ -538,6 +538,10 @@ class ResilientTheiaMixin(object): ...@@ -538,6 +538,10 @@ class ResilientTheiaMixin(object):
def getPartitionPath(cls, instance_type='export', *paths): def getPartitionPath(cls, instance_type='export', *paths):
return os.path.join(cls.slap._instance_root, cls.getPartitionId(instance_type), *paths) return os.path.join(cls.slap._instance_root, cls.getPartitionId(instance_type), *paths)
@classmethod
def getPath(cls, *components): # patch getPath
return cls.getPartitionPath('export', *components)
@classmethod @classmethod
def _getSlapos(cls, instance_type='export'): def _getSlapos(cls, instance_type='export'):
return cls.getPartitionPath(instance_type, 'srv', 'runner', 'bin', 'slapos') return cls.getPartitionPath(instance_type, 'srv', 'runner', 'bin', 'slapos')
......
...@@ -27,6 +27,7 @@ ...@@ -27,6 +27,7 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import errno import errno
import json
import os import os
import re import re
import shutil import shutil
...@@ -38,11 +39,15 @@ import requests ...@@ -38,11 +39,15 @@ import requests
from slapos.proxy.db_version import DB_VERSION from slapos.proxy.db_version import DB_VERSION
from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE
from slapos.testing.testcase import SlapOSNodeCommandError, installSoftwareUrlList from slapos.testing.testcase import SlapOSNodeCommandError, installSoftwareUrlList
import test
from test import TheiaTestCase, ResilientTheiaMixin, theia_software_release_url from test import TheiaTestCase, ResilientTheiaMixin, theia_software_release_url
dummy_software_url = os.path.abspath( dummy_software_url = os.path.abspath(
os.path.join('resilience_dummy', 'software.cfg')) os.path.join('resilience_dummy', 'software.cfg'))
...@@ -82,8 +87,7 @@ class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase): ...@@ -82,8 +87,7 @@ class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
cls.checkSlapos('node', 'instance', instance_type=instance_type) cls.checkSlapos('node', 'instance', instance_type=instance_type)
@classmethod @classmethod
def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, instance_type='export'): def _processEmbeddedSoftware(cls, retries=0, instance_type='export'):
cls.callSlapos('supply', software_url, 'slaprunner', instance_type=instance_type)
for _ in range(retries): for _ in range(retries):
try: try:
output = cls.captureSlapos('node', 'software', instance_type=instance_type, stderr=subprocess.STDOUT) output = cls.captureSlapos('node', 'software', instance_type=instance_type, stderr=subprocess.STDOUT)
...@@ -96,6 +100,11 @@ class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase): ...@@ -96,6 +100,11 @@ class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
print("Wait before running slapos node software one last time") print("Wait before running slapos node software one last time")
time.sleep(120) time.sleep(120)
cls.checkSlapos('node', 'software', instance_type=instance_type) cls.checkSlapos('node', 'software', instance_type=instance_type)
@classmethod
def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, instance_type='export'):
cls.callSlapos('supply', software_url, 'slaprunner', instance_type=instance_type)
cls._processEmbeddedSoftware(retries, instance_type)
cls.callSlapos('request', instance_name, software_url, instance_type=instance_type) cls.callSlapos('request', instance_name, software_url, instance_type=instance_type)
cls._processEmbeddedInstance(retries, instance_type) cls._processEmbeddedInstance(retries, instance_type)
...@@ -489,8 +498,18 @@ class TakeoverMixin(ExportAndImportMixin): ...@@ -489,8 +498,18 @@ class TakeoverMixin(ExportAndImportMixin):
self.assertIn("Success", resp.text, "An Error occured: %s" % resp.text) self.assertIn("Success", resp.text, "An Error occured: %s" % resp.text)
return resp.text return resp.text
def _doTakeover(self):
# Takeover
takeover_url, takeover_password = self._getTakeoverUrlAndPassword()
self._requestTakeover(takeover_url, takeover_password)
class TheiaSyncMixin(ResilienceMixin, TakeoverMixin): # Wait for import instance to become export instance and new import to be allocated
# This also checks that all promises of theia instances succeed
self.slap.waitForInstance(self.instance_max_retry)
self.computer_partition = self.requestDefaultInstance()
class TheiaSyncMixin(TakeoverMixin, ResilienceMixin):
def _doSync(self, max_tries=None, wait_interval=None): def _doSync(self, max_tries=None, wait_interval=None):
max_tries = max_tries or self.backup_max_tries max_tries = max_tries or self.backup_max_tries
wait_interval = wait_interval or self.backup_wait_interval wait_interval = wait_interval or self.backup_wait_interval
...@@ -538,17 +557,10 @@ class TestTheiaResilience(TheiaSyncMixin, ResilientTheiaTestCase): ...@@ -538,17 +557,10 @@ class TestTheiaResilience(TheiaSyncMixin, ResilientTheiaTestCase):
def _checkSync(self): def _checkSync(self):
# Check that ~/etc still contains everything it did before # Check that ~/etc still contains everything it did before
etc_listdir = os.listdir(self.getPartitionPath('import', 'etc')) etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
try:
self.assertTrue(set(self.etc_listdir).issubset(etc_listdir)) self.assertTrue(set(self.etc_listdir).issubset(etc_listdir))
except AssertionError:
def _doTakeover(self): breakpoint()
# Takeover
takeover_url, takeover_password = self._getTakeoverUrlAndPassword()
self._requestTakeover(takeover_url, takeover_password)
# Wait for import instance to become export instance and new import to be allocated
# This also checks that all promises of theia instances succeed
self.slap.waitForInstance(self.instance_max_retry)
self.computer_partition = self.requestDefaultInstance()
def _checkTakeover(self): def _checkTakeover(self):
# Check that there is an export, import and frozen instance and get their new partition IDs # Check that there is an export, import and frozen instance and get their new partition IDs
...@@ -602,3 +614,43 @@ class TestTheiaFrontendForwarding(TheiaSyncMixin, ResilientTheiaTestCase): ...@@ -602,3 +614,43 @@ class TestTheiaFrontendForwarding(TheiaSyncMixin, ResilientTheiaTestCase):
def _doTakeover(self): def _doTakeover(self):
# do nothing # do nothing
pass pass
class TestTheiaResilienceWithInitialInstance(TestTheiaResilience, test.TestTheiaWithEmbeddedInstance):
backup_max_tries = 70
backup_wait_interval = 10
sr_url = dummy_software_url
sr_type = DEFAULT_SOFTWARE_TYPE
sr_config = {}
@classmethod
def getInstanceParameterDict(cls, sr_url=None, sr_type=None, sr_config=None):
d = test.TestTheiaWithEmbeddedInstance.getInstanceParameterDict.__func__(
cls, sr_url, sr_type, sr_config)
d.update(autorun='stopped')
return d
def _prepareExport(self):
# Check that there is an export and import instance and get their partition IDs
self.export_id = self.getPartitionId('export')
self.import_id = self.getPartitionId('import')
# Remember content of ~/etc in the import theia
self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
# Check initial embedded instance
test.TestTheiaWithEmbeddedInstance.test(self)
self._processEmbeddedSoftware()
self._processEmbeddedInstance()
def _checkTakeover(self):
# Check takeover
TestTheiaResilience._checkTakeover(self)
# Check that embedded instance still exists
test.TestTheiaWithEmbeddedInstance.test(self)
self._processEmbeddedSoftware()
self._processEmbeddedInstance()
import json
import os
from slapos.testing.testcase import (
installSoftwareUrlList,
makeModuleSetUpAndTestCaseClass,
SlapOSNodeCommandError,
)
import test
import test_resiliency
stable_software_url = "https://lab.nexedi.com/nexedi/slapos/raw/1.0.324/software/theia/software.cfg"
dev_software_url = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
software_url_list = [stable_software_url, dev_software_url]
_, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(stable_software_url)
class UpgradeTestCase(SlapOSInstanceTestCase):
_current_software_url = stable_software_url
@classmethod
def getSoftwareURL(cls):
return cls._current_software_url
@classmethod
def upgrade(cls):
# request instance on dev software
cls._current_software_url = dev_software_url
cls.logger.debug('Requesting instance on dev software')
cls.requestDefaultInstance()
# wait for slapos node instance
snapshot_name = "{}.{}.dev.setUpClass".format(cls.__module__, cls.__name__)
with cls._snapshotManager(snapshot_name):
try:
for _ in range(2): # propagation
cls.waitForInstance()
cls.logger.debug("Instance on dev software done")
except BaseException:
cls.logger.exception("Error during instance on dev software")
raise
cls.computer_partition = cls.requestDefaultInstance()
@classmethod
def beforeUpgrade(cls):
pass
@classmethod
def setUpClass(cls):
# request and instantiate with old software url
super().setUpClass()
# before upgrade hook
cls.beforeUpgrade()
# upgrade
cls.upgrade()
def setUpModule():
installSoftwareUrlList(
SlapOSInstanceTestCase,
software_url_list,
debug=SlapOSInstanceTestCase._debug,
)
class TestTheia(UpgradeTestCase, test.TestTheia):
pass
class TestTheiaWithEmbeddedInstance(
UpgradeTestCase,
test.TestTheiaWithEmbeddedInstance):
pass
class TestTheiaResilientInterface(
UpgradeTestCase,
test.TestTheiaResilientInterface):
pass
class TestTheiaResilientWithEmbeddedInstance(
UpgradeTestCase,
test.TestTheiaResilientWithEmbeddedInstance):
pass
class TestTheiaResilienceWithInitialInstance(
UpgradeTestCase,
test_resiliency.TestTheiaResilienceWithInitialInstance):
@classmethod
def beforeUpgrade(cls):
# Check initial embedded instance
test.TestTheiaWithEmbeddedInstance.test(cls())
class TestResilientTheiaUpgradeWithInitialInstance(
UpgradeTestCase,
test_resiliency.ResilientTheiaTestCase,
test_resiliency.TheiaSyncMixin):
backup_max_tries = 70
backup_wait_interval = 10
old_flag_file = os.path.join('etc', 'embedded-instance-config.json.done')
old_exitcode_file = os.path.join('etc', 'embedded-request-exitcode')
flag_file = os.path.join('var', 'state', 'standalone-ran-before.flag')
exitcode_file = os.path.join('var', 'state', 'embedded-request.exitcode')
@classmethod
def getInstanceParameterDict(cls):
return {
'initial-embedded-instance': json.dumps({
'software-url': test_resiliency.dummy_software_url
}),
}
def assertExists(self, path):
self.assertTrue(os.path.exists(path))
def assertNotFound(self, path):
self.assertFalse(os.path.exists(path))
@classmethod
def beforeUpgrade(cls):
self = cls()
self.assertExists(cls.getPartitionPath('export', self.old_flag_file))
self.assertExists(cls.getPartitionPath('export', self.old_exitcode_file))
self.assertExists(self.getPartitionPath('import', self.old_flag_file))
self.assertNotFound(self.getPartitionPath('import', self.old_exitcode_file))
def _prepareExport(self): # after upgrade
self.assertNotFound(self.getPartitionPath('export', self.old_flag_file))
self.assertNotFound(self.getPartitionPath('export', self.old_exitcode_file))
self.assertNotFound(self.getPartitionPath('import', self.old_flag_file))
self.assertNotFound(self.getPartitionPath('import', self.old_exitcode_file))
self.assertExists(self.getPartitionPath('export', self.flag_file))
self.assertExists(self.getPartitionPath('export', self.exitcode_file))
self.assertExists(self.getPartitionPath('import', self.flag_file))
self.assertNotFound(self.getPartitionPath('import', self.exitcode_file))
def _checkSync(self):
self.assertExists(self.getPartitionPath('export', self.flag_file))
self.assertExists(self.getPartitionPath('export', self.exitcode_file))
self.assertExists(self.getPartitionPath('import', self.flag_file))
self.assertExists(self.getPartitionPath('import', self.exitcode_file))
def _checkTakeover(self):
self.assertNotFound(self.getPartitionPath('export', self.old_flag_file))
self.assertNotFound(self.getPartitionPath('export', self.old_exitcode_file))
self.assertNotFound(self.getPartitionPath('import', self.old_flag_file))
self.assertNotFound(self.getPartitionPath('import', self.old_exitcode_file))
self.assertExists(self.getPartitionPath('export', self.flag_file))
self.assertExists(self.getPartitionPath('export', self.exitcode_file))
self.assertExists(self.getPartitionPath('import', self.flag_file))
self.assertNotFound(self.getPartitionPath('import', self.exitcode_file))
class TestResilientTheiaUpgradeWithInitialInstanceAndSync(
TestResilientTheiaUpgradeWithInitialInstance):
@classmethod
def beforeUpgrade(cls):
cls()._doSync()
def _prepareExport(self): # after upgrade
self.assertNotFound(self.getPartitionPath('export', self.old_flag_file))
self.assertNotFound(self.getPartitionPath('export', self.old_exitcode_file))
self.assertNotFound(self.getPartitionPath('import', self.old_flag_file))
self.assertNotFound(self.getPartitionPath('import', self.old_exitcode_file))
self.assertExists(self.getPartitionPath('export', self.flag_file))
self.assertExists(self.getPartitionPath('export', self.exitcode_file))
self.assertExists(self.getPartitionPath('import', self.flag_file))
self.assertExists(self.getPartitionPath('import', self.exitcode_file))
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment