software/theia: Fix takoever of a theia with initial embedded instance

See merge request nexedi/slapos!1399

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = d8fa4bed6b7d7c77b0aec53abae32426
+md5sum = b406265f591f54a0d5a6aa3ff8522764
 
 [instance]
 _update_hash_filename_ = instance.cfg.in
@@ -23,11 +23,11 @@ md5sum = f322033a7670b9be29b1bf1bf9024b87
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in
-md5sum = a343818079d4fc106594e5850cc1853a
+md5sum = 6520d2aa0c1c6094cbf276080594ec1a
 
 [instance-export]
 _update_hash_filename_ = instance-export.cfg.jinja.in
-md5sum = 84ceb4c9ee0f07fce8518ef7517ce1d4
+md5sum = bb6d26c56b4bb9cf553c130fdd51000d
 
 [instance-resilient]
 _update_hash_filename_ = instance-resilient.cfg.jinja
@@ -35,7 +35,7 @@ md5sum = ad9499e7355ded4975ad313442cecb7a
 
 [slapos-standalone-script]
 _update_hash_filename_ = slapos_standalone_script.py.jinja
-md5sum = 93c492cc5c186989f104f37de43f25c5
+md5sum = 3572b3fa458505ae25ebcea9b1ed3267
 
 [theia-common]
 _update_hash_filename_ = theia_common.py

software/theia/instance-export.cfg.jinja.in

@@ -42,6 +42,7 @@ context =
   raw slapos_cfg $${directory:runner}/etc/slapos.cfg
   raw project_path $${directory:project}
   raw public_path $${directory:frontend-static-public}
+  raw statefiles_path $${directory:statefiles}
   key exitfile :exitcode-file
   key errorfile :error-file
 {%- raw %}
@@ -55,6 +56,7 @@ inline =
   --cfg {{ slapos_cfg }} \
   --dirs {{ project_path }} \
   --dirs {{ public_path }} \
+  --dirs {{ statefiles_path }} \
   --exitfile {{ exitfile }} \
   --errorfile {{ errorfile }}
 {%- endraw %}

software/theia/instance-import.cfg.jinja.in

@@ -102,6 +102,7 @@ context =
   raw slapos_cfg $${directory:runner}/etc/slapos.cfg
   raw project_path $${directory:project}
   raw public_path $${directory:frontend-static-public}
+  raw statefiles_path $${directory:statefiles}
   key exitfile :exitcode-file
   key errorfile :error-file
 {%- raw %}
@@ -122,6 +123,7 @@ inline =
   --cfg {{ slapos_cfg }} \
   --dirs {{ project_path }} \
   --dirs {{ public_path }} \
+  --dirs {{ statefiles_path }} \
   --exitfile {{ exitfile }} \
   --errorfile {{ errorfile }}
 {%- endraw %}

software/theia/instance-theia.cfg.jinja.in

@@ -50,6 +50,7 @@ bin = $${:home}/bin
 tmp = $${:home}/tmp
 dot-theia = $${:home}/.theia/
 pidfiles = $${:var}/run
+statefiles = $${:var}/state
 
 services = $${:etc}/service
 runner = $${:srv}/runner
@@ -589,7 +590,8 @@ inline =
 [slapos-standalone-script]
 recipe = slapos.recipe.template:jinja2
 output = $${directory:bin}/$${:_buildout_section_name_}
-embedded-request-exitcode-file = $${directory:etc}/embedded-request-exitcode
+embedded-request-exitcode-file = $${directory:statefiles}/embedded-request.exitcode
+standalone-ran-before-flag = $${directory:statefiles}/standalone-ran-before.flag
 shared-part-list =
   {{ """${buildout:shared-part-list}""" | indent(2) }}
 context =
@@ -599,6 +601,7 @@ context =
   key request_script_template request-script-example:inline
   key shared_part_list :shared-part-list
   key embedded_request_exitcode_file :embedded-request-exitcode-file
+  key standalone_ran_before_flag :standalone-ran-before-flag
   key embedded_instance_config embedded-instance-config:output
   key home_path directory:home
   key forward_frontend_requests :forward-frontend-requests

software/theia/slapos_standalone_script.py.jinja

@@ -153,10 +153,28 @@ def main():
   with setupStandalone() as standalone:
 
     config_json_file = {{ repr(embedded_instance_config) }}
-    done_file = config_json_file + '.done'
 
-    if not os.path.exists(done_file):
-      with open(done_file, 'w'): pass
+    # backwards compatibility
+    old_flag_file = config_json_file + '.done'
+    old_exitcode_file = os.path.join(
+      {{ repr(home_path) }}, 'etc', 'embedded-request-exitcode')
+
+    # new state filesstandalone_ran_before_flag
+    standalone_ran_before_flag = {{ repr(standalone_ran_before_flag) }}
+    exitcode_file = {{ repr(embedded_request_exitcode_file) }}
+
+    # backwards compatibility
+    if os.path.exists(old_flag_file):
+      logging.info("Moving old %s to %s", old_flag_file, standalone_ran_before_flag)
+      os.rename(old_flag_file, standalone_ran_before_flag)
+      if os.path.exists(old_exitcode_file):
+        logging.info("Moving old %s to %s", old_exitcode_file, exitcode_file)
+        os.rename(old_exitcode_file, exitcode_file)
+
+    elif not os.path.exists(standalone_ran_before_flag):
+      logging.info("First run!")
+      logging.info("Creating flag file %s", standalone_ran_before_flag)
+      with open(standalone_ran_before_flag, 'x'): pass
       try:
         config = parseEmbeddedInstanceConfig(config_json_file)
       except Exception:
@@ -172,7 +190,7 @@ def main():
             'SLAPOS_CONFIGURATION': standalone._slapos_config,
             'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config,
           })
-          with open({{ repr(embedded_request_exitcode_file) }}, 'w') as f:
+          with open(exitcode_file, 'w') as f:
             f.write(str(exitcode))
         except Exception:
           logging.info("Failed to request embedded instance", exc_info=True)

software/theia/test/test.py

@@ -538,6 +538,10 @@ class ResilientTheiaMixin(object):
   def getPartitionPath(cls, instance_type='export', *paths):
     return os.path.join(cls.slap._instance_root, cls.getPartitionId(instance_type), *paths)
 
+  @classmethod
+  def getPath(cls, *components): # patch getPath
+    return cls.getPartitionPath('export', *components)
+
   @classmethod
   def _getSlapos(cls, instance_type='export'):
     return cls.getPartitionPath(instance_type, 'srv', 'runner', 'bin', 'slapos')

software/theia/test/test_resiliency.py

@@ -27,6 +27,7 @@
 from __future__ import unicode_literals
 
 import errno
+import json
 import os
 import re
 import shutil
@@ -38,11 +39,15 @@ import requests
 
 from slapos.proxy.db_version import DB_VERSION
 
+from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE
+
 from slapos.testing.testcase import SlapOSNodeCommandError, installSoftwareUrlList
 
+import test
 from test import TheiaTestCase, ResilientTheiaMixin, theia_software_release_url
 
 
+
 dummy_software_url = os.path.abspath(
   os.path.join('resilience_dummy', 'software.cfg'))
 
@@ -82,8 +87,7 @@ class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
       cls.checkSlapos('node', 'instance', instance_type=instance_type)
 
   @classmethod
-  def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, instance_type='export'):
-    cls.callSlapos('supply', software_url, 'slaprunner', instance_type=instance_type)
+  def _processEmbeddedSoftware(cls, retries=0, instance_type='export'):
     for _ in range(retries):
       try:
         output = cls.captureSlapos('node', 'software', instance_type=instance_type, stderr=subprocess.STDOUT)
@@ -96,6 +100,11 @@ class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
         print("Wait before running slapos node software one last time")
         time.sleep(120)
       cls.checkSlapos('node', 'software', instance_type=instance_type)
+
+  @classmethod
+  def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, instance_type='export'):
+    cls.callSlapos('supply', software_url, 'slaprunner', instance_type=instance_type)
+    cls._processEmbeddedSoftware(retries, instance_type)
     cls.callSlapos('request', instance_name, software_url, instance_type=instance_type)
     cls._processEmbeddedInstance(retries, instance_type)
 
@@ -489,8 +498,18 @@ class TakeoverMixin(ExportAndImportMixin):
     self.assertIn("Success", resp.text, "An Error occured: %s" % resp.text)
     return resp.text
 
+  def _doTakeover(self):
+    # Takeover
+    takeover_url, takeover_password = self._getTakeoverUrlAndPassword()
+    self._requestTakeover(takeover_url, takeover_password)
 
-class TheiaSyncMixin(ResilienceMixin, TakeoverMixin):
+    # Wait for import instance to become export instance and new import to be allocated
+    # This also checks that all promises of theia instances succeed
+    self.slap.waitForInstance(self.instance_max_retry)
+    self.computer_partition = self.requestDefaultInstance()
+
+
+class TheiaSyncMixin(TakeoverMixin, ResilienceMixin):
   def _doSync(self, max_tries=None, wait_interval=None):
     max_tries = max_tries or self.backup_max_tries
     wait_interval = wait_interval or self.backup_wait_interval
@@ -538,17 +557,10 @@ class TestTheiaResilience(TheiaSyncMixin, ResilientTheiaTestCase):
   def _checkSync(self):
     # Check that ~/etc still contains everything it did before
     etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
+    try:
       self.assertTrue(set(self.etc_listdir).issubset(etc_listdir))
-
-  def _doTakeover(self):
-    # Takeover
-    takeover_url, takeover_password = self._getTakeoverUrlAndPassword()
-    self._requestTakeover(takeover_url, takeover_password)
-
-    # Wait for import instance to become export instance and new import to be allocated
-    # This also checks that all promises of theia instances succeed
-    self.slap.waitForInstance(self.instance_max_retry)
-    self.computer_partition = self.requestDefaultInstance()
+    except AssertionError:
+      breakpoint()
 
   def _checkTakeover(self):
     # Check that there is an export, import and frozen instance and get their new partition IDs
@@ -602,3 +614,43 @@ class TestTheiaFrontendForwarding(TheiaSyncMixin, ResilientTheiaTestCase):
   def _doTakeover(self):
     # do nothing
     pass
+
+
+class TestTheiaResilienceWithInitialInstance(TestTheiaResilience, test.TestTheiaWithEmbeddedInstance):
+  backup_max_tries = 70
+  backup_wait_interval = 10
+
+  sr_url = dummy_software_url
+  sr_type = DEFAULT_SOFTWARE_TYPE
+  sr_config = {}
+
+  @classmethod
+  def getInstanceParameterDict(cls, sr_url=None, sr_type=None, sr_config=None):
+    d = test.TestTheiaWithEmbeddedInstance.getInstanceParameterDict.__func__(
+      cls, sr_url, sr_type, sr_config)
+    d.update(autorun='stopped')
+    return d
+
+  def _prepareExport(self):
+    # Check that there is an export and import instance and get their partition IDs
+    self.export_id = self.getPartitionId('export')
+    self.import_id = self.getPartitionId('import')
+
+    # Remember content of ~/etc in the import theia
+    self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
+
+    # Check initial embedded instance
+    test.TestTheiaWithEmbeddedInstance.test(self)
+
+    self._processEmbeddedSoftware()
+    self._processEmbeddedInstance()
+
+  def _checkTakeover(self):
+    # Check takeover
+    TestTheiaResilience._checkTakeover(self)
+
+    # Check that embedded instance still exists
+    test.TestTheiaWithEmbeddedInstance.test(self)
+
+    self._processEmbeddedSoftware()
+    self._processEmbeddedInstance()

software/theia/test/upgrade_tests.py

+import json
+import os
+
+from slapos.testing.testcase import (
+  installSoftwareUrlList,
+  makeModuleSetUpAndTestCaseClass,
+  SlapOSNodeCommandError,
+)
+
+import test
+import test_resiliency
+
+
+stable_software_url = "https://lab.nexedi.com/nexedi/slapos/raw/1.0.324/software/theia/software.cfg"
+dev_software_url = os.path.abspath(
+  os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
+
+
+software_url_list = [stable_software_url, dev_software_url]
+
+_, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(stable_software_url)
+
+
+class UpgradeTestCase(SlapOSInstanceTestCase):
+  _current_software_url = stable_software_url
+
+  @classmethod
+  def getSoftwareURL(cls):
+    return cls._current_software_url
+
+  @classmethod
+  def upgrade(cls):
+    # request instance on dev software
+    cls._current_software_url = dev_software_url
+    cls.logger.debug('Requesting instance on dev software')
+    cls.requestDefaultInstance()
+
+    # wait for slapos node instance
+    snapshot_name = "{}.{}.dev.setUpClass".format(cls.__module__, cls.__name__)
+    with cls._snapshotManager(snapshot_name):
+      try:
+        for _ in range(2): # propagation
+          cls.waitForInstance()
+        cls.logger.debug("Instance on dev software done")
+      except BaseException:
+        cls.logger.exception("Error during instance on dev software")
+        raise
+
+    cls.computer_partition = cls.requestDefaultInstance()
+
+  @classmethod
+  def beforeUpgrade(cls):
+    pass
+
+  @classmethod
+  def setUpClass(cls):
+    # request and instantiate with old software url
+    super().setUpClass()
+
+    # before upgrade hook
+    cls.beforeUpgrade()
+
+    # upgrade
+    cls.upgrade()
+
+
+def setUpModule():
+  installSoftwareUrlList(
+    SlapOSInstanceTestCase,
+    software_url_list,
+    debug=SlapOSInstanceTestCase._debug,
+  )
+
+
+class TestTheia(UpgradeTestCase, test.TestTheia):
+  pass
+
+
+class TestTheiaWithEmbeddedInstance(
+    UpgradeTestCase,
+    test.TestTheiaWithEmbeddedInstance):
+  pass
+
+
+class TestTheiaResilientInterface(
+    UpgradeTestCase,
+    test.TestTheiaResilientInterface):
+  pass
+
+
+class TestTheiaResilientWithEmbeddedInstance(
+    UpgradeTestCase,
+    test.TestTheiaResilientWithEmbeddedInstance):
+  pass
+
+
+class TestTheiaResilienceWithInitialInstance(
+    UpgradeTestCase,
+    test_resiliency.TestTheiaResilienceWithInitialInstance):
+
+  @classmethod
+  def beforeUpgrade(cls):
+    # Check initial embedded instance
+    test.TestTheiaWithEmbeddedInstance.test(cls())
+
+
+class TestResilientTheiaUpgradeWithInitialInstance(
+    UpgradeTestCase,
+    test_resiliency.ResilientTheiaTestCase,
+    test_resiliency.TheiaSyncMixin):
+
+  backup_max_tries = 70
+  backup_wait_interval = 10
+
+  old_flag_file = os.path.join('etc', 'embedded-instance-config.json.done')
+  old_exitcode_file = os.path.join('etc', 'embedded-request-exitcode')
+
+  flag_file = os.path.join('var', 'state', 'standalone-ran-before.flag')
+  exitcode_file = os.path.join('var', 'state', 'embedded-request.exitcode')
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'initial-embedded-instance': json.dumps({
+        'software-url': test_resiliency.dummy_software_url
+      }),
+    }
+
+  def assertExists(self, path):
+    self.assertTrue(os.path.exists(path))
+
+  def assertNotFound(self, path):
+    self.assertFalse(os.path.exists(path))
+
+  @classmethod
+  def beforeUpgrade(cls):
+    self = cls()
+    self.assertExists(cls.getPartitionPath('export', self.old_flag_file))
+    self.assertExists(cls.getPartitionPath('export', self.old_exitcode_file))
+    self.assertExists(self.getPartitionPath('import', self.old_flag_file))
+    self.assertNotFound(self.getPartitionPath('import', self.old_exitcode_file))
+
+  def _prepareExport(self): # after upgrade
+    self.assertNotFound(self.getPartitionPath('export', self.old_flag_file))
+    self.assertNotFound(self.getPartitionPath('export', self.old_exitcode_file))
+    self.assertNotFound(self.getPartitionPath('import', self.old_flag_file))
+    self.assertNotFound(self.getPartitionPath('import', self.old_exitcode_file))
+
+    self.assertExists(self.getPartitionPath('export', self.flag_file))
+    self.assertExists(self.getPartitionPath('export', self.exitcode_file))
+    self.assertExists(self.getPartitionPath('import', self.flag_file))
+    self.assertNotFound(self.getPartitionPath('import', self.exitcode_file))
+
+  def _checkSync(self):
+    self.assertExists(self.getPartitionPath('export', self.flag_file))
+    self.assertExists(self.getPartitionPath('export', self.exitcode_file))
+    self.assertExists(self.getPartitionPath('import', self.flag_file))
+    self.assertExists(self.getPartitionPath('import', self.exitcode_file))
+
+  def _checkTakeover(self):
+    self.assertNotFound(self.getPartitionPath('export', self.old_flag_file))
+    self.assertNotFound(self.getPartitionPath('export', self.old_exitcode_file))
+    self.assertNotFound(self.getPartitionPath('import', self.old_flag_file))
+    self.assertNotFound(self.getPartitionPath('import', self.old_exitcode_file))
+
+    self.assertExists(self.getPartitionPath('export', self.flag_file))
+    self.assertExists(self.getPartitionPath('export', self.exitcode_file))
+    self.assertExists(self.getPartitionPath('import', self.flag_file))
+    self.assertNotFound(self.getPartitionPath('import', self.exitcode_file))
+
+
+class TestResilientTheiaUpgradeWithInitialInstanceAndSync(
+    TestResilientTheiaUpgradeWithInitialInstance):
+
+  @classmethod
+  def beforeUpgrade(cls):
+    cls()._doSync()
+
+  def _prepareExport(self): # after upgrade
+    self.assertNotFound(self.getPartitionPath('export', self.old_flag_file))
+    self.assertNotFound(self.getPartitionPath('export', self.old_exitcode_file))
+    self.assertNotFound(self.getPartitionPath('import', self.old_flag_file))
+    self.assertNotFound(self.getPartitionPath('import', self.old_exitcode_file))
+
+    self.assertExists(self.getPartitionPath('export', self.flag_file))
+    self.assertExists(self.getPartitionPath('export', self.exitcode_file))
+    self.assertExists(self.getPartitionPath('import', self.flag_file))
+    self.assertExists(self.getPartitionPath('import', self.exitcode_file))