Fix Gitlab resiliency tests

See merge request nexedi/slapos!1677

software/gitlab/buildout.hash.cfg

@@ -54,7 +54,7 @@ md5sum = d769ea27820e932c596c35bbbf3f2902
 
 [instance-gitlab.cfg.in]
 _update_hash_filename_ = instance-gitlab.cfg.in
-md5sum = e17bfe96bb9a0f4666d90c877a663e88
+md5sum = 35bb9f1d8f4fd6675bd768d8a7e1253c
 
 [instance-gitlab-export.cfg.in]
 _update_hash_filename_ = instance-gitlab-export.cfg.in

software/gitlab/instance-gitlab.cfg.in

@@ -916,8 +916,6 @@ rake    = ${gitlab-rake:wrapper-path}
 # run command on every reinstantiation
 update-command = ${:command}
 
-# https://gitlab.com/gitlab-org/gitlab-foss/issues/38457
-# we need to manually install ajv@^4.0.0 with yarn to fix the bug 'yarn check failed!'
 command =
     ${:rake} gitlab:assets:clean  &&
     ${:rake} gettext:compile RAILS_ENV=production &&

software/theia/buildout.hash.cfg

@@ -15,11 +15,11 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 23b498618bce83a6eb8df0470417f59e
+md5sum = b4e87cff99a8521e6d0b911e3ef35b30
 
 [instance]
 _update_hash_filename_ = instance.cfg.in
-md5sum = 5aab12790cdb1981cb0caf00d389a227
+md5sum = 837eb2786f185ddb5a28d29e271652f7
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in

software/theia/instance-theia.cfg.jinja.in

@@ -53,7 +53,11 @@ pidfiles = $${:var}/run
 statefiles = $${:var}/state
 
 services = $${:etc}/service
+{% if parameter_dict['testing-short-embedded-instance-path'] %}
+runner = $${:home}/r
+{% else %}
 runner = $${:srv}/runner
+{% endif %}
 backup = $${:srv}/backup/theia
 project = $${:srv}/project
 
@@ -567,8 +571,13 @@ ipv4 = {{ ipv4_random }}
 ipv6 = {{ slap_resource.get('ipv6-range-network') or ipv6_theia }}
 port = $${slapos-standalone-port:port}
 base-directory = $${directory:runner}
+{% if parameter_dict['testing-short-embedded-instance-path'] %}
+software-root = $${directory:runner}/s
+instance-root = $${directory:runner}/i
+{% else %}
 software-root = $${directory:runner}/software
 instance-root = $${directory:runner}/instance
+{% endif %}
 local-software-release-root = $${directory:home}
 slapos-bin = ${buildout:bin-directory}/slapos
 slapos-configuration = $${directory:runner}/etc/slapos.cfg

software/theia/instance.cfg.in

@@ -55,7 +55,8 @@ default-parameters =
     "additional-frontend-sr": "$${:frontend-sr}",
     "additional-frontend-sr-type": "default",
     "additional-frontend-guid": null,
-    "monitor-httpd-port": 8386
+    "monitor-httpd-port": 8386,
+    "testing-short-embedded-instance-path": null
   }
 frontend-sr = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 

software/theia/test/project_tests.py

@@ -24,7 +24,6 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from __future__ import unicode_literals
 
 import gzip
 import json
@@ -32,13 +31,12 @@ import os
 import re
 import subprocess
 import time
-import unittest
 import shutil
 import requests
 import tempfile
 
 from datetime import datetime, timedelta
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 from mimetypes import guess_type
 from json.decoder import JSONDecodeError
 
@@ -67,6 +65,21 @@ def setUpModule():
   )
 
 
+class TestTheiaResilienceWithShortPaths(test_resiliency.TestTheiaResilience):
+  """TestTheiaResilience, but with shorter paths for embedded slapos, to
+  overcome OS limit with the length of unix sockets or #! "shebang" lines.
+  """
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return dict(
+      super().getInstanceParameterDict(),
+      **{'testing-short-embedded-instance-path': 'true'})
+
+  @classmethod
+  def _getSlapos(cls, instance_type='export'):
+    return cls.getPartitionPath(instance_type, 'r', 'bin', 'slapos')
+
+
 class ERP5Mixin(object):
   _test_software_url = erp5_software_release_url
   _connexion_parameters_regex = re.compile(r"{.*}", re.DOTALL)
@@ -125,7 +138,7 @@ class ERP5Mixin(object):
       instance_type, 'srv', 'runner', 'instance', partition, *paths)
 
 
-class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
+class TestTheiaResilienceERP5(ERP5Mixin, TestTheiaResilienceWithShortPaths):
   test_instance_max_retries = 12
   backup_max_tries = 480
   backup_wait_interval = 60
@@ -242,7 +255,8 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
     out = subprocess.check_output((mysql_bin, 'erp5', '-e', query), universal_newlines=True)
     self.assertIn(self._erp5_new_title, out, 'Mariadb catalog is not properly restored')
 
-class TestTheiaResiliencePeertube(test_resiliency.TestTheiaResilience):
+
+class TestTheiaResiliencePeertube(TestTheiaResilienceWithShortPaths):
   test_instance_max_retries = 12
   backup_max_tries = 480
   backup_wait_interval = 60
@@ -446,9 +460,10 @@ class TestTheiaResiliencePeertube(test_resiliency.TestTheiaResilience):
   def _getPeertubePartitionPath(self, instance_type, servicename, *paths):
     partition = self._getPeertubePartition(servicename)
     return self.getPartitionPath(
-      instance_type, 'srv', 'runner', 'instance', partition, *paths)
+      instance_type, 'r', 'i', partition, *paths)
+
 
-class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience):
+class TestTheiaResilienceGitlab(TestTheiaResilienceWithShortPaths):
   test_instance_max_retries = 50  # puma takes time to be ready
   backup_max_tries = 480
   backup_wait_interval = 60
@@ -467,7 +482,7 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience):
       stderr=subprocess.STDOUT,
       text=True,
     )
-    print(out)
+    self.logger.info("_getGitlabConnectionParameters output: %s", out)
     return json.loads(self._connection_parameters_regex.search(out).group(0).replace("'", '"'))
 
   def test_twice(self):
@@ -499,7 +514,7 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience):
 
     # Create a new project
     print("Gitlab create a project")
-    path = '/api/v3/projects'
+    path = '/api/v4/projects'
     parameter_dict = {'name': 'sample-test', 'namespace': 'open'}
     # Token can be set manually
     headers = {"PRIVATE-TOKEN" : 'SLurtnxPscPsU-SDm4oN'}
@@ -508,14 +523,14 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience):
 
     # Check the project is exist
     print("Gitlab check project is exist")
-    path = '/api/v3/projects'
-    response = requests.get(backend_url + path, headers=headers, verify=False)
+    path = '/api/v4/projects'
+    response = requests.get(backend_url + path, params={'search': 'sample-test'}, headers=headers, verify=False)
     try:
       projects = response.json()
     except JSONDecodeError:
       self.fail("No json file returned! Maybe your Gitlab URL is incorrect.")
 
-    # Only one project exist
+    # Only one project matches the search
     self.assertEqual(len(projects), 1)
     # The project name is sample-test, which we created above.
     self.assertIn("sample-test", projects[0]['name_with_namespace'])
@@ -543,12 +558,14 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience):
     output = subprocess.check_output(('git', 'push', 'origin'), cwd=repo_path, universal_newlines=True)
 
     # Do a fake periodically update
-    # Compute backup date in the near future
-    soon = (datetime.now() + timedelta(minutes=4))
+    # Compute backup date in the future
+    # During slapos node instance, the static assets are recompiled, which takes a lot
+    # of time, so we give it at least 20 minutes.
+    soon = (datetime.now() + timedelta(minutes=20))
     frequency = "%d * * * *" % soon.minute
     params = 'backup_frequency=%s' % frequency
 
-    # Update Peertube parameters
+    # Update Gitlab parameters
     print('Requesting Gitlab with parameters %s' % params)
     self.checkSlapos('request', 'test_instance', self._test_software_url, '--parameters', params)
 
@@ -557,8 +574,8 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience):
     self.callSlapos('node', 'restart', 'all')
 
     # Wait until after the programmed backup date, and a bit more
-    t = (soon - datetime.now()).total_seconds()
-    time.sleep(t + 240)
+    t = ((soon - datetime.now()) + timedelta(minutes=10)).total_seconds()
+    time.sleep(t)
     self.callSlapos('node', 'status')
 
     os.chdir(self.temp_clone_dir)
@@ -583,9 +600,9 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience):
 
     # Check the project is exist
     print("Gitlab check project is exist")
-    path = '/api/v3/projects'
+    path = '/api/v4/projects'
     headers = {"PRIVATE-TOKEN" : 'SLurtnxPscPsU-SDm4oN'}
-    response = requests.get(backend_url + path, headers=headers, verify=False)
+    response = requests.get(backend_url + path, params={'search': 'sample-test'}, headers=headers, verify=False)
     try:
       projects = response.json()
     except JSONDecodeError:
@@ -623,4 +640,4 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience):
   def _getGitlabPartitionPath(self, instance_type, servicename, *paths):
     partition = self._getGitlabPartition(servicename)
     return self.getPartitionPath(
-      instance_type, 'srv', 'runner', 'instance', partition, *paths)
+      instance_type, 'r', 'i', partition, *paths)

software/theia/test/test_resiliency.py

@@ -72,34 +72,42 @@ def setUpModule():
 class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
   @classmethod
   def _processEmbeddedInstance(cls, retries=0, instance_type='export'):
-    for _ in range(retries):
+    for retry in range(retries):
       try:
-        output = cls.captureSlapos('node', 'instance', instance_type=instance_type, stderr=subprocess.STDOUT)
+        output = cls.captureSlapos('node', 'instance', instance_type=instance_type, stderr=subprocess.STDOUT, text=True)
       except subprocess.CalledProcessError:
         continue
-      print(output)
+      cls.logger.info("_processEmbeddedInstance retry=%s output=%s", retry, output)
       break
     else:
       if retries:
         # Sleep a bit as an attempt to workaround monitoring boostrap not being ready
         print("Wait before running slapos node instance one last time")
         time.sleep(120)
-      cls.checkSlapos('node', 'instance', instance_type=instance_type)
+      try:
+        cls.checkSlapos('node', 'instance', instance_type=instance_type, text=True)
+      except subprocess.CalledProcessError as e:
+        cls.logger.error(e.output, exc_info=True)
+        raise
 
   @classmethod
   def _processEmbeddedSoftware(cls, retries=0, instance_type='export'):
-    for _ in range(retries):
+    for retry in range(retries):
       try:
-        output = cls.captureSlapos('node', 'software', instance_type=instance_type, stderr=subprocess.STDOUT)
+        output = cls.captureSlapos('node', 'software', instance_type=instance_type, stderr=subprocess.STDOUT, text=True)
       except subprocess.CalledProcessError:
         continue
-      print(output)
+      cls.logger.info("_processEmbeddedSoftware retry=%s output=%s", retry, output)
       break
     else:
       if retries:
         print("Wait before running slapos node software one last time")
         time.sleep(120)
-      cls.checkSlapos('node', 'software', instance_type=instance_type)
+      try:
+        cls.checkSlapos('node', 'software', instance_type=instance_type, text=True)
+      except subprocess.CalledProcessError as e:
+        cls.logger.error(e.output, exc_info=True)
+        raise
 
   @classmethod
   def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, instance_type='export'):

stack/resilient/buildout.hash.cfg

@@ -30,7 +30,7 @@ md5sum = 1b8645835f04081861266436505fd28f
 
 [template-replicated]
 filename = template-replicated.cfg.in
-md5sum = 67c863b15dbfa937babdbd620f95c1ff
+md5sum = 743012b9e8d318712187621867613bd4
 
 [template-parts]
 filename = template-parts.cfg.in

stack/resilient/template-replicated.cfg.in

@@ -87,7 +87,7 @@ return = ssh-public-key resilient-ssh-url notification-url ip takeover-url takeo
 
 pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-push
 
-{% for extra_parameter_for_pseudo_replicating_instance in ["software-root", "buildout-shared-folder"] %}
+{% for extra_parameter_for_pseudo_replicating_instance in ["software-root", "buildout-shared-folder", "testing-short-embedded-instance-path"] %}
 {% if slapparameter_dict.get(extra_parameter_for_pseudo_replicating_instance) %}
 config-{{ extra_parameter_for_pseudo_replicating_instance }} = {{ slapparameter_dict.get(extra_parameter_for_pseudo_replicating_instance) }}
 {% endif %}