From eaaa854c55dc59debe4ed4998255ae30ba57015a Mon Sep 17 00:00:00 2001 From: Nicolas Wavrant Date: Thu, 12 Dec 2019 10:05:51 +0100 Subject: [PATCH 01/11] slaprunner: test support of multiple software release And the upgrade from one to another --- software/slapos-sr-testing/software.cfg | 4 +- software/slaprunner/test/setup.py | 1 + .../slaprunner/test/software_v1/instance.cfg | 9 + .../slaprunner/test/software_v1/software.cfg | 22 +++ .../slaprunner/test/software_v2/instance.cfg | 9 + .../slaprunner/test/software_v2/software.cfg | 22 +++ software/slaprunner/test/test.py | 179 +++++++++++++++++- 7 files changed, 241 insertions(+), 5 deletions(-) create mode 100644 software/slaprunner/test/software_v1/instance.cfg create mode 100644 software/slaprunner/test/software_v1/software.cfg create mode 100644 software/slaprunner/test/software_v2/instance.cfg create mode 100644 software/slaprunner/test/software_v2/software.cfg diff --git a/software/slapos-sr-testing/software.cfg b/software/slapos-sr-testing/software.cfg index 058952da2c..ec1380f8a5 100644 --- a/software/slapos-sr-testing/software.cfg +++ b/software/slapos-sr-testing/software.cfg @@ -291,4 +291,6 @@ mock = 2.0.0 testfixtures = 6.11 funcsigs = 1.0.2 PyPDF2 = 1.26.0 -mysqlclient = 1.3.12 \ No newline at end of file +mysqlclient = 1.3.12 + +BeautifulSoup = 3.2.2 diff --git a/software/slaprunner/test/setup.py b/software/slaprunner/test/setup.py index dca232fe3e..b5ad7e06d5 100644 --- a/software/slaprunner/test/setup.py +++ b/software/slaprunner/test/setup.py @@ -41,6 +41,7 @@ setup(name=name, url="https://lab.nexedi.com/nexedi/slapos", packages=find_packages(), install_requires=[ + 'BeautifulSoup', 'slapos.core', 'slapos.cookbook', 'slapos.libnetworkcache', diff --git a/software/slaprunner/test/software_v1/instance.cfg b/software/slaprunner/test/software_v1/instance.cfg new file mode 100644 index 0000000000..ea55dbf7df --- /dev/null +++ b/software/slaprunner/test/software_v1/instance.cfg @@ -0,0 +1,9 @@ +[buildout] +parts = instance + +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} + +[instance] +recipe = plone.recipe.command +command = echo -n 1 > $${buildout:directory}/version.txt diff --git a/software/slaprunner/test/software_v1/software.cfg b/software/slaprunner/test/software_v1/software.cfg new file mode 100644 index 0000000000..43de445675 --- /dev/null +++ b/software/slaprunner/test/software_v1/software.cfg @@ -0,0 +1,22 @@ +[buildout] +parts = + eggs + instance + +extends = + ../../../../stack/slapos.cfg + +[eggs] +recipe = zc.recipe.egg +eggs = + plone.recipe.command + +[instance] +recipe = slapos.recipe.template +filename = instance.cfg +url = ${:_profile_base_location_}/${:filename} +output = ${buildout:directory}/${:filename} +#md5sum = 869d5362b3daaa04db81129ce4f77c30 + +[versions] +slapos.recipe.template = 4.3 diff --git a/software/slaprunner/test/software_v2/instance.cfg b/software/slaprunner/test/software_v2/instance.cfg new file mode 100644 index 0000000000..68058b8279 --- /dev/null +++ b/software/slaprunner/test/software_v2/instance.cfg @@ -0,0 +1,9 @@ +[buildout] +parts = instance + +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} + +[instance] +recipe = plone.recipe.command +command = echo -n 2 > $${buildout:directory}/version.txt diff --git a/software/slaprunner/test/software_v2/software.cfg b/software/slaprunner/test/software_v2/software.cfg new file mode 100644 index 0000000000..43de445675 --- /dev/null +++ b/software/slaprunner/test/software_v2/software.cfg @@ -0,0 +1,22 @@ +[buildout] +parts = + eggs + instance + +extends = + ../../../../stack/slapos.cfg + +[eggs] +recipe = zc.recipe.egg +eggs = + plone.recipe.command + +[instance] +recipe = slapos.recipe.template +filename = instance.cfg +url = ${:_profile_base_location_}/${:filename} +output = ${buildout:directory}/${:filename} +#md5sum = 869d5362b3daaa04db81129ce4f77c30 + +[versions] +slapos.recipe.template = 4.3 diff --git a/software/slaprunner/test/test.py b/software/slaprunner/test/test.py index a2f718d9c9..96ce80125f 100644 --- a/software/slaprunner/test/test.py +++ b/software/slaprunner/test/test.py @@ -25,13 +25,19 @@ # ############################################################################## -import os -import unittest -import paramiko -import contextlib import base64 +import contextlib import hashlib +import json +import os +import paramiko +import requests import subprocess +import time +import unittest + +from lxml import etree +from lxml.html import soupparser from six.moves.urllib.parse import urlparse from six.moves.urllib.parse import quote @@ -315,3 +321,168 @@ class TestCustomFrontend(SlaprunnerTestCase): self.assertEqual( parameter_dict['custom-frontend-url'], 'https://www.erp5.com') + +class TestSlapProxyIntegration(SlaprunnerTestCase): + instance_max_retry = 5 + + @classmethod + def getInstanceParameterDict(cls): + return { + 'autorun': True, + 'auto-deploy': True, + 'slapos-repository': 'https://lab.nexedi.com/nexedi/slapos.git', + 'slapos-reference': 'runner-multi-sr', + } + + def setUp(self): + self.parameter_dict = parameter_dict = self.computer_partition.getConnectionParameterDict() + self.base_url = parameter_dict['url'] + + def _call(self, path, method="GET", data=None, assert_result_code=False): + resp = { + "GET": requests.get, + "POST": requests.post, + }[method]( + "%s/%s" % (self.base_url, path), + verify=False, + auth=(self.parameter_dict['init-user'], self.parameter_dict['init-password']), + data=data + ) + if assert_result_code: + try: + self.assertEqual(json.loads(resp.text)['code'], 1) + except ValueError: + raise ValueError("call to %s failed with HTTP code %s" % ( + path, result.status_code + )) + return resp + + def _buildAndRun(self): + self.logger.debug('Running Build&Run') + self._call('runSoftwareProfile') + while True: + time.sleep(10) + try: + response = self._call('slapgridResult') + result = json.loads(response.text) + except ValueError: + raise ValueError("couldnt decode JSON result of call (code: %s) : %s" % (response.status_code, response.text)) + self.logger.debug('%s', result) + + # If software or instance is currently under process, wait + if result['software']['state'] or result['instance']['state']: + self.logger.debug('Currently doing Build&Run') + continue + + if result['software']['success'] == 0: + if result['instance']['success'] == 0: + self.logger.debug('Instance is up!') + # Done + return + else: + self.logger.debug('Starting Run') + self._call('runInstanceProfile') + else: + self.logger.debug('Try to Build one more time') + self._call('runSoftwareProfile') + + def test_upgrade_of_software_release(self): + """ + Test that 2 Software Releases can be built at the same time in a webrunner, + and that the instance can be moved from one to another. + Also check border cases, including : + * ordering a new SR won't delete the existing ones + * only SR correctly built can be chosen for the instance + * the SR in-use can't be deleted + """ + # 1: Request 2 Software Releases, and do build&run + result = self._call( + 'supplySoftwareRelease', + method="POST", + data={"path": "workspace/slapos/software/slaprunner/test/software_v1/"}, + assert_result_code=True, + ) + result = self._call( + 'supplySoftwareRelease', + method="POST", + data={"path": "workspace/slapos/software/slaprunner/test/software_v2/"}, + assert_result_code=True, + ) + self._buildAndRun() + + # 2: Check that the Software Release of the instance is the 1st requested + result = self._call('inspectInstance') + root = soupparser.fromstring(result.text) + select_el = root.find(".//select[@name='software_release']") + self.assertEqual( + select_el.value_options, + [ + 'workspace/slapos/software/slaprunner/test/software_v1/software.cfg', + 'workspace/slapos/software/slaprunner/test/software_v2/software.cfg', + ] + ) + result = self._call( + 'getFileContent', + method="POST", + data={"file": "instance_root/slappart0/version.txt"}, + assert_result_code=True, + ) + self.assertEqual(json.loads(result.text)['result'], u'1') + + # 3: Check that that only the unused Software Release can be deleted + delete_form_el, = root.findall(".//form[@action='/destroySoftwareRelease']") + self.assertEqual( + delete_form_el.fields['uri'], + 'workspace/slapos/software/slaprunner/test/software_v2/software.cfg', + ) + + # 4: Request the instance with the 2nd Software Release, and check modification is registered + result = self._call( + 'saveParameterXml', + method="POST", + data={ + "software_release": "workspace/slapos/software/slaprunner/test/software_v2/software.cfg", + "software_type": "default", + "parameter": '\n\n', + }, + assert_result_code=True, + ) + result = self._call('inspectInstance') + root = soupparser.fromstring(result.text) + select_el = root.find(".//select[@name='software_release']") + self.assertEqual(select_el.value_options[0], "workspace/slapos/software/slaprunner/test/software_v2/software.cfg") + + # 5: Do build&run, and check that the instance was updated with the profile of the 2nd Software Release + self._buildAndRun() + result = self._call( + 'getFileContent', + method="POST", + data={"file": "instance_root/slappart0/version.txt"}, + assert_result_code=True, + ) + self.assertEqual(json.loads(result.text)['result'], u'2') + + # 6: Check that that only the first Software Release can be deleted, and delete it + delete_form_el, = root.findall(".//form[@action='/destroySoftwareRelease']") + self.assertEqual( + delete_form_el.fields['uri'], + 'workspace/slapos/software/slaprunner/test/software_v1/software.cfg', + ) + self._call('destroySoftwareRelease', method="POST", data={ + "uri": "workspace/slapos/software/slaprunner/test/software_v1/software.cfg" + }) + + # 7: Check that the first Software Release can't be chosen anymore + # for the instance, neither is shown as deletable + result = self._call('inspectInstance') + root = soupparser.fromstring(result.text) + select_el = root.find(".//select[@name='software_release']") + self.assertEqual( + select_el.value_options, + [ + 'workspace/slapos/software/slaprunner/test/software_v2/software.cfg', + ] + ) + self.assertTrue( + len(root.findall(".//form[@action='/destroySoftwareRelease']")) == 0 + ) -- 2.30.9 From 848e980df7e71577fba9de7526fdaf9c2c2459b3 Mon Sep 17 00:00:00 2001 From: Nicolas Wavrant Date: Wed, 8 Apr 2020 17:38:42 +0900 Subject: [PATCH 02/11] slaprunner/test: create needed directories so PromiseLauncher don't crash --- software/slaprunner/test/software_v1/instance.cfg | 11 ++++++++++- software/slaprunner/test/software_v2/instance.cfg | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/software/slaprunner/test/software_v1/instance.cfg b/software/slaprunner/test/software_v1/instance.cfg index ea55dbf7df..c7cea76c14 100644 --- a/software/slaprunner/test/software_v1/instance.cfg +++ b/software/slaprunner/test/software_v1/instance.cfg @@ -1,9 +1,18 @@ [buildout] -parts = instance +parts = + directory + instance eggs-directory = ${buildout:eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory} +[directory] +recipe = plone.recipe.command +home = $${buildout:directory} +etc = $${:home}/etc +plugins = $${:etc}/plugin +command = mkdir $${:etc} $${:plugins} + [instance] recipe = plone.recipe.command command = echo -n 1 > $${buildout:directory}/version.txt diff --git a/software/slaprunner/test/software_v2/instance.cfg b/software/slaprunner/test/software_v2/instance.cfg index 68058b8279..186be33bed 100644 --- a/software/slaprunner/test/software_v2/instance.cfg +++ b/software/slaprunner/test/software_v2/instance.cfg @@ -1,9 +1,18 @@ [buildout] -parts = instance +parts = + directory + instance eggs-directory = ${buildout:eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory} +[directory] +recipe = plone.recipe.command +home = $${buildout:directory} +etc = $${:home}/etc +plugins = $${:etc}/plugin +command = mkdir $${:etc} $${:plugins} + [instance] recipe = plone.recipe.command command = echo -n 2 > $${buildout:directory}/version.txt -- 2.30.9 From 77a19086d84bea2eb7c165c832eb81794322b002 Mon Sep 17 00:00:00 2001 From: Nicolas Wavrant Date: Mon, 6 Apr 2020 11:58:07 +0900 Subject: [PATCH 03/11] pycurl version is already defined in stack/slapos.cfg --- software/agent/software.cfg | 1 - software/kvm/software.cfg | 1 - software/neoppod/software-common.cfg | 1 - software/slapos-testing/software.cfg | 1 - software/slaprunner/software.cfg | 1 - stack/erp5/buildout.cfg | 1 - 6 files changed, 6 deletions(-) diff --git a/software/agent/software.cfg b/software/agent/software.cfg index 277419f0be..138757ae5f 100644 --- a/software/agent/software.cfg +++ b/software/agent/software.cfg @@ -46,7 +46,6 @@ eggs = ecdsa = 0.13 gitdb = 0.6.4 pycrypto = 2.6.1 -pycurl = 7.43.0 slapos.recipe.download = 1.0 slapos.recipe.template = 4.4 smmap = 0.9.0 diff --git a/software/kvm/software.cfg b/software/kvm/software.cfg index c68d0bb5cb..a6252d59e8 100644 --- a/software/kvm/software.cfg +++ b/software/kvm/software.cfg @@ -189,7 +189,6 @@ websockify = 0.9.0 collective.recipe.environment = 0.2.0 gitdb = 0.6.4 -pycurl = 7.43.0 slapos.recipe.template = 4.4 smmap = 0.9.0 numpy = 1.16.4 diff --git a/software/neoppod/software-common.cfg b/software/neoppod/software-common.cfg index 22ec396422..7ce4cc4a73 100644 --- a/software/neoppod/software-common.cfg +++ b/software/neoppod/software-common.cfg @@ -150,7 +150,6 @@ msgpack-python = 0.5.6 mysqlclient = 1.3.12 persistent = 4.5.0 pycrypto = 2.6.1 -pycurl = 7.43.0 setproctitle = 1.1.10 slapos.recipe.template = 4.4 smmap2 = 2.0.1 diff --git a/software/slapos-testing/software.cfg b/software/slapos-testing/software.cfg index db6188218d..258c7da7a2 100644 --- a/software/slapos-testing/software.cfg +++ b/software/slapos-testing/software.cfg @@ -210,7 +210,6 @@ manuel = 1.9.0 mock = 2.0.0 testfixtures = 6.11.0 pem = 18.1.0 -pycurl = 7.43.0.2 pyflakes = 2.0.0 smmap2 = 2.0.4 zope.testing = 4.6.2 diff --git a/software/slaprunner/software.cfg b/software/slaprunner/software.cfg index 321302a8ad..4079c775ba 100644 --- a/software/slaprunner/software.cfg +++ b/software/slaprunner/software.cfg @@ -171,7 +171,6 @@ futures = 3.0.5 gitdb = 0.6.4 gunicorn = 19.10.0 prettytable = 0.7.2 -pycurl = 7.43.0 slapos.recipe.template = 4.4 collective.recipe.environment = 0.2.0 smmap = 0.9.0 diff --git a/stack/erp5/buildout.cfg b/stack/erp5/buildout.cfg index b3e35f6034..86b37151e7 100644 --- a/stack/erp5/buildout.cfg +++ b/stack/erp5/buildout.cfg @@ -660,7 +660,6 @@ pprofile = 2.0.4 pyasn1-modules = 0.0.8 pycountry = 17.1.8 pycrypto = 2.6.1 -pycurl = 7.43.0 pyflakes = 1.5.0 python-memcached = 1.58 pytracemalloc = 1.2 -- 2.30.9 From 0c77ac5375e68f8ebe2b45d42794eaae8badd114 Mon Sep 17 00:00:00 2001 From: Nicolas Wavrant Date: Mon, 6 Apr 2020 14:23:50 +0900 Subject: [PATCH 04/11] slapos-sr-testing: pycurl egg must be set before slapos.toolbox --- software/slapos-sr-testing/software.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/software/slapos-sr-testing/software.cfg b/software/slapos-sr-testing/software.cfg index ec1380f8a5..a9c92d3b78 100644 --- a/software/slapos-sr-testing/software.cfg +++ b/software/slapos-sr-testing/software.cfg @@ -5,6 +5,7 @@ extends = ../../component/curl/buildout.cfg ../../component/git/buildout.cfg ../../component/pillow/buildout.cfg + ../../component/pycurl/buildout.cfg ../../component/python-cryptography/buildout.cfg ../../component/python-mysqlclient/buildout.cfg ../../component/python-pynacl/buildout.cfg @@ -156,6 +157,7 @@ setup = ${slapos.core-repository:location} [eggs] recipe = zc.recipe.egg eggs = + ${pycurl:egg} ${lxml-python:egg} ${slapos.core-setup:egg} ${pillow-python:egg} -- 2.30.9 From 5678e7c03a7c91f37e1b6b96049db2086410dcc5 Mon Sep 17 00:00:00 2001 From: Nicolas Wavrant Date: Tue, 14 Apr 2020 11:30:01 +0900 Subject: [PATCH 05/11] slaprunner/test: snapshot more files to ease debugging --- software/slaprunner/test/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/software/slaprunner/test/test.py b/software/slaprunner/test/test.py index 96ce80125f..8ad532d4d5 100644 --- a/software/slaprunner/test/test.py +++ b/software/slaprunner/test/test.py @@ -324,6 +324,7 @@ class TestCustomFrontend(SlaprunnerTestCase): class TestSlapProxyIntegration(SlaprunnerTestCase): instance_max_retry = 5 + _save_instance_file_pattern_list = SlaprunnerTestCase._save_instance_file_pattern_list + ('*/srv/runner/*.json',) @classmethod def getInstanceParameterDict(cls): -- 2.30.9 From 591e5b1420069085c05cab17e6c0d1aa131ff5f2 Mon Sep 17 00:00:00 2001 From: Nicolas Wavrant Date: Tue, 14 Jan 2020 04:18:16 +0100 Subject: [PATCH 06/11] DONOTMERGE: changes for running tests on correct branches + debug --- software/slapos-sr-testing/software.cfg | 27 ++----------------------- software/slaprunner/buildout.hash.cfg | 2 +- software/slaprunner/instance-runner.cfg | 2 +- stack/slapos.cfg | 17 ++++++++++++++-- 4 files changed, 19 insertions(+), 29 deletions(-) diff --git a/software/slapos-sr-testing/software.cfg b/software/slapos-sr-testing/software.cfg index a9c92d3b78..e137bd278f 100644 --- a/software/slapos-sr-testing/software.cfg +++ b/software/slapos-sr-testing/software.cfg @@ -159,6 +159,7 @@ recipe = zc.recipe.egg eggs = ${pycurl:egg} ${lxml-python:egg} + ${slapos.toolbox:egg} ${slapos.core-setup:egg} ${pillow-python:egg} erp5.util @@ -207,7 +208,7 @@ forbid-download-cache = true [slapos-repository] <= git-clone-repository repository = https://lab.nexedi.com/nexedi/slapos.git -branch = master +branch = runner-multi-sr [template] recipe = slapos.recipe.template:jinja2 @@ -223,31 +224,7 @@ context = key curl_location curl:location key tests :tests tests = - ${slapos.test.kvm-setup:setup} ${slapos.test.slaprunner-setup:setup} - ${:extra} -extra = - ${slapos.cookbook-setup:setup} - ${slapos.test.backupserver-setup:setup} - ${slapos.test.caddy-frontend-setup:setup} - ${slapos.test.erp5-setup:setup} - ${slapos.test.htmlvalidatorserver-setup:setup} - ${slapos.test.slapos-master-setup:setup} - ${slapos.test.plantuml-setup:setup} - ${slapos.test.powerdns-setup:setup} - ${slapos.test.proftpd-setup:setup} - ${slapos.test.re6stnet-setup:setup} - ${slapos.test.seleniumserver-setup:setup} - ${slapos.test.helloworld-setup:setup} - ${slapos.test.jstestnode-setup:setup} - ${slapos.test.jupyter-setup:setup} - ${slapos.test.nextcloud-setup:setup} - ${slapos.test.turnserver-setup:setup} - ${slapos.test.theia-setup:setup} - ${slapos.test.grafana-setup:setup} - ${slapos.test.gitlab-setup:setup} - ${slapos.test.cloudooo-setup:setup} - ${slapos.test.dream-setup:setup} [versions] # slapos.core is used from the clone always diff --git a/software/slaprunner/buildout.hash.cfg b/software/slaprunner/buildout.hash.cfg index f7977fbac3..b4a01e6757 100644 --- a/software/slaprunner/buildout.hash.cfg +++ b/software/slaprunner/buildout.hash.cfg @@ -18,7 +18,7 @@ md5sum = ec70348dd71b319590a5c5837f3d2e45 [template-runner] filename = instance-runner.cfg -md5sum = f342daca2e87f8fa44fd7b6a2a1c22f8 +md5sum = e7d59488e5f67011985211d775a9c499 [template-runner-import-script] filename = template/runner-import.sh.jinja2 diff --git a/software/slaprunner/instance-runner.cfg b/software/slaprunner/instance-runner.cfg index a802cb90dc..7e0eae5c7c 100644 --- a/software/slaprunner/instance-runner.cfg +++ b/software/slaprunner/instance-runner.cfg @@ -640,7 +640,7 @@ frontend-domain = slapos-repository = https://lab.nexedi.com/nexedi/slapos.git slapos-software = slapos-software-type = -slapos-reference = 1.0 +slapos-reference = runner-multi-sr auto-deploy = false auto-deploy-instance = true autorun = false diff --git a/stack/slapos.cfg b/stack/slapos.cfg index 753922ef5c..6ee96d17ac 100644 --- a/stack/slapos.cfg +++ b/stack/slapos.cfg @@ -77,6 +77,19 @@ versions = versions # Define networkcache with shacache.org networkcache-section = networkcache +[slapos.toolbox-repository] +recipe = slapos.recipe.build:gitclone +repository = https://lab.nexedi.com/Nicolas/slapos.toolbox.git +revision = +branch = runner-multi-sr +git-executable = ${git:location}/bin/git +develop = true + +[slapos.toolbox] +recipe = zc.recipe.egg:develop +egg = slapos.toolbox +setup = ${slapos.toolbox-repository:location} + [slapos.cookbook-repository] recipe = slapos.recipe.build:gitclone repository = https://lab.nexedi.com/nexedi/slapos.git @@ -107,7 +120,7 @@ eggs = ${lxml-python:egg} ${pycurl:egg} ${python-cryptography:egg} - slapos.toolbox + ${slapos.toolbox:egg} # Install a slapos command with networkcache enabled in ${buildout:bin-directory} [slapos-command] @@ -163,7 +176,7 @@ slapos.libnetworkcache = 0.20 slapos.rebootstrap = 4.4 slapos.recipe.build = 0.44 slapos.recipe.cmmi = 0.13 -slapos.toolbox = 0.109 +slapos.toolbox = stevedore = 1.21.0 subprocess32 = 3.5.3 unicodecsv = 0.14.1 -- 2.30.9 From 948f5f54ec74e8adafad15299a35e96931b58b88 Mon Sep 17 00:00:00 2001 From: Nicolas Wavrant Date: Wed, 22 Apr 2020 17:12:56 +0900 Subject: [PATCH 07/11] fixup! slaprunner: test support of multiple software release --- software/slaprunner/test/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/software/slaprunner/test/test.py b/software/slaprunner/test/test.py index 8ad532d4d5..3f07acd2a7 100644 --- a/software/slaprunner/test/test.py +++ b/software/slaprunner/test/test.py @@ -323,14 +323,14 @@ class TestCustomFrontend(SlaprunnerTestCase): 'https://www.erp5.com') class TestSlapProxyIntegration(SlaprunnerTestCase): - instance_max_retry = 5 - _save_instance_file_pattern_list = SlaprunnerTestCase._save_instance_file_pattern_list + ('*/srv/runner/*.json',) + _save_instance_file_pattern_list = SlaprunnerTestCase._save_instance_file_pattern_list + ('*/srv/runner/*.json', '*/srv/runner/*.log') @classmethod def getInstanceParameterDict(cls): return { 'autorun': True, 'auto-deploy': True, + 'auto-deploy-instance': True, 'slapos-repository': 'https://lab.nexedi.com/nexedi/slapos.git', 'slapos-reference': 'runner-multi-sr', } -- 2.30.9 From b7a2dc55798ad5dd9dd857faec130e21d6031151 Mon Sep 17 00:00:00 2001 From: Nicolas Wavrant Date: Wed, 22 Apr 2020 17:47:12 +0900 Subject: [PATCH 08/11] fixup! slaprunner: test support of multiple software release --- software/slaprunner/test/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/software/slaprunner/test/test.py b/software/slaprunner/test/test.py index 3f07acd2a7..e46f5f90dc 100644 --- a/software/slaprunner/test/test.py +++ b/software/slaprunner/test/test.py @@ -36,6 +36,7 @@ import subprocess import time import unittest +from datetime import datetime from lxml import etree from lxml.html import soupparser @@ -359,6 +360,7 @@ class TestSlapProxyIntegration(SlaprunnerTestCase): return resp def _buildAndRun(self): + start_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S') self.logger.debug('Running Build&Run') self._call('runSoftwareProfile') while True: @@ -376,7 +378,7 @@ class TestSlapProxyIntegration(SlaprunnerTestCase): continue if result['software']['success'] == 0: - if result['instance']['success'] == 0: + if result['instance']['success'] == 0 and start_date < result['instance']['last_build'].encode(): self.logger.debug('Instance is up!') # Done return -- 2.30.9 From 8aaae8aeb44b899ff9bd57933002766a801568cd Mon Sep 17 00:00:00 2001 From: Nicolas Wavrant Date: Fri, 24 Apr 2020 14:14:09 +0900 Subject: [PATCH 09/11] slaprunner: wait for gunicorn socket to start --- software/slaprunner/buildout.hash.cfg | 2 +- software/slaprunner/instance-runner.cfg | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/software/slaprunner/buildout.hash.cfg b/software/slaprunner/buildout.hash.cfg index b4a01e6757..5cc0fad568 100644 --- a/software/slaprunner/buildout.hash.cfg +++ b/software/slaprunner/buildout.hash.cfg @@ -18,7 +18,7 @@ md5sum = ec70348dd71b319590a5c5837f3d2e45 [template-runner] filename = instance-runner.cfg -md5sum = e7d59488e5f67011985211d775a9c499 +md5sum = bc2c65705d71325c0e7ba91e5c223335 [template-runner-import-script] filename = template/runner-import.sh.jinja2 diff --git a/software/slaprunner/instance-runner.cfg b/software/slaprunner/instance-runner.cfg index 7e0eae5c7c..92b93f1536 100644 --- a/software/slaprunner/instance-runner.cfg +++ b/software/slaprunner/instance-runner.cfg @@ -507,6 +507,8 @@ recipe = slapos.cookbook:wrapper command-line = $${directory:bin}/nginx-frontend wrapper-path = $${directory:services}/nginx-frontend hash-existing-files = $${buildout:directory}/software_release/buildout.cfg +wait-for-files = + $${gunicorn:socket} [certificate-authority-service] recipe = slapos.cookbook:wrapper -- 2.30.9 From 2bed73f7bac6a86f8944e135a117a5c2c75ec040 Mon Sep 17 00:00:00 2001 From: Nicolas Wavrant Date: Tue, 28 Apr 2020 07:41:11 +0200 Subject: [PATCH 10/11] monitor: improve httpd conf log accesses (the logrotate conf already exists for them), and remove the pattern "" which was used when this apache conf was extended by the software's apache extending the monitor stack --- stack/monitor/buildout.hash.cfg | 2 +- stack/monitor/templates/monitor-httpd.conf.in | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/stack/monitor/buildout.hash.cfg b/stack/monitor/buildout.hash.cfg index 2073d70b7c..31d2b8bc52 100644 --- a/stack/monitor/buildout.hash.cfg +++ b/stack/monitor/buildout.hash.cfg @@ -18,7 +18,7 @@ md5sum = 84bc2cf29e34b48c51116d93e2be7636 [monitor-httpd-conf] _update_hash_filename_ = templates/monitor-httpd.conf.in -md5sum = b5f42503799e7e770afce4097d3b75ae +md5sum = 0540fc5cc439a06079e9e724a5a55a70 [monitor-template-wrapper] _update_hash_filename_ = templates/wrapper.in diff --git a/stack/monitor/templates/monitor-httpd.conf.in b/stack/monitor/templates/monitor-httpd.conf.in index 183fbc6e6c..3d11235511 100644 --- a/stack/monitor/templates/monitor-httpd.conf.in +++ b/stack/monitor/templates/monitor-httpd.conf.in @@ -8,12 +8,13 @@ ThreadsPerChild 4 ServerName example.com ServerAdmin someone@email - Listen [{{ parameter_dict.get('listening-ip') }}]:{{ parameter_dict.get('port') }} -Define MonitorPort - DocumentRoot "{{ directory.get('private') }}" ErrorLog "{{ parameter_dict.get('error-log') }}" +LogFormat "%h %l %u %t \"%r\" %>s %b" common +CustomLog "{{ parameter_dict.get('access-log') }}" common + +LoadModule log_config_module modules/mod_log_config.so LoadModule unixd_module modules/mod_unixd.so LoadModule access_compat_module modules/mod_access_compat.so LoadModule authz_core_module modules/mod_authz_core.so -- 2.30.9 From fbe7ee1cb48398117ef1418cfd849bc62921bfc5 Mon Sep 17 00:00:00 2001 From: Nicolas Wavrant Date: Thu, 30 Apr 2020 10:22:30 +0900 Subject: [PATCH 11/11] slaprunner/test: improve a bit test SRs no need to extend stack/slapos.cfg anymore (command recipe is used), and set md5sum --- software/slaprunner/test/software_v1/software.cfg | 5 +---- software/slaprunner/test/software_v2/software.cfg | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/software/slaprunner/test/software_v1/software.cfg b/software/slaprunner/test/software_v1/software.cfg index 43de445675..f7fe9f2edb 100644 --- a/software/slaprunner/test/software_v1/software.cfg +++ b/software/slaprunner/test/software_v1/software.cfg @@ -3,9 +3,6 @@ parts = eggs instance -extends = - ../../../../stack/slapos.cfg - [eggs] recipe = zc.recipe.egg eggs = @@ -16,7 +13,7 @@ recipe = slapos.recipe.template filename = instance.cfg url = ${:_profile_base_location_}/${:filename} output = ${buildout:directory}/${:filename} -#md5sum = 869d5362b3daaa04db81129ce4f77c30 +md5sum = 4f1a8976944ac5a48b96705fc8874c31 [versions] slapos.recipe.template = 4.3 diff --git a/software/slaprunner/test/software_v2/software.cfg b/software/slaprunner/test/software_v2/software.cfg index 43de445675..dbdfd98f23 100644 --- a/software/slaprunner/test/software_v2/software.cfg +++ b/software/slaprunner/test/software_v2/software.cfg @@ -3,9 +3,6 @@ parts = eggs instance -extends = - ../../../../stack/slapos.cfg - [eggs] recipe = zc.recipe.egg eggs = @@ -16,7 +13,7 @@ recipe = slapos.recipe.template filename = instance.cfg url = ${:_profile_base_location_}/${:filename} output = ${buildout:directory}/${:filename} -#md5sum = 869d5362b3daaa04db81129ce4f77c30 +md5sum = 069716c5679c3fae6c96a80275382db7 [versions] slapos.recipe.template = 4.3 -- 2.30.9