diff --git a/software/agent/software.cfg b/software/agent/software.cfg
index 277419f0be54e9a73782bbd2cf4ebb50bc893c17..138757ae5f9887bbca858f7b97f699044aebb990 100644
--- a/software/agent/software.cfg
+++ b/software/agent/software.cfg
@@ -46,7 +46,6 @@ eggs =
ecdsa = 0.13
gitdb = 0.6.4
pycrypto = 2.6.1
-pycurl = 7.43.0
slapos.recipe.download = 1.0
slapos.recipe.template = 4.4
smmap = 0.9.0
diff --git a/software/kvm/software.cfg b/software/kvm/software.cfg
index c68d0bb5cb6e2ee39b533b7ddeb0624f95407371..a6252d59e80fb0426b2451ebd57920068ac69ab0 100644
--- a/software/kvm/software.cfg
+++ b/software/kvm/software.cfg
@@ -189,7 +189,6 @@ websockify = 0.9.0
collective.recipe.environment = 0.2.0
gitdb = 0.6.4
-pycurl = 7.43.0
slapos.recipe.template = 4.4
smmap = 0.9.0
numpy = 1.16.4
diff --git a/software/neoppod/software-common.cfg b/software/neoppod/software-common.cfg
index 22ec396422593e347bbf35f1e7f4a7cea1bce495..7ce4cc4a73405bb5b55915641795eb8c9036e5dc 100644
--- a/software/neoppod/software-common.cfg
+++ b/software/neoppod/software-common.cfg
@@ -150,7 +150,6 @@ msgpack-python = 0.5.6
mysqlclient = 1.3.12
persistent = 4.5.0
pycrypto = 2.6.1
-pycurl = 7.43.0
setproctitle = 1.1.10
slapos.recipe.template = 4.4
smmap2 = 2.0.1
diff --git a/software/slapos-sr-testing/software.cfg b/software/slapos-sr-testing/software.cfg
index 058952da2c8e56238083b2831893116910f2ff2a..e137bd278f42b357bcbb0d29003af3770de13c85 100644
--- a/software/slapos-sr-testing/software.cfg
+++ b/software/slapos-sr-testing/software.cfg
@@ -5,6 +5,7 @@ extends =
../../component/curl/buildout.cfg
../../component/git/buildout.cfg
../../component/pillow/buildout.cfg
+ ../../component/pycurl/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../component/python-mysqlclient/buildout.cfg
../../component/python-pynacl/buildout.cfg
@@ -156,7 +157,9 @@ setup = ${slapos.core-repository:location}
[eggs]
recipe = zc.recipe.egg
eggs =
+ ${pycurl:egg}
${lxml-python:egg}
+ ${slapos.toolbox:egg}
${slapos.core-setup:egg}
${pillow-python:egg}
erp5.util
@@ -205,7 +208,7 @@ forbid-download-cache = true
[slapos-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.git
-branch = master
+branch = runner-multi-sr
[template]
recipe = slapos.recipe.template:jinja2
@@ -221,31 +224,7 @@ context =
key curl_location curl:location
key tests :tests
tests =
- ${slapos.test.kvm-setup:setup}
${slapos.test.slaprunner-setup:setup}
- ${:extra}
-extra =
- ${slapos.cookbook-setup:setup}
- ${slapos.test.backupserver-setup:setup}
- ${slapos.test.caddy-frontend-setup:setup}
- ${slapos.test.erp5-setup:setup}
- ${slapos.test.htmlvalidatorserver-setup:setup}
- ${slapos.test.slapos-master-setup:setup}
- ${slapos.test.plantuml-setup:setup}
- ${slapos.test.powerdns-setup:setup}
- ${slapos.test.proftpd-setup:setup}
- ${slapos.test.re6stnet-setup:setup}
- ${slapos.test.seleniumserver-setup:setup}
- ${slapos.test.helloworld-setup:setup}
- ${slapos.test.jstestnode-setup:setup}
- ${slapos.test.jupyter-setup:setup}
- ${slapos.test.nextcloud-setup:setup}
- ${slapos.test.turnserver-setup:setup}
- ${slapos.test.theia-setup:setup}
- ${slapos.test.grafana-setup:setup}
- ${slapos.test.gitlab-setup:setup}
- ${slapos.test.cloudooo-setup:setup}
- ${slapos.test.dream-setup:setup}
[versions]
# slapos.core is used from the clone always
@@ -291,4 +270,6 @@ mock = 2.0.0
testfixtures = 6.11
funcsigs = 1.0.2
PyPDF2 = 1.26.0
-mysqlclient = 1.3.12
\ No newline at end of file
+mysqlclient = 1.3.12
+
+BeautifulSoup = 3.2.2
diff --git a/software/slapos-testing/software.cfg b/software/slapos-testing/software.cfg
index db6188218d725795d8733d9a0d0804a0124cfb2f..258c7da7a20e6372c4cf8dbe16aa21eea0b0cf35 100644
--- a/software/slapos-testing/software.cfg
+++ b/software/slapos-testing/software.cfg
@@ -210,7 +210,6 @@ manuel = 1.9.0
mock = 2.0.0
testfixtures = 6.11.0
pem = 18.1.0
-pycurl = 7.43.0.2
pyflakes = 2.0.0
smmap2 = 2.0.4
zope.testing = 4.6.2
diff --git a/software/slaprunner/buildout.hash.cfg b/software/slaprunner/buildout.hash.cfg
index f7977fbac3653a5690e2ac6da6241a854577fc00..5cc0fad5685bb1c9b6f0a1c95169534872b4a5c6 100644
--- a/software/slaprunner/buildout.hash.cfg
+++ b/software/slaprunner/buildout.hash.cfg
@@ -18,7 +18,7 @@ md5sum = ec70348dd71b319590a5c5837f3d2e45
[template-runner]
filename = instance-runner.cfg
-md5sum = f342daca2e87f8fa44fd7b6a2a1c22f8
+md5sum = bc2c65705d71325c0e7ba91e5c223335
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
diff --git a/software/slaprunner/instance-runner.cfg b/software/slaprunner/instance-runner.cfg
index a802cb90dca164254917c04f4f7be61097f290e5..92b93f15364d8c9bdc78f3e8bf1e6a7e4daa9823 100644
--- a/software/slaprunner/instance-runner.cfg
+++ b/software/slaprunner/instance-runner.cfg
@@ -507,6 +507,8 @@ recipe = slapos.cookbook:wrapper
command-line = $${directory:bin}/nginx-frontend
wrapper-path = $${directory:services}/nginx-frontend
hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
+wait-for-files =
+ $${gunicorn:socket}
[certificate-authority-service]
recipe = slapos.cookbook:wrapper
@@ -640,7 +642,7 @@ frontend-domain =
slapos-repository = https://lab.nexedi.com/nexedi/slapos.git
slapos-software =
slapos-software-type =
-slapos-reference = 1.0
+slapos-reference = runner-multi-sr
auto-deploy = false
auto-deploy-instance = true
autorun = false
diff --git a/software/slaprunner/software.cfg b/software/slaprunner/software.cfg
index 321302a8adec529ca2f42d34c630b3d8c07af480..4079c775ba862ff4989815ad372ed038580ffb0e 100644
--- a/software/slaprunner/software.cfg
+++ b/software/slaprunner/software.cfg
@@ -171,7 +171,6 @@ futures = 3.0.5
gitdb = 0.6.4
gunicorn = 19.10.0
prettytable = 0.7.2
-pycurl = 7.43.0
slapos.recipe.template = 4.4
collective.recipe.environment = 0.2.0
smmap = 0.9.0
diff --git a/software/slaprunner/test/setup.py b/software/slaprunner/test/setup.py
index dca232fe3e6eb3cb896efac5ab20d4a2aff7f137..b5ad7e06d56cc7c0258db0e57635c2a0987c299a 100644
--- a/software/slaprunner/test/setup.py
+++ b/software/slaprunner/test/setup.py
@@ -41,6 +41,7 @@ setup(name=name,
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
+ 'BeautifulSoup',
'slapos.core',
'slapos.cookbook',
'slapos.libnetworkcache',
diff --git a/software/slaprunner/test/software_v1/instance.cfg b/software/slaprunner/test/software_v1/instance.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..c7cea76c1450f99157a57f4c7e13c05eb065e580
--- /dev/null
+++ b/software/slaprunner/test/software_v1/instance.cfg
@@ -0,0 +1,18 @@
+[buildout]
+parts =
+ directory
+ instance
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+[directory]
+recipe = plone.recipe.command
+home = $${buildout:directory}
+etc = $${:home}/etc
+plugins = $${:etc}/plugin
+command = mkdir $${:etc} $${:plugins}
+
+[instance]
+recipe = plone.recipe.command
+command = echo -n 1 > $${buildout:directory}/version.txt
diff --git a/software/slaprunner/test/software_v1/software.cfg b/software/slaprunner/test/software_v1/software.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..f7fe9f2edb51c8ecd63f5ee8d5fd0b2746a34ba5
--- /dev/null
+++ b/software/slaprunner/test/software_v1/software.cfg
@@ -0,0 +1,19 @@
+[buildout]
+parts =
+ eggs
+ instance
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+ plone.recipe.command
+
+[instance]
+recipe = slapos.recipe.template
+filename = instance.cfg
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:filename}
+md5sum = 4f1a8976944ac5a48b96705fc8874c31
+
+[versions]
+slapos.recipe.template = 4.3
diff --git a/software/slaprunner/test/software_v2/instance.cfg b/software/slaprunner/test/software_v2/instance.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..186be33beda14aafdc57efc0e6cbd2f6a16e7719
--- /dev/null
+++ b/software/slaprunner/test/software_v2/instance.cfg
@@ -0,0 +1,18 @@
+[buildout]
+parts =
+ directory
+ instance
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+[directory]
+recipe = plone.recipe.command
+home = $${buildout:directory}
+etc = $${:home}/etc
+plugins = $${:etc}/plugin
+command = mkdir $${:etc} $${:plugins}
+
+[instance]
+recipe = plone.recipe.command
+command = echo -n 2 > $${buildout:directory}/version.txt
diff --git a/software/slaprunner/test/software_v2/software.cfg b/software/slaprunner/test/software_v2/software.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..dbdfd98f2371e98adc1c655f0054d738b6c7997e
--- /dev/null
+++ b/software/slaprunner/test/software_v2/software.cfg
@@ -0,0 +1,19 @@
+[buildout]
+parts =
+ eggs
+ instance
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+ plone.recipe.command
+
+[instance]
+recipe = slapos.recipe.template
+filename = instance.cfg
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:filename}
+md5sum = 069716c5679c3fae6c96a80275382db7
+
+[versions]
+slapos.recipe.template = 4.3
diff --git a/software/slaprunner/test/test.py b/software/slaprunner/test/test.py
index a2f718d9c9e6e3224e73de5a3810edf707cfca03..e46f5f90dc013a9d53cac10afb0a1f0effa0b04c 100644
--- a/software/slaprunner/test/test.py
+++ b/software/slaprunner/test/test.py
@@ -25,13 +25,20 @@
#
##############################################################################
-import os
-import unittest
-import paramiko
-import contextlib
import base64
+import contextlib
import hashlib
+import json
+import os
+import paramiko
+import requests
import subprocess
+import time
+import unittest
+
+from datetime import datetime
+from lxml import etree
+from lxml.html import soupparser
from six.moves.urllib.parse import urlparse
from six.moves.urllib.parse import quote
@@ -315,3 +322,170 @@ class TestCustomFrontend(SlaprunnerTestCase):
self.assertEqual(
parameter_dict['custom-frontend-url'],
'https://www.erp5.com')
+
+class TestSlapProxyIntegration(SlaprunnerTestCase):
+ _save_instance_file_pattern_list = SlaprunnerTestCase._save_instance_file_pattern_list + ('*/srv/runner/*.json', '*/srv/runner/*.log')
+
+ @classmethod
+ def getInstanceParameterDict(cls):
+ return {
+ 'autorun': True,
+ 'auto-deploy': True,
+ 'auto-deploy-instance': True,
+ 'slapos-repository': 'https://lab.nexedi.com/nexedi/slapos.git',
+ 'slapos-reference': 'runner-multi-sr',
+ }
+
+ def setUp(self):
+ self.parameter_dict = parameter_dict = self.computer_partition.getConnectionParameterDict()
+ self.base_url = parameter_dict['url']
+
+ def _call(self, path, method="GET", data=None, assert_result_code=False):
+ resp = {
+ "GET": requests.get,
+ "POST": requests.post,
+ }[method](
+ "%s/%s" % (self.base_url, path),
+ verify=False,
+ auth=(self.parameter_dict['init-user'], self.parameter_dict['init-password']),
+ data=data
+ )
+ if assert_result_code:
+ try:
+ self.assertEqual(json.loads(resp.text)['code'], 1)
+ except ValueError:
+ raise ValueError("call to %s failed with HTTP code %s" % (
+ path, result.status_code
+ ))
+ return resp
+
+ def _buildAndRun(self):
+ start_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+ self.logger.debug('Running Build&Run')
+ self._call('runSoftwareProfile')
+ while True:
+ time.sleep(10)
+ try:
+ response = self._call('slapgridResult')
+ result = json.loads(response.text)
+ except ValueError:
+ raise ValueError("couldnt decode JSON result of call (code: %s) : %s" % (response.status_code, response.text))
+ self.logger.debug('%s', result)
+
+ # If software or instance is currently under process, wait
+ if result['software']['state'] or result['instance']['state']:
+ self.logger.debug('Currently doing Build&Run')
+ continue
+
+ if result['software']['success'] == 0:
+ if result['instance']['success'] == 0 and start_date < result['instance']['last_build'].encode():
+ self.logger.debug('Instance is up!')
+ # Done
+ return
+ else:
+ self.logger.debug('Starting Run')
+ self._call('runInstanceProfile')
+ else:
+ self.logger.debug('Try to Build one more time')
+ self._call('runSoftwareProfile')
+
+ def test_upgrade_of_software_release(self):
+ """
+ Test that 2 Software Releases can be built at the same time in a webrunner,
+ and that the instance can be moved from one to another.
+ Also check border cases, including :
+ * ordering a new SR won't delete the existing ones
+ * only SR correctly built can be chosen for the instance
+ * the SR in-use can't be deleted
+ """
+ # 1: Request 2 Software Releases, and do build&run
+ result = self._call(
+ 'supplySoftwareRelease',
+ method="POST",
+ data={"path": "workspace/slapos/software/slaprunner/test/software_v1/"},
+ assert_result_code=True,
+ )
+ result = self._call(
+ 'supplySoftwareRelease',
+ method="POST",
+ data={"path": "workspace/slapos/software/slaprunner/test/software_v2/"},
+ assert_result_code=True,
+ )
+ self._buildAndRun()
+
+ # 2: Check that the Software Release of the instance is the 1st requested
+ result = self._call('inspectInstance')
+ root = soupparser.fromstring(result.text)
+ select_el = root.find(".//select[@name='software_release']")
+ self.assertEqual(
+ select_el.value_options,
+ [
+ 'workspace/slapos/software/slaprunner/test/software_v1/software.cfg',
+ 'workspace/slapos/software/slaprunner/test/software_v2/software.cfg',
+ ]
+ )
+ result = self._call(
+ 'getFileContent',
+ method="POST",
+ data={"file": "instance_root/slappart0/version.txt"},
+ assert_result_code=True,
+ )
+ self.assertEqual(json.loads(result.text)['result'], u'1')
+
+ # 3: Check that that only the unused Software Release can be deleted
+ delete_form_el, = root.findall(".//form[@action='/destroySoftwareRelease']")
+ self.assertEqual(
+ delete_form_el.fields['uri'],
+ 'workspace/slapos/software/slaprunner/test/software_v2/software.cfg',
+ )
+
+ # 4: Request the instance with the 2nd Software Release, and check modification is registered
+ result = self._call(
+ 'saveParameterXml',
+ method="POST",
+ data={
+ "software_release": "workspace/slapos/software/slaprunner/test/software_v2/software.cfg",
+ "software_type": "default",
+ "parameter": '\n\n',
+ },
+ assert_result_code=True,
+ )
+ result = self._call('inspectInstance')
+ root = soupparser.fromstring(result.text)
+ select_el = root.find(".//select[@name='software_release']")
+ self.assertEqual(select_el.value_options[0], "workspace/slapos/software/slaprunner/test/software_v2/software.cfg")
+
+ # 5: Do build&run, and check that the instance was updated with the profile of the 2nd Software Release
+ self._buildAndRun()
+ result = self._call(
+ 'getFileContent',
+ method="POST",
+ data={"file": "instance_root/slappart0/version.txt"},
+ assert_result_code=True,
+ )
+ self.assertEqual(json.loads(result.text)['result'], u'2')
+
+ # 6: Check that that only the first Software Release can be deleted, and delete it
+ delete_form_el, = root.findall(".//form[@action='/destroySoftwareRelease']")
+ self.assertEqual(
+ delete_form_el.fields['uri'],
+ 'workspace/slapos/software/slaprunner/test/software_v1/software.cfg',
+ )
+ self._call('destroySoftwareRelease', method="POST", data={
+ "uri": "workspace/slapos/software/slaprunner/test/software_v1/software.cfg"
+ })
+
+ # 7: Check that the first Software Release can't be chosen anymore
+ # for the instance, neither is shown as deletable
+ result = self._call('inspectInstance')
+ root = soupparser.fromstring(result.text)
+ select_el = root.find(".//select[@name='software_release']")
+ self.assertEqual(
+ select_el.value_options,
+ [
+ 'workspace/slapos/software/slaprunner/test/software_v2/software.cfg',
+ ]
+ )
+ self.assertTrue(
+ len(root.findall(".//form[@action='/destroySoftwareRelease']")) == 0
+ )
diff --git a/stack/erp5/buildout.cfg b/stack/erp5/buildout.cfg
index b3e35f6034a188164362bb3e830e89fe108c4278..86b37151e76a14a52906db949ec1075689e3f07c 100644
--- a/stack/erp5/buildout.cfg
+++ b/stack/erp5/buildout.cfg
@@ -660,7 +660,6 @@ pprofile = 2.0.4
pyasn1-modules = 0.0.8
pycountry = 17.1.8
pycrypto = 2.6.1
-pycurl = 7.43.0
pyflakes = 1.5.0
python-memcached = 1.58
pytracemalloc = 1.2
diff --git a/stack/monitor/buildout.hash.cfg b/stack/monitor/buildout.hash.cfg
index 2073d70b7c94d63263b27fdab14022532ea8a9d3..31d2b8bc52a6c1f8a45af05d7ebc2cb1a9abb025 100644
--- a/stack/monitor/buildout.hash.cfg
+++ b/stack/monitor/buildout.hash.cfg
@@ -18,7 +18,7 @@ md5sum = 84bc2cf29e34b48c51116d93e2be7636
[monitor-httpd-conf]
_update_hash_filename_ = templates/monitor-httpd.conf.in
-md5sum = b5f42503799e7e770afce4097d3b75ae
+md5sum = 0540fc5cc439a06079e9e724a5a55a70
[monitor-template-wrapper]
_update_hash_filename_ = templates/wrapper.in
diff --git a/stack/monitor/templates/monitor-httpd.conf.in b/stack/monitor/templates/monitor-httpd.conf.in
index 183fbc6e6ce0d82ae8b802fca4033cf0a1f7d587..3d11235511c4c019ac34d20a2cf764753b2f09f2 100644
--- a/stack/monitor/templates/monitor-httpd.conf.in
+++ b/stack/monitor/templates/monitor-httpd.conf.in
@@ -8,12 +8,13 @@ ThreadsPerChild 4
ServerName example.com
ServerAdmin someone@email
-
Listen [{{ parameter_dict.get('listening-ip') }}]:{{ parameter_dict.get('port') }}
-Define MonitorPort
-
DocumentRoot "{{ directory.get('private') }}"
ErrorLog "{{ parameter_dict.get('error-log') }}"
+LogFormat "%h %l %u %t \"%r\" %>s %b" common
+CustomLog "{{ parameter_dict.get('access-log') }}" common
+
+LoadModule log_config_module modules/mod_log_config.so
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule authz_core_module modules/mod_authz_core.so
diff --git a/stack/slapos.cfg b/stack/slapos.cfg
index 753922ef5c65cb5f228969cf60a9a89180f6edcd..6ee96d17ac181bac94f9553c396fbd2cd9481d9a 100644
--- a/stack/slapos.cfg
+++ b/stack/slapos.cfg
@@ -77,6 +77,19 @@ versions = versions
# Define networkcache with shacache.org
networkcache-section = networkcache
+[slapos.toolbox-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/Nicolas/slapos.toolbox.git
+revision =
+branch = runner-multi-sr
+git-executable = ${git:location}/bin/git
+develop = true
+
+[slapos.toolbox]
+recipe = zc.recipe.egg:develop
+egg = slapos.toolbox
+setup = ${slapos.toolbox-repository:location}
+
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.git
@@ -107,7 +120,7 @@ eggs =
${lxml-python:egg}
${pycurl:egg}
${python-cryptography:egg}
- slapos.toolbox
+ ${slapos.toolbox:egg}
# Install a slapos command with networkcache enabled in ${buildout:bin-directory}
[slapos-command]
@@ -163,7 +176,7 @@ slapos.libnetworkcache = 0.20
slapos.rebootstrap = 4.4
slapos.recipe.build = 0.44
slapos.recipe.cmmi = 0.13
-slapos.toolbox = 0.109
+slapos.toolbox =
stevedore = 1.21.0
subprocess32 = 3.5.3
unicodecsv = 0.14.1