Update Release Candidate

component/caddy/buildout.cfg

-[buildout]
-extends =
-  ../../component/golang/buildout.cfg
-
-parts =
-  gowork
-  caddy
-
-[caddy-get]
-<= go-git-package
-go.importpath = github.com/caddyserver/caddy
-repository    = https://lab.nexedi.com/nexedi/caddy.git
-revision      = nxd-v1.0.3-1-03fba31bf
-
-[gowork]
-golang  = ${golang1.17:location}
-install =
-  ${caddy-get:location}:./...
-
-[caddy]
-recipe = plone.recipe.command
-command = exit 0
-update-command = ${:command}
-output = ${gowork:bin}/caddy
-location = ${:output}

component/haproxy/buildout.cfg

@@ -13,8 +13,8 @@ parts = haproxy
 [haproxy]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.haproxy.org/download/2.6/src/haproxy-2.6.7.tar.gz
-md5sum = cfa36413f2bc5187ab34ffcdf71914d4
+url = https://www.haproxy.org/download/2.6/src/haproxy-2.6.8.tar.gz
+md5sum = 0b506378f1ade633d5b921f5806a5e82
 configure-command = true
 # for Linux kernel 2.6.28 and above, we use "linux-glibc" as the TARGET,
 # otherwise use "generic".

software/erp5testnode/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = 5bd72da73a8b84f70f568686924d7d6c
 
 [template-default]
 filename = instance-default.cfg.jinja.in
-md5sum = 2aafd7708aac85847da3a8679b84fe34
+md5sum = 96fb2221c8a91fb44dc5b931c29e0af9

software/erp5testnode/instance-default.cfg.jinja.in

@@ -107,23 +107,43 @@ inline =
     --unixdomain-only=$${:socket}:$(id -u):$(id -g):0600 \
     --service "/:$(id -u):$(id -g):HOME:$${shell-environment:shell} -l"
 
+[shellinabox-frontend-certificate]
+recipe = plone.recipe.command
+command =
+  if [ ! -e $${:cert-file} ]
+  then
+    ${openssl-output:openssl} req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \
+      -newkey rsa:2048 -keyout $${:cert-file} \
+      -out $${:cert-file}
+  fi
+update-command = $${:command}
+cert-file = $${directory:var}/$${:_buildout_section_name_}.pem
+common-name = $${:ipv6}
+location =
+  $${:cert-file}
+ipv6 = {{ partition_ipv6 }}
+
 [shellinabox-frontend-config]
-recipe = slapos.recipe.template
+recipe = slapos.recipe.template:jinja2
+url = ${stack-haproxy-default-backend-config:target}
 output = $${directory:etc}/$${:_buildout_section_name_}
-inline =
-  https://$${:hostname}:$${:port} {
-    bind $${:ipv6}
-    tls self_signed
-    gzip
-    log stdout
-    errors stderr
-    proxy / unix:$${shellinabox:socket}
-    basicauth $${:username} $${:passwd} {
-      realm "Test Node $${testnode:test-node-title}"
-      /
-    }
-  }
-ipv6 = {{ partition_ipv6 }}
+context =
+  key pidfile :pidfile
+  key content :content
+pidfile = $${:pidfile}
+content =
+  userlist auth
+    user $${:username} insecure-password $${:passwd}
+
+  listen app
+    log global
+    acl auth_ok http_auth(auth)
+    http-request auth realm "Test Node $${testnode:test-node-title}" unless auth_ok
+    bind $${:ipv6}:$${:port} ssl crt $${shellinabox-frontend-certificate:cert-file} alpn h2,http/1.1
+    server app unix@$${shellinabox:socket}
+
+ipv6 = $${shellinabox-frontend-certificate:ipv6}
 hostname = [$${:ipv6}]
 port = 8080
 username = testnode
@@ -131,24 +151,24 @@ passwd = $${pwgen:passwd}
 cert-file = $${directory:shellinabox}/public.crt
 key-file = $${directory:shellinabox}/private.key
 backend-url = https://$${:username}:$${:passwd}@$${:hostname}:$${:port}
-
+pidfile = $${basedirectory:run}/shellinabox-haproxy.pid
 
 [shellinabox-frontend]
 recipe = slapos.cookbook:wrapper
 wrapper-path = $${rootdirectory:bin}/$${:_buildout_section_name_}
 command-line =
-  ${caddy:output} -conf $${shellinabox-frontend-config:output} -pidfile $${:pidfile}
+  ${haproxy:location}/sbin/haproxy -f $${shellinabox-frontend-config:output}
 url =  $${shellinabox-frontend-config:backend-url}
 hostname = $${shellinabox-frontend-config:ipv6}
 port = $${shellinabox-frontend-config:port}
-pidfile = $${basedirectory:run}/$${:_buildout_section_name_}.pid
+pidfile = $${shellinabox-frontend-config:pidfile}
 
 [shellinabox-frontend-reload]
 recipe = slapos.cookbook:wrapper
 wrapper-path = $${basedirectory:services}/$${:_buildout_section_name_}
 command-line =
   ${bash:location}/bin/bash -c
-  "kill -s USR1 $$(${coreutils:location}/bin/cat $${shellinabox-frontend:pidfile}) \
+  "kill -s USR2 $$(${coreutils:location}/bin/cat $${shellinabox-frontend:pidfile}) \
     && ${coreutils:location}/bin/sleep infinity"
 hash-files =
   $${shellinabox-frontend-config:output}

software/erp5testnode/software.cfg

@@ -2,11 +2,11 @@
 extends =
   buildout.hash.cfg
   ../../stack/slapos.cfg
+  ../../stack/haproxy/default-backend.cfg
   ../../component/git/buildout.cfg
   ../../component/lxml-python/buildout.cfg
   ../../component/zip/buildout.cfg
   ../../component/bash/buildout.cfg
-  ../../component/caddy/buildout.cfg
   ../../component/coreutils/buildout.cfg
   ../../component/shellinabox/buildout.cfg
   ../../component/pwgen/buildout.cfg

software/erp5testnode/testsuite/slaprunner.cfg

-[buildout]
-extends = 
-   ../../slaprunner/development.cfg
-   testsuite.cfg
-
-parts += template-erp5testnode
-
-[template-resilient-test]
-filename = instance-resilient-test.cfg.jinja2 
-md5sum = e4b04aa6fd3413bc6ae38823abdcc8fa
-
-# We have to use an extra level of indentation here because this is substituted
-# during software buildout to generate instance buildout, but the
-# slapos.recipe.template recipe doing the substitution does string replacements
-# without knowledge of the buildout syntax, so we want the second line to be
-# indented in the final generated instance buildout.
-extra-context =
-    raw slapos_repository_url ${slapos.cookbook-repository:repository}
-        raw slapos_repository_branch ${slapos.cookbook-repository:branch}
-
-[exporter-default-configuration]
-# Define shorter interaction to speed up tests
-backup_wait_time = 1

software/helloworld/software.cfg

@@ -29,9 +29,6 @@ parts =
   helloweb-go
 
 
-[gowork]
-golang = ${golang1.17:location}
-
 # Macro for jinja templates. The filename is set in buildout.hash.cfg
 # in the section using this template
 [jinja-template]

software/kvm/README.rst

@@ -194,8 +194,12 @@ removing files:
 
 from the partition (typically ``/srv/slapgrid/slappartNN/`` directory).
 
+They will reappear automatically after some time, but as the old
+``external-disk-amount`` approach is now disabled, they won't be updated.
+
 The failure observed to confirm the situation can be found in
 ``.slappartNN_kvm-HASH.log`` with presence of message like::
 
-  qemu-system-x86_64: -drive file=/<instance_storage_home>/dataX/slappartNN/kvm_virtual_disk.qcow2,if=virtio,cache=writeback: Failed to get "write" lock
-  Is another process using the image [/<instance_storage_home>/dataX/slappartNN/kvm_virtual_disk.qcow2]?
+  ValueError: external-disk problems: conflicts with external-disk-number = XX, conflicts with already configured disks amount XX in /srv/slapgrid/slappartNN/etc/.data-disk-amount
+
+Where ``XX`` is the previously used ``external-disk-number`` and ``NN`` is the partition.

software/kvm/buildout.hash.cfg

@@ -59,7 +59,7 @@ md5sum = 6328f99728284847b8dd1146aadeae1b
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = 4ce3fc8072e1e010ee99651cb01d3b3d
+md5sum = 1663af08ea7afa8d3fa091bf0c2ea1ca
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/development.cfg

-# Development profile of slaprunner.
+# Development profile of kvm.
 # Exactly the same as software.cfg, but fetch the slapos.cookbook and
 # slapos.toolbox from git repository instead of fetching stable version,
 # allowing to play with bleeding edge environment.

software/kvm/template/template-kvm-run.in

@@ -48,13 +48,25 @@ external_disk_number = {{ parameter_dict.get("external-disk-number") }}
 external_disk_size = {{ parameter_dict.get("external-disk-size") }}
 external_disk_format = {{ repr(parameter_dict["external-disk-format"]) }}
 external_disk = {{ parameter_dict['external-disk'] }}
-if int(external_disk_number) > 0 and len(external_disk) > 0:
-  raise ValueError("external-disk-number and external-disk are mutually exclusive.")
+etc_directory = '{{ parameter_dict.get("etc-directory") }}'.strip()
+last_disk_num_f = os.path.join(etc_directory, '.data-disk-amount')
+if os.path.exists(last_disk_num_f):
+  with open(last_disk_num_f, 'r') as lf:
+    last_amount = int(lf.readline())
+else:
+  last_amount = 0
+if len(external_disk) > 0:
+  conflict_list = []
+  if int(external_disk_number) > 0:
+    conflict_list.append('conflicts with external-disk-number = %s' % (external_disk_number,))
+  if last_amount > 0:
+    conflict_list.append('conflicts with already configured disks amount %s in %s' % (last_amount, last_disk_num_f))
+  if len(conflict_list) > 0:
+    raise ValueError('external-disk problems: ' + ', '.join(conflict_list))
 instance_root = '{{ parameter_dict['instance-root'] }}'
 disk_storage_dict = {}
 disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
 map_storage_list = []
-etc_directory = '{{ parameter_dict.get("etc-directory") }}'.strip()
 httpd_port = {{ parameter_dict.get("httpd-port") }}
 netcat_bin = '{{ parameter_dict.get("netcat-binary") }}'.strip()
 cluster_doc_host = '{{ parameter_dict.get("cluster-doc-host") }}'

software/kvm/test/test.py

@@ -1875,38 +1875,6 @@ class TestParameterCluster(TestParameterDefault):
 
 
 class ExternalDiskMixin(KvmMixin):
-  def getRunningDriveList(self, kvm_instance_partition):
-    _match_drive = re.compile('file.*if=virtio.*').match
-    with self.slap.instance_supervisor_rpc as instance_supervisor:
-      kvm_pid = next(q for q in instance_supervisor.getAllProcessInfo()
-                     if 'kvm-' in q['name'])['pid']
-    drive_list = []
-    for entry in psutil.Process(kvm_pid).cmdline():
-      m = _match_drive(entry)
-      if m:
-        path = m.group(0)
-        drive_list.append(
-          path.replace(kvm_instance_partition, '${partition}')
-        )
-    return drive_list
-
-
-@skipUnlessKvm
-class TestExternalDisk(InstanceTestCase, ExternalDiskMixin):
-  __partition_reference__ = 'ed'
-  kvm_instance_partition_reference = 'ed0'
-
-  @classmethod
-  def getInstanceSoftwareType(cls):
-    return 'default'
-
-  @classmethod
-  def getInstanceParameterDict(cls):
-    return {
-      'external-disk-number': 2,
-      'external-disk-size': 1
-    }
-
   @classmethod
   def _prepareExternalStorageList(cls):
     external_storage_path = os.path.join(cls.working_directory, 'STORAGE')
@@ -1966,6 +1934,38 @@ class TestExternalDisk(InstanceTestCase, ExternalDiskMixin):
     with open(cls.slap._slapos_config, 'w') as fh:
       fh.write(''.join(slapos_config))
 
+  def getRunningDriveList(self, kvm_instance_partition):
+    _match_drive = re.compile('file.*if=virtio.*').match
+    with self.slap.instance_supervisor_rpc as instance_supervisor:
+      kvm_pid = next(q for q in instance_supervisor.getAllProcessInfo()
+                     if 'kvm-' in q['name'])['pid']
+    drive_list = []
+    for entry in psutil.Process(kvm_pid).cmdline():
+      m = _match_drive(entry)
+      if m:
+        path = m.group(0)
+        drive_list.append(
+          path.replace(kvm_instance_partition, '${partition}')
+        )
+    return drive_list
+
+
+@skipUnlessKvm
+class TestExternalDisk(InstanceTestCase, ExternalDiskMixin):
+  __partition_reference__ = 'ed'
+  kvm_instance_partition_reference = 'ed0'
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'default'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'external-disk-number': 2,
+      'external-disk-size': 1
+    }
+
   @classmethod
   def _setUpClass(cls):
     super()._setUpClass()
@@ -2015,19 +2015,29 @@ class TestExternalDiskJson(
   pass
 
 
-@skipUnlessKvm
-class TestExternalDiskModern(InstanceTestCase, ExternalDiskMixin):
+class ExternalDiskModernMixin(object):
   __partition_reference__ = 'edm'
   kvm_instance_partition_reference = 'edm0'
 
-  @classmethod
-  def getInstanceSoftwareType(cls):
-    return 'default'
-
   @classmethod
   def setUpClass(cls):
     super().setUpClass()
 
+  @classmethod
+  def _setUpClass(cls):
+    super()._setUpClass()
+    cls.working_directory = tempfile.mkdtemp()
+    # setup the external_storage_list, to mimic part of slapformat
+    cls._prepareExternalStorageList()
+    # re-run the instance, as information has been updated
+    cls.waitForInstance()
+
+  @classmethod
+  def tearDownClass(cls):
+    cls._dropExternalStorageList()
+    super().tearDownClass()
+    shutil.rmtree(cls.working_directory)
+
   def getExternalDiskInstanceParameterDict(
     self, first, second, third, update_dict=None):
     parameter_dict = {
@@ -2052,39 +2062,50 @@ class TestExternalDiskModern(InstanceTestCase, ExternalDiskMixin):
       parameter_dict.update(update_dict)
     return parameter_dict
 
-  def test(self):
+  def prepareEnv(self):
     # Disks can't be created in /tmp, as it's specially mounted on testnodes
     # and then KVM can't use them:
     # -drive file=/tmp/tmpX/third_disk,if=virtio,cache=none: Could not open
     # '/tmp/tmpX/third_disk': filesystem does not support O_DIRECT
     self.working_directory = tempfile.mkdtemp(dir=self.slap.instance_directory)
     self.addCleanup(shutil.rmtree, self.working_directory)
-    kvm_instance_partition = os.path.join(
+    self.kvm_instance_partition = os.path.join(
       self.slap.instance_directory, self.kvm_instance_partition_reference)
     # find qemu_img from the tested SR via it's partition parameter, as
     # otherwise qemu-kvm would be dependency of test suite
     with open(
       os.path.join(self.computer_partition_root_path, 'buildout.cfg')) as fh:
-      qemu_img = [
+      self.qemu_img = [
         q for q in fh.readlines()
         if 'raw qemu_img_executable_location' in q][0].split()[-1]
-
     self.first_disk = os.path.join(self.working_directory, 'first_disk')
     subprocess.check_call([
-      qemu_img, "create", "-f", "qcow", self.first_disk, "1M"])
-    second_disk = 'second_disk'
-    self.second_disk = os.path.join(kvm_instance_partition, second_disk)
+      self.qemu_img, "create", "-f", "qcow", self.first_disk, "1M"])
+    self.second_disk_name = 'second_disk'
+    self.second_disk = os.path.join(
+      self.kvm_instance_partition, self.second_disk_name)
     subprocess.check_call([
-      qemu_img, "create", "-f", "qcow2", os.path.join(
-        kvm_instance_partition, self.second_disk), "1M"])
+      self.qemu_img, "create", "-f", "qcow2", os.path.join(
+        self.kvm_instance_partition, self.second_disk), "1M"])
     self.third_disk = os.path.join(self.working_directory, 'third_disk')
     subprocess.check_call([
-      qemu_img, "create", "-f", "qcow2", self.third_disk, "1M"])
+      self.qemu_img, "create", "-f", "qcow2", self.third_disk, "1M"])
     self.rerequestInstance({'_': json.dumps(
         self.getExternalDiskInstanceParameterDict(
-          self.first_disk, second_disk, self.third_disk))})
+          self.first_disk, self.second_disk_name, self.third_disk))})
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'default'
+
+
+@skipUnlessKvm
+class TestExternalDiskModern(
+  ExternalDiskModernMixin, InstanceTestCase, ExternalDiskMixin):
+  def test(self):
+    self.prepareEnv()
     self.waitForInstance()
-    drive_list = self.getRunningDriveList(kvm_instance_partition)
+    drive_list = self.getRunningDriveList(self.kvm_instance_partition)
     self.assertEqual(
       drive_list,
       [
@@ -2097,16 +2118,39 @@ class TestExternalDiskModern(InstanceTestCase, ExternalDiskMixin):
           self.working_directory)
       ]
     )
-    update_dict = {
+
+
+@skipUnlessKvm
+class TestExternalDiskModernConflictAssurance(
+  ExternalDiskModernMixin, InstanceTestCase, ExternalDiskMixin):
+  def test(self):
+    self.prepareEnv()
+    # Create conflicting configuration
+    parameter_dict = {
       "external-disk-number": 1,
-      "external-disk-size": 100,
+      "external-disk-size": 10,
       "external-disk-format": "qcow2",
     }
-    parameter_dict = self.getExternalDiskInstanceParameterDict(
-      self.first_disk, second_disk, self.third_disk, update_dict)
-    # assert mutual exclusivity
+    self.rerequestInstance({'_': json.dumps(parameter_dict)})
+    self.waitForInstance()
+    data_disk_ids = os.path.join(
+      self.kvm_instance_partition, 'etc', '.data-disk-ids')
+    data_disk_amount = os.path.join(
+      self.kvm_instance_partition, 'etc', '.data-disk-amount')
+    self.assertTrue(os.path.exists(data_disk_ids))
+    self.assertTrue(os.path.exists(data_disk_amount))
+    with open(data_disk_amount) as fh:
+      self.assertEqual(1, int(fh.read()))
+    parameter_dict.update(self.getExternalDiskInstanceParameterDict(
+      self.first_disk, self.second_disk_name, self.third_disk))
+    parameter_dict["external-disk-number"] = 0
+    # assert mutual exclusivity of old and modern
     self.rerequestInstance({'_': json.dumps(parameter_dict)})
     self.raising_waitForInstance(3)
+    # Fix the situation
+    with open(data_disk_amount, 'w') as fh:
+      fh.write("0")
+    self.waitForInstance()
 
 
 @skipUnlessKvm

software/seleniumserver/buildout.hash.cfg

@@ -15,8 +15,8 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = 9b41238f67ea8e12f8ea9590fd552b33
+md5sum = 10e19df182c692b71ea552da183a0bcf
 
 [template-selenium]
 filename = instance-selenium.cfg.in
-md5sum = 9b4742e8a249aef38c7c8c12d74b0605
+md5sum = 2d84d5fb9306b88ef71e101acb46f684

software/seleniumserver/instance-selenium.cfg.in

@@ -145,44 +145,74 @@ username = selenium
 bytes = 12
 
 
-[selenium-server-frontend-config]
-recipe = slapos.recipe.template
-output = $${directory:etc}/$${:_buildout_section_name_}
-inline =
-  https://[$${:ip}]:$${:port} {
-    bind $${:ip}
-    tls self_signed # TODO
-    proxy / $${selenium-server-hub-instance:base-url} {
-      transparent
-    }
-    basicauth $${selenium-server-admin-password:username} $${selenium-server-admin-password:passwd} {
-      realm "Grid Admin"
-      $${:path-admin}
-    }
-    basicauth $${selenium-server-selenium-password:username} $${selenium-server-selenium-password:passwd} {
-      realm "Selenium Server"
-      $${:path-hub}
-    }
-  }
+[selenium-server-frontnend-certificate]
+recipe = plone.recipe.command
+command =
+  if [ ! -e $${:cert-file} ]
+  then
+    ${openssl-output:openssl} req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \
+      -newkey rsa:2048 -keyout $${:cert-file} \
+      -out $${:cert-file}
+  fi
+update-command = $${:command}
+cert-file = $${directory:var}/$${:_buildout_section_name_}.pem
+common-name = $${selenium-server-frontend-configuration:ip}
+location =
+  $${:cert-file}
+
+[selenium-server-frontend-configuration]
 ip = $${slap-configuration:ipv6-random}
 hostname = [$${:ip}]
 port = 9443
 path-admin = /grid/console
 path-hub = /wd/hub
+pidfile = $${directory:run}/haproxy.pid
 
+[selenium-server-frontend-config]
+recipe = slapos.recipe.template:jinja2
+url = ${stack-haproxy-default-backend-config:target}
+output = $${directory:etc}/$${:_buildout_section_name_}
+context =
+  key pidfile selenium-server-frontend-configuration:pidfile
+  key content :content
+
+content =
+  frontend app
+    log global
+    bind $${selenium-server-frontend-configuration:ip}:$${selenium-server-frontend-configuration:port} ssl crt $${selenium-server-frontnend-certificate:cert-file} alpn h2,http/1.1
+    use_backend hub if { path_beg $${selenium-server-frontend-configuration:path-hub} }
+    use_backend admin if { path_beg $${selenium-server-frontend-configuration:path-admin} }
+
+  userlist hub
+    user $${selenium-server-selenium-password:username} insecure-password $${selenium-server-selenium-password:passwd}
+  backend hub
+    acl auth_ok http_auth(hub)
+    http-request auth realm "Selenium Server" unless auth_ok
+    server hub $${selenium-server-hub-instance:hostname}:$${selenium-server-hub-instance:port}
+
+  userlist admin
+    user $${selenium-server-admin-password:username} insecure-password $${selenium-server-admin-password:passwd}
+  backend admin
+    acl auth_ok http_auth(admin)
+    http-request auth realm "Grid Admin" unless auth_ok
+    server admin $${selenium-server-hub-instance:hostname}:$${selenium-server-hub-instance:port}
 
 [selenium-server-frontend-instance]
 recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:services}/$${:_buildout_section_name_}
 command-line =
-  ${caddy:output} -conf $${selenium-server-frontend-config:output}
+  ${haproxy:location}/sbin/haproxy -f $${selenium-server-frontend-config:output}
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
+hash-files =
+  $${selenium-server-frontend-config:output}
 
-ip = $${selenium-server-frontend-config:ip}
-hostname = $${selenium-server-frontend-config:hostname}
-port = $${selenium-server-frontend-config:port}
+ip = $${selenium-server-frontend-configuration:ip}
+hostname = $${selenium-server-frontend-configuration:hostname}
+port = $${selenium-server-frontend-configuration:port}
 
-admin-url = https://$${selenium-server-admin-password:username}:$${selenium-server-admin-password:passwd}@$${:hostname}:$${:port}$${selenium-server-frontend-config:path-admin}
-url = https://$${selenium-server-selenium-password:username}:$${selenium-server-selenium-password:passwd}@$${:hostname}:$${:port}$${selenium-server-frontend-config:path-hub}
+admin-url = https://$${selenium-server-admin-password:username}:$${selenium-server-admin-password:passwd}@$${:hostname}:$${:port}$${selenium-server-frontend-configuration:path-admin}
+url = https://$${selenium-server-selenium-password:username}:$${selenium-server-selenium-password:passwd}@$${:hostname}:$${:port}$${selenium-server-frontend-configuration:path-hub}
 
 
 [userinfo]

software/seleniumserver/instance.cfg.in

@@ -24,3 +24,5 @@ recipe = slapos.cookbook:slapconfiguration
 computer = $${slap-connection:computer-id}
 partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}

software/seleniumserver/software.cfg

@@ -10,11 +10,11 @@ extends =
    ../../component/chromedriver/buildout.cfg
    ../../component/coreutils/buildout.cfg
    ../../component/java/buildout.cfg
-   ../../component/caddy/buildout.cfg
    ../../component/openssh/buildout.cfg
    ../../component/fonts/buildout.cfg
    ../../component/fontconfig/buildout.cfg
    ../../stack/slapos.cfg
+   ../../stack/haproxy/default-backend.cfg
    ./buildout.hash.cfg
    ../../stack/monitor/buildout.cfg
 

software/slapos-sr-testing/README.md

@@ -12,7 +12,7 @@ The results of this test suite running on Nexedi ERP5 are published as `SlapOS.S
 ## Running test locally
 
 Here's an example session of how a developer could use this software release in
-slaprunner to develop a slapos profile, in the example `helloworld`, make
+Theia to develop a slapos profile, in the example `helloworld`, make
 changes to the code, run tests and publish changes.
 
 ```bash

software/slapos-sr-testing/software.cfg

@@ -117,11 +117,6 @@ setup = ${slapos-repository:location}/software/restic-rest-server/test/
 egg = slapos.test.seleniumserver
 setup = ${slapos-repository:location}/software/seleniumserver/test/
 
-[slapos.test.slaprunner-setup]
-<= setup-develop-egg
-egg = slapos.test.slaprunner
-setup = ${slapos-repository:location}/software/slaprunner/test/
-
 [slapos.test.metabase-setup]
 <= setup-develop-egg
 egg = slapos.test.metabase
@@ -337,7 +332,6 @@ eggs +=
   ${slapos.test.restic_rest_server-setup:egg}
   ${slapos.test.seleniumserver-setup:egg}
   ${slapos.test.slapos-master-setup:egg}
-  ${slapos.test.slaprunner-setup:egg}
   ${slapos.test.theia-setup:egg}
   ${slapos.test.turnserver-setup:egg}
   ${slapos.test.upgrade_erp5-setup:egg}
@@ -428,7 +422,6 @@ tests =
   restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
   seleniumserver ${slapos.test.seleniumserver-setup:setup}
   slapos-master ${slapos.test.slapos-master-setup:setup}
-  slaprunner ${slapos.test.slaprunner-setup:setup}
   theia ${slapos.test.theia-setup:setup}
   turnserver ${slapos.test.turnserver-setup:setup}
   upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}

software/slapos-testing/README.md

@@ -10,7 +10,7 @@ The results of this test suite running on Nexedi ERP5 are published as
 
 
 Here's an example session of how a developer could use this software release in
-slaprunner to develop a slapos egg, in the example `slapos.core`, to make
+Theia to develop a slapos egg, in the example `slapos.core`, to make
 changes to the code, run tests and publish changes.
 
 ```bash

software/slaprunner/README.rst

-slaprunner
-==========
-
-Introduction
-------------
-
-This software release is used to deploy Slaprunner instances.
-Slaprunner is an all-in-one IDE used to develop and test profiles and recipes for SlapOS.
-
-This software is only running in Python2 and has been deprecated in favor of theia.
-See https://lab.nexedi.com/nexedi/slapos/tree/master/software/theia
-
-

software/slaprunner/TODO.rst

-- resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance
-- add test for parameter auto-deploy-instance
-- Add download facility in file browser

software/slaprunner/buildout.hash.cfg

-# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
-# The only allowed lines here are (regexes):
-# - "^#" comments, copied verbatim
-# - "^[" section beginings, copied verbatim
-# - lines containing an "=" sign which must fit in the following categorie.
-#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
-#     Copied verbatim.
-#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
-#     by the re-generation script.
-#     Re-generated.
-# - other lines are copied verbatim
-# Substitution (${...:...}), extension ([buildout] extends = ...) and
-# section inheritance (< = ...) are NOT supported (but you should really
-# not need these here).
-[template]
-filename = instance.cfg
-md5sum = 473325442cba8ee32642d1e5e3fa94f4
-
-[template-runner]
-filename = instance-runner.cfg
-md5sum = 03d3774fcb01a8ec68a23de31fb346b8
-
-[template-runner-import-script]
-filename = template/runner-import.sh.jinja2
-md5sum = f2e2493bc5da90a53f86e5bcf64d2d57
-
-[instance-runner-import]
-filename = instance-runner-import.cfg.in
-md5sum = f4d71d97cf16ef379b41c4c22352727c
-
-[instance-runner-export]
-filename = instance-runner-export.cfg.in
-md5sum = 5f57249108e61976d97465624cc5a2a1
-
-[template-resilient]
-filename = instance-resilient.cfg.jinja2
-md5sum = bd0ad0b80d2b39189f9665c48f1b3830
-
-[template_nginx_conf]
-filename = nginx_conf.in
-md5sum = a042e52e0594ec0486cff7dc896cb6ce
-
-[template_httpd_conf]
-filename = httpd_conf.in
-md5sum = 35f5f5c6e183af6064398c88fa85de59
-
-[template_launcher]
-filename = launcher.in
-md5sum = 7a2f5e25a818cb29964666ada7852a5c
-
-[template-slapos-cfg]
-filename = template/slapos.cfg.in
-md5sum = 03322b8a0f42ea579d35981211359190
-
-[template-slapformat-definition.cfg]
-filename = template/slapformat-definition.cfg.in
-md5sum = bbb767dee9730f5a816ce4341403d89b
-
-[template-parameters]
-filename = parameters.xml.in
-md5sum = 0d7b2432aa809b859509ca1114accd8f
-
-[template-bash-profile]
-filename = template/bash_profile.in
-md5sum = d0ff3c86c76168d17d27b47b93a9b7aa
-
-[template-supervisord]
-filename = template/supervisord.conf.in
-md5sum = 5e6c84098440c6bc163898dcafca8c9b
-
-[template-listener-slapgrid]
-filename = template/listener_slapgrid.py.in
-md5sum = 49d50410cf7467175a841eb7cd0d93d4
-
-[monitor-check-webrunner-internal-instance]
-filename = template/monitor-check-webrunner-internal-instances.py
-md5sum = acaac32cf1bd45714272468a89f4f119
-
-[template-resilient-software-release-information]
-filename = template/resilient_software_release_information.py.in
-md5sum = 869a3afbf3c2fff6c72602662c6a8730
-
-[template-slapuser-script]
-filename = template/slapos-slapuser-script.in
-md5sum = 75aab99c995ca841f93fc77fc9116c37
-
-[template-buildout-shared-part-list]
-filename = template/buildout-shared-part-list.in
-md5sum = f619c8c5897c4851442b7090c8509758

software/slaprunner/development.cfg

-# Development profile of slaprunner.
-# Exactly the same as software.cfg, but fetch the slapos.cookbook and
-# slapos.toolbox from git repository instead of fetching stable version,
-# allowing to play with bleeding edge environment.
-
-[buildout]
-extends = software.cfg
-  ../../stack/slapos-dev.cfg
-
-parts +=
-# Development eggs
-  ${:common-parts}
-
-[versions]
-slapos.cookbook =
-slapos.core =
-slapos.toolbox =

software/slaprunner/httpd_conf.in

-PidFile "{{ parameters.path_pid }}"
-ServerName example.com
-ServerAdmin someone@email
-
-<IfDefine !HTTPDPort>
-  Listen [{{ parameters.global_ip }}]:{{ parameters.global_port }}
-  Define HTTPDPort
-</IfDefine>
-
-LoadModule unixd_module modules/mod_unixd.so
-LoadModule access_compat_module modules/mod_access_compat.so
-LoadModule auth_basic_module modules/mod_auth_basic.so
-LoadModule authz_core_module modules/mod_authz_core.so
-LoadModule authz_user_module modules/mod_authz_user.so
-LoadModule authz_host_module modules/mod_authz_host.so
-LoadModule authn_core_module modules/mod_authn_core.so
-LoadModule authn_file_module modules/mod_authn_file.so
-LoadModule mime_module modules/mod_mime.so
-LoadModule ssl_module modules/mod_ssl.so
-LoadModule alias_module modules/mod_alias.so
-LoadModule env_module modules/mod_env.so
-LoadModule rewrite_module modules/mod_rewrite.so
-LoadModule headers_module modules/mod_headers.so
-LoadModule log_config_module modules/mod_log_config.so
-LoadModule dav_module modules/mod_dav.so
-LoadModule dav_fs_module modules/mod_dav_fs.so
-LoadModule cache_module modules/mod_cache.so
-LoadModule file_cache_module modules/mod_file_cache.so
-LoadModule setenvif_module modules/mod_setenvif.so
-LoadModule dir_module modules/mod_dir.so
-LoadModule autoindex_module modules/mod_autoindex.so
-
-ErrorLog "{{ parameters.path_error_log }}"
-LogFormat "%h %l %u %t \"%r\" %>s %b" common
-CustomLog "{{ parameters.path_access_log }}" common
-
-# SSL Configuration
-Define SSLConfigured
-SSLCertificateFile {{ parameters.cert_file }}
-SSLCertificateKeyFile {{ parameters.key_file }}
-SSLRandomSeed startup builtin
-SSLRandomSeed connect builtin
-SSLRandomSeed startup /dev/urandom 256
-SSLRandomSeed connect builtin
-SSLProtocol all -SSLv2 -SSLv3
-SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
-SSLHonorCipherOrder on
-SSLEngine   On
-
-Include {{ parameters.httpd_cors_file }}
-
-DocumentRoot {{ parameters.runner_home }}/public
-
-# Directory protection
-<Directory />
-    Options FollowSymLinks
-    AllowOverride None
-    Require all denied
-</Directory>
-
-Alias /public {{ parameters.runner_home }}/public
-<Directory {{ parameters.runner_home }}/public>
-    <Limit GET HEAD OPTIONS REPORT PROPFIND>
-      Order Allow,Deny
-      Allow from all
-      AllowOverride All
-      Satisfy Any
-      Options Indexes FollowSymLinks
-      DirectoryIndex index.html
-    </Limit>
-    <Files .htaccess>
-        order allow,deny
-        deny from all
-    </Files>
-</Directory>
-
-DavLockDB {{ parameters.dav_lock }}
-Alias /share {{ parameters.runner_home }}
-<Directory {{ parameters.runner_home }}>
-    DirectoryIndex disabled
-    DAV On
-    Options Indexes FollowSymLinks
-    AuthType Basic
-    AuthName "Webrunner Dav"
-    AuthUserFile "{{ parameters.htpasswd_file }}"
-
-    # Prevent using the web browser cache if requesting  the same document
-    # from different domains or with different users
-    Header set Cache-Control "private, max-age=0, must-revalidate"
-    Header set Vary "Origin,Cookie,Authorization"
-
-    <LimitExcept OPTIONS>
-        Require valid-user
-    </LimitExcept>
-</Directory>

software/slaprunner/instance-resilient.cfg.jinja2

-# vim: set ft=cfg:
-
-# Default parameters
-# NOTE: in case of resilient instance, auto-deploy-instance
-# set to false (else it could be dangerous)
-{% do slapparameter_dict.__setitem__('auto-deploy', slapparameter_dict.get('auto-deploy', 'true')) %}
-{% do slapparameter_dict.__setitem__('auto-deploy-instance', slapparameter_dict.get('auto-deploy-instance', 'false')) %}
-{% do slapparameter_dict.__setitem__('instance-type', 'resilient') %}
-# let decide the number of clones we want. We can't have more than 2 clones
-{% set number_of_instances = slapparameter_dict.get('resilient-clone-number', 1)|int %}
-{% if number_of_instances > 2 %}
-  {% set number_of_instances = 2 %}
-{% endif %}
-
-{% set slaprunner_return = ['init-user', 'init-password', 'url', 'ssh-public-key', 'resilient-ssh-url', 'notification-id', 'ip', 'backend-url', 'url', 'ssh-url', 'ssh-command', 'webdav-url', 'public-url'] -%}
-{% set monitor_return = ['monitor-base-url', 'monitor-url', 'monitor-user', 'monitor-password'] -%}
-{% set monitor_parameter = {'monitor-cors-domains': slapparameter_dict.pop('monitor-cors-domains', "monitor.app.officejs.com")} -%}
-{% set monitor_dict = {'parameter': monitor_parameter, 'return': monitor_return, 'set-monitor-url': True} -%}
-{% set monitor_interface_url = slapparameter_dict.pop('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
-
-{% import 'parts' as parts %}
-{% import 'replicated' as replicated with context %}
-
-[buildout]
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
-
-extends =
-  {{ monitor_template }}
-
-# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
-parts +=
-  publish-early
-  {{ parts.replicate("runner", number_of_instances + 1) }}
-  publish-connection-information
-
-[monitor-htpasswd]
-recipe = slapos.cookbook:generate.password
-storage-path = ${directory:etc}/.monitor_user
-bytes = 8
-username = admin
-
-[publish-early]
-recipe = slapos.cookbook:publish-early
--init =
-  init-password monitor-htpasswd:passwd
-
-{%  do monitor_parameter.__setitem__('monitor-username', slapparameter_dict.get('monitor-username', 'admin'))%}
-{%  do monitor_parameter.__setitem__('monitor-password', slapparameter_dict.get('monitor-password', '${publish-early:init-password}'))%}
-
-{{ replicated.replicate("runner", number_of_instances + 1, "runner-export", "runner-import", slapparameter_dict=slapparameter_dict, monitor_parameter_dict=monitor_dict) }}
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-etc = ${buildout:directory}/etc
-
-
-# Bubble up the parameters
-[request-runner]
-{% if slapparameter_dict.get('custom-frontend-backend-url') -%}
-{%   do slaprunner_return.append('custom-frontend-url') -%}
-{% endif -%}
-return =  {{ slaprunner_return | join(' ')}} {{ monitor_return | join(' ') }}
-
-[publish-connection-information]
-recipe = slapos.cookbook:publish
-backend-url = ${request-runner:connection-backend-url}
-url = ${request-runner:connection-url}
-init-user = ${request-runner:connection-init-user}
-init-password = ${publish-early:init-password}
-ssh-command = ${request-runner:connection-ssh-command}
-webdav-url = ${request-runner:connection-webdav-url}
-public-url = ${request-runner:connection-public-url}
-{% if slapparameter_dict.get('custom-frontend-backend-url') -%}
-custom-frontend-url = ${request-runner:connection-custom-frontend-url}
-{% endif %}
-monitor-base-url = ${request-runner:connection-monitor-base-url}
-monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${request-runner:connection-monitor-url}&username=${request-runner:connection-monitor-user}&password=${request-runner:connection-monitor-password}
-
-[slap-parameter]
-# Default parameters for distributed deployment
-# I.e state "backup1 of maria should go there, ..."
-# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated.
-{% for nb in range(1,number_of_instances+1) %}
-runner{{nb}}-computer-guid =
-pbs-runner{{nb}}-computer-guid =
-{% endfor %}
-# XXX-Cedric: Hardcoded parameters. Should be dynamically generated.
-domain =
-authorized-key =
-instance-amount = 10
-debug = false
-resilient-clone-number = 1

software/slaprunner/instance-runner-export.cfg.in

-[buildout]
-extends = {{ template_runner_path }}
-          {{ pbsready_export_template_path }}
-
-# parts from {{ template_runner_path }} are overriden by
-# {{ pbsready_export_template_path }} so we reinclude them here
-parts += ${:common-runner-parts}
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-# XXX - keep srv path with slash at the end.
-srv = ${:home}/srv/
-
-[proxy-free-port]
-recipe = slapos.cookbook:free_port
-minimum = 49980
-maximum = 49989
-ip = ${slap-network-information:local-ipv4}
-
-[runner-free-port]
-recipe = slapos.cookbook:free_port
-minimum = 50005
-maximum = 50014
-ip = ${slap-network-information:global-ipv6}
-
-[slaprunner]
-proxy_port = ${proxy-free-port:port}
-runner_port = ${runner-free-port:port}
-
-[supervisord-free-port]
-recipe = slapos.cookbook:free_port
-minimum = 39986
-maximum = 39995
-ip = ${slaprunner:ipv4}
-
-[supervisord]
-port = ${supervisord-free-port:port}
-
-[exporter-raw]
-recipe = slapos.cookbook:wrapper
-wrapper-path = ${directory:bin}/${slap-parameter:namebase}-exporter-raw
-command-line = {{ software_release_bin }}/${slap-parameter:namebase}-exporter --srv-path ${directory:srv} --etc-path ${directory:etc} --backup-path ${directory:backup} --backup-wait-time {{ backup_wait_time }}
-environment =
-  PATH={{ rsync_bin_folder }}:/bin:/usr/bin
-
-[exporter]
-recipe = slapos.cookbook:wrapper
-command-line = {{ bash_executable_location }} -c "exec ${exporter-raw:wrapper-path} > >(tee -ai "${directory:log}/resilient.log") 2>&1"
-wrapper-path = ${directory:bin}/${slap-parameter:namebase}-exporter
-# wrapper parameter is needed by resilience stack
-wrapper = ${:wrapper-path}
-
-[monitor-httpd-free-port]
-recipe = slapos.cookbook:free_port
-minimum = 8437
-maximum = 8446
-ip = ${slap-network-information:global-ipv6}
-
-# XXX Redefine Monitor parameters for runner-export
-[monitor-instance-parameter]
-monitor-httpd-port = ${monitor-httpd-free-port:port}
-{% if slapparameter_dict.get('name', '') -%}
-monitor-title = {{ slapparameter_dict['name'] }}
-{% endif -%}
-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
-{% if slapparameter_dict.get('monitor-username', '') -%}
-username = {{ slapparameter_dict['monitor-username'] }}
-{% endif -%}
-{% if slapparameter_dict.get('monitor-password', '') -%}
-password = {{ slapparameter_dict['monitor-password'] }}
-{% endif -%}
-{% if slapparameter_dict.get('monitor-url-list', '') -%}
-monitor-url-list = {{ slapparameter_dict['monitor-url-list'] }}
-{% endif -%}
-instance-configuration =
-  httpdcors cors-domain ${slaprunner-httpd-cors:location} ${httpd-graceful-wrapper:output}
-configuration-file-path = ${buildout:directory}/knowledge0.cfg
-
-[monitor-conf-parameters]
-private-path-list +=
-  $${logrotate-directory:logrotate-backup}
-
-# Extends publish section with resilient parameters
-[publish-connection-information]
-<= resilient-publish-connection-parameter
-
-[monitor-check-resilient-feed-file]
-recipe = slapos.recipe.template:jinja2
-url = {{ monitor_check_resilient_feed_template_path }}
-output = ${monitor-directory:bin}/check-create-resilient-feed-files
-context =
-  key input_feed_directory directory:notifier-feeds
-  key monitor_feed_directory monitor-directory:public
-  raw base_url http://[${notifier:host}]:${notifier:port}/get/
-  raw python_executable ${buildout:executable}

software/slaprunner/instance-runner-import.cfg.in

-[buildout]
-extends = {{ template_runner_path }}
-          {{ pbsready_import_template_path }}
-
-parts +=
-  nginx_conf
-  nginx-launcher
-  certificate-authority
-  ca-nginx
-  certificate-authority-service
-  ca-nginx-service
-  gunicorn-launcher
-  gunicorn-graceful
-  slaprunner-promise
-  slaprunner-supervisord-wrapper
-  runner-sshd-add-authorized-key
-  runner-sshd-promise
-  runner-sshd-service
-  runtestsuite
-  shellinabox
-  shellinabox-service
-  slapos-cfg
-  cron-entry-prepare-software
-  deploy-instance-parameters
-  instance-software-type
-  bash-profile
-  supervisord-wrapper
-  importer-consistency-promise
-  software-release-deployment-promise
-  template-slapuser-script
-
-  resilient-software-release-information
-
-  monitor-base
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-# XXX - keep srv path with slash at the end.
-srv = ${:home}/srv/
-
-
-# For the needs of importer, we run the full slaprunner
-# In case both exporter and importer (aka main instance and clone instance)
-# run with the same IP (usually for testing purposes),
-# run slaprunner using different ports.
-[proxy-free-port]
-recipe = slapos.cookbook:free_port
-minimum = 49990
-maximum = 49999
-ip = ${slap-network-information:local-ipv4}
-
-[runner-free-port]
-recipe = slapos.cookbook:free_port
-minimum = 50015
-maximum = 50024
-ip = ${slap-network-information:global-ipv6}
-
-[slaprunner]
-proxy_port = ${proxy-free-port:port}
-runner_port = ${runner-free-port:port}
-
-[supervisord-free-port]
-recipe = slapos.cookbook:free_port
-minimum = 39996
-maximum = 40005
-ip = ${slaprunner:ipv4}
-
-[supervisord]
-port = ${supervisord-free-port:port}
-
-[runner-sshd-port]
-minimum = 22232
-maximum = 22241
-
-# Deactivate the call to prepareSoftware, and let the importer script
-# handle the build&run of the instance.
-[cron-entry-prepare-software]
-recipe =
-
-[importer]
-recipe = slapos.recipe.template:jinja2
-url = {{ importer_script_path }}
-output = ${directory:bin}/${slap-parameter:namebase}-importer
-# backward compatibility for resilient stack
-wrapper = ${:output}
-restore-exit-code-file = ${directory:srv}/${:restore-exit-code-file-basename}
-restore-exit-code-file-basename = importer-exit-code-file
-restore-error-message-file = ${directory:srv}/${:restore-error-message-file-basename}
-restore-error-message-file-basename = importer-error-message-file
-resilient-log-basename = resilient.log
-context =
-  import sys sys
-  import easy_install zc.buildout.easy_install
-  key backend_url slaprunner:access-url
-  key ipv4 slaprunner:ipv4
-  key ipv6 slaprunner:ipv6
-  key proxy_port slaprunner:proxy_port
-  key instance_folder slaprunner:instance_root
-  section directory directory
-  section supervisord supervisord
-  raw  output_log_file ${directory:log}/${:resilient-log-basename}
-  raw  shell_binary {{ bash_executable_location }}
-  raw  sqlite3_binary {{ sqlite3_executable_location }}
-  raw  rsync_binary {{ rsync_executable_location }}
-  raw  restore_exit_code_file ${:restore-exit-code-file}
-  raw  restore_error_message_file ${:restore-error-message-file}
-
-[importer-consistency-promise-bin]
-# Test that the importer script and "after-import" subscripts
-# are not older than 2 days (1 day + some slack), and have succeeded
-recipe = collective.recipe.template
-input = inline: #!/bin/sh
-  EXIT_CODE_FILE="${importer:restore-exit-code-file}"
-  RECENT_EXIT_CODE_FILE=$(find ${directory:srv} -maxdepth 1 -name "${importer:restore-exit-code-file-basename}" -mtime -2)
-  RESILIENT_LOG_URL=${monitor-publish-parameters:monitor-base-url}/share/private/log/${importer:resilient-log-basename}
-  if [ ! -f "$EXIT_CODE_FILE" ]; then
-    exit 0;
-  else
-    if [ -z "$RECENT_EXIT_CODE_FILE" ]; then
-       echo "Consistency check is too old.";
-       exit 1;
-    else
-      if [ -f "${importer:restore-error-message-file}" ]; then
-        cat ${importer:restore-error-message-file}
-      fi
-      echo "More information can be found here : $RESILIENT_LOG_URL";
-      exit $(cat $EXIT_CODE_FILE);
-    fi
-  fi
-  exit 1; # Something else went wrong
-output = ${directory:bin}/importer-consistency-promise
-mode = 755
-
-[importer-consistency-promise]
-<= monitor-promise-base
-promise = check_command_execute
-name = importer-consistency-promise.py
-config-command = ${importer-consistency-promise-bin:output}
-
-[software-release-deployment-bin]
-recipe = collective.recipe.template
-input = inline: #!/bin/sh
-  PROJECT_FILE=$(find "${directory:etc}" -maxdepth 1 -name .project)
-  if [ -z "$PROJECT_FILE" ]; then
-    exit 0;
-  else
-    INSTALLATION_FILE=$(find "${runnerdirectory:software-root}" -mindepth 2 -maxdepth 2 -name .completed)
-    if [ -n "$INSTALLATION_FILE" ]; then
-      exit 0;
-    else
-      exit 1;
-    fi
-  fi
-  exit 1
-output = ${directory:bin}/software-release-deployment-promise
-mode = 755
-
-[software-release-deployment-promise]
-<= monitor-promise-base
-promise = check_command_execute
-name = software-release-deployment-promise.py
-config-command =${software-release-deployment-bin:output}
-
-[resilient-software-release-information]
-recipe = slapos.recipe.template
-url = {{ software_release_information_template }}
-output = ${directory:cgi-bin}/resilient_software_release_information.py
-resilient-log-url = ${monitor-publish-parameters:monitor-base-url}/log/${importer:resilient-log-basename}
-
-[slap-parameter]
-auto-deploy-instance = false
-auto-deploy = true
-
-[resilient-publish-connection-parameter]
-monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
-monitor-url = ${monitor-publish-parameters:monitor-url}
-monitor-user = ${monitor-publish-parameters:monitor-user}
-monitor-password = ${monitor-publish-parameters:monitor-password}
-
-[monitor-instance-parameter]
-monitor-httpd-port = 8360
-monitor-title = {{ slapparameter_dict.get('name', 'Webrunner import') }}
-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
-{% if slapparameter_dict.get('monitor-username', '') -%}
-username = {{ slapparameter_dict['monitor-username'] }}
-{% endif -%}
-{% if slapparameter_dict.get('monitor-password', '') -%}
-password = {{ slapparameter_dict['monitor-password'] }}
-{% endif -%}
-instance-configuration =
-  raw takeover-url http://[${resilient-web-takeover-httpd-configuration-file:listening-ip}]:${resilient-web-takeover-httpd-configuration-file:listening-port}/
-  raw takeover-password ${resilient-web-takeover-password:passwd}
-configuration-file-path = ${buildout:directory}/knowledge0.cfg
-
-[monitor-conf-parameters]
-private-path-list +=
-  $${logrotate-directory:logrotate-backup}
-
-[post-notification-run]
-recipe = slapos.cookbook:wrapper
-command-line = {{ software_release_bin }}/runner-importer-post-notification-run --diff-file ${:diff-file} --proof-signature-file ${:proof-signature-file} --srv-path ${directory:srv} --backup-path ${directory:backup}
-wrapper-path = ${rootdirectory:bin}/post-notification-run
-output = ${:wrapper-path}
-mode = 0700

software/slaprunner/instance-runner-input-schema.json

-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "type": "object",
-  "properties": {
-    "instance-name": {
-      "title": "Instance Name",
-      "description": "Name of the instance, to show in the window title",
-      "type": "string"
-    },
-    "custom-frontend-backend-url": {
-      "title": "Custom Frontend Backend URL",
-      "description": "return an ipv4 frontend of the given ipv6(+optional port)",
-      "type": "string",
-      "format": "uri"
-    },
-    "custom-frontend-backend-type": {
-      "title": "Custom Frontend Backend Type",
-      "description": "The type of the frontend slave instance to ask",
-      "type": "string",
-      "enum": [
-        "zope"
-      ]
-    },
-    "user-authorized-key": {
-      "title": "User Authorized Key",
-      "description": "SSH public key in order to connect to the SSH server of this runner instance.",
-      "textarea": true,
-      "type": "string"
-    },
-    "instance-amount": {
-      "title": "Partition Amount",
-      "description": "Number of slappart to deploy inside the runner (default is 10). Needs instance to be restarted.",
-      "type": "integer",
-      "minimum": 1,
-      "maximum": 40
-    },
-    "auto-deploy": {
-      "title": "Automatically Deploy Software",
-      "description": "Authorizes the software declared with 'slapos-software' to be automatically deployed, or not. Needs instance to be restarted. (default is false)",
-      "type": "boolean"
-    },
-    "auto-deploy-instance": {
-      "title": "Automatically Deploy Instances",
-      "description": "Prevent the runner from deploying and starting instances. Needs instance to be restarted. It is set to false for instances of type 'import' in resiliency in any case (default is false)",
-      "type": "boolean"
-    },
-    "autorun": {
-      "title": "Automatically Run Sofware/Instance",
-      "description": "Let automaticaly build and run a declared software with 'slapos-software'. Only works if 'slapos-software' is set, and 'auto-deploy' is true. Needs instance to be restarted. (default is false)",
-      "type": "boolean"
-    },
-    "slapos-software": {
-      "title": "Pre-selected Software Release",
-      "description": "a relative path from the slapos git repo to a folder containing a software release, which will be automaticaly deployed while the runner instanciation, and only if the parameter auto-deploy is set to 'true'. For example: 'software/helloworld",
-      "type": "string"
-    },
-    "slapos-repository": {
-      "title": "SlapOS Git Repository URL",
-      "description": "url of the default git repository that will be download by the runner while its instanciation. Will be cloned in a directory named 'slapos' (default is https://lab.nexedi.com/nexedi/slapos.git)",
-      "type": "string",
-      "format": "uri",
-      "pattern": "^(http|https|ftp)://"
-    },
-    "slapos-reference": {
-      "title": "SlapOS Git Branch Name",
-      "description": "Branch or hash on which the default repository will checkout (default is master)",
-      "type": "string"
-    },
-    "slapos-software-type": {
-      "title": "Deployed Instance Software Type",
-      "description": "Software type of your instance inside the runner",
-      "type": "string"
-    },
-    "cpu-usage-ratio": {
-      "title": "CPU Usage Ratio",
-      "description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
-      "type": "integer",
-      "default": 4
-    },
-    "no-ipv4-frontend": {
-      "title": "No IPv4 frontend",
-      "description": "Prevent the slaprunner to order an IPv4 frontend for itself",
-      "enum": [
-        "true",
-        "false"
-      ],
-      "default": "false"
-    },
-    "custom-frontend-basic-auth": {
-      "title": "Custom Frontend Basic Auth",
-      "description": "if the ip given with 'custom-frontend-backend-url' is secure, set it to true for the promise do not fail",
-      "type": "boolean"
-    },
-    "custom-frontend-instance-guid": {
-      "title": "Custom Frontend Instance GUID",
-      "description": "Instance guid of the frontend you whish to use",
-      "type": "string"
-    },
-    "custom-frontend-software-type": {
-      "title": "Custom Frontend Software Type",
-      "description": "SoftwareType of the frontend you request (default is RootSoftwareInstance)",
-      "type": "string",
-      "enum": [
-        "RootSoftwareInstance",
-        "default"
-      ]
-    },
-    "custom-frontend-software-url": {
-      "title": "Custom Frontend Software URL",
-      "description": "Software Url of the frontend you request (ie.: http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg)",
-      "type": "string",
-      "format": "uri"
-    },
-    "check-custom-frontend-promise": {
-      "title": "Check Custom Frontend Promise",
-      "description": "Enable a promise to check that HTTP frontend created from custom-frontend-backend-url is available",
-      "type": "string",
-      "enum": [
-        "true",
-        "false"
-      ],
-      "default": "false"
-    },
-    "slaprunner-httpd-port": {
-      "title": "Webrunner Server Port",
-      "description": "Port of the Apache server serving the Webrunner interface.",
-      "type": "integer",
-      "default": 9686
-    },
-    "monitor-httpd-port": {
-      "title": "Monitor Port",
-      "description": "Port of the Apache server serving the monitoring interface.",
-      "type": "integer",
-      "default": 8386
-    },
-    "monitor-interface-url": {
-      "title": "Monitor Web Interface URL",
-      "description": "Give Url of HTML web interface that will be used to render this monitor instance.",
-      "type": "string",
-      "format": "uri",
-      "default": "https://monitor.app.officejs.com"
-    },
-    "monitor-cors-domains": {
-      "title": "Monitor CORS domains",
-      "description": "List of cors domains separated with space. Needed for ajax query on this monitor instance from a different domain.",
-      "type": "string",
-      "default": "monitor.app.officejs.com"
-    }
-  }
-}

software/slaprunner/instance-runner-output-schema.json

-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "description": "Values returned by Runner instanciation",
-  "properties": {
-    "backend_url": {
-      "description": "ipv6 url to directly access your runner",
-      "type": "string"
-    },
-    "access_url": {
-      "description": "Url to set up your account and then login into your webrunner",
-      "type": "string"
-    },
-    "url": {
-      "description": "Url to access your runner once you set your account",
-      "type": "string"
-    },
-    "ssh_command": {
-      "description": "SSH command used to access your runner in ssh when you provided a ssh public key",
-      "type": "string"
-    },
-    "monitor_url": {
-      "description": "Url to access the monitoring interface of your instance. Use this url to get the recovery code needed to set up your runner",
-      "type": "string"
-    },
-    "webdav_url": {
-      "description": "Url for webdav access of your runner it access the root of your runner. Same users as the one for the runner",
-      "type": "string"
-    },
-    "public_url": {
-      "description": "Url of the public folder of your runner",
-      "type": "string"
-    }
-  },
-  "type": "object"
-}

software/slaprunner/instance-runner-resilient-input-schema.json

-{
-  "type": "object",
-  "$schema": "http://json-schema.org/draft-04/schema",
-  "allOf": [
-    {
-      "$ref": "instance-runner-input-schema.json#/"
-    },
-    {
-      "properties": {
-        "resilient-clone-number": {
-          "title": "Amount of backup(s) to create",
-          "description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
-          "type": "integer",
-          "default": 1,
-          "minimum": 0,
-          "maximum": 2,
-          "optional": true
-        },
-        "-sla-runner0-computer_guid": {
-          "title": "Target computer for main instance",
-          "description": "Target computer GUID for main instance.",
-          "type": "string",
-          "optional": true
-        },
-        "-sla-runner1-computer_guid": {
-          "title": "Target computer for first clone",
-          "description": "Target computer for first clone instance.",
-          "type": "string",
-          "optional": true
-        },
-        "-sla-pbs1-computer_guid": {
-          "title": "Target computer for first PBS",
-          "description": "Target computer for first PBS instance.",
-          "type": "string",
-          "optional": true
-        },
-        "-sla-runner2-computer_guid": {
-          "title": "Target computer for second clone",
-          "description": "Target computer for second clone instance.",
-          "type": "string",
-          "optional": true
-        },
-        "-sla-pbs2-computer_guid": {
-          "title": "Target computer for second PBS",
-          "description": "Target computer for second PBS instance.",
-          "type": "string",
-          "optional": true
-        },
-        "resiliency-backup-periodicity": {
-          "title": "Periodicity of backup",
-          "description": "Periodicity of backup, in cron format.",
-          "type": "string",
-          "optional": true
-        },
-        "remove-backup-older-than": {
-          "title": "Remove backups older than...",
-          "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
-          "type": "string",
-          "default": "2W",
-          "optional": true
-        }
-      }
-    }
-  ]
-}

software/slaprunner/instance.cfg

-[buildout]
-parts =
-  switch_softwaretype
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-
-extends =
-  ${template-resilient-templates:output}
-
-[switch_softwaretype]
-recipe = slapos.cookbook:switch-softwaretype
-default = $${:runner}
-resilient = instance-resilient:output
-runner = instance-base-runner:output
-runner-import = template-runner-import:output
-runner-export = template-runner-export:output
-
-frozen = instance-frozen:output
-pull-backup = template-pull-backup:output
-
-# BBB
-RootSoftwareInstance = $${:default}
-
-[instance-base-runner]
-recipe = slapos.recipe.template:jinja2
-url = ${template-runner:output}
-output = $${buildout:directory}/template-runner.cfg
-extensions = jinja2.ext.do
-context = key buildout buildout:bin-directory
-          key develop_eggs_directory buildout:develop-eggs-directory
-          key eggs_directory buildout:eggs-directory
-          key slapparameter_dict slap-configuration:configuration
-          raw curl_executable_location ${curl:location}/bin/curl
-          raw dash_executable_location ${dash:location}/bin/dash
-          raw dcron_executable_location ${dcron:location}/sbin/crond
-
-[instance-resilient]
-recipe = slapos.recipe.template:jinja2
-url = ${template-resilient:target}
-output = $${buildout:directory}/instance-resilient.cfg
-extensions = jinja2.ext.do
-context = key buildout buildout:bin-directory
-          key develop_eggs_directory buildout:develop-eggs-directory
-          key eggs_directory buildout:eggs-directory
-          key slapparameter_dict slap-configuration:configuration
-          raw monitor_template ${monitor-template:output}
-template-parts-destination = ${template-parts:target}
-template-replicated-destination = ${template-replicated:target}
-import-list = file parts :template-parts-destination
-              file replicated :template-replicated-destination
-
-[template-runner-export]
-recipe = slapos.recipe.template:jinja2
-url = ${instance-runner-export:target}
-output = $${buildout:directory}/instance-runner-export.cfg
-context =
-  key pbsready_export_template_path template-pbsready-export:output
-  key template_runner_path instance-base-runner:output
-  key slapparameter_dict slap-configuration:configuration
-  raw software_release_bin ${buildout:bin-directory}
-  raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
-  raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:target}
-  raw buildout_executable_location ${buildout:executable}
-  raw bash_executable_location ${bash:location}/bin/bash
-  raw rsync_bin_folder ${rsync:location}/bin
-
-[template-runner-import]
-recipe = slapos.recipe.template:jinja2
-url = ${instance-runner-import:target}
-output = $${buildout:directory}/instance-runner-import.cfg
-context =
-  key pbsready_import_template_path template-pbsready-import:output
-  key template_runner_path instance-base-runner:output
-  key slapparameter_dict slap-configuration:configuration
-  raw software_release_bin ${buildout:bin-directory}
-  raw importer_script_path ${template-runner-import-script:target}
-  raw buildout_executable_location ${buildout:executable}
-  raw bash_executable_location ${bash:location}/bin/bash
-  raw sqlite3_executable_location ${sqlite3:location}/bin/sqlite3
-  raw rsync_executable_location ${rsync:location}/bin/rsync
-  raw software_release_information_template ${template-resilient-software-release-information:target}
-
-[slap-configuration]
-recipe = slapos.cookbook:slapconfiguration
-computer = $${slap-connection:computer-id}
-partition = $${slap-connection:partition-id}
-url = $${slap-connection:server-url}
-key = $${slap-connection:key-file}
-cert = $${slap-connection:cert-file}

software/slaprunner/launcher.in

-#! {{ param_nginx_frontend['path_shell'] }}
-# BEWARE: This file is operated by slapos node
-# BEWARE: It will be overwritten automatically
-
-# Run nginx
-exec {{ param_nginx_frontend['bin_nginx'] }} \
-    -p {{ param_nginx_frontend['nginx_prefix'] }} \
-    -c {{ param_nginx_frontend['path_nginx_conf'] }}

software/slaprunner/nginx_conf.in

-worker_processes {{ param_nginx_frontend['nb_workers'] }};
-
-pid {{ param_nginx_frontend['path_pid'] }};
-error_log {{ param_nginx_frontend['path_error_log'] }};
-
-daemon off;
-
-events {
-  worker_connections 1024;
-  accept_mutex off;
-}
-
-http {
-     default_type application/octet-stream;
-     access_log {{ param_nginx_frontend['path_access_log'] }} combined;
-     client_max_body_size 10M;
-     map $http_upgrade $connection_upgrade {
-        default upgrade;
-        ''      close;
-     }
-
-     server {
-        listen [{{ param_nginx_frontend['global-ip'] }}]:{{ param_nginx_frontend['global-port'] }} ssl;
-        server_name _;
-        ssl_certificate     {{ param_nginx_frontend['ssl-certificate'] }};
-        ssl_certificate_key {{ param_nginx_frontend['ssl-key'] }};
-        ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
-        ssl_ciphers         ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5;
-        ssl_prefer_server_ciphers on;
-        keepalive_timeout 90s;
-        client_body_temp_path {{ param_tempdir['client_body_temp_path'] }};
-        proxy_temp_path {{ param_tempdir['proxy_temp_path'] }};
-        fastcgi_temp_path {{ param_tempdir['fastcgi_temp_path'] }};
-        uwsgi_temp_path {{ param_tempdir['uwsgi_temp_path'] }};
-        scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
-
-        location / {
-            auth_basic "Restricted";
-            auth_basic_user_file {{ param_nginx_frontend['etc_dir'] }}/.htpasswd;
-            proxy_redirect off;
-            proxy_set_header   X-Forwarded-Proto $scheme;
-            proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
-            proxy_set_header   X-Forwarded-Host  $http_host;
-            proxy_set_header   Host              $http_host;
-            proxy_connect_timeout 200;
-            proxy_send_timeout    200;
-            proxy_read_timeout    200;
-            send_timeout          200;
-
-            proxy_pass http://unix:{{ socket }};
-        }
-        location ~ ^(/login|/doLogin|/static|/slapgridResult|/isSRReady) {
-            proxy_redirect off;
-            proxy_set_header   X-Forwarded-Proto $scheme;
-            proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
-            proxy_set_header   X-Forwarded-Host  $http_host;
-            proxy_set_header   Host              $http_host;
-
-            proxy_pass http://unix:{{ socket }};
-        }
-        location /shellinabox {
-            proxy_pass http://unix:{{ shellinabox_socket }}:/;
-            proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
-            auth_basic "Restricted";
-            auth_basic_user_file {{ param_nginx_frontend['etc_dir'] }}/.htpasswd;
-            proxy_redirect off;
-            proxy_buffering off;
-            proxy_set_header        X-Real-IP         $remote_addr;
-            proxy_set_header        X-Forwarded-Proto $scheme;
-            proxy_set_header        X-Forwarded-For   $proxy_add_x_forwarded_for;
-            proxy_set_header        X-Forwarded-Host  $http_host;
-      }
-    }
-}

software/slaprunner/parameters.xml.in

-{% set inst_parameter_dict = {} -%}
-{% if slapparameter_dict is defined -%}
-{%   for key in slapparameter_dict.keys() -%}
-{%     if key.startswith('parameter-') -%}
-{%       do inst_parameter_dict.__setitem__(key[10:], slapparameter_dict.pop(key)) -%}
-{%     endif -%}
-{%   endfor -%}
-{% endif -%}
-
-<?xml version='1.0' encoding='utf-8'?>
-<instance>
-{% if slapparameter_dict is defined %}
-  {% for parameter_name in inst_parameter_dict.keys() %}
-  <parameter id="{{ parameter_name }}">{{ inst_parameter_dict[parameter_name] }}</parameter>
-  {% endfor %}
-{% endif %}
-</instance>

software/slaprunner/software.cfg

-[buildout]
-extends =
-  buildout.hash.cfg
-  ../../component/bash/buildout.cfg
-  ../../component/fish-shell/buildout.cfg
-  ../../component/tmux/buildout.cfg
-  ../../component/busybox/buildout.cfg
-  ../../component/curl/buildout.cfg
-  ../../component/dash/buildout.cfg
-  ../../component/dcron/buildout.cfg
-  ../../component/git/buildout.cfg
-  ../../component/tig/buildout.cfg
-  ../../component/logrotate/buildout.cfg
-  ../../component/lxml-python/buildout.cfg
-  ../../component/nano/buildout.cfg
-  ../../component/nginx/buildout.cfg
-  ../../component/openssh/buildout.cfg
-  ../../component/mosh/buildout.cfg
-  ../../component/rsync/buildout.cfg
-  ../../component/pycurl/buildout.cfg
-  ../../component/screen/buildout.cfg
-  ../../component/shellinabox/buildout.cfg
-  ../../component/vim/buildout.cfg
-  ../../component/zip/buildout.cfg
-  ../../stack/slapos.cfg
-  ../../stack/flask.cfg
-  ../../stack/resilient/buildout.cfg
-  ../../stack/logrotate/buildout.cfg
-  ../../stack/monitor/buildout.cfg
-
-# make sure shared-part-list is available, even for old versions
-# of slapos who do not set that.
-shared-part-list =
-
-# stacks are listed from most generic to most specific,
-# to avoid versioning issues
-
-common-parts =
-  slapos-command
-  slapos-cookbook
-  template
-  instance-runner-import
-  instance-runner-export
-  template-slapos-cfg
-  template-slapformat-definition.cfg
-  template-slapuser-script
-# XXX: we have to manually add this for resilience
-  pbs-recipe-egg
-
-parts =
-  ${:common-parts}
-
-[python]
-part = python2.7
-
-[template-base]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/${:filename}
-
-[download-base]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/${:filename}
-
-[template]
-< = template-base
-output = ${buildout:directory}/template.cfg
-
-[template-runner]
-< = template-base
-output = ${buildout:directory}/template-runner.cfg.in
-
-[template-runner-import-script]
-< = download-base
-
-[instance-runner-import]
-< = download-base
-
-[instance-runner-export]
-< = download-base
-
-[template-resilient]
-< = download-base
-
-[template_nginx_conf]
-< = download-base
-
-[template_httpd_conf]
-< = download-base
-
-[template_launcher]
-< = download-base
-
-[template-slapos-cfg]
-< = download-base
-
-[template-slapformat-definition.cfg]
-< = download-base
-
-[template-parameters]
-< = download-base
-
-[template-bash-profile]
-< = download-base
-
-[template-supervisord]
-< = download-base
-
-[template-listener-slapgrid]
-< = download-base
-
-[monitor-check-webrunner-internal-instance]
-< = download-base
-
-[template-resilient-software-release-information]
-< = download-base
-
-[template-slapuser-script]
-< = download-base
-
-[template-buildout-shared-part-list]
-< = template-base
-output = ${buildout:directory}/buildout-shared-part-list
-
-[python-with-eggs]
-recipe = zc.recipe.egg
-interpreter = ${:_buildout_section_name_}
-eggs =
-  ${slapos-toolbox:eggs}
-  erp5.util
-  lock-file
-  slapos.recipe.build
-  slapos.toolbox[flask_auth]
-  gunicorn
-# for gunicorn[gthread]
-  futures
-  supervisor
-# BBB: eggs used as recipe should be kept otherwise sections depending
-# on it can't be uninstalled
-  collective.recipe.shelloutput
-scripts =
-  gunicorn
-  supervisord
-  supervisorctl
-
-
-[versions]
-Flask-Auth = 0.85
-futures = 3.0.5
-gunicorn = 19.10.0
-prettytable = 0.7.2
-pycurl = 7.43.0

software/slaprunner/software.cfg.json

-{
-  "name": "Web Runner",
-  "description": "Web Runner",
-  "serialisation": "xml",
-  "software-type": {
-    "default": {
-      "title": "Default",
-      "description": "Standalone Runner",
-      "request": "instance-runner-input-schema.json",
-      "response": "instance-runner-output-schema.json",
-      "index": 0
-    },
-    "resilient": {
-      "title": "Resilient",
-      "description": "Resilient Runner",
-      "request": "instance-runner-resilient-input-schema.json",
-      "response": "instance-runner-output-schema.json",
-      "index": 1
-    }
-  }
-}

software/slaprunner/template/bash_profile.in

-# Beware, this file is automatically processed by slapgrid
-# Do not modify it, your changes will be lost
-# If you want to load your custom bash configuration, please use a .bashrc file
-
-cd {{ workdir }}
-export HOME={{- home }}
-export PATH={{- path }}
-export SHELL={{- shell }}
-{%- if instance_name %}
-export PROMPT_COMMAND='echo -en "\033]0;{{-instance_name}}\a"'
-{% endif %}
-export PS1="$ "
-
-export TERMINFO={{- terminfo }}
-
-if [ -f "$HOME/.bashrc" ] ; then
-  source $HOME/.bashrc
-fi
-
-# If not running interactively, don't do anything
-case $- in
-    *i*) ;;
-      *) return;;
-esac
-
-# enable color support
-if [ -x /usr/bin/dircolors ]; then
-    test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
-    alias ls='ls --color=auto'
-fi
-
-# some more ls aliases
-alias ll='ls -l'
-alias la='ls -A'
-alias l='ls -CF'
-
-echo "Welcome to SlapOS slaprunner shell"
-echo

software/slaprunner/template/buildout-shared-part-list.in

-${buildout:shared-part-list}

software/slaprunner/template/cors-domain.jinja

-#!{{ python_executable }}
-
-import cgi
-import cgitb
-import ConfigParser
-import os
-import re
-import subprocess
-
-cgitb.enable(display=0, logdir="/tmp/cgi.log")
-form = cgi.FieldStorage()
-
-apache_file = "{{ apache_file }}"
-config_file = "{{ config_cfg }}"
-
-
-if not os.path.exists(config_file):
-  print "Your software does <b>not</b> embed 0-knowledge. \
-  This interface is useless in this case</body></html>"
-  exit(0)
-
-
-parser = ConfigParser.ConfigParser()
-parser.read(config_file)
-if not parser.has_section("cors"):
-  parser.add_section("cors")
-
-if not parser.has_option("cors", "cors-domain"):
-  parser.set("cors", "cors-domain", "")
-
-if "cors-domain" in form:
-  parser.set("cors", "cors-domain", form["cors-domain"].value)
-
-cors_domain_parameter = parser.get("cors", "cors-domain")
-if cors_domain_parameter:
-  cors_domain_list = cors_domain_parameter.split()
-  cors_string = ""
-  for domain in cors_domain_list:
-    if cors_string:
-      cors_string += '|'
-    cors_string += re.escape(domain)
-  with open(apache_file, 'w') as file:
-    file.write('SetEnvIf Origin "^http(s)?://(.+\.)?(%s)$" origin_is=$0\n' % cors_string)
-    file.write('Header always set Access-Control-Allow-Origin %{origin_is}e env=origin_is')
-  subprocess.call('{{ httpd_graceful }}')
-
-with open(config_file, 'w') as file:
-  parser.write(file)
-
-print "<html><head>"
-print "<link rel=\"stylesheet\" href=\"static/pure-min.css\">"
-print "<link rel=\"stylesheet\" href=\"static/style.css\">"
-print "</head><body>"
-print "<h1>Cors Domains :</h1>"
-print "<p>Enter domain names separated by space to authorized CORS on your dav storage for these domains</p>"
-print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
-print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
-print "<div class=\"pure-control-group\">"
-print '<label for="cors-domain">CORS Domains</label>'
-print '<input type="text" name="cors-domain" value="%s">' % cors_domain_parameter
-print "</div>"
-print "<div class=\"pure-controls\"><button type=\"submit\" class=\"pure-button \
- pure-button-primary\">Save</button></div></form>"
-print "</body></html>"

software/slaprunner/template/listener_slapgrid.py.in

-#!{{ python_executable }}
-
-import datetime
-import json
-import sys
-import xmlrpclib
-
-from supervisor import childutils
-
-
-def write_stdout(s):
-    sys.stdout.write(s)
-    sys.stdout.flush()
-
-def write_stderr(s):
-    sys.stderr.write(s)
-    sys.stderr.flush()
-
-
-def write_slapgrid_result(process):
-    server = xmlrpclib.Server("http://{{- supervisord['server'] -}}")
-    # Tuple of tuples containing 2 elements : process name and the path of its info file
-    json_files = dict([("slapgrid-sr", "{{- slaprunner['software_info_json'] -}}"),
-                       ("slapgrid-cp", "{{- slaprunner['instance_info_json'] -}}")])
-    info = server.supervisor.getProcessInfo(process)
-    result = dict()
-    result['last_build'] = datetime.datetime.fromtimestamp(info['stop']).strftime("%Y-%m-%d %H:%M:%S")
-    result['success'] = info['exitstatus']
-    open(json_files[process], 'w').write(json.dumps(result))
-
-
-def main():
-    while 1:
-        headers, payload = writer.wait()
-        try:
-            processname = dict(x.split(':') for x in payload.split())['processname']
-            write_slapgrid_result(processname)
-        except:
-            pass
-        writer.ok()
-
-if __name__ == '__main__':
-    writer = childutils.EventListenerProtocol()
-    main()

software/slaprunner/template/monitor-check-webrunner-internal-instances.py

-#!/usr/bin/python
-
-import os
-import subprocess
-import sys
-
-def runPromise(promise_path):
-  promise_relative_path = promise_path.replace(os.path.expanduser('~'), '~')
-  print 'Running promise %s...' % promise_relative_path
-  promise_process = subprocess.Popen(promise_path, stderr=subprocess.PIPE)
-  stdout, stderr = promise_process.communicate()
-  return_code = promise_process.returncode
-  if return_code == 0:
-    print 'Success.'
-    return True
-  else:
-    sys.stderr.write('Failure while running promise %s. %s\n' % (promise_relative_path, stderr))
-
-def getPromisePathListFromPartitionPath(partition_path):
-  promise_directory_path = os.path.join(partition_path, 'etc/promise')
-  try:
-    promise_name_list = os.listdir(promise_directory_path)
-    return [os.path.join(promise_directory_path, promise_name) for promise_name in promise_name_list]
-  except OSError:
-    return []
-
-def main():
-  # XXX hardcoded
-  partition_root_path = os.path.expanduser('~/srv/runner/instance')
-  success = True
-  for partition_name in os.listdir(partition_root_path):
-    partition_path = os.path.join(partition_root_path, partition_name)
-    for promise_path in getPromisePathListFromPartitionPath(partition_path):
-      result = runPromise(promise_path)
-      if not result:
-        success = False
-  if not success:
-    sys.exit(1)
-
-if __name__ == '__main__':
-  main()
-

software/slaprunner/template/resilient_software_release_information.py.in

-# Provides information related to the Webrunner Software Release to the
-# takeover interface of the Resilient stack
-
-def main():
-  return {
-    'Read the log from the importer': '<a href="${:resilient-log-url}">${:resilient-log-url}</a>',
-  }

software/slaprunner/template/runner-import.sh.jinja2

-#!{{ shell_binary }}
-LC_ALL=C
-export LC_ALL
-umask 077
-
-# Exit on any error, to prevent inconsistent backup
-# Error on unset variable expansion
-set -eu
-
-# Redirect output to log
-exec > >(tee -ai '{{ output_log_file }}')
-exec 2>&1
-
-echo -e "\n\n$0 run at : $(date)"
-
-srv_directory='{{ directory["srv"] }}'
-backup_directory='{{ directory["backup"] }}'
-etc_directory='{{ directory["etc"] }}'
-
-RESTORE_EXIT_CODE_FILE='{{ restore_exit_code_file }}'
-RESTORE_ERROR_MESSAGE_FILE='{{ restore_error_message_file }}'
-ERROR_MESSAGE=""
-
-fail_with_exit_code () {
-  echo 1 > $RESTORE_EXIT_CODE_FILE
-  echo -e "Failure during step : $ERROR_MESSAGE" > $RESTORE_ERROR_MESSAGE_FILE
-  exit 1
-}
-trap fail_with_exit_code ERR
-
-log_message () {
-    ERROR_MESSAGE=$1
-    echo -e $1
-}
-# Delete the error message file, to not keep it even after a successful build
-rm "$RESTORE_ERROR_MESSAGE_FILE" || true
-
-rsync () {
-  set -x
-  '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete "$@"
-  set +x
-}
-
-log_message "Restoring WebRunner content..."
-(
-  # XXX: code duplication with runner-export.sh.jinja2
-  path=$srv_directory/runner
-  backup_path=$backup_directory/runner/
-  cd "$backup_path"
-
-  if [ -d instance ]; then
-    # Concatenate the exclude file of each partition of webrunner
-    # to create a global exclude file.
-    # Also, ignore all buildout-managed files.
-    exclude=$({{ sys.executable }} - "$path" <<EOF
-if 1:
-        import glob, errno, os, sys
-        sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
-        from zc.buildout.configparser import parse
-        path = sys.argv[1]
-
-        def print_relative(path_list):
-            for p in path_list:
-                p = p.strip()
-                if p:
-                    print(os.path.relpath(p, path))
-        print("*.sock")
-        print("*.socket")
-        print("*.pid")
-        print(".installed*.cfg")
-        for partition in glob.glob(path + "/instance/slappart*"):
-            try:
-              os.chdir(partition)
-            except OSError as e:
-              if e.errno != errno.ENOTDIR:
-                raise
-              continue
-            try:
-                with open("srv/exporter.exclude") as f:
-                    exclude = f.readlines()
-            except IOError as e:
-                if e.errno != errno.ENOENT:
-                    raise
-            else:
-                print_relative(exclude)
-            for installed in glob.glob(".installed*.cfg"):
-                try:
-                    with open(installed) as f:
-                        installed = parse(f, installed)
-                except IOError as e:
-                    if e.errno != errno.ENOENT:
-                        raise
-                else:
-                    for section in installed.itervalues():
-                        print_relative(section.get(
-                            '__buildout_installed__', '').splitlines())
-EOF
-)
-    echo "$exclude" |rsync --exclude-from=- instance "$path"
-  fi
-
-  test -d project  && rsync project "$path"
-  test -d public  && rsync public "$path"
-  test -f proxy.db && rsync proxy.db "$path"
-)
-
-log_message "Restoring WebRunner config (etc directory)..."
-(
-  cd "$backup_directory"/etc/
-  rsync config.json "$etc_directory"
-  # Hidden files are related to the webrunner's internals
-  cp -r .??* "$etc_directory"
-)
-
-# Invoke arbitrary script to perform specific restoration
-# procedure.
-runner_import_restore=$srv_directory/runner-import-restore
-if [ -x "$runner_import_restore" ]; then
-  log_message "Running $runner_import_restore..."
-  "$srv_directory/runner-import-restore"
-fi
-
-# If no "etc/.project" neither "srv/runner/proxy.db", we can safely assume
-# that there is no instance deployed on runner0
-if [ ! -f "$etc_directory/.project" -a ! -f "$srv_directory/runner/proxy.db" ]; then
-  log_message "No Software Requested... Writing status file... End"
-  echo 0 > $RESTORE_EXIT_CODE_FILE
-  exit 0
-fi
-
-log_message "Updating slapproxy database..."
-HOME='{{ directory["home"] }}'
-# XXX Hardcoded
-export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-export MAKEFLAGS=-j4
-SLAPOS='{{ directory["bin"] }}'/slapos
-SQLITE3="{{ sqlite3_binary }}"
-DATABASE="$HOME/srv/runner/proxy.db"
-db_query () {
-  # Try opening locked tables for 5 seconds to prevent "database is locked" error
-"$SQLITE3" "$DATABASE" <<EOF
-.timeout 5000
-$@
-EOF
-}
-# If slapproxy database is empty then no software release was opened
-if [ ! -s "$DATABASE" ]; then
-  log_message "Slapproxy database empty, no Software Requested... Writing status file... End"
-  echo 0 > $RESTORE_EXIT_CODE_FILE
-  exit 0
-fi
-
-# Slap proxy table contain version number, find the table name dynamically.
-# This is known to work with version 11 or 12 of tables, but it will probably
-# work with earlier versions as well.
-DB_PARTITION_TABLE=$(db_query ".table partition__")
-DB_PARTITION_NETWORK_TABLE=$(db_query ".table partition\_network__")
-DB_SOFTWARE_TABLE=$(db_query ".table software__")
-
-# Change slapproxy database to point instances to new software release
-# XXX hardcoded
-PARTITION=$(basename $HOME)
-OLD_SOFTWARE_RELEASE=$(db_query "select software_release from $DB_PARTITION_TABLE where reference='slappart0';")
-if [ "$OLD_SOFTWARE_RELEASE" == "" ];
-then
-   echo "No instance configured"
-   echo 0 > $RESTORE_EXIT_CODE_FILE
-   exit 0
-fi
-SOFTWARE_RELEASE=$({{ sys.executable }} - $OLD_SOFTWARE_RELEASE $PARTITION <<EOF
-if 1:
-  import os, re, sys
-
-  # We want to replace the last occurence only
-  old_software_release, partition = sys.argv[1], sys.argv[2]
-  for match in re.finditer("(slappart|test0-|s)[0-9][0-9]*", old_software_release):
-    start, end = match.start(), match.end()
-
-  print old_software_release[:start] + partition + old_software_release[end:]
-EOF
-)
-
-db_query "update $DB_PARTITION_TABLE set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;"
-db_query "update $DB_SOFTWARE_TABLE set url='$SOFTWARE_RELEASE' where url='$OLD_SOFTWARE_RELEASE';" || db_query "delete from $DB_SOFTWARE_TABLE where url='$OLD_SOFTWARE_RELEASE';"
-# Change slapproxy database to have all instances stopped
-db_query "update $DB_PARTITION_TABLE set requested_state='stopped';"
-# Change slapproxy database to get correct IPs
-IPV4='{{ ipv4 }}'
-IPV6='{{ ipv6 }}'
-db_query "update $DB_PARTITION_NETWORK_TABLE set address='$IPV4' where netmask='255.255.255.255';"
-db_query "update $DB_PARTITION_NETWORK_TABLE set address='$IPV6' where netmask='ffff:ffff:ffff::';"
-
-MASTERURL='http://{{ ipv4 }}:{{ proxy_port }}'
-
-log_message "Removing old supervisord service description files..."
-# XXX: Path hardcoded in slapos.core
-rm '{{ instance_folder }}'/etc/supervisord.conf.d/* || true
-
-SLAPOSCFG='{{ supervisord["slapos-cfg"] }}'
-SLAPGRIDSRLOG='{{ supervisord["slapgrid-sr-log"] }}'
-SLAPGRIDCPLOG='{{ supervisord["slapgrid-cp-log"] }}'
-
-contain_software_release=0
-
-SOFTWARE_RELEASES_COUNT=$(db_query "SELECT count(1) FROM $DB_SOFTWARE_TABLE WHERE url != '';")
-if [ $SOFTWARE_RELEASES_COUNT -gt 0 ]; then
-  contain_software_release=1
-fi
-
-if [ $contain_software_release -eq 0 ]; then
-  log_message "No Software Release were deployed, so skip to continue..."
-  echo 0 > $RESTORE_EXIT_CODE_FILE
-  exit 0
-fi
-
-log_message "Building newest Software Release..."
-"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
-"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
-"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
-(tail -n 200 "$SLAPGRIDSRLOG" && false)
-
-contain_instance=0
-for folder in $srv_directory/runner/instance/slappart*/; do
-  if [ -f "$folder/buildout.cfg" ]; then
-    contain_instance=1
-  fi
-done
-
-# If instance do not contains template.cfg it means the user contains no instance.
-# so it is safer to assume that he is using slaprunner for develop buildout rather them slapos.
-if [ $contain_instance -eq 0 ]; then
-  log_message "None Instance were deployed with this software release, so skip to continue..."
-  echo 0 > $RESTORE_EXIT_CODE_FILE
-  exit 0
-fi
-
-# Remove defined scripts to force buildout to recreate them to have updated paths
-rm "$srv_directory"/runner/instance/slappart*/srv/runner-import-restore || true
-log_message "Fixing Instances as needed after import..."
-# XXX hardcoded
-"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 ||
-"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 ||
-"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 ||
-(tail -n 200 "$SLAPGRIDCPLOG" && false)
-
-# Invoke defined scripts for each partition inside of slaprunner
-log_message "Invoke custom import scripts defined by each instances..."
-for partition in "$srv_directory"/runner/instance/slappart*/
-do
-  script=$partition/srv/runner-import-restore
-  if [ -x "$script" ]; then
-    log_message "Running custom instance script : $script..."
-    "$script"
-  fi
-done
-
-# Change back slapproxy database to have all instances started
-DB_PARTITION_TABLE=$(db_query ".table partition__")
-log_message "Set instances as to start after takeover..."
-db_query "update $DB_PARTITION_TABLE set requested_state='started';"
-
-# Write exit code to an arbitrary file that will be checked by promise/monitor
-log_message "Writing status file... End"
-echo 0 > $RESTORE_EXIT_CODE_FILE
-exit 0

software/slaprunner/template/slapformat-definition.cfg.in

-[computer]
-# Address example: 2001:67c:1254:27::1/ffff:ffff:ffff:ffff::/64 
-address = {{ slaprunner['ipv6'] }}/64
-
-{% for partition_index in range(int(slaprunner['partition-amount'])) %}
-[partition_{{ partition_index }}]
-address =  {{ slaprunner['ipv4'] }}/255.255.255.255 {{ slaprunner['ipv6'] }}/64
-user = {{ partition_user }}
-pathname = slappart{{ partition_index }}
-network_interface =
-
-{% endfor %}

software/slaprunner/template/slapos-slapuser-script.in

-#!/bin/sh
-# run slapos command inside slaprunner with appropriate config
-
-export SLAPOS_CONFIGURATION=${SLAPOS_CONFIGURATION-"{{ config_location }}"}
-export SLAPOS_CLIENT_CONFIGURATION=${SLAPOS_CLIENT_CONFIGURATION-$SLAPOS_CONFIGURATION}
-exec {{ slapos_python_file_location }} "$@"

software/slaprunner/template/slapos.cfg.in

-[slapos]
-software_root = {{ slaprunner['software_root'] }}
-instance_root = {{ slaprunner['instance_root'] }}
-shared_part_list =
-  {#- buildout_shared_part_list_dump is ${buildout:shared-part-list} rendered as a
-  template during software step.
-  Because it can contain new lines, it's not possible to use it directly when generating
-  buildout config files from buildout, because the newlines don't get indented and cause
-  the instance buildout to be invalid ini file.
-  So we had to dump it in a simple text file and now we parse again that text file. -#}
-  {%- for line in codecs.open(buildout_shared_part_list_dump).readlines() %}
-  {{   line.strip() }}
-  {%- endfor %}
-  {{ slaprunner['shared_root'] }}
-master_url = http://{{ slaprunner['ipv4'] }}:{{ slaprunner['proxy_port'] }}
-master_rest_url = http://{{ slaprunner['ipv4'] }}:{{ slaprunner['proxy_port'] }}/hateoas
-computer_id = slaprunner
-maximal_delay = 0
-root_check = {{ slaprunner['root_check'] }}
-forbid_supervisord_automatic_launch = true
-pidfile_software =  {{slaprunner['pidfile-software']}}
-pidfile_instance =  {{slaprunner['pidfile-instance']}}
-
-[slapformat]
-input_definition_file = {{ slaprunner['slapformat-definition.cfg'] }}
-partition_amount = {{ slaprunner['partition-amount'] }}
-alter_user = false
-alter_network = false
-create_tap = false
-create_tun = false
-computer_xml = {{ slaprunner['etc_dir'] }}/slapos.xml
-
-[slaprunner]
-slapos = {{ slaprunner['slapos'] }}
-slapos_cfg = {{ slaprunner['slapos.cfg'] }}
-slapproxy = {{ slaprunner['slapproxy'] }}
-supervisor = {{ slaprunner['supervisor'] }}
-supervisord_config = {{ slaprunner['supervisord_config'] }}
-supervisord_server = {{ slaprunner['supervisord_server'] }}
-runner_workdir = {{ slaprunner['working-directory'] }}
-runner_host = {{ slaprunner['ipv4'] }}
-runner_port = {{ slaprunner['runner_port'] }}
-instance_monitoring_url = {{ slaprunner['instance-monitor-url'] }}
-ipv4_address = {{ slaprunner['ipv4'] }}
-ipv6_address = {{ slaprunner['ipv6'] }}
-etc_dir = {{ slaprunner['etc_dir'] }}
-run_dir = {{ slaprunner['run_dir'] }}
-log_dir = {{ slaprunner['log_dir'] }}
-console = {{ slaprunner['console'] }}
-verbose = {{ slaprunner['verbose'] }}
-debug = {{ slaprunner['debug'] }}
-auto_deploy = {{ slaprunner['auto_deploy'] }}
-auto_deploy_instance = {{ slaprunner['auto_deploy_instance'] }}
-autorun = {{ slaprunner['autorun'] }}
-knowledge0_cfg = {{ slaprunner['knowledge0_file'] }}
-minishell_cwd_file = {{ slaprunner['minishell_cwd_file'] }}
-minishell_history_file = {{ slaprunner['minishell_history_file'] }}
-path = {{ slaprunner['path'] }}
-instance_name = {{ slaprunner['instance_name'] }}
-default_repository = {{ slaprunner['default_repository'] }}
-default_repository_branch = {{ slaprunner['default_repository_branch'] }}
-
-[slapproxy]
-host = {{ slaprunner['ipv4'] }}
-port = {{ slaprunner['proxy_port'] }}
-database_uri = {{ slaprunner['proxy_database'] }}
-
-[gitclient]
-git = {{ slaprunner['git-binary'] }}
-
-[sshkeys_authority]
-public_key = {{ slaprunner['public_key'] }}

software/slaprunner/template/supervisord.conf.in

-[unix_http_server]
-file = {{ supervisord['socket_path'] }}
-
-[supervisorctl]
-serverurl = {{ supervisord['socket_name'] }}
-
-[inet_http_server]
-port = {{ supervisord['server'] }}
-
-[supervisord]
-logfile = {{ supervisord['logfile'] }}
-pidfile = {{ supervisord['pidfile'] }}
-
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[program:{{- supervisord['slapgrid-sr'] -}}]
-command = {{ supervisord['slapgrid-sr-command'] }}
-process_name = {{ supervisord['slapgrid-sr'] }}
-numprocs = {{ supervisord['numprocs'] }}
-autostart = {{ supervisord['autostart'] }}
-exitcodes = {{ supervisord['exitcodes'] }}
-startretries = {{ supervisord['slapgrid-sr-startretries'] }}
-startsecs = {{ supervisord['startsecs'] }}
-autorestart = {{ supervisord['autorestart'] }}
-stdout_logfile = {{ supervisord['no_logfile'] }}
-stderr_logfile = {{ supervisord['no_logfile'] }}
-directory = {{ supervisord['directory'] }}
-{# how many parallel build jobs to spawn when compiling software -#}
-{% set njobs = max(1, (multiprocessing.cpu_count() // int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -%}
-environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{ njobs }}",NPY_NUM_BUILD_JOBS="{{ njobs }}",BUNDLE_JOBS="{{ njobs }}"
-
-[program:{{- supervisord['slapgrid-cp'] -}}]
-command = {{ supervisord['slapgrid-cp-command'] }}
-process_name = {{ supervisord['slapgrid-cp'] }}
-numprocs = {{ supervisord['numprocs'] }}
-autostart = {{ supervisord['autostart'] }}
-exitcodes = {{ supervisord['exitcodes'] }}
-startretries = {{ supervisord['slapgrid-cp-startretries'] }}
-startsecs = {{ supervisord['startsecs'] }}
-autorestart = {{ supervisord['autorestart'] }}
-stdout_logfile = {{ supervisord['no_logfile'] }}
-stderr_logfile = {{ supervisord['no_logfile'] }}
-directory = {{ supervisord['directory'] }}
-environment = PATH="{{- supervisord['path'] -}}"
-
-[program:{{- supervisord['slapformat'] -}}]
-command = {{ supervisord['slapformat-command'] }}
-process_name = {{ supervisord['slapformat'] }}
-numprocs = {{ supervisord['numprocs'] }}
-autostart = {{ supervisord['autostart'] }}
-exitcodes = {{ supervisord['exitcodes'] }}
-startretries = {{ supervisord['slapformat-startretries'] }}
-startsecs = {{ supervisord['startsecs'] }}
-autorestart = {{ supervisord['autorestart'] }}
-stdout_logfile = {{ supervisord['no_logfile'] }}
-stderr_logfile = {{ supervisord['no_logfile'] }}
-directory = {{ supervisord['directory'] }}
-environment = PATH="{{- supervisord['path'] -}}"
-
-[program:{{- supervisord['slapboot'] -}}]
-command = {{ supervisord['slapboot-command'] }}
-process_name = {{ supervisord['slapboot'] }}
-numprocs = {{ supervisord['numprocs'] }}
-autostart = {{ supervisord['autostart'] }}
-exitcodes = {{ supervisord['exitcodes'] }}
-startretries = {{ supervisord['slapboot-startretries'] }}
-startsecs = {{ supervisord['startsecs'] }}
-autorestart = {{ supervisord['autorestart'] }}
-stdout_logfile = {{ supervisord['no_logfile'] }}
-stderr_logfile = {{ supervisord['no_logfile'] }}
-directory = {{ supervisord['directory'] }}
-environment = PATH="{{- supervisord['path'] -}}"
-
-[program:{{- supervisord['slapbang'] -}}]
-command = {{ supervisord['slapbang-command'] }}
-process_name = {{ supervisord['slapbang'] }}
-numprocs = {{ supervisord['numprocs'] }}
-autostart = {{ supervisord['autostart'] }}
-exitcodes = {{ supervisord['exitcodes'] }}
-startretries = {{ supervisord['slapbang-startretries'] }}
-startsecs = {{ supervisord['startsecs'] }}
-autorestart = {{ supervisord['autorestart'] }}
-stdout_logfile = {{ supervisord['no_logfile'] }}
-stderr_logfile = {{ supervisord['no_logfile'] }}
-directory = {{ supervisord['directory'] }}
-environment = PATH="{{- supervisord['path'] -}}"
-
-[program:{{- supervisord['slapproxy'] -}}]
-command = {{ supervisord['slapproxy-command'] }}
-process_name = {{ supervisord['slapproxy'] }}
-numprocs = {{ supervisord['numprocs'] }}
-autostart = {{ supervisord['slapproxy-autostart'] }}
-startsecs = {{ supervisord['slapproxy-startsecs'] }}
-exitcodes = {{ supervisord['exitcodes'] }}
-autorestart = {{ supervisord['slapproxy-autorestart'] }}
-stdout_logfile = {{ supervisord['no_logfile'] }}
-stderr_logfile = {{ supervisord['no_logfile'] }}
-directory = {{ supervisord['directory'] }}
-environment = PATH="{{- supervisord['path'] -}}"
-
-[eventlistener:listener-slapgrid]
-command = {{ listener_slapgrid }}
-events = PROCESS_STATE_EXITED
-stdout_logfile = {{ supervisord['no_logfile'] }}
-stderr_logfile = {{ supervisord['stderr_logfile'] }}

software/slaprunner/test/README.md

-Tests for slaprunner Software Release

software/slaprunner/test/setup.py

-##############################################################################
-#
-# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-from setuptools import setup, find_packages
-
-version = '0.0.1.dev0'
-name = 'slapos.test.slaprunner'
-with open("README.md") as f:
-  long_description = f.read()
-
-setup(name=name,
-      version=version,
-      description="Test for SlapOS' slaprunner",
-      long_description=long_description,
-      long_description_content_type='text/markdown',
-      maintainer="Nexedi",
-      maintainer_email="info@nexedi.com",
-      url="https://lab.nexedi.com/nexedi/slapos",
-      packages=find_packages(),
-      install_requires=[
-          'slapos.core',
-          'slapos.cookbook',
-          'slapos.libnetworkcache',
-          'erp5.util',
-          'supervisor',
-          'psutil',
-          'paramiko',
-          'requests',
-      ],
-      zip_safe=True,
-      test_suite='test',
-      )

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = ca0f353d00131851a992cd3519bec5b0
+md5sum = 4b69822605e3f20fcec8f44a0bdf80bb
 
 [instance]
 _update_hash_filename_ = instance.cfg.in

software/theia/instance-theia.cfg.jinja.in

@@ -14,7 +14,6 @@ theia-environment-parts =
 
 theia-parts =
   frontend-instance
-  frontend-instance-rsyslogd
   python-server
   promises
 
@@ -98,7 +97,6 @@ instance-promises =
   {% if additional_frontend %}
   $${remote-additional-frontend-url-available-promise:name}
   {% endif %}
-  $${frontend-instance-rsyslogd-promise:name}
   $${slapos-standalone-listen-promise:name}
   $${slapos-standalone-ready-promise:name}
   $${slapos-autorun-promise:name}
@@ -153,12 +151,6 @@ config-url = $${remote-additional-frontend:connection-secure_access}
 config-http-code = 401
 {% endif %}
 
-[frontend-instance-rsyslogd-promise]
-<= monitor-promise-base
-promise = check_command_execute
-name = rsyslogd_listen_promise.py
-config-command = test -S $${frontend-instance-rsyslogd-config:log-socket}
-
 [slapos-standalone-listen-promise]
 <= monitor-promise-base
 promise = check_socket_listening
@@ -232,39 +224,6 @@ recipe = slapos.cookbook:generate.password
 username = admin
 storage-path = $${buildout:parts-directory}/.$${:_buildout_section_name_}
 
-[frontend-instance-rsyslogd-config]
-recipe = slapos.recipe.template
-output = $${directory:etc}/$${:_buildout_section_name_}
-log-file = $${directory:log}/frontend-instance.log
-log-socket = $${directory:run}/rsyslog.sock
-pidfile = $${directory:pidfiles}/rsyslogd.pid
-inline =
-  module(
-    load="imuxsock"
-    SysSock.Name="$${:log-socket}")
-
-  # Just simply output the raw line without any additional information, as
-  # haproxy emits enough information by itself
-  # Also cut out first empty space in msg, which is related to rsyslogd
-  # internal and end up cutting on 8k, as it's default of $MaxMessageSize
-  template(name="rawoutput" type="string" string="%msg:2:8192%\n")
-  $ActionFileDefaultTemplate rawoutput
-
-  $FileCreateMode 0600
-  $DirCreateMode 0700
-  $Umask 0022
-
-  $WorkDirectory $${directory:run}
-
-  *.* $${:log-file};rawoutput
-
-
-[frontend-instance-rsyslogd]
-recipe = slapos.cookbook:wrapper
-command-line = ${rsyslogd:location}/sbin/rsyslogd -i $${frontend-instance-rsyslogd-config:pidfile} -n -f $${frontend-instance-rsyslogd-config:output}
-wrapper-path = $${directory:services}/$${:_buildout_section_name_}
-hash-files = $${frontend-instance-rsyslogd-config:output}
-
 [frontend-instance-port]
 recipe = slapos.cookbook:free_port
 minimum = 3000
@@ -278,7 +237,7 @@ command =
   then
     ${openssl-output:openssl} req -x509 -nodes -days 3650 \
       -subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \
-      -newkey rsa:1024 -keyout $${:cert-file} \
+      -newkey rsa:2048 -keyout $${:cert-file} \
       -out $${:cert-file}
   fi
 update-command = $${:command}
@@ -288,34 +247,13 @@ location =
   $${:cert-file}
 
 [frontend-instance-config]
-recipe = slapos.recipe.template
+recipe = slapos.recipe.template:jinja2
+url = ${stack-haproxy-default-backend-config:target}
 output = $${directory:etc}/$${:_buildout_section_name_}
-blankline =
-inline =
-  global
-    maxconn 4096
-    master-worker
-    pidfile $${frontend-instance:pidfile}
-    log $${frontend-instance-rsyslogd-config:log-socket} local0  info
-
-  defaults
-    log global
-    option httplog
-    mode http
-    retries 1
-    option redispatch
-    maxconn 2000
-    balance roundrobin
-    timeout connect 10s
-    timeout queue 60s
-    timeout server 305s
-    timeout client 305s
-
-    # compress some content types
-    compression algo gzip
-    compression type application/font-woff application/font-woff2 application/hal+json application/javascript application/json application/rss+xml application/wasm application/x-font-opentype application/x-font-ttf application/x-javascript application/xml image/svg+xml text/cache-manifest text/css text/html text/javascript text/plain text/xml
-
-
+context =
+  key pidfile frontend-instance:pidfile
+  key content :content
+content =
   userlist basic-auth-list
     user $${frontend-instance-password:username} insecure-password $${frontend-instance-password:passwd}
 
@@ -340,7 +278,6 @@ inline =
   backend static
     log global
     server static_backend $${python-server-port:ip}:$${python-server-port:port}
-  $${:blankline}
 
 ip = $${frontend-instance-port:ip}
 hostname = [$${:ip}]

software/theia/software.cfg

 [buildout]
 extends =
-  ../../component/haproxy/buildout.cfg
-  ../../component/rsyslogd/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/bash/buildout.cfg
   ../../component/bash-completion/buildout.cfg
@@ -14,6 +12,7 @@ extends =
   ../../component/fonts/buildout.cfg
   ../../component/theia/buildout.cfg
   ../../component/pygolang/buildout.cfg
+  ../../stack/haproxy/default-backend.cfg
   ../../stack/slapos.cfg
   ../../stack/monitor/buildout.cfg
   ../../stack/resilient/buildout.cfg

stack/haproxy/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[stack-haproxy-default-backend-config]
+filename = default-backend.cfg.in
+md5sum = dd336cb9ea7dcee94127a994bbe0a2e8

stack/haproxy/default-backend.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../../component/haproxy/buildout.cfg
+
+[stack-haproxy-default-backend-config]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}

stack/haproxy/default-backend.cfg.in

+global
+  maxconn 4096
+  master-worker
+  pidfile {{ pidfile }}
+  log stdout format raw daemon
+
+defaults
+  log global
+  option httplog
+  mode http
+  retries 1
+  option redispatch
+  maxconn 2000
+  balance roundrobin
+  timeout connect 10s
+  timeout queue 60s
+  timeout server 305s
+  timeout client 305s
+
+  compression algo gzip
+  compression type application/font-woff application/font-woff2 application/hal+json application/javascript application/json application/rss+xml application/wasm application/x-font-opentype application/x-font-ttf application/x-javascript application/xml image/svg+xml text/cache-manifest text/css text/html text/javascript text/plain text/xml
+
+{{ content }}
+{# BLANKLINE #}

stack/monitor/README.md

@@ -78,8 +78,8 @@ If you have sub-instances, you should collect the base monitor url from all inst
 Also, all monitors of the sub instances need to have same password as the password of the root instance monitor.
 
 NB: You should use double $ (ex: $${monitor-template:output}) instead of one $ in your instance template file if it's not a jinja template. See:
-- Jinja template file exemple, use one $: https://lab.nexedi.com/nexedi/slapos/blob/master/software/slaprunner/instance-resilient-test.cfg.jinja2
-- Non Jinja template file, use $$: https://lab.nexedi.com/nexedi/slapos/blob/master/software/slaprunner/instance.cfg
+- Jinja template file exemple, use one $: https://lab.nexedi.com/nexedi/slapos/blob/master/software/theia/instance-resilient.cfg.jinja
+- Non Jinja template file, use $$: https://lab.nexedi.com/nexedi/slapos/blob/master/software/theia/instance.cfg.in
 
 Add a promise
 -------------

stack/slapos.cfg

@@ -285,7 +285,7 @@ simplegeneric = 0.8.1
 singledispatch = 3.4.0.3
 six = 1.16.0
 slapos.cookbook = 1.0.297
-slapos.core = 1.8.5
+slapos.core = 1.8.6
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.25
 slapos.rebootstrap = 4.5
@@ -346,6 +346,8 @@ smmap2 = 2.0.5
 traitlets = 4.3.3
 Werkzeug = 1.0.1
 wheel = 0.35.1:whl
+# for slapos.core
+selectors34 = 1.2
 
 [versions:sys.version_info < (3,8)]
 MarkupSafe = 1.0