Update Release Candidate

component/babeld/buildout.cfg

@@ -6,7 +6,7 @@ recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/babeld.git
 branch = master
 git-executable = ${git:location}/bin/git
-revision = hmac-nxd1
+revision = hmac-nxd2
 
 [babeld]
 recipe = slapos.recipe.cmmi

component/dcron/buildout.cfg

@@ -8,6 +8,7 @@ parts = dcron-output
 
 [dcron]
 recipe = slapos.recipe.cmmi
+shared = false
 url = http://www.jimpryor.net/linux/releases/dcron-4.4.tar.gz
 md5sum = 02d848ba043a9df5bf2102a9f4bc04bd
 configure-command = true

component/mosh/buildout.cfg

@@ -9,6 +9,7 @@ extends =
 
 [mosh]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://mosh.org/mosh-1.3.0.tar.gz
 md5sum = d961276995936953bf2d5a794068b076
 configure-options =

component/nano/buildout.cfg

@@ -8,8 +8,10 @@ extends =
 
 [nano]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://www.nano-editor.org/dist/v2.8/nano-2.8.4.tar.xz
 md5sum = 02ff28870194178595b287fc16fa611b
+location = @@LOCATION@@
 # The dummy PKG_CONFIG is in the case that both pkg-config and ncursesw
 # are installed on the system.
 environment=
@@ -17,7 +19,6 @@ environment=
   NCURSESW_CONFIG=${ncurses:location}/bin/ncursesw6-config
   CPPFLAGS=-I${file:location}/include
   LDFLAGS=-L${file:location}/lib/ -Wl,-rpath=${file:location}/lib/
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 post-install =
   cd ${:location} && mkdir etc &&
   echo include "${:location}/share/nano/*.nanorc" > etc/nanorc

component/nginx/buildout.cfg

@@ -12,11 +12,13 @@ parts = nginx-output
 
 [nginx-common]
 recipe = slapos.recipe.cmmi
+shared = false
 url = https://nginx.org/download/nginx-1.17.1.tar.gz
 md5sum = 51021f3e8204a5fc809f5e695a4508db
 
 [nginx]
 <= nginx-common
+shared = true
 configure-options=
   --with-http_ssl_module
   --with-http_v2_module

component/nodejs/buildout.cfg

@@ -15,9 +15,14 @@ parts =
 [nodejs]
 <= nodejs-8.9.4
 
+[nodejs-10.6.0]
+<= nodejs-base
+openssl_location = ${openssl:location}
+version = v10.6.0
+md5sum = 9df233b86244ebda1ded1f91694fbe86
+
 [nodejs-8.9.4]
 <= nodejs-base
-recipe = slapos.recipe.cmmi
 version = v8.9.4
 md5sum = 4ddc1daff327d7e6f63da57fdfc24f55
 
@@ -26,24 +31,27 @@ md5sum = 4ddc1daff327d7e6f63da57fdfc24f55
 version = v8.6.0
 md5sum = 0c95e08220667d8a18b97ecec8218ac6
 
+
 [nodejs-base]
 # Server-side Javascript.
 version =
 md5sum =
+openssl_location = ${openssl-1.0:location}
 
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://nodejs.org/dist/${:version}/node-${:version}.tar.gz
 configure-options =
   --shared-openssl
-  --shared-openssl-includes=${openssl-1.0:location}/include
-  --shared-openssl-libpath=${openssl-1.0:location}/lib
+  --shared-openssl-includes=${:openssl_location}/include
+  --shared-openssl-libpath=${:openssl_location}/lib
 environment =
-  HOME=${buildout:parts-directory}/${:_buildout_section_name_}
+  HOME=@@LOCATION@@
   PATH=${gcc:location}/bin:${pkgconfig:location}/bin:${python2.7:location}/bin/:%(PATH)s
-  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig/
+  PKG_CONFIG_PATH=${:openssl_location}/lib/pkgconfig/
   CPPFLAGS=-I${zlib:location}/include
-  LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -Wl,-rpath=${openssl-1.0:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-  LD_LIBRARY_PATH=${openssl-1.0:location}/lib
+  LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -Wl,-rpath=${:openssl_location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LD_LIBRARY_PATH=${:openssl_location}/lib
 
 [nodejs-8.6.0-output]
 # Shared binary location to ease migration

component/openssh/buildout.cfg

@@ -15,7 +15,9 @@ parts =
 
 [openssh]
 recipe = slapos.recipe.cmmi
+shared = true
 md5sum = 68ba883aff6958297432e5877e9a0fe2
+location = @@LOCATION@@
 url = https://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.7p1.tar.gz
 patch-binary = ${patch:location}/bin/patch
 patch-options = -p1
@@ -25,9 +27,9 @@ environment =
   CPPFLAGS=-I${zlib:location}/include -I${openssl-1.0:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl-1.0:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib
 configure-options = 
-  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
-  --exec-prefix=${buildout:parts-directory}/${:_buildout_section_name_}
-  --with-privsep-path=${buildout:parts-directory}/${:_buildout_section_name_}/var/empty
+  --prefix=${:location}
+  --exec-prefix=${:location}
+  --with-privsep-path=${:location}/var/empty
 
 [openssh-output]
 # Shared binary location to ease migration

component/protobuf/buildout.cfg

@@ -3,10 +3,12 @@ parts = protobuf
 
 [protobuf]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-python-3.4.0.tar.gz
 md5sum = 0820cc2e56d71aef8e99794fcbd184cd
 
 [protobuf-cpp]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
 md5sum = 6d59dad503bea5ad420fd09ddad84481

component/rsync/buildout.cfg

@@ -7,9 +7,10 @@ parts =
 
 [rsync]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://download.samba.org/pub/rsync/src/rsync-3.1.3.tar.gz
 md5sum = 1581a588fde9d89f6bc6201e8129afaf
 make-options =
-  PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
+  PREFIX=@@LOCATION@@
 environment =
   PATH=${perl:location}/bin:%(PATH)s

component/screen/buildout.cfg

@@ -7,6 +7,7 @@ extends =
 
 [screen]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://ftp.gnu.org/gnu/screen/screen-${:version}.tar.gz
 version = 4.5.1
 md5sum = a8c5da2f42f8a18fa4dada2419d1549b

component/texinfo/buildout.cfg

@@ -9,6 +9,7 @@ extends =
 
 [texinfo4]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://ftp.gnu.org/gnu/texinfo/texinfo-4.13.tar.gz
 md5sum = 71ba711519209b5fb583fed2b3d86fcb
 configure-options =

component/tig/buildout.cfg

@@ -7,6 +7,7 @@ extends =
 
 [tig]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://jonas.nitro.dk/tig/releases/tig-2.1.tar.gz
 md5sum = d6c237aba2c03d85897da79789fd6104
 environment =

component/vim/buildout.cfg

@@ -8,6 +8,7 @@ extends =
 
 [vim]
 recipe = slapos.recipe.cmmi
+shared = true
 url = ftp://ftp.vim.org/pub/vim/unix/vim-8.0.586.tar.bz2
 md5sum = b35e794140c196ff59b492b56c1e73db
 environment=

component/zip/buildout.cfg

@@ -3,8 +3,8 @@ parts = zip
 
 [zip]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://downloads.sourceforge.net/project/infozip/Zip%203.x%20%28latest%29/3.0/zip30.tar.gz
 md5sum = 7b74551e63f8ee6aab6fbc86676c0d37
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 configure-command = true
-make-options = prefix=${:location} NO_BZIP2_SUPPORT=1 -f unix/Makefile generic
+make-options = prefix=@@LOCATION@@ NO_BZIP2_SUPPORT=1 -f unix/Makefile generic

software/erp5/instance-erp5-input-schema.json

@@ -4,9 +4,7 @@
   "additionalProperties": false,
   "definitions": {
     "tcpv4port": {
-      "minimum": 0,
-      "maximum": 65535,
-      "type": "integer"
+      "$ref": "./schemas-definitions.json#/tcpv4port"
     }
   },
   "properties": {
@@ -194,16 +192,34 @@
     },
     "kumofs": {
       "description": "Persistent memcached service",
-      "additionalProperties": {
-        "$ref": "./instance-kumofs-schema.json"
-      },
+      "allOf": [
+        {
+          "$ref": "./instance-kumofs-schema.json"
+        },
+        {
+          "properties": {
+            "tcpv4-port": {
+              "default": 2000
+            }
+          }
+        }
+      ],
       "type": "object"
     },
     "memcached": {
       "description": "Volatile memcached service",
-      "additionalProperties": {
-        "$ref": "./instance-kumofs-schema.json"
-      },
+      "allOf": [
+        {
+          "$ref": "./instance-kumofs-schema.json"
+        },
+        {
+          "properties": {
+            "tcpv4-port": {
+              "default": 2010
+            }
+          }
+        }
+      ],
       "type": "object"
     },
     "cloudooo-url": {
@@ -218,16 +234,34 @@
     },
     "smtp": {
       "description": "Mail queuing and relay service",
-      "additionalProperties": {
-        "$ref": "./instance-smtp-schema.json"
-      },
+      "allOf": [
+        {
+          "$ref": "./instance-smtp-schema.json"
+        },
+        {
+          "properties": {
+            "tcpv4-port": {
+              "default": 2010
+            }
+          }
+        }
+      ],
       "type": "object"
     },
     "mariadb": {
       "description": "Relational database service",
-      "additionalProperties": {
-        "$ref": "./instance-mariadb-schema.json"
-      },
+      "allOf": [
+        {
+          "$ref": "./instance-mariadb-schema.json"
+        },
+        {
+          "properties": {
+            "tcpv4-port": {
+              "default": 2099
+            }
+          }
+        }
+      ],
       "type": "object"
     },
     "zodb-zeo": {
@@ -257,7 +291,7 @@
       "type": "object"
     },
     "zodb": {
-      "description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.",
+      "description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/4/src/ZODB/component.xml for extra options.",
       "items": {
         "required": [
           "type"
@@ -282,8 +316,13 @@
                 "type": "boolean"
               }
             },
+            "patternProperties": {
+              ".!$": {
+                "$ref": "#/properties/zodb/items/patternProperties/.!$"
+              }
+            },
             "additionalProperties": {
-              "type": "string"
+              "$ref": "#/properties/zodb/items/additionalProperties"
             },
             "type": "object"
           },
@@ -313,13 +352,38 @@
               },
               "server": {
                 "description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB. Partitions references are 'neo-0', 'neo-1', ...",
-                "$ref": "../neoppod/instance-neo-input-schema.json"
+                "$ref": "../neoppod/instance-neo-input-schema.json#/definitions/neo-cluster"
               }
             }
           }
         ],
+        "patternProperties": {
+          ".!$": {
+            "description": "Override with the value of the first item whose zope id matches against the pattern.",
+            "items": {
+              "items": [
+                {
+                  "description": "Override pattern (Python regular expression).",
+                  "type": "string"
+                },
+                {
+                  "description": "Override value (parameter for maching nodes).",
+                  "type": [
+                    "integer",
+                    "string"
+                  ]
+                }
+              ],
+              "type": "array"
+            },
+            "type": "array"
+          }
+        },
         "additionalProperties": {
-          "type": "string"
+          "type": [
+            "integer",
+            "string"
+          ]
         },
         "type": "object"
       },

software/erp5/instance-kumofs-schema.json

@@ -8,7 +8,7 @@
     "tcpv4-port": {
       "allOf": [
         {
-          "$ref": "#/definitions/tcpv4port"
+          "$ref": "./schemas-definitions.json#/tcpv4port"
         },
         {
           "description": "Start allocating ports at this value, going upward"

software/erp5/instance-mariadb-schema.json

 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "required": [
-    "tcpv4-port"
-  ],
   "type": "object",
   "properties": {
     "tcpv4-port": {
       "allOf": [
         {
-          "$ref": "#/definitions/tcpv4port"
+          "$ref": "./schemas-definitions.json#/tcpv4port"
         },
         {
           "description": "Start allocating ports at this value, going downward"

software/erp5/instance-smtp-schema.json

 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "extends": "./schema-definitions.json#",
   "required": [
     "tcpv4-port"
   ],
@@ -9,7 +8,7 @@
     "tcpv4-port": {
       "allOf": [
         {
-          "$ref": "#/definitions/tcpv4port"
+          "$ref": "./schemas-definitions.json#/tcpv4port"
         },
         {
           "description": "Start allocating ports at this value, going upward"

software/erp5/schemas-definitions.json

+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "tcpv4port": {
+    "minimum": 0,
+    "maximum": 65535,
+    "type": "integer"
+  }
+}

software/erp5/test/test/__init__.py

@@ -46,3 +46,13 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase):
     """Return the output paramters from the root partition"""
     return json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])
+
+  def getComputerPartition(self, partition_reference):
+    for computer_partition in self.slap.computer.getComputerPartitionList():
+      if partition_reference == computer_partition.getInstanceParameter(
+          'instance_title'):
+        return computer_partition
+
+  def getComputerPartitionPath(self, partition_reference):
+    partition_id = self.getComputerPartition(partition_reference).getId()
+    return os.path.join(self.slap._instance_root, partition_id)

software/erp5/test/test/test_erp5.py

@@ -195,8 +195,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     """
     # self.computer_partition_root_path is the path of root partition.
     # we want to assert that no scripts exist in any partition.
-    bin_programs = [os.path.basename(path) for path in
-      glob.glob("{}/../*/bin/*".format(self.computer_partition_root_path))]
+    bin_programs = map(os.path.basename,
+      glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
 
     self.assertTrue(bin_programs) # just to check the glob was correct.
     self.assertNotIn('runUnitTest', bin_programs)
@@ -210,8 +210,99 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     apache_process = psutil.Process(process_info['pid'])
     self.assertEqual(
         sorted([socket.AF_INET, socket.AF_INET6]),
-        sorted([
+        sorted(
             c.family
             for c in apache_process.connections()
             if c.status == 'LISTEN'
-        ]))
+        ))
+
+class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test override zope node parameters
+  """
+  __partition_reference__ = 'override'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    # The following example includes the most commonly used options,
+    # but not necessarily in a meaningful way.
+    return {'_': json.dumps({
+      "zodb": [{
+        "type": "zeo",
+        "server": {},
+        "cache-size-bytes": "20MB",
+        "cache-size-bytes!": [
+          ("bb-0", 1<<20),
+          ("bb-.*", "500MB"),
+        ],
+        "pool-timeout": "10m",
+        "storage-dict": {
+          "cache-size!": [
+            ("a-.*", "50MB"),
+          ],
+        },
+      }],
+      "zope-partition-dict": {
+          "a": {
+              "instance-count": 3,
+          },
+          "bb": {
+              "instance-count": 5,
+              "port-base": 2300,
+          },
+      },
+    })}
+
+  def test_zope_conf(self):
+    zeo_addr = json.loads(
+        self.getComputerPartition('zodb').getConnectionParameter('_')
+      )["storage-dict"]["root"]["server"]
+
+    def checkParameter(line, kw):
+      k, v = line.split()
+      self.assertFalse(k.endswith('!'), k)
+      try:
+        expected = kw.pop(k)
+      except KeyError:
+        if k == 'server':
+          return
+      self.assertIsNotNone(expected)
+      self.assertEqual(str(expected), v)
+
+    def checkConf(zodb, storage):
+      zodb["mount-point"] = "/"
+      zodb["pool-size"] = 4
+      zodb["pool-timeout"] = "10m"
+      storage["storage"] = "root"
+      storage["server"] = zeo_addr
+      with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
+        conf = map(str.strip, f.readlines())
+      i = conf.index("<zodb_db root>") + 1
+      conf = iter(conf[i:conf.index("</zodb_db>", i)])
+      for line in conf:
+        if line == '<zeoclient>':
+          for line in conf:
+            if line == '</zeoclient>':
+              break
+            checkParameter(line, storage)
+          for k, v in storage.iteritems():
+            self.assertIsNone(v, k)
+          del storage
+        else:
+          checkParameter(line, zodb)
+      for k, v in zodb.iteritems():
+        self.assertIsNone(v, k)
+
+    partition = self.getComputerPartitionPath('zope-a')
+    for zope in xrange(3):
+      checkConf({
+          "cache-size-bytes": "20MB",
+        }, {
+          "cache-size": "50MB",
+        })
+    partition = self.getComputerPartitionPath('zope-bb')
+    for zope in xrange(5):
+      checkConf({
+          "cache-size-bytes": "500MB" if zope else 1<<20,
+        }, {
+          "cache-size": None,
+        })

software/erp5testnode/testsuite/deploy-test/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [deploy-script-controller-script]
 filename = deploy-script-controller
-md5sum = f0f5dd379361eb37f84e0bc7639f645f
+md5sum = 8288e59eb442c662544daffbf446a033
 
 [template-deploy-test]
 filename = instance.cfg.in

software/erp5testnode/testsuite/deploy-test/deploy-script-controller

@@ -72,6 +72,8 @@ function add_checks ()
   echo 'iptables-save' >> $LOG_FILE 2>&1
   iptables-save >> $LOG_FILE 2>&1
   for f in /tmp/playbook-* ; do echo $f ; cat $f; echo; done >> $LOG_FILE 2>&1
+  echo 'slapos node status' >> $LOG_FILE 2>&1
+  slapos node status >> $LOG_FILE 2>&1
 }
 function upload ()
 {

software/gitlab/buildout.hash.cfg

@@ -66,7 +66,7 @@ md5sum = a56a44e96f65f5ed20211bb6a54279f4
 
 [nginx-gitlab-http.conf.in]
 _update_hash_filename_ = template/nginx-gitlab-http.conf.in
-md5sum = abcc5eda03e10b26c74619f299a7f6a8
+md5sum  = e74695aa1be60f0ffac64ddbe1c8eaf1
 
 [nginx.conf.in]
 _update_hash_filename_ = template/nginx.conf.in

software/gitlab/template/nginx-gitlab-http.conf.in

@@ -121,6 +121,13 @@ server {
     # Set CORS header
     add_header 'Access-Control-Allow-Origin' {{ cfg('nginx_header_allow_origin') }};
     add_header 'Access-Control-Allow-Credentials' true;
+    if ($request_method = OPTIONS ) {
+        add_header Allow "GET, OPTIONS";
+        add_header Content-Type text/plain;
+        add_header 'Access-Control-Allow-Origin' $http_origin;
+        add_header Access-Control-Allow-Headers "Origin, X-Requested-With, Authorization, Content-Type, Accept";
+        return 200;
+    }
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
     {{ 'gzip off;' if cfg_https else ''}}

software/kvm/buildout.hash.cfg

@@ -47,7 +47,7 @@ md5sum = b617d64de73de1eed518185f310bbc82
 
 [template-nbd]
 filename = instance-nbd.cfg.in
-md5sum = a05b581d65768ac55faf3b06d4aec447
+md5sum = c3667c9bbaa8627dad90f1abd62d2f16
 
 [template-ansible-promise]
 filename = template/ansible-promise.in

software/kvm/instance-kvm-cluster-input-schema.json

@@ -384,8 +384,7 @@
               "title": "NBD hostname or IP",
               "description": "hostname (or IP) of the NBD server containing the boot image.",
               "type": "string",
-              "format": "internet-address",
-              "default": "debian.nbd.vifib.net"
+              "format": "internet-address"
             },
             "nbd-port": {
               "title": "NBD port",

software/kvm/instance-kvm-cluster-simplified-input-schema.json

@@ -64,8 +64,7 @@
               "title": "NBD hostname or IP",
               "description": "hostname (or IP) of the NBD server containing the boot image.",
               "type": "string",
-              "format": "internet-address",
-              "default": "debian.nbd.vifib.net"
+              "format": "internet-address"
             },
             "nbd-port": {
               "title": "NBD port",

software/kvm/instance-kvm-input-schema.json

@@ -172,8 +172,7 @@
       "title": "NBD hostname",
       "description": "hostname (or IP) of the NBD server containing the boot image.",
       "type": "string",
-      "format": "internet-address",
-      "default": "debian.nbd.vifib.net"
+      "format": "internet-address"
     },
     "nbd-port": {
       "title": "NBD port",

software/kvm/instance-nbd.cfg.in

@@ -8,7 +8,7 @@ parts =
   nbd-promise
   onetimeupload-promise
   publish-connection-information
-
+extends = ${monitor-template:output}
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true

software/neoppod/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [instance-common]
 filename = instance-common.cfg.in
-md5sum = 0a3a54fcc7be0bbd63cbd64f006ceebc
+md5sum = 80599fcc6e5d07270d7900aebfd62139
 
 [root-common]
 filename = root-common.cfg.in
-md5sum = ccc6e33412259415ec6c3452d37b77cc
+md5sum = c03fbfc9df9edc1ef60be970e0627c5e
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
@@ -38,7 +38,7 @@ md5sum = 9f6f8f2b5f4cb0d97d50ffc1d3837e2f
 
 [template-neo]
 filename = instance.cfg.in
-md5sum = 83dc9faca482b2ddbd3d5fa968af7c33
+md5sum = 9e63e16eda75e73ad4ffb50afde0505d
 
 [cluster]
 filename = cluster.cfg.in

software/neoppod/instance-common.cfg.in

@@ -15,7 +15,9 @@ cert = ${slap-connection:cert-file}
 recipe = slapos.recipe.template:jinja2
 filename = ${:_buildout_section_name_}.cfg
 rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
-extensions = jinja2.ext.do
+extensions =
+    jinja2.ext.do
+    jinja2.ext.loopcontrols
 extra-context =
 context =
     key ipv4_set slap-configuration:ipv4

software/neoppod/instance-neo-input-schema.json

 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "Parameters to instantiate a NEO cluster. See https://lab.nexedi.com/nexedi/neoppod/blob/master/neo.conf for more information.",
-  "additionalProperties": false,
-  "require": [
-    "cluster"
-  ],
-  "properties": {
-    "cluster": {
-      "description": "Cluster unique identifier. Your last line of defense against mixing up NEO clusters and corrupting your data. Choose a unique value for each of your cluster. Space not allowed.",
-      "type": "string"
-    },
-    "partitions": {
-      "description": "Number of partitions. You cannot change this value once you created a cluster.",
-      "default": 12,
-      "type": "integer"
-    },
-    "replicas": {
-      "description": "Number of replicates.",
-      "default": 0,
-      "type": "integer"
-    },
-    "upstream-cluster": {
-      "description": "Identifier of the cluster to backup.",
-      "type": "string"
-    },
-    "upstream-masters": {
-      "description": "Master nodes in the cluster to backup.",
-      "type": "string"
-    },
-    "monitor": {
-      "description": "Parameters for monitoring.",
+  "definitions": {
+    "neo-cluster": {
+      "additionalProperties": false,
       "properties": {
-        "backup": {
-          "description": "List of cluster names that are expected to backup this cluster.",
-          "type": "array",
-          "items": {
-            "type": "string"
-          },
-          "uniqueItems": true
+        "cluster": {
+          "description": "Cluster unique identifier. Your last line of defense against mixing up NEO clusters and corrupting your data. Choose a unique value for each of your cluster. Space not allowed.",
+          "type": "string"
         },
-        "periodicity": {
-          "description": "Periodicity in minutes at which the cluster health is checked.",
-          "default": 10,
-          "type": "number"
-        }
-      },
-      "type": "object"
-    },
-    "sla-dict": {
-      "description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...). The prefix 'node-' is mandatory and the number must start from 0. The total number of nodes here must be equal to the length of node-list.",
-      "additionalProperties": {
-        "type": "array",
-        "items": {
+        "partitions": {
+          "description": "Number of partitions. You cannot change this value once you created a cluster.",
+          "default": 12,
+          "type": "integer"
+        },
+        "replicas": {
+          "description": "Number of replicates.",
+          "default": 0,
+          "type": "integer"
+        },
+        "upstream-cluster": {
+          "description": "Identifier of the cluster to backup.",
           "type": "string"
         },
-        "uniqueItems": true
-      },
-      "type": "object"
-    },
-    "ssl": {
-      "description": "Enable SSL. All nodes look for 3 files in ~/etc: ca.crt, neo.crt, neo.key. Waiting that SlapOS provides a way to manage certificates, the user must deploy them manually, or use the temporary _ca/_cert/_key parameters.",
-      "default": true,
-      "type": "boolean"
-    },
-    "_ca": {
-      "type": "string"
-    },
-    "_cert": {
-      "type": "string"
-    },
-    "_key": {
-      "type": "string"
-    },
-    "node-list": {
-      "description": "List of dictionaries containing parameters for each node.",
-      "items": {
-        "description": "Dictionary containing parameters required to configure individual nodes.",
-        "default": {},
-        "additionalProperties": false,
-        "properties": {
-          "admin": {
-            "description": "Port of admin node. 0 to disable.",
-            "default": 2050,
-            "type": "integer"
-          },
-          "master": {
-            "description": "Port of master node. 0 to disable.",
-            "default": 2051,
-            "type": "integer"
-          },
-          "storage-count": {
-            "description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage.",
-            "default": 1,
-            "type": "integer"
-          },
-          "data-deduplication": {
-            "description": "Set the --dedup option for storage nodes.",
-            "default": false,
-            "type": "boolean"
-          },
-          "disable-drop-partitions": {
-            "description": "Set the --disable-drop-partitions option for storage nodes.",
-            "default": false,
-            "type": "boolean"
+        "upstream-masters": {
+          "description": "Master nodes in the cluster to backup.",
+          "type": "string"
+        },
+        "monitor": {
+          "description": "Parameters for monitoring.",
+          "properties": {
+            "backup": {
+              "description": "List of cluster names that are expected to backup this cluster.",
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "uniqueItems": true
+            },
+            "periodicity": {
+              "description": "Periodicity in minutes at which the cluster health is checked.",
+              "default": 10,
+              "type": "number"
+            }
           },
-          "storage-type": {
-            "description": "Storage type. Defaults to MySQL if available, else SQLite.",
-            "enum": [
-              "MySQL",
-              "SQLite"
-            ],
-            "type": "string"
+          "type": "object"
+        },
+        "sla-dict": {
+          "description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...). The prefix 'node-' is mandatory and the number must start from 0. The total number of nodes here must be equal to the length of node-list.",
+          "additionalProperties": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "uniqueItems": true
           },
-          "mysql": {
-            "description": "Dictionary containing parameters for MySQL.",
-            "default": {},
-            "properties": {
-              "relaxed-writes": {
-                "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous",
-                "default": false,
-                "type": "boolean"
-              }
+          "type": "object"
+        },
+        "ssl": {
+          "description": "Enable SSL. All nodes look for 3 files in ~/etc: ca.crt, neo.crt, neo.key. Waiting that SlapOS provides a way to manage certificates, the user must deploy them manually, or use the temporary _ca/_cert/_key parameters.",
+          "default": true,
+          "type": "boolean"
+        },
+        "_ca": {
+          "type": "string"
+        },
+        "_cert": {
+          "type": "string"
+        },
+        "_key": {
+          "type": "string"
+        },
+        "node-count": {
+          "description": "Number of nodes to deploy.",
+          "default": 1,
+          "type": "integer"
+        },
+        "node": {
+          "description": "Default node parameters.",
+          "default": {},
+          "additionalProperties": false,
+          "properties": {
+            "admin": {
+              "description": "Port of admin node. 0 to disable.",
+              "default": 2050,
+              "type": "integer"
             },
-            "additionalProperties": {
-              "description": "To configure important parameters like innodb_buffer_pool_size, tokudb_cache_size, etc.",
+            "master": {
+              "description": "Port of master node. 0 to disable.",
+              "default": 2051,
+              "type": "integer"
+            },
+            "storage-count": {
+              "description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage.",
+              "default": 1,
+              "type": "integer"
+            },
+            "data-deduplication": {
+              "description": "Set the --dedup option for storage nodes.",
+              "default": false,
+              "type": "boolean"
+            },
+            "disable-drop-partitions": {
+              "description": "Set the --disable-drop-partitions option for storage nodes.",
+              "default": false,
+              "type": "boolean"
+            },
+            "storage-type": {
+              "description": "Storage type. Defaults to MySQL if available, else SQLite.",
+              "enum": [
+                "MySQL",
+                "SQLite"
+              ],
               "type": "string"
             },
-            "type": "object"
+            "mysql": {
+              "description": "Dictionary containing parameters for MySQL.",
+              "default": {},
+              "properties": {
+                "relaxed-writes": {
+                  "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous",
+                  "default": false,
+                  "type": "boolean"
+                }
+              },
+              "additionalProperties": {
+                "description": "To configure important parameters like innodb_buffer_pool_size, tokudb_cache_size, etc.",
+                "type": "string"
+              },
+              "type": "object"
+            },
+            "engine": {
+              "description": "Configures storage engine, currently only InnoDB, RocksDB and TokuDB are supported. Defaults to NEO's default.",
+              "type": "string"
+            }
           },
-          "engine": {
-            "description": "Configures storage engine, currently only InnoDB, RocksDB and TokuDB are supported. Defaults to NEO's default.",
-            "type": "string"
-          }
+          "type": "object"
         },
-        "type": "object"
+        "node!": {
+          "description": "Node parameters are overridden by the value of the first item whose partition reference matches against the pattern.",
+          "items": {
+            "items": [
+              {
+                "description": "Override pattern (Python regular expression).",
+                "type": "string"
+              },
+              {
+                "allOf": [
+                  {
+                    "$ref": "#/definitions/neo-cluster/properties/node"
+                  },
+                  {
+                    "description": "Override value (parameters for maching nodes)."
+                  }
+                ]
+              }
+            ],
+            "type": "array"
+          },
+          "type": "array"
+        }
       },
-      "type": "array"
+      "type": "object"
     }
   },
-  "type": "object"
+  "allOf": [
+    {
+      "$ref": "#/definitions/neo-cluster"
+    },
+    {
+      "required": [
+        "cluster"
+      ]
+    }
+  ]
 }

software/neoppod/instance.cfg.in

@@ -8,6 +8,7 @@ rendered = ${buildout:parts-directory}/${:_buildout_section_name_}.cfg
 <= jinja2-template-base
 template = {{ cluster }}
 extra-context =
+    import re re
     import urlparse urlparse
 import-list =
     rawfile root_common {{ root_common }}

software/neoppod/root-common.cfg.in

@@ -16,6 +16,20 @@ sla-computer_guid = ${slap-connection:computer-id}
 {% endif -%}
 {% endmacro -%}
 
+{% macro apply_overrides(dict_, reference) -%}
+{% for key in list(dict_) -%}
+{%   if key.endswith('!') -%}
+{%     for pattern, value in dict_.pop(key, ()) -%}
+{%       set m = re.match(pattern, reference) -%}
+{%       if m and m.group() == reference %}{# PY3: fullmatch -#}
+{%         do dict_.__setitem__(key[:-1], value) -%}
+{%         break -%}
+{%       endif -%}
+{%     endfor -%}
+{%   endif -%}
+{% endfor -%}
+{% endmacro -%}
+
 {% macro common_section() -%}
 [request-common-base]
 recipe = slapos.cookbook:request.serialised
@@ -52,7 +66,22 @@ config-ssl = {{ dumps((
 config-upstream-cluster = {{ dumps(parameter_dict.get('upstream-cluster', '')) }}
 config-upstream-masters = {{ dumps(parameter_dict.get('upstream-masters', '')) }}
 software-type = {{ software_type }}
-{% set node_list = parameter_dict.get('node-list', ({},)) -%}
+
+{% set node_list = parameter_dict.get('node-list') -%}
+{% if node_list == None -%}
+{%   set node_list = [] -%}
+{%   for i in range(parameter_dict.get('node-count', 1)) -%}
+{%     set x = parameter_dict.copy() -%}
+{%     do apply_overrides(x, prefix ~ i) -%}
+{%     do node_list.append(x.get('node', {})) -%}
+{%   endfor -%}
+{% else %}{# BBB -#}
+{%   do assert('node-count' not in parameter_dict) -%}
+{%   do assert('node' not in parameter_dict) -%}
+{%   do assert('node!' not in parameter_dict) -%}
+{% endif -%}
+{% do assert(node_list) -%}
+
 {% set storage_count = [] -%}
 {% for node in node_list -%}
 {%   do storage_count.append(node.get('storage-count', 1)) -%}

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 3d12778d8dc910180ff3a7fcae8b37f9
+md5sum = 4664f7dae66d3f582e34cec2ca627501

software/slapos-sr-testing/instance.cfg

@@ -28,7 +28,7 @@ bin = $${buildout:directory}/bin
 working-dir = $${buildout:directory}/tmp
 
 [test-list]
-path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup}
+path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup}
 
 [slapos-test-runner]
 recipe = slapos.cookbook:wrapper

software/slapos-sr-testing/software.cfg

@@ -102,6 +102,11 @@ setup = ${slapos-repository:location}/software/nextcloud/test/
 egg = slapos.test.nextcloud
 setup = ${slapos-repository:location}/software/turnserver/test/
 
+[slapos.test.theia-setup]
+<= setup-develop-egg
+egg = slapos.test.theia
+setup = ${slapos-repository:location}/software/theia/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git

software/theia/README.md

+# Theia software release
+
+Theia is a cloud (and desktop) IDE https://www.theia-ide.org
+
+This version comes pre-configured with a few plugins, but does not come with python plugin, to let
+you choose between theia and vscode one.
+
+## Installing vscode extensions
+
+To install vscode's python plugin ( https://code.visualstudio.com/docs/languages/python ):
+
+<key>F1</key> `Deploy plugin by id` and enter `vscode:extension/ms-python.python` and wait for a while.
+Check theia's log files in case of issues.
+
+see https://github.com/theia-ide/theia/wiki/Testing-VS-Code-extensions for more details.
+
+## jedi
+
+[jedi](https://github.com/davidhalter/jedi) which is used by both thiea and vscode python plugins has
+some support for `zc.buildout`. It looks up for a `buildout.cfg` file and if found will load all scripts
+from the bin directory from this buildout to add eggs to sys.path. In webrunner we have almost 100 scripts
+in bin directory, with maybe 30 eggs in each scripts, so this makes jedi so slow it's unusable. Also, if
+an error occurs parsing these scripts, jedi won't be usable. This issue is tracked in
+https://github.com/davidhalter/jedi/issues/1325
+
+A simple workaround is to create and empty `buildout.cfg` file at the root of project folder.
\ No newline at end of file

software/theia/buildout.hash.cfg

@@ -15,12 +15,12 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 6a0a65ff9f81f907d53c42c65b54442a
+md5sum = 7c9444fbe8dc8faea67ede2b77e188ed
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = fc2f20d496911241a0d9001c445d6158
+md5sum = b63a993a13e7c3b16b66c25fac5ac8b9
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt
-md5sum = 242c8359be6893725ad2ac8e3c864eb0
+md5sum = ce87c9818d64a16fce49c84ed5dba8fc

software/theia/instance.cfg.in

@@ -2,14 +2,13 @@
 parts =
   monitor-base
   promises
-  .bashrc
   frontend-reload
   publish-connection-parameter
 
 extends = ${monitor-template:rendered}
 
 eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory} 
+develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 
@@ -79,37 +78,15 @@ recipe = slapos.cookbook:userinfo
 recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:services}/$${:_buildout_section_name_}
 command-line =
-  env -i HOME=$${directory:home} LC_ALL=C.UTF-8 USER=$${user:pw-name} LOGNAME=$${user:pw-name} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port}
+  env LC_ALL=C.UTF-8 TMP=$${directory:tmp} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port}
 
 ip =  $${instance-parameter:ipv4-random}
 hostname =  $${:ip}
 port = 3000
 base-url = http://$${:hostname}:$${:port}/
-
-[.bashrc]
-recipe = slapos.recipe.template:jinja2
-rendered = $${directory:home}/$${:_buildout_section_name_}
-template = inline:
-  export PS1="$ " # because we are in a gowork workspace
-  # XXX .bash_profile is not executed, so we introduce a bashrc.theia file
-  # to allow customizations.
-  if [ -f "$HOME/.bashrc.theia" ] ; then
-    source $HOME/.bashrc.theia
-  fi
-  # If not running interactively, don't do anything
-  case $- in
-      *i*) ;;
-        *) return;;
-  esac
-  # enable color support
-  if [ -x /usr/bin/dircolors ]; then
-      test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
-      alias ls='ls --color=auto'
-  fi
-  # common ls aliases
-  alias ll='ls -l'
-  alias la='ls -A'
-  alias l='ls -CF'
+hash-existing-files =
+  ${yarn.lock:output}
+  ${theia-wrapper:rendered}
 
 [promises]
 recipe =
@@ -156,4 +133,4 @@ pidfiles = $${:var}/run
 services = $${:etc}/service
 framebuffer = $${:srv}/framebuffer
 fonts = $${:srv}/fonts
-home = $${:srv}/home
+home = $${:srv}/home
\ No newline at end of file

software/theia/python-language-server-requirements.txt

@@ -20,4 +20,4 @@ snowballstemmer==1.2.1
 toml==0.10.0
 typed-ast==1.2.0
 typeshed==0.0.1
-yapf==0.25.0
+yapf==0.28.0

software/theia/software.cfg

@@ -24,7 +24,7 @@ parts =
 
 
 [nodejs]
-<= nodejs-8.9.4
+<= nodejs-10.6.0
 
 [python3]
 <= python3.6.6
@@ -34,10 +34,10 @@ parts =
 # this could become a component, but it needs to be invoked from nodejs explicitly,
 # otherwise it uses system's nodejs
 # XXX why don't we build a wrapper ?
-version = 1.11.0
+version = 1.17.3
 recipe = slapos.recipe.build:download-unpacked
 url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
-md5sum = d4f05075f534dd9a0a8c18c650b55f0d
+md5sum = 4a02e1687a150113ad6b0215f9afdb3e
 
 
 [python-language-server]
@@ -60,8 +60,9 @@ mode = 0644
 [theia]
 recipe = plone.recipe.command
 command = ${bash:location}/bin/bash -c "
-  export PATH=${nodejs:location}/bin/:${python2.7:location}/bin/:$PATH &&
+  export TMPDIR=${:location}/tmp PATH=${nodejs:location}/bin/:${python2.7:location}/bin/:$PATH &&
   mkdir -p ${:location} && \
+  mkdir -p \$TMPDIR && \
   cd ${:location} && \
   cp ${package.json:rendered} . &&
   cp ${yarn.lock:output} . &&
@@ -69,6 +70,7 @@ command = ${bash:location}/bin/bash -c "
   ${yarn:location}/bin/yarn theia build"
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 stop-on-error = true
+uses = ${yarn.lock:recipe}
 
 [yarn.lock]
 recipe = slapos.recipe.template
@@ -78,59 +80,91 @@ mode = 0644
 
 [package.json]
 recipe = slapos.recipe.template:jinja2
-# this comes from https://github.com/theia-ide/theia-apps/blob/598d4dc9d4f9e0514869273c069f734a41f20207/theia-full-docker/next.package.json
+# this comes from https://github.com/theia-ide/theia-apps/blob/def6d29a5fb3739aebc17c6a5b67f1b5c16caacd/theia-full-docker/next.package.json
 # but "@theia/php": "next" was removed, because we don't have php/composer component so installation failed
 template =
   inline:
-  {
-    "private": true,
-    "dependencies": {
-      "typescript": "latest",
-      "@theia/callhierarchy": "next",
-      "@theia/core": "next",
-      "@theia/cpp": "next",
-      "@theia/docker": "next",
-      "@theia/editor": "next",
-      "@theia/editorconfig": "next",
-      "@theia/extension-manager": "next",
-      "@theia/file-search": "next",
-      "@theia/filesystem": "next",
-      "@theia/git": "next",
-      "@theia/go": "next",
-      "@theia/java": "next",
-      "@theia/json": "next",
-      "@theia/keymaps": "next",
-      "@theia/languages": "next",
-      "@theia/markers": "next",
-      "@theia/merge-conflicts": "next",
-      "@theia/messages": "next",
-      "@theia/metrics": "next",
-      "@theia/mini-browser": "next",
-      "@theia/monaco": "next",
-      "@theia/navigator": "next",
-      "@theia/outline-view": "next",
-      "@theia/output": "next",
-      "@theia/plantuml": "next",
-      "@theia/preferences": "next",
-      "@theia/preview": "next",
-      "@theia/process": "next",
-      "@theia/python": "next",
-      "@theia/ruby": "next",
-      "@theia/rust": "next",
-      "@theia/search-in-workspace": "next",
-      "@theia/task": "next",
-      "@theia/terminal": "next",
-      "@theia/textmate-grammars": "next",
-      "@theia/tslint": "next",
-      "@theia/typescript": "next",
-      "@theia/userstorage": "next",
-      "@theia/variable-resolver": "next",
-      "@theia/workspace": "next",
-      "theia-yang-extension": "next"
-    },
-    "devDependencies": {
-      "@theia/cli": "next"
-    }
+    {
+      "private": true,
+      "theia": {
+          "frontend": {
+              "config": {
+                  "applicationName": "Theia SlapOS",
+                  "preferences": {
+                      "files.enableTrash": false,
+                      "files.exclude": {
+                          "**.pyc": true,
+                          "**.egg-info": true,
+                          "__pycache__": true,
+                          ".git": true,
+                          ".env": true
+                      },
+                      "files.watcherExclude": {
+                        "**/.eggs/**": true,
+                        "**/.env/**": true,
+                        "**/.git/**": true,
+                        "**/node_modules/**": true
+                      },
+                      "editor.multiCursorModifier": "ctrlCmd",
+                      "plantuml.webservice": "//plantuml.host.vifib.net/svg/",
+                      "plantuml.monochrome": false,
+                      "editor.tabSize": 2
+                  }
+              }
+          }
+      },
+      "dependencies": {
+          "@theia/callhierarchy": "next",
+          "@theia/core": "next",
+          "@theia/cpp": "next",
+          "@theia/debug": "next",
+          "@theia/debug-nodejs": "next",
+          "@theia/docker": "next",
+          "@theia/editor": "next",
+          "@theia/editorconfig": "next",
+          "@theia/editor-preview": "next",
+          "@theia/file-search": "next",
+          "@theia/filesystem": "next",
+          "@theia/git": "next",
+          "@theia/go": "next",
+          "@theia/getting-started": "next",
+          "@theia/java": "next",
+          "@theia/json": "next",
+          "@theia/keymaps": "next",
+          "@theia/languages": "next",
+          "@theia/markers": "next",
+          "@theia/merge-conflicts": "next",
+          "@theia/messages": "next",
+          "@theia/metrics": "next",
+          "@theia/mini-browser": "next",
+          "@theia/monaco": "next",
+          "@theia/navigator": "next",
+          "@theia/outline-view": "next",
+          "@theia/output": "next",
+          "@theia/plantuml": "next",
+          "@theia/plugin": "next",
+          "@theia/plugin-ext": "next",
+          "@theia/plugin-ext-vscode": "next",
+          "@theia/preferences": "next",
+          "@theia/preview": "next",
+          "@theia/process": "next",
+          "@theia/ruby": "next",
+          "@theia/rust": "next",
+          "@theia/search-in-workspace": "next",
+          "@theia/task": "next",
+          "@theia/terminal": "next",
+          "@theia/textmate-grammars": "next",
+          "@theia/tslint": "next",
+          "@theia/typescript": "next",
+          "@theia/userstorage": "next",
+          "@theia/variable-resolver": "next",
+          "@theia/workspace": "next",
+          "theia-yang-extension": "next",
+          "typescript": "latest"
+      },
+      "devDependencies": {
+          "@theia/cli": "next"
+      }
   }
 rendered = ${buildout:directory}/${:_buildout_section_name_}
 mode = 0644
@@ -169,6 +203,8 @@ template =
   export PATH=${nodejs:location}/bin/:${python-language-server:location}/bin/:${bash:location}/bin/:${git:location}/bin/:$PATH
   . ${gowork:env.sh}
   export SHELL=bash
+  # reset PS1 from gowork
+  export PS1='$ '
   cd ${theia:location}
   exec ${yarn:location}/bin/yarn theia start $@
 

software/theia/test/README.md

+Tests for Theia software release

software/theia/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.theia'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' Theia",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'supervisor',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/theia/test/test.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import textwrap
+import logging
+import tempfile
+import time
+from six.moves.urllib.parse import urlparse
+
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestTheia(SlapOSInstanceTestCase):
+  def setUp(self):
+    self.theia_url = self.computer_partition.getConnectionParameterDict(
+    )['url']
+
+  def test_http_get(self):
+    resp = requests.get(self.theia_url, verify=False)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+
+    # without login/password, this is unauthorized
+    parsed_url = urlparse(self.theia_url)
+    resp = requests.get(
+        parsed_url._replace(
+            netloc='[{}]:{}'.format(
+                parsed_url.hostname,
+                parsed_url.port)).geturl(),
+        verify=False)
+    self.assertEqual(requests.codes.unauthorized, resp.status_code)

stack/erp5/buildout.hash.cfg

@@ -34,7 +34,7 @@ md5sum = e91c0fbd0df441884f7422fa7976053c
 
 [template-zope-conf]
 filename = zope.conf.in
-md5sum = 114e0ac43281b943931754ed317ebc36
+md5sum = 762897486b1e7e28b614224a9a577125
 
 [site-zcml]
 filename = site.zcml
@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
 
 [template]
 filename = instance.cfg.in
-md5sum = f6c6820f9b3653d0d5c29708606fc591
+md5sum = 520b6bf3461dddc9c8b862e50b14465d
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -86,7 +86,7 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = b1685783f4c93da918ccc83702559e6f
+md5sum = 8b4a15dca7e30ba5a792f1a9622216b0
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-zope.cfg.in

@@ -267,6 +267,11 @@ timerserver-interval = {{ dumps(timerserver_interval) }}
 [zope-conf-base]
 < = jinja2-template-base
 template = {{ parameter_dict['zope-conf-template'] }}
+extensions =
+    jinja2.ext.do
+    jinja2.ext.loopcontrols
+import-list =
+    rawfile root_common {{ root_common }}
 
 {% macro zope(
   index,
@@ -312,10 +317,10 @@ longrequest-logger-file =
 [{{ conf_name }}]
 < = zope-conf-base
 rendered = ${directory:etc}/{{ name }}.conf
-extensions = jinja2.ext.do
 context =
   section parameter_dict {{ conf_parameter_name }}
   import os os
+  import re re
 
 [{{ section(name) }}]
 < = runzope-base

stack/erp5/instance.cfg.in

@@ -46,6 +46,7 @@ extra-context =
     key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     key openssl_location :openssl-location
+    import re re
     import urlparse urlparse
 import-list =
     file root_common context:root-common
@@ -111,6 +112,7 @@ template = {{ template_zope }}
 filename = instance-zope.cfg
 extra-context =
     key buildout_directory buildout:directory
+    key root_common context:root-common
     section parameter_dict dynamic-template-zope-parameters
     import urlparse urlparse
     import hashlib hashlib

stack/erp5/zope.conf.in

+{% set slapparameter_dict = {} %}{# dummy -#}
+{% import "root_common" as root_common with context -%}
+{% set node_id = parameter_dict['node-id'] -%}
 # Note: Environment is setup in running wrapper script, as zope.conf is read
 # too late for some components.
 %define INSTANCE {{ parameter_dict['instance'] }}
@@ -65,7 +68,7 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
 
 {% endif -%}
 <product-config CMFActivity>
-  node-id {{ parameter_dict['node-id'] }}
+  node-id {{ node_id }}
 </product-config>
 
 {% set timerserver_interval = parameter_dict['timerserver-interval'] -%}
@@ -145,13 +148,15 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
 <zodb_db {{ name }}>
 {%- set storage_type = type_dict[zodb_dict.pop('type')] %}
 {%- set storage_dict = zodb_dict.pop('storage-dict') %}
+{%- do root_common.apply_overrides(zodb_dict, node_id) %}
 {%- for key, value in zodb_dict.iteritems() %}
   {{ key }} {{ value }}
-{%-   endfor %}
+{%- endfor %}
   <{{ storage_type }}>
-{%-   for key, value in storage_dict.iteritems() %}
+{%- do root_common.apply_overrides(storage_dict, node_id) %}
+{%- for key, value in storage_dict.iteritems() %}
     {{ key }} {{ value }}
-{%-   endfor %}
+{%- endfor %}
   </{{ storage_type }}>
 </zodb_db>
 

stack/slapos.cfg

@@ -140,11 +140,11 @@ slapos.cookbook = 1.0.123
 slapos.core = 1.5.4
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
-slapos.libnetworkcache = 0.19
+slapos.libnetworkcache = 0.20
 slapos.rebootstrap = 4.2
 slapos.recipe.build = 0.42
-slapos.recipe.cmmi = 0.11
-slapos.toolbox = 0.98
+slapos.recipe.cmmi = 0.12
+slapos.toolbox = 0.100
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1