Update Release Candidate

component/liburing/buildout.cfg

@@ -5,5 +5,5 @@ parts =
 [liburing]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://github.com/axboe/liburing/archive/refs/tags/liburing-2.0.tar.gz
-md5sum = 022bb540e8ab5c9916609145f020926f
+url = https://github.com/axboe/liburing/archive/refs/tags/liburing-2.3.tar.gz
+md5sum = 2e8c3c23795415475654346484f5c4b8

software/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = e1dd16a6f50468959b5c4572b8c82f23
+md5sum = cfb3bf67b11e5b1278d94f7e729d740c
 
 [json-test-template]
 _update_hash_filename_ = json-test-template.json.in.jinja2
@@ -26,7 +26,7 @@ md5sum = 6b933beb0744d97c7760e4601298e137
 
 [template-node-monitoring]
 _update_hash_filename_ = instance-node-monitoring.jinja2.cfg
-md5sum = 2d8bd1224472983e54f36770d3e3f969
+md5sum = 6dbc34d9052989225ada3b2a2d0b588c
 
 [network-bench-cfg]
 filename = network_bench.cfg.in

software/monitor/instance-default-input-schema.json

@@ -146,6 +146,21 @@
       "title": "Boolean to display prediction (unit: N/A)",
       "description": "Enable prediction display by setting boolean to True (unit: N/A)",
       "type": "boolean"
+    },
+    "promise_re6stnet_config_directory": {
+      "default": "/etc/re6stnet/",
+      "title": "Directory of re6stnet configuration on the node",
+      "type": "string"
+    },
+    "promise_re6stnet_certificate_file": {
+      "default": "cert.crt",
+      "title": "Filename of the re6stnet certificate in the re6stnet directory",
+      "type": "string"
+    },
+    "re6stnet_certificate_expiration_delay": {
+      "default": 15,
+      "title": "Days before expiration until certificate is considered valid",
+      "type": "number"
     }
   }
 }

software/monitor/instance-node-monitoring.jinja2.cfg

@@ -8,6 +8,7 @@ parts =
   check-network-errors.py
   check-network-transit.py
   check-cpu-load.py
+  check-re6stnet-certificate.py
   publish-connection-information
 
 extends = {{ monitor_template }}
@@ -105,3 +106,15 @@ config-last-transit-file = ${directory:var}/promise_network_last_transit_file
 promise = check_server_cpu_load
 config-frequency = {{ slapparameter_dict.get("promise_cpu_load_frequency", 3) }}
 config-cpu-load-threshold = {{ slapparameter_dict.get("promise_cpu_load_threshold", 1.5) }}
+
+[check-re6stnet-certificate.py]
+<= macro.promise
+{% set RE6STNET_CONFIG_DIR = slapparameter_dict.get('promise_re6stnet_config_directory', '/etc/re6stnet') %}
+{% if os_module.path.exists(os_module.path.join(RE6STNET_CONFIG_DIR, 're6stnet.conf')) %}
+promise = check_certificate
+config-certificate = {{ os_module.path.join(RE6STNET_CONFIG_DIR, slapparameter_dict.get('promise_re6stnet_certificate_file', 'cert.crt')) }}
+config-certificate-expiration-days = {{ slapparameter_dict.get('re6stnet_certificate_expiration_delay', '15') }}
+{% else %}
+promise = check_command_execute
+config-command = echo "re6stnet disabled on the node"
+{% endif %}

software/monitor/instance.cfg

@@ -30,6 +30,7 @@ context =
 <= instance-template
 url = ${template-node-monitoring:target}
 extra-context =
+    import os_module os
     raw buildout_directory ${buildout:directory}
     section slap_connection slap-connection
 

software/monitor/test/test.py

@@ -25,16 +25,26 @@
 #
 ##############################################################################
 
+import datetime
 import glob
 import hashlib
 import json
 import os
 import re
 import requests
+import shutil
 import subprocess
+import tempfile
 import xml.etree.ElementTree as ET
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509.oid import NameOID
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.util import bytes2str
 
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
@@ -571,3 +581,131 @@ class TestNodeMonitoring(SlapOSInstanceTestCase):
 
   def test_node_monitoring_instance(self):
     pass
+
+
+class TestNodeMonitoringRe6stCertificate(SlapOSInstanceTestCase):
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'default'
+
+  def reRequestInstance(self, partition_parameter_kw=None, state='started'):
+    if partition_parameter_kw is None:
+      partition_parameter_kw = {}
+    software_url = self.getSoftwareURL()
+    software_type = self.getInstanceSoftwareType()
+    return self.slap.request(
+        software_release=software_url,
+        software_type=software_type,
+        partition_reference=self.default_partition_reference,
+        partition_parameter_kw=partition_parameter_kw,
+        state=state)
+
+  def test_default(self):
+    self.reRequestInstance()
+    self.slap.waitForInstance()
+    promise = os.path.join(
+      self.computer_partition_root_path, 'etc', 'plugin',
+      'check-re6stnet-certificate.py')
+    self.assertTrue(os.path.exists(promise))
+    with open(promise) as fh:
+      promise_content = fh.read()
+    # this test depends on OS level configuration
+    if os.path.exists('/etc/re6stnet/cert.crt'):
+      self.assertIn(
+        "extra_config_dict = {'certificate': '/etc/re6stnet/cert.crt', "
+        "'certificate-expiration-days': '15'}", promise_content)
+      self.assertIn(
+        "from slapos.promise.plugin.check_certificate import RunPromise",
+        promise_content)
+    else:
+      self.assertIn(
+        "extra_config_dict = {'command': 'echo \"re6stnet disabled on the "
+        "node\"'}", promise_content)
+      self.assertIn(
+        "from slapos.promise.plugin.check_command_execute import RunPromise",
+        promise_content)
+
+  def createKey(self):
+    key = rsa.generate_private_key(
+      public_exponent=65537, key_size=2048, backend=default_backend())
+    key_pem = key.private_bytes(
+      encoding=serialization.Encoding.PEM,
+      format=serialization.PrivateFormat.TraditionalOpenSSL,
+      encryption_algorithm=serialization.NoEncryption()
+    )
+    return key, key_pem
+
+  def createCertificate(self, key, days=30):
+    subject = issuer = x509.Name([
+      x509.NameAttribute(NameOID.COUNTRY_NAME, u"FR"),
+      x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, u"Nord"),
+      x509.NameAttribute(NameOID.LOCALITY_NAME, u"Lille"),
+      x509.NameAttribute(NameOID.ORGANIZATION_NAME, u"Nexedi"),
+      x509.NameAttribute(NameOID.COMMON_NAME, u"Common"),
+    ])
+    certificate = x509.CertificateBuilder().subject_name(
+        subject
+    ).issuer_name(
+        issuer
+    ).public_key(
+        key.public_key()
+    ).serial_number(
+        x509.random_serial_number()
+    ).not_valid_before(
+        datetime.datetime.utcnow()
+    ).not_valid_after(
+        datetime.datetime.utcnow() + datetime.timedelta(days)
+    ).sign(key, hashes.SHA256(), default_backend())
+    certificate_pem = certificate.public_bytes(
+      encoding=serialization.Encoding.PEM)
+    return certificate, certificate_pem
+
+  def createKeyCertificate(self, certificate_path):
+    key, key_pem = self.createKey()
+    certificate, certificate_pem = self.createCertificate(key, 30)
+    with open(certificate_path, 'w') as fh:
+      fh.write(bytes2str(key_pem))
+    with open(certificate_path, 'a') as fh:
+      fh.write(bytes2str(certificate_pem))
+
+  def setUp(self):
+    super().setUp()
+    self.re6st_dir = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, self.re6st_dir)
+
+  def test_re6st_dir(self, days=None, filename='cert.crt'):
+    self.createKeyCertificate(os.path.join(self.re6st_dir, filename))
+    with open(os.path.join(self.re6st_dir, 're6stnet.conf'), 'w') as fh:
+      fh.write("")
+    partition_parameter_kw = {
+      'promise_re6stnet_config_directory': self.re6st_dir
+    }
+    if filename != 'cert.crt':
+      partition_parameter_kw['promise_re6stnet_certificate_file'] = filename
+    if days is not None:
+      partition_parameter_kw['re6stnet_certificate_expiration_delay'] = days
+    self.reRequestInstance(
+      partition_parameter_kw={'_': json.dumps(partition_parameter_kw)})
+    self.slap.waitForInstance()
+    promise = os.path.join(
+      self.computer_partition_root_path, 'etc', 'plugin',
+      'check-re6stnet-certificate.py')
+    self.assertTrue(os.path.exists(promise))
+    with open(promise) as fh:
+      promise_content = fh.read()
+    self.assertIn(
+      """extra_config_dict = { 'certificate': '%(re6st_dir)s/%(filename)s',
+  'certificate-expiration-days': '%(days)s'}""" % {
+       're6st_dir': self.re6st_dir,
+       'days': days or 15,
+       'filename': filename},
+      promise_content)
+    self.assertIn(
+      "from slapos.promise.plugin.check_certificate import RunPromise",
+      promise_content)
+
+  def test_re6st_dir_expiration(self):
+    self.test_re6st_dir(days=10)
+
+  def test_re6st_dir_filename(self):
+    self.test_re6st_dir(filename="cert.pem")

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 3648844f372a96974582e7281c9987dd
+md5sum = a9d4ace568acdd5002d587816ab91737
 
 [instance]
 _update_hash_filename_ = instance.cfg.in

software/theia/instance-theia.cfg.jinja.in

@@ -15,7 +15,6 @@ theia-environment-parts =
 
 theia-parts =
   frontend-instance
-  python-server
   promises
 
 parts =
@@ -123,8 +122,7 @@ config-port = $${frontend-instance:port}
 <= monitor-promise-base
 promise = check_socket_listening
 name = $${:_buildout_section_name_}.py
-config-host = $${python-server-port:ip}
-config-port = $${python-server-port:port}
+config-pathname = $${python-server:socket}
 
 [frontend-authentication-promise]
 <= monitor-promise-base
@@ -262,13 +260,14 @@ content =
     log global
     bind $${:ip}:$${:port} ssl crt $${frontend-instance-certificate:cert-file} alpn h2,http/1.1
     # writing twice the same ACL is doing OR
+    acl is_public path_beg /public/
     acl is_public path /$${frontend-instance-favicon.ico:filename}
     acl is_public path /$${frontend-instance-theia.webmanifest:filename}
     acl is_public path /$${frontend-instance-theia-serviceworker.js:filename}
     acl auth_ok http_auth(basic-auth-list)
-    # No authentication for some files
+    # No authentication for public folder
     http-request auth unless auth_ok || is_public
-    use_backend static if { path_beg /$${frontend-instance-fonts:folder-name} } || { path_beg /$${frontend-instance-slapos.css:folder-name} } || { path /$${frontend-instance-logo:filename} } || { path_beg /public/ } || is_public
+    use_backend static if { path_beg /$${frontend-instance-fonts:folder-name} } || { path_beg /$${frontend-instance-slapos.css:folder-name} } || { path /$${frontend-instance-logo:filename} } || is_public
     default_backend nodejs
 
   backend nodejs
@@ -277,7 +276,9 @@ content =
 
   backend static
     log global
-    server static_backend $${python-server-port:ip}:$${python-server-port:port}
+    server static_backend $${python-server:socket}
+    option forwardfor
+    http-response set-header Content-Security-Policy "default-src 'self'; img-src 'self' data:; script-src 'none'"
 
 ip = $${frontend-instance-port:ip}
 hostname = [$${:ip}]
@@ -387,17 +388,28 @@ filename = favicon.ico
 # Local Python Server
 # -------------------
 
-[python-server-port]
-recipe = slapos.cookbook:free_port
-minimum = 3000
-maximum = 3100
-ip = {{ ipv4_random }}
-
 [python-server]
-recipe = slapos.cookbook:wrapper
-wrapper-path = $${directory:services}/$${:_buildout_section_name_}
-command-line = $${buildout:executable} -m http.server $${python-server-port:port} --bind $${python-server-port:ip} --directory $${directory:frontend-static}
-
+recipe = slapos.recipe.template
+output = $${directory:services}/$${:_buildout_section_name_}
+socket = $${directory:run}/$${:_buildout_section_name_}.sock
+inline =
+  #!$${buildout:executable}
+  import atexit, os, socketserver
+  from http import server
+  class Server(socketserver.ThreadingUnixStreamServer):
+    daemon_threads = True
+  class Handler(server.SimpleHTTPRequestHandler):
+    def address_string(self): # insecure but ok for logging
+      return self.headers.get("X-Forwarded-For", "local")
+  s = "$${:socket}"
+  os.chdir("$${directory:frontend-static}")
+  def cleanup():
+    try:
+      os.remove(s)
+    except FileNotFoundError:
+      pass
+  atexit.register(cleanup)()
+  Server(s, Handler).serve_forever()
 
 # Common Environment
 # ------------------

software/theia/test/test.py

@@ -146,16 +146,16 @@ class TestTheia(TheiaTestCase):
         )).geturl()
     self.get(authenticated_url)
 
-    # there's a public folder to serve file
-    with open('{}/srv/frontend-static/public/test_file'.format(
-        self.getPath()), 'w') as f:
+    # there's a public folder to serve file (no need for authentication)
+    with open(self.getPath() + '/srv/frontend-static/public/test_file',
+              'w') as f:
       f.write("hello")
-    resp = self.get(urljoin(authenticated_url, '/public/'))
-    self.assertIn('test_file', resp.text)
-    resp = self.get(urljoin(authenticated_url, '/public/test_file'))
-    self.assertEqual('hello', resp.text)
-    # make sure public folder is protected
-    resp = self.get(urljoin(url, '/public/test_file'), requests.codes.unauthorized)
+    def get(path_info):
+      resp = self.get(urljoin(url, path_info))
+      self.assertIn('Content-Security-Policy', resp.headers)
+      return resp.text
+    self.assertIn('test_file', get('/public/'))
+    self.assertEqual('hello', get('/public/test_file'))
 
     # there's a (not empty) favicon (no need for authentication)
     resp = self.get(urljoin(url, '/favicon.ico'))

stack/slapos.cfg

@@ -302,7 +302,7 @@ slapos.rebootstrap = 4.5
 slapos.recipe.build = 0.56
 slapos.recipe.cmmi = 0.19
 slapos.recipe.template = 5.1
-slapos.toolbox = 0.132
+slapos.toolbox = 0.134
 smmap = 5.0.0
 sniffio = 1.3.0
 sortedcontainers = 2.4.0