dufs

https://github.com/sigoden/dufs

New software to serve static files

See merge request nexedi/slapos!1270

component/cmake/buildout.cfg

@@ -9,8 +9,8 @@ parts =
 [cmake]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://cmake.org/files/v3.22/cmake-3.22.1.tar.gz
-md5sum = 83802be08c114bb34729300206488321
+url = https://cmake.org/files/v3.23/cmake-3.23.2.tar.gz
+md5sum = ab3c9ed9578cdb496a252c6733989d78
 environment =
   CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
   CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib

component/llvm/buildout.cfg

 [buildout]
 extends =
+    ../xz-utils/buildout.cfg
     ../cmake/buildout.cfg
     ../git/buildout.cfg
 
 parts = llvm
 
+[llvm-cmake]
+recipe = slapos.recipe.build:download-unpacked
+shared = true
+url = https://github.com/llvm/llvm-project/releases/download/llvmorg-${:version}/cmake-${:version}.src.tar.xz
+version = 15.0.0
+md5sum = e2e69ec36e163293f0b558e0382660cb
+environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s
+
 [llvm]
 recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/llvm/llvm-project/releases/download/llvmorg-${:version}/llvm-${:version}.src.tar.xz
-version = 11.0.0
-md5sum = 85844102335b2e01b3c64b6734fb56f2
+version = 15.0.0
+md5sum = 7b406c97f87b94cda4224aa53968c9d5
+pre-configure =
+  cp ${llvm-cmake:location}/Modules/* ./cmake/modules/
 configure-command = ${cmake:location}/bin/cmake
 configure-options =
     -Bbuild
     -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
     -DCMAKE_BUILD_TYPE=Release
     -DLLVM_INSTALL_UTILS=ON
+    -DLLVM_INCLUDE_BENCHMARKS=OFF
     -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
     -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
 make-options = -C build
 CMAKE_CFLAGS = -I${libxml2:location}/include/libxml2 -I${ncurses:location}/include -I${zlib:location}/include
 environment =
-    PATH=${git:location}/bin:%(PATH)s
+    PATH=${xz-utils:location}/bin:${git:location}/bin:%(PATH)s
     LDFLAGS=-L${libxml2:location}/lib -L${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${zlib:location}/lib

component/rust/buildout.cfg

@@ -9,8 +9,8 @@ parts = rustc
 [rustc]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://static.rust-lang.org/dist/rustc-1.47.0-src.tar.gz
-md5sum = a460bed79b92f6a7833ba6e6390ee6ae
+url = https://static.rust-lang.org/dist/rustc-1.64.0-src.tar.gz
+md5sum = 948ecb62b82ed3543f03ebf598501796
 # --sysconfdir is a workaround for https://github.com/rust-lang/rust/issues/63915
 configure-options =
     --enable-extended

software/dufs/README.md

+# dufs
+
+https://github.com/sigoden/dufs
+
+A file server that supports static serving, uploading, searching, accessing control, webdav...
+
+## slapos configuration
+
+This software release creates an instance of dufs  exposing the `srv/www` folder from the instance (password protected) and `srv/www/pub` with read-only access for anonymous users.

software/dufs/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance.cfg.in]
+filename = instance.cfg.in
+md5sum = 9553d3f24d4201d3320408d8a3154049

software/dufs/instance.cfg.in

+{% import "caucase" as caucase with context %}
+
+[buildout]
+parts =
+  promises
+  publish-connection-parameter
+extends =
+  {{ template_monitor }}
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[slap-configuration]
+# frontend reads from from a part named [slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+srv = ${:home}/srv
+tmp = ${:home}/tmp
+var-log = ${:var}/log
+service = ${:etc}/service
+promise = ${:etc}/promise
+dufs-data-dir = ${:srv}/www
+dufs-data-dir-pub = ${:dufs-data-dir}/pub
+backup-caucased = ${:srv}/backup/caucased/
+
+# Macros
+[check-port-listening-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promise}/${:_buildout_section_name_}
+
+[check-url-available-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promise}/${:_buildout_section_name_}
+dash_path = {{ dash_bin }}
+curl_path = {{ curl_bin }}
+
+# Caucase
+[dufs-certificate]
+key-file = ${directory:etc}/${:_buildout_section_name_}.key
+cert-file = ${directory:etc}/${:_buildout_section_name_}.crt
+common-name = ${:_buildout_section_name_}
+ca-file = ${directory:etc}/${:_buildout_section_name_}.ca.crt
+crl-file = ${directory:etc}/${:_buildout_section_name_}.crl
+
+{{
+caucase.updater(
+    prefix='dufs-certificate',
+    buildout_bin_directory=buildout['bin-directory'],
+    updater_path='${directory:service}/dufs-certificate-updater',
+    url='${caucased:url}',
+    data_dir='${directory:srv}/caucase-updater',
+    crt_path='${dufs-certificate:cert-file}',
+    ca_path='${dufs-certificate:ca-file}',
+    crl_path='${dufs-certificate:crl-file}',
+    key_path='${dufs-certificate:key-file}',
+    template_csr='${dufs-certificate-prepare-csr:csr}',
+    openssl=openssl_bin,
+)}}
+
+[dufs-certificate-csr-config]
+recipe = slapos.recipe.template
+inline =
+  [req]
+  prompt = no
+  req_extensions = req_ext
+  distinguished_name = dn
+  [ dn ]
+  CN = dufs
+  [ req_ext ]
+  subjectAltName = @alt_names
+  [ alt_names ]
+  IP.1 = ${instance-parameter:ipv4-random}
+  IP.2 = ${instance-parameter:ipv6-random}
+
+output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
+
+[dufs-certificate-prepare-csr]
+recipe = plone.recipe.command
+command =
+  if [ ! -f '${:csr}' ] ; then
+   {{ openssl_bin }} req \
+      -newkey rsa \
+      -batch \
+      -new \
+      -nodes \
+      -keyout /dev/null \
+      -config '${dufs-certificate-csr-config:output}' \
+      -out '${:csr}'
+  fi
+stop-on-error = true
+csr = ${directory:srv}/${:_buildout_section_name_}.csr.pem
+
+[caucased]
+port = 19980
+ip = ${instance-parameter:ipv6-random}
+netloc = [${:ip}]:${:port}
+url = http://${:netloc}/
+
+{{
+caucase.caucased(
+    prefix='caucased',
+    buildout_bin_directory=buildout['bin-directory'],
+    caucased_path='${directory:service}/caucased',
+    backup_dir='${directory:backup-caucased}',
+    data_dir='${directory:srv}/caucased',
+    netloc='${caucased:netloc}',
+    tmp='${directory:tmp}',
+    service_auto_approve_count=1,
+    user_auto_approve_count=0,
+    key_len=2048,
+)}}
+
+[admin-password]
+recipe = slapos.cookbook:generate.password
+user = admin
+
+
+[dufs-server]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ dufs_bin }}
+    --enable-cors
+    --bind ${:ip}
+    --port ${:port}
+    --allow-all
+    --auth /@${admin-password:user}:${admin-password:passwd}
+    --auth /pub@${admin-password:user}:${admin-password:passwd}@*
+    --tls-cert ${dufs-certificate:cert-file}
+    --tls-key ${dufs-certificate:key-file}
+    ${directory:dufs-data-dir}
+wrapper-path = ${directory:service}/${:_buildout_section_name_}
+
+port = 19080
+ip = ${instance-parameter:ipv6-random}
+url = https://[${:ip}]:${:port}
+
+[dufs-listen-promise]
+<= check-port-listening-promise
+hostname= ${dufs-server:ip}
+port = ${dufs-server:port}
+
+[frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = dufs Server Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = ${dufs-server:url}
+return = domain secure_access
+
+[frontend-available-promise]
+<= check-url-available-promise
+url = ${frontend:connection-secure_access}
+check-secure = 1
+
+
+[promises]
+recipe =
+instance-promises =
+  ${caucased-promise:recipe}
+  ${dufs-certificate-promise:recipe}
+  ${dufs-listen-promise:path}
+  ${frontend-available-promise:path}
+
+
+[frontend-url]
+recipe = slapos.recipe.build
+frontend-url = ${frontend:connection-secure_access}
+admin-user = ${admin-password:user}
+admin-password = ${admin-password:passwd}
+init =
+  from urllib.parse import urlparse
+  frontend_url = urlparse(self.options['frontend-url'])
+  admin_user = self.options['admin-user']
+  admin_password = self.options['admin-password']
+  self.options['public-url'] = frontend_url._replace(path='/pub').geturl()
+  assert not frontend_url.username
+  self.options['upload-url'] = frontend_url._replace(
+    netloc=f'{admin_user}:{admin_password}@{frontend_url.netloc}').geturl()
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+upload-url = ${frontend-url:upload-url}
+public-url = ${frontend-url:public-url}
+caucase-url = ${caucased:url}
+backend-url = ${dufs-server:url}

software/dufs/software.cfg

+[buildout]
+extends =
+  ../../component/rust/buildout.cfg
+  ../../stack/caucase/buildout.cfg
+  ../../stack/slapos.cfg
+  buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  caucase-eggs
+  instance.cfg.in
+
+
+[dufs]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/sigoden/dufs/archive/refs/tags/v0.30.0.tar.gz
+md5sum = a146cb32028d025143667c3a70e2b696
+configure-command = :
+make-binary = cargo install --root=%(location)s --path .
+make-targets =
+environment =
+  PATH=${rustc:location}/bin:%(PATH)s
+
+
+[instance.cfg.in]
+recipe = slapos.recipe.template:jinja2
+output = ${buildout:directory}/instance.cfg
+url = ${:_profile_base_location_}/${:filename}
+context =
+  section buildout buildout
+  raw openssl_bin ${openssl:location}/bin/openssl
+  raw dash_bin ${dash:location}/bin/dash
+  raw curl_bin ${curl:location}/bin/curl
+  raw dufs_bin ${dufs:location}/bin/dufs
+  key template_monitor monitor2-template:output
+import-list =
+  file caucase caucase-jinja2-library:target

software/dufs/test/README.md

+Tests for dufs software release

software/dufs/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2022 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.dufs'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' dufs",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/dufs/test/test.py

+##############################################################################
+#
+# Copyright (c) 2022 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import io
+import os
+import tempfile
+import urllib.parse
+
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+  os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestFileServer(SlapOSInstanceTestCase):
+  def setUp(self):
+    self.connection_parameters = \
+        self.computer_partition.getConnectionParameterDict()
+    self.ca_cert = self._getCaucaseServiceCACertificate()
+
+  def _getCaucaseServiceCACertificate(self):
+    ca_cert = tempfile.NamedTemporaryFile(
+      prefix="ca.crt.pem",
+      mode="w",
+      delete=False,
+    )
+    ca_cert.write(
+      requests.get(
+        urllib.parse.urljoin(
+          self.connection_parameters['caucase-url'],
+          '/cas/crt/ca.crt.pem',
+        )).text)
+    self.addCleanup(os.unlink, ca_cert.name)
+    return ca_cert.name
+
+  def test_anonymous_can_only_access_public(self):
+    resp = requests.get(
+      self.connection_parameters['public-url'],
+      verify=self.ca_cert,
+    )
+    self.assertEqual(resp.status_code, requests.codes.ok)
+
+    resp = requests.get(
+      urllib.parse.urljoin(self.connection_parameters['public-url'], '..'),
+      verify=self.ca_cert,
+    )
+    self.assertEqual(resp.status_code, requests.codes.unauthorized)
+
+  def test_upload_file_refused_without_digest_auth(self):
+    resp = requests.put(
+      urllib.parse.urljoin(self.connection_parameters['upload-url'], 'hello.txt'),
+      data=io.BytesIO(b'hello'),
+      verify=self.ca_cert,
+    )
+    self.assertEqual(resp.status_code, requests.codes.unauthorized)
+
+  def test_upload_file(self):
+    parsed_url = urllib.parse.urlparse(self.connection_parameters['upload-url'])
+    auth = requests.auth.HTTPDigestAuth(
+      parsed_url.username,
+      parsed_url.password,
+    )
+
+    resp = requests.put(
+      urllib.parse.urljoin(self.connection_parameters['upload-url'], 'hello.txt'),
+      data=io.BytesIO(b'hello'),
+      auth=auth,
+      verify=self.ca_cert,
+    )
+    self.assertEqual(resp.status_code, requests.codes.created)
+
+    resp = requests.get(
+      urllib.parse.urljoin(self.connection_parameters['upload-url'], 'hello.txt'),
+      auth=auth,
+      verify=self.ca_cert,
+    )
+    self.assertEqual(resp.text, 'hello')
+    self.assertEqual(resp.status_code, requests.codes.ok)

software/slapos-sr-testing/software.cfg

@@ -45,6 +45,11 @@ setup = ${slapos-repository:location}/software/backupserver/test/
 egg = slapos.test.caddy-frontend
 setup = ${slapos-repository:location}/software/caddy-frontend/test/
 
+[slapos.test.dufs-setup]
+<= setup-develop-egg
+egg = slapos.test.dufs
+setup = ${slapos-repository:location}/software/dufs/test/
+
 [slapos.test.erp5-setup]
 <= setup-develop-egg
 egg = slapos.test.erp5
@@ -270,6 +275,7 @@ eggs +=
   ${slapos.test.caucase-setup:egg}
   ${slapos.test.cloudooo-setup:egg}
   ${slapos.test.dream-setup:egg}
+  ${slapos.test.dufs-setup:egg}
   ${slapos.test.erp5-setup:egg}
   ${slapos.test.erp5testnode-setup:egg}
   ${slapos.test.fluentd-setup:egg}
@@ -356,6 +362,7 @@ tests =
   caucase ${slapos.test.caucase-setup:setup}
   cloudooo ${slapos.test.cloudooo-setup:setup}
   dream ${slapos.test.dream-setup:setup}
+  dufs ${slapos.test.dufs-setup:setup}
   erp5 ${slapos.test.erp5-setup:setup}
   erp5testnode ${slapos.test.erp5testnode-setup:setup}
   fluentd ${slapos.test.fluentd-setup:setup}