Commit 64b718a7 authored by Jérome Perrin's avatar Jérome Perrin

dufs

https://github.com/sigoden/dufs

New software to serve static files

See merge request nexedi/slapos!1270
parents 6b8cd790 b1f0342f
Pipeline #24231 failed with stage
in 0 seconds
...@@ -9,8 +9,8 @@ parts = ...@@ -9,8 +9,8 @@ parts =
[cmake] [cmake]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
shared = true shared = true
url = https://cmake.org/files/v3.22/cmake-3.22.1.tar.gz url = https://cmake.org/files/v3.23/cmake-3.23.2.tar.gz
md5sum = 83802be08c114bb34729300206488321 md5sum = ab3c9ed9578cdb496a252c6733989d78
environment = environment =
CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib
[buildout] [buildout]
extends = extends =
../xz-utils/buildout.cfg
../cmake/buildout.cfg ../cmake/buildout.cfg
../git/buildout.cfg ../git/buildout.cfg
parts = llvm parts = llvm
[llvm-cmake]
recipe = slapos.recipe.build:download-unpacked
shared = true
url = https://github.com/llvm/llvm-project/releases/download/llvmorg-${:version}/cmake-${:version}.src.tar.xz
version = 15.0.0
md5sum = e2e69ec36e163293f0b558e0382660cb
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
[llvm] [llvm]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
shared = true shared = true
url = https://github.com/llvm/llvm-project/releases/download/llvmorg-${:version}/llvm-${:version}.src.tar.xz url = https://github.com/llvm/llvm-project/releases/download/llvmorg-${:version}/llvm-${:version}.src.tar.xz
version = 11.0.0 version = 15.0.0
md5sum = 85844102335b2e01b3c64b6734fb56f2 md5sum = 7b406c97f87b94cda4224aa53968c9d5
pre-configure =
cp ${llvm-cmake:location}/Modules/* ./cmake/modules/
configure-command = ${cmake:location}/bin/cmake configure-command = ${cmake:location}/bin/cmake
configure-options = configure-options =
-Bbuild -Bbuild
-DCMAKE_INSTALL_PREFIX=@@LOCATION@@ -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
-DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_TYPE=Release
-DLLVM_INSTALL_UTILS=ON -DLLVM_INSTALL_UTILS=ON
-DLLVM_INCLUDE_BENCHMARKS=OFF
-DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}" -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
-DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}" -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
make-options = -C build make-options = -C build
CMAKE_CFLAGS = -I${libxml2:location}/include/libxml2 -I${ncurses:location}/include -I${zlib:location}/include CMAKE_CFLAGS = -I${libxml2:location}/include/libxml2 -I${ncurses:location}/include -I${zlib:location}/include
environment = environment =
PATH=${git:location}/bin:%(PATH)s PATH=${xz-utils:location}/bin:${git:location}/bin:%(PATH)s
LDFLAGS=-L${libxml2:location}/lib -L${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${zlib:location}/lib LDFLAGS=-L${libxml2:location}/lib -L${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${zlib:location}/lib
...@@ -9,8 +9,8 @@ parts = rustc ...@@ -9,8 +9,8 @@ parts = rustc
[rustc] [rustc]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
shared = true shared = true
url = https://static.rust-lang.org/dist/rustc-1.47.0-src.tar.gz url = https://static.rust-lang.org/dist/rustc-1.64.0-src.tar.gz
md5sum = a460bed79b92f6a7833ba6e6390ee6ae md5sum = 948ecb62b82ed3543f03ebf598501796
# --sysconfdir is a workaround for https://github.com/rust-lang/rust/issues/63915 # --sysconfdir is a workaround for https://github.com/rust-lang/rust/issues/63915
configure-options = configure-options =
--enable-extended --enable-extended
......
# dufs
https://github.com/sigoden/dufs
A file server that supports static serving, uploading, searching, accessing control, webdav...
## slapos configuration
This software release creates an instance of dufs exposing the `srv/www` folder from the instance (password protected) and `srv/www/pub` with read-only access for anonymous users.
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance.cfg.in]
filename = instance.cfg.in
md5sum = 9553d3f24d4201d3320408d8a3154049
{% import "caucase" as caucase with context %}
[buildout]
parts =
promises
publish-connection-parameter
extends =
{{ template_monitor }}
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
[instance-parameter]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-configuration]
# frontend reads from from a part named [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
srv = ${:home}/srv
tmp = ${:home}/tmp
var-log = ${:var}/log
service = ${:etc}/service
promise = ${:etc}/promise
dufs-data-dir = ${:srv}/www
dufs-data-dir-pub = ${:dufs-data-dir}/pub
backup-caucased = ${:srv}/backup/caucased/
# Macros
[check-port-listening-promise]
recipe = slapos.cookbook:check_port_listening
path = ${directory:promise}/${:_buildout_section_name_}
[check-url-available-promise]
recipe = slapos.cookbook:check_url_available
path = ${directory:promise}/${:_buildout_section_name_}
dash_path = {{ dash_bin }}
curl_path = {{ curl_bin }}
# Caucase
[dufs-certificate]
key-file = ${directory:etc}/${:_buildout_section_name_}.key
cert-file = ${directory:etc}/${:_buildout_section_name_}.crt
common-name = ${:_buildout_section_name_}
ca-file = ${directory:etc}/${:_buildout_section_name_}.ca.crt
crl-file = ${directory:etc}/${:_buildout_section_name_}.crl
{{
caucase.updater(
prefix='dufs-certificate',
buildout_bin_directory=buildout['bin-directory'],
updater_path='${directory:service}/dufs-certificate-updater',
url='${caucased:url}',
data_dir='${directory:srv}/caucase-updater',
crt_path='${dufs-certificate:cert-file}',
ca_path='${dufs-certificate:ca-file}',
crl_path='${dufs-certificate:crl-file}',
key_path='${dufs-certificate:key-file}',
template_csr='${dufs-certificate-prepare-csr:csr}',
openssl=openssl_bin,
)}}
[dufs-certificate-csr-config]
recipe = slapos.recipe.template
inline =
[req]
prompt = no
req_extensions = req_ext
distinguished_name = dn
[ dn ]
CN = dufs
[ req_ext ]
subjectAltName = @alt_names
[ alt_names ]
IP.1 = ${instance-parameter:ipv4-random}
IP.2 = ${instance-parameter:ipv6-random}
output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
[dufs-certificate-prepare-csr]
recipe = plone.recipe.command
command =
if [ ! -f '${:csr}' ] ; then
{{ openssl_bin }} req \
-newkey rsa \
-batch \
-new \
-nodes \
-keyout /dev/null \
-config '${dufs-certificate-csr-config:output}' \
-out '${:csr}'
fi
stop-on-error = true
csr = ${directory:srv}/${:_buildout_section_name_}.csr.pem
[caucased]
port = 19980
ip = ${instance-parameter:ipv6-random}
netloc = [${:ip}]:${:port}
url = http://${:netloc}/
{{
caucase.caucased(
prefix='caucased',
buildout_bin_directory=buildout['bin-directory'],
caucased_path='${directory:service}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:srv}/caucased',
netloc='${caucased:netloc}',
tmp='${directory:tmp}',
service_auto_approve_count=1,
user_auto_approve_count=0,
key_len=2048,
)}}
[admin-password]
recipe = slapos.cookbook:generate.password
user = admin
[dufs-server]
recipe = slapos.cookbook:wrapper
command-line =
{{ dufs_bin }}
--enable-cors
--bind ${:ip}
--port ${:port}
--allow-all
--auth /@${admin-password:user}:${admin-password:passwd}
--auth /pub@${admin-password:user}:${admin-password:passwd}@*
--tls-cert ${dufs-certificate:cert-file}
--tls-key ${dufs-certificate:key-file}
${directory:dufs-data-dir}
wrapper-path = ${directory:service}/${:_buildout_section_name_}
port = 19080
ip = ${instance-parameter:ipv6-random}
url = https://[${:ip}]:${:port}
[dufs-listen-promise]
<= check-port-listening-promise
hostname= ${dufs-server:ip}
port = ${dufs-server:port}
[frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = dufs Server Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = ${dufs-server:url}
return = domain secure_access
[frontend-available-promise]
<= check-url-available-promise
url = ${frontend:connection-secure_access}
check-secure = 1
[promises]
recipe =
instance-promises =
${caucased-promise:recipe}
${dufs-certificate-promise:recipe}
${dufs-listen-promise:path}
${frontend-available-promise:path}
[frontend-url]
recipe = slapos.recipe.build
frontend-url = ${frontend:connection-secure_access}
admin-user = ${admin-password:user}
admin-password = ${admin-password:passwd}
init =
from urllib.parse import urlparse
frontend_url = urlparse(self.options['frontend-url'])
admin_user = self.options['admin-user']
admin_password = self.options['admin-password']
self.options['public-url'] = frontend_url._replace(path='/pub').geturl()
assert not frontend_url.username
self.options['upload-url'] = frontend_url._replace(
netloc=f'{admin_user}:{admin_password}@{frontend_url.netloc}').geturl()
[publish-connection-parameter]
recipe = slapos.cookbook:publish
upload-url = ${frontend-url:upload-url}
public-url = ${frontend-url:public-url}
caucase-url = ${caucased:url}
backend-url = ${dufs-server:url}
[buildout]
extends =
../../component/rust/buildout.cfg
../../stack/caucase/buildout.cfg
../../stack/slapos.cfg
buildout.hash.cfg
parts =
slapos-cookbook
caucase-eggs
instance.cfg.in
[dufs]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/sigoden/dufs/archive/refs/tags/v0.30.0.tar.gz
md5sum = a146cb32028d025143667c3a70e2b696
configure-command = :
make-binary = cargo install --root=%(location)s --path .
make-targets =
environment =
PATH=${rustc:location}/bin:%(PATH)s
[instance.cfg.in]
recipe = slapos.recipe.template:jinja2
output = ${buildout:directory}/instance.cfg
url = ${:_profile_base_location_}/${:filename}
context =
section buildout buildout
raw openssl_bin ${openssl:location}/bin/openssl
raw dash_bin ${dash:location}/bin/dash
raw curl_bin ${curl:location}/bin/curl
raw dufs_bin ${dufs:location}/bin/dufs
key template_monitor monitor2-template:output
import-list =
file caucase caucase-jinja2-library:target
Tests for dufs software release
##############################################################################
#
# Copyright (c) 2022 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.dufs'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' dufs",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
#
# Copyright (c) 2022 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import io
import os
import tempfile
import urllib.parse
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestFileServer(SlapOSInstanceTestCase):
def setUp(self):
self.connection_parameters = \
self.computer_partition.getConnectionParameterDict()
self.ca_cert = self._getCaucaseServiceCACertificate()
def _getCaucaseServiceCACertificate(self):
ca_cert = tempfile.NamedTemporaryFile(
prefix="ca.crt.pem",
mode="w",
delete=False,
)
ca_cert.write(
requests.get(
urllib.parse.urljoin(
self.connection_parameters['caucase-url'],
'/cas/crt/ca.crt.pem',
)).text)
self.addCleanup(os.unlink, ca_cert.name)
return ca_cert.name
def test_anonymous_can_only_access_public(self):
resp = requests.get(
self.connection_parameters['public-url'],
verify=self.ca_cert,
)
self.assertEqual(resp.status_code, requests.codes.ok)
resp = requests.get(
urllib.parse.urljoin(self.connection_parameters['public-url'], '..'),
verify=self.ca_cert,
)
self.assertEqual(resp.status_code, requests.codes.unauthorized)
def test_upload_file_refused_without_digest_auth(self):
resp = requests.put(
urllib.parse.urljoin(self.connection_parameters['upload-url'], 'hello.txt'),
data=io.BytesIO(b'hello'),
verify=self.ca_cert,
)
self.assertEqual(resp.status_code, requests.codes.unauthorized)
def test_upload_file(self):
parsed_url = urllib.parse.urlparse(self.connection_parameters['upload-url'])
auth = requests.auth.HTTPDigestAuth(
parsed_url.username,
parsed_url.password,
)
resp = requests.put(
urllib.parse.urljoin(self.connection_parameters['upload-url'], 'hello.txt'),
data=io.BytesIO(b'hello'),
auth=auth,
verify=self.ca_cert,
)
self.assertEqual(resp.status_code, requests.codes.created)
resp = requests.get(
urllib.parse.urljoin(self.connection_parameters['upload-url'], 'hello.txt'),
auth=auth,
verify=self.ca_cert,
)
self.assertEqual(resp.text, 'hello')
self.assertEqual(resp.status_code, requests.codes.ok)
...@@ -45,6 +45,11 @@ setup = ${slapos-repository:location}/software/backupserver/test/ ...@@ -45,6 +45,11 @@ setup = ${slapos-repository:location}/software/backupserver/test/
egg = slapos.test.caddy-frontend egg = slapos.test.caddy-frontend
setup = ${slapos-repository:location}/software/caddy-frontend/test/ setup = ${slapos-repository:location}/software/caddy-frontend/test/
[slapos.test.dufs-setup]
<= setup-develop-egg
egg = slapos.test.dufs
setup = ${slapos-repository:location}/software/dufs/test/
[slapos.test.erp5-setup] [slapos.test.erp5-setup]
<= setup-develop-egg <= setup-develop-egg
egg = slapos.test.erp5 egg = slapos.test.erp5
...@@ -270,6 +275,7 @@ eggs += ...@@ -270,6 +275,7 @@ eggs +=
${slapos.test.caucase-setup:egg} ${slapos.test.caucase-setup:egg}
${slapos.test.cloudooo-setup:egg} ${slapos.test.cloudooo-setup:egg}
${slapos.test.dream-setup:egg} ${slapos.test.dream-setup:egg}
${slapos.test.dufs-setup:egg}
${slapos.test.erp5-setup:egg} ${slapos.test.erp5-setup:egg}
${slapos.test.erp5testnode-setup:egg} ${slapos.test.erp5testnode-setup:egg}
${slapos.test.fluentd-setup:egg} ${slapos.test.fluentd-setup:egg}
...@@ -356,6 +362,7 @@ tests = ...@@ -356,6 +362,7 @@ tests =
caucase ${slapos.test.caucase-setup:setup} caucase ${slapos.test.caucase-setup:setup}
cloudooo ${slapos.test.cloudooo-setup:setup} cloudooo ${slapos.test.cloudooo-setup:setup}
dream ${slapos.test.dream-setup:setup} dream ${slapos.test.dream-setup:setup}
dufs ${slapos.test.dufs-setup:setup}
erp5 ${slapos.test.erp5-setup:setup} erp5 ${slapos.test.erp5-setup:setup}
erp5testnode ${slapos.test.erp5testnode-setup:setup} erp5testnode ${slapos.test.erp5testnode-setup:setup}
fluentd ${slapos.test.fluentd-setup:setup} fluentd ${slapos.test.fluentd-setup:setup}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment