software/restic-rest-server: new software

https://github.com/restic/rest-server

software/restic-rest-server/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance.cfg.in]
+filename = instance.cfg.in
+md5sum = fee2097f3d12fd4bcfbc2698ecf49afc

software/restic-rest-server/instance.cfg.in

+{% import "caucase" as caucase with context %}
+
+[buildout]
+parts =
+  promises
+  publish-connection-parameter
+extends =
+  {{ template_monitor }}
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[slap-configuration]
+# frontend reads from from a part named [slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+srv = ${:home}/srv
+tmp = ${:home}/tmp
+var-log = ${:var}/log
+service = ${:etc}/service
+promise = ${:etc}/promise
+rest-server-data-dir = ${:srv}/restic
+backup-caucased = ${:srv}/backup/caucased/
+
+# Macros
+[check-port-listening-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promise}/${:_buildout_section_name_}
+
+[check-url-available-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promise}/${:_buildout_section_name_}
+dash_path = {{ dash_bin }}
+curl_path = {{ curl_bin }}
+
+# Caucase
+[rest-server-certificate]
+key-file = ${directory:etc}/${:_buildout_section_name_}.key
+cert-file = ${directory:etc}/${:_buildout_section_name_}.crt
+common-name = ${:_buildout_section_name_}
+ca-file = ${directory:etc}/${:_buildout_section_name_}.ca.crt
+crl-file = ${directory:etc}/${:_buildout_section_name_}.crl
+
+{{
+caucase.updater(
+    prefix='rest-server-certificate',
+    buildout_bin_directory=buildout['bin-directory'],
+    updater_path='${directory:service}/rest-server-certificate-updater',
+    url='${caucased:url}',
+    data_dir='${directory:srv}/caucase-updater',
+    crt_path='${rest-server-certificate:cert-file}',
+    ca_path='${rest-server-certificate:ca-file}',
+    crl_path='${rest-server-certificate:crl-file}',
+    key_path='${rest-server-certificate:key-file}',
+    template_csr='${rest-server-certificate-prepare-csr:csr}',
+    openssl=openssl_bin,
+)}}
+
+[rest-server-certificate-csr-config]
+recipe = slapos.recipe.template:jinja2
+mode = 644
+template = inline:
+  [req]
+  prompt = no
+  req_extensions = req_ext
+  distinguished_name = dn
+  [ dn ]
+  CN = restic-rest-server
+  [ req_ext ]
+  subjectAltName = @alt_names
+  [ alt_names ]
+  IP.1 = ${instance-parameter:ipv4-random}
+  IP.2 = ${instance-parameter:ipv6-random}
+
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
+
+[rest-server-certificate-prepare-csr]
+recipe = plone.recipe.command
+command =
+  if [ ! -f '${:csr}' ] ; then
+   {{ openssl_bin }} req \
+      -newkey rsa:2048 \
+      -batch \
+      -new \
+      -nodes \
+      -keyout /dev/null \
+      -config '${rest-server-certificate-csr-config:rendered}' \
+      -out '${:csr}'
+  fi
+stop-on-error = true
+csr = ${directory:srv}/${:_buildout_section_name_}.csr.pem
+
+[caucased]
+port = 18080
+ip = ${instance-parameter:ipv6-random}
+netloc = [${:ip}]:${:port}
+url = http://${:netloc}/
+
+{{
+caucase.caucased(
+    prefix='caucased',
+    buildout_bin_directory=buildout['bin-directory'],
+    caucased_path='${directory:service}/caucased',
+    backup_dir='${directory:backup-caucased}',
+    data_dir='${directory:srv}/caucased',
+    netloc='${caucased:netloc}',
+    tmp='${directory:tmp}',
+    service_auto_approve_count=1,
+    user_auto_approve_count=0,
+    key_len=2048,
+)}}
+
+[rest-server-password]
+recipe = slapos.cookbook:generate.password
+user = backup
+
+[rest-server-htpassword]
+recipe = plone.recipe.command
+command = 
+  if [ ! -f '${:htpassword}' ] ; then
+    {{ htpasswd_bin }} \
+      -b \
+      -B \
+      -c ${:htpassword} \
+      ${rest-server-password:user} \
+      ${rest-server-password:passwd}
+  fi
+htpassword = ${directory:rest-server-data-dir}/.htpasswd
+stop-on-error = true
+
+[rest-server]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ gowork_bin }}/rest-server \
+    --listen [${instance-parameter:ipv6-random}]:${:port}
+    --log ${directory:var-log}/${:_buildout_section_name_}-access.log
+    --path ${directory:rest-server-data-dir}
+    --tls
+    --tls-cert ${rest-server-certificate:cert-file}
+    --tls-key ${rest-server-certificate:key-file}
+    --prometheus
+wrapper-path = ${directory:service}/rest-server
+
+port = 19080
+ip = ${instance-parameter:ipv6-random}
+url = https://[${:ip}]:${:port}
+depends =
+  ${rest-server-htpassword:recipe}
+
+[rest-server-listen-promise]
+<= check-port-listening-promise
+hostname= ${rest-server:ip}
+port = ${rest-server:port}
+
+[frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Rest Server Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = ${rest-server:url}
+return = domain secure_access
+
+[frontend-available-promise]
+<= check-url-available-promise
+url = ${frontend:connection-secure_access}
+check-secure = 1
+
+
+[promises]
+recipe =
+instance-promises =
+  ${caucased-promise:recipe}
+  ${rest-server-certificate-promise:recipe}
+  ${rest-server-listen-promise:path}
+  ${frontend-available-promise:path}
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+rest-server-user = ${rest-server-password:user}
+rest-server-password = ${rest-server-password:passwd}
+url = ${frontend:connection-secure_access}
+caucase-url = ${caucased:url}

software/restic-rest-server/software.cfg

+[buildout]
+
+extends =
+  ../../component/openssl/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/golang/buildout.cfg
+  ../../component/restic/buildout.cfg
+  ../../stack/caucase/buildout.cfg
+  ../../stack/slapos.cfg
+  buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  caucase-eggs
+  instance.cfg.in
+  restic
+  gowork
+
+[python]
+part = python3
+
+[gowork]
+install +=
+  ${git.github.com_restic_rest-server:location}:./cmd/...
+
+[git.github.com_restic_rest-server]
+<= go-git-package
+go.importpath = github.com/restic/rest-server
+repository =  https://github.com/restic/rest-server
+revision = v0.10.0-0-g9313f19
+
+[instance.cfg.in]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/instance.cfg
+template = ${:_profile_base_location_}/${:filename}
+mode = 0644
+context =
+  section buildout buildout
+  key gowork_bin gowork:bin
+  raw openssl_bin ${openssl:location}/bin/openssl
+  raw htpasswd_bin ${apache:location}/bin/htpasswd
+  raw dash_bin ${dash:location}/bin/dash
+  raw curl_bin ${curl:location}/bin/curl
+  key template_monitor monitor2-template:rendered
+import-list =
+  file caucase caucase-jinja2-library:target

software/restic-rest-server/test/README.md

+Tests for restic ReST Server software release

software/restic-rest-server/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.restic_rest_server'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' restic Rest Server",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/restic-rest-server/test/test.py

+##############################################################################
+#
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import glob
+import os
+import subprocess
+import tempfile
+import urllib.parse
+
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestResticRestServer(SlapOSInstanceTestCase):
+
+  def setUp(self):
+    self.connection_parameters = \
+        self.computer_partition.getConnectionParameterDict()
+    parsed_url = urllib.parse.urlparse(self.connection_parameters['url'])
+    self.url_with_credentials = parsed_url._replace(
+        netloc='{}:{}@[{}]:{}'.format(
+            self.connection_parameters['rest-server-user'],
+            self.connection_parameters['rest-server-password'],
+            parsed_url.hostname,
+            parsed_url.port,
+        )).geturl()
+
+    self.ca_cert = self._getCaucaseServiceCACertificate()
+
+  def _getCaucaseServiceCACertificate(self):
+    ca_cert = tempfile.NamedTemporaryFile(
+        prefix="ca.crt.pem",
+        mode="w",
+        delete=False,
+    )
+    ca_cert.write(
+        requests.get(
+            urllib.parse.urljoin(
+                self.connection_parameters['caucase-url'],
+                '/cas/crt/ca.crt.pem',
+            )).text)
+    self.addCleanup(os.unlink, ca_cert.name)
+    return ca_cert.name
+
+  def test_http_get(self):
+    resp = requests.get(self.connection_parameters['url'], verify=self.ca_cert)
+    self.assertEqual(resp.status_code, requests.codes.unauthorized)
+
+    resp = requests.get(
+        urllib.parse.urljoin(
+            self.url_with_credentials,
+            '/metrics',
+        ),
+        verify=self.ca_cert,
+    )
+    # a random metric
+    self.assertIn('process_cpu_seconds_total', resp.text)
+    resp.raise_for_status()
+
+  def test_backup_scenario(self):
+    restic_bin = os.path.join(
+        self.computer_partition_root_path,
+        'software_release',
+        'go.work',
+        'bin',
+        'restic',
+    )
+
+    def get_restic_output(*args, **kw):
+      return subprocess.check_output(
+          (
+              restic_bin,
+              '--cacert',
+              self.ca_cert,
+              '--password-file',
+              password_file.name,
+              '--repo',
+              'rest:' + self.url_with_credentials,
+          ) + args,
+          universal_newlines=True,
+          **kw,
+      )
+
+    with tempfile.TemporaryDirectory() as work_directory,\
+        tempfile.NamedTemporaryFile(mode='w') as password_file:
+      password_file.write('secret')
+      password_file.flush()
+
+      with open(os.path.join(work_directory, 'data'), 'w') as f:
+        f.write('data to backup')
+      with self.assertRaises(subprocess.CalledProcessError) as exc_context:
+        get_restic_output('snapshots', stderr=subprocess.PIPE)
+      self.assertIn('Is there a repository at the following location?',
+                    exc_context.exception.stderr)
+
+      out = get_restic_output('init')
+      self.assertIn('created restic repository', out)
+
+      out = get_restic_output('backup', work_directory)
+      self.assertIn('Added to the repo', out)
+
+      out = get_restic_output('snapshots')
+      self.assertEqual(out.splitlines()[-1], '1 snapshots')
+      snapshot_id = out.splitlines()[2].split()[0]
+      backup_path = out.splitlines()[2].split()[-1]
+      restore_directory = os.path.join(work_directory, 'restore')
+
+      out = get_restic_output(
+          'restore',
+          snapshot_id,
+          '--target',
+          restore_directory,
+      )
+      self.assertIn('restoring <Snapshot', out)
+      with open(os.path.join(restore_directory, backup_path, 'data')) as f:
+        self.assertEqual(f.read(), 'data to backup')