ProFTPd: More Authentication methods

* version up
 * ssh keys support
 * mod_auth_web support

See merge request !993

component/proftpd/buildout.cfg

@@ -23,16 +23,15 @@ recipe = collective.recipe.grp
 
 [proftpd]
 recipe = slapos.recipe.cmmi
-md5sum = 4040f6a6b86173e2a03f4ccdb9b9af6e
-url = ftp://ftp.proftpd.org/distrib/source/proftpd-1.3.6b.tar.gz
+md5sum = 4a9b8877b2e9b08d70e71ad56c19e2c9
+url = ftp://ftp.proftpd.org/distrib/source/proftpd-1.3.7a.tar.gz
 configure-options =
   --enable-openssl
   --enable-nls
   --enable-ctrls
   --enable-dso
   --disable-cap
-  --with-modules=mod_sftp:mod_ban
-  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+  --with-modules=mod_sftp:mod_ban:mod_rewrite
 environment =
   CFLAGS=-DPR_RUN_DIR=\"/proc/self/cwd/var\"
   CPPFLAGS=-I${zlib:location}/include -I${openssl:location}/include
@@ -47,11 +46,8 @@ patches =
 # mod_auth_web: a proftpd module to authenticate users against an HTTP service
 [proftpd-mod_auth_web-repository]
 recipe = slapos.recipe.build:gitclone
-#repository = https://github.com/proftpd/mod_auth_web
-# XXX until https://github.com/proftpd/mod_auth_web/pull/1 gets merged, we use
-# a copy of this repository on nexedi gitlab
-repository = https://lab.nexedi.com/jerome/mod_auth_web
-revision = dec090bd0e287544a34be156ee17f715bd4286f9
+repository = https://github.com/proftpd/mod_auth_web
+revision = e36105808b7d07d843b11f428a666a8f3cec35e4
 git-executable = ${git:location}/bin/git
 
 [proftpd-mod_auth_web]

software/proftpd/README.md

@@ -4,15 +4,14 @@ http://www.proftpd.org/docs/
 
 # Features
 
- * sftp only is enabled
+ * sftp only is enabled, with authentication by key or password
  * partially uploadloaded are not visible thanks to [`HiddenStores`](http://proftpd.org/docs/directives/linked/config_ref_HiddenStores.html) ( in fact they are, but name starts with `.` )
  * 5 failed login attempts will cause the host to be temporary banned
+ * support authentication against an external web service
 
 
 # TODO
 
- * only password login is enabled. enabling [`SFTPAuthorizedUserKeys`](http://www.proftpd.org/docs/contrib/mod_sftp.html#SFTPAuthorizedUserKeys) seems to break password only login
  * log rotation
  * make sure SFTPLog is useful (seems very verbose and does not contain more than stdout)
- * make it easier to manage users ( using `mod_auth_web` against an ERP5 endpoint or accepting a list of user/password as instance parameter )
  * allow configuring webhooks when new file is uploaded

software/proftpd/buildout.hash.cfg

@@ -19,8 +19,8 @@ md5sum = efb4238229681447aa7fe73898dffad4
 
 [instance-default]
 filename = instance-default.cfg.in
-md5sum = 2a2c066d7d40dd8545f3008f434ee842
+md5sum = dae19ec06f8da9fa2980a6d2bdf3da54
 
 [proftpd-config-file]
 filename = proftpd-config-file.cfg.in
-md5sum = a7c0f4607c378b640379cc258a8aadfa
+md5sum = 82cc600f4fce9852370f9d1f7c4cd3a6

software/proftpd/instance-default.cfg.in

@@ -66,16 +66,24 @@ ban-log=${directory:log}/proftpd-ban.log
 ssh-host-rsa-key=${ssh-host-rsa-key:output}
 ssh-host-dsa-key=${ssh-host-dsa-key:output}
 ssh-host-ecdsa-key=${ssh-host-ecdsa-key:output}
-ssh-authorized-keys-dir = ${directory:ssh-authorized-keys-dir}
+ssh-authorized-key = ${ssh-authorized-keys:rendered}
 ban-table=${directory:srv}/proftpd-ban-table
 control-socket=${directory:var}/proftpd.sock
 auth-user-file=${auth-user-file:output}
+authentication-url = {{ slapparameter_dict.get('authentication-url', '')}}
 
 recipe = slapos.cookbook:wrapper
 command-line =
    {{ proftpd_bin }} --nodaemon --config ${proftpd-config-file:rendered}
 wrapper-path = ${directory:service}/proftpd
 
+[ssh-authorized-keys]
+rendered = ${directory:ssh-authorized-keys-dir}/authorized_keys
+{% if slapparameter_dict.get('ssh-key') %}
+recipe = slapos.recipe.template:jinja2
+template = inline:{{ slapparameter_dict['ssh-key'] | indent }}
+{% endif %}
+
 [proftpd-listen-promise]
 <= monitor-promise-base
 module = check_port_listening
@@ -133,5 +141,9 @@ instance-promises =
 [publish-connection-parameter]
 recipe = slapos.cookbook:publish
 url = ${proftpd:url}
+{% if not slapparameter_dict.get('authentication-url') %}
 username = ${proftpd-password:username}
+{%   if not slapparameter_dict.get('ssh-key') %}
 password = ${proftpd-password:passwd}
+{%   endif %}
+{% endif %}

software/proftpd/instance-input-schema.json

 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "description": "Parameters to instantiate PoFTPd",
+  "description": "Parameters to instantiate ProFTPd",
   "additionalProperties": false,
   "properties": {
     "port": {
-      "description": "Port number to listen to - default to 8022",
-      "type": "number"
+      "description": "Port number to listen to",
+      "type": "number",
+      "default": 8022
+    },
+    "ssh-key": {
+      "description": "SSH public key, in RFC4716 format. Note that this is not the default format used by openssh and that openssh keys must be converted with `ssh-keygen -e -f ~/.ssh/id_rsa.pub`",
+      "type": "string"
+    },
+    "authentication-url": {
+      "description": "URL of an HTTP endpoint to authenticate users. Endoint recieve a `application/x-www-form-urlencoded` POST request with `login` and `password` arguments and must respond with a `X-Proftpd-Authentication-Result: Success` header to signal successful authentication",
+      "type": "string"
     }
   }
 }

software/proftpd/instance-output-schema.json

@@ -14,7 +14,7 @@
       "optional": true
     },
     "password": {
-      "description": "Password for default username",
+      "description": "Password for default username, when not using ssh-key",
       "type": "string",
       "optional": true
     }

software/proftpd/proftpd-config-file.cfg.in

@@ -20,7 +20,7 @@ SFTPEngine on
 SFTPHostKey {{ proftpd['ssh-host-rsa-key'] }}
 SFTPHostKey {{ proftpd['ssh-host-dsa-key'] }}
 SFTPHostKey {{ proftpd['ssh-host-ecdsa-key'] }}
-#SFTPAuthorizedUserKeys file:{{ proftpd['ssh-authorized-keys-dir'] }}%u
+SFTPAuthorizedUserKeys file:{{ proftpd['ssh-authorized-key'] }}
 
 
 # Logging
@@ -34,6 +34,23 @@ RequireValidShell off
 AuthUserFile {{ proftpd['auth-user-file'] }}
 
 
+# http authentication
+{% if proftpd['authentication-url'] %}
+LoadModule mod_auth_web.c
+AuthWebURL {{ proftpd['authentication-url'] }}
+AuthWebRequireHeader "X-Proftpd-Authentication-Result: Success"
+AuthWebUsernameParamName login
+AuthWebPasswordParamName password
+AuthWebLocalUser {{ proftpd['user'] }}
+# mod_auth_web only read /etc/passwd to know the home of the users, 
+# so we rewrite the relative paths to be relative to the data dir.
+LoadModule mod_rewrite.c
+RewriteEngine on
+RewriteCondition %m !USER
+RewriteRule ^([^/]+.*) {{ proftpd['data-dir'] }}$1
+{% endif %}
+
+
 # Prevent partially uploaded files to be visible
 HiddenStores on
 DeleteAbortedStores on

software/proftpd/software.cfg.json

 {
   "name": "ProFTPd",
   "description": "ProFTPd as a SFTP server with virtual users",
-  "serialisation": "json-in-xml",
+  "serialisation": "xml",
   "software-type": {
     "default": {
       "title": "Default",

software/proftpd/test/test.py

@@ -27,18 +27,23 @@
 
 import os
 import shutil
-from urllib.parse import urlparse
+from urllib.parse import urlparse, parse_qs
 import tempfile
 import io
 import subprocess
+from http.server import BaseHTTPRequestHandler
+import logging
 
 import pysftp
 import psutil
+import paramiko
 from paramiko.ssh_exception import SSHException
 from paramiko.ssh_exception import AuthenticationException
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 from slapos.testing.utils import findFreeTCPPort
+from slapos.testing.utils import ManagedHTTPServer
+
 
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
@@ -176,9 +181,9 @@ class TestUserManagement(ProFTPdTestCase):
 class TestBan(ProFTPdTestCase):
   def test_client_are_banned_after_5_wrong_passwords(self):
     # Simulate failed 5 login attempts
-    for i in range(5):
+    for _ in range(5):
       with self.assertRaisesRegex(AuthenticationException,
-                                   'Authentication failed'):
+                                  'Authentication failed'):
         self._getConnection(password='wrong')
 
     # after that, even with a valid password we cannot connect
@@ -237,3 +242,119 @@ class TestFilesAndSocketsInInstanceDir(ProFTPdTestCase):
             s for s in self.proftpdProcess.connections('unix')
             if not s.laddr.startswith(self.computer_partition_root_path)
         ])
+
+
+class TestSSHKey(TestSFTPOperations):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    cls.ssh_key = paramiko.DSSKey.generate(1024)
+    return {
+        'ssh-key':
+        '---- BEGIN SSH2 PUBLIC KEY ----\n{}\n---- END SSH2 PUBLIC KEY ----'.
+        format(cls.ssh_key.get_base64())
+    }
+
+  def _getConnection(self, username=None):
+    """Override to log in with the SSH key
+    """
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    sftp_url = urlparse(parameter_dict['url'])
+    username = username or parameter_dict['username']
+
+    cnopts = pysftp.CnOpts()
+    cnopts.hostkeys = None
+
+    with tempfile.NamedTemporaryFile(mode='w') as keyfile:
+      self.ssh_key.write_private_key(keyfile)
+      keyfile.flush()
+      return pysftp.Connection(
+          sftp_url.hostname,
+          port=sftp_url.port,
+          cnopts=cnopts,
+          username=username,
+          private_key=keyfile.name,
+      )
+
+  def test_authentication_failure(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    sftp_url = urlparse(parameter_dict['url'])
+
+    with self.assertRaisesRegex(AuthenticationException,
+                                'Authentication failed'):
+      self._getConnection(username='wrong username')
+
+    cnopts = pysftp.CnOpts()
+    cnopts.hostkeys = None
+
+    # wrong private key
+    with tempfile.NamedTemporaryFile(mode='w') as keyfile:
+      paramiko.DSSKey.generate(1024).write_private_key(keyfile)
+      keyfile.flush()
+      with self.assertRaisesRegex(AuthenticationException,
+                                  'Authentication failed'):
+        pysftp.Connection(
+            sftp_url.hostname,
+            port=sftp_url.port,
+            cnopts=cnopts,
+            username=parameter_dict['username'],
+            private_key=keyfile.name,
+        )
+
+  def test_published_parameters(self):
+    # no password is published, we only login with key
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    self.assertIn('username', parameter_dict)
+    self.assertNotIn('password', parameter_dict)
+
+
+class TestAuthenticationURL(TestSFTPOperations):
+  class AuthenticationServer(ManagedHTTPServer):
+    class RequestHandler(BaseHTTPRequestHandler):
+      def do_POST(self):
+        # type: () -> None
+        assert self.headers[
+            'Content-Type'] == 'application/x-www-form-urlencoded', self.headers[
+                'Content-Type']
+        posted_data = dict(
+            parse_qs(
+                self.rfile.read(int(self.headers['Content-Length'])).decode()))
+        if posted_data['login'] == ['login'] and posted_data['password'] == [
+            'password'
+        ]:
+          self.send_response(200)
+          self.send_header("X-Proftpd-Authentication-Result", "Success")
+          self.end_headers()
+          return self.wfile.write(b"OK")
+        self.send_response(401)
+        return self.wfile.write(b"Forbidden")
+
+      log_message = logging.getLogger(__name__ + '.AuthenticationServer').info
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+        'authentication-url':
+        cls.getManagedResource('authentication-server',
+                               TestAuthenticationURL.AuthenticationServer).url
+    }
+
+  def _getConnection(self, username='login', password='password'):
+    """Override to log in with the HTTP credentials by default.
+    """
+    return super()._getConnection(username=username, password=password)
+
+  def test_authentication_success(self):
+    with self._getConnection() as sftp:
+      self.assertEqual(sftp.listdir('.'), [])
+
+  def test_authentication_failure(self):
+    with self.assertRaisesRegex(AuthenticationException,
+                                'Authentication failed'):
+      self._getConnection(username='login', password='wrong')
+
+  def test_published_parameters(self):
+    # no login or password are published, logins are defined by their
+    # user name
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    self.assertNotIn('username', parameter_dict)
+    self.assertNotIn('password', parameter_dict)