Allow sorting files in FTPConnector

Extend`FTPConnector.listFiles` to sort files based on some server-side computed attributes of files.

This uses [`listdir_attr`](http://docs.paramiko.org/en/2.4/api/sftp.html#paramiko.sftp_client.SFTPClient.listdir_attr) method of paramiko. `sort_on` parameter is directly attributes of [`paramiko.sftp_attr.SFTPAttributes`]( http://docs.paramiko.org/en/2.4/api/sftp.html#paramiko.sftp_attr.SFTPAttributes).

At the same time, adds a minimal tests, that will only run if `testSFTPConnection_SFTP_URL` environment variable is set to an URI of a running sftp server supporting password authentication - which means that in our current tests infrastructure it will not run.

/cc @vpelletier @aurel @rafael 

/reviewed-on nexedi/erp5!680

bt5/erp5_web_service/DocumentTemplateItem/portal_components/document.erp5.FTPConnector.py

@@ -79,11 +79,11 @@ class FTPConnector(XMLObject):
     finally:
       conn.logout()
 
-  def listFiles(self, path="."):
+  def listFiles(self, path=".", sort_on=None):
     """ List file of a directory """
     conn = self.getConnection()
     try:
-      return conn.getDirectoryContent(path)
+      return conn.getDirectoryContent(path, sort_on=sort_on)
     finally:
       conn.logout()
 

bt5/erp5_web_service/TestTemplateItem/portal_components/test.erp5.testFTPConnection.py

+##############################################################################
+#
+# Copyright (c) 2018- Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+
+import os
+import unittest
+import urlparse
+
+from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
+
+
+class TestSFTPConnection(ERP5TypeTestCase):
+  if os.environ.get("testSFTPConnection_SFTP_URL"):
+    def afterSetUp(self):
+      url = os.environ["testSFTPConnection_SFTP_URL"]
+      parsed_url = urlparse.urlparse(url)
+      self.connection = self.portal.portal_web_services.newContent(
+          portal_type='FTP Connector',
+          reference=self.id(),
+          user_id=parsed_url.username,
+          password=parsed_url.password,
+          url_string=url,
+          url_protocol='sftp',
+          use_temporary_file_on_write=False)
+
+    def beforeTearDown(self):
+      for f in self.connection.listFiles("."):
+        self.connection.removeFile(f)
+      self.assertEqual([], self.connection.listFiles("."))
+
+    def test_create_read_delete_file(self):
+      self.connection.putFile("filename", "file content")
+      self.assertEqual(
+        "file content",
+        self.connection.getFile("filename")
+      )
+      self.connection.removeFile("filename")
+
+      # after file is removed, an IOError is raised when trying to read it.
+      self.assertRaises(
+        IOError,
+        self.connection.getFile, "filename"
+      )
+
+    def test_put_rename(self):
+      self.connection.putFile("filename", "file content")
+      self.connection.renameFile("filename", "new name")
+      self.assertEqual(
+        "file content",
+        self.connection.getFile("new name")
+      )
+
+    def test_list_dir(self):
+      self.connection.putFile("first_file", "first file content ( a bit bigger )")
+      self.connection.putFile("second_file", "second file content")
+      # by default, ordering is not specified
+      self.assertItemsEqual(
+          ["first_file", "second_file"],
+          self.connection.listFiles(".")
+      )
+      # but we can sort by modification date
+      self.assertEqual(
+          ["first_file", "second_file"],
+          self.connection.listFiles(".", sort_on="st_mtime")
+      )
+      # or by file size
+      self.assertEqual(
+          ["second_file" , "first_file"],
+          self.connection.listFiles(".", sort_on="st_size")
+      )
+
+  else:
+    def test_no_SFTP_URL_in_environ(self):
+      raise unittest.SkipTest(
+        """This test needs the environment variable testSFTPConnection_SFTP_URL set to the URL of a SFTP connection.
+
+        The URL must contain login and password, such as sftp://user:pass@[::1]:8022
+        The directory from this URL must be empty and writeable.
+        """
+      )
\ No newline at end of file

bt5/erp5_web_service/TestTemplateItem/portal_components/test.erp5.testFTPConnection.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Test Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>testFTPConnection</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>test.erp5.testFTPConnection</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Test Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
+    </pickle>
+    <pickle>
+      <tuple>
+        <none/>
+        <list>
+          <dictionary>
+            <item>
+                <key> <string>action</string> </key>
+                <value> <string>validate</string> </value>
+            </item>
+            <item>
+                <key> <string>validation_state</string> </key>
+                <value> <string>validated</string> </value>
+            </item>
+          </dictionary>
+        </list>
+      </tuple>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_web_service/bt/template_test_id_list

+test.erp5.testFTPConnection
\ No newline at end of file

product/ERP5Type/ConnectionPlugin/SFTPConnection.py

@@ -28,6 +28,7 @@
 ##############################################################################
 
 import os, socket
+import operator
 from urlparse import urlparse
 from socket import gaierror, error, socket, getaddrinfo, AF_UNSPEC, SOCK_STREAM
 from xmlrpclib import Binary
@@ -136,9 +137,17 @@ class SFTPConnection:
     # normalise CRLF/CR/LF like FTP's ASCII mode transfer.
     return os.linesep.join(self._getFile(filepath).splitlines())
 
-  def getDirectoryContent(self, path):
-    """retrieve all entries in a givan path as a list"""
+  def getDirectoryContent(self, path, sort_on=None):
+    """retrieve all entries in a givan path as a list.
+
+    `sort_on` parameter allows to retrieve the directory content in a sorted
+    order, it understands all parameters from
+    paramiko.sftp_attr.SFTPAttributes, the most useful being `st_mtime` to sort
+    by modification date.
+    """
     try:
+      if sort_on:
+        return [x.filename for x in sorted(self.conn.listdir_attr(path), key=operator.attrgetter(sort_on))]
       return self.conn.listdir(path)
     except (EOFError, error), msg:
       raise SFTPError(str(msg) + ' while trying to list %s on %s' % (path, self.url))