testnode: support chmod'ed files during directories cleanups

Because some tests or softwares can chmod'ed some files to make them
readonly, testnode should try to chmod the files to deletable state if
deleting them fail in the first place.

Add some bug reproduction tests for deletions of tempfiles and no longer
used software.

erp5/tests/testERP5TestNode.py

@@ -489,6 +489,22 @@ shared = true
     test_node.purgeOldTestSuite(test_suite_data)
     self.assertEquals(['foo'], os.listdir(self.working_directory))
 
+  def test_purgeOldTestSuiteChmod(self):
+    """Old test suites can be deleted even when some files/directories have
+    been chmod'd to make read only.  """
+    test_node = self.getTestNode()
+    test_suite_data = self.getTestSuiteData(add_third_repository=True)
+    os.mkdir(os.path.join(self.working_directory, 'bar'))
+    non_writable_file = open(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 'w')
+    non_writable_file.close()
+    # make this file and directory non writeable
+    os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o000)
+    os.chmod(os.path.join(self.working_directory, 'bar'), 0o000)
+
+    test_node.purgeOldTestSuite(test_suite_data) # should not fail
+    self.assertEqual([], os.listdir(self.working_directory))
+
+
   def test_09_runTestSuite(self, my_test_type='UnitTest'):
     """
     Check parameters passed to runTestSuite
@@ -913,16 +929,24 @@ shared = true
            "%r not contained by %r" % (file_list, directory_dir))
     check([])
     os.mkdir(os.path.join(temp_directory, 'buildoutA'))
-    os.mkdir(os.path.join(temp_directory, 'something'))
+    os.mkdir(os.path.join(temp_directory, 'something')) # this will be kept, as it's not a buildout tempfile
     os.mkdir(os.path.join(temp_directory, 'tmpC'))
-    check(set(['buildoutA', 'something', 'tmpC']))
+    os.mkdir(os.path.join(temp_directory, 'tmp-cannot-delete'), 0o000)
+
+    check(set(['buildoutA', 'something', 'tmpC', 'tmp-cannot-delete']))
     # default log file time is 15 days, so nothing is going to be deleted
     test_node._cleanupTemporaryFiles()
-    check(set(['buildoutA', 'something', 'tmpC']))
+    check(set(['buildoutA', 'something', 'tmpC', 'tmp-cannot-delete']))
     # then we set keep time to 0, folder will be deleted
     test_node.max_temp_time = 0
     test_node._cleanupTemporaryFiles()
+    # "something" is kept, as it is not a buildout related tmpfile
     check(set(['something']))
+    # other buildout related files are all deleted now.
+    self.assertEqual(
+        set([]) ,
+        set(['buildoutA', 'tmpC', 'tmp-cannot-delete']).intersection(
+            set(os.listdir(temp_directory))))
 
   def test_18_resetSoftwareAfterManyBuildFailures(self, my_test_type='UnitTest'):
     """

erp5/util/testnode/Updater.py

@@ -27,9 +27,9 @@
 import errno
 import os
 import re
-import shutil
 from . import logger
 from .ProcessManager import SubprocessError
+from .Utils import rmtree
 
 SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$')
 SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE)
@@ -149,7 +149,7 @@ class Updater(object):
   def deleteRepository(self):
     logger.info("Wrong repository or wrong url, deleting repos %s",
              self.repository_path)
-    shutil.rmtree(self.repository_path)
+    rmtree(self.repository_path)
 
   def checkRepository(self):
     # make sure that the repository is like we expect

erp5/util/testnode/Utils.py

 import os
+import stat
 import shutil
 
+def rmtree(path):
+  """Delete a path recursively.
+
+  Like shutil.rmtree, but supporting the case that some files or folder
+  might have been marked read only.  """
+  def chmod_retry(func, path, _):
+    """Make sure the file is writeable / the directory is executable.
+    """
+    if not os.path.exists(path):
+      # because we are calling again rmtree on listdir errors, this path might
+      # have been already deleted by the recursive call to rmtree.
+      return
+
+    os.chmod(path, 0o777)
+    if func is os.listdir:
+      # corner case to handle errors in listing directories.
+      # https://bugs.python.org/issue8523
+      # This might raises MaxRecursionError when the directory cannot be listed
+      # for other reasons than "user does not have read permssion"
+      return shutil.rmtree(path, onerror=chmod_retry)
+    func(path)
+  shutil.rmtree(path, onerror=chmod_retry)
+
+
 def createFolder(folder, clean=False):
   if os.path.exists(folder):
     if not clean:
       return
-    shutil.rmtree(folder)
+    rmtree(folder)
   os.mkdir(folder)
 
 def deunicodeData(data):

erp5/util/testnode/testnode.py

@@ -27,7 +27,6 @@
 import os
 import json
 import time
-import shutil
 import logging
 from contextlib import contextmanager
 from slapos.slap.slap import ConnectionError
@@ -39,6 +38,7 @@ from .NodeTestSuite import NodeTestSuite, SlapOSInstance
 from .ScalabilityTestRunner import ScalabilityTestRunner
 from .UnitTestRunner import UnitTestRunner
 from .Utils import deunicodeData
+from .Utils import rmtree
 from .. import taskdistribution
 
 MAX_LOG_TIME = 15 # time in days we should keep logs that we can see through
@@ -74,7 +74,7 @@ class TestNode(object):
       self.node_test_suite_dict.pop(reference, None)
       logger.info("testnode.purgeOldTestSuite, DELETING : %r", fpath)
       if os.path.isdir(fpath):
-        shutil.rmtree(fpath)
+        rmtree(fpath)
       else:
         os.remove(fpath)
   
@@ -218,7 +218,7 @@ shared = true
       if os.path.isdir(folder_path):
         if os.stat(folder_path).st_mtime < prune_time:
           logger.debug("deleting log directory %r", folder_path)
-          shutil.rmtree(folder_path)
+          rmtree(folder_path)
 
   def _cleanupTemporaryFiles(self):
     """
@@ -237,7 +237,7 @@ shared = true
           if stat.st_uid == user_id and stat.st_mtime < prune_time:
             logger.debug("deleting temp directory %r", folder_path)
             if os.path.isdir(folder_path):
-              shutil.rmtree(folder_path)
+              rmtree(folder_path)
             else:
               os.remove(folder_path)
         except OSError: