fixup! testnode: support chmod'ed files during directories cleanups

I made a mistake rebasing & applying the patch to master, so the test
added in
nexedi/erp5!726 (comment 65067) did not
reach master.

This reapplies this part.

Thanks Seb for noticing !

erp5/tests/testERP5TestNode.py

@@ -489,6 +489,22 @@ shared = true
     test_node.purgeOldTestSuite(test_suite_data)
     self.assertEquals(['foo'], os.listdir(self.working_directory))
 
+  def test_purgeOldTestSuiteChmod(self):
+    """Old test suites can be deleted even when some files/directories have
+    been chmod'd to make read only.  """
+    test_node = self.getTestNode()
+    test_suite_data = self.getTestSuiteData(add_third_repository=True)
+    os.mkdir(os.path.join(self.working_directory, 'bar'))
+    non_writable_file = open(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 'w')
+    non_writable_file.close()
+    # make this file and directory non writeable
+    os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o000)
+    os.chmod(os.path.join(self.working_directory, 'bar'), 0o000)
+
+    test_node.purgeOldTestSuite(test_suite_data) # should not fail
+    self.assertEqual([], os.listdir(self.working_directory))
+
+
   def test_09_runTestSuite(self, my_test_type='UnitTest'):
     """
     Check parameters passed to runTestSuite
@@ -913,16 +929,24 @@ shared = true
            "%r not contained by %r" % (file_list, directory_dir))
     check([])
     os.mkdir(os.path.join(temp_directory, 'buildoutA'))
-    os.mkdir(os.path.join(temp_directory, 'something'))
+    os.mkdir(os.path.join(temp_directory, 'something')) # this will be kept, as it's not a buildout tempfile
     os.mkdir(os.path.join(temp_directory, 'tmpC'))
-    check(set(['buildoutA', 'something', 'tmpC']))
+    os.mkdir(os.path.join(temp_directory, 'tmp-cannot-delete'), 0o000)
+
+    check(set(['buildoutA', 'something', 'tmpC', 'tmp-cannot-delete']))
     # default log file time is 15 days, so nothing is going to be deleted
     test_node._cleanupTemporaryFiles()
-    check(set(['buildoutA', 'something', 'tmpC']))
+    check(set(['buildoutA', 'something', 'tmpC', 'tmp-cannot-delete']))
     # then we set keep time to 0, folder will be deleted
     test_node.max_temp_time = 0
     test_node._cleanupTemporaryFiles()
+    # "something" is kept, as it is not a buildout related tmpfile
     check(set(['something']))
+    # other buildout related files are all deleted now.
+    self.assertEqual(
+        set([]) ,
+        set(['buildoutA', 'tmpC', 'tmp-cannot-delete']).intersection(
+            set(os.listdir(temp_directory))))
 
   def test_18_resetSoftwareAfterManyBuildFailures(self, my_test_type='UnitTest'):
     """

erp5/util/testnode/Utils.py

@@ -10,7 +10,18 @@ def rmtree(path):
   def chmod_retry(func, path, _):
     """Make sure the file is writeable / the directory is executable.
     """
+    if not os.path.exists(path):
+      # because we are calling again rmtree on listdir errors, this path might
+      # have been already deleted by the recursive call to rmtree.
+      return
+
     os.chmod(path, 0o777)
+    if func is os.listdir:
+      # corner case to handle errors in listing directories.
+      # https://bugs.python.org/issue8523
+      # This might raises MaxRecursionError when the directory cannot be listed
+      # for other reasons than "user does not have read permssion"
+      return shutil.rmtree(path, onerror=chmod_retry)
     func(path)
   shutil.rmtree(path, onerror=chmod_retry)
 

[Julien Muchembled]

Still buggy:

  File "/opt/slapgrid/b9f9e0967ab8491759399f306d65239a/eggs/erp5.util-0.4.53-py2.7.egg/erp5/util/testnode/Utils.py", line 25, in chmod_retry
    func(path)
OSError: [Errno 13] Permission denied: '/srv/slapgrid/slappart3/srv/testnode/ij/inst/test0-0/parts/slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/ec5e4ab3ceaca5375d908ea5f307fa97/.slapos.recipe.cmmi.signature'

# ls -la /srv/slapgrid/slappart3/srv/testnode/ij/inst/test0-0/parts/slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/ec5e4ab3ceaca5375d908ea5f307fa97/
total 12
dr-xr-xr-x 2 slapuser3 slapuser3 4096 Sep 13 22:38 .
drwxr-xr-x 4 slapuser3 slapuser3 4096 Sep 13 22:38 ..
-rwxrwxrwx 1 slapuser3 slapuser3  207 Sep 13 22:38 .slapos.recipe.cmmi.signature

[Julien Muchembled]

/cc @seb

[Jérome Perrin]

Still buggy:

I'll look into this, thanks for notifying