software/erp5/test: test clean shutdown WIP 🚧

testing strategy:
 - use an external method with a transaction manager which sleeps
  for 20 seconds during tpc_vote and tpc_commit
 - commit such a transaction
 - stop zope
 - check the stop happens after the tpc_commit

software/erp5/test/test/test_erp5.py

@@ -506,6 +506,7 @@ class ZopeSkinsMixin:
             script_id)
       except xmlrpc.client.ProtocolError as e:
         if e.errcode != 302:
+          breakpoint()
           raise
       getattr(custom, script_id).ZPythonScriptHTML_editAction(
           '',
@@ -697,6 +698,7 @@ class ZopeTestMixin(ZopeSkinsMixin, CrontabMixin):
     self.assertFalse(os.path.exists(rotated_log_file))
 
   def test_access_log_rotation(self):
+    import pdb; pdb.set_trace()
     requests.get(
         self.zope_base_url,
         verify=False,
@@ -879,6 +881,64 @@ class TestZopeWSGI(ZopeTestMixin, ERP5InstanceTestCase):
   pass
 
 
+
+class TestZopeShutdown(ZopeSkinsMixin, ERP5InstanceTestCase):
+  __partition_reference__ = 's'
+
+  @classmethod
+  def _setUpClass(cls):
+    super()._setUpClass()
+    cls.zope_base_url = cls._getAuthenticatedZopeUrl('')
+    param_dict = cls.getRootPartitionConnectionParameterDict()
+
+    # a python script to schedule activities
+    cls._addPythonScript(
+        script_id='ERP5Site_runActivityLoop',
+        params='mode="start"',
+        body='''if 1:
+          from time import sleep
+          portal = context.getPortalObject()
+          if mode == "run":
+            for i in range(1000):
+              for j in range(1000):
+                i * j # use CPU
+            portal.log("ERP5Site_runActivityLoop: %s" % portal.getTitle())
+            portal.setTitle(portal.getTitle() + '.')
+            return
+          if mode == "start":
+            for i in range(100):
+              portal.portal_templates.activate(activity="SQLQueue", group_method_id=None).ERP5Site_runActivityLoop(mode="run")
+            return "started"
+          raise ValueError("Unknown mode: %s" % mode)
+        ''',
+    )
+    cls.zope_verify_activity_processing_url = urllib.parse.urljoin(
+        cls.zope_base_url,
+        'ERP5Site_runActivityLoop',
+    )
+
+  def test_shutdown(self):
+    requests.get(self.zope_verify_activity_processing_url, verify=False).raise_for_status()
+
+    # XXX wait for a first execution
+    time.sleep(2)
+
+    with self.slap.instance_supervisor_rpc as supervisor:
+      zope_info, = [info for info in supervisor.getAllProcessInfo() if 'zope' in info['name']]
+      zope_name = f'{zope_info["group"]}:{zope_info["name"]}'
+      supervisor.stopProcess(zope_name)
+
+#    breakpoint()
+
+    for _ in range(20):
+      with self.slap.instance_supervisor_rpc as supervisor:
+        if supervisor.getProcessInfo(zope_name)['statename'] == 'STOPPED':
+          break
+      time.sleep(0.1)
+    else:
+      time.sleep(60)
+      self.fail("did not stop in time")
+
 class TestZopePublisherTimeout(ZopeSkinsMixin, ERP5InstanceTestCase):
   __partition_reference__ = 't'