upgrader: refactor dependencies between each steps

This should fix some race conditions which made post-upgrade steps not
executed sometimes.

* each steps use an defaultActivateParameterDict so that activity they
  may create inherits the same tag, so that next step also runs after
  this). This makes it possible for example to use post-upgrade step on
  documents newly installed in upgrade step.
* use tag / after tag between upgrader steps instead of checking if
  previous alarm had errors. We also now allow force running only one
  step regardless of the previous step completion state, there was no
  strong reason to prevernt this.
* do not use two different active sense and fix method for alarms, just
  use an active sense method understanding `fixit` parameter

bt5/erp5_upgrader/PathTemplateItem/portal_alarms/promise_check_upgrade.xml

@@ -8,7 +8,7 @@
       <dictionary>
         <item>
             <key> <string>active_sense_method_id</string> </key>
-            <value> <string>Alarm_senseFullUpgradeNeed</string> </value>
+            <value> <string>Alarm_runFullUpgrader</string> </value>
         </item>
         <item>
             <key> <string>alarm_notification_mode</string> </key>
@@ -18,6 +18,10 @@
               </tuple>
             </value>
         </item>
+        <item>
+            <key> <string>automatic_solve</string> </key>
+            <value> <int>0</int> </value>
+        </item>
         <item>
             <key> <string>description</string> </key>
             <value> <string>This alarm checks that the upgrader is required. If yes, the upgrader have three steps:\n
@@ -106,7 +110,9 @@ This step will run all constraints that have contraint_type equal post_upgrade.<
         </item>
         <item>
             <key> <string>solve_method_id</string> </key>
-            <value> <string>Alarm_runFullUpgrader</string> </value>
+            <value>
+              <none/>
+            </value>
         </item>
         <item>
             <key> <string>title</string> </key>

bt5/erp5_upgrader/PathTemplateItem/portal_alarms/upgrader_check_post_upgrade.xml

@@ -8,7 +8,7 @@
       <dictionary>
         <item>
             <key> <string>active_sense_method_id</string> </key>
-            <value> <string>Alarm_senseUpgradeNeed</string> </value>
+            <value> <string>Alarm_runPostUpgrade</string> </value>
         </item>
         <item>
             <key> <string>alarm_notification_mode</string> </key>
@@ -18,6 +18,10 @@
               </tuple>
             </value>
         </item>
+        <item>
+            <key> <string>automatic_solve</string> </key>
+            <value> <int>0</int> </value>
+        </item>
         <item>
             <key> <string>description</string> </key>
             <value> <string>This step will run all constraints that have contraint_type equal post_upgrade.</string> </value>
@@ -97,7 +101,9 @@
         </item>
         <item>
             <key> <string>solve_method_id</string> </key>
-            <value> <string>Alarm_runPostUpgrade</string> </value>
+            <value>
+              <none/>
+            </value>
         </item>
         <item>
             <key> <string>title</string> </key>

bt5/erp5_upgrader/PathTemplateItem/portal_alarms/upgrader_check_pre_upgrade.xml

@@ -8,7 +8,7 @@
       <dictionary>
         <item>
             <key> <string>active_sense_method_id</string> </key>
-            <value> <string>Alarm_senseUpgradeNeed</string> </value>
+            <value> <string>Alarm_runPreUpgrade</string> </value>
         </item>
         <item>
             <key> <string>alarm_notification_mode</string> </key>
@@ -18,6 +18,10 @@
               </tuple>
             </value>
         </item>
+        <item>
+            <key> <string>automatic_solve</string> </key>
+            <value> <int>0</int> </value>
+        </item>
         <item>
             <key> <string>description</string> </key>
             <value> <string>This step will run all constraints that have contraint_type equal pre_upgrade.</string> </value>
@@ -97,7 +101,9 @@
         </item>
         <item>
             <key> <string>solve_method_id</string> </key>
-            <value> <string>Alarm_runPreUpgrade</string> </value>
+            <value>
+              <none/>
+            </value>
         </item>
         <item>
             <key> <string>title</string> </key>

bt5/erp5_upgrader/PathTemplateItem/portal_alarms/upgrader_check_upgrader.xml

@@ -8,7 +8,7 @@
       <dictionary>
         <item>
             <key> <string>active_sense_method_id</string> </key>
-            <value> <string>Alarm_senseUpgradeNeed</string> </value>
+            <value> <string>Alarm_runUpgrader</string> </value>
         </item>
         <item>
             <key> <string>alarm_notification_mode</string> </key>
@@ -18,6 +18,10 @@
               </tuple>
             </value>
         </item>
+        <item>
+            <key> <string>automatic_solve</string> </key>
+            <value> <int>0</int> </value>
+        </item>
         <item>
             <key> <string>description</string> </key>
             <value> <string>This step will run all constraints that have contraint_type equal upgrader.</string> </value>
@@ -97,7 +101,9 @@
         </item>
         <item>
             <key> <string>solve_method_id</string> </key>
-            <value> <string>Alarm_runUpgrader</string> </value>
+            <value>
+              <none/>
+            </value>
         </item>
         <item>
             <key> <string>title</string> </key>

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Alarm_runFullUpgrader.py

-"""
-  Run upgrader
+"""Run all upgrader steps:
+
+ - pre-upgrade
+ - upgrade
+ - post-upgrade
+
 """
 portal = context.getPortalObject()
 portal_alarms = portal.portal_alarms
 
-def launchUpgraderAlarm(alarm_id, after_tag=None):
-  """ Get the alarm and use sense and solve """
-  if after_tag is None:
-    after_tag = []
-  upgrader_alarm = getattr(portal_alarms, alarm_id, None)
-  if upgrader_alarm is not None and (force or upgrader_alarm.sense()):
-    # call solve method
-    tag = alarm_id
-    activate_kw = dict(tag=tag)
-    activate_kw["after_tag"] = after_tag
-    method_id = upgrader_alarm.getSolveMethodId()
-    if method_id not in (None, ''):
-      method = getattr(upgrader_alarm.activate(**activate_kw), method_id)
-      method(force=force, activate_kw=activate_kw)
-    return [tag] + after_tag
-  return after_tag
-
-previous_tag = launchUpgraderAlarm('upgrader_check_pre_upgrade')
-
-previous_tag = launchUpgraderAlarm('upgrader_check_upgrader',
-                                   after_tag=previous_tag)
-
-previous_tag = launchUpgraderAlarm('upgrader_check_post_upgrade',
-                                   after_tag=previous_tag)
+pre_upgrade_tag = '%s-preupgrade' % tag
+upgrade_tag = '%s-upgrade' % tag
+post_upgrade_tag = '%s-postupgrade' % tag
+
+portal_alarms.upgrader_check_pre_upgrade.activate(
+  activity='SQLQueue',
+  tag=pre_upgrade_tag,
+).activeSense(fixit=fixit, params={'tag': pre_upgrade_tag})
+
+portal_alarms.upgrader_check_upgrader.activate(
+  activity='SQLQueue',
+  tag=upgrade_tag,
+  after_tag=pre_upgrade_tag,
+ ).activeSense(fixit=fixit, params={'tag': upgrade_tag})
+
+portal_alarms.upgrader_check_post_upgrade.activate(
+  activity='SQLQueue',
+  tag=post_upgrade_tag,
+  after_tag=upgrade_tag,
+).activeSense(fixit=fixit, params={'tag': post_upgrade_tag})
+
+
+# start another activity to collect the results from each upgrader step
+active_process = context.newActiveProcess()
+context.activate(after_tag=post_upgrade_tag).Alarm_postFullUpgradeNeed(
+  active_process=active_process.getRelativeUrl())
 
 # Nothing else to do, so we can disable.
 context.setEnabled(False)
-return

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Alarm_runFullUpgrader.xml

@@ -50,7 +50,7 @@
         </item>
         <item>
             <key> <string>_params</string> </key>
-            <value> <string>force=0, **kw</string> </value>
+            <value> <string>tag, fixit=False</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Alarm_runPostUpgrade.py

 """
   Run Post upgrade
 """
-portal_alarms = context.getPortalObject().portal_alarms
-active_process = context.newActiveProcess()
+activate_kw = params or {}
 
-# We should not run post upgrade if upgrader was not solved or never executed
-alarm = getattr(portal_alarms, 'upgrader_check_upgrader')
-if not(force) and alarm.sense() in (None, True):
-  active_process.postActiveResult(summary=context.getTitle(),
-      severity=1,
-      detail=["Is required run upgrade before solve it. You need run active sense once at least on this alarm"])
-  return
+with context.defaultActivateParameterDict(activate_kw, placeless=True):
+  active_process = context.newActiveProcess()
 
-context.ERP5Site_checkUpgraderConsistency(fixit=True,
+context.ERP5Site_checkUpgraderConsistency(fixit=fixit,
   activate_kw=activate_kw,
   active_process=active_process,
   filter_dict={"constraint_type": "post_upgrade"})
 
 context.setEnabled(False)
-return

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Alarm_runPostUpgrade.xml

@@ -50,7 +50,7 @@
         </item>
         <item>
             <key> <string>_params</string> </key>
-            <value> <string>force=0, activate_kw={}, **kw</string> </value>
+            <value> <string>tag, fixit, params</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Alarm_runPreUpgrade.py

 """
   Run Pre upgrade
 """
-context.ERP5Site_checkUpgraderConsistency(fixit=True,
+activate_kw = params or {}
+
+with context.defaultActivateParameterDict(activate_kw, placeless=True):
+  active_process = context.newActiveProcess()
+
+context.ERP5Site_checkUpgraderConsistency(fixit=fixit,
   activate_kw=activate_kw,
-  active_process=context.newActiveProcess(),
+  active_process=active_process,
   filter_dict={"constraint_type": "pre_upgrade"})
 
 context.setEnabled(False)
-return

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Alarm_runPreUpgrade.xml

@@ -50,7 +50,7 @@
         </item>
         <item>
             <key> <string>_params</string> </key>
-            <value> <string>force=0, activate_kw={}, **kw</string> </value>
+            <value> <string>tag, fixit, params</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Alarm_runUpgrader.py

 """
-  Run Upgrader
+  Run Upgrade step
 
   IMPORTANT: Don't use the constraint_type upgrader to data migration or big amount of objects,
   because this step is suppose to run all constraints in the same transaction. 
@@ -9,15 +9,6 @@
 REINDEX_SPLIT_COUNT = 100
 portal = context.getPortalObject()
 portal_alarms = portal.portal_alarms
-active_process = context.newActiveProcess()
-
-# We should not run upgrader if pre upgrade was not solved or never executed 
-alarm = getattr(portal_alarms, 'upgrader_check_pre_upgrade')
-if not(force) and alarm.sense() in (None, True):
-  active_process.postActiveResult(summary=context.getTitle(),
-      severity=1,
-      detail=["Is required solve Pre Upgrade first. You need run active sense once at least on this alarm"])
-  return
 
 _, type_per_constraint_type = context.Base_getConstraintTypeListPerPortalType()
 portal_type_list = type_per_constraint_type.get('upgrader', [])
@@ -26,12 +17,19 @@ tool_portal_type = 'Template Tool'
 if tool_portal_type in portal_type_list:
   portal_type_list.remove(tool_portal_type)
 
-method_kw = {'fixit': True,
+activate_kw = params or {}
+
+with context.defaultActivateParameterDict(activate_kw, placeless=True):
+  active_process = context.newActiveProcess()
+
+  method_kw = {'fixit': fixit,
     'filter': {"constraint_type": 'upgrader'},
-  'active_process': active_process.getRelativeUrl()}
+    'active_process': active_process.getRelativeUrl(),
+    'activate_kw': activate_kw,
+  }
 
-portal.portal_templates.Base_postCheckConsistencyResult(**method_kw)
-for portal_type in portal_type_list:
+  portal.portal_templates.Base_postCheckConsistencyResult(**method_kw)
+  for portal_type in portal_type_list:
     if portal.portal_catalog.countResults(
         portal_type=portal_type_list)[0][0] > REINDEX_SPLIT_COUNT:
       portal.portal_catalog.searchAndActivate('Base_postCheckConsistencyResult',
@@ -43,4 +41,3 @@ for portal_type in portal_type_list:
         result.Base_postCheckConsistencyResult(**method_kw)
 
 context.setEnabled(False)
-return

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Alarm_runUpgrader.xml

@@ -50,7 +50,7 @@
         </item>
         <item>
             <key> <string>_params</string> </key>
-            <value> <string>force=0, activate_kw={}, **kw</string> </value>
+            <value> <string>tag, fixit, params</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Alarm_senseFullUpgradeNeed.py

-"""
-  Run upgrader
-"""
-portal = context.getPortalObject()
-portal_alarms = portal.portal_alarms
-
-after_method_id = 'Base_postCheckConsistencyResult'
-def launchSenseAlarm(alarm_id, after_tag=[]):
-  """ Get the alarm and use sense"""
-  upgrader_alarm = getattr(portal_alarms, alarm_id, None)
-  if upgrader_alarm is not None:
-    # call solve method
-    kw = {"tag": alarm_id,
-      "after_method_id": after_method_id}
-    if len(after_tag) > 0:
-      kw["after_tag"] = after_tag
-    method_id = upgrader_alarm.getActiveSenseMethodId()
-    if method_id not in (None, ''):
-      method = getattr(upgrader_alarm.activate(**kw), method_id)
-      method()
-    return [alarm_id,]
-  return after_tag
-
-previous_tag = launchSenseAlarm('upgrader_check_pre_upgrade')
-
-previous_tag = launchSenseAlarm('upgrader_check_upgrader',
-                                   after_tag=previous_tag)
-
-previous_tag = launchSenseAlarm('upgrader_check_post_upgrade',
-                 after_tag=previous_tag)
-
-active_process = context.newActiveProcess()
-context.activate(after_tag=previous_tag,
-  after_method_id=after_method_id).Alarm_postFullUpgradeNeed(
-  active_process=active_process.getRelativeUrl())

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Alarm_senseFullUpgradeNeed.xml

-<?xml version="1.0"?>
-<ZopeData>
-  <record id="1" aka="AAAAAAAAAAE=">
-    <pickle>
-      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
-        <item>
-            <key> <string>_bind_names</string> </key>
-            <value>
-              <object>
-                <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
-                </klass>
-                <tuple/>
-                <state>
-                  <dictionary>
-                    <item>
-                        <key> <string>_asgns</string> </key>
-                        <value>
-                          <dictionary>
-                            <item>
-                                <key> <string>name_container</string> </key>
-                                <value> <string>container</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_context</string> </key>
-                                <value> <string>context</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_m_self</string> </key>
-                                <value> <string>script</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_subpath</string> </key>
-                                <value> <string>traverse_subpath</string> </value>
-                            </item>
-                          </dictionary>
-                        </value>
-                    </item>
-                  </dictionary>
-                </state>
-              </object>
-            </value>
-        </item>
-        <item>
-            <key> <string>_params</string> </key>
-            <value> <string>REQUEST=None, **kw</string> </value>
-        </item>
-        <item>
-            <key> <string>id</string> </key>
-            <value> <string>Alarm_senseFullUpgradeNeed</string> </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
-</ZopeData>

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Alarm_senseUpgradeNeed.py

-"""
-  Check if upgrader is required
-"""
-constraint_type = context.getId().replace("upgrader_check_", "")
-context.ERP5Site_checkUpgraderConsistency(
-    active_process=context.newActiveProcess(),
-    filter_dict={"constraint_type": constraint_type})

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Alarm_senseUpgradeNeed.xml

-<?xml version="1.0"?>
-<ZopeData>
-  <record id="1" aka="AAAAAAAAAAE=">
-    <pickle>
-      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
-        <item>
-            <key> <string>_bind_names</string> </key>
-            <value>
-              <object>
-                <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
-                </klass>
-                <tuple/>
-                <state>
-                  <dictionary>
-                    <item>
-                        <key> <string>_asgns</string> </key>
-                        <value>
-                          <dictionary>
-                            <item>
-                                <key> <string>name_container</string> </key>
-                                <value> <string>container</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_context</string> </key>
-                                <value> <string>context</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_m_self</string> </key>
-                                <value> <string>script</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_subpath</string> </key>
-                                <value> <string>traverse_subpath</string> </value>
-                            </item>
-                          </dictionary>
-                        </value>
-                    </item>
-                  </dictionary>
-                </state>
-              </object>
-            </value>
-        </item>
-        <item>
-            <key> <string>_params</string> </key>
-            <value> <string>REQUEST=None, **kw</string> </value>
-        </item>
-        <item>
-            <key> <string>id</string> </key>
-            <value> <string>Alarm_senseUpgradeNeed</string> </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
-</ZopeData>

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Base_postCheckConsistencyResult.py

@@ -4,7 +4,8 @@ active_process = context.getPortalObject().restrictedTraverse(active_process)
 if active_process.ActiveProcess_sense() and not fixit:
   return
 
-constraint_message_list = context.checkConsistency(
+with context.defaultActivateParameterDict(activate_kw, placeless=True):
+  constraint_message_list = context.checkConsistency(
     fixit=fixit, filter=filter,)
 
 if constraint_message_list and not active_process.getResultList():

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/Base_postCheckConsistencyResult.xml

@@ -50,7 +50,7 @@
         </item>
         <item>
             <key> <string>_params</string> </key>
-            <value> <string>fixit, filter, active_process</string> </value>
+            <value> <string>fixit, filter, active_process, activate_kw</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>

bt5/erp5_upgrader/SkinTemplateItem/portal_skins/erp5_upgrader/ERP5Site_checkUpgraderConsistency.py

@@ -18,5 +18,6 @@ if portal_type_list:
       'fixit': fixit,
       'filter': filter_dict,
       'active_process': active_process.getRelativeUrl(),
+      'activate_kw': activate_kw,
     },
   )