From e8b41f91f4f5fdfab3adb1bd26e6375bae0c3cf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
Date: Fri, 2 Nov 2018 10:38:46 +0000
Subject: [PATCH] trade: split "Lines Report" in activities

Instead of generating the report in one big activity, split the calculation
in several batches, so that the processing can be distributed on zope nodes.

The limit for the number of lines was also increased from 10000 to 100000
---
 .../DeliveryModule_getDeliveryLineList.py     | 18 +----------
 .../DeliveryModule_getDeliveryLineList.xml    |  2 +-
 ...Module_getDeliveryLineReportSectionList.py | 32 +++++++++++++++++--
 3 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/DeliveryModule_getDeliveryLineList.py b/bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/DeliveryModule_getDeliveryLineList.py
index 5dd2e131d40..a764843bc4a 100644
--- a/bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/DeliveryModule_getDeliveryLineList.py
+++ b/bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/DeliveryModule_getDeliveryLineList.py
@@ -2,22 +2,6 @@ from Products.PythonScripts.standard import Object
 line_list = []
 portal = context.getPortalObject()
 
-# XXX use a larger limit
-saved_selection_params = context.getPortalObject().portal_selections.getSelectionParamsFor(module_selection_name)
-selection_params = saved_selection_params.copy()
-selection_params['limit'] = 10000
-context.getPortalObject().portal_selections.setSelectionParamsFor(module_selection_name, selection_params)
-
-try:
-  checked_uid_list = portal.portal_selections.getSelectionCheckedUidsFor(module_selection_name)
-  if checked_uid_list:
-    getObject = portal.portal_catalog.getObject
-    delivery_list = [getObject(uid) for uid in checked_uid_list]
-  else:
-    delivery_list = portal.portal_selections.callSelectionFor(module_selection_name, context=context)
-finally:
-  context.getPortalObject().portal_selections.setSelectionParamsFor(module_selection_name, saved_selection_params)
-
 account_title_cache = {}
 def getAccountTitle(relative_url):
   try:
@@ -32,7 +16,7 @@ def getAccountTitle(relative_url):
     return title
 
 
-for delivery in delivery_list:
+for delivery in portal.portal_catalog(uid=uid_list or -1):
   delivery = delivery.getObject()
   for movement in delivery.getMovementList(portal_type=portal_type):
     line_list.append(Object(
diff --git a/bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/DeliveryModule_getDeliveryLineList.xml b/bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/DeliveryModule_getDeliveryLineList.xml
index 492e7ef117c..56b2f25756f 100644
--- a/bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/DeliveryModule_getDeliveryLineList.xml
+++ b/bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/DeliveryModule_getDeliveryLineList.xml
@@ -50,7 +50,7 @@
         </item>
         <item>
             <key> <string>_params</string> </key>
-            <value> <string>portal_type, module_selection_name, **kw</string> </value>
+            <value> <string>uid_list, portal_type, **kw</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>
diff --git a/bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/DeliveryModule_getDeliveryLineReportSectionList.py b/bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/DeliveryModule_getDeliveryLineReportSectionList.py
index 82a3bdd6fff..2861ee09ce3 100644
--- a/bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/DeliveryModule_getDeliveryLineReportSectionList.py
+++ b/bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/DeliveryModule_getDeliveryLineReportSectionList.py
@@ -1,6 +1,34 @@
 from Products.ERP5Form.Report import ReportSection
+portal = context.getPortalObject()
+
 request = container.REQUEST
-return ReportSection(form_id='DeliveryModule_viewDeliveryLineList',
+module_selection_name = request['selection_name']
+
+# so that ods style does not repeat header on each section
+request.set('merge_report_section_list', 1)
+
+# XXX use a larger limit
+saved_selection_params = portal.portal_selections.getSelectionParamsFor(module_selection_name)
+selection_params = saved_selection_params.copy()
+selection_params['limit'] = 100000
+context.getPortalObject().portal_selections.setSelectionParamsFor(module_selection_name, selection_params)
+
+report_section_list = []
+try:
+  all_uid_list = portal.portal_selections.getSelectionCheckedUidsFor(module_selection_name)
+  if not all_uid_list:
+    all_uid_list = [ x.uid for x in
+      portal.portal_selections.callSelectionFor(module_selection_name, context=context)]
+
+  # split in chunks
+  batch_size = 1000
+  for uid_list in [all_uid_list[x:x+batch_size] for x in xrange(0, len(all_uid_list), batch_size)]:
+    report_section_list.append(
+      ReportSection(form_id='DeliveryModule_viewDeliveryLineList',
                      path=context.getPhysicalPath(),
                      selection_params=dict(portal_type=request['portal_type'],
-                                           module_selection_name=request['selection_name'])),
+                                           uid_list=uid_list,
+                                           module_selection_name=module_selection_name)))
+  return report_section_list
+finally:
+  context.getPortalObject().portal_selections.setSelectionParamsFor(module_selection_name, saved_selection_params)
-- 
2.30.9