Analysis execution methods supporting transient in- and output

bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataAnalysis_executeDataOperation.py

+portal = context.getPortalObject()
+operation = None
+use = None
+parameter_dict = {}
+context.checkConsistency(fixit=True)
+initial_product = context.getSpecialiseValue(portal_type="Data Transformation").getResourceValue()
+for analysis_line in context.objectValues(portal_type="Data Analysis Line"):
+  resource = analysis_line.getResourceValue()
+  if resource == initial_product:
+    use = analysis_line.getUse()
+  if resource is not None:
+    resource_portal_type = resource.getPortalType()
+  else:
+    resource_portal_type = ''
+  if resource_portal_type == 'Data Operation':
+    operation_analysis_line = analysis_line
+    operation = analysis_line.getResourceValue()
+  else:
+    parameter = {}
+    for portal_type in ["Data Array", "Progress Indicator"] + \
+                        list(portal.getPortalDataSinkTypeList()) + \
+                        list(portal.getPortalDataDescriptorTypeList()):
+      value = analysis_line.getAggregateValue(portal_type=portal_type)
+      if value is not None:
+        parameter[portal_type] = value
+    if analysis_line.getQuantity() < 0 and analysis_line.getUse() == "big_data/analysis/transient":
+      # at the moment we only support transient data arrays
+      parameter['Data Array'] = transient_input_item
+    if analysis_line.getQuantity() > 0 and analysis_line.getUse() == "big_data/analysis/transient":
+      # at the moment we only support transient data arrays
+      transient_output_item = portal.data_array_module.newContent(portal_type='Data Array',
+                                                                  temp_object=True)
+      parameter['Data Array'] = transient_output_item
+    for base_category in analysis_line.getVariationRangeBaseCategoryList():
+      parameter[base_category] = analysis_line.getVariationCategoryItemList(
+                                   base_category_list=(base_category,))[0][0]
+    reference = analysis_line.getReference()
+    # several lines with same reference wil turn the parameter into a list
+    if reference in parameter_dict:
+      if not isinstance(parameter_dict[reference], list):
+        parameter_dict[reference] = [parameter_dict[reference]]
+      parameter_dict[reference].append(parameter)
+    else:
+      parameter_dict[reference] = parameter
+script_id = operation.getScriptId()
+out = getattr(operation_analysis_line, script_id)(**parameter_dict)
+
+for consuming_analysis in consuming_analysis_list:
+  portal.restrictedTraverse(consuming_analysis).DataAnalysis_executeDataOperation(transient_input_item = transient_output_item)
+
+if out == 1:
+  context.activate(serialization_tag=str(context.getUid())).DataAnalysis_executeDataOperation(consuming_analysis_list)
+else:
+  # only stop batch ingestions
+  if use == "big_data/ingestion/batch":
+    context.stop()

bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataAnalysis_executeDataOperation.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string>consuming_analysis_list=[], transient_input_item=None</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>DataAnalysis_executeDataOperation</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/ERP5Site_executeDataAnalysisList.py

+portal = context.getPortalObject()
+
+consuming_analysis_list_dict = {}
+
+def add_consuming_analysis(producing_analysis_relative_url, consuming_analysis_relative_url):
+  consuming_analysis_list = consuming_analysis_list_dict.setdefault(producing_analysis_relative_url, [])
+  consuming_analysis_list.append(consuming_analysis_relative_url)
+
+analysis_list = []
+for analysis in portal.portal_catalog(portal_type = "Data Analysis",
+                                           simulation_state = "started"):
+  has_transient_input = False
+  for line in analysis.objectValues(portal_type="Data Analysis Line"):
+    if line.getUse() == "big_data/analysis/transient" and line.getQuantity() < 0:
+      has_transient_input = True
+      add_consuming_analysis(line.getParentValue().getCausality(), line.getParentRelativeUrl())
+  if not has_transient_input:
+    analysis_list.append(analysis)
+
+for analysis in analysis_list:
+  if not analysis.hasActivity():
+    consuming_analysis_list = consuming_analysis_list_dict.get(analysis.getRelativeUrl(), [])
+    analysis.activate(serialization_tag=str(analysis.getUid()))\
+      .DataAnalysis_executeDataOperation(consuming_analysis_list)

bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/ERP5Site_executeDataAnalysisList.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>ERP5Site_executeDataAnalysisList</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>