WIP: Inventory api: interpolation + group by time sequence

jerome/erp5!6 rebased and in a big
commit so that we can start using that
SimulationTool: implement "linear" flow API

Conflicts:
	product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Resource_zGetInventoryList.xml

testInventoryAPI: silent pyflakes warnings

more interpolation implementation

inventory_list interpolation flow: support at_date & to_date

more and more interpolation API

interpolation

interpolation

group_by_time_sequence_list

testInventoryAPI: move inventory valuation methods in a dedicated test class

This should not be in test inventory list.

test: move Tracking API to a dedicated test module

inventory api: dirty way of disabling cache if keys such as node_category are used.

interpolation: security on SimulationTool_zGetInterpolationMethod

product/ERP5/Tool/SimulationTool.py

@@ -560,6 +560,8 @@ class SimulationTool(BaseTool):
         omit_output=0,
         omit_asset_increase=0,
         omit_asset_decrease=0,
+        # interpolation_method
+        interpolation_method='default',
         # group by
         group_by_node=0,
         group_by_node_category=0,
@@ -596,6 +598,7 @@ class SimulationTool(BaseTool):
         group_by_function_category=0,
         group_by_function_category_strict_membership=0,
         group_by_date=0,
+        group_by_time_sequence_list=None,
         # sort_on
         sort_on=None,
         group_by=None,
@@ -649,6 +652,43 @@ class SimulationTool(BaseTool):
           date_dict['range'] = 'ngt'
         if date_dict:
           column_value_dict['date'] = date_dict
+        if interpolation_method != 'default':
+          if not group_by_time_sequence_list:
+            if not (from_date and (to_date or at_date)):
+              raise ValueError("date_range is required to use interpolation_method")
+            # if we consider flow, we also select movement whose mirror date is
+            # in the from_date/to_date range and movement whose
+            # start_date/stop_date contains the report range.
+            # The selected range is wider, but the selected movements will have an
+            # "interpolation_ratio" applied to their quantity and prices.
+            if to_date:
+              column_value_dict['date'] = ComplexQuery(
+                    Query(date=(from_date, to_date), range='minmax'),
+                    Query(mirror_date=(from_date, to_date), range='minmax'),
+                    ComplexQuery(
+                        Query(mirror_date=from_date, range='min'),
+                        Query(date=to_date, range='max'),
+                        operator="AND"),
+                    ComplexQuery(
+                        Query(date=from_date, range='min'),
+                        Query(mirror_date=to_date, range='max'),
+                        operator="AND"),
+                    operator="OR"
+                )
+            else:
+              column_value_dict['date'] = ComplexQuery(
+                    Query(date=(from_date, at_date), range='minngt'),
+                    Query(mirror_date=(from_date, at_date), range='minngt'),
+                    ComplexQuery(
+                        Query(mirror_date=from_date, range='min'),
+                        Query(date=at_date, range='ngt'),
+                        operator="AND"),
+                    ComplexQuery(
+                        Query(date=from_date, range='min'),
+                        Query(mirror_date=at_date, range='ngt'),
+                        operator="AND"),
+                    operator="OR"
+                )
       else:
         column_value_dict['date'] = {'query': [to_date], 'range': 'ngt'}
         column_value_dict['mirror_date'] = {'query': [from_date], 'range': 'nlt'}
@@ -948,6 +988,8 @@ class SimulationTool(BaseTool):
         new_kw['related_key_select_expression_list'] =\
                 related_key_select_expression_list
 
+# XXX
+      sql_kw['group_by_time_sequence_list'] = group_by_time_sequence_list
       return sql_kw, new_kw
 
     #######################################################
@@ -1032,6 +1074,17 @@ class SimulationTool(BaseTool):
       output_simulation_state - only take rows with specified simulation_state
                         and quantity < 0
 
+      interpolation_method - Method to consider movements when calculating flows.
+        * (default): Consider the movement decreases 100% of source stock on
+          start date and increase 100% of the destination node stock on stop
+          date.
+        * linear: consider the movement decreases source stock and increase
+          destination stock linearly between start date and stop date.
+        * all_or_nothing: consider only movement who starts after the beginning of
+          the query period and finishes after the end of the query period.
+        * one_for_all: consider the movement fully as long as it is partially
+          contained in the query period.
+
       ignore_variation -  do not take into account variation in inventory
                         calculation (useless on getInventory, but useful on
                         getInventoryList)
@@ -1158,6 +1211,7 @@ class SimulationTool(BaseTool):
         group_by_section_category=0,
         group_by_section_category_strict_membership=0,
         group_by_resource=None,
+        group_by_time_sequence_list=(),
         group_by=None,
         **ignored):
       """
@@ -1177,7 +1231,8 @@ class SimulationTool(BaseTool):
            group_by_function or group_by_mirror_section or group_by_payment or \
            group_by_sub_variation or group_by_variation or \
            group_by_movement or group_by_date or group_by_section_category or\
-           group_by_section_category_strict_membership:
+           group_by_section_category_strict_membership or \
+           group_by_time_sequence_list:
           if group_by_resource is None:
             group_by_resource = 1
           new_group_by_dict['group_by_resource'] = group_by_resource
@@ -1194,6 +1249,7 @@ class SimulationTool(BaseTool):
                          omit_simulation=0,
                          only_accountable=True,
                          default_stock_table='stock',
+                         interpolation_method='default',
                          selection_domain=None, selection_report=None,
                          statistic=0, inventory_list=1,
                          precision=None, connection_id=None,
@@ -1272,6 +1328,7 @@ class SimulationTool(BaseTool):
         'stock_table_id': default_stock_table,
         'src__': src__,
         'ignore_variation': ignore_variation,
+        'interpolation_method': interpolation_method,
         'standardise': standardise,
         'omit_simulation': omit_simulation,
         'only_accountable': only_accountable,
@@ -1287,7 +1344,9 @@ class SimulationTool(BaseTool):
       # Get cached data
       if getattr(self, "Resource_zGetInventoryCacheResult", None) is not None and \
               optimisation__ and (not kw.get('from_date')) and \
-              'transformed_resource' not in kw:
+              'transformed_resource' not in kw \
+              and "category" not in str(kw) \
+              and "group_by_time_sequence_list" not in kw:
         # Here is the different kind of date
         # from_date : >=
         # to_date   : <
@@ -1317,7 +1376,7 @@ class SimulationTool(BaseTool):
           kw['from_date'] = cached_date
       else:
         cached_result = []
-      sql_kw, new_kw = self._generateKeywordDict(**kw)
+      sql_kw, new_kw = self._generateKeywordDict(interpolation_method=interpolation_method, **kw)
       # Copy kw content as _generateSQLKeywordDictFromKeywordDict
       # remove some values from it
       try:
@@ -1330,6 +1389,25 @@ class SimulationTool(BaseTool):
       stock_sql_kw = self._generateSQLKeywordDictFromKeywordDict(
           table=default_stock_table, sql_kw=sql_kw, new_kw=new_kw_copy)
       stock_sql_kw.update(base_inventory_kw)
+
+        # TODO: move in _generateSQLKeywordDictFromKeywordDict
+      if interpolation_method in ('linear', 'all_or_nothing', 'one_for_all'):
+          # XXX only DateTime instance are supported
+        from_date = kw.get('from_date')
+        if from_date:
+          from_date = from_date.toZone("UTC")
+        to_date = kw.get('to_date')
+        if to_date:
+          to_date = to_date.toZone("UTC")
+        at_date = kw.get('at_date')
+        if at_date:
+          at_date = at_date.toZone("UTC")
+        stock_sql_kw['interpolation_method_from_date'] = from_date
+        stock_sql_kw['interpolation_method_to_date'] = to_date
+        stock_sql_kw['interpolation_method_at_date'] = at_date
+      elif interpolation_method != 'default':
+        raise ValueError("Unsupported interpolation_method %r" % (interpolation_method,))
+
       delta_result = self.Resource_zGetInventoryList(
           **stock_sql_kw)
       if src__:

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Resource_zGetInventoryList.sql

+<dtml-let interpolation_ratio="SimulationTool_zGetInterpolationMethod(
+  stock_table_id=stock_table_id,
+  interpolation_method=interpolation_method,
+  interpolation_method_from_date=interpolation_method_from_date,
+  interpolation_method_to_date=interpolation_method_to_date,
+  interpolation_method_at_date=interpolation_method_at_date,
+  group_by_time_sequence_list=group_by_time_sequence_list,
+  src__=1)">
+
 SELECT
 <dtml-if expr="precision is not None">
-  SUM(ROUND(<dtml-var stock_table_id>.quantity
-    <dtml-if transformed_uid> * transformation.quantity</dtml-if>, <dtml-var precision>)) AS inventory,
-  SUM(ROUND(<dtml-var stock_table_id>.quantity
-    <dtml-if transformed_uid> * transformation.quantity</dtml-if>, <dtml-var precision>)) AS total_quantity,
+  SUM(ROUND(
+    <dtml-var stock_table_id>.quantity
+    <dtml-if transformed_uid> * transformation.quantity</dtml-if>
+    * <dtml-var interpolation_ratio>, <dtml-var precision>)) AS inventory,
+  SUM(ROUND(
+    <dtml-var stock_table_id>.quantity
+    <dtml-if transformed_uid> * transformation.quantity</dtml-if>
+    * <dtml-var interpolation_ratio>, <dtml-var precision>)) AS total_quantity,
   <dtml-if convert_quantity_result>
     SUM(ROUND(<dtml-var stock_table_id>.quantity * measure.quantity
       <dtml-if quantity_unit_uid> / quantity_unit_conversion.quantity</dtml-if>
-      <dtml-if transformed_uid> * transformation.quantity</dtml-if>, <dtml-var precision>))
+    * <dtml-var interpolation_ratio>, <dtml-var precision>))
     AS converted_quantity,
   </dtml-if>
-  IFNULL(SUM(ROUND(<dtml-var stock_table_id>.total_price, <dtml-var precision>)), 0) AS total_price
+
+  IFNULL(SUM(ROUND(
+    <dtml-var stock_table_id>.total_price * <dtml-var interpolation_ratio>, <dtml-var precision>)), 0) AS total_price
 <dtml-else>
-  SUM(<dtml-var stock_table_id>.quantity <dtml-if transformed_uid> * transformation.quantity</dtml-if>) AS inventory,
-  SUM(<dtml-var stock_table_id>.quantity <dtml-if transformed_uid> * transformation.quantity</dtml-if>) AS total_quantity,
+  SUM(<dtml-var stock_table_id>.quantity
+      <dtml-if transformed_uid> * transformation.quantity</dtml-if>
+      * <dtml-var interpolation_ratio>
+     ) AS inventory,
+  SUM(<dtml-var stock_table_id>.quantity
+      <dtml-if transformed_uid> * transformation.quantity</dtml-if>
+      * <dtml-var interpolation_ratio>
+     ) AS total_quantity,
   <dtml-if convert_quantity_result>
     ROUND(SUM(<dtml-var stock_table_id>.quantity * measure.quantity
       <dtml-if quantity_unit_uid> / quantity_unit_conversion.quantity</dtml-if>
-      <dtml-if transformed_uid> * transformation.quantity</dtml-if>), 12)
+      <dtml-if transformed_uid> * transformation.quantity</dtml-if> * <dtml-var interpolation_ratio>), 12)
     AS converted_quantity,
   </dtml-if>
-  IFNULL(SUM(<dtml-var stock_table_id>.total_price), 0) AS total_price
+  IFNULL(SUM(<dtml-var stock_table_id>.total_price * <dtml-var interpolation_ratio>), 0) AS total_price
 </dtml-if>
 <dtml-if inventory_list>
   ,
@@ -56,6 +77,8 @@ SELECT
   COUNT(DISTINCT <dtml-var stock_table_id>.uid) AS stock_uid,
   MAX(<dtml-var stock_table_id>.date) AS date
 </dtml-if>
+<dtml-if group_by_time_sequence_list>, slot_index </dtml-if> <dtml-comment>XXX is this really needed? are empty slots returned ? </dtml-comment>
+
 <dtml-if select_expression>, <dtml-var select_expression></dtml-if>
 
 FROM
@@ -69,6 +92,55 @@ FROM
   </dtml-if>
 </dtml-in>
 , <dtml-var stock_table_id>
+
+   <dtml-if group_by_time_sequence_list>
+     RIGHT JOIN
+       ( <dtml-in prefix="time_slot" expr="_.list(_.enumerate(group_by_time_sequence_list))">
+         SELECT
+           <dtml-sqlvar expr="time_slot_key" type="int"> slot_index,
+           <dtml-sqlvar expr="time_slot_item.get('from_date')" type="datetime" optional> slot_from_date,
+           <dtml-sqlvar expr="time_slot_item.get('at_date')" type="datetime" optional> slot_at_date,
+           <dtml-sqlvar expr="time_slot_item.get('to_date')" type="datetime" optional> slot_to_date
+
+         <dtml-unless time_slot_end>UNION ALL</dtml-unless>
+       </dtml-in> ) slots
+     ON
+     <dtml-if group_by_time_sequence_list>
+     (
+       ( slot_from_date is not null AND
+        ( slot_at_date is not null AND
+         GREATEST(`stock`.`date`, `stock`.`mirror_date`) >= slot_from_date AND
+         LEAST(`stock`.`date`, `stock`.`mirror_date`) <= slot_at_date
+        ) OR (
+          (
+            slot_to_date is not null AND
+            GREATEST(`stock`.`date`, `stock`.`mirror_date`) >= slot_from_date AND
+            LEAST(`stock`.`date`, `stock`.`mirror_date`) < slot_to_date
+          ) OR (
+            GREATEST(`stock`.`date`, `stock`.`mirror_date`) >= slot_from_date AND
+            slot_at_date is null AND slot_to_date is null
+          )
+        )
+       ) OR (
+         slot_from_date is null AND (
+           ( slot_at_date is not null AND
+            ( LEAST(`stock`.`date`, `stock`.`mirror_date`) <= slot_at_date )
+           ) OR  LEAST(`stock`.`date`, `stock`.`mirror_date`) < slot_to_date
+         )
+       )
+     )
+     <dtml-else>
+     (
+       ( slot_from_date is null OR stock.date >= slot_from_date )
+       AND ( slot_at_date is null OR stock.date <= slot_at_date )
+       AND ( slot_to_date is null OR stock.date < slot_to_date )
+     )
+     </dtml-if>
+   </dtml-if>
+
+
+
+
 </dtml-if>
 <dtml-if quantity_unit_uid> <dtml-comment>XXX quantity unit conversion will not work when using implict_join=False</dtml-comment>
   LEFT JOIN quantity_unit_conversion ON 
@@ -116,9 +188,16 @@ WHERE
 <dtml-if group_by_expression>
 GROUP BY
     <dtml-if transformed_uid>transformation.transformed_uid,</dtml-if>
+    <dtml-if group_by_time_sequence_list>slot_index,</dtml-if>
     <dtml-var group_by_expression>
+
 </dtml-if>
 <dtml-if order_by_expression>
 ORDER BY
   <dtml-var order_by_expression>
+<dtml-else>
+  <dtml-if group_by_time_sequence_list>
+    ORDER BY slot_index
+  </dtml-if>
 </dtml-if>
+</dtml-let>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Resource_zGetInventoryList.xml

@@ -45,7 +45,12 @@ convert_quantity_result\r\n
 quantity_unit_uid\r\n
 stock_table_id=stock\r\n
 transformed_uid\r\n
-transformed_variation_text</string> </value>
+transformed_variation_text\r\n
+group_by_time_sequence_list:list\r\n
+interpolation_method\r\n
+interpolation_method_from_date=not_applicable\r\n
+interpolation_method_to_date=not_applicable\r\n
+interpolation_method_at_date=not_applicable</string> </value>
         </item>
         <item>
             <key> <string>cache_time_</string> </key>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/SimulationTool_zGetInterpolationMethod.sql

+<dtml-if expr="interpolation_method == 'linear'">
+  <dtml-if group_by_time_sequence_list>
+    CASE
+     WHEN <dtml-var stock_table_id>.mirror_date = <dtml-var stock_table_id>.date THEN 1
+     ELSE (
+       UNIX_TIMESTAMP(
+         IFNULL(
+           LEAST(
+             IFNULL(slot_at_date, slot_to_date),
+             GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
+           ),
+           GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
+         )
+       )
+       - UNIX_TIMESTAMP(
+         GREATEST(
+           IFNULL(
+             slot_from_date, TIMESTAMP(0)
+           ),
+           LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
+         )
+       )
+     )
+     / (
+       UNIX_TIMESTAMP(GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) -
+       UNIX_TIMESTAMP(LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) ) END
+  <dtml-else>
+  CASE
+     WHEN <dtml-var stock_table_id>.mirror_date = <dtml-var stock_table_id>.date THEN 1
+     ELSE (
+      UNIX_TIMESTAMP(LEAST(
+        <dtml-if interpolation_method_at_date>
+          <dtml-sqlvar interpolation_method_at_date type="datetime">
+        <dtml-else>
+          <dtml-sqlvar interpolation_method_to_date type="datetime">
+        </dtml-if>,
+            GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date) ))
+       - UNIX_TIMESTAMP(GREATEST(<dtml-sqlvar interpolation_method_from_date type="datetime">,
+                  LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date))))
+      / ( UNIX_TIMESTAMP(GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) -
+         UNIX_TIMESTAMP(LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) ) END
+  </dtml-if>
+<dtml-elif expr="interpolation_method == 'all_or_nothing'">
+  CASE
+    WHEN (
+      -- movement contained in time frame
+      LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
+        >= <dtml-sqlvar interpolation_method_from_date type="datetime"> AND
+      GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
+      <dtml-if interpolation_method_at_date>
+        <= <dtml-sqlvar interpolation_method_at_date type="datetime">
+      <dtml-else>
+        < <dtml-sqlvar interpolation_method_to_date type="datetime">
+      </dtml-if>
+    ) THEN 1
+    ELSE 0
+  END
+<dtml-elif expr="interpolation_method == 'one_for_all'">
+  CASE
+    WHEN (
+      -- movement overlaps with time frame
+      GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
+        <= <dtml-sqlvar interpolation_method_from_date type="datetime"> OR
+      LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
+      <dtml-if interpolation_method_at_date>
+        >= <dtml-sqlvar interpolation_method_at_date type="datetime">
+      <dtml-else>
+        > <dtml-sqlvar interpolation_method_to_date type="datetime">
+      </dtml-if>
+    ) THEN 0
+    ELSE 1
+  END
+<dtml-else>
+  1
+</dtml-if>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/SimulationTool_zGetInterpolationMethod.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="SQL" module="Products.ZSQLMethods.SQL"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_Use_Database_Methods_Permission</string> </key>
+            <value>
+              <list>
+                <string>Member</string>
+              </list>
+            </value>
+        </item>
+        <item>
+            <key> <string>arguments_src</string> </key>
+            <value> <string>stock_table_id\r\n
+interpolation_method\r\n
+interpolation_method_from_date\r\n
+interpolation_method_to_date\r\n
+interpolation_method_at_date\r\n
+group_by_time_sequence_list:list</string> </value>
+        </item>
+        <item>
+            <key> <string>connection_id</string> </key>
+            <value> <string>cmf_activity_sql_connection</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>SimulationTool_zGetInterpolationMethod</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>