Skip to content

erp5
erp5
Commit cf53db45 authored by Aurel's avatar Aurel
Browse files
Options

Implement cache of getInventory results 

Create a sql cache for getInventory call
Remove full-inventory optimisation as it now useless
Implement full-inventory feature directly into stock
Make erp5_pdm depends on this optimisation
Install optimisation for all unit tests
parent 3f484980
Show whitespace changes
Inline Side-by-side
Showing with 1649 additions and 1554 deletions
bt5/erp5_banking_core/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_inventory_list.catalog_keys.xml deleted 100644 → 0
View file @ 3f484980
<catalog_method>
<item key="sql_catalog_object_list" type="int">
<value>1</value>
</item>
</catalog_method>
bt5/erp5_banking_core/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_inventory_list.xml deleted 100644 → 0
View file @ 3f484980
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>isInventory\r\n
uid\r\n
getDestinationUid\r\n
getDestinationSectionUid\r\n
getDestinationPaymentUid\r\n
getStartDate\r\n
isFullInventory\r\n
getSimulationState</string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_catalog_inventory_list</string> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
<dtml-let row_list="[]">\n
\n
<dtml-in prefix="loop" expr="_.range(_.len(uid))">\n
<dtml-if expr="isInventory[loop_item]">\n
<dtml-if expr="getStartDate[loop_item] is not None">\n
<dtml-call expr="row_list.append((\n
uid[loop_item],\n
getDestinationUid[loop_item],\n
getDestinationSectionUid[loop_item],\n
getDestinationPaymentUid[loop_item],\n
getStartDate[loop_item],\n
isFullInventory[loop_item],\n
getSimulationState[loop_item]))">\n
</dtml-if>\n
</dtml-if>\n
</dtml-in>\n
\n
<dtml-if expr="len(row_list)">\n
REPLACE INTO\n
inventory\n
(`uid`, `node_uid`, `section_uid`, `payment_uid`, `date`, `is_full_inventory`, `simulation_state`)\n
VALUES\n
<dtml-in prefix="row" expr="row_list">\n
(<dtml-sqlvar expr="row_item[0]" type="int">,\n
<dtml-sqlvar expr="row_item[1]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[2]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[3]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[4]" type="datetime">,\n
<dtml-sqlvar expr="row_item[5] or 0" type="int">,\n
<dtml-sqlvar expr="row_item[6]" type="string">\n
)<dtml-if sequence-end><dtml-else>,</dtml-if>\n
</dtml-in>\n
</dtml-if>\n
</dtml-let>
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_pdm/bt/dependency_list
View file @ cf53db45
erp5_base erp5_base
erp5_stock_cache
bt5/erp5_pdm/bt/revision
View file @ cf53db45
571 573
\ No newline at end of file \ No newline at end of file
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_inventory_stock_list.catalog_keys.xml → bt5/erp5_stock_cache/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_trimInventoryCacheOnCatalog.catalog_keys.xml
View file @ cf53db45
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
<value>1</value> <value>1</value>
</item> </item>
<item key="_filter_expression_archive" type="str"> <item key="_filter_expression_archive" type="str">
<value>python: context.providesIMovement() and context.isInventoryMovement()</value> <value>python: context.providesIMovement() and not context.isInventoryMovement()</value>
</item> </item>
<item key="_filter_expression_cache_key_archive" type="tuple"> <item key="_filter_expression_cache_key_archive" type="tuple">
<value>portal_type</value> <value>portal_type</value>
......
bt5/erp5_stock_cache/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_trimInventoryCacheOnCatalog.xml 0 → 100644
View file @ cf53db45
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>from Products.ERP5Type.Errors import ProgrammingError\n
\n
min_date = None\n
for loop_item in xrange(len(uid)):\n
if not isInventoryMovement[loop_item] and isMovement[loop_item] and getResourceUid[loop_item]:\n
if getDestinationUid[loop_item] and getStopDate[loop_item]:\n
if min_date:\n
min_date = min(min_date, getStopDate[loop_item])\n
else:\n
min_date = getStopDate[loop_item]\n
if getSourceUid[loop_item] and getStartDate[loop_item]:\n
if min_date:\n
min_date = min(min_date, getStartDate[loop_item])\n
else:\n
min_date = getStartDate[loop_item]\n
if min_date:\n
try:\n
context.SimulationTool_zTrimInventoryCacheFromDateOnCatalog(date=min_date)\n
except ProgrammingError:\n
# Create table if it does not exits\n
# Then no need to flush an empty table\n
context.SimulationTool_zCreateInventoryCache()\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>uid, isMovement, isInventoryMovement, getResourceUid, getDestinationUid, getStopDate, getSourceUid, getStartDate</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_trimInventoryCacheOnCatalog</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_stock_cache/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_trimInventoryCacheOnUncatalog.xml 0 → 100644
View file @ cf53db45
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>from Products.ERP5Type.Errors import ProgrammingError\n
\n
try:\n
context.SimulationTool_zTrimInventoryCacheFromDateOnUncatalog(uid=uid)\n
except ProgrammingError:\n
# Create table if it does not exits\n
# Then no need to flush an empty table\n
context.SimulationTool_zCreateInventoryCache()\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>uid</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_trimInventoryCacheOnUncatalog</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache.xml 0 → 100644
View file @ cf53db45
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Folder" module="OFS.Folder"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_objects</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>erp5_stock_cache</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Resource_zGetFullInventoryDate.xml → bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/Resource_zGetInventoryCacheResult.xml
View file @ cf53db45
...@@ -6,14 +6,32 @@ ...@@ -6,14 +6,32 @@
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item> <item>
<key> <string>arguments_src</string> </key> <key> <string>arguments_src</string> </key>
<value> <string>column_value_list_list:list\r\n <value> <string>query\r\n
column_id_list:list\r\n date</string> </value>
group_by_expression\r\n </item>
date\r\n <item>
simulation_state=delivered\r\n <key> <string>cache_time_</string> </key>
where_expression</string> </value> <value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value> <string></string> </value>
</item> </item>
<item> <item>
<key> <string>connection_id</string> </key> <key> <string>connection_id</string> </key>
...@@ -21,39 +39,31 @@ where_expression</string> </value> ...@@ -21,39 +39,31 @@ where_expression</string> </value>
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>Resource_zGetFullInventoryDate</string> </value> <value> <string>Resource_zGetInventoryCacheResult</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1</int> </value>
</item> </item>
<item> <item>
<key> <string>src</string> </key> <key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[ <value> <string encoding="cdata"><![CDATA[
SELECT\n SELECT \n
MAX(date) as date,\n date, \n
<dtml-var group_by_expression>\n result \n
FROM\n FROM \n
inventory\n inventory_cache \n
WHERE\n WHERE \n
is_full_inventory = TRUE\n inventory_cache.query=<dtml-sqlvar query type="string"> \n
<dtml-if simulation_state>\n AND \n
AND simulation_state = <dtml-sqlvar simulation_state type="string">\n inventory_cache.date <= <dtml-sqlvar date type="datetime"> \n
</dtml-if>\n ORDER BY \n
<dtml-in prefix="loop" expr="_.range(_.len(column_id_list))">\n date DESC
AND <dtml-var expr="column_id_list[loop_item]"> IN (\n
<dtml-in expr="column_value_list_list[loop_item]">\n
<dtml-sqlvar sequence-item type="int">\n
<dtml-if sequence-end><dtml-else>,</dtml-if>\n
</dtml-in>\n
)\n
</dtml-in>\n
<dtml-if date>\n
AND date <= <dtml-sqlvar date type="string">\n
</dtml-if>\n
<dtml-if where_expression>\n
AND <dtml-var where_expression>\n
</dtml-if>\n
GROUP BY\n
<dtml-var group_by_expression>\n
]]></string> </value> ]]></string> </value>
</item> </item>
......
bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/Resource_zInsertInventoryCacheResult.xml 0 → 100644
View file @ cf53db45
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>query\r\n
date\r\n
result</string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Resource_zInsertInventoryCacheResult</string> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
Insert into inventory_cache(`query`, `date`, `result`) values (<dtml-sqlvar query type="string">, <dtml-sqlvar date type="datetime">, <dtml-sqlvar result type="string">)
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/SimulationTool_flushInventoryCache.xml 0 → 100644
View file @ cf53db45
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string># XXX To be implemented\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SimulationTool_flushInventoryCache</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/SimulationTool_getInventoryCacheLag.xml 0 → 100644
View file @ cf53db45
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>"""\n
Returns a duration, in days, for stock cache management.\n
If data in stock cache is older than lag compared to query\'s date\n
(at_date or to_date), then it becomes a "soft miss": use found value,\n
but add a new entry to cache at query\'s date minus half the lag.\n
So this value should be:\n
- Small enough that few enough rows need to be table-scanned for\n
verage queries (probably queries against current date).\n
- Large enough that few enough documents get modified past that date,\n
therwise cache entries would be removed from cache all the time.\n
"""\n
\n
return 60\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SimulationTool_getInventoryCacheLag</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_inventory.xml → bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/SimulationTool_zCreateInventoryCache.xml
View file @ cf53db45
...@@ -16,11 +16,17 @@ ...@@ -16,11 +16,17 @@
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>z0_drop_inventory</string> </value> <value> <string>SimulationTool_zCreateInventoryCache</string> </value>
</item> </item>
<item> <item>
<key> <string>src</string> </key> <key> <string>src</string> </key>
<value> <string>DROP TABLE IF EXISTS inventory</string> </value> <value> <string>Create table `inventory_cache` (\n
`query` BINARY(16) NOT NULL,\n
`date` datetime NOT NULL,\n
`result` LONGBLOB NOT NULL,\n
PRIMARY KEY (`query`, `date`),\n
KEY (`date`)\n
) Engine=InnoDB</string> </value>
</item> </item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_inventory_stock.xml → bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/SimulationTool_zDropInventoryCache.xml
View file @ cf53db45
...@@ -6,6 +6,12 @@ ...@@ -6,6 +6,12 @@
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item> <item>
<key> <string>arguments_src</string> </key> <key> <string>arguments_src</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
...@@ -16,11 +22,11 @@ ...@@ -16,11 +22,11 @@
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>z0_drop_inventory_stock</string> </value> <value> <string>SimulationTool_zDropInventoryCache</string> </value>
</item> </item>
<item> <item>
<key> <string>src</string> </key> <key> <string>src</string> </key>
<value> <string>DROP TABLE IF EXISTS inventory_stock</string> </value> <value> <string>DROP TABLE IF EXISTS inventory_cache</string> </value>
</item> </item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_inventory_stock.xml → bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/SimulationTool_zTrimInventoryCacheFromDateOnCatalog.xml
View file @ cf53db45
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
<dictionary> <dictionary>
<item> <item>
<key> <string>arguments_src</string> </key> <key> <string>arguments_src</string> </key>
<value> <string>uid</string> </value> <value> <string>date</string> </value>
</item> </item>
<item> <item>
<key> <string>connection_id</string> </key> <key> <string>connection_id</string> </key>
...@@ -16,13 +16,16 @@ ...@@ -16,13 +16,16 @@
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>z0_uncatalog_inventory_stock</string> </value> <value> <string>SimulationTool_zTrimInventoryCacheFromDateOnCatalog</string> </value>
</item> </item>
<item> <item>
<key> <string>src</string> </key> <key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[ <value> <string encoding="cdata"><![CDATA[
DELETE FROM inventory_stock WHERE <dtml-sqltest uid op=eq type=int> DELETE FROM \n
inventory_cache \n
WHERE \n
date > <dtml-sqlvar expr="date" type="datetime">
]]></string> </value> ]]></string> </value>
</item> </item>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_inventory.xml → bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/SimulationTool_zTrimInventoryCacheFromDateOnUncatalog.xml
View file @ cf53db45
...@@ -16,13 +16,13 @@ ...@@ -16,13 +16,13 @@
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>z0_uncatalog_inventory</string> </value> <value> <string>SimulationTool_zTrimInventoryCacheFromDateOnUncatalog</string> </value>
</item> </item>
<item> <item>
<key> <string>src</string> </key> <key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[ <value> <string encoding="cdata"><![CDATA[
DELETE FROM inventory WHERE <dtml-sqltest uid op=eq type=int> DELETE FROM inventory_cache WHERE date > (SELECT min(date) from stock where <dtml-sqltest uid op=eq type=int>)
]]></string> </value> ]]></string> </value>
</item> </item>
......
bt5/erp5_stock_cache/bt/change_log 0 → 100644
View file @ cf53db45
2013-02-05 Aurel
Initial version
\ No newline at end of file
bt5/erp5_stock_cache/bt/copyright_list 0 → 100644
View file @ cf53db45
Copyright (c) 2001-2013 Nexedi SA
\ No newline at end of file
bt5/erp5_stock_cache/bt/dependency_list 0 → 100644
View file @ cf53db45
erp5_core
\ No newline at end of file
bt5/erp5_stock_cache/bt/description 0 → 100644
View file @ cf53db45
This bt5 contains code to activate the stock caching optimisation
\ No newline at end of file
bt5/erp5_stock_cache/bt/license 0 → 100644
View file @ cf53db45
GPL
\ No newline at end of file
bt5/erp5_stock_cache/bt/maintainer_list 0 → 100644
View file @ cf53db45
aurel
\ No newline at end of file
bt5/erp5_stock_cache/bt/revision 0 → 100644
View file @ cf53db45
4
\ No newline at end of file
bt5/erp5_stock_cache/bt/template_catalog_method_id_list 0 → 100644
View file @ cf53db45
erp5_mysql_innodb/SQLCatalog_trimInventoryCacheOnCatalog
erp5_mysql_innodb/SQLCatalog_trimInventoryCacheOnUncatalog
\ No newline at end of file
bt5/erp5_stock_cache/bt/template_skin_id_list 0 → 100644
View file @ cf53db45
erp5_stock_cache
\ No newline at end of file
bt5/erp5_stock_cache/bt/title 0 → 100644
View file @ cf53db45
erp5_stock_cache
\ No newline at end of file
bt5/erp5_stock_cache/bt/version 0 → 100644
View file @ cf53db45
1.0
\ No newline at end of file
product/ERP5/Document/Inventory.py
View file @ cf53db45
...@@ -165,8 +165,6 @@ class Inventory(Delivery): ...@@ -165,8 +165,6 @@ class Inventory(Delivery):
current_inventory_dict = {} current_inventory_dict = {}
current_inventory_key_id_list = [x["key"] for x in inventory_calculation_dict['first_level']] current_inventory_key_id_list = [x["key"] for x in inventory_calculation_dict['first_level']]
for line in current_inventory_list: for line in current_inventory_list:
current_inventory_key = [line[x] for x in current_inventory_key_id_list] current_inventory_key = [line[x] for x in current_inventory_key_id_list]
for x in xrange(len(current_inventory_key)): for x in xrange(len(current_inventory_key)):
if current_inventory_key[x] is None: if current_inventory_key[x] is None:
...@@ -189,6 +187,9 @@ class Inventory(Delivery): ...@@ -189,6 +187,9 @@ class Inventory(Delivery):
current_inventory_dict[current_inventory_key] = line['total_quantity'] current_inventory_dict[current_inventory_key] = line['total_quantity']
# Browse all movements on inventory and create diff line when necessary # Browse all movements on inventory and create diff line when necessary
if self.isFullInventory():
not_used_inventory_dict = current_inventory_dict
else:
not_used_inventory_dict = {} not_used_inventory_dict = {}
inventory_id = self.getId() inventory_id = self.getId()
list_method = inventory_calculation_dict['list_method'] list_method = inventory_calculation_dict['list_method']
...@@ -274,6 +275,7 @@ class Inventory(Delivery): ...@@ -274,6 +275,7 @@ class Inventory(Delivery):
for first_level_key in not_used_inventory_dict.keys(): for first_level_key in not_used_inventory_dict.keys():
inventory_value = \ inventory_value = \
not_used_inventory_dict[tuple(first_level_key)] not_used_inventory_dict[tuple(first_level_key)]
# XXX-Aurel : this code does not work with only one level of variation
for second_level_key in inventory_value.keys(): for second_level_key in inventory_value.keys():
diff_quantity = - inventory_value[tuple(second_level_key)] diff_quantity = - inventory_value[tuple(second_level_key)]
......
product/ERP5/Tool/SimulationTool.py
View file @ cf53db45
...@@ -36,7 +36,7 @@ from Products.ERP5Type.Tool.BaseTool import BaseTool ...@@ -36,7 +36,7 @@ from Products.ERP5Type.Tool.BaseTool import BaseTool
from Products.ERP5 import _dtmldir from Products.ERP5 import _dtmldir
from zLOG import LOG, PROBLEM from zLOG import LOG, PROBLEM, WARNING, INFO
from Products.ERP5.Capacity.GLPK import solve from Products.ERP5.Capacity.GLPK import solve
from numpy import zeros, resize from numpy import zeros, resize
...@@ -50,8 +50,19 @@ from Products.ZSQLCatalog.SQLCatalog import Query, ComplexQuery ...@@ -50,8 +50,19 @@ from Products.ZSQLCatalog.SQLCatalog import Query, ComplexQuery
from Shared.DC.ZRDB.Results import Results from Shared.DC.ZRDB.Results import Results
from Products.ERP5Type.Utils import mergeZRDBResults from Products.ERP5Type.Utils import mergeZRDBResults
from App.Extensions import getBrain
from MySQLdb import ProgrammingError
from hashlib import md5
from warnings import warn from warnings import warn
from cPickle import loads, dumps
from copy import deepcopy
from sys import exc_info
MYSQL_MIN_DATETIME_RESOLUTION = 1/86400.
class StockOptimisationError(Exception):
pass
class SimulationTool(BaseTool): class SimulationTool(BaseTool):
""" """
...@@ -1278,214 +1289,246 @@ class SimulationTool(BaseTool): ...@@ -1278,214 +1289,246 @@ class SimulationTool(BaseTool):
sql_source_list = [] sql_source_list = []
# If no group at all, give a default sort group by # If no group at all, give a default sort group by
kw.update(self._getDefaultGroupByParameters(**kw)) kw.update(self._getDefaultGroupByParameters(**kw))
base_inventory_kw = {
'stock_table_id': default_stock_table,
'src__': src__,
'ignore_variation': ignore_variation,
'standardise': standardise,
'omit_simulation': omit_simulation,
'only_accountable': only_accountable,
'selection_domain': selection_domain,
'selection_report': selection_report,
'precision': precision,
'inventory_list': inventory_list,
'connection_id': connection_id,
'statistic': statistic,
'convert_quantity_result': convert_quantity_result,
'quantity_unit_uid': quantity_unit_uid,
}
# Get cached data
if getattr(self, "Resource_zGetInventoryCacheResult", None) is not None and \
optimisation__ and 'from_date' not in kw and \
(('at_date' in kw) ^ ('to_date' in kw)) and \
'transformed_resource' not in kw:
# Here is the different kind of date
# from_date : >=
# to_date : <
# at_date : <=
# As we just have from_date, it means that we must use
# the to_date for the cache in order to avoid double computation
# of the same line
at_date = kw.pop("at_date", None)
if at_date is None:
to_date = kw.pop("to_date")
else:
# add one second so that we can use to_date
to_date = at_date + MYSQL_MIN_DATETIME_RESOLUTION
try:
cached_result, cached_date = self._getCachedInventoryList(
to_date=to_date,
sql_kw=kw,
**base_inventory_kw)
except StockOptimisationError:
cached_result = []
kw['to_date'] = to_date
else:
if src__:
sql_source_list.extend(cached_result)
# Now must generate query for date diff
kw['to_date'] = to_date
kw['from_date'] = cached_date
else:
cached_result = []
sql_kw, new_kw = self._generateKeywordDict(**kw) sql_kw, new_kw = self._generateKeywordDict(**kw)
# Copy kw content as _generateSQLKeywordDictFromKeywordDict
# remove some values from it
try:
new_kw_copy = deepcopy(new_kw)
except TypeError:
# new_kw contains wrong parameters
# as optimisation has already been disable we
# do not care about the deepcopy
new_kw_copy = new_kw
stock_sql_kw = self._generateSQLKeywordDictFromKeywordDict( stock_sql_kw = self._generateSQLKeywordDictFromKeywordDict(
table=default_stock_table, sql_kw=sql_kw, new_kw=new_kw) table=default_stock_table, sql_kw=sql_kw, new_kw=new_kw_copy)
Resource_zGetFullInventoryDate = \ stock_sql_kw.update(base_inventory_kw)
getattr(self, 'Resource_zGetFullInventoryDate', None) delta_result = self.Resource_zGetInventoryList(
EQUAL_DATE_TABLE_ID = 'inventory_stock'
GREATER_THAN_DATE_TABLE_ID = 'stock'
buildSQLQuery = self.getPortalObject().portal_catalog.buildSQLQuery
optimisation_success = optimisation__ and ('from_date' not in kw) and \
Resource_zGetFullInventoryDate is not None and \
(GREATER_THAN_DATE_TABLE_ID == default_stock_table)
# Generate first query parameter dict
if optimisation_success:
def getFirstQueryParameterDict(query_generator_kw):
optimisation_success = True
AVAILABLE_CRITERIONS_IN_INVENTORY_TABLE = ['node_uid',
'section_uid',
'payment_uid']
# Only column group by are supported in full inventories
group_by_list = query_generator_kw.get('column_group_by', [])
column_value_dict = query_generator_kw.get('column_value_dict', {})
new_group_by_list = []
new_column_value_dict = {}
for criterion_id in AVAILABLE_CRITERIONS_IN_INVENTORY_TABLE:
criterion_value_list = column_value_dict.get(criterion_id, [])
if not isinstance(criterion_value_list, (list, tuple)):
criterion_value_list = [criterion_value_list]
if len(criterion_value_list) > 0:
if len(criterion_value_list) > 1:
# Impossible to optimise if there is more than one possible
# value per criterion.
optimisation_success = False
break
new_column_value_dict[criterion_id] = criterion_value_list
new_group_by_list.append(criterion_id)
elif criterion_id in group_by_list:
new_group_by_list.append(criterion_id)
group_by_expression = ', '.join(new_group_by_list)
column_id_list = new_column_value_dict.keys()
column_value_list_list = new_column_value_dict.values()
date_value_list = column_value_dict.get('date', {}).get('query', [])
where_expression = None
if len(date_value_list) == 1:
date = date_value_list[0]
# build a query for date to take range into account
date_query_result = buildSQLQuery(**{
'inventory.date': {
'query': date,
'range': column_value_dict.get('date', {}).get('range', [])
},
'query_table': None,
})
if date_query_result['where_expression'] not in ('',None):
where_expression = date_query_result['where_expression']
elif len(date_value_list) > 1:
# When more than one date is provided, we must not optimise.
# Also, as we should never end up here (the only currently known
# case where there are 2 dates is when a from_date is provided
# along with either an at_date or a to_date, and we disable
# optimisation when from_date is given), emit a log.
# This can happen if there are more date parameters than mentioned
# above.
LOG('SimulationTool', PROBLEM, 'There is more than one date condition'
' so optimisation got disabled. The result of this call will be'
' correct but it requires investigation as some cases might'
' have gone unnoticed and produced wrong results.')
optimisation_success = False
return {'group_by_expression': group_by_expression,
'column_id_list': column_id_list,
'column_value_list_list': column_value_list_list,
'where_expression' : where_expression,}, optimisation_success
first_query_param_dict, optimisation_success = getFirstQueryParameterDict(new_kw)
if optimisation_success:
if len(first_query_param_dict['column_id_list']):
inventory_date_line_list = self.Resource_zGetFullInventoryDate(
**first_query_param_dict)
if src__:
sql_source_list.append(
self.Resource_zGetFullInventoryDate(src__=src__,
**first_query_param_dict))
# Check that all expected uids have been found, otherwise a full
# inventory of a node/section/payment might be missing.
if len(inventory_date_line_list) >= max([len(x) for x in \
first_query_param_dict['column_value_list_list']]):
# Generate a where expression which filters on dates retrieved
# in the first query to be used in the second query.
# Also, generate a where expression to use in the third query,
# since it is based on the same data.
# XXX: uggly duplicated query generation code
# XXX: duplicates SQL variable formatting present in
# ERP5Type/patches/sqlvar.py about datetime SQL columns.
# Note: This code can generate queries like:
# date = 2000/01/01 and date >= 2001/01/01
# When latest full inventory is at 2000/01/01 and given
# from_date is 2001/01/01.
# It is not a serious problem since MySQL detects incompatible
# conditions and immediately returns (with 0 rows).
# get search key definitions from portal_catalog
ctool = getToolByName(self, 'portal_catalog')
portal_catalog = ctool.getSQLCatalog()
keyword_search_keys = list(portal_catalog.sql_catalog_keyword_search_keys)
datetime_search_keys = list(portal_catalog.sql_catalog_datetime_search_keys)
full_text_search_keys = list(portal_catalog.sql_catalog_full_text_search_keys)
search_key_mapping = dict(key_alias_dict = None,
keyword_search_keys = keyword_search_keys,
datetime_search_keys = datetime_search_keys,
full_text_search_keys = full_text_search_keys)
equal_date_query_list = []
greater_than_date_query_list = []
for inventory_date_line_dict in \
inventory_date_line_list.dictionaries():
date = inventory_date_line_dict['date']
non_date_value_dict = dict([(k, v) for k, v \
in inventory_date_line_dict.iteritems() if k != 'date'])
equal_date_query_list.append(
ComplexQuery(
ComplexQuery(operator='AND',
*[Query(**{'%s.%s' % (EQUAL_DATE_TABLE_ID, k): v}) \
for k, v in non_date_value_dict.iteritems()]),
Query(**{'%s.date' % (EQUAL_DATE_TABLE_ID, ): date}),
operator='AND'))
greater_than_date_query_list.append(
ComplexQuery(
ComplexQuery(operator='AND',
*[Query(**{'%s.%s' % (GREATER_THAN_DATE_TABLE_ID, k): \
v}) \
for k, v in non_date_value_dict.iteritems()]),
# 'Use explicitly Universal' otherwise DateTime
# search key will convert it to UTC one more time
Query(**{'%s.date' % (GREATER_THAN_DATE_TABLE_ID, ): date,
'range': 'nlt'}),
operator='AND'))
assert len(equal_date_query_list) == \
len(greater_than_date_query_list)
assert len(equal_date_query_list) > 0
equal_date_query = buildSQLQuery(query=ComplexQuery(operator='OR', *equal_date_query_list), query_table=None)['where_expression']
greater_than_date_query = buildSQLQuery(query=ComplexQuery(operator='OR', *greater_than_date_query_list), query_table=None)['where_expression']
inventory_stock_sql_kw = \
self._generateSQLKeywordDictFromKeywordDict(
table=EQUAL_DATE_TABLE_ID, sql_kw=sql_kw, new_kw=new_kw)
inventory_stock_where_query = \
inventory_stock_sql_kw.get('where_expression', '(1)')
assert isinstance(inventory_stock_where_query, basestring) \
and len(inventory_stock_where_query)
inventory_stock_sql_kw['where_expression'] = '(%s) AND (%s)' % \
(inventory_stock_where_query, equal_date_query)
where_query = stock_sql_kw.get('where_expression', '(1)')
assert isinstance(where_query, basestring) and len(where_query)
stock_sql_kw['where_expression'] = '(%s) AND (%s)' % \
(where_query, greater_than_date_query)
# Get initial inventory amount
initial_inventory_line_list = self.Resource_zGetInventoryList(
stock_table_id=EQUAL_DATE_TABLE_ID,
src__=src__, ignore_variation=ignore_variation,
standardise=standardise, omit_simulation=omit_simulation,
only_accountable=only_accountable,
selection_domain=selection_domain,
selection_report=selection_report, precision=precision,
inventory_list=inventory_list,
statistic=statistic,
quantity_unit_uid=quantity_unit_uid,
convert_quantity_result=convert_quantity_result,
**inventory_stock_sql_kw)
# Get delta inventory
delta_inventory_line_list = self.Resource_zGetInventoryList(
stock_table_id=GREATER_THAN_DATE_TABLE_ID,
src__=src__, ignore_variation=ignore_variation,
standardise=standardise, omit_simulation=omit_simulation,
only_accountable=only_accountable,
selection_domain=selection_domain,
selection_report=selection_report, precision=precision,
inventory_list=inventory_list,
statistic=statistic,
quantity_unit_uid=quantity_unit_uid,
convert_quantity_result=convert_quantity_result,
**stock_sql_kw) **stock_sql_kw)
# Match & add initial and delta inventories
if src__: if src__:
sql_source_list.extend((initial_inventory_line_list, sql_source_list.append(delta_result)
delta_inventory_line_list)) result = ';\n-- NEXT QUERY\n'.join(sql_source_list)
else:
if cached_result:
result = self._addBrainResults(delta_result, cached_result, new_kw)
else: else:
if 'column_group_by' in new_kw: result = delta_result
return result
def getInventoryCacheLag(self):
"""
Returns a duration, in days, for stock cache management.
If data in stock cache is older than lag compared to query's date
(at_date or to_date), then it becomes a "soft miss": use found value,
but add a new entry to cache at query's date minus half the lag.
So this value should be:
- Small enough that few enough rows need to be table-scanned for
average queries (probably queries against current date).
- Large enough that few enough documents get modified past that date,
otherwise cache entries would be removed from cache all the time.
"""
return self.SimulationTool_getInventoryCacheLag()
def _getCachedInventoryList(self, to_date, sql_kw, stock_table_id, src__=False, **kw):
"""
Try to get a cached inventory list result
If not existing, fill the cache
"""
Resource_zGetInventoryList = self.Resource_zGetInventoryList
# Generate the SQL source without date parameter
# This will be the cache key
try:
no_date_kw = deepcopy(sql_kw)
except TypeError:
LOG("SimulationTool._getCachedInventoryList", WARNING,
"Failed copying sql_kw, disabling stock cache",
error=exc_info())
raise StockOptimisationError
no_date_sql_kw, no_date_new_kw = self._generateKeywordDict(**no_date_kw)
no_date_stock_sql_kw = self._generateSQLKeywordDictFromKeywordDict(
table=stock_table_id, sql_kw=no_date_sql_kw,
new_kw=no_date_new_kw)
kw.update(no_date_stock_sql_kw)
if src__:
sql_source_list = []
# Generate the cache key (md5 of query source)
sql_text_hash = md5(Resource_zGetInventoryList(
stock_table_id=stock_table_id,
src__=1,
**kw)).digest()
# Try to get result from cache
Resource_zGetInventoryCacheResult = self.Resource_zGetInventoryCacheResult
inventory_cache_kw = {
'query': sql_text_hash,
'date': to_date,
}
try:
cached_sql_result = Resource_zGetInventoryCacheResult(**inventory_cache_kw)
except ProgrammingError:
# First use of the optimisation, we need to create the table
LOG("SimulationTool._getCachedInventoryList", INFO,
"Creating inventory cache stock")
if src__:
sql_source_list.append(self.SimulationTool_zCreateInventoryCache(src__=1))
else:
self.SimulationTool_zCreateInventoryCache()
cached_sql_result = None
if src__:
sql_source_list.append(Resource_zGetInventoryCacheResult(src__=1, **inventory_cache_kw))
if cached_sql_result:
brain_result = loads(cached_sql_result[0].result)
# Rebuild the brains
cached_result = Results(
(brain_result['items'], brain_result['data']),
brains=getBrain(
Resource_zGetInventoryList.class_file_,
Resource_zGetInventoryList.class_name_,
),
parent=self,
)
else:
cached_result = []
cache_lag = self.getInventoryCacheLag()
if cached_sql_result and to_date - DateTime(cached_sql_result[0].date) < cache_lag:
cached_date = DateTime(cached_sql_result[0].date)
result = cached_result
else:
# Cache miss, or hit with old data: store a new entry in cache.
# Don't store it at to_date, as it risks being flushed soon (ie, when
# any document older than to_date gets reindexed in stock table).
# Don't store it at to_date - cache_lag, as it would risk expiring
# soon as we store it (except if to_date is fixed for many queries,
# which we cannot tell here).
# So store it at half the cache_lag before to_date.
cached_date = to_date - cache_lag / 2
new_cache_kw = deepcopy(sql_kw)
if cached_result:
# We can use cached result to generate new cache result
new_cache_kw['from_date'] = DateTime(cached_sql_result[0].date)
sql_kw, new_kw = self._generateKeywordDict(
to_date=cached_date,
**new_cache_kw)
kw.update(self._generateSQLKeywordDictFromKeywordDict(
table=stock_table_id,
sql_kw=sql_kw,
new_kw=new_kw,
)
)
new_result = Resource_zGetInventoryList(
stock_table_id=stock_table_id,
src__=src__,
**kw)
if src__:
sql_source_list.append(new_result)
else:
result = self._addBrainResults(new_result, cached_result, new_kw)
self.Resource_zInsertInventoryCacheResult(
query=sql_text_hash,
date=cached_date,
result=dumps({
'items': result.__items__,
'data': result._data,
}),
)
if src__:
result = sql_source_list
return result, cached_date
def _addBrainResults(self, first_result, second_result, new_kw):
"""
Build a Results which is the addition of two other result
"""
# This part defined key to group lines from different Results
group_by_id_list = [] group_by_id_list = []
group_by_id_list_append = group_by_id_list.append group_by_id_list_append = group_by_id_list.append
for group_by_id in new_kw['column_group_by']:
for group_by_id in new_kw.get('column_group_by', []):
if group_by_id == 'uid': if group_by_id == 'uid':
group_by_id_list_append('stock_uid') group_by_id_list_append('stock_uid')
else: else:
group_by_id_list_append(group_by_id) group_by_id_list_append(group_by_id)
# Add related key group by
if 'select_list' in new_kw.get("related_key_dict_passthrough", []):
for group_by_id in new_kw["related_key_dict_passthrough"]['group_by']:
if group_by_id in new_kw["related_key_dict_passthrough"]["select_list"]:
group_by_id_list_append(group_by_id)
else:
# XXX-Aurel : to review & change, must prevent coming here before
raise ValueError, "Impossible to group by %s" %(group_by_id)
elif "group_by" in new_kw.get("related_key_dict_passthrough", []):
raise ValueError, "Impossible to group by %s" %(new_kw["related_key_dict_passthrough"]['group_by'],)
if len(group_by_id_list):
def getInventoryListKey(line): def getInventoryListKey(line):
""" """
Generate a key based on values used in SQL group_by Generate a key based on values used in SQL group_by
""" """
return tuple([line[x] for x in group_by_id_list]) return tuple([line[x] for x in group_by_id_list])
else: else:
def getInventoryListKey(line): def getInventoryListKey(line):
""" """
No group by criterion, regroup everything. Return a dummy key, all line will be summed
""" """
return 'dummy_key' return "dummy"
result_column_id_dict['inventory'] = None result_column_id_dict = {
result_column_id_dict['total_quantity'] = None 'inventory': None,
result_column_id_dict['total_price'] = None 'total_quantity': None,
'total_price': None
}
def addLineValues(line_a=None, line_b=None): def addLineValues(line_a=None, line_b=None):
""" """
Addition columns of 2 lines and return a line with same Add columns of 2 lines and return a line with same
schema. If one of the parameters is None, returns the schema. If one of the parameters is None, returns the
other parameters. other parameters.
...@@ -1504,9 +1547,14 @@ class SimulationTool(BaseTool): ...@@ -1504,9 +1547,14 @@ class SimulationTool(BaseTool):
Result = line_a.__class__ Result = line_a.__class__
parent = line_a.aq_parent parent = line_a.aq_parent
result = Result((), parent) result = Result((), parent)
try:
# We must copy the path so that getObject works
setattr(result, 'path', line_a.path)
except ValueError: # XXX: ValueError ? really ?
# getInventory return no object, so no path available
pass
if parent is not None: if parent is not None:
result = result.__of__(parent) result = result.__of__(parent)
for key in line_a.__record_schema__: for key in line_a.__record_schema__:
value = line_a[key] value = line_a[key]
if key in result_column_id_dict: if key in result_column_id_dict:
...@@ -1522,23 +1570,20 @@ class SimulationTool(BaseTool): ...@@ -1522,23 +1570,20 @@ class SimulationTool(BaseTool):
elif key not in ('date', 'stock_uid', 'path'): elif key not in ('date', 'stock_uid', 'path'):
LOG('InventoryTool.getInventoryList.addLineValues', LOG('InventoryTool.getInventoryList.addLineValues',
PROBLEM, PROBLEM,
'mismatch for %s column: %s and %s' % \ 'mismatch for %s column: %s and %s' % (
(key, line_a[key], line_b[key])) key, line_a[key], line_b[key]))
return result return result
# Add lines
inventory_list_dict = {} inventory_list_dict = {}
for line_list in (initial_inventory_line_list, for line_list in (first_result, second_result):
delta_inventory_line_list):
for line in line_list: for line in line_list:
line_key = getInventoryListKey(line) line_key = getInventoryListKey(line)
line_a = inventory_list_dict.get(line_key) line_a = inventory_list_dict.get(line_key)
inventory_list_dict[line_key] = addLineValues(line_a, inventory_list_dict[line_key] = addLineValues(line_a, line)
line)
## XXX: Returns a dict instead of an <r> instance
## As long as they are accessed like dicts it's ok, but...
#result = inventory_list_dict.values()
sorted_inventory_list = inventory_list_dict.values() sorted_inventory_list = inventory_list_dict.values()
sort_on = new_kw.get('sort_on', tuple()) # Sort results manually when required
if len(sort_on) != 0: sort_on = new_kw.get('sort_on')
if sort_on:
def cmp_inventory_line(line_a, line_b): def cmp_inventory_line(line_a, line_b):
""" """
Compare 2 inventory lines and sort them according to Compare 2 inventory lines and sort them according to
...@@ -1546,14 +1591,13 @@ class SimulationTool(BaseTool): ...@@ -1546,14 +1591,13 @@ class SimulationTool(BaseTool):
""" """
result = 0 result = 0
for key, sort_direction in sort_on: for key, sort_direction in sort_on:
if not(key in line_a and key in line_b): try:
raise Exception, "Impossible to sort result since " \
"columns sort happens on are not available in " \
"result."
result = cmp(line_a[key], line_b[key]) result = cmp(line_a[key], line_b[key])
if result != 0: except KeyError:
if len(sort_direction[0]) and \ raise Exception('Impossible to sort result since columns sort '
sort_direction[0].upper() != 'A': 'happens on are not available in result: %r' % (key, ))
if result:
if not sort_direction.upper().startswith('A'):
# Default sort is ascending, if a sort is given and # Default sort is ascending, if a sort is given and
# it does not start with an 'A' then reverse sort. # it does not start with an 'A' then reverse sort.
# Tedious syntax checking is MySQL's job, and # Tedious syntax checking is MySQL's job, and
...@@ -1562,33 +1606,20 @@ class SimulationTool(BaseTool): ...@@ -1562,33 +1606,20 @@ class SimulationTool(BaseTool):
break break
return result return result
sorted_inventory_list.sort(cmp_inventory_line) sorted_inventory_list.sort(cmp_inventory_line)
result = Results((delta_inventory_line_list.\ # Brain is rebuild properly using tuple not r instance
_searchable_result_columns(), column_list = first_result._searchable_result_columns()
tuple(sorted_inventory_list))) column_name_list = [x['name'] for x in column_list]
else: # Rebuild a result object based on added results
# Not all required full inventories are found Resource_zGetInventoryList = self.Resource_zGetInventoryList
optimisation_success = False return Results(
else: (column_list, tuple([tuple([getattr(y, x) for x in column_name_list]) \
# Not enough criterions to trigger optimisation for y in sorted_inventory_list])),
optimisation_success = False parent=self,
if not optimisation_success: brains=getBrain(
result = self.Resource_zGetInventoryList( Resource_zGetInventoryList.class_file_,
stock_table_id=default_stock_table, Resource_zGetInventoryList.class_name_,
src__=src__, ignore_variation=ignore_variation, ),
standardise=standardise, omit_simulation=omit_simulation, )
only_accountable=only_accountable,
selection_domain=selection_domain,
selection_report=selection_report, precision=precision,
inventory_list=inventory_list, connection_id=connection_id,
statistic=statistic,
quantity_unit_uid=quantity_unit_uid,
convert_quantity_result=convert_quantity_result,
**stock_sql_kw)
if src__:
sql_source_list.append(result)
if src__:
result = ';\n-- NEXT QUERY\n'.join(sql_source_list)
return result
security.declareProtected(Permissions.AccessContentsInformation, security.declareProtected(Permissions.AccessContentsInformation,
'getConvertedInventoryList') 'getConvertedInventoryList')
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogDateTimeKeyTemplateItem/datetime_key_list.xml
View file @ cf53db45
...@@ -15,9 +15,6 @@ ...@@ -15,9 +15,6 @@
<key>effective_date</key> <key>effective_date</key>
<key>expiration_date</key> <key>expiration_date</key>
<key>grouping_date</key> <key>grouping_date</key>
<key>inventory.date</key>
<key>inventory_stock.date</key>
<key>inventory_stock.mirror_date</key>
<key>item.date</key> <key>item.date</key>
<key>mirror_date</key> <key>mirror_date</key>
<key>modification_date</key> <key>modification_date</key>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_inventory.catalog_keys.xml deleted 100644 → 0
View file @ 3f484980
<catalog_method>
<item key="sql_uncatalog_object" type="int">
<value>1</value>
</item>
<item key="_is_filtered_archive" type="int">
<value>1</value>
</item>
<item key="_filter_expression_archive" type="str">
<value>python: context.isInventory()</value>
</item>
<item key="_filter_expression_cache_key_archive" type="tuple">
<value>portal_type</value>
</item>
</catalog_method>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_inventory_list.catalog_keys.xml deleted 100644 → 0
View file @ 3f484980
<catalog_method>
<item key="sql_catalog_object_list" type="int">
<value>1</value>
</item>
</catalog_method>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_inventory_list.xml deleted 100644 → 0
View file @ 3f484980
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>isInventory\r\n
uid\r\n
getDestinationUid\r\n
getDestinationSectionUid\r\n
getDestinationPaymentUid\r\n
getStartDate\r\n
isFullInventory\r\n
getSimulationState</string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_catalog_inventory_list</string> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
<dtml-let row_list="[]">\n
\n
<dtml-in prefix="loop" expr="_.range(_.len(uid))">\n
<dtml-if expr="isInventory[loop_item]">\n
<dtml-if expr="getStartDate[loop_item] is not None">\n
<dtml-call expr="row_list.append((\n
uid[loop_item],\n
getDestinationUid[loop_item],\n
getDestinationSectionUid[loop_item],\n
getDestinationPaymentUid[loop_item],\n
getStartDate[loop_item],\n
isFullInventory[loop_item],\n
getSimulationState[loop_item]))">\n
</dtml-if>\n
</dtml-if>\n
</dtml-in>\n
\n
<dtml-if expr="len(row_list)">\n
REPLACE INTO\n
inventory\n
(`uid`, `node_uid`, `section_uid`, `payment_uid`, `date`, `is_full_inventory`, `simulation_state`)\n
VALUES\n
<dtml-in prefix="row" expr="row_list">\n
(<dtml-sqlvar expr="row_item[0]" type="int">,\n
<dtml-sqlvar expr="row_item[1]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[2]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[3]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[4]" type="datetime">,\n
<dtml-sqlvar expr="row_item[5] or 0" type="int">,\n
<dtml-sqlvar expr="row_item[6]" type="string">\n
)<dtml-if sequence-end><dtml-else>,</dtml-if>\n
</dtml-in>\n
</dtml-if>\n
</dtml-let>
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_inventory_stock_list.xml deleted 100644 → 0
View file @ 3f484980
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid\r\n
getResourceUid\r\n
getInventoriatedQuantity\r\n
getSourceUid\r\n
getDestinationUid\r\n
getSourceSectionUid\r\n
getDestinationSectionUid\r\n
isMovement\r\n
getSourcePaymentUid\r\n
getDestinationPaymentUid\r\n
getSourceFunctionUid\r\n
getDestinationFunctionUid\r\n
getSourceProjectUid\r\n
getDestinationProjectUid\r\n
getSourceFundingUid\r\n
getDestinationFundingUid\r\n
getSourcePaymentRequestUid\r\n
getDestinationPaymentRequestUid\r\n
getSimulationState\r\n
getSourceInventoriatedTotalAssetPrice\r\n
getDestinationInventoriatedTotalAssetPrice\r\n
getStartDate\r\n
getStopDate\r\n
isAccountable\r\n
getPortalType\r\n
getVariationText\r\n
getSubVariationText</string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_catalog_inventory_stock_list</string> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
DELETE FROM\n
inventory_stock\n
WHERE\n
<dtml-in uid>\n
uid=<dtml-sqlvar sequence-item type="int"><dtml-if sequence-end><dtml-else> OR </dtml-if>\n
</dtml-in>\n
\n
<dtml-var sql_delimiter>\n
\n
<dtml-let row_list="[]" uid_dict="{}">\n
<dtml-in prefix="loop" expr="_.range(_.len(uid))">\n
<dtml-if "isMovement[loop_item] and isAccountable[loop_item] and getResourceUid[loop_item]">\n
<dtml-if "getDestinationUid[loop_item]">\n
<dtml-call expr="uid_dict.update({uid[loop_item]: uid_dict.get(uid[loop_item], -1) + 1})">\n
<dtml-call expr="row_list.append([\n
uid[loop_item], \n
uid_dict[uid[loop_item]],\n
getDestinationUid[loop_item],\n
getDestinationSectionUid[loop_item],\n
getDestinationPaymentUid[loop_item],\n
getDestinationFunctionUid[loop_item],\n
getDestinationProjectUid[loop_item], \n
getDestinationFundingUid[loop_item], \n
getDestinationPaymentRequestUid[loop_item], \n
getSourceSectionUid[loop_item], \n
getSourceUid[loop_item], \n
getResourceUid[loop_item],\n
getInventoriatedQuantity[loop_item],\n
getStopDate[loop_item], \n
getStartDate[loop_item], \n
getDestinationInventoriatedTotalAssetPrice[loop_item], \n
getPortalType[loop_item], \n
getSimulationState[loop_item], \n
getVariationText[loop_item],\n
getSubVariationText[loop_item]])">\n
</dtml-if>\n
<dtml-if "getSourceUid[loop_item]">\n
<dtml-call expr="uid_dict.update({uid[loop_item]: uid_dict.get(uid[loop_item], -1) + 1})">\n
<dtml-call expr="row_list.append([\n
uid[loop_item], \n
uid_dict[uid[loop_item]],\n
getSourceUid[loop_item],\n
getSourceSectionUid[loop_item],\n
getSourcePaymentUid[loop_item],\n
getSourceFunctionUid[loop_item],\n
getSourceProjectUid[loop_item], \n
getSourceFundingUid[loop_item], \n
getSourcePaymentRequestUid[loop_item], \n
getDestinationSectionUid[loop_item], \n
getDestinationUid[loop_item], \n
getResourceUid[loop_item],\n
-(getInventoriatedQuantity[loop_item] or 0), \n
getStartDate[loop_item], \n
getStopDate[loop_item],\n
getSourceInventoriatedTotalAssetPrice[loop_item], \n
getPortalType[loop_item], \n
getSimulationState[loop_item], \n
getVariationText[loop_item],\n
getSubVariationText[loop_item]])">\n
</dtml-if>\n
</dtml-if>\n
</dtml-in> \n
\n
<dtml-if "row_list">\n
INSERT INTO\n
inventory_stock\n
VALUES\n
<dtml-in prefix="row" expr="row_list">\n
(\n
<dtml-sqlvar expr="row_item[0]" type="int">,\n
<dtml-sqlvar expr="row_item[1]" type="int">,\n
<dtml-sqlvar expr="row_item[2]" type="int">,\n
<dtml-sqlvar expr="row_item[3]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[4]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[5]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[6]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[7]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[8]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[9]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[10]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[11]" type="int">,\n
<dtml-sqlvar expr="row_item[12]" type="float" optional>,\n
1, <dtml-comment>only accountable</dtml-comment>\n
<dtml-sqlvar expr="row_item[13]" type="datetime" optional>,\n
<dtml-sqlvar expr="row_item[14]" type="datetime" optional>,\n
<dtml-sqlvar expr="row_item[15]" type="float" optional>,\n
<dtml-sqlvar expr="row_item[16]" type="string" optional>,\n
<dtml-sqlvar expr="row_item[17]" type="string" optional>,\n
<dtml-sqlvar expr="row_item[18]" type="string" optional>,\n
<dtml-sqlvar expr="row_item[19]" type="string" optional>\n
)\n
<dtml-if sequence-end><dtml-else>,</dtml-if>\n
</dtml-in>\n
</dtml-if>\n
</dtml-let>\n
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_stock_list.xml
View file @ cf53db45
...@@ -92,7 +92,6 @@ WHERE\n ...@@ -92,7 +92,6 @@ WHERE\n
;\n ;\n
\n \n
<dtml-var "\'\\0\'">\n <dtml-var "\'\\0\'">\n
\n
<dtml-let row_list="[]" uid_dict="{}">\n <dtml-let row_list="[]" uid_dict="{}">\n
<dtml-in prefix="loop" expr="_.range(_.len(uid))">\n <dtml-in prefix="loop" expr="_.range(_.len(uid))">\n
<dtml-if "not(isInventoryMovement[loop_item]) and isMovement[loop_item] and getResourceUid[loop_item]">\n <dtml-if "not(isInventoryMovement[loop_item]) and isMovement[loop_item] and getResourceUid[loop_item]">\n
...@@ -152,7 +151,6 @@ WHERE\n ...@@ -152,7 +151,6 @@ WHERE\n
</dtml-if>\n </dtml-if>\n
</dtml-if>\n </dtml-if>\n
</dtml-in> \n </dtml-in> \n
\n
<dtml-if "row_list">\n <dtml-if "row_list">\n
INSERT INTO\n INSERT INTO\n
stock\n stock\n
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_inventory.xml deleted 100644 → 0
View file @ 3f484980
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_create_inventory</string> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string>CREATE TABLE `inventory` (\n
`uid` BIGINT(20) UNSIGNED NOT NULL,\n
`node_uid` BIGINT(20) UNSIGNED NULL,\n
`section_uid` BIGINT(20) UNSIGNED NULL,\n
`payment_uid` BIGINT(20) UNSIGNED NULL,\n
`date` DATETIME NOT NULL,\n
`is_full_inventory` BOOL NOT NULL DEFAULT FALSE,\n
`simulation_state` VARCHAR(255) NOT NULL DEFAULT \'\',\n
PRIMARY KEY `uid` (`uid`),\n
KEY `node_index` (`is_full_inventory`, `simulation_state`, `node_uid`),\n
KEY `section_index` (`is_full_inventory`, `simulation_state`, `section_uid`),\n
KEY `payment_index` (`is_full_inventory`, `simulation_state`, `payment_uid`)\n
) ENGINE=InnoDB</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_inventory_stock.xml deleted 100644 → 0
View file @ 3f484980
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_create_inventory_stock</string> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string>CREATE TABLE `inventory_stock` (\n
`uid` BIGINT UNSIGNED NOT NULL,\n
`order_id` TINYINT UNSIGNED NOT NULL,\n
`node_uid` BIGINT UNSIGNED,\n
`section_uid` BIGINT UNSIGNED,\n
`payment_uid` BIGINT UNSIGNED,\n
`function_uid` BIGINT UNSIGNED,\n
`project_uid` BIGINT UNSIGNED,\n
`funding_uid` BIGINT UNSIGNED,\n
`payment_request_uid` BIGINT UNSIGNED,\n
`mirror_section_uid` BIGINT UNSIGNED,\n
`mirror_node_uid` BIGINT UNSIGNED,\n
`resource_uid` BIGINT UNSIGNED,\n
`quantity` real ,\n
`is_accountable` BOOLEAN,\n
`date` datetime,\n
`mirror_date` datetime,\n
`total_price` real ,\n
`portal_type` VARCHAR(255),\n
`simulation_state` varchar(255) default \'\',\n
`variation_text` VARCHAR(255),\n
`sub_variation_text` VARCHAR(255),\n
PRIMARY KEY (`uid`, `order_id`),\n
KEY `quantity` (`quantity`),\n
KEY `section_uid` (`section_uid`),\n
KEY `mirror_section_uid` (`mirror_section_uid`),\n
KEY `mirror_node_uid` (`mirror_node_uid`),\n
KEY `node_uid` (`node_uid`),\n
KEY `payment_uid` (`payment_uid`),\n
KEY `function_uid` (`function_uid`),\n
KEY `project_uid` (`project_uid`),\n
KEY `simulation_state` (`simulation_state`),\n
KEY `resource_node_uid` (`resource_uid`, `node_uid`),\n
KEY `resource_section_node_uid` (`resource_uid`, `section_uid`, `node_uid`, `simulation_state`),\n
KEY `date` (`date`)\n
) ENGINE=InnoDB\n
</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogResultTableTemplateItem/result_table_list.xml
View file @ cf53db45
...@@ -3,8 +3,6 @@ ...@@ -3,8 +3,6 @@
<key>catalog</key> <key>catalog</key>
<key>category</key> <key>category</key>
<key>delivery</key> <key>delivery</key>
<key>inventory</key>
<key>inventory_stock</key>
<key>item</key> <key>item</key>
<key>predicate</key> <key>predicate</key>
<key>predicate_category</key> <key>predicate_category</key>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/template_catalog_datetime_key_list
View file @ cf53db45
...@@ -12,9 +12,6 @@ delivery.start_date_range_min ...@@ -12,9 +12,6 @@ delivery.start_date_range_min
delivery.stop_date delivery.stop_date
delivery.stop_date_range_max delivery.stop_date_range_max
delivery.stop_date_range_min delivery.stop_date_range_min
inventory.date
inventory_stock.date
inventory_stock.mirror_date
item.date item.date
mirror_date mirror_date
modification_date modification_date
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/template_catalog_method_id_list
View file @ cf53db45
...@@ -7,8 +7,6 @@ erp5_mysql_innodb/z0_drop_alarm ...@@ -7,8 +7,6 @@ erp5_mysql_innodb/z0_drop_alarm
erp5_mysql_innodb/z0_drop_catalog erp5_mysql_innodb/z0_drop_catalog
erp5_mysql_innodb/z0_drop_category erp5_mysql_innodb/z0_drop_category
erp5_mysql_innodb/z0_drop_delivery erp5_mysql_innodb/z0_drop_delivery
erp5_mysql_innodb/z0_drop_inventory
erp5_mysql_innodb/z0_drop_inventory_stock
erp5_mysql_innodb/z0_drop_item erp5_mysql_innodb/z0_drop_item
erp5_mysql_innodb/z0_drop_measure erp5_mysql_innodb/z0_drop_measure
erp5_mysql_innodb/z0_drop_predicate erp5_mysql_innodb/z0_drop_predicate
...@@ -23,8 +21,6 @@ erp5_mysql_innodb/z0_drop_translation ...@@ -23,8 +21,6 @@ erp5_mysql_innodb/z0_drop_translation
erp5_mysql_innodb/z0_drop_versioning erp5_mysql_innodb/z0_drop_versioning
erp5_mysql_innodb/z0_uncatalog_alarm erp5_mysql_innodb/z0_uncatalog_alarm
erp5_mysql_innodb/z0_uncatalog_category erp5_mysql_innodb/z0_uncatalog_category
erp5_mysql_innodb/z0_uncatalog_inventory
erp5_mysql_innodb/z0_uncatalog_inventory_stock
erp5_mysql_innodb/z0_uncatalog_item erp5_mysql_innodb/z0_uncatalog_item
erp5_mysql_innodb/z0_uncatalog_measure erp5_mysql_innodb/z0_uncatalog_measure
erp5_mysql_innodb/z0_uncatalog_predicate erp5_mysql_innodb/z0_uncatalog_predicate
...@@ -35,8 +31,6 @@ erp5_mysql_innodb/z0_uncatalog_transformation ...@@ -35,8 +31,6 @@ erp5_mysql_innodb/z0_uncatalog_transformation
erp5_mysql_innodb/z0_uncatalog_versioning erp5_mysql_innodb/z0_uncatalog_versioning
erp5_mysql_innodb/z_catalog_alarm_list erp5_mysql_innodb/z_catalog_alarm_list
erp5_mysql_innodb/z_catalog_delivery_list erp5_mysql_innodb/z_catalog_delivery_list
erp5_mysql_innodb/z_catalog_inventory_list
erp5_mysql_innodb/z_catalog_inventory_stock_list
erp5_mysql_innodb/z_catalog_item_list erp5_mysql_innodb/z_catalog_item_list
erp5_mysql_innodb/z_catalog_measure_list erp5_mysql_innodb/z_catalog_measure_list
erp5_mysql_innodb/z_catalog_movement_category_list erp5_mysql_innodb/z_catalog_movement_category_list
...@@ -58,8 +52,6 @@ erp5_mysql_innodb/z_create_alarm ...@@ -58,8 +52,6 @@ erp5_mysql_innodb/z_create_alarm
erp5_mysql_innodb/z_create_catalog erp5_mysql_innodb/z_create_catalog
erp5_mysql_innodb/z_create_category erp5_mysql_innodb/z_create_category
erp5_mysql_innodb/z_create_delivery erp5_mysql_innodb/z_create_delivery
erp5_mysql_innodb/z_create_inventory
erp5_mysql_innodb/z_create_inventory_stock
erp5_mysql_innodb/z_create_item erp5_mysql_innodb/z_create_item
erp5_mysql_innodb/z_create_measure erp5_mysql_innodb/z_create_measure
erp5_mysql_innodb/z_create_predicate erp5_mysql_innodb/z_create_predicate
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/template_catalog_result_table_list
View file @ cf53db45
...@@ -2,8 +2,6 @@ catalog ...@@ -2,8 +2,6 @@ catalog
category category
alarm alarm
delivery delivery
inventory
inventory_stock
item item
predicate predicate
predicate_category predicate_category
......
product/ERP5/tests/testInventory.py
View file @ cf53db45
...@@ -184,7 +184,7 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase): ...@@ -184,7 +184,7 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase):
inventory_list.append(inventory) inventory_list.append(inventory)
sequence.edit(inventory_list = inventory_list) sequence.edit(inventory_list = inventory_list)
def createInventory(self, sequence=None): def createInventory(self, sequence=None, full=False):
""" """
""" """
portal = self.getPortal() portal = self.getPortal()
...@@ -193,7 +193,8 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase): ...@@ -193,7 +193,8 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase):
inventory = inventory_module.newContent(portal_type = self.inventory_portal_type) inventory = inventory_module.newContent(portal_type = self.inventory_portal_type)
inventory.edit(destination_value = sequence.get('node'), inventory.edit(destination_value = sequence.get('node'),
destination_section_value = sequence.get('section'), destination_section_value = sequence.get('section'),
start_date = DateTime() + 1 start_date = DateTime() + 1,
full_inventory=full,
) )
inventory_list.append(inventory) inventory_list.append(inventory)
sequence.edit(inventory_list=inventory_list) sequence.edit(inventory_list=inventory_list)
...@@ -213,6 +214,66 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase): ...@@ -213,6 +214,66 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase):
inventory_list.append(inventory) inventory_list.append(inventory)
sequence.edit(inventory_list=inventory_list) sequence.edit(inventory_list=inventory_list)
def stepCreateFullInventory(self, sequence=None, sequence_list=None, **kw):
"""
Create a full Inventory object for Inventory Module testing
"""
inventory = self.createInventory(sequence=sequence)
inventory_list = sequence.get('inventory_list',[])
inventory.edit(full_inventory=True)
inventory_line = inventory.newContent(
portal_type = self.inventory_line_portal_type,
resource_value = sequence.get("second_resource"),
inventory = 101)
inventory.deliver()
inventory_list.append(inventory)
sequence.edit(inventory_list=inventory_list)
def stepCreatePartialInventoryMultipleResource(self, sequence=None, sequence_list=None, **kw):
"""
Create a partial inventory object for one resource
"""
inventory = self.createInventory(sequence=sequence)
inventory_list = sequence.get('inventory_list',[])
inventory.edit(full_inventory=False)
inventory_line = inventory.newContent(
portal_type = self.inventory_line_portal_type,
resource_value = sequence.get("second_resource"),
inventory = 101)
inventory.deliver()
inventory_list.append(inventory)
sequence.edit(inventory_list=inventory_list)
def stepTestPartialInventoryMultipleResource(self, sequence=None, sequence_list=None, **kw):
"""
Test partial inventory behavior with multiple resource
"""
inventory_list = sequence.get('inventory_list')
simulation = self.getPortal().portal_simulation
# First resource, must not have changed
inventory = simulation.getCurrentInventory(
resource = sequence.get("resource").getRelativeUrl(),
section = sequence.get('section').getRelativeUrl(),
node = sequence.get('node').getRelativeUrl(),
)
self.assertEquals(inventory, 100.,
'section=%s, node=%s' % (
sequence.get('section').getRelativeUrl(),
sequence.get('node').getRelativeUrl()))
# second resource, must be 101
inventory = simulation.getCurrentInventory(
resource = sequence.get("second_resource").getRelativeUrl(),
section = sequence.get('section').getRelativeUrl(),
node = sequence.get('node').getRelativeUrl(),
)
self.assertEquals(inventory, 101.,
'section=%s, node=%s' % (
sequence.get('section').getRelativeUrl(),
sequence.get('node').getRelativeUrl()))
def stepCreateSingleVariatedInventory(self, sequence=None, sequence_list=None, **kw): def stepCreateSingleVariatedInventory(self, sequence=None, sequence_list=None, **kw):
""" """
Create a single Inventory object for Inventory Module testing Create a single Inventory object for Inventory Module testing
...@@ -231,19 +292,32 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase): ...@@ -231,19 +292,32 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase):
variation_category_list = category_list, variation_category_list = category_list,
mapped_value_property_list = ['quantity'], mapped_value_property_list = ['quantity'],
) )
category_list = sequence.get('variation_2') # When checking the not full inventory function, quantity must remain the same if
# no inventory line defined for a variation
inventory.deliver()
def stepCreateFullVariatedInventory(self, sequence=None, sequence_list=None, **kw):
"""
Create a single full Inventory object for Inventory Module testing
"""
inventory = self.createInventory(sequence=sequence, full=True)
inventory_line = inventory.newContent(portal_type = self.inventory_line_portal_type) inventory_line = inventory.newContent(portal_type = self.inventory_line_portal_type)
category_list = sequence.get('variation_1')
inventory_line.edit(resource_value = sequence.get('resource'), inventory_line.edit(resource_value = sequence.get('resource'),
variation_category_list=category_list variation_category_list=category_list
) )
cell = inventory_line.newCell(base_id='movement',*category_list) cell = inventory_line.newCell(base_id='movement',*category_list)
quantity=0
cell.edit( cell.edit(
quantity = quantity, quantity = 55,
predicate_category_list = category_list, predicate_category_list = category_list,
variation_category_list = category_list, variation_category_list = category_list,
mapped_value_property_list = ['quantity'], mapped_value_property_list = ['quantity'],
) )
inventory_line = inventory.newContent(
portal_type = self.inventory_line_portal_type,
resource_value = sequence.get("second_resource"),
inventory = 101)
inventory.deliver() inventory.deliver()
def stepCreatePackingListForModule(self, sequence=None, def stepCreatePackingListForModule(self, sequence=None,
...@@ -1762,9 +1836,10 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase): ...@@ -1762,9 +1836,10 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase):
variation_category_list.sort() variation_category_list.sort()
variation_text = '\n'.join(variation_category_list) variation_text = '\n'.join(variation_category_list)
inventory = simulation.getCurrentInventory( inventory = simulation.getCurrentInventory(
section=sequence.get('section').getRelativeUrl(), resource = sequence.get("resource").getRelativeUrl(),
node=sequence.get('node').getRelativeUrl(), section = sequence.get('section').getRelativeUrl(),
variation_text=variation_text node = sequence.get('node').getRelativeUrl(),
variation_text = variation_text
) )
self.assertEquals(inventory, quantity) self.assertEquals(inventory, quantity)
...@@ -1806,10 +1881,51 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase): ...@@ -1806,10 +1881,51 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase):
self.checkVariatedInventory(variation_category_list=variation_category_list, self.checkVariatedInventory(variation_category_list=variation_category_list,
quantity=quantity,sequence=sequence) quantity=quantity,sequence=sequence)
variation_category_list = sequence.get('variation_2') variation_category_list = sequence.get('variation_2')
quantity = 3
self.checkVariatedInventory(variation_category_list=variation_category_list,
quantity=quantity,sequence=sequence)
def stepTestVariatedInventoryNonDefaultQuantityUnitAfterInventory(self, sequence=None, sequence_list=None, **kw):
"""
Test Inventory Module behavior
"""
resource = sequence.get('resource')
variation_category_list = sequence.get('variation_1')
quantity = 5
self.checkVariatedInventory(variation_category_list=variation_category_list,
quantity=quantity,sequence=sequence)
variation_category_list = sequence.get('variation_2')
quantity = 300
self.checkVariatedInventory(variation_category_list=variation_category_list,
quantity=quantity,sequence=sequence)
def stepTestFullVariatedInventory(self, sequence=None, sequence_list=None, **kw):
"""
Test full inventory with variated resource
"""
resource = sequence.get('resource')
variation_category_list = sequence.get('variation_1')
# Test first resource
quantity = 55
self.checkVariatedInventory(variation_category_list=variation_category_list,
quantity=quantity,sequence=sequence)
variation_category_list = sequence.get('variation_2')
quantity = 0 quantity = 0
self.checkVariatedInventory(variation_category_list=variation_category_list, self.checkVariatedInventory(variation_category_list=variation_category_list,
quantity=quantity,sequence=sequence) quantity=quantity,sequence=sequence)
# second resource, must be 101
simulation = self.getPortal().portal_simulation
inventory = simulation.getCurrentInventory(
resource = sequence.get("second_resource").getRelativeUrl(),
section = sequence.get('section').getRelativeUrl(),
node = sequence.get('node').getRelativeUrl(),
)
self.assertEquals(inventory, 101.,
'section=%s, node=%s' % (
sequence.get('section').getRelativeUrl(),
sequence.get('node').getRelativeUrl()))
def stepTestInventoryModule(self, sequence=None, sequence_list=None, **kw): def stepTestInventoryModule(self, sequence=None, sequence_list=None, **kw):
""" """
Test Inventory Module behavior Test Inventory Module behavior
...@@ -1832,6 +1948,36 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase): ...@@ -1832,6 +1948,36 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase):
step += 1 step += 1
sequence.edit(step=step) sequence.edit(step=step)
def stepTestFullInventory(self, sequence=None, sequence_list=None, **kw):
"""
Test Full inventory behavior
"""
inventory_list = sequence.get('inventory_list')
simulation = self.getPortal().portal_simulation
# First resource, must be zero
inventory = simulation.getCurrentInventory(
resource = sequence.get("resource").getRelativeUrl(),
section = sequence.get('section').getRelativeUrl(),
node = sequence.get('node').getRelativeUrl(),
)
self.assertEquals(inventory, 0.,
'section=%s, node=%s' % (
sequence.get('section').getRelativeUrl(),
sequence.get('node').getRelativeUrl()))
# second resource, must be 101
inventory = simulation.getCurrentInventory(
resource = sequence.get("second_resource").getRelativeUrl(),
section = sequence.get('section').getRelativeUrl(),
node = sequence.get('node').getRelativeUrl(),
)
self.assertEquals(inventory, 101.,
'section=%s, node=%s' % (
sequence.get('section').getRelativeUrl(),
sequence.get('node').getRelativeUrl()))
def stepModifyFirstInventory(self, sequence=None, sequence_list=None, **kw): def stepModifyFirstInventory(self, sequence=None, sequence_list=None, **kw):
""" """
Modify the first entered Inventory, to test the quantity change Modify the first entered Inventory, to test the quantity change
...@@ -1845,6 +1991,27 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase): ...@@ -1845,6 +1991,27 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase):
quantity=sum([x.getQuantity() for x in aggregate_value_list])) quantity=sum([x.getQuantity() for x in aggregate_value_list]))
def stepCreateNotVariatedSecondResource(self,sequence=None,
sequence_list=None,
**kw):
"""
Create a second resource with no variation
"""
portal = self.getPortal()
resource_module = portal.getDefaultModule(self.resource_portal_type)
resource = resource_module.newContent(portal_type=self.resource_portal_type)
resource.edit(
title = "NotVariatedSecondResource%s" % resource.getId(),
industrial_phase_list=["phase1", "phase2"],
product_line = 'apparel'
)
sequence.edit( second_resource = resource )
resource_list = sequence.get('resource_list',default=[])
resource_list.append(resource)
sequence.edit( resource_list = resource_list )
def test_01_getInventory(self, quiet=0, run=run_all_test): def test_01_getInventory(self, quiet=0, run=run_all_test):
""" """
Test the getInventory methods Test the getInventory methods
...@@ -1966,7 +2133,7 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase): ...@@ -1966,7 +2133,7 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase):
stepTestInitialVariatedNonDefaultQuantityUnitInventory \ stepTestInitialVariatedNonDefaultQuantityUnitInventory \
stepCreateSingleVariatedInventory \ stepCreateSingleVariatedInventory \
stepTic \ stepTic \
stepTestVariatedInventoryAfterInventory \ stepTestVariatedInventoryNonDefaultQuantityUnitAfterInventory \
' '
sequence_list.addSequenceString(sequence_string) sequence_list.addSequenceString(sequence_string)
sequence_list.play(self) sequence_list.play(self)
...@@ -2017,6 +2184,83 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase): ...@@ -2017,6 +2184,83 @@ class TestInventory(TestOrderMixin, ERP5TypeTestCase):
resource_uid=product.getUid()), resource_uid=product.getUid()),
0) 0)
def test_06_FullInventory(self, quiet=0, run=run_all_test):
"""
Test the full inventory behavior
"""
if not run: return
sequence_list = SequenceList()
sequence_string = 'stepCreateOrganisationsForModule \
stepCreateNotVariatedResource \
stepCreateNotVariatedSecondResource \
stepCreateItemList \
stepCreatePackingListForModule \
stepTic \
stepCreatePackingListLine \
stepTic \
stepDeliverPackingList \
stepTic \
stepCreateFullInventory \
stepTic \
stepTestFullInventory \
'
sequence_list.addSequenceString(sequence_string)
sequence_list.play(self)
def test_07_FullVariatedInventory(self, quiet=0, run=run_all_test):
"""
Test the full inventory behavior with variated resource
"""
if not run: return
sequence_list = SequenceList()
sequence_string = 'stepCreateOrganisationsForModule \
stepCreateVariatedResource \
stepCreateNotVariatedSecondResource \
stepCreateItemList \
stepCreatePackingListForModule \
stepTic \
stepCreateVariatedPackingListLine \
stepTic \
stepDeliverPackingList \
stepTic \
stepCreateFullVariatedInventory \
stepTic \
stepTestFullVariatedInventory \
'
sequence_list.addSequenceString(sequence_string)
sequence_list.play(self)
def test_08_PartialInventoryMultipleResource(self, quiet=0, run=run_all_test):
"""
Test behaviour of partial inventory with multiple resource
defining inventory of resource B must not modify inventory of resource A
"""
if not run: return
sequence_list = SequenceList()
sequence_string = 'stepCreateOrganisationsForModule \
stepCreateNotVariatedResource \
stepCreateNotVariatedSecondResource \
stepCreateItemList \
stepCreatePackingListForModule \
stepTic \
stepCreatePackingListLine \
stepTic \
stepDeliverPackingList \
stepTic \
stepCreatePartialInventoryMultipleResource \
stepTic \
stepTestPartialInventoryMultipleResource \
'
sequence_list.addSequenceString(sequence_string)
sequence_list.play(self)
def test_suite(): def test_suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestInventory)) suite.addTest(unittest.makeSuite(TestInventory))
......
product/ERP5/tests/testInventoryAPI.py
View file @ cf53db45
...@@ -38,14 +38,13 @@ import unittest ...@@ -38,14 +38,13 @@ import unittest
from AccessControl.SecurityManagement import newSecurityManager from AccessControl.SecurityManagement import newSecurityManager
from DateTime import DateTime from DateTime import DateTime
from Testing import ZopeTestCase from MySQLdb import ProgrammingError
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ERP5Type.tests.utils import reindex from Products.ERP5Type.tests.utils import reindex
from Products.ERP5Type.tests.backportUnittest import expectedFailure from Products.ERP5Type.tests.backportUnittest import expectedFailure
from Products.DCWorkflow.DCWorkflow import ValidationFailed from Products.ERP5.Tool.SimulationTool import MYSQL_MIN_DATETIME_RESOLUTION
from Products.ERP5Type.Base import _aq_reset
from Products.ERP5Type.tests.utils import createZODBPythonScript
class InventoryAPITestCase(ERP5TypeTestCase): class InventoryAPITestCase(ERP5TypeTestCase):
"""Base class for Inventory API Tests {{{ """Base class for Inventory API Tests {{{
...@@ -182,10 +181,11 @@ class InventoryAPITestCase(ERP5TypeTestCase): ...@@ -182,10 +181,11 @@ class InventoryAPITestCase(ERP5TypeTestCase):
""" erp5_trade is required for transit_simulation_state """ erp5_trade is required for transit_simulation_state
erp5_apparel is required for item erp5_apparel is required for item
""" """
return ('erp5_base', 'erp5_pdm', 'erp5_dummy_movement', 'erp5_simulation', return ('erp5_core_proxy_field_legacy', 'erp5_base', 'erp5_pdm',
'erp5_dummy_movement', 'erp5_simulation',
'erp5_trade', 'erp5_apparel', 'erp5_project', 'erp5_trade', 'erp5_apparel', 'erp5_project',
'erp5_configurator_standard_trade_template', 'erp5_configurator_standard_trade_template',
'erp5_simulation_test') 'erp5_simulation_test', 'erp5_stock_cache')
# TODO: move this to a base class {{{ # TODO: move this to a base class {{{
@reindex @reindex
...@@ -1033,7 +1033,7 @@ class TestInventoryList(InventoryAPITestCase): ...@@ -1033,7 +1033,7 @@ class TestInventoryList(InventoryAPITestCase):
self._makeMovement(quantity=7, use='use2') self._makeMovement(quantity=7, use='use2')
self._makeMovement(quantity=4, use='use2') self._makeMovement(quantity=4, use='use2')
# note that grouping by related key only make sense if you group by strict # note that grouping by related key only make sense if you group by strict
# memebership related keys # membership related keys
inventory_list = getInventoryList(node_uid=(self.node.getUid(), inventory_list = getInventoryList(node_uid=(self.node.getUid(),
self.other_node.getUid()), self.other_node.getUid()),
group_by=('strict_use_uid', )) group_by=('strict_use_uid', ))
...@@ -2351,133 +2351,62 @@ class TestTrackingList(InventoryAPITestCase): ...@@ -2351,133 +2351,62 @@ class TestTrackingList(InventoryAPITestCase):
'%s=now - %i, aggregate should be at node %i but is at node %i' % \ '%s=now - %i, aggregate should be at node %i but is at node %i' % \
(param_id, now - date, node_uid_to_node_number[location_uid], node_uid_to_node_number[uid_list[0]])) (param_id, now - date, node_uid_to_node_number[location_uid], node_uid_to_node_number[uid_list[0]]))
class TestInventoryDocument(InventoryAPITestCase): class TestInventoryCacheTable(InventoryAPITestCase):
""" Test impact of creating full inventories of stock points on inventory """ Test impact of creating cache entries into inventory_cache table
lookup. This is an optimisation to regular inventory system to avoid This is an optimisation on stock results to avoid reading all stock entries
reading all stock entries since a node/section/payment is used when for the same request.
gathering its amounts of resources.
""" Cache description:
def _createAutomaticInventoryAtDate(self, date, override_inventory=None, A cache entry is defined by:
full_inventory=False): - a request identifier (implementation detail: MD5 of query source code)
""" - the date of first possible stock row *excluded* from cache entry (even if
getInventoryList is tested to work in another unit test. there is no row at that exact date)
If full_inventory is false, only inventoriate the first resource - cache data (implementation detail: pickle of a dict containing selected
found. properties of a query result)
Cache filling:
Cache might be filled by any request with either an at_date or a to_date,
and without a from_date. Cache entry date is chosen as "{at,to}_date -
cache_lag / 2".
No cache entry is created if a cache entry newer than "{at,to}_date -
cache_lag" exists.
Cache flush:
Cache is flushed by any modification (actually triggered on indexation or
de-indexation) on a stock-indexed document (aka movement). Only cache
entries with a date greater than any modified stock line are dropped.
""" """
self.tic() # Tic so that grabbed inventory is up to date. def afterSetUp(self):
getInventoryList = self.getSimulationTool().getInventoryList InventoryAPITestCase.afterSetUp(self)
portal = self.getPortal() self.CACHE_LAG = cache_lag = self.getSimulationTool().getInventoryCacheLag()
inventory_module = portal.getDefaultModule(portal_type='Inventory') min_lag = cache_lag / 2
inventory = inventory_module.newContent(portal_type='Inventory') self.NOW = now = DateTime(DateTime().strftime("%Y-%m-%d %H:%M:%S UTC"))
inventory.edit(destination_value=self.node, self.CACHE_DATE = cache_date = now - min_lag
destination_section_value=self.section, self.LAST_CACHED_MOVEMENT_DATE = last_cached_movement_date = \
start_date=date, cache_date - MYSQL_MIN_DATETIME_RESOLUTION
full_inventory=full_inventory) # First movement, won't be into cache
inventory_list = getInventoryList(node_uid=self.node.getUid(), self.INVENTORY_DATE_3 = INVENTORY_DATE_3 = now - 10
at_date=date,
omit_output=1)
if full_inventory:
inventory_list = [inventory_list[0]]
# TODO: Define a second resource which will only be present in full
# inventories. This will allow testing getInventoryList.
#else:
# inventory_list.append({'resource_relative_url': '','total_quantity': 50,'variation_text': ''})
for inventory_line in inventory_list:
line = inventory.newContent(portal_type='Inventory Line')
if override_inventory is None:
total_quantity = inventory_line['total_quantity']
else:
total_quantity = override_inventory
line.edit(resource=inventory_line['resource_relative_url'],
inventory=total_quantity,
variation_text=inventory_line['variation_text'])
# TODO: pass more properties through from calculated inventory to
# inventory lines if needed.
inventory.deliver()
return inventory
def _populateInventoryModule(self):
"""
Create 3 inventories:
Type Deviation Date (see stepCreateInitialMovements)
- partial 1000
- full 10000
- full 100000
"""
self.BASE_QUANTITY = BASE_QUANTITY = 1
# TODO: It would be better to strip numbers below seconds instead of below
# days.
self.MAX_DATE = MAX_DATE = DateTime(DateTime().Date()) - 1
self.DUPLICATE_INVENTORY_DATE = MAX_DATE - 8 # Newest
self.INVENTORY_DATE_3 = INVENTORY_DATE_3 = MAX_DATE - 10 # Newest
self.INVENTORY_QUANTITY_3 = INVENTORY_QUANTITY_3 = 100000 self.INVENTORY_QUANTITY_3 = INVENTORY_QUANTITY_3 = 100000
self.INVENTORY_DATE_2 = INVENTORY_DATE_2 = INVENTORY_DATE_3 - 10 # Second movement, won't be into cache, just at the limit of cache min_lag
self.INVENTORY_DATE_2 = INVENTORY_DATE_2 = cache_date
self.INVENTORY_QUANTITY_2 = INVENTORY_QUANTITY_2 = 10000 self.INVENTORY_QUANTITY_2 = INVENTORY_QUANTITY_2 = 10000
self.INVENTORY_DATE_1 = INVENTORY_DATE_1 = INVENTORY_DATE_2 - 10 # Oldest # Next will be stored as cache result after first getInventory
self.INVENTORY_DATE_1 = INVENTORY_DATE_1 = last_cached_movement_date
self.INVENTORY_QUANTITY_1 = INVENTORY_QUANTITY_1 = 1000 self.INVENTORY_QUANTITY_1 = INVENTORY_QUANTITY_1 = 1000
# Create movements
# "actual" quantities are the quantities which will end up in the stock self._makeMovement(
# table. quantity=INVENTORY_QUANTITY_1,
self.ACTUAL_INVENTORY_QUANTITY_1 = INVENTORY_QUANTITY_1 - \ start_date=INVENTORY_DATE_1,
BASE_QUANTITY simulation_state='delivered',
self.ACTUAL_INVENTORY_QUANTITY_2 = INVENTORY_QUANTITY_2 - \ )
(self.INVENTORY_QUANTITY_1 + BASE_QUANTITY) self._makeMovement(
self.ACTUAL_INVENTORY_QUANTITY_3 = INVENTORY_QUANTITY_3 - \ quantity=INVENTORY_QUANTITY_2,
(self.INVENTORY_QUANTITY_2 + BASE_QUANTITY) start_date=INVENTORY_DATE_2,
simulation_state='delivered',
self.movement_uid_list = movement_uid_list = [] )
# Initial movement of 1 self._makeMovement(
movement = self._makeMovement(quantity=BASE_QUANTITY, quantity=INVENTORY_QUANTITY_3,
start_date=INVENTORY_DATE_1 - 1, start_date=INVENTORY_DATE_3,
simulation_state='delivered') simulation_state='delivered',
movement_uid_list.append(movement.getUid()) )
# First (partial) inventory of 1 000
partial_inventory = self._createAutomaticInventoryAtDate(
date=INVENTORY_DATE_1, override_inventory=INVENTORY_QUANTITY_1)
# Second movement of 1
movement = self._makeMovement(quantity=BASE_QUANTITY,
start_date=INVENTORY_DATE_2 - 1,
simulation_state='delivered')
movement_uid_list.append(movement.getUid())
# Second (full) inventory of 10 000
self._createAutomaticInventoryAtDate(date=INVENTORY_DATE_2,
override_inventory=INVENTORY_QUANTITY_2,
full_inventory=True)
# Third movement of 1
movement = self._makeMovement(quantity=BASE_QUANTITY,
start_date=INVENTORY_DATE_3 - 1,
simulation_state='delivered')
movement_uid_list.append(movement.getUid())
# Third (full) inventory of 100 000
self._createAutomaticInventoryAtDate(date=INVENTORY_DATE_3,
override_inventory=INVENTORY_QUANTITY_3,
full_inventory=True)
# Fourth movement of 1
movement = self._makeMovement(quantity=BASE_QUANTITY,
start_date=INVENTORY_DATE_3 + 1,
simulation_state='delivered')
movement_uid_list.append(movement.getUid())
self.tic()
manage_test = self.getPortal().erp5_sql_transactionless_connection.manage_test
def executeSQL(query):
manage_test("BEGIN\x00%s\x00COMMIT" % (query, ))
# Make stock table inconsistent with inventory_stock to make sure
# inventory_stock is actually tested.
executeSQL("UPDATE stock SET quantity=quantity*2 WHERE uid IN (%s)" %
(', '.join([str(x) for x in movement_uid_list]), ))
self.BASE_QUANTITY *= 2
# Make inventory_stock table inconsistent with stock to make sure
# inventory_stock is actually not used when checking that partial
# inventory is not taken into account.
executeSQL("UPDATE inventory_stock SET quantity=quantity*2 WHERE "\
"uid IN (%s)" % (', '.join([str(x.getUid()) for x in \
partial_inventory.objectValues()]),
))
def afterSetUp(self):
InventoryAPITestCase.afterSetUp(self)
self._populateInventoryModule()
simulation_tool = self.getSimulationTool() simulation_tool = self.getSimulationTool()
self.getInventory = simulation_tool.getInventory self.getInventory = simulation_tool.getInventory
self.getInventoryList = simulation_tool.getInventoryList self.getInventoryList = simulation_tool.getInventoryList
...@@ -2537,586 +2466,458 @@ class TestInventoryDocument(InventoryAPITestCase): ...@@ -2537,586 +2466,458 @@ class TestInventoryDocument(InventoryAPITestCase):
# Leads to rasing exception instead of calling self.assert[...] method. # Leads to rasing exception instead of calling self.assert[...] method.
if not success: if not success:
if ordered_check: if ordered_check:
raise AssertionError, 'Line %r do not match %r' % \ raise AssertionError, 'Line %r\ndo not match\n %r' % \
(inventory_list[inventory_position], (inventory_list[inventory_position],
criterion_dict) criterion_dict)
else: else:
raise AssertionError, 'No line in %r match %r' % \ raise AssertionError, 'No line in %r\n match\n %r' % \
(inventory_list, criterion_dict) (inventory_list, criterion_dict)
# Check all expected lines have been found. # Check all expected lines have been found.
self.assertFalse(inventory_list) self.assertFalse(inventory_list)
def _fillCache(self, inventory_list=False):
"""
Calling getInventoryXXX will fill the cache for us
"""
if inventory_list:
result_list = self.getInventoryList(node_uid=self.node_uid, to_date=self.NOW)
result = 0
for line in result_list:
result += line.quantity
else:
result = self.getInventory(node_uid=self.node_uid, to_date=self.NOW)
self.assertEqual(result, self.INVENTORY_QUANTITY_1 + \
self.INVENTORY_QUANTITY_2 + self.INVENTORY_QUANTITY_3)
return result
def doubleStockValue(self):
"""
Make stock table inconsistent so that we can check that
optimisation is well used
"""
# in the test, this will always be the date at which an entry is put into
# inventory_cache
self.getPortalObject().erp5_sql_transactionless_connection.manage_test(
"BEGIN\0"
"UPDATE stock SET quantity=quantity*2 WHERE date < '%s'\0"
"COMMIT" % ((self.CACHE_DATE).ISO(), ))
self.commit()
def assertInventoryEquals(self, value, inventory_kw): def assertInventoryEquals(self, value, inventory_kw):
""" """
Check that optimised getInventory call is equal to given value Check that optimised getInventory call is equal to given value
and that unoptimised call is *not* equal to thi value. and that unoptimised call is *not* equal to thi value.
""" """
# Make stock table inconsistent to be sure it uses cache
self.doubleStockValue()
# Check it use cache
self.assertEquals(value, self.getInventory(**inventory_kw)) self.assertEquals(value, self.getInventory(**inventory_kw))
self.assertNotEquals(value, self.assertNotEquals(value,
self.getInventory(optimisation__=False, self.getInventory(optimisation__=False,
**inventory_kw)) **inventory_kw))
def setUpDefaultInventoryCalculationList(self): def test_01_CurrentInventory(self):
createZODBPythonScript(self.portal.portal_skins.custom,
'Inventory_getDefaultInventoryCalculationList', '',
'''return ({
'inventory_params':{
'section_uid':context.getDestinationSectionUid(),
'node_uid':context.getDestinationUid(),
'group_by_variation':1,
'group_by_resource':1},
'list_method':'getMovementList',
'first_level':({'key':'resource_relative_url',
'getter':'getResource',
'setter':('appendToCategoryList', 'resource')},
{'key':'variation_text',
'getter':'getVariationText',
'setter':'splitAndExtendToCategoryList'},
),
},)
''')
self.commit()
def clearAllInventoryAndSetUpTwoInventory(self):
self.portal.inventory_module.manage_delObjects(list(self.portal.inventory_module.objectIds()))
self.folder.manage_delObjects(list(self.folder.objectIds()))
self.resource.setProductLine('level1/level2')
self.other_resource.setProductLine('anotherlevel')
self.section.setGroup('level1/level2')
self.other_section.setGroup('anotherlevel')
self.node.setRegion('level1')
self.other_node.setGroup('anotherlevel')
full_inventory = self.portal.inventory_module.newContent(portal_type='Inventory')
full_inventory.edit(destination_section_value=self.section,
destination_value=self.node,
full_inventory=1,
start_date=DateTime('2012/05/18 00:00:00 GMT+9'))
line = full_inventory.newContent(portal_type='Inventory Line')
line.setResourceValue(self.resource)
line.setQuantity(1)
line = full_inventory.newContent(portal_type='Inventory Line')
line.setResourceValue(self.other_resource)
line.setQuantity(10)
full_inventory.deliver()
full_inventory = self.portal.inventory_module.newContent(portal_type='Inventory')
full_inventory.edit(destination_section_value=self.other_section,
destination_value=self.node,
full_inventory=1,
start_date=DateTime('2012/05/18 00:00:00 GMT+9'))
line = full_inventory.newContent(portal_type='Inventory Line')
line.setResourceValue(self.resource)
line.setQuantity(100)
line = full_inventory.newContent(portal_type='Inventory Line')
line.setResourceValue(self.other_resource)
line.setQuantity(1000)
full_inventory.deliver()
full_inventory = self.portal.inventory_module.newContent(portal_type='Inventory')
full_inventory.edit(destination_section_value=self.section,
destination_value=self.other_node,
full_inventory=1,
start_date=DateTime('2012/05/18 00:00:00 GMT+9'))
line = full_inventory.newContent(portal_type='Inventory Line')
line.setResourceValue(self.resource)
line.setQuantity(10000)
line = full_inventory.newContent(portal_type='Inventory Line')
line.setResourceValue(self.other_resource)
line.setQuantity(100000)
full_inventory.deliver()
full_inventory = self.portal.inventory_module.newContent(portal_type='Inventory')
full_inventory.edit(destination_section_value=self.other_section,
destination_value=self.other_node,
full_inventory=1,
start_date=DateTime('2012/05/18 00:00:00 GMT+9'))
line = full_inventory.newContent(portal_type='Inventory Line')
line.setResourceValue(self.resource)
line.setQuantity(1000000)
line = full_inventory.newContent(portal_type='Inventory Line')
line.setResourceValue(self.other_resource)
line.setQuantity(10000000)
full_inventory.deliver()
self.commit()
self.tic()
def test_01_CurrentInventoryWithFullInventory(self):
"""
Check that inventory optimisation is executed when querying current
amount (there is a usable full inventory which is the latest).
"""
self.assertInventoryEquals(value=self.INVENTORY_QUANTITY_3 + \
self.BASE_QUANTITY,
inventory_kw={'node_uid': self.node_uid})
def test_02_InventoryAtLatestFullInventoryDate(self):
""" """
Check that inventory optimisation is executed when querying an amount Check that optimisation is executed when querying current
at the exact time of latest usable full inventory. amount
""" """
self.assertInventoryEquals(value=self.INVENTORY_QUANTITY_3, self.assertInventoryEquals(
inventory_kw={'node_uid': self.node_uid, self._fillCache(),
'at_date': self.INVENTORY_DATE_3}) inventory_kw={
'node_uid': self.node_uid,
'to_date': self.NOW,
}
)
def test_03_InventoryAtEarlierFullInventoryDate(self): def test_02_InventoryAtCacheDate(self):
""" """
Check that inventory optimisation is executed when querying past Check that optimisation is executed when querying an amount
amount (there is a usable full inventory which is not the latest). at the exact time of the cache of result
""" """
self.assertInventoryEquals(value=self.INVENTORY_QUANTITY_2 + \ self._fillCache()
self.BASE_QUANTITY, # We got results from cache + results from stock
inventory_kw={'node_uid': self.node_uid, self.assertInventoryEquals(
'at_date': self.INVENTORY_DATE_3 - \ self.INVENTORY_QUANTITY_2 + self.INVENTORY_QUANTITY_1,
1}) inventory_kw={
'node_uid': self.node_uid,
'at_date': self.CACHE_DATE,
},
)
def test_04_InventoryBeforeFullInventoryAfterPartialInventory(self): def test_03_InventoryToCacheDate(self):
""" """
Check that optimisation is not executed when querying past amount Check that optimisation is executed when querying an amount
with no usable full inventory. that will only take into account cache data.
If optimisation was executed,
self.INVENTORY_QUANTITY_1 * 2 + self.BASE_QUANTITY * 2
would be found.
""" """
self.assertEquals(self.ACTUAL_INVENTORY_QUANTITY_1 + \ self._fillCache()
self.BASE_QUANTITY * 2, # We got only results from cache
self.getInventory(node_uid=self.node_uid, self.assertInventoryEquals(
at_date=self.INVENTORY_DATE_2 - 1)) self.INVENTORY_QUANTITY_1,
inventory_kw={
'node_uid': self.node_uid,
'to_date': self.CACHE_DATE,
},
)
def test_05_InventoryListWithFullInventory(self): def test_04_InventoryList(self):
""" """
Check that inventory optimisation is executed when querying current Check that optimisation is executed when querying current
amount list (there is a usable full inventory which is the latest). amount list
""" """
inventory = self.getInventoryList(node_uid=self.node_uid) self._fillCache(True)
reference_inventory = [ # Check we got all results
{'date': self.INVENTORY_DATE_3, self._checkInventoryList(
self.getInventoryList(node_uid=self.node_uid, to_date=self.NOW),
[{
'date': self.INVENTORY_DATE_3,
'inventory': self.INVENTORY_QUANTITY_3, 'inventory': self.INVENTORY_QUANTITY_3,
'node_uid': self.node_uid}, 'node_uid': self.node_uid,
{'date': self.INVENTORY_DATE_3 + 1, }, {
'inventory': self.BASE_QUANTITY, 'date': self.INVENTORY_DATE_2,
'node_uid': self.node_uid} 'inventory': self.INVENTORY_QUANTITY_2,
] 'node_uid': self.node_uid,
self._checkInventoryList(inventory, reference_inventory) }, {
'date': self.INVENTORY_DATE_1,
'inventory': self.INVENTORY_QUANTITY_1,
'node_uid': self.node_uid,
}],
)
def test_06_InventoryListAtLatestFullInventoryDate(self): def test_05_InventoryListAtCacheDate(self):
""" """
Check that inventory optimisation is executed when querying past Check that optimisation is executed when querying an amount list
amount list (there is a usable full inventory which is not the latest). at the exact time of the cache of result
""" """
inventory = self.getInventoryList(node_uid=self.node_uid, self._fillCache(True)
at_date=self.INVENTORY_DATE_3) # We got results from cache + results from stock
reference_inventory = [ self._checkInventoryList(
{'date': self.INVENTORY_DATE_3, self.getInventoryList(node_uid=self.node_uid, at_date=self.CACHE_DATE),
'inventory': self.INVENTORY_QUANTITY_3, [{
'node_uid': self.node_uid} 'date': self.INVENTORY_DATE_1,
] 'inventory': self.INVENTORY_QUANTITY_1,
self._checkInventoryList(inventory, reference_inventory) 'node_uid': self.node_uid,
}, {
'date': self.INVENTORY_DATE_2,
'inventory': self.INVENTORY_QUANTITY_2,
'node_uid': self.node_uid,
}],
)
def test_07_InventoryListAtEarlierFullInventoryDate(self): def test_06_InventoryListToCacheDate(self):
""" """
Check that inventory optimisation is executed when querying past Check that optimisation is executed when querying an amount list
amount list (there is a usable full inventory which is not the latest). that will only take into account cache data.
""" """
inventory = self.getInventoryList(node_uid=self.node_uid, self._fillCache(True)
at_date=self.INVENTORY_DATE_3 - 1) # We got only results from cache
reference_inventory = [ self._checkInventoryList(
{'date': self.INVENTORY_DATE_2, self.getInventoryList(node_uid=self.node_uid, to_date=self.CACHE_DATE),
'inventory': self.INVENTORY_QUANTITY_2, [{
'node_uid': self.node_uid}, 'date': self.INVENTORY_DATE_1,
{'date': self.INVENTORY_DATE_3 - 1, 'inventory': self.INVENTORY_QUANTITY_1,
'inventory': self.BASE_QUANTITY, 'node_uid': self.node_uid,
'node_uid': self.node_uid} }],
] )
self._checkInventoryList(inventory, reference_inventory)
def test_08_InventoryListBeforeFullInventoryAfterPartialInventory(self):
"""
Check that optimisation is not executed when querying past amount list
with no usable full inventory.
"""
inventory = self.getInventoryList(node_uid=self.node_uid,
at_date=self.INVENTORY_DATE_2 - 1)
reference_inventory = [
{'date': self.INVENTORY_DATE_1 - 1,
'inventory': self.BASE_QUANTITY,
'node_uid': self.node_uid},
{'date': self.INVENTORY_DATE_1,
'inventory': self.ACTUAL_INVENTORY_QUANTITY_1,
'node_uid': self.node_uid},
{'date': self.INVENTORY_DATE_2 - 1,
'inventory': self.BASE_QUANTITY,
'node_uid': self.node_uid}
]
self._checkInventoryList(inventory, reference_inventory)
def test_09_InventoryListGroupedByResource(self): def test_07_InventoryListGroupedByResource(self):
""" """
Group inventory list by resource explicitely, used inventory is the Check that optimisation is executed when grouping inventory list
latest. by resource explicitely
""" """
inventory = self.getInventoryList(node_uid=self.node_uid, inventory_kw = {
group_by_resource=1) 'node_uid': self.node_uid,
reference_inventory = [ 'to_date': self.NOW,
{'inventory': self.INVENTORY_QUANTITY_3 + self.BASE_QUANTITY, 'group_by_resource': 1,
}
# Fill cache
self.getInventoryList(**inventory_kw)
# Check we got all results
self._checkInventoryList(
self.getInventoryList(**inventory_kw),
[{
'inventory': self.INVENTORY_QUANTITY_3 + self.INVENTORY_QUANTITY_2 + \
self.INVENTORY_QUANTITY_1,
'resource_uid': self.resource.getUid(), 'resource_uid': self.resource.getUid(),
'node_uid': self.node_uid} 'node_uid': self.node_uid,
] }],
self._checkInventoryList(inventory, reference_inventory) )
def test_10_InventoryListGroupedByResourceBeforeLatestFullInventoryDate(self): def test_08_InventoryListGroupedByResourceAtCacheDate(self):
""" """
Group inventory list by resource explicitely, used inventory is not the Check that optimisation is executed when grouping inventory list
latest. by resource explicitely
""" """
inventory = self.getInventoryList(node_uid=self.node_uid, # Fill cache
group_by_resource=1, self.getInventoryList(node_uid=self.node_uid, to_date=self.NOW,
at_date=self.INVENTORY_DATE_3 - 1) group_by_resource=1)
reference_inventory = [ # We got results from cache + results from stock
{'inventory': self.INVENTORY_QUANTITY_2 + self.BASE_QUANTITY, self._checkInventoryList(
self.getInventoryList(node_uid=self.node_uid, at_date=self.CACHE_DATE,
group_by_resource=1),
[{
'inventory': self.INVENTORY_QUANTITY_2 + self.INVENTORY_QUANTITY_1,
'resource_uid': self.resource.getUid(), 'resource_uid': self.resource.getUid(),
'node_uid': self.node_uid} 'node_uid': self.node_uid,
] }],
self._checkInventoryList(inventory, reference_inventory) )
def test_11_InventoryListAroundLatestInventoryDate(self): def test_09_InventoryListGroupByResourceToCacheDate(self):
""" """
Test getInventoryList with a min and a max date around latest full Check that optimisation is executed when grouping inventory list
inventory. A full inventory is used and is not the latest. by resource explicitely
""" """
inventory = self.getInventoryList(node_uid=self.node_uid, self.getInventoryList(node_uid=self.node_uid, to_date=self.NOW, group_by_resource=1)
from_date=self.INVENTORY_DATE_3 - 1, # We got only results from cache
at_date=self.INVENTORY_DATE_3 + 1) self._checkInventoryList(
reference_inventory = [ self.getInventoryList(node_uid=self.node_uid, to_date=self.CACHE_DATE,
{'inventory': self.BASE_QUANTITY, group_by_resource=1),
'resource_uid': self.resource.getUid(), [{
'node_uid': self.node_uid, 'inventory': self.INVENTORY_QUANTITY_1,
'date': self.INVENTORY_DATE_3 - 1},
{'inventory': self.ACTUAL_INVENTORY_QUANTITY_3,
'resource_uid': self.resource.getUid(),
'node_uid': self.node_uid,
'date': self.INVENTORY_DATE_3},
{'inventory': self.BASE_QUANTITY,
'resource_uid': self.resource.getUid(), 'resource_uid': self.resource.getUid(),
'node_uid': self.node_uid, 'node_uid': self.node_uid,
'date': self.INVENTORY_DATE_3 + 1} }],
] )
self._checkInventoryList(inventory, reference_inventory)
def test_12_InventoryListWithOrderByDate(self): def test_10_InventoryListWithOrderByDate(self):
""" """
Test order_by is preserved by optimisation on date column. Test order_by is preserved by optimisation on date column.
Also sort on total_quantity column because there are inventory lines """
which are on the same date but with distinct quantities. inventory_kw={
""" 'node_uid': self.node_uid,
inventory = self.getInventoryList(node_uid=self.node_uid, 'to_date': self.NOW,
from_date=self.INVENTORY_DATE_3 - 1, }
at_date=self.INVENTORY_DATE_3 + 1, sort_on = (('date', 'ASC'), ('total_quantity', 'DESC'))
sort_on=(('date', 'ASC'), reversed_sort_on = (('date', 'DESC'), ('total_quantity', 'ASC'))
('total_quantity', 'DESC'))) reference_inventory = [{
reference_inventory = [ 'inventory': self.INVENTORY_QUANTITY_1,
{'inventory': self.BASE_QUANTITY,
'resource_uid': self.resource.getUid(), 'resource_uid': self.resource.getUid(),
'node_uid': self.node_uid, 'node_uid': self.node_uid,
'date': self.INVENTORY_DATE_3 - 1}, 'date': self.INVENTORY_DATE_1,
{'inventory': self.ACTUAL_INVENTORY_QUANTITY_3, }, {
'inventory': self.INVENTORY_QUANTITY_2,
'resource_uid': self.resource.getUid(), 'resource_uid': self.resource.getUid(),
'node_uid': self.node_uid, 'node_uid': self.node_uid,
'date': self.INVENTORY_DATE_3}, 'date': self.INVENTORY_DATE_2
{'inventory': self.BASE_QUANTITY, }, {
'inventory': self.INVENTORY_QUANTITY_3,
'resource_uid': self.resource.getUid(), 'resource_uid': self.resource.getUid(),
'node_uid': self.node_uid, 'node_uid': self.node_uid,
'date': self.INVENTORY_DATE_3 + 1} 'date': self.INVENTORY_DATE_3,
] }]
self._checkInventoryList(inventory, reference_inventory, # Fill cache
ordered_check=True) self.getInventoryList(sort_on=sort_on, **inventory_kw)
inventory = self.getInventoryList(node_uid=self.node_uid, # Check it in fist order
from_date=self.INVENTORY_DATE_3 - 1, self._checkInventoryList(
at_date=self.INVENTORY_DATE_3 + 1, self.getInventoryList(sort_on=sort_on, **inventory_kw),
sort_on=(('date', 'DESC'), reference_inventory,
('total_quantity', 'ASC'))) ordered_check=True,
)
# Check it in reverse order
reference_inventory.reverse() reference_inventory.reverse()
self._checkInventoryList(inventory, reference_inventory, self._checkInventoryList(
ordered_check=True) self.getInventoryList(sort_on=reversed_sort_on, **inventory_kw),
reference_inventory,
ordered_check=True,
)
def test_13_InventoryAfterModificationInPast(self): def test_11_InventoryWithFromDate(self):
""" """
Test inventory after adding a new movement in past and reindex all inventory Test that getInventory called with from date does not generate cache entry
""" """
movement = self._makeMovement(quantity=self.BASE_QUANTITY*2, inventory_kw = {
start_date=self.INVENTORY_DATE_3 - 2, 'node_uid': self.node_uid,
simulation_state='delivered') 'to_date': self.NOW,
# reindex inventory module, although we modified table by hand 'from_date': self.NOW - self.CACHE_LAG - 10,
# everything must be consistent after reindexation }
inventory_module = self.getPortal().getDefaultModule(portal_type='Inventory') # Double stock value and check that we got same result
inventory_module.recursiveReindexObject() # with and without optimisation
self.tic() value = self.INVENTORY_QUANTITY_3 + self.INVENTORY_QUANTITY_2 + \
inventory_kw={'node_uid': self.node_uid, 2 * self.INVENTORY_QUANTITY_1
'at_date': self.INVENTORY_DATE_3} self.doubleStockValue()
value=self.INVENTORY_QUANTITY_3
# use optimisation
self.assertEquals(value, self.getInventory(**inventory_kw)) self.assertEquals(value, self.getInventory(**inventory_kw))
# without optimisation
self.assertEquals(value, self.assertEquals(value,
self.getInventory(optimisation__=False, self.getInventory(optimisation__=False,
**inventory_kw)) **inventory_kw))
def test_14_TwoInventoryWithSameDateAndResourceAndNode(self): def test_12_CheckCacheFlush(self):
""" """
It makes no sense to validate two inventories with same date, Test the cache is flushed when indexing a movement into stock
same resource, and same node. The calculation of inventories
will not work in such case. So here we test that a constraint
does not allow such things
""" """
portal = self.getPortal() inventory_kw = {
self._addPropertySheet('Inventory', 'InventoryConstraint') 'node_uid': self.node_uid,
try: 'to_date': self.NOW,
inventory_module = portal.getDefaultModule(portal_type='Inventory') }
inventory = inventory_module.newContent(portal_type='Inventory') # Fill cache and make stock inconsistent
date = self.DUPLICATE_INVENTORY_DATE value = self.getInventory(**inventory_kw)
inventory.edit(destination_value=self.node, self.doubleStockValue()
destination_section_value=self.section, # Create a movement after CACHE DATE
start_date=date) # Cache is cleared for all entry > movement date
inventory_line = inventory.newContent( # as at a cache date D, it contains results from stock
resource_value = self.resource, # for all line < D
quantity = 1) INVENTORY_QUANTITY_4 = 5000
self.workflow_tool = portal.portal_workflow INVENTORY_DATE_4 = self.CACHE_DATE
workflow_id = 'inventory_workflow' movement = self._makeMovement(
transition_id = 'deliver_action' quantity=INVENTORY_QUANTITY_4,
workflow_id= 'inventory_workflow' start_date=INVENTORY_DATE_4,
self.workflow_tool.doActionFor(inventory, transition_id, simulation_state='delivered',
wf_id=workflow_id) )
self.assertEquals('delivered', inventory.getSimulationState())
self.tic() self.tic()
# Optimisation must still be used
# We should detect the previous inventory and fails self.assertEquals(
new_inventory = inventory.Base_createCloneDocument(batch_mode=1) value + INVENTORY_QUANTITY_4,
self.assertRaises(ValidationFailed, self.workflow_tool.doActionFor, self.getInventory(**inventory_kw),
new_inventory, transition_id, wf_id=workflow_id) )
workflow_history = self.workflow_tool.getInfoFor(ob=new_inventory, # Edit start date so that cache table is cleared
name='history', wf_id=workflow_id) movement.edit(start_date=self.LAST_CACHED_MOVEMENT_DATE)
workflow_error_message = str(workflow_history[-1]['error_message'])
self.assertTrue(len(workflow_error_message))
self.assertTrue(len([x for x in workflow_error_message \
if x.find('There is already an inventory')]))
# Add a case in order to check a bug when the other inventory at the
# same date does not change stock values
new_inventory = inventory.Base_createCloneDocument(batch_mode=1)
new_inventory.setStartDate(self.DUPLICATE_INVENTORY_DATE + 1)
self.workflow_tool.doActionFor(new_inventory, transition_id,
wf_id=workflow_id)
self.assertEquals('delivered', new_inventory.getSimulationState())
self.tic() self.tic()
self.assertEquals(
value + INVENTORY_QUANTITY_4 + self.INVENTORY_QUANTITY_1,
self.getInventory(**inventory_kw),
)
self.doubleStockValue()
# Cache hit again
self.assertEquals(
value + INVENTORY_QUANTITY_4 + self.INVENTORY_QUANTITY_1,
self.getInventory(**inventory_kw),
)
# Delete movement, so it gets unindexed
self.folder.manage_delObjects(ids=[movement.getId(), ])
self.tic()
self.assertEquals(
value + 3 * self.INVENTORY_QUANTITY_1,
self.getInventory(**inventory_kw),
)
self.doubleStockValue()
# Cache hit again
self.assertEquals(
value + 3 * self.INVENTORY_QUANTITY_1,
self.getInventory(**inventory_kw),
)
new_inventory = new_inventory.Base_createCloneDocument(batch_mode=1) def test_13_CacheCreatedFromCache(self):
self.assertRaises(ValidationFailed, self.workflow_tool.doActionFor,
new_inventory, transition_id, wf_id=workflow_id)
workflow_history = self.workflow_tool.getInfoFor(ob=new_inventory,
name='history', wf_id=workflow_id)
workflow_error_message = str(workflow_history[-1]['error_message'])
self.assertTrue(len(workflow_error_message))
self.assertTrue(len([x for x in workflow_error_message \
if x.find('There is already an inventory')]))
finally:
# remove all property sheets we added to type informations
ttool = self.getTypesTool()
for ti_name, psheet_list in self._added_property_sheets.iteritems():
ti = ttool.getTypeInfo(ti_name)
property_sheet_set = set(ti.getTypePropertySheetList())
property_sheet_set.difference_update(psheet_list)
ti._setTypePropertySheetList(list(property_sheet_set))
self.commit()
_aq_reset()
def test_15_InventoryAfterModificationInFuture(self):
""" """
Test inventory after adding a new movement in future Test that a new cache entry is created from previous cache entry if it exists
""" """
movement = self._makeMovement(quantity=self.BASE_QUANTITY*2, # Create an old movement
start_date=self.INVENTORY_DATE_3 + 2, INVENTORY_QUANTITY_4 = 100
INVENTORY_DATE_4 = self.NOW - 3 * self.CACHE_LAG
movement = self._makeMovement(quantity=INVENTORY_QUANTITY_4,
start_date=INVENTORY_DATE_4,
simulation_state='delivered') simulation_state='delivered')
self.tic() # Get inventory in past so that cache is filled
inventory_kw={'node_uid': self.node_uid,
def getCurrentInventoryPathList(resource, **kw): "to_date" : self.NOW - 2 * self.CACHE_LAG,}
# the brain is not a zsqlbrain instance here, so it does not value = self.getInventory(**inventory_kw)
# have getPath(). self.assertInventoryEquals(value, inventory_kw)
return [x.path for x in resource.getCurrentInventoryList(**kw)]
# Now compute wanted value manually as we screwed the stock table
# use optimisation # As we double every movement < CACHE_LAG/2, inventory_1 is doubled
self.assertEquals(True,movement.getPath() in # like inventory_4, but inventory_4 must be retrieved from cache with
[x.path for x in self.resource.getInventoryList( # its initial value
mirror_uid=self.mirror_node.getUid())]) wanted_value = 2 * self.INVENTORY_QUANTITY_1 + self.INVENTORY_QUANTITY_2 + \
self.INVENTORY_QUANTITY_3 + INVENTORY_QUANTITY_4
inventory_kw={'node_uid': self.node_uid,
"to_date" : self.NOW,}
value = self.getInventory(**inventory_kw)
self.assertEqual(value, wanted_value)
# Make sure it has filled a new cache
self.assertInventoryEquals(wanted_value, inventory_kw)
# without optimisation
self.assertEquals(True,movement.getPath() in
[x.path for x in self.resource.getInventoryList(
optimisation__=False,
mirror_uid=self.mirror_node.getUid())])
@expectedFailure def test_14_CacheTableCreatedOnGetInventoryCall(self):
def test_MultipleSectionAndFullInventory(self): """
"""Make sure that getInventoryList works in the situation which Check that getInventory does not fail it cache table does not exist
two sections use the same node and one section has full inventory for and that it create the table and add an entry in it
the node.
""" """
# In this test we do not need doucments made by afterSetUp. self.portal.SimulationTool_zDropInventoryCache()
self.portal.inventory_module.manage_delObjects(list(self.portal.inventory_module.objectIds())) # Make sure it is dropped
self.folder.manage_delObjects(list(self.folder.objectIds())) self.assertRaises(ProgrammingError,
self.portal.SimulationTool_zTrimInventoryCacheFromDateOnCatalog,
date=DateTime())
# Check that src__ call still works
inventory_kw={
'node_uid': self.node_uid,
'to_date': self.NOW,
}
self.getInventory(src__=1, **inventory_kw)
# Table is still not created
self.assertRaises(ProgrammingError,
self.portal.SimulationTool_zTrimInventoryCacheFromDateOnCatalog,
date=DateTime())
# This call should not fail
# It will create table, fill it and check optimisation is used
self.assertInventoryEquals(
self._fillCache(),
inventory_kw=inventory_kw,
)
self.commit() def test_15_CacheTableCreatedOnIndexation(self):
"""
Check that getInventory does not fail it cache table does not exist
and that it create the table and add an entry in it
"""
self.portal.SimulationTool_zDropInventoryCache()
# Make sure it is dropped
self.assertRaises(ProgrammingError,
self.portal.SimulationTool_zTrimInventoryCacheFromDateOnCatalog,
date=DateTime())
# Create a new movement, indexation should not fail
INVENTORY_QUANTITY_4 = 5000
INVENTORY_DATE_4 = self.CACHE_DATE
movement = self._makeMovement(
quantity=INVENTORY_QUANTITY_4,
start_date=INVENTORY_DATE_4,
simulation_state='delivered',
)
self.tic() self.tic()
createZODBPythonScript(self.portal.portal_skins.custom, # Optimisation must then be used
'Inventory_getDefaultInventoryCalculationList', '', inventory_kw={
'''return ({ 'node_uid': self.node_uid,
'inventory_params':{ 'to_date': self.NOW,
'section_uid':context.getDestinationSectionUid(), }
'node_uid':context.getDestinationUid(), value = self.getInventory(**inventory_kw)
'group_by_variation':1, self.assertInventoryEquals(
'group_by_resource':1}, value,
'list_method':'getMovementList', inventory_kw=inventory_kw,
'first_level':({'key':'resource_relative_url', )
'getter':'getResource',
'setter':('appendToCategoryList', 'resource')},
{'key':'variation_text',
'getter':'getVariationText',
'setter':'splitAndExtendToCategoryList'},
),
},)
''')
self.commit()
getCurrentInventoryList = self.getSimulationTool().getCurrentInventoryList
# Add movements for section
self._makeMovement(source_section_value=None,
source_value=None,
destination_section_value=self.section,
destination_value=self.node,
start_date=DateTime('2012/07/18 00:00:00 GMT+9'),
simulation_state='delivered',
resource_value=self.resource,
quantity=1)
self._makeMovement(source_section_value=None,
source_value=None,
destination_section_value=self.section,
destination_value=self.node,
start_date=DateTime('2012/07/21 00:00:00 GMT+9'),
simulation_state='delivered',
resource_value=self.resource,
quantity=2)
# Add movemnets for other section def test_16_CacheTableCreatedOnUnindexation(self):
self._makeMovement(source_section_value=None, """
source_value=None, Check that getInventory does not fail it cache table does not exist
destination_section_value=self.other_section, and that it create the table and add an entry in it
destination_value=self.node, """
start_date=DateTime('2012/07/19 00:00:00 GMT+9'), # Create a new movement
simulation_state='delivered', INVENTORY_QUANTITY_4 = 5000
resource_value=self.resource, INVENTORY_DATE_4 = self.CACHE_DATE
quantity=3) movement = self._makeMovement(
self._makeMovement(source_section_value=None, quantity=INVENTORY_QUANTITY_4,
source_value=None, start_date=INVENTORY_DATE_4,
destination_section_value=self.other_section,
destination_value=self.node,
start_date=DateTime('2012/07/20 00:00:00 GMT+9'),
simulation_state='delivered', simulation_state='delivered',
resource_value=self.resource, )
quantity=4)
self.commit()
self.tic() self.tic()
# Check it also works on unindexation
# Check inventory self.portal.SimulationTool_zDropInventoryCache()
result = {} # Make sure it is dropped
for brain in getCurrentInventoryList(node_uid=self.node.getUid(), self.assertRaises(ProgrammingError,
group_by_resource=1, self.portal.SimulationTool_zTrimInventoryCacheFromDateOnCatalog,
group_by_node=1, date=DateTime())
group_by_section=1): # Delete movement
key = (brain.section_uid, brain.node_uid, brain.resource_uid) self.folder.manage_delObjects(ids=[movement.getId(), ])
if not key in result:
result[key] = 0
result[key] = result[key] + brain.inventory
self.assertEqual(result,
{(self.section.getUid(), self.node.getUid(), self.resource.getUid()):3,
(self.other_section.getUid(), self.node.getUid(), self.resource.getUid()):7})
# Add full inventory for section, not for other section
full_inventory1 = self.portal.inventory_module.newContent(portal_type='Inventory')
full_inventory1.edit(destination_section_value=self.section,
destination_value=self.node,
full_inventory=1,
start_date=DateTime('2012/07/20 00:00:00 GMT+9'))
line = full_inventory1.newContent(portal_type='Inventory Line')
line.setResourceValue(self.resource)
line.setQuantity(100)
full_inventory1.deliver()
self.commit()
self.tic() self.tic()
# This call must not fail as table has been created
self.portal.SimulationTool_zTrimInventoryCacheFromDateOnCatalog(date=DateTime())
# This call should not fail
# It will create table, fill it and check optimisation is used
self.assertInventoryEquals(
self._fillCache(),
inventory_kw={
'node_uid': self.node_uid,
'to_date': self.NOW,
}
)
# Check inventory again. This time, full inventory should change
# section's inventory. It should not change other section's inventory.
result = {}
for brain in getCurrentInventoryList(node_uid=self.node.getUid(),
group_by_resource=1,
group_by_node=1,
group_by_section=1):
key = (brain.section_uid, brain.node_uid, brain.resource_uid)
if not key in result:
result[key] = 0
result[key] = result[key] + brain.inventory
self.assertEqual(result,
{(self.section.getUid(), self.node.getUid(), self.resource.getUid()):102,
(self.other_section.getUid(), self.node.getUid(), self.resource.getUid()):7})
@expectedFailure
def test_ResourceCategory(self):
"""Make sure that resource category works when full inventory exists."""
# In this test we do not need doucments made by afterSetUp.
self.setUpDefaultInventoryCalculationList()
self.clearAllInventoryAndSetUpTwoInventory()
self.assertEquals(1,
self.getInventory(section_uid=self.section.getUid(),
node_uid=self.node.getUid(),
resource_category='product_line/level1',
optimisation__=False))
self.assertEquals(1,
self.getInventory(section_uid=self.section.getUid(),
node_uid=self.node.getUid(),
resource_category='product_line/level1',
optimisation__=True))
@expectedFailure
def test_SectionCategory(self):
"""Make sure that section category works when full inventory exists."""
# In this test we do not need doucments made by afterSetUp.
self.clearAllInventoryAndSetUpTwoInventory()
self.assertEquals(11,
self.getInventory(node_uid=self.node.getUid(),
section_category='group/level1/level2',
optimisation__=False))
self.assertEquals(11,
self.getInventory(node_uid=self.node.getUid(),
section_category='group/level1/level2',
optimisation__=True))
@expectedFailure
def test_NodeCategory(self):
# In this test we do not need doucments made by afterSetUp.
self.clearAllInventoryAndSetUpTwoInventory()
self.assertEquals(11,
self.getInventory(section_uid=self.section.getUid(),
node_category='region/level1',
optimisation__=False))
self.assertEquals(11,
self.getInventory(section_uid=self.section.getUid(),
node_category='region/level1',
optimisation__=True))
class BaseTestUnitConversion(InventoryAPITestCase): class BaseTestUnitConversion(InventoryAPITestCase):
QUANTITY_UNIT_DICT = {} QUANTITY_UNIT_DICT = {}
...@@ -3543,10 +3344,10 @@ def test_suite(): ...@@ -3543,10 +3344,10 @@ def test_suite():
suite.addTest(unittest.makeSuite(TestInventoryStat)) suite.addTest(unittest.makeSuite(TestInventoryStat))
suite.addTest(unittest.makeSuite(TestNextNegativeInventoryDate)) suite.addTest(unittest.makeSuite(TestNextNegativeInventoryDate))
suite.addTest(unittest.makeSuite(TestTrackingList)) suite.addTest(unittest.makeSuite(TestTrackingList))
suite.addTest(unittest.makeSuite(TestInventoryDocument)) suite.addTest(unittest.makeSuite(TestInventoryCacheTable))
suite.addTest(unittest.makeSuite(TestUnitConversion)) suite.addTest(unittest.makeSuite(TestUnitConversion))
suite.addTest(unittest.makeSuite(TestUnitConversionDefinition)) suite.addTest(unittest.makeSuite(TestUnitConversionDefinition))
suite.addTest(unittest.makeSuite(TestUnitConversionBackwardCompatibility)) suite.addTest(unittest.makeSuite(TestUnitConversionBackwardCompatibility))
return suite return suite
# vim: foldmethod=marker
product/ERP5Catalog/tests/testERP5Catalog.py
View file @ cf53db45
...@@ -1554,8 +1554,12 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -1554,8 +1554,12 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
# Do a hot reindex in the reverse way, but this time a more # Do a hot reindex in the reverse way, but this time a more
# complicated hot reindex # complicated hot reindex
portal_catalog.manage_hotReindexAll(self.new_catalog_id, portal_catalog.manage_hotReindexAll(
self.original_catalog_id) source_sql_catalog_id=self.new_catalog_id,
destination_sql_catalog_id=self.original_catalog_id,
source_sql_connection_id_list=destination_sql_connection_id_list,
destination_sql_connection_id_list=source_sql_connection_id_list,
update_destination_sql_catalog=True)
self.commit() self.commit()
self.assertEquals(portal_catalog.getHotReindexingState(), self.assertEquals(portal_catalog.getHotReindexingState(),
HOT_REINDEXING_RECORDING_STATE) HOT_REINDEXING_RECORDING_STATE)
...@@ -1735,6 +1739,17 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -1735,6 +1739,17 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
# Check more cached methods of SQLCatalog by building SQLQuery # Check more cached methods of SQLCatalog by building SQLQuery
query = portal_catalog.getSQLCatalog().buildQuery(kw={'dummy.dummy_title': 'Foo'}) query = portal_catalog.getSQLCatalog().buildQuery(kw={'dummy.dummy_title': 'Foo'})
self.assertTrue(query.query_list) self.assertTrue(query.query_list)
# We need to reset SQL connections in skin folder's zsql methods
sql_connection_id_dict = {}
for destination_sql_connection_id, source_sql_connection_id in \
zip(destination_sql_connection_id_list,
source_sql_connection_id_list):
if source_sql_connection_id != destination_sql_connection_id:
sql_connection_id_dict[destination_sql_connection_id] = \
source_sql_connection_id
portal_catalog.changeSQLConnectionIds(
folder=portal.portal_skins,
sql_connection_id_dict = sql_connection_id_dict)
def test_47_Unrestricted(self, quiet=quiet, run=run_all_test): def test_47_Unrestricted(self, quiet=quiet, run=run_all_test):
"""test unrestricted search/count results. """test unrestricted search/count results.
......
product/ERP5Type/Errors.py
View file @ cf53db45
...@@ -6,7 +6,9 @@ ...@@ -6,7 +6,9 @@
from Products.PythonScripts.Utility import allow_class from Products.PythonScripts.Utility import allow_class
from Products.CMFCore.WorkflowCore import WorkflowException from Products.CMFCore.WorkflowCore import WorkflowException
from MySQLdb import ProgrammingError
allow_class(ProgrammingError)
class DeferredCatalogError(Exception): class DeferredCatalogError(Exception):
...@@ -53,3 +55,4 @@ allow_class(WorkflowException) ...@@ -53,3 +55,4 @@ allow_class(WorkflowException)
allow_class(UnsupportedWorkflowMethod) allow_class(UnsupportedWorkflowMethod)
allow_class(TransformationRuleError) allow_class(TransformationRuleError)
allow_class(SimulationError) allow_class(SimulationError)
product/ERP5Type/tests/ERP5TypeTestCase.py
View file @ cf53db45
...@@ -14,7 +14,6 @@ import os ...@@ -14,7 +14,6 @@ import os
import random import random
import re import re
import socket import socket
import shutil
import sys import sys
import time import time
import traceback import traceback
...@@ -26,7 +25,6 @@ from glob import glob ...@@ -26,7 +25,6 @@ from glob import glob
from hashlib import md5 from hashlib import md5
from warnings import warn from warnings import warn
from ExtensionClass import pmc_init_of from ExtensionClass import pmc_init_of
from ZTUtils import make_query
from DateTime import DateTime from DateTime import DateTime
# XXX make sure that get_request works. # XXX make sure that get_request works.
...@@ -818,7 +816,8 @@ class ERP5TypeCommandLineTestCase(ERP5TypeTestCaseMixin): ...@@ -818,7 +816,8 @@ class ERP5TypeCommandLineTestCase(ERP5TypeTestCaseMixin):
if update_business_templates and erp5_load_data_fs: if update_business_templates and erp5_load_data_fs:
update_only = os.environ.get('update_only', None) update_only = os.environ.get('update_only', None)
template_list = (erp5_catalog_storage, 'erp5_property_sheets', template_list = (erp5_catalog_storage, 'erp5_property_sheets',
'erp5_core', 'erp5_xhtml_style') + tuple(template_list) 'erp5_core', 'erp5_xhtml_style') \
+ tuple(template_list)
# Update only specified business templates, regular expression # Update only specified business templates, regular expression
# can be used. # can be used.
if update_only is not None: if update_only is not None:
...@@ -837,6 +836,10 @@ class ERP5TypeCommandLineTestCase(ERP5TypeTestCaseMixin): ...@@ -837,6 +836,10 @@ class ERP5TypeCommandLineTestCase(ERP5TypeTestCaseMixin):
light_install = self.enableLightInstall() light_install = self.enableLightInstall()
create_activities = self.enableActivityTool() create_activities = self.enableActivityTool()
hot_reindexing = self.enableHotReindexing() hot_reindexing = self.enableHotReindexing()
# We want to always have optimisation available
if "erp5_stock_cache" not in template_list:
template_list = list(template_list)
template_list.append("erp5_stock_cache")
self.setUpERP5Site(business_template_list=template_list, self.setUpERP5Site(business_template_list=template_list,
light_install=light_install, light_install=light_install,
create_activities=create_activities, create_activities=create_activities,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7