Skip to content

GitLab

Commit cf53db45 authored by Aurel's avatar Aurel
Browse files

Implement cache of getInventory results 

Create a sql cache for getInventory call
Remove full-inventory optimisation as it now useless
Implement full-inventory feature directly into stock
Make erp5_pdm depends on this optimisation
Install optimisation for all unit tests
parent 3f484980
Hide whitespace changes
Inline Side-by-side
Showing with 482 additions and 95 deletions
bt5/erp5_banking_core/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_inventory_list.catalog_keys.xml deleted 100644 → 0
View file @ 3f484980
<catalog_method>
<item key="sql_catalog_object_list" type="int">
<value>1</value>
</item>
</catalog_method>
bt5/erp5_banking_core/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_inventory_list.xml deleted 100644 → 0
View file @ 3f484980
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>isInventory\r\n
uid\r\n
getDestinationUid\r\n
getDestinationSectionUid\r\n
getDestinationPaymentUid\r\n
getStartDate\r\n
isFullInventory\r\n
getSimulationState</string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_catalog_inventory_list</string> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
<dtml-let row_list="[]">\n
\n
<dtml-in prefix="loop" expr="_.range(_.len(uid))">\n
<dtml-if expr="isInventory[loop_item]">\n
<dtml-if expr="getStartDate[loop_item] is not None">\n
<dtml-call expr="row_list.append((\n
uid[loop_item],\n
getDestinationUid[loop_item],\n
getDestinationSectionUid[loop_item],\n
getDestinationPaymentUid[loop_item],\n
getStartDate[loop_item],\n
isFullInventory[loop_item],\n
getSimulationState[loop_item]))">\n
</dtml-if>\n
</dtml-if>\n
</dtml-in>\n
\n
<dtml-if expr="len(row_list)">\n
REPLACE INTO\n
inventory\n
(`uid`, `node_uid`, `section_uid`, `payment_uid`, `date`, `is_full_inventory`, `simulation_state`)\n
VALUES\n
<dtml-in prefix="row" expr="row_list">\n
(<dtml-sqlvar expr="row_item[0]" type="int">,\n
<dtml-sqlvar expr="row_item[1]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[2]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[3]" type="int" optional>,\n
<dtml-sqlvar expr="row_item[4]" type="datetime">,\n
<dtml-sqlvar expr="row_item[5] or 0" type="int">,\n
<dtml-sqlvar expr="row_item[6]" type="string">\n
)<dtml-if sequence-end><dtml-else>,</dtml-if>\n
</dtml-in>\n
</dtml-if>\n
</dtml-let>
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_banking_core/bt/revision
View file @ cf53db45
587
\ No newline at end of file
587
bt5/erp5_pdm/bt/dependency_list
View file @ cf53db45
erp5_base
\ No newline at end of file
erp5_base
erp5_stock_cache
bt5/erp5_pdm/bt/revision
View file @ cf53db45
571
\ No newline at end of file
573
\ No newline at end of file
bt5/erp5_simulation/bt/dependency_list
View file @ cf53db45
erp5_base
\ No newline at end of file
erp5_base
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_inventory_stock_list.catalog_keys.xml → bt5/erp5_stock_cache/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_trimInventoryCacheOnCatalog.catalog_keys.xml
View file @ cf53db45
......@@ -6,7 +6,7 @@
<value>1</value>
</item>
<item key="_filter_expression_archive" type="str">
<value>python: context.providesIMovement() and context.isInventoryMovement()</value>
<value>python: context.providesIMovement() and not context.isInventoryMovement()</value>
</item>
<item key="_filter_expression_cache_key_archive" type="tuple">
<value>portal_type</value>
......
bt5/erp5_stock_cache/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_trimInventoryCacheOnCatalog.xml 0 → 100644
View file @ cf53db45
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>from Products.ERP5Type.Errors import ProgrammingError\n
\n
min_date = None\n
for loop_item in xrange(len(uid)):\n
if not isInventoryMovement[loop_item] and isMovement[loop_item] and getResourceUid[loop_item]:\n
if getDestinationUid[loop_item] and getStopDate[loop_item]:\n
if min_date:\n
min_date = min(min_date, getStopDate[loop_item])\n
else:\n
min_date = getStopDate[loop_item]\n
if getSourceUid[loop_item] and getStartDate[loop_item]:\n
if min_date:\n
min_date = min(min_date, getStartDate[loop_item])\n
else:\n
min_date = getStartDate[loop_item]\n
if min_date:\n
try:\n
context.SimulationTool_zTrimInventoryCacheFromDateOnCatalog(date=min_date)\n
except ProgrammingError:\n
# Create table if it does not exits\n
# Then no need to flush an empty table\n
context.SimulationTool_zCreateInventoryCache()\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>uid, isMovement, isInventoryMovement, getResourceUid, getDestinationUid, getStopDate, getSourceUid, getStartDate</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_trimInventoryCacheOnCatalog</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_stock_cache/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_trimInventoryCacheOnUncatalog.xml 0 → 100644
View file @ cf53db45
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>from Products.ERP5Type.Errors import ProgrammingError\n
\n
try:\n
context.SimulationTool_zTrimInventoryCacheFromDateOnUncatalog(uid=uid)\n
except ProgrammingError:\n
# Create table if it does not exits\n
# Then no need to flush an empty table\n
context.SimulationTool_zCreateInventoryCache()\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>uid</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_trimInventoryCacheOnUncatalog</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache.xml 0 → 100644
View file @ cf53db45
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Folder" module="OFS.Folder"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_objects</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>erp5_stock_cache</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Resource_zGetFullInventoryDate.xml → bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/Resource_zGetInventoryCacheResult.xml
View file @ cf53db45
......@@ -6,14 +6,32 @@
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>column_value_list_list:list\r\n
column_id_list:list\r\n
group_by_expression\r\n
date\r\n
simulation_state=delivered\r\n
where_expression</string> </value>
<value> <string>query\r\n
date</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
......@@ -21,39 +39,31 @@ where_expression</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Resource_zGetFullInventoryDate</string> </value>
<value> <string>Resource_zGetInventoryCacheResult</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
SELECT\n
MAX(date) as date,\n
<dtml-var group_by_expression>\n
FROM\n
inventory\n
WHERE\n
is_full_inventory = TRUE\n
<dtml-if simulation_state>\n
AND simulation_state = <dtml-sqlvar simulation_state type="string">\n
</dtml-if>\n
<dtml-in prefix="loop" expr="_.range(_.len(column_id_list))">\n
AND <dtml-var expr="column_id_list[loop_item]"> IN (\n
<dtml-in expr="column_value_list_list[loop_item]">\n
<dtml-sqlvar sequence-item type="int">\n
<dtml-if sequence-end><dtml-else>,</dtml-if>\n
</dtml-in>\n
)\n
</dtml-in>\n
<dtml-if date>\n
AND date <= <dtml-sqlvar date type="string">\n
</dtml-if>\n
<dtml-if where_expression>\n
AND <dtml-var where_expression>\n
</dtml-if>\n
GROUP BY\n
<dtml-var group_by_expression>\n
SELECT \n
date, \n
result \n
FROM \n
inventory_cache \n
WHERE \n
inventory_cache.query=<dtml-sqlvar query type="string"> \n
AND \n
inventory_cache.date <= <dtml-sqlvar date type="datetime"> \n
ORDER BY \n
date DESC
]]></string> </value>
</item>
......
bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/Resource_zInsertInventoryCacheResult.xml 0 → 100644
View file @ cf53db45
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>query\r\n
date\r\n
result</string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Resource_zInsertInventoryCacheResult</string> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
Insert into inventory_cache(`query`, `date`, `result`) values (<dtml-sqlvar query type="string">, <dtml-sqlvar date type="datetime">, <dtml-sqlvar result type="string">)
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/SimulationTool_flushInventoryCache.xml 0 → 100644
View file @ cf53db45
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string># XXX To be implemented\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SimulationTool_flushInventoryCache</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/SimulationTool_getInventoryCacheLag.xml 0 → 100644
View file @ cf53db45
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>"""\n
Returns a duration, in days, for stock cache management.\n
If data in stock cache is older than lag compared to query\'s date\n
(at_date or to_date), then it becomes a "soft miss": use found value,\n
but add a new entry to cache at query\'s date minus half the lag.\n
So this value should be:\n
- Small enough that few enough rows need to be table-scanned for\n
verage queries (probably queries against current date).\n
- Large enough that few enough documents get modified past that date,\n
therwise cache entries would be removed from cache all the time.\n
"""\n
\n
return 60\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SimulationTool_getInventoryCacheLag</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_inventory.xml → bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/SimulationTool_zCreateInventoryCache.xml
View file @ cf53db45
......@@ -16,11 +16,17 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_drop_inventory</string> </value>
<value> <string>SimulationTool_zCreateInventoryCache</string> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string>DROP TABLE IF EXISTS inventory</string> </value>
<value> <string>Create table `inventory_cache` (\n
`query` BINARY(16) NOT NULL,\n
`date` datetime NOT NULL,\n
`result` LONGBLOB NOT NULL,\n
PRIMARY KEY (`query`, `date`),\n
KEY (`date`)\n
) Engine=InnoDB</string> </value>
</item>
<item>
<key> <string>title</string> </key>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_inventory_stock.xml → bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/SimulationTool_zDropInventoryCache.xml
View file @ cf53db45
......@@ -6,6 +6,12 @@
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
......@@ -16,11 +22,11 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_drop_inventory_stock</string> </value>
<value> <string>SimulationTool_zDropInventoryCache</string> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string>DROP TABLE IF EXISTS inventory_stock</string> </value>
<value> <string>DROP TABLE IF EXISTS inventory_cache</string> </value>
</item>
<item>
<key> <string>title</string> </key>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_inventory_stock.xml → bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/SimulationTool_zTrimInventoryCacheFromDateOnCatalog.xml
View file @ cf53db45
......@@ -8,7 +8,7 @@
<dictionary>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid</string> </value>
<value> <string>date</string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
......@@ -16,13 +16,16 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_uncatalog_inventory_stock</string> </value>
<value> <string>SimulationTool_zTrimInventoryCacheFromDateOnCatalog</string> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
DELETE FROM inventory_stock WHERE <dtml-sqltest uid op=eq type=int>
DELETE FROM \n
inventory_cache \n
WHERE \n
date > <dtml-sqlvar expr="date" type="datetime">
]]></string> </value>
</item>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_inventory.xml → bt5/erp5_stock_cache/SkinTemplateItem/portal_skins/erp5_stock_cache/SimulationTool_zTrimInventoryCacheFromDateOnUncatalog.xml
View file @ cf53db45
......@@ -16,13 +16,13 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_uncatalog_inventory</string> </value>
<value> <string>SimulationTool_zTrimInventoryCacheFromDateOnUncatalog</string> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
DELETE FROM inventory WHERE <dtml-sqltest uid op=eq type=int>
DELETE FROM inventory_cache WHERE date > (SELECT min(date) from stock where <dtml-sqltest uid op=eq type=int>)
]]></string> </value>
</item>
......
bt5/erp5_stock_cache/bt/change_log 0 → 100644
View file @ cf53db45
2013-02-05 Aurel
Initial version
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7