Commit 2b9a5e82 authored by Nicolas Dumazet's avatar Nicolas Dumazet

Split Transformation indexation in several activate()'ed chunks

Since the number of possible variations for a resource can be very big,
it is not reasonable to process the indexation of all those lines in
a single big transaction.
Split the process and use activities; one activity per (resource, variation)
couple.


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@33435 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent fbcedc58
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<tuple>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
<tuple/>
</tuple>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string encoding="cdata"><![CDATA[
# This script indexes the preferred Transformation to produce\n
# a variation of a product.\n
# Only the transformation is passed as a parameter, the resource being\n
# transformation.getResourceValue()\n
\n
from Products.ERP5Type.Document import newTempMovement\n
\n
transformation = context.restrictedTraverse(transformation_relative_url)\n
resource = transformation.getResourceValue()\n
\n
row_dict_list = []\n
movement = newTempMovement(resource, \'temp\',\n
variation_category_list=variation_list,\n
resource=resource.getRelativeUrl(),\n
quantity=1.0)\n
\n
base_row = dict(uid=resource.getUid(), variation_text=movement.getVariationText())\n
\n
for amount in transformation.getAggregatedAmountList(movement):\n
transformed_resource_uid = amount.getResourceUid()\n
quantity = amount.getQuantity()\n
if transformed_resource_uid is not None and quantity is not None:\n
row_dict = base_row.copy()\n
row_dict.update(transformed_uid=transformed_resource_uid,\n
transformed_variation_text=amount.getVariationText(),\n
quantity=quantity)\n
row_dict_list.append(row_dict)\n
\n
if len(row_dict_list) > 0:\n
context.z_catalog_transformation(row_dict_list=row_dict_list)\n
]]></string> </value>
</item>
<item>
<key> <string>_code</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>transformation_relative_url, variation_list</string> </value>
</item>
<item>
<key> <string>errors</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>func_code</string> </key>
<value>
<object>
<klass>
<global name="FuncCode" module="Shared.DC.Scripts.Signature"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>co_argcount</string> </key>
<value> <int>2</int> </value>
</item>
<item>
<key> <string>co_varnames</string> </key>
<value>
<tuple>
<string>transformation_relative_url</string>
<string>variation_list</string>
<string>Products.ERP5Type.Document</string>
<string>newTempMovement</string>
<string>_getattr_</string>
<string>context</string>
<string>transformation</string>
<string>resource</string>
<string>row_dict_list</string>
<string>movement</string>
<string>dict</string>
<string>base_row</string>
<string>_getiter_</string>
<string>amount</string>
<string>transformed_resource_uid</string>
<string>quantity</string>
<string>None</string>
<string>row_dict</string>
<string>len</string>
</tuple>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>func_defaults</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_catalogTransformation</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Index Transformation required to produce a specific variation of a Resource</string> </value>
</item>
<item>
<key> <string>warnings</string> </key>
<value>
<tuple/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<tuple>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
<tuple/>
</tuple>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string># Indexing all transformation lines for all possible variations of a Resource can be very costly.\n
# Avoid doing this in a single transaction, and split the operation.\n
#\n
# One activity is generated per (resource, variation_category_list) possibility\n
\n
for i, transformation in enumerate(getDefaultConversionTransformationValue):\n
if transformation is None:\n
continue\n
transformation_relative_url = transformation.getRelativeUrl()\n
for variation_category_list in getTransformationVariationCategoryCartesianProduct[i]:\n
context.activate(activity=\'SQLQueue\').SQLCatalog_catalogTransformation(transformation_relative_url,\n
variation_category_list)\n
</string> </value>
</item>
<item>
<key> <string>_code</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>getDefaultConversionTransformationValue, getTransformationVariationCategoryCartesianProduct</string> </value>
</item>
<item>
<key> <string>errors</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>func_code</string> </key>
<value>
<object>
<klass>
<global name="FuncCode" module="Shared.DC.Scripts.Signature"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>co_argcount</string> </key>
<value> <int>2</int> </value>
</item>
<item>
<key> <string>co_varnames</string> </key>
<value>
<tuple>
<string>getDefaultConversionTransformationValue</string>
<string>getTransformationVariationCategoryCartesianProduct</string>
<string>_getiter_</string>
<string>enumerate</string>
<string>i</string>
<string>transformation</string>
<string>None</string>
<string>_getattr_</string>
<string>transformation_relative_url</string>
<string>_getitem_</string>
<string>variation_category_list</string>
<string>context</string>
</tuple>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>func_defaults</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_catalogTransformationList</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Catalog the ways to produce all variations of a Resource</string> </value>
</item>
<item>
<key> <string>warnings</string> </key>
<value>
<tuple/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -24,13 +24,7 @@ ...@@ -24,13 +24,7 @@
<value> <value>
<dictionary> <dictionary>
<item> <item>
<key> <string>getTransformationRowList</string> </key> <key> <string>row_dict_list</string> </key>
<value>
<dictionary/>
</value>
</item>
<item>
<key> <string>uid</string> </key>
<value> <value>
<dictionary/> <dictionary/>
</value> </value>
...@@ -42,8 +36,7 @@ ...@@ -42,8 +36,7 @@
<key> <string>_keys</string> </key> <key> <string>_keys</string> </key>
<value> <value>
<list> <list>
<string>uid</string> <string>row_dict_list</string>
<string>getTransformationRowList</string>
</list> </list>
</value> </value>
</item> </item>
...@@ -60,8 +53,7 @@ ...@@ -60,8 +53,7 @@
</item> </item>
<item> <item>
<key> <string>arguments_src</string> </key> <key> <string>arguments_src</string> </key>
<value> <string>uid\r\n <value> <string>row_dict_list</string> </value>
getTransformationRowList</string> </value>
</item> </item>
<item> <item>
<key> <string>cache_time_</string> </key> <key> <string>cache_time_</string> </key>
...@@ -87,7 +79,7 @@ getTransformationRowList</string> </value> ...@@ -87,7 +79,7 @@ getTransformationRowList</string> </value>
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>z_catalog_transformation_list</string> </value> <value> <string>z_catalog_transformation</string> </value>
</item> </item>
<item> <item>
<key> <string>max_cache_</string> </key> <key> <string>max_cache_</string> </key>
...@@ -102,19 +94,14 @@ getTransformationRowList</string> </value> ...@@ -102,19 +94,14 @@ getTransformationRowList</string> </value>
<value> <string encoding="cdata"><![CDATA[ <value> <string encoding="cdata"><![CDATA[
DELETE FROM `transformation` WHERE\n DELETE FROM `transformation` WHERE\n
<dtml-sqltest uid type="int" multiple>;\n <dtml-sqltest expr="row_dict_list[0][\'uid\']" column="uid" type="int">\n
\n AND\n
<dtml-let transformation_list="[]">\n <dtml-sqltest expr="row_dict_list[0][\'variation_text\']" column="variation_text" type="string">;\n
<dtml-in getTransformationRowList prefix="loop">\n <dtml-var sql_delimiter>\n
<dtml-call expr="loop_item and transformation_list.extend(loop_item)">\n
</dtml-in>\n
\n
<dtml-if transformation_list>\n
<dtml-var sql_delimiter>\n
\n \n
INSERT INTO `transformation`\n INSERT INTO `transformation`\n
VALUES\n VALUES\n
<dtml-in transformation_list prefix="loop">\n <dtml-in row_dict_list prefix="loop">\n
(\n (\n
<dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n <dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n
<dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n <dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n
...@@ -123,11 +110,7 @@ VALUES\n ...@@ -123,11 +110,7 @@ VALUES\n
<dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n <dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n
)\n )\n
<dtml-unless sequence-end>,</dtml-unless>\n <dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in>\n </dtml-in>
\n
</dtml-if>\n
</dtml-let>\n
]]></string> </value> ]]></string> </value>
</item> </item>
...@@ -168,19 +151,14 @@ VALUES\n ...@@ -168,19 +151,14 @@ VALUES\n
<value> <string encoding="cdata"><![CDATA[ <value> <string encoding="cdata"><![CDATA[
DELETE FROM `transformation` WHERE\n DELETE FROM `transformation` WHERE\n
<dtml-sqltest uid type="int" multiple>;\n <dtml-sqltest expr="row_dict_list[0][\'uid\']" column="uid" type="int">\n
\n AND\n
<dtml-let transformation_list="[]">\n <dtml-sqltest expr="row_dict_list[0][\'variation_text\']" column="variation_text" type="string">;\n
<dtml-in getTransformationRowList prefix="loop">\n <dtml-var sql_delimiter>\n
<dtml-call expr="loop_item and transformation_list.extend(loop_item)">\n
</dtml-in>\n
\n
<dtml-if transformation_list>\n
<dtml-var sql_delimiter>\n
\n \n
INSERT INTO `transformation`\n INSERT INTO `transformation`\n
VALUES\n VALUES\n
<dtml-in transformation_list prefix="loop">\n <dtml-in row_dict_list prefix="loop">\n
(\n (\n
<dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n <dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n
<dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n <dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n
...@@ -189,11 +167,7 @@ VALUES\n ...@@ -189,11 +167,7 @@ VALUES\n
<dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n <dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n
)\n )\n
<dtml-unless sequence-end>,</dtml-unless>\n <dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in>\n </dtml-in>
\n
</dtml-if>\n
</dtml-let>\n
]]></string> </value> ]]></string> </value>
</item> </item>
......
<catalog_method>
<item key="sql_catalog_object_list" type="int">
<value>1</value>
</item>
<item key="_is_filtered_archive" type="int">
<value>1</value>
</item>
<item key="_filter_expression_archive" type="str">
<value>python: context.isResourceType()</value>
</item>
<item key="_filter_expression_cache_key_archive" type="tuple">
<value>portal_type</value>
</item>
</catalog_method>
...@@ -66,7 +66,7 @@ ...@@ -66,7 +66,7 @@
KEY `transformed_resource` (`transformed_uid`, `transformed_variation_text`)\n KEY `transformed_resource` (`transformed_uid`, `transformed_variation_text`)\n
)\n )\n
\n \n
</string> </value> </string> </value>
</item> </item>
<item> <item>
<key> <string>template</string> </key> <key> <string>template</string> </key>
...@@ -112,7 +112,7 @@ ...@@ -112,7 +112,7 @@
KEY `transformed_resource` (`transformed_uid`, `transformed_variation_text`)\n KEY `transformed_resource` (`transformed_uid`, `transformed_variation_text`)\n
)\n )\n
\n \n
</string> </value> </string> </value>
</item> </item>
</dictionary> </dictionary>
</state> </state>
......
180 181
\ No newline at end of file \ No newline at end of file
erp5_mysql_innodb/SQLCatalog_catalogTransformation
erp5_mysql_innodb/SQLCatalog_catalogTransformationList
erp5_mysql_innodb/SQLCatalog_makeQuickSearchQuery erp5_mysql_innodb/SQLCatalog_makeQuickSearchQuery
erp5_mysql_innodb/z0_drop_alarm erp5_mysql_innodb/z0_drop_alarm
erp5_mysql_innodb/z0_drop_catalog erp5_mysql_innodb/z0_drop_catalog
...@@ -46,7 +48,7 @@ erp5_mysql_innodb/z_catalog_predicate_category_list ...@@ -46,7 +48,7 @@ erp5_mysql_innodb/z_catalog_predicate_category_list
erp5_mysql_innodb/z_catalog_predicate_list erp5_mysql_innodb/z_catalog_predicate_list
erp5_mysql_innodb/z_catalog_roles_and_users_list erp5_mysql_innodb/z_catalog_roles_and_users_list
erp5_mysql_innodb/z_catalog_stock_list erp5_mysql_innodb/z_catalog_stock_list
erp5_mysql_innodb/z_catalog_transformation_list erp5_mysql_innodb/z_catalog_transformation
erp5_mysql_innodb/z_catalog_translation_list erp5_mysql_innodb/z_catalog_translation_list
erp5_mysql_innodb/z_catalog_versioning_list erp5_mysql_innodb/z_catalog_versioning_list
erp5_mysql_innodb/z_clear_reserved erp5_mysql_innodb/z_clear_reserved
...@@ -125,4 +127,4 @@ erp5_mysql_innodb/z_show_index ...@@ -125,4 +127,4 @@ erp5_mysql_innodb/z_show_index
erp5_mysql_innodb/z_show_tables erp5_mysql_innodb/z_show_tables
erp5_mysql_innodb/z_uncatalog_delivery erp5_mysql_innodb/z_uncatalog_delivery
erp5_mysql_innodb/z_uncatalog_object erp5_mysql_innodb/z_uncatalog_object
erp5_mysql_innodb/z_unique_values erp5_mysql_innodb/z_unique_values
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment