Skip to content

erp5
erp5
Commit 4e6c6d8b authored by Nicolas Dumazet's avatar Nicolas Dumazet
Browse files
Options

group transformation indexation to avoid using many (too small) transactions 


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@33897 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 3552e3c8
Show whitespace changes
Inline Side-by-side
Showing with 111 additions and 57 deletions
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_catalogTransformation.xml
View file @ 4e6c6d8b
...@@ -53,27 +53,37 @@ ...@@ -53,27 +53,37 @@
</item> </item>
<item> <item>
<key> <string>_body</string> </key> <key> <string>_body</string> </key>
<value> <string encoding="cdata"><![CDATA[ <value> <string># This script indexes the preferred Transformation to produce\n
# This script indexes the preferred Transformation to produce\n
# a variation of a product.\n # a variation of a product.\n
# Only the transformation is passed as a parameter, the resource being\n # transformation_item_list is a list of:\n
# transformation.getResourceValue()\n # transformation, variation_list_list\n
# transformation is a transformation to index, while variation_list_list\n
# is a list of variation categories that are relevant for the produced resource\n
\n
\n \n
from Products.ERP5Type.Document import newTempMovement\n from Products.ERP5Type.Document import newTempMovement\n
\n \n
transformation = context.restrictedTraverse(transformation_relative_url)\n # List of dictionaries:\n
resource = transformation.getResourceValue()\n # { id:resource_id,\n
# variation_text: resource_variation_text,\n
# row_dict_list: list of rows to insert; each row is represented as a dict.}\n
row_dict_dict_list = []\n
\n
for transformation_relative_url, variation_list_list in transformation_item_list:\n
transformation = context.restrictedTraverse(transformation_relative_url)\n
resource = transformation.getResourceValue()\n
\n \n
row_dict_list = []\n if resource is None:\n
movement = newTempMovement(resource, \'temp\',\n continue\n
for variation_list in variation_list_list:\n
movement = newTempMovement(resource, \'temp\',\n
variation_category_list=variation_list,\n variation_category_list=variation_list,\n
resource=resource.getRelativeUrl(),\n resource=resource.getRelativeUrl(),\n
quantity=1.0)\n quantity=1.0)\n
base_row = dict(uid=resource.getUid(), variation_text=movement.getVariationText())\n
\n \n
base_row = dict(uid=resource.getUid(), variation_text=movement.getVariationText())\n row_dict_list = []\n
\n for amount in transformation.getAggregatedAmountList(movement):\n
for amount in transformation.getAggregatedAmountList(movement):\n
transformed_resource_uid = amount.getResourceUid()\n transformed_resource_uid = amount.getResourceUid()\n
quantity = amount.getQuantity()\n quantity = amount.getQuantity()\n
if transformed_resource_uid is not None and quantity is not None:\n if transformed_resource_uid is not None and quantity is not None:\n
...@@ -83,11 +93,11 @@ for amount in transformation.getAggregatedAmountList(movement):\n ...@@ -83,11 +93,11 @@ for amount in transformation.getAggregatedAmountList(movement):\n
quantity=quantity)\n quantity=quantity)\n
row_dict_list.append(row_dict)\n row_dict_list.append(row_dict)\n
\n \n
if len(row_dict_list) > 0:\n base_row[\'row_dict_list\'] = row_dict_list\n
context.z_catalog_transformation(row_dict_list=row_dict_list)\n row_dict_dict_list.append(base_row)\n
\n
context.z_catalog_transformation_list(row_dict_dict_list=row_dict_dict_list)\n
]]></string> </value> </string> </value>
</item> </item>
<item> <item>
<key> <string>_code</string> </key> <key> <string>_code</string> </key>
...@@ -97,7 +107,7 @@ if len(row_dict_list) > 0:\n ...@@ -97,7 +107,7 @@ if len(row_dict_list) > 0:\n
</item> </item>
<item> <item>
<key> <string>_params</string> </key> <key> <string>_params</string> </key>
<value> <string>transformation_relative_url, variation_list</string> </value> <value> <string>transformation_item_list</string> </value>
</item> </item>
<item> <item>
<key> <string>errors</string> </key> <key> <string>errors</string> </key>
...@@ -117,31 +127,34 @@ if len(row_dict_list) > 0:\n ...@@ -117,31 +127,34 @@ if len(row_dict_list) > 0:\n
<dictionary> <dictionary>
<item> <item>
<key> <string>co_argcount</string> </key> <key> <string>co_argcount</string> </key>
<value> <int>2</int> </value> <value> <int>1</int> </value>
</item> </item>
<item> <item>
<key> <string>co_varnames</string> </key> <key> <string>co_varnames</string> </key>
<value> <value>
<tuple> <tuple>
<string>transformation_relative_url</string> <string>transformation_item_list</string>
<string>variation_list</string>
<string>Products.ERP5Type.Document</string> <string>Products.ERP5Type.Document</string>
<string>newTempMovement</string> <string>newTempMovement</string>
<string>row_dict_dict_list</string>
<string>_getiter_</string>
<string>transformation_relative_url</string>
<string>variation_list_list</string>
<string>_getattr_</string> <string>_getattr_</string>
<string>context</string> <string>context</string>
<string>transformation</string> <string>transformation</string>
<string>resource</string> <string>resource</string>
<string>row_dict_list</string> <string>None</string>
<string>variation_list</string>
<string>movement</string> <string>movement</string>
<string>dict</string> <string>dict</string>
<string>base_row</string> <string>base_row</string>
<string>_getiter_</string> <string>row_dict_list</string>
<string>amount</string> <string>amount</string>
<string>transformed_resource_uid</string> <string>transformed_resource_uid</string>
<string>quantity</string> <string>quantity</string>
<string>None</string>
<string>row_dict</string> <string>row_dict</string>
<string>len</string> <string>_write_</string>
</tuple> </tuple>
</value> </value>
</item> </item>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_catalogTransformationList.xml
View file @ 4e6c6d8b
...@@ -53,19 +53,36 @@ ...@@ -53,19 +53,36 @@
</item> </item>
<item> <item>
<key> <string>_body</string> </key> <key> <string>_body</string> </key>
<value> <string># Indexing all transformation lines for all possible variations of a Resource can be very costly.\n <value> <string encoding="cdata"><![CDATA[
# Indexing all transformation lines for all possible variations of a Resource can be very costly.\n
# Avoid doing this in a single transaction, and split the operation.\n # Avoid doing this in a single transaction, and split the operation.\n
#\n \n
# One activity is generated per (resource, variation_category_list) possibility\n batch_size = 100\n
current_batch = []\n
current_size = 0\n
\n \n
for i, transformation in enumerate(getDefaultConversionTransformationValue):\n for i, transformation in enumerate(getDefaultConversionTransformationValue):\n
if transformation is None:\n if transformation is None:\n
continue\n continue\n
transformation_relative_url = transformation.getRelativeUrl()\n transformation_relative_url = transformation.getRelativeUrl()\n
for variation_category_list in getTransformationVariationCategoryCartesianProduct[i]:\n variation_list_list = getTransformationVariationCategoryCartesianProduct[i]\n
context.activate(activity=\'SQLQueue\').SQLCatalog_catalogTransformation(transformation_relative_url,\n size = len(transformation)*len(variation_list_list)\n
variation_category_list)\n \n
</string> </value> if size + current_size < batch_size:\n
current_batch.append((transformation_relative_url, variation_list_list))\n
current_size += size\n
else:\n
context.activate(activity=\'SQLQueue\').SQLCatalog_catalogTransformation(current_batch)\n
current_batch = (transformation_relative_url, variation_list_list),\n
current_size = size\n
\n
\n
if current_batch:\n
context.activate(activity=\'SQLQueue\').SQLCatalog_catalogTransformation(current_batch)\n
]]></string> </value>
</item> </item>
<item> <item>
<key> <string>_code</string> </key> <key> <string>_code</string> </key>
...@@ -103,6 +120,9 @@ for i, transformation in enumerate(getDefaultConversionTransformationValue):\n ...@@ -103,6 +120,9 @@ for i, transformation in enumerate(getDefaultConversionTransformationValue):\n
<tuple> <tuple>
<string>getDefaultConversionTransformationValue</string> <string>getDefaultConversionTransformationValue</string>
<string>getTransformationVariationCategoryCartesianProduct</string> <string>getTransformationVariationCategoryCartesianProduct</string>
<string>batch_size</string>
<string>current_batch</string>
<string>current_size</string>
<string>_getiter_</string> <string>_getiter_</string>
<string>enumerate</string> <string>enumerate</string>
<string>i</string> <string>i</string>
...@@ -111,7 +131,10 @@ for i, transformation in enumerate(getDefaultConversionTransformationValue):\n ...@@ -111,7 +131,10 @@ for i, transformation in enumerate(getDefaultConversionTransformationValue):\n
<string>_getattr_</string> <string>_getattr_</string>
<string>transformation_relative_url</string> <string>transformation_relative_url</string>
<string>_getitem_</string> <string>_getitem_</string>
<string>variation_category_list</string> <string>variation_list_list</string>
<string>len</string>
<string>size</string>
<string>_inplacevar_</string>
<string>context</string> <string>context</string>
</tuple> </tuple>
</value> </value>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_transformation.xml → product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_transformation_list.xml
View file @ 4e6c6d8b
...@@ -24,7 +24,7 @@ ...@@ -24,7 +24,7 @@
<value> <value>
<dictionary> <dictionary>
<item> <item>
<key> <string>row_dict_list</string> </key> <key> <string>row_dict_dict_list</string> </key>
<value> <value>
<dictionary/> <dictionary/>
</value> </value>
...@@ -36,7 +36,7 @@ ...@@ -36,7 +36,7 @@
<key> <string>_keys</string> </key> <key> <string>_keys</string> </key>
<value> <value>
<list> <list>
<string>row_dict_list</string> <string>row_dict_dict_list</string>
</list> </list>
</value> </value>
</item> </item>
...@@ -53,7 +53,7 @@ ...@@ -53,7 +53,7 @@
</item> </item>
<item> <item>
<key> <string>arguments_src</string> </key> <key> <string>arguments_src</string> </key>
<value> <string>row_dict_list</string> </value> <value> <string>row_dict_dict_list</string> </value>
</item> </item>
<item> <item>
<key> <string>cache_time_</string> </key> <key> <string>cache_time_</string> </key>
...@@ -79,7 +79,7 @@ ...@@ -79,7 +79,7 @@
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>z_catalog_transformation</string> </value> <value> <string>z_catalog_transformation_list</string> </value>
</item> </item>
<item> <item>
<key> <string>max_cache_</string> </key> <key> <string>max_cache_</string> </key>
...@@ -93,15 +93,22 @@ ...@@ -93,15 +93,22 @@
<key> <string>src</string> </key> <key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[ <value> <string encoding="cdata"><![CDATA[
<dtml-let row_list="[]">\n
<dtml-in row_dict_dict_list prefix="outer">\n
DELETE FROM `transformation` WHERE\n DELETE FROM `transformation` WHERE\n
<dtml-sqltest expr="row_dict_list[0][\'uid\']" column="uid" type="int">\n <dtml-sqltest expr="outer_item[\'uid\']" column="uid" type="int">\n
AND\n AND\n
<dtml-sqltest expr="row_dict_list[0][\'variation_text\']" column="variation_text" type="string">;\n <dtml-sqltest expr="outer_item[\'variation_text\']" column="variation_text" type="string">;\n
<dtml-var sql_delimiter>\n
<dtml-call "row_list.extend(outer_item[\'row_dict_list\'])">\n
</dtml-in>\n
\n
<dtml-if "len(row_list)>0">\n
<dtml-var sql_delimiter>\n <dtml-var sql_delimiter>\n
\n \n
INSERT INTO `transformation`\n INSERT INTO `transformation`\n
VALUES\n VALUES\n
<dtml-in row_dict_list prefix="loop">\n <dtml-in row_list prefix="loop">\n
(\n (\n
<dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n <dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n
<dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n <dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n
...@@ -110,7 +117,9 @@ VALUES\n ...@@ -110,7 +117,9 @@ VALUES\n
<dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n <dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n
)\n )\n
<dtml-unless sequence-end>,</dtml-unless>\n <dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in> </dtml-in>\n
</dtml-if>\n
</dtml-let>
]]></string> </value> ]]></string> </value>
</item> </item>
...@@ -150,15 +159,22 @@ VALUES\n ...@@ -150,15 +159,22 @@ VALUES\n
<key> <string>raw</string> </key> <key> <string>raw</string> </key>
<value> <string encoding="cdata"><![CDATA[ <value> <string encoding="cdata"><![CDATA[
<dtml-let row_list="[]">\n
<dtml-in row_dict_dict_list prefix="outer">\n
DELETE FROM `transformation` WHERE\n DELETE FROM `transformation` WHERE\n
<dtml-sqltest expr="row_dict_list[0][\'uid\']" column="uid" type="int">\n <dtml-sqltest expr="outer_item[\'uid\']" column="uid" type="int">\n
AND\n AND\n
<dtml-sqltest expr="row_dict_list[0][\'variation_text\']" column="variation_text" type="string">;\n <dtml-sqltest expr="outer_item[\'variation_text\']" column="variation_text" type="string">;\n
<dtml-var sql_delimiter>\n
<dtml-call "row_list.extend(outer_item[\'row_dict_list\'])">\n
</dtml-in>\n
\n
<dtml-if "len(row_list)>0">\n
<dtml-var sql_delimiter>\n <dtml-var sql_delimiter>\n
\n \n
INSERT INTO `transformation`\n INSERT INTO `transformation`\n
VALUES\n VALUES\n
<dtml-in row_dict_list prefix="loop">\n <dtml-in row_list prefix="loop">\n
(\n (\n
<dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n <dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n
<dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n <dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n
...@@ -167,7 +183,9 @@ VALUES\n ...@@ -167,7 +183,9 @@ VALUES\n
<dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n <dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n
)\n )\n
<dtml-unless sequence-end>,</dtml-unless>\n <dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in> </dtml-in>\n
</dtml-if>\n
</dtml-let>
]]></string> </value> ]]></string> </value>
</item> </item>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/revision
View file @ 4e6c6d8b
183 184
\ No newline at end of file \ No newline at end of file
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/template_catalog_method_id_list
View file @ 4e6c6d8b
...@@ -48,7 +48,7 @@ erp5_mysql_innodb/z_catalog_predicate_category_list ...@@ -48,7 +48,7 @@ erp5_mysql_innodb/z_catalog_predicate_category_list
erp5_mysql_innodb/z_catalog_predicate_list erp5_mysql_innodb/z_catalog_predicate_list
erp5_mysql_innodb/z_catalog_roles_and_users_list erp5_mysql_innodb/z_catalog_roles_and_users_list
erp5_mysql_innodb/z_catalog_stock_list erp5_mysql_innodb/z_catalog_stock_list
erp5_mysql_innodb/z_catalog_transformation erp5_mysql_innodb/z_catalog_transformation_list
erp5_mysql_innodb/z_catalog_translation_list erp5_mysql_innodb/z_catalog_translation_list
erp5_mysql_innodb/z_catalog_versioning_list erp5_mysql_innodb/z_catalog_versioning_list
erp5_mysql_innodb/z_clear_reserved erp5_mysql_innodb/z_clear_reserved
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7