From f502dfe453bf26bcf39d1062249e1d9466b19731 Mon Sep 17 00:00:00 2001 From: Vincent Pelletier <vincent@nexedi.com> Date: Wed, 14 Feb 2007 09:51:48 +0000 Subject: [PATCH] Remove useless redirection when clearing catalog in ERP5Site_reindexAll. Remove useless spaces at end of line (sorry for poluting annotate with code I don't actually change). Create marker activity in ERP5Site_reindexAll and InventoryModule_reindexMovementList to be able to check when activities created by those scripts are over. Folder_reindexAll: Separate actualy activity parameters from activate()'s parameters. Remove duplicate definition of [...]_kw dict. Factorise activate() calls. Define a constant for object bundle length. Don't use objectValues to get object ids. Use list slice to generate a list of a fixed number of elements. Folder_reindexObjectList: Separate actualy activity parameters from activate()'s parameters. Add a check to avoid reindexation failure when an object scheduled for indexation is deleted when indexation takes place. git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@12691 20353a03-c40f-0410-a6d1-a30d3c3de9de --- .../erp5_core/ERP5Site_reindexAll.xml | 61 ++++++------- .../erp5_core/Folder_reindexAll.xml | 90 +++++++++---------- .../erp5_core/Folder_reindexObjectList.xml | 25 +++--- .../InventoryModule_reindexMovementList.xml | 19 +++- product/ERP5/bootstrap/erp5_core/bt/revision | 2 +- 5 files changed, 100 insertions(+), 97 deletions(-) diff --git a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_reindexAll.xml b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_reindexAll.xml index e4a9a12579..ed4688c38f 100644 --- a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_reindexAll.xml +++ b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_reindexAll.xml @@ -69,32 +69,23 @@ <item> <key> <string>_body</string> </key> <value> <string>portal = context.getPortalObject() \n -if clear_catalog: \n - # clear the catalog before reindexing \n +if clear_catalog:\n + # clear the catalog before reindexing\n catalog = portal.portal_catalog.getSQLCatalog(sql_catalog_id)\n catalog.manage_catalogClear()\n - # redirect to commit transaction\n - from ZTUtils import make_query\n - query_kw={\'passive_commit\': passive_commit} \n - if sql_catalog_id:\n - # XXX make_query doesn\'t support None as an argument \n - query_kw[\'sql_catalog_id\'] = sql_catalog_id \n - qstring = make_query(**query_kw) \n - return context.REQUEST.RESPONSE.redirect( \n - \'%s?%s\' % (script.absolute_url(), qstring)) \n - \n +\n # Reindex person module\n -print "#### Indexing person_module, stage 1 ####" \n +print "#### Indexing person_module, stage 1 ####"\n person_module=getattr(portal, \'person_module\', None)\n -if person_module is not None : \n - tag = \'person_stage_1\' \n - person_module.activate(tag=tag).Folder_reindexAll( \n - folder_tag=tag, \n - object_tag=tag, \n - sql_catalog_id=sql_catalog_id, \n - passive_commit=passive_commit) \n +if person_module is not None :\n + tag = \'person_stage_1\'\n + person_module.activate(tag=tag).Folder_reindexAll(\n + folder_tag=tag,\n + object_tag=tag,\n + sql_catalog_id=sql_catalog_id,\n + passive_commit=passive_commit)\n \n -print "#### Indexing translations ####" \n +print "#### Indexing translations ####"\n context.ERP5Site_updateTranslationTable(sql_catalog_id=sql_catalog_id)\n \n # Reindex categories\n @@ -222,7 +213,18 @@ context.portal_simulation.activate(\n priority=3\n ).InventoryModule_reindexMovementList(\n sql_catalog_id=sql_catalog_id,\n - passive_commit=passive_commit)\n + passive_commit=passive_commit,\n + final_activity_tag=\'last_inventory_activity\')\n +\n +if final_activity_tag is not None:\n + # Start a dummy activity which will get discarded when all started activities\n + # (and all activities they trigger) are over.\n + # Started on portal_simulation because activate does not work on portal object...\n + # No idea if there is a better place.\n + context.portal_simulation.activate(tag=final_activity_tag,\n + after_tag=(\'module\', \'inventory\', \'simulation\', \'person_stage_1\',\n + \'group_person_stage_1\', \'last_inventory_activity\', \'document\')\n + ).getId()\n \n return printed\n </string> </value> @@ -247,7 +249,7 @@ return printed\n </item> <item> <key> <string>_params</string> </key> - <value> <string>sql_catalog_id=None, passive_commit=1, clear_catalog=0</string> </value> + <value> <string>sql_catalog_id=None, passive_commit=1, clear_catalog=0, final_activity_tag=None</string> </value> </item> <item> <key> <string>errors</string> </key> @@ -267,7 +269,7 @@ return printed\n <dictionary> <item> <key> <string>co_argcount</string> </key> - <value> <int>3</int> </value> + <value> <int>4</int> </value> </item> <item> <key> <string>co_varnames</string> </key> @@ -276,19 +278,13 @@ return printed\n <string>sql_catalog_id</string> <string>passive_commit</string> <string>clear_catalog</string> + <string>final_activity_tag</string> <string>_print_</string> <string>_print</string> <string>_getattr_</string> -<string>context</string> + <string>context</string> <string>portal</string> <string>catalog</string> - <string>ZTUtils</string> - <string>make_query</string> - <string>query_kw</string> - <string>_write_</string> - <string>_apply_</string> - <string>qstring</string> -<string>script</string> <string>getattr</string> <string>None</string> <string>person_module</string> @@ -315,6 +311,7 @@ return printed\n <none/> <int>1</int> <int>0</int> + <none/> </tuple> </value> </item> diff --git a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexAll.xml b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexAll.xml index 4bf83bca3a..af6593335e 100644 --- a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexAll.xml +++ b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexAll.xml @@ -68,22 +68,22 @@ </item> <item> <key> <string>_body</string> </key> - <value> <string>folder = context\n + <value> <string encoding="cdata"><![CDATA[ + +folder = context\n \n -# Reindex folder\n -folder_kw = {\n +activate_kw = {\n \'tag\': folder_tag,\n \'after_tag\': folder_after_tag,\n - \'sql_catalog_id\': sql_catalog_id,\n \'passive_commit\': passive_commit\n }\n -for key, value in folder_kw.items():\n +for key, value in activate_kw.items():\n if value is None:\n - folder_kw.pop(key)\n + activate_kw.pop(key)\n \n -folder.reindexObject(**folder_kw)\n +# Reindex folder immediately\n +folder.reindexObject(sql_catalog_id=sql_catalog_id)\n \n -# Reindex folder content\n group_tag = None\n group_after_tag = object_after_tag\n if object_tag is not None:\n @@ -93,45 +93,33 @@ if object_tag is not None:\n else:\n group_after_tag = (object_tag, )\n \n -object_kw = {\n - \'tag\': folder_tag,\n - \'after_tag\': folder_after_tag,\n - \'sql_catalog_id\': sql_catalog_id,\n - \'passive_commit\': passive_commit\n -}\n -for key, value in object_kw.items():\n - if value is None:\n - object_kw.pop(key)\n -\n -\n -id_list = []\n -len = 0\n -for obj in folder.objectValues():\n - id_list.append(obj.getId())\n - len += 1\n - if len==100:\n +# Spawn activities for bundles of content objects.\n +# Bundle size, in object count\n +BUNDLE_ITEM_COUNT=100\n \n - folder.activate(activity=\'SQLQueue\', priority=object_priority, **object_kw).Folder_reindexObjectList(\n - id_list,\n - object_priority=object_priority,\n - object_tag=object_tag,\n - object_after_tag=object_after_tag,\n - sql_catalog_id=sql_catalog_id,\n - passive_commit=passive_commit,\n - )\n - id_list = []\n - len = 0\n +def Folder_reindexObjectList(id_list):\n + """\n + Create an activity calling Folder_reindexObjectList.\n + """\n + folder.activate(activity=\'SQLQueue\', priority=object_priority, **activate_kw).Folder_reindexObjectList(\n + id_list=id_list,\n + object_priority=object_priority,\n + object_tag=object_tag,\n + object_after_tag=object_after_tag,\n + sql_catalog_id=sql_catalog_id,\n + passive_commit=passive_commit,\n + )\n \n +id_list = [x for x in folder.objectIds()]\n +for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):\n + Folder_reindexObjectList(id_list=id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])\n \n -folder.activate(activity=\'SQLQueue\', priority=object_priority, **object_kw).Folder_reindexObjectList(\n - id_list,\n - object_priority=object_priority,\n - object_tag=object_tag,\n - object_after_tag=object_after_tag,\n - sql_catalog_id=sql_catalog_id,\n - passive_commit=passive_commit,\n - )\n -</string> </value> +remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT\n +if remaining_object_id_count > 0:\n + Folder_reindexObjectList(id_list=id_list[-remaining_object_id_count:])\n + + +]]></string> </value> </item> <item> <key> <string>_code</string> </key> @@ -188,19 +176,25 @@ folder.activate(activity=\'SQLQueue\', priority=object_priority, **object_kw).Fo <string>passive_commit</string> <string>context</string> <string>folder</string> - <string>folder_kw</string> + <string>activate_kw</string> <string>_getiter_</string> <string>_getattr_</string> <string>key</string> <string>value</string> <string>None</string> - <string>_apply_</string> <string>group_tag</string> <string>group_after_tag</string> - <string>object_kw</string> + <string>BUNDLE_ITEM_COUNT</string> + <string>Folder_reindexObjectList</string> + <string>append</string> + <string>$append0</string> + <string>x</string> <string>id_list</string> + <string>xrange</string> <string>len</string> - <string>obj</string> + <string>bundle_index</string> + <string>_getitem_</string> + <string>remaining_object_id_count</string> </tuple> </value> </item> diff --git a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexObjectList.xml b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexObjectList.xml index 2de5128bfe..0308590e20 100644 --- a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexObjectList.xml +++ b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexObjectList.xml @@ -70,20 +70,21 @@ <key> <string>_body</string> </key> <value> <string>folder = context\n \n -object_kw = {\n +activate_kw = {\n \'tag\': object_tag,\n \'after_tag\': object_after_tag,\n - \'sql_catalog_id\': sql_catalog_id,\n - \'passive_commit\': passive_commit\n + \'passive_commit\': passive_commit,\n + \'priority\': object_priority\n }\n -for key, value in object_kw.items():\n +for key, value in activate_kw.items():\n if value is None:\n - object_kw.pop(key)\n + activate_kw.pop(key)\n \n for id in id_list:\n - obj = folder.restrictedTraverse(id)\n - obj.recursiveReindexObject(priority=object_priority,\n - **object_kw)\n + obj = getattr(folder, id, None)\n + if obj is not None:\n + obj.recursiveReindexObject(activate_kw=activate_kw,\n + sql_catalog_id=sql_catalog_id)\n </string> </value> </item> <item> @@ -138,17 +139,17 @@ for id in id_list:\n <string>object_priority</string> <string>sql_catalog_id</string> <string>passive_commit</string> -<string>context</string> + <string>context</string> <string>folder</string> - <string>object_kw</string> + <string>activate_kw</string> <string>_getiter_</string> <string>_getattr_</string> <string>key</string> <string>value</string> <string>None</string> <string>id</string> + <string>getattr</string> <string>obj</string> - <string>_apply_</string> </tuple> </value> </item> @@ -170,7 +171,7 @@ for id in id_list:\n </value> </item> <item> - <key> <string>id</string> </key> + <key> <string>id</string> </key> <value> <string>Folder_reindexObjectList</string> </value> </item> <item> diff --git a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/InventoryModule_reindexMovementList.xml b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/InventoryModule_reindexMovementList.xml index 32f5ba998e..cde940a9fe 100644 --- a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/InventoryModule_reindexMovementList.xml +++ b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/InventoryModule_reindexMovementList.xml @@ -68,7 +68,9 @@ </item> <item> <key> <string>_body</string> </key> - <value> <string># look in the catalog to all inventory movement sort on date\n + <value> <string encoding="cdata"><![CDATA[ + +# look in the catalog to all inventory movement sort on date\n count = 1\n \n # We have to reindex all inventory in the order of the date\n @@ -85,7 +87,14 @@ for o in [ x.getObject() for x in context.portal_catalog(\n , passive_commit=passive_commit\n )\n count += 1\n -</string> </value> +\n +if final_activity_tag is not None and count > 1:\n + # Dummy activity used to determine if the previously started activities are over.\n + context.activate(tag=final_activity_tag\n + ,after_tag=\'inventory_%i\' % (count - 1, )).getId()\n + + +]]></string> </value> </item> <item> <key> <string>_code</string> </key> @@ -101,7 +110,7 @@ for o in [ x.getObject() for x in context.portal_catalog(\n </item> <item> <key> <string>_params</string> </key> - <value> <string>sql_catalog_id=None, passive_commit=1, **kw</string> </value> + <value> <string>sql_catalog_id=None, passive_commit=1, final_activity_tag=None, **kw</string> </value> </item> <item> <key> <string>errors</string> </key> @@ -121,7 +130,7 @@ for o in [ x.getObject() for x in context.portal_catalog(\n <dictionary> <item> <key> <string>co_argcount</string> </key> - <value> <int>2</int> </value> + <value> <int>3</int> </value> </item> <item> <key> <string>co_varnames</string> </key> @@ -129,6 +138,7 @@ for o in [ x.getObject() for x in context.portal_catalog(\n <tuple> <string>sql_catalog_id</string> <string>passive_commit</string> + <string>final_activity_tag</string> <string>kw</string> <string>count</string> <string>_getiter_</string> @@ -153,6 +163,7 @@ for o in [ x.getObject() for x in context.portal_catalog(\n <tuple> <none/> <int>1</int> + <none/> </tuple> </value> </item> diff --git a/product/ERP5/bootstrap/erp5_core/bt/revision b/product/ERP5/bootstrap/erp5_core/bt/revision index f123c4f4b5..2197544d04 100644 --- a/product/ERP5/bootstrap/erp5_core/bt/revision +++ b/product/ERP5/bootstrap/erp5_core/bt/revision @@ -1 +1 @@ -251 \ No newline at end of file +252 -- 2.30.9