Skip to content

erp5
erp5
Commit 998daccd authored by Sebastien Robin's avatar Sebastien Robin
Browse files
Options

- Added Folder_reindexTreeObject made by Aurélien. I improved it so that 

it starts once and then call himself until all sub trees are parsed.
- Increased from 100 to 1000 the number of objects in a bundle, this is
not that much because 1000 can be parsed in less than 20 seconds.
- Make Folder_reindexObjectList starts once and then call himself until
it finishes a list of lists of ids.
- make sure that ERP5Site_reindexAll still reindex objects in several modules by the
same time.
- Added additionnal_priority in order to increase default priority values, this
is very nice while doing hot reindexing.
- Added simple tag dependency in order to make sure the amount of reindexed objects
stay limited. Also decrease a lot the number of Folder_reindexObjectList activities
by the same time, like this performance is increased.
- Add a condition when adding after_tag parameter while calling Folder_riendexObjectList,
so we wait that the previous work was finished only after adding 1000 * nb_nodes
objects reindex activities.
- explain the most important points with comments

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@17070 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 59b8e7c8
Show whitespace changes
Inline Side-by-side
Showing with 430 additions and 63 deletions
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_reindexAll.xml
View file @ 998daccd
...@@ -3,11 +3,8 @@ ...@@ -3,11 +3,8 @@
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<tuple> <tuple>
<tuple> <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
<string>Products.PythonScripts.PythonScript</string> <tuple/>
<string>PythonScript</string>
</tuple>
<none/>
</tuple> </tuple>
</pickle> </pickle>
<pickle> <pickle>
...@@ -77,11 +74,14 @@ if clear_catalog:\n ...@@ -77,11 +74,14 @@ if clear_catalog:\n
# Reindex person module\n # Reindex person module\n
print "#### Indexing person_module, stage 1 ####"\n print "#### Indexing person_module, stage 1 ####"\n
person_module=getattr(portal, \'person_module\', None)\n person_module=getattr(portal, \'person_module\', None)\n
higher_priority = 1 + additional_priority\n
if person_module is not None :\n if person_module is not None :\n
tag = \'person_stage_1\'\n tag = \'person_stage_1\'\n
person_module.activate(tag=tag).Folder_reindexAll(\n person_module.activate(tag=tag,\n
priority=higher_priority).Folder_reindexAll(\n
folder_tag=tag,\n folder_tag=tag,\n
object_tag=tag,\n object_tag=tag,\n
object_priority=higher_priority,\n
sql_catalog_id=sql_catalog_id,\n sql_catalog_id=sql_catalog_id,\n
passive_commit=passive_commit)\n passive_commit=passive_commit)\n
\n \n
...@@ -97,11 +97,13 @@ object_after_tag = folder_after_tag\n ...@@ -97,11 +97,13 @@ object_after_tag = folder_after_tag\n
\n \n
context.portal_categories.activate(\n context.portal_categories.activate(\n
tag=folder_tag,\n tag=folder_tag,\n
priority=higher_priority,\n
after_tag=folder_after_tag).Folder_reindexAll(\n after_tag=folder_after_tag).Folder_reindexAll(\n
folder_tag=folder_tag,\n folder_tag=folder_tag,\n
folder_after_tag=folder_after_tag,\n folder_after_tag=folder_after_tag,\n
object_tag=object_tag,\n object_tag=object_tag,\n
object_after_tag=object_after_tag,\n object_after_tag=object_after_tag,\n
object_priority=higher_priority,\n
sql_catalog_id=sql_catalog_id,\n sql_catalog_id=sql_catalog_id,\n
passive_commit=passive_commit)\n passive_commit=passive_commit)\n
\n \n
...@@ -111,10 +113,11 @@ folder_tag = \'module\'\n ...@@ -111,10 +113,11 @@ folder_tag = \'module\'\n
folder_after_tag = (\'category\', \'person_stage_1\', \'group_person_stage_1\')\n folder_after_tag = (\'category\', \'person_stage_1\', \'group_person_stage_1\')\n
object_tag = \'document\'\n object_tag = \'document\'\n
object_after_tag = folder_after_tag\n object_after_tag = folder_after_tag\n
object_priority = 2\n object_priority = 2 + additional_priority\n
for folder in [context.portal_alarms, context.portal_preferences]:\n for folder in [context.portal_alarms, context.portal_preferences]:\n
folder.activate(\n folder.activate(\n
tag=folder_tag,\n tag=folder_tag,\n
priority=object_priority,\n
after_tag=folder_after_tag).Folder_reindexAll(\n after_tag=folder_after_tag).Folder_reindexAll(\n
folder_tag=folder_tag,\n folder_tag=folder_tag,\n
folder_after_tag=folder_after_tag,\n folder_after_tag=folder_after_tag,\n
...@@ -131,14 +134,16 @@ folder_tag = \'module\'\n ...@@ -131,14 +134,16 @@ folder_tag = \'module\'\n
folder_after_tag = (\'category\', \'document\', \'person_stage_1\', \'group_person_stage_1\')\n folder_after_tag = (\'category\', \'document\', \'person_stage_1\', \'group_person_stage_1\')\n
object_tag = \'simulation\'\n object_tag = \'simulation\'\n
object_after_tag = folder_after_tag\n object_after_tag = folder_after_tag\n
object_priority = 3\n object_priority = 3 + additional_priority\n
context.portal_simulation.activate(\n context.portal_simulation.activate(\n
tag=folder_tag,\n tag=folder_tag,\n
priority=higher_priority,\n
after_tag=folder_after_tag).Folder_reindexAll(\n after_tag=folder_after_tag).Folder_reindexAll(\n
folder_tag=folder_tag,\n folder_tag=folder_tag,\n
folder_after_tag=folder_after_tag,\n folder_after_tag=folder_after_tag,\n
object_tag=object_tag,\n object_tag=object_tag,\n
object_after_tag=object_after_tag,\n object_after_tag=object_after_tag,\n
object_priority=higher_priority,\n
sql_catalog_id=sql_catalog_id,\n sql_catalog_id=sql_catalog_id,\n
passive_commit=passive_commit)\n passive_commit=passive_commit)\n
\n \n
...@@ -159,12 +164,13 @@ folder_tag = \'module\'\n ...@@ -159,12 +164,13 @@ folder_tag = \'module\'\n
folder_after_tag = (\'category\', \'person_stage_1\', \'group_person_stage_1\')\n folder_after_tag = (\'category\', \'person_stage_1\', \'group_person_stage_1\')\n
object_tag = \'document\'\n object_tag = \'document\'\n
object_after_tag = folder_after_tag\n object_after_tag = folder_after_tag\n
object_priority = 2\n object_priority = 2 + additional_priority\n
for folder in [context.portal_templates, context.portal_trash,\n for folder in [context.portal_templates, context.portal_trash,\n
context.portal_domains, context.portal_rules,\n context.portal_domains, context.portal_rules,\n
context.portal_deliveries, context.portal_orders]:\n context.portal_deliveries, context.portal_orders]:\n
folder.activate(\n folder.activate(\n
tag=folder_tag,\n tag=folder_tag,\n
priority=object_priority,\n
after_tag=folder_after_tag).Folder_reindexAll(\n after_tag=folder_after_tag).Folder_reindexAll(\n
folder_tag=folder_tag,\n folder_tag=folder_tag,\n
folder_after_tag=folder_after_tag,\n folder_after_tag=folder_after_tag,\n
...@@ -181,18 +187,7 @@ for folder in portal.objectValues(("ERP5 Folder",)):\n ...@@ -181,18 +187,7 @@ for folder in portal.objectValues(("ERP5 Folder",)):\n
\n \n
folder.activate(\n folder.activate(\n
tag=folder_tag,\n tag=folder_tag,\n
after_tag=folder_after_tag).Folder_reindexAll(\n priority=object_priority,\n
folder_tag=folder_tag,\n
folder_after_tag=folder_after_tag,\n
object_tag=object_tag,\n
object_after_tag=object_after_tag,\n
object_priority=object_priority,\n
sql_catalog_id=sql_catalog_id,\n
passive_commit=passive_commit)\n
# reindex portal_caches plugins\n
print "#### Indexing portal_caches tool ####"\n
context.portal_caches.activate(\n
tag=folder_tag,\n
after_tag=folder_after_tag).Folder_reindexAll(\n after_tag=folder_after_tag).Folder_reindexAll(\n
folder_tag=folder_tag,\n folder_tag=folder_tag,\n
folder_after_tag=folder_after_tag,\n folder_after_tag=folder_after_tag,\n
...@@ -210,6 +205,7 @@ for folder in portal.objectValues(("ERP5 Folder",)):\n ...@@ -210,6 +205,7 @@ for folder in portal.objectValues(("ERP5 Folder",)):\n
print "#### Indexing contents inside folder %s ####" % folder.id\n print "#### Indexing contents inside folder %s ####" % folder.id\n
folder.activate(\n folder.activate(\n
tag=folder_tag,\n tag=folder_tag,\n
priority=object_priority,\n
after_tag=folder_after_tag).Folder_reindexAll(\n after_tag=folder_after_tag).Folder_reindexAll(\n
folder_tag=folder_tag,\n folder_tag=folder_tag,\n
folder_after_tag=folder_after_tag,\n folder_after_tag=folder_after_tag,\n
...@@ -222,7 +218,7 @@ for folder in portal.objectValues(("ERP5 Folder",)):\n ...@@ -222,7 +218,7 @@ for folder in portal.objectValues(("ERP5 Folder",)):\n
# start activty from simulation because the erp5site is not an active object\n # start activty from simulation because the erp5site is not an active object\n
context.portal_simulation.activate(\n context.portal_simulation.activate(\n
after_tag=(\'inventory\', \'simulation\', \'person_stage_1\', \'group_person_stage_1\'),\n after_tag=(\'inventory\', \'simulation\', \'person_stage_1\', \'group_person_stage_1\'),\n
priority=3\n priority=3 + additional_priority\n
).InventoryModule_reindexMovementList(\n ).InventoryModule_reindexMovementList(\n
sql_catalog_id=sql_catalog_id,\n sql_catalog_id=sql_catalog_id,\n
passive_commit=passive_commit,\n passive_commit=passive_commit,\n
...@@ -234,6 +230,7 @@ if final_activity_tag is not None:\n ...@@ -234,6 +230,7 @@ if final_activity_tag is not None:\n
# Started on portal_simulation because activate does not work on portal object...\n # Started on portal_simulation because activate does not work on portal object...\n
# No idea if there is a better place.\n # No idea if there is a better place.\n
context.portal_simulation.activate(tag=final_activity_tag,\n context.portal_simulation.activate(tag=final_activity_tag,\n
priority=3 + additional_priority,\n
after_tag=(\'module\', \'inventory\', \'simulation\', \'person_stage_1\',\n after_tag=(\'module\', \'inventory\', \'simulation\', \'person_stage_1\',\n
\'group_person_stage_1\', \'last_inventory_activity\', \'document\')\n \'group_person_stage_1\', \'last_inventory_activity\', \'document\')\n
).getId()\n ).getId()\n
...@@ -261,7 +258,7 @@ return printed\n ...@@ -261,7 +258,7 @@ return printed\n
</item> </item>
<item> <item>
<key> <string>_params</string> </key> <key> <string>_params</string> </key>
<value> <string>sql_catalog_id=None, passive_commit=1, clear_catalog=0, final_activity_tag=None</string> </value> <value> <string>sql_catalog_id=None, additional_priority=0, passive_commit=1, clear_catalog=0, final_activity_tag=None</string> </value>
</item> </item>
<item> <item>
<key> <string>errors</string> </key> <key> <string>errors</string> </key>
...@@ -281,13 +278,14 @@ return printed\n ...@@ -281,13 +278,14 @@ return printed\n
<dictionary> <dictionary>
<item> <item>
<key> <string>co_argcount</string> </key> <key> <string>co_argcount</string> </key>
<value> <int>4</int> </value> <value> <int>5</int> </value>
</item> </item>
<item> <item>
<key> <string>co_varnames</string> </key> <key> <string>co_varnames</string> </key>
<value> <value>
<tuple> <tuple>
<string>sql_catalog_id</string> <string>sql_catalog_id</string>
<string>additional_priority</string>
<string>passive_commit</string> <string>passive_commit</string>
<string>clear_catalog</string> <string>clear_catalog</string>
<string>final_activity_tag</string> <string>final_activity_tag</string>
...@@ -300,6 +298,7 @@ return printed\n ...@@ -300,6 +298,7 @@ return printed\n
<string>getattr</string> <string>getattr</string>
<string>None</string> <string>None</string>
<string>person_module</string> <string>person_module</string>
<string>higher_priority</string>
<string>tag</string> <string>tag</string>
<string>folder_tag</string> <string>folder_tag</string>
<string>folder_after_tag</string> <string>folder_after_tag</string>
...@@ -321,6 +320,7 @@ return printed\n ...@@ -321,6 +320,7 @@ return printed\n
<value> <value>
<tuple> <tuple>
<none/> <none/>
<int>0</int>
<int>1</int> <int>1</int>
<int>0</int> <int>0</int>
<none/> <none/>
......
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexAll.xml
View file @ 998daccd
...@@ -3,11 +3,8 @@ ...@@ -3,11 +3,8 @@
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<tuple> <tuple>
<tuple> <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
<string>Products.PythonScripts.PythonScript</string> <tuple/>
<string>PythonScript</string>
</tuple>
<none/>
</tuple> </tuple>
</pickle> </pickle>
<pickle> <pickle>
...@@ -73,7 +70,7 @@ ...@@ -73,7 +70,7 @@
folder = context\n folder = context\n
\n \n
activate_kw = {\n activate_kw = {\n
\'tag\': folder_tag,\n \'tag\': object_tag,\n
\'after_tag\': folder_after_tag,\n \'after_tag\': folder_after_tag,\n
\'passive_commit\': passive_commit\n \'passive_commit\': passive_commit\n
}\n }\n
...@@ -84,25 +81,18 @@ for key, value in activate_kw.items():\n ...@@ -84,25 +81,18 @@ for key, value in activate_kw.items():\n
# Reindex folder immediately\n # Reindex folder immediately\n
folder.reindexObject(sql_catalog_id=sql_catalog_id)\n folder.reindexObject(sql_catalog_id=sql_catalog_id)\n
\n \n
group_tag = None\n
group_after_tag = object_after_tag\n
if object_tag is not None:\n
group_tag = "group_%s" % object_tag\n
if object_after_tag is not None:\n
group_after_tag = object_after_tag + (object_tag,)\n
else:\n
group_after_tag = (object_tag, )\n
\n
# Spawn activities for bundles of content objects.\n # Spawn activities for bundles of content objects.\n
# Bundle size, in object count\n # Bundle size, in object count\n
BUNDLE_ITEM_COUNT=100\n BUNDLE_ITEM_COUNT=1000\n
\n \n
def Folder_reindexObjectList(id_list):\n def Folder_reindexObjectList(id_list_list):\n
"""\n """\n
Create an activity calling Folder_reindexObjectList.\n Create an activity calling Folder_reindexObjectList.\n
"""\n """\n
folder.activate(activity=\'SQLQueue\', priority=object_priority, **activate_kw).Folder_reindexObjectList(\n folder.activate(activity=\'SQLQueue\', priority=object_priority, \n
id_list=id_list,\n round_robin_scheduling=1,**activate_kw).Folder_reindexObjectList(\n
id_list=None,\n
id_list_list=id_list_list,\n
object_priority=object_priority,\n object_priority=object_priority,\n
object_tag=object_tag,\n object_tag=object_tag,\n
object_after_tag=object_after_tag,\n object_after_tag=object_after_tag,\n
...@@ -110,13 +100,64 @@ def Folder_reindexObjectList(id_list):\n ...@@ -110,13 +100,64 @@ def Folder_reindexObjectList(id_list):\n
passive_commit=passive_commit,\n passive_commit=passive_commit,\n
)\n )\n
\n \n
id_list = [x for x in folder.objectIds()]\n try:\n
for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):\n tree_id_list = folder.getTreeIdList()\n
except NotImplementedError:\n
# Build a list of list, like this we parse ids only one time,\n
# and then Folder_reinexObjectList will work with one list at\n
# a time and remove it from the list of list\n
# This id_list_list can be quite big and generate quite big\n
# activities, but the effect is limited, because if we have too\n
# much objects (like millions), we should use HBTree Folders, and\n
# then the work will be splitted\n
id_list = [x for x in folder.objectIds()]\n
id_list_list = []\n
for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):\n
Folder_reindexObjectList(id_list=id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])\n Folder_reindexObjectList(id_list=id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])\n
id_list_list.append(id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])\n
\n
remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT\n
if remaining_object_id_count > 0:\n
id_list_list.append(id_list[-remaining_object_id_count:])\n
Folder_reindexObjectList(id_list_list)\n
else:\n
if folder_after_tag is None:\n
folder_after_tag = ()\n
i = 0\n
folder_id = folder.getId()\n
\n
tree_tag = "%s_tree" % folder_id\n
bundle_tag = "%s_bundle_reindex" % folder_id\n
bundle_object_tag = "%s_reindex" % folder_id\n
\n
# Say to Folder_reindexTreeObjectList to call himself again and\n
# again until all tree_id_list are parsed. Also, make sure that\n
# the work of the previous Folder_reindexTreeObjectList is completely\n
# done\n
tree_after_tag = folder_after_tag + (bundle_tag, bundle_object_tag)\n
folder.activate(activity=\'SQLQueue\', priority=object_priority, \n
after_tag=tree_after_tag, tag=tree_tag,\n
).Folder_reindexTreeObjectList(\n
tree_id=None,\n
tree_id_list=tree_id_list,\n
folder_tag=bundle_tag,\n
folder_after_tag=bundle_object_tag,\n
object_priority=object_priority,\n
sql_catalog_id=sql_catalog_id,\n
passive_commit=passive_commit,\n
object_tag=bundle_object_tag,\n
tree_after_tag=tree_after_tag,\n
tree_tag=tree_tag,\n
)\n
\n
# Start an activity wich will wait the end of the module\n
folder_id_after_tag = folder_after_tag + (tree_tag, bundle_tag, bundle_object_tag)\n
id_activate_kw = {}\n
if object_tag is not None:\n
id_activate_kw[\'tag\'] = object_tag\n
\n \n
remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT\n folder.activate(activity=\'SQLDict\', priority=object_priority, \n
if remaining_object_id_count > 0:\n after_tag=folder_id_after_tag, **id_activate_kw).getId()\n
Folder_reindexObjectList(id_list=id_list[-remaining_object_id_count:])\n
]]></string> </value> ]]></string> </value>
...@@ -182,19 +223,30 @@ if remaining_object_id_count > 0:\n ...@@ -182,19 +223,30 @@ if remaining_object_id_count > 0:\n
<string>key</string> <string>key</string>
<string>value</string> <string>value</string>
<string>None</string> <string>None</string>
<string>group_tag</string>
<string>group_after_tag</string>
<string>BUNDLE_ITEM_COUNT</string> <string>BUNDLE_ITEM_COUNT</string>
<string>Folder_reindexObjectList</string> <string>Folder_reindexObjectList</string>
<string>tree_id_list</string>
<string>NotImplementedError</string>
<string>append</string> <string>append</string>
<string>$append0</string> <string>$append0</string>
<string>x</string> <string>x</string>
<string>id_list</string> <string>id_list</string>
<string>id_list_list</string>
<string>xrange</string> <string>xrange</string>
<string>len</string> <string>len</string>
<string>bundle_index</string> <string>bundle_index</string>
<string>_getitem_</string> <string>_getitem_</string>
<string>remaining_object_id_count</string> <string>remaining_object_id_count</string>
<string>i</string>
<string>folder_id</string>
<string>tree_tag</string>
<string>bundle_tag</string>
<string>bundle_object_tag</string>
<string>tree_after_tag</string>
<string>folder_id_after_tag</string>
<string>id_activate_kw</string>
<string>_write_</string>
<string>_apply_</string>
</tuple> </tuple>
</value> </value>
</item> </item>
......
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexObjectList.xml
View file @ 998daccd
...@@ -3,11 +3,8 @@ ...@@ -3,11 +3,8 @@
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<tuple> <tuple>
<tuple> <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
<string>Products.PythonScripts.PythonScript</string> <tuple/>
<string>PythonScript</string>
</tuple>
<none/>
</tuple> </tuple>
</pickle> </pickle>
<pickle> <pickle>
...@@ -68,7 +65,17 @@ ...@@ -68,7 +65,17 @@
</item> </item>
<item> <item>
<key> <string>_body</string> </key> <key> <string>_body</string> </key>
<value> <string>folder = context\n <value> <string encoding="cdata"><![CDATA[
folder = context\n
\n
# Keep compatibility with id_list\n
if id_list_list is not None:\n
if id_list is not None:\n
raise ValueError, "both id_list and id_list_list can be defined"\n
if len(id_list_list) == 0:\n
return None\n
id_list = id_list_list.pop()\n
\n \n
activate_kw = {\n activate_kw = {\n
\'tag\': object_tag,\n \'tag\': object_tag,\n
...@@ -85,7 +92,39 @@ for id in id_list:\n ...@@ -85,7 +92,39 @@ for id in id_list:\n
if obj is not None:\n if obj is not None:\n
obj.recursiveReindexObject(activate_kw=activate_kw,\n obj.recursiveReindexObject(activate_kw=activate_kw,\n
sql_catalog_id=sql_catalog_id)\n sql_catalog_id=sql_catalog_id)\n
</string> </value> \n
\n
if id_list_list is not None:\n
if len(id_list_list) > 0:\n
if count is None:\n
count = 1\n
new_activity_kw = {}\n
# We do not want to wait until there is enough activities\n
# So add BUNDLE_ITEM_COUNT * node_len activities before waiting\n
node_len = len(context.portal_activities.getProcessingNodeList())\n
if count % node_len == 0:\n
new_activity_kw[\'after_tag\'] = folder_after_tag\n
count = 0\n
count += 1\n
\n
# By calling again and again, we improve performance and we have\n
# less activities by the same time\n
folder.activate(activity=\'SQLQueue\',\n
priority=object_priority,\n
tag=folder_tag, **new_activity_kw).Folder_reindexObjectList(\n
None,\n
id_list_list=id_list_list,\n
object_priority=object_priority,\n
object_tag=object_tag,\n
sql_catalog_id=sql_catalog_id,\n
passive_commit=passive_commit,\n
folder_tag=folder_tag,\n
folder_after_tag=folder_after_tag,\n
count=count,\n
)\n
]]></string> </value>
</item> </item>
<item> <item>
<key> <string>_code</string> </key> <key> <string>_code</string> </key>
...@@ -107,7 +146,7 @@ for id in id_list:\n ...@@ -107,7 +146,7 @@ for id in id_list:\n
</item> </item>
<item> <item>
<key> <string>_params</string> </key> <key> <string>_params</string> </key>
<value> <string>id_list, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, passive_commit=1</string> </value> <value> <string>id_list, id_list_list=None, folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, passive_commit=1, count=None</string> </value>
</item> </item>
<item> <item>
<key> <string>errors</string> </key> <key> <string>errors</string> </key>
...@@ -127,29 +166,40 @@ for id in id_list:\n ...@@ -127,29 +166,40 @@ for id in id_list:\n
<dictionary> <dictionary>
<item> <item>
<key> <string>co_argcount</string> </key> <key> <string>co_argcount</string> </key>
<value> <int>6</int> </value> <value> <int>10</int> </value>
</item> </item>
<item> <item>
<key> <string>co_varnames</string> </key> <key> <string>co_varnames</string> </key>
<value> <value>
<tuple> <tuple>
<string>id_list</string> <string>id_list</string>
<string>id_list_list</string>
<string>folder_tag</string>
<string>folder_after_tag</string>
<string>object_tag</string> <string>object_tag</string>
<string>object_after_tag</string> <string>object_after_tag</string>
<string>object_priority</string> <string>object_priority</string>
<string>sql_catalog_id</string> <string>sql_catalog_id</string>
<string>passive_commit</string> <string>passive_commit</string>
<string>count</string>
<string>context</string> <string>context</string>
<string>folder</string> <string>folder</string>
<string>None</string>
<string>ValueError</string>
<string>len</string>
<string>_getattr_</string>
<string>activate_kw</string> <string>activate_kw</string>
<string>_getiter_</string> <string>_getiter_</string>
<string>_getattr_</string>
<string>key</string> <string>key</string>
<string>value</string> <string>value</string>
<string>None</string>
<string>id</string> <string>id</string>
<string>getattr</string> <string>getattr</string>
<string>obj</string> <string>obj</string>
<string>new_activity_kw</string>
<string>node_len</string>
<string>_write_</string>
<string>_inplacevar_</string>
<string>_apply_</string>
</tuple> </tuple>
</value> </value>
</item> </item>
...@@ -162,11 +212,15 @@ for id in id_list:\n ...@@ -162,11 +212,15 @@ for id in id_list:\n
<key> <string>func_defaults</string> </key> <key> <string>func_defaults</string> </key>
<value> <value>
<tuple> <tuple>
<none/>
<none/>
<none/>
<none/> <none/>
<none/> <none/>
<int>1</int> <int>1</int>
<none/> <none/>
<int>1</int> <int>1</int>
<none/>
</tuple> </tuple>
</value> </value>
</item> </item>
......
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexTreeObjectList.xml 0 → 100644
View file @ 998daccd
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<tuple>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
<tuple/>
</tuple>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Python_magic</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>__ac_local_roles__</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string encoding="cdata"><![CDATA[
folder = context\n
\n
# Keep compatibility with tree_id\n
if tree_id_list is not None:\n
context.log(\'tree_id\', tree_id)\n
context.log(\'tree_id_list\', tree_id_list)\n
if tree_id is not None:\n
raise ValueError, "both tree and tree_id_list should not be defined"\n
tree_id = tree_id_list.pop()\n
\n
activate_kw = {\n
\'passive_commit\': passive_commit\n
}\n
for key, value in activate_kw.items():\n
if value is None:\n
activate_kw.pop(key)\n
\n
\n
# Spawn activities for bundles of content objects.\n
# Bundle size, in object count\n
BUNDLE_ITEM_COUNT = 1000\n
\n
folder_id = folder.getId()\n
def Folder_reindexObjectList(id_list_list):\n
"""\n
Create an activity calling Folder_reindexObjectList.\n
"""\n
folder.activate(activity=\'SQLQueue\', priority=object_priority, \n
after_tag=object_tag,\n
tag=folder_tag, **activate_kw).Folder_reindexObjectList(\n
None,\n
id_list_list=id_list_list,\n
object_priority=object_priority,\n
object_tag=object_tag,\n
sql_catalog_id=sql_catalog_id,\n
passive_commit=passive_commit,\n
folder_tag=folder_tag,\n
folder_after_tag=folder_after_tag,\n
)\n
\n
# HBTree folder\n
id_list = [x for x in folder.objectIds(base_id=tree_id)]\n
# Build a list of list, like this we parse ids only one time,\n
# and then Folder_reinexObjectList will work with one list at\n
# a time and remove it from the list of list\n
# This id_list_list can be quite big and generate quite big\n
# activities, but the effect is limited, because the work is\n
# splitted for each base_id of the HBTree.\n
id_list_list = []\n
for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):\n
id_list_list.append(id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])\n
\n
remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT\n
if remaining_object_id_count > 0:\n
id_list_list.append(id_list[-remaining_object_id_count:])\n
Folder_reindexObjectList(id_list_list=id_list_list)\n
\n
if tree_id_list is not None:\n
if len(tree_id_list) > 0:\n
# Calling again and again the same script allow to decrease the\n
# number of activities by the same time and increase performance.\n
folder.activate(activity=\'SQLQueue\', priority=object_priority,\n
after_tag=tree_after_tag, \n
tag=tree_tag).Folder_reindexTreeObjectList(\n
tree_id=None,\n
tree_id_list=tree_id_list,\n
folder_tag=folder_tag,\n
folder_after_tag=folder_after_tag,\n
object_priority=object_priority,\n
sql_catalog_id=sql_catalog_id,\n
object_tag=object_tag,\n
passive_commit=passive_commit,\n
tree_after_tag=tree_after_tag,\n
tree_tag=tree_tag,\n
)\n
]]></string> </value>
</item>
<item>
<key> <string>_code</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>_filepath</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>tree_id=None, tree_id_list=None, folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, passive_commit=1, tree_tag=None, tree_after_tag=None</string> </value>
</item>
<item>
<key> <string>errors</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>func_code</string> </key>
<value>
<object>
<klass>
<global name="FuncCode" module="Shared.DC.Scripts.Signature"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>co_argcount</string> </key>
<value> <int>11</int> </value>
</item>
<item>
<key> <string>co_varnames</string> </key>
<value>
<tuple>
<string>tree_id</string>
<string>tree_id_list</string>
<string>folder_tag</string>
<string>folder_after_tag</string>
<string>object_tag</string>
<string>object_after_tag</string>
<string>object_priority</string>
<string>sql_catalog_id</string>
<string>passive_commit</string>
<string>tree_tag</string>
<string>tree_after_tag</string>
<string>context</string>
<string>folder</string>
<string>None</string>
<string>_getattr_</string>
<string>ValueError</string>
<string>activate_kw</string>
<string>_getiter_</string>
<string>key</string>
<string>value</string>
<string>BUNDLE_ITEM_COUNT</string>
<string>folder_id</string>
<string>Folder_reindexObjectList</string>
<string>append</string>
<string>$append0</string>
<string>x</string>
<string>id_list</string>
<string>id_list_list</string>
<string>xrange</string>
<string>len</string>
<string>bundle_index</string>
<string>_getitem_</string>
<string>remaining_object_id_count</string>
</tuple>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>func_defaults</string> </key>
<value>
<tuple>
<none/>
<none/>
<none/>
<none/>
<none/>
<none/>
<int>1</int>
<none/>
<int>1</int>
<none/>
<none/>
</tuple>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Folder_reindexTreeObjectList</string> </value>
</item>
<item>
<key> <string>warnings</string> </key>
<value>
<tuple/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_core/bt/revision
View file @ 998daccd
527 536
\ No newline at end of file \ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7