Commit 92c93bdf authored by Vincent Pelletier's avatar Vincent Pelletier

CategoryTool_importCategoryFile: Refactor.

Factorise code paths between simulation and "real" modes,
fixing occasional non-representativeness of simulation mode.
Factorise code (down from 300 to 200 lines).
Simplify code (ex: use set instead of None-value dict).
Merge consecutive loops:
- delete/expire immediately instead of building a list to process later
- process each Base Category entirely (create, edit, delete/expire/keep)
before going to the next
This merging should overall reduce memory usage significantly.
parent 4aa2fbb8
...@@ -52,43 +52,33 @@ ...@@ -52,43 +52,33 @@
<key> <string>_body</string> </key> <key> <string>_body</string> </key>
<value> <string>from Products.ERP5Type.Message import translateString\n <value> <string>from Products.ERP5Type.Message import translateString\n
from Products.ERP5Type.Document import newTempBase\n from Products.ERP5Type.Document import newTempBase\n
from zExceptions import Redirect\n
portal = context.getPortalObject()\n
\n \n
# Initialise some general variables\n # XXX: allow simulation_mode without detailed_report ?\n
detailed_report |= simulation_mode\n
\n
portal = context.getPortalObject()\n
REQUEST = portal.REQUEST\n
base_category_property_id_set = portal.portal_types[\'Base Category\'].getInstancePropertySet()\n
category_property_id_set = portal.portal_types.Category.getInstancePropertySet()\n
portal_categories = portal.portal_categories\n
resolveCategory = portal_categories.resolveCategory\n
getRelatedValueList = portal_categories.getRelatedValueList\n
isTransitionPossible = portal.portal_workflow.isTransitionPossible\n
detailed_report_result = []\n detailed_report_result = []\n
detailed_report_append = detailed_report_result.append\n detailed_report_append = detailed_report_result.append\n
base_category_id_list = []\n def report(field_type, message, mapping=None, field_category=\'\', level=None):\n
category_path_dict = {}\n if level and level not in displayed_report:\n
simulation_new_category_id_list = []\n return\n
\n detailed_report_append(newTempBase(\n
base_category_property_id_set = getattr(context.portal_types, \'Base Category\').getInstancePropertySet()\n folder=context,\n
category_property_id_set = context.portal_types.Category.getInstancePropertySet()\n id=\'item\',\n
\n field_type=field_type,\n
getRelatedValueList = portal.portal_categories.getRelatedValueList\n field_category=field_category,\n
def Object_hasRelation(obj):\n field_message=translateString(\n
# Check if there is some related objets.\n message,\n
result = 0\n mapping=mapping,\n
for o in obj.getIndexableChildValueList():\n ),\n
for related in getRelatedValueList(o):\n ))\n
related_url = related.getRelativeUrl()\n
if related_url.startswith(obj.getRelativeUrl()):\n
continue\n
elif related_url.startswith(\'portal_trash\'):\n
continue\n
else:\n
result = 1\n
break\n
return result\n
\n
\n
def isValidID(id, is_base_category):\n
if is_base_category:\n
return id not in base_category_property_id_set\n
else:\n
return id not in category_property_id_set\n
\n
# Some statistics\n
new_category_counter = 0\n new_category_counter = 0\n
updated_category_counter = 0\n updated_category_counter = 0\n
total_category_counter = 0\n total_category_counter = 0\n
...@@ -97,258 +87,188 @@ deleted_category_counter = 0\n ...@@ -97,258 +87,188 @@ deleted_category_counter = 0\n
kept_category_counter = 0\n kept_category_counter = 0\n
expired_category_counter = 0\n expired_category_counter = 0\n
\n \n
filename = getattr(import_file, \'filename\', \'?\')\n def hasRelation(obj):\n
\n # Tests if there is any sensible related objet.\n
# The list of error from simulation mode\n for o in obj.getIndexableChildValueList():\n
error_list = []\n for related in getRelatedValueList(o):\n
if simulation_mode:\n related_url = related.getRelativeUrl()\n
def invalid_category_spreadsheet_handler(message):\n if not related_url.startswith(obj.getRelativeUrl()) and not related_url.startswith(\'portal_trash\'):\n
error = newTempBase(context, \'item\')\n return True\n
error.edit(field_type=\'Error\',\n return False\n
field_message=message)\n
error_list.append(error)\n
return True # means continue processing the rest of the document\n
else:\n
def invalid_category_spreadsheet_handler(message):\n
# action taken when an invalid spreadsheet is provided.\n
# we *raise* a Redirect, because we don\'t want the transaction to succeed\n
# note, we could make a dialog parameter to allow import invalid spreadsheet:\n
raise Redirect(\'%s/view?portal_status_message=%s\' % (\n
context.portal_categories.absolute_url(),\n
message))\n
\n
category_list_spreadsheet_mapping = context.Base_getCategoriesSpreadSheetMapping(import_file,\n
invalid_spreadsheet_error_handler=invalid_category_spreadsheet_handler)\n
\n \n
if simulation_mode and error_list:\n def invalid_category_spreadsheet_handler(message):\n
context.REQUEST.other[\'category_import_report\'] = error_list\n report(\n
return context.CategoryTool_viewImportReport()\n field_type=\'Error\',\n
message=message,\n
)\n
return True\n
category_list_spreadsheet_dict = context.Base_getCategoriesSpreadSheetMapping(\n
import_file,\n
invalid_spreadsheet_error_handler=invalid_category_spreadsheet_handler,\n
)\n
if detailed_report_result:\n
REQUEST.RESPONSE.write(portal_categories.CategoryTool_viewImportReport())\n
raise Exception(\'Spreadsheet contains errors\')\n
\n \n
for base_category, category_list in \\\n for base_category, category_list in category_list_spreadsheet_dict.iteritems():\n
category_list_spreadsheet_mapping.items():\n
base_category_id_list.append(base_category)\n
# Create categories\n
total_category_counter += len(category_list)\n total_category_counter += len(category_list)\n
category_path_set = set()\n
for category in category_list:\n for category in category_list:\n
is_new_category = False\n is_new_category = False\n
key_list = category.keys()\n category_path = category.pop(\'path\')\n
if \'path\' in key_list:\n category.pop(\'id\', None)\n
base_path_obj = context.portal_categories\n try:\n
category_id_list = category[\'path\'].split(\'/\')\n container_path, category_id = category_path.rsplit(\'/\', 1)\n
parent_path = category[\'path\'].split(\'/\')[0]+\'/\'\n except ValueError:\n
category_id = category_path\n
container = portal_categories\n
is_base_category = True\n is_base_category = True\n
is_valid_category = True\n
for category_id in category_id_list:\n
category_id = str(category_id)\n
# The current category is not existing\n
if not base_path_obj.hasContent(category_id) and (not {\'category\':category_id,\'path\':parent_path} in simulation_new_category_id_list):\n
# Create the category\n
if is_base_category:\n
category_type = \'Base Category\'\n category_type = \'Base Category\'\n
category_type_property_id_set = base_category_property_id_set\n
else:\n else:\n
container = resolveCategory(container_path)\n
is_base_category = False\n
category_type = \'Category\'\n category_type = \'Category\'\n
if isValidID(category_id, is_base_category):\n category_type_property_id_set = category_property_id_set\n
if simulation_mode:\n try:\n
simulation_new_category_id_list.append({\'category\':category_id,\'path\':parent_path})\n category_value = container[category_id]\n
if \'created\' in displayed_report:\n except KeyError:\n
report_line = newTempBase(context, \'item\')\n if category_id in category_type_property_id_set:\n
report_line.edit(field_type = \'Creation\', field_category = category[\'path\'].split(category_id)[0]+category_id, field_message = translateString("Will be created new ${type}", mapping=dict(type=category_type)))\n report(\n
detailed_report_append(report_line)\n level=\'warning\',\n
else:\n field_type=\'WARNING\',\n
base_path_obj.newContent( portal_type = category_type\n message="found invalid ID ${id} ",\n
, id = category_id\n mapping={\'id\':category_id},\n
, effective_date = effective_date\n
)\n )\n
if \'created\' in displayed_report:\n
report_line = newTempBase(context, \'item\')\n
report_line.edit(field_type = \'Creation\', field_category = base_path_obj[category_id].getRelativeUrl(), field_message = translateString("Created new ${type}", mapping=dict(type=category_type)))\n
detailed_report_append(report_line)\n
else:\n
# The ID is invalid, we must break the loop\n
invalid_category_id_counter += 1\n invalid_category_id_counter += 1\n
is_valid_category = False\n continue\n
if \'warning\' in displayed_report:\n
report_line = newTempBase(context, \'item\')\n
report_line.edit(field_type = \'WARNING\', field_category = \'\', field_message = translateString("found invalid ID ${id} ", mapping=dict(id=category_id)))\n
detailed_report_append(report_line)\n
break\n
is_new_category = True\n
new_category_counter += 1\n new_category_counter += 1\n
if {\'category\':category_id,\'path\':parent_path} not in simulation_new_category_id_list:\n category_value = container.newContent(\n
base_path_obj = base_path_obj[category_id]\n portal_type=category_type,\n
category_path_dict[base_path_obj.getRelativeUrl()] = None\n id=category_id,\n
is_base_category = False\n effective_date=effective_date,\n
parent_path = category[\'path\'].split(category_id)[0] +category_id\n )\n
report(\n
level=\'created\',\n
field_type=\'Creation\',\n
field_category=category_value.getRelativeUrl(),\n
message="Created new ${type}",\n
mapping={\'type\': category_type},\n
)\n
is_new_category = True\n
category_path_set.add(category_value.getRelativeUrl())\n
\n \n
property_id_list = base_path_obj.propertyIds() # XXX could be cached\n
if is_valid_category:\n
# Only try to update a valid category\n
new_category = base_path_obj\n
# Set the category properties\n
category_update_dict = {}\n category_update_dict = {}\n
first_update_reported = True\n for key, value in category.iteritems():\n
for key in key_list:\n if not create_local_property and key not in category_type_property_id_set:\n
key = str(key)\n report(\n
if key not in [\'path\', \'id\']:\n field_type=\'Update\',\n
value = category[key]\n field_category=category_value.getRelativeUrl(),\n
\n message="Ignoring local property ${key} with value ${value}",\n
if not create_local_property and key not in property_id_list:\n mapping={\'key\': key, \'value\': value},\n
# if we do not create local properties, then we skip properties\n )\n
# that are not in category ids\n elif is_new_category or (\n
report_line = newTempBase(context, \'item\')\n value not in (\'\', None) and\n
report_line.edit(field_type=\'Update\',\n not category_value.hasProperty(key)\n
field_category=new_category.getRelativeUrl(),\n ) or (\n
field_message=translateString("Ignoring local property ${key} with value ${value}",\n update_existing_property and\n
mapping=dict(key=key, value=value)))\n str(category_value.getProperty(key)) != value\n
detailed_report_append(report_line)\n ):\n
continue\n
\n
if is_new_category:\n
# Always update properties if this a new category\n
category_update_dict[key] = value\n
elif update_existing_property:\n
# Update if update existing property and the property is not the same\n
if new_category.hasProperty(key):\n
if str(new_category.getProperty(key)) != value :\n
category_update_dict[key] = value\n
if not is_new_category and \'updated\' in displayed_report:\n
if first_update_reported:\n
field_type = \'Update\'\n
field_category = new_category.getRelativeUrl()\n
first_update_reported = False\n
else:\n
field_type = \'\'\n
field_category = \'\'\n
report_line = newTempBase(context, \'item\')\n
report_line.edit(field_type=field_type, field_category=field_category, field_message=translateString("Updated ${key} with value ${value} ", mapping=dict(key=key, value=value)))\n
detailed_report_append(report_line)\n
else:\n
if value not in (\'\', None) :\n
category_update_dict[key] = value\n
if not is_new_category and \'updated\' in displayed_report:\n
if first_update_reported:\n
field_type = \'Update\'\n
field_category = new_category.getRelativeUrl()\n
first_update_reported = False\n
else:\n
field_type = \'\'\n
field_category = \'\'\n
report_line = newTempBase(context, \'item\')\n
report_line.edit(field_type=field_type, field_category=field_category, field_message=translateString("Updated ${key} with value ${value} ", mapping=dict(key=key, value=value)))\n
detailed_report_append(report_line)\n
elif value not in (\'\', None) and not new_category.hasProperty(key):\n
# Only set properties which are not already defined\n
category_update_dict[key] = value\n category_update_dict[key] = value\n
if \'updated\' in displayed_report:\n if not is_new_category:\n
if first_update_reported:\n report(\n
field_type = \'Update\'\n level=\'updated\',\n
field_category = new_category.getRelativeUrl()\n field_type=\'Update\',\n
first_update_reported = False\n field_category=category_value.getRelativeUrl(),\n
else:\n message="Updated ${key} with value ${value} ",\n
field_type = \'\'\n mapping={\'key\': key, \'value\': value},\n
field_category = \'\'\n )\n
report_line = newTempBase(context, \'item\')\n if category_update_dict:\n
report_line.edit(field_type=field_type, field_category=field_category, field_message=translateString("Updated ${key} with value ${value} ", mapping=dict(key=key, value=value)))\n if not is_new_category:\n
detailed_report_append(report_line)\n
if not is_new_category and category_update_dict:\n
updated_category_counter += 1\n updated_category_counter += 1\n
\n
if not simulation_mode:\n
# force_update=1 is required here because\n # force_update=1 is required here because\n
# edit(short_title=\'foo\', title=\'foo\') only stores short_title property.\n # edit(short_title=\'foo\', title=\'foo\') only stores short_title property.\n
new_category.edit(force_update=1, **category_update_dict)\n category_value.edit(force_update=1, **category_update_dict)\n
else:\n
raise KeyError, \'path was not defined for a category, this should never happen.\'\n
\n \n
# Find categories to delete\n to_do_list = [portal_categories[base_category]]\n
category_to_delete_list = []\n while to_do_list:\n
for base_category_id in base_category_id_list:\n category = to_do_list.pop()\n
base_category = context.portal_categories[base_category_id]\n recurse = True\n
for category in base_category.getCategoryChildValueList(is_self_excluded=False):\n if category.getRelativeUrl() in category_path_set:\n
if not category_path_dict.has_key(category.getRelativeUrl()):\n pass\n
if existing_category_list == \'keep\' or Object_hasRelation(category):\n elif existing_category_list == \'keep\':\n
if Object_hasRelation(category):\n report(\n
# TODO: add a dialog parameter allowing to delete this path\n level=\'kept\',\n
if \'warning\' in displayed_report:\n field_type=\'Keep\',\n
report_line = newTempBase(context, \'item\')\n
report_line.edit(field_type=\'Warning\',\n
field_category=category.getRelativeUrl(),\n field_category=category.getRelativeUrl(),\n
field_message=translateString("Category is used and can not be deleted or expired "))\n message="Kept category",\n
detailed_report_append(report_line)\n )\n
else:\n
if existing_category_list == \'keep\' and \'kept\' in displayed_report:\n
report_line = newTempBase(context, \'item\')\n
report_line.edit(field_type=\'Keep\',\n
field_category=category.getRelativeUrl(),\n
field_message=translateString("Kept category"))\n
detailed_report_append(report_line)\n
kept_category_counter += 1\n kept_category_counter += 1\n
else:\n elif hasRelation(category):\n
if existing_category_list == \'delete\' and \'deleted\' in displayed_report:\n # TODO: add a dialog parameter allowing to delete this path\n
report_line = newTempBase(context, \'item\')\n report(\n
report_line.edit(field_type=\'Delete\',\n level=\'warning\',\n
field_type=\'Warning\',\n
field_category=category.getRelativeUrl(),\n field_category=category.getRelativeUrl(),\n
field_message=translateString("Deleted category"))\n message="Category is used and can not be deleted or expired ",\n
detailed_report_append(report_line)\n )\n
elif existing_category_list == \'delete\':\n
recurse = False\n
deleted_category_counter += 1\n deleted_category_counter += 1\n
if existing_category_list == \'expire\' and \'expired\' in displayed_report:\n report(\n
report_line = newTempBase(context, \'item\')\n level=\'deleted\',\n
report_line.edit(field_type=\'Expire\',\n field_type=\'Delete\',\n
field_category=category.getRelativeUrl(),\n field_category=category.getRelativeUrl(),\n
field_message=translateString("Expired category"))\n message="Deleted category",\n
detailed_report_append(report_line)\n )\n
expired_category_counter += 1\n category.getParentValue().deleteContent(category.getId())\n
category_to_delete_list.append(category.getRelativeUrl())\n elif existing_category_list == \'expire\':\n
\n report(\n
if not simulation_mode:\n level=\'expired\',\n
# Delete unused categories\n field_type=\'Expire\',\n
if existing_category_list == \'delete\':\n field_category=category.getRelativeUrl(),\n
for category in category_to_delete_list:\n message="Expired category",\n
category = context.portal_categories.resolveCategory(category)\n )\n
if category is not None:\n
parent = category.getParentValue()\n
category_id = category.getId()\n
parent.deleteContent(category_id)\n
if existing_category_list == \'expire\':\n
if expiration_date:\n if expiration_date:\n
for category in category_to_delete_list:\n expired_category_counter += 1\n
category = context.portal_categories.resolveCategory(category)\n category.edit(expiration_date=expiration_date)\n
if category is not None:\n elif isTransitionPossible(category, \'expire\'):\n
category.edit(**{\'expiration_date\':expiration_date})\n expired_category_counter += 1\n
else:\n
isTransitionPossible = context.getPortalObject().portal_workflow \\\n
.isTransitionPossible\n
for category in category_to_delete_list:\n
category = context.portal_categories.resolveCategory(category)\n
if category is not None \\\n
and isTransitionPossible(category, \'expire\'):\n
category.expire()\n category.expire()\n
# Report failure otherwise ?\n
# Report failure on unexpected value ?\n
if recurse:\n
to_do_list.extend(category.objectValues())\n
\n \n
portal.portal_caches.clearAllCache()\n
\n \n
\n # TODO: translate\n
context.portal_caches.clearAllCache()\n portal_status_message = \'%s categories found in %s: %s created, %s updated, %s untouched, %s invalid ID. %s existing categories: %s deleted, %s expired, %s kept.%s\' % (\n
\n total_category_counter,\n
if (detailed_report or simulation_mode) and detailed_report_result:\n getattr(import_file, \'filename\', \'?\'),\n
# Return a detailed report if requested\n new_category_counter,\n
context.REQUEST.other[\'category_import_report\'] = detailed_report_result\n updated_category_counter,\n
return context.CategoryTool_viewImportReport()\n total_category_counter - new_category_counter - updated_category_counter,\n
\n invalid_category_id_counter,\n
# Import is a success, go back to the portal_categories tool\n deleted_category_counter + kept_category_counter + expired_category_counter,\n
return context.REQUEST.RESPONSE.redirect(\n deleted_category_counter,\n
# TODO translate\n expired_category_counter,\n
"%s/view?portal_status_message=%s+categories+found+in+%s:+%s+created,+%s+updated,+%s+untouched,+%s+invalid+ID.+%s+existing+categories:+%s+deleted,+%s+expired,+%s+kept."\n kept_category_counter,\n
% ( context.portal_categories.absolute_url()\n \' (nothing done, simulation mode enabled)\' if simulation_mode else \'\',\n
, total_category_counter\n )\n
, filename\n if detailed_report:\n
, new_category_counter\n REQUEST.other[\'portal_status_message\'] = portal_status_message\n
, updated_category_counter\n REQUEST.other[\'category_import_report\'] = detailed_report_result\n
, total_category_counter - new_category_counter - updated_category_counter\n result = portal_categories.CategoryTool_viewImportReport().encode(\'utf-8\')\n
, invalid_category_id_counter\n if simulation_mode:\n
, deleted_category_counter + kept_category_counter + expired_category_counter\n REQUEST.RESPONSE.write(result)\n
, deleted_category_counter\n raise Exception(\'Dry run\') \n
, expired_category_counter\n return result\n
, kept_category_counter\n portal_categories.Base_redirect(\n
)\n keep_items={\n
\'portal_status_message\': portal_status_message,\n
},\n
abort_transaction=simulation_mode,\n
)\n )\n
</string> </value> </string> </value>
</item> </item>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment