Reimplement migration of persistent objects with obsolete classes

This fixes several issues:
- Some classes like XMLObject were outside Products.ERP5Type.Document
  and there were not migrated.
- Persistent migration using _delOb/_setOb does not work with mount points.

A new Base.migrateToPortalTypeClass method is also provided to migrate objects
persistently. Note however that migration of HBTrees requires additional work,
using PickleUpdater method.

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@44780 20353a03-c40f-0410-a6d1-a30d3c3de9de

product/ERP5/Document/BusinessTemplate.py

@@ -58,7 +58,7 @@ from Products.ERP5Type.Utils import readLocalExtension, \
 from Products.ERP5Type.Utils import readLocalTest, \
                                     writeLocalTest, \
                                     removeLocalTest
-from Products.ERP5Type.Utils import convertToUpperCase, PersistentMigrationMixin
+from Products.ERP5Type.Utils import convertToUpperCase
 from Products.ERP5Type import Permissions, PropertySheet, interfaces
 from Products.ERP5Type.XMLObject import XMLObject
 from Products.ERP5Type.dynamic.portal_type_class import synchronizeDynamicModules
@@ -546,23 +546,6 @@ class BaseTemplateItem(Implicit, Persistent):
   def importFile(self, bta, **kw):
     bta.importFiles(item=self)
 
-  def migrateToPortalTypeClass(self, obj):
-    klass = obj.__class__
-    if klass.__module__ == 'erp5.portal_type':
-      return obj
-    portal_type = getattr(aq_base(obj), 'portal_type', None)
-    if portal_type is None:
-      portal_type = getattr(klass, 'portal_type', None)
-
-    if portal_type is None:
-      # ugh?
-      return obj
-    import erp5.portal_type
-    newklass = getattr(erp5.portal_type, portal_type)
-    assert klass != newklass
-    obj.__class__ = newklass
-    return obj
-
   def removeProperties(self, obj, export, keep_workflow_history=False):
     """
     Remove unneeded properties for export
@@ -590,8 +573,6 @@ class BaseTemplateItem(Implicit, Persistent):
         attr_set.update(('last_max_id_dict', 'last_id_dict'))
       elif classname == 'Types Tool' and klass.__module__ == 'erp5.portal_type':
         attr_set.add('type_provider_list')
-    else:
-      obj = self.migrateToPortalTypeClass(obj)
 
     for attr in obj.__dict__.keys():
       if attr in attr_set or attr.startswith('_cache_cookie_'):
@@ -840,12 +821,6 @@ class ObjectTemplateItem(BaseTemplateItem):
         else: # new object
           modified_object_list[path] = 'New', type_name
 
-        # if that's an old style class, use a portal type class instead
-        migrateme = getattr(obj, '_migrateToPortalTypeClass', None)
-        if migrateme is not None:
-          migrateme()
-        self._objects[path] = obj
-
       # update _p_jar property of objects cleaned by removeProperties
       transaction.savepoint(optimistic=True)
       for path, old_object in upgrade_list:
@@ -1054,14 +1029,6 @@ class ObjectTemplateItem(BaseTemplateItem):
 
           # install object
           obj = self._objects[path]
-          if isinstance(self, PortalTypeTemplateItem):
-            # if that's an old style class, use a portal type class instead
-            # XXX PortalTypeTemplateItem-specific
-            migrateme = getattr(obj, '_migrateToPortalTypeClass', None)
-            if migrateme is not None:
-              migrateme()
-              self._objects[path] = obj
-
           # XXX Following code make Python Scripts compile twice, because
           #     _getCopy returns a copy without the result of the compilation.
           #     A solution could be to add a specific _getCopy method to
@@ -1924,9 +1891,8 @@ class PortalTypeTemplateItem(ObjectTemplateItem):
       if score is None:
         obj = self._objects[path]
         klass = obj.__class__
-        if klass.__module__.startswith('Products.ERP5Type.Document.'):
+        if klass.__module__ != 'erp5.portal_type':
           portal_type = obj.portal_type
-          obj._p_deactivate()
         else:
           portal_type = klass.__name__
         depend = path_dict.get(portal_type)
@@ -1938,11 +1904,7 @@ class PortalTypeTemplateItem(ObjectTemplateItem):
           return 0, path
         cache[path] = score = depend and 1 + solveDependency(depend)[0] or 0
       return score, path
-    PersistentMigrationMixin._no_migration += 1
-    try:
-      object_key_list.sort(key=solveDependency)
-    finally:
-      PersistentMigrationMixin._no_migration -= 1
+    object_key_list.sort(key=solveDependency)
     return object_key_list
 
   # XXX : this method is kept temporarily, but can be removed once all bt5 are
@@ -2858,12 +2820,6 @@ class ActionTemplateItem(ObjectTemplateItem):
         for name, obj in action_dict.iteritems():
           imported_action = container._importOldAction(obj).aq_base
 
-          # if that's an old style class, use a portal type class instead
-          # XXX PortalTypeTemplateItem-specific
-          migrateme = getattr(imported_action, '_migrateToPortalTypeClass', None)
-          if migrateme is not None:
-            migrateme()
-
     else:
       BaseTemplateItem.install(self, context, trashbin, **kw)
       p = context.getPortalObject()

product/ERP5/ERP5Site.py

@@ -34,7 +34,6 @@ from Products.ERP5Type.Accessor.Constant import PropertyGetter as ConstantGetter
 from Products.ERP5Type.Cache import caching_instance_method
 from Products.ERP5Type.Cache import CachingMethod, CacheCookieMixin
 from Products.ERP5Type.ERP5Type import ERP5TypeInformation
-from Products.ERP5.Document.BusinessTemplate import BusinessTemplate
 from Products.ERP5Type.Log import log as unrestrictedLog
 from Products.CMFActivity.Errors import ActivityPendingError
 import ERP5Defaults
@@ -1433,44 +1432,24 @@ class ERP5Site(FolderMixIn, CMFSite, CacheCookieMixin):
     if self.getERP5SiteGlobalId() in [None, '']:
       self.erp5_site_global_id = global_id
 
-  security.declareProtected(Permissions.ManagePortal, 'migrateToPortalTypeClass')
-  def migrateToPortalTypeClass(self, REQUEST=None):
-    """Compatibility code that allows migrating a site to portal type classes.
-    
-    We consider that a Site is migrated if its Types Tool is migrated
-    (it will always be migrated last)"""
-    types_tool = getattr(self, 'portal_types', None)
-    if types_tool is None:
-      # empty site
-      return
-    if types_tool.__class__.__module__ == 'erp5.portal_type':
-      # nothing to do, already migrated
-      if REQUEST is not None:
-        return REQUEST.RESPONSE.redirect(
-          '%s?portal_status_message=' \
-          'Nothing to do, already migrated.' % \
-          self.absolute_url())
-      return
-
-    # note that the site itself is not migrated (ERP5Site is not a portal type)
-    # only the tools and top level modules are.
-    # Normally, PersistentMigrationMixin should take care of the rest.
-    id_list = self.objectIds()
-
-    # make sure that Types Tool is migrated last
-    id_list.remove('portal_types')
-    id_list.append('portal_types')
-    for id in id_list:
-      method = getattr(self[id], '_migrateToPortalTypeClass', None)
-      if method is None:
-        continue
-      method()
-
-    if REQUEST is not None:
-      return REQUEST.RESPONSE.redirect(
-        '%s?portal_status_message=' \
-        'Successfully migrated tools and types to portal type classes.' % \
-        self.absolute_url())
+  security.declareProtected(Permissions.ManagePortal,
+                            'migrateToPortalTypeClass')
+  def migrateToPortalTypeClass(self):
+    from Products.ERP5Type.dynamic.persistent_migration import PickleUpdater
+    from Products.ERP5Type.Tool.BaseTool import BaseTool
+    PickleUpdater(self)
+    for tool in self.objectValues():
+      if isinstance(tool, BaseTool):
+        tool_id = tool.id
+        if tool_id != 'portal_property_sheets':
+          if tool_id in ('portal_categories', ):
+            tool = tool.activate()
+          tool.migrateToPortalTypeClass(tool_id not in (
+            'portal_activities', 'portal_simulation', 'portal_templates',
+            'portal_trash'))
+          if tool_id in ('portal_trash',):
+            for obj in tool.objectValues():
+              obj.migrateToPortalTypeClass()
 
 Globals.InitializeClass(ERP5Site)
 

product/ERP5Type/Base.py

@@ -3578,25 +3578,6 @@ class Base( CopyContainer,
   def isItem(self):
     return self.portal_type in self.getPortalItemTypeList()
 
-  def _migrateToPortalTypeClass(self):
-    klass = self.__class__
-    portal_type = self.getPortalType()
-    if klass.__module__ not in ('erp5.portal_type', 'erp5.temp_portal_type'):
-      import erp5.portal_type
-      newklass = getattr(erp5.portal_type, portal_type)
-      assert klass != newklass
-      self.__class__ = newklass
-      self._p_changed = True
-      # this might look useless, but it is necessary to explicitely record
-      # the change in the parent container, because the class has changed
-      try:
-        parent = self.getParentValue()
-      except AttributeError:
-        return
-      id = self.getId()
-      parent._delOb(id)
-      parent._setOb(id, self)
-
   security.declareProtected(Permissions.DeletePortalContent,
                             'migratePortalType')
   def migratePortalType(self, portal_type):

product/ERP5Type/ERP5Type.py

@@ -723,7 +723,8 @@ class ERP5TypeInformation(XMLObject,
 
       This is used to update an existing site or to import a BT.
       """
-      from Products.ERP5Type.Document.ActionInformation import ActionInformation
+      import erp5.portal_type
+      ActionInformation = getattr(erp5.portal_type, 'Action Information')
       old_action = old_action.__getstate__()
       action_type = old_action.pop('category', None)
       action = ActionInformation(self.generateNewId())

product/ERP5Type/Tool/BaseTool.py

@@ -92,35 +92,23 @@ class BaseTool (UniqueObject, Folder):
                 meta_types.append(meta_type)
         return meta_types
 
-    def _migrateToPortalTypeClass(self):
-      portal_type = self.getPortalType()
+    def _fixPortalTypeBeforeMigration(self, portal_type):
       # Tools are causing problems: they used to have no type_class, or wrong
       # type_class, or sometimes have no type definitions at all.
-      # Check that everything is alright before trying
-      # to migrate the tool:
+      # Fix type definition if possible before any migration.
       from Products.ERP5.ERP5Site import getSite
       types_tool = getSite().portal_types
       type_definition = getattr(types_tool, portal_type, None)
-      if type_definition is None:
-        LOG('BaseTool._migrateToPortalTypeClass', WARNING,
-            "No portal type definition was found for Tool '%s'"
-            " (class %s, portal_type '%s')"
-            % (self.getId(), self.__class__.__name__, portal_type))
-        return
-
-      type_class = type_definition.getTypeClass()
-      if type_class in ('Folder', None):
+      if type_definition is not None and \
+         type_definition.getTypeClass() in ('Folder', None):
         # wrong type_class, fix it manually:
         from Products.ERP5Type import document_class_registry
-        document_class_name = portal_type.replace(' ', '')
-        if document_class_name in document_class_registry:
-          type_definition.type_class = document_class_name
-        else:
-          LOG('BaseTool._migrateToPortalTypeClass', WARNING,
-              'No document class could be found for portal type %s'
+        try:
+          type_definition.type_class = document_class_registry[
+            portal_type.replace(' ', '')]
+        except KeyError:
+          LOG('BaseTool._migratePortalType', WARNING,
+              'No document class could be found for portal type %r'
               % portal_type)
-          return
-
-      return super(BaseTool, self)._migrateToPortalTypeClass()
 
 InitializeClass(BaseTool)

product/ERP5Type/Tool/TypesTool.py

@@ -128,7 +128,7 @@ class TypesTool(TypeProvider):
       'Standard Property',
       'Acquired Property',
       'Dummy Class Tool',
-      # the following ones are required by '_migrateToPortalTypeClass'
+      # XXX the following ones are required by '_migrateToPortalTypeClass'
       'Types Tool',
       'Property Sheet Tool',
       # the following ones are required to upgrade an existing site
@@ -387,11 +387,6 @@ class TypesTool(TypeProvider):
       trashbin = UnrestrictedMethod(trash_tool.newTrashBin)(self.id)
       trashbin._setOb(old_types_tool.id, old_types_tool)
 
-  def _migrateToPortalTypeClass(self):
-    for type_definition in self.objectValues():
-      type_definition._migrateToPortalTypeClass()
-    return super(TypesTool, self)._migrateToPortalTypeClass()
-
 # Compatibility code to access old "ERP5 Role Information" objects.
 OldRoleInformation = imp.new_module('Products.ERP5Type.RoleInformation')
 sys.modules[OldRoleInformation.__name__] = OldRoleInformation

product/ERP5Type/Utils.py

@@ -893,44 +893,6 @@ def setDefaultClassProperties(property_holder):
         )
       }
 
-class PersistentMigrationMixin(object):
-  """
-  All classes issued from ERP5Type.Document.XXX submodules
-  will gain with mixin as a base class.
-
-  It allows us to migrate ERP5Type.Document.XXX.YYY classes to
-  erp5.portal_type.ZZZ namespace
-
-  Note that migration can be disabled by setting the '_no_migration'
-  class attribute to a nonzero value, as all old objects in the system
-  should inherit from this mixin
-  """
-  _no_migration = 0
-
-  def __setstate__(self, value):
-    klass = self.__class__
-    if PersistentMigrationMixin._no_migration \
-        or klass.__module__ in ('erp5.portal_type', 'erp5.temp_portal_type'):
-      super(PersistentMigrationMixin, self).__setstate__(value)
-      return
-
-    portal_type = value.get('portal_type')
-    if portal_type is None:
-      portal_type = getattr(klass, 'portal_type', None)
-    if portal_type is None:
-      LOG('ERP5Type', PROBLEM,
-          "no portal type was found for %s (class %s)" \
-               % (self, klass))
-      super(PersistentMigrationMixin, self).__setstate__(value)
-    else:
-      # proceed with migration
-      import erp5.portal_type
-      newklass = getattr(erp5.portal_type, portal_type)
-      assert self.__class__ != newklass
-      self.__class__ = newklass
-      self.__setstate__(value)
-      LOG('ERP5Type', TRACE, "Migration for object %s" % self)
-
 from Globals import Persistent, PersistentMapping
 
 def importLocalDocument(class_id, path=None, class_path=None):
@@ -968,30 +930,8 @@ def importLocalDocument(class_id, path=None, class_path=None):
 
   ### Migration
   module_name = "Products.ERP5Type.Document.%s" % class_id
-
-  # Most of Document modules define a single class
-  # (ERP5Type.Document.Person.Person)
-  # but some (eek) need to act as module to find other documents,
-  # e.g. ERP5Type.Document.BusinessTemplate.SkinTemplateItem
-  #
-  def migrate_me_document_loader(document_name):
-    klass = getattr(module, document_name)
-    if issubclass(klass, (Persistent, PersistentMapping)):
-      setDefaultClassProperties(klass)
-      InitializeClass(klass)
-
-      class MigrateMe(PersistentMigrationMixin, klass):
-        pass
-      MigrateMe.__name__ = document_name
-      MigrateMe.__module__ = module_name
-      return MigrateMe
-    else:
-      return klass
-  from dynamic.dynamic_module import registerDynamicModule
-  document_module = registerDynamicModule(module_name,
-                                          migrate_me_document_loader)
-
-  setattr(Products.ERP5Type.Document, class_id, document_module)
+  sys.modules[module_name] = module
+  setattr(Products.ERP5Type.Document, class_id, module)
 
   ### newTempFoo
   from Products.ERP5Type.ERP5Type import ERP5TypeInformation

product/ERP5Type/dynamic/lazy_class.py

@@ -18,6 +18,7 @@ from zLOG import LOG, WARNING, BLATHER
 
 from portal_type_class import generatePortalTypeClass
 from accessor_holder import AccessorHolderType
+import persistent_migration
 
 # PersistentBroken can't be reused directly
 # because its « layout differs from 'GhostPortalType' »
@@ -183,6 +184,7 @@ class PortalTypeMetaClass(GhostBaseMetaClass, PropertyHolder):
       for attr in cls.__dict__.keys():
         if attr not in ('__module__',
                         '__doc__',
+                        '__setstate__',
                         'workflow_method_registry',
                         '__isghost__',
                         'portal_type'):
@@ -306,6 +308,11 @@ class PortalTypeMetaClass(GhostBaseMetaClass, PropertyHolder):
         for key, value in attribute_dict.iteritems():
           setattr(klass, key, value)
 
+        if getattr(klass.__setstate__, 'im_func', None) is \
+           persistent_migration.__setstate__:
+          # optimization to reduce overhead of compatibility code
+          klass.__setstate__ = persistent_migration.Base__setstate__
+
         for interface in interface_list:
           classImplements(klass, interface)
 

product/ERP5Type/dynamic/persistent_migration.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SARL and Contributors. All Rights Reserved.
+#                    Julien Muchembled <jm@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+
+# The class of an object is first fixed non-persistently by __setstate__:
+# - It can't be done before because its portal_type maybe different from
+#   the one specified on the old class.
+# - If done later, some methods may be wrong or missing.
+# By default, objects are not migrated persistently, mainly because the old
+# class may be copied in the pickle of the container, and we can't access it
+# from __setstate__.
+
+import re
+from AccessControl import ClassSecurityInfo
+from Acquisition import aq_base
+from OFS.Folder import Folder as OFS_Folder
+from persistent import Persistent, wref
+from zLOG import LOG, PROBLEM, DEBUG, TRACE
+from ZODB.serialize import ObjectWriter, ObjectReader
+from Products.ERP5Type import Permissions
+from Products.ERP5Type.Base import Base, WorkflowMethod
+
+isOldBTree = re.compile(r'BTrees\._(..)BTree\.(\1)BTree$').match
+
+class Ghost(object):
+
+  def __init__(self, oid):
+    self._p_oid = oid
+
+class LazyPersistent(object):
+
+  def __call__(self, oid):
+    return Ghost(oid)
+
+class LazyBTree(LazyPersistent):
+  """Fake class to prevent loading too many objects while migrating BTrees
+
+  When we don't migrate recursively, we don't want to migrate values of BTrees,
+  and for performance reasons, we don't even want to load them.
+  So the only remaining way to know if a BTree contains BTrees/Buckets or values
+  is to look at how the state is structured.
+  """
+
+  def getOidList(self, state):
+    if state and len(state) > 1:
+      # return oid of first/next bucket
+      return state[1]._p_oid,
+    return ()
+
+class PickleUpdater(ObjectReader, ObjectWriter, object):
+  """Function-like class to update obsolete references in pickle"""
+
+  def __new__(cls, obj, recursive=False):
+    assert cls.get, "Persistent migration of pickle requires ZODB >= 3.5"
+    self = object.__new__(cls)
+    obj = aq_base(obj)
+    connection = obj._p_jar
+    ObjectReader.__init__(self, connection, connection._cache,
+                                connection._db.classFactory)
+    ObjectWriter.__init__(self, obj)
+    migrated_oid_set = set()
+    oid_set = set((obj._p_oid,))
+    while oid_set:
+      oid = oid_set.pop()
+      obj = self.get(oid)
+      obj._p_activate()
+      klass = obj.__class__
+      self.lazy = None
+      if not recursive:
+        _setOb = getattr(klass, '_setOb', None)
+        if _setOb:
+          if isinstance(_setOb, WorkflowMethod):
+            _setOb = _setOb._m
+          if _setOb.im_func is OFS_Folder._setOb.im_func:
+            self.lazy = Ghost
+        elif klass.__module__[:7] == 'BTrees.' and klass.__name__ != 'Length':
+          self.lazy = LazyBTree()
+      self.oid_dict = {}
+      self.oid_set = set()
+      p, serial = self._conn._storage.load(oid, '')
+      unpickler = self._get_unpickler(p)
+      def find_global(*args):
+        self.do_migrate = args != (klass.__module__, klass.__name__) and \
+                          not isOldBTree('%s.%s' % args)
+        unpickler.find_global = self._get_class
+        return self._get_class(*args)
+      unpickler.find_global = find_global
+      unpickler.load() # class
+      state = unpickler.load()
+      if isinstance(self.lazy, LazyPersistent):
+        self.oid_set.update(self.lazy.getOidList(state))
+      migrated_oid_set.add(oid)
+      oid_set |= self.oid_set - migrated_oid_set
+      self.oid_set = None
+      if self.do_migrate:
+        LOG('PickleUpdater', DEBUG, 'migrate %r (%r)' % (obj, klass))
+        self.setGhostState(obj, self.serialize(obj))
+        obj._p_changed = 1
+
+  get = getattr(ObjectReader, 'load_oid', None)
+
+  def getOid(self, obj):
+    if isinstance(obj, (Persistent, type, wref.WeakRef)):
+      return getattr(obj, '_p_oid', None)
+
+  def load_oid(self, oid):
+    if self.oid_set is not None:
+      if self.lazy:
+        return self.lazy(oid)
+      self.oid_set.add(oid)
+    return self.get(oid)
+
+  def load_persistent(self, oid, klass):
+    obj = ObjectReader.load_persistent(self, oid, klass)
+    if self.oid_set is not None:
+      if not self.lazy:
+        self.oid_set.add(oid)
+      obj._p_activate()
+      self.oid_dict[oid] = oid_klass = ObjectWriter.persistent_id(self, obj)
+      if oid_klass != (oid, klass):
+        self.do_migrate = True
+    return obj
+
+  def persistent_id(self, obj):
+    assert type(obj) is not Ghost
+    oid = self.getOid(obj)
+    if type(oid) is str:
+      try:
+        return self.oid_dict[oid]
+      except KeyError:
+        obj._p_activate()
+    return ObjectWriter.persistent_id(self, obj)
+
+if 1:
+  from Products.ERP5Type.Core.Folder import Folder
+  from Products.ERP5.Tool.CategoryTool import CategoryTool
+
+  Base__setstate__ = Base.__setstate__
+
+  def __setstate__(self, value):
+    klass = self.__class__
+
+    if klass.__module__ in ('erp5.portal_type', 'erp5.temp_portal_type'):
+      return Base__setstate__(self, value)
+    try:
+      portal_type = value.get('portal_type') or klass.portal_type
+    except AttributeError:
+      LOG('ERP5Type', PROBLEM,
+          "no portal type was found for %r (class %s)" % (self, klass))
+      return Base__setstate__(self, value)
+    if portal_type == 'Dummy Class Tool':
+      return Base__setstate__(self, value)
+    # proceed with migration
+    self._fixPortalTypeBeforeMigration(portal_type)
+    import erp5.portal_type
+    newklass = getattr(erp5.portal_type, portal_type)
+    assert self.__class__ is not newklass
+    self.__class__ = newklass
+    self.__setstate__(value)
+    LOG('Base.__setstate__', TRACE, "migrate %r" % self)
+
+  def migrateToPortalTypeClass(self, recursive=False):
+    """Migrate persistently all referenced classes
+
+    When 'recursive' is False, subobjects (read objectValues) are not migrated.
+    So a typical migration of a big folder using activities would be:
+
+      folder.migrateToPortalTypeClass()
+      for obj in folder.objectValues():
+        obj.activate().migrateToPortalTypeClass(True)
+
+    Note however this pattern does not work for HBTrees, because sub-btrees are
+    treated like subobjects for PickleUpdater.
+    """
+    PickleUpdater(self, recursive)
+
+  Base.__setstate__ = __setstate__
+  Folder.__setstate__ = CategoryTool.__setstate__ = __setstate__
+  Base._fixPortalTypeBeforeMigration = lambda self, portal_type: None
+  Base.migrateToPortalTypeClass = migrateToPortalTypeClass
+  Base.security.declareProtected(Permissions.ManagePortal,
+                                 'migrateToPortalTypeClass')
+
+else:
+  __setstate__ = None

product/ERP5Type/dynamic/portal_type_class.py

@@ -183,11 +183,14 @@ def generatePortalTypeClass(site, portal_type_name):
     raise AttributeError('Document class is not defined on Portal Type %s' \
             % portal_type_name)
 
-  type_class_path = document_class_registry.get(type_class)
-  if type_class_path is None:
-    raise AttributeError('Document class %s has not been registered:' \
-                         ' cannot import it as base of Portal Type %s' \
-                         % (type_class, portal_type_name))
+  if '.' in type_class:
+    type_class_path = type_class
+  else:
+    type_class_path = document_class_registry.get(type_class)
+    if type_class_path is None:
+      raise AttributeError('Document class %s has not been registered:'
+                           ' cannot import it as base of Portal Type %s'
+                           % (type_class, portal_type_name))
 
   klass = _importClass(type_class_path)
 
@@ -338,6 +341,7 @@ def synchronizeDynamicModules(context, force=False):
       bootstrap = None
       from Products.ERP5Type.Tool.PropertySheetTool import PropertySheetTool
       from Products.ERP5Type.Tool.TypesTool import TypesTool
+      import erp5.portal_type
       for tool_class in TypesTool, PropertySheetTool:
         # if the instance has no property sheet tool, or incomplete
         # property sheets, we need to import some data to bootstrap
@@ -345,10 +349,6 @@ def synchronizeDynamicModules(context, force=False):
         tool_id = tool_class.id
         tool = getattr(portal, tool_id, None)
         if tool is None:
-          # Create a "non-migrated" (types) tool, so that
-          # ERP5Site.migrateToPortalTypeClass doesn't think there nothing to do.
-          # On the other hand, we must make sure TypesTool._bootstrap installs
-          # the needed portal types in order to migrate this bootstrap tool.
           tool = tool_class()
           try:
             portal._setObject(tool_id, tool, set_owner=False, suppress_events=True)
@@ -366,18 +366,16 @@ def synchronizeDynamicModules(context, force=False):
         try:
           os.chdir(bootstrap)
           tool._bootstrap()
+          tool.__class__ = getattr(erp5.portal_type, tool.portal_type)
         finally:
           os.chdir(cwd)
 
-      if bootstrap:
-        if not getattr(portal, '_v_bootstrapping', False):
-          LOG('ERP5Site', INFO, 'Transition successful, please update your'
-              ' business templates')
-        # XXX: if some portal types are missing, for instance
-        # if some Tools have no portal types, this is likely to fail with an
-        # error. On the other hand, we can't proceed without this change,
-        # and if we dont import the xml, the instance wont start.
-        portal.migrateToPortalTypeClass()
+      if bootstrap and not getattr(portal, '_v_bootstrapping', False):
+        from Products.ERP5Type.dynamic.persistent_migration import PickleUpdater
+        if PickleUpdater.get:
+          portal.migrateToPortalTypeClass()
+        LOG('ERP5Site', INFO, 'Transition successful, please update your'
+            ' business templates')
 
       _bootstrapped.add(portal.id)
 

product/ERP5Type/tests/testDynamicClassGeneration.py

@@ -28,9 +28,11 @@
 #
 ##############################################################################
 
+import gc
 import unittest
 import transaction
 
+from persistent import Persistent
 from Products.ERP5Type.dynamic.portal_type_class import synchronizeDynamicModules
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from Products.ERP5Type.tests.backportUnittest import expectedFailure, skip
@@ -42,75 +44,101 @@ class TestPortalTypeClass(ERP5TypeTestCase):
   def getBusinessTemplateList(self):
     return 'erp5_base',
 
-  def testImportNonMigratedPerson(self):
+  def testMigrateOldObject(self):
     """
-    Import a .xml containing a Person created with an old
-    Products.ERP5Type.Document.Person.Person type
+    Check migration of persistent objects with old classes
+    like Products.ERP5(Type).Document.Person.Person
     """
+    from Products.ERP5Type.Document.Person import Person
     person_module = self.portal.person_module
+    connection = person_module._p_jar
+    newId = self.portal.person_module.generateNewId
+
+    def unload(id):
+      oid = person_module._tree[id]._p_oid
+      person_module._tree._p_deactivate()
+      connection._cache.invalidate(oid)
+      gc.collect()
+      # make sure we manage to remove the object from memory
+      assert connection._cache.get(oid, None) is None
+      return oid
+
+    def check(migrated):
+      klass = old_object.__class__
+      self.assertEqual(klass.__module__,
+        migrated and 'erp5.portal_type' or 'Products.ERP5.Document.Person')
+      self.assertEqual(klass.__name__, 'Person')
+      self.assertEqual(klass.__setstate__ is Persistent.__setstate__, migrated)
+
+    # Import a .xml containing a Person created with an old
+    # Products.ERP5Type.Document.Person.Person type
     self.importObjectFromFile(person_module, 'non_migrated_person.xml')
     transaction.commit()
+    unload('non_migrated_person')
+    old_object = person_module.non_migrated_person
+    # object unpickling should have instanciated a new style object directly
+    check(1)
 
-    non_migrated_person = person_module.non_migrated_person
-    # check that object unpickling instanciated a new style object
-    person_class = self.portal.portal_types.getPortalTypeClass('Person')
-    self.assertEquals(non_migrated_person.__class__, person_class)
-
-  @expectedFailure
-  def testImportNonMigratedDocumentUsingContentClass(self):
-    """
-    Import a .xml containing a Base Type with old Document path
-    Products.ERP5Type.ERP5Type.ERP5TypeInformation
-
-    This Document class is different because it's a content_class,
-    i.e. it was not in Products.ERP5Type.Document.** but was
-    imported directly as such.
-    """
-    self.importObjectFromFile(self.portal, 'Category.xml')
-    transaction.commit()
-
-    non_migrated_type = self.portal.Category
-    # check that object unpickling instanciated a new style object
-    base_type_class = self.portal.portal_types.getPortalTypeClass('Base Type')
-    self.assertEquals(non_migrated_type.__class__, base_type_class)
-
-  def testMigrateOldObjectFromZODB(self):
-    """
-    Load an object with ERP5Type.Document.Person.Person from the ZODB
-    and check that migration works well
-    """
-    from Products.ERP5Type.Document.Person import Person
-
-    # remove temporarily the migration
-    from Products.ERP5Type.Utils import PersistentMigrationMixin
-    PersistentMigrationMixin.migrate = 0
-
-    person_module = self.getPortal().person_module
-    obj_id = "this_object_is_old"
-    old_object = Person(obj_id)
-    person_module._setObject(obj_id, old_object)
-    old_object = person_module._getOb(obj_id)
-
+    obj_id = newId()
+    person_module._setObject(obj_id, Person(obj_id))
     transaction.commit()
-    self.assertEquals(old_object.__class__.__module__, 'Products.ERP5Type.Document.Person')
-    self.assertEquals(old_object.__class__.__name__, 'Person')
-
-    self.assertTrue(hasattr(old_object.__class__, '__setstate__'))
-
-    # unload/deactivate the object
-    old_object._p_invalidate()
-
+    unload(obj_id)
+    old_object = person_module[obj_id]
     # From now on, everything happens as if the object was a old, non-migrated
-    # object with an old Products.ERP5Type.Document.Person.Person
-
-    # now turn on migration
-    PersistentMigrationMixin.migrate = 1
-
+    # object with an old Products.ERP5(Type).Document.Person.Person
+    check(0)
     # reload the object
     old_object._p_activate()
+    check(1)
+    # automatic migration is not persistent
+    old_object = None
+    # (note we get back the object directly from its oid to make sure we test
+    # the class its pickle and not the one in its container)
+    old_object = connection.get(unload(obj_id))
+    check(0)
+
+    try:
+      from ZODB import __version__
+
+    except ImportError: # recent ZODB
+      # Test persistent migration
+      old_object.migrateToPortalTypeClass()
+      old_object = None
+      transaction.commit()
+      old_object = connection.get(unload(obj_id))
+      check(1)
+      # but the container still have the old class
+      old_object = None
+      unload(obj_id)
+      old_object = person_module[obj_id]
+      check(0)
+
+      # Test persistent migration of containers
+      obj_id = newId()
+      person_module._setObject(obj_id, Person(obj_id))
+      transaction.commit()
+      unload(obj_id)
+      person_module.migrateToPortalTypeClass()
+      transaction.commit()
+      unload(obj_id)
+      old_object = person_module[obj_id]
+      check(1)
+      # not recursive by default
+      old_object = None
+      old_object = connection.get(unload(obj_id))
+      check(0)
+
+      # Test recursive migration
+      old_object = None
+      unload(obj_id)
+      person_module.migrateToPortalTypeClass(True)
+      transaction.commit()
+      old_object = connection.get(unload(obj_id))
+      check(1)
 
-    self.assertEquals(old_object.__class__.__module__, 'erp5.portal_type')
-    self.assertEquals(old_object.__class__.__name__, 'Person')
+    else: # Zope 2.8
+      # compatibility code not implemented
+      self.assertRaises(AssertionError, old_object.migrateToPortalTypeClass)
 
   def testChangeMixin(self):
     """