Commit 6305ce9d authored by Roque's avatar Roque

Improvements on folder migration

See merge request nexedi/erp5!1394
parents 6058e5f3 7ca55964
...@@ -481,6 +481,34 @@ class TestFolderMigration(ERP5TypeTestCase, LogInterceptor): ...@@ -481,6 +481,34 @@ class TestFolderMigration(ERP5TypeTestCase, LogInterceptor):
self.assertEqual(self.folder.isBTree(), False) self.assertEqual(self.folder.isBTree(), False)
self.assertEqual(self.folder.isHBTree(), True) self.assertEqual(self.folder.isHBTree(), True)
def test_15_checkMigrationWorksIfIdsDontChange(self):
"""
migrate folder using a script that leaves some objects with same ids
"""
# Create some objects
self.assertEqual(self.folder.getIdGenerator(), '')
self.assertEqual(len(self.folder), 0)
obj1 = self.newContent()
self.assertEqual(obj1.getId(), '1')
obj2 = self.newContent()
self.assertEqual(obj2.getId(), '2')
obj3 = self.newContent(id='custom-id')
self.assertEqual(obj3.getId(), 'custom-id')
self.tic()
# call migration script Base_generateIdFromCreationDate that only changes int ids
self.folder.migrateToHBTree(migration_generate_id_method="Base_generateIdFromCreationDate",
new_generate_id_method="_generatePerDayId")
self.tic()
# check object ids
from DateTime import DateTime
date = DateTime().Date()
date = date.replace("/", "")
#1 y 2 should have new format id (because old ids were int)
self.assertEqual(obj1.getId(), '%s-1' % date)
self.assertEqual(obj2.getId(), '%s-2' % date)
#3 should have the same old id
self.assertEqual(obj3.getId(), 'custom-id')
def test_suite(): def test_suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestFolderMigration)) suite.addTest(unittest.makeSuite(TestFolderMigration))
......
# retrieve a date from object and return the new id
if obj is None:
obj = context
# don't change custom (non-int) ids
old_id = obj.getId()
if not old_id.isdigit():
return old_id
date = obj.getCreationDate()
if date is None:
from DateTime import DateTime
date = DateTime()
date = date.Date().replace('/', '')
return "%s-%s" %(date, old_id)
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>obj=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_generateIdFromCreationDate</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -4,5 +4,6 @@ method = getattr(context, method) ...@@ -4,5 +4,6 @@ method = getattr(context, method)
for id_ in id_list: for id_ in id_list:
ob = folder.get(id_) ob = folder.get(id_)
new_id = method(ob) new_id = method(ob)
if new_id != id_:
ob.setDefaultActivateParameterDict(activate_kw) ob.setDefaultActivateParameterDict(activate_kw)
ob.setId(new_id) ob.setId(new_id)
...@@ -795,14 +795,13 @@ class Folder(FolderMixIn, CopyContainer, ObjectManager, Base, OFSFolder2, CMFBTr ...@@ -795,14 +795,13 @@ class Folder(FolderMixIn, CopyContainer, ObjectManager, Base, OFSFolder2, CMFBTr
return self._folder_handler == HBTREE_HANDLER return self._folder_handler == HBTREE_HANDLER
security.declareProtected( Permissions.ManagePortal, 'migrateToHBTree' ) security.declareProtected( Permissions.ManagePortal, 'migrateToHBTree' )
def migrateToHBTree(self, migration_generate_id_method=None, new_generate_id_method='_generatePerDayId', REQUEST=None): def migrateToHBTree(self, migration_generate_id_method=None, new_generate_id_method='_generatePerDayId', bundle_count=10, REQUEST=None):
""" """
Function to migrate from a BTree folder to HBTree folder. Function to migrate from a BTree folder to HBTree folder.
It will first call setId on all folder objects to have right id It will first call setId on all folder objects to have right id
to be used with an hbtreefolder. to be used with an hbtreefolder.
Then it will migrate foder from btree to hbtree. Then it will migrate foder from btree to hbtree.
""" """
BUNDLE_COUNT = 10
# if folder is already migrated or migration process is in progress # if folder is already migrated or migration process is in progress
# do not do anything beside logging # do not do anything beside logging
...@@ -822,15 +821,15 @@ class Folder(FolderMixIn, CopyContainer, ObjectManager, Base, OFSFolder2, CMFBTr ...@@ -822,15 +821,15 @@ class Folder(FolderMixIn, CopyContainer, ObjectManager, Base, OFSFolder2, CMFBTr
tag = "%s/%s/migrate" %(self.getId(),migration_generate_id_method) tag = "%s/%s/migrate" %(self.getId(),migration_generate_id_method)
id_list = list(self.objectIds()) id_list = list(self.objectIds())
# set new id by bundle # set new id by bundle
for x in xrange(len(self) / BUNDLE_COUNT): for x in xrange(len(self) / bundle_count):
self.activate(activity="SQLQueue", tag=tag).ERP5Site_setNewIdPerBundle( self.activate(activity="SQLQueue", tag=tag, priority=3, serialization_tag='ERP5Site_setNewIdPerBundle').ERP5Site_setNewIdPerBundle(
self.getPath(), self.getPath(),
id_list[x*BUNDLE_COUNT:(x+1)*BUNDLE_COUNT], id_list[x*bundle_count:(x+1)*bundle_count],
migration_generate_id_method, tag) migration_generate_id_method, tag)
remaining_id_count = len(self) % BUNDLE_COUNT remaining_id_count = len(self) % bundle_count
if remaining_id_count: if remaining_id_count:
self.activate(activity="SQLQueue", tag=tag).ERP5Site_setNewIdPerBundle( self.activate(activity="SQLQueue", tag=tag, priority=3, serialization_tag='ERP5Site_setNewIdPerBundle').ERP5Site_setNewIdPerBundle(
self.getPath(), self.getPath(),
id_list[-remaining_id_count:], id_list[-remaining_id_count:],
migration_generate_id_method, tag) migration_generate_id_method, tag)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment