Commit e508b2eb authored by Aurel's avatar Aurel

add integration of HBTreeFolder into ERP5 Folder class

add a new method to generate id per day
add  unit test for migration from one btree to an hbtree


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@16295 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 0128fc04
......@@ -46,9 +46,16 @@ try:
from Products.CMFCore.CMFBTreeFolder import CMFBTreeFolder
except ImportError:
from Products.BTreeFolder2.CMFBTreeFolder import CMFBTreeFolder
from Products.BTreeFolder2.BTreeFolder2 import BTreeFolder2Base
from Products.HBTreeFolder2.CMFHBTreeFolder import CMFHBTreeFolder
from Products.HBTreeFolder2.HBTreeFolder2 import HBTreeFolder2Base
from AccessControl import getSecurityManager
from Products.ERP5Type import Permissions
from MethodObject import Method
from DateTime import DateTime
from random import randint
import os
......@@ -140,6 +147,22 @@ class FolderMixIn(ExtensionClass.Base):
raise TypeError, 'deleteContent only accepts string or list, '\
'not %s' % type(id)
def _generatePerDayId(self):
"""
Generate id base on date, useful for HBTreeFolder
"""
current_date = str(DateTime().Date()).replace("/", "")
try:
my_id = int(self.getLastId())
except TypeError:
my_id = 1
while self.hasContent("%s-%s" %(current_date, my_id)):
my_id = my_id + 1
my_id = str(my_id)
self._setLastId(my_id) # Make sure no reindexing happens
return "%s-%s" %(current_date, my_id)
def _generateRandomId(self):
"""
Generate a random Id.
......@@ -295,7 +318,23 @@ class FolderMixIn(ExtensionClass.Base):
"""
return self.countFolder(**kw)[0][0]
class Folder( CopyContainer, CMFBTreeFolder, Base, FolderMixIn, WebDAVFolder):
class FolderMethodWrapper(Method):
"""
This a wrapper between folder method and folder type method
"""
def __init__(self, method_id):
self.method_id = method_id
def __call__(self, folder, *args, **kw):
try:
method = getattr(folder._plugin, self.method_id)
return method(folder, *args, **kw)
except AttributeError:
LOG("Folder call %s" %self.method_id, WARNING, "raise AttributeError on %s with plugin %s" %(folder, folder._plugin))
pass
class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn, WebDAVFolder):
"""
A Folder is a subclass of Base but not of XMLObject.
Folders are not considered as documents and are therefore
......@@ -358,12 +397,164 @@ class Folder( CopyContainer, CMFBTreeFolder, Base, FolderMixIn, WebDAVFolder):
# XXX Prevent inheritance from PortalFolderBase
description = None
# Per default we use BTree folder
_isHBTree = False
_isBTree = True
_plugin = None # plugin store the folder type class use by the folder
# Overload __init__ so that we do not take into account title
# This is required for test_23_titleIsNotDefinedByDefault
def __init__(self, id):
self.id = id
BTreeFolder2Base.__init__(self, id)
# do not do any other initialisation here,
# wait for new content call to do init
method_id_list = ["initBTrees", "manage_fixCount", "manage_cleanup",
"getBatchObjectListing", "manage_object_workspace",
"tpValues", "objectCount", "has_key", "objectIds",
"objectItems", "objectMap", "objectIds_d", "objectMap_d",
"keys", "values", "items", "hasObject", "get", "generateId",
"__len__", "allowedContentTypes", "_delOb",
"_getOb", "_setObject", "_initBTrees", "_populateFromFolder",
"_fixCount", "_cleanup", "_setOb", "_checkId", "_delObject",]
def _generatePluginMethod(self):
""" Will generate alias to Tree method depending
on configuration """
for method_id in self.method_id_list:
method_wrapper = FolderMethodWrapper(method_id)
setattr(self, method_id, method_wrapper)
def newContent(self, *args, **kw):
""" Create a new content """
# Create data structure if none present
if self._plugin is None:
if self._isHBTree:
self._plugin = CMFHBTreeFolder
elif self._isBTree:
self._plugin = CMFBTreeFolder
else:
raise ValueError, "No plugin defined"
self._generatePluginMethod()
self._plugin.__init__(self, self.id)
return FolderMixIn.newContent(self, *args, **kw)
security.declareProtected(Permissions.View, 'isBTree')
def isBTree(self):
""" Return if folder is a BTree or not """
return self._isBTree
security.declareProtected(Permissions.View, 'isHBTree')
def isHBTree(self):
""" Return if folder is a HBTree or not """
return self._isHBTree
def _setHBTree(self,):
""" Define this folder use HBTree stucture """
self._isHBTree = True
self._isBTree = False
def _setBTree(self,):
""" Define this folder use BTree stucture """
self._isHBTree = False
self._isBTree = True
def hashId(self, id):
""" id hashing can be override with a script """
script = self._getTypeBasedMethod('hashId')
if script is not None:
return script(self, id)
return self._plugin.hashId(self, id)
security.declareProtected( Permissions.ManagePortal, 'migrateToHBTree' )
def migrateToHBTree(self, migration_generate_id_method=None, new_generate_id_method=None, REQUEST=None):
"""
Function to migrate from a BTree folder to HBTree folder.
It will first call setId on all folder objects to have right id
to be used with an hbtreefolder.
Then it will migrate foder from btree to hbtree.
"""
if self._plugin is None:
# make sure to have all method right defined
self._plugin = CMFBTreeFolder
self._generatePluginMethod()
BUNDLE_COUNT = 10
# we may want to change all objects ids before migrating to new folder type
if migration_generate_id_method not in (None, ''):
# set new id generator here so that object created while migration
# got a right id
self.setIdGenerator(new_generate_id_method)
tag = "%s/%s/migrate" %(self.getId(),migration_generate_id_method)
id_list = list(self.objectIds())
# set new id by bundle
for x in xrange(len(self) / BUNDLE_COUNT):
self.activate(activity="SQLQueue", tag=tag).ERP5Site_setNewIdPerBundle(
self.getPath(),
id_list[x*BUNDLE_COUNT:(x+1)*BUNDLE_COUNT],
migration_generate_id_method, tag)
remaining_id_count = len(self) % BUNDLE_COUNT
if remaining_id_count:
self.activate(activity="SQLQueue", tag=tag).ERP5Site_setNewIdPerBundle(
self.getPath(),
id_list[-remaining_id_count:],
migration_generate_id_method, tag)
else:
tag = ''
# copy from btree to hbtree
self.activate(activity="SQLQueue", after_tag=tag)._launchCopyObjectToHBTree(tag)
if REQUEST is not None:
psm = N_('Migration to HBTree is running.',)
ret_url = '%s/%s?portal_status_message=%s' % \
(self.absolute_url(),
REQUEST.get('form_id', 'view'), psm)
return REQUEST.RESPONSE.redirect( ret_url )
def _finishCopyObjectToHBTree(self):
"""
Remove remaining attributes from previous btree
and migration
"""
delattr(self, "_tree")
delattr(self, "_former_plugin")
def _launchCopyObjectToHBTree(self, tag):
"""
Launch activity per bundle to move object
from a btree to an hbtree
"""
# migrate folder from btree to hbtree
id_list = list(self.objectIds())
self._former_plugin = self._plugin
self._setHBTree()
self._plugin = CMFHBTreeFolder
self._generatePluginMethod()
self._plugin.__init__(self, self.id)
# launch activity per bundle to copy/paste to hbtree
BUNDLE_COUNT = 100
for x in xrange(len(id_list) / BUNDLE_COUNT):
self.activate(activity="SQLQueue", tag=tag)._copyObjectToHBTree(
id_list=id_list[x*BUNDLE_COUNT:(x+1)*BUNDLE_COUNT],)
remaining_id_count = len(id_list) % BUNDLE_COUNT
if remaining_id_count:
self.activate(activity="SQLQueue", tag=tag)._copyObjectToHBTree(
id_list=id_list[-remaining_id_count:],)
# remove uneeded attribute
self.activate(activity="SQLQueue", after_tag=tag)._finishCopyObjectToHBTree()
def _copyObjectToHBTree(self, id_list=None,):
"""
Move object from a btree container to
a hbtree one
"""
getOb = self._former_plugin._getOb
setOb = self._setOb
for id in id_list:
obj = getOb(self, id)
setOb(id, obj)
# Override Zope default by folder id generation
def _get_id(self, id):
......@@ -394,7 +585,10 @@ class Folder( CopyContainer, CMFBTreeFolder, Base, FolderMixIn, WebDAVFolder):
security.declareProtected(Permissions.View, 'hasContent')
def hasContent(self,id):
return self.hasObject(id)
try:
return self._plugin.hasObject(self, id)
except AttributeError:
return False
security.declareProtected( Permissions.ModifyPortalContent, 'exportAll' )
def exportAll(self,dir=None):
......@@ -727,6 +921,10 @@ class Folder( CopyContainer, CMFBTreeFolder, Base, FolderMixIn, WebDAVFolder):
def _getVisibleAllowedContentTypeList():
hidden_type_list = portal.portal_types.getTypeInfo(self)\
.getHiddenContentTypeList()
try:
return [ ti.id for ti in self._plugin.allowedContentTypes(self)
if ti.id not in hidden_type_list ]
except AttributeError:
return [ ti.id for ti in CMFBTreeFolder.allowedContentTypes(self)
if ti.id not in hidden_type_list ]
......@@ -772,6 +970,9 @@ class Folder( CopyContainer, CMFBTreeFolder, Base, FolderMixIn, WebDAVFolder):
# account i18n into consideration.
# XXX So sorting should be done in skins, after translation is performed.
def compareTypes(a, b): return cmp(a.title or a.id, b.title or b.id)
try:
type_list = self._plugin.allowedContentTypes(self)
except AttributeError:
type_list = CMFBTreeFolder.allowedContentTypes(self)
type_list.sort(compareTypes)
return ['/'.join(x.getPhysicalPath()) for x in type_list]
......@@ -796,7 +997,14 @@ class Folder( CopyContainer, CMFBTreeFolder, Base, FolderMixIn, WebDAVFolder):
view = Base.view
# Aliases
getObjectIds = CMFBTreeFolder.objectIds
security.declareProtected(Permissions.AccessContentsInformation,
'getObjectIds')
def getObjectIds(self, *args, **kw):
try:
self._plugin.objectIds(args, kw)
except AttributeError:
CMFBTreeFolder.objectIds(args, kw)
# Overloading
security.declareProtected( Permissions.AccessContentsInformation,
......@@ -862,7 +1070,7 @@ class Folder( CopyContainer, CMFBTreeFolder, Base, FolderMixIn, WebDAVFolder):
security.declareProtected( Permissions.AccessContentsInformation,
'objectValues' )
def objectValues(self, spec=None, meta_type=None, portal_type=None,
sort_on=None, sort_order=None, **kw):
sort_on=None, sort_order=None, base_id=None,**kw):
"""
Returns a list containing object contained in this folder.
"""
......@@ -876,6 +1084,12 @@ class Folder( CopyContainer, CMFBTreeFolder, Base, FolderMixIn, WebDAVFolder):
except AttributeError:
from Products.BTreeFolder2.BTreeFolder2 import BTreeFolder2Base
BTreeFolder2Base.__init__(self, self.getId())
try:
if self.isBTree():
object_list = self._plugin.objectValues(self, spec=spec)
else:
object_list = self._plugin.objectValues(self, base_id=base_id)
except AttributeError:
object_list = CMFBTreeFolder.objectValues(self, spec=spec)
if portal_type is not None:
if type(portal_type) == type(''):
......@@ -899,6 +1113,9 @@ class Folder( CopyContainer, CMFBTreeFolder, Base, FolderMixIn, WebDAVFolder):
kw['portal_type'] = portal_type
filter = kw.pop('filter', {}) or {}
kw.update(filter)
try:
object_list = self._plugin.contentValues(self, spec=spec, filter=kw)
except AttributeError:
object_list = CMFBTreeFolder.contentValues(self, spec=spec, filter=kw)
object_list = sortValueList(object_list, sort_on, sort_order, **kw)
return object_list
......
##############################################################################
#
# Copyright (c) 2007 Nexedi SARL and Contributors. All Rights Reserved.
# Aurlien Calonne <aurel@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os, sys
if __name__ == '__main__':
execfile(os.path.join(sys.path[0], 'framework.py'))
# Needed in order to have a log file inside the current folder
os.environ['EVENT_LOG_FILE'] = os.path.join(os.getcwd(), 'zLOG.log')
os.environ['EVENT_LOG_SEVERITY'] = '-300'
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from zLOG import LOG
from Products.CMFCore.tests.base.testcase import LogInterceptor
from Products.ERP5Type.tests.utils import createZODBPythonScript
from Products.ERP5Type.ERP5Type import ERP5TypeInformation
from Products.ERP5Type.Cache import clearCache
class TestFolderMigration(ERP5TypeTestCase, LogInterceptor):
# Some helper methods
def getTitle(self):
return "Folder Migration"
def getBusinessTemplateList(self):
"""
Return the list of business templates.
"""
return tuple()
def afterSetUp(self):
"""
Executed before each test_*.
"""
self.login()
self.folder = self.getPortal().newContent(id='TestFolder',
portal_type='Folder')
def beforeTearDown(self):
"""
Executed after each test_*.
"""
self.getPortal().manage_delObjects(ids=[self.folder.getId(),])
clearCache()
def newContent(self):
"""
Create an object in self.folder and return it.
"""
return self.folder.newContent(portal_type='Folder')
def test_01_folderIsBtree(self, quiet=0, run=1):
"""
Test the folder is a BTree
"""
if not run : return
if not quiet:
message = 'Test folderIsBtree'
LOG('Testing... ', 0, message)
self.assertRaises(AttributeError, self.folder.getTreeIdList)
self.assertEqual(self.folder.isBTree(), True)
self.assertEqual(self.folder.isHBTree(), False)
def test_02_migrateFolder(self, quiet=0, run=1):
"""
migrate folder from btree to hbtree
"""
if not run : return
if not quiet:
message = 'Test migrateFolder'
LOG('Testing... ', 0, message)
# Create some objects
self.assertEquals(self.folder.getIdGenerator(), '')
self.assertEquals(len(self.folder), 0)
obj1 = self.newContent()
self.assertEquals(obj1.getId(), '1')
obj2 = self.newContent()
self.assertEquals(obj2.getId(), '2')
obj3 = self.newContent()
self.assertEquals(obj3.getId(), '3')
get_transaction().commit()
self.tic()
# call migration script
self.folder.migrateToHBTree(migration_generate_id_method="Base_generateIdFromStopDate",
new_generate_id_method="_generatePerDayId")
get_transaction().commit()
self.tic()
# check we now have a hbtree
self.assertEqual(self.folder.isBTree(), False)
self.assertEqual(self.folder.isHBTree(), True)
self.assertEqual(len(self.folder.getTreeIdList()), 1)
self.assertEqual(len(self.folder.objectIds()), 3)
# check object ids
from DateTime import DateTime
date = DateTime().Date()
date = date.replace("/", "")
self.assertEquals(obj1.getId(), '%s-1' %date)
self.assertEquals(obj2.getId(), '%s-2' %date)
self.assertEquals(obj3.getId(), '%s-3' %date)
if __name__ == '__main__':
framework()
else:
import unittest
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestFolderMigration))
return suite
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment