Commit 055d0a69 authored by Julien Muchembled's avatar Julien Muchembled

HBTreeFolder2: make object{Ids,Values,Items} really lazy

Lazy* classes from ZCatalog product aren't used anymore because they don't
accept iterators.

Remove objectMap* methods instead of update them because they never worked.

Add assertion to prevent passing a meta type, which was never supported
for this type of folder.
Signed-off-by: Vincent Pelletier's avatarVincent Pelletier <vincent@nexedi.com>
parent cd74aef3
No related merge requests found
...@@ -914,6 +914,7 @@ class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn): ...@@ -914,6 +914,7 @@ class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn):
if self._folder_handler == HBTREE_HANDLER: if self._folder_handler == HBTREE_HANDLER:
if self._htree is None: if self._htree is None:
return [] return []
assert spec is None
if kw.has_key("base_id"): if kw.has_key("base_id"):
return CMFHBTreeFolder.objectIds(self, base_id=kw["base_id"]) return CMFHBTreeFolder.objectIds(self, base_id=kw["base_id"])
else: else:
...@@ -927,6 +928,7 @@ class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn): ...@@ -927,6 +928,7 @@ class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn):
if self._folder_handler == HBTREE_HANDLER: if self._folder_handler == HBTREE_HANDLER:
if self._htree is None: if self._htree is None:
return [] return []
assert spec is None
if kw.has_key("base_id"): if kw.has_key("base_id"):
return CMFHBTreeFolder.objectItems(self, base_id=kw["base_id"]) return CMFHBTreeFolder.objectItems(self, base_id=kw["base_id"])
else: else:
...@@ -936,16 +938,6 @@ class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn): ...@@ -936,16 +938,6 @@ class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn):
return [] return []
return CMFBTreeFolder.objectItems(self, spec) return CMFBTreeFolder.objectItems(self, spec)
def objectMap(self):
if self._folder_handler == HBTREE_HANDLER:
if self._htree is None:
return []
return CMFHBTreeFolder.objectMap(self)
else:
if self._tree is None:
return []
return CMFBTreeFolder.objectIMap(self)
def objectIds_d(self, t=None): def objectIds_d(self, t=None):
if self._folder_handler == HBTREE_HANDLER: if self._folder_handler == HBTREE_HANDLER:
if self._htree is None: if self._htree is None:
...@@ -956,16 +948,6 @@ class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn): ...@@ -956,16 +948,6 @@ class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn):
return {} return {}
return CMFBTreeFolder.objectIds_d(self, t) return CMFBTreeFolder.objectIds_d(self, t)
def objectMap_d(self, t=None):
if self._folder_handler == HBTREE_HANDLER:
if self._htree is None:
return {}
return CMFHBTreeFolder.objectMap_d(self, t)
else:
if self._tree is None:
return {}
return CMFBTreeFolder.objectMap_d(self, t)
def _checkId(self, id, allow_dup=0): def _checkId(self, id, allow_dup=0):
if self._folder_handler == HBTREE_HANDLER: if self._folder_handler == HBTREE_HANDLER:
return CMFHBTreeFolder._checkId(self, id, allow_dup) return CMFHBTreeFolder._checkId(self, id, allow_dup)
...@@ -1545,6 +1527,7 @@ class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn): ...@@ -1545,6 +1527,7 @@ class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn):
elif self._folder_handler == HBTREE_HANDLER: elif self._folder_handler == HBTREE_HANDLER:
if self._htree is None: if self._htree is None:
return [] return []
assert spec is None
if 'base_id' in kw: if 'base_id' in kw:
object_list = CMFHBTreeFolder.objectValues(self, base_id=kw['base_id']) object_list = CMFHBTreeFolder.objectValues(self, base_id=kw['base_id'])
else: else:
......
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
import sys import sys
from cgi import escape from cgi import escape
from itertools import chain, islice
from urllib import quote from urllib import quote
from random import randint from random import randint
from types import StringType from types import StringType
...@@ -32,7 +33,6 @@ from AccessControl import getSecurityManager, ClassSecurityInfo ...@@ -32,7 +33,6 @@ from AccessControl import getSecurityManager, ClassSecurityInfo
from AccessControl.Permissions import access_contents_information, \ from AccessControl.Permissions import access_contents_information, \
view_management_screens view_management_screens
from zLOG import LOG, INFO, ERROR, WARNING from zLOG import LOG, INFO, ERROR, WARNING
from Products.ZCatalog.Lazy import LazyMap, LazyFilter, LazyCat, LazyValues
manage_addHBTreeFolder2Form = DTMLFile('folderAdd', globals()) manage_addHBTreeFolder2Form = DTMLFile('folderAdd', globals())
...@@ -67,6 +67,69 @@ class ExhaustedUniqueIdsError (Exception): ...@@ -67,6 +67,69 @@ class ExhaustedUniqueIdsError (Exception):
pass pass
class HBTreeObjectIds(object):
_index = float('inf')
def __init__(self, tree, base_id=_marker):
self._tree = tree
if base_id is _marker:
tree_id_list = tree.getTreeIdList()
self._count = tree._count
else:
tree_id_list = base_id,
check = tree._checkObjectId
self._keys = lambda: (x for base_id in tree_id_list
for x in (tree._htree if base_id is None else
tree._getTree(base_id)).keys()
if check((base_id, x)))
def _count(self):
count = sum(1 for x in self._keys())
self._count = lambda: count
return count
def __len__(self):
return self._count()
def __iter__(self):
return self._keys()
def __getitem__(self, item):
if item < 0:
item += self._count()
i = self._index
self._index = item + 1
i = item - i
try:
if i < 0:
self._ikeys = keys = self._keys()
return islice(keys, item, None).next()
return (islice(self._ikeys, i, None) if i else self._ikeys).next()
except StopIteration:
del self._index, self._ikeys
raise IndexError
class HBTreeObjectItems(HBTreeObjectIds):
def __iter__(self):
getOb = self._tree._getOb
return ((x, getOb(x)) for x in self._keys())
def __getitem__(self, item):
object_id = HBTreeObjectIds.__getitem__(self, item)
return object_id, self._tree._getOb(object_id)
class HBTreeObjectValues(HBTreeObjectIds):
def __iter__(self):
getOb = self._tree._getOb
return (getOb(x) for x in self._keys())
def __getitem__(self, item):
return self._tree._getOb(HBTreeObjectIds.__getitem__(self, item))
class HBTreeFolder2Base (Persistent): class HBTreeFolder2Base (Persistent):
"""Base for BTree-based folders. """Base for BTree-based folders.
""" """
...@@ -198,6 +261,7 @@ class HBTreeFolder2Base (Persistent): ...@@ -198,6 +261,7 @@ class HBTreeFolder2Base (Persistent):
def hashId(self, id): def hashId(self, id):
"""Return a tuple of ids """Return a tuple of ids
""" """
# XXX: why tolerate non-string ids ?
id_list = str(id).split(H_SEPARATOR) # We use '-' as the separator by default id_list = str(id).split(H_SEPARATOR) # We use '-' as the separator by default
if len(id_list) > 1: if len(id_list) > 1:
return tuple(id_list) return tuple(id_list)
...@@ -288,7 +352,6 @@ class HBTreeFolder2Base (Persistent): ...@@ -288,7 +352,6 @@ class HBTreeFolder2Base (Persistent):
b_count = int(REQUEST.get('b_count', 1000)) b_count = int(REQUEST.get('b_count', 1000))
b_end = b_start + b_count - 1 b_end = b_start + b_count - 1
url = self.absolute_url() + '/manage_main' url = self.absolute_url() + '/manage_main'
idlist = self.objectIds() # Pre-sorted.
count = self.objectCount() count = self.objectCount()
if b_end < count: if b_end < count:
...@@ -302,10 +365,9 @@ class HBTreeFolder2Base (Persistent): ...@@ -302,10 +365,9 @@ class HBTreeFolder2Base (Persistent):
else: else:
prev_url = '' prev_url = ''
formatted = [] formatted = [listtext0 % pref_rows]
formatted.append(listtext0 % pref_rows) for optID in islice(self.objectIds(), b_start - 1, b_end):
for i in range(b_start - 1, b_end): optID = escape(optID)
optID = escape(idlist[i])
formatted.append(listtext1 % (escape(optID, quote=1), optID)) formatted.append(listtext1 % (escape(optID, quote=1), optID))
formatted.append(listtext2) formatted.append(listtext2)
return {'b_start': b_start, 'b_end': b_end, return {'b_start': b_start, 'b_end': b_end,
...@@ -404,7 +466,7 @@ class HBTreeFolder2Base (Persistent): ...@@ -404,7 +466,7 @@ class HBTreeFolder2Base (Persistent):
""" Return a list of subtree ids """ Return a list of subtree ids
""" """
tree = self._getTree(base_id=base_id) tree = self._getTree(base_id=base_id)
return [x for x in self._htree.keys() if isinstance(self._htree[x], OOBTree)] return [k for k, v in self._htree.items() if isinstance(v, OOBTree)]
def _getTree(self, base_id): def _getTree(self, base_id):
...@@ -448,31 +510,6 @@ class HBTreeFolder2Base (Persistent): ...@@ -448,31 +510,6 @@ class HBTreeFolder2Base (Persistent):
self._tree_list[tree] = None self._tree_list[tree] = None
return sorted(self._tree_list.keys()) return sorted(self._tree_list.keys())
def _treeObjectValues(self, base_id=None):
""" return object values for a given btree
"""
if base_id is not None:
return LazyFilter(self._isNotBTree, self._getTree("%s" %base_id).values())
else:
return LazyFilter(self._isNotBTree, self._htree.values())
def _treeObjectIds(self, base_id=None):
""" return object ids for a given btree
"""
if base_id is not None:
return LazyValues(LazyFilter(self._checkObjectId, [(base_id, x) for x in self._getTree("%s" %base_id).keys()]))
else:
return LazyValues(LazyFilter(self._checkObjectId, [(base_id, x) for x in self._htree.keys()]))
def _isNotBTree(self, obj):
""" test object is not a btree
"""
if isinstance(obj, OOBTree):
return False
else:
return True
def _checkObjectId(self, ids): def _checkObjectId(self, ids):
""" test id is not in btree id list """ test id is not in btree id list
""" """
...@@ -483,36 +520,21 @@ class HBTreeFolder2Base (Persistent): ...@@ -483,36 +520,21 @@ class HBTreeFolder2Base (Persistent):
security.declareProtected(access_contents_information, security.declareProtected(access_contents_information,
'objectValues') 'objectValues')
def objectValues(self, base_id=_marker, spec=None): def objectValues(self, base_id=_marker):
return LazyMap(self._getOb, self.objectIds(base_id=base_id)) return HBTreeObjectValues(self, base_id)
security.declareProtected(access_contents_information, security.declareProtected(access_contents_information,
'objectIds') 'objectIds')
def objectIds(self, base_id=_marker, spec=None): def objectIds(self, base_id=_marker):
if base_id is _marker: return HBTreeObjectIds(self, base_id)
return LazyCat(LazyMap(self._treeObjectIds, self.getTreeIdList()))
else:
return self._treeObjectIds(base_id=base_id)
security.declareProtected(access_contents_information, security.declareProtected(access_contents_information,
'objectItems') 'objectItems')
def objectItems(self, base_id=_marker, spec=None): def objectItems(self, base_id=_marker):
# Returns a list of (id, subobject) tuples of the current object. # Returns a list of (id, subobject) tuples of the current object.
# If 'spec' is specified, returns only objects whose meta_type match # If 'spec' is specified, returns only objects whose meta_type match
# 'spec' # 'spec'
return LazyMap(lambda id, _getOb=self._getOb: (id, _getOb(id)), return HBTreeObjectItems(self, base_id)
self.objectIds(base_id=base_id))
security.declareProtected(access_contents_information,
'objectMap')
def objectMap(self):
# Returns a tuple of mappings containing subobject meta-data.
return LazyMap(lambda (k, v):
{'id': k, 'meta_type': getattr(v, 'meta_type', None)},
self._htree.items(), self._count())
# superValues() looks for the _objects attribute, but the implementation # superValues() looks for the _objects attribute, but the implementation
# would be inefficient, so superValues() support is disabled. # would be inefficient, so superValues() support is disabled.
...@@ -522,18 +544,7 @@ class HBTreeFolder2Base (Persistent): ...@@ -522,18 +544,7 @@ class HBTreeFolder2Base (Persistent):
security.declareProtected(access_contents_information, security.declareProtected(access_contents_information,
'objectIds_d') 'objectIds_d')
def objectIds_d(self, t=None): def objectIds_d(self, t=None):
ids = self.objectIds(t) return dict.fromkeys(self.objectIds(t), 1)
res = {}
for id in ids:
res[id] = 1
return res
security.declareProtected(access_contents_information,
'objectMap_d')
def objectMap_d(self, t=None):
return self.objectMap()
def _checkId(self, id, allow_dup=0): def _checkId(self, id, allow_dup=0):
if not allow_dup and self.has_key(id): if not allow_dup and self.has_key(id):
......
...@@ -84,14 +84,6 @@ class HBTreeFolder2Tests(ERP5TypeTestCase): ...@@ -84,14 +84,6 @@ class HBTreeFolder2Tests(ERP5TypeTestCase):
self.assertEqual(list(self.f.objectIds()), []) self.assertEqual(list(self.f.objectIds()), [])
self.assertEqual(self.f.objectCount(), 0) self.assertEqual(self.f.objectCount(), 0)
def testObjectMap(self):
map = self.f.objectMap()
self.assertEqual(list(map), [{'id': 'item', 'meta_type':
self.ff.meta_type}])
# I'm not sure why objectMap_d() exists, since it appears to be
# the same as objectMap(), but it's implemented by Folder.
self.assertEqual(list(self.f.objectMap_d()), list(self.f.objectMap()))
def testObjectIds_d(self): def testObjectIds_d(self):
self.assertEqual(self.f.objectIds_d(), {'item': 1}) self.assertEqual(self.f.objectIds_d(), {'item': 1})
...@@ -145,7 +137,7 @@ class HBTreeFolder2Tests(ERP5TypeTestCase): ...@@ -145,7 +137,7 @@ class HBTreeFolder2Tests(ERP5TypeTestCase):
f2 = HBTreeFolder2('somefolder') f2 = HBTreeFolder2('somefolder')
self.f._setObject(f2.id, f2) self.f._setObject(f2.id, f2)
# Hack in an absolute_url() method that works without context. # Hack in an absolute_url() method that works without context.
self.f.absolute_url = lambda: '' self.f.absolute_url = str
info = self.f.getBatchObjectListing() info = self.f.getBatchObjectListing()
self.assertEqual(info['b_start'], 1) self.assertEqual(info['b_start'], 1)
self.assertEqual(info['b_end'], 2) self.assertEqual(info['b_end'], 2)
...@@ -177,7 +169,7 @@ class HBTreeFolder2Tests(ERP5TypeTestCase): ...@@ -177,7 +169,7 @@ class HBTreeFolder2Tests(ERP5TypeTestCase):
name = " some folder " name = " some folder "
f2 = HBTreeFolder2(name) f2 = HBTreeFolder2(name)
self.f._setObject(f2.id, f2) self.f._setObject(f2.id, f2)
self.f.absolute_url = lambda: '' self.f.absolute_url = str
info = self.f.getBatchObjectListing() info = self.f.getBatchObjectListing()
expect = '<option value="%s">%s</option>' % (name, name) expect = '<option value="%s">%s</option>' % (name, name)
self.assert_(info['formatted_list'].find(expect) > 0) self.assert_(info['formatted_list'].find(expect) > 0)
...@@ -231,7 +223,7 @@ class HBTreeFolder2Tests(ERP5TypeTestCase): ...@@ -231,7 +223,7 @@ class HBTreeFolder2Tests(ERP5TypeTestCase):
h._delOb(i) h._delOb(i)
@expectedFailure @expectedFailure
def testPerformanceInDepth(self): def _testPerformanceInDepth(self):
""" """
Check HBTreeFolder2 GET performance with the depth and the number of Check HBTreeFolder2 GET performance with the depth and the number of
documents. documents.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment