Commit 7865c58d authored by Julien Muchembled's avatar Julien Muchembled

CMFCategory: add support for ZODB indexing of related documents

parent a47a9bbd
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
ERP portal_categories tool. ERP portal_categories tool.
""" """
from collections import deque from collections import deque
from BTrees.OOBTree import OOTreeSet
from OFS.Folder import Folder from OFS.Folder import Folder
from Products.CMFCore.utils import UniqueObject from Products.CMFCore.utils import UniqueObject
from Products.ERP5Type.Globals import InitializeClass, DTMLFile from Products.ERP5Type.Globals import InitializeClass, DTMLFile
...@@ -40,9 +41,11 @@ from Acquisition import aq_base, aq_inner ...@@ -40,9 +41,11 @@ from Acquisition import aq_base, aq_inner
from Products.ERP5Type import Permissions from Products.ERP5Type import Permissions
from Products.ERP5Type.Base import Base from Products.ERP5Type.Base import Base
from Products.ERP5Type.Cache import getReadOnlyTransactionCache from Products.ERP5Type.Cache import getReadOnlyTransactionCache
from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
from Products.CMFCategory import _dtmldir from Products.CMFCategory import _dtmldir
from Products.CMFCore.PortalFolder import ContentFilter from Products.CMFCore.PortalFolder import ContentFilter
from Products.CMFCategory.Renderer import Renderer from Products.CMFCategory.Renderer import Renderer
from Products.CMFCategory.Category import Category, BaseCategory
from OFS.Traversable import NotFound from OFS.Traversable import NotFound
import types import types
...@@ -55,6 +58,34 @@ _marker = object() ...@@ -55,6 +58,34 @@ _marker = object()
class CategoryError( Exception ): class CategoryError( Exception ):
pass pass
class RelatedIndex(): # persistent.Persistent can be added
# without breaking compatibility
def __repr__(self):
try:
contents = ', '.join('%s=%r' % (k, list(v))
for (k, v) in self.__dict__.iteritems())
except Exception:
contents = '...'
return '<%s(%s) at 0x%x>' % (self.__class__.__name__, contents, id(self))
def __nonzero__(self):
return any(self.__dict__.itervalues())
def add(self, base, relative_url):
try:
getattr(self, base).add(relative_url)
except AttributeError:
setattr(self, base, OOTreeSet((relative_url,)))
def remove(self, base, relative_url):
try:
getattr(self, base).remove(relative_url)
except (AttributeError, KeyError):
pass
class CategoryTool( UniqueObject, Folder, Base ): class CategoryTool( UniqueObject, Folder, Base ):
""" """
The CategoryTool object is the placeholder for all methods The CategoryTool object is the placeholder for all methods
...@@ -1174,7 +1205,28 @@ class CategoryTool( UniqueObject, Folder, Base ): ...@@ -1174,7 +1205,28 @@ class CategoryTool( UniqueObject, Folder, Base ):
security.declareProtected( Permissions.ModifyPortalContent, '_setCategoryList' ) security.declareProtected( Permissions.ModifyPortalContent, '_setCategoryList' )
def _setCategoryList(self, context, value): def _setCategoryList(self, context, value):
context.categories = tuple(value) old = set(getattr(aq_base(context), 'categories', ()))
context.categories = value = tuple(value)
if context.isTempDocument():
return
value = set(value)
relative_url = context.getRelativeUrl()
for edit, value in ("remove", old - value), ("add", value - old):
for path in value:
base = self.getBaseCategoryId(path)
try:
if self[base].isRelatedLocallyIndexed():
path = self._removeDuplicateBaseCategoryIdInCategoryPath(base, path)
ob = aq_base(self.unrestrictedTraverse(path))
try:
related = ob._related_index
except AttributeError:
if edit is "remove":
continue
related = ob._related_index = RelatedIndex()
getattr(related, edit)(base, relative_url)
except KeyError:
pass
security.declareProtected( Permissions.AccessContentsInformation, 'getAcquiredCategoryList' ) security.declareProtected( Permissions.AccessContentsInformation, 'getAcquiredCategoryList' )
def getAcquiredCategoryList(self, context): def getAcquiredCategoryList(self, context):
...@@ -1287,50 +1339,153 @@ class CategoryTool( UniqueObject, Folder, Base ): ...@@ -1287,50 +1339,153 @@ class CategoryTool( UniqueObject, Folder, Base ):
portal_type = kw.get('portal_type') portal_type = kw.get('portal_type')
if isinstance(portal_type, str): if isinstance(portal_type, str):
portal_type = [portal_type] portal_type = portal_type,
# Base Category may not be related, besides sub categories # Base Category may not be related, besides sub categories
if context.getPortalType() == 'Base Category': relative_url = context.getRelativeUrl()
category_list = [context.getRelativeUrl()] local_index_dict = {}
if isinstance(context, BaseCategory):
category_list = relative_url,
else: else:
category_list = []
if isinstance(base_category_list, str): if isinstance(base_category_list, str):
base_category_list = [base_category_list] base_category_list = base_category_list,
elif base_category_list is () or base_category_list is None: elif base_category_list is () or base_category_list is None:
base_category_list = self.getBaseCategoryList() base_category_list = self.getBaseCategoryList()
category_list = []
for base_category in base_category_list: for base_category in base_category_list:
category_list.append("%s/%s" % (base_category, context.getRelativeUrl())) if self[base_category].isRelatedLocallyIndexed():
category = base_category + '/'
brain_result = self.Base_zSearchRelatedObjectsByCategoryList( local_index_dict[base_category] = '' \
category_list=category_list, if relative_url.startswith(category) else category
else:
category_list.append("%s/%s" % (base_category, relative_url))
search = self.getPortalObject().Base_zSearchRelatedObjectsByCategoryList
if local_index_dict:
# For some base categories, lookup indexes in ZODB.
recurse = isinstance(context, Category) and not strict_membership
result_dict = {}
def check_local():
r = set(getattr(related, base_category, ()))
r.difference_update(result_dict)
for r in r:
try:
ob = self.unrestrictedTraverse(r)
if category in aq_base(ob).categories:
result_dict[r] = ob
continue
# Do not add 'r' to result_dict, because 'ob' may be linked in
# another way.
except (AttributeError, KeyError):
result_dict[r] = None
related.remove(base_category, r)
tv = getTransactionalVariable().setdefault(
'CategoriesTool.getRelatedValueList', {})
try:
related = aq_base(context)._related_index
except AttributeError:
related = RelatedIndex()
include_self = False
for base_category, category in local_index_dict.iteritems():
if not category:
# Categories are member of themselves.
include_self = True
result_dict[relative_url] = context
category += relative_url
if tv.get(category, -1) < recurse:
# Update local index with results from catalog for backward
# compatibility. But no need to do it several times in the same
# transaction.
for r in search(category_list=category,
portal_type=None,
strict_membership=strict_membership):
r = r.relative_url
# relative_url is empty if object is deleted (but not yet
# unindexed). Nothing specific to do in such case because
# category tool won't match.
try:
ob = self.unrestrictedTraverse(r)
categories = aq_base(ob).categories
except (AttributeError, KeyError):
result_dict[r] = None
continue
if category in categories:
related.add(base_category, r)
result_dict[r] = ob
elif recurse:
for p in categories:
if p.startswith(category + '/'):
try:
o = self.unrestrictedTraverse(p)
p = aq_base(o)._related_index
except KeyError:
continue
except AttributeError:
p = o._related_index = RelatedIndex()
result_dict[r] = ob
p.add(base_category, r)
tv[category] = recurse
# Get and check all objects referenced by local index for the base
# category that is currently considered.
check_local()
# Modify context only if it's worth it.
if related and not hasattr(aq_base(context), '_related_index'):
context._related_index = related
# In case of non-strict membership search, include all objects that
# are linked to a subobject of context.
if recurse:
r = [context]
while r:
for ob in r.pop().objectValues():
r.append(ob)
relative_url = ob.getRelativeUrl()
if include_self:
result_dict[relative_url] = ob
try:
related = aq_base(ob)._related_index
except AttributeError:
continue
for base_category, category in local_index_dict.iteritems():
category += relative_url
check_local()
# Filter out objects that are not of requested portal type.
result = [ob for ob in result_dict.itervalues() if ob is not None and (
not portal_type or ob.getPortalType() in portal_type)]
# Finish with base categories that are only indexed in catalog,
# making sure we don't return duplicate values.
if category_list:
for r in search(category_list=category_list,
portal_type=portal_type, portal_type=portal_type,
strict_membership=strict_membership) strict_membership=strict_membership):
if r.relative_url not in result_dict:
try:
result.append(self.unrestrictedTraverse(r.path))
except KeyError:
pass
else:
# Catalog-only search.
result = [] result = []
for r in search(category_list=category_list,
portal_type=portal_type,
strict_membership=strict_membership):
try:
result.append(self.unrestrictedTraverse(r.path))
except KeyError:
pass
if checked_permission is None: if checked_permission is None:
# No permission to check return result
for b in brain_result:
o = b.getObject()
if o is not None:
result.append(o)
else:
# Check permissions on object # Check permissions on object
if isinstance(checked_permission, str): if isinstance(checked_permission, str):
checked_permission = (checked_permission, ) checked_permission = checked_permission,
checkPermission = self.portal_membership.checkPermission checkPermission = self.portal_membership.checkPermission
for b in brain_result: def check(ob):
obj = b.getObject()
if obj is not None:
for permission in checked_permission: for permission in checked_permission:
if not checkPermission(permission, obj): if checkPermission(permission, ob):
break return True
result.append(obj) return filter(check, result)
return result
# XXX missing filter and **kw stuff
#return self.search_category(category_list=category_list,
# portal_type=spec)
# future implementation with brains, much more efficient
security.declareProtected( Permissions.AccessContentsInformation, security.declareProtected( Permissions.AccessContentsInformation,
'getRelatedPropertyList' ) 'getRelatedPropertyList' )
......
...@@ -60,6 +60,11 @@ class TestCMFCategory(ERP5TypeTestCase): ...@@ -60,6 +60,11 @@ class TestCMFCategory(ERP5TypeTestCase):
), ),
resource = dict( resource = dict(
), ),
test0 = dict(
),
test1 = dict(
contents=('a', ('ab', 'ac', ('acd',))),
),
) )
def getTitle(self): def getTitle(self):
...@@ -104,7 +109,8 @@ class TestCMFCategory(ERP5TypeTestCase): ...@@ -104,7 +109,8 @@ class TestCMFCategory(ERP5TypeTestCase):
acquisition_copy_value=0, acquisition_copy_value=0,
acquisition_append_value=0, acquisition_append_value=0,
acquisition_mask_value=0, acquisition_mask_value=0,
acquisition_portal_type_list="python: []") acquisition_portal_type_list="python: []",
related_locally_indexed=0)
edit_kw.update(kw) edit_kw.update(kw)
queue = deque(((bc, edit_kw.pop('contents', ())),)) queue = deque(((bc, edit_kw.pop('contents', ())),))
bc.edit(**edit_kw) bc.edit(**edit_kw)
...@@ -129,6 +135,7 @@ class TestCMFCategory(ERP5TypeTestCase): ...@@ -129,6 +135,7 @@ class TestCMFCategory(ERP5TypeTestCase):
ti = self.getTypesTool().getTypeInfo(portal_type) ti = self.getTypesTool().getTypeInfo(portal_type)
ti.filter_content_types = 0 ti.filter_content_types = 0
self._original_categories[portal_type] = x = ti.getTypeBaseCategoryList() self._original_categories[portal_type] = x = ti.getTypeBaseCategoryList()
x += 'test0', 'test1'
ti._setTypeBaseCategoryList(x + categories) ti._setTypeBaseCategoryList(x + categories)
# Make persons. # Make persons.
...@@ -1094,6 +1101,55 @@ class TestCMFCategory(ERP5TypeTestCase): ...@@ -1094,6 +1101,55 @@ class TestCMFCategory(ERP5TypeTestCase):
self.assertEqual(get(bc.id), list('bab')) self.assertEqual(get(bc.id), list('bab'))
_set(bc.id, ()) _set(bc.id, ())
def test_relatedIndex(self):
category_tool = self.getCategoriesTool()
newOrganisation = self.getOrganisationModule().newContent
organisation = newOrganisation()
other_organisation = newOrganisation(destination_value=organisation)
person = self.getPersonModule().newContent(test0_value=organisation,
test1='a/ac/acd')
self.tic()
get = organisation.getTest0RelatedValueList
a = category_tool.test1.a
def check():
self.assertEqual([person, other_organisation],
category_tool.getRelatedValueList(organisation))
self.assertEqual([person], get())
self.assertEqual([person], get(portal_type='Person'))
self.assertEqual([], get(portal_type='Organisation'))
self.assertEqual([person], a.getTest1RelatedValueList(
portal_type='Person'))
self.assertEqual([a], a.getTest1RelatedValueList(
strict_membership=True))
self.assertEqual([person], a.ac.acd.getTest1RelatedValueList(
portal_type='Person', strict_membership=True))
category_tool.test0._setRelatedLocallyIndexed(True)
category_tool.test1._setRelatedLocallyIndexed(True)
check()
related_list = sorted(a.getTest1RelatedList())
self.assertTrue(person.getRelativeUrl() in related_list)
self.assertEqual(related_list, sorted(x.getRelativeUrl()
for x in self.portal.portal_catalog(test1_uid=a.getUid())))
related = organisation._related_index
self.assertTrue(related)
self.assertEqual([person.getRelativeUrl()], list(related.test0))
person.unindexObject()
self.tic()
category_tool.test0._setRelatedLocallyIndexed(False)
self.assertEqual([], get())
category_tool.test0._setRelatedLocallyIndexed(True)
check()
person.categories = tuple(x for x in person.categories
if not x.startswith('test0/'))
self.assertEqual([], get())
self.assertFalse(related)
self.assertEqual([], list(related.test0))
related = a.ac.acd._related_index.test1
self.assertEqual(list(related), [person.getRelativeUrl()])
person._setTest1Value(a)
self.assertEqual(list(related), [])
def test_suite(): def test_suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestCMFCategory)) suite.addTest(unittest.makeSuite(TestCMFCategory))
......
...@@ -544,7 +544,8 @@ class BaseTemplateItem(Implicit, Persistent): ...@@ -544,7 +544,8 @@ class BaseTemplateItem(Implicit, Persistent):
klass = obj.__class__ klass = obj.__class__
classname = klass.__name__ classname = klass.__name__
attr_set = set(('_dav_writelocks', '_filepath', '_owner', 'last_id', 'uid', attr_set = set(('_dav_writelocks', '_filepath', '_owner', '_related_index',
'last_id', 'uid',
'__ac_local_roles__', '__ac_local_roles_group_id_dict__')) '__ac_local_roles__', '__ac_local_roles_group_id_dict__'))
if export: if export:
if not keep_workflow_history: if not keep_workflow_history:
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>return not (value and (\n
request.other[\'field_my_acquisition_object_id_list\'] or\n
request.other[\'field_my_acquisition_base_category_list\']))\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>value, request</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>BaseCategory_validateRelatedLocallyIndexed</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -105,6 +105,7 @@ ...@@ -105,6 +105,7 @@
<key> <string>right</string> </key> <key> <string>right</string> </key>
<value> <value>
<list> <list>
<string>my_related_locally_indexed</string>
<string>my_acquisition_copy_value</string> <string>my_acquisition_copy_value</string>
<string>my_acquisition_mask_value</string> <string>my_acquisition_mask_value</string>
<string>my_acquisition_append_value</string> <string>my_acquisition_append_value</string>
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>external_validator</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>delegated_message_list</string> </key>
<value>
<list>
<string>external_validator_failed</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_related_locally_indexed</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>Local index is incompatible with category acquision.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_checkbox</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Index Related Documents Locally</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Method" module="Products.Formulator.MethodField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>method_name</string> </key>
<value> <string>BaseCategory_validateRelatedLocallyIndexed</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -58,15 +58,11 @@ order_by_expression</string> </value> ...@@ -58,15 +58,11 @@ order_by_expression</string> </value>
<key> <string>src</string> </key> <key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[ <value> <string encoding="cdata"><![CDATA[
SELECT DISTINCT catalog.uid, catalog.path, portal_type\n SELECT DISTINCT catalog.uid, path, relative_url, portal_type\n
FROM catalog, category\n FROM catalog, category\n
WHERE catalog.uid = category.uid\n WHERE catalog.uid = category.uid\n
<dtml-if portal_type>\n <dtml-if portal_type>\n
AND\n AND <dtml-sqltest portal_type type="string" multiple>\n
(<dtml-in portal_type>\n
<dtml-unless sequence-start> OR </dtml-unless>\n
catalog.portal_type=\'<dtml-var sequence-item>\'\n
</dtml-in>)\n
</dtml-if>\n </dtml-if>\n
AND (<dtml-var "portal_categories.buildSQLSelector(category_list)">)\n AND (<dtml-var "portal_categories.buildSQLSelector(category_list)">)\n
<dtml-if strict_membership>\n <dtml-if strict_membership>\n
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Standard Property" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_local_properties</string> </key>
<value>
<tuple>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>mode</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>elementary_type/boolean</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Determines if related values should be indexed on target documents (i.e. in ZODB) in addition to catalog.\n
This is incompatible with category acquisition.</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>related_locally_indexed_property</string> </value>
</item>
<item>
<key> <string>mode</string> </key>
<value> <string>w</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Standard Property</string> </value>
</item>
<item>
<key> <string>property_default</string> </key>
<value> <string>python: 0</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment