Commit 7865c58d authored by Julien Muchembled's avatar Julien Muchembled

CMFCategory: add support for ZODB indexing of related documents

parent a47a9bbd
......@@ -31,6 +31,7 @@
ERP portal_categories tool.
"""
from collections import deque
from BTrees.OOBTree import OOTreeSet
from OFS.Folder import Folder
from Products.CMFCore.utils import UniqueObject
from Products.ERP5Type.Globals import InitializeClass, DTMLFile
......@@ -40,9 +41,11 @@ from Acquisition import aq_base, aq_inner
from Products.ERP5Type import Permissions
from Products.ERP5Type.Base import Base
from Products.ERP5Type.Cache import getReadOnlyTransactionCache
from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
from Products.CMFCategory import _dtmldir
from Products.CMFCore.PortalFolder import ContentFilter
from Products.CMFCategory.Renderer import Renderer
from Products.CMFCategory.Category import Category, BaseCategory
from OFS.Traversable import NotFound
import types
......@@ -55,6 +58,34 @@ _marker = object()
class CategoryError( Exception ):
pass
class RelatedIndex(): # persistent.Persistent can be added
# without breaking compatibility
def __repr__(self):
try:
contents = ', '.join('%s=%r' % (k, list(v))
for (k, v) in self.__dict__.iteritems())
except Exception:
contents = '...'
return '<%s(%s) at 0x%x>' % (self.__class__.__name__, contents, id(self))
def __nonzero__(self):
return any(self.__dict__.itervalues())
def add(self, base, relative_url):
try:
getattr(self, base).add(relative_url)
except AttributeError:
setattr(self, base, OOTreeSet((relative_url,)))
def remove(self, base, relative_url):
try:
getattr(self, base).remove(relative_url)
except (AttributeError, KeyError):
pass
class CategoryTool( UniqueObject, Folder, Base ):
"""
The CategoryTool object is the placeholder for all methods
......@@ -1174,7 +1205,28 @@ class CategoryTool( UniqueObject, Folder, Base ):
security.declareProtected( Permissions.ModifyPortalContent, '_setCategoryList' )
def _setCategoryList(self, context, value):
context.categories = tuple(value)
old = set(getattr(aq_base(context), 'categories', ()))
context.categories = value = tuple(value)
if context.isTempDocument():
return
value = set(value)
relative_url = context.getRelativeUrl()
for edit, value in ("remove", old - value), ("add", value - old):
for path in value:
base = self.getBaseCategoryId(path)
try:
if self[base].isRelatedLocallyIndexed():
path = self._removeDuplicateBaseCategoryIdInCategoryPath(base, path)
ob = aq_base(self.unrestrictedTraverse(path))
try:
related = ob._related_index
except AttributeError:
if edit is "remove":
continue
related = ob._related_index = RelatedIndex()
getattr(related, edit)(base, relative_url)
except KeyError:
pass
security.declareProtected( Permissions.AccessContentsInformation, 'getAcquiredCategoryList' )
def getAcquiredCategoryList(self, context):
......@@ -1287,50 +1339,153 @@ class CategoryTool( UniqueObject, Folder, Base ):
portal_type = kw.get('portal_type')
if isinstance(portal_type, str):
portal_type = [portal_type]
portal_type = portal_type,
# Base Category may not be related, besides sub categories
if context.getPortalType() == 'Base Category':
category_list = [context.getRelativeUrl()]
relative_url = context.getRelativeUrl()
local_index_dict = {}
if isinstance(context, BaseCategory):
category_list = relative_url,
else:
category_list = []
if isinstance(base_category_list, str):
base_category_list = [base_category_list]
base_category_list = base_category_list,
elif base_category_list is () or base_category_list is None:
base_category_list = self.getBaseCategoryList()
category_list = []
for base_category in base_category_list:
category_list.append("%s/%s" % (base_category, context.getRelativeUrl()))
brain_result = self.Base_zSearchRelatedObjectsByCategoryList(
category_list=category_list,
if self[base_category].isRelatedLocallyIndexed():
category = base_category + '/'
local_index_dict[base_category] = '' \
if relative_url.startswith(category) else category
else:
category_list.append("%s/%s" % (base_category, relative_url))
search = self.getPortalObject().Base_zSearchRelatedObjectsByCategoryList
if local_index_dict:
# For some base categories, lookup indexes in ZODB.
recurse = isinstance(context, Category) and not strict_membership
result_dict = {}
def check_local():
r = set(getattr(related, base_category, ()))
r.difference_update(result_dict)
for r in r:
try:
ob = self.unrestrictedTraverse(r)
if category in aq_base(ob).categories:
result_dict[r] = ob
continue
# Do not add 'r' to result_dict, because 'ob' may be linked in
# another way.
except (AttributeError, KeyError):
result_dict[r] = None
related.remove(base_category, r)
tv = getTransactionalVariable().setdefault(
'CategoriesTool.getRelatedValueList', {})
try:
related = aq_base(context)._related_index
except AttributeError:
related = RelatedIndex()
include_self = False
for base_category, category in local_index_dict.iteritems():
if not category:
# Categories are member of themselves.
include_self = True
result_dict[relative_url] = context
category += relative_url
if tv.get(category, -1) < recurse:
# Update local index with results from catalog for backward
# compatibility. But no need to do it several times in the same
# transaction.
for r in search(category_list=category,
portal_type=None,
strict_membership=strict_membership):
r = r.relative_url
# relative_url is empty if object is deleted (but not yet
# unindexed). Nothing specific to do in such case because
# category tool won't match.
try:
ob = self.unrestrictedTraverse(r)
categories = aq_base(ob).categories
except (AttributeError, KeyError):
result_dict[r] = None
continue
if category in categories:
related.add(base_category, r)
result_dict[r] = ob
elif recurse:
for p in categories:
if p.startswith(category + '/'):
try:
o = self.unrestrictedTraverse(p)
p = aq_base(o)._related_index
except KeyError:
continue
except AttributeError:
p = o._related_index = RelatedIndex()
result_dict[r] = ob
p.add(base_category, r)
tv[category] = recurse
# Get and check all objects referenced by local index for the base
# category that is currently considered.
check_local()
# Modify context only if it's worth it.
if related and not hasattr(aq_base(context), '_related_index'):
context._related_index = related
# In case of non-strict membership search, include all objects that
# are linked to a subobject of context.
if recurse:
r = [context]
while r:
for ob in r.pop().objectValues():
r.append(ob)
relative_url = ob.getRelativeUrl()
if include_self:
result_dict[relative_url] = ob
try:
related = aq_base(ob)._related_index
except AttributeError:
continue
for base_category, category in local_index_dict.iteritems():
category += relative_url
check_local()
# Filter out objects that are not of requested portal type.
result = [ob for ob in result_dict.itervalues() if ob is not None and (
not portal_type or ob.getPortalType() in portal_type)]
# Finish with base categories that are only indexed in catalog,
# making sure we don't return duplicate values.
if category_list:
for r in search(category_list=category_list,
portal_type=portal_type,
strict_membership=strict_membership)
strict_membership=strict_membership):
if r.relative_url not in result_dict:
try:
result.append(self.unrestrictedTraverse(r.path))
except KeyError:
pass
else:
# Catalog-only search.
result = []
for r in search(category_list=category_list,
portal_type=portal_type,
strict_membership=strict_membership):
try:
result.append(self.unrestrictedTraverse(r.path))
except KeyError:
pass
if checked_permission is None:
# No permission to check
for b in brain_result:
o = b.getObject()
if o is not None:
result.append(o)
else:
return result
# Check permissions on object
if isinstance(checked_permission, str):
checked_permission = (checked_permission, )
checked_permission = checked_permission,
checkPermission = self.portal_membership.checkPermission
for b in brain_result:
obj = b.getObject()
if obj is not None:
def check(ob):
for permission in checked_permission:
if not checkPermission(permission, obj):
break
result.append(obj)
return result
# XXX missing filter and **kw stuff
#return self.search_category(category_list=category_list,
# portal_type=spec)
# future implementation with brains, much more efficient
if checkPermission(permission, ob):
return True
return filter(check, result)
security.declareProtected( Permissions.AccessContentsInformation,
'getRelatedPropertyList' )
......
......@@ -60,6 +60,11 @@ class TestCMFCategory(ERP5TypeTestCase):
),
resource = dict(
),
test0 = dict(
),
test1 = dict(
contents=('a', ('ab', 'ac', ('acd',))),
),
)
def getTitle(self):
......@@ -104,7 +109,8 @@ class TestCMFCategory(ERP5TypeTestCase):
acquisition_copy_value=0,
acquisition_append_value=0,
acquisition_mask_value=0,
acquisition_portal_type_list="python: []")
acquisition_portal_type_list="python: []",
related_locally_indexed=0)
edit_kw.update(kw)
queue = deque(((bc, edit_kw.pop('contents', ())),))
bc.edit(**edit_kw)
......@@ -129,6 +135,7 @@ class TestCMFCategory(ERP5TypeTestCase):
ti = self.getTypesTool().getTypeInfo(portal_type)
ti.filter_content_types = 0
self._original_categories[portal_type] = x = ti.getTypeBaseCategoryList()
x += 'test0', 'test1'
ti._setTypeBaseCategoryList(x + categories)
# Make persons.
......@@ -1094,6 +1101,55 @@ class TestCMFCategory(ERP5TypeTestCase):
self.assertEqual(get(bc.id), list('bab'))
_set(bc.id, ())
def test_relatedIndex(self):
category_tool = self.getCategoriesTool()
newOrganisation = self.getOrganisationModule().newContent
organisation = newOrganisation()
other_organisation = newOrganisation(destination_value=organisation)
person = self.getPersonModule().newContent(test0_value=organisation,
test1='a/ac/acd')
self.tic()
get = organisation.getTest0RelatedValueList
a = category_tool.test1.a
def check():
self.assertEqual([person, other_organisation],
category_tool.getRelatedValueList(organisation))
self.assertEqual([person], get())
self.assertEqual([person], get(portal_type='Person'))
self.assertEqual([], get(portal_type='Organisation'))
self.assertEqual([person], a.getTest1RelatedValueList(
portal_type='Person'))
self.assertEqual([a], a.getTest1RelatedValueList(
strict_membership=True))
self.assertEqual([person], a.ac.acd.getTest1RelatedValueList(
portal_type='Person', strict_membership=True))
category_tool.test0._setRelatedLocallyIndexed(True)
category_tool.test1._setRelatedLocallyIndexed(True)
check()
related_list = sorted(a.getTest1RelatedList())
self.assertTrue(person.getRelativeUrl() in related_list)
self.assertEqual(related_list, sorted(x.getRelativeUrl()
for x in self.portal.portal_catalog(test1_uid=a.getUid())))
related = organisation._related_index
self.assertTrue(related)
self.assertEqual([person.getRelativeUrl()], list(related.test0))
person.unindexObject()
self.tic()
category_tool.test0._setRelatedLocallyIndexed(False)
self.assertEqual([], get())
category_tool.test0._setRelatedLocallyIndexed(True)
check()
person.categories = tuple(x for x in person.categories
if not x.startswith('test0/'))
self.assertEqual([], get())
self.assertFalse(related)
self.assertEqual([], list(related.test0))
related = a.ac.acd._related_index.test1
self.assertEqual(list(related), [person.getRelativeUrl()])
person._setTest1Value(a)
self.assertEqual(list(related), [])
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestCMFCategory))
......
......@@ -544,7 +544,8 @@ class BaseTemplateItem(Implicit, Persistent):
klass = obj.__class__
classname = klass.__name__
attr_set = set(('_dav_writelocks', '_filepath', '_owner', 'last_id', 'uid',
attr_set = set(('_dav_writelocks', '_filepath', '_owner', '_related_index',
'last_id', 'uid',
'__ac_local_roles__', '__ac_local_roles_group_id_dict__'))
if export:
if not keep_workflow_history:
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>return not (value and (\n
request.other[\'field_my_acquisition_object_id_list\'] or\n
request.other[\'field_my_acquisition_base_category_list\']))\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>value, request</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>BaseCategory_validateRelatedLocallyIndexed</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
......@@ -105,6 +105,7 @@
<key> <string>right</string> </key>
<value>
<list>
<string>my_related_locally_indexed</string>
<string>my_acquisition_copy_value</string>
<string>my_acquisition_mask_value</string>
<string>my_acquisition_append_value</string>
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>external_validator</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>delegated_message_list</string> </key>
<value>
<list>
<string>external_validator_failed</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_related_locally_indexed</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>Local index is incompatible with category acquision.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_checkbox</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Index Related Documents Locally</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Method" module="Products.Formulator.MethodField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>method_name</string> </key>
<value> <string>BaseCategory_validateRelatedLocallyIndexed</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
......@@ -58,15 +58,11 @@ order_by_expression</string> </value>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
SELECT DISTINCT catalog.uid, catalog.path, portal_type\n
SELECT DISTINCT catalog.uid, path, relative_url, portal_type\n
FROM catalog, category\n
WHERE catalog.uid = category.uid\n
<dtml-if portal_type>\n
AND\n
(<dtml-in portal_type>\n
<dtml-unless sequence-start> OR </dtml-unless>\n
catalog.portal_type=\'<dtml-var sequence-item>\'\n
</dtml-in>)\n
AND <dtml-sqltest portal_type type="string" multiple>\n
</dtml-if>\n
AND (<dtml-var "portal_categories.buildSQLSelector(category_list)">)\n
<dtml-if strict_membership>\n
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Standard Property" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_local_properties</string> </key>
<value>
<tuple>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>mode</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>elementary_type/boolean</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Determines if related values should be indexed on target documents (i.e. in ZODB) in addition to catalog.\n
This is incompatible with category acquisition.</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>related_locally_indexed_property</string> </value>
</item>
<item>
<key> <string>mode</string> </key>
<value> <string>w</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Standard Property</string> </value>
</item>
<item>
<key> <string>property_default</string> </key>
<value> <string>python: 0</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment