Commit 57691407 authored by Andreas Jung's avatar Andreas Jung

added TopicIndexes to trunk (merge from ajung-topicindex branch)

parent 7ff5ee51
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
__version__ = '$Id: FilteredSet.py,v 1.2 2002/02/28 15:31:41 andreasjung Exp $'
from BTrees.IIBTree import IISet
from Persistence import Persistent
from Globals import DTMLFile
from zLOG import WARNING,LOG
import sys
class FilteredSetBase(Persistent):
def __init__(self, id, expr):
self.id = id
self.expr = expr
self.clear()
def clear(self):
self.ids = IISet()
def index_object(self, documentId, obj):
raise RuntimeError,'index_object not defined'
def unindex_object(self,documentId):
try: self.ids.remove(Id)
except: pass
def getId(self): return self.id
def getExpression(self): return self.expr
def getIds(self): return self.ids
def getType(self): return self.meta_type
def setExpression(self, expr): self.expr = expr
def __repr__(self):
return '%s: (%s) %s' % (self.id,self.expr,map(None,self.ids))
__str__ = __repr__
class AttributeFilteredSet(FilteredSetBase):
""" The implementation of this FS is currently nonsense """
meta_type = 'AttributeFilteredSet'
def index_object(self, documentId, o):
if hasattr(o,self.id):
attr = getattr(o,self.id)
if callable(attr):
attr = attr()
try:
if attr in eval(self.expr):
self.ids.insert(documentId)
except:
pass
class PythonFilteredSet(FilteredSetBase):
meta_type = 'PythonFilteredSet'
def index_object(self, documentId, o):
try:
if eval(self.expr): self.ids.insert(documentId)
except:
LOG('FilteredSet',WARNING,'eval() failed',\
'Object: %s, expr: %s' % (o.getId(),self.expr),\
sys.exc_info())
class CatalogFilteredSet(FilteredSetBase):
meta_type = 'CatalogFilteredSet'
def index_object(self, documentId, obj):
raise RuntimeError, 'not implemented yet'
def factory(f_id, f_type, expr):
""" factory function for FilteredSets """
if f_type=='PythonFilteredSet':
return PythonFilteredSet(f_id, expr)
elif f_type=='AttributeFilteredSet':
return AttributeFilteredSet(f_id, expr)
elif f_type=='CatalogFilteredSet':
return CatalogFilteredSet(f_id, expr)
else:
raise TypeError,'unknown type for FilteredSets: %s' % f_type
TopicIndex
Reference: http://dev.zope.org/Wikis/DevSite/Proposals/TopicIndexes
A TopicIndex is a container for so-called FilteredSet. A FilteredSet
consists of an expression and a set of internal ZCatalog document
identifiers that represent a pre-calculated result list for performance
reasons. Instead of executing the same query on a ZCatalog multiple times
it is much faster to use a TopicIndex instead.
Building up FilteredSets happens on the fly when objects are cataloged
and uncatalogued. Every indexed object is evaluated against the expressions
of every FilteredSet. An object is added to a FilteredSet if the expression
with the object evaluates to 1. Uncatalogued objects are removed from the
FilteredSet.
Types of FilteredSet
PythonFilteredSet
A PythonFilteredSet evaluates using the eval() function inside the
context of the FilteredSet class. The object to be indexes must
be referenced inside the expression using "o.".
Examples::
"o.meta_type=='DTML Method'"
Queries on TopicIndexes
A TopicIndex is queried in the same way as other ZCatalog Indexes and supports
usage of the 'operator' parameter to specify how to combine search results.
API
The TopicIndex implements the API for pluggable Indexes.
Additionall it provides the following functions to manage FilteredSets
-- addFilteredSet(Id, filterType, expression):
-- Id: unique Id for the FilteredSet
-- filterType: 'PythonFilteredSet'
-- expression: Python expression
-- delFilteredSet(Id):
-- clearFilteredSet(Id):
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
__version__ = '$Id: TopicIndex.py,v 1.2 2002/02/28 15:31:41 andreasjung Exp $'
from Products.PluginIndexes import PluggableIndex
from Products.PluginIndexes.common.util import parseIndexRequest
from Globals import Persistent, DTMLFile
from Acquisition import Implicit
from OFS.SimpleItem import SimpleItem
from BTrees.OOBTree import OOBTree,OOSet
from BTrees.IIBTree import IISet,difference,intersection,union
from types import StringType, ListType, TupleType
import FilteredSet
_marker = []
class TopicIndex(PluggableIndex.PluggableIndex, Persistent,
Implicit, SimpleItem):
""" A TopicIndex maintains a set of FilteredSet objects.
Every FilteredSet object consists of an expression and
and IISet with all Ids of indexed objects that eval with
this expression to 1.
"""
__implements__ = (PluggableIndex.PluggableIndexInterface,)
meta_type="TopicIndex"
manage_options= (
{'label': 'FilteredSets',
'action': 'manage_workspace',
'help': ('TopicIndex','TopicIndex_searchResults.stx')},
)
manage_workspace = DTMLFile('dtml/manageTopicIndex',globals())
query_options = ('query','operator')
def __init__(self,id,caller=None):
self.id = id
self.filteredSets = OOBTree()
# experimental code for specifing the operator
self.operators = ('or','and')
self.defaultOperator = 'or'
def clear(self):
""" clear everything """
self.filteredSets = OOBTree()
def index_object(self, documentId, obj ,threshold=100):
""" hook for (Z)Catalog """
for fid, filteredSet in self.filteredSets.items():
filteredSet.index_object(documentId,obj)
return 1
def unindex_object(self,documentId):
""" hook for (Z)Catalog """
for fs in self.filteredSets.values():
fs.unindex_object(documentId)
return 1
def __len__(self):
""" len """
n=0
for fs in self.filteredSets.values():
n = n + len(fs.getIds())
return n
numObjects = "does not apply"
def keys(self): pass
def values(self): pass
def items(self): pass
def search(self,filterId):
if self.filteredSets.has_key(filterId):
return self.filteredSets[filterId].getIds()
def _apply_index(self, request, cid=''):
""" hook for (Z)Catalog
request mapping type (usually {"topic": "..." }
cid ???
"""
record = parseIndexRequest(request,self.id,self.query_options)
if record.keys==None: return None
# experimental code for specifing the operator
operator = record.get('operator',self.defaultOperator).lower()
# depending on the operator we use intersection of union
if operator=="or": set_func = union
else: set_func = intersection
res = None
for filterId in record.keys:
rows = self.search(filterId)
res = set_func(res,rows)
if res:
return res, (self.id,)
else:
return IISet(), (self.id,)
def uniqueValues(self,name=None,withLength=0):
""" needed to be consistent with the interface """
return self.filteredSets.keys()
def getEntryForObject(self,documentId,default=_marker):
""" Takes a document ID and returns all the information we have
on that specific object. """
return self.filteredSets.keys()
def addFilteredSet(self, filterId, typeFilteredSet, expr):
if self.filteredSets.has_key(filterId):
raise KeyError,\
'A FilteredSet with this name already exists: %s' % filterId
self.filteredSets[filterId] = \
FilteredSet.factory(filterId, typeFilteredSet, expr)
def delFilteredSet(self,filterId):
if not self.filteredSets.has_key(filterId):
raise KeyError,\
'no such FilteredSet: %s' % filterId
del self.filteredSets[filterId]
def clearFilteredSet(self,filterId):
if not self.filteredSets.has_key(filterId):
raise KeyError,\
'no such FilteredSet: %s' % filterId
self.filteredSets[filterId].clear()
def manage_addFilteredSet(self, filterId, typeFilteredSet, expr, URL1, \
REQUEST=None,RESPONSE=None):
""" add a new filtered set """
if len(filterId)==0: raise RuntimeError,'Length of ID too short'
if len(expr)==0: raise RuntimeError,'Length of expression too short'
self.addFilteredSet(filterId, typeFilteredSet, expr)
if RESPONSE:
RESPONSE.redirect(URL1+'/manage_workspace?manage_tabs_message=FilteredSet%20added')
def manage_delFilteredSet(self, filterIds=[], URL1=None, \
REQUEST=None,RESPONSE=None):
""" delete a list of FilteredSets"""
for filterId in filterIds:
self.delFilteredSet(filterId)
if RESPONSE:
RESPONSE.redirect(URL1+'/manage_workspace?manage_tabs_message=FilteredSet(s)%20deleted')
def manage_saveFilteredSet(self,filterId, expr, URL1=None,\
REQUEST=None,RESPONSE=None):
""" save expression for a FilteredSet """
self.filteredSets[filterId].setExpression(expr)
if RESPONSE:
RESPONSE.redirect(URL1+'/manage_workspace?manage_tabs_message=FilteredSet(s)%20updated')
def manage_clearFilteredSet(self, filterIds=[], URL1=None, \
REQUEST=None,RESPONSE=None):
""" clear a list of FilteredSets"""
for filterId in filterIds:
self.clearFilteredSet(filterId)
if RESPONSE:
RESPONSE.redirect(URL1+'/manage_workspace?manage_tabs_message=FilteredSet(s)%20cleared')
editFilteredSet = DTMLFile('dtml/editFilteredSet',globals())
index_html = DTMLFile('dtml/index', globals())
manage_addTopicIndexForm = DTMLFile('dtml/addTopicIndex', globals())
def manage_addTopicIndex(self, id, REQUEST=None, RESPONSE=None, URL3=None):
"""Add a TopicIndex"""
return self.manage_addIndex(id, 'TopicIndex', extra=None, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add TopicIndex',
)">
<p class="form-help">
A <em>TopicIndex</em> is a container for so-called <em>FilteredSets</em>
that consist of an expression and a set of internal ZCatalog document
identifiers that fulfill this expession. <em>TopicIndexes</em> are
usefull for performance reasons when search queries take too long
and pre-calculated resultsets offer a better performance.
</p>
<form action="manage_addTopicIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
TopicIndex
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p>
<dtml-with "filteredSets[filteredSet]">
<form action="manage_saveFilteredSet" method="post" enctype="multipart/form-data">
<input type="hidden" name="filterId" value="<dtml-var getId url_quote>" >
<table cellspacing="0" cellpadding="2" border="1" width="90%" align="center">
<tr>
<th colspan="2">Edit FilteredSet</th>
</tr>
<tr>
<th>FilteredSet Id</th>
<td>
<dtml-var getId>
</td>
</tr>
<tr>
<th>FilteredSet Type</th>
<td><dtml-var getType></td>
</tr>
<tr>
<th>FilteredSet Expression</th>
<td>
<textarea name="expr" cols="60" rows="5"><dtml-var getExpression></textarea>
</td>
</tr>
<tr>
<td colspan="2" align="center">
<input class="form-element" type="submit" value=" Save " />
</td>
</tr>
</table>
</form>
</dtml-with>
<dtml-var manage_page_footer>
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<form action="&dtml-URL1;/" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="1" width="90%" align="center">
<tr>
<th colspan="5">
Defined FilteredSets
</th>
</tr>
<dtml-if "_.len(filteredSets.values())>0">
<tr>
<th>&nbsp;</th>
<th>FilteredSet Id</th>
<th>FilteredSet Type</th>
<th>Expression</th>
<th># entries</th>
</tr>
<dtml-in expr="filteredSets.values()">
<dtml-call "REQUEST.set('fs',_['sequence-item'])">
<tr>
<td align="center">
<input type="checkbox" name="filterIds:list" value="<dtml-var "fs.getId()">">
</td>
<td align="center" valign="top">
<div class="form-label">
<a href="editFilteredSet?filteredSet=&dtml-id;"><dtml-var getId> </a>
</div>
</td>
<td align="center" valign="top">
<div class="form-label">
<dtml-var getType>
</div>
</td>
<td align="left" valign="top">
<div class="form-label">
<dtml-var getExpression>
</div>
</td>
<td align="center" valign="top">
<div class="form-label">
<dtml-var "_.len(fs.getIds())">
</div>
</td>
</tr>
</dtml-in>
<tr>
<td colspan="5" align="center">
<input class="form-element" type="submit" name="manage_delFilteredSet:method"
value=" Remove " />
<input class="form-element" type="submit" name="manage_clearFilteredSet:method"
value=" Clear " />
</td>
</tr>
<dtml-else>
<tr>
<td colspan="5" align="center">
<em>no FilteredSets defined </em>
</td>
</tr>
</dtml-if>
</table>
</form>
<hr>
<form action="manage_addFilteredSet" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id for FilteredSet
</div>
</td>
<td align="left" valign="top">
<input type="text" name="filterId" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Type of FilteredSet
</div>
</td>
<td align="left" valign="top">
<select name="typeFilteredSet">
<option value="PythonFilteredSet">PythonFilteredSet
<dtml-comment>
<option value="AttributeFilteredSet">AttributeFilteredSet
</dtml-comment>
</select>
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Expression
</div>
</td>
<td align="left" valign="top">
<textarea type="text" name="expr" cols="60" rows="5" />
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
ZCatalog - searchResults: specifying parameters for a search query
The searchResults() method of the ZCatalog accepts parameters that
define a query to be made on that catalog. A query can either be
passed as keyword argument to searchResults(), as a mapping, or as
part of a Zope REQUEST object, typically from HTML forms.
The index of the catalog to query is either the name of the
keyword argument, a key in a mapping, or an attribute of a record
object.
Attributes of record objects
'query' -- either a sequence of objects or a single value to be
passed as query to the index (mandatory)
'operator' -- specifies the combination of search results when
query is a sequence of values. (optional, default: 'or').
Allowed values:
'and', 'or'
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import Zope
import os,sys,re,unittest
from Products.PluginIndexes.TopicIndex.TopicIndex import TopicIndex
from Products.ZCatalog.ZCatalog import ZCatalog
class Obj:
def __init__(self,id,meta_type=''):
self.id = id
self.meta_type = meta_type
def getId(self): return self.id
def getPhysicalPath(self): return self.id
class TestBase(unittest.TestCase):
def _searchAnd(self,query,expected):
return self._search(query,'and',expected)
def _searchOr(self,query,expected):
return self._search(query,'or',expected)
def _search(self,query,operator,expected):
res = self.TI._apply_index({'topic':{'query':query,'operator':operator}})
rows = list(res[0])
rows.sort()
expected.sort()
self.assertEqual(rows,expected,query)
return rows
class TestTopicIndex(TestBase):
def setUp(self):
self.TI = TopicIndex("topic")
self.TI.addFilteredSet("doc1","PythonFilteredSet","o.meta_type=='doc1'")
self.TI.addFilteredSet("doc2","PythonFilteredSet","o.meta_type=='doc2'")
self.TI.index_object(0 , Obj('0',))
self.TI.index_object(1 , Obj('1','doc1'))
self.TI.index_object(2 , Obj('2','doc1'))
self.TI.index_object(3 , Obj('3','doc2'))
self.TI.index_object(4 , Obj('4','doc2'))
self.TI.index_object(5 , Obj('5','doc3'))
self.TI.index_object(6 , Obj('6','doc3'))
def testOr(self):
""" test 1 """
self._searchOr('doc1',[1,2])
self._searchOr(['doc1'],[1,2])
self._searchOr('doc2',[3,4]),
self._searchOr(['doc2'],[3,4])
self._searchOr(['doc1','doc2'], [1,2,3,4])
def testAnd(self):
""" test 1 """
self._searchAnd('doc1',[1,2])
self._searchAnd(['doc1'],[1,2])
self._searchAnd('doc2',[3,4])
self._searchAnd(['doc2'],[3,4])
self._searchAnd(['doc1','doc2'],[])
class ZCatalogTopicTests(TestBase):
def setUp(self):
self.cat = ZCatalog('catalog')
self.cat.addIndex('topic','TopicIndex')
self.TI = self.cat._catalog.indexes['topic']
self.TI.addFilteredSet("doc1","PythonFilteredSet","o.meta_type=='doc1'")
self.TI.addFilteredSet("doc2","PythonFilteredSet","o.meta_type=='doc2'")
self.cat.catalog_object(Obj('0'))
self.cat.catalog_object(Obj('1','doc1'))
self.cat.catalog_object(Obj('2','doc1'))
self.cat.catalog_object(Obj('3','doc2'))
self.cat.catalog_object(Obj('4','doc2'))
self.cat.catalog_object(Obj('5','doc3'))
self.cat.catalog_object(Obj('6','doc3'))
def testOr(self):
""" testing or (catalog)"""
self._searchOr('doc1',[1,2])
self._searchOr('doc2',[3,4])
self._searchOr(['doc1','doc2'],[1,2,3,4])
def testAnd(self):
""" testing And (catalog)"""
self._searchAnd('doc1',[1,2])
self._searchAnd('doc2',[3,4])
self._searchAnd(['doc1','doc2'],[])
def _search(self,query,operator,expected):
res = self.cat.searchResults({'topic':{'query':query,'operator':operator}})
rows = [int(x.id) for x in res ]
rows.sort()
expected.sort()
self.assertEqual(rows,expected,query)
return rows,res
def test_suite():
return unittest.TestSuite( (
unittest.makeSuite(TestTopicIndex),
unittest.makeSuite(ZCatalogTopicTests),
))
def main():
unittest.TextTestRunner().run(test_suite())
if __name__ == '__main__':
main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment