Added KeywordIndexes, and optimized subtransaction processing to kick

the cache

lib/python/SearchIndex/UnKeywordIndex.py

+from UnIndex import UnIndex, MV, intSet
+from types import ListType, TupleType
+
+class UnKeywordIndex(UnIndex):
+    """
+    Like an UnIndex only it indexs sequences of items
+    
+    Searches match any keyword.
+    """
+
+    def index_object(self, i, obj, threshold=None):
+        """ index an object 'obj' with integer id 'i'"""
+
+        index = self._index
+        unindex = self._unindex
+
+        id = self.id
+
+        try:
+            kws=getattr(obj, id)
+            if callable(kws):
+                kws = kws()
+        except:
+            kws = [MV] 
+        
+        # index each item in the sequence
+        for kw in kws:
+            set = index.get(kw)
+            if set is None:
+                index[kw] = set = intSet()
+            set.insert(i)
+        
+        unindex[i] = kws
+
+        self._index = index
+        self._unindex = unindex
+
+        return 1
+    
+
+    def unindex_object(self, i):
+        """ Unindex the object with integer id 'i' """
+        index = self._index
+        unindex = self._unindex
+
+        kws = unindex[i]
+        if kws is None:
+            return None
+        
+        for kw in kws:
+            set = index.get(kw)
+            if set is not None: set.remove(i)
+        
+        del unindex[i]
+        
+        self._index = index
+        self._unindex = unindex
+
+

lib/python/SearchIndex/UnTextIndex.py

@@ -87,7 +87,7 @@
 
 
 """
-__version__='$Revision: 1.9 $'[11:-2]
+__version__='$Revision: 1.10 $'[11:-2]
 
 from Globals import Persistent
 import BTree, IIBTree, IOBTree, OIBTree
@@ -162,7 +162,7 @@ class UnTextIndex(Persistent):
 
 
     def index_object(self, i, obj, threshold=None, tupleType=type(()),
-                     dictType=type({}), callable=callable):
+                     dictType=type({}), strType=type(""), callable=callable):
         
         """ Please document """
 
@@ -199,9 +199,14 @@ class UnTextIndex(Persistent):
 
         times = 0
         for word,score in d.items():
-            if times > threshold:
-                get_transaction().commit(1)
-                times = 0
+            if threshold is not None:
+                if times > threshold:
+                    # commit a subtransaction
+                    get_transaction().commit(1)
+                    # kick the cache
+                    self._p_jar.cacheFullSweep(1)
+                    times = 0
+                    
             r = get(word)
             if r is not None:
                 r = index[word]