Vastly improved mass indexing speed and memory usage. Indexed 56MB of

text in 8MB

lib/python/Products/ZCatalog/Catalog.py

@@ -570,3 +570,6 @@ class Catalog(Persistent, Acquisition.Implicit):
 
 
 
+
+
+

lib/python/Products/ZCatalog/ZCatalog.py

@@ -109,7 +109,7 @@ def manage_addZCatalog(self,id,title,REQUEST=None):
         return self.manage_main(self,REQUEST)
 
 
-class ZCatalog(Folder, FindSupport, Persistent, Implicit):
+class ZCatalog(Folder, Persistent, Implicit):
     """ZCatalog object
 
     A ZCatalog contains arbirary index like references to Zope
@@ -287,23 +287,21 @@ class ZCatalog(Folder, FindSupport, Persistent, Implicit):
 	c_elapse = time.clock()
 	
         words = 0
-        results = self.ZopeFind(REQUEST.PARENTS[1],
-                                obj_metatypes=obj_metatypes,
-                                obj_ids=obj_ids,
-                                obj_searchterm=obj_searchterm,
-                                obj_expr=obj_expr,
-                                obj_mtime=obj_mtime,
-                                obj_mspec=obj_mspec,
-                                obj_permission=obj_permission,
-                                obj_roles=obj_roles,
-                                search_sub=1,
-                                REQUEST=REQUEST)
-
-        for n in results:
-	    if REQUEST.script != "":
-		abs_path = string.split((URL2 +'/'+n[0]),
-					REQUEST.script)[1][1:]
-            self.catalog_object(n[1], abs_path)
+        path=string.split(URL2, REQUEST.script)[1][1:]
+        
+        results = self.ZopeFindAndApply(REQUEST.PARENTS[1],
+                                        obj_metatypes=obj_metatypes,
+                                        obj_ids=obj_ids,
+                                        obj_searchterm=obj_searchterm,
+                                        obj_expr=obj_expr,
+                                        obj_mtime=obj_mtime,
+                                        obj_mspec=obj_mspec,
+                                        obj_permission=obj_permission,
+                                        obj_roles=obj_roles,
+                                        search_sub=1,
+                                        REQUEST=REQUEST,
+                                        apply_func=self.catalog_object,
+                                        apply_path=path)
 
 	elapse = time.time() - elapse
 	c_elapse = time.clock() - c_elapse
@@ -450,6 +448,100 @@ class ZCatalog(Folder, FindSupport, Persistent, Implicit):
         roles.sort()
         return roles
 
+    def ZopeFindAndApply(self, obj, obj_ids=None, obj_metatypes=None,
+                         obj_searchterm=None, obj_expr=None,
+                         obj_mtime=None, obj_mspec=None,
+                         obj_permission=None, obj_roles=None,
+                         search_sub=0,
+                         REQUEST=None, result=None, pre='',
+                         apply_func=None, apply_path=''):
+        """Zope Find interface and apply"""
+
+        if result is None:
+            result=[]
+
+            if obj_metatypes and 'all' in obj_metatypes:
+                obj_metatypes=None
+                
+            if obj_mtime and type(obj_mtime)==type('s'):
+                obj_mtime=DateTime(obj_mtime).timeTime()
+
+            if obj_permission:
+                obj_permission=p_name(obj_permission)
+
+            if obj_roles and type(obj_roles) is type('s'):
+                obj_roles=[obj_roles]
+                
+            if obj_expr:
+                # Setup expr machinations
+                md=td()
+                if hasattr(REQUEST, 'AUTHENTICATED_USER'):
+                    md.AUTHENTICATED_USER=REQUEST.AUTHENTICATED_USER
+                obj_expr=(Eval(obj_expr, expr_globals), md, md._push, md._pop)
+
+        base=obj
+        if hasattr(obj, 'aq_base'):
+            base=obj.aq_base
+
+        if not hasattr(base, 'objectItems'):
+            return result
+        try:    items=base.objectItems()
+        except: return result
+
+        try: add_result=result.append
+        except:
+            raise AttributeError, `result`
+
+        for id, ob in items:
+            if pre: p="%s/%s" % (pre, id)
+            else:   p=id
+            
+            dflag=0
+            if hasattr(ob, '_p_changed') and (ob._p_changed == None):
+                dflag=1
+
+            if hasattr(ob, 'aq_base'):
+                bs=ob.aq_base
+            else: bs=ob
+
+            if (
+                (not obj_ids or absattr(bs.id) in obj_ids)
+                and
+                (not obj_metatypes or (hasattr(bs, 'meta_type') and
+                 bs.meta_type in obj_metatypes))
+                and
+                (not obj_searchterm or
+                 (hasattr(ob, 'PrincipiaSearchSource') and
+                  find(ob.PrincipiaSearchSource(), obj_searchterm) >= 0
+                  ))
+                and
+                (not obj_expr or expr_match(ob, obj_expr))
+                and
+                (not obj_mtime or mtime_match(ob, obj_mtime, obj_mspec))
+                and
+                ( (not obj_permission or not obj_roles) or \
+                   role_match(ob, obj_permission, obj_roles)
+                )
+                ):
+                if apply_func:
+                    apply_func(ob, (apply_path+'/'+p))
+                else:
+                    add_result((p, ob))
+                    dflag=0
+                    
+            if search_sub and hasattr(bs, 'objectItems'):
+                self.ZopeFindAndApply(ob, obj_ids, obj_metatypes,
+                                      obj_searchterm, obj_expr,
+                                      obj_mtime, obj_mspec,
+                                      obj_permission, obj_roles,
+                                      search_sub,
+                                      REQUEST, result, p,
+                                      apply_func, apply_path)
+            if dflag: ob._p_deactivate()
+
+        return result
+
+
 
     def resolve_url(self, path, REQUEST):
         """