merge ajung-zcatalog-progress branch 26525:26606

doc/CHANGES.txt

@@ -12,10 +12,6 @@ Zope Changes
        changes in C, brining the Python and C implementations back in
        sync.  See lib/python/AccessControl/PermissionRole.py.
 
-     - Renable C Zope security policy by implementing recent Python
-       changes in C, brining the Python and C implementations back in
-       sync.  See lib/python/AccessControl/ZopeSecurityPolicy.py.
-
      - Add cyclic-garbage collection support to C extension classes,
        especially to acquisition wrappers.
 
@@ -26,6 +22,10 @@ Zope Changes
 
     Features added
 
+     - ZCatalog: added a new configuration option in the "Advanced" tab
+       to provide optional logging of the progress of long running 
+       reindexing or recataloging operations.
+
      - made Zope.configure return the starter instance to enable other
        methods to be called, such as starter.setupConfiguredLoggers()
 

lib/python/Products/ZCatalog/Catalog.py

@@ -11,27 +11,27 @@
 #
 ##############################################################################
 
-from Persistence import Persistent
+import types
+import logging
+from bisect import bisect
+from random import randint
+
 import Acquisition
 import ExtensionClass
-from MultiMapping import MultiMapping
-import Record
 from Missing import MV
-import logging
+from Persistence import Persistent
 
-from Lazy import LazyMap, LazyFilter, LazyCat, LazyValues
-from CatalogBrains import AbstractCatalogBrain, NoBrainer
+import BTrees.Length
 from BTrees.IIBTree import intersection, weightedIntersection, IISet
 from BTrees.OIBTree import OIBTree
 from BTrees.IOBTree import IOBTree
-import BTrees.Length
+from Lazy import LazyMap, LazyCat, LazyValues
+from CatalogBrains import AbstractCatalogBrain, NoBrainer
 
-import time, sys, types
-from bisect import bisect
-from random import randint
 
 LOG = logging.getLogger('Zope.ZCatalog')
 
+
 try:
     from DocumentTemplate.cDocumentTemplate import safe_callable
 except ImportError:
@@ -278,7 +278,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
 
         indexes = self.indexes
 
-        if isinstance(index_type, types.StringType):
+        if isinstance(index_type, str):
             raise TypeError,"""Catalog addIndex now requires the index type to
             be resolved prior to adding; create the proper index in the caller."""
 
@@ -755,7 +755,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
         reverse = 0
         if sort_index is not None:
             order = self._get_sort_attr("order", args)
-            if (isinstance(order, types.StringType) and
+            if (isinstance(order, str) and
                 order.lower() in ('reverse', 'descending')):
                 reverse = 1
         # Perform searches with indexes and sort_index

lib/python/Products/ZCatalog/CatalogPathAwareness.py

@@ -13,7 +13,6 @@
 
 """ZCatalog Findable class"""
 
-import urllib
 from Globals import DTMLFile
 from Acquisition import aq_base
 

lib/python/Products/ZCatalog/ProgressHandler.py

+##############################################################################
+#
+# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+"""
+$Id: ZCatalog.py 25050 2004-05-27 15:06:40Z chrisw $
+"""
+
+import time, sys
+from zLOG import LOG, INFO
+
+from Interface import Interface
+
+class IProgressHandler(Interface):
+    """ A handler to log progress informations for long running 
+        operations.
+    """
+
+    def init(ident, max):
+        """ Called at the start of the long running process.
+
+            'ident' -- a string identifying the operation
+            'max' -- maximum number of objects to be processed (int)
+        """ 
+
+    def info(text):
+        """ Log some 'text'"""
+
+    def finish():
+        """ Called up termination """
+
+    def report(current, *args, **kw):
+        """ Called for every iteration.
+
+            'current' -- an integer representing the number of objects 
+                         processed so far.
+        """
+
+    def output(text):
+        """ Log 'text' to some output channel """        
+
+
+class StdoutHandler:
+    """ A simple progress handler """
+
+    __implements__ = IProgressHandler
+
+    def __init__(self, steps=100):
+        self._steps = steps
+
+    def init(self, ident, max):
+        self._ident = ident
+        self._max = max
+        self._start = time.time()
+        self.fp = sys.stdout
+        self.output('Process started (%d objects to go)' % self._max)
+
+    def info(self, text):
+        self.output(text)
+
+    def finish(self):
+        self.output('Process terminated. Duration: %0.2f seconds' % \
+                    (time.time() -self._start))
+
+    def report(self, current, *args, **kw):
+        if current % self._steps == 0: 
+            self.output('%d/%d (%.2f%%)' % (current, self._max, 
+                                           (100.0 * current / self._max)))
+
+    def output(self, text):
+        print >>self.fp, '%s: %s' % (self._ident, text)
+
+
+class ZLogHandler(StdoutHandler):
+    """ Use zLOG """
+
+    __implements__ = IProgressHandler
+
+    def output(self, text):
+        LOG(self._ident, INFO, text)
+

lib/python/Products/ZCatalog/README.txt

@@ -52,3 +52,11 @@ ZCatalog
        encoding (set in site.py of the Python installation) if any
        to the keyword uses a non-ascii encoding (e.g. using accented
        characters). 
+
+  Notes for Zope 2.8:
+
+       reindexIndex() and refreshCatalog() accept a new optional parameter 
+       'pghandler' which must be a handler instance implementing the 
+       IProgressHandler interface (see ProgressHandler.py). This can be useful
+       to provide logging during long running operations.
+ 

lib/python/Products/ZCatalog/ZCatalog.py

@@ -15,6 +15,8 @@
 $Id$
 """
 
+import urllib, time, sys, string,logging
+
 from Globals import DTMLFile, MessageDialog
 import Globals
 
@@ -35,9 +37,8 @@ from ZODB.POSException import ConflictError
 from Products.PluginIndexes.common.PluggableIndex \
      import PluggableIndexInterface
 from Products.PluginIndexes.TextIndex import Splitter
-import urllib, time, sys
-import string,logging
 from IZCatalog import IZCatalog
+from ProgressHandler import ZLogHandler
 
 LOG = logging.getLogger('Zope.ZCatalog')
 
@@ -127,6 +128,7 @@ class ZCatalog(Folder, Persistent, Implicit):
           'manage_catalogClear', 'manage_addColumn', 'manage_delColumn',
           'manage_addIndex', 'manage_delIndex', 'manage_clearIndex',
           'manage_reindexIndex', 'manage_main', 'availableSplitters',
+          'manage_setProgress',
 
           # these two are deprecated:
           'manage_delColumns', 'manage_deleteIndex'
@@ -250,7 +252,9 @@ class ZCatalog(Folder, Persistent, Implicit):
         elapse = time.time()
         c_elapse = time.clock()
 
-        self.refreshCatalog(clear=1)
+        pgthreshold = self._getProgressThreshold()
+        handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
+        self.refreshCatalog(clear=1, pghandler=handler)
 
         elapse = time.time() - elapse
         c_elapse = time.clock() - c_elapse
@@ -263,7 +267,7 @@ class ZCatalog(Folder, Persistent, Implicit):
                          'Total CPU time: %s' % (`elapse`, `c_elapse`)))
 
 
-    def refreshCatalog(self, clear=0):
+    def refreshCatalog(self, clear=0, pghandler=None):
         """ re-index everything we can find """
 
         cat = self._catalog
@@ -272,26 +276,27 @@ class ZCatalog(Folder, Persistent, Implicit):
             paths = tuple(paths)
             cat.clear()
 
-        LOG('ZCatalog', BLATHER, 'Starting recataloging of ZCatalog at %s' % 
-             self.absolute_url(1))
         num_objects = len(paths)
+        if pghandler:
+            pghandler.init('Refreshing catalog: %s' % self.absolute_url(1), num_objects)
+
         for i in xrange(num_objects):
+            if pghandler: pghandler.report(i)
+
             p = paths[i]
             obj = self.resolve_path(p)
             if not obj:
                 obj = self.resolve_url(p, self.REQUEST)
             if obj is not None:
                 try:
-                    LOG('ZCatalog', BLATHER, 'Recataloging object %s (%d/%d)' %
-                         (p, i, num_objects))
-                    self.catalog_object(obj, p)
+                    self.catalog_object(obj, p, pghandler=pghandler)
                 except ConflictError:
                     raise
                 except:
-                    LOG('ZCatalog', ERROR, 'Recataloging object at %s failed' % p,
-                              error=sys.exc_info())
+                    LOG.error('Recataloging object at %s failed' % p,
+                              exc_info=sys.exc_info())
 
-        LOG('ZCatalog', BLATHER, 'Recataloging of ZCatalog at %s terminated' % self.absolute_url(1))
+        if pghandler: pghandler.finish()
 
     def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None):
         """ clears the whole enchilada """
@@ -460,10 +465,21 @@ class ZCatalog(Folder, Persistent, Implicit):
                 '/manage_catalogIndexes?manage_tabs_message=Index%20Cleared')
 
 
-    def reindexIndex(self, name, REQUEST):
+    def reindexIndex(self, name, REQUEST, pghandler=None):
         if isinstance(name, str):
             name = (name,)
-        for p in self._catalog.uids.keys():
+
+        paths = self._catalog.uids.keys()
+        num_paths = len(paths)   # inefficient        
+
+        i = 0
+        if pghandler:
+            pghandler.init('reindexing %s' % name, num_paths)
+
+        for p in paths:
+            i+=1
+            if pghandler: pghandler.report(i)
+
             obj = self.resolve_path(p)
             if not obj:
                 obj = self.resolve_url(p, REQUEST)
@@ -475,7 +491,7 @@ class ZCatalog(Folder, Persistent, Implicit):
                 # index via the UI
                 try:
                     self.catalog_object(obj, p, idxs=name,
-                                        update_metadata=0)
+                                        update_metadata=0, pghandler=pghandler)
                 except TypeError:
                     # Fall back to Zope 2.6.2 interface. This is necessary for
                     # products like CMF 1.4.2 and earlier that subclass from
@@ -485,7 +501,10 @@ class ZCatalog(Folder, Persistent, Implicit):
                     warn('catalog_object interface of %s not up to date'
                          % self.__class__.__name__,
                          DeprecationWarning)
-                    self.catalog_object(obj, p, idxs=name)
+                    self.catalog_object(obj, p, idxs=name, pghandler=pghandler)
+
+        if pghandler:
+            pghandler.finish()
 
     def manage_reindexIndex(self, ids=None, REQUEST=None, RESPONSE=None,
                             URL1=None):
@@ -495,7 +514,9 @@ class ZCatalog(Folder, Persistent, Implicit):
                 message='No items were specified!',
                 action = "./manage_catalogIndexes",)
 
-        self.reindexIndex(ids, REQUEST)
+        pgthreshold = self._getProgressThreshold()
+        handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
+        self.reindexIndex(ids, REQUEST, handler)
 
         if REQUEST and RESPONSE:
             RESPONSE.redirect(
@@ -509,7 +530,7 @@ class ZCatalog(Folder, Persistent, Implicit):
         return Splitter.availableSplitters
 
 
-    def catalog_object(self, obj, uid=None, idxs=None, update_metadata=1):
+    def catalog_object(self, obj, uid=None, idxs=None, update_metadata=1, pghandler=None):
         """ wrapper around catalog """
 
         if uid is None:
@@ -551,6 +572,8 @@ class ZCatalog(Folder, Persistent, Implicit):
                 get_transaction().commit(1)
                 self._p_jar.cacheGC()
                 self._v_total = 0
+                if pghandler:
+                    pghandler.info('commiting subtransaction')
 
     def uncatalog_object(self, uid):
         """Wrapper around catalog """
@@ -861,6 +884,21 @@ class ZCatalog(Folder, Persistent, Implicit):
                   '%s unchanged.' % (len(fixed), len(removed), unchanged),
           action='./manage_main')
 
+    def manage_setProgress(self, pgthreshold=0, RESPONSE=None, URL1=None):
+        """Set parameter to perform logging of reindexing operations very 
+           'pgthreshold' objects
+        """
+
+        self.pgthreshold = pgthreshold
+        if RESPONSE:
+            RESPONSE.redirect(
+                URL1 + '/manage_main?manage_tabs_message=Catalog%20Changed')
+
+    def _getProgressThreshold(self):
+        if not hasattr(self, 'pgthreshold'):
+            self.pgthreshold = 0
+        return self.pgthreshold
+
     def manage_convertBTrees(self, threshold=200):
         """Convert the catalog's data structures to use BTrees package"""
         assert type(threshold) is type(0)

lib/python/Products/ZCatalog/ZCatalogIndexes.py

@@ -14,20 +14,15 @@
 """$Id$
 """
 
+
+from Acquisition import Implicit
+from Persistence import Persistent
 from Globals import DTMLFile, InitializeClass
 from AccessControl.SecurityInfo import ClassSecurityInfo
-from AccessControl.Permissions import delete_objects, manage_zcatalog_indexes
-import Globals
+from AccessControl.Permissions import manage_zcatalog_indexes
 from OFS.Folder import Folder
-from OFS.FindSupport import FindSupport
-from OFS.History import Historical
 from OFS.SimpleItem import SimpleItem
-from OFS.ObjectManager import ObjectManager, IFAwareObjectManager
-
-import os, sys, time 
-
-from Acquisition import Implicit
-from Persistence import Persistent
+from OFS.ObjectManager import IFAwareObjectManager
 
 from Products.PluginIndexes.common.PluggableIndex import PluggableIndexInterface
 

lib/python/Products/ZCatalog/__init__.py

@@ -13,7 +13,7 @@
 
 """ZCatalog product"""
 
-import ZCatalog, Catalog, CatalogAwareness, CatalogPathAwareness, ZClasses
+import ZCatalog, CatalogAwareness, CatalogPathAwareness
 from Products.PluginIndexes.TextIndex import Vocabulary
 from ZClasses import createZClassForBase
 

lib/python/Products/ZCatalog/dtml/catalogAdvanced.dtml

@@ -36,6 +36,19 @@
 </form>
   </td>
 </tr>
+<tr>
+  <td align="left" valign="top">
+  <p class="form-help">Log progress of reindexing every N objects to the Zope logger (set to 0 to disabled logging)
+  </p>
+  </td>
+  <td align="right" valign="top">
+<form action="&dtml-URL1;">
+<input type="text" name="pgthreshold:int" value="<dtml-var pgthreshold missing="0">">
+<input class="form-element" type="submit" 
+ name="manage_setProgress:method" value=" Change ">
+</form>
+  </td>
+</tr>
 <tr>
   <td>