Merge transience changes from chrism-pre27-branch.

lib/python/Products/Transience/HowTransienceWorks.stx

@@ -42,6 +42,7 @@ Data Structures Maintained by a Transient Object Container
     inside of the "_data" structure.  There is a concept of a
     "current" bucket, which is the bucket that is contained within the
     _data structured with a key equal to the "current" timeslice.
+    A current bucket must always exist (this is an invariant).
 
   - A "max_timeslice" integer, which is equal to the "largest"
     timeslice for which there exists a bucket in the _data structure.
@@ -74,10 +75,13 @@ Housekeeping: Finalization, Garbage Collection, and Bucket
 Replentishing
 
   The TOC performs "finalization", "garbage collection", and "bucket
-  replentishing".  It performs these tasks "in-band".  This means that
-  the TOC does not maintain a separate thread that wakes up every so
-  often to do these housekeeping tasks.  Instead, during the course of
-  normal operations, the TOC opportunistically performs them.
+  replentishing".  It typically performs these tasks "in-band"
+  (although it is possible to do the housekeeping tasks "out of band"
+  as well: see the methods of the Transient Object Container with
+  "housekeep" in their names).  "In band" housekeeping implies that
+  the TOC does not maintain a separate thread or process that wakes up
+  every so often to clean up.  Instead, during the course of normal
+  operations, the TOC opportunistically performs housekeeping functions.
 
   Finalization is defined as optionally calling a function at bucket
   expiration time against all transient objects contained within that

lib/python/Products/Transience/TransactionHelper.py

+import time
+
+class PreventTransactionCommit(Exception):
+    def __init__(self, reason):
+        self. reason = reason
+
+    def __str__(self):
+        return "Uncommittable transaction: " % self.reason
+    
+class UncommittableJar:
+    """ A jar that cannot be committed """
+    def __init__(self, reason):
+        self.reason = reason
+        self.time = time.time()
+        
+    def sort_key(self):
+        return self.time()
+
+    def tpc_begin(self, *arg, **kw):
+        pass
+
+    def commit(self, obj, transaction):
+        pass
+
+    def tpc_vote(self, transaction):
+        raise PreventTransactionCommit(self.reason)
+
+class makeTransactionUncommittable:
+    """
+    - register an uncommittable object with the provided transaction
+      which prevents the commit of that transaction
+    """
+    def __init__(self, transaction, reason):
+        self._p_jar = UncommittableJar(reason)
+        transaction.register(self)
+        

lib/python/Products/Transience/TransientObject.py

@@ -16,6 +16,8 @@ Simple ZODB-based transient object implementation.
 $Id$
 """
 
+__version__='$Revision: 1.9.68.5 $'[11:-2]
+
 from Persistence import Persistent
 from Acquisition import Implicit
 import time, random, sys, os
@@ -192,69 +194,59 @@ class TransientObject(Persistent, Implicit):
     # Other non interface code
     #
 
-    def _p_independent(self):
-        # My state doesn't depend on or materially effect the state of
-        # other objects (eliminates read conflicts).
-        return 1
-
     def _p_resolveConflict(self, saved, state1, state2):
         DEBUG and TLOG('entering TO _p_rc')
         DEBUG and TLOG('states: sv: %s, s1: %s, s2: %s' % (
             saved, state1, state2))
-        try:
-            states = [saved, state1, state2]
-
-            # We can clearly resolve the conflict if one state is invalid,
-            # because it's a terminal state.
-            for state in states:
-                if state.has_key('_invalid'):
-                    DEBUG and TLOG('TO _p_rc: a state was invalid')
-                    return state
-            # The only other times we can clearly resolve the conflict is if
-            # the token, the id, or the creation time don't differ between
-            # the three states, so we check that here.  If any differ, we punt
-            # by raising ConflictError.
-            attrs = ['token', 'id', '_created']
-            for attr in attrs:
-                svattr = saved.get(attr)
-                s1attr = state1.get(attr)
-                s2attr = state2.get(attr)
-                DEBUG and TLOG('TO _p_rc: attr %s: sv: %s s1: %s s2: %s' %
-                               (attr, svattr, s1attr, s2attr))
-                if not svattr==s1attr==s2attr:
-                    DEBUG and TLOG('TO _p_rc: cant resolve conflict')
-                    raise ConflictError
-
-            # Now we need to do real work.
-            #
-            # Data in our _container dictionaries might conflict.  To make
-            # things simple, we intentionally create a race condition where the
-            # state which was last modified "wins".  It would be preferable to
-            # somehow merge our _containers together, but as there's no
-            # generally acceptable way to union their states, there's not much
-            # we can do about it if we want to be able to resolve this kind of
-            # conflict.
-
-            # We return the state which was most recently modified, if
-            # possible.
-            states.sort(lastmodified_sort)
-            if states[0].get('_last_modified'):
-                DEBUG and TLOG('TO _p_rc: returning last mod state')
-                return states[0]
-
-            # If we can't determine which object to return on the basis
-            # of last modification time (no state has been modified), we return
-            # the object that was most recently accessed (last pulled out of
-            # our parent).  This will return an essentially arbitrary state if
-            # all last_accessed values are equal.
-            states.sort(lastaccessed_sort)
-            DEBUG and TLOG('TO _p_rc: returning last_accessed state')
+        states = [saved, state1, state2]
+
+        # We can clearly resolve the conflict if one state is invalid,
+        # because it's a terminal state.
+        for state in states:
+            if state.has_key('_invalid'):
+                DEBUG and TLOG('TO _p_rc: a state was invalid')
+                return state
+
+        # The only other times we can clearly resolve the conflict is if
+        # the token, the id, or the creation time don't differ between
+        # the three states, so we check that here.  If any differ, we punt
+        # by raising ConflictError.
+        attrs = ['token', 'id', '_created']
+        for attr in attrs:
+            svattr = saved.get(attr)
+            s1attr = state1.get(attr)
+            s2attr = state2.get(attr)
+            DEBUG and TLOG('TO _p_rc: attr %s: sv: %s s1: %s s2: %s' %
+                           (attr, svattr, s1attr, s2attr))
+            if not svattr==s1attr==s2attr:
+                DEBUG and TLOG('TO _p_rc: cant resolve conflict')
+                raise ConflictError
+
+        # Now we need to do real work.
+        #
+        # Data in our _container dictionaries might conflict.  To make
+        # things simple, we intentionally create a race condition where the
+        # state which was last modified "wins".  It would be preferable to
+        # somehow merge our _containers together, but as there's no
+        # generally acceptable way to union their states, there's not much
+        # we can do about it if we want to be able to resolve this kind of
+        # conflict.
+
+        # We return the state which was most recently modified, if
+        # possible.
+        states.sort(lastmodified_sort)
+        if states[0].get('_last_modified'):
+            DEBUG and TLOG('TO _p_rc: returning last mod state')
             return states[0]
-        except ConflictError:
-            raise
-        except:
-            LOG.info('Conflict resolution error in TransientObject',
-                      exc_info=sys.exc_info())
+
+        # If we can't determine which object to return on the basis
+        # of last modification time (no state has been modified), we return
+        # the object that was most recently accessed (last pulled out of
+        # our parent).  This will return an essentially arbitrary state if
+        # all last_accessed values are equal.
+        states.sort(lastaccessed_sort)
+        DEBUG and TLOG('TO _p_rc: returning last_accessed state')
+        return states[0]
 
     getName = getId # this is for SQLSession compatibility
 

lib/python/Products/Transience/dtml/manageTransientObjectContainer.dtml

@@ -13,7 +13,7 @@ Transient data will persist, but only for a user-specified period of time
 (the "data object timeout") after which it will be flushed.
 </p>
 
-<dtml-call nudge><!-- turn the buckets if necessary -->
+<dtml-call housekeep><!-- turn the buckets if necessary -->
 
 <p class="form-label">
 <font color="green">

lib/python/Products/Transience/tests/testCounters.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+import os
+from unittest import TestCase, TestSuite, makeSuite
+from ZODB.POSException import ConflictError
+from ZODB.FileStorage import FileStorage
+from ZODB.DB import DB
+
+from Products.Transience.Transience import Length2, Increaser
+
+class Base(TestCase):
+    db = None
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        if self.db is not None:
+            self.db.close()
+            self.storage.cleanup()
+
+    def openDB(self):
+        n = 'fs_tmp__%s' % os.getpid()
+        self.storage = FileStorage(n)
+        self.db = DB(self.storage)
+
+class TestLength2(Base):
+
+    def testConflict(self):
+        # this test fails on the HEAD (MVCC?)
+        self.openDB()
+        length = Length2(0)
+
+        r1 = self.db.open().root()
+        r1['ob'] = length
+        get_transaction().commit()
+
+        r2 = self.db.open().root()
+        copy = r2['ob']
+        # The following ensures that copy is loaded.
+        self.assertEqual(copy(),0)
+
+        # First transaction.
+        length.increment(10)
+        length.decrement(1)
+        get_transaction().commit()
+
+        # Second transaction.
+        length = copy
+        length.increment(20)
+        length.decrement(2)
+        get_transaction().commit()
+
+        self.assertEqual(length(), 10+20-max(1,2))
+
+class TestIncreaser(Base):
+
+    def testConflict(self):
+        self.openDB()
+        increaser = Increaser(0)
+
+        r1 = self.db.open().root()
+        r1['ob'] = increaser
+        get_transaction().commit()
+
+        r2 = self.db.open().root()
+        copy = r2['ob']
+        # The following ensures that copy is loaded.
+        self.assertEqual(copy(),0)
+
+        # First transaction.
+        increaser.set(10)
+        get_transaction().commit()
+
+
+        # Second transaction.
+        increaser = copy
+        increaser.set(20)
+        get_transaction().commit()
+
+        self.assertEqual(increaser(), 20)
+
+def test_suite():
+    suite = TestSuite()
+    suite.addTest(makeSuite(TestLength2))
+    suite.addTest(makeSuite(TestIncreaser))
+    return suite

lib/python/Products/Transience/tests/testTransactionHelper.py

+##############################################################################
+#
+# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import sys, os, time, random, unittest
+
+if __name__ == "__main__":
+    sys.path.insert(0, '../../..')
+
+import ZODB
+from unittest import TestCase, TestSuite, TextTestRunner, makeSuite
+from Products.Transience.TransactionHelper import PreventTransactionCommit, \
+         makeTransactionUncommittable
+
+class TestTransactionHelper(TestCase):
+    def setUp(self):
+        self.t = get_transaction()
+
+    def tearDown(self):
+        self.t = None
+
+    def testUncommittable(self):
+        makeTransactionUncommittable(self.t, "test")
+        self.assertRaises(PreventTransactionCommit, get_transaction().commit)
+        
+def test_suite():
+    suite = makeSuite(TestTransactionHelper, 'test')
+    return suite
+
+if __name__ == '__main__':
+    runner = TextTestRunner(verbosity=9)
+    runner.run(test_suite())

lib/python/Products/Transience/tests/testTransientObjectContainer.py

@@ -17,7 +17,7 @@ if __name__ == "__main__":
 
 import ZODB
 from Products.Transience.Transience import TransientObjectContainer,\
-     MaxTransientObjectsExceeded
+     MaxTransientObjectsExceeded, SPARE_BUCKETS, getCurrentTimeslice
 from Products.Transience.TransientObject import TransientObject
 import Products.Transience.Transience
 import Products.Transience.TransientObject
@@ -380,6 +380,18 @@ class TestTransientObjectContainer(TestBase):
         fauxtime.sleep(180)
         self.assertEqual(len(self.t.keys()), 100)
 
+    def testGarbageCollection(self):
+        # this is pretty implementation-dependent :-(
+        for x in range(0, 100):
+            self.t[x] = x
+        sleeptime = self.period * SPARE_BUCKETS
+        fauxtime.sleep(sleeptime)
+        self.t._invoke_finalize_and_gc()
+        max_ts = self.t._last_finalized_timeslice()
+        keys = list(self.t._data.keys())
+        for k in keys:
+            self.assert_(k > max_ts, "k %s < max_ts %s" % (k, max_ts))
+
     def _maxOut(self):
         for x in range(11):
             self.t.new(str(x))