Merge 4.x branch

src/ZODB/ConflictResolution.py

@@ -28,7 +28,7 @@ from pickle import PicklingError
 
 logger = logging.getLogger('ZODB.ConflictResolution')
 
-ResolvedSerial = b'rs'
+ResolvedSerial = b'rs' # deprecated: store/tpc_finish should just use True
 
 class BadClassName(Exception):
     pass

src/ZODB/Connection.py

@@ -589,7 +589,7 @@ class Connection(ExportImport, object):
 
             self._handle_serial(oid, s)
 
-    def _handle_serial(self, oid, serial, change=True):
+    def _handle_serial(self, oid, serial=True, change=True):
 
         # if we write an object, we don't want to check if it was read
         # while current.  This is a convenient choke point to do this.
@@ -597,7 +597,9 @@ class Connection(ExportImport, object):
 
         if not serial:
             return
-        if not isinstance(serial, bytes):
+        if serial is True:
+            serial = ResolvedSerial
+        elif not isinstance(serial, bytes):
             raise serial
         obj = self._cache.get(oid, None)
         if obj is None:
@@ -605,6 +607,7 @@ class Connection(ExportImport, object):
         if serial == ResolvedSerial:
             del obj._p_changed # transition from changed to ghost
         else:
+            self._warn_about_returned_serial()
             if change:
                 obj._p_changed = 0 # transition from changed to up-to-date
             obj._p_serial = serial
@@ -674,17 +677,44 @@ class Connection(ExportImport, object):
             raise
 
         if s:
+            if type(s[0]) is bytes:
+                for oid in s:
+                    self._handle_serial(oid)
+                return
+            self._warn_about_returned_serial()
             for oid, serial in s:
                 self._handle_serial(oid, serial)
 
     def tpc_finish(self, transaction):
         """Indicate confirmation that the transaction is done.
         """
-        # XXX someday, we'll care about the *real* tid we get back via
-        # the callback (that we're no longer calling).
-        self._storage.tpc_finish(transaction)
+        serial = self._storage.tpc_finish(transaction)
+        if serial is not None:
+            assert type(serial) is bytes, repr(serial)
+            for oid_iterator in self._modified, self._creating:
+                for oid in oid_iterator:
+                    obj = self._cache.get(oid)
+                    # Ignore missing objects and don't update ghosts.
+                    if obj is not None and obj._p_changed is not None:
+                        obj._p_changed = 0
+                        obj._p_serial = serial
+        else:
+            self._warn_about_returned_serial()
         self._tpc_cleanup()
 
+    def _warn_about_returned_serial(self):
+        # Do not warn about own implementations of ZODB.
+        # We're aware and the user can't do anything about it.
+        if self._normal_storage.__module__.startswith("_ZODB."):
+            self._warn_about_returned_serial = lambda: None
+        else:
+            warnings.warn(
+                "In ZODB 5+, the new API for the returned value of"
+                " store/tpc_vote/tpc_finish will be mandatory."
+                " See IStorage for more information.",
+                DeprecationWarning, 2)
+            Connection._warn_about_returned_serial = lambda self: None
+
     def sortKey(self):
         """Return a consistent sort key for this connection."""
         return "%s:%s" % (self._storage.sortKey(), id(self))

src/ZODB/interfaces.py

@@ -722,6 +722,12 @@ class IStorage(Interface):
         called while the storage transaction lock is held.  It takes
         the new transaction id generated by the transaction.
 
+        The return value must be the committed tid. It is used to set the
+        serial for objects whose ids were passed to previous store calls
+        in the same transaction.
+
+        For compatibility, the return value can also be None, in which case
+        store/tpc_vote must return the serial of stored objects.
         """
 
     def tpc_vote(transaction):
@@ -737,17 +743,18 @@ class IStorage(Interface):
         without an error, then there must not be an error if
         tpc_finish or tpc_abort is called subsequently.
 
-        The return value can be either None or a sequence of object-id
-        and serial pairs giving new serials for objects who's ids were
-        passed to previous store calls in the same transaction.
-        After the tpc_vote call, new serials must have been returned,
-        either from tpc_vote or store for objects passed to store.
+        The return value can be either None or a sequence of oids for which
+        a conflict was resolved.
 
-        A serial returned in a sequence of oid/serial pairs, may be
-        the special value ZODB.ConflictResolution.ResolvedSerial to
-        indicate that a conflict occured and that the object should be
+        For compatibility, the return value can also be a sequence of object-id
+        and serial pairs giving new serials for objects whose ids were
+        passed to previous store calls in the same transaction. The serial
+        can be the special value ZODB.ConflictResolution.ResolvedSerial to
+        indicate that a conflict occurred and that the object should be
         invalidated.
 
+        After the tpc_vote call, all solved conflicts must have been notified,
+        either from tpc_vote or store for objects passed to store.
         """
 
 class IStorageRestoreable(IStorage):

src/ZODB/mvccadapter.py

@@ -172,7 +172,7 @@ class MVCCAdapterInstance(Base):
             self._base._invalidate_finish(modified, self)
             func(tid)
 
-        self._storage.tpc_finish(transaction, invalidate_finish)
+        return self._storage.tpc_finish(transaction, invalidate_finish)
 
 def read_only_writer(self, *a, **kw):
     raise POSException.ReadOnlyError

src/ZODB/tests/BasicStorage.py

@@ -71,8 +71,10 @@ class BasicStorage:
         r1 = self._storage.store(oid, None, zodb_pickle(MinPO(11)),
                                        '', txn)
         r2 = self._storage.tpc_vote(txn)
-        self._storage.tpc_finish(txn)
+        serial = self._storage.tpc_finish(txn)
         newrevid = handle_serials(oid, r1, r2)
+        if newrevid is None and serial is not None:
+            newrevid = serial
         data, revid = utils.load_current(self._storage, oid)
         value = zodb_unpickle(data)
         eq(value, MinPO(11))

src/ZODB/tests/ConflictResolution.py

@@ -13,11 +13,10 @@
 ##############################################################################
 """Tests for application-level conflict resolution."""
 
+from ZODB import DB
 from ZODB.POSException import ConflictError, UndoError
 from persistent import Persistent
-from transaction import Transaction
-
-from ZODB.utils import load_current
+from transaction import Transaction, TransactionManager
 
 from ZODB.tests.StorageTestBase import zodb_unpickle, zodb_pickle
 
@@ -28,8 +27,8 @@ class PCounter(Persistent):
     def __repr__(self):
         return "<PCounter %d>" % self._value
 
-    def inc(self):
-        self._value = self._value + 1
+    def inc(self, n=1):
+        self._value = self._value + n
 
     def _p_resolveConflict(self, oldState, savedState, newState):
         savedDiff = savedState['_value'] - oldState['_value']
@@ -57,46 +56,38 @@ class PCounter4(PCounter):
 
 class ConflictResolvingStorage:
 
-    def checkResolve(self):
-        obj = PCounter()
-        obj.inc()
-
-        oid = self._storage.new_oid()
-
-        revid1 = self._dostoreNP(oid, data=zodb_pickle(obj))
+    def checkResolve(self, resolvable=True):
+        db = DB(self._storage)
 
-        obj.inc()
-        obj.inc()
-        # The effect of committing two transactions with the same
-        # pickle is to commit two different transactions relative to
-        # revid1 that add two to _value.
-        revid2 = self._dostoreNP(oid, revid=revid1, data=zodb_pickle(obj))
-        revid3 = self._dostoreNP(oid, revid=revid1, data=zodb_pickle(obj))
+        t1 = TransactionManager()
+        c1 = db.open(t1)
+        o1 = c1.root()['p'] = (PCounter if resolvable else PCounter2)()
+        o1.inc()
+        t1.commit()
 
-        data, serialno = load_current(self._storage, oid)
-        inst = zodb_unpickle(data)
-        self.assertEqual(inst._value, 5)
+        t2 = TransactionManager()
+        c2 = db.open(t2)
+        o2 = c2.root()['p']
+        o2.inc(2)
+        t2.commit()
 
-    def checkUnresolvable(self):
-        obj = PCounter2()
-        obj.inc()
-
-        oid = self._storage.new_oid()
-
-        revid1 = self._dostoreNP(oid, data=zodb_pickle(obj))
-
-        obj.inc()
-        obj.inc()
-        # The effect of committing two transactions with the same
-        # pickle is to commit two different transactions relative to
-        # revid1 that add two to _value.
-        revid2 = self._dostoreNP(oid, revid=revid1, data=zodb_pickle(obj))
+        o1.inc(3)
         try:
-            self._dostoreNP(oid, revid=revid1, data=zodb_pickle(obj))
+            t1.commit()
         except ConflictError as err:
-            self.assertTrue("PCounter2" in str(err))
+            self.assertIn(".PCounter2,", str(err))
+            self.assertEqual(o1._value, 3)
         else:
-            self.fail("Expected ConflictError")
+            self.assertTrue(resolvable, "Expected ConflictError")
+            self.assertEqual(o1._value, 6)
+
+        t2.begin()
+        self.assertEqual(o2._value, o1._value)
+
+        db.close()
+
+    def checkUnresolvable(self):
+        self.checkResolve(False)
 
     def checkZClassesArentResolved(self):
         from ZODB.ConflictResolution import find_global, BadClassName

src/ZODB/tests/HistoryStorage.py

@@ -34,7 +34,7 @@ class HistoryStorage:
         for data in data:
             if sys.platform == 'win32':
                 # time.time() has a precision of 1ms on Windows.
-                sleep(0.002)
+                sleep(0.001)
             revids.append(self._dostore(oid, revids[-1], MinPO(data)))
         revids.reverse()
         del revids[-1]
@@ -43,6 +43,10 @@ class HistoryStorage:
             h = self._storage.history(oid, size=i)
             self.assertEqual([d['tid'] for d in h], revids[:i])
         # Check results are sorted by timestamp, in descending order.
+        if sys.platform == 'win32':
+            # Same as above. This is also required in case this method is
+            # called several times for the same storage.
+            sleep(0.001)
         a = time()
         for d in h:
             b = a

src/ZODB/tests/MTStorage.py

@@ -155,10 +155,12 @@ class StorageClientThread(TestThread):
         r2 = self.storage.tpc_vote(t)
         self.pause()
 
-        self.storage.tpc_finish(t)
+        serial = self.storage.tpc_finish(t)
         self.pause()
 
         revid = handle_serials(oid, r1, r2)
+        if serial is not None and revid is None:
+            revid = serial
         self.oids[oid] = revid
 
 class ExtStorageClientThread(StorageClientThread):

src/ZODB/tests/RevisionStorage.py

@@ -150,10 +150,12 @@ class RevisionStorage:
                 # Finish the transaction
                 r2 = self._storage.tpc_vote(t)
                 newrevid = handle_serials(oid, r1, r2)
-                self._storage.tpc_finish(t)
+                serial = self._storage.tpc_finish(t)
             except:
                 self._storage.tpc_abort(t)
                 raise
+            if serial is not None and newrevid is None:
+                newrevid = serial
             return newrevid
         revid1 = helper(1, None, 1)
         revid2 = helper(2, revid1, 2)

src/ZODB/tests/StorageTestBase.py

@@ -132,7 +132,7 @@ def handle_serials(oid, *args):
 
     A helper for function _handle_all_serials().
     """
-    return handle_all_serials(oid, *args)[oid]
+    return handle_all_serials(oid, *args).get(oid)
 
 def import_helper(name):
     __import__(name)
@@ -189,7 +189,9 @@ class StorageTestBase(ZODB.tests.util.TestCase):
             # Finish the transaction
             r2 = self._storage.tpc_vote(t)
             revid = handle_serials(oid, r1, r2)
-            self._storage.tpc_finish(t)
+            serial = self._storage.tpc_finish(t)
+            if serial is not None and revid is None:
+                revid = serial
         except:
             self._storage.tpc_abort(t)
             raise
@@ -209,8 +211,8 @@ class StorageTestBase(ZODB.tests.util.TestCase):
         self._storage.tpc_begin(t)
         undo_result = self._storage.undo(tid, t)
         vote_result = self._storage.tpc_vote(t)
-        self._storage.tpc_finish(t)
-        if expected_oids is not None:
+        serial = self._storage.tpc_finish(t)
+        if expected_oids is not None and serial is None:
             oids = list(undo_result[1]) if undo_result else []
             oids.extend(oid for (oid, _) in vote_result or ())
             self.assertEqual(len(oids), len(expected_oids), repr(oids))

src/ZODB/tests/TransactionalUndoStorage.py

@@ -73,6 +73,12 @@ class TransactionalUndoStorage:
     def _transaction_newserial(self, oid):
         return self.__serials[oid]
 
+    def _transaction_finish(self, t, oid_list):
+        tid = self._storage.tpc_finish(t)
+        if tid is not None:
+            for oid in oid_list:
+                self.__serials[oid] = tid
+
     def _multi_obj_transaction(self, objs):
         newrevs = {}
         t = Transaction()
@@ -82,7 +88,7 @@ class TransactionalUndoStorage:
             self._transaction_store(oid, rev, data, '', t)
             newrevs[oid] = None
         self._transaction_vote(t)
-        self._storage.tpc_finish(t)
+        self._transaction_finish(t, [x[0] for x in objs])
         for oid in newrevs.keys():
             newrevs[oid] = self._transaction_newserial(oid)
         return newrevs
@@ -218,9 +224,9 @@ class TransactionalUndoStorage:
         self._transaction_store(oid2, revid2, p51, '', t)
         # Finish the transaction
         self._transaction_vote(t)
+        self._transaction_finish(t, [oid1, oid2])
         revid1 = self._transaction_newserial(oid1)
         revid2 = self._transaction_newserial(oid2)
-        self._storage.tpc_finish(t)
         eq(revid1, revid2)
         # Update those same two objects
         t = Transaction()
@@ -230,9 +236,9 @@ class TransactionalUndoStorage:
         self._transaction_store(oid2, revid2, p52, '', t)
         # Finish the transaction
         self._transaction_vote(t)
+        self._transaction_finish(t, [oid1, oid2])
         revid1 = self._transaction_newserial(oid1)
         revid2 = self._transaction_newserial(oid2)
-        self._storage.tpc_finish(t)
         eq(revid1, revid2)
         # Make sure the objects have the current value
         data, revid1 = load_current(self._storage, oid1)
@@ -288,10 +294,11 @@ class TransactionalUndoStorage:
         tid1 = info[1]['id']
         t = Transaction()
         oids = self._begin_undos_vote(t, tid, tid1)
-        self._storage.tpc_finish(t)
+        serial = self._storage.tpc_finish(t)
         # We may get the finalization stuff called an extra time,
         # depending on the implementation.
-        self.assertEqual(set(oids), set((oid1, oid2)))
+        if serial is None:
+            self.assertEqual(set(oids), {oid1, oid2})
         data, revid1 = load_current(self._storage, oid1)
         eq(zodb_unpickle(data), MinPO(30))
         data, revid2 = load_current(self._storage, oid2)
@@ -325,7 +332,7 @@ class TransactionalUndoStorage:
         self._transaction_store(oid2, revid2, p52, '', t)
         # Finish the transaction
         self._transaction_vote(t)
-        self._storage.tpc_finish(t)
+        self._transaction_finish(t, [oid1, oid2])
         revid1 = self._transaction_newserial(oid1)
         revid2 = self._transaction_newserial(oid2)
         eq(revid1, revid2)
@@ -345,7 +352,7 @@ class TransactionalUndoStorage:
         self._transaction_store(oid2, revid2, p53, '', t)
         # Finish the transaction
         self._transaction_vote(t)
-        self._storage.tpc_finish(t)
+        self._transaction_finish(t, [oid1, oid2])
         revid1 = self._transaction_newserial(oid1)
         revid2 = self._transaction_newserial(oid2)
         eq(revid1, revid2)
@@ -357,10 +364,11 @@ class TransactionalUndoStorage:
         tid = info[1]['id']
         t = Transaction()
         oids = self._begin_undos_vote(t, tid)
-        self._storage.tpc_finish(t)
-        eq(len(oids), 1)
-        self.assertTrue(oid1 in oids)
-        self.assertTrue(not oid2 in oids)
+        serial = self._storage.tpc_finish(t)
+        if serial is None:
+            eq(len(oids), 1)
+            self.assertTrue(oid1 in oids)
+            self.assertTrue(not oid2 in oids)
         data, revid1 = load_current(self._storage, oid1)
         eq(zodb_unpickle(data), MinPO(33))
         data, revid2 = load_current(self._storage, oid2)
@@ -397,7 +405,7 @@ class TransactionalUndoStorage:
         self._transaction_store(oid1, revid1, p81, '', t)
         self._transaction_store(oid2, revid2, p91, '', t)
         self._transaction_vote(t)
-        self._storage.tpc_finish(t)
+        self._transaction_finish(t, [oid1, oid2])
         revid1 = self._transaction_newserial(oid1)
         revid2 = self._transaction_newserial(oid2)
         eq(revid1, revid2)

src/ZODB/tests/testDemoStorage.py

@@ -42,6 +42,45 @@ from ZODB.utils import load_current
 
 from zope.testing import renormalizing
 
+# With the following monkey-patch, we can test the different ways
+# to update _p_changed/_p_serial status of committed oids.
+
+from ZODB.ConflictResolution import ResolvedSerial
+
+class DemoStorage(ZODB.DemoStorage.DemoStorage):
+
+    delayed_store = False
+
+    def tpc_begin(self, *args):
+        super(DemoStorage, self).tpc_begin(*args)
+        self.__stored = []
+
+    def store(self, oid, *args):
+        s = super(DemoStorage, self).store(oid, *args)
+        if s != ResolvedSerial:
+            assert type(s) is bytes, s
+            return
+        if not self.delayed_store:
+            return True
+        self.__stored.append(oid)
+
+    tpc_vote = property(lambda self: self._tpc_vote, lambda *_: None)
+
+    def _tpc_vote(self, transaction):
+        s = self.changes.tpc_vote(transaction)
+        assert s is None, s
+        return self.__stored if self.delayed_store else s
+
+    def tpc_finish(self, transaction, func = lambda tid: None):
+        r = []
+        def callback(tid):
+            func(tid)
+            r.append(tid)
+        tid = super(DemoStorage, self).tpc_finish(transaction, callback)
+        assert tid is None, tid
+        return r[0]
+
+ZODB.DemoStorage.DemoStorage = DemoStorage
 
 class DemoStorageTests(
     StorageTestBase.StorageTestBase,
@@ -107,6 +146,10 @@ class DemoStorageTests(
         self._checkHistory(base_and_changes())
         self._storage = self._storage.pop()
 
+    def checkResolveLate(self):
+        self._storage.delayed_store = True
+        self.checkResolve()
+
 
 class DemoStorageHexTests(DemoStorageTests):