Merge remote-tracking branch 'origin/master' into no-more-load

CHANGES.rst

@@ -26,6 +26,11 @@
   transaction.
   https://github.com/zopefoundation/ZODB/pull/52
 
+- DemoStorage: add support for conflict resolution and fix history()
+  https://github.com/zopefoundation/ZODB/pull/58
+
+- Fixed: FileStorage loadBefore didn't handle deleted/undone data correctly.
+
 4.2.0 (2015-06-02)
 ==================
 

setup.py

@@ -23,23 +23,8 @@ interface, rich transaction support, and undo.
 version = "4.3.0.dev0"
 
 import os
-import sys
 from setuptools import setup, find_packages
 
-if sys.version_info < (2, 6):
-    print("This version of ZODB requires Python 2.6 or higher")
-    sys.exit(0)
-
-if (3,) < sys.version_info < (3, 2):
-    print("This version of ZODB requires Python 3.2 or higher")
-    sys.exit(0)
-
-
-# The (non-obvious!) choices for the Trove Development Status line:
-# Development Status :: 5 - Production/Stable
-# Development Status :: 4 - Beta
-# Development Status :: 3 - Alpha
-
 classifiers = """\
 Development Status :: 4 - Beta
 Intended Audience :: Developers

src/ZODB/ConflictResolution.py

@@ -313,4 +313,9 @@ class ConflictResolvingStorage(object):
     def registerDB(self, wrapper):
         self._crs_untransform_record_data = wrapper.untransform_record_data
         self._crs_transform_record_data = wrapper.transform_record_data
-        super(ConflictResolvingStorage, self).registerDB(wrapper)
+        try:
+            m = super(ConflictResolvingStorage, self).registerDB
+        except AttributeError:
+            pass
+        else:
+            m(wrapper)

src/ZODB/DemoStorage.py

@@ -31,12 +31,13 @@ import ZODB.MappingStorage
 import ZODB.POSException
 import ZODB.utils
 import zope.interface
+from .ConflictResolution import ConflictResolvingStorage, ResolvedSerial
 
 @zope.interface.implementer(
         ZODB.interfaces.IStorage,
         ZODB.interfaces.IStorageIteration,
         )
-class DemoStorage(object):
+class DemoStorage(ConflictResolvingStorage):
 
 
     def __init__(self, name=None, base=None, changes=None,
@@ -112,7 +113,7 @@ class DemoStorage(object):
     def _copy_methods_from_changes(self, changes):
         for meth in (
             '_lock_acquire', '_lock_release',
-            'getSize', 'history', 'isReadOnly', 'registerDB',
+            'getSize', 'isReadOnly',
             'sortKey', 'tpc_transaction', 'tpc_vote',
             ):
             setattr(self, meth, getattr(changes, meth))
@@ -137,6 +138,20 @@ class DemoStorage(object):
         except ZODB.POSException.POSKeyError:
             return self.base.getTid(oid)
 
+    def history(self, oid, size=1):
+        try:
+            r = self.changes.history(oid, size)
+        except ZODB.POSException.POSKeyError:
+            r = []
+        size -= len(r)
+        if size:
+            try:
+                r += self.base.history(oid, size)
+            except ZODB.POSException.POSKeyError:
+                if not r:
+                    raise
+        return r
+
     def iterator(self, start=None, end=None):
         for t in self.base.iterator(start, end):
             yield t
@@ -281,8 +296,9 @@ class DemoStorage(object):
                 old = serial
 
         if old != serial:
-            raise ZODB.POSException.ConflictError(
-                oid=oid, serials=(old, serial)) # XXX untested branch
+            rdata = self.tryToResolveConflict(oid, old, serial, data)
+            self.changes.store(oid, old, rdata, '', transaction)
+            return ResolvedSerial
 
         return self.changes.store(oid, serial, data, '', transaction)
 

src/ZODB/FileStorage/FileStorage.py

@@ -490,11 +490,13 @@ class FileStorage(
                 if not pos:
                     return None
 
-            if h.back:
+            if h.plen:
+                return _file.read(h.plen), h.tid, end_tid
+            elif h.back:
                 data, _, _, _ = self._loadBack_impl(oid, h.back, _file=_file)
                 return data, h.tid, end_tid
             else:
-                return _file.read(h.plen), h.tid, end_tid
+                raise POSKeyError(oid)
 
     def store(self, oid, oldserial, data, version, transaction):
         if self._is_read_only:
@@ -1645,7 +1647,7 @@ def read_index(file, name, index, tindex, stop=b'\377'*8,
         maxoid = index.maxKey()
     except ValueError:
         # The index is empty.
-        maxoid == z64
+        pass # maxoid is already equal to z64
 
     return pos, maxoid, ltid
 

src/ZODB/MappingStorage.py

@@ -105,7 +105,7 @@ class MappingStorage(object):
         tids.reverse()
         return [
             dict(
-                time = ZODB.TimeStamp.TimeStamp(tid),
+                time = ZODB.TimeStamp.TimeStamp(tid).timeTime(),
                 tid = tid,
                 serial = tid,
                 user_name = self._transactions[tid].user,

src/ZODB/tests/HistoryStorage.py

@@ -17,45 +17,31 @@ Any storage that supports the history() method should be able to pass
 all these tests.
 """
 
+from time import time
 from ZODB.tests.MinPO import MinPO
 
 class HistoryStorage:
     def checkSimpleHistory(self):
-        eq = self.assertEqual
+        self._checkHistory((11, 12, 13))
+
+    def _checkHistory(self, data):
+        start = time()
         # Store a couple of revisions of the object
         oid = self._storage.new_oid()
         self.assertRaises(KeyError,self._storage.history,oid)
-        revid1 = self._dostore(oid, data=MinPO(11))
-        revid2 = self._dostore(oid, revid=revid1, data=MinPO(12))
-        revid3 = self._dostore(oid, revid=revid2, data=MinPO(13))
+        revids = [None]
+        for data in data:
+            revids.append(self._dostore(oid, revids[-1], MinPO(data)))
+        revids.reverse()
+        del revids[-1]
         # Now get various snapshots of the object's history
-        h = self._storage.history(oid, size=1)
-        eq(len(h), 1)
-        d = h[0]
-        eq(d['tid'], revid3)
-        # Try to get 2 historical revisions
-        h = self._storage.history(oid, size=2)
-        eq(len(h), 2)
-        d = h[0]
-        eq(d['tid'], revid3)
-        d = h[1]
-        eq(d['tid'], revid2)
-        # Try to get all 3 historical revisions
-        h = self._storage.history(oid, size=3)
-        eq(len(h), 3)
-        d = h[0]
-        eq(d['tid'], revid3)
-        d = h[1]
-        eq(d['tid'], revid2)
-        d = h[2]
-        eq(d['tid'], revid1)
-        # There should be no more than 3 revisions
-        h = self._storage.history(oid, size=4)
-        eq(len(h), 3)
-        d = h[0]
-        eq(d['tid'], revid3)
-        d = h[1]
-        eq(d['tid'], revid2)
-        d = h[2]
-        eq(d['tid'], revid1)
-
+        for i in range(1, 1 + len(revids)):
+            h = self._storage.history(oid, size=i)
+            self.assertEqual([d['tid'] for d in h], revids[:i])
+        # Check results are sorted by timestamp, in descending order.
+        a = time()
+        for d in h:
+            b = a
+            a = d['time']
+            self.assertLess(a, b)
+        self.assertLess(start, a)

src/ZODB/tests/TransactionalUndoStorage.py

@@ -179,6 +179,10 @@ class TransactionalUndoStorage:
         info = self._storage.undoInfo()
         self._undo(info[2]['id'], [oid])
         self.assertRaises(KeyError, self._storage.load, oid, '')
+
+        # Loading current data via loadBefore should raise a POSKeyError too:
+        self.assertRaises(KeyError, self._storage.loadBefore, oid,
+                          b'\x7f\xff\xff\xff\xff\xff\xff\xff')
         self._iterate()
 
     def checkUndoCreationBranch2(self):
@@ -286,10 +290,9 @@ class TransactionalUndoStorage:
         t = Transaction()
         oids = self._begin_undos_vote(t, tid, tid1)
         self._storage.tpc_finish(t)
-        # We get the finalization stuff called an extra time:
-        eq(len(oids), 4)
-        unless(oid1 in oids)
-        unless(oid2 in oids)
+        # We may get the finalization stuff called an extra time,
+        # depending on the implementation.
+        self.assertEqual(set(oids), set((oid1, oid2)))
         data, revid1 = self._storage.load(oid1, '')
         eq(zodb_unpickle(data), MinPO(30))
         data, revid2 = self._storage.load(oid2, '')

src/ZODB/tests/testDemoStorage.py

@@ -14,6 +14,7 @@
 from ZODB.DB import DB
 from ZODB.tests import (
     BasicStorage,
+    ConflictResolution,
     HistoryStorage,
     IteratorStorage,
     MTStorage,
@@ -42,7 +43,7 @@ from zope.testing import renormalizing
 class DemoStorageTests(
     StorageTestBase.StorageTestBase,
     BasicStorage.BasicStorage,
-
+    ConflictResolution.ConflictResolvingStorage,
     HistoryStorage.HistoryStorage,
     IteratorStorage.ExtendedIteratorStorage,
     IteratorStorage.IteratorStorage,
@@ -86,6 +87,23 @@ class DemoStorageTests(
         pass # we don't support undo yet
     checkUndoZombie = checkLoadBeforeUndo
 
+    def checkBaseHistory(self):
+        def base_only():
+            yield 11
+            yield 12
+            yield 13
+            self._storage = self._storage.push()
+        self._checkHistory(base_only())
+        self._storage = self._storage.pop()
+        def base_and_changes():
+            yield 11
+            yield 12
+            self._storage = self._storage.push()
+            yield 13
+            yield 14
+        self._checkHistory(base_and_changes())
+        self._storage = self._storage.pop()
+
 
 class DemoStorageHexTests(DemoStorageTests):
 
@@ -144,11 +162,11 @@ def testSomeDelegation():
     >>> class S:
     ...     def __init__(self, name):
     ...         self.name = name
-    ...     def registerDB(self, db):
-    ...         six.print_(self.name, db)
+    ...     def getSize(self):
+    ...         six.print_(self.name, 'size')
     ...     def close(self):
     ...         six.print_(self.name, 'closed')
-    ...     sortKey = getSize = __len__ = history = getTid = None
+    ...     sortKey = __len__ = getTid = None
     ...     tpc_finish = tpc_vote = tpc_transaction = None
     ...     _lock_acquire = _lock_release = lambda self: None
     ...     getName = lambda self: 'S'
@@ -165,8 +183,8 @@ def testSomeDelegation():
     >>> from ZODB.DemoStorage import DemoStorage
     >>> storage = DemoStorage(base=S(1), changes=S(2))
 
-    >>> storage.registerDB(1)
-    2 1
+    >>> storage.getSize()
+    2 size
 
     >>> storage.close()
     1 closed