pack

neo/client/Storage.py

@@ -20,6 +20,7 @@ from zope.interface import implementer
 import ZODB.interfaces
 
 from neo.lib import logging
+from neo.lib.util import tidFromTime
 from .app import Application
 from .exception import NEOStorageNotFoundError, NEOStorageDoesNotExistError
 
@@ -235,7 +236,7 @@ class Storage(BaseStorage.BaseStorage,
             logging.warning('Garbage Collection is not available in NEO,'
                 ' please use an external tool. Packing without GC.')
         try:
-            self.app.pack(t)
+            self.app.pack(tidFromTime(t))
         except Exception:
             logging.exception('pack_time=%r', t)
             raise

neo/client/__init__.py

@@ -28,19 +28,24 @@ def patch():
     # successful commit (which ends with a response from the master) already
     # acts as a "network barrier".
     # BBB: What this monkey-patch does has been merged in ZODB5.
-    if not hasattr(Connection, '_flush_invalidations'):
-        return
-
-    assert H(Connection.afterCompletion) in (
-        'cd3a080b80fd957190ff3bb867149448', # Python 2.7
-        'b1d9685c13967d4b6d74c7ef86f68f17', # PyPy 2.7
-        )
-
-    def afterCompletion(self, *ignored):
-        self._readCurrent.clear()
-        # PATCH: do not call sync()
-        self._flush_invalidations()
-    Connection.afterCompletion = afterCompletion
+    if hasattr(Connection, '_flush_invalidations'):
+        assert H(Connection.afterCompletion) in (
+            'cd3a080b80fd957190ff3bb867149448', # Python 2.7
+            'b1d9685c13967d4b6d74c7ef86f68f17', # PyPy 2.7
+            )
+        def afterCompletion(self, *ignored):
+            self._readCurrent.clear()
+            # PATCH: do not call sync()
+            self._flush_invalidations()
+        Connection.afterCompletion = afterCompletion
+
+    global TransactionMetaData
+    try:
+        from ZODB.Connection import TransactionMetaData
+    except ImportError: # BBB: ZODB < 5
+        from ZODB.BaseStorage import TransactionRecord
+        TransactionMetaData = lambda user='', description='', extension=None: \
+            TransactionRecord(None, None, user, description, extension)
 
 patch()
 

neo/client/app.py

@@ -25,7 +25,6 @@ except ImportError:
     from cPickle import dumps, loads
     _protocol = 1
 from ZODB.POSException import UndoError, ConflictError, ReadConflictError
-from persistent.TimeStamp import TimeStamp
 
 from neo.lib import logging
 from neo.lib.compress import decompress_list, getCompress
@@ -35,6 +34,7 @@ from neo.lib.util import makeChecksum, dump
 from neo.lib.locking import Empty, Lock
 from neo.lib.connection import MTClientConnection, ConnectionClosed
 from neo.lib.exception import NodeNotReady
+from . import TransactionMetaData
 from .exception import (NEOStorageError, NEOStorageCreationUndoneError,
     NEOStorageReadRetry, NEOStorageNotFoundError, NEOPrimaryMasterLost)
 from .handlers import storage, master
@@ -49,6 +49,8 @@ CHECKED_SERIAL = object()
 # failed in the past.
 MAX_FAILURE_AGE = 600
 
+TXN_PACK_DESC = 'IStorage.pack'
+
 try:
     from Signals.Signals import SignalHandler
 except ImportError:
@@ -64,6 +66,8 @@ class Application(ThreadedApplication):
     # the transaction is really committed, no matter for how long the master
     # is unreachable.
     max_reconnection_to_master = float('inf')
+    # For tests only. See end of pack() method.
+    wait_for_pack = False
 
     def __init__(self, master_nodes, name, compress=True, cache_size=None,
                  **kw):
@@ -590,7 +594,8 @@ class Application(ThreadedApplication):
         # user and description are cast to str in case they're unicode.
         # BBB: This is not required anymore with recent ZODB.
         packet = Packets.AskStoreTransaction(ttid, str(transaction.user),
-            str(transaction.description), ext, list(txn_context.cache_dict))
+            str(transaction.description), ext, list(txn_context.cache_dict),
+            txn_context.pack)
         queue = txn_context.queue
         conn_dict = txn_context.conn_dict
         # Ask in parallel all involved storage nodes to commit object metadata.
@@ -705,7 +710,7 @@ class Application(ThreadedApplication):
             del cache_dict[oid]
         ttid = txn_context.ttid
         p = Packets.AskFinishTransaction(ttid, list(cache_dict),
-                                         checked_list)
+                                         checked_list, txn_context.pack)
         try:
             tid = self._askPrimary(p, cache_dict=cache_dict, callback=f)
             assert tid
@@ -977,12 +982,16 @@ class Application(ThreadedApplication):
     def sync(self):
         self._askPrimary(Packets.Ping())
 
-    def pack(self, t):
-        tid = TimeStamp(*time.gmtime(t)[:5] + (t % 60, )).raw()
-        if tid == ZERO_TID:
-            raise NEOStorageError('Invalid pack time')
-        self._askPrimary(Packets.AskPack(tid))
-        # XXX: this is only needed to make ZODB unit tests pass.
+    def pack(self, tid, _oids=None): # TODO: API for partial pack
+        transaction = TransactionMetaData(description=TXN_PACK_DESC)
+        self.tpc_begin(None, transaction)
+        self._txn_container.get(transaction).pack = _oids and sorted(_oids), tid
+        tid = self.tpc_finish(transaction)
+        if not self.wait_for_pack:
+            return
+        # Waiting for pack to be finished is only needed
+        # to make ZODB unit tests pass.
+        self._askPrimary(Packets.WaitForPack(tid))
         # It should not be otherwise required (clients should be free to load
         # old data as long as it is available in cache, event if it was pruned
         # by a pack), so don't bother invalidating on other clients.

neo/client/exception.py

@@ -37,6 +37,13 @@ class NEOStorageCreationUndoneError(NEOStorageDoesNotExistError):
     some object existed at some point, but its creation was undone.
     """
 
+class NEOUndoPackError(NEOStorageNotFoundError):
+    """Race condition between undo & pack
+
+    While undoing a transaction, an oid record disappeared.
+    This can happen if the storage node is packing.
+    """
+
 # TODO: Inherit from transaction.interfaces.TransientError
 #       (not recognized yet by ERP5 as a transient error).
 class NEOPrimaryMasterLost(POSException.ReadConflictError):

neo/client/handlers/master.py

@@ -174,3 +174,6 @@ class PrimaryAnswersHandler(AnswerBaseHandler):
 
     def answerFinalTID(self, conn, tid):
         self.app.setHandlerData(tid)
+
+    def waitedForPack(self, conn):
+        pass

neo/client/handlers/storage.py

@@ -25,8 +25,10 @@ from neo.lib.exception import NodeNotReady
 from neo.lib.handler import MTEventHandler
 from . import AnswerBaseHandler
 from ..transactions import Transaction
-from ..exception import NEOStorageError, NEOStorageNotFoundError
-from ..exception import NEOStorageReadRetry, NEOStorageDoesNotExistError
+from ..exception import (
+    NEOStorageError, NEOStorageNotFoundError, NEOUndoPackError,
+    NEOStorageReadRetry, NEOStorageDoesNotExistError,
+)
 
 @apply
 class _DeadlockPacket(object):
@@ -194,6 +196,9 @@ class StorageAnswersHandler(AnswerBaseHandler):
         # This can happen when requiring txn informations
         raise NEOStorageNotFoundError(message)
 
+    def undoPackError(self, conn, message):
+        raise NEOUndoPackError(message)
+
     def nonReadableCell(self, conn, message):
         logging.info('non readable cell')
         raise NEOStorageReadRetry(True)

neo/client/transactions.py

@@ -31,6 +31,7 @@ class Transaction(object):
     voted = False
     ttid = None     # XXX: useless, except for testBackupReadOnlyAccess
     lockless_dict = None                    # {partition: {uuid}}
+    pack = None
 
     def __init__(self, txn):
         self.queue = SimpleQueue()

neo/lib/connection.py

@@ -600,11 +600,40 @@ class Connection(BaseConnection):
         packet.setId(self.peer_id)
         self._addPacket(packet)
 
+    def delayedAnswer(self, packet):
+        return DelayedAnswer(self, packet)
+
     def _connected(self):
         self.connecting = False
         self.getHandler().connectionCompleted(self)
 
 
+class DelayedAnswer(object):
+
+    def __init__(self, conn, packet):
+        assert packet.isResponse() and not packet.isError(), packet
+        self.conn = conn
+        self.packet = packet
+        self.msg_id = conn.peer_id
+
+    def __call__(self, *args):
+        # Same behaviour as Connection.answer for closed connections.
+        # Not more tolerant, because connections are expected to be properly
+        # cleaned up when they're closed (__eq__/__hash__ help to identify
+        # instances that are related to the connection being closed).
+        try:
+            self.conn.send(self.packet(*args), self.msg_id)
+        except ConnectionClosed:
+            if self.packet.ignoreOnClosedConnection():
+                raise
+
+    def __hash__(self):
+        return hash(self.conn)
+
+    def __eq__(self, other):
+        return self is other or self.conn is other
+
+
 class ClientConnection(Connection):
     """A connection from this node to a remote node."""
 

neo/lib/event.py

@@ -63,6 +63,8 @@ class EpollEventManager(object):
         assert fd == -1, fd
         self.epoll.register(r, EPOLLIN)
         self._trigger_lock = Lock()
+        self.lock = l = Lock()
+        l.acquire()
         close_list = []
         self._closeAppend = close_list.append
         l = Lock()
@@ -207,6 +209,15 @@ class EpollEventManager(object):
             # granularity of 1ms and Python 2.7 rounds the timeout towards zero.
             # See also https://bugs.python.org/issue20452 (fixed in Python 3).
             blocking = .001 + max(0, timeout - time()) if timeout else -1
+            def poll(blocking):
+                l = self.lock
+                l.release()
+                try:
+                    return self.epoll.poll(blocking)
+                finally:
+                    l.acquire()
+        else:
+            poll = self.epoll.poll
         # From this point, and until we have processed all fds returned by
         # epoll, we must prevent any fd from being closed, because they could
         # be reallocated by new connection, either by this thread or by another.
@@ -214,7 +225,7 @@ class EpollEventManager(object):
         # 'finally' clause.
         self._closeAcquire()
         try:
-            event_list = self.epoll.poll(blocking)
+            event_list = poll(blocking)
         except IOError, exc:
             if exc.errno in (0, EAGAIN):
                 logging.info('epoll.poll triggered undocumented error %r',

neo/lib/exception.py

@@ -56,3 +56,6 @@ class NonReadableCell(NeoException):
 
     On such event, the client must retry, preferably another cell.
     """
+
+class UndoPackError(NeoException):
+    pass

neo/lib/protocol.py

@@ -173,6 +173,7 @@ def ErrorCodes():
     NON_READABLE_CELL
     READ_ONLY_ACCESS
     INCOMPLETE_TRANSACTION
+    UNDO_PACK_ERROR
 
 @Enum
 def NodeStates():
@@ -273,21 +274,24 @@ class Packet(object):
         assert isinstance(other, Packet)
         return self._code == other._code
 
-    def isError(self):
-        return self._code == RESPONSE_MASK
+    @classmethod
+    def isError(cls):
+        return cls._code == RESPONSE_MASK
 
-    def isResponse(self):
-        return self._code & RESPONSE_MASK
+    @classmethod
+    def isResponse(cls):
+        return cls._code & RESPONSE_MASK
 
     def getAnswerClass(self):
         return self._answer
 
-    def ignoreOnClosedConnection(self):
+    @classmethod
+    def ignoreOnClosedConnection(cls):
         """
         Tells if this packet must be ignored when its connection is closed
         when it is handled.
         """
-        return self._ignore_when_closed
+        return cls._ignore_when_closed
 
 
 class PacketRegistryFactory(dict):
@@ -669,11 +673,31 @@ class Packets(dict):
         :nodes: C -> S
         """)
 
-    AskPack, AnswerPack = request("""
-        Request a pack at given TID.
+    WaitForPack, WaitedForPack = request("""
+        Wait until pack given by tid is completed.
 
-        :nodes: C -> M -> S
-        """, ignore_when_closed=False)
+        :nodes: C -> M
+        """)
+
+    AskPackOrders, AnswerPackOrders = request("""
+        Request list of pack orders excluding oldest completed ones.
+
+        :nodes: M -> S; C, S -> M
+        """)
+
+    NotifyPackSigned = notify("""
+        Send ids of pack orders to be processed. Also used to fix replicas
+        that may have lost them.
+
+        :nodes: M -> S, backup
+        """)
+
+    NotifyPackCompleted = notify("""
+        Notify the master node that partitions have been successfully
+        packed up to the given ids.
+
+        :nodes: S -> M
+        """)
 
     CheckReplicas = request("""
         Ask the cluster to search for mismatches between replicas, metadata

neo/lib/pt.py

@@ -30,9 +30,11 @@ class PartitionTableException(Exception):
 class Cell(object):
     """This class represents a cell in a partition table."""
 
+    state = CellStates.DISCARDED
+
     def __init__(self, node, state = CellStates.UP_TO_DATE):
         self.node = node
-        self.state = state
+        self.setState(state)
 
     def __repr__(self):
         return "<Cell(uuid=%s, address=%s, state=%s)>" % (

neo/lib/util.py

@@ -101,6 +101,9 @@ def datetimeFromTID(tid):
     seconds, lower = divmod(lower * 60, TID_LOW_OVERFLOW)
     return datetime(*(higher + (seconds, int(lower * MICRO_FROM_UINT32))))
 
+def timeFromTID(tid, _epoch=datetime.utcfromtimestamp(0)):
+    return (datetimeFromTID(tid) - _epoch).total_seconds()
+
 def addTID(ptid, offset):
     """
     Offset given packed TID.

neo/master/app.py

@@ -42,6 +42,7 @@ def monotonic_time():
 
 from .backup_app import BackupApplication
 from .handlers import identification, administration, client, master, storage
+from .pack import PackManager
 from .pt import PartitionTable
 from .recovery import RecoveryManager
 from .transactions import TransactionManager
@@ -51,7 +52,6 @@ from .verification import VerificationManager
 @buildOptionParser
 class Application(BaseApplication):
     """The master node application."""
-    packing = None
     storage_readiness = 0
     # Latest completely committed TID
     last_transaction = ZERO_TID
@@ -101,6 +101,7 @@ class Application(BaseApplication):
         super(Application, self).__init__(
             config.get('ssl'), config.get('dynamic_master_list'))
         self.tm = TransactionManager(self.onTransactionCommitted)
+        self.pm = PackManager()
 
         self.name = config['cluster']
         self.server = config['bind']
@@ -317,6 +318,8 @@ class Application(BaseApplication):
                     truncate = Packets.Truncate(*e.args) if e.args else None
                     # Automatic restart except if we truncate or retry to.
                     self._startup_allowed = not (self.truncate_tid or truncate)
+                finally:
+                    self.pm.reset()
                 self.storage_readiness = 0
                 self.storage_ready_dict.clear()
                 self.storage_starting_set.clear()
@@ -560,7 +563,8 @@ class Application(BaseApplication):
         tid = txn.getTID()
         transaction_node = txn.getNode()
         invalidate_objects = Packets.InvalidateObjects(tid, txn.getOIDList())
-        for client_node in self.nm.getClientList(only_identified=True):
+        client_list = self.nm.getClientList(only_identified=True)
+        for client_node in client_list:
             if client_node is transaction_node:
                 client_node.send(Packets.AnswerTransactionFinished(ttid, tid),
                                  msg_id=txn.getMessageId())
@@ -570,9 +574,26 @@ class Application(BaseApplication):
         # Unlock Information to relevant storage nodes.
         notify_unlock = Packets.NotifyUnlockInformation(ttid)
         getByUUID = self.nm.getByUUID
-        for storage_uuid in txn.getUUIDList():
+        txn_storage_list = txn.getUUIDList()
+        for storage_uuid in txn_storage_list:
             getByUUID(storage_uuid).send(notify_unlock)
 
+        # Notify storage nodes about new pack order if any.
+        pack = self.pm.packs.get(tid)
+        if pack is not None is not pack.approved:
+            # We could exclude those that store transaction metadata, because
+            # they can deduce it upon NotifyUnlockInformation: quite simple but
+            # for the moment, let's optimize the case where there's no pack.
+            # We're only there in case of automatic approval.
+            assert pack.approved
+            pack = Packets.NotifyPackSigned((tid,), ())
+            for uuid in self.getStorageReadySet():
+                getByUUID(uuid).send(pack)
+            # Notify backup clusters.
+            for node in client_list:
+                if node.extra.get('backup'):
+                    node.send(pack)
+
         # Notify storage that have replications blocked by this transaction,
         # and clients that try to recover from a failure during tpc_finish.
         notify_finished = Packets.NotifyTransactionFinished(ttid, tid)
@@ -612,6 +633,9 @@ class Application(BaseApplication):
         assert uuid not in self.storage_ready_dict, self.storage_ready_dict
         self.storage_readiness = self.storage_ready_dict[uuid] = \
             self.storage_readiness + 1
+        pack = self.pm.getApprovedRejected()
+        if any(pack):
+            self.nm.getByUUID(uuid).send(Packets.NotifyPackSigned(*pack))
         self.tm.executeQueuedEvents()
 
     def isStorageReady(self, uuid):
@@ -629,3 +653,11 @@ class Application(BaseApplication):
             getByUUID = self.nm.getByUUID
             for uuid in uuid_set:
                 getByUUID(uuid).send(p)
+
+    def updateCompletedPackId(self):
+        try:
+            pack_id = min(node.completed_pack_id
+                for node in self.pt.getNodeSet(True))
+        except AttributeError:
+            return
+        self.pm.notifyCompleted(pack_id)

neo/master/backup_app.py

@@ -75,6 +75,7 @@ class BackupApplication(object):
         self.nm.createMasters(master_addresses)
 
     em = property(lambda self: self.app.em)
+    pm = property(lambda self: self.app.pm)
     ssl = property(lambda self: self.app.ssl)
 
     def close(self):
@@ -117,8 +118,19 @@ class BackupApplication(object):
                     app.changeClusterState(ClusterStates.BACKINGUP)
                     del bootstrap, node
                     self.ignore_invalidations = True
+                    self.ignore_pack_notifications = True
                     conn.setHandler(BackupHandler(self))
                     conn.ask(Packets.AskLastTransaction())
+                    assert app.backup_tid == pt.getBackupTid()
+                    min_tid = add64(app.backup_tid, 1)
+                    p = app.pm.packs
+                    for tid in sorted(p):
+                        if min_tid <= tid:
+                            break
+                        if p[tid].approved is None:
+                            min_tid = tid
+                            break
+                    conn.ask(Packets.AskPackOrders(min_tid), min_tid=min_tid)
                     # debug variable to log how big 'tid_list' can be.
                     self.debug_tid_count = 0
                     while True:
@@ -375,3 +387,12 @@ class BackupApplication(object):
         if node_list:
             min(node_list, key=lambda node: node.getUUID()).send(
                 Packets.NotifyUpstreamAdmin(addr))
+
+    def broadcastApprovedRejected(self, min_tid):
+        app = self.app
+        p = app.pm.getApprovedRejected(min_tid)
+        if any(p):
+            getByUUID = app.nm.getByUUID
+            p = Packets.NotifyPackSigned(*p)
+            for uuid in app.getStorageReadySet():
+                getByUUID(uuid).send(p)

neo/master/handlers/__init__.py

@@ -15,10 +15,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from ..app import monotonic_time
+from ..pack import RequestOld
 from neo.lib import logging
 from neo.lib.exception import StoppedOperation
 from neo.lib.handler import EventHandler
-from neo.lib.protocol import Packets
+from neo.lib.protocol import Packets, ZERO_TID
 
 class MasterHandler(EventHandler):
     """This class implements a generic part of the event handlers."""
@@ -40,12 +41,21 @@ class MasterHandler(EventHandler):
 
     def askLastIDs(self, conn):
         tm = self.app.tm
-        conn.answer(Packets.AnswerLastIDs(tm.getLastOID(), tm.getLastTID()))
+        conn.answer(Packets.AnswerLastIDs(tm.getLastTID(), tm.getLastOID()))
 
     def askLastTransaction(self, conn):
         conn.answer(Packets.AnswerLastTransaction(
             self.app.getLastTransaction()))
 
+    def _askPackOrders(self, conn, pack_id, only_first_approved):
+        app = self.app
+        if pack_id is not None is not app.pm.max_completed >= pack_id:
+            RequestOld(app, pack_id, only_first_approved,
+                conn.delayedAnswer(Packets.AnswerPackOrders))
+        else:
+            conn.answer(Packets.AnswerPackOrders(
+                app.pm.dump(pack_id or ZERO_TID, only_first_approved)))
+
     def _notifyNodeInformation(self, conn):
         app = self.app
         node = app.nm.getByUUID(conn.getUUID())

neo/master/handlers/backup.py

@@ -16,7 +16,7 @@
 
 from neo.lib.exception import PrimaryFailure
 from neo.lib.handler import EventHandler
-from neo.lib.protocol import NodeTypes, NodeStates, Packets, ZERO_TID
+from neo.lib.protocol import NodeTypes, NodeStates, Packets
 from neo.lib.pt import PartitionTable
 
 class BackupHandler(EventHandler):
@@ -72,3 +72,41 @@ class BackupHandler(EventHandler):
         partition_set.add(getPartition(tid))
         prev_tid = app.app.getLastTransaction()
         app.invalidatePartitions(tid, prev_tid, partition_set)
+
+    # The following 2 methods:
+    # - keep the PackManager up-to-date;
+    # - replicate the status of pack orders when they're known after the
+    #   storage nodes have fetched related transactions.
+
+    def notifyPackSigned(self, conn, approved, rejected):
+        backup_app = self.app
+        if backup_app.ignore_pack_notifications:
+            return
+        app = backup_app.app
+        packs = app.pm.packs
+        ask_tid = min_tid = None
+        for approved, tid in (True, approved), (False, rejected):
+            try:
+                packs[tid].approved = approved
+            except KeyError:
+                if not ask_tid or tid < ask_tid:
+                    ask_tid = tid
+            else:
+                if not min_tid or tid < min_tid:
+                    min_tid = tid
+        if min_tid:
+            backup_app.broadcastApprovedRejected(min_tid)
+        if ask_tid:
+            conn.ask(Packets.AskPackOrders(ask_tid), min_tid=ask_tid)
+
+    def answerPackOrders(self, conn, pack_list, min_tid):
+        backup_app = self.app
+        app = backup_app.app
+        add = app.pm.add
+        for pack_order in pack_list:
+            add(*pack_order)
+        if min_tid < app.getLastTransaction():
+            backup_app.broadcastApprovedRejected(min_tid)
+        backup_app.ignore_pack_notifications = False
+
+    ###

neo/master/handlers/client.py

@@ -32,6 +32,7 @@ class ClientServiceHandler(MasterHandler):
         app = self.app
         node = app.nm.getByUUID(conn.getUUID())
         assert node is not None, conn
+        app.pm.clientLost(conn)
         for x in app.tm.clientLost(node):
             app.notifyTransactionAborted(*x)
         node.setUnknown()
@@ -63,7 +64,7 @@ class ClientServiceHandler(MasterHandler):
         conn.answer((Errors.Ack if app.tm.vote(app, *args) else
                      Errors.IncompleteTransaction)())
 
-    def askFinishTransaction(self, conn, ttid, oid_list, checked_list):
+    def askFinishTransaction(self, conn, ttid, oid_list, checked_list, pack):
         app = self.app
         tid, node_list = app.tm.prepare(
             app,
@@ -73,7 +74,8 @@ class ClientServiceHandler(MasterHandler):
             conn.getPeerId(),
         )
         if tid:
-            p = Packets.AskLockInformation(ttid, tid)
+            p = Packets.AskLockInformation(ttid, tid,
+                app.pm.new(tid, *pack) if pack else False)
             for node in node_list:
                 node.ask(p)
         else:
@@ -100,18 +102,6 @@ class ClientServiceHandler(MasterHandler):
             tid = MAX_TID
         conn.answer(Packets.AnswerFinalTID(tid))
 
-    def askPack(self, conn, tid):
-        app = self.app
-        if app.packing is None:
-            storage_list = app.nm.getStorageList(only_identified=True)
-            app.packing = (conn, conn.getPeerId(),
-                {x.getUUID() for x in storage_list})
-            p = Packets.AskPack(tid)
-            for storage in storage_list:
-                storage.getConnection().ask(p)
-        else:
-            conn.answer(Packets.AnswerPack(False))
-
     def abortTransaction(self, conn, tid, uuid_list):
         # Consider a failure when the connection between the storage and the
         # client breaks while the answer to the first write is sent back.
@@ -126,6 +116,16 @@ class ClientServiceHandler(MasterHandler):
         involved.update(uuid_list)
         app.notifyTransactionAborted(tid, involved)
 
+    def askPackOrders(self, conn, pack_id):
+        return self._askPackOrders(conn, pack_id, False)
+
+    def waitForPack(self, conn, tid):
+        try:
+            pack = self.app.pm.packs[tid]
+        except KeyError:
+            conn.answer(Packets.WaitedForPack())
+        else:
+            pack.waitForPack(conn.delayedAnswer(Packets.WaitedForPack))
 
 # like ClientServiceHandler but read-only & only for tid <= backup_tid
 class ClientReadOnlyServiceHandler(ClientServiceHandler):

neo/master/handlers/storage.py

@@ -43,14 +43,14 @@ class StorageServiceHandler(BaseServiceHandler):
         super(StorageServiceHandler, self).connectionLost(conn, new_state)
         app.setStorageNotReady(uuid)
         app.tm.storageLost(uuid)
+        app.pm.connectionLost(conn)
+        app.updateCompletedPackId()
         if (app.getClusterState() == ClusterStates.BACKINGUP
             # Also check if we're exiting, because backup_app is not usable
             # in this case. Maybe cluster state should be set to something
             # else, like STOPPING, during cleanup (__del__/close).
             and app.listening_conn):
             app.backup_app.nodeLost(node)
-        if app.packing is not None:
-            self.answerPack(conn, False)
 
     def askUnfinishedTransactions(self, conn, offset_list):
         app = self.app
@@ -108,13 +108,13 @@ class StorageServiceHandler(BaseServiceHandler):
                       uuid_str(uuid), offset, dump(tid))
         self.app.broadcastPartitionChanges(cell_list)
 
-    def answerPack(self, conn, status):
+    def notifyPackCompleted(self, conn, pack_id):
         app = self.app
-        if app.packing is not None:
-            client, msg_id, uid_set = app.packing
-            uid_set.remove(conn.getUUID())
-            if not uid_set:
-                app.packing = None
-                if not client.isClosed():
-                    client.send(Packets.AnswerPack(True), msg_id)
+        app.nm.getByUUID(conn.getUUID()).completed_pack_id = pack_id
+        app.updateCompletedPackId()
 
+    def askPackOrders(self, conn, pack_id):
+        return self._askPackOrders(conn, pack_id, True)
+
+    def answerPackOrders(self, conn, pack_list, process):
+        process(pack_list)

neo/master/pack.py

+#
+# Copyright (C) 2021  Nexedi SA
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# IDEA: Keep minimal information to avoid useless memory usage, e.g. with
+#       arbitrary data large like a list of OIDs. Only {tid: id} is important:
+#       everything could be queried from storage nodes when needed. Note
+#       however that extra information allows the master to automatically drop
+#       redundant pack orders: keeping partial/time may be an acceptable cost.
+
+from collections import defaultdict
+from functools import partial
+from operator import attrgetter
+from weakref import proxy
+from neo.lib.protocol import Packets, ZERO_TID
+from neo.lib.util import add64
+
+
+class Pack(object):
+
+    def __init__(self, tid, approved, partial, oids, time):
+        self.tid = tid
+        self.approved = approved
+        self.partial = partial
+        self.oids = oids
+        self.time = time
+        self._waiting = []
+
+    @property
+    def waitForPack(self):
+        return self._waiting.append
+
+    def completed(self):
+        for callback in self._waiting:
+            callback()
+        del self._waiting
+
+    def connectionLost(self, conn):
+        try:
+            self._waiting.remove(conn)
+        except ValueError:
+            pass
+
+
+class RequestOld(object):
+
+    caller = None
+
+    def __init__(self, app, pack_id, only_first_approved, caller):
+        self.app = proxy(app)
+        self.caller = caller
+        self.pack_id = pack_id
+        self.only_first_approved = only_first_approved
+        self.offsets = set(xrange(app.pt.getPartitions()))
+        self.packs = []
+        # In case that the PT changes, we may ask a node again before it
+        # replies to previous requests, so we can't simply use its id as key.
+        self.querying = set()
+        app.pm.old.append(self)
+        self._ask()
+
+    def connectionLost(self, conn):
+        if self.caller != conn:
+            nid = conn.getUUID()
+            x = [x for x in self.querying if x[0] == nid]
+            if x:
+                self.querying.difference_update(x)
+                self._ask()
+            return True
+        self.__dict__.clear()
+
+    def _ask(self):
+        getCellList = self.app.pt.getCellList
+        readable = defaultdict(list)
+        for offset in self.offsets:
+            for cell in getCellList(offset, True):
+                readable[cell.getUUID()].append(offset)
+        offsets = self.offsets.copy()
+        for x in self.querying:
+            offsets.difference_update(x[1])
+        p = Packets.AskPackOrders(self.pack_id)
+        while offsets:
+            node = getCellList(offsets.pop(), True)[0].getNode()
+            nid = node.getUUID()
+            x = tuple(readable.pop(nid))
+            offsets.difference_update(x)
+            x = nid, x
+            self.querying.add(x)
+            node.ask(p, process=partial(self._answer, x))
+
+    def _answer(self, nid_offsets, pack_list):
+        caller = self.caller
+        if caller:
+            self.querying.remove(nid_offsets)
+            self.offsets.difference_update(nid_offsets[1])
+            self.packs += pack_list
+            if self.offsets:
+                self._ask()
+            else:
+                del self.caller
+                app = self.app
+                pm = app.pm
+                tid = self.pack_id
+                pm.max_completed = add64(tid, -1)
+                for pack_order in self.packs:
+                    pm.add(*pack_order)
+                caller(pm.dump(tid, self.only_first_approved))
+                app.updateCompletedPackId()
+
+
+class PackManager(object):
+
+    autosign = True
+
+    def __init__(self):
+        self.max_completed = None
+        self.packs = {}
+        self.old = []
+
+    reset = __init__
+
+    def add(self, tid, *args):
+        p = self.packs.get(tid)
+        if p is None:
+            self.packs[tid] = Pack(tid, *args)
+            if None is not self.max_completed > tid:
+                self.max_completed = add64(tid, -1)
+        elif p.approved is None:
+            p.approved = args[0]
+
+    @apply
+    def dump():
+        by_tid = attrgetter('tid')
+        def dump(self, pack_id, only_first_approved):
+            if only_first_approved:
+                try:
+                    p = min((p for p in self.packs.itervalues()
+                           if p.approved and p.tid >= pack_id),
+                        key=by_tid),
+                except ValueError:
+                    p = ()
+            else:
+                p = sorted(
+                    (p for p in self.packs.itervalues() if p.tid >= pack_id),
+                    key=by_tid)
+            return [(p.tid, p.approved, p.partial, p.oids, p.time) for p in p]
+        return dump
+
+    def new(self, tid, oids, time):
+        autosign = self.autosign and None not in (
+            p.approved for p in self.packs.itervalues())
+        self.packs[tid] = Pack(tid, autosign or None, bool(oids), oids, time)
+        return autosign
+
+    def getApprovedRejected(self, above_tid=ZERO_TID):
+        r = [], []
+        for tid, p in self.packs.iteritems():
+            if above_tid < tid:
+                approved = p.approved
+                if approved is not None:
+                    r[0 if approved else 1].append(tid)
+        if any(r):
+            return r
+        return (self.max_completed,) if self.max_completed else (), ()
+
+    def notifyCompleted(self, pack_id):
+        for tid in list(self.packs):
+            if tid <= pack_id:
+                self.packs.pop(tid).completed()
+                if self.max_completed is None or self.max_completed < tid:
+                    self.max_completed = tid
+
+    def clientLost(self, conn):
+        for p in self.packs.itervalues():
+            p.connectionLost(conn)
+        self.connectionLost(conn)
+
+    def connectionLost(self, conn):
+        self.old = [old for old in self.old if old.connectionLost(conn)]

neo/master/verification.py

@@ -70,6 +70,10 @@ class VerificationManager(BaseServiceHandler):
         app.setLastTransaction(app.tm.getLastTID())
         # Just to not return meaningless information in AnswerRecovery.
         app.truncate_tid = None
+        # Set up pack manager.
+        node_set = app.pt.getNodeSet(readable=True)
+        pack_id = min(node.completed_pack_id for node in node_set)
+        self._askStorageNodesAndWait(Packets.AskPackOrders(pack_id), node_set)
 
     def verifyData(self):
         app = self.app
@@ -126,11 +130,20 @@ class VerificationManager(BaseServiceHandler):
                 for node in getIdentifiedList(pool_set=uuid_set):
                     node.send(packet)
 
-    def answerLastIDs(self, conn, loid, ltid):
+    def notifyPackCompleted(self, conn, pack_id):
+        self.app.nm.getByUUID(conn.getUUID()).completed_pack_id = pack_id
+
+    def answerLastIDs(self, conn, ltid, loid):
         self._uuid_set.remove(conn.getUUID())
         tm = self.app.tm
-        tm.setLastOID(loid)
         tm.setLastTID(ltid)
+        tm.setLastOID(loid)
+
+    def answerPackOrders(self, conn, pack_list):
+        self._uuid_set.remove(conn.getUUID())
+        add = self.app.pm.add
+        for pack_order in pack_list:
+            add(*pack_order)
 
     def answerLockedTransactions(self, conn, tid_dict):
         uuid = conn.getUUID()

neo/neoctl/app.py

@@ -103,7 +103,7 @@ class TerminalNeoCTL(object):
             r = "backup_tid = 0x%x (%s)" % (u64(backup_tid),
                                             datetimeFromTID(backup_tid))
         else:
-            loid, ltid = self.neoctl.getLastIds()
+            ltid, loid = self.neoctl.getLastIds()
             r = "last_oid = 0x%x" % (u64(loid))
         return r + "\nlast_tid = 0x%x (%s)\nlast_ptid = %s" % \
                                     (u64(ltid), datetimeFromTID(ltid), ptid)

neo/scripts/runner.py

@@ -48,16 +48,13 @@ UNIT_TEST_MODULES = [
     'neo.tests.testUtil',
     'neo.tests.testPT',
     # master application
-    'neo.tests.master.testClientHandler',
     'neo.tests.master.testMasterApp',
     'neo.tests.master.testMasterPT',
-    'neo.tests.master.testStorageHandler',
     'neo.tests.master.testTransactions',
     # storage application
     'neo.tests.storage.testClientHandler',
     'neo.tests.storage.testMasterHandler',
     'neo.tests.storage.testStorage' + os.getenv('NEO_TESTS_ADAPTER', 'SQLite'),
-    'neo.tests.storage.testTransactions',
     # client application
     'neo.tests.client.testClientApp',
     'neo.tests.client.testMasterHandler',
@@ -66,6 +63,7 @@ UNIT_TEST_MODULES = [
     'neo.tests.threaded.test',
     'neo.tests.threaded.testConfig',
     'neo.tests.threaded.testImporter',
+    'neo.tests.threaded.testPack',
     'neo.tests.threaded.testReplication',
     'neo.tests.threaded.testSSL',
 ]

neo/storage/app.py

@@ -19,11 +19,12 @@ from collections import deque
 
 from neo.lib import logging
 from neo.lib.app import BaseApplication, buildOptionParser
-from neo.lib.protocol import CellStates, ClusterStates, NodeTypes, Packets
+from neo.lib.protocol import CellStates, ClusterStates, NodeTypes, Packets, \
+    ZERO_TID
 from neo.lib.connection import ListeningConnection
 from neo.lib.exception import StoppedOperation, PrimaryFailure
 from neo.lib.pt import PartitionTable
-from neo.lib.util import dump
+from neo.lib.util import add64, dump
 from neo.lib.bootstrap import BootstrapManager
 from .checker import Checker
 from .database import buildDatabaseManager, DATABASE_MANAGERS
@@ -132,6 +133,7 @@ class Application(BaseApplication):
                 logging.node(self.name, self.uuid)
 
         registerLiveDebugger(on_log=self.log)
+        self.dm.lock.release()
 
     def close(self):
         self.listening_conn = None
@@ -190,7 +192,8 @@ class Application(BaseApplication):
 
     def run(self):
         try:
-            self._run()
+            with self.dm.lock:
+                self._run()
         except Exception:
             logging.exception('Pre-mortem data:')
             self.log()
@@ -216,6 +219,8 @@ class Application(BaseApplication):
             if self.master_node is None:
                 # look for the primary master
                 self.connectToPrimary()
+            self.completed_pack_id = ZERO_TID
+            self.last_pack_id = None
             self.checker = Checker(self)
             self.replicator = Replicator(self)
             self.tm = TransactionManager(self)
@@ -281,17 +286,24 @@ class Application(BaseApplication):
 
         self.task_queue = task_queue = deque()
         try:
-            self.dm.doOperation(self)
-            while True:
-                while task_queue:
-                    try:
-                        while isIdle():
-                            next(task_queue[-1]) or task_queue.rotate()
-                            _poll(0)
-                        break
-                    except StopIteration:
-                        task_queue.pop()
-                poll()
+            with self.dm.operational(self):
+                with self.dm.lock:
+                    self.maybePack()
+                while True:
+                    if task_queue and isIdle():
+                        with self.dm.lock:
+                            while True:
+                                try:
+                                    next(task_queue[-1]) or task_queue.rotate()
+                                except StopIteration:
+                                    task_queue.pop()
+                                    if not task_queue:
+                                        break
+                                else:
+                                    _poll(0)
+                                if not isIdle():
+                                    break
+                    poll()
         finally:
             del self.task_queue
 
@@ -320,3 +332,36 @@ class Application(BaseApplication):
             self.dm.erase()
         logging.info("Application has been asked to shut down")
         sys.exit()
+
+    def notifyPackCompleted(self):
+        packed = self.dm.getPackedIDs()
+        if packed:
+            pack_id = min(packed.itervalues())
+            if self.completed_pack_id != pack_id:
+                self.completed_pack_id = pack_id
+                self.master_conn.send(Packets.NotifyPackCompleted(pack_id))
+
+    def maybePack(self, info=None, min_id=None):
+        ready = self.dm.isReadyToStartPack()
+        if ready:
+            packed_dict = self.dm.getPackedIDs(True)
+            if packed_dict:
+                packed = min(packed_dict.itervalues())
+                if packed < self.last_pack_id:
+                    if packed == ready[1]:
+                        pack_id = ready[0]
+                    elif packed == min_id:
+                        pack_id = info[0]
+                    else:
+                        self.master_conn.ask(
+                            Packets.AskPackOrders(add64(packed, 1)),
+                            pack_id=packed)
+                        return
+                    self.dm.pack(self, info, packed,
+                        self.replicator.filterPackable(pack_id,
+                            (k for k, v in packed_dict.iteritems()
+                                if v == packed)))
+                else:
+                    self.dm.pack(self, None, None, ()) # for cleanup
+            else:
+                assert not self.pt.getReadableOffsetList(self.uuid)

neo/storage/database/__init__.py

@@ -14,6 +14,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import time
+
 LOG_QUERIES = False
 
 def useMySQLdb():
@@ -65,5 +67,25 @@ DATABASE_MANAGERS = tuple(sorted(
 def buildDatabaseManager(name, args=(), kw={}):
     return getAdapterKlass(name)(*args, **kw)
 
+
 class DatabaseFailure(Exception):
-    pass
+
+    transient_failure = False
+
+    if __debug__:
+        def getFailingDatabaseManager(self):
+            pass
+
+    def logTransientFailure(self):
+        raise NotImplementedError
+
+    def checkTransientFailure(self, dm):
+        if dm.LOCK or not self.transient_failure:
+            raise
+        assert dm is self.getFailingDatabaseManager()
+        dm.close()
+        self.logTransientFailure()
+        # Avoid reconnecting too often.
+        # Since this is used when wrapping an arbitrary long process and
+        # not just a single query, we can't limit the number of retries.
+        time.sleep(5)

neo/storage/database/importer.py

@@ -18,6 +18,7 @@ import os
 import pickle, sys, time
 from bisect import bisect, insort
 from collections import deque
+from contextlib import contextmanager
 from cStringIO import StringIO
 from ConfigParser import SafeConfigParser
 from ZConfig import loadConfigFile
@@ -370,13 +371,14 @@ class ImporterDatabaseManager(DatabaseManager):
     """Proxy that transparently imports data from a ZODB storage
     """
     _writeback = None
-    _last_commit = 0
 
     def __init__(self, *args, **kw):
-        super(ImporterDatabaseManager, self).__init__(*args, **kw)
-        implements(self, """_getNextTID checkSerialRange checkTIDRange
-            deleteObject deleteTransaction _dropPartition _getLastTID
-            getReplicationObjectList _getTIDList nonempty""".split())
+        super(ImporterDatabaseManager, self).__init__(
+            background_worker_class=lambda: None,
+            *args, **kw)
+        implements(self, """_getNextTID checkSerialRange checkTIDRange _pack
+            deleteObject deleteTransaction _dropPartition _getLastTID nonempty
+            getReplicationObjectList _getTIDList _setPartitionPacked""".split())
 
     _getPartition = property(lambda self: self.db._getPartition)
     _getReadablePartition = property(lambda self: self.db._getReadablePartition)
@@ -409,7 +411,9 @@ class ImporterDatabaseManager(DatabaseManager):
                     updateCellTID getUnfinishedTIDDict dropUnfinishedData
                     abortTransaction storeTransaction lockTransaction
                     loadData storeData getOrphanList _pruneData deferCommit
-                    _getDevPath dropPartitionsTemporary
+                    _getDevPath dropPartitionsTemporary lock
+                    getPackedIDs updateCompletedPackByReplication
+                    _getPackOrders storePackOrder signPackOrders
                  """.split():
             setattr(self, x, getattr(db, x))
         if self._writeback:
@@ -417,7 +421,6 @@ class ImporterDatabaseManager(DatabaseManager):
         db_commit = db.commit
         def commit():
             db_commit()
-            self._last_commit = time.time()
             if self._writeback:
                 self._writeback.committed()
         self.commit = db.commit = commit
@@ -477,9 +480,11 @@ class ImporterDatabaseManager(DatabaseManager):
             else:
                 self._import = self._import()
 
-    def doOperation(self, app):
+    @contextmanager
+    def operational(self, app):
         if self._import:
             app.newTask(self._import)
+        yield
 
     def _import(self):
         p64 = util.p64
@@ -542,7 +547,7 @@ class ImporterDatabaseManager(DatabaseManager):
             " your configuration to use the native backend and restart.")
         self._import = None
         for x in """getObject getReplicationTIDList getReplicationObjectList
-                    _fetchObject _getDataTID getLastObjectTID
+                    _fetchObject _getObjectHistoryForUndo getLastObjectTID
                  """.split():
             setattr(self, x, getattr(self.db, x))
         for zodb in self.zodb:
@@ -729,13 +734,15 @@ class ImporterDatabaseManager(DatabaseManager):
         raise AssertionError
 
     getLastObjectTID = Fallback.getLastObjectTID.__func__
-    _getDataTID = Fallback._getDataTID.__func__
 
-    def getObjectHistory(self, *args, **kw):
-        raise BackendNotImplemented(self.getObjectHistory)
+    def _getObjectHistoryForUndo(self, *args, **kw):
+        raise BackendNotImplemented(self._getObjectHistoryForUndo)
+
+    def getObjectHistoryWithLength(self, *args, **kw):
+        raise BackendNotImplemented(self.getObjectHistoryWithLength)
 
-    def pack(self, *args, **kw):
-        raise BackendNotImplemented(self.pack)
+    def isReadyToStartPack(self):
+        pass # disable pack
 
 
 class WriteBack(object):
@@ -844,7 +851,7 @@ class WriteBack(object):
 class TransactionRecord(BaseStorage.TransactionRecord):
 
     def __init__(self, db, tid):
-        self._oid_list, user, desc, ext, _, _ = db.getTransaction(tid)
+        self._oid_list, user, desc, ext, _, _, _ = db.getTransaction(tid)
         super(TransactionRecord, self).__init__(tid, ' ', user, desc,
             loads(ext) if ext else {})
         self._db = db

neo/storage/database/manager.py

@@ -14,11 +14,10 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import os, errno, socket, sys, threading
+import os, errno, socket, sys, thread, threading, weakref
 from collections import defaultdict
 from contextlib import contextmanager
 from copy import copy
-from functools import wraps
 from time import time
 from neo.lib import logging, util
 from neo.lib.exception import NonReadableCell
@@ -44,22 +43,404 @@ class CreationUndone(Exception):
 class Fallback(object):
     pass
 
+class BackgroundWorker(object):
+
+    _processing = None, None
+    _exc_info = _thread = None
+    _packing = _stop = False
+    _orphan = _packed = None
+    _pack_info = None,
+
+    def __init__(self):
+        self._stat_dict = {}
+        self._drop_set = set()
+        self._pack_set = set()
+
+    def _delattr(self, *attrs):
+        for attr in attrs:
+            try:
+                delattr(self, attr)
+            except AttributeError:
+                assert hasattr(self, attr)
+
+    def _join(self, app, thread):
+        l = app.em.lock
+        l.release()
+        thread.join()
+        l.acquire()
+        del self._thread
+        self._delattr('_packing', '_processing', '_stop')
+        exc_info = self._exc_info
+        if exc_info:
+            del self._exc_info
+            etype, value, tb = exc_info
+            raise etype, value, tb
+
+    @contextmanager
+    def _maybeResume(self, app):
+        assert app.dm.lock._is_owned()
+        if self._stop:
+            self._join(app, self._thread)
+        yield
+        if app.operational and self._thread is None:
+            t = self._thread = threading.Thread(
+                name=self.__class__.__name__,
+                target=self._worker,
+                args=(weakref.ref(app),))
+            t.daemon = 1
+            t.start()
+
+    @contextmanager
+    def operational(self, app):
+        assert app.em.lock.locked()
+        try:
+            with self._maybeResume(app):
+                pass
+            app.dm.lock.release()
+            yield
+        finally:
+            thread = self._thread
+            if thread is not None:
+                self._stop = True
+                logging.info("waiting for background tasks to interrupt")
+                self._join(app, thread)
+            locked = app.dm.lock.acquire(0)
+            assert locked
+            self._pack_set.clear()
+            self._delattr('_pack_info', '_packed')
+
+    def _stats(self, task, dm, what='deleted'):
+        period = .01 if dm.LOCK else .1
+        stats = self._stat_dict
+        before = elapsed, count = stats.setdefault(task, (1e-99, 0))
+        while True:
+            start = time()
+            # Do not process too few lines or the efficiency would drop.
+            count += yield max(100, int(period * count / elapsed))
+            elapsed += time() - start
+            stats[task] = elapsed, count
+            end = yield
+            if end:
+                break
+        logging.info("%s (time: %ss/%ss, %s: %s/%s)",
+            end, round(elapsed - before[0], 3), round(elapsed, 3),
+            what, count - before[1], count)
+
+    def _worker(self, weak_app):
+        try:
+            em_lock = weak_app().em.lock
+            dm = dm2 = weak_app().dm
+            try:
+                mvcc = isinstance(dm, MVCCDatabaseManager)
+                while True:
+                    if mvcc:
+                        stats = self._stats('prune', dm)
+                        log = False
+                        while True:
+                            with em_lock:
+                                # Tasks shall not leave uncommitted changes,
+                                # so pass a dummy value as dm2 parameter
+                                # to avoid a useless commit.
+                                self._checkStop(dm, dm)
+                            with dm.lock:
+                                data_id_list = dm._dataIdsToPrune(next(stats))
+                                if not data_id_list:
+                                    break
+                                if not log:
+                                    logging.info(
+                                        "deferred pruning: processing...")
+                                    log = True
+                                stats.send(dm._pruneData(data_id_list))
+                                dm.commitFromTimeToTime()
+                        if log:
+                            stats.send(0)
+                            try:
+                                stats.send("deferred pruning: processed")
+                            except StopIteration:
+                                pass
+                    with dm.lock:
+                        if self._drop_set:
+                            task = self._task_drop
+                        elif self._pack_set:
+                            task = self._task_pack
+                            self._packing = True
+                        elif self._orphan is not None:
+                            task = self._task_orphan
+                        else:
+                            assert not dm.nonempty('todel')
+                            self._stop = True
+                        if self._stop:
+                            break
+                        if mvcc and dm is dm2:
+                            # The following commit is actually useless for
+                            # drop & orphan tasks. On the other hand, it is
+                            # required if a 0-day pack is requested whereas
+                            # the deferred commit for the latest tpc_finish
+                            # affecting this node hasn't been processed yet.
+                            dm.commit()
+                            dm2 = copy(dm)
+                    try:
+                        task(weak_app, dm, dm2)
+                    except DatabaseFailure, e:
+                        e.checkTransientFailure(dm2)
+                        with dm:
+                            dm.commit()
+                            dm2 = copy(dm)
+            finally:
+                dm is dm2 or dm2.close()
+        except SystemExit:
+            pass
+        except:
+            self._exc_info = sys.exc_info()
+        finally:
+            logging.info("background tasks stopped")
+            thread = self._thread
+            weak_app().em.wakeup(lambda: self._thread is thread
+                and self._join(weak_app(), thread))
+
+    def _checkStop(self, dm, dm2):
+        # Either em or dm lock shall be locked.
+        if self._stop:
+            dm is dm2 or dm2.commit()
+            with dm.lock:
+                dm.commit()
+            thread.exit()
+
+    def _dm21(self, dm, dm2):
+        if dm is dm2:
+            return threading.Lock # faster than contextlib.nullcontext
+        dm_lock = dm.lock
+        dm2_commit = dm2.commit
+        def dm21():
+            dm2_commit()
+            with dm_lock:
+                yield
+        return contextmanager(dm21)
+
+    def _task_drop(self, weak_app, dm, dm2):
+        stats = self._stats('drop', dm2)
+        dropped = 0
+        parts = self._drop_set
+        em_lock = weak_app().em.lock
+        dm21 = self._dm21(dm, dm2)
+        while True:
+            lock = threading.Lock()
+            with lock:
+                with em_lock:
+                    try:
+                        offset = min(parts) # same as in _task_pack
+                    except ValueError:
+                        if dropped:
+                            try:
+                                stats.send("%s partition(s) dropped" % dropped)
+                            except StopIteration:
+                                pass
+                        break
+                    self._processing = offset, lock
+                logging.info("dropping partition %s...", offset)
+                while True:
+                    with em_lock:
+                        self._checkStop(dm, dm2)
+                        if offset not in parts: # partition reassigned
+                            break
+                    with dm2.lock:
+                        deleted = dm2._dropPartition(offset, next(stats))
+                        if type(deleted) is list:
+                            try:
+                                deleted.remove(None)
+                                pass # XXX: not covered
+                            except ValueError:
+                                pass
+                            pruned = dm2._pruneData(deleted)
+                            stats.send(len(deleted) if pruned is None else
+                                       pruned)
+                        else:
+                            stats.send(deleted)
+                            if not deleted:
+                                with dm21():
+                                    try:
+                                        parts.remove(offset)
+                                    except KeyError:
+                                        pass
+                                    else:
+                                        dropped += 1
+                                        dm.commit()
+                                break
+                        dm2.commitFromTimeToTime()
+            if dm is not dm2:
+                # Process deferred pruning before dropping another partition.
+                parts = ()
+
+    def _task_pack(self, weak_app, dm, dm2):
+        stats = self._stats('pack', dm2)
+        pack_id, approved, partial, _oids, tid = self._pack_info
+        assert approved, self._pack_info
+        tid = util.u64(tid)
+        packed = 0
+        parts = self._pack_set
+        em_lock = weak_app().em.lock
+        dm21 = self._dm21(dm, dm2)
+        while True:
+            lock = threading.Lock()
+            with lock:
+                with em_lock, dm.lock:
+                    try:
+                        # Better than next(iter(...)) to resume work
+                        # on a partially-processed partition.
+                        offset = min(parts)
+                    except ValueError:
+                        if packed:
+                            try:
+                                stats.send(
+                                    "%s partition(s) processed for pack %s"
+                                    % (packed, util.dump(pack_id)))
+                            except StopIteration:
+                                pass
+                            weak_app().notifyPackCompleted()
+                        self._packing = False
+                        if not (self._stop or self._pack_set):
+                            weak_app().maybePack()
+                        break
+                    self._processing = offset, lock
+                if partial:
+                    np = dm.np
+                    oid_index = 0
+                    oids = [oid for oid in _oids if oid % np == offset]
+                    logging.info(
+                        "partial pack %s @%016x: partition %s (%s oids)...",
+                        util.dump(pack_id), tid, offset, len(oids))
+                else:
+                    oid = -1
+                    logging.info(
+                        "pack %s @%016x: partition %s...",
+                        util.dump(pack_id), tid, offset)
+                while True:
+                    with em_lock:
+                        self._checkStop(dm, dm2)
+                        if offset not in parts: # partition not readable anymore
+                            break
+                    with dm2.lock:
+                        limit = next(stats)
+                        if partial:
+                            i = oid_index + limit
+                            deleted = dm2._pack(offset,
+                                oids[oid_index:i], tid)[1]
+                            oid_index = i
+                        else:
+                            oid, deleted = dm2._pack(offset, oid+1, tid, limit)
+                        stats.send(deleted)
+                        if oid_index >= len(oids) if partial else oid is None:
+                            with dm21():
+                                try:
+                                    parts.remove(offset)
+                                except ValueError:
+                                    pass
+                                else:
+                                    packed += 1
+                                    dm._setPartitionPacked(offset,
+                                                           util.u64(pack_id))
+                                    dm.commit()
+                            break
+                        dm2.commitFromTimeToTime()
+            if dm is not dm2:
+                # Process deferred pruning before packing another partition.
+                parts = ()
+
+    def _task_orphan(self, weak_app, dm, dm2):
+        dm21 = self._dm21(dm, dm2)
+        logging.info("searching for orphan records...")
+        with dm2.lock:
+            data_id_list = dm2.getOrphanList()
+            logging.info("found %s records that may be orphan",
+                         len(data_id_list))
+            if data_id_list and not self._orphan:
+                deleted = dm2._pruneData(data_id_list)
+                if deleted is not None:
+                    logging.info("deleted %s orphan records", deleted)
+                with dm21():
+                    dm.commit()
+            self._orphan = None
+
+    def checkNotProcessing(self, app, offset, newly_assigned):
+        assert offset not in self._drop_set, offset
+        assert offset not in self._pack_set, offset
+        processing, lock = self._processing
+        if processing == offset:
+            if not lock.acquire(0):
+                assert newly_assigned
+                dm_lock = app.dm.lock
+                em_lock = app.em.lock
+                dm_lock.release()
+                em_lock.release()
+                lock.acquire()
+                em_lock.acquire()
+                dm_lock.acquire()
+            lock.release()
+
+    @contextmanager
+    def dropPartitions(self, app):
+        if app.disable_drop_partitions:
+            drop_set = set()
+            yield drop_set
+            if drop_set:
+                logging.info("don't drop data for partitions %r",
+                             sorted(drop_set))
+        else:
+            with self._maybeResume(app):
+                drop_set = self._drop_set
+                yield drop_set
+        self._pack_set -= drop_set
+
+    def isReadyToStartPack(self):
+        if not (self._packing or self._pack_set):
+            return self._pack_info[0], self._packed
+
+    def pack(self, app, info, packed, offset_list):
+        assert app.operational
+        parts = self._pack_set
+        assert not parts
+        with self._maybeResume(app):
+            parts.update(offset_list)
+            if parts:
+                if info:
+                    pack_id, approved, partial, oids, tid = info
+                    self._pack_info = (pack_id, approved, partial,
+                        oids and map(util.u64, oids), tid)
+                    self._packed = packed
+                else:
+                    assert self._packed == packed
+            elif not packed:
+                # Release memory: oids may take several MB.
+                try:
+                    del self._pack_info, self._packed
+                except AttributeError:
+                    pass
+
+    def pruneOrphan(self, app, dry_run):
+        with self._maybeResume(app):
+            if self._orphan is None:
+                self._orphan = dry_run
+            else:
+                logging.error('already repairing')
+
+
 class DatabaseManager(object):
     """This class only describes an interface for database managers."""
 
     ENGINES = ()
+    TEST_IDENT = None
     UNSAFE = False
 
-    __lock = None
+    __lockFile = None
     LOCK = "neostorage"
     LOCKED = "error: database is locked"
 
-    _deferred = 0
-    _drop_stats = 0, 0
-    _dropping = None
-    _repairing = None
+    _deferred_commit = 0
+    _last_commit = 0
+    _uncommitted_data = () # for secondary connections
 
-    def __init__(self, database, engine=None, wait=None):
+    def __init__(self, database, engine=None, wait=None,
+                 background_worker_class=BackgroundWorker):
         """
             Initialize the object.
         """
@@ -73,6 +454,9 @@ class DatabaseManager(object):
         self._wait = wait or 0
         self._parse(database)
         self._init_attrs = tuple(self.__dict__)
+        self._background_worker = background_worker_class()
+        self.lock = threading.RLock()
+        self.lock.acquire()
         self._connect()
 
     def __getstate__(self):
@@ -86,18 +470,12 @@ class DatabaseManager(object):
         #self._init_attrs = tuple(self.__dict__)
         # Secondary connections don't lock.
         self.LOCK = None
+        self.lock = threading.RLock() # dummy lock
+        self.lock.acquire()
         self._connect()
 
-    @contextmanager
-    def _duplicate(self):
-        db = copy(self)
-        try:
-            yield db
-        finally:
-            db.close()
-
     _cached_attr_list = (
-        '_readable_set', '_getPartition', '_getReadablePartition')
+        'pt', '_readable_set', '_getPartition', '_getReadablePartition')
 
     def __getattr__(self, attr):
         if attr in self._cached_attr_list:
@@ -118,7 +496,7 @@ class DatabaseManager(object):
     def __exit__(self, t, v, tb):
         if v is None:
             # Deferring commits make no sense for secondary connections.
-            assert not self._deferred
+            assert not self._deferred_commit
             self._commit()
 
     @abstract
@@ -132,17 +510,16 @@ class DatabaseManager(object):
     def autoReconnect(self, f):
         """
         Placeholder for backends that may lose connection to the underlying
-        database: although a primary connection is reestablished transparently
-        when possible, secondary connections use transactions and they must
-        restart from the beginning.
+        database.
         For other backends, there's no expected transient failure so the
         default implementation is to execute the given task exactly once.
         """
-        f()
+        assert not self.LOCK, "not a secondary connection"
+        return f()
 
-    def lock(self, db_path):
+    def lockFile(self, db_path):
         if self.LOCK:
-            assert self.__lock is None, self.__lock
+            assert self.__lockFile is None, self.__lockFile
             # For platforms that don't support anonymous sockets,
             # we can either use zc.lockfile or an empty SQLite db
             # (with BEGIN EXCLUSIVE).
@@ -152,7 +529,7 @@ class DatabaseManager(object):
                 if e.errno != errno.ENOENT:
                     raise
                 return # in-memory or temporary database
-            s = self.__lock = socket.socket(socket.AF_UNIX)
+            s = self.__lockFile = socket.socket(socket.AF_UNIX)
             try:
                 s.bind('\0%s:%s:%s' % (self.LOCK, stat.st_dev, stat.st_ino))
             except socket.error as e:
@@ -215,41 +592,53 @@ class DatabaseManager(object):
             getattr(self, '_migrate%s' % version)(*args, **kw)
             self.setConfiguration("version", version)
 
-    def doOperation(self, app):
-        if self._dropping:
-            self._dropPartitions(app)
+    @property
+    def operational(self):
+        return self._background_worker.operational
+
+    @property
+    def checkNotProcessing(self):
+        return self._background_worker.checkNotProcessing
 
     def _close(self):
         """Backend-specific code to close the database"""
 
     @requires(_close)
     def close(self):
-        self._deferredCommit()
+        if self._deferred_commit:
+            self.commit()
         self._close()
-        if self.__lock:
-            self.__lock.close()
-            del self.__lock
+        if self.__lockFile:
+            self.__lockFile.close()
+            del self.__lockFile
 
     def _commit(self):
         """Backend-specific code to commit the pending changes"""
 
     @requires(_commit)
     def commit(self):
+        assert self.lock._is_owned() or self.TEST_IDENT == thread.get_ident()
         logging.debug('committing...')
         self._commit()
+        self._last_commit = time()
         # Instead of cancelling a timeout that would be set to defer a commit,
         # we simply use to a boolean so that _deferredCommit() does nothing.
         # IOW, epoll may wait wake up for nothing but that should be rare,
         # because most immediate commits are usually quickly followed by
         # deferred commits.
-        self._deferred = 0
+        self._deferred_commit = 0
 
     def deferCommit(self):
-        self._deferred = 1
+        self._deferred_commit = 1
         return self._deferredCommit
 
     def _deferredCommit(self):
-        if self._deferred:
+        with self.lock:
+            if self._deferred_commit:
+                self.commit()
+
+    def commitFromTimeToTime(self):
+        if self._last_commit + 1 < time():
             self.commit()
 
     @abstract
@@ -278,8 +667,9 @@ class DatabaseManager(object):
 
     def _getPartitionTable(self):
         """Return a whole partition table as a sequence of rows. Each row
-        is again a tuple of an offset (row ID), the NID of a storage
-        node, and a cell state."""
+        is again a tuple of an offset (row ID), the NID of a storage node,
+        either a tid or the negative of a cell state, and a pack id.
+        """
 
     def getUUID(self):
         """
@@ -297,8 +687,8 @@ class DatabaseManager(object):
         old_nid = self.getUUID()
         if nid != old_nid:
             if old_nid:
-                self._changePartitionTable((offset, x, tid)
-                    for offset, x, tid in self._getPartitionTable()
+                self._changePartitionTable((offset, x, tid, pack)
+                    for offset, x, tid, pack in self._getPartitionTable()
                     if x == old_nid
                     for x, tid in ((x, None), (nid, tid)))
             self.setConfiguration('nid', str(nid))
@@ -347,15 +737,6 @@ class DatabaseManager(object):
         logging.debug('truncate_tid = %s', tid)
         return self._setConfiguration('truncate_tid', tid)
 
-    def _setPackTID(self, tid):
-        self._setConfiguration('_pack_tid', tid)
-
-    def _getPackTID(self):
-        try:
-            return int(self.getConfiguration('_pack_tid'))
-        except TypeError:
-            return -1
-
     # XXX: Consider splitting getLastIDs/_getLastIDs because
     #      sometimes the last oid is not wanted.
 
@@ -399,6 +780,46 @@ class DatabaseManager(object):
                     None if oid is None else util.p64(oid))
         return None, None
 
+    def _getPackOrders(self, min_completed):
+      """Request list of pack orders excluding oldest completed ones.
+      """
+
+    @requires(_getPackOrders)
+    def getPackOrders(self, min_completed):
+        p64 = util.p64
+        return [(
+            p64(id),
+            None if approved is None else bool(approved),
+            bool(partial),
+            oids,
+            p64(tid),
+        ) for id, approved, partial, oids, tid in self._getPackOrders(
+            util.u64(min_completed))]
+
+    @abstract
+    def getPackedIDs(self, up_to_date=False):
+        """"""
+
+    @abstract
+    def _setPartitionPacked(self, partition, pack_id):
+        """"""
+
+    @abstract
+    def updateCompletedPackByReplication(self, pack_id, partition=None):
+        """"""
+
+    @property
+    def pack(self):
+        return self._background_worker.pack
+
+    @property
+    def isReadyToStartPack(self):
+        return self._background_worker.isReadyToStartPack
+
+    @property
+    def repair(self):
+        return self._background_worker.pruneOrphan
+
     def _getUnfinishedTIDDict(self):
         """"""
 
@@ -513,13 +934,14 @@ class DatabaseManager(object):
     @requires(_getPartitionTable)
     def iterAssignedCells(self):
         my_nid = self.getUUID()
-        return ((offset, tid) for offset, nid, tid in self._getPartitionTable()
-                              if my_nid == nid)
+        return ((offset, tid)
+            for offset, nid, tid, pack in self._getPartitionTable()
+            if my_nid == nid)
 
     @requires(_getPartitionTable)
     def getPartitionTable(self):
-        return [(offset, nid, max(0, -state))
-            for offset, nid, state in self._getPartitionTable()]
+        return [(offset, nid, max(0, -tid))
+            for offset, nid, tid, pack in self._getPartitionTable()]
 
     @contextmanager
     def replicated(self, offset):
@@ -543,7 +965,7 @@ class DatabaseManager(object):
     def _updateReadable(self, reset=True):
         if reset:
             readable_set = self._readable_set = set()
-            np = 1 + self._getMaxPartition()
+            np = self.np = 1 + self._getMaxPartition()
             def _getPartition(x, np=np):
                 return x % np
             def _getReadablePartition(x, np=np, r=readable_set):
@@ -574,40 +996,49 @@ class DatabaseManager(object):
         if backup_tid:
             backup_tid = util.u64(backup_tid)
         max_offset = -1
-        dropping = self._dropping or set()
         assigned = []
         cells = []
-        for offset, nid, state in cell_list:
-            if max_offset < offset:
-                max_offset = offset
-            if state == CellStates.DISCARDED:
-                if nid == my_nid:
-                    dropping.add(offset)
-                tid = None
-            else:
-                if nid == my_nid:
-                    assigned.append(offset)
-                if nid != my_nid or state != CellStates.OUT_OF_DATE:
-                    tid = -state
+        pack_set = self._background_worker._pack_set
+        app_last_pack_id = app.last_pack_id
+        if app_last_pack_id:
+            app_last_pack_id = util.u64(app_last_pack_id)
+        with self._background_worker.dropPartitions(app) as drop_set:
+            for offset, nid, state in cell_list:
+                if max_offset < offset:
+                    max_offset = offset
+                pack = None
+                if state == CellStates.DISCARDED:
+                    if nid == my_nid:
+                        drop_set.add(offset)
+                        pack_set.discard(offset)
+                    tid = None
                 else:
-                    tid = pt.get(offset, 0)
-                    if tid < 0:
-                        tid = -tid in READABLE and (backup_tid or
-                            max(self._getLastIDs(offset)[0],
-                                self._getLastTID(offset))) or 0
-            cells.append((offset, nid, tid))
-        if reset:
-            dropping.update(xrange(max_offset + 1))
-            dropping.difference_update(assigned)
-        self._changePartitionTable(cells, reset)
-        self._updateReadable(reset)
-        assert isinstance(ptid, (int, long)), ptid
-        self._setConfiguration('ptid', str(ptid))
-        self._setConfiguration('replicas', str(num_replicas))
-        if dropping and not self._dropping:
-            self._dropping = dropping
-            if app.operational:
-                self._dropPartitions(app)
+                    if nid == my_nid:
+                        assigned.append(offset)
+                        if state in READABLE:
+                            assert app_last_pack_id is None or not reset, (
+                                reset, app_last_pack_id, cell_list)
+                            pack = 0
+                        else:
+                            pack_set.discard(offset)
+                            pack = app_last_pack_id
+                    if nid != my_nid or state != CellStates.OUT_OF_DATE:
+                        tid = -state
+                    else:
+                        tid = pt.get(offset, 0)
+                        if tid < 0:
+                            tid = -tid in READABLE and (backup_tid or
+                                max(self._getLastIDs(offset)[0],
+                                    self._getLastTID(offset))) or 0
+                cells.append((offset, nid, tid, pack))
+            if reset:
+                drop_set.update(xrange(max_offset + 1))
+            drop_set.difference_update(assigned)
+            self._changePartitionTable(cells, reset)
+            self._updateReadable(reset)
+            assert isinstance(ptid, (int, long)), ptid
+            self._setConfiguration('ptid', str(ptid))
+            self._setConfiguration('replicas', str(num_replicas))
 
     @requires(_changePartitionTable)
     def updateCellTID(self, partition, tid):
@@ -624,7 +1055,7 @@ class DatabaseManager(object):
         # we may end up in a special situation where an OUT_OF_DATE cell
         # is actually more up-to-date than an UP_TO_DATE one.
         assert t < tid or self.getBackupTID()
-        self._changePartitionTable([(partition, self.getUUID(), tid)])
+        self._changePartitionTable([(partition, self.getUUID(), tid, None)])
 
     def iterCellNextTIDs(self):
         p64 = util.p64
@@ -652,61 +1083,6 @@ class DatabaseManager(object):
                 else:
                     yield offset, None
 
-    def _dropPartitions(self, app):
-        if app.disable_drop_partitions:
-            logging.info("don't drop data for partitions %r", self._dropping)
-            return
-        def dropPartitions():
-            dropping = self._dropping
-            before = drop_count, drop_time = self._drop_stats
-            commit = dropped = 0
-            while dropping:
-                offset = next(iter(dropping))
-                log = dropped
-                while True:
-                    yield 1
-                    if offset not in dropping:
-                        break
-                    start = time()
-                    if 0 < commit < start:
-                        self.commit()
-                        logging.debug('drop: committed')
-                        commit = 0
-                        continue
-                    data_id_list = self._dropPartition(offset,
-                        # The efficiency drops when the number of lines to
-                        # delete is too small so do not delete too few.
-                        max(100, int(.1 * drop_count / drop_time))
-                        if drop_time else 1000)
-                    if data_id_list:
-                        if not commit:
-                            commit = time() + 1
-                        if log == dropped:
-                            dropped += 1
-                            logging.info("dropping partition %s...", offset)
-                        if type(data_id_list) is list:
-                            try:
-                                data_id_list.remove(None)
-                                pass # XXX: not covered
-                            except ValueError:
-                                pass
-                            logging.debug('drop: pruneData(%s)',
-                                          len(data_id_list))
-                            drop_count += self._pruneData(data_id_list)
-                            drop_time += time() - start
-                            self._drop_stats = drop_count, drop_time
-                            continue
-                    dropping.remove(offset)
-                    break
-            if dropped:
-                if commit:
-                    self.commit()
-                logging.info("%s partition(s) dropped"
-                    " (stats: count: %s/%s, time: %.4s/%.4s)",
-                    dropped, drop_count - before[0], drop_count,
-                    round(drop_time - before[1], 3), round(drop_time, 3))
-        app.newTask(dropPartitions())
-
     @abstract
     def _dropPartition(self, offset, count):
         """Delete rows for given partition
@@ -813,28 +1189,15 @@ class DatabaseManager(object):
             else:
                 del refcount[data_id]
         if prune:
-            return self._pruneData(data_id_list)
-
-    @fallback
-    @requires(_getObject)
-    def _getDataTID(self, oid, tid=None, before_tid=None):
-        """
-        Return a 2-tuple:
-        tid (int)
-            tid corresponding to received parameters
-        serial
-            data tid of the found record
-
-        (None, None) is returned if requested object and transaction
-        could not be found.
+            self._pruneData(data_id_list)
 
-        This method only exists for performance reasons, by not returning data:
-        _getObject already returns these values but it is slower.
-        """
-        r = self._getObject(oid, tid, before_tid)
-        return (r[0], r[-1]) if r else (None, None)
+    def _getObjectHistoryForUndo(self, oid, undo_tid):
+        """Return (undone_tid, history) where 'undone_tid' is the greatest tid
+        before 'undo_tid' and 'history' is the list of (tid, value_tid) after
+        'undo_tid'. If there's no record at 'undo_tid', return None."""
 
-    def findUndoTID(self, oid, ltid, undone_tid, current_tid):
+    @requires(_getObjectHistoryForUndo)
+    def findUndoTID(self, oid, ltid, undo_tid, current_tid):
         """
         oid
             Object OID
@@ -843,7 +1206,7 @@ class DatabaseManager(object):
         ltid
             Upper (excluded) bound of transactions visible to transaction doing
             the undo.
-        undone_tid
+        undo_tid
             Transaction to undo
         current_tid
             Serial of object data from memory, if it was modified by running
@@ -855,58 +1218,100 @@ class DatabaseManager(object):
             see. This is used later to detect current conflicts (eg, another
             client modifying the same object in parallel)
         data_tid (int)
-            TID containing (without indirection) the data prior to undone
-            transaction.
+            TID containing the data prior to undone transaction.
             None if object doesn't exist prior to transaction being undone
               (its creation is being undone).
         is_current (bool)
             False if object was modified by later transaction (ie, data_tid is
             not current), True otherwise.
+
+        When undoing several times in such a way that several data_tid are
+        possible, the implementation guarantees to return the greatest one,
+        which makes undo compatible with pack without having to update the
+        value_tid of obj records. IOW, all records that are undo-identical
+        constitute a simply-linked list; after a pack, the value_tid of the
+        record with the smallest TID points to nowhere.
+
+        With a different implementation, it could fail as follows:
+            tid value_tid
+            10  -
+            20  10
+            30  10
+            40  20
+        After packing at 30, the DB would lose the information that 30 & 40
+        are undo-identical.
+
+        TODO: Since ZODB requires nothing about how undo-identical records are
+              linked, imported databases may not be packable without breaking
+              undo information. Same for existing databases because older NEO
+              implementation linked records differently. A background task to
+              fix value_tid should be implemented; for example, it would be
+              used automatically once Importer has finished, if it has seen
+              non-null value_tid.
         """
         u64 = util.u64
-        oid = u64(oid)
-        undone_tid = u64(undone_tid)
-        def getDataTID(tid=None, before_tid=None):
-            tid, data_tid = self._getDataTID(oid, tid, before_tid)
-            current_tid = tid
-            while data_tid:
-                if data_tid < tid:
-                    tid, data_tid = self._getDataTID(oid, data_tid)
-                    if tid is not None:
-                        continue
-                logging.error("Incorrect data serial for oid %s at tid %s",
-                              oid, current_tid)
-                return current_tid, current_tid
-            return current_tid, tid
-        found_undone_tid, undone_data_tid = getDataTID(tid=undone_tid)
-        if found_undone_tid is None:
-            return
+        undo_tid = u64(undo_tid)
+        history = self._getObjectHistoryForUndo(u64(oid), undo_tid)
+        if not history:
+            return # nothing to undo for this oid at undo_tid
+        undone_tid, history = history
         if current_tid:
-            current_data_tid = u64(current_tid)
+            current = u64(current_tid)
         else:
-            if ltid:
-                ltid = u64(ltid)
-            current_tid, current_data_tid = getDataTID(before_tid=ltid)
-            if current_tid is None:
-                return None, None, False
-            current_tid = util.p64(current_tid)
-        # Load object data as it was before given transaction.
-        # It can be None, in which case it means we are undoing object
-        # creation.
-        _, data_tid = getDataTID(before_tid=undone_tid)
-        if data_tid is not None:
-            data_tid = util.p64(data_tid)
-        return current_tid, data_tid, undone_data_tid == current_data_tid
+            ltid = u64(ltid) if ltid else float('inf')
+            for current, _ in reversed(history):
+                if current < ltid:
+                    break
+            else:
+                if ltid <= undo_tid:
+                    return None, None, False
+                current = undo_tid
+            current_tid = util.p64(current)
+        is_current = current == undo_tid
+        for tid, data_tid in history:
+            if data_tid is not None:
+                if data_tid == undone_tid:
+                    undone_tid = tid
+                elif data_tid == undo_tid:
+                    if current == tid:
+                        is_current = True
+                    else:
+                        undo_tid = tid
+        return (current_tid,
+                None if undone_tid is None else util.p64(undone_tid),
+                is_current)
 
     @abstract
-    def lockTransaction(self, tid, ttid):
+    def storePackOrder(self, tid, approved, partial, oid_list, pack_tid):
+        """"""
+
+    def _signPackOrders(self, approved, rejected):
+        """"""
+
+    @requires(_signPackOrders)
+    def signPackOrders(self, approved, rejected, auto_commit=True):
+        u64 = util.u64
+        changed = map(util.p64, self._signPackOrders(
+            map(u64, approved), map(u64, rejected)))
+        if changed:
+            if auto_commit:
+                self.commit()
+            def _(signed):
+                signed = set(signed)
+                signed.difference_update(changed)
+                return sorted(signed)
+            return _(approved), _(rejected)
+        return approved, rejected
+
+    @abstract
+    def lockTransaction(self, tid, ttid, pack):
         """Mark voted transaction 'ttid' as committed with given 'tid'
 
         All pending changes are committed just before returning to the caller.
         """
 
     @abstract
-    def unlockTransaction(self, tid, ttid, trans, obj):
+    def unlockTransaction(self, tid, ttid, trans, obj, pack):
         """Finalize a transaction by moving data to a finished area."""
 
     @abstract
@@ -941,44 +1346,13 @@ class DatabaseManager(object):
                         self.commit()
                     commit = time() + 10
                 if state > tid:
-                    cell_list.append((partition, my_nid, tid))
+                    cell_list.append((partition, my_nid, tid, None))
                 self._deleteRange(partition, tid)
             if cell_list:
                 self._changePartitionTable(cell_list)
             self._setTruncateTID(None)
             self.commit()
 
-    def repair(self, weak_app, dry_run):
-        t = self._repairing
-        if t and t.is_alive():
-            logging.error('already repairing')
-            return
-        def repair():
-            l = threading.Lock()
-            l.acquire()
-            def finalize():
-                try:
-                    if data_id_list and not dry_run:
-                        self.commit()
-                        logging.info("repair: deleted %s orphan records",
-                                     self._pruneData(data_id_list))
-                        self.commit()
-                finally:
-                    l.release()
-            try:
-                with self._duplicate() as db:
-                    data_id_list = db.getOrphanList()
-                logging.info("repair: found %s records that may be orphan",
-                             len(data_id_list))
-                weak_app().em.wakeup(finalize)
-                l.acquire()
-            finally:
-                del self._repairing
-            logging.info("repair: done")
-        t = self._repairing = threading.Thread(target=repair)
-        t.daemon = 1
-        t.start()
-
     @abstract
     def getTransaction(self, tid, all = False):
         """Return a tuple of the list of OIDs, user information,
@@ -988,7 +1362,7 @@ class DatabaseManager(object):
         area as well."""
 
     @abstract
-    def getObjectHistory(self, oid, offset, length):
+    def getObjectHistoryWithLength(self, oid, offset, length):
         """Return a list of serials and sizes for a given object ID.
         The length specifies the maximum size of such a list. Result starts
         with latest serial, and the list must be sorted in descending order.
@@ -1022,20 +1396,12 @@ class DatabaseManager(object):
         passed to filter out non-applicable TIDs."""
 
     @abstract
-    def pack(self, tid, updateObjectDataForPack):
-        """Prune all non-current object revisions at given tid.
-        updateObjectDataForPack is a function called for each deleted object
-        and revision with:
-        - OID
-        - packed TID
-        - new value_serial
-            If object data was moved to an after-pack-tid revision, this
-            parameter contains the TID of that revision, allowing to backlink
-            to it.
-        - getObjectData function
-            To call if value_serial is None and an object needs to be updated.
-            Takes no parameter, returns a 3-tuple: compression, data_id,
-            value
+    def _pack(self, offset, oid, tid, limit=None):
+        """
+        The undo feature is implemented in such a way that value_tid does not
+        have to be updated. This is important for performance reasons, but also
+        because pack must be idempotent to guarantee that up-to-date replicas
+        are identical.
         """
 
     @abstract
@@ -1078,3 +1444,12 @@ class DatabaseManager(object):
               record read)
               ZERO_TID if no record found
         """
+
+
+class MVCCDatabaseManager(DatabaseManager):
+    """This class only describes an interface for database managers."""
+
+    @abstract
+    def _dataIdsToPrune(self, limit):
+        """
+        """

neo/storage/database/mysql.py

@@ -14,9 +14,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import os, re, string, struct, sys, time
+import os, re, string, struct, sys, thread, time, weakref
 from binascii import a2b_hex
-from collections import OrderedDict
+from collections import defaultdict, OrderedDict
 from functools import wraps
 from hashlib import sha1
 from . import useMySQLdb
@@ -50,10 +50,11 @@ else:
     from pymysql import NotSupportedError
     from pymysql.constants.ER import BAD_DB_ERROR, UNKNOWN_STORAGE_ENGINE
 from . import LOG_QUERIES, DatabaseFailure
-from .manager import DatabaseManager, splitOIDField
+from .manager import MVCCDatabaseManager, splitOIDField
 from neo.lib import logging, util
+from neo.lib.exception import UndoPackError
 from neo.lib.interfaces import implements
-from neo.lib.protocol import ZERO_OID, ZERO_TID, ZERO_HASH
+from neo.lib.protocol import CellStates, ZERO_OID, ZERO_TID, ZERO_HASH
 
 
 class MysqlError(DatabaseFailure):
@@ -61,6 +62,7 @@ class MysqlError(DatabaseFailure):
     def __init__(self, exc, query=None):
         self.exc = exc
         self.query = query
+        self.transient_failure = exc.args[0] in (SERVER_GONE_ERROR, SERVER_LOST)
 
     code = property(lambda self: self.exc.args[0])
 
@@ -69,6 +71,9 @@ class MysqlError(DatabaseFailure):
         return msg if self.query is None else '%s\nQuery: %s' % (
             msg, getPrintableQuery(self.query[:1000]))
 
+    def logTransientFailure(self):
+        logging.info('the MySQL server is gone; reconnecting')
+
 
 def getPrintableQuery(query, max=70):
     return ''.join(c if c in string.printable and c not in '\t\x0b\x0c\r'
@@ -88,14 +93,13 @@ def auto_reconnect(wrapped):
                 # XXX:  However, this would another case of failure that would
                 #       be unnoticed by other nodes (ADMIN & MASTER). When
                 #       there are replicas, it may be preferred to not retry.
-                if (self._active
-                    or SERVER_GONE_ERROR != m.args[0] != SERVER_LOST
-                    or not retry):
-                    if self.LOCK:
-                        raise MysqlError(m, *args)
-                    raise # caught upper for secondary connections
-                logging.info('the MySQL server is gone; reconnecting')
-                assert not self._deferred
+                e = MysqlError(m, *args)
+                if self._active or not (e.transient_failure and retry):
+                    if __debug__:
+                        e.getFailingDatabaseManager = weakref.ref(self)
+                    raise e
+                e.logTransientFailure()
+                assert not self._deferred_commit
                 self.close()
                 retry -= 1
     return wraps(wrapped)(wrapper)
@@ -106,10 +110,10 @@ def splitList(x, n):
 
 
 @implements
-class MySQLDatabaseManager(DatabaseManager):
+class MySQLDatabaseManager(MVCCDatabaseManager):
     """This class manages a database on MySQL."""
 
-    VERSION = 3
+    VERSION = 4
     ENGINES = "InnoDB", "RocksDB"
     _engine = ENGINES[0] # default engine
 
@@ -196,29 +200,18 @@ class MySQLDatabaseManager(DatabaseManager):
             if e.args[0] != NO_SUCH_TABLE:
                 raise
             self._dedup = None
-        if not self.LOCK:
-            # Prevent automatic reconnection for secondary connections.
-            self._active = 1
-            self._commit = self.conn.commit
+        if self.LOCK:
+            self._todel_min_id = 0
 
     _connect = auto_reconnect(_tryConnect)
 
     def autoReconnect(self, f):
-        assert self._active and not self.LOCK
-        @auto_reconnect
-        def try_once(self):
-            if self._active:
-                try:
-                    f()
-                finally:
-                    self._active = 0
-                return True
-        while not try_once(self):
-            # Avoid reconnecting too often.
-            # Since this is used to wrap an arbitrary long process and
-            # not just a single query, we can't limit the number of retries.
-            time.sleep(5)
-            self._connect()
+        assert not self.LOCK, "not a secondary connection"
+        while True:
+            try:
+                return f()
+            except DatabaseFailure, e:
+                e.checkTransientFailure(self)
 
     def _commit(self):
         # XXX: Should we translate OperationalError into MysqlError ?
@@ -228,18 +221,23 @@ class MySQLDatabaseManager(DatabaseManager):
     @auto_reconnect
     def query(self, query):
         """Query data from a database."""
+        assert self.lock._is_owned() or self.TEST_IDENT == thread.get_ident()
         if LOG_QUERIES:
-            logging.debug('querying %s...',
-                getPrintableQuery(query.split('\n', 1)[0][:70]))
+            logging.debug('querying %s...', getPrintableQuery(query
+                    .split('\n', 1)[0][:70]
+                ))
         conn = self.conn
         conn.query(query)
         if query.startswith("SELECT "):
             return fetch_all(conn)
         r = query.split(None, 1)[0]
-        if r in ("INSERT", "REPLACE", "DELETE", "UPDATE"):
+        if r in ("INSERT", "REPLACE", "DELETE", "UPDATE", "SET"):
             self._active = 1
-        else:
-            assert r in ("ALTER", "CREATE", "DROP"), query
+        else: # DDL (implicit commits)
+            assert r in ("ALTER", "CREATE", "DROP", "TRUNCATE"), query
+            assert self.LOCK, "not a primary connection"
+            self._last_commit = time.time()
+            self._active = self._deferred_commit = 0
 
     @property
     def escape(self):
@@ -253,7 +251,7 @@ class MySQLDatabaseManager(DatabaseManager):
 
     def erase(self):
         self.query("DROP TABLE IF EXISTS"
-            " config, pt, trans, obj, data, bigdata, ttrans, tobj")
+            " config, pt, pack, trans, obj, data, bigdata, ttrans, tobj, todel")
 
     def nonempty(self, table):
         try:
@@ -283,18 +281,19 @@ class MySQLDatabaseManager(DatabaseManager):
         self._alterTable(schema_dict, 'obj')
 
     def _migrate3(self, schema_dict):
-        self._alterTable(schema_dict, 'pt', "rid as `partition`, nid,"
+        x = 'pt'
+        self._alterTable({x: schema_dict[x].replace('pack', '-- pack')}, x,
+            "rid AS `partition`, nid,"
             " CASE state"
             " WHEN 0 THEN -1"  # UP_TO_DATE
             " WHEN 2 THEN -2"  # FEEDING
             " ELSE 1-state"
-            " END as tid")
+            " END AS tid")
 
-    # Let's wait for a more important change to clean up,
-    # so that users can still downgrade.
-    if 0:
-      def _migrate4(self, schema_dict):
+    def _migrate4(self, schema_dict):
         self._setConfiguration('partitions', None)
+        self._alterTable(schema_dict, 'pt', "*,"
+            " IF(nid=%s, 0, NULL) AS pack" % self.getUUID())
 
     def _setup(self, dedup=False):
         self._config.clear()
@@ -314,9 +313,18 @@ class MySQLDatabaseManager(DatabaseManager):
                  `partition` SMALLINT UNSIGNED NOT NULL,
                  nid INT NOT NULL,
                  tid BIGINT NOT NULL,
+                 pack BIGINT UNSIGNED,
                  PRIMARY KEY (`partition`, nid)
              ) ENGINE=""" + engine
 
+        schema_dict['pack'] = """CREATE TABLE %s (
+                  tid BIGINT UNSIGNED NOT NULL PRIMARY KEY,
+                  approved BOOLEAN, -- NULL if not signed
+                  partial BOOLEAN NOT NULL,
+                  oids MEDIUMBLOB, -- same format as trans.oids
+                  pack_tid BIGINT UNSIGNED
+             ) ENGINE=""" + engine
+
         if engine == "RocksDB":
             cf = lambda name, rev=False: " COMMENT '%scf_neo_%s'" % (
                 'rev:' if rev else '', name)
@@ -391,6 +399,12 @@ class MySQLDatabaseManager(DatabaseManager):
                  PRIMARY KEY (tid, oid){}
              ) ENGINE={}""".format(cf('no_comp'), p)
 
+        # The table "todel" is used for deferred deletion of data rows.
+        schema_dict['todel'] = """CREATE TABLE %s (
+                 data_id BIGINT UNSIGNED NOT NULL,
+                 PRIMARY KEY (data_id){}
+             ) ENGINE={}""".format(cf('no_comp'), p)
+
         if self.nonempty('config') is None:
             q(schema_dict.pop('config') % 'config')
             self._setConfiguration('version', self.VERSION)
@@ -458,6 +472,32 @@ class MySQLDatabaseManager(DatabaseManager):
         (tid,), = q("SELECT MAX(tid) FROM obj FORCE INDEX (tid)" + x)
         return tid, oid
 
+    def _getPackOrders(self, min_completed):
+        return self.query(
+            "SELECT * FROM pack WHERE tid >= %s AND tid %% %s IN (%s)"
+            % (min_completed, self.np, ','.join(map(str, self._readable_set))))
+
+    def getPackedIDs(self, up_to_date=False):
+        return {offset: util.p64(pack) for offset, pack in self.query(
+            "SELECT `partition`, pack FROM pt WHERE pack IS NOT NULL"
+            + (" AND tid=-%u" % CellStates.UP_TO_DATE if up_to_date else ""))}
+
+    def _setPartitionPacked(self, partition, pack_id):
+        q = self.query
+        nid = self.getUUID()
+        (i,), = q("SELECT pack FROM pt WHERE `partition`=%s AND nid=%s"
+            % (partition, nid))
+        assert i is not None and i < pack_id, (i, pack_id)
+        q("UPDATE pt SET pack=%s WHERE `partition`=%s AND nid=%s"
+            % (pack_id, partition, nid))
+
+    def updateCompletedPackByReplication(self, pack_id, partition=None):
+        pack_id = util.u64(pack_id)
+        self.query(
+            "UPDATE pt SET pack=%s WHERE nid=%s AND (pack IS NULL OR pack>%s)"
+            % (pack_id, self.getUUID(), pack_id)
+            + ("" if partition is None else " AND `partition`=%s" % partition))
+
     def _getDataLastId(self, partition):
         return self.query("SELECT MAX(id) FROM data WHERE %s <= id AND id < %s"
             % (partition << 48, (partition + 1) << 48))[0][0]
@@ -519,18 +559,19 @@ class MySQLDatabaseManager(DatabaseManager):
 
     def _changePartitionTable(self, cell_list, reset=False):
         q = self.query
-        if reset:
-            q("DELETE FROM pt")
-        for offset, nid, tid in cell_list:
-            # TODO: this logic should move out of database manager
-            # add 'dropCells(cell_list)' to API and use one query
+        delete = set(q("SELECT `partition`, nid FROM pt")) if reset else set()
+        for offset, nid, tid, pack in cell_list:
+            key = offset, nid
             if tid is None:
-                q("DELETE FROM pt WHERE `partition` = %d AND nid = %d"
-                  % (offset, nid))
+                delete.add(key)
             else:
-                q("INSERT INTO pt VALUES (%d, %d, %d)"
-                  " ON DUPLICATE KEY UPDATE tid = %d"
-                  % (offset, nid, tid, tid))
+                delete.discard(key)
+                q("INSERT INTO pt VALUES (%d, %d, %d, %s)"
+                  " ON DUPLICATE KEY UPDATE tid=%d"
+                  % (offset, nid, tid, 'NULL' if pack is None else pack, tid))
+        if delete:
+            q("DELETE FROM pt WHERE " + " OR ".join(
+                map("`partition`=%s AND nid=%s".__mod__, delete)))
 
     def _dropPartition(self, offset, count):
         q = self.query
@@ -543,7 +584,8 @@ class MySQLDatabaseManager(DatabaseManager):
             q("DELETE FROM obj" + where)
             return [x for x, in x]
         logging.debug("drop: trans")
-        q("DELETE FROM trans WHERE `partition`=%s" % offset)
+        q("DELETE trans, pack FROM trans LEFT JOIN pack USING(tid)"
+          " WHERE `partition`=%s" % offset)
         (x,), = q('SELECT ROW_COUNT()')
         return x
 
@@ -556,7 +598,7 @@ class MySQLDatabaseManager(DatabaseManager):
             " WHERE `partition` IN (%s)" % ','.join(map(str, offset_list))
         q = self.query
         q("DELETE FROM tobj" + where)
-        q("DELETE FROM ttrans" + where)
+        q("DELETE ttrans, pack FROM ttrans LEFT JOIN pack USING(tid)" + where)
 
     def storeTransaction(self, tid, object_list, transaction, temporary = True):
         e = self.escape
@@ -604,18 +646,39 @@ class MySQLDatabaseManager(DatabaseManager):
 
     def getOrphanList(self):
         return [x for x, in self.query(
-            "SELECT id FROM data LEFT JOIN obj ON (id=data_id)"
-            " WHERE data_id IS NULL")]
+            "SELECT id FROM data"
+            " LEFT JOIN obj ON (id=obj.data_id)"
+            " LEFT JOIN todel ON (id=todel.data_id)"
+            " WHERE obj.data_id IS NULL"
+            " AND todel.data_id IS NULL")]
+
+    def _dataIdsToPrune(self, limit):
+        min_id = self._todel_min_id
+        data_id_list = [data_id for data_id, in self.query(
+            "SELECT data_id FROM todel WHERE data_id>=%s"
+            " ORDER BY data_id LIMIT %s" % (min_id, limit))]
+        if data_id_list:
+            self._todel_min_id = data_id_list[-1] + 1
+        elif min_id:
+            self._todel_min_id = 0
+            self.query("TRUNCATE TABLE todel")
+        return data_id_list
 
     def _pruneData(self, data_id_list):
         data_id_list = set(data_id_list).difference(self._uncommitted_data)
         if data_id_list:
+            # Split the query to avoid exceeding max_allowed_packet.
+            # Each id is 20 chars maximum.
+            data_id_list = splitList(sorted(data_id_list), 1000000)
             q = self.query
+            if self.LOCK is None:
+                for data_id_list in data_id_list:
+                    q("REPLACE INTO todel VALUES (%s)"
+                      % "),(".join(map(str, data_id_list)))
+                return
             id_list = []
             bigid_list = []
-            # Split the query to avoid exceeding max_allowed_packet.
-            # Each id is 20 chars maximum.
-            for data_id_list in splitList(sorted(data_id_list), 1000000):
+            for data_id_list in data_id_list:
                 for id, value in q(
                         "SELECT id, IF(compression < 128, NULL, value)"
                         " FROM data LEFT JOIN obj ON (id = data_id)"
@@ -655,6 +718,8 @@ class MySQLDatabaseManager(DatabaseManager):
                     " WHERE `partition`=%s AND oid=%s AND tid=%s"
                     % (p, oid, util.u64(data_tid))):
                 return r
+            if p in self._readable_set: # and not checksum:
+                raise UndoPackError
         if not checksum:
             return # delete
         e = self.escape
@@ -725,18 +790,42 @@ class MySQLDatabaseManager(DatabaseManager):
         r = self.query(sql)
         return r[0] if r else (None, None)
 
-    def lockTransaction(self, tid, ttid):
+    def storePackOrder(self, tid, approved, partial, oid_list, pack_tid):
+        u64 = util.u64
+        self.query("INSERT INTO pack VALUES (%s,%s,%s,%s,%s)" % (
+            u64(tid),
+            'NULL' if approved is None else approved,
+            partial,
+            'NULL' if oid_list is None else
+                "'%s'" % self.escape(''.join(oid_list)),
+            u64(pack_tid)))
+
+    def _signPackOrders(self, approved, rejected):
+        def isTID(x):
+            return "tid IN (%s)" % ','.join(map(str, x)) if x else 0
+        approved = isTID(approved)
+        where = " WHERE %s OR %s" % (approved, isTID(rejected))
+        changed = [tid for tid, in self.query("SELECT tid FROM pack" + where)]
+        if changed:
+            self.query("UPDATE pack SET approved = %s%s" % (approved, where))
+        return changed
+
+    def lockTransaction(self, tid, ttid, pack):
         u64 = util.u64
         self.query("UPDATE ttrans SET tid=%d WHERE ttid=%d LIMIT 1"
                    % (u64(tid), u64(ttid)))
+        if pack:
+            self.query("UPDATE pack SET approved=1 WHERE tid=%d" % u64(ttid))
         self.commit()
 
-    def unlockTransaction(self, tid, ttid, trans, obj):
+    def unlockTransaction(self, tid, ttid, trans, obj, pack):
         q = self.query
         u64 = util.u64
         tid = u64(tid)
         if trans:
             q("INSERT INTO trans SELECT * FROM ttrans WHERE tid=%d" % tid)
+            if pack:
+                q("UPDATE pack SET tid=%s WHERE tid=%d" % (tid, u64(ttid)))
             q("DELETE FROM ttrans WHERE tid=%d" % tid)
             if not obj:
                 return
@@ -756,7 +845,9 @@ class MySQLDatabaseManager(DatabaseManager):
 
     def deleteTransaction(self, tid):
         tid = util.u64(tid)
-        self.query("DELETE FROM trans WHERE `partition`=%s AND tid=%s" %
+        self.query("DELETE trans, pack"
+            " FROM trans LEFT JOIN pack USING(tid)"
+            " WHERE `partition`=%s AND tid=%s" %
             (self._getPartition(tid), tid))
 
     def deleteObject(self, oid, serial=None):
@@ -779,7 +870,7 @@ class MySQLDatabaseManager(DatabaseManager):
         if max_tid is not None:
             sql += " AND tid <= %d" % max_tid
         q = self.query
-        q("DELETE FROM trans" + sql)
+        q("DELETE trans, pack FROM trans LEFT JOIN pack USING(tid)" + sql)
         sql = " FROM obj" + sql
         data_id_list = [x for x, in q(
             "SELECT DISTINCT data_id%s AND data_id IS NOT NULL" % sql)]
@@ -789,18 +880,45 @@ class MySQLDatabaseManager(DatabaseManager):
     def getTransaction(self, tid, all = False):
         tid = util.u64(tid)
         q = self.query
-        r = q("SELECT oids, user, description, ext, packed, ttid"
-              " FROM trans WHERE `partition` = %d AND tid = %d"
+        r = q("SELECT trans.oids, user, description, ext, packed, ttid,"
+                " approved, partial, pack.oids, pack_tid"
+              " FROM trans LEFT JOIN pack USING (tid)"
+              " WHERE `partition` = %d AND tid = %d"
               % (self._getReadablePartition(tid), tid))
-        if not r and all:
-            r = q("SELECT oids, user, description, ext, packed, ttid"
-                  " FROM ttrans WHERE tid = %d" % tid)
-        if r:
-            oids, user, desc, ext, packed, ttid = r[0]
-            oid_list = splitOIDField(tid, oids)
-            return oid_list, user, desc, ext, bool(packed), util.p64(ttid)
-
-    def getObjectHistory(self, oid, offset, length):
+        if not r:
+            if not all:
+                return
+            r = q("SELECT ttrans.oids, user, description, ext, packed, ttid,"
+                    " approved, partial, pack.oids, pack_tid"
+                  " FROM ttrans LEFT JOIN pack USING (tid)"
+                  " WHERE tid = %d" % tid)
+            if not r:
+                return
+        oids, user, desc, ext, packed, ttid, \
+            approved, pack_partial, pack_oids, pack_tid = r[0]
+        return (
+            splitOIDField(tid, oids),
+            user, desc, ext,
+            bool(packed), util.p64(ttid),
+            None if pack_partial is None else (
+                None if approved is None else bool(approved),
+                bool(pack_partial),
+                None if pack_oids is None else splitOIDField(tid, pack_oids),
+                util.p64(pack_tid)))
+
+    def _getObjectHistoryForUndo(self, oid, undo_tid):
+        q = self.query
+        args = self._getReadablePartition(oid), oid, undo_tid
+        undo = iter(q("SELECT tid FROM obj"
+                      " WHERE `partition`=%s AND oid=%s AND tid<=%s"
+                      " ORDER BY tid DESC LIMIT 2" % args))
+        if next(undo, (None,))[0] == undo_tid:
+            return next(undo, (None,))[0], q(
+                "SELECT tid, value_tid FROM obj"
+                " WHERE `partition`=%s AND oid=%s AND tid>%s"
+                " ORDER BY tid" % args)
+
+    def getObjectHistoryWithLength(self, oid, offset, length):
         # FIXME: This method doesn't take client's current transaction id as
         # parameter, which means it can return transactions in the future of
         # client's transaction.
@@ -810,10 +928,9 @@ class MySQLDatabaseManager(DatabaseManager):
             "  CAST(CONV(HEX(SUBSTR(value, 5, 4)), 16, 10) AS INT))"
             " FROM obj FORCE INDEX(PRIMARY)"
             " LEFT JOIN data ON (obj.data_id = data.id)"
-            " WHERE `partition` = %d AND oid = %d AND tid >= %d"
+            " WHERE `partition` = %d AND oid = %d"
             " ORDER BY tid DESC LIMIT %d, %d" %
-            (self._getReadablePartition(oid), oid,
-             self._getPackTID(), offset, length))
+            (self._getReadablePartition(oid), oid, offset, length))
         if r:
             return [(p64(tid), length or 0) for tid, length in r]
 
@@ -861,67 +978,31 @@ class MySQLDatabaseManager(DatabaseManager):
             '' if length is None else ' LIMIT %s' % length))
         return [p64(t[0]) for t in r]
 
-    def _updatePackFuture(self, oid, orig_serial, max_serial):
+    def _pack(self, offset, oid, tid, limit=None):
         q = self.query
-        # Before deleting this objects revision, see if there is any
-        # transaction referencing its value at max_serial or above.
-        # If there is, copy value to the first future transaction. Any further
-        # reference is just updated to point to the new data location.
-        value_serial = None
-        kw = {
-          'partition': self._getReadablePartition(oid),
-          'oid': oid,
-          'orig_tid': orig_serial,
-          'max_tid': max_serial,
-          'new_tid': 'NULL',
-        }
-        for kw['table'] in 'obj', 'tobj':
-            for kw['tid'], in q('SELECT tid FROM %(table)s'
-                  ' WHERE `partition`=%(partition)d AND oid=%(oid)d'
-                  ' AND tid>=%(max_tid)d AND value_tid=%(orig_tid)d'
-                  ' ORDER BY tid ASC' % kw):
-                q('UPDATE %(table)s SET value_tid=%(new_tid)s'
-                  ' WHERE `partition`=%(partition)d AND oid=%(oid)d'
-                  ' AND tid=%(tid)d' % kw)
-                if value_serial is None:
-                    # First found, mark its serial for future reference.
-                    kw['new_tid'] = value_serial = kw['tid']
-        return value_serial
-
-    def pack(self, tid, updateObjectDataForPack):
-        # TODO: unit test (along with updatePackFuture)
-        p64 = util.p64
-        tid = util.u64(tid)
-        updatePackFuture = self._updatePackFuture
-        getPartition = self._getReadablePartition
-        q = self.query
-        self._setPackTID(tid)
-        for count, oid, max_serial in q("SELECT COUNT(*) - 1, oid, MAX(tid)"
-                                        " FROM obj FORCE INDEX(PRIMARY)"
-                                        " WHERE tid <= %d GROUP BY oid"
-                                        % tid):
-            partition = getPartition(oid)
-            if q("SELECT 1 FROM obj WHERE `partition` = %d"
-                 " AND oid = %d AND tid = %d AND data_id IS NULL"
-                 % (partition, oid, max_serial)):
-                max_serial += 1
-            elif not count:
-                continue
-            # There are things to delete for this object
-            data_id_set = set()
-            sql = ' FROM obj WHERE `partition`=%d AND oid=%d' \
-                ' AND tid<%d' % (partition, oid, max_serial)
-            for serial, data_id in q('SELECT tid, data_id' + sql):
-                data_id_set.add(data_id)
-                new_serial = updatePackFuture(oid, serial, max_serial)
-                if new_serial:
-                    new_serial = p64(new_serial)
-                updateObjectDataForPack(p64(oid), p64(serial),
-                                        new_serial, data_id)
-            q('DELETE' + sql)
-            data_id_set.discard(None)
-            self._pruneData(data_id_set)
-        self.commit()
+        data_id_set = set()
+        sql = ("SELECT obj.oid,"
+                " IF(data_id IS NULL OR n>1, tid + (data_id IS NULL), NULL)"
+            " FROM (SELECT COUNT(*) AS n, oid, MAX(tid) AS max_tid"
+                " FROM obj FORCE INDEX(PRIMARY)"
+                " WHERE `partition`=%s AND oid%s AND tid<=%s"
+                " GROUP BY oid%s) AS t"
+            " JOIN obj ON `partition`=%s AND t.oid=obj.oid AND tid=max_tid") % (
+            offset,
+            ">=%s" % oid if limit else " IN (%s)" % ','.join(map(str, oid)),
+            tid,
+            " LIMIT %s" % limit if limit else "",
+            offset)
+        oid = None
+        for oid, tid in q(sql):
+            if tid is not None:
+                sql = " FROM obj WHERE `partition`=%s AND oid=%s AND tid<%s" % (
+                    offset, oid, tid)
+                data_id_set.update(*zip(*q("SELECT DISTINCT data_id" + sql)))
+                q("DELETE" + sql)
+        data_id_set.discard(None)
+        self._pruneData(data_id_set)
+        return limit and oid, len(data_id_set)
 
     def checkTIDRange(self, partition, length, min_tid, max_tid):
         count, tid_checksum, max_tid = self.query(

neo/storage/database/sqlite.py

@@ -14,7 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from collections import OrderedDict
+from collections import defaultdict, OrderedDict
 import os
 import sqlite3
 from hashlib import sha1
@@ -24,8 +24,9 @@ import traceback
 from . import LOG_QUERIES
 from .manager import DatabaseManager, splitOIDField
 from neo.lib import logging, util
+from neo.lib.exception import UndoPackError
 from neo.lib.interfaces import implements
-from neo.lib.protocol import ZERO_OID, ZERO_TID, ZERO_HASH
+from neo.lib.protocol import CellStates, ZERO_OID, ZERO_TID, ZERO_HASH
 
 def unique_constraint_message(table, *columns):
     c = sqlite3.connect(":memory:")
@@ -68,7 +69,7 @@ class SQLiteDatabaseManager(DatabaseManager):
              never be used for small requests.
     """
 
-    VERSION = 3
+    VERSION = 4
 
     def _parse(self, database):
         self.db = os.path.expanduser(database)
@@ -80,7 +81,7 @@ class SQLiteDatabaseManager(DatabaseManager):
         logging.info('connecting to SQLite database %r', self.db)
         self.conn = sqlite3.connect(self.db, check_same_thread=False)
         self.conn.text_factory = str
-        self.lock(self.db)
+        self.lockFile(self.db)
         if self.UNSAFE:
             q = self.query
             q("PRAGMA synchronous = OFF").fetchall()
@@ -94,19 +95,20 @@ class SQLiteDatabaseManager(DatabaseManager):
         retry_if_locked(self.conn.commit)
 
     if LOG_QUERIES:
-        def query(self, query):
+        def query(self, query, *args):
             printable_char_list = []
             for c in query.split('\n', 1)[0][:70]:
                 if c not in string.printable or c in '\t\x0b\x0c\r':
                     c = '\\x%02x' % ord(c)
                 printable_char_list.append(c)
             logging.debug('querying %s...', ''.join(printable_char_list))
-            return self.conn.execute(query)
+            return self.conn.execute(query, *args)
     else:
         query = property(lambda self: self.conn.execute)
 
     def erase(self):
-        for t in 'config', 'pt', 'trans', 'obj', 'data', 'ttrans', 'tobj':
+        for t in ('config', 'pt', 'pack', 'trans',
+                  'obj', 'data', 'ttrans', 'tobj'):
             self.query('DROP TABLE IF EXISTS ' + t)
 
     def nonempty(self, table):
@@ -142,16 +144,18 @@ class SQLiteDatabaseManager(DatabaseManager):
         self._alterTable(schema_dict, 'obj')
 
     def _migrate3(self, schema_dict, index_dict):
-        self._alterTable(schema_dict, 'pt', "rid, nid, CASE state"
+        x = 'pt'
+        self._alterTable({x: schema_dict[x].replace('pack', '-- pack')}, x,
+            "rid, nid, CASE state"
             " WHEN 0 THEN -1"  # UP_TO_DATE
             " WHEN 2 THEN -2"  # FEEDING
             " ELSE 1-state END")
 
-    # Let's wait for a more important change to clean up,
-    # so that users can still downgrade.
-    if 0:
-      def _migrate4(self, schema_dict, index_dict):
+    def _migrate4(self, schema_dict, index_dict):
         self._setConfiguration('partitions', None)
+        self._alterTable(schema_dict, 'pt', "*, CASE"
+            " WHEN nid=%s THEN 0"
+            " ELSE NULL END" % self.getUUID())
 
     def _setup(self, dedup=False):
         # BBB: SQLite has transactional DDL but before Python 3.6,
@@ -175,10 +179,19 @@ class SQLiteDatabaseManager(DatabaseManager):
                  partition INTEGER NOT NULL,
                  nid INTEGER NOT NULL,
                  tid INTEGER NOT NULL,
+                 pack INTEGER,
                  PRIMARY KEY (partition, nid)
             ) WITHOUT ROWID
             """
 
+        schema_dict['pack'] = """CREATE TABLE %s (
+                 tid INTEGER PRIMARY KEY,
+                 approved BOOLEAN, -- NULL if not signed
+                 partial BOOLEAN NOT NULL,
+                 oids BLOB, -- same format as trans.oids
+                 pack_tid INTEGER)
+            """
+
         # The table "trans" stores information on committed transactions.
         schema_dict['trans'] = """CREATE TABLE %s (
                  partition INTEGER NOT NULL,
@@ -297,6 +310,35 @@ class SQLiteDatabaseManager(DatabaseManager):
         (tid,), = q("SELECT MAX(tid) FROM obj WHERE `partition`=?", args)
         return tid, oid
 
+    def _getPackOrders(self, min_completed):
+        return self.query(
+            "SELECT * FROM pack WHERE tid >= ? AND tid %% %s IN (%s)"
+            % (self.np, ','.join(map(str, self._readable_set))),
+            (min_completed,))
+
+    def getPackedIDs(self, up_to_date=False):
+        return {offset: util.p64(pack) for offset, pack in self.query(
+            "SELECT partition, pack FROM pt WHERE pack IS NOT NULL"
+            + (" AND tid=-%u" % CellStates.UP_TO_DATE if up_to_date else ""))}
+
+    def _setPartitionPacked(self, partition, pack_id):
+        q = self.query
+        nid = self.getUUID()
+        (i,), = q("SELECT pack FROM pt WHERE partition=? AND nid=?",
+                  (partition, nid))
+        assert i is not None and i < pack_id, (i, pack_id)
+        q("UPDATE pt SET pack=? WHERE partition=? AND nid=?",
+          (pack_id, partition, nid))
+
+    def updateCompletedPackByReplication(self, pack_id, partition=None):
+        pack_id = util.u64(pack_id)
+        sql = "UPDATE pt SET pack=? WHERE nid=? AND (pack IS NULL OR pack>?)"
+        args = [pack_id, self.getUUID(), pack_id]
+        if partition is not None:
+            sql += " AND partition=?"
+            args.append(partition)
+        self.query(sql, args)
+
     def _getDataLastId(self, partition):
         return self.query("SELECT MAX(id) FROM data WHERE %s <= id AND id < %s"
             % (partition << 48, (partition + 1) << 48)).fetchone()[0]
@@ -354,19 +396,22 @@ class SQLiteDatabaseManager(DatabaseManager):
 
     def _changePartitionTable(self, cell_list, reset=False):
         q = self.query
-        if reset:
-            q("DELETE FROM pt")
-        for offset, nid, state in cell_list:
-            # TODO: this logic should move out of database manager
-            # add 'dropCells(cell_list)' to API and use one query
-            # WKRD: Why does SQLite need a statement journal file
-            #       whereas we try to replace only 1 value ?
-            #       We don't want to remove the 'NOT NULL' constraint
-            #       so we must simulate a "REPLACE OR FAIL".
-            q("DELETE FROM pt WHERE partition=? AND nid=?", (offset, nid))
-            if state is not None:
-                q("INSERT OR FAIL INTO pt VALUES (?,?,?)",
-                  (offset, nid, int(state)))
+        delete = set(q("SELECT partition, nid FROM pt")) if reset else set()
+        for cell in cell_list:
+            key = cell[:2]
+            tid = cell[2]
+            if tid is None:
+                delete.add(key)
+            else:
+                delete.discard(key)
+                if q("SELECT 1 FROM pt WHERE partition=? AND nid=?",
+                       key).fetchone():
+                    q("UPDATE pt SET tid=? WHERE partition=? AND nid=?",
+                      (tid,) + key)
+                else:
+                    q("INSERT OR FAIL INTO pt VALUES (?,?,?,?)", cell)
+        for key in delete:
+            q("DELETE FROM pt WHERE partition=? AND nid=?", key)
 
     def _dropPartition(self, *args):
         q = self.query
@@ -375,7 +420,10 @@ class SQLiteDatabaseManager(DatabaseManager):
         if x:
             q("DELETE" + where, args)
             return [x for x, in x]
-        return q("DELETE FROM trans WHERE partition=?", args[:1]).rowcount
+        x = args[:1]
+        q("DELETE FROM pack WHERE tid IN ("
+            "SELECT tid FROM trans JOIN pack USING (tid) WHERE partition=?)", x)
+        return q("DELETE FROM trans WHERE partition=?", x).rowcount
 
     def _getUnfinishedDataIdList(self):
         return [x for x, in self.query(
@@ -386,6 +434,8 @@ class SQLiteDatabaseManager(DatabaseManager):
             " WHERE `partition` IN (%s)" % ','.join(map(str, offset_list))
         q = self.query
         q("DELETE FROM tobj" + where)
+        q("DELETE FROM pack WHERE tid IN ("
+            "SELECT tid FROM ttrans JOIN pack USING (tid)%s)" % where)
         q("DELETE FROM ttrans" + where)
 
     def storeTransaction(self, tid, object_list, transaction, temporary=True):
@@ -446,6 +496,8 @@ class SQLiteDatabaseManager(DatabaseManager):
                     " WHERE partition=? AND oid=? AND tid=?",
                     (p, oid, util.u64(data_tid))):
                 return r
+            if p in self._readable_set: # and not checksum:
+                raise UndoPackError
         if not checksum:
             return # delete
         H = buffer(checksum)
@@ -487,18 +539,39 @@ class SQLiteDatabaseManager(DatabaseManager):
         r = r.fetchone()
         return r or (None, None)
 
-    def lockTransaction(self, tid, ttid):
+    def storePackOrder(self, tid, approved, partial, oid_list, pack_tid):
+        u64 = util.u64
+        self.query("INSERT INTO pack VALUES (?,?,?,?,?)", (
+            u64(tid), approved, partial,
+            None if oid_list is None else buffer(''.join(oid_list)),
+            u64(pack_tid)))
+
+    def _signPackOrders(self, approved, rejected):
+        def isTID(x):
+            return "tid IN (%s)" % ','.join(map(str, x)) if x else 0
+        approved = isTID(approved)
+        where = " WHERE %s OR %s" % (approved, isTID(rejected))
+        changed = [tid for tid, in self.query("SELECT tid FROM pack" + where)]
+        if changed:
+            self.query("UPDATE pack SET approved = %s%s" % (approved, where))
+        return changed
+
+    def lockTransaction(self, tid, ttid, pack):
         u64 = util.u64
         self.query("UPDATE ttrans SET tid=? WHERE ttid=?",
                    (u64(tid), u64(ttid)))
+        if pack:
+            self.query("UPDATE pack SET approved=1 WHERE tid=?", (u64(ttid),))
         self.commit()
 
-    def unlockTransaction(self, tid, ttid, trans, obj):
+    def unlockTransaction(self, tid, ttid, trans, obj, pack):
         q = self.query
         u64 = util.u64
         tid = u64(tid)
         if trans:
             q("INSERT INTO trans SELECT * FROM ttrans WHERE tid=?", (tid,))
+            if pack:
+                q("UPDATE pack SET tid=? WHERE tid=?", (tid, u64(ttid)))
             q("DELETE FROM ttrans WHERE tid=?", (tid,))
             if not obj:
                 return
@@ -519,7 +592,9 @@ class SQLiteDatabaseManager(DatabaseManager):
 
     def deleteTransaction(self, tid):
         tid = util.u64(tid)
-        self.query("DELETE FROM trans WHERE partition=? AND tid=?",
+        q = self.query
+        q("DELETE FROM pack WHERE tid=?", (tid,))
+        q("DELETE FROM trans WHERE partition=? AND tid=?",
             (self._getPartition(tid), tid))
 
     def deleteObject(self, oid, serial=None):
@@ -545,6 +620,8 @@ class SQLiteDatabaseManager(DatabaseManager):
             sql += " AND tid <= ?"
             args.append(max_tid)
         q = self.query
+        q("DELETE FROM pack WHERE tid IN ("
+            "SELECT tid FROM trans JOIN pack USING (tid)%s)" % sql, args)
         q("DELETE FROM trans" + sql, args)
         sql = " FROM obj" + sql
         data_id_list = [x for x, in q(
@@ -555,30 +632,56 @@ class SQLiteDatabaseManager(DatabaseManager):
     def getTransaction(self, tid, all=False):
         tid = util.u64(tid)
         q = self.query
-        r = q("SELECT oids, user, description, ext, packed, ttid"
-              " FROM trans WHERE partition=? AND tid=?",
+        r = q("SELECT trans.oids, user, description, ext, packed, ttid,"
+                " approved, partial, pack.oids, pack_tid"
+              " FROM trans LEFT JOIN pack USING (tid)"
+              " WHERE partition=? AND tid=?",
               (self._getReadablePartition(tid), tid)).fetchone()
-        if not r and all:
-            r = q("SELECT oids, user, description, ext, packed, ttid"
-                  " FROM ttrans WHERE tid=?", (tid,)).fetchone()
-        if r:
-            oids, user, description, ext, packed, ttid = r
-            return splitOIDField(tid, oids), str(user), \
-                str(description), str(ext), packed, util.p64(ttid)
-
-    def getObjectHistory(self, oid, offset, length):
+        if not r:
+            if not all:
+                return
+            r = q("SELECT ttrans.oids, user, description, ext, packed, ttid,"
+                    " approved, partial, pack.oids, pack_tid"
+                  " FROM ttrans LEFT JOIN pack USING (tid)"
+                  " WHERE tid=?", (tid,)).fetchone()
+            if not r:
+                return
+        oids, user, desc, ext, packed, ttid, \
+            approved, pack_partial, pack_oids, pack_tid = r
+        return (
+            splitOIDField(tid, oids),
+            str(user), str(desc), str(ext),
+            bool(packed), util.p64(ttid),
+            None if pack_partial is None else (
+                None if approved is None else bool(approved),
+                bool(pack_partial),
+                None if pack_oids is None else splitOIDField(tid, pack_oids),
+                util.p64(pack_tid)))
+
+    def _getObjectHistoryForUndo(self, oid, undo_tid):
+        q = self.query
+        args = self._getReadablePartition(oid), oid, undo_tid
+        undo = q("SELECT tid FROM obj"
+                 " WHERE partition=? AND oid=? AND tid<=?"
+                 " ORDER BY tid DESC LIMIT 2", args).fetchall()
+        if undo and undo.pop(0)[0] == undo_tid:
+            return undo[0][0] if undo else None, q(
+                "SELECT tid, value_tid FROM obj"
+                " WHERE partition=? AND oid=? AND tid>?"
+                " ORDER BY tid", args).fetchall()
+
+    def getObjectHistoryWithLength(self, oid, offset, length):
         # FIXME: This method doesn't take client's current transaction id as
         # parameter, which means it can return transactions in the future of
         # client's transaction.
         p64 = util.p64
         oid = util.u64(oid)
-        return [(p64(tid), length or 0) for tid, length in self.query("""\
-            SELECT tid, LENGTH(value)
-                FROM obj LEFT JOIN data ON obj.data_id = data.id
-                WHERE partition=? AND oid=? AND tid>=?
-                ORDER BY tid DESC LIMIT ?,?""",
-            (self._getReadablePartition(oid), oid,
-             self._getPackTID(), offset, length))
+        return [(p64(tid), length or 0) for tid, length in self.query(
+            "SELECT tid, LENGTH(value)"
+            " FROM obj LEFT JOIN data ON obj.data_id = data.id"
+            " WHERE partition=? AND oid=?"
+            " ORDER BY tid DESC LIMIT ?,?",
+            (self._getReadablePartition(oid), oid, offset, length))
             ] or None
 
     def _fetchObject(self, oid, tid):
@@ -625,60 +728,31 @@ class SQLiteDatabaseManager(DatabaseManager):
             ORDER BY tid ASC LIMIT ?""",
             (partition, min_tid, max_tid, length))]
 
-    def _updatePackFuture(self, oid, orig_serial, max_serial):
-        # Before deleting this objects revision, see if there is any
-        # transaction referencing its value at max_serial or above.
-        # If there is, copy value to the first future transaction. Any further
-        # reference is just updated to point to the new data location.
-        partition = self._getReadablePartition(oid)
-        value_serial = None
+    _pack = " FROM obj WHERE partition=? AND oid=? AND tid<?"
+    def _pack(self, offset, oid, tid, limit=None,
+            _select_data_id_sql="SELECT DISTINCT data_id" + _pack,
+            _delete_obj_sql="DELETE" + _pack):
         q = self.query
-        for T in '', 't':
-            update = """UPDATE OR FAIL %sobj SET value_tid=?
-                         WHERE partition=? AND oid=? AND tid=?""" % T
-            for serial, in q("""SELECT tid FROM %sobj
-                    WHERE partition=? AND oid=? AND tid>=? AND value_tid=?
-                    ORDER BY tid ASC""" % T,
-                    (partition, oid, max_serial, orig_serial)):
-                q(update, (value_serial, partition, oid, serial))
-                if value_serial is None:
-                    # First found, mark its serial for future reference.
-                    value_serial = serial
-        return value_serial
-
-    def pack(self, tid, updateObjectDataForPack):
-        # TODO: unit test (along with updatePackFuture)
-        p64 = util.p64
-        tid = util.u64(tid)
-        updatePackFuture = self._updatePackFuture
-        getPartition = self._getReadablePartition
-        q = self.query
-        self._setPackTID(tid)
-        for count, oid, max_serial in q("SELECT COUNT(*) - 1, oid, MAX(tid)"
-                                        " FROM obj WHERE tid<=? GROUP BY oid",
-                                        (tid,)):
-            partition = getPartition(oid)
-            if q("SELECT 1 FROM obj WHERE partition=?"
-                 " AND oid=? AND tid=? AND data_id IS NULL",
-                 (partition, oid, max_serial)).fetchone():
-                max_serial += 1
-            elif not count:
-                continue
-            # There are things to delete for this object
-            data_id_set = set()
-            sql = " FROM obj WHERE partition=? AND oid=? AND tid<?"
-            args = partition, oid, max_serial
-            for serial, data_id in q("SELECT tid, data_id" + sql, args):
-                data_id_set.add(data_id)
-                new_serial = updatePackFuture(oid, serial, max_serial)
-                if new_serial:
-                    new_serial = p64(new_serial)
-                updateObjectDataForPack(p64(oid), p64(serial),
-                                        new_serial, data_id)
-            q("DELETE" + sql, args)
-            data_id_set.discard(None)
-            self._pruneData(data_id_set)
-        self.commit()
+        data_id_set = set()
+        value_dict = defaultdict(list)
+        sql = ("SELECT COUNT(*), oid, MAX(tid) FROM obj"
+               " WHERE partition=%s AND tid<=%s AND oid%s GROUP BY oid%s") % (
+            offset, tid,
+            ">=%s" % oid if limit else " IN (%s)" % ','.join(map(str, oid)),
+            " LIMIT %s" % limit if limit else "")
+        oid = None
+        for x, oid, max_tid in q(sql):
+            for x in q("SELECT tid + (data_id IS NULL) FROM obj"
+                  " WHERE partition=? AND oid=? AND tid=?"
+                    " AND (data_id IS NULL OR ?>1)",
+                  (offset, oid, max_tid, x)):
+                x = (offset, oid) + x
+                data_id_set.update(*zip(*q(_select_data_id_sql, x)))
+                q(_delete_obj_sql, x)
+                break
+        data_id_set.discard(None)
+        self._pruneData(data_id_set)
+        return limit and oid, len(data_id_set)
 
     def checkTIDRange(self, partition, length, min_tid, max_tid):
         # XXX: SQLite's GROUP_CONCAT is slow (looks like quadratic)

neo/storage/handlers/__init__.py

@@ -14,12 +14,20 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import weakref
+from functools import partial
 from neo.lib import logging
 from neo.lib.handler import EventHandler
 from neo.lib.exception import PrimaryFailure, ProtocolError, StoppedOperation
 from neo.lib.protocol import uuid_str, NodeStates, NodeTypes, Packets
 
+
+class EventHandler(EventHandler):
+
+    def packetReceived(self, *args):
+        with self.app.dm.lock:
+            self.dispatch(*args)
+
+
 class BaseHandler(EventHandler):
 
     def notifyTransactionFinished(self, conn, ttid, max_tid):
@@ -30,6 +38,7 @@ class BaseHandler(EventHandler):
     def abortTransaction(self, conn, ttid, _):
         self.notifyTransactionFinished(conn, ttid, None)
 
+
 class BaseMasterHandler(BaseHandler):
 
     def connectionLost(self, conn, new_state):
@@ -64,21 +73,52 @@ class BaseMasterHandler(BaseHandler):
                 # See comment in ClientOperationHandler.connectionClosed
                 self.app.tm.abortFor(uuid, even_if_voted=True)
 
+    def notifyPackSigned(self, conn, approved, rejected):
+        app = self.app
+        app.replicator.keepPendingSignedPackOrders(
+            *app.dm.signPackOrders(approved, rejected))
+        if approved:
+            pack_id = max(approved)
+            if app.last_pack_id is None:
+                app.dm.updateCompletedPackByReplication(pack_id)
+            elif app.last_pack_id >= pack_id:
+                return
+            app.last_pack_id = pack_id
+            if app.operational:
+                app.maybePack()
+
     def notifyPartitionChanges(self, conn, ptid, num_replicas, cell_list):
         """This is very similar to Send Partition Table, except that
        the information is only about changes from the previous."""
         app = self.app
         if ptid != 1 + app.pt.getID():
             raise ProtocolError('wrong partition table id')
+        if app.operational:
+            getOutdatedOffsetList = partial(
+                app.pt.getOutdatedOffsetListFor, app.uuid)
+            were_outdated = set(getOutdatedOffsetList())
         app.pt.update(ptid, num_replicas, cell_list, app.nm)
         app.dm.changePartitionTable(app, ptid, num_replicas, cell_list)
         if app.operational:
             app.replicator.notifyPartitionChanges(cell_list)
+            # The U -> !U case is already handled by dm.changePartitionTable.
+            # XXX: What about CORRUPTED cells?
+            were_outdated.difference_update(getOutdatedOffsetList())
+            if were_outdated: # O -> !O
+                # After a cell is discarded,
+                # the smallest pt.pack may be greater.
+                app.notifyPackCompleted()
+                # And we may start processing the next pack order.
+                app.maybePack()
         app.dm.commit()
 
     def askFinalTID(self, conn, ttid):
         conn.answer(Packets.AnswerFinalTID(self.app.dm.getFinalTID(ttid)))
 
+    def askPackOrders(self, conn, min_completed_id):
+        conn.answer(Packets.AnswerPackOrders(
+            self.app.dm.getPackOrders(min_completed_id)))
+
     def notifyRepair(self, conn, *args):
         app = self.app
-        app.dm.repair(weakref.ref(app), *args)
+        app.dm.repair(app, *args)

neo/storage/handlers/client.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from neo.lib import logging
-from neo.lib.exception import NonReadableCell, ProtocolError
+from neo.lib.exception import NonReadableCell, ProtocolError, UndoPackError
 from neo.lib.handler import DelayEvent
 from neo.lib.util import dump, makeChecksum, add64
 from neo.lib.protocol import Packets, Errors, \
@@ -46,7 +46,8 @@ class ClientOperationHandler(BaseHandler):
             #   not releasing write-locks now would lead to a deadlock.
             # - A client node may be disconnected from the master, whereas
             #   there are still voted (and not locked) transactions to abort.
-            app.tm.abortFor(conn.getUUID())
+            with app.dm.lock:
+                app.tm.abortFor(conn.getUUID())
 
     def askTransactionInformation(self, conn, tid):
         t = self.app.dm.getTransaction(tid)
@@ -54,7 +55,7 @@ class ClientOperationHandler(BaseHandler):
             p = Errors.TidNotFound('%s does not exist' % dump(tid))
         else:
             p = Packets.AnswerTransactionInformation(tid, t[1], t[2], t[3],
-                    bool(t[4]), t[0])
+                    t[4], t[0])
         conn.answer(p)
 
     def getEventQueue(self):
@@ -106,6 +107,10 @@ class ClientOperationHandler(BaseHandler):
                     dump(oid), dump(serial), dump(ttid),
                     dump(self.app.tm.getLockingTID(oid)))
             locked = ZERO_TID
+        except UndoPackError:
+            conn.answer(Errors.UndoPackError(
+                'Could not undo for oid %s' % dump(oid)))
+            return
         else:
             if request_time and SLOW_STORE is not None:
                 duration = time.time() - request_time
@@ -199,7 +204,8 @@ class ClientOperationHandler(BaseHandler):
         app = self.app
         if app.tm.loadLocked(oid):
             raise DelayEvent
-        history_list = app.dm.getObjectHistory(oid, first, last - first)
+        history_list = app.dm.getObjectHistoryWithLength(
+            oid, first, last - first)
         if history_list is None:
             p = Errors.OidNotFound(dump(oid))
         else:
@@ -300,5 +306,5 @@ class ClientReadOnlyOperationHandler(ClientOperationHandler):
     # (askObjectUndoSerial is used in undo() but itself is read-only query)
 
     # FIXME askObjectHistory to limit tid <= backup_tid
-    # TODO dm.getObjectHistory has to be first fixed for this
+    # TODO dm.getObjectHistoryWithLength has to be first fixed for this
     #def askObjectHistory(self, conn, oid, first, last):

neo/storage/handlers/identification.py

@@ -16,8 +16,8 @@
 
 from neo.lib import logging
 from neo.lib.exception import NotReadyError, ProtocolError
-from neo.lib.handler import EventHandler
 from neo.lib.protocol import NodeTypes, Packets
+from . import EventHandler
 from .storage import StorageOperationHandler
 from .client import ClientOperationHandler, ClientReadOnlyOperationHandler
 

neo/storage/handlers/initialization.py

@@ -15,9 +15,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from . import BaseMasterHandler
-from neo.lib import logging
 from neo.lib.exception import ProtocolError
-from neo.lib.protocol import Packets, ZERO_TID
+from neo.lib.protocol import Packets
 
 class InitializationHandler(BaseMasterHandler):
 
@@ -50,8 +49,11 @@ class InitializationHandler(BaseMasterHandler):
     def askLastIDs(self, conn):
         dm = self.app.dm
         dm.truncate()
-        ltid, loid = dm.getLastIDs()
-        conn.answer(Packets.AnswerLastIDs(loid, ltid))
+        packed = dm.getPackedIDs()
+        if packed:
+            self.app.completed_pack_id = pack_id = min(packed.itervalues())
+            conn.send(Packets.NotifyPackCompleted(pack_id))
+        conn.answer(Packets.AnswerLastIDs(*dm.getLastIDs()))
 
     def askPartitionTable(self, conn):
         pt = self.app.pt
@@ -64,8 +66,8 @@ class InitializationHandler(BaseMasterHandler):
 
     def validateTransaction(self, conn, ttid, tid):
         dm = self.app.dm
-        dm.lockTransaction(tid, ttid)
-        dm.unlockTransaction(tid, ttid, True, True)
+        dm.lockTransaction(tid, ttid, True)
+        dm.unlockTransaction(tid, ttid, True, True, True)
         dm.commit()
 
     def startOperation(self, conn, backup):

neo/storage/handlers/master.py

@@ -28,19 +28,16 @@ class MasterOperationHandler(BaseMasterHandler):
         assert self.app.operational and backup
         self.app.replicator.startOperation(backup)
 
-    def askLockInformation(self, conn, ttid, tid):
-        self.app.tm.lock(ttid, tid)
+    def askLockInformation(self, conn, ttid, tid, pack):
+        self.app.tm.lock(ttid, tid, pack)
         conn.answer(Packets.AnswerInformationLocked(ttid))
 
     def notifyUnlockInformation(self, conn, ttid):
         self.app.tm.unlock(ttid)
 
-    def askPack(self, conn, tid):
-        app = self.app
-        logging.info('Pack started, up to %s...', dump(tid))
-        app.dm.pack(tid, app.tm.updateObjectDataForPack)
-        logging.info('Pack finished.')
-        conn.answer(Packets.AnswerPack(True))
+    def answerPackOrders(self, conn, pack_list, pack_id):
+        if pack_list:
+            self.app.maybePack(pack_list[0], pack_id)
 
     def answerUnfinishedTransactions(self, conn, *args, **kw):
         self.app.replicator.setUnfinishedTIDList(*args, **kw)

neo/storage/handlers/storage.py

@@ -18,8 +18,9 @@ import weakref
 from functools import wraps
 from neo.lib.connection import ConnectionClosed
 from neo.lib.exception import ProtocolError
-from neo.lib.handler import DelayEvent, EventHandler
+from neo.lib.handler import DelayEvent
 from neo.lib.protocol import Errors, Packets, ZERO_HASH
+from . import EventHandler
 
 def checkConnectionIsReplicatorConnection(func):
     def wrapper(self, conn, *args, **kw):
@@ -47,16 +48,17 @@ class StorageOperationHandler(EventHandler):
     def connectionLost(self, conn, new_state):
         app = self.app
         if app.operational and conn.isClient():
-            uuid = conn.getUUID()
-            if uuid:
-                node = app.nm.getByUUID(uuid)
-            else:
-                node = app.nm.getByAddress(conn.getAddress())
-                node.setUnknown()
-            replicator = app.replicator
-            if replicator.current_node is node:
-                replicator.abort()
-            app.checker.connectionLost(conn)
+            with app.dm.lock:
+                uuid = conn.getUUID()
+                if uuid:
+                    node = app.nm.getByUUID(uuid)
+                else:
+                    node = app.nm.getByAddress(conn.getAddress())
+                    node.setUnknown()
+                replicator = app.replicator
+                if replicator.current_node is node:
+                    replicator.abort()
+                app.checker.connectionLost(conn)
 
     # Client
 
@@ -69,33 +71,36 @@ class StorageOperationHandler(EventHandler):
         self.app.checker.connected(node)
 
     @checkConnectionIsReplicatorConnection
-    def answerFetchTransactions(self, conn, pack_tid, next_tid, tid_list):
+    def answerFetchTransactions(self, conn, next_tid, tid_list, completed_pack):
+        app = self.app
         if tid_list:
-            deleteTransaction = self.app.dm.deleteTransaction
+            deleteTransaction = app.dm.deleteTransaction
             for tid in tid_list:
                 deleteTransaction(tid)
-        assert not pack_tid, "TODO"
+        if completed_pack is not None:
+            app.dm.updateCompletedPackByReplication(
+                completed_pack, app.replicator.current_partition)
         if next_tid:
-            self.app.replicator.fetchTransactions(next_tid)
+            app.replicator.fetchTransactions(next_tid)
         else:
-            self.app.replicator.fetchObjects()
+            app.replicator.fetchObjects()
 
     @checkConnectionIsReplicatorConnection
     def addTransaction(self, conn, tid, user, desc, ext, packed, ttid,
-                                   oid_list):
+                                   oid_list, pack):
         # Directly store the transaction.
         self.app.dm.storeTransaction(tid, (),
             (oid_list, user, desc, ext, packed, ttid), False)
+        if pack:
+            self.app.dm.storePackOrder(tid, *pack)
 
     @checkConnectionIsReplicatorConnection
-    def answerFetchObjects(self, conn, pack_tid, next_tid,
-                                       next_oid, object_dict):
+    def answerFetchObjects(self, conn, next_tid, next_oid, object_dict):
         if object_dict:
             deleteObject = self.app.dm.deleteObject
             for serial, oid_list in object_dict.iteritems():
                 for oid in oid_list:
                     deleteObject(oid, serial)
-        assert not pack_tid, "TODO"
         if next_tid:
             # TODO also provide feedback to master about current replication state (tid)
             self.app.replicator.fetchObjects(next_tid, next_oid)
@@ -176,7 +181,7 @@ class StorageOperationHandler(EventHandler):
 
     @checkFeedingConnection(check=False)
     def askFetchTransactions(self, conn, partition, length, min_tid, max_tid,
-            tid_list):
+            tid_list, ask_pack_info):
         app = self.app
         if app.tm.isLockedTid(max_tid):
             # Wow, backup cluster is fast. Requested transactions are still in
@@ -190,12 +195,12 @@ class StorageOperationHandler(EventHandler):
         conn = weakref.proxy(conn)
         peer_tid_set = set(tid_list)
         dm = app.dm
+        completed_pack = dm.getPackedIDs()[partition] if ask_pack_info else None
         tid_list = dm.getReplicationTIDList(min_tid, max_tid, length + 1,
             partition)
         next_tid = tid_list.pop() if length < len(tid_list) else None
         def push():
             try:
-                pack_tid = None # TODO
                 for tid in tid_list:
                     if tid in peer_tid_set:
                         peer_tid_set.remove(tid)
@@ -206,11 +211,11 @@ class StorageOperationHandler(EventHandler):
                                 "partition %u dropped"
                                 % partition), msg_id)
                             return
-                        oid_list, user, desc, ext, packed, ttid = t
+                        oid_list, user, desc, ext, packed, ttid, pack = t
                         # Sending such packet does not mark the connection
                         # for writing if there's too little data in the buffer.
                         conn.send(Packets.AddTransaction(tid, user,
-                            desc, ext, bool(packed), ttid, oid_list), msg_id)
+                            desc, ext, packed, ttid, oid_list, pack), msg_id)
                         # To avoid delaying several connections simultaneously,
                         # and also prevent the backend from scanning different
                         # parts of the DB at the same time, we ask the
@@ -219,7 +224,7 @@ class StorageOperationHandler(EventHandler):
                         # is flushing another one for a concurrent connection.
                         yield conn.buffering
                 conn.send(Packets.AnswerFetchTransactions(
-                    pack_tid, next_tid, peer_tid_set), msg_id)
+                    next_tid, peer_tid_set, completed_pack), msg_id)
                 yield
             except (weakref.ReferenceError, ConnectionClosed):
                 pass
@@ -242,7 +247,6 @@ class StorageOperationHandler(EventHandler):
             next_tid = next_oid = None
         def push():
             try:
-                pack_tid = None # TODO
                 for serial, oid in object_list:
                     oid_set = object_dict.get(serial)
                     if oid_set:
@@ -265,7 +269,7 @@ class StorageOperationHandler(EventHandler):
                     conn.send(Packets.AddObject(oid, *object), msg_id)
                     yield conn.buffering
                 conn.send(Packets.AnswerFetchObjects(
-                    pack_tid, next_tid, next_oid, object_dict), msg_id)
+                    next_tid, next_oid, object_dict), msg_id)
                 yield
             except (weakref.ReferenceError, ConnectionClosed):
                 pass

neo/storage/replicator.py

@@ -93,7 +93,7 @@ from neo.lib import logging
 from neo.lib.protocol import CellStates, NodeTypes, NodeStates, \
     Packets, INVALID_TID, ZERO_TID, ZERO_OID
 from neo.lib.connection import ClientConnection, ConnectionClosed
-from neo.lib.util import add64, dump, p64
+from neo.lib.util import add64, dump, p64, u64
 from .handlers.storage import StorageOperationHandler
 
 FETCH_COUNT = 1000
@@ -101,7 +101,10 @@ FETCH_COUNT = 1000
 
 class Partition(object):
 
-    __slots__ = 'next_trans', 'next_obj', 'max_ttid'
+    __slots__ = 'next_trans', 'next_obj', 'max_ttid', 'pack'
+
+    def __init__(self):
+        self.pack = [], [] # approved, rejected
 
     def __repr__(self):
         return '<%s(%s) at 0x%x>' % (self.__class__.__name__,
@@ -365,11 +368,13 @@ class Replicator(object):
         assert self.current_node.getConnection().isClient(), self.current_node
         offset = self.current_partition
         p = self.partition_dict[offset]
+        dm = self.app.dm
         if min_tid:
             # More than one chunk ? This could be a full replication so avoid
             # restarting from the beginning by committing now.
-            self.app.dm.commit()
+            dm.commit()
             p.next_trans = min_tid
+            ask_pack_info = False
         else:
             try:
                 addr, name = self.source_dict[offset]
@@ -383,11 +388,13 @@ class Replicator(object):
             logging.debug("starting replication of <partition=%u"
                 " min_tid=%s max_tid=%s> from %r", offset, dump(min_tid),
                 dump(self.replicate_tid), self.current_node)
+            ask_pack_info = True
+            dm.checkNotProcessing(self.app, offset, min_tid == ZERO_TID)
         max_tid = self.replicate_tid
-        tid_list = self.app.dm.getReplicationTIDList(min_tid, max_tid,
+        tid_list = dm.getReplicationTIDList(min_tid, max_tid,
             FETCH_COUNT, offset)
         self._conn_msg_id = self.current_node.ask(Packets.AskFetchTransactions(
-            offset, FETCH_COUNT, min_tid, max_tid, tid_list))
+            offset, FETCH_COUNT, min_tid, max_tid, tid_list, ask_pack_info))
 
     def fetchObjects(self, min_tid=None, min_oid=ZERO_OID):
         offset = self.current_partition
@@ -398,10 +405,12 @@ class Replicator(object):
             p.next_obj = min_tid
             self.updateBackupTID()
             dm.updateCellTID(offset, add64(min_tid, -1))
-            dm.commit() # like in fetchTransactions
         else:
             min_tid = p.next_obj
             p.next_trans = add64(max_tid, 1)
+        if any(p.pack): # only useful in backup mode
+            p.pack = self.app.dm.signPackOrders(*p.pack, auto_commit=False)
+        dm.commit()
         object_dict = {}
         for serial, oid in dm.getReplicationObjectList(min_tid,
                 max_tid, FETCH_COUNT, offset, min_oid):
@@ -429,6 +438,8 @@ class Replicator(object):
             app.tm.replicated(offset, tid)
         logging.debug("partition %u replicated up to %s from %r",
                       offset, dump(tid), self.current_node)
+        if app.pt.getCell(offset, app.uuid).isUpToDate():
+            app.maybePack() # only useful in backup mode
         self.getCurrentConnection().setReconnectionNoDelay()
         self._nextPartition()
 
@@ -476,3 +487,22 @@ class Replicator(object):
                              ' up to %s', offset, addr, dump(tid))
         # Make UP_TO_DATE cells really UP_TO_DATE
         self._nextPartition()
+
+    def filterPackable(self, tid, parts):
+        backup = self.app.dm.getBackupTID()
+        for offset in parts:
+            if backup:
+                p = self.partition_dict[offset]
+                if (None is not p.next_trans <= tid or
+                    None is not p.next_obj <= tid):
+                    continue
+            yield offset
+
+    def keepPendingSignedPackOrders(self, *args):
+        np = self.app.pt.getPartitions()
+        for i, x in enumerate(args):
+            for x in x:
+                try:
+                    self.partition_dict[u64(x) % np].pack[i].append(x)
+                except KeyError:
+                    pass

neo/storage/transactions.py

@@ -42,6 +42,7 @@ class Transaction(object):
         Container for a pending transaction
     """
     _delayed = {}
+    pack = False
     tid = None
     voted = 0
 
@@ -231,17 +232,22 @@ class TransactionManager(EventQueue):
             raise ProtocolError("unknown ttid %s" % dump(ttid))
         object_list = transaction.store_dict.itervalues()
         if txn_info:
-            user, desc, ext, oid_list = txn_info
+            user, desc, ext, oid_list, pack = txn_info
             txn_info = oid_list, user, desc, ext, False, ttid
             transaction.voted = 2
         else:
+            pack = None
             transaction.voted = 1
         # store metadata to temporary table
         dm = self._app.dm
         dm.storeTransaction(ttid, object_list, txn_info)
+        if pack:
+            transaction.pack = True
+            oid_list, pack_tid = pack
+            dm.storePackOrder(ttid, None, bool(oid_list), oid_list, pack_tid)
         dm.commit()
 
-    def lock(self, ttid, tid):
+    def lock(self, ttid, tid, pack):
         """
             Lock a transaction
         """
@@ -256,7 +262,7 @@ class TransactionManager(EventQueue):
         self._load_lock_dict.update(
             dict.fromkeys(transaction.store_dict, ttid))
         if transaction.voted == 2:
-            self._app.dm.lockTransaction(tid, ttid)
+            self._app.dm.lockTransaction(tid, ttid, pack)
         else:
             assert transaction.voted
 
@@ -273,7 +279,8 @@ class TransactionManager(EventQueue):
         dm = self._app.dm
         dm.unlockTransaction(tid, ttid,
             transaction.voted == 2,
-            transaction.store_dict)
+            transaction.store_dict,
+            transaction.pack)
         self._app.em.setTimeout(time() + 1, dm.deferCommit())
         self.abort(ttid, even_if_locked=True)
 
@@ -564,10 +571,3 @@ class TransactionManager(EventQueue):
             logging.info('    %s by %s', dump(oid), dump(ttid))
         self.logQueuedEvents()
         self.read_queue.logQueuedEvents()
-
-    def updateObjectDataForPack(self, oid, orig_serial, new_serial, data_id):
-        lock_tid = self.getLockingTID(oid)
-        if lock_tid is not None:
-            transaction = self._transaction_dict[lock_tid]
-            if transaction.store_dict[oid][2] == orig_serial:
-                transaction.store(oid, data_id, new_serial)

neo/tests/__init__.py

@@ -25,6 +25,7 @@ import socket
 import subprocess
 import sys
 import tempfile
+import thread
 import unittest
 import weakref
 import transaction
@@ -38,10 +39,12 @@ except ImportError:
     from cPickle import Unpickler
 from functools import wraps
 from inspect import isclass
+from itertools import islice
 from .mock import Mock
 from neo.lib import debug, event, logging
 from neo.lib.protocol import NodeTypes, Packet, Packets, UUID_NAMESPACES
 from neo.lib.util import cached_property
+from neo.storage.database.manager import DatabaseManager
 from time import time, sleep
 from struct import pack, unpack
 from unittest.case import _ExpectedFailure, _UnexpectedSuccess
@@ -77,6 +80,8 @@ DB_INSTALL = os.getenv('NEO_DB_INSTALL', 'mysql_install_db')
 DB_MYSQLD = os.getenv('NEO_DB_MYSQLD', '/usr/sbin/mysqld')
 DB_MYCNF = os.getenv('NEO_DB_MYCNF')
 
+DatabaseManager.TEST_IDENT = thread.get_ident()
+
 adapter = os.getenv('NEO_TESTS_ADAPTER')
 if adapter:
     from neo.storage.database import getAdapterKlass
@@ -629,6 +634,10 @@ class Patch(object):
     def __exit__(self, t, v, tb):
         self.__del__()
 
+def consume(iterator, n):
+    """Advance the iterator n-steps ahead and returns the last consumed item"""
+    return next(islice(iterator, n-1, n))
+
 def unpickle_state(data):
     unpickler = Unpickler(StringIO(data))
     unpickler.persistent_load = PersistentReferenceFactory().persistent_load

neo/tests/master/testClientHandler.py

-#
-# Copyright (C) 2009-2019  Nexedi SA
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import unittest
-from ..mock import Mock
-from .. import NeoUnitTestBase
-from neo.lib.util import p64
-from neo.lib.protocol import NodeTypes, NodeStates, Packets
-from neo.master.app import Application
-from neo.master.handlers.client import ClientServiceHandler
-
-class MasterClientHandlerTests(NeoUnitTestBase):
-
-    def setUp(self):
-        NeoUnitTestBase.setUp(self)
-        # create an application object
-        config = self.getMasterConfiguration(master_number=1, replicas=1)
-        self.app = Application(config)
-        self.app.em.close()
-        self.app.em = Mock()
-        self.app.loid = '\0' * 8
-        self.app.tm.setLastTID('\0' * 8)
-        self.service = ClientServiceHandler(self.app)
-        # define some variable to simulate client and storage node
-        self.client_port = 11022
-        self.storage_port = 10021
-        self.client_address = ('127.0.0.1', self.client_port)
-        self.storage_address = ('127.0.0.1', self.storage_port)
-        self.storage_uuid = self.getStorageUUID()
-        # register the storage
-        self.app.nm.createStorage(
-            uuid=self.storage_uuid,
-            address=self.storage_address,
-        )
-
-    def identifyToMasterNode(self, node_type=NodeTypes.STORAGE, ip="127.0.0.1",
-                             port=10021):
-        """Do first step of identification to MN """
-        # register the master itself
-        uuid = self.getNewUUID(node_type)
-        self.app.nm.createFromNodeType(
-            node_type,
-            address=(ip, port),
-            uuid=uuid,
-            state=NodeStates.RUNNING,
-        )
-        return uuid
-
-    def test_askPack(self):
-        self.assertEqual(self.app.packing, None)
-        self.app.nm.createClient()
-        tid = self.getNextTID()
-        peer_id = 42
-        conn = self.getFakeConnection(peer_id=peer_id)
-        storage_uuid = self.storage_uuid
-        storage_conn = self.getFakeConnection(storage_uuid,
-            self.storage_address, is_server=True)
-        self.app.nm.getByUUID(storage_uuid).setConnection(storage_conn)
-        self.service.askPack(conn, tid)
-        self.checkNoPacketSent(conn)
-        ptid = self.checkAskPacket(storage_conn, Packets.AskPack)._args[0]
-        self.assertEqual(ptid, tid)
-        self.assertTrue(self.app.packing[0] is conn)
-        self.assertEqual(self.app.packing[1], peer_id)
-        self.assertEqual(self.app.packing[2], {storage_uuid})
-        # Asking again to pack will cause an immediate error
-        storage_uuid = self.identifyToMasterNode(port=10022)
-        storage_conn = self.getFakeConnection(storage_uuid,
-            self.storage_address, is_server=True)
-        self.app.nm.getByUUID(storage_uuid).setConnection(storage_conn)
-        self.service.askPack(conn, tid)
-        self.checkNoPacketSent(storage_conn)
-        status = self.checkAnswerPacket(conn, Packets.AnswerPack)._args[0]
-        self.assertFalse(status)
-
-if __name__ == '__main__':
-    unittest.main()
-

neo/tests/master/testStorageHandler.py

-#
-# Copyright (C) 2009-2019  Nexedi SA
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import unittest
-from ..mock import Mock
-from .. import NeoUnitTestBase
-from neo.lib.protocol import NodeTypes, Packets
-from neo.master.app import Application
-from neo.master.handlers.storage import StorageServiceHandler
-
-class MasterStorageHandlerTests(NeoUnitTestBase):
-
-    def setUp(self):
-        NeoUnitTestBase.setUp(self)
-        # create an application object
-        config = self.getMasterConfiguration(master_number=1, replicas=1)
-        self.app = Application(config)
-        self.app.em.close()
-        self.app.em = Mock()
-        self.service = StorageServiceHandler(self.app)
-
-    def _allocatePort(self):
-        self.port = getattr(self, 'port', 1000) + 1
-        return self.port
-
-    def _getStorage(self):
-        return self.identifyToMasterNode(node_type=NodeTypes.STORAGE,
-                ip='127.0.0.1', port=self._allocatePort())
-
-    def identifyToMasterNode(self, node_type=NodeTypes.STORAGE, ip="127.0.0.1",
-                             port=10021):
-        """Do first step of identification to MN
-        """
-        nm = self.app.nm
-        uuid = self.getNewUUID(node_type)
-        node = nm.createFromNodeType(node_type, address=(ip, port),
-                uuid=uuid)
-        conn = self.getFakeConnection(node.getUUID(), node.getAddress(), True)
-        node.setConnection(conn)
-        return (node, conn)
-
-    def test_answerPack(self):
-        # Note: incoming status has no meaning here, so it's left to False.
-        node1, conn1 = self._getStorage()
-        node2, conn2 = self._getStorage()
-        self.app.packing = None
-        # Does nothing
-        self.service.answerPack(None, False)
-
-        client_conn = Mock({
-            'getPeerId': 512,
-        })
-        client_peer_id = 42
-        self.app.packing = (client_conn, client_peer_id,
-                            {conn1.getUUID(), conn2.getUUID()})
-        self.service.answerPack(conn1, False)
-        self.checkNoPacketSent(client_conn)
-        self.assertEqual(self.app.packing[2], {conn2.getUUID()})
-        self.service.answerPack(conn2, False)
-        packet = self.checkNotifyPacket(client_conn, Packets.AnswerPack)
-        # TODO: verify packet peer id
-        self.assertTrue(packet._args[0])
-        self.assertEqual(self.app.packing, None)
-
-if __name__ == '__main__':
-    unittest.main()
-

neo/tests/protocol

@@ -3,14 +3,14 @@ AbortTransaction(p64,[int])
 AcceptIdentification(NodeTypes,?int,?int)
 AddObject(p64,p64,int,bin,bin,?p64)
 AddPendingNodes([int])
-AddTransaction(p64,bin,bin,bin,bool,p64,[p64])
+AddTransaction(p64,bin,bin,bin,bool,p64,[p64],?(?bool,bool,?[p64],p64))
 AnswerBeginTransaction(p64)
 AnswerCheckCurrentSerial(?p64)
 AnswerCheckSerialRange(int,bin,p64,bin,p64)
 AnswerCheckTIDRange(int,bin,p64)
 AnswerClusterState(?ClusterStates)
-AnswerFetchObjects(?,?p64,?p64,{:})
-AnswerFetchTransactions(?,?p64,[])
+AnswerFetchObjects(?p64,?p64,{:})
+AnswerFetchTransactions(?p64,[],?p64)
 AnswerFinalTID(p64)
 AnswerInformationLocked(p64)
 AnswerLastIDs(?p64,?p64)
@@ -22,7 +22,7 @@ AnswerNodeList([(NodeTypes,?(bin,int),?int,NodeStates,?float)])
 AnswerObject(p64,p64,?p64,?int,bin,bin,?p64)
 AnswerObjectHistory(p64,[(p64,int)])
 AnswerObjectUndoSerial({p64:(p64,?p64,bool)})
-AnswerPack(bool)
+AnswerPackOrders([(p64,?bool,bool,?[p64],p64)])
 AnswerPartitionList(int,int,[[(int,CellStates)]])
 AnswerPartitionTable(int,int,[[(int,CellStates)]])
 AnswerPrimary(int)
@@ -43,12 +43,12 @@ AskCheckSerialRange(int,int,p64,p64,p64)
 AskCheckTIDRange(int,int,p64,p64)
 AskClusterState()
 AskFetchObjects(int,int,p64,p64,p64,{p64:[p64]})
-AskFetchTransactions(int,int,p64,p64,[p64])
+AskFetchTransactions(int,int,p64,p64,[p64],bool)
 AskFinalTID(p64)
-AskFinishTransaction(p64,[p64],[p64])
+AskFinishTransaction(p64,[p64],[p64],?(?[p64],p64))
 AskLastIDs()
 AskLastTransaction()
-AskLockInformation(p64,p64)
+AskLockInformation(p64,p64,bool)
 AskLockedTransactions()
 AskMonitorInformation()
 AskNewOIDs(int)
@@ -56,14 +56,14 @@ AskNodeList(NodeTypes)
 AskObject(p64,?p64,?p64)
 AskObjectHistory(p64,int,int)
 AskObjectUndoSerial(p64,p64,p64,[p64])
-AskPack(p64)
+AskPackOrders(p64)
 AskPartitionList(int,int,?)
 AskPartitionTable()
 AskPrimary()
 AskRecovery()
 AskRelockObject(p64,p64)
 AskStoreObject(p64,p64,int,bin,bin,?p64,?p64)
-AskStoreTransaction(p64,bin,bin,bin,[p64])
+AskStoreTransaction(p64,bin,bin,bin,[p64],?(?[p64],p64))
 AskTIDs(int,int,int)
 AskTIDsFrom(p64,p64,int,int)
 AskTransactionInformation(p64)
@@ -79,6 +79,8 @@ NotifyClusterInformation(ClusterStates)
 NotifyDeadlock(p64,p64)
 NotifyMonitorInformation({bin:any})
 NotifyNodeInformation(float,[(NodeTypes,?(bin,int),?int,NodeStates,?float)])
+NotifyPackCompleted(p64)
+NotifyPackSigned([p64],[p64])
 NotifyPartitionChanges(int,int,[(int,int,CellStates)])
 NotifyPartitionCorrupted(int,[int])
 NotifyReady()
@@ -101,3 +103,5 @@ StopOperation()
 Truncate(p64)
 TweakPartitionTable(bool,[int])
 ValidateTransaction(p64,p64)
+WaitForPack(p64)
+WaitedForPack()

neo/tests/storage/testStorageDBTests.py

@@ -14,12 +14,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import string, unittest
 from binascii import a2b_hex
 from contextlib import closing, contextmanager
-import unittest
-from neo.lib.util import add64, p64, u64
+from copy import copy
+from neo.lib.util import add64, p64, u64, makeChecksum
 from neo.lib.protocol import CellStates, ZERO_HASH, ZERO_OID, ZERO_TID, MAX_TID
+from neo.storage.database.manager import MVCCDatabaseManager
 from .. import NeoUnitTestBase
+from ..mock import Mock
 
 
 class StorageDBTests(NeoUnitTestBase):
@@ -49,7 +52,9 @@ class StorageDBTests(NeoUnitTestBase):
         uuid = self.getStorageUUID()
         db.setUUID(uuid)
         self.assertEqual(uuid, db.getUUID())
-        db.changePartitionTable(None, 1, 0,
+        app = Mock()
+        app.last_pack_id = None
+        db.changePartitionTable(app, 1, 0,
             [(i, uuid, CellStates.UP_TO_DATE) for i in xrange(num_partitions)],
             reset=True)
         self.assertEqual(num_partitions, 1 + db._getMaxPartition())
@@ -67,10 +72,10 @@ class StorageDBTests(NeoUnitTestBase):
     def commitTransaction(self, tid, objs, txn, commit=True):
         ttid = txn[-1]
         self.db.storeTransaction(ttid, objs, txn)
-        self.db.lockTransaction(tid, ttid)
+        self.db.lockTransaction(tid, ttid, None)
         yield
         if commit:
-            self.db.unlockTransaction(tid, ttid, True, objs)
+            self.db.unlockTransaction(tid, ttid, True, objs, False)
             self.db.commit()
         elif commit is not None:
             self.db.abortTransaction(ttid)
@@ -189,25 +194,25 @@ class StorageDBTests(NeoUnitTestBase):
         with self.commitTransaction(tid1, objs1, txn1), \
              self.commitTransaction(tid2, objs2, txn2):
             self.assertEqual(self.db.getTransaction(tid1, True),
-                             ([oid1], 'user', 'desc', 'ext', False, p64(1)))
+                ([oid1], 'user', 'desc', 'ext', False, p64(1), None))
             self.assertEqual(self.db.getTransaction(tid2, True),
-                             ([oid2], 'user', 'desc', 'ext', False, p64(2)))
+                ([oid2], 'user', 'desc', 'ext', False, p64(2), None))
             self.assertEqual(self.db.getTransaction(tid1, False), None)
             self.assertEqual(self.db.getTransaction(tid2, False), None)
-        result = self.db.getTransaction(tid1, True)
-        self.assertEqual(result, ([oid1], 'user', 'desc', 'ext', False, p64(1)))
-        result = self.db.getTransaction(tid2, True)
-        self.assertEqual(result, ([oid2], 'user', 'desc', 'ext', False, p64(2)))
-        result = self.db.getTransaction(tid1, False)
-        self.assertEqual(result, ([oid1], 'user', 'desc', 'ext', False, p64(1)))
-        result = self.db.getTransaction(tid2, False)
-        self.assertEqual(result, ([oid2], 'user', 'desc', 'ext', False, p64(2)))
+        self.assertEqual(self.db.getTransaction(tid1, True),
+            ([oid1], 'user', 'desc', 'ext', False, p64(1), None))
+        self.assertEqual(self.db.getTransaction(tid2, True),
+            ([oid2], 'user', 'desc', 'ext', False, p64(2), None))
+        self.assertEqual(self.db.getTransaction(tid1, False),
+            ([oid1], 'user', 'desc', 'ext', False, p64(1), None))
+        self.assertEqual(self.db.getTransaction(tid2, False),
+            ([oid2], 'user', 'desc', 'ext', False, p64(2), None))
 
     def test_deleteTransaction(self):
         txn, objs = self.getTransaction([])
         tid = txn[-1]
         self.db.storeTransaction(tid, objs, txn, False)
-        self.assertEqual(self.db.getTransaction(tid), txn)
+        self.assertEqual(self.db.getTransaction(tid), txn + (None,))
         self.db.deleteTransaction(tid)
         self.assertEqual(self.db.getTransaction(tid), None)
 
@@ -265,13 +270,13 @@ class StorageDBTests(NeoUnitTestBase):
         with self.commitTransaction(tid1, objs1, txn1), \
              self.commitTransaction(tid2, objs2, txn2, None):
             pass
-        result = self.db.getTransaction(tid1, True)
-        self.assertEqual(result, ([oid1], 'user', 'desc', 'ext', False, p64(1)))
-        result = self.db.getTransaction(tid2, True)
-        self.assertEqual(result, ([oid2], 'user', 'desc', 'ext', False, p64(2)))
+        self.assertEqual(self.db.getTransaction(tid1, True),
+            ([oid1], 'user', 'desc', 'ext', False, p64(1), None))
+        self.assertEqual(self.db.getTransaction(tid2, True),
+            ([oid2], 'user', 'desc', 'ext', False, p64(2), None))
         # get from non-temporary only
-        result = self.db.getTransaction(tid1, False)
-        self.assertEqual(result, ([oid1], 'user', 'desc', 'ext', False, p64(1)))
+        self.assertEqual(self.db.getTransaction(tid1, False),
+            ([oid1], 'user', 'desc', 'ext', False, p64(1), None))
         self.assertEqual(self.db.getTransaction(tid2, False), None)
 
     def test_getObjectHistory(self):
@@ -282,17 +287,17 @@ class StorageDBTests(NeoUnitTestBase):
         txn3, objs3 = self.getTransaction([oid])
         # one revision
         self.db.storeTransaction(tid1, objs1, txn1, False)
-        result = self.db.getObjectHistory(oid, 0, 3)
+        result = self.db.getObjectHistoryWithLength(oid, 0, 3)
         self.assertEqual(result, [(tid1, 0)])
-        result = self.db.getObjectHistory(oid, 1, 1)
+        result = self.db.getObjectHistoryWithLength(oid, 1, 1)
         self.assertEqual(result, None)
         # two revisions
         self.db.storeTransaction(tid2, objs2, txn2, False)
-        result = self.db.getObjectHistory(oid, 0, 3)
+        result = self.db.getObjectHistoryWithLength(oid, 0, 3)
         self.assertEqual(result, [(tid2, 0), (tid1, 0)])
-        result = self.db.getObjectHistory(oid, 1, 3)
+        result = self.db.getObjectHistoryWithLength(oid, 1, 3)
         self.assertEqual(result, [(tid1, 0)])
-        result = self.db.getObjectHistory(oid, 2, 3)
+        result = self.db.getObjectHistoryWithLength(oid, 2, 3)
         self.assertEqual(result, None)
 
     def _storeTransactions(self, count):
@@ -439,5 +444,31 @@ class StorageDBTests(NeoUnitTestBase):
             db.findUndoTID(oid1, tid4, tid1, None),
             (tid3, None, True))
 
+    def testDeferredPruning(self):
+        self.setupDB(1, True)
+        db = self.db
+        if isinstance(db, MVCCDatabaseManager):
+            self.assertFalse(db.nonempty('todel'))
+            self.assertEqual([
+                db.storeData(makeChecksum(x), ZERO_OID, x, 0, None)
+                for x in string.digits
+            ], range(0, 10))
+            db2 = copy(db)
+            for x in (3, 9, 4), (4, 7, 6):
+                self.assertIsNone(db2._pruneData(x))
+            db.commit()
+            db2.commit()
+            for expected in (3, 4, 6), (7, 9):
+                self.assertTrue(db.nonempty('todel'))
+                x = db._dataIdsToPrune(3)
+                self.assertEqual(tuple(x), expected)
+                self.assertEqual(db._pruneData(x), len(expected))
+            self.assertFalse(db._dataIdsToPrune(3))
+            self.assertFalse(db2.nonempty('todel'))
+            self.assertEqual(db._pruneData(range(10)), 5)
+            self.assertFalse(db.nonempty('todel'))
+        else:
+            self.assertIsNone(db.nonempty('todel'))
+
 if __name__ == "__main__":
     unittest.main()

neo/tests/storage/testStorageMySQL.py

@@ -30,13 +30,24 @@ from neo.storage.database.mysql import (MySQLDatabaseManager,
 class ServerGone(object):
 
     @contextmanager
-    def __new__(cls, db):
+    def __new__(cls, db, once):
         self = object.__new__(cls)
-        with Patch(db, conn=self) as self._p:
-            yield self._p
+        with Patch(db, conn=self) as p:
+            if once:
+                self.__revert = p.revert
+                try:
+                    yield p
+                finally:
+                    del self.__revert
+            else:
+                with Patch(db, close=lambda orig: None):
+                    yield
+
+    def __revert(self):
+        pass
 
     def query(self, *args):
-        self._p.revert()
+        self.__revert()
         raise OperationalError(SERVER_GONE_ERROR, 'this is a test')
 
 
@@ -67,7 +78,7 @@ class StorageMySQLdbTests(StorageDBTests):
         return db
 
     def test_ServerGone(self):
-        with ServerGone(self.db) as p:
+        with ServerGone(self.db, True) as p:
             self.assertRaises(ProgrammingError, self.db.query, 'QUERY')
             self.assertFalse(p.applied)
 

neo/tests/storage/testTransactions.py

-#
-# Copyright (C) 2010-2019  Nexedi SA
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import unittest
-from ..mock import Mock
-from .. import NeoUnitTestBase
-from neo.lib.util import p64
-from neo.storage.transactions import TransactionManager
-
-
-class TransactionManagerTests(NeoUnitTestBase):
-
-    def setUp(self):
-        NeoUnitTestBase.setUp(self)
-        self.app = Mock()
-        # no history
-        self.app.dm = Mock({'getObjectHistory': []})
-        self.app.pt = Mock({'isAssigned': True, 'getPartitions': 2})
-        self.app.em = Mock({'setTimeout': None})
-        self.manager = TransactionManager(self.app)
-
-    def register(self, uuid, ttid):
-        self.manager.register(Mock({'getUUID': uuid}), ttid)
-
-    def test_updateObjectDataForPack(self):
-        ram_serial = self.getNextTID()
-        oid = p64(1)
-        orig_serial = self.getNextTID()
-        uuid = self.getClientUUID()
-        locking_serial = self.getNextTID()
-        other_serial = self.getNextTID()
-        new_serial = self.getNextTID()
-        data_id = (1 << 48) + 2
-        self.register(uuid, locking_serial)
-        # Object not known, nothing happens
-        self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
-            oid), None)
-        self.manager.updateObjectDataForPack(oid, orig_serial, None, data_id)
-        self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
-            oid), None)
-        self.manager.abort(locking_serial, even_if_locked=True)
-        # Object known, but doesn't point at orig_serial, it is not updated
-        self.register(uuid, locking_serial)
-        self.manager.storeObject(locking_serial, ram_serial, oid, 0, "3" * 20,
-            'bar', None)
-        holdData = self.app.dm.mockGetNamedCalls('holdData')
-        self.assertEqual(holdData.pop(0).params,
-                         ("3" * 20, oid, 'bar', 0, None))
-        orig_object = self.manager.getObjectFromTransaction(locking_serial,
-            oid)
-        self.manager.updateObjectDataForPack(oid, orig_serial, None, data_id)
-        self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
-            oid), orig_object)
-        self.manager.abort(locking_serial, even_if_locked=True)
-
-        self.register(uuid, locking_serial)
-        self.manager.storeObject(locking_serial, ram_serial, oid, None, None,
-            None, other_serial)
-        orig_object = self.manager.getObjectFromTransaction(locking_serial,
-            oid)
-        self.manager.updateObjectDataForPack(oid, orig_serial, None, data_id)
-        self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
-            oid), orig_object)
-        self.manager.abort(locking_serial, even_if_locked=True)
-        # Object known and points at undone data it gets updated
-        self.register(uuid, locking_serial)
-        self.manager.storeObject(locking_serial, ram_serial, oid, None, None,
-            None, orig_serial)
-        self.manager.updateObjectDataForPack(oid, orig_serial, new_serial,
-            data_id)
-        self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
-            oid), (oid, data_id, new_serial))
-        self.manager.abort(locking_serial, even_if_locked=True)
-
-        self.register(uuid, locking_serial)
-        self.manager.storeObject(locking_serial, ram_serial, oid, None, None,
-            None, orig_serial)
-        self.manager.updateObjectDataForPack(oid, orig_serial, None, data_id)
-        self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
-            oid), (oid, data_id, None))
-        self.manager.abort(locking_serial, even_if_locked=True)
-
-if __name__ == "__main__":
-    unittest.main()

neo/tests/stress.py

@@ -200,7 +200,7 @@ class StressApplication(AdminApplication):
         if conn:
             conn.ask(Packets.AskLastIDs())
 
-    def answerLastIDs(self, loid, ltid):
+    def answerLastIDs(self, ltid, loid):
         self.loid = loid
         self.ltid = ltid
         self.em.setTimeout(int(time.time() + 1), self.askLastIDs)

neo/tests/threaded/__init__.py

@@ -555,8 +555,12 @@ class LoggerThreadName(str):
         return id(self)
 
     def __str__(self):
+        t = threading.currentThread()
+        if t.name == 'BackgroundWorker':
+            t, = t._Thread__args
+            return t().node_name
         try:
-            return threading.currentThread().node_name
+            return t.node_name
         except AttributeError:
             return str.__str__(self)
 
@@ -1078,6 +1082,20 @@ class NEOCluster(object):
         self.storage_list[:] = (x[r] for r in r)
         return self.storage_list
 
+    def ticAndJoinStorageTasks(self):
+        while True:
+            Serialized.tic()
+            for s in self.storage_list:
+                try:
+                    join = s.dm._background_worker._thread.join
+                    break
+                except AttributeError:
+                    pass
+            else:
+                break
+            join()
+
+
 class NEOThreadedTest(NeoTestBase):
 
     __run_count = {}

neo/tests/threaded/test.py

@@ -798,7 +798,9 @@ class Test(NEOThreadedTest):
     def testStorageUpgrade1(self, cluster):
         storage = cluster.storage
         # Disable migration steps that aren't idempotent.
-        with Patch(storage.dm.__class__, _migrate3=lambda *_: None):
+        def noop(*_): pass
+        with Patch(storage.dm.__class__, _migrate3=noop), \
+             Patch(storage.dm.__class__, _migrate4=noop):
             t, c = cluster.getTransaction()
             storage.dm.setConfiguration("version", None)
             c.root()._p_changed = 1
@@ -1776,7 +1778,7 @@ class Test(NEOThreadedTest):
                 for e, s in zip(expected, cluster.storage_list):
                     while 1:
                         self.tic()
-                        if s.dm._repairing is None:
+                        if s.dm._background_worker._orphan is None:
                             break
                         time.sleep(.1)
                     self.assertEqual(e, s.getDataLockInfo())
@@ -2696,7 +2698,6 @@ class Test(NEOThreadedTest):
         big_id_list = ('\x7c' * 8, '\x7e' * 8), ('\x7b' * 8, '\x7d' * 8)
         for i in 0, 1:
             dm = cluster.storage_list[i].dm
-            expected = dm.getLastTID(u64(MAX_TID)), dm.getLastIDs()
             oid, tid = big_id_list[i]
             for j, expected in (
                     (1 - i, (dm.getLastTID(u64(MAX_TID)), dm.getLastIDs())),
@@ -2721,7 +2722,6 @@ class Test(NEOThreadedTest):
             dump_dict[s.uuid] = dm.dump()
             with open(path % (s.getAdapter(), s.uuid)) as f:
                 dm.restore(f.read())
-            dm.setConfiguration('partitions', None) # XXX: see dm._migrate4
         with NEOCluster(storage_count=3, partitions=3, replicas=1,
                         name=self._testMethodName) as cluster:
             s1, s2, s3 = cluster.storage_list

neo/tests/threaded/testImporter.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from contextlib import contextmanager
 from cPickle import Pickler, Unpickler
 from cStringIO import StringIO
 from itertools import izip_longest
@@ -213,8 +214,8 @@ class ImporterTests(NEOThreadedTest):
             # does not import data too fast and we test read/write access
             # by the client during the import.
             dm = cluster.storage.dm
-            def doOperation(app):
-                del dm.doOperation
+            def operational(app):
+                del dm.operational
                 try:
                     while True:
                         if app.task_queue:
@@ -222,7 +223,9 @@ class ImporterTests(NEOThreadedTest):
                         app._poll()
                 except StopIteration:
                     app.task_queue.pop()
-            dm.doOperation = doOperation
+                assert not app.task_queue
+                yield
+            dm.operational = contextmanager(operational)
             cluster.start()
             t, c = cluster.getTransaction()
             r = c.root()['tree']
@@ -234,12 +237,14 @@ class ImporterTests(NEOThreadedTest):
             storage._cache.clear()
             storage.loadBefore(r._p_oid, r._p_serial)
             ##
-            self.assertRaisesRegexp(NotImplementedError, " getObjectHistory$",
+            self.assertRaisesRegexp(NotImplementedError,
+                                    " getObjectHistoryWithLength$",
                                     c.db().history, r._p_oid)
             h = random_tree.hashTree(r)
             h(30)
             logging.info("start migration")
-            dm.doOperation(cluster.storage)
+            with dm.operational(cluster.storage):
+                pass
             # Adjust if needed. Must remain > 0.
             beforeCheck(h, 22)
             # New writes after the switch to NEO.
@@ -285,16 +290,18 @@ class ImporterTests(NEOThreadedTest):
                 x = type(db).__name__
                 if x == 'MySQLDatabaseManager':
                     from neo.tests.storage.testStorageMySQL import ServerGone
-                    with ServerGone(db):
+                    with ServerGone(db, False):
                         orig(db, *args)
                     self.fail()
                 else:
                     assert x == 'SQLiteDatabaseManager'
-                    tid_list.append(None)
+                    tid_list.insert(-1, None)
                     p.revert()
             return orig(db, *args)
         def sleep(orig, seconds):
+            logging.info("sleep(%s)", seconds)
             self.assertEqual(len(tid_list), 5)
+            tid_list[-1] = None
             p.revert()
         with Patch(importer, FORK=False), \
              Patch(TransactionRecord, __init__=__init__), \
@@ -303,6 +310,7 @@ class ImporterTests(NEOThreadedTest):
             self._importFromFileStorage()
             self.assertFalse(p.applied)
         self.assertEqual(len(tid_list), 13)
+        self.assertIsNone(tid_list[4])
 
     def testThreadedWritebackWithUnbalancedPartitions(self):
         N = 7
@@ -409,7 +417,7 @@ class ImporterTests(NEOThreadedTest):
             storage = cluster.storage
             dm = storage.dm
             with storage.patchDeferred(dm._finished):
-                with storage.patchDeferred(dm.doOperation):
+                with storage.patchDeferred(storage.newTask):
                     cluster.start()
                     s = cluster.getZODBStorage()
                     check() # before import

neo/tests/threaded/testPack.py

+#
+# Copyright (C) 2021  Nexedi SA
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import random, threading, unittest
+from bisect import bisect
+from collections import defaultdict, deque
+from contextlib import contextmanager
+from time import time
+import transaction
+from persistent import Persistent
+from ZODB.POSException import UndoError
+from neo.client.exception import NEOUndoPackError
+from neo.lib import logging
+from neo.lib.protocol import Packets
+from neo.storage.database.manager import BackgroundWorker
+from .. import consume, Patch
+from . import ConnectionFilter, NEOThreadedTest, with_cluster
+
+class PCounter(Persistent):
+    value = 0
+
+class PackTests(NEOThreadedTest):
+
+    @contextmanager
+    def assertPackOperationCount(self, cluster, *counts):
+        packs = defaultdict(dict)
+        def _pack(orig, dm, offset, *args):
+            p = packs[dm.getUUID()]
+            tid = args[1]
+            try:
+                tids = p[offset]
+            except KeyError:
+                p[offset] = [tid]
+            else:
+                self.assertLessEqual(tids[-1], tid)
+                tids.append(tid)
+            return orig(dm, offset, *args)
+        storage_list = cluster.storage_list
+        cls, = {type(s.dm) for s in storage_list}
+        with Patch(cls, _pack=_pack):
+            yield
+            cluster.ticAndJoinStorageTasks()
+        self.assertSequenceEqual(counts,
+            tuple(sum(len(set(x)) for x in packs.pop(s.uuid, {}).itervalues())
+                  for s in storage_list))
+        self.assertFalse(packs)
+
+    def countAskPackOrders(self, connection_filter):
+        counts = defaultdict(int)
+        @connection_filter.add
+        def _(conn, packet):
+            if isinstance(packet, Packets.AskPackOrders):
+                counts[self.getConnectionApp(conn).uuid] += 1
+        return counts
+
+    def populate(self, cluster):
+        t, c = cluster.getTransaction()
+        r = c.root()
+        for x in 'ab', 'ac', 'ab', 'bd', 'c':
+            for x in x:
+                try:
+                    r[x].value += 1
+                except KeyError:
+                    r[x] = PCounter()
+            t.commit()
+            yield cluster.client.last_tid
+        c.close()
+
+    @with_cluster(partitions=3, replicas=1, storage_count=3)
+    def testOutdatedNodeIsBack(self, cluster):
+        s0 = cluster.storage_list[0]
+        populate = self.populate(cluster)
+        tid = consume(populate, 3)
+        with self.assertPackOperationCount(cluster, 0, 4, 4), \
+             ConnectionFilter() as f:
+            counts = self.countAskPackOrders(f)
+            def _worker(orig, self, weak_app):
+                if weak_app() is s0:
+                    logging.info("do not pack partitions %s",
+                        ', '.join(map(str, self._pack_set)))
+                    self._stop = True
+                orig(self, weak_app)
+            with Patch(BackgroundWorker, _worker=_worker):
+                cluster.client.pack(tid)
+                tid = consume(populate, 2)
+                cluster.client.pack(tid)
+                s0.stop()
+                cluster.join((s0,))
+        # First storage node stops any pack-related work after the first
+        # response to AskPackOrders. Other storage nodes process a pack order
+        # for all cells before asking the master for the next pack order.
+        self.assertEqual(counts, {s.uuid: 1 if s is s0 else 2
+                                  for s in cluster.storage_list})
+        s0.resetNode()
+        with ConnectionFilter() as f, \
+             self.assertPackOperationCount(cluster, 4, 0, 0):
+            counts = self.countAskPackOrders(f)
+            deque(populate, 0)
+            s0.start()
+        # The master queries 2 storage nodes for old pack orders and remember
+        # those that s0 has not completed. s0 processes all orders for the first
+        # replicated cell and ask them again when the second is up-to-date.
+        self.assertIn(counts.pop(s0.uuid), (2, 3, 4))
+        self.assertEqual(counts, {cluster.master.uuid: 2})
+        self.checkReplicas(cluster)
+
+    @with_cluster(replicas=1)
+    def testValueSerialVsReplication(self, cluster):
+        t, c = cluster.getTransaction()
+        ob = c.root()[''] = PCounter()
+        t.commit()
+        s0 = cluster.storage_list[0]
+        s0.stop()
+        cluster.join((s0,))
+        ob.value += 1
+        t.commit()
+        ob.value += 1
+        t.commit()
+        s0.resetNode()
+        with ConnectionFilter() as f:
+            f.delayAskFetchTransactions()
+            s0.start()
+            c.db().undo(ob._p_serial, t.get())
+            t.commit()
+            c.db().storage.pack(time(), None)
+            self.tic()
+        cluster.ticAndJoinStorageTasks()
+        self.checkReplicas(cluster)
+
+    @with_cluster()
+    def _testValueSerialMultipleUndo(self, cluster, race, *undos):
+        t, c = cluster.getTransaction()
+        r = c.root()
+        ob = r[''] = PCounter()
+        t.commit()
+        tids = []
+        for x in xrange(2):
+            ob.value += 1
+            t.commit()
+            tids.append(ob._p_serial)
+        db = c.db()
+        def undo(i):
+            db.undo(tids[i], t.get())
+            t.commit()
+            tids.append(db.lastTransaction())
+        undo(-1)
+        for i in undos:
+            undo(i)
+        if race:
+            l1 = threading.Lock(); l1.acquire()
+            l2 = threading.Lock(); l2.acquire()
+            def _task_pack(orig, *args):
+                l1.acquire()
+                orig(*args)
+                l2.release()
+            def answerObjectUndoSerial(orig, *args, **kw):
+                orig(*args, **kw)
+                l1.release()
+                l2.acquire()
+            with Patch(cluster.client.storage_handler,
+                       answerObjectUndoSerial=answerObjectUndoSerial), \
+                 Patch(BackgroundWorker, _task_pack=_task_pack):
+                cluster.client.pack(tids[-1])
+                self.tic()
+                self.assertRaises(NEOUndoPackError, undo, 2)
+        else:
+            cluster.client.pack(tids[-1])
+            cluster.ticAndJoinStorageTasks()
+            undo(2) # empty transaction
+
+    def testValueSerialMultipleUndo1(self):
+        self._testValueSerialMultipleUndo(False, 0, -1)
+
+    def testValueSerialMultipleUndo2(self):
+        self._testValueSerialMultipleUndo(True, -1, 1)
+
+    @with_cluster(partitions=3)
+    def testPartial(self, cluster):
+        N = 256
+        T = 40
+        rnd = random.Random(0)
+        t, c = cluster.getTransaction()
+        r = c.root()
+        for i in xrange(T):
+            for x in xrange(40):
+                x = rnd.randrange(0, N)
+                try:
+                    r[x].value += 1
+                except KeyError:
+                    r[x] = PCounter()
+            t.commit()
+            if i == 30:
+                self.assertEqual(len(r), N-1)
+                tid = c.db().lastTransaction()
+        self.assertEqual(len(r), N)
+        oids = []
+        def tids(oid, pack=False):
+            tids = [x['tid'] for x in c.db().history(oid, T)]
+            self.assertLess(len(tids), T)
+            tids.reverse()
+            if pack:
+                oids.append(oid)
+                return tids[bisect(tids, tid)-1:]
+            return tids
+        expected = [tids(r._p_oid, True)]
+        for x in xrange(N):
+            expected.append(tids(r[x]._p_oid, x % 2))
+        self.assertNotEqual(sorted(oids), oids)
+        client = c.db().storage.app
+        client.wait_for_pack = True
+        with self.assertPackOperationCount(cluster, 3):
+            client.pack(tid, oids)
+        result = [tids(r._p_oid)]
+        for x in xrange(N):
+            result.append(tids(r[x]._p_oid))
+        self.assertEqual(expected, result)
+
+
+if __name__ == "__main__":
+    unittest.main()

neo/tests/threaded/testReplication.py

@@ -681,7 +681,7 @@ class ReplicationTests(NEOThreadedTest):
         self.assertEqual(3, s0.sqlCount('obj'))
         cluster.enableStorageList((s1,))
         cluster.neoctl.tweakPartitionTable()
-        self.tic()
+        cluster.ticAndJoinStorageTasks()
         self.assertEqual(1, s1.sqlCount('obj'))
         self.assertEqual(2, s0.sqlCount('obj'))
 
@@ -732,7 +732,7 @@ class ReplicationTests(NEOThreadedTest):
         self.assertEqual(tids, getTIDList())
         t0_next = add64(tids[0], 1)
         self.assertEqual(ask, [
-            (t0_next, tids[2], tids[2:]),
+            (t0_next, tids[2], tids[2:], True),
             (t0_next, tids[2], ZERO_OID, {tids[2]: [ZERO_OID]}),
         ])
 
@@ -855,9 +855,9 @@ class ReplicationTests(NEOThreadedTest):
             t1_next = add64(tids[1], 1)
             self.assertEqual(ask, [
                 # trans
-                (0, 1, t1_next, tids[4], []),
-                (0, 1, tids[3], tids[4], []),
-                (0, 1, tids[4], tids[4], []),
+                (0, 1, t1_next, tids[4], [], True),
+                (0, 1, tids[3], tids[4], [], False),
+                (0, 1, tids[4], tids[4], [], False),
                 # obj
                 (0, 1, t1_next, tids[4], ZERO_OID, {}),
                 (0, 1, tids[2], tids[4], p64(2), {}),
@@ -871,9 +871,9 @@ class ReplicationTests(NEOThreadedTest):
             n = replicator.FETCH_COUNT
             t4_next = add64(tids[4], 1)
             self.assertEqual(ask, [
-                (0, n, t4_next, tids[5], []),
+                (0, n, t4_next, tids[5], [], True),
                 (0, n, tids[3], tids[5], ZERO_OID, {tids[3]: [ZERO_OID]}),
-                (1, n, t1_next, tids[5], []),
+                (1, n, t1_next, tids[5], [], True),
                 (1, n, t1_next, tids[5], ZERO_OID, {}),
             ])
         self.tic()

neo/tests/zodb/__init__.py

@@ -16,7 +16,8 @@
 
 import os
 
-from .. import DB_PREFIX
+from neo.client.app import Application as ClientApplication, TXN_PACK_DESC
+from .. import DB_PREFIX, Patch
 functional = int(os.getenv('NEO_TEST_ZODB_FUNCTIONAL', 0))
 if functional:
     from ..functional import NEOCluster, NEOFunctionalTest as TestCase
@@ -29,6 +30,16 @@ else:
 
 class ZODBTestCase(TestCase):
 
+    def undoLog(orig, *args, **kw):
+        return [txn for txn in orig(*args, **kw)
+                    if txn['description'] != TXN_PACK_DESC]
+
+    _patches = (
+        Patch(ClientApplication, undoLog=undoLog),
+        Patch(ClientApplication, wait_for_pack=True),
+    )
+    del undoLog
+
     def setUp(self):
         super(ZODBTestCase, self).setUp()
         storages = int(os.getenv('NEO_TEST_ZODB_STORAGES', 1))
@@ -41,6 +52,8 @@ class ZODBTestCase(TestCase):
         if functional:
             kw['temp_dir'] = self.getTempDirectory()
         self.neo = NEOCluster(**kw)
+        for p in self._patches:
+            p.apply()
 
     def __init__(self, methodName):
         super(ZODBTestCase, self).__init__(methodName)
@@ -51,7 +64,20 @@ class ZODBTestCase(TestCase):
                 self.neo.start()
                 self.open()
                 test(self)
-                if not functional:
+                if functional:
+                    dm = self._getDatabaseManager()
+                    try:
+                        @self.neo.expectCondition
+                        def _(last_try):
+                            dm.commit()
+                            dm.setup()
+                            x = dm._deferred_pruning, dm._uncommitted_data
+                            return not any(x), x
+                        orphan = dm.getOrphanList()
+                    finally:
+                        dm.close()
+                else:
+                    self.neo.ticAndJoinStorageTasks()
                     orphan = self.neo.storage.dm.getOrphanList()
                 failed = False
             finally:
@@ -60,24 +86,22 @@ class ZODBTestCase(TestCase):
                     self.neo.stop(ignore_errors=failed)
                 else:
                     self.neo.stop(None)
-            if functional:
-                dm = self.neo.getSQLConnection(*self.neo.db_list)
-                try:
-                    dm.setup()
-                    orphan = set(dm.getOrphanList())
-                    orphan.difference_update(dm._uncommitted_data)
-                finally:
-                    dm.close()
             self.assertFalse(orphan)
         setattr(self, methodName, runTest)
 
     def _tearDown(self, success):
+        for p in self._patches:
+            p.revert()
         del self.neo
         super(ZODBTestCase, self)._tearDown(success)
 
     assertEquals = failUnlessEqual = TestCase.assertEqual
     assertNotEquals = failIfEqual = TestCase.assertNotEqual
 
+    if functional:
+        def _getDatabaseManager(self):
+            return self.neo.getSQLConnection(*self.neo.db_list)
+
     def open(self, **kw):
         self._open(_storage=self.neo.getZODBStorage(**kw))
 

neo/tests/zodb/testZODB.py

@@ -22,7 +22,7 @@ from ZODB.tests import testZODB
 from neo.storage import database as database_module
 from neo.storage.database.importer import ImporterDatabaseManager
 from .. import expectedFailure, getTempDirectory, Patch
-from . import ZODBTestCase
+from . import functional, ZODBTestCase
 
 class NEOZODBTests(ZODBTestCase, testZODB.ZODBTests):
 
@@ -64,9 +64,15 @@ class NEOZODBImporterTests(NEOZODBTests):
 
     def run(self, *args, **kw):
         with Patch(database_module, getAdapterKlass=lambda *args:
-                partial(DummyImporter, self._importer_config, *args)):
+                partial(DummyImporter, self._importer_config, *args)) as p:
+            self._importer_patch = p
             super(ZODBTestCase, self).run(*args, **kw)
 
+    if functional:
+        def _getDatabaseManager(self):
+            self._importer_patch.revert()
+            return super(NEOZODBImporterTests, self)._getDatabaseManager()
+
     checkMultipleUndoInOneTransaction = expectedFailure(IndexError)(
         NEOZODBTests.checkMultipleUndoInOneTransaction)
 

tools/stress

@@ -4,9 +4,10 @@ from __future__ import division, print_function
 import argparse, curses, errno, os, random, select
 import signal, socket, subprocess, sys, threading, time
 from contextlib import contextmanager
+from ctypes import c_ulonglong
 from datetime import datetime
 from functools import partial
-from multiprocessing import Lock, RawArray
+from multiprocessing import Array, Lock, RawArray
 from multiprocessing.queues import SimpleQueue
 from struct import Struct
 from netfilterqueue import NetfilterQueue
@@ -17,7 +18,7 @@ from neo.lib.connector import SocketConnector
 from neo.lib.debug import PdbSocket
 from neo.lib.node import Node
 from neo.lib.protocol import NodeTypes
-from neo.lib.util import datetimeFromTID, p64, u64
+from neo.lib.util import datetimeFromTID, timeFromTID, p64, u64
 from neo.storage.app import DATABASE_MANAGERS, \
     Application as StorageApplication
 from neo.tests import getTempDirectory, mysql_pool
@@ -87,6 +88,7 @@ class Client(Process):
     def __init__(self, command, thread_count, **kw):
         super(Client, self).__init__(command)
         self.config = kw
+        self.ltid = Array(c_ulonglong, thread_count)
         self.count = RawArray('I', thread_count)
         self.thread_count = thread_count
 
@@ -136,6 +138,7 @@ class Client(Process):
             while 1:
                 txn = transaction_begin()
                 try:
+                    self.ltid[i] = u64(db.lastTransaction())
                     data = pack(j, name)
                     for log in random.sample(logs, 2):
                         log.append(data)
@@ -318,12 +321,14 @@ class Application(StressApplication):
 
     def __init__(self, client_count, thread_count,
                  fault_probability, restart_ratio, kill_mysqld,
-                 logrotate, *args, **kw):
+                 pack_period, pack_keep, logrotate, *args, **kw):
         self.client_count = client_count
         self.thread_count = thread_count
         self.logrotate = logrotate
         self.fault_probability = fault_probability
         self.restart_ratio = restart_ratio
+        self.pack_period = pack_period
+        self.pack_keep = pack_keep
         self.cluster = cluster = NEOCluster(*args, **kw)
         logging.setup(os.path.join(cluster.temp_dir, 'stress.log'))
         # Make the firewall also affect connections between storage nodes.
@@ -417,6 +422,10 @@ class Application(StressApplication):
                 **config)
             process_list.append(p)
             p.start()
+        if self.pack_period:
+            t = threading.Thread(target=self._pack_thread)
+            t.daemon = 1
+            t.start()
         if self.logrotate:
             t = threading.Thread(target=self._logrotate_thread)
             t.daemon = 1
@@ -444,6 +453,19 @@ class Application(StressApplication):
         except KeyError:
             pass
 
+    def _pack_thread(self):
+        process_dict = self.cluster.process_dict
+        storage = self.cluster.getZODBStorage()
+        try:
+            while 1:
+                time.sleep(self.pack_period)
+                if self._stress:
+                    storage.pack(timeFromTID(p64(self._getPackableTid()))
+                                 - self.pack_keep, None)
+        except:
+            if storage.app is not None: # closed ?
+                raise
+
     def _logrotate_thread(self):
         try:
             import zstd
@@ -530,13 +552,24 @@ class Application(StressApplication):
 
     _ids_height = 4
 
+    def _getPackableTid(self):
+        return min(min(client.ltid)
+            for client in self.cluster.process_dict[Client])
+
     def refresh_ids(self, y):
         attr = curses.A_NORMAL, curses.A_BOLD
         stdscr = self.stdscr
+        htid = self._getPackableTid()
         ltid = self.ltid
         stdscr.addstr(y, 0,
-            'last oid: 0x%x\nlast tid: 0x%x (%s)\nclients: '
-            % (u64(self.loid), u64(ltid), datetimeFromTID(ltid)))
+            'last oid: 0x%x\n'
+            'last tid: 0x%x (%s)\n'
+            'packable tid: 0x%x (%s)\n'
+            'clients: ' % (
+            u64(self.loid),
+            u64(ltid), datetimeFromTID(ltid),
+            htid, datetimeFromTID(p64(htid)),
+        ))
         before = after = 0
         for i, p in enumerate(self.cluster.process_dict[Client]):
             if i:
@@ -622,6 +655,11 @@ def main():
         help='number of thread workers per client process')
     _('-f', '--fault-probability', type=ratio, default=1, metavar='P',
         help='probability to cause faults every second')
+    _('-p', '--pack-period', type=float, default=10, metavar='N',
+        help='during stress, pack every N seconds, 0 to disable')
+    _('-P', '--pack-keep', type=float, default=0, metavar='N',
+        help='when packing, keep N seconds of history, relative to packable tid'
+             ' (which the oldest tid an ongoing transaction is reading)')
     _('-r', '--restart-ratio', type=ratio, default=.5, metavar='RATIO',
         help='probability to kill/restart a storage node, rather than just'
              ' RSTing a TCP connection with this node')
@@ -680,6 +718,7 @@ def main():
             parser.error('--kill-mysqld: ' + error)
         app = Application(args.clients, args.threads,
             args.fault_probability, args.restart_ratio, args.kill_mysqld,
+            args.pack_period, args.pack_keep,
             int(round(args.logrotate * 3600, 0)), **kw)
         t = threading.Thread(target=console, args=(args.console, app))
         t.daemon = 1