storage: fix replication of creation undone

For records that undo object creation, None values are used at the backend
level whereas the protocol is not designed to serialize None for any field.

Therefore, a dance done in many places around packet serialization, using the
specific 0/ZERO_HASH/'' triplet to represent a deleted oid. For replication,
it was missing at the sender side, leading to the following crash:

  Traceback (most recent call last):
    File "neo/storage/app.py", line 147, in run
      self._run()
    File "neo/storage/app.py", line 178, in _run
      self.doOperation()
    File "neo/storage/app.py", line 257, in doOperation
      next(task_queue[-1]) or task_queue.rotate()
    File "neo/storage/handlers/storage.py", line 271, in push
      conn.send(Packets.AddObject(oid, *object), msg_id)
    File "neo/lib/protocol.py", line 234, in __init__
      self._fmt.encode(buf.write, args)
    File "neo/lib/protocol.py", line 345, in encode
      return self._trace(self._encode, writer, items)
    File "neo/lib/protocol.py", line 334, in _trace
      return method(*args)
    File "neo/lib/protocol.py", line 367, in _encode
      item.encode(writer, value)
    File "neo/lib/protocol.py", line 345, in encode
      return self._trace(self._encode, writer, items)
    File "neo/lib/protocol.py", line 342, in _trace
      raise ParseError(self, trace)
  ParseError: at add_object/checksum:
    File "neo/lib/protocol.py", line 553, in _encode
      assert len(checksum) == 20, (len(checksum), checksum)
  TypeError: object of type 'NoneType' has no len()

neo/storage/handlers/storage.py

@@ -267,6 +267,8 @@ class StorageOperationHandler(EventHandler):
                             "partition %u dropped or truncated"
                             % partition), msg_id)
                         return
+                    if not object[2]: # creation undone
+                        object = object[0], 0, ZERO_HASH, '', object[4]
                     # Same as in askFetchTransactions.
                     conn.send(Packets.AddObject(oid, *object), msg_id)
                     yield conn.buffering

neo/tests/threaded/test.py

@@ -137,27 +137,6 @@ class Test(NEOThreadedTest):
                     self.assertRaises(POSException.POSKeyError,
                         storage.load, oid, '')
 
-    @with_cluster()
-    def testCreationUndoneHistory(self, cluster):
-        if 1:
-            storage = cluster.getZODBStorage()
-            oid = storage.new_oid()
-            txn = transaction.Transaction()
-            storage.tpc_begin(txn)
-            storage.store(oid, None, 'foo', '', txn)
-            storage.tpc_vote(txn)
-            tid1 = storage.tpc_finish(txn)
-            storage.tpc_begin(txn)
-            storage.undo(tid1, txn)
-            tid2 = storage.tpc_finish(txn)
-            storage.tpc_begin(txn)
-            storage.undo(tid2, txn)
-            tid3 = storage.tpc_finish(txn)
-            expected = [(tid1, 3), (tid2, 0), (tid3, 3)]
-            for x in storage.history(oid, 10):
-                self.assertEqual((x['tid'], x['size']), expected.pop())
-            self.assertFalse(expected)
-
     def _testUndoConflict(self, cluster, *inc):
         def waitResponses(orig, *args):
             orig(*args)

neo/tests/threaded/testReplication.py

@@ -343,6 +343,35 @@ class ReplicationTests(NEOThreadedTest):
                 self.tic()
                 self.assertTrue(backup.master.is_alive())
 
+    @backup_test()
+    def testCreationUndone(self, backup):
+        """
+        Check both IStorage.history and replication when the DB contains a
+        deletion record.
+
+        XXX: This test reveals that without --dedup, the replication does not
+             preserve the deduplication that is done by the 'undo' code.
+        """
+        storage = backup.upstream.getZODBStorage()
+        oid = storage.new_oid()
+        txn = transaction.Transaction()
+        storage.tpc_begin(txn)
+        storage.store(oid, None, 'foo', '', txn)
+        storage.tpc_vote(txn)
+        tid1 = storage.tpc_finish(txn)
+        storage.tpc_begin(txn)
+        storage.undo(tid1, txn)
+        tid2 = storage.tpc_finish(txn)
+        storage.tpc_begin(txn)
+        storage.undo(tid2, txn)
+        tid3 = storage.tpc_finish(txn)
+        expected = [(tid1, 3), (tid2, 0), (tid3, 3)]
+        for x in storage.history(oid, 10):
+            self.assertEqual((x['tid'], x['size']), expected.pop())
+        self.assertFalse(expected)
+        self.tic()
+        self.assertEqual(1, self.checkBackup(backup))
+
     @backup_test()
     def testBackupTid(self, backup):
         """