X wcfs/xbtree: ΔBtail tests += ø -> Tree; Tree -> ø

We need to test this because the following case happens often in
practice: the tree did not exist when ΔBtail was started, later the tree
is created and ΔBtail is recomputed. Since we are testing tree transitions
that involves delete, test it uniformly for all cases.

Currently Update test, if Fatal("TODO") is removed fails with:

    --- FAIL: TestΔBTail/Update/T2,5/B1:a-B2:b,4:d-B8:h→ø (0.00s)
panic: ΔBtail update 03e036b7f59f86cc -> 03e036b7f5a28899: treediff 03e036b7f59f86cc..03e036b7f5a28899 000000000000000c: diffT 03e036b7f59f86cc:000000000000000c 03e036b7f5a28899:000000000000000c: BTrees.LOBTree.LOBTree(000000000000000c): activate: file:///tmp/δBTail851157667/1.fs: load 03e036b7f5a28899:000000000000000c: 000000000000000c: object was deleted @03e036b7f5a28899 [recovered]

wcfs/internal/xbtree/testprog/treegen.py

@@ -119,7 +119,7 @@ import sys
 from golang import func, defer, panic
 from golang import time
 from ZODB import DB
-from ZODB.Connection import Connection
+from ZODB.Connection import Connection, TransactionMetaData
 from ZODB.MappingStorage import MappingStorage
 import transaction
 import itertools
@@ -202,6 +202,14 @@ def TreesSrv(zstor, r):
     treetxtPrev = zctx.ztreetxt(ztree)
 
     for treetxt in xreadlines(r):
+        subj = "treegen/tree: %s" % treetxt
+
+        # ø commands to delete the tree
+        if treetxt == "ø":
+            head = commitDelete(ztree, subj)
+            xprint("%s" % ashex(head))
+            continue
+
         # mark tree as changed if the same topology is requested twice.
         # this ensures we can actually make a non-empty commit
         if treetxt == treetxtPrev:
@@ -230,7 +238,7 @@ def TreesSrv(zstor, r):
         xbtree.Restructure(ztree, tree)
 
         # commit tree to storage
-        head = commit("treegen/tree: %s" % treetxt)
+        head = commit(subj)
 
         # verify what was persisted to storage is indeed what we wanted to persist
         zctx.zconn.cacheMinimize()
@@ -444,6 +452,32 @@ def commit(description, skipIfEmpty=False): # -> tid | None
     txn.commit()
     return obj._p_serial
 
+# commitDelete commits deletion of obj with description.
+def commitDelete(obj, description): # -> tid
+    txn = transaction.get()
+    zstor = obj._p_jar._db.storage
+
+    # deleteObject works only at IStorage level, and at that low level
+    # zstor requires ZODB.IStorageTransactionMetaData not txn (ITransaction)
+    txn_stormeta = TransactionMetaData(txn.user, description, txn.extension)
+    zstor.tpc_begin(txn_stormeta)
+    zstor.deleteObject(obj._p_oid, obj._p_serial, txn_stormeta)
+    zstor.tpc_vote(txn_stormeta)
+    _ = []
+    zstor.tpc_finish(txn_stormeta, lambda tid: _.append(tid))
+    assert len(_) == 1, _
+    tid = _[0]
+
+    # object in the database is now current as of committed tid
+    # adjust obj's serial so that the next change to it (possibly recreating
+    # the object), does not fail with ConflictError.
+    obj._p_serial = tid
+
+	# reset transaction to a new one
+    transaction.begin()
+
+    return tid
+
 
 # ztreetxt returns text representation of a ZODB tree.
 @func(ZCtx)

wcfs/internal/xbtree/δbtail_test.go

@@ -608,6 +608,14 @@ func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb
 	// verify transition at1->at2 for all initial states of tracked {keys} from kv1 + kv2 + ∞
 
 	t.Run(fmt.Sprintf("Update/%s→%s", t1.tree, t2.tree), func(t *testing.T) {
+		if t1.tree == DEL {
+			// Track(k) -> Update  ; requires that tree exists at Track time
+			t.Skip("Track(tree1) needs tree1 to exist")
+		}
+		if t2.tree == DEL {
+			t.Fatal("TODO")
+		}
+
 		allKeys := allTestKeys(t1, t2)
 		allKeyv := allKeys.SortedElements()
 
@@ -820,6 +828,10 @@ func (δbtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK, track
 // t0->t1 exercises from-scratch rebuild,
 // t1->t2 further exercises incremental rebuild.
 func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1, t2 *tTreeCommit) {
+	// XXX handle DEL
+	// XXX Update -> Track -> rebuild ...
+	// XXX can start with non-existing tree
+
 //	t1 := t2.prev
 //	t0 := t1.prev
 	t.Run(fmt.Sprintf("rebuild/%s→%s", t0.tree, t1.tree), func(t *testing.T) {
@@ -1220,7 +1232,25 @@ func testΔBTail(t *testing.T, testq chan ΔBTestEntry) {
 			exc.Raisef("treegen -> %s  ; watchq -> %s", tid, δZ)
 		}
 
-		xkv := XGetTree(db, δZ.Tid, tg.treeRoot)
+		// load tree structure from the db
+		// if the tree does not exist yet - report its structure as empty
+		var xkv RBucketSet
+		if tree != DEL {
+			xkv = XGetTree(db, δZ.Tid, tg.treeRoot)
+		} else {
+			xkv = RBucketSet{
+				&RBucket{
+					oid:    zodb.InvalidOid,
+					parent: &RTree{
+						oid:    zodb.InvalidOid,
+						parent: nil,
+					},
+					lo:     KeyMin,
+					hi_:    KeyMax,
+					kv:     map[Key]string{},
+				},
+			}
+		}
 
 		return &tTreeCommit{
 			tree: tree,
@@ -1232,7 +1262,7 @@ func testΔBTail(t *testing.T, testq chan ΔBTestEntry) {
 
 	var t0 *tTreeCommit
 	t1 := &tTreeCommit{
-		tree: "ø",	// initial	XXX ok?
+		tree: "T/B:",	// treegen.py creates the tree as initially empty
 		prev: nil,	// XXX ok?
 		at:   tg.head,
 		xkv:  XGetTree(db, tg.head, tg.treeRoot),
@@ -1410,6 +1440,13 @@ func TestΔBTail(t *testing.T) {
 		"T3/B1:a,2:b-B4:d,8:h",
 		"T2,5/B1:a-B2:b,4:d-B8:h", // XXX add A
 
+		// tree deletion
+		// having ø in the middle of the test cases exercises all:
+		// * `ø -> Tree ...`		(tree is created anew),
+		// * `... Tree -> ø`		(tree is deleted), and
+		// * `Tree -> ø -> Tree`	(tree is deleted and then recreated)
+		DEL,
+
 		// found by AllStructs ([1] is not changed, but because B1 is
 		// unlinked and 1 migrates to other bucket, changes in that
 		// other bucket must be included into δT)
@@ -1671,7 +1708,7 @@ func TestIntSets(t *testing.T) {
 
 
 // kvdiff returns difference in between kv1 and kv2.
-var DEL = "ø" // DEL means key deletion
+var DEL = "ø" // DEL means deletion
 type Δstring struct {
 	Old string
 	New string