X rebuild: tests: Don't access ZODB in XGetδKV

Preload data when tTreeCommit is created and use that data in
the inner loops instead of accessing ZODB from those inner loops.

130s -> 95s

wcfs/internal/xbtree/δbtail_test.go

@@ -403,7 +403,7 @@ func XGetTree(db *zodb.DB, at zodb.Tid, root zodb.Oid) RBucketSet {
 	xwalkDFS(ctx, KeyMin, KeyMax, ztree, func(rb *RBucket) {
 		rbucketv = append(rbucketv, rb)
 	})
-	if len(rbucketv) == 0 { // empty tree -> [-∞, ∞){}
+	if len(rbucketv) == 0 { // empty tree -> [-∞,∞){}
 		etree   := &RTree{
 				oid:    root,
 				parent: nil,
@@ -474,24 +474,15 @@ func _xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, rparent *RTree, bv
 	}
 }
 
-// XGetδKV translates {k -> δ<oid>} to {k -> δ(ZBlk(oid).data)} according to db@at1..at2 snapshots.
-func XGetδKV(db *zodb.DB, at1, at2 zodb.Tid, δkvOid map[Key]ΔValue) map[Key]Δstring {
-	defer exc.Contextf("%s: get δkv %s..%s %v", db.Storage().URL(), at1, at2, δkvOid)
-	X := exc.Raiseif
-
-	txn, ctx := transaction.New(context.Background())
-	defer txn.Abort()
-	zconn1, err := db.Open(ctx, &zodb.ConnOptions{At: at1}); X(err)
-	zconn2, err := db.Open(ctx, &zodb.ConnOptions{At: at2}); X(err)
-
-	δkv := map[Key]Δstring{}
+// XGetδKV translates {k -> δ<oid>} to {k -> δ(ZBlk(oid).data)} according to t1..t2 db snapshots.
+func XGetδKV(t1, t2 *tTreeCommit, δkvOid map[Key]ΔValue) map[Key]Δstring {
+	δkv := make(map[Key]Δstring, len(δkvOid))
 	for k, δvOid := range δkvOid {
 		δkv[k] = Δstring{
-			Old: xzgetBlkData(ctx, zconn1, δvOid.Old),
-			New: xzgetBlkData(ctx, zconn2, δvOid.New),
+			Old: t1.xgetBlkData(δvOid.Old),
+			New: t2.xgetBlkData(δvOid.New),
 		}
 	}
-
 	return δkv
 }
 
@@ -529,7 +520,7 @@ type KAdjMatrix map[Key]SetKey
 
 // Map returns kadj·keys .
 func (kadj KAdjMatrix) Map(keys SetKey) SetKey {
-	res := SetKey{}
+	res := make(SetKey, len(keys))
 	for k := range keys {
 		to, ok := kadj[k]
 		if !ok {
@@ -691,7 +682,7 @@ func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb
 			// this t.Run allocates and keeps too much memory in -verylong
 			// also it is not so useful as above "Update/t1->t2"
 			//t.Run(fmt.Sprintf(" track=%s", keys), func(t *testing.T) {
-				xverifyΔBTail_Update1(t, subj, db, treeRoot, t1.at,t2.at, t1.xkv,t2.xkv, t2.δZ, t2.δxkv, keys, kadj12)
+				xverifyΔBTail_Update1(t, subj, db, treeRoot, t1,t2, keys, kadj12)
 			//})
 		}
 	})
@@ -699,10 +690,13 @@ func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb
 
 // xverifyΔBTail_Update1 verifies how ΔBTail handles ZODB update at1->at2 from initial
 // tracked state defined by initialTrackedKeys.
-func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, at1,at2 zodb.Tid, xkv1,xkv2 RBucketSet, δZ *zodb.EventCommit, d12 map[Key]Δstring, initialTrackedKeys SetKey, kadj KAdjMatrix) {
+func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1,t2 *tTreeCommit, initialTrackedKeys SetKey, kadj KAdjMatrix) {
 	X := exc.Raiseif
 	//t.Logf("\n>>> Track=%s\n", initialTrackedKeys)
 
+	δZ  := t2.δZ
+	d12 := t2.δxkv
+
 	var TrackedδZ SetKey = nil
 	var kadjTrackedδZ SetKey = nil
 	var δT, δTok map[Key]Δstring = nil, nil
@@ -733,18 +727,18 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 
 
 	// δbtail @at1 with initial tracked set
-	δbtail := NewΔBtail(at1, db)
+	δbtail := NewΔBtail(t1.at, db)
 	xtrackKeys(δbtail, treeRoot, initialTrackedKeys)
 
 	// TrackedδZ = Tracked ^ δZ  (i.e. a tracked node has changed, or its coverage was changed)
 	TrackedδZ = SetKey{}
 	for k := range initialTrackedKeys {
-		leaf1 := xkv1.Get(k)
+		leaf1 := t1.xkv.Get(k)
 		oid1 := leaf1.oid
 		if oid1 == zodb.InvalidOid { // embedded bucket
 			oid1 = leaf1.parent.oid
 		}
-		leaf2 := xkv2.Get(k)
+		leaf2 := t2.xkv.Get(k)
 		oid2 := leaf2.oid
 		if oid2 == zodb.InvalidOid { // embedded bucket
 			oid2 = leaf2.parent.oid
@@ -782,8 +776,8 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 
 	// trackSet1 = xkv1[tracked1]
 	// trackSet2 = xkv2[tracked2]  ( = xkv2[kadj[tracked1]]
-	trackSet1, tkeyCov1 := xkv1.trackSetWithCov(initialTrackedKeys)
-	trackSet2, tkeyCov2 := xkv2.trackSetWithCov(initialTrackedKeys.Union(kadjTrackedδZ))
+	trackSet1, tkeyCov1 := t1.xkv.trackSetWithCov(initialTrackedKeys)
+	trackSet2, tkeyCov2 := t2.xkv.trackSetWithCov(initialTrackedKeys.Union(kadjTrackedδZ))
 
 	// verify δbtail.trackSet against @at1
 	δbtail.assertTrack(t, "1", ø, trackSet1)
@@ -844,8 +838,8 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 
 
 	// δT <- δB
-	δToid := δB.ΔByRoot[treeRoot]        // {} k -> δoid
-	δT     = XGetδKV(db, at1,at2, δToid) // {} k -> δ(ZBlk(oid).data)
+	δToid := δB.ΔByRoot[treeRoot]   // {} k -> δoid
+	δT     = XGetδKV(t1,t2, δToid) // {} k -> δ(ZBlk(oid).data)
 
 	// δT must be subset of d12.
 	// changed keys, that are
@@ -988,7 +982,7 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
 				xverifyΔBTail_rebuild_U(t, δbtail, treeRoot, t0, t1, xat,
 					/*trackSet=*/ø,
 					/*vδT=ø*/)
-				xverifyΔBTail_rebuild_TR(t, db, δbtail, t1, treeRoot, xat,
+				xverifyΔBTail_rebuild_TR(t, δbtail, t1, treeRoot, xat,
 					// after Track(keys1)
 					keys1,
 					/*trackSet=*/ ø,
@@ -1083,7 +1077,7 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
 						// t.Run is expensive at this level of nest
 						//t.Run(" T"+keys2.String()+";R", func(t *testing.T) {
 							δbtail_ := δbtail.Clone()
-							xverifyΔBTail_rebuild_TR(t, db, δbtail_, t2, treeRoot, xat,
+							xverifyΔBTail_rebuild_TR(t, δbtail_, t2, treeRoot, xat,
 								// after Track(keys2)
 								keys2,
 								/*trackSet*/ t2.xkv.trackSet(keys1R2),
@@ -1140,7 +1134,7 @@ func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, treeRoot zodb.Oid,
 	}
 	δToid, ok := δB.ΔByRoot[treeRoot]
 	if ok {
-		δT = XGetδKV(δbtail.db, ti.at, tj.at, δToid)
+		δT = XGetδKV(ti, tj, δToid)
 	}
 	if δB.Rev != tj.at {
 		t.Errorf("%s: δB.Rev: have %s  ; want %s", subj, δB.Rev, tj.at)
@@ -1154,7 +1148,7 @@ func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, treeRoot zodb.Oid,
 }
 
 // xverifyΔBTail_rebuild_TR verifies ΔBtail state after Track(keys) + rebuild.
-func xverifyΔBTail_rebuild_TR(t *testing.T, db *zodb.DB, δbtail *ΔBtail, tj *tTreeCommit, treeRoot zodb.Oid, xat map[zodb.Tid]string, keys SetKey, trackSet PPTreeSubSet, trackNew, trackSetAfterRebuild PPTreeSubSet, vδTok ...map[Key]Δstring) {
+func xverifyΔBTail_rebuild_TR(t *testing.T, δbtail *ΔBtail, tj *tTreeCommit, treeRoot zodb.Oid, xat map[zodb.Tid]string, keys SetKey, trackSet PPTreeSubSet, trackNew, trackSetAfterRebuild PPTreeSubSet, vδTok ...map[Key]Δstring) {
 	t.Helper()
 	ø := PPTreeSubSet{}
 
@@ -1183,6 +1177,7 @@ func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit,
 	l := len(vδTok)
 	var vatOK  []zodb.Tid
 	var vδTok_ []map[Key]Δstring
+	at2t := map[zodb.Tid]*tTreeCommit{tj.at: tj}
 	t0 := tj
 	for i := 0; i<l; i++ {
 		// empty vδTok entries means they should be absent in vδT
@@ -1191,6 +1186,7 @@ func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit,
 			vδTok_ = append([]map[Key]Δstring{δTok}, vδTok_...)
 		}
 		t0 = t0.prev
+		at2t[t0.at] = t0
 	}
 	vδTok = vδTok_
 	δTtail, ok := δbtail.vδTbyRoot[treeRoot]
@@ -1205,7 +1201,7 @@ func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit,
 	atPrev := t0.at
 	for _, δToid := range vδToid {
 		vat = append(vat, δToid.Rev)
-		δT := XGetδKV(δbtail.db, atPrev, δToid.Rev, δToid.ΔKV) // {} k -> δ(ZBlk(oid).data)
+		δT := XGetδKV(at2t[atPrev], at2t[δToid.Rev], δToid.ΔKV) // {} k -> δ(ZBlk(oid).data)
 		vδT = append(vδT, δT)
 		atPrev = δToid.Rev
 	}
@@ -1376,12 +1372,13 @@ func ΔBTest(xtest interface{}) ΔBTestEntry {
 
 // tTreeCommit represent test commit changing a tree.
 type tTreeCommit struct {
-	tree string            // the tree in toplogy-encoding
-	prev *tTreeCommit      // previous commit
-	at   zodb.Tid          // commit revision
-	δZ   *zodb.EventCommit // raw ZODB changes; δZ.tid == at
-	xkv  RBucketSet        // full tree state as of @at
-	δxkv map[Key]Δstring   // full tree-diff against parent
+	tree       string              // the tree in toplogy-encoding
+	prev       *tTreeCommit        // previous commit
+	at         zodb.Tid            // commit revision
+	δZ         *zodb.EventCommit   // raw ZODB changes; δZ.tid == at
+	xkv        RBucketSet          // full tree state as of @at
+	δxkv       map[Key]Δstring     // full tree-diff against parent
+	blkDataTab map[zodb.Oid]string // full snapshot of all ZBlk data @at
 }
 
 // tZODBCacheEverything is workaround for ZODB/go not implementing real
@@ -1393,6 +1390,68 @@ func (_ *tZODBCacheEverything) PCacheClassify(_ zodb.IPersistent) zodb.PCachePol
 	return zodb.PCachePinObject | zodb.PCacheKeepState
 }
 
+// XGetBlkDataTab loads all ZBlk from db@at.
+//
+// it returns {} oid -> blkdata.
+func XGetBlkDataTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]string {
+	defer exc.Contextf("%s: @%s: get blkdatatab", db.Storage().URL(), at)
+	X := exc.Raiseif
+
+	blkDataTab := map[zodb.Oid]string{}
+
+	txn, ctx := transaction.New(context.Background())
+	defer txn.Abort()
+	zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)
+
+	xzroot, err := zconn.Get(ctx, 0); X(err)
+	zroot, ok := xzroot.(*zodb.Map)
+	if !ok {
+		exc.Raisef("root: expected %s, got %s", xzodb.TypeOf(zroot), xzodb.TypeOf(xzroot))
+	}
+
+	err = zroot.PActivate(ctx); X(err)
+	defer zroot.PDeactivate()
+
+	xzblkdir, ok := zroot.Data["treegen/values"]
+	if !ok {
+		exc.Raisef("root['treegen/values'] missing")
+	}
+	zblkdir, ok := xzblkdir.(*zodb.Map)
+	if !ok {
+		exc.Raisef("root['treegen/values']: expected %s, got %s", xzodb.TypeOf(zblkdir), xzodb.TypeOf(xzblkdir))
+	}
+
+	err = zblkdir.PActivate(ctx); X(err)
+	defer zblkdir.PDeactivate()
+
+	for k, xzblk := range zblkdir.Data {
+		zblk, ok := xzblk.(zodb.IPersistent)
+		if !ok {
+			exc.Raisef("root['treegen/values'][%q]: expected %s, got %s", k, xzodb.TypeOf(zblk), xzodb.TypeOf(xzblk))
+		}
+
+		oid := zblk.POid()
+		data := xzgetBlkData(ctx, zconn, oid)
+		blkDataTab[oid] = data
+	}
+
+	return blkDataTab
+}
+
+// xgetBlkData loads blk data for ZBlk<oid> @t.at
+//
+// For speed the load is done via preloaded t.blkDataTab instead of access to the DB.
+func (t *tTreeCommit) xgetBlkData(oid zodb.Oid) string {
+	if oid == VDEL {
+		return DEL
+	}
+	data, ok := t.blkDataTab[oid]
+	if !ok {
+		exc.Raisef("getBlkData ZBlk<%s> @%s: no such ZBlk", oid, t.at)
+	}
+	return data
+}
+
 // testΔBTail verifies ΔBTail on sequence of tree topologies coming from testq.
 func testΔBTail(t *testing.T, testq chan ΔBTestEntry) {
 	X := exc.Raiseif
@@ -1474,21 +1533,23 @@ func testΔBTail(t *testing.T, testq chan ΔBTestEntry) {
 		}
 
 		return &tTreeCommit{
-			tree: tree,
-			at:   δZ.Tid,
-			δZ:   δZ,
-			xkv:  xkv,
+			tree:       tree,
+			at:         δZ.Tid,
+			δZ:         δZ,
+			xkv:        xkv,
+			blkDataTab: XGetBlkDataTab(db, δZ.Tid),
 		}
 	}
 
 	var t0 *tTreeCommit
 	t1 := &tTreeCommit{
-		tree: "T/B:",	// treegen.py creates the tree as initially empty
-		prev: nil,
-		at:   tg.head,
-		xkv:  XGetTree(db, tg.head, tg.treeRoot),
-		δZ:   nil,
-		δxkv: nil,
+		tree:       "T/B:",	// treegen.py creates the tree as initially empty
+		prev:       nil,
+		at:         tg.head,
+		xkv:        XGetTree(db, tg.head, tg.treeRoot),
+		blkDataTab: XGetBlkDataTab(db, tg.head),
+		δZ:         nil,
+		δxkv:       nil,
 	}
 	for test := range testq {
 		t2 := XCommitTree(test.tree)