X xbtree: Factor testing infrastructure bits into xbtree/xbtreetest

This way ΔFtail tests will be able to reuse it.

* t3:
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  Revert "."
  .
  .
  .
  .

wcfs/internal/xbtree/blib/blib.go

+// Copyright (C) 2021  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package blib provides utilities related to BTrees.
+package blib
+
+import (
+	"fmt"
+	"math"
+
+	"lab.nexedi.com/kirr/neo/go/zodb/btree"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
+)
+
+// XXX instead of generics
+type Tree   = btree.LOBTree
+type Bucket = btree.LOBucket
+type Node   = btree.LONode
+type TreeEntry   = btree.LOEntry
+type BucketEntry = btree.LOBucketEntry
+
+type Key    = int64
+const KeyMax Key = math.MaxInt64
+const KeyMin Key = math.MinInt64
+
+type setOid = set.Oid
+
+
+// kstr formats key as string.
+func kstr(k Key) string {
+	if k == KeyMin {
+		return "-∞"
+	}
+	if k == KeyMax {
+		return "∞"
+	}
+	return fmt.Sprintf("%d", k)
+}
+
+
+func panicf(format string, argv ...interface{}) {
+	panic(fmt.Sprintf(format, argv...))
+}

wcfs/internal/xbtree/pptreesubset.go → wcfs/internal/xbtree/blib/pptreesubset.go

@@ -17,7 +17,7 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package xbtree
+package blib
 // PP-connected subset of tree nodes.
 
 import (
@@ -61,6 +61,16 @@ type nodeInTree struct {
 	nchild  int      // number of direct children in PPTreeSubSet referring to this node
 }
 
+// Parent returns parent of this node.
+func (n *nodeInTree) Parent() zodb.Oid {
+	return n.parent
+}
+
+// NChild returns number of children of this node in the tree subset.
+func (n *nodeInTree) NChild() int {
+	return n.nchild
+}
+
 // Has returns whether node is in the set.
 func (S PPTreeSubSet) Has(oid zodb.Oid) bool {
 	_, ok := S[oid]
@@ -102,7 +112,7 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
 
 	// normalize path: remove embedded bucket and check whether it was an
 	// artificial empty tree.
-	path = normPath(path)
+	path = NormPath(path)
 
 	// go through path and add nodes to the set
 	parent := zodb.InvalidOid
@@ -132,11 +142,11 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
 	}
 }
 
-// normPath normalizes path.
+// NormPath normalizes path.
 //
 // It removes embedded buckets and artificial empty trees.
 // Returned slice is subslice of path and aliases its memory.
-func normPath(path []zodb.Oid) []zodb.Oid {
+func NormPath(path []zodb.Oid) []zodb.Oid {
 	l := len(path)
 
 	// don't keep track of artificial empty tree
@@ -464,20 +474,20 @@ func (t nodeInTree) String() string {
 //     The second component with "-22" builds from leaf, but the first
 //     component with "-43" builds from non-leaf node.
 //
-//         δnchildNonLeafs = {43: +1}
+//         ΔnchildNonLeafs = {43: +1}
 //
 //     Only complete result of applying all
 //
-//         - xfixup(-1, δnchildNonLeafs)
+//         - xfixup(-1, ΔnchildNonLeafs)
 //         - δ.Del,
 //         - δ.Add, and
-//         - xfixup(+1, δnchildNonLeafs)
+//         - xfixup(+1, ΔnchildNonLeafs)
 //
 //     produces correctly PP-connected set.
 type ΔPPTreeSubSet struct {
 	Del PPTreeSubSet
 	Add PPTreeSubSet
-	δnchildNonLeafs map[zodb.Oid]int
+	ΔnchildNonLeafs map[zodb.Oid]int
 }
 
 // NewΔPPTreeSubSet creates new empty ΔPPTreeSubSet.
@@ -485,7 +495,7 @@ func NewΔPPTreeSubSet() *ΔPPTreeSubSet {
 	return &ΔPPTreeSubSet{
 		Del: PPTreeSubSet{},
 		Add: PPTreeSubSet{},
-		δnchildNonLeafs: map[zodb.Oid]int{},
+		ΔnchildNonLeafs: map[zodb.Oid]int{},
 	}
 }
 
@@ -493,15 +503,15 @@ func NewΔPPTreeSubSet() *ΔPPTreeSubSet {
 func (δ *ΔPPTreeSubSet) Update(δ2 *ΔPPTreeSubSet) {
 	δ.Del.UnionInplace(δ2.Del)
 	δ.Add.UnionInplace(δ2.Add)
-	for oid, δnc := range δ2.δnchildNonLeafs {
-		δ.δnchildNonLeafs[oid] += δnc
+	for oid, δnc := range δ2.ΔnchildNonLeafs {
+		δ.ΔnchildNonLeafs[oid] += δnc
 	}
 }
 
 // Reverse changes δ=diff(A->B) to δ'=diff(A<-B).
 func (δ *ΔPPTreeSubSet) Reverse() {
 	δ.Del, δ.Add = δ.Add, δ.Del
-	// δnchildNonLeafs stays the same
+	// ΔnchildNonLeafs stays the same
 }
 
 
@@ -514,7 +524,7 @@ func (S PPTreeSubSet) ApplyΔ(δ *ΔPPTreeSubSet) {
 		fmt.Printf("  A: %s\n", S)
 		fmt.Printf("  -: %s\n", δ.Del)
 		fmt.Printf("  +: %s\n", δ.Add)
-		fmt.Printf("  x: %v\n", δ.δnchildNonLeafs)
+		fmt.Printf("  x: %v\n", δ.ΔnchildNonLeafs)
 		defer fmt.Printf("\n->B: %s\n", S)
 	}
 
@@ -523,8 +533,8 @@ func (S PPTreeSubSet) ApplyΔ(δ *ΔPPTreeSubSet) {
 	δ.Add.verify()
 	defer S.verify()
 
-	S.xfixup(-1, δ.δnchildNonLeafs)
+	S.xfixup(-1, δ.ΔnchildNonLeafs)
 	S.xDifferenceInplace(δ.Del)
 	S.xUnionInplace(δ.Add)
-	S.xfixup(+1, δ.δnchildNonLeafs)
+	S.xfixup(+1, δ.ΔnchildNonLeafs)
 }

wcfs/internal/xbtree/pptreesubset_test.go → wcfs/internal/xbtree/blib/pptreesubset_test.go

@@ -17,7 +17,7 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package xbtree
+package blib
 
 import (
 	"strings"

wcfs/internal/xbtree/rangeset.go → wcfs/internal/xbtree/blib/rangeset.go

@@ -17,7 +17,7 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package xbtree
+package blib
 // set of [lo,hi) Key ranges.
 
 import (
@@ -30,8 +30,8 @@ const debugRangeSet = false
 
 // KeyRange represents [lo,hi) Key range.
 type KeyRange struct {
-	lo  Key
-	hi_ Key // NOTE _not_ hi) to avoid overflow at ∞;  hi = hi_ + 1
+	Lo  Key
+	Hi_ Key // NOTE _not_ hi) to avoid overflow at ∞;  hi = hi_ + 1
 }
 
 // RangedKeySet is set of Keys with adjacent keys coalesced into Ranges.
@@ -47,23 +47,23 @@ type RangedKeySet struct {
 
 // Has returns whether key k belongs to the range.
 func (r *KeyRange) Has(k Key) bool {
-	return (r.lo <= k && k <= r.hi_)
+	return (r.Lo <= k && k <= r.Hi_)
 }
 
 
 // Add adds key k to the set.
 func (S *RangedKeySet) Add(k Key) {
-	S.AddRange(KeyRange{lo: k, hi_: k})
+	S.AddRange(KeyRange{Lo: k, Hi_: k})
 }
 
 // Del removes key k from the set.
 func (S *RangedKeySet) Del(k Key) {
-	S.DelRange(KeyRange{lo: k, hi_: k})
+	S.DelRange(KeyRange{Lo: k, Hi_: k})
 }
 
 // Has returns whether key k belongs to the set.
 func (S *RangedKeySet) Has(k Key) bool {
-	return S.HasRange(KeyRange{lo: k, hi_: k})
+	return S.HasRange(KeyRange{Lo: k, Hi_: k})
 }
 
 
@@ -79,10 +79,10 @@ func (S *RangedKeySet) AddRange(r KeyRange) {
 	S.verify()
 	defer S.verify()
 
-	// find first ilo: r.lo < [ilo].hi
+	// find first ilo: r.Lo < [ilo].hi
 	l := len(S.rangev)
 	ilo := sort.Search(l, func(i int) bool {
-		 return r.lo <= S.rangev[i].hi_
+		 return r.Lo <= S.rangev[i].Hi_
 	})
 	debugfRSet("\tilo: %d\n", ilo)
 
@@ -92,58 +92,58 @@ func (S *RangedKeySet) AddRange(r KeyRange) {
 		debugfRSet("\tappend %s\t-> %s\n", r, S)
 	}
 
-	// find last jhi: [jhi].lo < r.hi
+	// find last jhi: [jhi].Lo < r.hi
 	jhi := ilo
 	for ;; jhi++ {
 		if jhi == l {
 			break
 		}
-		if S.rangev[jhi].lo <= r.hi_ {
+		if S.rangev[jhi].Lo <= r.Hi_ {
 			continue
 		}
 		break
 	}
 	debugfRSet("\tjhi: %d\n", jhi)
 
-	// entries in [ilo:jhi) ∈ [r.lo,r.hi) and should be merged into one
+	// entries in [ilo:jhi) ∈ [r.Lo,r.hi) and should be merged into one
 	if (jhi - ilo) > 1 {
-		lo  := S.rangev[ilo].lo
-		hi_ := S.rangev[jhi-1].hi_
+		lo  := S.rangev[ilo].Lo
+		hi_ := S.rangev[jhi-1].Hi_
 		vReplaceSlice(&S.rangev, ilo,jhi, KeyRange{lo,hi_})
 		debugfRSet("\tmerge S[%d:%d]\t-> %s\n", ilo, jhi, S)
 	}
 	jhi = -1 // no longer valid
 
 	// if [r.lo,r.hi) was outside of any entry - create new entry
-	if r.hi_ < S.rangev[ilo].lo {
+	if r.Hi_ < S.rangev[ilo].Lo {
 		vInsert(&S.rangev, ilo, r)
 		debugfRSet("\tinsert %s\t-> %s\n", r, S)
 	}
 
 	// now we have covered entries merged as needed into [ilo]
 	// extend this entry if r coverage is wider
-	if r.lo < S.rangev[ilo].lo {
-		S.rangev[ilo].lo = r.lo
+	if r.Lo < S.rangev[ilo].Lo {
+		S.rangev[ilo].Lo = r.Lo
 		debugfRSet("\textend left\t-> %s\n", S)
 	}
-	if r.hi_ > S.rangev[ilo].hi_ {
-		S.rangev[ilo].hi_ = r.hi_
+	if r.Hi_ > S.rangev[ilo].Hi_ {
+		S.rangev[ilo].Hi_ = r.Hi_
 		debugfRSet("\textend right\t-> %s\n", S)
 	}
 
 	// and check if we should merge it with right/left neighbours
 	if ilo+1 < len(S.rangev) { // right
-		if S.rangev[ilo].hi_+1 == S.rangev[ilo+1].lo {
+		if S.rangev[ilo].Hi_+1 == S.rangev[ilo+1].Lo {
 			vReplaceSlice(&S.rangev, ilo,ilo+2,
-				KeyRange{S.rangev[ilo].lo, S.rangev[ilo+1].hi_})
+				KeyRange{S.rangev[ilo].Lo, S.rangev[ilo+1].Hi_})
 			debugfRSet("\tmerge right\t-> %s\n", S)
 		}
 	}
 
 	if ilo > 0 { // left
-		if S.rangev[ilo-1].hi_+1 == S.rangev[ilo].lo {
+		if S.rangev[ilo-1].Hi_+1 == S.rangev[ilo].Lo {
 			vReplaceSlice(&S.rangev, ilo-1,ilo+1,
-				KeyRange{S.rangev[ilo-1].lo, S.rangev[ilo].hi_})
+				KeyRange{S.rangev[ilo-1].Lo, S.rangev[ilo].Hi_})
 			debugfRSet("\tmerge left\t-> %s\n", S)
 		}
 	}
@@ -163,10 +163,10 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
 	S.verify()
 	defer S.verify()
 
-	// find first ilo: r.lo < [ilo].hi
+	// find first ilo: r.Lo < [ilo].hi
 	l := len(S.rangev)
 	ilo := sort.Search(l, func(i int) bool {
-		 return r.lo <= S.rangev[i].hi_
+		 return r.Lo <= S.rangev[i].Hi_
 	})
 	debugfRSet("\tilo: %d\n", ilo)
 
@@ -175,13 +175,13 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
 		return
 	}
 
-	// find last jhi: [jhi].lo < r.hi
+	// find last jhi: [jhi].Lo < r.hi
 	jhi := ilo
 	for ;; jhi++ {
 		if jhi == l {
 			break
 		}
-		if S.rangev[jhi].lo <= r.hi_ {
+		if S.rangev[jhi].Lo <= r.Hi_ {
 			continue
 		}
 		break
@@ -196,19 +196,19 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
 	// [ilo+1:jhi-1] should be deleted
 	// [ilo] and [jhi-1] overlap with [r.lo,r.hi) - they should be deleted, or shrinked,
 	// or split+shrinked if ilo==jhi-1 and r is inside [ilo]
-	if jhi-ilo == 1 && S.rangev[ilo].lo < r.lo && r.hi_ < S.rangev[ilo].hi_ {
+	if jhi-ilo == 1 && S.rangev[ilo].Lo < r.Lo && r.Hi_ < S.rangev[ilo].Hi_ {
 		x := S.rangev[ilo]
 		vInsert(&S.rangev, ilo, x)
 		jhi++
 		debugfRSet("\tpresplit copy %s\t-> %s\n", x, S)
 	}
-	if S.rangev[ilo].lo < r.lo { // shrink left
-		S.rangev[ilo] = KeyRange{S.rangev[ilo].lo, r.lo-1}
+	if S.rangev[ilo].Lo < r.Lo { // shrink left
+		S.rangev[ilo] = KeyRange{S.rangev[ilo].Lo, r.Lo-1}
 		ilo++
 		debugfRSet("\tshrink [%d] left\t-> %s\n", ilo, S)
 	}
-	if r.hi_ < S.rangev[jhi-1].hi_ { // shrink right
-		S.rangev[jhi-1] = KeyRange{r.hi_+1, S.rangev[jhi-1].hi_}
+	if r.Hi_ < S.rangev[jhi-1].Hi_ { // shrink right
+		S.rangev[jhi-1] = KeyRange{r.Hi_+1, S.rangev[jhi-1].Hi_}
 		jhi--
 		debugfRSet("\tshrink [%d] right\t-> %s\n", jhi-1, S)
 	}
@@ -237,7 +237,7 @@ func (S *RangedKeySet) HasRange(r KeyRange) (yes bool) {
 	// find first ilo: r.lo < [ilo].hi
 	l := len(S.rangev)
 	ilo := sort.Search(l, func(i int) bool {
-		 return r.lo <= S.rangev[i].hi_
+		 return r.Lo <= S.rangev[i].Hi_
 	})
 	debugfRSet("\tilo: %d\n", ilo)
 
@@ -246,7 +246,7 @@ func (S *RangedKeySet) HasRange(r KeyRange) (yes bool) {
 	}
 
 	// all keys from r are in S if r ∈ [ilo]
-	return (S.rangev[ilo].lo <= r.lo && r.hi_ <= S.rangev[ilo].hi_)
+	return (S.rangev[ilo].Lo <= r.Lo && r.Hi_ <= S.rangev[ilo].Hi_)
 }
 
 
@@ -317,13 +317,13 @@ func (S *RangedKeySet) verify() {
 	hi_Prev := KeyMin
 	for i, r := range S.rangev {
 		hiPrev := hi_Prev + 1
-		if i > 0 && !(hiPrev < r.lo) { // NOTE not ≤ - adjacent ranges must be merged
+		if i > 0 && !(hiPrev < r.Lo) { // NOTE not ≤ - adjacent ranges must be merged
 			badf("[%d]: !(hiPrev < r.lo)", i)
 		}
-		if !(r.lo <= r.hi_) {
+		if !(r.Lo <= r.Hi_) {
 			badf("[%d]: !(r.lo <= r.hi_)", i)
 		}
-		hi_Prev = r.hi_
+		hi_Prev = r.Hi_
 	}
 }
 
@@ -378,12 +378,12 @@ func (S RangedKeySet) String() string {
 
 func (r KeyRange) String() string {
 	var shi string
-	if r.hi_ == KeyMax {
-		shi = kstr(r.hi_) // ∞
+	if r.Hi_ == KeyMax {
+		shi = kstr(r.Hi_) // ∞
 	} else {
-		shi = fmt.Sprintf("%d", r.hi_+1)
+		shi = fmt.Sprintf("%d", r.Hi_+1)
 	}
-	return fmt.Sprintf("[%s,%s)", kstr(r.lo), shi)
+	return fmt.Sprintf("[%s,%s)", kstr(r.Lo), shi)
 }
 
 

wcfs/internal/xbtree/rangeset_test.go → wcfs/internal/xbtree/blib/rangeset_test.go

@@ -17,7 +17,7 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package xbtree
+package blib
 
 import (
 	"testing"

wcfs/internal/xbtree/xbtree.go

@@ -5,25 +5,24 @@ package xbtree
 
 import (
 	"fmt"
-	"math"
 
 	"lab.nexedi.com/kirr/neo/go/zodb"
-	"lab.nexedi.com/kirr/neo/go/zodb/btree"
 
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
 )
 
 
 // XXX instead of generics
-type Tree   = btree.LOBTree
-type Bucket = btree.LOBucket
-type Node   = btree.LONode
-type TreeEntry   = btree.LOEntry
-type BucketEntry = btree.LOBucketEntry
+type Tree   = blib.Tree
+type Bucket = blib.Bucket
+type Node   = blib.Node
+type TreeEntry   = blib.TreeEntry
+type BucketEntry = blib.BucketEntry
 
-type Key    = int64
-const KeyMax Key = math.MaxInt64
-const KeyMin Key = math.MinInt64
+type Key    = blib.Key
+const KeyMax = blib.KeyMax
+const KeyMin = blib.KeyMin
 
 // value is assumed to be persistent reference.
 // deletion is represented as VDEL.
@@ -74,3 +73,7 @@ func kstr(k Key) string {
 	}
 	return fmt.Sprintf("%d", k)
 }
+
+func panicf(format string, argv ...interface{}) {
+	panic(fmt.Sprintf(format, argv...))
+}

wcfs/internal/xbtree/xbtreetest/init/init.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package xbtreetest/init (ex imported from package A) should be imported in
+// addition to xbtreetest (from package A_test) to initialize xbtreetest at runtime.
+package init
+
+// ZBlk-related part of δbtail_test
+
+import (
+	"context"
+	"fmt"
+
+	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
+)
+
+type Tree = xbtreetest.Tree
+type Node = xbtreetest.Node
+type Key  = xbtreetest.Key
+
+type ZBlk = zdata.ZBlk
+
+// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
+// XXX +return blkRevMax and use it ?
+func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
+	path = []Node{}
+	xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
+		path = append(path, node)
+	})
+	if err != nil {
+		return nil, false, nil, err
+	}
+
+	if ok {
+		zblk, ok = xzblk.(ZBlk)
+		if !ok {
+			return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk))  // XXX errctx
+		}
+	}
+
+	return zblk, ok, path, nil
+}
+
+
+func init() {
+	xbtreetest.ZTreeGetBlkData = _ZTreeGetBlkData
+	xbtreetest.ZGetBlkData     = _ZGetBlkData
+}
+
+// _ZTreeGetBlkData returns block data from block pointed to by ztree[k].
+func _ZTreeGetBlkData(ctx context.Context, ztree *Tree, k Key) (data string, ok bool, path []Node, err error) {
+	defer xerr.Contextf(&err, "@%s: tree<%s>: get blkdata from [%d]", ztree.PJar().At(), ztree.POid(), k)
+
+	zblk, ok, path, err := ztreeGetBlk(ctx, ztree, k)
+	if err != nil || !ok {
+		return "", ok, path, err
+	}
+
+	bdata, _, err := zblk.LoadBlkData(ctx)
+	if err != nil {
+		return "", false, nil, err
+	}
+
+	return string(bdata), true, path, nil
+}
+
+// _ZGetBlkData loads block data from ZBlk object specified by its oid.
+func _ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
+	defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
+
+	xblk, err := zconn.Get(ctx, zblkOid)
+	if err != nil {
+		return "", err
+	}
+	zblk, ok := xblk.(ZBlk)
+	if !ok {
+		return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
+	}
+	bdata, _, err := zblk.LoadBlkData(ctx)
+	if err != nil {
+		return "", err
+	}
+	return string(bdata), nil
+}

wcfs/internal/xbtree/xbtreetest/kvdiff.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+// kvdiff + friends
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// kvdiff returns difference in between kv1 and kv2.
+const DEL = "ø" // DEL means deletion
+type Δstring struct {
+	Old string
+	New string
+}
+func kvdiff(kv1, kv2 map[Key]string) map[Key]Δstring {
+	delta := map[Key]Δstring{}
+	keys := setKey{}
+	for k := range kv1 { keys.Add(k) }
+	for k := range kv2 { keys.Add(k) }
+
+	for k := range keys {
+		v1, ok := kv1[k]
+		if !ok { v1 = DEL }
+		v2, ok := kv2[k]
+		if !ok { v2 = DEL }
+		if v1 != v2 {
+			delta[k] = Δstring{v1,v2}
+		}
+	}
+
+	return delta
+}
+
+// kvtxt returns string representation of {} kv.
+func kvtxt(kv map[Key]string) string {
+	if len(kv) == 0 {
+		return "ø"
+	}
+
+	keyv := []Key{}
+	for k := range kv { keyv = append(keyv, k) }
+	sort.Slice(keyv, func(i,j int) bool { return keyv[i] < keyv[j] })
+
+	sv := []string{}
+	for _, k := range keyv {
+		v := kv[k]
+		if strings.ContainsAny(v, " \n\t,:") {
+			panicf("[%v]=%q: invalid value", k, v)
+		}
+		sv = append(sv, fmt.Sprintf("%v:%s", k, v))
+	}
+
+	return strings.Join(sv, ",")
+}

wcfs/internal/xbtree/misc.go → wcfs/internal/xbtree/xbtreetest/kvdiff_test.go

-// Copyright (C) 2018-2021  Nexedi SA and Contributors.
+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -17,12 +17,28 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package xbtree
+package xbtreetest
 
 import (
-	"fmt"
+	"reflect"
+	"testing"
 )
 
-func panicf(format string, argv ...interface{}) {
-	panic(fmt.Sprintf(format, argv...))
+func TestKVDiff(t *testing.T) {
+	kv1 := map[Key]string{1:"a", 3:"c", 4:"d"}
+	kv2 := map[Key]string{1:"b",        4:"d", 5:"e"}
+	got  := kvdiff(kv1, kv2)
+	want := map[Key]Δstring{1:{"a","b"}, 3:{"c",DEL}, 5:{DEL,"e"}}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("error:\ngot:  %v\nwant: %v", got, want)
+	}
+}
+
+func TestKVTxt(t *testing.T) {
+	kv := map[Key]string{3:"hello", 1:"zzz", 4:"world"}
+	got  := kvtxt(kv)
+	want := "1:zzz,3:hello,4:world"
+	if got != want {
+		t.Fatalf("error:\ngot:  %q\nwant: %q", got, want)
+	}
 }

wcfs/internal/xbtree/xbtreetest/rtree.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+
+import (
+	"fmt"
+	"sort"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
+)
+
+// RTree represents Tree node covering [lo, hi_] key range in its parent tree.
+// XXX actually no coverage here -> kill? -> change to just `path []zodb.Oid` in RBucket?
+type RTree struct {
+	Oid    zodb.Oid
+	Parent *RTree
+	// XXX +children?
+}
+
+// RBucket represents Bucket node covering [lo, hi_] key range in its Tree.
+// NOTE it is not [lo,hi) but [lo,hi_] instead to avoid overflow at KeyMax.
+type RBucket struct {
+	Oid     zodb.Oid
+	Parent  *RTree
+	Keycov  blib.KeyRange
+	KV      map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
+}
+
+// Path returns path to this bucket from tree root.
+func (rb *RBucket) Path() []zodb.Oid {
+	path := []zodb.Oid{rb.Oid}
+	p := rb.Parent
+	for p != nil {
+		path = append([]zodb.Oid{p.Oid}, path...)
+		p = p.Parent
+	}
+	return path
+}
+
+// RBucketSet represents set of buckets covering whole [-∞,∞) range.
+type RBucketSet []*RBucket // k↑
+
+
+// Get returns RBucket which covers key k.
+func (rbs RBucketSet) Get(k Key) *RBucket {
+	i := sort.Search(len(rbs), func(i int) bool {
+		return k <= rbs[i].Keycov.Hi_
+	})
+	if i == len(rbs) {
+		panicf("BUG: key %v not covered;  coverage: %s", k, rbs.coverage())
+	}
+
+	rb := rbs[i]
+	if !rb.Keycov.Has(k) {
+		panicf("BUG: get(%v) -> %s;  coverage: %s", k, rb.Keycov, rbs.coverage())
+	}
+
+	return rb
+}
+
+// coverage returns string representation of rbs coverage structure.
+func (rbs RBucketSet) coverage() string {
+	if len(rbs) == 0 {
+		return "ø"
+	}
+	s := ""
+	for _, rb := range rbs {
+		if s != "" {
+			s += " "
+		}
+		s += fmt.Sprintf("%s", rb.Keycov)
+	}
+	return s
+}
+
+// Flatten converts xkv with bucket structure into regular dict.
+func (xkv RBucketSet) Flatten() map[Key]string {
+	kv := make(map[Key]string)
+	for _, b := range xkv {
+		for k,v := range b.KV {
+			kv[k] = v
+		}
+	}
+	return kv
+}
+
+func (b *RBucket) String() string {
+	return fmt.Sprintf("%sB%s{%s}", b.Keycov, b.Oid, kvtxt(b.KV))
+}

wcfs/internal/xbtree/xbtreetest/treegen.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+// treegen.go provides functionality:
+//
+//   - to commit a particular BTree topology into ZODB, and
+//   - to generate set of random tree topologies that all correspond to particular {k->v} dict.
+//
+// treegen.py is used as helper for both tasks.
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"lab.nexedi.com/kirr/go123/my"
+	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+
+// TreeGenSrv represents connection to running `treegen ...` server.
+type TreeGenSrv struct {
+	argv      []string
+	pysrv     *exec.Cmd      // spawned `treegen ...`
+	pyin      io.WriteCloser // input to pysrv
+	pyoutRaw  io.ReadCloser  // output from pysrv
+	pyout     *bufio.Reader  // buffered ^^^
+}
+
+// TreeSrv represents connection to running `treegen trees` server.
+//
+// Create it with StartTreeSrv(zurl).
+// - Commit(treeTopology) -> tid
+type TreeSrv struct {
+	*TreeGenSrv
+	zurl	  string
+	treeRoot  zodb.Oid       // oid of the tree treegen works on
+	head      zodb.Tid       // last made commit
+}
+
+// AllStructsSrv represents connection to running `treegen allstructs` server.
+//
+// Create it with StartAllStructsSrv().
+// - AllStructs(maxdepth, maxsplit, n, seed, kv1, kv2)
+type AllStructsSrv struct {
+	*TreeGenSrv
+}
+
+// StartTreeGenSrv spawns `treegen ...` server.
+func StartTreeGenSrv(argv ...string) (_ *TreeGenSrv, hello string, err error) {
+	defer xerr.Contextf(&err, "treesrv %v: start", argv)
+
+	// spawn `treegen ...`
+	tg := &TreeGenSrv{argv: argv}
+	tg.pysrv = exec.Command(filepath.Dir(my.File())+"/treegen.py", argv...)
+	tg.pyin, err = tg.pysrv.StdinPipe()
+	if err != nil {
+		return nil, "", err
+	}
+	tg.pyoutRaw, err = tg.pysrv.StdoutPipe()
+	if err != nil {
+		return nil, "", err
+	}
+	tg.pyout = bufio.NewReader(tg.pyoutRaw)
+	tg.pysrv.Stderr = os.Stderr // no redirection
+
+	err = tg.pysrv.Start()
+	if err != nil {
+		return nil, "", err
+	}
+
+	// wait for hello message and return it
+	defer func() {
+		if err != nil {
+			tg.Close() // ignore error
+		}
+	}()
+	defer xerr.Context(&err, "handshake")
+	hello, err = tg.pyout.ReadString('\n')
+	if err != nil {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		return nil, "", err
+	}
+	hello = strings.TrimSuffix(hello, "\n")
+
+	return tg, hello, nil
+}
+
+// Close shutdowns treegen server.
+func (tg *TreeGenSrv) Close() (err error) {
+	defer xerr.Contextf(&err, "treegen %v: close", tg.argv)
+
+	err1 := tg.pyin.Close()
+	err2 := tg.pyoutRaw.Close()
+	err3 := tg.pysrv.Wait()
+	return xerr.Merge(err1, err2, err3)
+}
+
+
+// StartTreeSrv spawns `treegen trees` server.
+func StartTreeSrv(zurl string) (_ *TreeSrv, err error) {
+	defer xerr.Contextf(&err, "tree.srv %s: start", zurl)
+	tgSrv, hello, err := StartTreeGenSrv("trees", zurl)
+	if err != nil {
+		return nil, err
+	}
+
+	tg := &TreeSrv{TreeGenSrv: tgSrv, zurl: zurl}
+	defer func() {
+		if err != nil {
+			tgSrv.Close() // ignore error
+		}
+	}()
+
+	// tree.srv start @<at> tree=<root>
+	defer xerr.Contextf(&err, "invalid hello %q", hello)
+	startRe := regexp.MustCompile(`^tree.srv start @([^ ]+) root=([^ ]+)$`)
+	m := startRe.FindStringSubmatch(hello)
+	if m == nil {
+		return nil, fmt.Errorf("unexpected format")
+	}
+	tg.head, err = zodb.ParseTid(m[1]) // <at>
+	if err != nil {
+		return nil, fmt.Errorf("tid: %s", err)
+	}
+	tg.treeRoot, err = zodb.ParseOid(m[2]) // <root>
+	if err != nil {
+		return nil, fmt.Errorf("root: %s", err)
+	}
+
+	return tg, nil
+}
+
+// StartAllStructsSrv spawns `treegen allstructs` server.
+func StartAllStructsSrv() (_ *AllStructsSrv, err error) {
+	defer xerr.Context(&err, "allstructs.srv: start")
+
+	tgSrv, hello, err := StartTreeGenSrv("allstructs")
+	if err != nil {
+		return nil, err
+	}
+
+	sg := &AllStructsSrv{TreeGenSrv: tgSrv}
+	defer func() {
+		if err != nil {
+			tgSrv.Close() // ignore error
+		}
+	}()
+
+	defer xerr.Contextf(&err, "invalid hello %q", hello)
+	if hello != "# allstructs.srv start" {
+		return nil, fmt.Errorf("unexpected format")
+	}
+
+	return sg, nil
+}
+
+
+// Commit creates new commit with underlying tree changed to specified tree topology.
+func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
+	defer xerr.Contextf(&err, "tree.srv %s: commit %s", tg.zurl, tree)
+
+	_, err = io.WriteString(tg.pyin, tree + "\n")
+	if err != nil {
+		return zodb.InvalidTid, err
+	}
+
+	reply, err := tg.pyout.ReadString('\n')
+	if err != nil {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		return zodb.InvalidTid, err
+	}
+	reply = strings.TrimSuffix(reply, "\n")
+
+	tid, err := zodb.ParseTid(reply)
+	if err != nil {
+		return zodb.InvalidTid, fmt.Errorf("invalid reply: %s", err)
+	}
+	tg.head = tid
+	return tid, nil
+}
+
+// AllStructs returns response from `treegen allstructs`
+func (tg *AllStructsSrv) AllStructs(kv map[Key]string, maxdepth, maxsplit, n int, seed int64) (_ []string, err error) {
+	req := fmt.Sprintf("%d %d %d/%d %s", maxdepth, maxsplit, n, seed, kvtxt(kv))
+	defer xerr.Contextf(&err, "allstructs.srv: %s ", req)
+
+	_, err = io.WriteString(tg.pyin, req + "\n")
+	if err != nil {
+		return nil, err
+	}
+
+	structv := []string{}
+	for {
+		reply, err := tg.pyout.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			return nil, err
+		}
+		reply = strings.TrimSuffix(reply, "\n")
+
+		if reply == "# ----" {
+			return structv, nil // end of response
+		}
+		if strings.HasPrefix(reply, "#") {
+			continue // comment
+		}
+
+		structv = append(structv, reply)
+	}
+}

wcfs/internal/xbtree/xbtreetest/xbtreetest.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package xbtreetest provides infrastructure for testing LOBTree with ZBlk values.
+// XXX -> treetest?
+package xbtreetest
+
+import (
+	"fmt"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
+)
+
+// XXX instead of generics
+type Tree   = blib.Tree
+type Bucket = blib.Bucket
+type Node   = blib.Node
+type TreeEntry   = blib.TreeEntry
+type BucketEntry = blib.BucketEntry
+
+type Key    = blib.Key
+const KeyMax = blib.KeyMax
+const KeyMin = blib.KeyMin
+
+type setKey = set.I64
+
+// XXX dup from xbtree  (to avoid import cycle)
+const VDEL  = zodb.InvalidOid
+
+
+func panicf(format string, argv ...interface{}) {
+	panic(fmt.Sprintf(format, argv...))
+}

wcfs/internal/xbtree/xbtreetest/zdata.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+// access to ZBlk data
+
+import (
+	"context"
+
+	"lab.nexedi.com/kirr/go123/exc"
+
+	"lab.nexedi.com/kirr/neo/go/transaction"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
+)
+
+// ZBlk-related functions are imported at runtime by package xbtreetest/init
+var (
+	ZTreeGetBlkData func(context.Context, *Tree, Key) (string, bool, []Node, error)
+	ZGetBlkData     func(context.Context, *zodb.Connection, zodb.Oid) (string, error)
+)
+
+func zassertInitDone() {
+	if ZTreeGetBlkData == nil {
+		panic("xbtreetest/zdata not initialized -> import xbtreetest/init to fix")
+	}
+}
+
+// xzgetBlkData loads block data from ZBlk object specified by its oid.
+func xzgetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) string {
+	zassertInitDone()
+	X := exc.Raiseif
+
+	if zblkOid == VDEL {
+		return DEL
+	}
+
+	data, err := ZGetBlkData(ctx, zconn, zblkOid); X(err)
+	return string(data)
+}
+
+// xzgetBlkDataAt loads block data from ZBlk object specified by oid@at.
+func xzgetBlkDataAt(db *zodb.DB, zblkOid zodb.Oid, at zodb.Tid) string {
+	zassertInitDone()
+	X := exc.Raiseif
+
+	txn, ctx := transaction.New(context.Background())
+	defer txn.Abort()
+
+	zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at});  X(err)
+	return xzgetBlkData(ctx, zconn, zblkOid)
+}

wcfs/internal/xbtree/δbtail.go

@@ -30,6 +30,7 @@ import (
 	"lab.nexedi.com/kirr/neo/go/transaction"
 	"lab.nexedi.com/kirr/neo/go/zodb"
 
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xtail"
 )
 
@@ -96,7 +97,7 @@ type ΔBtail struct {
 	// For this set all vδT are fully computed.
 	// The set of nodes that were requested to be tracked, but were not yet
 	// taken into account, is kept in ΔTtail.trackNew & co.
-	trackSet PPTreeSubSet
+	trackSet blib.PPTreeSubSet
 
 	// set of trees for which .trackNew is non-empty
 	trackNewRoots setOid
@@ -114,7 +115,7 @@ type ΔTtail struct {
 
 	// set of nodes that were requested to be tracked in this tree, but for
 	// which vδT was not yet rebuilt
-	trackNew PPTreeSubSet
+	trackNew blib.PPTreeSubSet
 	// XXX + trackNewKeys RangedKeySet
 
 	// {}k/v @tail for keys that are changed in (tail, head].
@@ -158,7 +159,7 @@ func NewΔBtail(at0 zodb.Tid, db *zodb.DB) *ΔBtail {
 		δZtail:        zodb.NewΔTail(at0),
 		vδBroots:      nil,
 		vδTbyRoot:     map[zodb.Oid]*ΔTtail{},
-		trackSet:      PPTreeSubSet{},
+		trackSet:      blib.PPTreeSubSet{},
 		trackNewRoots: setOid{},
 		db:            db,
 	}
@@ -167,7 +168,7 @@ func NewΔBtail(at0 zodb.Tid, db *zodb.DB) *ΔBtail {
 // newΔTtail creates new empty ΔTtail object.
 func newΔTtail() *ΔTtail {
 	return &ΔTtail{
-		trackNew:  PPTreeSubSet{},
+		trackNew:  blib.PPTreeSubSet{},
 		KVAtTail:  make(map[Key]Value),
 		lastRevOf: make(map[Key]zodb.Tid),
 	}
@@ -271,7 +272,7 @@ func (δBtail *ΔBtail) track(key Key, path []zodb.Oid) error {
 	// empty artificial tree. We need to do the normalization because we
 	// later check whether leaf path[-1] ∈ trackSet and without
 	// normalization path[-1] can be InvalidOid.
-	path = normPath(path)
+	path = blib.NormPath(path)
 	if len(path) == 0 {
 		return nil // empty tree
 	}
@@ -334,7 +335,7 @@ func (δBtail *ΔBtail) rebuildAll() (err error) {
 // - set of revisions for which new entries in .vδT have been created.
 //
 // XXX place
-func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet PPTreeSubSet, δrevSet setTid, err error) {
+func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet blib.PPTreeSubSet, δrevSet setTid, err error) {
 	defer xerr.Context(&err, "ΔTtail rebuild")
 	// XXX locking
 
@@ -342,7 +343,7 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
 	tracefΔBtail("trackNew:       %v\n", δTtail.trackNew)
 
 	trackNew := δTtail.trackNew
-	δTtail.trackNew = PPTreeSubSet{}
+	δTtail.trackNew = blib.PPTreeSubSet{}
 
 	if len(trackNew) == 0 {
 		return nil, nil, nil
@@ -353,7 +354,7 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
 	// go backwards and merge vδT <- treediff(lo..hi/trackNew)
 	vδZ := δZtail.Data()
 	for {
-		δtkeycov := &RangedKeySet{}     // all keys coming into tracking set during this lo<-hi scan
+		δtkeycov := &blib.RangedKeySet{}     // all keys coming into tracking set during this lo<-hi scan
 		trackNewCur := trackNew.Clone() // trackNew adjusted as of when going to i<- entry
 		for i := len(vδZ)-1; i>=0; i-- {
 			δZ := vδZ[i]
@@ -411,7 +412,7 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
 
 // widenTrackNew widens trackNew to cover δtkeycov.
 // XXX -> widenTrackSet?
-func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid, at zodb.Tid, db *zodb.DB) (err error) {
+func widenTrackNew(trackNew blib.PPTreeSubSet, δtkeycov *blib.RangedKeySet, root zodb.Oid, at zodb.Tid, db *zodb.DB) (err error) {
 	// XXX errctx, debug
 	defer xerr.Contextf(&err, "widenTrackNew tree<%s> @%s +%s", root, at, δtkeycov)
 
@@ -427,19 +428,19 @@ func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid
 	}
 	tree := xtree.(*Tree) // must succeed	XXX better explicit panic?
 
-	top := &nodeInRange{prefix: nil, lo: KeyMin, hi_: KeyMax, node: tree}
+	top := &nodeInRange{prefix: nil, keycov: blib.KeyRange{KeyMin, KeyMax}, node: tree}
 	V   := rangeSplit{top}
 	for _, r := range δtkeycov.AllRanges() {
-		lo := r.lo
+		lo := r.Lo
 		for {
 			b, err := V.GetToLeaf(ctx, lo);  /*X*/ if err != nil { return err }
 			trackNew.AddPath(b.Path())
 
 			// continue with next right bucket until r coverage is complete
-			if r.hi_ <= b.hi_ {
+			if r.Hi_ <= b.keycov.Hi_ {
 				break
 			}
-			lo = b.hi_ + 1
+			lo = b.keycov.Hi_ + 1
 		}
 	}
 	return nil
@@ -450,7 +451,7 @@ func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid
 //
 // δtrackNew/δtkeycov represents how trackNew changes when going through `atPrev <- δZ.Rev` .
 // newRevEntry indicates whether δZ.Rev was not there before in .vδT and new corresponding δT entry was created.
-func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew PPTreeSubSet, db *zodb.DB) (δtrackNew *ΔPPTreeSubSet, δtkeycov *RangedKeySet, newRevEntry bool, err error) {
+func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew blib.PPTreeSubSet, db *zodb.DB) (δtrackNew *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, newRevEntry bool, err error) {
 	defer xerr.Contextf(&err, "rebuild1 %s<-%s", atPrev, δZ.Rev)
 
 	debugfΔBtail("\n  rebuild1 @%s <- @%s\n", atPrev, δZ.Rev)
@@ -467,7 +468,7 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
 
 	// skip opening DB connections if there is no change to this tree
 	if len(δtopsByRoot) == 0 {
-		return NewΔPPTreeSubSet(), &RangedKeySet{}, false, nil
+		return blib.NewΔPPTreeSubSet(), &blib.RangedKeySet{}, false, nil
 	}
 
 	if len(δtopsByRoot) != 1 {
@@ -560,7 +561,7 @@ func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
 
 		// δtkeycov1 != ø   ->  rebuild δTtail with trackNew ~= δtkeycov1
 		if !δT1.δtkeycov1.Empty() && δBtail.δZtail.Len() > 1 {
-			trackNew := PPTreeSubSet{}
+			trackNew := blib.PPTreeSubSet{}
 			err := widenTrackNew(trackNew, δT1.δtkeycov1, root, δBtail.Head(), δBtail.db)
 			if err != nil {
 				return ΔB{}, err
@@ -611,8 +612,8 @@ type _ΔBUpdate1 struct {
 	ByRoot map[zodb.Oid]*_ΔTUpdate1
 }
 type _ΔTUpdate1 struct {
-	δtkeycov1 *RangedKeySet   // {} root -> δtrackedKeys after first treediff (always grow)
-	δtrack    *ΔPPTreeSubSet  // XXX kill (not used)
+	δtkeycov1 *blib.RangedKeySet   // {} root -> δtrackedKeys after first treediff (always grow)
+	δtrack    *blib.ΔPPTreeSubSet  // XXX kill (not used)
 }
 func (δBtail *ΔBtail) _Update1(δZ *zodb.EventCommit) (δB1 _ΔBUpdate1, err error) {
 	headOld := δBtail.Head()

wcfs/internal/xbtree/δbtail_x_test.go

@@ -18,85 +18,7 @@
 // See https://www.nexedi.com/licensing for rationale and options.
 
 package xbtree_test
-// ZBlk-related part of δbtail_test
 
 import (
-	"context"
-	"fmt"
-
-	"lab.nexedi.com/kirr/go123/xerr"
-	"lab.nexedi.com/kirr/neo/go/zodb"
-
-	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree"
-	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
-	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
+	_ "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest/init"
 )
-
-type Tree = xbtree.Tree
-type Node = xbtree.Node
-type Key  = xbtree.Key
-
-type ZBlk = zdata.ZBlk
-
-// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
-// XXX +return blkRevMax and use it ?
-func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
-	path = []Node{}
-	xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
-		path = append(path, node)
-	})
-	if err != nil {
-		return nil, false, nil, err
-	}
-
-	if ok {
-		zblk, ok = xzblk.(ZBlk)
-		if !ok {
-			return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk))  // XXX errctx
-		}
-	}
-
-	return zblk, ok, path, nil
-}
-
-
-func init() {
-	xbtree.ZTreeGetBlkData = ZTreeGetBlkData
-	xbtree.ZGetBlkData     = ZGetBlkData
-}
-
-// ZTreeGetBlkData returns block data from block pointed to by ztree[k].
-func ZTreeGetBlkData(ctx context.Context, ztree *Tree, k Key) (data string, ok bool, path []Node, err error) {
-	defer xerr.Contextf(&err, "@%s: tree<%s>: get blkdata from [%d]", ztree.PJar().At(), ztree.POid(), k)
-
-	zblk, ok, path, err := ztreeGetBlk(ctx, ztree, k)
-	if err != nil || !ok {
-		return "", ok, path, err
-	}
-
-	bdata, _, err := zblk.LoadBlkData(ctx)
-	if err != nil {
-		return "", false, nil, err
-	}
-
-	return string(bdata), true, path, nil
-}
-
-// ZGetBlkData loads block data from ZBlk object specified by its oid.
-func ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
-	defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
-
-	xblk, err := zconn.Get(ctx, zblkOid)
-	if err != nil {
-		return "", err
-	}
-	zblk, ok := xblk.(ZBlk)
-	if !ok {
-		return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
-	}
-	bdata, _, err := zblk.LoadBlkData(ctx)
-	if err != nil {
-		return "", err
-	}
-	return string(bdata), nil
-}