.

wcfs/internal/xbtree/blib/pptreesubset.go

@@ -102,7 +102,7 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
 
 	// normalize path: remove embedded bucket and check whether it was an
 	// artificial empty tree.
-	path = normPath(path)
+	path = NormPath(path)
 
 	// go through path and add nodes to the set
 	parent := zodb.InvalidOid
@@ -132,11 +132,11 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
 	}
 }
 
-// normPath normalizes path.
+// NormPath normalizes path.
 //
 // It removes embedded buckets and artificial empty trees.
 // Returned slice is subslice of path and aliases its memory.
-func normPath(path []zodb.Oid) []zodb.Oid {
+func NormPath(path []zodb.Oid) []zodb.Oid {
 	l := len(path)
 
 	// don't keep track of artificial empty tree

wcfs/internal/xbtree/treediff.go

@@ -80,6 +80,7 @@ import (
 	"lab.nexedi.com/kirr/neo/go/zodb"
 
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
 )
 
 const traceDiff = false
@@ -113,7 +114,7 @@ func (δv ΔValue) String() string {
 // for example for e.g. t₀->t₁->b₂ if δZ/T={t₀ b₂} -> δZ/TC=δZ/T+{t₁}
 //
 // δtopsByRoot = {} root -> {top changed nodes in that tree}
-func δZConnectTracked(δZv []zodb.Oid, T PPTreeSubSet) (δZTC setOid, δtopsByRoot map[zodb.Oid]setOid) {
+func δZConnectTracked(δZv []zodb.Oid, T blib.PPTreeSubSet) (δZTC setOid, δtopsByRoot map[zodb.Oid]setOid) {
 	δZ  := setOid{};  for _, δ := range δZv { δZ.Add(δ) }
 	δZTC = setOid{}
 	δtopsByRoot = map[zodb.Oid]setOid{}
@@ -338,18 +339,18 @@ func (rs rangeSplit) String() string {
 // δtops is set of top nodes for changed subtrees.
 // δZTC is connected(δZ/T) - connected closure for subset of δZ(old..new) that
 // touches tracked nodes of T.
-func treediff(ctx context.Context, root zodb.Oid, δtops setOid, δZTC setOid, trackSet PPTreeSubSet, zconnOld, zconnNew *zodb.Connection) (δT map[Key]ΔValue, δtrack *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
+func treediff(ctx context.Context, root zodb.Oid, δtops setOid, δZTC setOid, trackSet blib.PPTreeSubSet, zconnOld, zconnNew *zodb.Connection) (δT map[Key]ΔValue, δtrack *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, err error) {
 	defer xerr.Contextf(&err, "treediff %s..%s %s", zconnOld.At(), zconnNew.At(), root)
 
 	δT = map[Key]ΔValue{}
-	δtrack = NewΔPPTreeSubSet()
-	δtkeycov = &RangedKeySet{}
+	δtrack = blib.NewΔPPTreeSubSet()
+	δtkeycov = &blib.RangedKeySet{}
 
 	tracefDiff("\ntreediff %s  δtops: %v  δZTC: %v\n", root, δtops, δZTC)
 	tracefDiff(" trackSet: %v\n", trackSet)
 	defer tracefDiff("\n-> δT: %v\nδtrack: %v\nδtkeycov: %v\n", δT, δtrack, δtkeycov)
 
-	δtrackv := []*ΔPPTreeSubSet{}
+	δtrackv := []*blib.ΔPPTreeSubSet{}
 
 	for top := range δtops { // XXX -> sorted?
 		a, err1 := zgetNodeOrNil(ctx, zconnOld, top)
@@ -401,7 +402,7 @@ func treediff(ctx context.Context, root zodb.Oid, δtops setOid, δZTC setOid, t
 // consistent with b (= a + δ).
 //
 // δtkeycov represents how δtrack grows (always grows) tracking set key coverage.
-func diffX(ctx context.Context, a, b Node, δZTC setOid, trackSet PPTreeSubSet) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
+func diffX(ctx context.Context, a, b Node, δZTC setOid, trackSet blib.PPTreeSubSet) (δ map[Key]ΔValue, δtrack *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, err error) {
 	if a==nil && b==nil {
 		panic("BUG: both a & b == nil")	// XXX -> not a bug e.g. for `ø ø T` sequence?
 	}
@@ -438,11 +439,11 @@ func diffX(ctx context.Context, a, b Node, δZTC setOid, trackSet PPTreeSubSet)
 	if isT {
 		return diffT(ctx, aT, bT, δZTC, trackSet)
 	} else {
-		var δtrack *ΔPPTreeSubSet
+		var δtrack *blib.ΔPPTreeSubSet
 		δ, err := diffB(ctx, aB, bB)
 		if δ != nil {
-			δtrack = NewΔPPTreeSubSet()
-			δtkeycov = &RangedKeySet{}
+			δtrack = blib.NewΔPPTreeSubSet()
+			δtkeycov = &blib.RangedKeySet{}
 		}
 		return δ, δtrack, δtkeycov, err
 	}
@@ -452,13 +453,13 @@ func diffX(ctx context.Context, a, b Node, δZTC setOid, trackSet PPTreeSubSet)
 //
 // a, b point to top of subtrees @old and @new revisions.
 // δZTC is connected set of objects covering δZT (objects changed in this tree in old..new).
-func diffT(ctx context.Context, A, B *Tree, δZTC setOid, trackSet PPTreeSubSet) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
+func diffT(ctx context.Context, A, B *Tree, δZTC setOid, trackSet blib.PPTreeSubSet) (δ map[Key]ΔValue, δtrack *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, err error) {
 	tracefDiff("  diffT %s %s\n", xidOf(A), xidOf(B))
 	defer xerr.Contextf(&err, "diffT %s %s", xidOf(A), xidOf(B))
 
 	δ = map[Key]ΔValue{}
-	δtrack = NewΔPPTreeSubSet()
-	δtkeycov = &RangedKeySet{}
+	δtrack = blib.NewΔPPTreeSubSet()
+	δtkeycov = &blib.RangedKeySet{}
 	defer func() {
 		tracefDiff("  -> δ: %v\n", δ)
 		tracefDiff("  -> δtrack: %v\n", δtrack)
@@ -564,22 +565,22 @@ ABcov:
 	Bv := rangeSplit{btop} // nodes expanded from B
 
 	// for phase 2:
-	Akqueue := &RangedKeySet{} // queue for keys in A to be processed for δ-
-	Bkqueue := &RangedKeySet{} // ----//---- in B for δ+
-	Akdone  := &RangedKeySet{} // already processed keys in A
-	Bkdone  := &RangedKeySet{} // ----//---- in B
-	Aktodo := func(r KeyRange) {
+	Akqueue := &blib.RangedKeySet{} // queue for keys in A to be processed for δ-
+	Bkqueue := &blib.RangedKeySet{} // ----//---- in B for δ+
+	Akdone  := &blib.RangedKeySet{} // already processed keys in A
+	Bkdone  := &blib.RangedKeySet{} // ----//---- in B
+	Aktodo := func(r blib.KeyRange) {
 		if !Akdone.HasRange(r) {
-			δtodo := &RangedKeySet{}
+			δtodo := &blib.RangedKeySet{}
 			δtodo.AddRange(r)
 			δtodo.DifferenceInplace(Akdone)
 			debugfDiff("    Akq <- %s\n", δtodo)
 			Akqueue.UnionInplace(δtodo)
 		}
 	}
-	Bktodo := func(r KeyRange) {
+	Bktodo := func(r blib.KeyRange) {
 		if !Bkdone.HasRange(r) {
-			δtodo := &RangedKeySet{}
+			δtodo := &blib.RangedKeySet{}
 			δtodo.AddRange(r)
 			δtodo.DifferenceInplace(Bkdone)
 			debugfDiff("    Bkq <- %s\n", δtodo)
@@ -595,8 +596,8 @@ ABcov:
 	}
 
 	// δtkeycov will be = BAdd \ ADel
-	δtkeycovADel := &RangedKeySet{}
-	δtkeycovBAdd := &RangedKeySet{}
+	δtkeycovADel := &blib.RangedKeySet{}
+	δtkeycovBAdd := &blib.RangedKeySet{}
 
 	// phase 1: expand A top->down driven by δZTC.
 	// by default a node contributes to δ-
@@ -620,7 +621,7 @@ ABcov:
 			// a is bucket -> δ-
 			δA, err := diffB(ctx, a, nil);  /*X*/if err != nil { return nil,nil,nil, err }
 			err = δMerge(δ, δA);		/*X*/if err != nil { return nil,nil,nil, err }
-			ar := KeyRange{ra.lo, ra.hi_}
+			ar := blib.KeyRange{ra.lo, ra.hi_}
 			δtrack.Del.AddPath(ra.Path())
 			δtkeycovADel.AddRange(ar)
 			debugfDiff("  δtrack - %s %v\n", ar, ra.Path())
@@ -632,7 +633,7 @@ ABcov:
 		case *Tree:
 			// empty tree - queue holes covered by it
 			if len(a.Entryv()) == 0 {
-				ar := KeyRange{ra.lo, ra.hi_}
+				ar := blib.KeyRange{ra.lo, ra.hi_}
 				δtrack.Del.AddPath(ra.Path())
 				δtkeycovADel.AddRange(ar)
 				debugfDiff("  δtrack - %s %v\n", ar, ra.Path())
@@ -691,8 +692,8 @@ ABcov:
 					}
 					if found {
 						// ac can be skipped if key coverage stays the same
-						ar := KeyRange{ac.lo, ac.hi_}
-						br := KeyRange{bc.lo, bc.hi_}
+						ar := blib.KeyRange{ac.lo, ac.hi_}
+						br := blib.KeyRange{bc.lo, bc.hi_}
 						if ar == br {
 							// adjust trackSet since path to the node could have changed
 							apath := ac.Path()
@@ -744,7 +745,7 @@ ABcov:
 		}
 
 		for _, r := range Bkqueue.AllRanges() {
-			lo := r.lo
+			lo := r.Lo
 			for {
 				b, err := Bv.GetToLeaf(ctx, lo);  /*X*/if err != nil { return nil,nil,nil, err }
 				debugfDiff("  B k%d -> %s\n", lo, b)
@@ -758,7 +759,7 @@ ABcov:
 
 					// δ <- δB
 					err = δMerge(δ, δB);	/*X*/if err != nil { return nil,nil,nil, err }
-					br := KeyRange{b.lo, b.hi_}
+					br := blib.KeyRange{b.lo, b.hi_}
 					δtrack.Add.AddPath(b.Path())
 					δtkeycovBAdd.AddRange(br)
 					debugfDiff("  δtrack + %s %v\n", br, b.Path())
@@ -771,7 +772,7 @@ ABcov:
 				}
 
 				// continue with next right bucket until r coverage is complete
-				if r.hi_ <= b.hi_ {
+				if r.Hi_ <= b.hi_ {
 					break
 				}
 				lo = b.hi_ + 1
@@ -782,7 +783,7 @@ ABcov:
 		debugfDiff("\n")
 		debugfDiff("  Akq: %s\n", Akqueue)
 		for _, r := range Akqueue.AllRanges() {
-			lo := r.lo
+			lo := r.Lo
 			for {
 				a, err := Av.GetToLeaf(ctx, lo);  /*X*/if err != nil { return nil,nil,nil, err }
 				debugfDiff("  A k%d -> %s\n", lo, a)
@@ -798,7 +799,7 @@ ABcov:
 					err = δMerge(δ, δA);	/*X*/if err != nil { return nil,nil,nil, err }
 					δtrack.Del.AddPath(a.Path())
 					// NOTE adjust δtkeycovADel only if a was originally tracked
-					ar := KeyRange{a.lo, a.hi_}
+					ar := blib.KeyRange{a.lo, a.hi_}
 					_, tracked := trackSet[a.node.POid()]
 					if tracked {
 						δtkeycovADel.AddRange(ar)
@@ -815,7 +816,7 @@ ABcov:
 				}
 
 				// continue with next right bucket until r coverage is complete
-				if r.hi_ <= a.hi_ {
+				if r.Hi_ <= a.hi_ {
 					break
 				}
 				lo = a.hi_ + 1
@@ -1009,7 +1010,7 @@ func xidOf(obj zodb.IPersistent) string {
 
 func (rn *nodeInRange) String() string {
 	done := " ";   if rn.done         { done = "*" }
-	return fmt.Sprintf("%s%s%s", done, KeyRange{rn.lo, rn.hi_}, vnode(rn.node))
+	return fmt.Sprintf("%s%s%s", done, blib.KeyRange{rn.lo, rn.hi_}, vnode(rn.node))
 }
 
 // push pushes element to node stack.

wcfs/internal/xbtree/xbtree.go

@@ -5,10 +5,8 @@ package xbtree
 
 import (
 	"fmt"
-	"math"
 
 	"lab.nexedi.com/kirr/neo/go/zodb"
-	"lab.nexedi.com/kirr/neo/go/zodb/btree"
 
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"

wcfs/internal/xbtree/xbtreetest/init/init.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package xbtreetest/init (ex imported from package A) should be imported in
+// addition to xbtreetest (from package A_test) to initialize xbtreetest at runtime.
+package init
+
+// ZBlk-related part of δbtail_test
+
+import (
+	"context"
+	"fmt"
+
+	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
+)
+
+type Tree = xbtree.Tree
+type Node = xbtree.Node
+type Key  = xbtree.Key
+
+type ZBlk = zdata.ZBlk
+
+// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
+// XXX +return blkRevMax and use it ?
+func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
+	path = []Node{}
+	xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
+		path = append(path, node)
+	})
+	if err != nil {
+		return nil, false, nil, err
+	}
+
+	if ok {
+		zblk, ok = xzblk.(ZBlk)
+		if !ok {
+			return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk))  // XXX errctx
+		}
+	}
+
+	return zblk, ok, path, nil
+}
+
+
+func init() {
+	xbtree.ZTreeGetBlkData = ZTreeGetBlkData
+	xbtree.ZGetBlkData     = ZGetBlkData
+}
+
+// ZTreeGetBlkData returns block data from block pointed to by ztree[k].
+func ZTreeGetBlkData(ctx context.Context, ztree *Tree, k Key) (data string, ok bool, path []Node, err error) {
+	defer xerr.Contextf(&err, "@%s: tree<%s>: get blkdata from [%d]", ztree.PJar().At(), ztree.POid(), k)
+
+	zblk, ok, path, err := ztreeGetBlk(ctx, ztree, k)
+	if err != nil || !ok {
+		return "", ok, path, err
+	}
+
+	bdata, _, err := zblk.LoadBlkData(ctx)
+	if err != nil {
+		return "", false, nil, err
+	}
+
+	return string(bdata), true, path, nil
+}
+
+// ZGetBlkData loads block data from ZBlk object specified by its oid.
+func ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
+	defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
+
+	xblk, err := zconn.Get(ctx, zblkOid)
+	if err != nil {
+		return "", err
+	}
+	zblk, ok := xblk.(ZBlk)
+	if !ok {
+		return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
+	}
+	bdata, _, err := zblk.LoadBlkData(ctx)
+	if err != nil {
+		return "", err
+	}
+	return string(bdata), nil
+}

wcfs/internal/xbtree/xbtreetest/rtree.go

@@ -31,27 +31,27 @@ import (
 // RTree represents Tree node covering [lo, hi_] key range in its parent tree.
 // XXX actually no coverage here -> kill? -> change to just `path []zodb.Oid` in RBucket?
 type RTree struct {
-	oid    zodb.Oid
-	parent *RTree
+	Oid    zodb.Oid
+	Parent *RTree
 	// XXX +children?
 }
 
 // RBucket represents Bucket node covering [lo, hi_] key range in its Tree.
 // NOTE it is not [lo,hi) but [lo,hi_] instead to avoid overflow at KeyMax.
 type RBucket struct {
-	oid     zodb.Oid
-	parent  *RTree
-	lo, hi_ Key	// XXX -> KeyRange ?
-	kv      map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
+	Oid     zodb.Oid
+	Parent  *RTree
+	Keycov  blib.KeyRange
+	KV      map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
 }
 
 // Path returns path to this bucket from tree root.
 func (rb *RBucket) Path() []zodb.Oid {
-	path := []zodb.Oid{rb.oid}
-	p := rb.parent
+	path := []zodb.Oid{rb.Oid}
+	p := rb.Parent
 	for p != nil {
-		path = append([]zodb.Oid{p.oid}, path...)
-		p = p.parent
+		path = append([]zodb.Oid{p.Oid}, path...)
+		p = p.Parent
 	}
 	return path
 }
@@ -63,15 +63,15 @@ type RBucketSet []*RBucket // k↑
 // Get returns RBucket which covers key k.
 func (rbs RBucketSet) Get(k Key) *RBucket {
 	i := sort.Search(len(rbs), func(i int) bool {
-		return k <= rbs[i].hi_
+		return k <= rbs[i].Keycov.Hi_
 	})
 	if i == len(rbs) {
 		panicf("BUG: key %v not covered;  coverage: %s", k, rbs.coverage())
 	}
 
 	rb := rbs[i]
-	if !(rb.lo <= k && k <= rb.hi_) {
-		panicf("BUG: get(%v) -> [%v, %v];  coverage: %s", k, rb.lo, rb.hi_, rbs.coverage())
+	if !(rb.Keycov.Lo <= k && k <= rb.Keycov.Hi_) {
+		panicf("BUG: get(%v) -> %s;  coverage: %s", k, rb.Keycov, rbs.coverage())
 	}
 
 	return rb
@@ -87,7 +87,7 @@ func (rbs RBucketSet) coverage() string {
 		if s != "" {
 			s += " "
 		}
-		s += fmt.Sprintf("[%v, %v]", rb.lo, rb.hi_)
+		s += fmt.Sprintf("%s", rb.Keycov)
 	}
 	return s
 }
@@ -96,7 +96,7 @@ func (rbs RBucketSet) coverage() string {
 func (xkv RBucketSet) Flatten() map[Key]string {
 	kv := make(map[Key]string)
 	for _, b := range xkv {
-		for k,v := range b.kv {
+		for k,v := range b.KV {
 			kv[k] = v
 		}
 	}
@@ -104,5 +104,5 @@ func (xkv RBucketSet) Flatten() map[Key]string {
 }
 
 func (b *RBucket) String() string {
-	return fmt.Sprintf("%sB%s{%s}", blib.KeyRange{b.lo, b.hi_}, b.oid, kvtxt(b.kv))
+	return fmt.Sprintf("%sB%s{%s}", b.Keycov, b.Oid, kvtxt(b.KV))
 }

wcfs/internal/xbtree/xbtreetest/treeenv.go

@@ -18,7 +18,7 @@
 // See https://www.nexedi.com/licensing for rationale and options.
 
 package xbtreetest
-// TreeEnv + friends
+// T + friends
 
 import (
 	"context"
@@ -28,14 +28,15 @@ import (
 	"lab.nexedi.com/kirr/neo/go/transaction"
 	"lab.nexedi.com/kirr/neo/go/zodb"
 
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
 )
 
-// tTreeEnv is tree-based testing environment.
+// T is tree-based testing environment.
 //
 // It combines TreeSrv and client side access to ZODB with committed trees.
-// It should be created it via tNewTreeEnv().
-type tTreeEnv struct {
+// It should be created it via NewT().
+type T struct {
 	*testing.T
 
 	work    string   // working directory
@@ -44,13 +45,13 @@ type tTreeEnv struct {
 	db      *zodb.DB
 
 	// all committed trees
-	commitv []*tTreeCommit
+	commitv []*Commit
 }
 
-// tTreeCommit represent test commit changing a tree.
-type tTreeCommit struct {
+// Commit represent test commit changing a tree.
+type Commit struct {
 	tree       string              // the tree in topology-encoding
-	prev       *tTreeCommit        // previous commit
+	prev       *Commit             // previous commit
 	at         zodb.Tid            // commit revision
 	δZ         *zodb.EventCommit   // raw ZODB changes; δZ.tid == at
 	xkv        RBucketSet          // full tree state as of @at
@@ -59,12 +60,12 @@ type tTreeCommit struct {
 //	δzblkData  map[zodb.Oid]Δstring // full diff for zblkData against parent XXX ?
 }
 
-// tNewTreeEnv creates new  tTreeEnv.
-func tNewTreeEnv(t *testing.T) *tTreeEnv {
+// NewT creates new T.
+func NewT(t *testing.T) *T {
 	X := exc.Raiseif
 	t.Helper()
 
-	tt := &tTreeEnv{T: t}
+	tt := &T{T: t}
 
 	var err error
 	work := t.TempDir()
@@ -91,7 +92,7 @@ func tNewTreeEnv(t *testing.T) *tTreeEnv {
 	})
 
 	head := tt.treeSrv.head
-	t1 := &tTreeCommit{
+	t1 := &Commit{
 		tree:       "T/B:",	// treegen.py creates the tree as initially empty
 		prev:       nil,
 		at:         head,
@@ -100,7 +101,7 @@ func tNewTreeEnv(t *testing.T) *tTreeEnv {
 		δZ:         nil,
 		δxkv:       nil,
 	}
-	tt.commitv = []*tTreeCommit{t1}
+	tt.commitv = []*Commit{t1}
 
 	return tt
 }
@@ -115,17 +116,18 @@ func (_ *tZODBCacheEverything) PCacheClassify(_ zodb.IPersistent) zodb.PCachePol
 }
 
 // Root returns OID of root tree node.
-func (t *tTreeEnv) Root() zodb.Oid {
+func (t *T) Root() zodb.Oid {
 	return t.treeSrv.treeRoot
 }
 
 // Head returns most-recently committed tree.
-func (t *tTreeEnv) Head() *tTreeCommit {
+func (t *T) Head() *Commit {
 	return t.commitv[len(t.commitv)-1]
 }
 
-// CommitTree calls t.treeSrv.Commit and returns tTreeCommit corresponding to committed transaction.
-func (t *tTreeEnv) CommitTree(tree string) *tTreeCommit {
+// CommitTree calls t.treeSrv.Commit and returns Commit corresponding to committed transaction.
+// XXX naming -> Commit ?
+func (t *T) CommitTree(tree string) *Commit {
 	// TODO X = FatalIf
 	X := exc.Raiseif
 	defer exc.Contextf("commit %s", tree)
@@ -164,19 +166,18 @@ func (t *tTreeEnv) CommitTree(tree string) *tTreeCommit {
 		// it comes to ->T2.
 		xkv = RBucketSet{
 			&RBucket{
-				oid:    zodb.InvalidOid,
-				parent: &RTree{
-					oid:    t.Root(), // NOTE oid is not InvalidOid
-					parent: nil,
+				Oid:    zodb.InvalidOid,
+				Parent: &RTree{
+					Oid:    t.Root(), // NOTE oid is not InvalidOid
+					Parent: nil,
 				},
-				lo:     KeyMin,
-				hi_:    KeyMax,
-				kv:     map[Key]string{},
+				Keycov: blib.KeyRange{KeyMin, KeyMax},
+				KV:     map[Key]string{},
 			},
 		}
 	}
 
-	ttree := &tTreeCommit{
+	ttree := &Commit{
 		tree:       tree,
 		at:         δZ.Tid,
 		δZ:         δZ,
@@ -244,7 +245,7 @@ func xGetBlkDataTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]string {
 // xgetBlkData loads blk data for ZBlk<oid> @t.at
 //
 // For speed the load is done via preloaded t.blkDataTab instead of access to the DB.
-func (t *tTreeCommit) xgetBlkData(oid zodb.Oid) string {
+func (t *Commit) xgetBlkData(oid zodb.Oid) string {
 	if oid == VDEL {
 		return DEL
 	}
@@ -282,15 +283,14 @@ func xGetTree(db *zodb.DB, at zodb.Tid, root zodb.Oid) RBucketSet {
 	})
 	if len(rbucketv) == 0 { // empty tree -> [-∞,∞){}
 		etree   := &RTree{
-				oid:    root,
-				parent: nil,
+				Oid:    root,
+				Parent: nil,
 		}
 		ebucket := &RBucket{
-				oid:    zodb.InvalidOid,
-				parent: etree,
-				lo:     KeyMin,
-				hi_:    KeyMax,
-				kv:     map[Key]string{},
+				Oid:    zodb.InvalidOid,
+				Parent: etree,
+				Keycov: blib.KeyRange{KeyMin, KeyMax},
+				KV:     map[Key]string{},
 		}
 		rbucketv = RBucketSet{ebucket}
 	}
@@ -307,7 +307,7 @@ func _xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, rparent *RTree, bv
 	err := ztree.PActivate(ctx); X(err)
 	defer ztree.PDeactivate()
 
-	rtree := &RTree{oid: ztree.POid(), parent: rparent}
+	rtree := &RTree{Oid: ztree.POid(), Parent: rparent}
 
 	// [i].Key ≤ [i].Child.*.Key < [i+1].Key  i ∈ [0, len([]))
 	//
@@ -346,7 +346,12 @@ func _xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, rparent *RTree, bv
 		}
 
 
-		b := &RBucket{oid: zbucket.POid(), parent: rtree, lo: xlo, hi_: xhi_, kv: bkv}
+		b := &RBucket{
+			Oid:    zbucket.POid(),
+			Parent: rtree,
+			Keycov: blib.KeyRange{xlo, xhi_},
+			KV:     bkv,
+		}
 		bvisit(b)
 	}
 }

wcfs/internal/xbtree/xbtreetest/xbtreetest.go

@@ -23,24 +23,27 @@ package xbtreetest
 
 import (
 	"fmt"
-	"math"
 
 	"lab.nexedi.com/kirr/neo/go/zodb"
-	"lab.nexedi.com/kirr/neo/go/zodb/btree"
 
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
-//	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
 )
 
-// XXX dup from xbtree  (to avoid import cycle)
-type Tree   = btree.LOBTree
-type Bucket = btree.LOBucket
+// XXX instead of generics
+type Tree   = blib.Tree
+type Bucket = blib.Bucket
+type Node   = blib.Node
+type TreeEntry   = blib.TreeEntry
+type BucketEntry = blib.BucketEntry
+
+type Key    = blib.Key
+const KeyMax = blib.KeyMax
+const KeyMin = blib.KeyMin
 
-type Key  = int64
-const KeyMax Key = math.MaxInt64
-const KeyMin Key = math.MinInt64
 type setKey = set.I64
 
+// XXX dup from xbtree  (to avoid import cycle)
 const VDEL  = zodb.InvalidOid
 
 

wcfs/internal/xbtree/xbtreetest/zdata.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+// access to ZBlk data
+
+import (
+	"context"
+
+	"lab.nexedi.com/kirr/go123/exc"
+
+	"lab.nexedi.com/kirr/neo/go/transaction"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
+)
+
+// ZBlk-related functions are imported at runtime by package xbtreetest/init
+var (
+	ZTreeGetBlkData func(context.Context, *Tree, Key) (string, bool, []Node, error)
+	ZGetBlkData     func(context.Context, *zodb.Connection, zodb.Oid) (string, error)
+)
+
+func zassertInitDone() {
+	if ZTreeGetBlkData == nil {
+		panic("xbtreetest/zdata not initialized -> import xbtreetest/init to fix")
+	}
+}
+
+// xzgetBlkData loads block data from ZBlk object specified by its oid.
+func xzgetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) string {
+	zassertInitDone()
+	X := exc.Raiseif
+
+	if zblkOid == VDEL {
+		return DEL
+	}
+
+	data, err := ZGetBlkData(ctx, zconn, zblkOid); X(err)
+	return string(data)
+}
+
+// xzgetBlkDataAt loads block data from ZBlk object specified by oid@at.
+func xzgetBlkDataAt(db *zodb.DB, zblkOid zodb.Oid, at zodb.Tid) string {
+	zassertInitDone()
+	X := exc.Raiseif
+
+	txn, ctx := transaction.New(context.Background())
+	defer txn.Abort()
+
+	zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at});  X(err)
+	return xzgetBlkData(ctx, zconn, zblkOid)
+}

wcfs/internal/xbtree/δbtail.go

@@ -30,6 +30,7 @@ import (
 	"lab.nexedi.com/kirr/neo/go/transaction"
 	"lab.nexedi.com/kirr/neo/go/zodb"
 
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xtail"
 )
 
@@ -96,7 +97,7 @@ type ΔBtail struct {
 	// For this set all vδT are fully computed.
 	// The set of nodes that were requested to be tracked, but were not yet
 	// taken into account, is kept in ΔTtail.trackNew & co.
-	trackSet PPTreeSubSet
+	trackSet blib.PPTreeSubSet
 
 	// set of trees for which .trackNew is non-empty
 	trackNewRoots setOid
@@ -114,7 +115,7 @@ type ΔTtail struct {
 
 	// set of nodes that were requested to be tracked in this tree, but for
 	// which vδT was not yet rebuilt
-	trackNew PPTreeSubSet
+	trackNew blib.PPTreeSubSet
 	// XXX + trackNewKeys RangedKeySet
 
 	// {}k/v @tail for keys that are changed in (tail, head].
@@ -158,7 +159,7 @@ func NewΔBtail(at0 zodb.Tid, db *zodb.DB) *ΔBtail {
 		δZtail:        zodb.NewΔTail(at0),
 		vδBroots:      nil,
 		vδTbyRoot:     map[zodb.Oid]*ΔTtail{},
-		trackSet:      PPTreeSubSet{},
+		trackSet:      blib.PPTreeSubSet{},
 		trackNewRoots: setOid{},
 		db:            db,
 	}
@@ -167,7 +168,7 @@ func NewΔBtail(at0 zodb.Tid, db *zodb.DB) *ΔBtail {
 // newΔTtail creates new empty ΔTtail object.
 func newΔTtail() *ΔTtail {
 	return &ΔTtail{
-		trackNew:  PPTreeSubSet{},
+		trackNew:  blib.PPTreeSubSet{},
 		KVAtTail:  make(map[Key]Value),
 		lastRevOf: make(map[Key]zodb.Tid),
 	}
@@ -271,7 +272,7 @@ func (δBtail *ΔBtail) track(key Key, path []zodb.Oid) error {
 	// empty artificial tree. We need to do the normalization because we
 	// later check whether leaf path[-1] ∈ trackSet and without
 	// normalization path[-1] can be InvalidOid.
-	path = normPath(path)
+	path = blib.NormPath(path)
 	if len(path) == 0 {
 		return nil // empty tree
 	}
@@ -334,7 +335,7 @@ func (δBtail *ΔBtail) rebuildAll() (err error) {
 // - set of revisions for which new entries in .vδT have been created.
 //
 // XXX place
-func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet PPTreeSubSet, δrevSet setTid, err error) {
+func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet blib.PPTreeSubSet, δrevSet setTid, err error) {
 	defer xerr.Context(&err, "ΔTtail rebuild")
 	// XXX locking
 
@@ -342,7 +343,7 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
 	tracefΔBtail("trackNew:       %v\n", δTtail.trackNew)
 
 	trackNew := δTtail.trackNew
-	δTtail.trackNew = PPTreeSubSet{}
+	δTtail.trackNew = blib.PPTreeSubSet{}
 
 	if len(trackNew) == 0 {
 		return nil, nil, nil
@@ -353,7 +354,7 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
 	// go backwards and merge vδT <- treediff(lo..hi/trackNew)
 	vδZ := δZtail.Data()
 	for {
-		δtkeycov := &RangedKeySet{}     // all keys coming into tracking set during this lo<-hi scan
+		δtkeycov := &blib.RangedKeySet{}     // all keys coming into tracking set during this lo<-hi scan
 		trackNewCur := trackNew.Clone() // trackNew adjusted as of when going to i<- entry
 		for i := len(vδZ)-1; i>=0; i-- {
 			δZ := vδZ[i]
@@ -411,7 +412,7 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
 
 // widenTrackNew widens trackNew to cover δtkeycov.
 // XXX -> widenTrackSet?
-func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid, at zodb.Tid, db *zodb.DB) (err error) {
+func widenTrackNew(trackNew blib.PPTreeSubSet, δtkeycov *blib.RangedKeySet, root zodb.Oid, at zodb.Tid, db *zodb.DB) (err error) {
 	// XXX errctx, debug
 	defer xerr.Contextf(&err, "widenTrackNew tree<%s> @%s +%s", root, at, δtkeycov)
 
@@ -430,13 +431,13 @@ func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid
 	top := &nodeInRange{prefix: nil, lo: KeyMin, hi_: KeyMax, node: tree}
 	V   := rangeSplit{top}
 	for _, r := range δtkeycov.AllRanges() {
-		lo := r.lo
+		lo := r.Lo
 		for {
 			b, err := V.GetToLeaf(ctx, lo);  /*X*/ if err != nil { return err }
 			trackNew.AddPath(b.Path())
 
 			// continue with next right bucket until r coverage is complete
-			if r.hi_ <= b.hi_ {
+			if r.Hi_ <= b.hi_ {
 				break
 			}
 			lo = b.hi_ + 1
@@ -450,7 +451,7 @@ func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid
 //
 // δtrackNew/δtkeycov represents how trackNew changes when going through `atPrev <- δZ.Rev` .
 // newRevEntry indicates whether δZ.Rev was not there before in .vδT and new corresponding δT entry was created.
-func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew PPTreeSubSet, db *zodb.DB) (δtrackNew *ΔPPTreeSubSet, δtkeycov *RangedKeySet, newRevEntry bool, err error) {
+func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew blib.PPTreeSubSet, db *zodb.DB) (δtrackNew *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, newRevEntry bool, err error) {
 	defer xerr.Contextf(&err, "rebuild1 %s<-%s", atPrev, δZ.Rev)
 
 	debugfΔBtail("\n  rebuild1 @%s <- @%s\n", atPrev, δZ.Rev)
@@ -467,7 +468,7 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
 
 	// skip opening DB connections if there is no change to this tree
 	if len(δtopsByRoot) == 0 {
-		return NewΔPPTreeSubSet(), &RangedKeySet{}, false, nil
+		return blib.NewΔPPTreeSubSet(), &blib.RangedKeySet{}, false, nil
 	}
 
 	if len(δtopsByRoot) != 1 {
@@ -560,7 +561,7 @@ func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
 
 		// δtkeycov1 != ø   ->  rebuild δTtail with trackNew ~= δtkeycov1
 		if !δT1.δtkeycov1.Empty() && δBtail.δZtail.Len() > 1 {
-			trackNew := PPTreeSubSet{}
+			trackNew := blib.PPTreeSubSet{}
 			err := widenTrackNew(trackNew, δT1.δtkeycov1, root, δBtail.Head(), δBtail.db)
 			if err != nil {
 				return ΔB{}, err
@@ -611,8 +612,8 @@ type _ΔBUpdate1 struct {
 	ByRoot map[zodb.Oid]*_ΔTUpdate1
 }
 type _ΔTUpdate1 struct {
-	δtkeycov1 *RangedKeySet   // {} root -> δtrackedKeys after first treediff (always grow)
-	δtrack    *ΔPPTreeSubSet  // XXX kill (not used)
+	δtkeycov1 *blib.RangedKeySet   // {} root -> δtrackedKeys after first treediff (always grow)
+	δtrack    *blib.ΔPPTreeSubSet  // XXX kill (not used)
 }
 func (δBtail *ΔBtail) _Update1(δZ *zodb.EventCommit) (δB1 _ΔBUpdate1, err error) {
 	headOld := δBtail.Head()

wcfs/internal/xbtree/δbtail_x_test.go

@@ -18,85 +18,7 @@
 // See https://www.nexedi.com/licensing for rationale and options.
 
 package xbtree_test
-// ZBlk-related part of δbtail_test
 
 import (
-	"context"
-	"fmt"
-
-	"lab.nexedi.com/kirr/go123/xerr"
-	"lab.nexedi.com/kirr/neo/go/zodb"
-
-	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree"
-	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
-	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
+	_ "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest/init"
 )
-
-type Tree = xbtree.Tree
-type Node = xbtree.Node
-type Key  = xbtree.Key
-
-type ZBlk = zdata.ZBlk
-
-// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
-// XXX +return blkRevMax and use it ?
-func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
-	path = []Node{}
-	xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
-		path = append(path, node)
-	})
-	if err != nil {
-		return nil, false, nil, err
-	}
-
-	if ok {
-		zblk, ok = xzblk.(ZBlk)
-		if !ok {
-			return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk))  // XXX errctx
-		}
-	}
-
-	return zblk, ok, path, nil
-}
-
-
-func init() {
-	xbtree.ZTreeGetBlkData = ZTreeGetBlkData
-	xbtree.ZGetBlkData     = ZGetBlkData
-}
-
-// ZTreeGetBlkData returns block data from block pointed to by ztree[k].
-func ZTreeGetBlkData(ctx context.Context, ztree *Tree, k Key) (data string, ok bool, path []Node, err error) {
-	defer xerr.Contextf(&err, "@%s: tree<%s>: get blkdata from [%d]", ztree.PJar().At(), ztree.POid(), k)
-
-	zblk, ok, path, err := ztreeGetBlk(ctx, ztree, k)
-	if err != nil || !ok {
-		return "", ok, path, err
-	}
-
-	bdata, _, err := zblk.LoadBlkData(ctx)
-	if err != nil {
-		return "", false, nil, err
-	}
-
-	return string(bdata), true, path, nil
-}
-
-// ZGetBlkData loads block data from ZBlk object specified by its oid.
-func ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
-	defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
-
-	xblk, err := zconn.Get(ctx, zblkOid)
-	if err != nil {
-		return "", err
-	}
-	zblk, ok := xblk.(ZBlk)
-	if !ok {
-		return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
-	}
-	bdata, _, err := zblk.LoadBlkData(ctx)
-	if err != nil {
-		return "", err
-	}
-	return string(bdata), nil
-}