wcfs: tests: Tree-based testing environment

Add treeenv.go that combines Treegen and client side access to ZODB with
committed trees as extension to testing.T . The environment allows to
easily see which tree update was committed, what is the difference in
terms of KV, what is the state of updated tree and state of pointed-to
ZBlk objects.

This will be used to test upcoming ΔBtail and ΔFtail.

Main functionality is in treeenv.go; the other added files are to
support that.

Some preliminary history:

kirr/wendelin.core@f07502fc    X xbtreetest: Teach T & Commit to automatically provide At in symbolic form
kirr/wendelin.core@0d62b05e    X Adjust to btree.VGet & friends signature change to include keycov in visit callback
kirr/wendelin.core@588a512a    X zdata: Switch SliceByFileRev not to clone Zinblk
kirr/wendelin.core@e9c4b619    X rebuild: tests: Random testing
kirr/wendelin.core@43090ac7    X tests: Factor-out tree-test-env into tTreeEnv
kirr/wendelin.core@d4a523b2    X δbtail: tests: Run much faster with live ZODB cache
kirr/wendelin.core@271d953d    X rebuild: tests: Move ΔBtail.Clone test out of hot inner loop into separate test
kirr/wendelin.core@c32055fc    X wcfs/xbtree: ΔBtail tests += ø -> Tree; Tree -> ø
kirr/wendelin.core@5324547c    X wcfs/xbtree: root(a) must stay in trackSet even after treediff(a,ø)
kirr/wendelin.core@8f6e2b1e    X rebuild: tests: Don't access ZODB in XGetδKV

wcfs/internal/xbtree/xbtreetest/init/init.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package xbtreetest/init (ex imported from package A_test) should be imported
+// in addition to xbtreetest (ex imported from package A) to initialize
+// xbtreetest at runtime.
+package init
+
+// ZBlk-related part of xbtreetest
+
+import (
+	"context"
+	"fmt"
+
+	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
+)
+
+type Tree     = xbtreetest.Tree
+type Node     = xbtreetest.Node
+type Key      = xbtreetest.Key
+type KeyRange = xbtreetest.KeyRange
+
+type ZBlk = zdata.ZBlk
+
+
+func init() {
+	xbtreetest.ZGetBlkData = _ZGetBlkData
+}
+
+
+// _ZGetBlkData loads block data from ZBlk object specified by its oid.
+func _ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
+	defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
+
+	xblk, err := zconn.Get(ctx, zblkOid)
+	if err != nil {
+		return "", err
+	}
+	zblk, ok := xblk.(ZBlk)
+	if !ok {
+		return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
+	}
+	bdata, _, err := zblk.LoadBlkData(ctx)
+	if err != nil {
+		return "", err
+	}
+	return string(bdata), nil
+}

wcfs/internal/xbtree/xbtreetest/kvdiff.go

@@ -18,6 +18,7 @@
 // See https://www.nexedi.com/licensing for rationale and options.
 
 package xbtreetest
+// kvdiff + friends
 
 import (
 	"fmt"
@@ -25,6 +26,31 @@ import (
 	"strings"
 )
 
+// KVDiff returns difference in between kv1 and kv2.
+const DEL = "ø" // DEL means deletion
+type Δstring struct {
+	Old string
+	New string
+}
+func KVDiff(kv1, kv2 map[Key]string) map[Key]Δstring {
+	delta := map[Key]Δstring{}
+	keys := setKey{}
+	for k := range kv1 { keys.Add(k) }
+	for k := range kv2 { keys.Add(k) }
+
+	for k := range keys {
+		v1, ok := kv1[k]
+		if !ok { v1 = DEL }
+		v2, ok := kv2[k]
+		if !ok { v2 = DEL }
+		if v1 != v2 {
+			delta[k] = Δstring{v1,v2}
+		}
+	}
+
+	return delta
+}
+
 // KVTxt returns string representation of {} kv.
 func KVTxt(kv map[Key]string) string {
 	if len(kv) == 0 {

wcfs/internal/xbtree/xbtreetest/kvdiff_test.go

@@ -20,9 +20,20 @@
 package xbtreetest
 
 import (
+	"reflect"
 	"testing"
 )
 
+func TestKVDiff(t *testing.T) {
+	kv1 := map[Key]string{1:"a", 3:"c", 4:"d"}
+	kv2 := map[Key]string{1:"b",        4:"d", 5:"e"}
+	got  := KVDiff(kv1, kv2)
+	want := map[Key]Δstring{1:{"a","b"}, 3:{"c",DEL}, 5:{DEL,"e"}}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("error:\ngot:  %v\nwant: %v", got, want)
+	}
+}
+
 func TestKVTxt(t *testing.T) {
 	kv := map[Key]string{3:"hello", 1:"zzz", 4:"world"}
 	got  := KVTxt(kv)

wcfs/internal/xbtree/xbtreetest/rtree.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+// RBucket + RBucketSet
+
+import (
+	"fmt"
+	"sort"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
+)
+
+// RBucketSet represents set of buckets covering whole [-∞,∞) range.
+type RBucketSet []*RBucket // k↑
+
+// RBucket represents Bucket node with values covering [lo, hi_] key range in its Tree.
+// NOTE it is not [lo,hi) but [lo,hi_] instead to avoid overflow at KeyMax.
+type RBucket struct {
+	Oid     zodb.Oid
+	Parent  *RTree
+	Keycov  blib.KeyRange
+	KV      map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
+}
+
+// RTree represents Tree node that RBucket refers to as parent.
+type RTree struct {
+	Oid    zodb.Oid
+	Parent *RTree
+}
+
+
+// Path returns path to this bucket from tree root.
+func (rb *RBucket) Path() []zodb.Oid {
+	path := []zodb.Oid{rb.Oid}
+	p := rb.Parent
+	for p != nil {
+		path = append([]zodb.Oid{p.Oid}, path...)
+		p = p.Parent
+	}
+	return path
+}
+
+// Get returns RBucket which covers key k.
+func (rbs RBucketSet) Get(k Key) *RBucket {
+	i := sort.Search(len(rbs), func(i int) bool {
+		return k <= rbs[i].Keycov.Hi_
+	})
+	if i == len(rbs) {
+		panicf("BUG: key %v not covered;  coverage: %s", k, rbs.coverage())
+	}
+
+	rb := rbs[i]
+	if !rb.Keycov.Has(k) {
+		panicf("BUG: get(%v) -> %s;  coverage: %s", k, rb.Keycov, rbs.coverage())
+	}
+
+	return rb
+}
+
+// coverage returns string representation of rbs coverage structure.
+func (rbs RBucketSet) coverage() string {
+	if len(rbs) == 0 {
+		return "ø"
+	}
+	s := ""
+	for _, rb := range rbs {
+		if s != "" {
+			s += " "
+		}
+		s += fmt.Sprintf("%s", rb.Keycov)
+	}
+	return s
+}
+
+// Flatten converts xkv with bucket structure into regular dict.
+func (xkv RBucketSet) Flatten() map[Key]string {
+	kv := make(map[Key]string)
+	for _, b := range xkv {
+		for k,v := range b.KV {
+			kv[k] = v
+		}
+	}
+	return kv
+}
+
+func (b *RBucket) String() string {
+	return fmt.Sprintf("%sB%s{%s}", b.Keycov, b.Oid, KVTxt(b.KV))
+}

wcfs/internal/xbtree/xbtreetest/xbtreetest.go

@@ -23,6 +23,9 @@ package xbtreetest
 import (
 	"fmt"
 
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
 )
 
@@ -38,6 +41,10 @@ type KeyRange = blib.KeyRange
 const KeyMax  = blib.KeyMax
 const KeyMin  = blib.KeyMin
 
+type setKey = set.I64
+
+const VDEL  = zodb.InvalidOid
+
 
 func panicf(format string, argv ...interface{}) {
 	panic(fmt.Sprintf(format, argv...))

wcfs/internal/xbtree/xbtreetest/zdata.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+// access to ZBlk data
+
+import (
+	"context"
+
+	"lab.nexedi.com/kirr/go123/exc"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
+)
+
+// ZBlk-related functions are imported at runtime by package xbtreetest/init
+var (
+	ZGetBlkData func(context.Context, *zodb.Connection, zodb.Oid) (string, error)
+)
+
+func zassertInitDone() {
+	if ZGetBlkData == nil {
+		panic("xbtreetest/zdata not initialized -> import xbtreetest/init to fix")
+	}
+}
+
+// xzgetBlkData loads block data from ZBlk object specified by its oid.
+func xzgetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) string {
+	zassertInitDone()
+	X := exc.Raiseif
+
+	if zblkOid == VDEL {
+		return DEL
+	}
+
+	data, err := ZGetBlkData(ctx, zconn, zblkOid); X(err)
+	return string(data)
+}