// Copyright (C) 2020  Nexedi SA and Contributors.
//                     Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.

package main
// tests for δbtail.go
// XXX doc  (2 ways of testing: explicit + allstructs), treegen py helper

// Adjacency matrix
//
// A,  B  - topologies			ex T/B1    T2/B1-B3
// Av, Bv - topologies with values	ex T/B1:a  T2/B1:b-B3:c
//
// δ(Av, Bv) - full diff {k -> v} for changed keys; DEL = k -> ø
//             ex δ(T/B1:a, T2/B1:b-B3:c) = {1:b 3:c}		XXX include "was" value too?
//
// Δ(T, Av, Bv) - subset of δ(Av, Bv) corresponding to initial tracking set T
//             ex Δ({1}, T/B1:a, T2/B1:b-B3:c) = {1:b}  (no 3:c)
//
// kadj(A,B) {} k -> {k'}:	- adjacency matrix
//	∃v1,v2:  k'∈ Δ({k}, Av1, Bv2)
//
//	ex kadj(T/B1, T2/B1-B3) = {1:{1} 3:{1,3} ∞:{1,3}}	k ∈ A+B+{∞}
//                                + {0:{1} 2:{1,3} + ... all possible keys}
//
// Δ(T, Av, Bv) = δ(Av, Bv)/kadj(A,B)[T]
//
// i.e. = δ(Av, Bv) for k: k ∈ U kadj(A,B)[·]
//                            ·∈T

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"reflect"
	"regexp"
	"sort"
	"strings"
	"testing"

	"lab.nexedi.com/kirr/go123/exc"
	"lab.nexedi.com/kirr/go123/xerr"
	"lab.nexedi.com/kirr/neo/go/transaction"
	"lab.nexedi.com/kirr/neo/go/zodb"
)

const kInf Key = 10000 // inf key (TeX hack)
//const kInf Key = KeyMax

// XXX move infrastructure -> δbtail_treegen_test.go ?

// TreeGenSrv represents connection to running `treegen ...` server.
type TreeGenSrv struct {
	argv      []string
	pysrv     *exec.Cmd      // spawned `treegen ...`
	pyin      io.WriteCloser // input to pysrv
	pyoutRaw  io.ReadCloser  // output from pysrv
	pyout     *bufio.Reader  // buffered ^^^
}

// TreeSrv represents connection to running `treegen trees` server.
//
// Create it with StartTreeSrv(zurl).
// - Commit(treeTopology) -> tid
type TreeSrv struct {
	*TreeGenSrv
	zurl	  string
	treeRoot  zodb.Oid       // oid of the tree treegen works on
	head      zodb.Tid       // last made commit
}

// AllStructsSrv represents connection to running `treegen allstructs` server.
//
// Create it with StartAllStructsSrv().
// - AllStructs(maxdepth, maxsplit, n, seed, kv1, kv2)
type AllStructsSrv struct {
	*TreeGenSrv
}

// StartTreeGenSrv spawns `treegen ...` server.
func StartTreeGenSrv(argv ...string) (_ *TreeGenSrv, hello string, err error) {
	defer xerr.Contextf(&err, "treesrv %v: start", argv)

	// spawn `treegen ...`
	tg := &TreeGenSrv{argv: argv}
	tg.pysrv = exec.Command("./testprog/treegen.py", argv...)
	tg.pyin, err = tg.pysrv.StdinPipe()
	if err != nil {
		return nil, "", err
	}
	tg.pyoutRaw, err = tg.pysrv.StdoutPipe()
	if err != nil {
		return nil, "", err
	}
	tg.pyout = bufio.NewReader(tg.pyoutRaw)
	tg.pysrv.Stderr = os.Stderr // no redirection

	err = tg.pysrv.Start()
	if err != nil {
		return nil, "", err
	}

	// wait for hello message and return it
	defer func() {
		if err != nil {
			tg.Close() // ignore error
		}
	}()
	defer xerr.Context(&err, "handshake")
	hello, err = tg.pyout.ReadString('\n')
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return nil, "", err
	}
	hello = strings.TrimSuffix(hello, "\n")

	return tg, hello, nil
}

// Close shutdowns treegen server.
func (tg *TreeGenSrv) Close() (err error) {
	defer xerr.Contextf(&err, "treegen %v: close", tg.argv)

	err1 := tg.pyin.Close()
	err2 := tg.pyoutRaw.Close()
	err3 := tg.pysrv.Wait()
	return xerr.Merge(err1, err2, err3)
}


// StartTreeSrv spawns `treegen trees` server.
func StartTreeSrv(zurl string) (_ *TreeSrv, err error) {
	defer xerr.Contextf(&err, "tree.srv %s: start", zurl)
	tgSrv, hello, err := StartTreeGenSrv("trees", zurl)
	if err != nil {
		return nil, err
	}

	tg := &TreeSrv{TreeGenSrv: tgSrv, zurl: zurl}
	defer func() {
		if err != nil {
			tgSrv.Close() // ignore error
		}
	}()

	// tree.srv start @<at> tree=<root>
	defer xerr.Contextf(&err, "invalid hello %q", hello)
	startRe := regexp.MustCompile(`^tree.srv start @([^ ]+) root=([^ ]+)$`)
	m := startRe.FindStringSubmatch(hello)
	if m == nil {
		return nil, fmt.Errorf("unexpected format")
	}
	tg.head, err = zodb.ParseTid(m[1]) // <at>
	if err != nil {
		return nil, fmt.Errorf("tid: %s", err)
	}
	tg.treeRoot, err = zodb.ParseOid(m[2]) // <root>
	if err != nil {
		return nil, fmt.Errorf("root: %s", err)
	}

	return tg, nil
}

// StartAllStructsSrv spawns `treegen allstructs` server.
func StartAllStructsSrv() (_ *AllStructsSrv, err error) {
	defer xerr.Context(&err, "allstructs.srv: start")

	tgSrv, hello, err := StartTreeGenSrv("allstructs")
	if err != nil {
		return nil, err
	}

	sg := &AllStructsSrv{TreeGenSrv: tgSrv}
	defer func() {
		if err != nil {
			tgSrv.Close() // ignore error
		}
	}()

	defer xerr.Contextf(&err, "invalid hello %q", hello)
	if hello != "# allstructs.srv start" {
		return nil, fmt.Errorf("unexpected format")
	}

	return sg, nil
}


// Commit creates new commit with underlying tree changed to specified tree topology.
func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
	defer xerr.Contextf(&err, "tree.srv %s: commit %s", tg.zurl, tree)

	_, err = io.WriteString(tg.pyin, tree + "\n")
	if err != nil {
		return zodb.InvalidTid, err
	}

	reply, err := tg.pyout.ReadString('\n')
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return zodb.InvalidTid, err
	}
	reply = strings.TrimSuffix(reply, "\n")

	tid, err := zodb.ParseTid(reply)
	if err != nil {
		return zodb.InvalidTid, fmt.Errorf("invalid reply: %s", err)
	}
	tg.head = tid
	return tid, nil
}

// AllStructs returns response from `treegen allstructs`
func (tg *AllStructsSrv) AllStructs(kv1, kv2 map[Key]string, maxdepth, maxsplit, n int, seed int64) (_ []string, err error) {
	req := fmt.Sprintf("%d %d %d/%d %s %s", maxdepth, maxsplit, n, seed, kvtxt(kv1), kvtxt(kv2))
	defer xerr.Contextf(&err, "allstructs.srv: %s", req)

	_, err = io.WriteString(tg.pyin, req + "\n")
	if err != nil {
		return nil, err
	}

	structv := []string{}
	for {
		reply, err := tg.pyout.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}
			return nil, err
		}
		reply = strings.TrimSuffix(reply, "\n")

		if reply == "# ----" {
			return structv, nil // end of response
		}
		if strings.HasPrefix(reply, "#") {
			continue // comment
		}

		structv = append(structv, reply)
	}

}

// RTree represents Tree node covering [lo, hi_] key range in its parent tree.
type RTree struct {
	oid    zodb.Oid
	parent *RTree
	// XXX +children?
}

// RBucket represents Bucket node covering [lo, hi_] key range in its Tree.
// NOTE it is not [lo,hi) but [lo,hi_] instead to avoid overflow at KeyMax.
type RBucket struct {
	oid     zodb.Oid
	parent  *RTree
	lo, hi_ Key
	kv      map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
}

// RBucketSet represents set of buckets covering whole [-∞,∞) range.
type RBucketSet []*RBucket // k↑


// Get returns RBucket which covers key k.
func (rbs RBucketSet) Get(k Key) *RBucket {
	i := sort.Search(len(rbs), func(i int) bool {
		return k <= rbs[i].hi_
	})
	if i == len(rbs) {
		panicf("BUG: key %v not covered;  coverage: %s", k, rbs.coverage())
	}

	rb := rbs[i]
	if !(rb.lo <= k && k <= rb.hi_) {
		panicf("BUG: get(%v) -> [%v, %v];  coverage: %s", k, rb.lo, rb.hi_, rbs.coverage())
	}

	return rb
}

// coverage returns string representation of rbs coverage structure.
func (rbs RBucketSet) coverage() string {
	if len(rbs) == 0 {
		return "ø"
	}
	s := ""
	for _, rb := range rbs {
		if s != "" {
			s += " "
		}
		s += fmt.Sprintf("[%v, %v]", rb.lo, rb.hi_)
	}
	return s
}

// holeIdx returns what should be ΔBtree.holeIdx for specified tracked key set.
func (rbs RBucketSet) holeIdx(tracked SetKey) SetKey {
	holes := SetKey{}
	for k := range tracked {
		_, keyin := rbs.Get(k).kv[k]
		if !keyin {
			holes.Add(k)
		}
	}
	return holes
}

// trackIdx returns what should be ΔBtree.trackIdx for specified tracked key set.
func (rbs RBucketSet) trackIdx(tracked SetKey) trackIndex {
	trackIdx := trackIndex{}
	for k := range tracked {
		kb := rbs.Get(k)
		// trackIdx explicitly records only regular buckets.
		// embedded buckets all have oid=zodb.InvalidOid and would lead to z
		newNode := false
		if kb.oid != zodb.InvalidOid {
			track, already := trackIdx[kb.oid]
			if !already {
				track = &nodeTrack{parent: kb.parent.oid, nchild: 0}
				trackIdx[kb.oid] = track
				newNode = true
			}
			if track.parent != kb.parent.oid {
				panicf("BUG: %s: B%s -> multiple parents: %s %s", rbs.coverage(), kb.oid, track.parent, kb.parent.oid)
			}
		}

		p := kb.parent
		for p != nil {
			ppoid := zodb.InvalidOid // oid of p.parent
			if p.parent != nil {
				ppoid = p.parent.oid
			}

			pt, already := trackIdx[p.oid]
			newParent := false
			if !already {
				pt = &nodeTrack{parent: ppoid, nchild: 0}
				trackIdx[p.oid] = pt
				newParent = true
			}
			if pt.parent != ppoid {
				panicf("BUG: %s: T%s -> multiple parents: %s %s", rbs.coverage(), p.oid, pt.parent, ppoid)
			}

			if newNode {
				pt.nchild++
			}
			p = p.parent
			newNode = newParent
		}
	}
	return trackIdx
}



// XGetTree loads Tree from zurl@at->obj<root>.
// Tree values must be ZBlk whose data is returned instead of references to ZBlk objects.
// The tree is returned structured by buckets as
//
//	[] [lo,hi){k->v}  k↑
func XGetTree(db *zodb.DB, at zodb.Tid, root zodb.Oid) RBucketSet {
	defer exc.Contextf("%s: @%s: get tree %s", db.Storage().URL(), at, root)
	X := exc.Raiseif

	txn, ctx := transaction.New(context.Background())
	defer txn.Abort()
	zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)

	xztree, err := zconn.Get(ctx, root); X(err)
	ztree, ok := xztree.(*Tree)
	if !ok {
		exc.Raisef("expected %s, got %s", typeOf(ztree), typeOf(xztree))
	}

	rbucketv := RBucketSet{}
	xwalkDFS(ctx, -kInf, kInf, ztree, func(rb *RBucket) {
		rbucketv = append(rbucketv, rb)
	})
	if len(rbucketv) == 0 { // empty tree -> [-∞, ∞){}
		etree   := &RTree{
				oid:    root,
				parent: nil,
		}
		ebucket := &RBucket{
				oid:    zodb.InvalidOid,
				parent: etree,
				lo:     -kInf,
				hi_:    kInf,
				kv:     map[Key]string{},
		}
		rbucketv = RBucketSet{ebucket}
	}
	return rbucketv
}

// xwalkDFS walks ztree in depth-first order emitting bvisit callback on visited bucket nodes.
func xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, bvisit func(*RBucket)) {
	_xwalkDFS(ctx, lo, hi_, ztree, /*rparent*/nil, bvisit)
}
func _xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, rparent *RTree, bvisit func(*RBucket)) {
	X := exc.Raiseif

	err := ztree.PActivate(ctx); X(err)
	defer ztree.PDeactivate()

	rtree := &RTree{oid: ztree.POid(), parent: rparent}

	// [i].Key ≤ [i].Child.*.Key < [i+1].Key  i ∈ [0, len([]))
	//
	// [0].Key       = -∞ ; always returned so
	// [len(ev)].Key = +∞ ; should be assumed so
	ev := ztree.Entryv()
	for i := range ev {
		xlo  := lo;  if i   > 0       { xlo  = ev[i].Key() }
		xhi_ := hi_; if i+1 < len(ev) { xhi_ = ev[i+1].Key() - 1 }

		tchild, ok := ev[i].Child().(*Tree)
		if ok {
			_xwalkDFS(ctx, xlo, xhi_, tchild, rtree, bvisit)
			continue
		}

		zbucket := ev[i].Child().(*Bucket)
		err = zbucket.PActivate(ctx); X(err)
		defer zbucket.PDeactivate()

		bkv := make(map[Key]string)
		bentryv := zbucket.Entryv()

		for _, __ := range bentryv {
			k  := __.Key()
			xv := __.Value()
			zv, ok := xv.(zBlk)
			if !ok {
				exc.Raisef("[%d] -> %s;  want ZBlk", k, typeOf(xv))
			}

			data, _, err := zv.loadBlkData(ctx); X(err)
			bkv[k] = string(data)
		}


		b := &RBucket{oid: zbucket.POid(), parent: rtree, lo: xlo, hi_: xhi_, kv: bkv}
		bvisit(b)
	}
}

// XGetδKV translates {k -> δ<oid>} to {k -> δ(ZBlk(oid).data)} according to db@at1..at2 snapshots.
func XGetδKV(db *zodb.DB, at1, at2 zodb.Tid, δkvOid map[Key]ΔValue) map[Key]Δstring {
	defer exc.Contextf("%s: get δkv %s..%s %v", db.Storage().URL(), at1, at2, δkvOid)
	X := exc.Raiseif

	txn, ctx := transaction.New(context.Background())
	defer txn.Abort()
	zconn1, err := db.Open(ctx, &zodb.ConnOptions{At: at1}); X(err)
	zconn2, err := db.Open(ctx, &zodb.ConnOptions{At: at2}); X(err)

	δkv := map[Key]Δstring{}
	for k, δvOid := range δkvOid {
		δkv[k] = Δstring{
			Old: xzgetBlkData(ctx, zconn1, δvOid.Old),
			New: xzgetBlkData(ctx, zconn2, δvOid.New),
		}
	}

	return δkv
}

// xzgetBlkData loads block data from ZBlk object specified by its oid.
func xzgetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) string {
	X := exc.Raiseif

	if zblkOid == VDEL {
		return DEL
	}

	xblk, err := zconn.Get(ctx, zblkOid); X(err)
	zblk := xblk.(zBlk)
	data, _, err := zblk.loadBlkData(ctx); X(err)
	return string(data)
}


// xverifyΔBTail verifies how ΔBTail handles ZODB update for a tree with changes in between at1->at2.
//
// it is known that @at1 and @at2 the tree has xkv1 and xkv2 values correspondingly.
// it is known that for at1->at2 ZODB-level change is δZ.
//
// kadjOK may be optionally provided. if kadjOK != nil computed adjacency
// matrix is verified against it.
func xverifyΔBTail(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, at1, at2 zodb.Tid, xkv1, xkv2 RBucketSet, δZ *zodb.EventCommit, kadjOK map[Key]SetKey) {
	// verify transition at1->at2 for all initial states of tracked {keys} from kv1 + kv2 + ∞
	allKeys := SetKey{}; allKeys.Add(kInf) // ∞ simulating ZBigFile.Size() query
	maxk1 := -kInf; maxk2 := -kInf
	for _, b := range xkv1 { for k := range b.kv {
		allKeys.Add(k)
		if k > maxk1 { maxk1 = k }
	}}
	for _, b := range xkv2 { for k := range b.kv {
		allKeys.Add(k)
		if k > maxk2 { maxk2 = k }
	}}
	allKeyv := allKeys.Elements()
	sort.Slice(allKeyv, func(i, j int) bool {
		return allKeyv[i] < allKeyv[j]
	})


	// kadj = {} k -> adjacent keys. if k is tracked -> changes to adjacents must be in δT.
	kadj := map[Key]SetKey{}
	for k := range allKeys {
		adj1 := SetKey{}
		adj2 := SetKey{}

		q1 := []Key{k}
		q2 := []Key{k}

		//tracef("\n")
		for len(q1) > 0 || len(q2) > 0 {
			//tracef("q1: %v\n", q1)
			//tracef("q2: %v\n", q2)
			if l1 := len(q1); l1 > 0 {
				k1 := q1[l1-1];  q1 = q1[:l1-1]
				if !adj1.Has(k1) {
					for k_ := range xkv1.Get(k1).kv {
						adj1.Add(k_)
						for k__ := range xkv2.Get(k_).kv {
							if !adj2.Has(k__) {
								q2 = append(q2, k__)
							}
						}
					}
				}
			}

			if l2 := len(q2); l2 > 0 {
				k2 := q2[l2-1];  q2 = q2[:l2-1]
				if !adj2.Has(k2) {
					for k_ := range xkv2.Get(k2).kv {
						adj2.Add(k_)
						for k__ := range xkv1.Get(k_).kv {
							if !adj1.Has(k__) {
								q1 = append(q1, k__)
							}
						}
					}
				}
			}
		}

		adj := SetKey{}; adj.Add(k); adj.Update(adj1); adj.Update(adj2)
		kadj[k] = adj
	}

	if kadjOK != nil && !reflect.DeepEqual(kadj, kadjOK) {
		t.Errorf("%s: BUG: computed kadj is wrong:\nkadjOK: %v\nkadj  : %v\n\n", subj, kadjOK, kadj)
	}


	// verify at1->at2 for all combination of initial tracked keys.
	for kidx := range IntSets(len(allKeyv)) {
		keys := SetKey{}
		for _, idx := range kidx {
			keys.Add(allKeyv[idx])
		}

		xverifyΔBTail1(t, subj, db, treeRoot, at1,at2, xkv1,xkv2, δZ, keys, kadj)
	}
}

// xverifyΔBTail1 verifies how ΔBTail handles ZODB update at1->at2 from initial
// tracked state defined by initialTrackedKeys.
// XXX initialTrackedKeys -> tracked1 ?
func xverifyΔBTail1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, at1,at2 zodb.Tid, xkv1,xkv2 RBucketSet, δZ *zodb.EventCommit, initialTrackedKeys SetKey, kadj map[Key]SetKey) {
	X := exc.Raiseif
	tracef("\n>>> Track=%s\n", initialTrackedKeys)

	d12 := kvdiff(xkvFlatten(xkv1), xkvFlatten(xkv2))

	var kadjTracked SetKey = nil
	var δT, δTok map[Key]Δstring = nil, nil

	// badf queues error message to be reported on return.
	var badv []string
	badf := func(format string, argv ...interface{}) {
		badv = append(badv, fmt.Sprintf(format, argv...))
	}
	defer func() {
		if badv != nil {
			emsg := fmt.Sprintf("%s  ; tracked=%v :\n\n", subj, initialTrackedKeys)
			emsg += fmt.Sprintf("d12:  %v\nδTok: %v\nδT:   %v\n\n", d12, δTok, δT)
			emsg += fmt.Sprintf("kadj[Tracked]: %v\nkadj: %v\n\n", kadjTracked, kadj)
			emsg += strings.Join(badv, "\n")
			emsg += "\n"

			t.Fatal(emsg)
		}
	}()


	txn, ctx := transaction.New(context.Background())
	defer txn.Abort()

	// zconn, δbtail @at1 with initial tracked set
	zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at1});	X(err)
	δbtail := NewΔBtail(zconn.At(), db)

	xtree, err := zconn.Get(ctx, treeRoot);	X(err)
	ztree := xtree.(*Tree)

	kadjTracked = SetKey{} // kadj[Tracked]  (all keys adjacent to tracked keys)
	for k := range initialTrackedKeys {
		path := []Node{}
		_, ok, err := ztree.VGet(ctx, k, func(node Node) {
			path = append(path, node)
		}); X(err)

		err = δbtail.Track(k, ok, path); X(err)

		kadjTracked.Update(kadj[k])
	}

	// assert Tracked ∈ kadj[Tracked]  except ∞
	trackedNotInKadj := SetKey{}
	for k := range initialTrackedKeys {
		if !kadjTracked.Has(k) {
			trackedNotInKadj.Add(k)
		}
	}
	trackedNotInKadj.Del(kInf)
	if len(trackedNotInKadj) > 0 {
		badf("BUG: Tracked ∉ kadj[Tracked]  ; extra=%v", trackedNotInKadj)
		return
	}

	//              k ∈ d12
	// k ∈ δT  <=>
	//              k ∈   U kadj[·]
	//                   ·∈tracking
	δTok = map[Key]Δstring{} // d12[all keys that should be present in δT]
	for k,δv := range d12 {
		if kadjTracked.Has(k) {
			δTok[k] = δv
		}
	}

	// verify δbtail.holeIdx  against @at1
	// holes1 = tracked1 \ kv1
	holes1 := xkv1.holeIdx(initialTrackedKeys)
	holeIdx := δbtail.holeIdxFor(treeRoot)
	if !reflect.DeepEqual(holes1, holeIdx.SetKey) {
		badf("δbtail.holeIdx1 wrong  ; holeIdx=%v  holeIdxOK=%v", holeIdx, holes1)
	}

	// verify δbtail.trackIdx against @at1
	// trackIdx1 = xkv1[tracked1]
	trackIdx1 := xkv1.trackIdx(initialTrackedKeys)
	if !reflect.DeepEqual(trackIdx1, δbtail.trackIdx) {
		badf("δbtail.trackIdx1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackIdx, trackIdx1)
	}


	// δB <- δZ
	δB, err := δbtail.Update(δZ); X(err)
	if δB.Rev != δZ.Tid {
		badf("δB: rev != δZ.Tid  ; rev=%s  δZ.Tid=%s", δB.Rev, δZ.Tid)
		return
	}

	// verify δbtail.holeIdx  against @at2
	// holes2 = tracked2 \ kv2  ( = kadj[tracked1] \ kv2)
	holes2 := xkv2.holeIdx(kadjTracked)
	if !reflect.DeepEqual(holes2, holeIdx.SetKey) {
		badf("δbtail.holeIdx2 wrong  ; holeIdx=%v  holeIdxOK=%v", holeIdx, holes2)
	}

	// verify δbtail.trackIdx  against @at2
	// trackIdx2 = xkv2[tracked2]  ( = xkv2[kadj[tracked1]]
	trackIdx2 := xkv2.trackIdx(kadjTracked)
	if !reflect.DeepEqual(trackIdx2, δbtail.trackIdx) {
		badf("δbtail.trackIdx2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackIdx, trackIdx2)
	}


	// assert δB.ByRoot == {treeRoot -> ...}  if δTok != ø
	//                  == ø                  if δTok == ø
	rootsOK := SetOid{}
	if len(δTok) > 0 {
		rootsOK.Add(treeRoot)
	}
	roots := SetOid{}
	for root := range δB.ByRoot {
		roots.Add(root)
	}
	if !reflect.DeepEqual(roots, rootsOK) {
		badf("δB: roots != rootsOK  ; roots=%v  rootsOK=%v", roots, rootsOK)
	}
	_, inδB := δB.ByRoot[treeRoot]
	if !inδB {
		return
	}


	// δT <- δB
	δToid := δB.ByRoot[treeRoot]         // {} k -> δoid
	δT     = XGetδKV(db, at1,at2, δToid) // {} k -> δ(ZBlk(oid).data)

	// δT must be subset of d12.
	// changed keys, that are
	// - in tracked set	 -> must be present in δT
	// - outside tracked set -> may  be present in δT  (kadj gives exact answer)

	// δT is subset of d12
	for _, k := range sortedKeys(δT) {
		_, ind12 := d12[k]
		if !ind12 {
			badf("δT[%v] ∉  d12", k)
		}
	}

	// k ∈ tracked set ->  must be present in δT
	// k ∉ tracked set ->  may  be present in δT  (kadj gives exact answer)
	for _, k := range sortedKeys(d12) {
		_, inδT   := δT[k]
		_, inδTok := δTok[k]
		if inδT && !inδTok {
			badf("δT[%v] ∉  δTok", k)
		}

		if !inδT && inδTok {
			badf("δT    ∌  δTok[%v]", k)
		}

		if inδT {
			if δT[k] != d12[k] {
				badf("δT[%v] ≠  δTok[%v]", k, k)
			}
		}
	}
}


// ΔBTestEntry represents one entry in ΔBTail tests.
type ΔBTestEntry struct {
	tree   string         // next tree topology
	kadjOK map[Key]SetKey // adjacency matrix against previous case (optional)
}

// ΔBTest converts xtest into ΔBTestEntry.
// xtest can be string|ΔBTestEntry.
func ΔBTest(xtest interface{}) ΔBTestEntry {
	var test ΔBTestEntry
	switch xtest := xtest.(type) {
	case string:
		test.tree   = xtest
		test.kadjOK = nil
	case ΔBTestEntry:
		test = xtest
	default:
		panicf("BUG: ΔBTest: bad type %T", xtest)
	}
	return test
}

// testΔBTail verifies ΔBTail on sequence of tree topologies coming from testq.
func testΔBTail(t *testing.T, testq chan ΔBTestEntry) {
	X := exc.Raiseif

	work, err := ioutil.TempDir("", "δBTail"); X(err)
	defer func() {
		err := os.RemoveAll(work); X(err)
	}()
	tg, err := StartTreeSrv(work + "/1.fs"); X(err)
	defer func() {
		err := tg.Close(); X(err)
	}()

	zstor, err := zodb.Open(context.Background(), tg.zurl, &zodb.OpenOptions{
					ReadOnly: true,
		      }); X(err)
	defer func() {
		err := zstor.Close(); X(err)
	}()

	db := zodb.NewDB(zstor)
	defer func() {
		err := db.Close(); X(err)
	}()

	// XCommitTree calls tg.Commit and returns δZ corresponding to committed transaction.
	XCommitTree := func(tree string) *zodb.EventCommit {
		defer exc.Contextf("commit %s", tree)

		watchq := make(chan zodb.Event)
		at0 := zstor.AddWatch(watchq)
		defer zstor.DelWatch(watchq)

		tid, err := tg.Commit(tree); X(err)

		if !(tid > at0) {
			exc.Raisef("treegen -> %s  ; want > %s", tid, at0)
		}

		zevent := <-watchq
		δZ := zevent.(*zodb.EventCommit)
		if δZ.Tid != tid {
			exc.Raisef("treegen -> %s  ; watchq -> %s", tid, δZ)
		}
		return δZ
	}

	at1  := tg.head
	xkv1 := XGetTree(db, at1, tg.treeRoot)
	tree1 := "ø" // initial
	for test := range testq {
		tree2 := test.tree
		δZ := XCommitTree(tree2)
		at2  := δZ.Tid
		xkv2 := XGetTree(db, at2, tg.treeRoot)

		subj := fmt.Sprintf("%s -> %s", tree1, tree2)
		tracef("\n\n\n**** %s ****\n\n", subj)
		xverifyΔBTail(t, subj, db, tg.treeRoot, at1,at2, xkv1,xkv2, δZ, test.kadjOK)

		at1   = at2
		xkv1  = xkv2
		tree1 = tree2
	}
}

// TestΔBTail verifies ΔBTail for explicitly provided tree topologies.
func TestΔBTail(t *testing.T) {
	// K is shorthand for SetKey
	K := func(keyv ...Key) SetKey {
		ks := SetKey{}
		for _, k := range keyv { ks.Add(k) }
		return ks
	}
	// oo is shorthand for kInf
	const oo = kInf
	// A is shorthand for map[Key]SetKey
	type A = map[Key]SetKey
	// Δ is shorthand for ΔBTestEntry
	Δ := func(tree string, kadjOK A) (test ΔBTestEntry) {
		test.tree   = tree
		test.kadjOK = kadjOK
		return test
	}

	// test known cases going through tree1 -> tree2 -> ...
	testv := []interface{} {
		// XXX TODO classify
		"T2/B1:a-B7:g", "T2,8/B1:a-B7:g-B9:i",

		"T2/B1:a-B2:b", "T/B1:a,2:b",
		"T2,3/B1:a-B2:b-B3:c", "T/B1:a,2:b",
		"T2,3/B1:a-B2:c-B3:c", "T/B1:a,2:b",

		"T2/B1:a-B2:c", "T2,3/B1:a-B2:c-B3:c",

		"T2/B1:a-B3:c",
		Δ("T2/T-T4/B1:b-B3:d-B99:h",
			A{1:  K(1),
			  3:  K(3),
			  99: K(3,99),
			  oo: K(3,99,oo)}),
		// XXX --------

		// tree rotation
		"T3/B2:b-B3:c,4:d",
		"T5/T3-T7/B2:a-B3:a,4:a-B6:a-B8:a",

		// XXX --------

		"T/B:",

		// +1
		Δ("T/B1:a",
			A{1:  K(1),
			  oo: K(1,oo)}),

		// +2
		Δ("T/B1:a,2:b",
			A{1:  K(1,2),
			  2:  K(1,2),
			  oo: K(1,2,oo)}),

		// -1
		Δ("T/B2:b",
			A{1:  K(1,2),
			  2:  K(1,2),
			  oo: K(1,2,oo)}),

		// 2: b->c
		Δ("T/B2:c",
			A{2:  K(2),
			  oo: K(2,oo)}),

		// +1 in new bucket (to the left)
		Δ("T2/B1:a-B2:c",
			A{1:  K(1,2),
			  2:  K(2),
			  oo: K(2,oo)}), // NOTE no ∞ -> 1, because 1 is added to the left

		// +3 in new bucket (to the right)
		Δ("T2,3/B1:a-B2:c-B3:c",
			A{1:  K(1),
			  2:  K(2),
			  3:  K(2,3),
			  oo: K(2,3,oo)}),

		// bucket split; +3 in new bucket
		"T/B1:a,2:b",
		Δ("T2/B1:a-B2:b,3:c",
			A{1:  K(1,2,3),
			  2:  K(1,2,3),
			  3:  K(1,2,3),
			  oo: K(1,2,3,oo)}),

		// bucket split; +3 in new bucket; +4 +5 in another new bucket
		// which remain not tracked unless previously accessed.
		"T/B1:a,2:b",
		Δ("T2,4/B1:a-B2:b,3:c-B4:d,5:e",
			A{1:  K(1,2,3),
			  2:  K(1,2,3),
			  3:  K(1,2,3),
			  4:  K(1,2,3,4,5),
			  5:  K(1,2,3,4,5),
			  oo: K(1,2,3,4,5,oo)}),

		// reflow of keys: even if tracked={1}, changes to all B nodes need to be rescanned:
		// +B12 forces to look in -B23 which adds -3 into δ, which
		// forces to look into +B34 and so on.
		"T2,4,6/B1:a-B2:b,3:c-B4:d,5:e-B6:f,7:g",
		Δ("T3,5,7/B1:g,2:f-B3:e,4:d-B5:c,6:b-B7:a",
			A{1:  K(1,2,3,4,5,6,7),
			  2:  K(1,2,3,4,5,6,7),
			  3:  K(1,2,3,4,5,6,7),
			  4:  K(1,2,3,4,5,6,7),
			  5:  K(1,2,3,4,5,6,7),
			  6:  K(1,2,3,4,5,6,7),
			  7:  K(1,2,3,4,5,6,7),
			  oo: K(1,2,3,4,5,6,7,oo)}),

		// depth=2; bucket split; +3 in new bucket; left T remain
		// _unchanged_ even though B under it is modified; right arm is
		// added to tracking set because 2 migrates there.
		"T/T/B1:a,2:b",
		Δ("T2/T-T/B1:a-B2:b,3:c",
			A{1:  K(1,2,3),
			  2:  K(1,2,3),
			  3:  K(1,2,3),
			  oo: K(1,2,3,oo)}),

		// depth=2; like prev. case, but additional right arm with +4
		// +5 is added which remain not tracked unless previously accessed.
		"T/T/B1:a,2:b",
		Δ("T2,4/T-T-T/B1:a-B2:b,3:c-B4:d,5:e",
			A{1:  K(1,2,3),
			  2:  K(1,2,3),
			  3:  K(1,2,3),
			  4:  K(1,2,3,4,5),
			  5:  K(1,2,3,4,5),
			  oo: K(1,2,3,4,5,oo)}),

		// depth=2; bucket split; +3 in new bucket; t0 and t1 split;
		// +right arm (T7/B45-B89) which remain not tracked unless previously accessed.
		"T/T/B1:a,2:b",
		Δ("T4/T2-T7/B1:a-B2:b,3:c-B4:d,5:e-B8:h,9:i",
			A{1:  K(1,2,3),
			  2:  K(1,2,3),
			  3:  K(1,2,3),
			  4:  K(1,2,3,4,5),
			  5:  K(1,2,3,4,5),
			  8:  K(1,2,3, 8,9),
			  9:  K(1,2,3, 8,9),
			  oo: K(1,2,3, 8,9,oo)}),


		// 2 reflow to right B neighbour; 8 split into new B; δ=ø
		"T3/B1:a,2:b-B4:d,8:h",
		"T2,5/B1:a-B2:b,4:d-B8:h", // XXX add A

		// found by AllStructs ([1] is not changed, but because B1 is
		// unlinked and 1 migrates to othe bucket, changes in that
		// other bucket must be included into δT)
		"T1,2/B0:e-B1:d-B2:g,3:a",
		"T1/B0:d-B1:d,2:d",

		// XXX the same issue as ^^^ but with depth=2
		"T1,2/T-T-T/B0:a-B1:b-B2:c,3:d",
		"T1/T-T/B0:e-B1:b,2:f",


		// XXX depth=3 (to verify recursion and selecting which tree children to follow or not)

		// XXX more

		// degenerate topology from ZODB tests
		"T4/T2-T/T-T-T6,10/B1:a-B3:b-T-T-T/T-B7:c-B11:d/B5:e",
		"T/B1:e,5:d,7:c,8:b,11:a", // -3 +8

		// was leading treegen to generate corrupt trees
		"T/T1/T-T/B0:g-B1:e,2:d,3:h",
		"T1/T-T3/B0:g-T-T/B1:e,2:d-B3:h",

		// was leading to wrongly computed trackIdx2 due to top not
		// being tracked to tree root.
		"T/T1/B0:a-B1:b",
		"T/T1/T-T/B0:c-B1:d",

		// was leading to wrongly computed trackIdx2: leaf bucket not
		// reparented to root.
		"T/T/B0:a",
		"T/B0:a",


//		"T/T/T/B1:a,2:b",
//		"T/T/B1:a",

		// found by AllStructs (trackIdx2 wrongly computed due to top
		// not being tracked to tree root)
		"T2/T1-T/B0:g-B1:b-T/B2:b,3:a",
		"T2/T1-T/T-T-B2:a/B0:c-B1:g",

		// found by AllStructs (unchanged node is reparented)
		"T1/B0:c-B1:f",
		"T1/T-T/B0:c-T/B1:h",

		// found by AllStructs (SIGSEGV in ApplyΔ)
		"T1/T-T2/T-B1:c-B2:c/B0:g",
		"T1/T-T/B0:g-T/B1:e",
	}
	// direct  tree_i -> tree_{i+1} -> _{i+2} ...   plus
	// reverse ... tree_i <- _{i+1} <- _{i+2}
	kadjOK := ΔBTest(testv[len(testv)-1]).kadjOK
	for i := len(testv)-2; i >= 0; i-- {
		test := ΔBTest(testv[i])
		kadjOK, test.kadjOK = test.kadjOK, kadjOK
		testv = append(testv, test)
	}

	testq := make(chan ΔBTestEntry)
	go func() {
		defer close(testq)
		for _, test := range testv {
			testq <- ΔBTest(test)
		}
	}()
	testΔBTail(t, testq)

}


// ---- misc ----


// IntSets generates all sets of integers in range [0,N)
func IntSets(N int) chan []int {
	ch := make(chan []int)
	go intSets(ch, 0, N)
	return ch
}

// intSets generates all sets of integers in range [lo,hi)
func intSets(ch chan []int, lo, hi int) {
	ch <- nil // ø

	if lo < hi {
		for i := lo; i < hi; i++ {
			chTail := make(chan []int)
			go intSets(chTail, i+1, hi)
			for tail := range chTail {
				ch <- append([]int{i}, tail...) // i + tail
			}
		}
	}

	close(ch)
}


// kvdiff returns difference in between kv1 and kv2.
var DEL = "ø" // DEL means key deletion
type Δstring struct {
	Old string
	New string
}
func kvdiff(kv1, kv2 map[Key]string) map[Key]Δstring {
	delta := map[Key]Δstring{}
	keys := SetKey{}
	for k := range kv1 { keys.Add(k) }
	for k := range kv2 { keys.Add(k) }

	for k := range keys {
		v1, ok := kv1[k]
		if !ok { v1 = DEL }
		v2, ok := kv2[k]
		if !ok { v2 = DEL }
		if v1 != v2 {
			delta[k] = Δstring{v1,v2}
		}
	}

	return delta
}

// kvtxt returns string representation of {} kv.
func kvtxt(kv map[Key]string) string {
	if len(kv) == 0 {
		return "ø"
	}

	keyv := []Key{}
	for k := range kv { keyv = append(keyv, k) }
	sort.Slice(keyv, func(i,j int) bool { return keyv[i] < keyv[j] })

	sv := []string{}
	for _, k := range keyv {
		v := kv[k]
		if strings.ContainsAny(v, " \n\t,:") {
			panicf("[%v]=%q: invalid value", k, v)
		}
		sv = append(sv, fmt.Sprintf("%v:%s", k, v))
	}

	return strings.Join(sv, ",")
}

// xkvFlatten converts xkv with bucket structure into regular dict.
func xkvFlatten(xkv RBucketSet) map[Key]string {
	kv := make(map[Key]string)
	for _, b := range xkv {
		for k,v := range b.kv {
			kv[k] = v
		}
	}
	return kv
}


// easies debugging / makes error output stable from run to run.
func (ks SetKey) SortedKeys() []Key {
	keyv := ks.Elements()
	sort.Slice(keyv, func(i, j int) bool {
		return keyv[i] < keyv[j]
	})
	return keyv
}
func (ks SetKey) String() string {
	strv := []string{}
	for _, k := range ks.SortedKeys() {
		strv = append(strv, fmt.Sprintf("%d", k))
	}
	return "{" + strings.Join(strv, " ") + "}"
}

func (os SetOid) SortedKeys() []Oid {
	oidv := os.Elements()
	sort.Slice(oidv, func(i, j int) bool {
		return oidv[i] < oidv[j]
	})
	return oidv
}
func (os SetOid) String() string {
	strv := []string{}
	for _, oid := range os.SortedKeys() {
		strv = append(strv, fmt.Sprintf("%s", oid))
	}
	return "{" + strings.Join(strv, " ") + "}"
}

func sortedKeys(kv map[Key]Δstring) []Key {
	keyv := []Key{}
	for k := range kv {
		keyv = append(keyv, k)
	}
	sort.Slice(keyv, func(i, j int) bool {
		return keyv[i] < keyv[j]
	})
	return keyv
}

// String is like default %v, but uses ø for VDEL.
func (δv ΔValue) String() string {
	old, new := DEL, DEL
	if δv.Old != VDEL {
		old = δv.Old.String()
	}
	if δv.New != VDEL {
		new = δv.New.String()
	}
	return fmt.Sprintf("{%s %s}", old, new)
}
