X wcfs: Goodby holeIdx

As a consequence rework treediff to work on key ranges instead of
individial keys.

Adjacency coverage gets wider, but that is ok, because in practice if
keys are added to a BTree - that means that something is added to
ZBigFile, and it is very likely that just added data will be accessed
again. In other words it means that wcfs will very likely track those
data on nearby future read access, so we don't loose anything if we
start to track those keys right at the beginning.

* t2: (74 commits)
  .
  .
  .
  .
  .
  .
  X kadj must be taken into account as kadj^δZ
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  ...

wcfs/internal/xbtree/pptreesubset.go

@@ -26,7 +26,7 @@ import (
 	"lab.nexedi.com/kirr/neo/go/zodb"
 )
 
-const debugPPSet = false
+const tracePPSet = false
 
 // PPTreeSubSet represents PP-connected subset of tree node objects.
 //
@@ -131,14 +131,23 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
 	}
 }
 
-// ---- Union/Difference ----
+// ---- Union/Difference/Intersetctior ----
+
+// Union returns U = PP(A.leafs | B.leafs)
+//
+// In other words it returns sum of A and B.
+func (A PPTreeSubSet) Union(B PPTreeSubSet) PPTreeSubSet {
+	U := A.Clone()
+	U.UnionInplace(B)
+	return U
+}
 
 // UnionInplace sets A = PP(A.leafs | B.leafs)
 //
 // In other words it adds B nodes to A.
 func (A PPTreeSubSet) UnionInplace(B PPTreeSubSet) {
-	if debugPPSet {
-		fmt.Printf("\n\nUnionInplace:\n")
+	if tracePPSet {
+		fmt.Printf("\n\nUnion:\n")
 		fmt.Printf("  A: %s\n", A)
 		fmt.Printf("  B: %s\n", B)
 		defer fmt.Printf("->U: %s\n", A)
@@ -151,12 +160,22 @@ func (A PPTreeSubSet) UnionInplace(B PPTreeSubSet) {
 	A.xUnionInplace(B)
 }
 
+// Difference returns D = PP(A.leafs \ B.leafs)
+//
+// In other words ... XXX
+func (A PPTreeSubSet) Difference(B PPTreeSubSet) PPTreeSubSet {
+	D := A.Clone()
+	D.DifferenceInplace(B)
+	return D
+}
+
+
 // DifferenceInplace sets A = PP(A.leafs \ B.leafs)
 //
 // In other words it removes B nodes from A while still maintaining A as PP-connected.
 func (A PPTreeSubSet) DifferenceInplace(B PPTreeSubSet) {
-	if debugPPSet {
-		fmt.Printf("\n\nDifferenceInplace:\n")
+	if tracePPSet {
+		fmt.Printf("\n\nDifference:\n")
 		fmt.Printf("  A: %s\n", A)
 		fmt.Printf("  B: %s\n", B)
 		defer fmt.Printf("->D: %s\n", A)
@@ -169,9 +188,11 @@ func (A PPTreeSubSet) DifferenceInplace(B PPTreeSubSet) {
 	A.xDifferenceInplace(B)
 }
 
+// XXX Intersection
+
 func (A PPTreeSubSet) xUnionInplace(B PPTreeSubSet) {
-	if debugPPSet {
-		fmt.Printf("\n\n  xUnionInplace:\n")
+	if tracePPSet {
+		fmt.Printf("\n\n  xUnion:\n")
 		fmt.Printf("    a: %s\n", A)
 		fmt.Printf("    b: %s\n", B)
 		defer fmt.Printf("  ->u: %s\n", A)
@@ -204,8 +225,8 @@ func (A PPTreeSubSet) xUnionInplace(B PPTreeSubSet) {
 }
 
 func (A PPTreeSubSet) xDifferenceInplace(B PPTreeSubSet) {
-	if debugPPSet {
-		fmt.Printf("\n\n  xDifferenceInplace:\n")
+	if tracePPSet {
+		fmt.Printf("\n\n  xDifference:\n")
 		fmt.Printf("    a: %s\n", A)
 		fmt.Printf("    b: %s\n", B)
 		defer fmt.Printf("  ->d: %s\n", A)
@@ -445,7 +466,7 @@ func (δ *ΔPPTreeSubSet) Reverse() {
 //
 // See ΔPPTreeSubSet documentation for details.
 func (S PPTreeSubSet) ApplyΔ(δ *ΔPPTreeSubSet) {
-	if debugPPSet {
+	if tracePPSet {
 		fmt.Printf("\n\nApplyΔ\n")
 		fmt.Printf("  A: %s\n", S)
 		fmt.Printf("  -: %s\n", δ.Del)

wcfs/internal/xbtree/pptreesubset_test.go

@@ -33,8 +33,8 @@ func TestPPTreeSubSetOps(t *testing.T) {
 		d
 		ø = zodb.InvalidOid
 	)
-	type  S = PPTreeSubSet
-	type  testEntry struct {
+	type S = PPTreeSubSet
+	type testEntry struct {
 		A, B       S
 		Union      S
 		Difference S
@@ -82,10 +82,8 @@ func TestPPTreeSubSetOps(t *testing.T) {
 	}
 
 	for _, tt := range testv {
-		U := tt.A.Clone()
-		U.UnionInplace(tt.B)
-		D := tt.A.Clone()
-		D.DifferenceInplace(tt.B)
+		U := tt.A.Union(tt.B)
+		D := tt.A.Difference(tt.B)
 
 		if !U.Equal(tt.Union) {
 			t.Errorf("Union:\n  A:   %s\n  B:   %s\n  ->u: %s\n  okU: %s\n", tt.A, tt.B, U, tt.Union)

wcfs/internal/xbtree/rangeset.go

+// Copyright (C) 2021  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtree
+// set of [lo,hi) Key ranges.
+
+import (
+	"fmt"
+	"sort"
+)
+
+const traceRangeSet = false
+const debugRangeSet = false
+
+// KeyRange represents [lo,hi) Key range.
+type KeyRange struct {
+	lo  Key
+	hi_ Key // NOTE _not_ hi) to avoid overflow at ∞;  hi = hi_ + 1
+}
+
+// RangedKeySet is set of Keys with adjacent keys coaleced into Ranges.
+//
+// Zero value represents empty set.
+type RangedKeySet struct {
+	// TODO rework to use BTree lo->hi_ instead if in practice in treediff,
+	// and other usage places, N(ranges) turns out to be not small
+	// (i.e. performance turns out to be not acceptable)
+	rangev []KeyRange // lo↑
+}
+
+
+// Has returns whether key k belongs to the range.
+func (r *KeyRange) Has(k Key) bool {
+	return (r.lo <= k && k <= r.hi_)
+}
+
+
+// Add adds key k to the set.
+func (S *RangedKeySet) Add(k Key) {
+	S.AddRange(KeyRange{lo: k, hi_: k})
+}
+
+// Del removes key k from the set.
+func (S *RangedKeySet) Del(k Key) {
+	S.DelRange(KeyRange{lo: k, hi_: k})
+}
+
+// Has returns whether key k belongs to the set.
+func (S *RangedKeySet) Has(k Key) bool {
+	return S.HasRange(KeyRange{lo: k, hi_: k})
+}
+
+
+// AddRange adds range r to the set.
+func (S *RangedKeySet) AddRange(r KeyRange) {
+	if traceRangeSet {
+		fmt.Printf("\n\nAddRange:\n")
+		fmt.Printf("  S: %s\n", S)
+		fmt.Printf("  r: %s\n", r)
+		defer fmt.Printf("->u: %s\n", S)
+	}
+
+	S.verify()
+	defer S.verify()
+
+	// find first ilo: r.lo < [ilo].hi
+	l := len(S.rangev)
+	ilo := sort.Search(l, func(i int) bool {
+		 return r.lo <= S.rangev[i].hi_
+	})
+	debugfRSet("\tilo: %d\n", ilo)
+
+	if ilo == l { // not found
+		S.rangev = append(S.rangev, r)
+		l++
+		debugfRSet("\tappend %s\t-> %s\n", r, S)
+	}
+
+	// find last jhi: [jhi].lo < r.hi
+	jhi := ilo
+	for ;; jhi++ {
+		if jhi == l {
+			break
+		}
+		if S.rangev[jhi].lo <= r.hi_ {
+			continue
+		}
+		break
+	}
+	debugfRSet("\tjhi: %d\n", jhi)
+
+	// entries in [ilo:jhi) ∈ [r.lo,r.hi) and should be merged into one
+	if (jhi - ilo) > 1 {
+		lo  := S.rangev[ilo].lo
+		hi_ := S.rangev[jhi-1].hi_
+		S.rangev = append(
+			S.rangev[:ilo],		append([]KeyRange{
+			KeyRange{lo, hi_}},
+			S.rangev[jhi:]...)...)
+		debugfRSet("\tmerge S[%d:%d]\t-> %s\n", ilo, jhi, S)
+	}
+	jhi = -1 // no longer valid
+
+	// if [r.lo,r.hi) was outside of any entry - create new entry
+	if r.hi_ < S.rangev[ilo].lo {
+		S.rangev = append(
+			S.rangev[:ilo],		append([]KeyRange{
+			r},
+			S.rangev[ilo:]...)...)
+		debugfRSet("\tinsert %s\t-> %s\n", r, S)
+	}
+
+	// now we have covered entries merged as needed into [ilo]
+	// extend this entry if r coverage is wider
+	if r.lo < S.rangev[ilo].lo {
+		S.rangev[ilo].lo = r.lo
+		debugfRSet("\textend left\t-> %s\n", S)
+	}
+	if r.hi_ > S.rangev[ilo].hi_ {
+		S.rangev[ilo].hi_ = r.hi_
+		debugfRSet("\textend right\t-> %s\n", S)
+	}
+
+	// and check if we should merge it with right/left neighbours
+	if ilo+1 < len(S.rangev) { // right
+		if S.rangev[ilo].hi_+1 == S.rangev[ilo+1].lo {
+			S.rangev = append(
+				S.rangev[:ilo],		append([]KeyRange{
+				KeyRange{S.rangev[ilo].lo, S.rangev[ilo+1].hi_}},
+				S.rangev[ilo+2:]...)...)
+			debugfRSet("\tmerge right\t-> %s\n", S)
+		}
+	}
+
+	if ilo > 0 { // left
+		if S.rangev[ilo-1].hi_+1 == S.rangev[ilo].lo {
+			S.rangev = append(
+				S.rangev[:ilo-1],	append([]KeyRange{
+				KeyRange{S.rangev[ilo-1].lo, S.rangev[ilo].hi_}},
+				S.rangev[ilo+1:]...)...)
+			debugfRSet("\tmerge left\t-> %s\n", S)
+		}
+	}
+
+	// done
+}
+
+// DelRange removes range r from the set.
+func (S *RangedKeySet) DelRange(r KeyRange) {
+	if traceRangeSet {
+		fmt.Printf("\n\nDelRange:\n")
+		fmt.Printf("  S: %s\n", S)
+		fmt.Printf("  r: %s\n", r)
+		defer fmt.Printf("->d: %s\n", S)
+	}
+
+	S.verify()
+	defer S.verify()
+
+	// find first ilo: r.lo < [ilo].hi
+	l := len(S.rangev)
+	ilo := sort.Search(l, func(i int) bool {
+		 return r.lo <= S.rangev[i].hi_
+	})
+	debugfRSet("\tilo: %d\n", ilo)
+
+	if ilo == l { // not found
+		debugfRSet("\tnon-overlap right\n")
+		return
+	}
+
+	// find last jhi: [jhi].lo < r.hi
+	jhi := ilo
+	for ;; jhi++ {
+		if jhi == l {
+			break
+		}
+		if S.rangev[jhi].lo <= r.hi_ {
+			continue
+		}
+		break
+	}
+	debugfRSet("\tjhi: %d\n", jhi)
+
+	if jhi == 0 {
+		debugfRSet("\tnon-overlap left\n")
+		return
+	}
+
+	// [ilo+1:jhi-1] should be deleted
+	// [ilo] and [jhi-1] overlap with [r.lo,r.hi) - they shuold be deleted, or shrinked,
+	// or split+shrinked if ilo==jhi-1 and r is inside [ilo]
+	if jhi-ilo == 1 && S.rangev[ilo].lo < r.lo && r.hi_ < S.rangev[ilo].hi_ {
+		x := S.rangev[ilo]
+		S.rangev = append(
+			S.rangev[:ilo],		append([]KeyRange{
+			x, x},
+			S.rangev[ilo+1:]...)...)
+		jhi++
+		debugfRSet("\tpresplit copy %s\t-> %s\n", x, S)
+	}
+	if S.rangev[ilo].lo < r.lo { // shrink left
+		S.rangev = append(
+			S.rangev[:ilo],		append([]KeyRange{
+			KeyRange{S.rangev[ilo].lo, r.lo-1}},
+			S.rangev[ilo+1:]...)...)
+		ilo++
+		debugfRSet("\tshrink [%d] left\t-> %s\n", ilo, S)
+	}
+	if r.hi_ < S.rangev[jhi-1].hi_ { // shrink right
+		S.rangev = append(
+			S.rangev[:jhi-1],	append([]KeyRange{
+			KeyRange{r.hi_+1, S.rangev[jhi-1].hi_}},
+			S.rangev[jhi:]...)...)
+		jhi--
+		debugfRSet("\tshrink [%d] right\t-> %s\n", jhi-1, S)
+	}
+
+	if (jhi - ilo) > 0 {
+		S.rangev = append(
+			S.rangev[:ilo],
+			S.rangev[jhi:]...)
+		debugfRSet("\tdelete S[%d:%d]\t-> %s\n", ilo, jhi, S)
+	}
+
+	// done
+}
+
+// HasRange returns whether all keys from range r belong to the set.
+func (S *RangedKeySet) HasRange(r KeyRange) (yes bool) {
+	if traceRangeSet {
+		fmt.Printf("\n\nHasRange:\n")
+		fmt.Printf("  S: %s\n", S)
+		fmt.Printf("  r: %s\n", r)
+		defer func() {
+			fmt.Printf("->·: %v\n", yes)
+		}()
+	}
+
+	S.verify()
+
+	// find first ilo: r.lo < [ilo].hi
+	l := len(S.rangev)
+	ilo := sort.Search(l, func(i int) bool {
+		 return r.lo <= S.rangev[i].hi_
+	})
+	debugfRSet("\tilo: %d\n", ilo)
+
+	if ilo == l { // not found
+		return false
+	}
+
+	// all keys from r are in S if r ∈ [ilo]
+	return (S.rangev[ilo].lo <= r.lo && r.hi_ <= S.rangev[ilo].hi_)
+}
+
+
+// Union returns RangedKeySet(A.keys | B.keys).
+func (A *RangedKeySet) Union(B *RangedKeySet) *RangedKeySet {
+	U := A.Clone()
+	U.UnionInplace(B)
+	return U
+}
+
+// Difference returns RangedKeySet(A.keys \ B.keys).
+func (A *RangedKeySet) Difference(B *RangedKeySet) *RangedKeySet {
+	D := A.Clone()
+	D.DifferenceInplace(B)
+	return D
+}
+
+// XXX Intersection
+
+func (A *RangedKeySet) UnionInplace(B *RangedKeySet) {
+	A.verify()
+	B.verify()
+	defer A.verify()
+
+	// XXX dumb
+	for _, r := range B.rangev {
+		A.AddRange(r)
+	}
+}
+
+func (A *RangedKeySet) DifferenceInplace(B *RangedKeySet) {
+	A.verify()
+	B.verify()
+	defer A.verify()
+
+	// XXX dumb
+	for _, r := range B.rangev {
+		if len(A.rangev) == 0 {
+			break
+		}
+		A.DelRange(r)
+	}
+}
+
+
+// --------
+
+// verify check RangedKeySet for internal consistency:
+// - ranges must be not overlapping nor adjacent and ↑
+func (S *RangedKeySet) verify() {
+	// XXX !debug -> return ?
+
+	var badv []string
+	badf := func(format string, argv ...interface{}) {
+		badv = append(badv, fmt.Sprintf(format, argv...))
+	}
+	defer func() {
+		if badv != nil {
+			emsg := fmt.Sprintf("S.verify: fail:\n\n")
+			for _, bad := range badv {
+				emsg += fmt.Sprintf("- %s\n", bad)
+			}
+			emsg += fmt.Sprintf("\nS: %s\n", S)
+			panic(emsg)
+		}
+	}()
+
+	hi_Prev := KeyMin
+	for i, r := range S.rangev {
+		hiPrev := hi_Prev + 1
+		if i > 0 && !(hiPrev < r.lo) { // NOTE not ≤ - adjacent ranges must be merged
+			badf("[%d]: !(hiPrev < r.lo)", i)
+		}
+		if !(r.lo <= r.hi_) {
+			badf("[%d]: !(r.lo <= r.hi_)", i)
+		}
+		hi_Prev = r.hi_
+	}
+}
+
+// Clone returns copy of the set.
+func (orig *RangedKeySet) Clone() *RangedKeySet {
+	klon := &RangedKeySet{}
+	klon.rangev = append(klon.rangev, orig.rangev...)
+	return klon
+}
+
+// Empty returns whether the set is empty.
+func (S *RangedKeySet) Empty() bool {
+	return len(S.rangev) == 0
+}
+
+// Equal returns whether A == B.
+func (A *RangedKeySet) Equal(B *RangedKeySet) bool {
+	if len(A.rangev) != len(B.rangev) {
+		return false
+	}
+	for i, ra := range A.rangev {
+		rb := B.rangev[i]
+		if ra != rb {
+			return false
+		}
+	}
+	return true
+}
+
+// Clear removes all elements from the set.
+func (S *RangedKeySet) Clear() {
+	S.rangev = nil
+}
+
+// AllRanges returns slice of all key ranges in the set.
+// XXX -> iter?
+func (S *RangedKeySet) AllRanges() /*readonly*/[]KeyRange {
+	return S.rangev
+}
+
+func (S RangedKeySet) String() string {
+	s := "{"
+	for i, r := range S.rangev {
+		if i > 0 {
+			s += " "
+		}
+		s += r.String()
+	}
+	s += "}"
+	return s
+}
+
+func (r KeyRange) String() string {
+	hi := r.hi_
+	if hi < KeyMax {
+		hi += 1
+	}
+	return fmt.Sprintf("[%s,%s)", kstr(r.lo), kstr(hi))
+}
+
+
+func debugfRSet(format string, argv ...interface{}) {
+	if !debugRangeSet {
+		return
+	}
+	fmt.Printf(format, argv...)
+}

wcfs/internal/xbtree/rangeset_test.go

+// Copyright (C) 2021  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtree
+
+import (
+	"testing"
+)
+
+func TestRangedKeySet(t *testing.T) {
+	const (
+		oo  = KeyMax
+		noo = KeyMin
+	)
+
+	type testEntry struct {
+		A, B       *RangedKeySet
+		Union      *RangedKeySet
+		Difference *RangedKeySet
+	}
+	E := func(A, B, U, D *RangedKeySet) testEntry {
+		return testEntry{A, B, U, D}
+	}
+
+	// S is shorthand to create RangedKeySet, e.g. S(1,2, 4,5) will return {[1,2) [4,5)}
+	S := func(kv ...Key) *RangedKeySet {
+		l := len(kv)
+		if l % 2 != 0 {
+			panic("odd number of keys")
+		}
+		S := &RangedKeySet{}
+		for i := 0; i < l/2; i++ {
+			// construct .rangev directly, not via AddRange
+			lo := kv[2*i]
+			hi := kv[2*i+1]
+			hi_ := hi
+			if hi_ != oo {
+				hi_--
+			}
+			S.rangev = append(S.rangev, KeyRange{lo, hi_})
+		}
+		S.verify()
+		return S
+	}
+
+	testv := []testEntry{
+		E(
+			S(),	// A
+			S(),	// B
+				S(),	// U
+				S()),	// D
+
+		E(
+			S(1,2),	// A
+			S(1,2),	// B
+				S(1,2),	// U
+				S()),	// D
+
+		// adjacent [1,3) [3,5)
+		E(
+			S(1,3), // A
+			S(3,5), // B
+				S(1,5),	 // U
+				S(1,3)), // D
+
+		// overlapping [1,3) [2,4)
+		E(
+			S(1,3),	// A
+			S(2,4),	// B
+				S(1,4),  // U
+				S(1,2)), // D
+
+		// [1,7) \ [3,5)  ->  [1,3) [5,7)
+		E(
+			S(1,7),	// A
+			S(3,5),	// B
+				S(1,7),
+				S(1,3, 5,7)),
+
+		// several ranges \ [-∞, ∞) -> ø
+		E(
+			S(1,3, 5,7, 11,100), // A
+			S(noo, oo),          // B
+				S(noo, oo),	// U
+				S()),		// D
+
+		// [1,3) [5,7) + insert [3,5) -> [1,7)
+		E(
+			S(1,3, 5,7),	// A
+			S(3,5),		// B
+				S(1,7),		// U
+				S(1,3, 5,7)),	// D
+
+		// delete covering several ranges
+		// [-1,0)  [1,3) [5,7) [9,11) [15,20)  [100,200)   \  [2,17)
+		E(
+			S(-1,0, 1,3, 5,7, 9,11, 15,20, 100,200),	// A
+			S(2,17),					// B
+				S(-1,0, 1,20, 100,200),			// U
+				S(-1,0, 1,2, 17,20, 100,200)),		// D
+	}
+
+	for _, tt := range testv {
+		A := tt.A
+		B := tt.B
+		U := A.Union(B)
+		D := A.Difference(B)
+
+		if !U.Equal(tt.Union) {
+			t.Errorf("Union:\n  A:   %s\n  B:   %s\n  ->u: %s\n  okU: %s\n", tt.A, tt.B, U, tt.Union)
+		}
+		if !D.Equal(tt.Difference) {
+			t.Errorf("Difference:\n  A:   %s\n  B:   %s\n  ->d: %s\n  okD: %s\n", tt.A, tt.B, D, tt.Difference)
+		}
+
+		// HasRange
+		assertHasRange(t, A.AllRanges(), A, true)
+		assertHasRange(t, B.AllRanges(), B, true)
+		assertHasRange(t, A.AllRanges(), U, true)
+		assertHasRange(t, B.AllRanges(), U, true)
+
+		Dab := D
+		Dba := B.Difference(A)
+		assertHasRange(t, Dab.AllRanges(), A, true)
+		assertHasRange(t, Dab.AllRanges(), B, false)
+		assertHasRange(t, Dba.AllRanges(), B, true)
+		assertHasRange(t, Dba.AllRanges(), A, false)
+	}
+}
+
+// assertHasRange asserts for all ranges from rangev that S.HasRange(r) == hasOK
+func assertHasRange(t *testing.T, rangev []KeyRange, S *RangedKeySet, hasOK bool) {
+	t.Helper()
+	for _, r := range rangev {
+		has := S.HasRange(r)
+		if has != hasOK {
+			t.Errorf("HasRange:\n  S:   %s\n  r:   %s\n  ->: %v\n  ok: %v\n", S, r, has, hasOK)
+		}
+	}
+}

wcfs/internal/xbtree/treediff.go

@@ -89,25 +89,6 @@ type ΔValue struct {
 }
 
 
-// treeSetKey represents ordered set of keys.
-// it can be point-queried and range-accessed.
-// TODO -> btree
-type treeSetKey struct {
-	SetKey
-}
-
-// InRange returns
-func (hi treeSetKey) GetInRange(lo, hi_ Key) SetKey {
-	// FIXME dumb O(n) -> TODO use cznic/b
-	ret := SetKey{}
-	for k := range hi.SetKey {
-		if lo <= k && k <= hi_ {
-			ret.Add(k)
-		}
-	}
-	return ret
-}
-
 // δZConnectTracked computes connected closure of δZ/T.
 //
 // δZ    - all changes in a ZODB transaction.
@@ -175,7 +156,7 @@ func δZConnectTracked(δZv []zodb.Oid, T PPTreeSubSet) (δZTC SetOid, δtopsByR
 // nodeInRange represents a Node coming under [lo, hi_] key range in its tree.
 type nodeInRange struct {
 	prefix   []zodb.Oid // path to this node goes via this objects
-	lo, hi_  Key  // [lo, hi_]	NOTE _not_ hi) not to overflow at ∞
+	lo, hi_  Key  // [lo, hi_]	NOTE _not_ hi) not to overflow at ∞	XXX -> Range
 	node     Node
 	done     bool // whether this node was already taken into account while computing diff
 }
@@ -192,7 +173,8 @@ func (n *nodeInRange) NodePath() []Node {
 }
 */
 func (n *nodeInRange) Path() []zodb.Oid {
-	return append(n.prefix, n.node.POid())
+	// return full copy - else .prefix can become aliased in between children of a node
+	return append([]zodb.Oid{}, append(n.prefix, n.node.POid())...)
 }
 
 // rangeSplit represents set of nodes covering a range.
@@ -341,10 +323,7 @@ func (rs rangeSplit) String() string {
 // δtops is set of top nodes for changed subtrees.
 // δZTC is connected(δZ/T) - connected closure for subset of δZ(old..new) that
 // touches tracked nodes of T.
-//
-// XXX holeIdx  is updated  XXX -> return similarly to δtrack
-// XXX ^^^ -> but better kill holeIdx and do everything only via trackSet
-func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, trackSet PPTreeSubSet, holeIdx treeSetKey, zconnOld, zconnNew *zodb.Connection) (δT map[Key]ΔValue, δtrack *ΔPPTreeSubSet, err error) {
+func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, trackSet PPTreeSubSet, zconnOld, zconnNew *zodb.Connection) (δT map[Key]ΔValue, δtrack *ΔPPTreeSubSet, err error) {
 	defer xerr.Contextf(&err, "treediff %s..%s %s", zconnOld.At(), zconnNew.At(), root)
 
 	tracef("\ntreediff %s  δtops: %v  δZTC: %v\n", root, δtops, δZTC)
@@ -361,7 +340,7 @@ func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, t
 			return nil, nil, err
 		}
 
-		δtop, δtrackTop, err := diffX(ctx, a, b, δZTC, trackSet, holeIdx)
+		δtop, δtrackTop, err := diffX(ctx, a, b, δZTC, trackSet)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -380,16 +359,6 @@ func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, t
 		δtrackv = append(δtrackv, δtrackTop)
 	}
 
-	// adjust holeIdx
-	for k, δv := range δT {
-		if δv.Old == VDEL {
-			holeIdx.Del(k)
-		}
-		if δv.New == VDEL {
-			holeIdx.Add(k)
-		}
-	}
-
 	// adjust trackSet by merge(δtrackTops)
 	δtrack = &ΔPPTreeSubSet{Del: PPTreeSubSet{}, Add: PPTreeSubSet{}, δnchildNonLeafs: map[zodb.Oid]int{}}
 	for _, δ := range δtrackv {
@@ -410,7 +379,7 @@ func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, t
 //
 // δtrack is trackSet δ that needs to be applied to trackSet to keep it
 // consistent with b (= a + δ).
-func diffX(ctx context.Context, a, b Node, δZTC SetOid, trackSet PPTreeSubSet, holeIdx treeSetKey) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, err error) {
+func diffX(ctx context.Context, a, b Node, δZTC SetOid, trackSet PPTreeSubSet) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, err error) {
 	if a==nil && b==nil {
 		panic("BUG: both a & b == nil")	// XXX -> not a bug e.g. for `ø ø T` sequence?
 	}
@@ -445,7 +414,7 @@ func diffX(ctx context.Context, a, b Node, δZTC SetOid, trackSet PPTreeSubSet,
 	}
 
 	if isT {
-		return diffT(ctx, aT, bT, δZTC, trackSet, holeIdx)
+		return diffT(ctx, aT, bT, δZTC, trackSet)
 	} else {
 		var δtrack *ΔPPTreeSubSet
 		δ, err := diffB(ctx, aB, bB)
@@ -460,7 +429,7 @@ func diffX(ctx context.Context, a, b Node, δZTC SetOid, trackSet PPTreeSubSet,
 //
 // a, b point to top of subtrees @old and @new revisions.
 // δZTC is connected set of objects covering δZT (objects changed in this tree in old..new).
-func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet, holeIdx treeSetKey) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, err error) {
+func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, err error) {
 	tracef("  diffT %s %s\n", xidOf(A), xidOf(B))
 	defer xerr.Contextf(&err, "diffT %s %s", xidOf(A), xidOf(B))
 
@@ -518,20 +487,26 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet,
 	Bv := rangeSplit{btop} // nodes expanded from B
 
 	// for phase 2:
-	Akqueue := SetKey{} // queue for keys in A to be processed for δ-
-	Bkqueue := SetKey{} // ----//---- in B for δ+
-	Akdone  := SetKey{} // already processed keys in A
-	Bkdone  := SetKey{} // ----//---- in B
-	Aktodo := func(k Key) {
-		if !Akdone.Has(k) {
-			tracef("    Akq <- %d\n", k)
-			Akqueue.Add(k)
+	Akqueue := &RangedKeySet{} // queue for keys in A to be processed for δ-
+	Bkqueue := &RangedKeySet{} // ----//---- in B for δ+
+	Akdone  := &RangedKeySet{} // already processed keys in A
+	Bkdone  := &RangedKeySet{} // ----//---- in B
+	Aktodo := func(r KeyRange) {
+		if !Akdone.HasRange(r) {
+			δtodo := &RangedKeySet{}
+			δtodo.AddRange(r)
+			δtodo.DifferenceInplace(Akdone)
+			tracef("    Akq <- %s\n", δtodo)
+			Akqueue.UnionInplace(δtodo)
 		}
 	}
-	Bktodo := func(k Key) {
-		if !Bkdone.Has(k) {
-			tracef("    Bkq <- %d\n", k)
-			Bkqueue.Add(k)
+	Bktodo := func(r KeyRange) {
+		if !Bkdone.HasRange(r) {
+			δtodo := &RangedKeySet{}
+			δtodo.AddRange(r)
+			δtodo.DifferenceInplace(Bkdone)
+			tracef("    Bkq <- %s\n", δtodo)
+			Bkqueue.UnionInplace(δtodo)
 		}
 	}
 
@@ -564,23 +539,14 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet,
 			err = δMerge(δ, δA);		/*X*/if err != nil { return nil,nil, err }
 			δtrack.Del.AddPath(ra.Path())
 
-			// Bkqueue <- δA
-			for k := range δA {
-				Akdone.Add(k)
-				Bktodo(k)
-			}
-			// Bkqueue <- holes(ra.range)
-			for k := range holeIdx.GetInRange(ra.lo, ra.hi_) {
-				Bktodo(k)
-			}
+			// Bkqueue <- ra.range
+			Bktodo(KeyRange{ra.lo, ra.hi_})
 			ra.done = true
 
 		case *Tree:
 			// empty tree - only queue holes covered by it
 			if len(a.Entryv()) == 0 {
-				for k := range holeIdx.GetInRange(ra.lo, ra.hi_) {
-					Bktodo(k)
-				}
+				Bktodo(KeyRange{ra.lo, ra.hi_})
 				continue
 			}
 
@@ -634,8 +600,6 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet,
 					}
 					if found {
 						// ac can be skipped
-						// XXX Bkqueue <- holes(ac.range \ bc.range)	XXX test for this
-
 						// adjust trackSet since path to the node could have changed
 						apath := trackSet.Path(acOid)
 						bpath := BtrackSet.Path(acOid)
@@ -672,9 +636,6 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet,
 	// * +B12, which queues A.2 and leads to
 	// * -B23, which queues B.3 and leads to
 	// * +B23, ...
-	//
-	// XXX inefficient: we process each key separately, while they can be
-	// processed in sorted batches.
 	tracef("\nphase 2:\n")
 	for {
 		tracef("\n")
@@ -683,69 +644,80 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet,
 
 		tracef("\n")
 		tracef("  Bkq: %s\n", Bkqueue)
-		if len(Bkqueue) == 0 {
+		if Bkqueue.Empty() {
 			break
 		}
 
-		for k := range Bkqueue {
-			b, err := Bv.GetToLeaf(ctx, k);  /*X*/if err != nil { return nil,nil, err }
-			tracef("  B k%d -> %s\n", k, b)
-			// +bucket if that bucket is reached for the first time
-			if !b.done {
-				var δB map[Key]ΔValue
-				bbucket, ok := b.node.(*Bucket)
-				if ok { // !ok means ø tree
-					δB, err = diffB(ctx, nil, bbucket);  /*X*/if err != nil { return nil,nil, err }
-				}
+		for _, r := range Bkqueue.AllRanges() {
+			lo := r.lo
+			for {
+				b, err := Bv.GetToLeaf(ctx, lo);  /*X*/if err != nil { return nil,nil, err }
+				tracef("  B k%d -> %s\n", lo, b)
+				// +bucket if that bucket is reached for the first time
+				if !b.done {
+					var δB map[Key]ΔValue
+					bbucket, ok := b.node.(*Bucket)
+					if ok { // !ok means ø tree
+						δB, err = diffB(ctx, nil, bbucket);  /*X*/if err != nil { return nil,nil, err }
+					}
 
-				// δ <- δB
-				err = δMerge(δ, δB);	/*X*/if err != nil { return nil,nil, err }
-				δtrack.Add.AddPath(b.Path())
+					// δ <- δB
+					err = δMerge(δ, δB);	/*X*/if err != nil { return nil,nil, err }
+					δtrack.Add.AddPath(b.Path())
 
-				// Akqueue <- δB
-				for k_ := range δB {
-					Bkdone.Add(k_)
-					Aktodo(k_)
+					// Akqueue <- δB
+					br := KeyRange{b.lo, b.hi_}
+					Bkdone.AddRange(br)
+					Aktodo(br)
+
+					b.done = true
 				}
 
-				b.done = true
+				// stop if r coverage is complete
+				if r.hi_ <= b.hi_ {
+					break
+				}
+				// continue with next right bucket
+				lo = b.hi_ + 1
 			}
-
-			// XXX k is not there -> hole	XXX test
 		}
-		Bkqueue = SetKey{}
+		Bkqueue.Clear()
 
 		tracef("\n")
 		tracef("  Akq: %s\n", Akqueue)
-		for k := range Akqueue {
-			a, err := Av.GetToLeaf(ctx, k);  /*X*/if err != nil { return nil,nil, err }
-			tracef("  A k%d -> %s\n", k, a)
-			// -bucket if that bucket is reached for the first time
-			if !a.done {
-				var δA map[Key]ΔValue
-				abucket, ok := a.node.(*Bucket)
-				if ok { // !ok means ø tree
-					δA, err = diffB(ctx, abucket, nil);  /*X*/if err != nil { return nil,nil, err }
-				}
+		for _, r := range Akqueue.AllRanges() {
+			lo := r.lo
+			for {
+				a, err := Av.GetToLeaf(ctx, lo);  /*X*/if err != nil { return nil,nil, err }
+				tracef("  A k%d -> %s\n", lo, a)
+				// -bucket if that bucket is reached for the first time
+				if !a.done {
+					var δA map[Key]ΔValue
+					abucket, ok := a.node.(*Bucket)
+					if ok { // !ok means ø tree
+						δA, err = diffB(ctx, abucket, nil);  /*X*/if err != nil { return nil,nil, err }
+					}
 
-				// δ <- δA
-				err = δMerge(δ, δA);	/*X*/if err != nil { return nil,nil, err }
-				δtrack.Del.AddPath(a.Path())
+					// δ <- δA
+					err = δMerge(δ, δA);	/*X*/if err != nil { return nil,nil, err }
+					δtrack.Del.AddPath(a.Path())
 
-				// Bkqueue <- δA
-				for k_ := range δA {
-					Akdone.Add(k_)
-					Bktodo(k_)
-				}
-				// Bkqueue <- holes(a.range)
-				for k_ := range holeIdx.GetInRange(a.lo, a.hi_) {
-					Bktodo(k_)
+					// Bkqueue <- a.range
+					ar := KeyRange{a.lo, a.hi_}
+					Akdone.AddRange(ar)
+					Bktodo(ar)
+
+					a.done = true
 				}
 
-				a.done = true
+				// continue with next right bucket until r coverage is complete
+				if r.hi_ <= a.hi_ {
+					break
+				}
+				lo = a.hi_ + 1
 			}
 		}
-		Akqueue = SetKey{}
+		Akqueue.Clear()
 	}
 
 	return δ, δtrack, nil
@@ -930,11 +902,13 @@ func xidOf(obj zodb.IPersistent) string {
 	return xid.String()
 }
 
-func (rn nodeInRange) String() string {
-	slo  := "-∞";  if rn.lo  > KeyMin { slo = fmt.Sprintf("%v", rn.lo) }
-	shi  := "∞";   if rn.hi_ < KeyMax { shi = fmt.Sprintf("%v", rn.hi_+1) }
+func (rn *nodeInRange) String() string {
 	done := " ";   if rn.done         { done = "*" }
-	return fmt.Sprintf("%s[%s,%s)%s", done, slo, shi, vnode(rn.node))
+	hi := rn.hi_
+	if hi < KeyMax {
+		hi += 1
+	}
+	return fmt.Sprintf("%s[%s,%s)%s", done, kstr(rn.lo), kstr(hi), vnode(rn.node))
 }
 
 // push pushes element to node stack.

wcfs/internal/xbtree/xbtree.go

@@ -68,3 +68,15 @@ func nodePathToPath(nodePath []Node) (path []zodb.Oid) {
 	}
 	return path
 }
+
+
+// kstr formats key as string.
+func kstr(k Key) string {
+	if k == KeyMin {
+		return "-∞"
+	}
+	if k == KeyMax {
+		return "∞"
+	}
+	return fmt.Sprintf("%d", k)
+}

wcfs/internal/xbtree/δbtail.go

@@ -101,11 +101,6 @@ type ΔBtail struct {
 
 	// set of node that were requested to be tracked, but for which vδB was not yet rebuilt
 	trackNew PPTreeSubSet
-
-	// XXX root -> tracked holes
-	// XXX move -> ΔTtail ?
-	holeIdxByRoot map[zodb.Oid]treeSetKey
-
 }
 
 
@@ -149,7 +144,6 @@ func NewΔBtail(at0 zodb.Tid, db *zodb.DB) *ΔBtail {
 		byRoot:        map[zodb.Oid]*ΔTtail{},
 		trackSet:      PPTreeSubSet{},
 		trackNew:      PPTreeSubSet{},
-		holeIdxByRoot: map[zodb.Oid]treeSetKey{},
 		db:       db,
 	}
 }
@@ -184,14 +178,6 @@ func (orig *ΔBtail) clone() *ΔBtail {
 		klon.byRoot[root] = klonΔTtail
 	}
 
-	// holeIdxByRoot
-	klon.holeIdxByRoot = make(map[zodb.Oid]treeSetKey, len(orig.holeIdxByRoot))
-	for root, origHoleIdx := range orig.holeIdxByRoot {
-		klonHoleIdx := treeSetKey{make(SetKey, len(origHoleIdx.SetKey))}
-		klonHoleIdx.Update(origHoleIdx.SetKey)
-		klon.holeIdxByRoot[root] = klonHoleIdx
-	}
-
 	return klon
 }
 
@@ -224,6 +210,7 @@ func (δBtail *ΔBtail) Tail() zodb.Tid { return δBtail.δZtail.Tail() }
 // XXX path -> []oid ?
 //
 // XXX catch cycles on add?
+// XXX no need to pass keyPresent since holeIdx was removed
 func (δBtail *ΔBtail) Track(key Key, keyPresent bool, nodePath []Node) error { // XXX Tree|Bucket; path[0] = root
 
 	path := nodePathToPath(nodePath)
@@ -241,21 +228,6 @@ func (δBtail *ΔBtail) track(key Key, keyPresent bool, path []zodb.Oid) error {
 
 	// track is track of path[-1]  (i.e. leaf)
 
-	// remember missing keys in track of leaf node (bucket or top-level ø tree)
-	holeIdx := δBtail.holeIdxFor(root)
-	if !keyPresent {
-		holeIdx.Add(key)
-		//track.holes.Add(key)
-	} else {
-		/*
-		if track.holes.Has(key) {
-			panicf("[%v] was previously requested to be tracked as ø", key)
-		}
-		*/
-		if holeIdx.Has(key) {
-			panicf("[%v] was previously requested to be tracked as ø", key)
-		}
-	}
 
 // XXX hack - until rebuild is implemented
 if XXX_killWhenRebuildWorks {
@@ -270,15 +242,6 @@ if XXX_killWhenRebuildWorks {
 	return nil
 }
 
-func (δBtail *ΔBtail) holeIdxFor(root zodb.Oid) treeSetKey {
-	holeIdx, ok := δBtail.holeIdxByRoot[root]
-	if !ok {
-		holeIdx = treeSetKey{SetKey{}}
-		δBtail.holeIdxByRoot[root] = holeIdx
-	}
-	return holeIdx
-}
-
 // rebuild rebuilds ΔBtail taking trackNew requests into account.
 // XXX place
 func (δBtail *ΔBtail) rebuild() (err error) {
@@ -332,9 +295,8 @@ func (δBtail *ΔBtail) rebuild() (err error) {
 		}
 
 		for root, δtops := range δtopsByRoot {
-			holeIdx := treeSetKey{SetKey{}} // XXX stub
 			// diff backwards curr -> prev
-			δT, δtrack, err := treediff(ctx, root, δtops, δZTC, trackNew, holeIdx, zconnCurr, zconnPrev)
+			δT, δtrack, err := treediff(ctx, root, δtops, δZTC, trackNew, zconnCurr, zconnPrev)
 			if err != nil {
 				return err
 			}
@@ -428,7 +390,6 @@ if XXX_killWhenRebuildWorks {
 	tracef("Update  @%s -> @%s\n", δBtail.Head(), δZ.Tid)
 	tracef("δZ:\t%v\n", δZ.Changev)
 	tracef("trackSet:       %v\n", δBtail.trackSet)
-	tracef("holeIdxByRoot:  %v\n", δBtail.holeIdxByRoot)
 
 	// XXX dup wrt rebuild?
 
@@ -457,8 +418,7 @@ if XXX_killWhenRebuildWorks {
 	}
 
 	for root, δtops := range δtopsByRoot {
-		holeIdx := δBtail.holeIdxByRoot[root]
-		δT, δtrack, err := treediff(ctx, root, δtops, δZTC, δBtail.trackSet, holeIdx, zconnOld, zconnNew)
+		δT, δtrack, err := treediff(ctx, root, δtops, δZTC, δBtail.trackSet, zconnOld, zconnNew)
 		if err != nil {
 			return ΔB{}, err
 		}

wcfs/internal/xbtree/δbtail_test.go

@@ -47,9 +47,6 @@ import (
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
 )
 
-const kInf Key = 10000 // inf key (TeX hack)
-//const kInf Key = KeyMax
-
 // XXX move infrastructure -> δbtail_treegen_test.go ?
 
 // TreeGenSrv represents connection to running `treegen ...` server.
@@ -263,7 +260,7 @@ type RTree struct {
 type RBucket struct {
 	oid     zodb.Oid
 	parent  *RTree
-	lo, hi_ Key
+	lo, hi_ Key	// XXX -> KeyRange ?
 	kv      map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
 }
 
@@ -303,18 +300,6 @@ func (rbs RBucketSet) coverage() string {
 	return s
 }
 
-// holeIdx returns what should be ΔBtree.holeIdx for specified tracked key set.
-func (rbs RBucketSet) holeIdx(tracked SetKey) SetKey {
-	holes := SetKey{}
-	for k := range tracked {
-		_, keyin := rbs.Get(k).kv[k]
-		if !keyin {
-			holes.Add(k)
-		}
-	}
-	return holes
-}
-
 // trackSet returns what should be ΔBtree.trackSet for specified tracked key set.
 func (rbs RBucketSet) trackSet(tracked SetKey) PPTreeSubSet {
 	trackSet := PPTreeSubSet{}
@@ -385,7 +370,7 @@ func XGetTree(db *zodb.DB, at zodb.Tid, root zodb.Oid) RBucketSet {
 	}
 
 	rbucketv := RBucketSet{}
-	xwalkDFS(ctx, -kInf, kInf, ztree, func(rb *RBucket) {
+	xwalkDFS(ctx, KeyMin, KeyMax, ztree, func(rb *RBucket) {
 		rbucketv = append(rbucketv, rb)
 	})
 	if len(rbucketv) == 0 { // empty tree -> [-∞, ∞){}
@@ -396,8 +381,8 @@ func XGetTree(db *zodb.DB, at zodb.Tid, root zodb.Oid) RBucketSet {
 		ebucket := &RBucket{
 				oid:    zodb.InvalidOid,
 				parent: etree,
-				lo:     -kInf,
-				hi_:    kInf,
+				lo:     KeyMin,
+				hi_:    KeyMax,
 				kv:     map[Key]string{},
 		}
 		rbucketv = RBucketSet{ebucket}
@@ -489,15 +474,15 @@ func XGetδKV(db *zodb.DB, at1, at2 zodb.Tid, δkvOid map[Key]ΔValue) map[Key]
 // Av, Bv - topologies with values	ex T/B1:a  T2/B1:b-B3:c
 //
 // δ(Av, Bv) - full diff {k -> v} for changed keys; DEL = k -> ø
-//             ex δ(T/B1:a, T2/B1:b-B3:c) = {1:b 3:c}		XXX include "was" value too?
+//             ex δ(T/B1:a, T2/B1:b-B3:c) = {-1:a +1:b +3:c}
 //
 // Δ(T, Av, Bv) - subset of δ(Av, Bv) corresponding to initial tracking set T
-//             ex Δ({1}, T/B1:a, T2/B1:b-B3:c) = {1:b}  (no 3:c)
+//             ex Δ({1}, T/B1:a, T2/B1:b-B3:c) = {-1:a +1:b}  (no +3:c)		XXX fix example
 //
 // kadj(A,B) {} k -> {k'}:	- adjacency matrix
 //	∃v1,v2:  k'∈ Δ({k}, Av1, Bv2)
 //
-//	ex kadj(T/B1, T2/B1-B3) = {1:{1} 3:{1,3} ∞:{1,3}}	k ∈ A+B+{∞}
+//	ex kadj(T/B1, T2/B1-B3) = {1:{1} 3:{1,3} ∞:{1,3}}	k ∈ A+B+{∞}	XXX adjust to new kadj
 //                                + {0:{1} 2:{1,3} + ... all possible keys}
 //
 // Δ(T, Av, Bv) = δ(Av, Bv)/kadj(A,B)[T]
@@ -505,6 +490,8 @@ func XGetδKV(db *zodb.DB, at1, at2 zodb.Tid, δkvOid map[Key]ΔValue) map[Key]
 // i.e. = δ(Av, Bv) for k: k ∈ U kadj(A,B)[·]
 //                            ·∈T
 //
+// XXX fix definition for "and changed"
+//
 // XXX adjacency matrix is symmetric:	XXX KAdj verifies this at runtime
 //
 //	kadj(A,B) == kadj(B,A)
@@ -534,12 +521,19 @@ func KAdj(t1, t2 *tTreeCommit, keysv ...SetKey) (kadj KAdjMatrix) {
 	kadj12 := _KAdj(t1,t2, keysv...)
 	kadj21 := _KAdj(t2,t1, keysv...)
 	if !reflect.DeepEqual(kadj12, kadj21) {
-		panicf("KAdj not symmetric:\nt1: %s\bt2: %s\nkadj12: %v\nkadj21: %v",
+		panicf("KAdj not symmetric:\nt1: %s\nt2: %s\nkadj12: %v\nkadj21: %v",
 			t1.tree, t2.tree, kadj12, kadj21)
 	}
 	return kadj12
 }
 
+const debugKAdj = false
+func debugfKAdj(format string, argv ...interface{}) {
+	if debugKAdj {
+		fmt.Printf(format, argv...)
+	}
+}
+
 func _KAdj(t1, t2 *tTreeCommit, keysv ...SetKey) (kadj KAdjMatrix) {
 	var keys SetKey
 	switch len(keysv) {
@@ -551,51 +545,90 @@ func _KAdj(t1, t2 *tTreeCommit, keysv ...SetKey) (kadj KAdjMatrix) {
 		panic("multiple key sets on the call")
 	}
 
+	debugfKAdj("\n\n_KAdj\n")
+	debugfKAdj("t1:   %s\n", t1.tree)
+	debugfKAdj("t2:   %s\n", t2.tree)
+	debugfKAdj("keys: %s\n", keys)
+	defer func() {
+		debugfKAdj("kadj -> %v\n", kadj)
+	}()
 
 	// kadj = {} k -> adjacent keys.
-	// if k is tracked -> changes to adjacents must be in Update(t1->t2).
+	// if k is tracked and covered by changed leaf -> changes to adjacents must be in Update(t1->t2).
 	kadj = KAdjMatrix{}
 	for k := range keys {
 		adj1 := SetKey{}
 		adj2 := SetKey{}
 
-		q1 := []Key{k}
-		q2 := []Key{k}
-
-		//tracef("\n")
-		for len(q1) > 0 || len(q2) > 0 {
-			//tracef("q1: %v\n", q1)
-			//tracef("q2: %v\n", q2)
-			if l1 := len(q1); l1 > 0 {
-				k1 := q1[l1-1];  q1 = q1[:l1-1]
-				if !adj1.Has(k1) {
-					for k_ := range t1.xkv.Get(k1).kv {
-						adj1.Add(k_)
-						for k__ := range t2.xkv.Get(k_).kv {
-							if !adj2.Has(k__) {
-								q2 = append(q2, k__)
-							}
+		q1 := &RangedKeySet{}; q1.Add(k)
+		q2 := &RangedKeySet{}; q2.Add(k)
+		done1 := &RangedKeySet{}
+		done2 := &RangedKeySet{}
+
+		debugfKAdj("\nk%s\n", kstr(k))
+		for !q1.Empty() || !q2.Empty() {
+			debugfKAdj("q1:   %s\tdone1:  %s\n", q1, done1)
+			debugfKAdj("q2:   %s\tdone2:  %s\n", q2, done2)
+			for _, r1 := range q1.AllRanges() {
+				lo1 := r1.lo
+				for {
+					b1 := t1.xkv.Get(lo1)
+					debugfKAdj("  b1: %s\n", b1)
+					for k_ := range keys {
+						if b1.lo <= k_ && k_ <= b1.hi_ {
+							adj1.Add(k_)
+							debugfKAdj("    adj1 += %s\t-> %s\n", kstr(k_), adj1)
 						}
 					}
+					b1r := KeyRange{b1.lo, b1.hi_}
+					done1.AddRange(b1r)
+					// q2 |= (b1.keyrange \ done2)
+					δq2 := &RangedKeySet{}
+					δq2.AddRange(b1r)
+					δq2.DifferenceInplace(done2)
+					q2.UnionInplace(δq2)
+					debugfKAdj("q2 += %s\t-> %s\n", δq2, q2)
+
+					// continue with next right bucket until r1 coverage is complete
+					if r1.hi_ <= b1.hi_ {
+						break
+					}
+					lo1 = b1.hi_ + 1
 				}
 			}
-
-			if l2 := len(q2); l2 > 0 {
-				k2 := q2[l2-1];  q2 = q2[:l2-1]
-				if !adj2.Has(k2) {
-					for k_ := range t2.xkv.Get(k2).kv {
-						adj2.Add(k_)
-						for k__ := range t1.xkv.Get(k_).kv {
-							if !adj1.Has(k__) {
-								q1 = append(q1, k__)
-							}
+			q1.Clear()
+
+			for _, r2 := range q2.AllRanges() {
+				lo2 := r2.lo
+				for {
+					b2 := t2.xkv.Get(lo2)
+					debugfKAdj("  b2: %s\n", b2)
+					for k_ := range keys {
+						if b2.lo <= k_ && k_ <= b2.hi_ {
+							adj2.Add(k_)
+							debugfKAdj("    adj2 += %s\t-> %s\n", kstr(k_), adj2)
 						}
 					}
+					b2r := KeyRange{b2.lo, b2.hi_}
+					done2.AddRange(b2r)
+					// q1 |= (b2.keyrange \ done1)
+					δq1 := &RangedKeySet{}
+					δq1.AddRange(b2r)
+					δq1.DifferenceInplace(done1)
+					q1.UnionInplace(δq1)
+					debugfKAdj("q1 += %s\t-> %s\n", δq1, q1)
+
+					// continue with next right bucket until r2 coverage is complete
+					if r2.hi_ <= b2.hi_ {
+						break
+					}
+					lo2 = b2.hi_ + 1
 				}
 			}
+			q2.Clear()
 		}
 
-		adj := SetKey{}; adj.Add(k); adj.Update(adj1); adj.Update(adj2)
+		adj := SetKey{}; adj.Update(adj1); adj.Update(adj2)
 		kadj[k] = adj
 	}
 
@@ -635,8 +668,13 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 	X := exc.Raiseif
 	tracef("\n>>> Track=%s\n", initialTrackedKeys)
 
-	var kadjTracked SetKey = nil
+	var TrackedδZ SetKey = nil
+	var kadjTrackedδZ SetKey = nil
 	var δT, δTok map[Key]Δstring = nil, nil
+	δZset := SetOid{}
+	for _, oid := range δZ.Changev {
+		δZset.Add(oid)
+	}
 
 	// badf queues error message to be reported on return.
 	var badv []string
@@ -647,7 +685,10 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 		if badv != nil {
 			emsg := fmt.Sprintf("%s  ; tracked=%v :\n\n", subj, initialTrackedKeys)
 			emsg += fmt.Sprintf("d12:  %v\nδTok: %v\nδT:   %v\n\n", d12, δTok, δT)
-			emsg += fmt.Sprintf("kadj[Tracked]: %v\nkadj: %v\n\n", kadjTracked, kadj)
+			emsg += fmt.Sprintf("δZ:               %v\n", δZset)
+			emsg += fmt.Sprintf("Tracked^δZ:       %v\n", TrackedδZ)
+			emsg += fmt.Sprintf("kadj[Tracked^δZ]: %v\n", kadjTrackedδZ)
+			emsg += fmt.Sprintf("kadj: %v\n\n", kadj)
 			emsg += strings.Join(badv, "\n")
 			emsg += "\n"
 
@@ -669,7 +710,6 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 		ztree = xtree.(*Tree)
 	}
 
-	kadjTracked = SetKey{} // kadj[Tracked]  (all keys adjacent to tracked keys)
 	for k := range initialTrackedKeys {
 		if ztree != nil {
 			_, ok, path, err := ZTreeGetBlkData(ctx, ztree, k);  X(err)
@@ -686,42 +726,56 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 			// T1->ø, we test what will happen on ø->T2.
 			err = δbtail.track(k, false, []zodb.Oid{treeRoot});  X(err)
 		}
-
-		kadjTracked.Update(kadj[k])
 	}
 
-	// assert Tracked ∈ kadj[Tracked]  except ∞   XXX -> initialTrackedKeys.Difference(kadjTracked)
-	trackedNotInKadj := SetKey{}
+	TrackedδZ = SetKey{} // Tracked ^ δZ
 	for k := range initialTrackedKeys {
-		if !kadjTracked.Has(k) {
-			trackedNotInKadj.Add(k)
+		leaf1 := xkv1.Get(k)
+		oid1 := leaf1.oid
+		if oid1 == zodb.InvalidOid { // embedded bucket
+			oid1 = leaf1.parent.oid
+		}
+		leaf2 := xkv2.Get(k)
+		oid2 := leaf2.oid
+		if oid2 == zodb.InvalidOid { // embedded bucket
+			oid2 = leaf2.parent.oid
+		}
+		if δZset.Has(oid1) || δZset.Has(oid2) {
+			TrackedδZ.Add(k)
 		}
 	}
-	trackedNotInKadj.Del(kInf)
-	if len(trackedNotInKadj) > 0 {
-		badf("BUG: Tracked ∉ kadj[Tracked]  ; extra=%v", trackedNotInKadj)
+
+	kadjTrackedδZ = SetKey{} // kadj[Tracked^δZ]  (all keys adjacent to tracked^δZ)
+	for k := range TrackedδZ {
+		kadjTrackedδZ.Update(kadj[k])
+	}
+
+
+	// XXX why "except ∞" ?
+	// assert TrackedδZ ∈ kadj[TrackedδZ]  except ∞   XXX -> initialTrackedKeys.Difference(kadjTrackedδZ)
+	trackNotInKadj := SetKey{}
+	for k := range TrackedδZ {
+		if !kadjTrackedδZ.Has(k) {
+			trackNotInKadj.Add(k)
+		}
+	}
+	trackNotInKadj.Del(KeyMax)
+	if len(trackNotInKadj) > 0 {
+		badf("BUG: Tracked^δZ ∉ kadj[Tracked^δZ]  ; extra=%v", trackNotInKadj)
 		return
 	}
 
 	//              k ∈ d12
 	// k ∈ δT  <=>
 	//              k ∈   U kadj[·]
-	//                   ·∈tracking
+	//                   ·∈tracking^δZ
 	δTok = map[Key]Δstring{} // d12[all keys that should be present in δT]
 	for k,δv := range d12 {
-		if kadjTracked.Has(k) {
+		if kadjTrackedδZ.Has(k) {
 			δTok[k] = δv
 		}
 	}
 
-	// verify δbtail.holeIdx  against @at1
-	// holes1 = tracked1 \ kv1
-	holes1 := xkv1.holeIdx(initialTrackedKeys)
-	holeIdx := δbtail.holeIdxFor(treeRoot)
-	if !reflect.DeepEqual(holes1, holeIdx.SetKey) {
-		badf("δbtail.holeIdx1 wrong  ; holeIdx=%v  holeIdxOK=%v", holeIdx, holes1)
-	}
-
 	ø := PPTreeSubSet{}
 
 	// verify δbtail.trackSet against @at1
@@ -746,16 +800,9 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 		return
 	}
 
-	// verify δbtail.holeIdx  against @at2
-	// holes2 = tracked2 \ kv2  ( = kadj[tracked1] \ kv2)
-	holes2 := xkv2.holeIdx(kadjTracked)
-	if !reflect.DeepEqual(holes2, holeIdx.SetKey) {
-		badf("δbtail.holeIdx2 wrong  ; holeIdx=%v  holeIdxOK=%v", holeIdx, holes2)
-	}
-
 	// verify δbtail.trackSet  against @at2
 	// trackSet2 = xkv2[tracked2]  ( = xkv2[kadj[tracked1]]
-	trackSet2 := xkv2.trackSet(kadjTracked)
+	trackSet2 := xkv2.trackSet(initialTrackedKeys.Union(kadjTrackedδZ))
 //	if !reflect.DeepEqual(trackSet2, δbtail.trackSet) {
 //		badf("δbtail.trackSet2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, trackSet2)
 //	}
@@ -841,8 +888,6 @@ func (δbtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK, track
 	t.Helper()
 	assertTrack(t, subj + ": trackSet", δbtail.trackSet, trackSetOK)
 	assertTrack(t, subj + ": trackNew", δbtail.trackNew, trackNewOK)
-
-// XXX also verify .holeIdx	XXX or better get rid of .holeIdx
 }
 
 // xverifyΔBTail_rebuild verifies δBtail.rebuild during t0->t1->t2 transition.
@@ -1341,8 +1386,8 @@ func TestΔBTail(t *testing.T) {
 		for _, k := range keyv { ks.Add(k) }
 		return ks
 	}
-	// oo is shorthand for kInf
-	const oo = kInf
+	// oo is shorthand for KeyMax
+	const oo = KeyMax
 	// A is shorthand for KAdjMatrix
 	type A = KAdjMatrix
 	// Δ is shorthand for ΔBTestEntry
@@ -1362,56 +1407,56 @@ func TestΔBTail(t *testing.T) {
 
 		// +1
 		Δ("T/B1:a",
-			A{1:  K(1),
+			A{1:  K(1,oo),
 			  oo: K(1,oo)}),
 
 		// +2
 		Δ("T/B1:a,2:b",
-			A{1:  K(1,2),
-			  2:  K(1,2),
+			A{1:  K(1,2,oo),
+			  2:  K(1,2,oo),
 			  oo: K(1,2,oo)}),
 
 		// -1
 		Δ("T/B2:b",
-			A{1:  K(1,2),
-			  2:  K(1,2),
+			A{1:  K(1,2,oo),
+			  2:  K(1,2,oo),
 			  oo: K(1,2,oo)}),
 
 		// 2: b->c
 		Δ("T/B2:c",
-			A{2:  K(2),
+			A{2:  K(2,oo),
 			  oo: K(2,oo)}),
 
 		// +1 in new bucket (to the left)
 		Δ("T2/B1:a-B2:c",
-			A{1:  K(1,2),
-			  2:  K(2),
-			  oo: K(2,oo)}), // NOTE no ∞ -> 1, because 1 is added to the left
+			A{1:  K(1,2,oo),
+			  2:  K(1,2,oo),
+			  oo: K(1,2,oo)}),
 
 		// +3 in new bucket (to the right)
 		Δ("T2,3/B1:a-B2:c-B3:c",
 			A{1:  K(1),
-			  2:  K(2),
-			  3:  K(2,3),
+			  2:  K(2,3,oo),
+			  3:  K(2,3,oo),
 			  oo: K(2,3,oo)}),
 
 		// bucket split; +3 in new bucket
 		"T/B1:a,2:b",
 		Δ("T2/B1:a-B2:b,3:c",
-			A{1:  K(1,2,3),
-			  2:  K(1,2,3),
-			  3:  K(1,2,3),
+			A{1:  K(1,2,3,oo),
+			  2:  K(1,2,3,oo),
+			  3:  K(1,2,3,oo),
 			  oo: K(1,2,3,oo)}),
 
 		// bucket split; +3 in new bucket; +4 +5 in another new bucket
-		// which remain not tracked unless previously accessed.
+		// everything becomes tracked because original bucket had [-∞,∞) coverage
 		"T/B1:a,2:b",
 		Δ("T2,4/B1:a-B2:b,3:c-B4:d,5:e",
-			A{1:  K(1,2,3),
-			  2:  K(1,2,3),
-			  3:  K(1,2,3),
-			  4:  K(1,2,3,4,5),
-			  5:  K(1,2,3,4,5),
+			A{1:  K(1,2,3,4,5,oo),
+			  2:  K(1,2,3,4,5,oo),
+			  3:  K(1,2,3,4,5,oo),
+			  4:  K(1,2,3,4,5,oo),
+			  5:  K(1,2,3,4,5,oo),
 			  oo: K(1,2,3,4,5,oo)}),
 
 		// reflow of keys: even if tracked={1}, changes to all B nodes need to be rescanned:
@@ -1419,13 +1464,13 @@ func TestΔBTail(t *testing.T) {
 		// forces to look into +B34 and so on.
 		"T2,4,6/B1:a-B2:b,3:c-B4:d,5:e-B6:f,7:g",
 		Δ("T3,5,7/B1:g,2:f-B3:e,4:d-B5:c,6:b-B7:a",
-			A{1:  K(1,2,3,4,5,6,7),
-			  2:  K(1,2,3,4,5,6,7),
-			  3:  K(1,2,3,4,5,6,7),
-			  4:  K(1,2,3,4,5,6,7),
-			  5:  K(1,2,3,4,5,6,7),
-			  6:  K(1,2,3,4,5,6,7),
-			  7:  K(1,2,3,4,5,6,7),
+			A{1:  K(1,2,3,4,5,6,7,oo),
+			  2:  K(1,2,3,4,5,6,7,oo),
+			  3:  K(1,2,3,4,5,6,7,oo),
+			  4:  K(1,2,3,4,5,6,7,oo),
+			  5:  K(1,2,3,4,5,6,7,oo),
+			  6:  K(1,2,3,4,5,6,7,oo),
+			  7:  K(1,2,3,4,5,6,7,oo),
 			  oo: K(1,2,3,4,5,6,7,oo)}),
 
 		// reflow of keys for rebuild: even if tracked1={}, tracked2={1}, changes to
@@ -1440,44 +1485,57 @@ func TestΔBTail(t *testing.T) {
 		"T2,5,8/B1:a-B2:b,3:c-B5:e,6:f-B8:h,9:i",
 
 		// depth=2; bucket split; +3 in new bucket; left T remain
-		// _unchanged_ even though B under it is modified; right arm is
-		// added to tracking set because 2 migrates there.
+		// _unchanged_ even though B under it is modified.
 		"T/T/B1:a,2:b",
 		Δ("T2/T-T/B1:a-B2:b,3:c",
-			A{1:  K(1,2,3),
-			  2:  K(1,2,3),
-			  3:  K(1,2,3),
+			A{1:  K(1,2,3,oo),
+			  2:  K(1,2,3,oo),
+			  3:  K(1,2,3,oo),
 			  oo: K(1,2,3,oo)}),
 
-		// depth=2; like prev. case, but additional right arm with +4
-		// +5 is added which remain not tracked unless previously accessed.
+		// depth=2; like prev. case, but additional right arm with +4 +5 is added.
 		"T/T/B1:a,2:b",
 		Δ("T2,4/T-T-T/B1:a-B2:b,3:c-B4:d,5:e",
-			A{1:  K(1,2,3),
-			  2:  K(1,2,3),
-			  3:  K(1,2,3),
-			  4:  K(1,2,3,4,5),
-			  5:  K(1,2,3,4,5),
+			A{1:  K(1,2,3,4,5,oo),
+			  2:  K(1,2,3,4,5,oo),
+			  3:  K(1,2,3,4,5,oo),
+			  4:  K(1,2,3,4,5,oo),
+			  5:  K(1,2,3,4,5,oo),
 			  oo: K(1,2,3,4,5,oo)}),
 
 		// depth=2; bucket split; +3 in new bucket; t0 and t1 split;
-		// +right arm (T7/B45-B89) which remain not tracked unless previously accessed.
+		// +right arm (T7/B45-B89).
 		"T/T/B1:a,2:b",
 		Δ("T4/T2-T7/B1:a-B2:b,3:c-B4:d,5:e-B8:h,9:i",
-			A{1:  K(1,2,3),
-			  2:  K(1,2,3),
-			  3:  K(1,2,3),
-			  4:  K(1,2,3,4,5),
-			  5:  K(1,2,3,4,5),
-			  8:  K(1,2,3, 8,9),
-			  9:  K(1,2,3, 8,9),
-			  oo: K(1,2,3, 8,9,oo)}),
+			A{1:  K(1,2,3,4,5,8,9,oo),
+			  2:  K(1,2,3,4,5,8,9,oo),
+			  3:  K(1,2,3,4,5,8,9,oo),
+			  4:  K(1,2,3,4,5,8,9,oo),
+			  5:  K(1,2,3,4,5,8,9,oo),
+			  8:  K(1,2,3,4,5,8,9,oo),
+			  9:  K(1,2,3,4,5,8,9,oo),
+			  oo: K(1,2,3,4,5,8,9,oo)}),
 
 
 		// 2 reflow to right B neighbour; 8 split into new B; δ=ø
 		"T3/B1:a,2:b-B4:d,8:h",
 		"T2,5/B1:a-B2:b,4:d-B8:h", // XXX add A
 
+		// case where kadj does not grow too much as leafs coverage remains stable
+		"T4,8/B1:a,2:b-B5:d,6:e-B10:g,11:h",
+		Δ("T4,8/B2:b,3:c-B6:e,7:f-B11:h,12:i",
+			A{1:  K(1,2,3),
+			  2:  K(1,2,3),
+			  3:  K(1,2,3),
+			  5:  K(5,6,7),
+			  6:  K(5,6,7),
+			  7:  K(5,6,7,),
+			  10: K(10,11,12,oo),
+			  11: K(10,11,12,oo),
+			  12: K(10,11,12,oo),
+			  oo: K(10,11,12,oo)}),
+
+
 		// tree deletion
 		// having ø in the middle of the test cases exercises all:
 		// * `ø -> Tree ...`		(tree is created anew),
@@ -1565,8 +1623,8 @@ func TestΔBTail(t *testing.T) {
 		"T2/B1:a-B3:c",
 		Δ("T2/T-T4/B1:b-B3:d-B99:h",
 			A{1:  K(1),
-			  3:  K(3),
-			  99: K(3,99),
+			  3:  K(3,99,oo),
+			  99: K(3,99,oo),
 			  oo: K(3,99,oo)}),
 		// XXX --------
 
@@ -1824,7 +1882,7 @@ func (xkv RBucketSet) Flatten() map[Key]string {
 
 // allTestKeys returns all keys from vt + ∞.
 func allTestKeys(vt ...*tTreeCommit) SetKey {
-	allKeys := SetKey{}; allKeys.Add(kInf) // ∞ simulating ZBigFile.Size() query
+	allKeys := SetKey{}; allKeys.Add(KeyMax) // ∞ simulating ZBigFile.Size() query
 	for _, t := range vt {
 		for _, b := range t.xkv {
 			for k := range b.kv {
@@ -1881,9 +1939,11 @@ func sortedKeys(kv map[Key]Δstring) []Key {
 
 func (b *RBucket) String() string {
 	// XXX dup wrt nodeInRange.String
-	slo  := "-∞";  if b.lo  > KeyMin { slo = fmt.Sprintf("%v", b.lo) }
-	shi  := "∞";   if b.hi_ < KeyMax { shi = fmt.Sprintf("%v", b.hi_+1) }
-	return fmt.Sprintf("[%s,%s)B%s{%s}", slo, shi, b.oid, kvtxt(b.kv))
+	hi := b.hi_
+	if hi < KeyMax {
+		hi += 1
+	}
+	return fmt.Sprintf("[%s,%s)B%s{%s}", kstr(b.lo), kstr(hi), b.oid, kvtxt(b.kv))
 
 }