X wcfs: Teach treediff to compute not only δtrack (set of nodes), but also δ for track-key coverage

This will, hopefully, be used to fix rebuild.

* t2:
  .
  .
  .
  .
  .
  .
  .
  X treediff ret += δtkeycov
  .
  .

wcfs/internal/xbtree/pptreesubset.go

@@ -54,10 +54,11 @@ const tracePPSet = false
 type PPTreeSubSet map[zodb.Oid]*nodeInTree
 
 // nodeInTree represents tracking information about a node.
+// XXX -> treeNode ?
 type nodeInTree struct {
 	parent  zodb.Oid // parent node | InvalidOid for root
 	nchild  int      // number of direct children in PPTreeSubSet referring to this node
-	// XXX + [lo,hi) range this node is coming under in its parent
+	// XXX + [lo,hi) range this node is coming under in its parent	XXX -> in its tree ?
 }
 
 

wcfs/internal/xbtree/rangeset.go

@@ -398,11 +398,13 @@ func (S RangedKeySet) String() string {
 }
 
 func (r KeyRange) String() string {
-	hi := r.hi_
-	if hi < KeyMax {
-		hi += 1
+	var shi string
+	if r.hi_ == KeyMax {
+		shi = kstr(r.hi_) // ∞
+	} else {
+		shi = fmt.Sprintf("%d", r.hi_+1)
 	}
-	return fmt.Sprintf("[%s,%s)", kstr(r.lo), kstr(hi))
+	return fmt.Sprintf("[%s,%s)", kstr(r.lo), shi)
 }
 
 

wcfs/internal/xbtree/xbtree.go

@@ -36,13 +36,6 @@ type SetOid = set.SetOid
 
 
 
-const debug = false
-func tracef(format string, argv ...interface{}) {
-	if debug {
-		fmt.Printf(format, argv...)
-	}
-}
-
 // nodePathToPath converts path from []Node to []Oid.
 // XXX place = ?
 func nodePathToPath(nodePath []Node) (path []zodb.Oid) {

wcfs/internal/xbtree/δbtail.go

@@ -22,6 +22,7 @@ package xbtree
 
 import (
 	"context"
+	"fmt"
 	"sort"
 	"strings"
 
@@ -36,6 +37,10 @@ import (
 // TODO kill this after rebuild is finished
 const XXX_killWhenRebuildWorks = true
 
+const traceΔBtail = false
+const debugΔBtail = false
+
+
 // ΔBtail represents tail of revisional changes to BTrees.
 //
 // It semantically consists of
@@ -99,7 +104,7 @@ type ΔBtail struct {
 	// set of tracked nodes as of @head state.
 	trackSet PPTreeSubSet
 
-	// set of node that were requested to be tracked, but for which vδB was not yet rebuilt
+	// set of nodes that were requested to be tracked, but for which vδB was not yet rebuilt
 	trackNew PPTreeSubSet
 }
 
@@ -217,18 +222,19 @@ func (δBtail *ΔBtail) Track(key Key, keyPresent bool, nodePath []Node) error {
 
 	pathv := []string{}
 	for _, node := range nodePath { pathv = append(pathv, vnode(node)) }
-	tracef("Track [%v] %s\n", key, strings.Join(pathv, " -> "))
+	tracefΔBtail("\nTrack [%v] %s\n", key, strings.Join(pathv, " -> "))
 
 	return δBtail.track(key, keyPresent, path)
 }
 
 func (δBtail *ΔBtail) track(key Key, keyPresent bool, path []zodb.Oid) error {
+	// XXX locking
+
 	root := path[0]
 	δBtail.trackNew.AddPath(path)
 
 	// track is track of path[-1]  (i.e. leaf)
 
-
 // XXX hack - until rebuild is implemented
 if XXX_killWhenRebuildWorks {
 	_, ok := δBtail.byRoot[root]
@@ -248,6 +254,10 @@ func (δBtail *ΔBtail) rebuild() (err error) {
 	defer xerr.Context(&err, "ΔBtail rebuild")
 	// XXX locking
 
+	tracefΔBtail("\nRebuild @%s .. @%s\n", δBtail.Tail(), δBtail.Head())
+	tracefΔBtail("trackSet:       %v\n", δBtail.trackSet)
+	tracefΔBtail("trackNew:       %v\n", δBtail.trackNew)
+
 	trackNew := δBtail.trackNew
 	δBtail.trackNew = PPTreeSubSet{}
 
@@ -272,11 +282,11 @@ func (δBtail *ΔBtail) rebuild() (err error) {
 
 		δZTC, δtopsByRoot := δZConnectTracked(δZ.Changev, trackNew)
 
-		tracef("\nrebuild @%s <- @%s\n", atPrev, δZ.Rev)
-		tracef("δZ:\t%v\n", δZ.Changev)
-		tracef("trackNew:  %v\n", trackNew)
-		tracef("trackSet:  %v\n", δBtail.trackSet)	// XXX needed?
-		defer tracef("\n\n")
+		debugfΔBtail("\n  rebuild @%s <- @%s\n", atPrev, δZ.Rev)
+		debugfΔBtail("  δZ:\t%v\n", δZ.Changev)
+		debugfΔBtail("  trackNew:  %v\n", trackNew)
+		debugfΔBtail("  trackSet:  %v\n", δBtail.trackSet)	// XXX needed?
+		defer debugfΔBtail("\n\n")
 
 		// XXX len(δtopsByRoot) == 0  -> skip
 
@@ -296,12 +306,15 @@ func (δBtail *ΔBtail) rebuild() (err error) {
 
 		for root, δtops := range δtopsByRoot {
 			// diff backwards curr -> prev
-			δT, δtrack, err := treediff(ctx, root, δtops, δZTC, trackNew, zconnCurr, zconnPrev)
+			δT, δtrack, δtkeycov, err := treediff(ctx, root, δtops, δZTC, trackNew, zconnCurr, zconnPrev)
 			if err != nil {
 				return err
 			}
 
-			tracef("-> root<%s>  δkv*: %v  δtrack*: %v\n", root, δT, δtrack)
+			// FIXME use δtkeycov to recompute track coverage
+			_ = δtkeycov
+
+			debugfΔBtail("  -> root<%s>  δkv*: %v  δtrack*: %v\n", root, δT, δtrack)
 
 			trackNew.ApplyΔ(δtrack)
 			vδtrack = append([]*ΔPPTreeSubSet{δtrack}, vδtrack...)
@@ -371,9 +384,24 @@ func (δBtail *ΔBtail) rebuild() (err error) {
 //
 // TODO optionally accept zconnOld/zconnNew from client
 func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
+	δB, _, err := δBtail._Update(δZ)
+	return δB, err
+}
+
+// _Update serves Update and also returns δtkeycov from treediff for testing.
+type _ΔTrackKeyCov struct {
+	ByRoot map[zodb.Oid]*RangedKeySet // {} root -> δtrackedKeys  (always grow)
+}
+func (δBtail *ΔBtail) _Update(δZ *zodb.EventCommit) (_ ΔB, δTKeyCov _ΔTrackKeyCov, err error) {
 	headOld := δBtail.Head()
 	defer xerr.Contextf(&err, "ΔBtail update %s -> %s", headOld, δZ.Tid)
 
+	tracefΔBtail("\nUpdate  @%s -> @%s    δZ: %v\n", δBtail.Head(), δZ.Tid, δZ.Changev)
+	tracefΔBtail("trackSet:       %v\n", δBtail.trackSet)
+	tracefΔBtail("trackNew:       %v\n", δBtail.trackNew)
+
+	δTKeyCov = _ΔTrackKeyCov{ByRoot: make(map[zodb.Oid]*RangedKeySet)}
+
 if XXX_killWhenRebuildWorks {
 // XXX hack - until vvv is reenabled
 	δBtail.trackSet.UnionInplace(δBtail.trackNew)
@@ -383,14 +411,10 @@ if XXX_killWhenRebuildWorks {
 	// update .trackSet and vδB from .trackNew
 	err = δBtail.rebuild()
 	if err != nil {
-		return ΔB{}, err
+		return ΔB{}, δTKeyCov, err
 	}
 }
 
-	tracef("Update  @%s -> @%s\n", δBtail.Head(), δZ.Tid)
-	tracef("δZ:\t%v\n", δZ.Changev)
-	tracef("trackSet:       %v\n", δBtail.trackSet)
-
 	// XXX dup wrt rebuild?
 
 	δBtail.δZtail.Append(δZ.Tid, δZ.Changev)
@@ -401,7 +425,7 @@ if XXX_killWhenRebuildWorks {
 
 	// skip opening DB connections if there is no change to any tree node
 	if len(δtopsByRoot) == 0 {
-		return δB, nil
+		return δB, δTKeyCov, nil
 	}
 
 	// open ZODB connections corresponding to "old" and "new" states
@@ -410,20 +434,20 @@ if XXX_killWhenRebuildWorks {
 	defer txn.Abort()
 	zconnOld, err := δBtail.db.Open(ctx, &zodb.ConnOptions{At: headOld})
 	if err != nil {
-		return ΔB{}, err
+		return ΔB{}, δTKeyCov, err
 	}
 	zconnNew, err := δBtail.db.Open(ctx, &zodb.ConnOptions{At: δZ.Tid})
 	if err != nil {
-		return ΔB{}, err
+		return ΔB{}, δTKeyCov, err
 	}
 
 	for root, δtops := range δtopsByRoot {
-		δT, δtrack, err := treediff(ctx, root, δtops, δZTC, δBtail.trackSet, zconnOld, zconnNew)
+		δT, δtrack, δtkeycov, err := treediff(ctx, root, δtops, δZTC, δBtail.trackSet, zconnOld, zconnNew)
 		if err != nil {
-			return ΔB{}, err
+			return ΔB{}, δTKeyCov, err
 		}
 
-		tracef("\n-> root<%s>  δkv: %v  δtrack: %v\n", root, δT, δtrack)
+		tracefΔBtail("\n-> root<%s>  δkv: %v  δtrack: %v  δtkeycov: %v\n", root, δT, δtrack, δtkeycov)
 
 		if len(δT) > 0 { // an object might be resaved without change
 			δB.ByRoot[root] = δT
@@ -441,9 +465,10 @@ if XXX_killWhenRebuildWorks {
 		}
 
 		δBtail.trackSet.ApplyΔ(δtrack)
+		δTKeyCov.ByRoot[root] = δtkeycov
 	}
 
-	return δB, nil
+	return δB, δTKeyCov, nil
 }
 
 
@@ -563,3 +588,16 @@ func (δBtail *ΔBtail) SliceByRootRev(root zodb.Oid, lo, hi zodb.Tid) /*readonl
 func (δBtail *ΔBtail) ΔZtail() /*readonly*/*zodb.ΔTail {
 	return δBtail.δZtail
 }
+
+
+func tracefΔBtail(format string, argv ...interface{}) {
+	if traceΔBtail {
+		fmt.Printf(format, argv...)
+	}
+}
+
+func debugfΔBtail(format string, argv ...interface{}) {
+	if debugΔBtail {
+		fmt.Printf(format, argv...)
+	}
+}

wcfs/internal/xbtree/δbtail_test.go

@@ -300,11 +300,19 @@ func (rbs RBucketSet) coverage() string {
 	return s
 }
 
-// trackSet returns what should be ΔBtree.trackSet for specified tracked key set.
+// trackSet returns what should be ΔBtree.trackSet coverage for specified tracked key set.
 func (rbs RBucketSet) trackSet(tracked SetKey) PPTreeSubSet {
-	trackSet := PPTreeSubSet{}
+	trackSet, _ := rbs.trackSetWithCov(tracked)
+	return trackSet
+}
+
+// trackSetWithCov returns what should be ΔBtree.trackSet and its key coverage for specified tracked key set.
+func (rbs RBucketSet) trackSetWithCov(tracked SetKey) (trackSet PPTreeSubSet, keyCover *RangedKeySet) {
+	trackSet = PPTreeSubSet{}
+	keyCover = &RangedKeySet{}
 	for k := range tracked {
 		kb := rbs.Get(k)
+		keyCover.AddRange(KeyRange{kb.lo, kb.hi_})
 		// trackSet explicitly records only regular buckets.
 		// embedded buckets all have oid=zodb.InvalidOid and would lead to z
 		newNode := false
@@ -345,7 +353,7 @@ func (rbs RBucketSet) trackSet(tracked SetKey) PPTreeSubSet {
 			p = p.parent
 		}
 	}
-	return trackSet
+	return trackSet, keyCover
 }
 
 
@@ -666,7 +674,7 @@ func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb
 // tracked state defined by initialTrackedKeys.
 func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, at1,at2 zodb.Tid, xkv1,xkv2 RBucketSet, δZ *zodb.EventCommit, d12 map[Key]Δstring, initialTrackedKeys SetKey, kadj KAdjMatrix) {
 	X := exc.Raiseif
-	tracef("\n>>> Track=%s\n", initialTrackedKeys)
+//	t.Logf("\n>>> Track=%s\n", initialTrackedKeys)
 
 	var TrackedδZ SetKey = nil
 	var kadjTrackedδZ SetKey = nil
@@ -780,21 +788,18 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 
 	// verify δbtail.trackSet against @at1
 	// trackSet1 = xkv1[tracked1]
-	trackSet1 := xkv1.trackSet(initialTrackedKeys)
-//	if !reflect.DeepEqual(trackSet1, δbtail.trackSet) {
-//		badf("δbtail.trackSet1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, trackSet1)
-//	}
+	trackSet1, tkeyCov1 := xkv1.trackSetWithCov(initialTrackedKeys)
 	if !δbtail.trackSet.Equal(ø) {
 		badf("δbtail.trackSet1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, ø)
 	}
 	if !δbtail.trackNew.Equal(trackSet1) {
 		badf("δbtail.trackNew1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackNew, trackSet1)
 	}
-//	δbtail.assertTrack(t, "1", /*ø*/PPTreeSubSet{}, trackSet1)
+//	δbtail.assertTrack(t, "1", ø, trackSet1)
 
 
 	// δB <- δZ
-	δB, err := δbtail.Update(δZ); X(err)
+	δB, δTKeyCov, err := δbtail._Update(δZ); X(err)
 	if δB.Rev != δZ.Tid {
 		badf("δB: rev != δZ.Tid  ; rev=%s  δZ.Tid=%s", δB.Rev, δZ.Tid)
 		return
@@ -802,17 +807,14 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 
 	// verify δbtail.trackSet  against @at2
 	// trackSet2 = xkv2[tracked2]  ( = xkv2[kadj[tracked1]]
-	trackSet2 := xkv2.trackSet(initialTrackedKeys.Union(kadjTrackedδZ))
-//	if !reflect.DeepEqual(trackSet2, δbtail.trackSet) {
-//		badf("δbtail.trackSet2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, trackSet2)
-//	}
+	trackSet2, tkeyCov2 := xkv2.trackSetWithCov(initialTrackedKeys.Union(kadjTrackedδZ))
 	if !δbtail.trackSet.Equal(trackSet2) {
 		badf("δbtail.trackSet2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, trackSet2)
 	}
 	if !δbtail.trackNew.Equal(ø) {
 		badf("δbtail.trackNew2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackNew, ø)
 	}
-//	δbtail.assertTrack(t, "2", trackSet2, /*ø*/PPTreeSubSet{})
+//	δbtail.assertTrack(t, "2", trackSet2, ø)
 
 
 	// assert δB.ByRoot == {treeRoot -> ...}  if δTok != ø
@@ -833,6 +835,13 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 		return
 	}
 
+	// assert δtkeycov == δ(tkeyCov1, tkeyCov2)
+	δtkeycovOK := tkeyCov2.Difference(tkeyCov1)
+	δtkeycov := δTKeyCov.ByRoot[treeRoot]
+	if !δtkeycov.Equal(δtkeycovOK) {
+		badf("δtkeycov wrong:\nhave: %s\nwant: %s", δtkeycov, δtkeycovOK)
+	}
+
 
 	// δT <- δB
 	δToid := δB.ByRoot[treeRoot]         // {} k -> δoid
@@ -1354,7 +1363,7 @@ func testΔBTail(t *testing.T, testq chan ΔBTestEntry) {
 		t2.prev = t1						// XXX ----//----
 
 		subj := fmt.Sprintf("%s -> %s", t1.tree, t2.tree)
-		tracef("\n\n\n**** %s ****\n\n", subj)
+//		t.Logf("\n\n\n**** %s ****\n\n", subj)
 
 		// KAdj
 		if kadjOK := test.kadjOK; kadjOK != nil {
@@ -1938,12 +1947,7 @@ func sortedKeys(kv map[Key]Δstring) []Key {
 }
 
 func (b *RBucket) String() string {
-	// XXX dup wrt nodeInRange.String
-	hi := b.hi_
-	if hi < KeyMax {
-		hi += 1
-	}
-	return fmt.Sprintf("[%s,%s)B%s{%s}", kstr(b.lo), kstr(hi), b.oid, kvtxt(b.kv))
+	return fmt.Sprintf("%sB%s{%s}", KeyRange{b.lo, b.hi_}, b.oid, kvtxt(b.kv))
 
 }