X ΔBtail.rebuild started to work draftly

* t2: (50 commits)
  .
  X rebuild: Serial tests now started to PASS draftly
  .
  X rebuild: Don't return nil for empty ΔPPTreeSubSet - that leads to SIGSEGV
  X treediff: Fix BUG while computing AB coverage
  X ΔBtail.Clone had bug that it was aliasing klon and orig data
  X Fix rebuild with ø @at2
  X Fix bug in PPTreeSubSet.Difference  - it was always leaving root node alive
  .
  .
  X found why TestΔBTailAllStructs was not effective to find δtkeycov bugs
  .
  .
  X wcfs: assert that keycov only grow
  .
  X another bug in δtkeyconv computation
  .
  .
  .
  .
  ...

wcfs/internal/set/set.go.in

@@ -136,6 +136,15 @@ func (a Set) Equal(b Set) bool {
 	return true
 }
 
+// Clone returns copy of the set.
+func (orig Set) Clone() Set {
+	klon := make(Set, len(orig))
+	for v := range orig {
+		klon.Add(v)
+	}
+	return klon
+}
+
 // --------
 
 func (s Set) SortedElements() []VALUE {

wcfs/internal/set/zset_i64.go

@@ -138,6 +138,15 @@ func (a SetI64) Equal(b SetI64) bool {
 	return true
 }
 
+// Clone returns copy of the set.
+func (orig SetI64) Clone() SetI64 {
+	klon := make(SetI64, len(orig))
+	for v := range orig {
+		klon.Add(v)
+	}
+	return klon
+}
+
 // --------
 
 func (s SetI64) SortedElements() []int64 {

wcfs/internal/set/zset_oid.go

@@ -138,6 +138,15 @@ func (a SetOid) Equal(b SetOid) bool {
 	return true
 }
 
+// Clone returns copy of the set.
+func (orig SetOid) Clone() SetOid {
+	klon := make(SetOid, len(orig))
+	for v := range orig {
+		klon.Add(v)
+	}
+	return klon
+}
+
 // --------
 
 func (s SetOid) SortedElements() []_Oid {

wcfs/internal/xbtree/pptreesubset.go

@@ -27,6 +27,7 @@ import (
 )
 
 const tracePPSet = false
+const debugPPSet = false
 
 // PPTreeSubSet represents PP-connected subset of tree node objects.
 //
@@ -58,9 +59,13 @@ type PPTreeSubSet map[zodb.Oid]*nodeInTree
 type nodeInTree struct {
 	parent  zodb.Oid // parent node | InvalidOid for root
 	nchild  int      // number of direct children in PPTreeSubSet referring to this node
-	// XXX + [lo,hi) range this node is coming under in its parent	XXX -> in its tree ?
 }
 
+// Has returns whether node is in the set.
+func (S PPTreeSubSet) Has(oid zodb.Oid) bool {
+	_, ok := S[oid]
+	return ok
+}
 
 // Path returns path leading to the node specified by oid.
 //
@@ -84,6 +89,8 @@ func (S PPTreeSubSet) Path(oid zodb.Oid) (path []zodb.Oid) {
 }
 
 // AddPath adds path to a node to the set.
+//
+// Note: embedded buckets (leaf node with InvalidOid) are removed from the path.
 func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
 	S.verify()
 	defer S.verify()
@@ -93,18 +100,11 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
 		panic("empty path")
 	}
 
-	// don't keep track of artificial empty tree
-	if l == 1 && path[0] == zodb.InvalidOid {
-		return
-	}
-
-	// don't explicitly keep track of embedded buckets - they all have
-	// InvalidOid, and thus, if kept in S, e.g. T/B1:a and another
-	// T/B2:b would lead to InvalidOid having multiple parents.
-	if l == 2 && path[1] == zodb.InvalidOid {
-		path = path[:1]
-	}
+	// normalize path: remove embedded bucket and check whether it was an
+	// artificial empty tree.
+	path = normPath(path)
 
+	// go through path and add nodes to the set
 	parent := zodb.InvalidOid
 	var pt *nodeInTree = nil
 	for _, oid := range path {
@@ -132,11 +132,33 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
 	}
 }
 
-// ---- Union/Difference/Intersetctior ----
+// normPath normalizes path.
+//
+// It removes embedded buckets and artificial empty trees.
+// Returned slice is subslice of path and aliases its memory.
+func normPath(path []zodb.Oid) []zodb.Oid {
+	l := len(path)
+
+	// don't keep track of artificial empty tree
+	if l == 1 && path[0] == zodb.InvalidOid {
+		return nil
+	}
+
+	// don't explicitly keep track of embedded buckets - they all have
+	// InvalidOid, and thus, if kept in S, e.g. T/B1:a and another
+	// T/B2:b would lead to InvalidOid having multiple parents.
+	if l == 2 && path[1] == zodb.InvalidOid {
+		return path[:1]
+	}
+
+	return path
+}
+
+// ---- Union/Difference/Intersection ----
 
 // Union returns U = PP(A.leafs | B.leafs)
 //
-// In other words it returns sum of A and B.
+// In other words it adds A and B nodes.
 func (A PPTreeSubSet) Union(B PPTreeSubSet) PPTreeSubSet {
 	U := A.Clone()
 	U.UnionInplace(B)
@@ -163,7 +185,7 @@ func (A PPTreeSubSet) UnionInplace(B PPTreeSubSet) {
 
 // Difference returns D = PP(A.leafs \ B.leafs)
 //
-// In other words ... XXX
+// In other words it removes B nodes from A while still maintaining A as PP-connected.
 func (A PPTreeSubSet) Difference(B PPTreeSubSet) PPTreeSubSet {
 	D := A.Clone()
 	D.DifferenceInplace(B)
@@ -189,7 +211,7 @@ func (A PPTreeSubSet) DifferenceInplace(B PPTreeSubSet) {
 	A.xDifferenceInplace(B)
 }
 
-// XXX Intersection
+// TODO Intersection
 
 func (A PPTreeSubSet) xUnionInplace(B PPTreeSubSet) {
 	if tracePPSet {
@@ -269,10 +291,16 @@ func (A PPTreeSubSet) fixup(δnchild map[zodb.Oid]int) {
 	A.xfixup(+1, δnchild)
 }
 func (A PPTreeSubSet) xfixup(sign int, δnchild map[zodb.Oid]int) {
-	//fmt.Printf("\nfixup:\n")
-	//fmt.Printf("    ·: %s\n", A)
-	//fmt.Printf("    δ: %v\n", δnchild)
-	//defer fmt.Printf("  ->·: %s\n\n", A)
+	if debugPPSet {
+		ssign := "+"
+		if sign < 0 {
+			ssign = "-"
+		}
+		fmt.Printf("\n  fixup:\n")
+		fmt.Printf("      ·: %s\n", A)
+		fmt.Printf("     %sδ: %v\n", ssign, δnchild)
+		defer fmt.Printf("    ->·: %s\n\n", A)
+	}
 
 	gcq := []zodb.Oid{}
 	for oid, δnc := range δnchild {
@@ -304,7 +332,7 @@ func (S PPTreeSubSet) gc1(oid zodb.Oid) {
 	for oid != zodb.InvalidOid {
 		t := S[oid]
 		t.nchild--
-		if t.nchild > 0 || /* root node */t.parent == zodb.InvalidOid {
+		if t.nchild > 0 {
 			break
 		}
 		delete(S, oid)
@@ -395,6 +423,11 @@ func (A PPTreeSubSet) Equal(B PPTreeSubSet) bool {
 	return true
 }
 
+// Empty returns whether set is empty.
+func (S PPTreeSubSet) Empty() bool {
+	return len(S) == 0
+}
+
 func (t nodeInTree) String() string {
 	return fmt.Sprintf("{p%s c%d}", t.parent, t.nchild)
 }
@@ -440,13 +473,22 @@ func (t nodeInTree) String() string {
 //         - δ.Add, and
 //         - xfixup(+1, δnchildNonLeafs)
 //
-//     produce correctly PP-connected set.
+//     produces correctly PP-connected set.
 type ΔPPTreeSubSet struct {
 	Del PPTreeSubSet
 	Add PPTreeSubSet
 	δnchildNonLeafs map[zodb.Oid]int
 }
 
+// NewΔPPTreeSubSet creates new empty ΔPPTreeSubSet.
+func NewΔPPTreeSubSet() *ΔPPTreeSubSet {
+	return &ΔPPTreeSubSet{
+		Del: PPTreeSubSet{},
+		Add: PPTreeSubSet{},
+		δnchildNonLeafs: map[zodb.Oid]int{},
+	}
+}
+
 // Update updates δ to be combination of δ+δ2.
 func (δ *ΔPPTreeSubSet) Update(δ2 *ΔPPTreeSubSet) {
 	δ.Del.UnionInplace(δ2.Del)

wcfs/internal/xbtree/pptreesubset_test.go

@@ -20,6 +20,7 @@
 package xbtree
 
 import (
+	"strings"
 	"testing"
 
 	"lab.nexedi.com/kirr/neo/go/zodb"
@@ -79,19 +80,44 @@ func TestPPTreeSubSetOps(t *testing.T) {
 			S{a:{ø,1}, b:{a,0}},	// B
 				S{a:{ø,1}, b:{a,0}},	// U
 				S{}),			// D
-	}
 
-	for _, tt := range testv {
-		U := tt.A.Union(tt.B)
-		D := tt.A.Difference(tt.B)
+		E(
+			S{a:{ø,1}, b:{a,1}, c:{b,0}},	// A
+			S{a:{ø,1}, b:{a,1}, c:{b,0}},	// B (=A)
+				S{a:{ø,1}, b:{a,1}, c:{b,0}},	// U (=A)
+				S{}),				// D
+	}
 
-		if !U.Equal(tt.Union) {
-			t.Errorf("Union:\n  A:   %s\n  B:   %s\n  ->u: %s\n  okU: %s\n", tt.A, tt.B, U, tt.Union)
+	// assert1 asserts that result of op(A,B) == resOK.
+	assert1 := func(op string, A, B, res, resOK S) {
+		t.Helper()
+		if res.Equal(resOK) {
+			return
 		}
-		if !D.Equal(tt.Difference) {
-			t.Errorf("Difference:\n  A:   %s\n  B:   %s\n  ->d: %s\n  okD: %s\n", tt.A, tt.B, D, tt.Difference)
+		op1 := op[0:1]
+		t.Errorf("%s:\n  A:   %s\n  B:   %s\n  ->%s: %s\n  ok%s: %s\n",
+			strings.Title(op), A, B, op1, res, strings.ToUpper(op1), resOK)
 	}
 
+	for _, tt := range testv {
+		Uab := tt.A.Union(tt.B)
+		Uba := tt.B.Union(tt.A)
+		Dab := tt.A.Difference(tt.B)
+
+		assert1("union",      tt.A, tt.B, Uab, tt.Union)
+		assert1("union",      tt.B, tt.A, Uba, tt.Union)
+		assert1("difference", tt.A, tt.B, Dab, tt.Difference)
+
+		Uaa := tt.A.Union(tt.A)
+		Ubb := tt.B.Union(tt.B)
+		Daa := tt.A.Difference(tt.A)
+		Dbb := tt.B.Difference(tt.B)
+
+		assert1("union",      tt.A, tt.A, Uaa, tt.A)
+		assert1("union",      tt.B, tt.B, Ubb, tt.B)
+		assert1("difference", tt.A, tt.A, Daa, S{})
+		assert1("difference", tt.B, tt.B, Dbb, S{})
+
 		// XXX also verify U/D properties like (A+B)\B + (A+B)\A + (A^B) == (A+B) ?
 	}
 }

wcfs/internal/xbtree/testprog/treegen.py

@@ -20,7 +20,7 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 """Program treegen provides infrastructure to generate ZODB BTree states.
 
-It is used as helper for ΔBTree tests.
+It is used as helper for ΔBtail tests.
 
 The following subcommands are provided:
 
@@ -319,6 +319,9 @@ def AllStructs(kv1txt, kv2txt, maxdepth, maxsplit, n, seed=None):
     t1structv.insert(0, t1struct0)
     t2structv.insert(0, t2struct0)
 
+    # XXX rework allstructs to accept only 1 kv and emit n structs for that one kv only
+    #     -> iterate through the pairs/triplets in the caller (TestΔBTailAllStructs)
+
     # emit topologies for tree1->tree2 and tree1<-tree2 transitions for all
     # combinations of tree1 and tree2.
     t12travel = list(bitravel2Way(t1structv, t2structv))

wcfs/internal/xbtree/treediff.go

@@ -159,7 +159,7 @@ func δZConnectTracked(δZv []zodb.Oid, T PPTreeSubSet) (δZTC SetOid, δtopsByR
 // nodeInRange represents a Node coming under [lo, hi_] key range in its tree.
 type nodeInRange struct {
 	prefix   []zodb.Oid // path to this node goes via this objects
-	lo, hi_  Key  // [lo, hi_]	NOTE _not_ hi) not to overflow at ∞	XXX -> Range
+	lo, hi_  Key  // [lo, hi_]	NOTE _not_ hi) not to overflow at ∞	XXX -> keycov KeyRange?
 	node     Node
 	done     bool // whether this node was already taken into account while computing diff
 }
@@ -330,7 +330,7 @@ func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, t
 	defer xerr.Contextf(&err, "treediff %s..%s %s", zconnOld.At(), zconnNew.At(), root)
 
 	δT = map[Key]ΔValue{}
-	δtrack = &ΔPPTreeSubSet{Del: PPTreeSubSet{}, Add: PPTreeSubSet{}, δnchildNonLeafs: map[zodb.Oid]int{}}
+	δtrack = NewΔPPTreeSubSet()
 	δtkeycov = &RangedKeySet{}
 
 	tracefDiff("\ntreediff %s  δtops: %v  δZTC: %v\n", root, δtops, δZTC)
@@ -429,7 +429,7 @@ func diffX(ctx context.Context, a, b Node, δZTC SetOid, trackSet PPTreeSubSet)
 		var δtrack *ΔPPTreeSubSet
 		δ, err := diffB(ctx, aB, bB)
 		if δ != nil {
-			δtrack = &ΔPPTreeSubSet{}
+			δtrack = NewΔPPTreeSubSet()
 			δtkeycov = &RangedKeySet{}
 		}
 		return δ, δtrack, δtkeycov, err
@@ -445,7 +445,7 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet)
 	defer xerr.Contextf(&err, "diffT %s %s", xidOf(A), xidOf(B))
 
 	δ = map[Key]ΔValue{}
-	δtrack = &ΔPPTreeSubSet{Del: PPTreeSubSet{}, Add: PPTreeSubSet{}, δnchildNonLeafs: map[zodb.Oid]int{}}
+	δtrack = NewΔPPTreeSubSet()
 	δtkeycov = &RangedKeySet{}
 	defer func() {
 		tracefDiff("  -> δ: %v\n", δ)
@@ -487,6 +487,9 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet)
 ABcov:
 	for i := len(ABpath)-2; i >= 0; i-- {
 		xparent, err := node.PJar().Get(ctx, ABpath[i]);  /*X*/if err != nil { return nil,nil,nil, err }
+		err = xparent.PActivate(ctx);  /*X*/if err != nil { return nil,nil,nil, err}
+		defer xparent.PDeactivate()
+
 		parent := xparent.(*Tree) // must succeed
 		// find node in parent children and constrain ABlo/ABhi accordingly
 		entryv := parent.Entryv()
@@ -512,7 +515,14 @@ ABcov:
 			}
 		}
 
-		panicf("BUG: T%s points to T%s as parent in trackSet, but not found in T%s children", node.POid(), parent.POid(), parent.POid())
+		emsg := fmt.Sprintf("BUG: T%s points to T%s as parent in trackSet, but not found in T%s children\n", node.POid(), parent.POid(), parent.POid())
+		children := []string{}
+		for _, entry := range entryv {
+			children = append(children, vnode(entry.Child()))
+		}
+		emsg += fmt.Sprintf("T%s children: %v\n", parent.POid(), children)
+		emsg += fmt.Sprintf("trackSet: %s\n", trackSet)
+		panic(emsg)
 	}
 
 
@@ -565,10 +575,11 @@ ABcov:
 		}
 	}
 
-	// {} oid -> parent for all nodes in Bv: current and previously expanded - up till top B
-	BtrackSet := PPTreeSubSet{}
+	// {} oid -> nodeInRange for all nodes we've came through in Bv:
+	//                       current and previously expanded - up till top B.
+	BnodeIdx := map[zodb.Oid]*nodeInRange{}
 	if !Bempty {
-		BtrackSet.AddPath(ABpath)
+		BnodeIdx[ABoid] = btop
 	}
 
 	// δtkeycov will be = BAdd \ ADel
@@ -579,7 +590,8 @@ ABcov:
 	// by default a node contributes to δ-
 	// a node ac does not contribute to δ- and can be skipped, if:
 	// - ac is not tracked, or
-	// - ac ∉ δZTC && ∃ bc from B: ac.oid == bc.oid	(ac+ac.children were not changed, and ac stays in the tree)
+	// - ac ∉ δZTC && ∃ bc from B: ac.oid == bc.oid && ac.keycov == bc.keycov
+	//   (ac+ac.children were not changed, ac stays in the tree with the same key range coverage)
 	Aq := []*nodeInRange{atop} // queue for A nodes that contribute to δ-
 	for len(Aq) > 0 {
 		debugfDiff("\n")
@@ -606,7 +618,7 @@ ABcov:
 			ra.done = true
 
 		case *Tree:
-			// empty tree - only queue holes covered by it
+			// empty tree - queue holes covered by it
 			if len(a.Entryv()) == 0 {
 				ar := KeyRange{ra.lo, ra.hi_}
 				δtrack.Del.AddPath(ra.Path())
@@ -632,7 +644,7 @@ ABcov:
 					// ( this does not give exact answer but should be a reasonable heuristic;
 					//   the diff is the same if heuristic does not work and we
 					//   look into and load more nodes to compute δ )
-					_, found := BtrackSet[acOid]
+					bc, found := BnodeIdx[acOid]
 					if !found {
 						for {
 							blo  := Bv.Get(ac.lo)
@@ -652,11 +664,12 @@ ABcov:
 							}
 
 							bchildren := Bv.Expand(blo)
-							for _, bc := range bchildren {
-									bcOid := bc.node.POid()
-									BtrackSet.AddPath(bc.Path())
-									if acOid == bcOid {
+							for _, bc_ := range bchildren {
+									bc_Oid := bc_.node.POid()
+									BnodeIdx[bc_Oid] = bc_
+									if acOid == bc_Oid {
 										found = true
+										bc = bc_
 									}
 							}
 							if found {
@@ -665,31 +678,25 @@ ABcov:
 						}
 					}
 					if found {
-						// ac can be skipped
+						// ac can be skipped if key coverage stays the same
+						ar := KeyRange{ac.lo, ac.hi_}
+						br := KeyRange{bc.lo, bc.hi_}
+						if ar == br {
 							// adjust trackSet since path to the node could have changed
-						apath := trackSet.Path(acOid)
-						bpath := BtrackSet.Path(acOid)
+							apath := ac.Path()
+							bpath := bc.Path()
 							if !pathEqual(apath, bpath) {
 								δtrack.Del.AddPath(apath)
 								δtrack.Add.AddPath(bpath)
 								if nc := at.nchild; nc != 0 {
 									δtrack.δnchildNonLeafs[acOid] = nc
-								// XXX debugDiff δtrack [) ...
-							} else {
-								// adjust δtkeycov only if it was leaf bucket
-								ar := KeyRange{ac.lo, ac.hi_}
-								bc := Bv.Get(ac.lo)
-								br := KeyRange{bc.lo, bc.hi_}
-								δtkeycovADel.AddRange(ar)
-								δtkeycovBAdd.AddRange(br)
-								debugfDiff("  δtrack - %s %v KKK\n", ar, apath)
-								debugfDiff("  δtrack + %s %v KKK\n", br, bpath)
 								}
 							}
 
 							continue
 						}
 					}
+				}
 
 				// ac cannot be skipped
 				push(&Aq, ac)
@@ -751,11 +758,10 @@ ABcov:
 					b.done = true
 				}
 
-				// stop if r coverage is complete
+				// continue with next right bucket until r coverage is complete
 				if r.hi_ <= b.hi_ {
 					break
 				}
-				// continue with next right bucket
 				lo = b.hi_ + 1
 			}
 		}

wcfs/internal/zdata/δftail.go

@@ -160,7 +160,7 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zb
 		// XXX blk = ∞ from beginning ?
 		blk = xbtree.KeyMax
 	}
-	err := δFtail.δBtail.Track(blk, zblk != nil, path)
+	err := δFtail.δBtail.Track(blk, path)
 	if err != nil {
 		panic(err) // XXX -> error? errctx
 	}
@@ -233,7 +233,7 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
 	δF := ΔF{Rev: δB.Rev, ByFile: make(map[*ZBigFile]*ΔFile)}
 
 	// take btree changes into account
-	for root, δt := range δB.ByRoot {
+	for root, δt := range δB.ΔByRoot {
 		files := δFtail.fileIdx[root]
 		if len(files) == 0 {
 			panicf("BUG: ΔFtail: root<%s> -> ø files", root)