X Get rebuild tests to run in a sane time; Add proper random-based testing for rebuild

Now both normal and random tests for all Update and rebuild could be
exercised as part of regular test runs.

* t2:
  X rebuild: tests: Random testing
  X rebuild: tests: Don't exercise keys from keys2 that already became tracked after Track(keys1) + Update
  X rebuild: tests: Inline _assertTrack
  X rebuild: tests: Don't compute keyCover in trackSet
  X rebuild: tests: Don't recompute trackSet(keys1R2) several times
  X rebuild: tests: Move ΔBtail.Clone test out of hot inner loop into separate test
  X tests: Factor-out tree-test-env into tTreeEnv
  X xbtree: Less copy/garbage in RangedKeySet ops
  X rebuild: tests: Precompute kadj10·kadj21
  X rebuild: tests: Don't access ZODB in xtrackKeys
  X rebuild: tests: Don't access ZODB in XGetδKV
  X rebuild: tests: Don't reflect.DeepEqual in inner loop
  .
  .
  .
  .
  .
  .

wcfs/.gitignore

 /wcfs
+*.test

wcfs/internal/xbtree/pptreesubset.go

@@ -363,12 +363,12 @@ func (S PPTreeSubSet) verify() {
 	}()
 
 	// recompute {} oid -> children  and verify .nchild against it
-	children := map[zodb.Oid]SetOid{}
+	children := make(map[zodb.Oid]SetOid, len(S))
 	for oid, t := range S {
 		if t.parent != zodb.InvalidOid {
 			cc, ok := children[t.parent]
 			if !ok {
-				cc = SetOid{}
+				cc = make(SetOid, 1)
 				children[t.parent] = cc
 			}
 			cc.Add(oid)

wcfs/internal/xbtree/rangeset.go

@@ -34,7 +34,7 @@ type KeyRange struct {
 	hi_ Key // NOTE _not_ hi) to avoid overflow at ∞;  hi = hi_ + 1
 }
 
-// RangedKeySet is set of Keys with adjacent keys coaleced into Ranges.
+// RangedKeySet is set of Keys with adjacent keys coalesced into Ranges.
 //
 // Zero value represents empty set.
 type RangedKeySet struct {
@@ -109,20 +109,14 @@ func (S *RangedKeySet) AddRange(r KeyRange) {
 	if (jhi - ilo) > 1 {
 		lo  := S.rangev[ilo].lo
 		hi_ := S.rangev[jhi-1].hi_
-		S.rangev = append(
-			S.rangev[:ilo],		append([]KeyRange{
-			KeyRange{lo, hi_}},
-			S.rangev[jhi:]...)...)
+		vReplaceSlice(&S.rangev, ilo,jhi, KeyRange{lo,hi_})
 		debugfRSet("\tmerge S[%d:%d]\t-> %s\n", ilo, jhi, S)
 	}
 	jhi = -1 // no longer valid
 
 	// if [r.lo,r.hi) was outside of any entry - create new entry
 	if r.hi_ < S.rangev[ilo].lo {
-		S.rangev = append(
-			S.rangev[:ilo],		append([]KeyRange{
-			r},
-			S.rangev[ilo:]...)...)
+		vInsert(&S.rangev, ilo, r)
 		debugfRSet("\tinsert %s\t-> %s\n", r, S)
 	}
 
@@ -140,20 +134,16 @@ func (S *RangedKeySet) AddRange(r KeyRange) {
 	// and check if we should merge it with right/left neighbours
 	if ilo+1 < len(S.rangev) { // right
 		if S.rangev[ilo].hi_+1 == S.rangev[ilo+1].lo {
-			S.rangev = append(
-				S.rangev[:ilo],		append([]KeyRange{
-				KeyRange{S.rangev[ilo].lo, S.rangev[ilo+1].hi_}},
-				S.rangev[ilo+2:]...)...)
+			vReplaceSlice(&S.rangev, ilo,ilo+2,
+				KeyRange{S.rangev[ilo].lo, S.rangev[ilo+1].hi_})
 			debugfRSet("\tmerge right\t-> %s\n", S)
 		}
 	}
 
 	if ilo > 0 { // left
 		if S.rangev[ilo-1].hi_+1 == S.rangev[ilo].lo {
-			S.rangev = append(
-				S.rangev[:ilo-1],	append([]KeyRange{
-				KeyRange{S.rangev[ilo-1].lo, S.rangev[ilo].hi_}},
-				S.rangev[ilo+1:]...)...)
+			vReplaceSlice(&S.rangev, ilo-1,ilo+1,
+				KeyRange{S.rangev[ilo-1].lo, S.rangev[ilo].hi_})
 			debugfRSet("\tmerge left\t-> %s\n", S)
 		}
 	}
@@ -204,38 +194,27 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
 	}
 
 	// [ilo+1:jhi-1] should be deleted
-	// [ilo] and [jhi-1] overlap with [r.lo,r.hi) - they shuold be deleted, or shrinked,
+	// [ilo] and [jhi-1] overlap with [r.lo,r.hi) - they should be deleted, or shrinked,
 	// or split+shrinked if ilo==jhi-1 and r is inside [ilo]
 	if jhi-ilo == 1 && S.rangev[ilo].lo < r.lo && r.hi_ < S.rangev[ilo].hi_ {
 		x := S.rangev[ilo]
-		S.rangev = append(
-			S.rangev[:ilo],		append([]KeyRange{
-			x, x},
-			S.rangev[ilo+1:]...)...)
+		vInsert(&S.rangev, ilo, x)
 		jhi++
 		debugfRSet("\tpresplit copy %s\t-> %s\n", x, S)
 	}
 	if S.rangev[ilo].lo < r.lo { // shrink left
-		S.rangev = append(
-			S.rangev[:ilo],		append([]KeyRange{
-			KeyRange{S.rangev[ilo].lo, r.lo-1}},
-			S.rangev[ilo+1:]...)...)
+		S.rangev[ilo] = KeyRange{S.rangev[ilo].lo, r.lo-1}
 		ilo++
 		debugfRSet("\tshrink [%d] left\t-> %s\n", ilo, S)
 	}
 	if r.hi_ < S.rangev[jhi-1].hi_ { // shrink right
-		S.rangev = append(
-			S.rangev[:jhi-1],	append([]KeyRange{
-			KeyRange{r.hi_+1, S.rangev[jhi-1].hi_}},
-			S.rangev[jhi:]...)...)
+		S.rangev[jhi-1] = KeyRange{r.hi_+1, S.rangev[jhi-1].hi_}
 		jhi--
 		debugfRSet("\tshrink [%d] right\t-> %s\n", jhi-1, S)
 	}
 
 	if (jhi - ilo) > 0 {
-		S.rangev = append(
-			S.rangev[:ilo],
-			S.rangev[jhi:]...)
+		vDeleteSlice(&S.rangev, ilo,jhi)
 		debugfRSet("\tdelete S[%d:%d]\t-> %s\n", ilo, jhi, S)
 	}
 
@@ -414,3 +393,32 @@ func debugfRSet(format string, argv ...interface{}) {
 	}
 	fmt.Printf(format, argv...)
 }
+
+
+// ---- slice ops ----
+
+// vInsert inserts r into *pv[i].
+func vInsert(pv *[]KeyRange, i int, r KeyRange) {
+	v := *pv
+	v = append(v, KeyRange{})
+	copy(v[i+1:], v[i:])
+	v[i] = r
+	*pv = v
+}
+
+// vDeleteSlice deletes *pv[lo:hi].
+func vDeleteSlice(pv *[]KeyRange, lo,hi int) {
+	v := *pv
+	n := copy(v[lo:], v[hi:])
+	v = v[:lo+n]
+	*pv = v
+}
+
+// vReplaceSlice replaces *pv[lo:hi] with r.
+func vReplaceSlice(pv *[]KeyRange, lo,hi int, r KeyRange) {
+	v := *pv
+	n := copy(v[lo+1:], v[hi:])
+	v[lo] = r
+	v = v[:lo+1+n]
+	*pv = v
+}

wcfs/internal/xbtree/testprog/treegen.py

@@ -25,8 +25,8 @@ It is used as helper for ΔBtail tests.
 The following subcommands are provided:
 
 - `trees` transition ZODB tree through requested tree states,
-- `allstructs` generates topologies for subset of all possible tree changes in
-  between two trees specified by two key->value dicts.
+- `allstructs` generate subset of all possible tree topologies for a tree
+  specified by key->value dict.
 
 Because python/pkg_resources startup is very slow(*) all subcommands can be
 used either in CLI or in server mode, where requests are continuously read from
@@ -50,10 +50,10 @@ zconn.root()['treegen/tree'].
 Trees protocol specification:
 
     S: tree.srv start @<head> root=<tree-root-oid>
-    C: <tree>
-    S: <tid>
-    C: <tree>
-    S: <tid>
+    C: <tree₁>
+    S: <tid₁>
+    C: <tree₂>
+    S: <tid₂>
     ...
 
 session example:
@@ -69,43 +69,43 @@ session example:
 allstructs
 ----------
 
-`treegen allstructs` generates topologies for subset of all possible tree
-changes in between two trees specified by two key->value dicts.
+`treegen allstructs` generates subset of all possible tree topologies for a tree
+specified by key->value dict.
 
-For every kv the following tree topologies are considered: 1) native (the one
-that ZODB would usually create natively via regular usage), and 2) n random
-ones. Then tree topologies are emitted corresponding to tree1->tree2 and
-tree1<-tree2 transitions for all combinations of (tree1, tree2) pairs.
+Given kv the following tree topologies are considered: 1) native (the one
+that ZODB would usually create natively via regular usage), and 2) n-1 random
+ones. Then those tree topologies are emitted.
 
 The output of `treegen allstructs` is valid input for `treegen trees`.
 
 Allstructs protocol specification:
 
     S: # allstructs.srv start
-    C: <maxdepth> <maxsplit> <n>(/<seed>) <kv1> <kv2>
-    S: # allstructs <kv1> <kv2>
+    C: <maxdepth> <maxsplit> <n>(/<seed>) <kv>
+    S: # allstructs <kv>
     S: # maxdepth=<maxdepth> maxsplit=<maxsplit> n=<n> seed=<seed>
-    S: <tree1₀>
-    S: <tree2₀>
-    S: <tree1₁>
+    S: <tree₀>
+    S: <tree₁>
+    S: <tree₂>
     ...
     S: # ----
 
 session example:
 
     # allstructs.srv start
-    1 1 10 1:a 2:b
-    # allstructs 1:a 2:b
-    # maxdepth=1 maxsplit=1 n=10 seed=1591369961
-    T/B1:a
-    T/B2:b
-    T/T/B1:a
-    T/B2:b
-    T/B1:a
-    T/T/B2:b
-    T/T/B1:a
-    T/T/B2:b
-    T/B1:a
+    1 1 10 1:a,2:b,3:c
+    # allstructs 1:a,2:b,3:c
+    # maxdepth=1 maxsplit=1 n=10 seed=1624901326
+    T2/B1:a-B2:b,3:c
+    T3/B1:a,2:b-B3:c
+    T2/T-T3/B1:a-B2:b-B3:c
+    T/T3/B1:a,2:b-B3:c
+    T/T2/B1:a-B2:b,3:c
+    T/B1:a,2:b,3:c
+    T3/T2-T/B1:a-B2:b-B3:c
+    T2/T-T/B1:a-B2:b,3:c
+    T/T/B1:a,2:b,3:c
+    T3/T-T/B1:a,2:b-B3:c
     # ----
 
 --------
@@ -258,8 +258,8 @@ def TreesSrv(zstor, r):
 def AllStructsSrv(r):
     xprint('# allstructs.srv start')
     for req in xreadlines(r):
-        # maxdepth maxsplit n(/seed) kv1 kv2
-        maxdepth, maxsplit, n, kv1txt, kv2txt = req.split()
+        # maxdepth maxsplit n(/seed) kv
+        maxdepth, maxsplit, n, kvtxt = req.split()
         maxdepth = int(maxdepth)
         maxsplit = int(maxsplit)
         seed = None
@@ -267,39 +267,32 @@ def AllStructsSrv(r):
             n, seeds = n.split('/')
             seed = int(seeds)
         n = int(n)
-        if kv1txt == 'ø': kv1txt = ''
-        if kv2txt == 'ø': kv2txt = ''
+        if kvtxt == 'ø': kvtxt = ''
 
-        AllStructs(kv1txt, kv2txt, maxdepth, maxsplit, n, seed)
+        AllStructs(kvtxt, maxdepth, maxsplit, n, seed)
         xprint('# ----')
 
-# AllStructs generates topologies for subset of all possible tree changes in
-# between kv1 and kv2. See top-level documentation for details.
+# AllStructs generates subset of all possible topologies for a tree specified by kv dict.
+# See top-level documentation for details.
 @func
-def AllStructs(kv1txt, kv2txt, maxdepth, maxsplit, n, seed=None):
+def AllStructs(kvtxt, maxdepth, maxsplit, n, seed=None):
     zstor = MappingStorage() # in RAM storage to create native ZODB topologies
     zctx = ZCtx(zstor)
     defer(zctx.close)
 
-    kv1 = kvDecode(kv1txt, zctx.vdecode)
-    kv2 = kvDecode(kv2txt, zctx.vdecode)
+    kv = kvDecode(kvtxt, zctx.vdecode)
 
-    print("# allstructs %s %s" % (kv1txt, kv2txt))
+    print("# allstructs %s" % kvtxt)
 
     # create the tree
     ztree = zctx.root['ztree'] = LOBTree()
     commit('init')
 
-    # initial kv1 and kv2 states with topologies prepared as ZODB would do natively
-    patch(ztree, diff({}, kv1), verify=kv1)
-    if kv1 == {}: ztree._p_changed = True # to avoid empty commit - see TreesSrv
-    commit('kv1')
-    t1struct0 = xbtree.StructureOf(ztree)
-
-    patch(ztree, diff(kv1, kv2), verify=kv2)
-    if kv2 == kv1: ztree._p_changed = True
-    commit('kv2')
-    t2struct0 = xbtree.StructureOf(ztree)
+    # initial kv state with topology prepared as ZODB would do natively
+    patch(ztree, diff({}, kv), verify=kv)
+    if kv == {}: ztree._p_changed = True # to avoid empty commit - see TreesSrv
+    commit('kv')
+    tstruct0 = xbtree.StructureOf(ztree)
 
     # seed
     if seed is None:
@@ -308,31 +301,15 @@ def AllStructs(kv1txt, kv2txt, maxdepth, maxsplit, n, seed=None):
     random.seed(seed)
     print("# maxdepth=%d maxsplit=%d n=%d seed=%d" % (maxdepth, maxsplit, n, seed))
 
-    # make 2·n random samples from all tree topologies that can represent kv1 and kv2
-    t1structv = rsample(xbtree.AllStructs(kv1.keys(), maxdepth, maxsplit, kv=kv1), n)
-    t2structv = rsample(xbtree.AllStructs(kv2.keys(), maxdepth, maxsplit, kv=kv2), n)
-
-    # all tree1 and tree2 topologies jumps in between we are going to emit:
-    # native + n random ones.
-    if t1struct0 in t1structv: t1structv.remove(t1struct0) # avoid dups
-    if t2struct0 in t2structv: t2structv.remove(t2struct0)
-    t1structv.insert(0, t1struct0)
-    t2structv.insert(0, t2struct0)
-
-    # XXX rework allstructs to accept only 1 kv and emit n structs for that one kv only
-    #     -> iterate through the pairs/triplets in the caller (TestΔBTailAllStructs)
-
-    # emit topologies for tree1->tree2 and tree1<-tree2 transitions for all
-    # combinations of tree1 and tree2.
-    t12travel = list(bitravel2Way(t1structv, t2structv))
-    for i,tstruct in enumerate(t12travel):
-        if i%2 == 0:
-            assert tstruct in t1structv
-        else:
-            assert tstruct in t2structv
+    # emit native + n-1 random samples from all tree topologies that can represent kv
+    tstructv = rsample(xbtree.AllStructs(kv.keys(), maxdepth, maxsplit, kv=kv), n-1)
+    if tstruct0 in tstructv: tstructv.remove(tstruct0) # avoid dups
+    tstructv.insert(0, tstruct0)
 
+    for tstruct in tstructv:
         print(zctx.TopoEncode(tstruct))
 
+
 # rsample returns k random samples from seq.
 # it differs from random.sample in that it does not keep whole list(seq) in memory.
 def rsample(seq, k): # -> [] of items; len <= k
@@ -352,31 +329,6 @@ def rsample(seq, k): # -> [] of items; len <= k
             sample[j] = item
     return sample
 
-# bitravel2Way generates travel path through all A<->B edges such
-# that all edges a->b and a<-b are traveled and exactly once.
-#
-# The travel starts from A[0].
-def bitravel2Way(A, B): # -> i[] of node
-    na = len(A); assert na > 0
-    nb = len(B); assert nb > 0
-
-    yield A[0]                  # A₀
-    for j in range(nb):
-        yield B[j]              # A₀ -> Bj
-        for i in range(1,na):
-            yield A[i]          # Ai <- Bj
-            yield B[j]          # Ai -> Bj
-        yield A[0]              # A₀ <- Bj
-
-def test_bitravel2Way():
-    a,b,c = 'a','b','c'
-    A = [a,b,c]
-    B = [1, 2]
-    got  = list(bitravel2Way(A, B))
-    want = [a,1,b,1,c,1,a,2,b,2,c,2,a]
-    assert got == want, (got, want)
-test_bitravel2Way()
-
 
 # kvEncode encodes key->value mapping into text.
 # e.g. {1:'a', 2:'b'} -> '1:a,2:b'

wcfs/internal/xbtree/treediff.go

@@ -91,6 +91,18 @@ type ΔValue struct {
 	New Value
 }
 
+// String is like default %v, but uses ø for VDEL.
+func (δv ΔValue) String() string {
+	old, new := "ø", "ø"
+	if δv.Old != VDEL {
+		old = δv.Old.String()
+	}
+	if δv.New != VDEL {
+		new = δv.New.String()
+	}
+	return fmt.Sprintf("{%s %s}", old, new)
+}
+
 
 // δZConnectTracked computes connected closure of δZ/T.
 //

wcfs/internal/xbtree/δbtail.go

@@ -184,6 +184,7 @@ func (orig *ΔBtail) Clone() *ΔBtail {
 	}
 
 	// vδBroots
+	klon.vδBroots = make([]ΔBroots, 0, len(orig.vδBroots))
 	for _, origδBroots := range orig.vδBroots {
 		klonδBroots := ΔBroots{
 			Rev:    origδBroots.Rev,
@@ -208,6 +209,7 @@ func (orig *ΔBtail) Clone() *ΔBtail {
 // Clone returns copy of ΔTtail.
 func (orig *ΔTtail) Clone() *ΔTtail {
 	klon := &ΔTtail{}
+	klon.vδT = make([]ΔTree, 0, len(orig.vδT))
 	for _, origδT := range orig.vδT {
 		klonδT := ΔTree{
 			Rev: origδT.Rev,