Skip to content

W
wendelin.core
Commit 00b6ac98 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

.

parent 84986acb
Show whitespace changes
Inline Side-by-side
Showing with 1204 additions and 1195 deletions
wcfs/internal/xbtree/δbtail_test.go
View file @ 00b6ac98
...@@ -61,47 +61,6 @@ import ( ...@@ -61,47 +61,6 @@ import (
type Δstring = xbtreetest.Δstring type Δstring = xbtreetest.Δstring
// trackSet returns what should be ΔBtail.trackSet coverage for specified tracked key set.
func trackSet(rbs xbtreetest.RBucketSet, tracked setKey) blib.PPTreeSubSet {
// nil = don't compute keyCover
// (trackSet is called from inside hot inner loop of rebuild test)
return _trackSetWithCov(rbs, tracked, nil)
}
// trackSetWithCov returns what should be ΔBtail.trackSet and its key coverage for specified tracked key set.
func trackSetWithCov(rbs xbtreetest.RBucketSet, tracked setKey) (trackSet blib.PPTreeSubSet, keyCover *blib.RangedKeySet) {
keyCover = &blib.RangedKeySet{}
trackSet = _trackSetWithCov(rbs, tracked, keyCover)
return trackSet, keyCover
}
func _trackSetWithCov(rbs xbtreetest.RBucketSet, tracked setKey, outKeyCover *blib.RangedKeySet) (trackSet blib.PPTreeSubSet) {
trackSet = blib.PPTreeSubSet{}
for k := range tracked {
kb := rbs.Get(k)
if outKeyCover != nil {
outKeyCover.AddRange(kb.Keycov)
}
trackSet.AddPath(kb.Path())
}
return trackSet
}
// XGetδKV translates {k -> δ<oid>} to {k -> δ(ZBlk(oid).data)} according to t1..t2 db snapshots.
func XGetδKV(t1, t2 *xbtreetest.Commit, δkvOid map[Key]ΔValue) map[Key]Δstring {
δkv := make(map[Key]Δstring, len(δkvOid))
for k, δvOid := range δkvOid {
δkv[k] = Δstring{
Old: t1.XGetBlkData(δvOid.Old),
New: t2.XGetBlkData(δvOid.New),
}
}
return δkv
}
// KAdjMatrix is adjacency matrix that describes how set of tracked keys // KAdjMatrix is adjacency matrix that describes how set of tracked keys
// changes (always grow) when tree topology is updated from A to B. // changes (always grow) when tree topology is updated from A to B.
// //
...@@ -129,1436 +88,1486 @@ func XGetδKV(t1, t2 *xbtreetest.Commit, δkvOid map[Key]ΔValue) map[Key]Δstri ...@@ -129,1436 +88,1486 @@ func XGetδKV(t1, t2 *xbtreetest.Commit, δkvOid map[Key]ΔValue) map[Key]Δstri
// //
// XXX fix definition for "and changed, or coverage changed" // XXX fix definition for "and changed, or coverage changed"
// //
// Use:
//
// - KAdj(A,B) to build adjacency matrix for A -> B transition.
// - kadj.Map(keys) to compute kadj·keys.
// - kadj1.Mul(kadj2) to compute kadj1·kadj2.
//
// Note: adjacency matrix is symmetric (KAdj verifies this at runtime): // Note: adjacency matrix is symmetric (KAdj verifies this at runtime):
// //
// kadj(A,B) == kadj(B,A) // KAdj(A,B) == KAdj(B,A)
type KAdjMatrix map[Key]setKey type KAdjMatrix map[Key]setKey
// Map returns kadj·keys .
func (kadj KAdjMatrix) Map(keys setKey) setKey {
res := make(setKey, len(keys))
for k := range keys {
to, ok := kadj[k]
if !ok {
panicf("kadj.Map: %d ∉ kadj\n\nkadj: %v", k, kadj)
}
res.Update(to)
}
return res
}
// Mul returns kadjA·kadjB .
//
// (kadjA·kadjB).Map(keys) = kadjA.Map(kadjB.Map(keys))
func (kadjA KAdjMatrix) Mul(kadjB KAdjMatrix) KAdjMatrix {
// ~ assert kadjA.keys == kadjB.keys
// check only len here; the rest will be asserted by Map
if len(kadjA) != len(kadjB) {
panicf("kadj.Mul: different keys:\n\nkadjA: %v\nkadjB: %v", kadjA, kadjB)
}
kadj := make(KAdjMatrix, len(kadjB)) // ΔBTestEntry represents one entry in ΔBTail tests.
for k, tob := range kadjB { type ΔBTestEntry struct {
kadj[k] = kadjA.Map(tob) tree string // next tree topology
} kadjOK KAdjMatrix // adjacency matrix against previous case (optional)
return kadj flags ΔBTestFlags
} }
// KAdj builds adjacency matrix for t1 -> t2 transition. type ΔBTestFlags int
// const ΔBTest_SkipUpdate ΔBTestFlags = 1 // skip verifying Update for this test entry
// The set of keys for which kadj matrix is computed can be optionally provided. const ΔBTest_SkipRebuild ΔBTestFlags = 2 // skip verifying rebuild for this test entry
// This set of keys defaults to allTestKeys(t1,t2).
//
// KAdj itself is verified by testΔBTail on entries with .kadjOK set.
func KAdj(t1, t2 *xbtreetest.Commit, keysv ...setKey) (kadj KAdjMatrix) {
// assert KAdj(A,B) == KAdj(B,A)
kadj12 := _KAdj(t1,t2, keysv...)
kadj21 := _KAdj(t2,t1, keysv...)
if !reflect.DeepEqual(kadj12, kadj21) {
panicf("KAdj not symmetric:\nt1: %s\nt2: %s\nkadj12: %v\nkadj21: %v",
t1.Tree, t2.Tree, kadj12, kadj21)
}
return kadj12
}
const debugKAdj = false // ΔBTest converts xtest into ΔBTestEntry.
func debugfKAdj(format string, argv ...interface{}) { // xtest can be string|ΔBTestEntry.
if debugKAdj { func ΔBTest(xtest interface{}) ΔBTestEntry {
fmt.Printf(format, argv...) var test ΔBTestEntry
switch xtest := xtest.(type) {
case string:
test.tree = xtest
test.kadjOK = nil
test.flags = 0
case ΔBTestEntry:
test = xtest
default:
panicf("BUG: ΔBTest: bad type %T", xtest)
} }
return test
} }
func _KAdj(t1, t2 *xbtreetest.Commit, keysv ...setKey) (kadj KAdjMatrix) { // TestΔBTail verifies ΔBTail for explicitly provided tree topologies.
var keys setKey func TestΔBTail(t *testing.T) {
switch len(keysv) { // K is shorthand for setKey
case 0: K := func(keyv ...Key) setKey {
keys = allTestKeys(t1, t2) ks := setKey{}
case 1: for _, k := range keyv { ks.Add(k) }
keys = keysv[0] return ks
default: }
panic("multiple key sets on the call") // oo is shorthand for KeyMax
const oo = KeyMax
// A is shorthand for KAdjMatrix
type A = KAdjMatrix
// Δ is shorthand for ΔBTestEntry
Δ := func(tree string, kadjOK A) (test ΔBTestEntry) {
test.tree = tree
test.kadjOK = kadjOK
return test
} }
debugfKAdj("\n\n_KAdj\n") // test known cases going through tree1 -> tree2 -> ...
debugfKAdj("t1: %s\n", t1.Tree) testv := []interface{} {
debugfKAdj("t2: %s\n", t2.Tree) // start from non-empty tree to verify both ->empty and empty-> transitions
debugfKAdj("keys: %s\n", keys) "T/B1:a,2:b",
defer func() {
debugfKAdj("kadj -> %v\n", kadj)
}()
// kadj = {} k -> adjacent keys. // empty
// if k is tracked and covered by changed leaf -> changes to adjacents must be in Update(t1->t2). "T/B:",
kadj = KAdjMatrix{}
for k := range keys {
adj1 := setKey{}
adj2 := setKey{}
q1 := &blib.RangedKeySet{}; q1.Add(k) // +1
q2 := &blib.RangedKeySet{}; q2.Add(k) Δ("T/B1:a",
done1 := &blib.RangedKeySet{} A{1: K(1,oo),
done2 := &blib.RangedKeySet{} oo: K(1,oo)}),
debugfKAdj("\nk%s\n", kstr(k)) // +2
for !q1.Empty() || !q2.Empty() { Δ("T/B1:a,2:b",
debugfKAdj("q1: %s\tdone1: %s\n", q1, done1) A{1: K(1,2,oo),
debugfKAdj("q2: %s\tdone2: %s\n", q2, done2) 2: K(1,2,oo),
for _, r1 := range q1.AllRanges() { oo: K(1,2,oo)}),
lo1 := r1.Lo
for {
b1 := t1.Xkv.Get(lo1)
debugfKAdj(" b1: %s\n", b1)
for k_ := range keys {
if b1.Keycov.Has(k_) {
adj1.Add(k_)
debugfKAdj(" adj1 += %s\t-> %s\n", kstr(k_), adj1)
}
}
done1.AddRange(b1.Keycov)
// q2 |= (b1.keyrange \ done2)
δq2 := &blib.RangedKeySet{}
δq2.AddRange(b1.Keycov)
δq2.DifferenceInplace(done2)
q2.UnionInplace(δq2)
debugfKAdj("q2 += %s\t-> %s\n", δq2, q2)
// continue with next right bucket until r1 coverage is complete // -1
if r1.Hi_ <= b1.Keycov.Hi_ { Δ("T/B2:b",
break A{1: K(1,2,oo),
} 2: K(1,2,oo),
lo1 = b1.Keycov.Hi_ + 1 oo: K(1,2,oo)}),
}
}
q1.Clear()
for _, r2 := range q2.AllRanges() { // 2: b->c
lo2 := r2.Lo Δ("T/B2:c",
for { A{2: K(2,oo),
b2 := t2.Xkv.Get(lo2) oo: K(2,oo)}),
debugfKAdj(" b2: %s\n", b2)
for k_ := range keys {
if b2.Keycov.Has(k_) {
adj2.Add(k_)
debugfKAdj(" adj2 += %s\t-> %s\n", kstr(k_), adj2)
}
}
done2.AddRange(b2.Keycov)
// q1 |= (b2.keyrange \ done1)
δq1 := &blib.RangedKeySet{}
δq1.AddRange(b2.Keycov)
δq1.DifferenceInplace(done1)
q1.UnionInplace(δq1)
debugfKAdj("q1 += %s\t-> %s\n", δq1, q1)
// continue with next right bucket until r2 coverage is complete // +1 in new bucket (to the left)
if r2.Hi_ <= b2.Keycov.Hi_ { Δ("T2/B1:a-B2:c",
break A{1: K(1,2,oo),
} 2: K(1,2,oo),
lo2 = b2.Keycov.Hi_ + 1 oo: K(1,2,oo)}),
}
}
q2.Clear()
}
adj := setKey{}; adj.Update(adj1); adj.Update(adj2) // +3 in new bucket (to the right)
kadj[k] = adj Δ("T2,3/B1:a-B2:c-B3:c",
} A{1: K(1),
2: K(2,3,oo),
3: K(2,3,oo),
oo: K(2,3,oo)}),
return kadj // bucket split; +3 in new bucket
} "T/B1:a,2:b",
Δ("T2/B1:a-B2:b,3:c",
A{1: K(1,2,3,oo),
2: K(1,2,3,oo),
3: K(1,2,3,oo),
oo: K(1,2,3,oo)}),
// bucket split; +3 in new bucket; +4 +5 in another new bucket
// everything becomes tracked because original bucket had [-∞,∞) coverage
"T/B1:a,2:b",
Δ("T2,4/B1:a-B2:b,3:c-B4:d,5:e",
A{1: K(1,2,3,4,5,oo),
2: K(1,2,3,4,5,oo),
3: K(1,2,3,4,5,oo),
4: K(1,2,3,4,5,oo),
5: K(1,2,3,4,5,oo),
oo: K(1,2,3,4,5,oo)}),
// xverifyΔBTail_Update verifies how ΔBTail handles ZODB update for a tree with changes in between t1->t2. // reflow of keys: even if tracked={1}, changes to all B nodes need to be rescanned:
// // +B12 forces to look in -B23 which adds -3 into δ, which
// Note: this test verifies only single treediff step of ΔBtail.Update. // forces to look into +B34 and so on.
// the cycling phase of update, that is responsible to recompute older "T2,4,6/B1:a-B2:b,3:c-B4:d,5:e-B6:f,7:g",
// entries when key coverage grows, is exercised by Δ("T3,5,7/B1:g,2:f-B3:e,4:d-B5:c,6:b-B7:a",
// xverifyΔBTail_rebuild. A{1: K(1,2,3,4,5,6,7,oo),
func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1, t2 *xbtreetest.Commit) { 2: K(1,2,3,4,5,6,7,oo),
// verify transition at1->at2 for all initial states of tracked {keys} from kv1 + kv2 + ∞ 3: K(1,2,3,4,5,6,7,oo),
t.Run(fmt.Sprintf("Update/%s→%s", t1.Tree, t2.Tree), func(t *testing.T) { 4: K(1,2,3,4,5,6,7,oo),
allKeys := allTestKeys(t1, t2) 5: K(1,2,3,4,5,6,7,oo),
allKeyv := allKeys.SortedElements() 6: K(1,2,3,4,5,6,7,oo),
7: K(1,2,3,4,5,6,7,oo),
kadj12 := KAdj(t1, t2) oo: K(1,2,3,4,5,6,7,oo)}),
// verify at1->at2 for all combination of initial tracked keys.
for kidx := range IntSets(len(allKeyv)) {
keys := setKey{}
for _, idx := range kidx {
keys.Add(allKeyv[idx])
}
// this t.Run allocates and keeps too much memory in -verylong
// also it is not so useful as above "Update/t1->t2"
//t.Run(fmt.Sprintf(" track=%s", keys), func(t *testing.T) {
xverifyΔBTail_Update1(t, subj, db, treeRoot, t1,t2, keys, kadj12)
//})
}
})
}
// xverifyΔBTail_Update1 verifies how ΔBTail handles ZODB update at1->at2 from initial
// tracked state defined by initialTrackedKeys.
func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1,t2 *xbtreetest.Commit, initialTrackedKeys setKey, kadj KAdjMatrix) {
X := exc.Raiseif
//t.Logf("\n>>> Track=%s\n", initialTrackedKeys)
δZ := t2.ΔZ // reflow of keys for rebuild: even if tracked1={}, tracked2={1}, changes to
d12 := t2.Δxkv // all A/B/C nodes need to be rescanned. Contrary to the above case the reflow
// is not detectable at separate diff(A,B) and diff(B,C) runs.
"T3,5,7/B1:a,2:b-B3:c,4:d-B5:e,6:f-B7:g,8:h",
"T/B1:b",
"T2,4,6/B1:a-B2:b,3:c-B4:d,5:e-B6:f,7:g",
// similar situation where rebuild has to detect reflow in between non-neighbour trees
"T3,6/B1:a,2:b-B3:c,4:d-B6:f,7:g",
"T4,7/B1:b-B4:d,5:e-B7:g,8:h",
"T2,5,8/B1:a-B2:b,3:c-B5:e,6:f-B8:h,9:i",
var TrackedδZ setKey = nil // depth=2; bucket split; +3 in new bucket; left T remain
var kadjTrackedδZ setKey = nil // _unchanged_ even though B under it is modified.
var δT, δTok map[Key]Δstring = nil, nil "T/T/B1:a,2:b",
δZset := setOid{} Δ("T2/T-T/B1:a-B2:b,3:c",
for _, oid := range δZ.Changev { A{1: K(1,2,3,oo),
δZset.Add(oid) 2: K(1,2,3,oo),
} 3: K(1,2,3,oo),
oo: K(1,2,3,oo)}),
// badf queues error message to be reported on return. // depth=2; like prev. case, but additional right arm with +4 +5 is added.
var badv []string "T/T/B1:a,2:b",
badf := func(format string, argv ...interface{}) { Δ("T2,4/T-T-T/B1:a-B2:b,3:c-B4:d,5:e",
badv = append(badv, fmt.Sprintf(format, argv...)) A{1: K(1,2,3,4,5,oo),
} 2: K(1,2,3,4,5,oo),
defer func() { 3: K(1,2,3,4,5,oo),
if badv != nil || t.Failed() { 4: K(1,2,3,4,5,oo),
emsg := fmt.Sprintf("%s ; tracked=%v :\n\n", subj, initialTrackedKeys) 5: K(1,2,3,4,5,oo),
emsg += fmt.Sprintf("d12: %v\nδTok: %v\nδT: %v\n\n", d12, δTok, δT) oo: K(1,2,3,4,5,oo)}),
emsg += fmt.Sprintf("δZ: %v\n", δZset)
emsg += fmt.Sprintf("Tracked^δZ: %v\n", TrackedδZ)
emsg += fmt.Sprintf("kadj[Tracked^δZ]: %v\n", kadjTrackedδZ)
emsg += fmt.Sprintf("kadj: %v\n\n", kadj)
emsg += strings.Join(badv, "\n")
emsg += "\n"
t.Fatal(emsg) // depth=2; bucket split; +3 in new bucket; t0 and t1 split;
} // +right arm (T7/B45-B89).
}() "T/T/B1:a,2:b",
Δ("T4/T2-T7/B1:a-B2:b,3:c-B4:d,5:e-B8:h,9:i",
A{1: K(1,2,3,4,5,8,9,oo),
2: K(1,2,3,4,5,8,9,oo),
3: K(1,2,3,4,5,8,9,oo),
4: K(1,2,3,4,5,8,9,oo),
5: K(1,2,3,4,5,8,9,oo),
8: K(1,2,3,4,5,8,9,oo),
9: K(1,2,3,4,5,8,9,oo),
oo: K(1,2,3,4,5,8,9,oo)}),
// δbtail @at1 with initial tracked set // 2 reflow to right B neighbour; 8 splits into new B; δ=ø
δbtail := NewΔBtail(t1.At, db) "T3/B1:a,2:b-B4:d,8:h",
xtrackKeys(δbtail, t1, initialTrackedKeys) "T2,5/B1:a-B2:b,4:d-B8:h",
// TrackedδZ = Tracked ^ δZ (i.e. a tracked node has changed, or its coverage was changed) // case where kadj does not grow too much as leafs coverage remains stable
TrackedδZ = setKey{} "T4,8/B1:a,2:b-B5:d,6:e-B10:g,11:h",
for k := range initialTrackedKeys { Δ("T4,8/B2:b,3:c-B6:e,7:f-B11:h,12:i",
leaf1 := t1.Xkv.Get(k) A{1: K(1,2,3),
oid1 := leaf1.Oid 2: K(1,2,3),
if oid1 == zodb.InvalidOid { // embedded bucket 3: K(1,2,3),
oid1 = leaf1.Parent.Oid 5: K(5,6,7),
} 6: K(5,6,7),
leaf2 := t2.Xkv.Get(k) 7: K(5,6,7,),
oid2 := leaf2.Oid 10: K(10,11,12,oo),
if oid2 == zodb.InvalidOid { // embedded bucket 11: K(10,11,12,oo),
oid2 = leaf2.Parent.Oid 12: K(10,11,12,oo),
} oo: K(10,11,12,oo)}),
if δZset.Has(oid1) || δZset.Has(oid2) || (leaf1.Keycov != leaf2.Keycov) {
TrackedδZ.Add(k)
}
}
kadjTrackedδZ = setKey{} // kadj[Tracked^δZ] (all keys adjacent to tracked^δZ)
for k := range TrackedδZ {
kadjTrackedδZ.Update(kadj[k])
}
// tree deletion
// having ø in the middle of the test cases exercises all:
// * `ø -> Tree ...` (tree is created anew),
// * `... Tree -> ø` (tree is deleted), and
// * `Tree -> ø -> Tree` (tree is deleted and then recreated)
xbtreetest.DEL,
// assert TrackedδZ ∈ kadj[TrackedδZ] // tree rotation
trackNotInKadj := TrackedδZ.Difference(kadjTrackedδZ) "T3/B2:b-B3:c,4:d",
if len(trackNotInKadj) > 0 { "T5/T3-T7/B2:a-B3:a,4:a-B6:a-B8:a",
badf("BUG: Tracked^δZ ∉ kadj[Tracked^δZ] ; extra=%v", trackNotInKadj)
return
}
// k ∈ d12 // found by AllStructs ([1] is not changed, but because B1 is
// k ∈ δT <=> // unlinked and 1 migrates to other bucket, changes in that
// k ∈ U kadj[·] // other bucket must be included into δT)
// ·∈tracking^δZ "T1,2/B0:e-B1:d-B2:g,3:a",
δTok = map[Key]Δstring{} // d12[all keys that should be present in δT] "T1/B0:d-B1:d,2:d",
for k,δv := range d12 { // ----//---- with depth=2
if kadjTrackedδZ.Has(k) { "T1,2/T-T-T/B0:a-B1:b-B2:c,3:d",
δTok[k] = δv "T1/T-T/B0:e-B1:b,2:f",
}
}
ø := blib.PPTreeSubSet{} // XXX depth=3 (to verify recursion and selecting which tree children to follow or not)
// trackSet1 = xkv1[tracked1]
// trackSet2 = xkv2[tracked2] ( = xkv2[kadj[tracked1]]
trackSet1, tkeyCov1 := trackSetWithCov(t1.Xkv, initialTrackedKeys)
trackSet2, tkeyCov2 := trackSetWithCov(t2.Xkv, initialTrackedKeys.Union(kadjTrackedδZ))
// verify δbtail.trackSet against @at1 // degenerate topology from ZODB tests
δbtail.assertTrack(t, "1", ø, trackSet1) // https://github.com/zopefoundation/ZODB/commit/6cd24e99f89b
// https://github.com/zopefoundation/BTrees/blob/4.7.2-1-g078ba60/BTrees/tests/testBTrees.py#L20-L57
"T4/T2-T/T-T-T6,10/B1:a-B3:b-T-T-T/T-B7:c-B11:d/B5:e",
"T/B1:e,5:d,7:c,8:b,11:a", // -3 +8
// δB <- δZ // was leading treegen to generate corrupt trees
// "T/T1/T-T/B0:g-B1:e,2:d,3:h",
// also call _Update1 directly to verify δtkeycov return from treediff "T1/T-T3/B0:g-T-T/B1:e,2:d-B3:h",
// the result of Update and _Update1 should be the same since δbtail is initially empty.
δbtail_ := δbtail.Clone()
δB1, err := δbtail_._Update1(δZ); X(err) // XXX don't compute treediff twice
// assert tkeyCov1 ⊂ tkeyCov2 // was leading to wrongly computed trackSet2 due to top not
dkeycov12 := tkeyCov1.Difference(tkeyCov2) // being tracked to tree root.
if !dkeycov12.Empty() { "T/T1/B0:a-B1:b",
t.Errorf("BUG: tkeyCov1 ⊄ tkeyCov2:\n\ttkeyCov1: %s\n\ttkeyCov2: %s\n\ttkeyCov1 \\ tkeyCov2: %s", tkeyCov1, tkeyCov2, dkeycov12) "T/T1/T-T/B0:c-B1:d",
}
// assert δtkeycov == δ(tkeyCov1, tkeyCov2) // was leading to wrongly computed trackSet2: leaf bucket not
δtkeycovOK := tkeyCov2.Difference(tkeyCov1) // reparented to root.
δtkeycov := &blib.RangedKeySet{} "T/T/B0:a",
if __, ok := δB1.ByRoot[treeRoot]; ok { "T/B0:a",
δtkeycov = __.δtkeycov1
}
if !δtkeycov.Equal(δtkeycovOK) {
badf("δtkeycov wrong:\nhave: %s\nwant: %s", δtkeycov, δtkeycovOK)
}
δB, err := δbtail.Update(δZ); X(err) // δtkeycov grows due to change in parent tree only
// XXX assert δB.roots == δTKeyCov roots "T3/B1:a-B8:c",
// XXX assert δBtail[root].vδT = δBtail_[root].vδT "T7/B1:a-B8:c",
// ----//----
"T3/B1:a,2:b-B8:c,9:d",
"T7/B1:a,2:b-B8:c,9:d",
// ----//---- depth=2
"T3/T-T/B1:a,2:b-B8:c,9:d",
"T7/T-T/B1:a,2:b-B8:c,9:d",
// ----//---- found by AllStructs
"T1,3/B0:d-B1:a-B3:d,4:g",
"T1,4/B0:e-B1:a-B4:c",
// ----//---- found by AllStructs
"T2,4/T-T-T/T1-T-B4:f/T-T-B3:f/B0:h-B1:f",
"T4/T-T/B3:f-T/B4:a",
if δB.Rev != δZ.Tid {
badf("δB: rev != δZ.Tid ; rev=%s δZ.Tid=%s", δB.Rev, δZ.Tid)
return
}
// verify δbtail.trackSet against @at2 // ---- found by AllStructs ----
δbtail.assertTrack(t, "2", trackSet2, ø)
// trackSet2 wrongly computed due to top not being tracked to tree root
"T2/T1-T/B0:g-B1:b-T/B2:b,3:a",
"T2/T1-T/T-T-B2:a/B0:c-B1:g",
// assert δB.ByRoot == {treeRoot -> ...} if δTok != ø // unchanged node is reparented
// == ø if δTok == ø "T1/B0:c-B1:f",
rootsOK := setOid{} "T1/T-T/B0:c-T/B1:h",
if len(δTok) > 0 {
rootsOK.Add(treeRoot)
}
roots := setOid{}
for root := range δB.ΔByRoot {
roots.Add(root)
}
if !reflect.DeepEqual(roots, rootsOK) {
badf("δB: roots != rootsOK ; roots=%v rootsOK=%v", roots, rootsOK)
}
_, inδB := δB.ΔByRoot[treeRoot]
if !inδB {
return
}
// SIGSEGV in ApplyΔ
"T1/T-T2/T-B1:c-B2:c/B0:g",
"T1/T-T/B0:g-T/B1:e",
// δT <- δB // trackSet corruption: oid is pointed by some .parent but is not present
δToid := δB.ΔByRoot[treeRoot] // {} k -> δoid "T1/T-T/B0:g-T2/B1:h-B2:g",
δT = XGetδKV(t1,t2, δToid) // {} k -> δ(ZBlk(oid).data) "T/T1/T-T2/B0:e-B1:f-B2:g",
// δT must be subset of d12. // ApplyΔ -> xunion: node is reachable from multiple parents
// changed keys, that are // ( because xdifference did not remove common non-leaf node
// - in tracked set -> must be present in δT // under which there were also other changed, but not initially
// - outside tracked set -> may be present in δT (kadj gives exact answer) // tracked, node )
"T4/T1-T/T-T2-B4:c/T-T-T/B0:f-B1:h-B2:g,3:b",
"T1/T-T/T-T2/T-T-T/B0:f-B1:h-B2:f",
// ----//----
"T3/T1-T/T-T2-T/B0:b-T-T-B3:h/B1:e-B2:a",
"T1/T-T4/T-T2-T/T-T-T-T/B0:b-B1:e-B2:a,3:c-B4:e",
// ----//----
"T/T1,3/T-T2-T4/B0:b-T-T-B3:g-B4:c/B1:b-B2:e",
"T1,4/T-T-T/T-T2-B4:f/T-T-T/B0:h-B1:b-B2:h,3:a",
// δT is subset of d12 "T2/B1:a-B7:g",
for _, k := range sortedKeys(δT) { "T2,8/B1:a-B7:g-B9:i",
_, ind12 := d12[k]
if !ind12 {
badf("δT[%v] ∉ d12", k)
}
}
// k ∈ tracked set -> must be present in δT "T2/B1:a-B2:b", "T/B1:a,2:b",
// k ∉ tracked set -> may be present in δT (kadj gives exact answer) "T2,3/B1:a-B2:b-B3:c", "T/B1:a,2:b",
for _, k := range sortedKeys(d12) { "T2,3/B1:a-B2:c-B3:c", "T/B1:a,2:b",
_, inδT := δT[k]
_, inδTok := δTok[k]
if inδT && !inδTok {
badf("δT[%v] ∉ δTok", k)
}
if !inδT && inδTok { "T2/B1:a-B2:c", "T2,3/B1:a-B2:c-B3:c",
badf("δT ∌ δTok[%v]", k)
}
if inδT { "T2/B1:a-B3:c",
if δT[k] != d12[k] { Δ("T2/T-T4/B1:b-B3:d-B99:h",
badf("δT[%v] ≠ δTok[%v]", k, k) A{1: K(1),
3: K(3,99,oo),
99: K(3,99,oo),
oo: K(3,99,oo)}),
} }
// direct tree_i -> tree_{i+1} -> _{i+2} ... plus
// reverse ... tree_i <- _{i+1} <- _{i+2}
kadjOK := ΔBTest(testv[len(testv)-1]).kadjOK
for i := len(testv)-2; i >= 0; i-- {
test := ΔBTest(testv[i])
kadjOK, test.kadjOK = test.kadjOK, kadjOK
testv = append(testv, test)
} }
testq := make(chan ΔBTestEntry)
go func() {
defer close(testq)
for _, test := range testv {
testq <- ΔBTest(test)
} }
}()
testΔBTail(t, testq)
} }
// assertTrack verifies state of .trackSet and ΔTtail.trackNew. // TestΔBTailRandom verifies ΔBtail on random tree topologies generated by AllStructs.
// it assumes that only one tree root is being tracked. func TestΔBTailRandom(t *testing.T) {
// XXX place X := exc.Raiseif
func (δBtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK blib.PPTreeSubSet, trackNewOK blib.PPTreeSubSet) {
t.Helper()
if !δBtail.trackSet.Equal(trackSetOK) {
t.Errorf("%s: trackSet:\n\thave: %v\n\twant: %v", subj, δBtail.trackSet, trackSetOK)
}
roots := setOid{} // considerations:
for root := range δBtail.vδTbyRoot { // - maxdepth↑ better for testing (more tricky topologies)
roots.Add(root) // - maxsplit↑ not so better for testing (leave s=1, max s=2)
} // - |kmin - kmax| affects N(variants) significantly
// -> keep key range small (dumb increase does not help testing)
// - N(keys) affects N(variants) significantly
// -> keep Nkeys reasonably small/medium (dumb increase does not help testing)
//
// - spawning python subprocess is very slow (takes 300-500ms for
// imports; https://github.com/pypa/setuptools/issues/510)
// -> we spawn `treegen allstructs` once and use request/response approach.
nrootsOK := 1 maxdepth := xbtreetest.N(2, 3, 4)
if trackSetOK.Empty() && trackNewOK.Empty() { maxsplit := xbtreetest.N(1, 2, 2)
nrootsOK = 0 n := xbtreetest.N(10,10,100)
} nkeys := xbtreetest.N(3, 5, 10)
if len(roots) != nrootsOK {
t.Errorf("%s: len(vδTbyRoot) != %d ; roots=%v", subj, nrootsOK, roots)
return
}
if nrootsOK == 0 {
return
}
root := roots.Elements()[0] // server to generate AllStructs(kv, ...)
sg, err := xbtreetest.StartAllStructsSrv(); X(err)
defer func() {
err := sg.Close(); X(err)
}()
δTtail := δBtail.vδTbyRoot[root] // random-number generator
rng, seed := xbtreetest.NewRand()
t.Logf("# maxdepth=%d maxsplit=%d nkeys=%d n=%d seed=%d", maxdepth, maxsplit, nkeys, n, seed)
trackNewRootsOK := setOid{} // generate (kv1, kv2, kv3) randomly
if !trackNewOK.Empty() {
trackNewRootsOK.Add(root)
}
if !δBtail.trackNewRoots.Equal(trackNewRootsOK) { // keysv1, keysv2 and keysv3 are random shuffle of IntSets
t.Errorf("%s: trackNewRoots:\n\thave: %v\n\twant: %v", subj, δBtail.trackNewRoots, trackNewRootsOK) var keysv1 [][]int
var keysv2 [][]int
var keysv3 [][]int
for keys := range IntSets(nkeys) {
keysv1 = append(keysv1, keys)
keysv2 = append(keysv2, keys)
keysv3 = append(keysv3, keys)
} }
v := keysv1
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] })
v = keysv2
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] })
v = keysv3
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] })
if !δTtail.trackNew.Equal(trackNewOK) { // given random (kv1, kv2, kv3) generate corresponding set of random tree
t.Errorf("%s: vδT.trackNew:\n\thave: %v\n\twant: %v", subj, δTtail.trackNew, trackNewOK) // topology sets (T1, T2, T3). Then iterate through T1->T2->T3->T1...
// elements such that all right-directed triplets are visited and only once.
// Test Update and rebuild on the generated tree sequences.
vv := "abcdefghij"
randv := func() string {
i := rng.Intn(len(vv))
return vv[i:i+1]
} }
}
// xverifyΔBTail_rebuild verifies ΔBtail.rebuild during t0->t1->t2 transition.
//
// t0->t1 exercises from-scratch rebuild,
// t1->t2 further exercises incremental rebuild.
//
// It also exercises rebuild phase of ΔBtail.Update.
func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1, t2 *xbtreetest.Commit) {
t.Run(fmt.Sprintf("rebuild/%s→%s", t0.Tree, t1.Tree), func(t *testing.T) {
tAllKeys := allTestKeys(t0, t1, t2)
tAllKeyv := tAllKeys.SortedElements()
// tid -> "at_i" // the number of pairs is 3·n^2
xat := map[zodb.Tid]string{ // the number of triplets is n^3
t0.At: "at0", //
t1.At: "at1", // limit n for emitted triplets, so that the amount of work for Update
t2.At: "at2", // and rebuild tests is approximately of the same order.
nrebuild := int(math.Ceil(math.Pow(3*float64(n*n), 1./3)))
// in non-short mode rebuild tests are exercising more keys variants, plus every test case
// takes more time. Compensate for that as well.
if !testing.Short() {
nrebuild -= 3
} }
//fmt.Printf("@%s: %v\n", xat[t0.At], t0.Xkv.Flatten()) testq := make(chan ΔBTestEntry)
//fmt.Printf("@%s: %v\n", xat[t1.At], t1.Xkv.Flatten()) go func() {
//fmt.Printf("@%s: %v\n", xat[t2.At], t2.Xkv.Flatten()) defer close(testq)
for i := range keysv1 {
kadj10 := KAdj(t1,t0, allTestKeys(t0,t1,t2)) keys1 := keysv1[i]
kadj21 := KAdj(t2,t1, allTestKeys(t0,t1,t2)) keys2 := keysv2[i]
kadj12 := KAdj(t1,t2, allTestKeys(t0,t1,t2)) keys3 := keysv3[i]
// kadj210 = kadj10·kadj21
kadj210 := kadj10.Mul(kadj21)
ø := blib.PPTreeSubSet{} kv1 := map[Key]string{}
kv2 := map[Key]string{}
kv3 := map[Key]string{}
for _, k := range keys1 { kv1[Key(k)] = randv() }
for _, k := range keys2 { kv2[Key(k)] = randv() }
for _, k := range keys3 { kv3[Key(k)] = randv() }
// verify t0 -> t1 Track(keys1) Rebuild -> t2 Track(keys2) Rebuild treev1, err1 := sg.AllStructs(kv1, maxdepth, maxsplit, n, rng.Int63())
// for all combinations of keys1 and keys2 treev2, err2 := sg.AllStructs(kv2, maxdepth, maxsplit, n, rng.Int63())
for k1idx := range IntSets(len(tAllKeyv)) { treev3, err3 := sg.AllStructs(kv3, maxdepth, maxsplit, n, rng.Int63())
keys1 := setKey{} err := xerr.Merge(err1, err2, err3)
for _, idx1 := range k1idx { if err != nil {
keys1.Add(tAllKeyv[idx1]) t.Fatal(err)
} }
// δkv1_1 = t1.δxkv / kadj10(keys1) emit := func(tree string, flags ΔBTestFlags) {
keys1_0 := kadj10.Map(keys1) // skip emitting this entry if both Update and
δkv1_1 := map[Key]Δstring{} // Rebuild are requested to be skipped.
for k := range keys1_0 { if flags == (ΔBTest_SkipUpdate | ΔBTest_SkipRebuild) {
δv, ok := t1.Δxkv[k] return
if ok {
δkv1_1[k] = δv
} }
testq <- ΔBTestEntry{tree, nil, flags}
} }
Tkeys1 := trackSet(t1.Xkv, keys1) URSkipIf := func(ucond, rcond bool) ΔBTestFlags {
Tkeys1_0 := trackSet(t1.Xkv, keys1_0) var flags ΔBTestFlags
if ucond {
t.Run(fmt.Sprintf(" T%s;R", keys1), func(t *testing.T) { flags |= ΔBTest_SkipUpdate
δbtail := NewΔBtail(t0.At, db) }
if rcond {
flags |= ΔBTest_SkipRebuild
}
return flags
}
// assert trackSet=ø, trackNew=ø, vδB=[] for j := range treev1 {
δbtail.assertTrack(t, "@at0", ø, ø) for k := range treev2 {
assertΔTtail(t, "@at0", δbtail, t0, treeRoot, xat, for l := range treev3 {
/*vδT=ø*/) // limit rebuild to subset of tree topologies,
// because #(triplets) grow as n^3. See nrebuild
// definition above for details.
norebuild := (j >= nrebuild ||
k >= nrebuild ||
l >= nrebuild)
xverifyΔBTail_rebuild_U(t, δbtail, treeRoot, t0, t1, xat, // C_{l-1} -> Aj (pair first seen on k=0)
/*trackSet=*/ø, emit(treev1[j], URSkipIf(k != 0, norebuild))
/*vδT=ø*/)
xverifyΔBTail_rebuild_TR(t, δbtail, t1, treeRoot, xat,
// after Track(keys1)
keys1,
/*trackSet=*/ ø,
/*trackNew=*/ Tkeys1,
// after rebuild // Aj -> Bk (pair first seen on l=0)
/*trackSet=*/ Tkeys1_0, emit(treev2[k], URSkipIf(l != 0, norebuild))
/*vδT=*/ δkv1_1)
t.Run((" →" + t2.Tree), func(t *testing.T) { // Bk -> Cl (pair first seen on j=0)
// keys1R2 is full set of keys that should become tracked after emit(treev3[l], URSkipIf(j != 0, norebuild))
// Update() (which includes rebuild)
keys1R2 := kadj12.Map(keys1)
for {
keys1R2_ := kadj210.Map(keys1R2)
if keys1R2.Equal(keys1R2_) {
break
}
keys1R2 = keys1R2_
} }
// δkvX_k1R2 = tX.δxkv / keys1R2
δkv1_k1R2 := map[Key]Δstring{}
δkv2_k1R2 := map[Key]Δstring{}
for k := range keys1R2 {
δv1, ok := t1.Δxkv[k]
if ok {
δkv1_k1R2[k] = δv1
} }
δv2, ok := t2.Δxkv[k]
if ok {
δkv2_k1R2[k] = δv2
} }
} }
}()
testΔBTail(t, testq)
}
// testΔBTail verifies ΔBTail on sequence of tree topologies coming from testq.
func testΔBTail(t_ *testing.T, testq chan ΔBTestEntry) {
t := xbtreetest.NewT(t_)
Tkeys1R2 := trackSet(t2.Xkv, keys1R2) var t0 *xbtreetest.Commit
for test := range testq {
t1 := t.Head()
t2 := t.CommitTree(test.tree)
xverifyΔBTail_rebuild_U(t, δbtail, treeRoot, t1, t2, xat, subj := fmt.Sprintf("%s -> %s", t1.Tree, t2.Tree)
/*trackSet=*/ Tkeys1R2, //t.Logf("\n\n\n**** %s ****\n\n", subj)
/*vδT=*/ δkv1_k1R2, δkv2_k1R2)
// tRestKeys2 = tAllKeys - keys1 // KAdj
// reduce that to = tAllKeys - keys1R2 in short mode if kadjOK := test.kadjOK; kadjOK != nil {
// ( if key from keys2 already became tracked after Track(keys1) + Update, t.Run(fmt.Sprintf("KAdj/%s→%s", t1.Tree, t2.Tree), func(t *testing.T) {
// adding Track(that-key), is not adding much testing coverage to recompute paths ) kadj := KAdj(t1, t2)
var tRestKeys2 setKey if !reflect.DeepEqual(kadj, kadjOK) {
if testing.Short() { t.Fatalf("BUG: computed kadj is wrong:\nkadjOK: %v\nkadj : %v\n\n", kadjOK, kadj)
tRestKeys2 = tAllKeys.Difference(keys1R2) }
} else { })
tRestKeys2 = tAllKeys.Difference(keys1)
} }
tRestKeyv2 := tRestKeys2.SortedElements() // ΔBTail.Update
for k2idx := range IntSets(len(tRestKeyv2)) { if test.flags & ΔBTest_SkipUpdate == 0 {
keys2 := setKey{} xverifyΔBTail_Update(t.T, subj, t.DB, t.Root(), t1,t2)
for _, idx2 := range k2idx {
keys2.Add(tRestKeyv2[idx2])
} }
// keys12R2 is full set of keys that should become tracked after // ΔBTail.rebuild
// Track(keys2) + rebuild if t0 != nil && (test.flags & ΔBTest_SkipRebuild == 0) {
keys12R2 := keys1R2.Union(keys2) xverifyΔBTail_rebuild(t.T, t.DB, t.Root(), t0,t1,t2)
for {
keys12R2_ := kadj210.Map(keys12R2)
if keys12R2.Equal(keys12R2_) {
break
} }
keys12R2 = keys12R2_
t0, t1 = t1, t2
} }
}
Tkeys2 := trackSet(t2.Xkv, keys2)
Tkeys12R2 := trackSet(t2.Xkv, keys12R2)
/*
fmt.Printf("\n\n\nKKK\nkeys1=%s keys2=%s\n", keys1, keys2)
fmt.Printf("keys1R2: %s\n", keys1R2)
fmt.Printf("keys12R2: %s\n", keys12R2)
fmt.Printf("t0.Xkv: %v\n", t0.Xkv) // xverifyΔBTail_Update verifies how ΔBTail handles ZODB update for a tree with changes in between t1->t2.
fmt.Printf("t1.Xkv: %v\n", t1.Xkv) //
fmt.Printf("t2.Xkv: %v\n", t2.Xkv) // Note: this test verifies only single treediff step of ΔBtail.Update.
fmt.Printf("kadj21: %v\n", kadj21) // the cycling phase of update, that is responsible to recompute older
fmt.Printf("kadj12: %v\n", kadj12) // entries when key coverage grows, is exercised by
fmt.Printf("Tkeys2 -> %s\n", Tkeys2) // xverifyΔBTail_rebuild.
fmt.Printf("Tkeys1R2 -> %s\n", Tkeys1R2) func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1, t2 *xbtreetest.Commit) {
fmt.Printf("Tkeys2 \\ Tkeys1R2 -> %s\n", Tkeys2.Difference(Tkeys1R2)) // verify transition at1->at2 for all initial states of tracked {keys} from kv1 + kv2 + ∞
fmt.Printf("\n\n\n") t.Run(fmt.Sprintf("Update/%s→%s", t1.Tree, t2.Tree), func(t *testing.T) {
*/ allKeys := allTestKeys(t1, t2)
allKeyv := allKeys.SortedElements()
kadj12 := KAdj(t1, t2)
// δkvX_k12R2 = tX.δxkv / keys12R2 // verify at1->at2 for all combination of initial tracked keys.
δkv1_k12R2 := make(map[Key]Δstring, len(t1.Δxkv)) for kidx := range IntSets(len(allKeyv)) {
δkv2_k12R2 := make(map[Key]Δstring, len(t2.Δxkv)) keys := setKey{}
for k := range keys12R2 { for _, idx := range kidx {
δv1, ok := t1.Δxkv[k] keys.Add(allKeyv[idx])
if ok {
δkv1_k12R2[k] = δv1
}
δv2, ok := t2.Δxkv[k]
if ok {
δkv2_k12R2[k] = δv2
}
} }
// t.Run is expensive at this level of nest // this t.Run allocates and keeps too much memory in -verylong
//t.Run(" T"+keys2.String()+";R", func(t *testing.T) { // also it is not so useful as above "Update/t1->t2"
δbtail_ := δbtail.Clone() //t.Run(fmt.Sprintf(" track=%s", keys), func(t *testing.T) {
xverifyΔBTail_rebuild_TR(t, δbtail_, t2, treeRoot, xat, xverifyΔBTail_Update1(t, subj, db, treeRoot, t1,t2, keys, kadj12)
// after Track(keys2)
keys2,
/*trackSet*/ Tkeys1R2,
/*trackNew*/ Tkeys2.Difference(
// trackNew should not cover ranges that are
// already in trackSet
Tkeys1R2),
// after rebuild
/* trackSet=*/ Tkeys12R2,
/*vδT=*/ δkv1_k12R2, δkv2_k12R2)
//}) //})
} }
}) })
})
}
})
} }
// xverifyΔBTail_rebuild_U verifies ΔBtail state after Update(ti->tj). // xverifyΔBTail_Update1 verifies how ΔBTail handles ZODB update at1->at2 from initial
func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, treeRoot zodb.Oid, ti, tj *xbtreetest.Commit, xat map[zodb.Tid]string, trackSet blib.PPTreeSubSet, vδTok ...map[Key]Δstring) { // tracked state defined by initialTrackedKeys.
t.Helper() func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1,t2 *xbtreetest.Commit, initialTrackedKeys setKey, kadj KAdjMatrix) {
X := exc.Raiseif X := exc.Raiseif
ø := blib.PPTreeSubSet{} //t.Logf("\n>>> Track=%s\n", initialTrackedKeys)
subj := fmt.Sprintf("after Update(@%s→@%s)", xat[ti.At], xat[tj.At])
// Update ati -> atj δZ := t2.ΔZ
δB, err := δbtail.Update(tj.ΔZ); X(err) d12 := t2.Δxkv
δbtail.assertTrack(t, subj, trackSet, ø)
assertΔTtail(t, subj, δbtail, tj, treeRoot, xat, vδTok...)
// assert δB = vδTok[-1] var TrackedδZ setKey = nil
var δT, δTok map[Key]Δstring var kadjTrackedδZ setKey = nil
if l := len(vδTok); l > 0 { var δT, δTok map[Key]Δstring = nil, nil
δTok = vδTok[l-1] δZset := setOid{}
for _, oid := range δZ.Changev {
δZset.Add(oid)
} }
if len(δTok) == 0 {
δTok = nil // badf queues error message to be reported on return.
var badv []string
badf := func(format string, argv ...interface{}) {
badv = append(badv, fmt.Sprintf(format, argv...))
} }
δrootsOK := 1 defer func() {
if δTok == nil { if badv != nil || t.Failed() {
δrootsOK = 0 emsg := fmt.Sprintf("%s ; tracked=%v :\n\n", subj, initialTrackedKeys)
emsg += fmt.Sprintf("d12: %v\nδTok: %v\nδT: %v\n\n", d12, δTok, δT)
emsg += fmt.Sprintf("δZ: %v\n", δZset)
emsg += fmt.Sprintf("Tracked^δZ: %v\n", TrackedδZ)
emsg += fmt.Sprintf("kadj[Tracked^δZ]: %v\n", kadjTrackedδZ)
emsg += fmt.Sprintf("kadj: %v\n\n", kadj)
emsg += strings.Join(badv, "\n")
emsg += "\n"
t.Fatal(emsg)
} }
}()
δroots := setOid{}
for root := range δbtail.vδTbyRoot { // δbtail @at1 with initial tracked set
δroots.Add(root) δbtail := NewΔBtail(t1.At, db)
xtrackKeys(δbtail, t1, initialTrackedKeys)
// TrackedδZ = Tracked ^ δZ (i.e. a tracked node has changed, or its coverage was changed)
TrackedδZ = setKey{}
for k := range initialTrackedKeys {
leaf1 := t1.Xkv.Get(k)
oid1 := leaf1.Oid
if oid1 == zodb.InvalidOid { // embedded bucket
oid1 = leaf1.Parent.Oid
} }
δToid, ok := δB.ΔByRoot[treeRoot] leaf2 := t2.Xkv.Get(k)
if ok { oid2 := leaf2.Oid
δT = XGetδKV(ti, tj, δToid) if oid2 == zodb.InvalidOid { // embedded bucket
oid2 = leaf2.Parent.Oid
} }
if δB.Rev != tj.At { if δZset.Has(oid1) || δZset.Has(oid2) || (leaf1.Keycov != leaf2.Keycov) {
t.Errorf("%s: δB.Rev: have %s ; want %s", subj, δB.Rev, tj.At) TrackedδZ.Add(k)
} }
if len(δB.ΔByRoot) != δrootsOK {
t.Errorf("%s: len(δB.ΔByRoot) != %d ; δroots=%v", subj, δrootsOK, δroots)
} }
if !δTEqual(δT, δTok) {
t.Errorf("%s: δB.ΔBByRoot[%s]:\nhave: %v\nwant: %v", subj, treeRoot, δT, δTok) kadjTrackedδZ = setKey{} // kadj[Tracked^δZ] (all keys adjacent to tracked^δZ)
for k := range TrackedδZ {
kadjTrackedδZ.Update(kadj[k])
}
// assert TrackedδZ ∈ kadj[TrackedδZ]
trackNotInKadj := TrackedδZ.Difference(kadjTrackedδZ)
if len(trackNotInKadj) > 0 {
badf("BUG: Tracked^δZ ∉ kadj[Tracked^δZ] ; extra=%v", trackNotInKadj)
return
}
// k ∈ d12
// k ∈ δT <=>
// k ∈ U kadj[·]
// ·∈tracking^δZ
δTok = map[Key]Δstring{} // d12[all keys that should be present in δT]
for k,δv := range d12 {
if kadjTrackedδZ.Has(k) {
δTok[k] = δv
}
} }
}
// xverifyΔBTail_rebuild_TR verifies ΔBtail state after Track(keys) + rebuild.
func xverifyΔBTail_rebuild_TR(t *testing.T, δbtail *ΔBtail, tj *xbtreetest.Commit, treeRoot zodb.Oid, xat map[zodb.Tid]string, keys setKey, trackSet blib.PPTreeSubSet, trackNew, trackSetAfterRebuild blib.PPTreeSubSet, vδTok ...map[Key]Δstring) {
t.Helper()
ø := blib.PPTreeSubSet{} ø := blib.PPTreeSubSet{}
// Track(keys) // trackSet1 = xkv1[tracked1]
xtrackKeys(δbtail, tj, keys) // trackSet2 = xkv2[tracked2] ( = xkv2[kadj[tracked1]]
trackSet1, tkeyCov1 := trackSetWithCov(t1.Xkv, initialTrackedKeys)
trackSet2, tkeyCov2 := trackSetWithCov(t2.Xkv, initialTrackedKeys.Union(kadjTrackedδZ))
subj := fmt.Sprintf("@%s: after Track%v", xat[tj.At], keys) // verify δbtail.trackSet against @at1
δbtail.assertTrack(t, subj, trackSet, trackNew) δbtail.assertTrack(t, "1", ø, trackSet1)
δbtail.rebuildAll() // δB <- δZ
//
// also call _Update1 directly to verify δtkeycov return from treediff
// the result of Update and _Update1 should be the same since δbtail is initially empty.
δbtail_ := δbtail.Clone()
δB1, err := δbtail_._Update1(δZ); X(err) // XXX don't compute treediff twice
// assert tkeyCov1 ⊂ tkeyCov2
dkeycov12 := tkeyCov1.Difference(tkeyCov2)
if !dkeycov12.Empty() {
t.Errorf("BUG: tkeyCov1 ⊄ tkeyCov2:\n\ttkeyCov1: %s\n\ttkeyCov2: %s\n\ttkeyCov1 \\ tkeyCov2: %s", tkeyCov1, tkeyCov2, dkeycov12)
}
// assert δtkeycov == δ(tkeyCov1, tkeyCov2)
δtkeycovOK := tkeyCov2.Difference(tkeyCov1)
δtkeycov := &blib.RangedKeySet{}
if __, ok := δB1.ByRoot[treeRoot]; ok {
δtkeycov = __.δtkeycov1
}
if !δtkeycov.Equal(δtkeycovOK) {
badf("δtkeycov wrong:\nhave: %s\nwant: %s", δtkeycov, δtkeycovOK)
}
subj += " + rebuild" δB, err := δbtail.Update(δZ); X(err)
δbtail.assertTrack(t, subj, trackSetAfterRebuild, ø) // XXX assert δB.roots == δTKeyCov roots
// XXX assert δBtail[root].vδT = δBtail_[root].vδT
// XXX verify Get -> XXX assertΔTtail ? if δB.Rev != δZ.Tid {
badf("δB: rev != δZ.Tid ; rev=%s δZ.Tid=%s", δB.Rev, δZ.Tid)
return
}
// verify δbtail.vδTbyRoot[treeRoot] // verify δbtail.trackSet against @at2
assertΔTtail(t, subj, δbtail, tj, treeRoot, xat, vδTok...) δbtail.assertTrack(t, "2", trackSet2, ø)
}
// assertΔTtail verifies state of ΔTtail that corresponds to treeRoot in δbtail.
// it also verifies that δbtail.vδBroots matches ΔTtail data.
func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *xbtreetest.Commit, treeRoot zodb.Oid, xat map[zodb.Tid]string, vδTok ...map[Key]Δstring) {
t.Helper()
// XXX +KVAtTail, +lastRevOf
l := len(vδTok) // assert δB.ByRoot == {treeRoot -> ...} if δTok != ø
var vatOK []zodb.Tid // == ø if δTok == ø
var vδTok_ []map[Key]Δstring rootsOK := setOid{}
at2t := map[zodb.Tid]*xbtreetest.Commit{tj.At: tj} if len(δTok) > 0 {
t0 := tj rootsOK.Add(treeRoot)
for i := 0; i<l; i++ {
// empty vδTok entries means they should be absent in vδT
if δTok := vδTok[l-i-1]; len(δTok) != 0 {
vatOK = append([]zodb.Tid{t0.At}, vatOK...)
vδTok_ = append([]map[Key]Δstring{δTok}, vδTok_...)
} }
t0 = t0.Prev roots := setOid{}
at2t[t0.At] = t0 for root := range δB.ΔByRoot {
roots.Add(root)
} }
vδTok = vδTok_ if !reflect.DeepEqual(roots, rootsOK) {
δTtail, ok := δbtail.vδTbyRoot[treeRoot] badf("δB: roots != rootsOK ; roots=%v rootsOK=%v", roots, rootsOK)
var vδToid []ΔTree
if ok {
vδToid = δTtail.vδT
} }
_, inδB := δB.ΔByRoot[treeRoot]
l = len(vδToid) if !inδB {
var vat []zodb.Tid return
var vδT []map[Key]Δstring
atPrev := t0.At
for _, δToid := range vδToid {
vat = append(vat, δToid.Rev)
δT := XGetδKV(at2t[atPrev], at2t[δToid.Rev], δToid.ΔKV) // {} k -> δ(ZBlk(oid).data)
vδT = append(vδT, δT)
atPrev = δToid.Rev
} }
var vatδB []zodb.Tid // δbtail.vδBroots/treeRoot
for _, δBroots := range δbtail.vδBroots { // δT <- δB
if δBroots.ΔRoots.Has(treeRoot) { δToid := δB.ΔByRoot[treeRoot] // {} k -> δoid
vatδB = append(vatδB, δBroots.Rev) δT = XGetδKV(t1,t2, δToid) // {} k -> δ(ZBlk(oid).data)
// δT must be subset of d12.
// changed keys, that are
// - in tracked set -> must be present in δT
// - outside tracked set -> may be present in δT (kadj gives exact answer)
// δT is subset of d12
for _, k := range sortedKeys(δT) {
_, ind12 := d12[k]
if !ind12 {
badf("δT[%v] ∉ d12", k)
} }
} }
tok := tidvEqual(vat, vatOK) && vδTEqual(vδT, vδTok) // k ∈ tracked set -> must be present in δT
bok := tidvEqual(vatδB, vatOK) // k ∉ tracked set -> may be present in δT (kadj gives exact answer)
if !(tok && bok) { for _, k := range sortedKeys(d12) {
emsg := fmt.Sprintf("%s: vδT:\n", subj) _, inδT := δT[k]
have := "" _, inδTok := δTok[k]
for i := 0; i<len(vδT); i++ { if inδT && !inδTok {
have += fmt.Sprintf("\n\t@%s: %v", xat[vat[i]], vδT[i]) badf("δT[%v] ∉ δTok", k)
} }
emsg += fmt.Sprintf("have: %s\n", have)
if !tok { if !inδT && inδTok {
want := "" badf("δT ∌ δTok[%v]", k)
for i := 0; i<len(vδTok); i++ {
want += fmt.Sprintf("\n\t@%s: %v", xat[vatOK[i]], vδTok[i])
}
emsg += fmt.Sprintf("want: %s\n", want)
} }
if !bok { if inδT {
vδb_root := "" if δT[k] != d12[k] {
for i := 0; i<len(vatδB); i++ { badf("δT[%v] ≠ δTok[%v]", k, k)
vδb_root += fmt.Sprintf("\n\t@%s", xat[vatδB[i]])
} }
emsg += fmt.Sprintf("vδb/root: %s\n", vδb_root)
} }
t.Error(emsg)
} }
} }
// xtrackKeys issues δbtail.Track requests for tree[keys]. // xverifyΔBTail_rebuild verifies ΔBtail.rebuild during t0->t1->t2 transition.
// XXX place //
func xtrackKeys(δbtail *ΔBtail, t *xbtreetest.Commit, keys setKey) { // t0->t1 exercises from-scratch rebuild,
X := exc.Raiseif // t1->t2 further exercises incremental rebuild.
head := δbtail.Head() //
if head != t.At { // It also exercises rebuild phase of ΔBtail.Update.
panicf("BUG: δbtail.head: %s ; t.at: %s", head, t.At) func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1, t2 *xbtreetest.Commit) {
} t.Run(fmt.Sprintf("rebuild/%s→%s", t0.Tree, t1.Tree), func(t *testing.T) {
tAllKeys := allTestKeys(t0, t1, t2)
tAllKeyv := tAllKeys.SortedElements()
for k := range keys { // tid -> "at_i"
// NOTE: if tree is deleted - the following adds it to tracked xat := map[zodb.Tid]string{
// set with every key being a hole. This aligns with the t0.At: "at0",
// following situation t1.At: "at1",
// t2.At: "at2",
// T1 -> ø -> T2
//
// where after T1->ø, even though the tree becomes deleted, its root
// continues to be tracked and all keys migrate to holes in the
// tracking set. By aligning initial state to the same as after
// T1->ø, we test what will happen on ø->T2.
b := t.Xkv.Get(k)
err := δbtail.track(k, b.Path()); X(err)
} }
}
// xverifyΔBTail_GetAt verifies δBtail.Get on series of vt ZODB changes. //fmt.Printf("@%s: %v\n", xat[t0.At], t0.Xkv.Flatten())
// XXX //fmt.Printf("@%s: %v\n", xat[t1.At], t1.Xkv.Flatten())
// XXX kill //fmt.Printf("@%s: %v\n", xat[t2.At], t2.Xkv.Flatten())
/*
func ___xverifyΔBTail_GetAt(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt ...*xbtreetest.Commit) {
subj := vt[0].Tree
for _, t := range vt[1:] {
subj += "→" + t.Tree
}
t.Run(fmt.Sprintf("Get/%s", subj), func(t *testing.T) { kadj10 := KAdj(t1,t0, allTestKeys(t0,t1,t2))
// tid -> "at_i" kadj21 := KAdj(t2,t1, allTestKeys(t0,t1,t2))
xat := map[zodb.Tid]string{} kadj12 := KAdj(t1,t2, allTestKeys(t0,t1,t2))
for i := range vt {
xat[vt[i].At] = fmt.Sprintf("at%d", i)
fmt.Printf("@%s: %v\n", xat[vt[i].At], vt[i].Xkv.Flatten()) // kadj210 = kadj10·kadj21
} kadj210 := kadj10.Mul(kadj21)
tkeys := allTestKeys(vt...) ø := blib.PPTreeSubSet{}
tkeyv := tkeys.SortedElements()
// verify t1->t2-> ... ->tn Track(keys) Get(keys, @at) // verify t0 -> t1 Track(keys1) Rebuild -> t2 Track(keys2) Rebuild
// for all combinations of tracked keys and at // for all combinations of keys1 and keys2
for kidx := range IntSets(len(tkeyv)) { for k1idx := range IntSets(len(tAllKeyv)) {
keys := setKey{} keys1 := setKey{}
for _, idx := range kidx { for _, idx1 := range k1idx {
keys.Add(tkeyv[idx]) keys1.Add(tAllKeyv[idx1])
} }
t.Run(fmt.Sprintf("track=%s", keys), func(t *testing.T) { // δkv1_1 = t1.δxkv / kadj10(keys1)
xverifyΔBTail_GetAt1(t, db, treeRoot, vt, xat, keys) keys1_0 := kadj10.Map(keys1)
}) δkv1_1 := map[Key]Δstring{}
for k := range keys1_0 {
δv, ok := t1.Δxkv[k]
if ok {
δkv1_1[k] = δv
}
} }
})
}
func xverifyΔBTail_GetAt1(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt []*xbtreetest.Commit, xat map[zodb.Tid]string, keys setKey) { Tkeys1 := trackSet(t1.Xkv, keys1)
X := exc.Raiseif Tkeys1_0 := trackSet(t1.Xkv, keys1_0)
// t1->t2-> ... -> tn t.Run(fmt.Sprintf(" T%s;R", keys1), func(t *testing.T) {
δbtail := NewΔBtail(vt[0].At, db) δbtail := NewΔBtail(t0.At, db)
for i := 1; i < len(vt); i++ {
_, err := δbtail.Update(vt[i].ΔZ); X(err) // assert trackSet=ø, trackNew=ø, vδB=[]
δbtail.assertTrack(t, "@at0", ø, ø)
assertΔTtail(t, "@at0", δbtail, t0, treeRoot, xat,
/*vδT=ø*/)
xverifyΔBTail_rebuild_U(t, δbtail, treeRoot, t0, t1, xat,
/*trackSet=*/ø,
/*vδT=ø*/)
xverifyΔBTail_rebuild_TR(t, δbtail, t1, treeRoot, xat,
// after Track(keys1)
keys1,
/*trackSet=*/ ø,
/*trackNew=*/ Tkeys1,
// after rebuild
/*trackSet=*/ Tkeys1_0,
/*vδT=*/ δkv1_1)
t.Run((" →" + t2.Tree), func(t *testing.T) {
// keys1R2 is full set of keys that should become tracked after
// Update() (which includes rebuild)
keys1R2 := kadj12.Map(keys1)
for {
keys1R2_ := kadj210.Map(keys1R2)
if keys1R2.Equal(keys1R2_) {
break
}
keys1R2 = keys1R2_
} }
// Track(keys) // δkvX_k1R2 = tX.δxkv / keys1R2
txn, ctx := transaction.New(context.Background()) δkv1_k1R2 := map[Key]Δstring{}
defer txn.Abort() δkv2_k1R2 := map[Key]Δstring{}
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: vt[len(vt)-1].At}); X(err) for k := range keys1R2 {
xtree, err := zconn.Get(ctx, treeRoot); X(err) δv1, ok := t1.Δxkv[k]
ztree := xtree.(*Tree) if ok {
δkv1_k1R2[k] = δv1
}
δv2, ok := t2.Δxkv[k]
if ok {
δkv2_k1R2[k] = δv2
}
}
Tkeys1R2 := trackSet(t2.Xkv, keys1R2)
xverifyΔBTail_rebuild_U(t, δbtail, treeRoot, t1, t2, xat,
/*trackSet=*/ Tkeys1R2,
/*vδT=*/ δkv1_k1R2, δkv2_k1R2)
for k := range keys { // tRestKeys2 = tAllKeys - keys1
_, _, path, err := ZTreeGetBlkData(ctx, ztree, k); X(err) // reduce that to = tAllKeys - keys1R2 in short mode
err = δbtail.Track(k, path); X(err) // ( if key from keys2 already became tracked after Track(keys1) + Update,
// adding Track(that-key), is not adding much testing coverage to recompute paths )
var tRestKeys2 setKey
if testing.Short() {
tRestKeys2 = tAllKeys.Difference(keys1R2)
} else {
tRestKeys2 = tAllKeys.Difference(keys1)
} }
// verify GetAt(k, @at) for all keys and @at tRestKeyv2 := tRestKeys2.SortedElements()
for i := 1; i < len(vt); i++ { for k2idx := range IntSets(len(tRestKeyv2)) {
at := vt[i].At keys2 := setKey{}
for _, k := range keys.SortedElements() { for _, idx2 := range k2idx {
vOid, ok, rev, revExact, err := δbtail.GetAt(ctx, ztree, k, at); X(err) keys2.Add(tRestKeyv2[idx2])
v := xzgetBlkDataAt(db, vOid, rev) }
v_, ok_ := vt[i].Xkv.Get(k).kv[k] // keys12R2 is full set of keys that should become tracked after
rev_, revExact_ := vt[i].At, false // Track(keys2) + rebuild
for j := i-1; j >= 0; j-- { keys12R2 := keys1R2.Union(keys2)
v__ := vt[j].Xkv.Get(k).kv[k] for {
if v__ != v_ { keys12R2_ := kadj210.Map(keys12R2)
rev_ = vt[j+1].At if keys12R2.Equal(keys12R2_) {
revExact_ = true
break break
} }
rev_ = vt[j].At keys12R2 = keys12R2_
} }
if v == "" { v = DEL } Tkeys2 := trackSet(t2.Xkv, keys2)
if v_ == "" { v_ = DEL } Tkeys12R2 := trackSet(t2.Xkv, keys12R2)
/*
fmt.Printf("\n\n\nKKK\nkeys1=%s keys2=%s\n", keys1, keys2)
fmt.Printf("keys1R2: %s\n", keys1R2)
fmt.Printf("keys12R2: %s\n", keys12R2)
if !(v == v_ && ok == ok_ && rev == rev_ && revExact == revExact_) { fmt.Printf("t0.Xkv: %v\n", t0.Xkv)
t.Errorf("Get(%d, @%s) ->\nhave: %s, %v, @%s, %v\nwant: %s, %v, @%s, %v", fmt.Printf("t1.Xkv: %v\n", t1.Xkv)
k, xat[at], fmt.Printf("t2.Xkv: %v\n", t2.Xkv)
v, ok, xat[rev], revExact, fmt.Printf("kadj21: %v\n", kadj21)
v_, ok_, xat[rev_], revExact_) fmt.Printf("kadj12: %v\n", kadj12)
} fmt.Printf("Tkeys2 -> %s\n", Tkeys2)
} fmt.Printf("Tkeys1R2 -> %s\n", Tkeys1R2)
} fmt.Printf("Tkeys2 \\ Tkeys1R2 -> %s\n", Tkeys2.Difference(Tkeys1R2))
} fmt.Printf("\n\n\n")
*/ */
// ---------------------------------------- // δkvX_k12R2 = tX.δxkv / keys12R2
δkv1_k12R2 := make(map[Key]Δstring, len(t1.Δxkv))
// ΔBTestEntry represents one entry in ΔBTail tests. δkv2_k12R2 := make(map[Key]Δstring, len(t2.Δxkv))
type ΔBTestEntry struct { for k := range keys12R2 {
tree string // next tree topology δv1, ok := t1.Δxkv[k]
kadjOK KAdjMatrix // adjacency matrix against previous case (optional) if ok {
flags ΔBTestFlags δkv1_k12R2[k] = δv1
} }
δv2, ok := t2.Δxkv[k]
if ok {
δkv2_k12R2[k] = δv2
}
}
type ΔBTestFlags int // t.Run is expensive at this level of nest
const ΔBTest_SkipUpdate ΔBTestFlags = 1 // skip verifying Update for this test entry //t.Run(" T"+keys2.String()+";R", func(t *testing.T) {
const ΔBTest_SkipRebuild ΔBTestFlags = 2 // skip verifying rebuild for this test entry δbtail_ := δbtail.Clone()
xverifyΔBTail_rebuild_TR(t, δbtail_, t2, treeRoot, xat,
// after Track(keys2)
keys2,
/*trackSet*/ Tkeys1R2,
/*trackNew*/ Tkeys2.Difference(
// trackNew should not cover ranges that are
// already in trackSet
Tkeys1R2),
// ΔBTest converts xtest into ΔBTestEntry. // after rebuild
// xtest can be string|ΔBTestEntry. /* trackSet=*/ Tkeys12R2,
func ΔBTest(xtest interface{}) ΔBTestEntry { /*vδT=*/ δkv1_k12R2, δkv2_k12R2)
var test ΔBTestEntry //})
switch xtest := xtest.(type) {
case string:
test.tree = xtest
test.kadjOK = nil
test.flags = 0
case ΔBTestEntry:
test = xtest
default:
panicf("BUG: ΔBTest: bad type %T", xtest)
} }
return test })
})
}
})
} }
// xverifyΔBTail_rebuild_U verifies ΔBtail state after Update(ti->tj).
func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, treeRoot zodb.Oid, ti, tj *xbtreetest.Commit, xat map[zodb.Tid]string, trackSet blib.PPTreeSubSet, vδTok ...map[Key]Δstring) {
t.Helper()
X := exc.Raiseif
ø := blib.PPTreeSubSet{}
// testΔBTail verifies ΔBTail on sequence of tree topologies coming from testq. subj := fmt.Sprintf("after Update(@%s→@%s)", xat[ti.At], xat[tj.At])
func testΔBTail(t_ *testing.T, testq chan ΔBTestEntry) {
t := xbtreetest.NewT(t_)
var t0 *xbtreetest.Commit
for test := range testq {
t1 := t.Head()
t2 := t.CommitTree(test.tree)
subj := fmt.Sprintf("%s -> %s", t1.Tree, t2.Tree) // Update ati -> atj
//t.Logf("\n\n\n**** %s ****\n\n", subj) δB, err := δbtail.Update(tj.ΔZ); X(err)
δbtail.assertTrack(t, subj, trackSet, ø)
assertΔTtail(t, subj, δbtail, tj, treeRoot, xat, vδTok...)
// KAdj // assert δB = vδTok[-1]
if kadjOK := test.kadjOK; kadjOK != nil { var δT, δTok map[Key]Δstring
t.Run(fmt.Sprintf("KAdj/%s→%s", t1.Tree, t2.Tree), func(t *testing.T) { if l := len(vδTok); l > 0 {
kadj := KAdj(t1, t2) δTok = vδTok[l-1]
if !reflect.DeepEqual(kadj, kadjOK) {
t.Fatalf("BUG: computed kadj is wrong:\nkadjOK: %v\nkadj : %v\n\n", kadjOK, kadj)
} }
}) if len(δTok) == 0 {
δTok = nil
} }
δrootsOK := 1
// ΔBTail.Update if δTok == nil {
if test.flags & ΔBTest_SkipUpdate == 0 { δrootsOK = 0
xverifyΔBTail_Update(t.T, subj, t.DB, t.Root(), t1,t2)
} }
// ΔBTail.rebuild δroots := setOid{}
if t0 != nil && (test.flags & ΔBTest_SkipRebuild == 0) { for root := range δbtail.vδTbyRoot {
xverifyΔBTail_rebuild(t.T, t.DB, t.Root(), t0,t1,t2) δroots.Add(root)
} }
δToid, ok := δB.ΔByRoot[treeRoot]
t0, t1 = t1, t2 if ok {
δT = XGetδKV(ti, tj, δToid)
} }
} if δB.Rev != tj.At {
t.Errorf("%s: δB.Rev: have %s ; want %s", subj, δB.Rev, tj.At)
// TestΔBTail verifies ΔBTail for explicitly provided tree topologies.
func TestΔBTail(t *testing.T) {
// K is shorthand for setKey
K := func(keyv ...Key) setKey {
ks := setKey{}
for _, k := range keyv { ks.Add(k) }
return ks
} }
// oo is shorthand for KeyMax if len(δB.ΔByRoot) != δrootsOK {
const oo = KeyMax t.Errorf("%s: len(δB.ΔByRoot) != %d ; δroots=%v", subj, δrootsOK, δroots)
// A is shorthand for KAdjMatrix
type A = KAdjMatrix
// Δ is shorthand for ΔBTestEntry
Δ := func(tree string, kadjOK A) (test ΔBTestEntry) {
test.tree = tree
test.kadjOK = kadjOK
return test
} }
if !δTEqual(δT, δTok) {
t.Errorf("%s: δB.ΔBByRoot[%s]:\nhave: %v\nwant: %v", subj, treeRoot, δT, δTok)
}
}
// test known cases going through tree1 -> tree2 -> ... // xverifyΔBTail_rebuild_TR verifies ΔBtail state after Track(keys) + rebuild.
testv := []interface{} { func xverifyΔBTail_rebuild_TR(t *testing.T, δbtail *ΔBtail, tj *xbtreetest.Commit, treeRoot zodb.Oid, xat map[zodb.Tid]string, keys setKey, trackSet blib.PPTreeSubSet, trackNew, trackSetAfterRebuild blib.PPTreeSubSet, vδTok ...map[Key]Δstring) {
// start from non-empty tree to verify both ->empty and empty-> transitions t.Helper()
"T/B1:a,2:b", ø := blib.PPTreeSubSet{}
// empty
"T/B:",
// +1
Δ("T/B1:a",
A{1: K(1,oo),
oo: K(1,oo)}),
// +2
Δ("T/B1:a,2:b",
A{1: K(1,2,oo),
2: K(1,2,oo),
oo: K(1,2,oo)}),
// -1 // Track(keys)
Δ("T/B2:b", xtrackKeys(δbtail, tj, keys)
A{1: K(1,2,oo),
2: K(1,2,oo),
oo: K(1,2,oo)}),
// 2: b->c subj := fmt.Sprintf("@%s: after Track%v", xat[tj.At], keys)
Δ("T/B2:c", δbtail.assertTrack(t, subj, trackSet, trackNew)
A{2: K(2,oo),
oo: K(2,oo)}),
// +1 in new bucket (to the left) δbtail.rebuildAll()
Δ("T2/B1:a-B2:c",
A{1: K(1,2,oo),
2: K(1,2,oo),
oo: K(1,2,oo)}),
// +3 in new bucket (to the right) subj += " + rebuild"
Δ("T2,3/B1:a-B2:c-B3:c", δbtail.assertTrack(t, subj, trackSetAfterRebuild, ø)
A{1: K(1),
2: K(2,3,oo),
3: K(2,3,oo),
oo: K(2,3,oo)}),
// bucket split; +3 in new bucket // XXX verify Get -> XXX assertΔTtail ?
"T/B1:a,2:b",
Δ("T2/B1:a-B2:b,3:c",
A{1: K(1,2,3,oo),
2: K(1,2,3,oo),
3: K(1,2,3,oo),
oo: K(1,2,3,oo)}),
// bucket split; +3 in new bucket; +4 +5 in another new bucket // verify δbtail.vδTbyRoot[treeRoot]
// everything becomes tracked because original bucket had [-∞,∞) coverage assertΔTtail(t, subj, δbtail, tj, treeRoot, xat, vδTok...)
"T/B1:a,2:b", }
Δ("T2,4/B1:a-B2:b,3:c-B4:d,5:e",
A{1: K(1,2,3,4,5,oo),
2: K(1,2,3,4,5,oo),
3: K(1,2,3,4,5,oo),
4: K(1,2,3,4,5,oo),
5: K(1,2,3,4,5,oo),
oo: K(1,2,3,4,5,oo)}),
// reflow of keys: even if tracked={1}, changes to all B nodes need to be rescanned: // xverifyΔBTail_GetAt verifies δBtail.Get on series of vt ZODB changes.
// +B12 forces to look in -B23 which adds -3 into δ, which // XXX
// forces to look into +B34 and so on. // XXX kill
"T2,4,6/B1:a-B2:b,3:c-B4:d,5:e-B6:f,7:g", /*
Δ("T3,5,7/B1:g,2:f-B3:e,4:d-B5:c,6:b-B7:a", func ___xverifyΔBTail_GetAt(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt ...*xbtreetest.Commit) {
A{1: K(1,2,3,4,5,6,7,oo), subj := vt[0].Tree
2: K(1,2,3,4,5,6,7,oo), for _, t := range vt[1:] {
3: K(1,2,3,4,5,6,7,oo), subj += "→" + t.Tree
4: K(1,2,3,4,5,6,7,oo), }
5: K(1,2,3,4,5,6,7,oo),
6: K(1,2,3,4,5,6,7,oo),
7: K(1,2,3,4,5,6,7,oo),
oo: K(1,2,3,4,5,6,7,oo)}),
// reflow of keys for rebuild: even if tracked1={}, tracked2={1}, changes to t.Run(fmt.Sprintf("Get/%s", subj), func(t *testing.T) {
// all A/B/C nodes need to be rescanned. Contrary to the above case the reflow // tid -> "at_i"
// is not detectable at separate diff(A,B) and diff(B,C) runs. xat := map[zodb.Tid]string{}
"T3,5,7/B1:a,2:b-B3:c,4:d-B5:e,6:f-B7:g,8:h", for i := range vt {
"T/B1:b", xat[vt[i].At] = fmt.Sprintf("at%d", i)
"T2,4,6/B1:a-B2:b,3:c-B4:d,5:e-B6:f,7:g",
// similar situation where rebuild has to detect reflow in between non-neighbour trees
"T3,6/B1:a,2:b-B3:c,4:d-B6:f,7:g",
"T4,7/B1:b-B4:d,5:e-B7:g,8:h",
"T2,5,8/B1:a-B2:b,3:c-B5:e,6:f-B8:h,9:i",
// depth=2; bucket split; +3 in new bucket; left T remain fmt.Printf("@%s: %v\n", xat[vt[i].At], vt[i].Xkv.Flatten())
// _unchanged_ even though B under it is modified. }
"T/T/B1:a,2:b",
Δ("T2/T-T/B1:a-B2:b,3:c",
A{1: K(1,2,3,oo),
2: K(1,2,3,oo),
3: K(1,2,3,oo),
oo: K(1,2,3,oo)}),
// depth=2; like prev. case, but additional right arm with +4 +5 is added. tkeys := allTestKeys(vt...)
"T/T/B1:a,2:b", tkeyv := tkeys.SortedElements()
Δ("T2,4/T-T-T/B1:a-B2:b,3:c-B4:d,5:e",
A{1: K(1,2,3,4,5,oo),
2: K(1,2,3,4,5,oo),
3: K(1,2,3,4,5,oo),
4: K(1,2,3,4,5,oo),
5: K(1,2,3,4,5,oo),
oo: K(1,2,3,4,5,oo)}),
// depth=2; bucket split; +3 in new bucket; t0 and t1 split; // verify t1->t2-> ... ->tn Track(keys) Get(keys, @at)
// +right arm (T7/B45-B89). // for all combinations of tracked keys and at
"T/T/B1:a,2:b", for kidx := range IntSets(len(tkeyv)) {
Δ("T4/T2-T7/B1:a-B2:b,3:c-B4:d,5:e-B8:h,9:i", keys := setKey{}
A{1: K(1,2,3,4,5,8,9,oo), for _, idx := range kidx {
2: K(1,2,3,4,5,8,9,oo), keys.Add(tkeyv[idx])
3: K(1,2,3,4,5,8,9,oo), }
4: K(1,2,3,4,5,8,9,oo),
5: K(1,2,3,4,5,8,9,oo),
8: K(1,2,3,4,5,8,9,oo),
9: K(1,2,3,4,5,8,9,oo),
oo: K(1,2,3,4,5,8,9,oo)}),
t.Run(fmt.Sprintf("track=%s", keys), func(t *testing.T) {
xverifyΔBTail_GetAt1(t, db, treeRoot, vt, xat, keys)
})
}
})
}
// 2 reflow to right B neighbour; 8 splits into new B; δ=ø func xverifyΔBTail_GetAt1(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt []*xbtreetest.Commit, xat map[zodb.Tid]string, keys setKey) {
"T3/B1:a,2:b-B4:d,8:h", X := exc.Raiseif
"T2,5/B1:a-B2:b,4:d-B8:h",
// case where kadj does not grow too much as leafs coverage remains stable // t1->t2-> ... -> tn
"T4,8/B1:a,2:b-B5:d,6:e-B10:g,11:h", δbtail := NewΔBtail(vt[0].At, db)
Δ("T4,8/B2:b,3:c-B6:e,7:f-B11:h,12:i", for i := 1; i < len(vt); i++ {
A{1: K(1,2,3), _, err := δbtail.Update(vt[i].ΔZ); X(err)
2: K(1,2,3), }
3: K(1,2,3),
5: K(5,6,7),
6: K(5,6,7),
7: K(5,6,7,),
10: K(10,11,12,oo),
11: K(10,11,12,oo),
12: K(10,11,12,oo),
oo: K(10,11,12,oo)}),
// Track(keys)
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: vt[len(vt)-1].At}); X(err)
xtree, err := zconn.Get(ctx, treeRoot); X(err)
ztree := xtree.(*Tree)
// tree deletion for k := range keys {
// having ø in the middle of the test cases exercises all: _, _, path, err := ZTreeGetBlkData(ctx, ztree, k); X(err)
// * `ø -> Tree ...` (tree is created anew), err = δbtail.Track(k, path); X(err)
// * `... Tree -> ø` (tree is deleted), and }
// * `Tree -> ø -> Tree` (tree is deleted and then recreated)
xbtreetest.DEL,
// tree rotation // verify GetAt(k, @at) for all keys and @at
"T3/B2:b-B3:c,4:d", for i := 1; i < len(vt); i++ {
"T5/T3-T7/B2:a-B3:a,4:a-B6:a-B8:a", at := vt[i].At
for _, k := range keys.SortedElements() {
vOid, ok, rev, revExact, err := δbtail.GetAt(ctx, ztree, k, at); X(err)
v := xzgetBlkDataAt(db, vOid, rev)
// found by AllStructs ([1] is not changed, but because B1 is v_, ok_ := vt[i].Xkv.Get(k).kv[k]
// unlinked and 1 migrates to other bucket, changes in that rev_, revExact_ := vt[i].At, false
// other bucket must be included into δT) for j := i-1; j >= 0; j-- {
"T1,2/B0:e-B1:d-B2:g,3:a", v__ := vt[j].Xkv.Get(k).kv[k]
"T1/B0:d-B1:d,2:d", if v__ != v_ {
// ----//---- with depth=2 rev_ = vt[j+1].At
"T1,2/T-T-T/B0:a-B1:b-B2:c,3:d", revExact_ = true
"T1/T-T/B0:e-B1:b,2:f", break
}
rev_ = vt[j].At
}
// XXX depth=3 (to verify recursion and selecting which tree children to follow or not) if v == "" { v = DEL }
if v_ == "" { v_ = DEL }
if !(v == v_ && ok == ok_ && rev == rev_ && revExact == revExact_) {
t.Errorf("Get(%d, @%s) ->\nhave: %s, %v, @%s, %v\nwant: %s, %v, @%s, %v",
k, xat[at],
v, ok, xat[rev], revExact,
v_, ok_, xat[rev_], revExact_)
}
}
}
}
*/
// degenerate topology from ZODB tests
// https://github.com/zopefoundation/ZODB/commit/6cd24e99f89b
// https://github.com/zopefoundation/BTrees/blob/4.7.2-1-g078ba60/BTrees/tests/testBTrees.py#L20-L57
"T4/T2-T/T-T-T6,10/B1:a-B3:b-T-T-T/T-B7:c-B11:d/B5:e",
"T/B1:e,5:d,7:c,8:b,11:a", // -3 +8
// was leading treegen to generate corrupt trees // ----------------------------------------
"T/T1/T-T/B0:g-B1:e,2:d,3:h",
"T1/T-T3/B0:g-T-T/B1:e,2:d-B3:h",
// was leading to wrongly computed trackSet2 due to top not func TestΔBtailForget(t_ *testing.T) {
// being tracked to tree root. t := xbtreetest.NewT(t_)
"T/T1/B0:a-B1:b", X := exc.Raiseif
"T/T1/T-T/B0:c-B1:d",
// was leading to wrongly computed trackSet2: leaf bucket not t0 := t.CommitTree("T/B:")
// reparented to root. t1 := t.CommitTree("T/B1:a")
"T/T/B0:a", t2 := t.CommitTree("T2/B1:a-B2:b")
"T/B0:a", t3 := t.CommitTree("T/B2:b")
// δtkeycov grows due to change in parent tree only δbtail := NewΔBtail(t0.At, t.DB)
"T3/B1:a-B8:c", _, err := δbtail.Update(t1.ΔZ); X(err)
"T7/B1:a-B8:c", _, err = δbtail.Update(t2.ΔZ); X(err)
// ----//----
"T3/B1:a,2:b-B8:c,9:d",
"T7/B1:a,2:b-B8:c,9:d",
// ----//---- depth=2
"T3/T-T/B1:a,2:b-B8:c,9:d",
"T7/T-T/B1:a,2:b-B8:c,9:d",
// ----//---- found by AllStructs
"T1,3/B0:d-B1:a-B3:d,4:g",
"T1,4/B0:e-B1:a-B4:c",
// ----//---- found by AllStructs
"T2,4/T-T-T/T1-T-B4:f/T-T-B3:f/B0:h-B1:f",
"T4/T-T/B3:f-T/B4:a",
// start tracking. everything becomes tracked because t1's T/B1:a has [-∞,∞) coverage
// By starting tracking after t2 we verify vδBroots update in both Update and rebuild
_0 := setKey{}; _0.Add(0)
xtrackKeys(δbtail, t2, _0)
// ---- found by AllStructs ---- _, err = δbtail.Update(t3.ΔZ); X(err)
// trackSet2 wrongly computed due to top not being tracked to tree root xat := map[zodb.Tid]string{
"T2/T1-T/B0:g-B1:b-T/B2:b,3:a", t0.At: "at0",
"T2/T1-T/T-T-B2:a/B0:c-B1:g", t1.At: "at1",
t2.At: "at2",
t3.At: "at3",
}
assertΔTtail(t.T, "init", δbtail, t3, t.Root(), xat, t1.Δxkv, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t0.At)
assertΔTtail(t.T, "forget ≤ at0", δbtail, t3, t.Root(), xat, t1.Δxkv, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t1.At)
assertΔTtail(t.T, "forget ≤ at1", δbtail, t3, t.Root(), xat, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t3.At)
assertΔTtail(t.T, "forget ≤ at3", δbtail, t3, t.Root(), xat, )
}
func TestΔBtailClone(t_ *testing.T) {
// ΔBtail.Clone had bug that aliased klon data to orig
t := xbtreetest.NewT(t_)
X := exc.Raiseif
// unchanged node is reparented t0 := t.CommitTree("T2/B1:a-B2:b")
"T1/B0:c-B1:f", t1 := t.CommitTree("T2/B1:c-B2:d")
"T1/T-T/B0:c-T/B1:h", δbtail := NewΔBtail(t0.At, t.DB)
_, err := δbtail.Update(t1.ΔZ); X(err)
_2 := setKey{}; _2.Add(2)
xtrackKeys(δbtail, t1, _2)
err = δbtail.rebuildAll(); X(err)
// SIGSEGV in ApplyΔ xat := map[zodb.Tid]string{
"T1/T-T2/T-B1:c-B2:c/B0:g", t0.At: "at0",
"T1/T-T/B0:g-T/B1:e", t1.At: "at1",
}
// trackSet corruption: oid is pointed by some .parent but is not present δkv1_1 := map[Key]Δstring{2:{"b","d"}}
"T1/T-T/B0:g-T2/B1:h-B2:g", assertΔTtail(t.T, "orig @at1", δbtail, t1, t.Root(), xat, δkv1_1)
"T/T1/T-T2/B0:e-B1:f-B2:g", δbklon := δbtail.Clone()
assertΔTtail(t.T, "klon @at1", δbklon, t1, t.Root(), xat, δkv1_1)
// ApplyΔ -> xunion: node is reachable from multiple parents t2 := t.CommitTree("T/B1:b,2:a")
// ( because xdifference did not remove common non-leaf node _, err = δbtail.Update(t2.ΔZ); X(err)
// under which there were also other changed, but not initially xat[t2.At] = "at2"
// tracked, node )
"T4/T1-T/T-T2-B4:c/T-T-T/B0:f-B1:h-B2:g,3:b",
"T1/T-T/T-T2/T-T-T/B0:f-B1:h-B2:f",
// ----//----
"T3/T1-T/T-T2-T/B0:b-T-T-B3:h/B1:e-B2:a",
"T1/T-T4/T-T2-T/T-T-T-T/B0:b-B1:e-B2:a,3:c-B4:e",
// ----//----
"T/T1,3/T-T2-T4/B0:b-T-T-B3:g-B4:c/B1:b-B2:e",
"T1,4/T-T-T/T-T2-B4:f/T-T-T/B0:h-B1:b-B2:h,3:a",
"T2/B1:a-B7:g", δkv1_2 := map[Key]Δstring{1:{"a","c"}, 2:{"b","d"}}
"T2,8/B1:a-B7:g-B9:i", δkv2_2 := map[Key]Δstring{1:{"c","b"}, 2:{"d","a"}}
assertΔTtail(t.T, "orig @at2", δbtail, t2, t.Root(), xat, δkv1_2, δkv2_2)
assertΔTtail(t.T, "klon @at1 after orig @at->@at2", δbklon, t1, t.Root(), xat, δkv1_1)
}
"T2/B1:a-B2:b", "T/B1:a,2:b",
"T2,3/B1:a-B2:b-B3:c", "T/B1:a,2:b",
"T2,3/B1:a-B2:c-B3:c", "T/B1:a,2:b",
"T2/B1:a-B2:c", "T2,3/B1:a-B2:c-B3:c", // -------- KAdj --------
"T2/B1:a-B3:c", // Map returns kadj·keys.
Δ("T2/T-T4/B1:b-B3:d-B99:h", func (kadj KAdjMatrix) Map(keys setKey) setKey {
A{1: K(1), res := make(setKey, len(keys))
3: K(3,99,oo), for k := range keys {
99: K(3,99,oo), to, ok := kadj[k]
oo: K(3,99,oo)}), if !ok {
panicf("kadj.Map: %d ∉ kadj\n\nkadj: %v", k, kadj)
} }
// direct tree_i -> tree_{i+1} -> _{i+2} ... plus res.Update(to)
// reverse ... tree_i <- _{i+1} <- _{i+2}
kadjOK := ΔBTest(testv[len(testv)-1]).kadjOK
for i := len(testv)-2; i >= 0; i-- {
test := ΔBTest(testv[i])
kadjOK, test.kadjOK = test.kadjOK, kadjOK
testv = append(testv, test)
} }
return res
}
testq := make(chan ΔBTestEntry) // Mul returns kadjA·kadjB.
go func() { //
defer close(testq) // (kadjA·kadjB).Map(keys) = kadjA.Map(kadjB.Map(keys))
for _, test := range testv { func (kadjA KAdjMatrix) Mul(kadjB KAdjMatrix) KAdjMatrix {
testq <- ΔBTest(test) // ~ assert kadjA.keys == kadjB.keys
// check only len here; the rest will be asserted by Map
if len(kadjA) != len(kadjB) {
panicf("kadj.Mul: different keys:\n\nkadjA: %v\nkadjB: %v", kadjA, kadjB)
} }
}()
testΔBTail(t, testq)
kadj := make(KAdjMatrix, len(kadjB))
for k, tob := range kadjB {
kadj[k] = kadjA.Map(tob)
}
return kadj
} }
// TestΔBTailRandom verifies ΔBtail on random tree topologies generated by AllStructs. // KAdj computes adjacency matrix for t1 -> t2 transition.
func TestΔBTailRandom(t *testing.T) { //
X := exc.Raiseif // The set of keys for which kadj matrix is computed can be optionally provided.
// This set of keys defaults to allTestKeys(t1,t2).
//
// KAdj itself is verified by testΔBTail on entries with .kadjOK set.
func KAdj(t1, t2 *xbtreetest.Commit, keysv ...setKey) (kadj KAdjMatrix) {
// assert KAdj(A,B) == KAdj(B,A)
kadj12 := _KAdj(t1,t2, keysv...)
kadj21 := _KAdj(t2,t1, keysv...)
if !reflect.DeepEqual(kadj12, kadj21) {
panicf("KAdj not symmetric:\nt1: %s\nt2: %s\nkadj12: %v\nkadj21: %v",
t1.Tree, t2.Tree, kadj12, kadj21)
}
return kadj12
}
// considerations: const debugKAdj = false
// - maxdepth↑ better for testing (more tricky topologies) func debugfKAdj(format string, argv ...interface{}) {
// - maxsplit↑ not so better for testing (leave s=1, max s=2) if debugKAdj {
// - |kmin - kmax| affects N(variants) significantly fmt.Printf(format, argv...)
// -> keep key range small (dumb increase does not help testing) }
// - N(keys) affects N(variants) significantly }
// -> keep Nkeys reasonably small/medium (dumb increase does not help testing)
//
// - spawning python subprocess is very slow (takes 300-500ms for
// imports; https://github.com/pypa/setuptools/issues/510)
// -> we spawn `treegen allstructs` once and use request/response approach.
maxdepth := xbtreetest.N(2, 3, 4) func _KAdj(t1, t2 *xbtreetest.Commit, keysv ...setKey) (kadj KAdjMatrix) {
maxsplit := xbtreetest.N(1, 2, 2) var keys setKey
n := xbtreetest.N(10,10,100) switch len(keysv) {
nkeys := xbtreetest.N(3, 5, 10) case 0:
keys = allTestKeys(t1, t2)
case 1:
keys = keysv[0]
default:
panic("multiple key sets on the call")
}
// server to generate AllStructs(kv, ...) debugfKAdj("\n\n_KAdj\n")
sg, err := xbtreetest.StartAllStructsSrv(); X(err) debugfKAdj("t1: %s\n", t1.Tree)
debugfKAdj("t2: %s\n", t2.Tree)
debugfKAdj("keys: %s\n", keys)
defer func() { defer func() {
err := sg.Close(); X(err) debugfKAdj("kadj -> %v\n", kadj)
}() }()
// random-number generator // kadj = {} k -> adjacent keys.
rng, seed := xbtreetest.NewRand() // if k is tracked and covered by changed leaf -> changes to adjacents must be in Update(t1->t2).
t.Logf("# maxdepth=%d maxsplit=%d nkeys=%d n=%d seed=%d", maxdepth, maxsplit, nkeys, n, seed) kadj = KAdjMatrix{}
for k := range keys {
adj1 := setKey{}
adj2 := setKey{}
// generate (kv1, kv2, kv3) randomly q1 := &blib.RangedKeySet{}; q1.Add(k)
q2 := &blib.RangedKeySet{}; q2.Add(k)
done1 := &blib.RangedKeySet{}
done2 := &blib.RangedKeySet{}
// keysv1, keysv2 and keysv3 are random shuffle of IntSets debugfKAdj("\nk%s\n", kstr(k))
var keysv1 [][]int for !q1.Empty() || !q2.Empty() {
var keysv2 [][]int debugfKAdj("q1: %s\tdone1: %s\n", q1, done1)
var keysv3 [][]int debugfKAdj("q2: %s\tdone2: %s\n", q2, done2)
for keys := range IntSets(nkeys) { for _, r1 := range q1.AllRanges() {
keysv1 = append(keysv1, keys) lo1 := r1.Lo
keysv2 = append(keysv2, keys) for {
keysv3 = append(keysv3, keys) b1 := t1.Xkv.Get(lo1)
debugfKAdj(" b1: %s\n", b1)
for k_ := range keys {
if b1.Keycov.Has(k_) {
adj1.Add(k_)
debugfKAdj(" adj1 += %s\t-> %s\n", kstr(k_), adj1)
} }
v := keysv1 }
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] }) done1.AddRange(b1.Keycov)
v = keysv2 // q2 |= (b1.keyrange \ done2)
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] }) δq2 := &blib.RangedKeySet{}
v = keysv3 δq2.AddRange(b1.Keycov)
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] }) δq2.DifferenceInplace(done2)
q2.UnionInplace(δq2)
debugfKAdj("q2 += %s\t-> %s\n", δq2, q2)
// given random (kv1, kv2, kv3) generate corresponding set of random tree // continue with next right bucket until r1 coverage is complete
// topology sets (T1, T2, T3). Then iterate through T1->T2->T3->T1... if r1.Hi_ <= b1.Keycov.Hi_ {
// elements such that all right-directed triplets are visited and only once. break
// Test Update and rebuild on the generated tree sequences.
vv := "abcdefghij"
randv := func() string {
i := rng.Intn(len(vv))
return vv[i:i+1]
} }
lo1 = b1.Keycov.Hi_ + 1
}
}
q1.Clear()
// the number of pairs is 3·n^2 for _, r2 := range q2.AllRanges() {
// the number of triplets is n^3 lo2 := r2.Lo
// for {
// limit n for emitted triplets, so that the amount of work for Update b2 := t2.Xkv.Get(lo2)
// and rebuild tests is approximately of the same order. debugfKAdj(" b2: %s\n", b2)
nrebuild := int(math.Ceil(math.Pow(3*float64(n*n), 1./3))) for k_ := range keys {
// in non-short mode rebuild tests are exercising more keys variants, plus every test case if b2.Keycov.Has(k_) {
// takes more time. Compensate for that as well. adj2.Add(k_)
if !testing.Short() { debugfKAdj(" adj2 += %s\t-> %s\n", kstr(k_), adj2)
nrebuild -= 3 }
}
done2.AddRange(b2.Keycov)
// q1 |= (b2.keyrange \ done1)
δq1 := &blib.RangedKeySet{}
δq1.AddRange(b2.Keycov)
δq1.DifferenceInplace(done1)
q1.UnionInplace(δq1)
debugfKAdj("q1 += %s\t-> %s\n", δq1, q1)
// continue with next right bucket until r2 coverage is complete
if r2.Hi_ <= b2.Keycov.Hi_ {
break
}
lo2 = b2.Keycov.Hi_ + 1
}
}
q2.Clear()
}
adj := setKey{}; adj.Update(adj1); adj.Update(adj2)
kadj[k] = adj
} }
testq := make(chan ΔBTestEntry) return kadj
go func() { }
defer close(testq)
for i := range keysv1 {
keys1 := keysv1[i]
keys2 := keysv2[i]
keys3 := keysv3[i]
kv1 := map[Key]string{}
kv2 := map[Key]string{}
kv3 := map[Key]string{}
for _, k := range keys1 { kv1[Key(k)] = randv() }
for _, k := range keys2 { kv2[Key(k)] = randv() }
for _, k := range keys3 { kv3[Key(k)] = randv() }
treev1, err1 := sg.AllStructs(kv1, maxdepth, maxsplit, n, rng.Int63()) // ----------------------------------------
treev2, err2 := sg.AllStructs(kv2, maxdepth, maxsplit, n, rng.Int63())
treev3, err3 := sg.AllStructs(kv3, maxdepth, maxsplit, n, rng.Int63())
err := xerr.Merge(err1, err2, err3)
if err != nil {
t.Fatal(err)
}
emit := func(tree string, flags ΔBTestFlags) { // assertΔTtail verifies state of ΔTtail that corresponds to treeRoot in δbtail.
// skip emitting this entry if both Update and // it also verifies that δbtail.vδBroots matches ΔTtail data.
// Rebuild are requested to be skipped. func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *xbtreetest.Commit, treeRoot zodb.Oid, xat map[zodb.Tid]string, vδTok ...map[Key]Δstring) {
if flags == (ΔBTest_SkipUpdate | ΔBTest_SkipRebuild) { t.Helper()
return // XXX +KVAtTail, +lastRevOf
}
testq <- ΔBTestEntry{tree, nil, flags}
}
URSkipIf := func(ucond, rcond bool) ΔBTestFlags { l := len(vδTok)
var flags ΔBTestFlags var vatOK []zodb.Tid
if ucond { var vδTok_ []map[Key]Δstring
flags |= ΔBTest_SkipUpdate at2t := map[zodb.Tid]*xbtreetest.Commit{tj.At: tj}
t0 := tj
for i := 0; i<l; i++ {
// empty vδTok entries means they should be absent in vδT
if δTok := vδTok[l-i-1]; len(δTok) != 0 {
vatOK = append([]zodb.Tid{t0.At}, vatOK...)
vδTok_ = append([]map[Key]Δstring{δTok}, vδTok_...)
} }
if rcond { t0 = t0.Prev
flags |= ΔBTest_SkipRebuild at2t[t0.At] = t0
} }
return flags vδTok = vδTok_
δTtail, ok := δbtail.vδTbyRoot[treeRoot]
var vδToid []ΔTree
if ok {
vδToid = δTtail.vδT
} }
for j := range treev1 { l = len(vδToid)
for k := range treev2 { var vat []zodb.Tid
for l := range treev3 { var vδT []map[Key]Δstring
// limit rebuild to subset of tree topologies, atPrev := t0.At
// because #(triplets) grow as n^3. See nrebuild for _, δToid := range vδToid {
// definition above for details. vat = append(vat, δToid.Rev)
norebuild := (j >= nrebuild || δT := XGetδKV(at2t[atPrev], at2t[δToid.Rev], δToid.ΔKV) // {} k -> δ(ZBlk(oid).data)
k >= nrebuild || vδT = append(vδT, δT)
l >= nrebuild) atPrev = δToid.Rev
}
// C_{l-1} -> Aj (pair first seen on k=0) var vatδB []zodb.Tid // δbtail.vδBroots/treeRoot
emit(treev1[j], URSkipIf(k != 0, norebuild)) for _, δBroots := range δbtail.vδBroots {
if δBroots.ΔRoots.Has(treeRoot) {
vatδB = append(vatδB, δBroots.Rev)
}
}
// Aj -> Bk (pair first seen on l=0) tok := tidvEqual(vat, vatOK) && vδTEqual(vδT, vδTok)
emit(treev2[k], URSkipIf(l != 0, norebuild)) bok := tidvEqual(vatδB, vatOK)
if !(tok && bok) {
emsg := fmt.Sprintf("%s: vδT:\n", subj)
have := ""
for i := 0; i<len(vδT); i++ {
have += fmt.Sprintf("\n\t@%s: %v", xat[vat[i]], vδT[i])
}
emsg += fmt.Sprintf("have: %s\n", have)
// Bk -> Cl (pair first seen on j=0) if !tok {
emit(treev3[l], URSkipIf(j != 0, norebuild)) want := ""
for i := 0; i<len(vδTok); i++ {
want += fmt.Sprintf("\n\t@%s: %v", xat[vatOK[i]], vδTok[i])
} }
emsg += fmt.Sprintf("want: %s\n", want)
} }
if !bok {
vδb_root := ""
for i := 0; i<len(vatδB); i++ {
vδb_root += fmt.Sprintf("\n\t@%s", xat[vatδB[i]])
} }
emsg += fmt.Sprintf("vδb/root: %s\n", vδb_root)
} }
}()
testΔBTail(t, testq) t.Error(emsg)
}
} }
// xtrackKeys issues δbtail.Track requests for tree[keys].
func TestΔBtailForget(t_ *testing.T) { func xtrackKeys(δbtail *ΔBtail, t *xbtreetest.Commit, keys setKey) {
t := xbtreetest.NewT(t_)
X := exc.Raiseif X := exc.Raiseif
head := δbtail.Head()
if head != t.At {
panicf("BUG: δbtail.head: %s ; t.at: %s", head, t.At)
}
t0 := t.CommitTree("T/B:") for k := range keys {
t1 := t.CommitTree("T/B1:a") // NOTE: if tree is deleted - the following adds it to tracked
t2 := t.CommitTree("T2/B1:a-B2:b") // set with every key being a hole. This aligns with the
t3 := t.CommitTree("T/B2:b") // following situation
//
δbtail := NewΔBtail(t0.At, t.DB) // T1 -> ø -> T2
_, err := δbtail.Update(t1.ΔZ); X(err) //
_, err = δbtail.Update(t2.ΔZ); X(err) // where after T1->ø, even though the tree becomes deleted, its root
// continues to be tracked and all keys migrate to holes in the
// tracking set. By aligning initial state to the same as after
// T1->ø, we test what will happen on ø->T2.
b := t.Xkv.Get(k)
err := δbtail.track(k, b.Path()); X(err)
}
}
// start tracking. everything becomes tracked because t1's T/B1:a has [-∞,∞) coverage // trackSet returns what should be ΔBtail.trackSet coverage for specified tracked key set.
// By starting tracking after t2 we verify vδBroots update in both Update and rebuild func trackSet(rbs xbtreetest.RBucketSet, tracked setKey) blib.PPTreeSubSet {
_0 := setKey{}; _0.Add(0) // nil = don't compute keyCover
xtrackKeys(δbtail, t2, _0) // (trackSet is called from inside hot inner loop of rebuild test)
return _trackSetWithCov(rbs, tracked, nil)
}
_, err = δbtail.Update(t3.ΔZ); X(err) // trackSetWithCov returns what should be ΔBtail.trackSet and its key coverage for specified tracked key set.
func trackSetWithCov(rbs xbtreetest.RBucketSet, tracked setKey) (trackSet blib.PPTreeSubSet, keyCover *blib.RangedKeySet) {
keyCover = &blib.RangedKeySet{}
trackSet = _trackSetWithCov(rbs, tracked, keyCover)
return trackSet, keyCover
}
xat := map[zodb.Tid]string{ func _trackSetWithCov(rbs xbtreetest.RBucketSet, tracked setKey, outKeyCover *blib.RangedKeySet) (trackSet blib.PPTreeSubSet) {
t0.At: "at0", trackSet = blib.PPTreeSubSet{}
t1.At: "at1", for k := range tracked {
t2.At: "at2", kb := rbs.Get(k)
t3.At: "at3", if outKeyCover != nil {
outKeyCover.AddRange(kb.Keycov)
} }
assertΔTtail(t.T, "init", δbtail, t3, t.Root(), xat, t1.Δxkv, t2.Δxkv, t3.Δxkv) trackSet.AddPath(kb.Path())
δbtail.ForgetPast(t0.At) }
assertΔTtail(t.T, "forget ≤ at0", δbtail, t3, t.Root(), xat, t1.Δxkv, t2.Δxkv, t3.Δxkv) return trackSet
δbtail.ForgetPast(t1.At)
assertΔTtail(t.T, "forget ≤ at1", δbtail, t3, t.Root(), xat, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t3.At)
assertΔTtail(t.T, "forget ≤ at3", δbtail, t3, t.Root(), xat, )
} }
// assertTrack verifies state of .trackSet and ΔTtail.trackNew.
// it assumes that only one tree root is being tracked.
func (δBtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK blib.PPTreeSubSet, trackNewOK blib.PPTreeSubSet) {
t.Helper()
if !δBtail.trackSet.Equal(trackSetOK) {
t.Errorf("%s: trackSet:\n\thave: %v\n\twant: %v", subj, δBtail.trackSet, trackSetOK)
}
func TestΔBtailClone(t_ *testing.T) { roots := setOid{}
// ΔBtail.Clone had bug that aliased klon data to orig for root := range δBtail.vδTbyRoot {
t := xbtreetest.NewT(t_) roots.Add(root)
X := exc.Raiseif }
t0 := t.CommitTree("T2/B1:a-B2:b") nrootsOK := 1
t1 := t.CommitTree("T2/B1:c-B2:d") if trackSetOK.Empty() && trackNewOK.Empty() {
δbtail := NewΔBtail(t0.At, t.DB) nrootsOK = 0
_, err := δbtail.Update(t1.ΔZ); X(err) }
_2 := setKey{}; _2.Add(2) if len(roots) != nrootsOK {
xtrackKeys(δbtail, t1, _2) t.Errorf("%s: len(vδTbyRoot) != %d ; roots=%v", subj, nrootsOK, roots)
err = δbtail.rebuildAll(); X(err) return
}
if nrootsOK == 0 {
return
}
xat := map[zodb.Tid]string{ root := roots.Elements()[0]
t0.At: "at0",
t1.At: "at1", δTtail := δBtail.vδTbyRoot[root]
trackNewRootsOK := setOid{}
if !trackNewOK.Empty() {
trackNewRootsOK.Add(root)
} }
δkv1_1 := map[Key]Δstring{2:{"b","d"}} if !δBtail.trackNewRoots.Equal(trackNewRootsOK) {
assertΔTtail(t.T, "orig @at1", δbtail, t1, t.Root(), xat, δkv1_1) t.Errorf("%s: trackNewRoots:\n\thave: %v\n\twant: %v", subj, δBtail.trackNewRoots, trackNewRootsOK)
δbklon := δbtail.Clone() }
assertΔTtail(t.T, "klon @at1", δbklon, t1, t.Root(), xat, δkv1_1)
t2 := t.CommitTree("T/B1:b,2:a") if !δTtail.trackNew.Equal(trackNewOK) {
_, err = δbtail.Update(t2.ΔZ); X(err) t.Errorf("%s: vδT.trackNew:\n\thave: %v\n\twant: %v", subj, δTtail.trackNew, trackNewOK)
xat[t2.At] = "at2" }
}
δkv1_2 := map[Key]Δstring{1:{"a","c"}, 2:{"b","d"}}
δkv2_2 := map[Key]Δstring{1:{"c","b"}, 2:{"d","a"}}
assertΔTtail(t.T, "orig @at2", δbtail, t2, t.Root(), xat, δkv1_2, δkv2_2) // XGetδKV translates {k -> δ<oid>} to {k -> δ(ZBlk(oid).data)} according to t1..t2 db snapshots.
assertΔTtail(t.T, "klon @at1 after orig @at->@at2", δbklon, t1, t.Root(), xat, δkv1_1) func XGetδKV(t1, t2 *xbtreetest.Commit, δkvOid map[Key]ΔValue) map[Key]Δstring {
δkv := make(map[Key]Δstring, len(δkvOid))
for k, δvOid := range δkvOid {
δkv[k] = Δstring{
Old: t1.XGetBlkData(δvOid.Old),
New: t2.XGetBlkData(δvOid.New),
}
}
return δkv
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7