Skip to content

W
wendelin.core
Commit c7f1e3c9 authored by Kirill Smelkov's avatar Kirill Smelkov
1 Browse files
Options

X xbtree: Factor testing infrastructure bits into xbtree/xbtreetest 

This way ΔFtail tests will be able to reuse it.

* t3:
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  Revert "."
  .
  .
  .
  .
parents af65001c 064efa3c
Show whitespace changes
Inline Side-by-side
Showing with 1405 additions and 1130 deletions
wcfs/internal/xbtree/blib/blib.go 0 → 100644
View file @ c7f1e3c9
// Copyright (C) 2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package blib provides utilities related to BTrees.
package blib
import (
"fmt"
"math"
"lab.nexedi.com/kirr/neo/go/zodb/btree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
)
// XXX instead of generics
type Tree = btree.LOBTree
type Bucket = btree.LOBucket
type Node = btree.LONode
type TreeEntry = btree.LOEntry
type BucketEntry = btree.LOBucketEntry
type Key = int64
const KeyMax Key = math.MaxInt64
const KeyMin Key = math.MinInt64
type setOid = set.Oid
// kstr formats key as string.
func kstr(k Key) string {
if k == KeyMin {
return "-∞"
}
if k == KeyMax {
return "∞"
}
return fmt.Sprintf("%d", k)
}
func panicf(format string, argv ...interface{}) {
panic(fmt.Sprintf(format, argv...))
}
wcfs/internal/xbtree/pptreesubset.go → wcfs/internal/xbtree/blib/pptreesubset.go
View file @ c7f1e3c9
......@@ -17,7 +17,7 @@
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree
package blib
// PP-connected subset of tree nodes.
import (
......@@ -61,6 +61,16 @@ type nodeInTree struct {
nchild int // number of direct children in PPTreeSubSet referring to this node
}
// Parent returns parent of this node.
func (n *nodeInTree) Parent() zodb.Oid {
return n.parent
}
// NChild returns number of children of this node in the tree subset.
func (n *nodeInTree) NChild() int {
return n.nchild
}
// Has returns whether node is in the set.
func (S PPTreeSubSet) Has(oid zodb.Oid) bool {
_, ok := S[oid]
......@@ -102,7 +112,7 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
// normalize path: remove embedded bucket and check whether it was an
// artificial empty tree.
path = normPath(path)
path = NormPath(path)
// go through path and add nodes to the set
parent := zodb.InvalidOid
......@@ -132,11 +142,11 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
}
}
// normPath normalizes path.
// NormPath normalizes path.
//
// It removes embedded buckets and artificial empty trees.
// Returned slice is subslice of path and aliases its memory.
func normPath(path []zodb.Oid) []zodb.Oid {
func NormPath(path []zodb.Oid) []zodb.Oid {
l := len(path)
// don't keep track of artificial empty tree
......@@ -464,20 +474,20 @@ func (t nodeInTree) String() string {
// The second component with "-22" builds from leaf, but the first
// component with "-43" builds from non-leaf node.
//
// δnchildNonLeafs = {43: +1}
// ΔnchildNonLeafs = {43: +1}
//
// Only complete result of applying all
//
// - xfixup(-1, δnchildNonLeafs)
// - xfixup(-1, ΔnchildNonLeafs)
// - δ.Del,
// - δ.Add, and
// - xfixup(+1, δnchildNonLeafs)
// - xfixup(+1, ΔnchildNonLeafs)
//
// produces correctly PP-connected set.
type ΔPPTreeSubSet struct {
Del PPTreeSubSet
Add PPTreeSubSet
δnchildNonLeafs map[zodb.Oid]int
ΔnchildNonLeafs map[zodb.Oid]int
}
// NewΔPPTreeSubSet creates new empty ΔPPTreeSubSet.
......@@ -485,7 +495,7 @@ func NewΔPPTreeSubSet() *ΔPPTreeSubSet {
return &ΔPPTreeSubSet{
Del: PPTreeSubSet{},
Add: PPTreeSubSet{},
δnchildNonLeafs: map[zodb.Oid]int{},
ΔnchildNonLeafs: map[zodb.Oid]int{},
}
}
......@@ -493,15 +503,15 @@ func NewΔPPTreeSubSet() *ΔPPTreeSubSet {
func (δ *ΔPPTreeSubSet) Update(δ2 *ΔPPTreeSubSet) {
δ.Del.UnionInplace(δ2.Del)
δ.Add.UnionInplace(δ2.Add)
for oid, δnc := range δ2.δnchildNonLeafs {
δ.δnchildNonLeafs[oid] += δnc
for oid, δnc := range δ2.ΔnchildNonLeafs {
δ.ΔnchildNonLeafs[oid] += δnc
}
}
// Reverse changes δ=diff(A->B) to δ'=diff(A<-B).
func (δ *ΔPPTreeSubSet) Reverse() {
δ.Del, δ.Add = δ.Add, δ.Del
// δnchildNonLeafs stays the same
// ΔnchildNonLeafs stays the same
}
......@@ -514,7 +524,7 @@ func (S PPTreeSubSet) ApplyΔ(δ *ΔPPTreeSubSet) {
fmt.Printf(" A: %s\n", S)
fmt.Printf(" -: %s\n", δ.Del)
fmt.Printf(" +: %s\n", δ.Add)
fmt.Printf(" x: %v\n", δ.δnchildNonLeafs)
fmt.Printf(" x: %v\n", δ.ΔnchildNonLeafs)
defer fmt.Printf("\n->B: %s\n", S)
}
......@@ -523,8 +533,8 @@ func (S PPTreeSubSet) ApplyΔ(δ *ΔPPTreeSubSet) {
δ.Add.verify()
defer S.verify()
S.xfixup(-1, δ.δnchildNonLeafs)
S.xfixup(-1, δ.ΔnchildNonLeafs)
S.xDifferenceInplace(δ.Del)
S.xUnionInplace(δ.Add)
S.xfixup(+1, δ.δnchildNonLeafs)
S.xfixup(+1, δ.ΔnchildNonLeafs)
}
wcfs/internal/xbtree/pptreesubset_test.go → wcfs/internal/xbtree/blib/pptreesubset_test.go
View file @ c7f1e3c9
......@@ -17,7 +17,7 @@
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree
package blib
import (
"strings"
......
wcfs/internal/xbtree/rangeset.go → wcfs/internal/xbtree/blib/rangeset.go
View file @ c7f1e3c9
......@@ -17,7 +17,7 @@
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree
package blib
// set of [lo,hi) Key ranges.
import (
......@@ -30,8 +30,8 @@ const debugRangeSet = false
// KeyRange represents [lo,hi) Key range.
type KeyRange struct {
lo Key
hi_ Key // NOTE _not_ hi) to avoid overflow at ∞; hi = hi_ + 1
Lo Key
Hi_ Key // NOTE _not_ hi) to avoid overflow at ∞; hi = hi_ + 1
}
// RangedKeySet is set of Keys with adjacent keys coalesced into Ranges.
......@@ -47,23 +47,23 @@ type RangedKeySet struct {
// Has returns whether key k belongs to the range.
func (r *KeyRange) Has(k Key) bool {
return (r.lo <= k && k <= r.hi_)
return (r.Lo <= k && k <= r.Hi_)
}
// Add adds key k to the set.
func (S *RangedKeySet) Add(k Key) {
S.AddRange(KeyRange{lo: k, hi_: k})
S.AddRange(KeyRange{Lo: k, Hi_: k})
}
// Del removes key k from the set.
func (S *RangedKeySet) Del(k Key) {
S.DelRange(KeyRange{lo: k, hi_: k})
S.DelRange(KeyRange{Lo: k, Hi_: k})
}
// Has returns whether key k belongs to the set.
func (S *RangedKeySet) Has(k Key) bool {
return S.HasRange(KeyRange{lo: k, hi_: k})
return S.HasRange(KeyRange{Lo: k, Hi_: k})
}
......@@ -79,10 +79,10 @@ func (S *RangedKeySet) AddRange(r KeyRange) {
S.verify()
defer S.verify()
// find first ilo: r.lo < [ilo].hi
// find first ilo: r.Lo < [ilo].hi
l := len(S.rangev)
ilo := sort.Search(l, func(i int) bool {
return r.lo <= S.rangev[i].hi_
return r.Lo <= S.rangev[i].Hi_
})
debugfRSet("\tilo: %d\n", ilo)
......@@ -92,58 +92,58 @@ func (S *RangedKeySet) AddRange(r KeyRange) {
debugfRSet("\tappend %s\t-> %s\n", r, S)
}
// find last jhi: [jhi].lo < r.hi
// find last jhi: [jhi].Lo < r.hi
jhi := ilo
for ;; jhi++ {
if jhi == l {
break
}
if S.rangev[jhi].lo <= r.hi_ {
if S.rangev[jhi].Lo <= r.Hi_ {
continue
}
break
}
debugfRSet("\tjhi: %d\n", jhi)
// entries in [ilo:jhi) ∈ [r.lo,r.hi) and should be merged into one
// entries in [ilo:jhi) ∈ [r.Lo,r.hi) and should be merged into one
if (jhi - ilo) > 1 {
lo := S.rangev[ilo].lo
hi_ := S.rangev[jhi-1].hi_
lo := S.rangev[ilo].Lo
hi_ := S.rangev[jhi-1].Hi_
vReplaceSlice(&S.rangev, ilo,jhi, KeyRange{lo,hi_})
debugfRSet("\tmerge S[%d:%d]\t-> %s\n", ilo, jhi, S)
}
jhi = -1 // no longer valid
// if [r.lo,r.hi) was outside of any entry - create new entry
if r.hi_ < S.rangev[ilo].lo {
if r.Hi_ < S.rangev[ilo].Lo {
vInsert(&S.rangev, ilo, r)
debugfRSet("\tinsert %s\t-> %s\n", r, S)
}
// now we have covered entries merged as needed into [ilo]
// extend this entry if r coverage is wider
if r.lo < S.rangev[ilo].lo {
S.rangev[ilo].lo = r.lo
if r.Lo < S.rangev[ilo].Lo {
S.rangev[ilo].Lo = r.Lo
debugfRSet("\textend left\t-> %s\n", S)
}
if r.hi_ > S.rangev[ilo].hi_ {
S.rangev[ilo].hi_ = r.hi_
if r.Hi_ > S.rangev[ilo].Hi_ {
S.rangev[ilo].Hi_ = r.Hi_
debugfRSet("\textend right\t-> %s\n", S)
}
// and check if we should merge it with right/left neighbours
if ilo+1 < len(S.rangev) { // right
if S.rangev[ilo].hi_+1 == S.rangev[ilo+1].lo {
if S.rangev[ilo].Hi_+1 == S.rangev[ilo+1].Lo {
vReplaceSlice(&S.rangev, ilo,ilo+2,
KeyRange{S.rangev[ilo].lo, S.rangev[ilo+1].hi_})
KeyRange{S.rangev[ilo].Lo, S.rangev[ilo+1].Hi_})
debugfRSet("\tmerge right\t-> %s\n", S)
}
}
if ilo > 0 { // left
if S.rangev[ilo-1].hi_+1 == S.rangev[ilo].lo {
if S.rangev[ilo-1].Hi_+1 == S.rangev[ilo].Lo {
vReplaceSlice(&S.rangev, ilo-1,ilo+1,
KeyRange{S.rangev[ilo-1].lo, S.rangev[ilo].hi_})
KeyRange{S.rangev[ilo-1].Lo, S.rangev[ilo].Hi_})
debugfRSet("\tmerge left\t-> %s\n", S)
}
}
......@@ -163,10 +163,10 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
S.verify()
defer S.verify()
// find first ilo: r.lo < [ilo].hi
// find first ilo: r.Lo < [ilo].hi
l := len(S.rangev)
ilo := sort.Search(l, func(i int) bool {
return r.lo <= S.rangev[i].hi_
return r.Lo <= S.rangev[i].Hi_
})
debugfRSet("\tilo: %d\n", ilo)
......@@ -175,13 +175,13 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
return
}
// find last jhi: [jhi].lo < r.hi
// find last jhi: [jhi].Lo < r.hi
jhi := ilo
for ;; jhi++ {
if jhi == l {
break
}
if S.rangev[jhi].lo <= r.hi_ {
if S.rangev[jhi].Lo <= r.Hi_ {
continue
}
break
......@@ -196,19 +196,19 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
// [ilo+1:jhi-1] should be deleted
// [ilo] and [jhi-1] overlap with [r.lo,r.hi) - they should be deleted, or shrinked,
// or split+shrinked if ilo==jhi-1 and r is inside [ilo]
if jhi-ilo == 1 && S.rangev[ilo].lo < r.lo && r.hi_ < S.rangev[ilo].hi_ {
if jhi-ilo == 1 && S.rangev[ilo].Lo < r.Lo && r.Hi_ < S.rangev[ilo].Hi_ {
x := S.rangev[ilo]
vInsert(&S.rangev, ilo, x)
jhi++
debugfRSet("\tpresplit copy %s\t-> %s\n", x, S)
}
if S.rangev[ilo].lo < r.lo { // shrink left
S.rangev[ilo] = KeyRange{S.rangev[ilo].lo, r.lo-1}
if S.rangev[ilo].Lo < r.Lo { // shrink left
S.rangev[ilo] = KeyRange{S.rangev[ilo].Lo, r.Lo-1}
ilo++
debugfRSet("\tshrink [%d] left\t-> %s\n", ilo, S)
}
if r.hi_ < S.rangev[jhi-1].hi_ { // shrink right
S.rangev[jhi-1] = KeyRange{r.hi_+1, S.rangev[jhi-1].hi_}
if r.Hi_ < S.rangev[jhi-1].Hi_ { // shrink right
S.rangev[jhi-1] = KeyRange{r.Hi_+1, S.rangev[jhi-1].Hi_}
jhi--
debugfRSet("\tshrink [%d] right\t-> %s\n", jhi-1, S)
}
......@@ -237,7 +237,7 @@ func (S *RangedKeySet) HasRange(r KeyRange) (yes bool) {
// find first ilo: r.lo < [ilo].hi
l := len(S.rangev)
ilo := sort.Search(l, func(i int) bool {
return r.lo <= S.rangev[i].hi_
return r.Lo <= S.rangev[i].Hi_
})
debugfRSet("\tilo: %d\n", ilo)
......@@ -246,7 +246,7 @@ func (S *RangedKeySet) HasRange(r KeyRange) (yes bool) {
}
// all keys from r are in S if r ∈ [ilo]
return (S.rangev[ilo].lo <= r.lo && r.hi_ <= S.rangev[ilo].hi_)
return (S.rangev[ilo].Lo <= r.Lo && r.Hi_ <= S.rangev[ilo].Hi_)
}
......@@ -317,13 +317,13 @@ func (S *RangedKeySet) verify() {
hi_Prev := KeyMin
for i, r := range S.rangev {
hiPrev := hi_Prev + 1
if i > 0 && !(hiPrev < r.lo) { // NOTE not ≤ - adjacent ranges must be merged
if i > 0 && !(hiPrev < r.Lo) { // NOTE not ≤ - adjacent ranges must be merged
badf("[%d]: !(hiPrev < r.lo)", i)
}
if !(r.lo <= r.hi_) {
if !(r.Lo <= r.Hi_) {
badf("[%d]: !(r.lo <= r.hi_)", i)
}
hi_Prev = r.hi_
hi_Prev = r.Hi_
}
}
......@@ -378,12 +378,12 @@ func (S RangedKeySet) String() string {
func (r KeyRange) String() string {
var shi string
if r.hi_ == KeyMax {
shi = kstr(r.hi_) // ∞
if r.Hi_ == KeyMax {
shi = kstr(r.Hi_) // ∞
} else {
shi = fmt.Sprintf("%d", r.hi_+1)
shi = fmt.Sprintf("%d", r.Hi_+1)
}
return fmt.Sprintf("[%s,%s)", kstr(r.lo), shi)
return fmt.Sprintf("[%s,%s)", kstr(r.Lo), shi)
}
......
wcfs/internal/xbtree/rangeset_test.go → wcfs/internal/xbtree/blib/rangeset_test.go
View file @ c7f1e3c9
......@@ -17,7 +17,7 @@
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree
package blib
import (
"testing"
......
wcfs/internal/xbtree/treediff.go
View file @ c7f1e3c9
......@@ -80,6 +80,7 @@ import (
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
)
const traceDiff = false
......@@ -113,7 +114,7 @@ func (δv ΔValue) String() string {
// for example for e.g. t₀->t₁->b₂ if δZ/T={t₀ b₂} -> δZ/TC=δZ/T+{t₁}
//
// δtopsByRoot = {} root -> {top changed nodes in that tree}
func δZConnectTracked(δZv []zodb.Oid, T PPTreeSubSet) (δZTC setOid, δtopsByRoot map[zodb.Oid]setOid) {
func δZConnectTracked(δZv []zodb.Oid, T blib.PPTreeSubSet) (δZTC setOid, δtopsByRoot map[zodb.Oid]setOid) {
δZ := setOid{}; for _, δ := range δZv { δZ.Add(δ) }
δZTC = setOid{}
δtopsByRoot = map[zodb.Oid]setOid{}
......@@ -131,7 +132,7 @@ func δZConnectTracked(δZv []zodb.Oid, T PPTreeSubSet) (δZTC setOid, δtopsByR
// if !root -> δZTC += path through which we reached another node (forming connection)
path := []zodb.Oid{}
node := δ
parent := track.parent
parent := track.Parent()
for {
// reached root
if parent == zodb.InvalidOid {
......@@ -159,7 +160,7 @@ func δZConnectTracked(δZv []zodb.Oid, T PPTreeSubSet) (δZTC setOid, δtopsByR
panicf("BUG: .p%s -> %s, but %s is not tracked", node, parent, parent)
}
node = parent
parent = trackUp.parent
parent = trackUp.Parent()
}
}
......@@ -171,7 +172,7 @@ func δZConnectTracked(δZv []zodb.Oid, T PPTreeSubSet) (δZTC setOid, δtopsByR
// nodeInRange represents a Node coming under [lo, hi_] key range in its tree.
type nodeInRange struct {
prefix []zodb.Oid // path to this node goes via this objects
lo, hi_ Key // [lo, hi_] NOTE _not_ hi) not to overflow at ∞ XXX -> keycov KeyRange?
keycov blib.KeyRange
node Node
done bool // whether this node was already taken into account while computing diff
}
......@@ -209,14 +210,14 @@ func (rs rangeSplit) Get(k Key) *nodeInRange {
// Get_ returns node covering key k.
func (rs rangeSplit) Get_(k Key) (rnode *nodeInRange, ok bool) {
i := sort.Search(len(rs), func(i int) bool {
return k <= rs[i].hi_
return k <= rs[i].keycov.Hi_
})
if i == len(rs) {
return nil, false // key not covered
}
rn := rs[i]
if !(rn.lo <= k && k <= rn.hi_) {
if !rn.keycov.Has(k) {
panicf("BUG: get(%v) -> %s; coverage: %s", k, rn, rs)
}
......@@ -233,7 +234,7 @@ func (rs rangeSplit) Get_(k Key) (rnode *nodeInRange, ok bool) {
func (prs *rangeSplit) Expand(rnode *nodeInRange) (children rangeSplit) {
rs := *prs
i := sort.Search(len(rs), func(i int) bool {
return rnode.hi_ <= rs[i].hi_
return rnode.keycov.Hi_ <= rs[i].keycov.Hi_
})
if i == len(rs) || rs[i] != rnode {
panicf("%s not in rangeSplit; coverage: %s", rnode, rs)
......@@ -247,19 +248,18 @@ func (prs *rangeSplit) Expand(rnode *nodeInRange) (children rangeSplit) {
treev := tree.Entryv()
children = make(rangeSplit, 0, len(treev)+1)
for i := range treev {
lo := rnode.lo
lo := rnode.keycov.Lo
if i > 0 {
lo = treev[i].Key()
}
hi_ := rnode.hi_
hi_ := rnode.keycov.Hi_
if i < len(treev)-1 {
hi_ = treev[i+1].Key()-1 // NOTE -1 because it is hi_] not hi)
}
children = append(children, &nodeInRange{
prefix: rnode.Path(),
lo: lo,
hi_: hi_,
keycov: blib.KeyRange{lo, hi_},
node: treev[i].Child(),
})
}
......@@ -338,18 +338,18 @@ func (rs rangeSplit) String() string {
// δtops is set of top nodes for changed subtrees.
// δZTC is connected(δZ/T) - connected closure for subset of δZ(old..new) that
// touches tracked nodes of T.
func treediff(ctx context.Context, root zodb.Oid, δtops setOid, δZTC setOid, trackSet PPTreeSubSet, zconnOld, zconnNew *zodb.Connection) (δT map[Key]ΔValue, δtrack *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
func treediff(ctx context.Context, root zodb.Oid, δtops setOid, δZTC setOid, trackSet blib.PPTreeSubSet, zconnOld, zconnNew *zodb.Connection) (δT map[Key]ΔValue, δtrack *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, err error) {
defer xerr.Contextf(&err, "treediff %s..%s %s", zconnOld.At(), zconnNew.At(), root)
δT = map[Key]ΔValue{}
δtrack = NewΔPPTreeSubSet()
δtkeycov = &RangedKeySet{}
δtrack = blib.NewΔPPTreeSubSet()
δtkeycov = &blib.RangedKeySet{}
tracefDiff("\ntreediff %s δtops: %v δZTC: %v\n", root, δtops, δZTC)
tracefDiff(" trackSet: %v\n", trackSet)
defer tracefDiff("\n-> δT: %v\nδtrack: %v\nδtkeycov: %v\n", δT, δtrack, δtkeycov)
δtrackv := []*ΔPPTreeSubSet{}
δtrackv := []*blib.ΔPPTreeSubSet{}
for top := range δtops { // XXX -> sorted?
a, err1 := zgetNodeOrNil(ctx, zconnOld, top)
......@@ -401,7 +401,7 @@ func treediff(ctx context.Context, root zodb.Oid, δtops setOid, δZTC setOid, t
// consistent with b (= a + δ).
//
// δtkeycov represents how δtrack grows (always grows) tracking set key coverage.
func diffX(ctx context.Context, a, b Node, δZTC setOid, trackSet PPTreeSubSet) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
func diffX(ctx context.Context, a, b Node, δZTC setOid, trackSet blib.PPTreeSubSet) (δ map[Key]ΔValue, δtrack *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, err error) {
if a==nil && b==nil {
panic("BUG: both a & b == nil") // XXX -> not a bug e.g. for `ø ø T` sequence?
}
......@@ -438,11 +438,11 @@ func diffX(ctx context.Context, a, b Node, δZTC setOid, trackSet PPTreeSubSet)
if isT {
return diffT(ctx, aT, bT, δZTC, trackSet)
} else {
var δtrack *ΔPPTreeSubSet
var δtrack *blib.ΔPPTreeSubSet
δ, err := diffB(ctx, aB, bB)
if δ != nil {
δtrack = NewΔPPTreeSubSet()
δtkeycov = &RangedKeySet{}
δtrack = blib.NewΔPPTreeSubSet()
δtkeycov = &blib.RangedKeySet{}
}
return δ, δtrack, δtkeycov, err
}
......@@ -452,13 +452,13 @@ func diffX(ctx context.Context, a, b Node, δZTC setOid, trackSet PPTreeSubSet)
//
// a, b point to top of subtrees @old and @new revisions.
// δZTC is connected set of objects covering δZT (objects changed in this tree in old..new).
func diffT(ctx context.Context, A, B *Tree, δZTC setOid, trackSet PPTreeSubSet) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
func diffT(ctx context.Context, A, B *Tree, δZTC setOid, trackSet blib.PPTreeSubSet) (δ map[Key]ΔValue, δtrack *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, err error) {
tracefDiff(" diffT %s %s\n", xidOf(A), xidOf(B))
defer xerr.Contextf(&err, "diffT %s %s", xidOf(A), xidOf(B))
δ = map[Key]ΔValue{}
δtrack = NewΔPPTreeSubSet()
δtkeycov = &RangedKeySet{}
δtrack = blib.NewΔPPTreeSubSet()
δtkeycov = &blib.RangedKeySet{}
defer func() {
tracefDiff(" -> δ: %v\n", δ)
tracefDiff(" -> δtrack: %v\n", δtrack)
......@@ -557,29 +557,30 @@ ABcov:
}
// initial split ranges for A and B
ABcov := blib.KeyRange{ABlo, ABhi_}
prefix := ABpath[:len(ABpath)-1]
atop := &nodeInRange{prefix: prefix, lo: ABlo, hi_: ABhi_, node: A} // [-∞, ∞)
btop := &nodeInRange{prefix: prefix, lo: ABlo, hi_: ABhi_, node: B} // [-∞, ∞)
atop := &nodeInRange{prefix: prefix, keycov: ABcov, node: A}
btop := &nodeInRange{prefix: prefix, keycov: ABcov, node: B}
Av := rangeSplit{atop} // nodes expanded from A
Bv := rangeSplit{btop} // nodes expanded from B
// for phase 2:
Akqueue := &RangedKeySet{} // queue for keys in A to be processed for δ-
Bkqueue := &RangedKeySet{} // ----//---- in B for δ+
Akdone := &RangedKeySet{} // already processed keys in A
Bkdone := &RangedKeySet{} // ----//---- in B
Aktodo := func(r KeyRange) {
Akqueue := &blib.RangedKeySet{} // queue for keys in A to be processed for δ-
Bkqueue := &blib.RangedKeySet{} // ----//---- in B for δ+
Akdone := &blib.RangedKeySet{} // already processed keys in A
Bkdone := &blib.RangedKeySet{} // ----//---- in B
Aktodo := func(r blib.KeyRange) {
if !Akdone.HasRange(r) {
δtodo := &RangedKeySet{}
δtodo := &blib.RangedKeySet{}
δtodo.AddRange(r)
δtodo.DifferenceInplace(Akdone)
debugfDiff(" Akq <- %s\n", δtodo)
Akqueue.UnionInplace(δtodo)
}
}
Bktodo := func(r KeyRange) {
Bktodo := func(r blib.KeyRange) {
if !Bkdone.HasRange(r) {
δtodo := &RangedKeySet{}
δtodo := &blib.RangedKeySet{}
δtodo.AddRange(r)
δtodo.DifferenceInplace(Bkdone)
debugfDiff(" Bkq <- %s\n", δtodo)
......@@ -595,8 +596,8 @@ ABcov:
}
// δtkeycov will be = BAdd \ ADel
δtkeycovADel := &RangedKeySet{}
δtkeycovBAdd := &RangedKeySet{}
δtkeycovADel := &blib.RangedKeySet{}
δtkeycovBAdd := &blib.RangedKeySet{}
// phase 1: expand A top->down driven by δZTC.
// by default a node contributes to δ-
......@@ -620,23 +621,21 @@ ABcov:
// a is bucket -> δ-
δA, err := diffB(ctx, a, nil); /*X*/if err != nil { return nil,nil,nil, err }
err = δMerge(δ, δA); /*X*/if err != nil { return nil,nil,nil, err }
ar := KeyRange{ra.lo, ra.hi_}
δtrack.Del.AddPath(ra.Path())
δtkeycovADel.AddRange(ar)
debugfDiff(" δtrack - %s %v\n", ar, ra.Path())
δtkeycovADel.AddRange(ra.keycov)
debugfDiff(" δtrack - %s %v\n", ra.keycov, ra.Path())
// Bkqueue <- ra.range
Bktodo(ar)
Bktodo(ra.keycov)
ra.done = true
case *Tree:
// empty tree - queue holes covered by it
if len(a.Entryv()) == 0 {
ar := KeyRange{ra.lo, ra.hi_}
δtrack.Del.AddPath(ra.Path())
δtkeycovADel.AddRange(ar)
debugfDiff(" δtrack - %s %v\n", ar, ra.Path())
Bktodo(ar)
δtkeycovADel.AddRange(ra.keycov)
debugfDiff(" δtrack - %s %v\n", ra.keycov, ra.Path())
Bktodo(ra.keycov)
continue
}
......@@ -659,8 +658,8 @@ ABcov:
bc, found := BnodeIdx[acOid]
if !found {
for {
blo := Bv.Get(ac.lo)
bhi_ := Bv.Get(ac.hi_)
blo := Bv.Get(ac.keycov.Lo)
bhi_ := Bv.Get(ac.keycov.Hi_)
if blo != bhi_ {
break
}
......@@ -691,17 +690,15 @@ ABcov:
}
if found {
// ac can be skipped if key coverage stays the same
ar := KeyRange{ac.lo, ac.hi_}
br := KeyRange{bc.lo, bc.hi_}
if ar == br {
if ac.keycov == bc.keycov {
// adjust trackSet since path to the node could have changed
apath := ac.Path()
bpath := bc.Path()
if !pathEqual(apath, bpath) {
δtrack.Del.AddPath(apath)
δtrack.Add.AddPath(bpath)
if nc := at.nchild; nc != 0 {
δtrack.δnchildNonLeafs[acOid] = nc
if nc := at.NChild(); nc != 0 {
δtrack.ΔnchildNonLeafs[acOid] = nc
}
}
......@@ -744,7 +741,7 @@ ABcov:
}
for _, r := range Bkqueue.AllRanges() {
lo := r.lo
lo := r.Lo
for {
b, err := Bv.GetToLeaf(ctx, lo); /*X*/if err != nil { return nil,nil,nil, err }
debugfDiff(" B k%d -> %s\n", lo, b)
......@@ -758,23 +755,22 @@ ABcov:
// δ <- δB
err = δMerge(δ, δB); /*X*/if err != nil { return nil,nil,nil, err }
br := KeyRange{b.lo, b.hi_}
δtrack.Add.AddPath(b.Path())
δtkeycovBAdd.AddRange(br)
debugfDiff(" δtrack + %s %v\n", br, b.Path())
δtkeycovBAdd.AddRange(b.keycov)
debugfDiff(" δtrack + %s %v\n", b.keycov, b.Path())
// Akqueue <- δB
Bkdone.AddRange(br)
Aktodo(br)
Bkdone.AddRange(b.keycov)
Aktodo(b.keycov)
b.done = true
}
// continue with next right bucket until r coverage is complete
if r.hi_ <= b.hi_ {
if r.Hi_ <= b.keycov.Hi_ {
break
}
lo = b.hi_ + 1
lo = b.keycov.Hi_ + 1
}
}
Bkqueue.Clear()
......@@ -782,7 +778,7 @@ ABcov:
debugfDiff("\n")
debugfDiff(" Akq: %s\n", Akqueue)
for _, r := range Akqueue.AllRanges() {
lo := r.lo
lo := r.Lo
for {
a, err := Av.GetToLeaf(ctx, lo); /*X*/if err != nil { return nil,nil,nil, err }
debugfDiff(" A k%d -> %s\n", lo, a)
......@@ -798,27 +794,26 @@ ABcov:
err = δMerge(δ, δA); /*X*/if err != nil { return nil,nil,nil, err }
δtrack.Del.AddPath(a.Path())
// NOTE adjust δtkeycovADel only if a was originally tracked
ar := KeyRange{a.lo, a.hi_}
_, tracked := trackSet[a.node.POid()]
if tracked {
δtkeycovADel.AddRange(ar)
debugfDiff(" δtrack - %s %v\n", ar, a.Path())
δtkeycovADel.AddRange(a.keycov)
debugfDiff(" δtrack - %s %v\n", a.keycov, a.Path())
} else {
debugfDiff(" δtrack - [) %v\n", a.Path())
}
// Bkqueue <- a.range
Akdone.AddRange(ar)
Bktodo(ar)
Akdone.AddRange(a.keycov)
Bktodo(a.keycov)
a.done = true
}
// continue with next right bucket until r coverage is complete
if r.hi_ <= a.hi_ {
if r.Hi_ <= a.keycov.Hi_ {
break
}
lo = a.hi_ + 1
lo = a.keycov.Hi_ + 1
}
}
Akqueue.Clear()
......@@ -1009,7 +1004,7 @@ func xidOf(obj zodb.IPersistent) string {
func (rn *nodeInRange) String() string {
done := " "; if rn.done { done = "*" }
return fmt.Sprintf("%s%s%s", done, KeyRange{rn.lo, rn.hi_}, vnode(rn.node))
return fmt.Sprintf("%s%s%s", done, rn.keycov, vnode(rn.node))
}
// push pushes element to node stack.
......
wcfs/internal/xbtree/xbtree.go
View file @ c7f1e3c9
......@@ -5,25 +5,24 @@ package xbtree
import (
"fmt"
"math"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/neo/go/zodb/btree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
)
// XXX instead of generics
type Tree = btree.LOBTree
type Bucket = btree.LOBucket
type Node = btree.LONode
type TreeEntry = btree.LOEntry
type BucketEntry = btree.LOBucketEntry
type Tree = blib.Tree
type Bucket = blib.Bucket
type Node = blib.Node
type TreeEntry = blib.TreeEntry
type BucketEntry = blib.BucketEntry
type Key = int64
const KeyMax Key = math.MaxInt64
const KeyMin Key = math.MinInt64
type Key = blib.Key
const KeyMax = blib.KeyMax
const KeyMin = blib.KeyMin
// value is assumed to be persistent reference.
// deletion is represented as VDEL.
......@@ -74,3 +73,7 @@ func kstr(k Key) string {
}
return fmt.Sprintf("%d", k)
}
func panicf(format string, argv ...interface{}) {
panic(fmt.Sprintf(format, argv...))
}
wcfs/internal/xbtree/xbtreetest/init/init.go 0 → 100644
View file @ c7f1e3c9
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package xbtreetest/init (ex imported from package A) should be imported in
// addition to xbtreetest (from package A_test) to initialize xbtreetest at runtime.
package init
// ZBlk-related part of δbtail_test
import (
"context"
"fmt"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
)
type Tree = xbtreetest.Tree
type Node = xbtreetest.Node
type Key = xbtreetest.Key
type ZBlk = zdata.ZBlk
// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
// XXX +return blkRevMax and use it ?
func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
path = []Node{}
xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
path = append(path, node)
})
if err != nil {
return nil, false, nil, err
}
if ok {
zblk, ok = xzblk.(ZBlk)
if !ok {
return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk)) // XXX errctx
}
}
return zblk, ok, path, nil
}
func init() {
xbtreetest.ZTreeGetBlkData = _ZTreeGetBlkData
xbtreetest.ZGetBlkData = _ZGetBlkData
}
// _ZTreeGetBlkData returns block data from block pointed to by ztree[k].
func _ZTreeGetBlkData(ctx context.Context, ztree *Tree, k Key) (data string, ok bool, path []Node, err error) {
defer xerr.Contextf(&err, "@%s: tree<%s>: get blkdata from [%d]", ztree.PJar().At(), ztree.POid(), k)
zblk, ok, path, err := ztreeGetBlk(ctx, ztree, k)
if err != nil || !ok {
return "", ok, path, err
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", false, nil, err
}
return string(bdata), true, path, nil
}
// _ZGetBlkData loads block data from ZBlk object specified by its oid.
func _ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
xblk, err := zconn.Get(ctx, zblkOid)
if err != nil {
return "", err
}
zblk, ok := xblk.(ZBlk)
if !ok {
return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", err
}
return string(bdata), nil
}
wcfs/internal/xbtree/xbtreetest/kvdiff.go 0 → 100644
View file @ c7f1e3c9
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// kvdiff + friends
import (
"fmt"
"sort"
"strings"
)
// kvdiff returns difference in between kv1 and kv2.
const DEL = "ø" // DEL means deletion
type Δstring struct {
Old string
New string
}
func kvdiff(kv1, kv2 map[Key]string) map[Key]Δstring {
delta := map[Key]Δstring{}
keys := setKey{}
for k := range kv1 { keys.Add(k) }
for k := range kv2 { keys.Add(k) }
for k := range keys {
v1, ok := kv1[k]
if !ok { v1 = DEL }
v2, ok := kv2[k]
if !ok { v2 = DEL }
if v1 != v2 {
delta[k] = Δstring{v1,v2}
}
}
return delta
}
// kvtxt returns string representation of {} kv.
func kvtxt(kv map[Key]string) string {
if len(kv) == 0 {
return "ø"
}
keyv := []Key{}
for k := range kv { keyv = append(keyv, k) }
sort.Slice(keyv, func(i,j int) bool { return keyv[i] < keyv[j] })
sv := []string{}
for _, k := range keyv {
v := kv[k]
if strings.ContainsAny(v, " \n\t,:") {
panicf("[%v]=%q: invalid value", k, v)
}
sv = append(sv, fmt.Sprintf("%v:%s", k, v))
}
return strings.Join(sv, ",")
}
wcfs/internal/xbtree/misc.go → wcfs/internal/xbtree/xbtreetest/kvdiff_test.go
View file @ c7f1e3c9
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -17,12 +17,28 @@
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree
package xbtreetest
import (
"fmt"
"reflect"
"testing"
)
func panicf(format string, argv ...interface{}) {
panic(fmt.Sprintf(format, argv...))
func TestKVDiff(t *testing.T) {
kv1 := map[Key]string{1:"a", 3:"c", 4:"d"}
kv2 := map[Key]string{1:"b", 4:"d", 5:"e"}
got := kvdiff(kv1, kv2)
want := map[Key]Δstring{1:{"a","b"}, 3:{"c",DEL}, 5:{DEL,"e"}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("error:\ngot: %v\nwant: %v", got, want)
}
}
func TestKVTxt(t *testing.T) {
kv := map[Key]string{3:"hello", 1:"zzz", 4:"world"}
got := kvtxt(kv)
want := "1:zzz,3:hello,4:world"
if got != want {
t.Fatalf("error:\ngot: %q\nwant: %q", got, want)
}
}
wcfs/internal/xbtree/xbtreetest/rtree.go 0 → 100644
View file @ c7f1e3c9
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
import (
"fmt"
"sort"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
)
// RTree represents Tree node covering [lo, hi_] key range in its parent tree.
// XXX actually no coverage here -> kill? -> change to just `path []zodb.Oid` in RBucket?
type RTree struct {
Oid zodb.Oid
Parent *RTree
// XXX +children?
}
// RBucket represents Bucket node covering [lo, hi_] key range in its Tree.
// NOTE it is not [lo,hi) but [lo,hi_] instead to avoid overflow at KeyMax.
type RBucket struct {
Oid zodb.Oid
Parent *RTree
Keycov blib.KeyRange
KV map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
}
// Path returns path to this bucket from tree root.
func (rb *RBucket) Path() []zodb.Oid {
path := []zodb.Oid{rb.Oid}
p := rb.Parent
for p != nil {
path = append([]zodb.Oid{p.Oid}, path...)
p = p.Parent
}
return path
}
// RBucketSet represents set of buckets covering whole [-∞,∞) range.
type RBucketSet []*RBucket // k↑
// Get returns RBucket which covers key k.
func (rbs RBucketSet) Get(k Key) *RBucket {
i := sort.Search(len(rbs), func(i int) bool {
return k <= rbs[i].Keycov.Hi_
})
if i == len(rbs) {
panicf("BUG: key %v not covered; coverage: %s", k, rbs.coverage())
}
rb := rbs[i]
if !rb.Keycov.Has(k) {
panicf("BUG: get(%v) -> %s; coverage: %s", k, rb.Keycov, rbs.coverage())
}
return rb
}
// coverage returns string representation of rbs coverage structure.
func (rbs RBucketSet) coverage() string {
if len(rbs) == 0 {
return "ø"
}
s := ""
for _, rb := range rbs {
if s != "" {
s += " "
}
s += fmt.Sprintf("%s", rb.Keycov)
}
return s
}
// Flatten converts xkv with bucket structure into regular dict.
func (xkv RBucketSet) Flatten() map[Key]string {
kv := make(map[Key]string)
for _, b := range xkv {
for k,v := range b.KV {
kv[k] = v
}
}
return kv
}
func (b *RBucket) String() string {
return fmt.Sprintf("%sB%s{%s}", b.Keycov, b.Oid, kvtxt(b.KV))
}
wcfs/internal/xbtree/xbtreetest/treeenv.go 0 → 100644
View file @ c7f1e3c9
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// T + friends
import (
"context"
"testing"
"lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/neo/go/transaction"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
)
// T is tree-based testing environment.
//
// It combines TreeSrv and client side access to ZODB with committed trees.
// It should be created it via NewT().
type T struct {
*testing.T
work string // working directory
treeSrv *TreeSrv
zstor zodb.IStorage
DB *zodb.DB
// all committed trees
commitv []*Commit
}
// Commit represent test commit changing a tree.
type Commit struct {
Tree string // the tree in topology-encoding
Prev *Commit // previous commit
At zodb.Tid // commit revision
ΔZ *zodb.EventCommit // raw ZODB changes; δZ.tid == at
Xkv RBucketSet // full tree state as of @at
Δxkv map[Key]Δstring // full tree-diff against parent
zblkDataTab map[zodb.Oid]string // full snapshot of all ZBlk data @at
// δzblkData map[zodb.Oid]Δstring // full diff for zblkData against parent XXX ?
}
// NewT creates new T.
func NewT(t *testing.T) *T {
X := exc.Raiseif
t.Helper()
tt := &T{T: t}
var err error
work := t.TempDir()
tt.treeSrv, err = StartTreeSrv(work + "/1.fs"); X(err)
t.Cleanup(func() {
err := tt.treeSrv.Close(); X(err)
})
tt.zstor, err = zodb.Open(context.Background(), tt.treeSrv.zurl, &zodb.OpenOptions{
ReadOnly: true,
}); X(err)
t.Cleanup(func() {
err := tt.zstor.Close(); X(err)
})
tt.DB = zodb.NewDB(tt.zstor, &zodb.DBOptions{
// We need objects to be cached, because otherwise it is too
// slow to run the test for many testcases, especially
// xverifyΔBTail_rebuild.
CacheControl: &tZODBCacheEverything{},
})
t.Cleanup(func() {
err := tt.DB.Close(); X(err)
})
head := tt.treeSrv.head
t1 := &Commit{
Tree: "T/B:", // treegen.py creates the tree as initially empty
Prev: nil,
At: head,
Xkv: xGetTree(tt.DB, head, tt.Root()),
zblkDataTab: xGetBlkDataTab(tt.DB, head),
ΔZ: nil,
Δxkv: nil,
}
tt.commitv = []*Commit{t1}
return tt
}
// tZODBCacheEverything is workaround for ZODB/go not implementing real
// live cache for now: Objects get dropped on PDeactivate if cache
// control does not say we need the object to stay in the cache.
// XXX place
type tZODBCacheEverything struct{}
func (_ *tZODBCacheEverything) PCacheClassify(_ zodb.IPersistent) zodb.PCachePolicy {
return zodb.PCachePinObject | zodb.PCacheKeepState
}
// Root returns OID of root tree node.
func (t *T) Root() zodb.Oid {
return t.treeSrv.treeRoot
}
// Head returns most-recently committed tree.
func (t *T) Head() *Commit {
return t.commitv[len(t.commitv)-1]
}
// CommitTree calls t.treeSrv.Commit and returns Commit corresponding to committed transaction.
// XXX naming -> Commit ?
func (t *T) CommitTree(tree string) *Commit {
// TODO X = FatalIf
X := exc.Raiseif
defer exc.Contextf("commit %s", tree)
watchq := make(chan zodb.Event)
at0 := t.zstor.AddWatch(watchq)
defer t.zstor.DelWatch(watchq)
tid, err := t.treeSrv.Commit(tree); X(err)
if !(tid > at0) {
exc.Raisef("treegen -> %s ; want > %s", tid, at0)
}
zevent := <-watchq
δZ := zevent.(*zodb.EventCommit)
if δZ.Tid != tid {
exc.Raisef("treegen -> %s ; watchq -> %s", tid, δZ)
}
// load tree structure from the db
// if the tree does not exist yet - report its structure as empty
var xkv RBucketSet
if tree != DEL {
xkv = xGetTree(t.DB, δZ.Tid, t.Root())
} else {
// empty tree with real treeRoot as oid even though the tree is
// deleted. Having real oid in the root tests that after deletion,
// root of the tree stays in the tracking set. We need root to stay
// in trackSet because e.g. in
//
// T1 -> ø -> T2
//
// where the tree is first deleted, then recreated, without root
// staying in trackSet after ->ø, treediff will notice nothing when
// it comes to ->T2.
xkv = RBucketSet{
&RBucket{
Oid: zodb.InvalidOid,
Parent: &RTree{
Oid: t.Root(), // NOTE oid is not InvalidOid
Parent: nil,
},
Keycov: blib.KeyRange{KeyMin, KeyMax},
KV: map[Key]string{},
},
}
}
ttree := &Commit{
Tree: tree,
At: δZ.Tid,
ΔZ: δZ,
Xkv: xkv,
zblkDataTab: xGetBlkDataTab(t.DB, δZ.Tid),
}
tprev := t.Head()
ttree.Prev = tprev
ttree.Δxkv = kvdiff(tprev.Xkv.Flatten(), ttree.Xkv.Flatten())
t.commitv = append(t.commitv, ttree)
return ttree
}
// xGetBlkDataTab loads all ZBlk from db@at.
//
// it returns {} oid -> blkdata.
func xGetBlkDataTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]string {
defer exc.Contextf("%s: @%s: get blkdatatab", db.Storage().URL(), at)
X := exc.Raiseif
blkDataTab := map[zodb.Oid]string{}
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)
xzroot, err := zconn.Get(ctx, 0); X(err)
zroot, ok := xzroot.(*zodb.Map)
if !ok {
exc.Raisef("root: expected %s, got %s", xzodb.TypeOf(zroot), xzodb.TypeOf(xzroot))
}
err = zroot.PActivate(ctx); X(err)
defer zroot.PDeactivate()
xzblkdir, ok := zroot.Data["treegen/values"]
if !ok {
exc.Raisef("root['treegen/values'] missing")
}
zblkdir, ok := xzblkdir.(*zodb.Map)
if !ok {
exc.Raisef("root['treegen/values']: expected %s, got %s", xzodb.TypeOf(zblkdir), xzodb.TypeOf(xzblkdir))
}
err = zblkdir.PActivate(ctx); X(err)
defer zblkdir.PDeactivate()
for k, xzblk := range zblkdir.Data {
zblk, ok := xzblk.(zodb.IPersistent)
if !ok {
exc.Raisef("root['treegen/values'][%q]: expected %s, got %s", k, xzodb.TypeOf(zblk), xzodb.TypeOf(xzblk))
}
oid := zblk.POid()
data := xzgetBlkData(ctx, zconn, oid)
blkDataTab[oid] = data
}
return blkDataTab
}
// XGetBlkData loads blk data for ZBlk<oid> @t.at
//
// For speed the load is done via preloaded t.blkDataTab instead of access to the DB.
func (t *Commit) XGetBlkData(oid zodb.Oid) string {
if oid == VDEL {
return DEL
}
data, ok := t.zblkDataTab[oid]
if !ok {
exc.Raisef("getBlkData ZBlk<%s> @%s: no such ZBlk", oid, t.At)
}
return data
}
// xGetTree loads Tree from zurl@at->obj<root>.
//
// Tree values must be ZBlk whose data is returned instead of references to ZBlk objects.
// The tree is returned structured by buckets as
//
// [] [lo,hi){k->v} k↑
func xGetTree(db *zodb.DB, at zodb.Tid, root zodb.Oid) RBucketSet {
defer exc.Contextf("%s: @%s: get tree %s", db.Storage().URL(), at, root)
X := exc.Raiseif
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)
xztree, err := zconn.Get(ctx, root); X(err)
ztree, ok := xztree.(*Tree)
if !ok {
exc.Raisef("expected %s, got %s", xzodb.TypeOf(ztree), xzodb.TypeOf(xztree))
}
rbucketv := RBucketSet{}
xwalkDFS(ctx, KeyMin, KeyMax, ztree, func(rb *RBucket) {
rbucketv = append(rbucketv, rb)
})
if len(rbucketv) == 0 { // empty tree -> [-∞,∞){}
etree := &RTree{
Oid: root,
Parent: nil,
}
ebucket := &RBucket{
Oid: zodb.InvalidOid,
Parent: etree,
Keycov: blib.KeyRange{KeyMin, KeyMax},
KV: map[Key]string{},
}
rbucketv = RBucketSet{ebucket}
}
return rbucketv
}
// xwalkDFS walks ztree in depth-first order emitting bvisit callback on visited bucket nodes.
func xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, bvisit func(*RBucket)) {
_xwalkDFS(ctx, lo, hi_, ztree, /*rparent*/nil, bvisit)
}
func _xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, rparent *RTree, bvisit func(*RBucket)) {
X := exc.Raiseif
err := ztree.PActivate(ctx); X(err)
defer ztree.PDeactivate()
rtree := &RTree{Oid: ztree.POid(), Parent: rparent}
// [i].Key ≤ [i].Child.*.Key < [i+1].Key i ∈ [0, len([]))
//
// [0].Key = -∞ ; always returned so
// [len(ev)].Key = +∞ ; should be assumed so
ev := ztree.Entryv()
for i := range ev {
xlo := lo; if i > 0 { xlo = ev[i].Key() }
xhi_ := hi_; if i+1 < len(ev) { xhi_ = ev[i+1].Key() - 1 }
tchild, ok := ev[i].Child().(*Tree)
if ok {
_xwalkDFS(ctx, xlo, xhi_, tchild, rtree, bvisit)
continue
}
zbucket := ev[i].Child().(*Bucket)
err = zbucket.PActivate(ctx); X(err)
defer zbucket.PDeactivate()
bkv := make(map[Key]string)
bentryv := zbucket.Entryv()
for _, __ := range bentryv {
k := __.Key()
xv := __.Value()
pv, ok := xv.(zodb.IPersistent)
if !ok {
exc.Raisef("[%d] -> %s; want IPersistent", k, xzodb.TypeOf(xv))
}
data, err := ZGetBlkData(ctx, pv.PJar(), pv.POid())
if err != nil {
exc.Raisef("[%d]: %s", k, err)
}
bkv[k] = data
}
b := &RBucket{
Oid: zbucket.POid(),
Parent: rtree,
Keycov: blib.KeyRange{xlo, xhi_},
KV: bkv,
}
bvisit(b)
}
}
wcfs/internal/xbtree/xbtreetest/treegen.go 0 → 100644
View file @ c7f1e3c9
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// treegen.go provides functionality:
//
// - to commit a particular BTree topology into ZODB, and
// - to generate set of random tree topologies that all correspond to particular {k->v} dict.
//
// treegen.py is used as helper for both tasks.
import (
"bufio"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"lab.nexedi.com/kirr/go123/my"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
)
// TreeGenSrv represents connection to running `treegen ...` server.
type TreeGenSrv struct {
argv []string
pysrv *exec.Cmd // spawned `treegen ...`
pyin io.WriteCloser // input to pysrv
pyoutRaw io.ReadCloser // output from pysrv
pyout *bufio.Reader // buffered ^^^
}
// TreeSrv represents connection to running `treegen trees` server.
//
// Create it with StartTreeSrv(zurl).
// - Commit(treeTopology) -> tid
type TreeSrv struct {
*TreeGenSrv
zurl string
treeRoot zodb.Oid // oid of the tree treegen works on
head zodb.Tid // last made commit
}
// AllStructsSrv represents connection to running `treegen allstructs` server.
//
// Create it with StartAllStructsSrv().
// - AllStructs(maxdepth, maxsplit, n, seed, kv1, kv2)
type AllStructsSrv struct {
*TreeGenSrv
}
// StartTreeGenSrv spawns `treegen ...` server.
func StartTreeGenSrv(argv ...string) (_ *TreeGenSrv, hello string, err error) {
defer xerr.Contextf(&err, "treesrv %v: start", argv)
// spawn `treegen ...`
tg := &TreeGenSrv{argv: argv}
tg.pysrv = exec.Command(filepath.Dir(my.File())+"/treegen.py", argv...)
tg.pyin, err = tg.pysrv.StdinPipe()
if err != nil {
return nil, "", err
}
tg.pyoutRaw, err = tg.pysrv.StdoutPipe()
if err != nil {
return nil, "", err
}
tg.pyout = bufio.NewReader(tg.pyoutRaw)
tg.pysrv.Stderr = os.Stderr // no redirection
err = tg.pysrv.Start()
if err != nil {
return nil, "", err
}
// wait for hello message and return it
defer func() {
if err != nil {
tg.Close() // ignore error
}
}()
defer xerr.Context(&err, "handshake")
hello, err = tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, "", err
}
hello = strings.TrimSuffix(hello, "\n")
return tg, hello, nil
}
// Close shutdowns treegen server.
func (tg *TreeGenSrv) Close() (err error) {
defer xerr.Contextf(&err, "treegen %v: close", tg.argv)
err1 := tg.pyin.Close()
err2 := tg.pyoutRaw.Close()
err3 := tg.pysrv.Wait()
return xerr.Merge(err1, err2, err3)
}
// StartTreeSrv spawns `treegen trees` server.
func StartTreeSrv(zurl string) (_ *TreeSrv, err error) {
defer xerr.Contextf(&err, "tree.srv %s: start", zurl)
tgSrv, hello, err := StartTreeGenSrv("trees", zurl)
if err != nil {
return nil, err
}
tg := &TreeSrv{TreeGenSrv: tgSrv, zurl: zurl}
defer func() {
if err != nil {
tgSrv.Close() // ignore error
}
}()
// tree.srv start @<at> tree=<root>
defer xerr.Contextf(&err, "invalid hello %q", hello)
startRe := regexp.MustCompile(`^tree.srv start @([^ ]+) root=([^ ]+)$`)
m := startRe.FindStringSubmatch(hello)
if m == nil {
return nil, fmt.Errorf("unexpected format")
}
tg.head, err = zodb.ParseTid(m[1]) // <at>
if err != nil {
return nil, fmt.Errorf("tid: %s", err)
}
tg.treeRoot, err = zodb.ParseOid(m[2]) // <root>
if err != nil {
return nil, fmt.Errorf("root: %s", err)
}
return tg, nil
}
// StartAllStructsSrv spawns `treegen allstructs` server.
func StartAllStructsSrv() (_ *AllStructsSrv, err error) {
defer xerr.Context(&err, "allstructs.srv: start")
tgSrv, hello, err := StartTreeGenSrv("allstructs")
if err != nil {
return nil, err
}
sg := &AllStructsSrv{TreeGenSrv: tgSrv}
defer func() {
if err != nil {
tgSrv.Close() // ignore error
}
}()
defer xerr.Contextf(&err, "invalid hello %q", hello)
if hello != "# allstructs.srv start" {
return nil, fmt.Errorf("unexpected format")
}
return sg, nil
}
// Commit creates new commit with underlying tree changed to specified tree topology.
func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
defer xerr.Contextf(&err, "tree.srv %s: commit %s", tg.zurl, tree)
_, err = io.WriteString(tg.pyin, tree + "\n")
if err != nil {
return zodb.InvalidTid, err
}
reply, err := tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return zodb.InvalidTid, err
}
reply = strings.TrimSuffix(reply, "\n")
tid, err := zodb.ParseTid(reply)
if err != nil {
return zodb.InvalidTid, fmt.Errorf("invalid reply: %s", err)
}
tg.head = tid
return tid, nil
}
// AllStructs returns response from `treegen allstructs`
func (tg *AllStructsSrv) AllStructs(kv map[Key]string, maxdepth, maxsplit, n int, seed int64) (_ []string, err error) {
req := fmt.Sprintf("%d %d %d/%d %s", maxdepth, maxsplit, n, seed, kvtxt(kv))
defer xerr.Contextf(&err, "allstructs.srv: %s ", req)
_, err = io.WriteString(tg.pyin, req + "\n")
if err != nil {
return nil, err
}
structv := []string{}
for {
reply, err := tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err
}
reply = strings.TrimSuffix(reply, "\n")
if reply == "# ----" {
return structv, nil // end of response
}
if strings.HasPrefix(reply, "#") {
continue // comment
}
structv = append(structv, reply)
}
}
wcfs/internal/xbtree/xbtreetest/xbtreetest.go 0 → 100644
View file @ c7f1e3c9
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package xbtreetest provides infrastructure for testing LOBTree with ZBlk values.
// XXX -> treetest?
package xbtreetest
import (
"fmt"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
)
// XXX instead of generics
type Tree = blib.Tree
type Bucket = blib.Bucket
type Node = blib.Node
type TreeEntry = blib.TreeEntry
type BucketEntry = blib.BucketEntry
type Key = blib.Key
const KeyMax = blib.KeyMax
const KeyMin = blib.KeyMin
type setKey = set.I64
// XXX dup from xbtree (to avoid import cycle)
const VDEL = zodb.InvalidOid
func panicf(format string, argv ...interface{}) {
panic(fmt.Sprintf(format, argv...))
}
wcfs/internal/xbtree/xbtreetest/zdata.go 0 → 100644
View file @ c7f1e3c9
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// access to ZBlk data
import (
"context"
"lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/neo/go/transaction"
"lab.nexedi.com/kirr/neo/go/zodb"
_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
)
// ZBlk-related functions are imported at runtime by package xbtreetest/init
var (
ZTreeGetBlkData func(context.Context, *Tree, Key) (string, bool, []Node, error)
ZGetBlkData func(context.Context, *zodb.Connection, zodb.Oid) (string, error)
)
func zassertInitDone() {
if ZTreeGetBlkData == nil {
panic("xbtreetest/zdata not initialized -> import xbtreetest/init to fix")
}
}
// xzgetBlkData loads block data from ZBlk object specified by its oid.
func xzgetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) string {
zassertInitDone()
X := exc.Raiseif
if zblkOid == VDEL {
return DEL
}
data, err := ZGetBlkData(ctx, zconn, zblkOid); X(err)
return string(data)
}
// xzgetBlkDataAt loads block data from ZBlk object specified by oid@at.
func xzgetBlkDataAt(db *zodb.DB, zblkOid zodb.Oid, at zodb.Tid) string {
zassertInitDone()
X := exc.Raiseif
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)
return xzgetBlkData(ctx, zconn, zblkOid)
}
wcfs/internal/xbtree/δbtail.go
View file @ c7f1e3c9
......@@ -30,6 +30,7 @@ import (
"lab.nexedi.com/kirr/neo/go/transaction"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xtail"
)
......@@ -96,7 +97,7 @@ type ΔBtail struct {
// For this set all vδT are fully computed.
// The set of nodes that were requested to be tracked, but were not yet
// taken into account, is kept in ΔTtail.trackNew & co.
trackSet PPTreeSubSet
trackSet blib.PPTreeSubSet
// set of trees for which .trackNew is non-empty
trackNewRoots setOid
......@@ -114,7 +115,7 @@ type ΔTtail struct {
// set of nodes that were requested to be tracked in this tree, but for
// which vδT was not yet rebuilt
trackNew PPTreeSubSet
trackNew blib.PPTreeSubSet
// XXX + trackNewKeys RangedKeySet
// {}k/v @tail for keys that are changed in (tail, head].
......@@ -158,7 +159,7 @@ func NewΔBtail(at0 zodb.Tid, db *zodb.DB) *ΔBtail {
δZtail: zodb.NewΔTail(at0),
vδBroots: nil,
vδTbyRoot: map[zodb.Oid]*ΔTtail{},
trackSet: PPTreeSubSet{},
trackSet: blib.PPTreeSubSet{},
trackNewRoots: setOid{},
db: db,
}
......@@ -167,7 +168,7 @@ func NewΔBtail(at0 zodb.Tid, db *zodb.DB) *ΔBtail {
// newΔTtail creates new empty ΔTtail object.
func newΔTtail() *ΔTtail {
return &ΔTtail{
trackNew: PPTreeSubSet{},
trackNew: blib.PPTreeSubSet{},
KVAtTail: make(map[Key]Value),
lastRevOf: make(map[Key]zodb.Tid),
}
......@@ -271,7 +272,7 @@ func (δBtail *ΔBtail) track(key Key, path []zodb.Oid) error {
// empty artificial tree. We need to do the normalization because we
// later check whether leaf path[-1] ∈ trackSet and without
// normalization path[-1] can be InvalidOid.
path = normPath(path)
path = blib.NormPath(path)
if len(path) == 0 {
return nil // empty tree
}
......@@ -334,7 +335,7 @@ func (δBtail *ΔBtail) rebuildAll() (err error) {
// - set of revisions for which new entries in .vδT have been created.
//
// XXX place
func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet PPTreeSubSet, δrevSet setTid, err error) {
func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet blib.PPTreeSubSet, δrevSet setTid, err error) {
defer xerr.Context(&err, "ΔTtail rebuild")
// XXX locking
......@@ -342,7 +343,7 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
tracefΔBtail("trackNew: %v\n", δTtail.trackNew)
trackNew := δTtail.trackNew
δTtail.trackNew = PPTreeSubSet{}
δTtail.trackNew = blib.PPTreeSubSet{}
if len(trackNew) == 0 {
return nil, nil, nil
......@@ -353,7 +354,7 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
// go backwards and merge vδT <- treediff(lo..hi/trackNew)
vδZ := δZtail.Data()
for {
δtkeycov := &RangedKeySet{} // all keys coming into tracking set during this lo<-hi scan
δtkeycov := &blib.RangedKeySet{} // all keys coming into tracking set during this lo<-hi scan
trackNewCur := trackNew.Clone() // trackNew adjusted as of when going to i<- entry
for i := len(vδZ)-1; i>=0; i-- {
δZ := vδZ[i]
......@@ -411,7 +412,7 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
// widenTrackNew widens trackNew to cover δtkeycov.
// XXX -> widenTrackSet?
func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid, at zodb.Tid, db *zodb.DB) (err error) {
func widenTrackNew(trackNew blib.PPTreeSubSet, δtkeycov *blib.RangedKeySet, root zodb.Oid, at zodb.Tid, db *zodb.DB) (err error) {
// XXX errctx, debug
defer xerr.Contextf(&err, "widenTrackNew tree<%s> @%s +%s", root, at, δtkeycov)
......@@ -427,19 +428,19 @@ func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid
}
tree := xtree.(*Tree) // must succeed XXX better explicit panic?
top := &nodeInRange{prefix: nil, lo: KeyMin, hi_: KeyMax, node: tree}
top := &nodeInRange{prefix: nil, keycov: blib.KeyRange{KeyMin, KeyMax}, node: tree}
V := rangeSplit{top}
for _, r := range δtkeycov.AllRanges() {
lo := r.lo
lo := r.Lo
for {
b, err := V.GetToLeaf(ctx, lo); /*X*/ if err != nil { return err }
trackNew.AddPath(b.Path())
// continue with next right bucket until r coverage is complete
if r.hi_ <= b.hi_ {
if r.Hi_ <= b.keycov.Hi_ {
break
}
lo = b.hi_ + 1
lo = b.keycov.Hi_ + 1
}
}
return nil
......@@ -450,7 +451,7 @@ func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid
//
// δtrackNew/δtkeycov represents how trackNew changes when going through `atPrev <- δZ.Rev` .
// newRevEntry indicates whether δZ.Rev was not there before in .vδT and new corresponding δT entry was created.
func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew PPTreeSubSet, db *zodb.DB) (δtrackNew *ΔPPTreeSubSet, δtkeycov *RangedKeySet, newRevEntry bool, err error) {
func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew blib.PPTreeSubSet, db *zodb.DB) (δtrackNew *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, newRevEntry bool, err error) {
defer xerr.Contextf(&err, "rebuild1 %s<-%s", atPrev, δZ.Rev)
debugfΔBtail("\n rebuild1 @%s <- @%s\n", atPrev, δZ.Rev)
......@@ -467,7 +468,7 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
// skip opening DB connections if there is no change to this tree
if len(δtopsByRoot) == 0 {
return NewΔPPTreeSubSet(), &RangedKeySet{}, false, nil
return blib.NewΔPPTreeSubSet(), &blib.RangedKeySet{}, false, nil
}
if len(δtopsByRoot) != 1 {
......@@ -560,7 +561,7 @@ func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
// δtkeycov1 != ø -> rebuild δTtail with trackNew ~= δtkeycov1
if !δT1.δtkeycov1.Empty() && δBtail.δZtail.Len() > 1 {
trackNew := PPTreeSubSet{}
trackNew := blib.PPTreeSubSet{}
err := widenTrackNew(trackNew, δT1.δtkeycov1, root, δBtail.Head(), δBtail.db)
if err != nil {
return ΔB{}, err
......@@ -611,8 +612,8 @@ type _ΔBUpdate1 struct {
ByRoot map[zodb.Oid]*_ΔTUpdate1
}
type _ΔTUpdate1 struct {
δtkeycov1 *RangedKeySet // {} root -> δtrackedKeys after first treediff (always grow)
δtrack *ΔPPTreeSubSet // XXX kill (not used)
δtkeycov1 *blib.RangedKeySet // {} root -> δtrackedKeys after first treediff (always grow)
δtrack *blib.ΔPPTreeSubSet // XXX kill (not used)
}
func (δBtail *ΔBtail) _Update1(δZ *zodb.EventCommit) (δB1 _ΔBUpdate1, err error) {
headOld := δBtail.Head()
......
wcfs/internal/xbtree/δbtail_test.go
View file @ c7f1e3c9
......@@ -38,23 +38,17 @@ package xbtree
//
// TestΔBTail and TestΔBTailAllStructs implement approaches "a" and "b" correspondingly.
//
// testprog/treegen.py is used as helper to both:
// testprog/treegen.py is used as helper to both: XXX moved to xbtreetest
//
// - commit a particular BTree topology into ZODB, and
// - to generate set of random tree topologies that all correspond to particular {k->v} dict.
import (
"bufio"
"context"
"flag"
"fmt"
"io"
"math"
"math/rand"
"os"
"os/exec"
"reflect"
"regexp"
"sort"
"strings"
"testing"
......@@ -62,445 +56,52 @@ import (
"lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/transaction"
"lab.nexedi.com/kirr/neo/go/zodb"
_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest"
)
// TreeGenSrv represents connection to running `treegen ...` server.
type TreeGenSrv struct {
argv []string
pysrv *exec.Cmd // spawned `treegen ...`
pyin io.WriteCloser // input to pysrv
pyoutRaw io.ReadCloser // output from pysrv
pyout *bufio.Reader // buffered ^^^
}
// TreeSrv represents connection to running `treegen trees` server.
//
// Create it with StartTreeSrv(zurl).
// - Commit(treeTopology) -> tid
type TreeSrv struct {
*TreeGenSrv
zurl string
treeRoot zodb.Oid // oid of the tree treegen works on
head zodb.Tid // last made commit
}
// AllStructsSrv represents connection to running `treegen allstructs` server.
//
// Create it with StartAllStructsSrv().
// - AllStructs(maxdepth, maxsplit, n, seed, kv1, kv2)
type AllStructsSrv struct {
*TreeGenSrv
}
// StartTreeGenSrv spawns `treegen ...` server.
func StartTreeGenSrv(argv ...string) (_ *TreeGenSrv, hello string, err error) {
defer xerr.Contextf(&err, "treesrv %v: start", argv)
// spawn `treegen ...`
tg := &TreeGenSrv{argv: argv}
tg.pysrv = exec.Command("./testprog/treegen.py", argv...)
tg.pyin, err = tg.pysrv.StdinPipe()
if err != nil {
return nil, "", err
}
tg.pyoutRaw, err = tg.pysrv.StdoutPipe()
if err != nil {
return nil, "", err
}
tg.pyout = bufio.NewReader(tg.pyoutRaw)
tg.pysrv.Stderr = os.Stderr // no redirection
err = tg.pysrv.Start()
if err != nil {
return nil, "", err
}
// wait for hello message and return it
defer func() {
if err != nil {
tg.Close() // ignore error
}
}()
defer xerr.Context(&err, "handshake")
hello, err = tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, "", err
}
hello = strings.TrimSuffix(hello, "\n")
return tg, hello, nil
}
// Close shutdowns treegen server.
func (tg *TreeGenSrv) Close() (err error) {
defer xerr.Contextf(&err, "treegen %v: close", tg.argv)
type Δstring = xbtreetest.Δstring
err1 := tg.pyin.Close()
err2 := tg.pyoutRaw.Close()
err3 := tg.pysrv.Wait()
return xerr.Merge(err1, err2, err3)
}
// StartTreeSrv spawns `treegen trees` server.
func StartTreeSrv(zurl string) (_ *TreeSrv, err error) {
defer xerr.Contextf(&err, "tree.srv %s: start", zurl)
tgSrv, hello, err := StartTreeGenSrv("trees", zurl)
if err != nil {
return nil, err
}
tg := &TreeSrv{TreeGenSrv: tgSrv, zurl: zurl}
defer func() {
if err != nil {
tgSrv.Close() // ignore error
}
}()
// tree.srv start @<at> tree=<root>
defer xerr.Contextf(&err, "invalid hello %q", hello)
startRe := regexp.MustCompile(`^tree.srv start @([^ ]+) root=([^ ]+)$`)
m := startRe.FindStringSubmatch(hello)
if m == nil {
return nil, fmt.Errorf("unexpected format")
}
tg.head, err = zodb.ParseTid(m[1]) // <at>
if err != nil {
return nil, fmt.Errorf("tid: %s", err)
}
tg.treeRoot, err = zodb.ParseOid(m[2]) // <root>
if err != nil {
return nil, fmt.Errorf("root: %s", err)
}
return tg, nil
}
// StartAllStructsSrv spawns `treegen allstructs` server.
func StartAllStructsSrv() (_ *AllStructsSrv, err error) {
defer xerr.Context(&err, "allstructs.srv: start")
tgSrv, hello, err := StartTreeGenSrv("allstructs")
if err != nil {
return nil, err
}
sg := &AllStructsSrv{TreeGenSrv: tgSrv}
defer func() {
if err != nil {
tgSrv.Close() // ignore error
}
}()
defer xerr.Contextf(&err, "invalid hello %q", hello)
if hello != "# allstructs.srv start" {
return nil, fmt.Errorf("unexpected format")
}
return sg, nil
}
// Commit creates new commit with underlying tree changed to specified tree topology.
func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
defer xerr.Contextf(&err, "tree.srv %s: commit %s", tg.zurl, tree)
_, err = io.WriteString(tg.pyin, tree + "\n")
if err != nil {
return zodb.InvalidTid, err
}
reply, err := tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return zodb.InvalidTid, err
}
reply = strings.TrimSuffix(reply, "\n")
tid, err := zodb.ParseTid(reply)
if err != nil {
return zodb.InvalidTid, fmt.Errorf("invalid reply: %s", err)
}
tg.head = tid
return tid, nil
}
// AllStructs returns response from `treegen allstructs`
func (tg *AllStructsSrv) AllStructs(kv map[Key]string, maxdepth, maxsplit, n int, seed int64) (_ []string, err error) {
req := fmt.Sprintf("%d %d %d/%d %s", maxdepth, maxsplit, n, seed, kvtxt(kv))
defer xerr.Contextf(&err, "allstructs.srv: %s ", req)
_, err = io.WriteString(tg.pyin, req + "\n")
if err != nil {
return nil, err
}
structv := []string{}
for {
reply, err := tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err
}
reply = strings.TrimSuffix(reply, "\n")
if reply == "# ----" {
return structv, nil // end of response
}
if strings.HasPrefix(reply, "#") {
continue // comment
}
structv = append(structv, reply)
}
}
// RTree represents Tree node covering [lo, hi_] key range in its parent tree.
// XXX actually no coverage here -> kill? -> change to just `path []zodb.Oid` in RBucket?
type RTree struct {
oid zodb.Oid
parent *RTree
// XXX +children?
}
// RBucket represents Bucket node covering [lo, hi_] key range in its Tree.
// NOTE it is not [lo,hi) but [lo,hi_] instead to avoid overflow at KeyMax.
type RBucket struct {
oid zodb.Oid
parent *RTree
lo, hi_ Key // XXX -> KeyRange ?
kv map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
}
// Path returns path to this bucket from tree root.
func (rb *RBucket) Path() []zodb.Oid {
path := []zodb.Oid{rb.oid}
p := rb.parent
for p != nil {
path = append([]zodb.Oid{p.oid}, path...)
p = p.parent
}
return path
}
// RBucketSet represents set of buckets covering whole [-∞,∞) range.
type RBucketSet []*RBucket // k↑
// Get returns RBucket which covers key k.
func (rbs RBucketSet) Get(k Key) *RBucket {
i := sort.Search(len(rbs), func(i int) bool {
return k <= rbs[i].hi_
})
if i == len(rbs) {
panicf("BUG: key %v not covered; coverage: %s", k, rbs.coverage())
}
rb := rbs[i]
if !(rb.lo <= k && k <= rb.hi_) {
panicf("BUG: get(%v) -> [%v, %v]; coverage: %s", k, rb.lo, rb.hi_, rbs.coverage())
}
return rb
}
// coverage returns string representation of rbs coverage structure.
func (rbs RBucketSet) coverage() string {
if len(rbs) == 0 {
return "ø"
}
s := ""
for _, rb := range rbs {
if s != "" {
s += " "
}
s += fmt.Sprintf("[%v, %v]", rb.lo, rb.hi_)
}
return s
}
// trackSet returns what should be ΔBtail.trackSet coverage for specified tracked key set.
func (rbs RBucketSet) trackSet(tracked setKey) PPTreeSubSet {
// XXX was method -> change name?
func trackSet(rbs xbtreetest.RBucketSet, tracked setKey) blib.PPTreeSubSet {
// nil = don't compute keyCover
// (trackSet is called from inside hot inner loop of rebuild test)
trackSet := rbs._trackSetWithCov(tracked, nil)
trackSet := _trackSetWithCov(rbs, tracked, nil)
return trackSet
}
// trackSetWithCov returns what should be ΔBtail.trackSet and its key coverage for specified tracked key set.
func (rbs RBucketSet) trackSetWithCov(tracked setKey) (trackSet PPTreeSubSet, keyCover *RangedKeySet) {
keyCover = &RangedKeySet{}
trackSet = rbs._trackSetWithCov(tracked, keyCover)
func trackSetWithCov(rbs xbtreetest.RBucketSet, tracked setKey) (trackSet blib.PPTreeSubSet, keyCover *blib.RangedKeySet) {
keyCover = &blib.RangedKeySet{}
trackSet = _trackSetWithCov(rbs, tracked, keyCover)
return trackSet, keyCover
}
func (rbs RBucketSet) _trackSetWithCov(tracked setKey, outKeyCover *RangedKeySet) (trackSet PPTreeSubSet) {
trackSet = PPTreeSubSet{}
func _trackSetWithCov(rbs xbtreetest.RBucketSet, tracked setKey, outKeyCover *blib.RangedKeySet) (trackSet blib.PPTreeSubSet) {
trackSet = blib.PPTreeSubSet{}
for k := range tracked {
kb := rbs.Get(k)
if outKeyCover != nil {
outKeyCover.AddRange(KeyRange{kb.lo, kb.hi_})
}
// trackSet explicitly records only regular buckets.
// embedded buckets all have oid=zodb.InvalidOid and would lead to z
newNode := false
if kb.oid != zodb.InvalidOid {
track, already := trackSet[kb.oid]
if !already {
track = &nodeInTree{parent: kb.parent.oid, nchild: 0}
trackSet[kb.oid] = track
newNode = true
}
if track.parent != kb.parent.oid {
panicf("BUG: %s: B%s -> multiple parents: %s %s", rbs.coverage(), kb.oid, track.parent, kb.parent.oid)
}
}
p := kb.parent
for p != nil {
ppoid := zodb.InvalidOid // oid of p.parent
if p.parent != nil {
ppoid = p.parent.oid
}
newParent := false
pt, already := trackSet[p.oid]
if !already {
pt = &nodeInTree{parent: ppoid, nchild: 0}
trackSet[p.oid] = pt
newParent = true
}
if pt.parent != ppoid {
panicf("BUG: %s: T%s -> multiple parents: %s %s", rbs.coverage(), p.oid, pt.parent, ppoid)
}
if newNode {
pt.nchild++
}
newNode = newParent
p = p.parent
outKeyCover.AddRange(kb.Keycov)
}
trackSet.AddPath(kb.Path())
}
return trackSet
}
// XGetTree loads Tree from zurl@at->obj<root>.
//
// Tree values must be ZBlk whose data is returned instead of references to ZBlk objects.
// The tree is returned structured by buckets as
//
// [] [lo,hi){k->v} k↑
func XGetTree(db *zodb.DB, at zodb.Tid, root zodb.Oid) RBucketSet {
defer exc.Contextf("%s: @%s: get tree %s", db.Storage().URL(), at, root)
X := exc.Raiseif
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)
xztree, err := zconn.Get(ctx, root); X(err)
ztree, ok := xztree.(*Tree)
if !ok {
exc.Raisef("expected %s, got %s", xzodb.TypeOf(ztree), xzodb.TypeOf(xztree))
}
rbucketv := RBucketSet{}
xwalkDFS(ctx, KeyMin, KeyMax, ztree, func(rb *RBucket) {
rbucketv = append(rbucketv, rb)
})
if len(rbucketv) == 0 { // empty tree -> [-∞,∞){}
etree := &RTree{
oid: root,
parent: nil,
}
ebucket := &RBucket{
oid: zodb.InvalidOid,
parent: etree,
lo: KeyMin,
hi_: KeyMax,
kv: map[Key]string{},
}
rbucketv = RBucketSet{ebucket}
}
return rbucketv
}
// xwalkDFS walks ztree in depth-first order emitting bvisit callback on visited bucket nodes.
func xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, bvisit func(*RBucket)) {
_xwalkDFS(ctx, lo, hi_, ztree, /*rparent*/nil, bvisit)
}
func _xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, rparent *RTree, bvisit func(*RBucket)) {
X := exc.Raiseif
err := ztree.PActivate(ctx); X(err)
defer ztree.PDeactivate()
rtree := &RTree{oid: ztree.POid(), parent: rparent}
// [i].Key ≤ [i].Child.*.Key < [i+1].Key i ∈ [0, len([]))
//
// [0].Key = -∞ ; always returned so
// [len(ev)].Key = +∞ ; should be assumed so
ev := ztree.Entryv()
for i := range ev {
xlo := lo; if i > 0 { xlo = ev[i].Key() }
xhi_ := hi_; if i+1 < len(ev) { xhi_ = ev[i+1].Key() - 1 }
tchild, ok := ev[i].Child().(*Tree)
if ok {
_xwalkDFS(ctx, xlo, xhi_, tchild, rtree, bvisit)
continue
}
zbucket := ev[i].Child().(*Bucket)
err = zbucket.PActivate(ctx); X(err)
defer zbucket.PDeactivate()
bkv := make(map[Key]string)
bentryv := zbucket.Entryv()
for _, __ := range bentryv {
k := __.Key()
xv := __.Value()
pv, ok := xv.(zodb.IPersistent)
if !ok {
exc.Raisef("[%d] -> %s; want IPersistent", k, xzodb.TypeOf(xv))
}
data, err := ZGetBlkData(ctx, pv.PJar(), pv.POid())
if err != nil {
exc.Raisef("[%d]: %s", k, err)
}
bkv[k] = data
}
b := &RBucket{oid: zbucket.POid(), parent: rtree, lo: xlo, hi_: xhi_, kv: bkv}
bvisit(b)
}
}
// XGetδKV translates {k -> δ<oid>} to {k -> δ(ZBlk(oid).data)} according to t1..t2 db snapshots.
func XGetδKV(t1, t2 *tTreeCommit, δkvOid map[Key]ΔValue) map[Key]Δstring {
func XGetδKV(t1, t2 *xbtreetest.Commit, δkvOid map[Key]ΔValue) map[Key]Δstring {
δkv := make(map[Key]Δstring, len(δkvOid))
for k, δvOid := range δkvOid {
δkv[k] = Δstring{
Old: t1.xgetBlkData(δvOid.Old),
New: t2.xgetBlkData(δvOid.New),
Old: t1.XGetBlkData(δvOid.Old),
New: t2.XGetBlkData(δvOid.New),
}
}
return δkv
......@@ -574,13 +175,13 @@ func (kadjA KAdjMatrix) Mul(kadjB KAdjMatrix) KAdjMatrix {
// This set of keys defaults to allTestKeys(t1,t2).
//
// KAdj itself is verified by testΔBTail on entries with .kadjOK set.
func KAdj(t1, t2 *tTreeCommit, keysv ...setKey) (kadj KAdjMatrix) {
func KAdj(t1, t2 *xbtreetest.Commit, keysv ...setKey) (kadj KAdjMatrix) {
// assert KAdj(A,B) == KAdj(B,A)
kadj12 := _KAdj(t1,t2, keysv...)
kadj21 := _KAdj(t2,t1, keysv...)
if !reflect.DeepEqual(kadj12, kadj21) {
panicf("KAdj not symmetric:\nt1: %s\nt2: %s\nkadj12: %v\nkadj21: %v",
t1.tree, t2.tree, kadj12, kadj21)
t1.Tree, t2.Tree, kadj12, kadj21)
}
return kadj12
}
......@@ -592,7 +193,7 @@ func debugfKAdj(format string, argv ...interface{}) {
}
}
func _KAdj(t1, t2 *tTreeCommit, keysv ...setKey) (kadj KAdjMatrix) {
func _KAdj(t1, t2 *xbtreetest.Commit, keysv ...setKey) (kadj KAdjMatrix) {
var keys setKey
switch len(keysv) {
case 0:
......@@ -604,8 +205,8 @@ func _KAdj(t1, t2 *tTreeCommit, keysv ...setKey) (kadj KAdjMatrix) {
}
debugfKAdj("\n\n_KAdj\n")
debugfKAdj("t1: %s\n", t1.tree)
debugfKAdj("t2: %s\n", t2.tree)
debugfKAdj("t1: %s\n", t1.Tree)
debugfKAdj("t2: %s\n", t2.Tree)
debugfKAdj("keys: %s\n", keys)
defer func() {
debugfKAdj("kadj -> %v\n", kadj)
......@@ -618,69 +219,67 @@ func _KAdj(t1, t2 *tTreeCommit, keysv ...setKey) (kadj KAdjMatrix) {
adj1 := setKey{}
adj2 := setKey{}
q1 := &RangedKeySet{}; q1.Add(k)
q2 := &RangedKeySet{}; q2.Add(k)
done1 := &RangedKeySet{}
done2 := &RangedKeySet{}
q1 := &blib.RangedKeySet{}; q1.Add(k)
q2 := &blib.RangedKeySet{}; q2.Add(k)
done1 := &blib.RangedKeySet{}
done2 := &blib.RangedKeySet{}
debugfKAdj("\nk%s\n", kstr(k))
for !q1.Empty() || !q2.Empty() {
debugfKAdj("q1: %s\tdone1: %s\n", q1, done1)
debugfKAdj("q2: %s\tdone2: %s\n", q2, done2)
for _, r1 := range q1.AllRanges() {
lo1 := r1.lo
lo1 := r1.Lo
for {
b1 := t1.xkv.Get(lo1)
b1 := t1.Xkv.Get(lo1)
debugfKAdj(" b1: %s\n", b1)
for k_ := range keys {
if b1.lo <= k_ && k_ <= b1.hi_ {
if b1.Keycov.Has(k_) {
adj1.Add(k_)
debugfKAdj(" adj1 += %s\t-> %s\n", kstr(k_), adj1)
}
}
b1r := KeyRange{b1.lo, b1.hi_}
done1.AddRange(b1r)
done1.AddRange(b1.Keycov)
// q2 |= (b1.keyrange \ done2)
δq2 := &RangedKeySet{}
δq2.AddRange(b1r)
δq2 := &blib.RangedKeySet{}
δq2.AddRange(b1.Keycov)
δq2.DifferenceInplace(done2)
q2.UnionInplace(δq2)
debugfKAdj("q2 += %s\t-> %s\n", δq2, q2)
// continue with next right bucket until r1 coverage is complete
if r1.hi_ <= b1.hi_ {
if r1.Hi_ <= b1.Keycov.Hi_ {
break
}
lo1 = b1.hi_ + 1
lo1 = b1.Keycov.Hi_ + 1
}
}
q1.Clear()
for _, r2 := range q2.AllRanges() {
lo2 := r2.lo
lo2 := r2.Lo
for {
b2 := t2.xkv.Get(lo2)
b2 := t2.Xkv.Get(lo2)
debugfKAdj(" b2: %s\n", b2)
for k_ := range keys {
if b2.lo <= k_ && k_ <= b2.hi_ {
if b2.Keycov.Has(k_) {
adj2.Add(k_)
debugfKAdj(" adj2 += %s\t-> %s\n", kstr(k_), adj2)
}
}
b2r := KeyRange{b2.lo, b2.hi_}
done2.AddRange(b2r)
done2.AddRange(b2.Keycov)
// q1 |= (b2.keyrange \ done1)
δq1 := &RangedKeySet{}
δq1.AddRange(b2r)
δq1 := &blib.RangedKeySet{}
δq1.AddRange(b2.Keycov)
δq1.DifferenceInplace(done1)
q1.UnionInplace(δq1)
debugfKAdj("q1 += %s\t-> %s\n", δq1, q1)
// continue with next right bucket until r2 coverage is complete
if r2.hi_ <= b2.hi_ {
if r2.Hi_ <= b2.Keycov.Hi_ {
break
}
lo2 = b2.hi_ + 1
lo2 = b2.Keycov.Hi_ + 1
}
}
q2.Clear()
......@@ -700,10 +299,10 @@ func _KAdj(t1, t2 *tTreeCommit, keysv ...setKey) (kadj KAdjMatrix) {
// the cycling phase of update, that is responsible to recompute older
// entries when key coverage grows, is exercised by
// xverifyΔBTail_rebuild.
func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1, t2 *tTreeCommit) {
func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1, t2 *xbtreetest.Commit) {
// verify transition at1->at2 for all initial states of tracked {keys} from kv1 + kv2 + ∞
t.Run(fmt.Sprintf("Update/%s→%s", t1.tree, t2.tree), func(t *testing.T) {
t.Run(fmt.Sprintf("Update/%s→%s", t1.Tree, t2.Tree), func(t *testing.T) {
allKeys := allTestKeys(t1, t2)
allKeyv := allKeys.SortedElements()
......@@ -727,12 +326,12 @@ func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb
// xverifyΔBTail_Update1 verifies how ΔBTail handles ZODB update at1->at2 from initial
// tracked state defined by initialTrackedKeys.
func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1,t2 *tTreeCommit, initialTrackedKeys setKey, kadj KAdjMatrix) {
func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1,t2 *xbtreetest.Commit, initialTrackedKeys setKey, kadj KAdjMatrix) {
X := exc.Raiseif
//t.Logf("\n>>> Track=%s\n", initialTrackedKeys)
δZ := t2.δZ
d12 := t2.δxkv
δZ := t2.ΔZ
d12 := t2.Δxkv
var TrackedδZ setKey = nil
var kadjTrackedδZ setKey = nil
......@@ -764,23 +363,23 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// δbtail @at1 with initial tracked set
δbtail := NewΔBtail(t1.at, db)
δbtail := NewΔBtail(t1.At, db)
xtrackKeys(δbtail, t1, initialTrackedKeys)
// TrackedδZ = Tracked ^ δZ (i.e. a tracked node has changed, or its coverage was changed)
TrackedδZ = setKey{}
for k := range initialTrackedKeys {
leaf1 := t1.xkv.Get(k)
oid1 := leaf1.oid
leaf1 := t1.Xkv.Get(k)
oid1 := leaf1.Oid
if oid1 == zodb.InvalidOid { // embedded bucket
oid1 = leaf1.parent.oid
oid1 = leaf1.Parent.Oid
}
leaf2 := t2.xkv.Get(k)
oid2 := leaf2.oid
leaf2 := t2.Xkv.Get(k)
oid2 := leaf2.Oid
if oid2 == zodb.InvalidOid { // embedded bucket
oid2 = leaf2.parent.oid
oid2 = leaf2.Parent.Oid
}
if δZset.Has(oid1) || δZset.Has(oid2) || (KeyRange{leaf1.lo,leaf1.hi_} != KeyRange{leaf2.lo,leaf2.hi_}) {
if δZset.Has(oid1) || δZset.Has(oid2) || (leaf1.Keycov != leaf2.Keycov) {
TrackedδZ.Add(k)
}
}
......@@ -809,12 +408,12 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
}
}
ø := PPTreeSubSet{}
ø := blib.PPTreeSubSet{}
// trackSet1 = xkv1[tracked1]
// trackSet2 = xkv2[tracked2] ( = xkv2[kadj[tracked1]]
trackSet1, tkeyCov1 := t1.xkv.trackSetWithCov(initialTrackedKeys)
trackSet2, tkeyCov2 := t2.xkv.trackSetWithCov(initialTrackedKeys.Union(kadjTrackedδZ))
trackSet1, tkeyCov1 := trackSetWithCov(t1.Xkv, initialTrackedKeys)
trackSet2, tkeyCov2 := trackSetWithCov(t2.Xkv, initialTrackedKeys.Union(kadjTrackedδZ))
// verify δbtail.trackSet against @at1
δbtail.assertTrack(t, "1", ø, trackSet1)
......@@ -834,7 +433,7 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// assert δtkeycov == δ(tkeyCov1, tkeyCov2)
δtkeycovOK := tkeyCov2.Difference(tkeyCov1)
δtkeycov := &RangedKeySet{}
δtkeycov := &blib.RangedKeySet{}
if __, ok := δB1.ByRoot[treeRoot]; ok {
δtkeycov = __.δtkeycov1
}
......@@ -915,7 +514,7 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// assertTrack verifies state of .trackSet and ΔTtail.trackNew.
// it assumes that only one tree root is being tracked.
// XXX place
func (δBtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK PPTreeSubSet, trackNewOK PPTreeSubSet) {
func (δBtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK blib.PPTreeSubSet, trackNewOK blib.PPTreeSubSet) {
t.Helper()
if !δBtail.trackSet.Equal(trackSetOK) {
t.Errorf("%s: trackSet:\n\thave: %v\n\twant: %v", subj, δBtail.trackSet, trackSetOK)
......@@ -962,21 +561,21 @@ func (δBtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK PPTree
// t1->t2 further exercises incremental rebuild.
//
// It also exercises rebuild phase of ΔBtail.Update.
func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1, t2 *tTreeCommit) {
t.Run(fmt.Sprintf("rebuild/%s→%s", t0.tree, t1.tree), func(t *testing.T) {
func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1, t2 *xbtreetest.Commit) {
t.Run(fmt.Sprintf("rebuild/%s→%s", t0.Tree, t1.Tree), func(t *testing.T) {
tAllKeys := allTestKeys(t0, t1, t2)
tAllKeyv := tAllKeys.SortedElements()
// tid -> "at_i"
xat := map[zodb.Tid]string{
t0.at: "at0",
t1.at: "at1",
t2.at: "at2",
t0.At: "at0",
t1.At: "at1",
t2.At: "at2",
}
//fmt.Printf("@%s: %v\n", xat[t0.at], t0.xkv.Flatten())
//fmt.Printf("@%s: %v\n", xat[t1.at], t1.xkv.Flatten())
//fmt.Printf("@%s: %v\n", xat[t2.at], t2.xkv.Flatten())
//fmt.Printf("@%s: %v\n", xat[t0.At], t0.Xkv.Flatten())
//fmt.Printf("@%s: %v\n", xat[t1.At], t1.Xkv.Flatten())
//fmt.Printf("@%s: %v\n", xat[t2.At], t2.Xkv.Flatten())
kadj10 := KAdj(t1,t0, allTestKeys(t0,t1,t2))
kadj21 := KAdj(t2,t1, allTestKeys(t0,t1,t2))
......@@ -985,7 +584,7 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
// kadj210 = kadj10·kadj21
kadj210 := kadj10.Mul(kadj21)
ø := PPTreeSubSet{}
ø := blib.PPTreeSubSet{}
// verify t0 -> t1 Track(keys1) Rebuild -> t2 Track(keys2) Rebuild
// for all combinations of keys1 and keys2
......@@ -999,17 +598,17 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
keys1_0 := kadj10.Map(keys1)
δkv1_1 := map[Key]Δstring{}
for k := range keys1_0 {
δv, ok := t1.δxkv[k]
δv, ok := t1.Δxkv[k]
if ok {
δkv1_1[k] = δv
}
}
Tkeys1 := t1.xkv.trackSet(keys1)
Tkeys1_0 := t1.xkv.trackSet(keys1_0)
Tkeys1 := trackSet(t1.Xkv, keys1)
Tkeys1_0 := trackSet(t1.Xkv, keys1_0)
t.Run(fmt.Sprintf(" T%s;R", keys1), func(t *testing.T) {
δbtail := NewΔBtail(t0.at, db)
δbtail := NewΔBtail(t0.At, db)
// assert trackSet=ø, trackNew=ø, vδB=[]
δbtail.assertTrack(t, "@at0", ø, ø)
......@@ -1029,7 +628,7 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
/*trackSet=*/ Tkeys1_0,
/*vδT=*/ δkv1_1)
t.Run((" →" + t2.tree), func(t *testing.T) {
t.Run((" →" + t2.Tree), func(t *testing.T) {
// keys1R2 is full set of keys that should become tracked after
// Update() (which includes rebuild)
keys1R2 := kadj12.Map(keys1)
......@@ -1045,17 +644,17 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
δkv1_k1R2 := map[Key]Δstring{}
δkv2_k1R2 := map[Key]Δstring{}
for k := range keys1R2 {
δv1, ok := t1.δxkv[k]
δv1, ok := t1.Δxkv[k]
if ok {
δkv1_k1R2[k] = δv1
}
δv2, ok := t2.δxkv[k]
δv2, ok := t2.Δxkv[k]
if ok {
δkv2_k1R2[k] = δv2
}
}
Tkeys1R2 := t2.xkv.trackSet(keys1R2)
Tkeys1R2 := trackSet(t2.Xkv, keys1R2)
xverifyΔBTail_rebuild_U(t, δbtail, treeRoot, t1, t2, xat,
/*trackSet=*/ Tkeys1R2,
......@@ -1090,16 +689,16 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
keys12R2 = keys12R2_
}
Tkeys2 := t2.xkv.trackSet(keys2)
Tkeys12R2 := t2.xkv.trackSet(keys12R2)
Tkeys2 := trackSet(t2.Xkv, keys2)
Tkeys12R2 := trackSet(t2.Xkv, keys12R2)
/*
fmt.Printf("\n\n\nKKK\nkeys1=%s keys2=%s\n", keys1, keys2)
fmt.Printf("keys1R2: %s\n", keys1R2)
fmt.Printf("keys12R2: %s\n", keys12R2)
fmt.Printf("t0.xkv: %v\n", t0.xkv)
fmt.Printf("t1.xkv: %v\n", t1.xkv)
fmt.Printf("t2.xkv: %v\n", t2.xkv)
fmt.Printf("t0.Xkv: %v\n", t0.Xkv)
fmt.Printf("t1.Xkv: %v\n", t1.Xkv)
fmt.Printf("t2.Xkv: %v\n", t2.Xkv)
fmt.Printf("kadj21: %v\n", kadj21)
fmt.Printf("kadj12: %v\n", kadj12)
fmt.Printf("Tkeys2 -> %s\n", Tkeys2)
......@@ -1110,14 +709,14 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
// δkvX_k12R2 = tX.δxkv / keys12R2
δkv1_k12R2 := make(map[Key]Δstring, len(t1.δxkv))
δkv2_k12R2 := make(map[Key]Δstring, len(t2.δxkv))
δkv1_k12R2 := make(map[Key]Δstring, len(t1.Δxkv))
δkv2_k12R2 := make(map[Key]Δstring, len(t2.Δxkv))
for k := range keys12R2 {
δv1, ok := t1.δxkv[k]
δv1, ok := t1.Δxkv[k]
if ok {
δkv1_k12R2[k] = δv1
}
δv2, ok := t2.δxkv[k]
δv2, ok := t2.Δxkv[k]
if ok {
δkv2_k12R2[k] = δv2
}
......@@ -1147,15 +746,15 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
}
// xverifyΔBTail_rebuild_U verifies ΔBtail state after Update(ti->tj).
func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, treeRoot zodb.Oid, ti, tj *tTreeCommit, xat map[zodb.Tid]string, trackSet PPTreeSubSet, vδTok ...map[Key]Δstring) {
func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, treeRoot zodb.Oid, ti, tj *xbtreetest.Commit, xat map[zodb.Tid]string, trackSet blib.PPTreeSubSet, vδTok ...map[Key]Δstring) {
t.Helper()
X := exc.Raiseif
ø := PPTreeSubSet{}
ø := blib.PPTreeSubSet{}
subj := fmt.Sprintf("after Update(@%s→@%s)", xat[ti.at], xat[tj.at])
subj := fmt.Sprintf("after Update(@%s→@%s)", xat[ti.At], xat[tj.At])
// Update ati -> atj
δB, err := δbtail.Update(tj.δZ); X(err)
δB, err := δbtail.Update(tj.ΔZ); X(err)
δbtail.assertTrack(t, subj, trackSet, ø)
assertΔTtail(t, subj, δbtail, tj, treeRoot, xat, vδTok...)
......@@ -1180,8 +779,8 @@ func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, treeRoot zodb.Oid,
if ok {
δT = XGetδKV(ti, tj, δToid)
}
if δB.Rev != tj.at {
t.Errorf("%s: δB.Rev: have %s ; want %s", subj, δB.Rev, tj.at)
if δB.Rev != tj.At {
t.Errorf("%s: δB.Rev: have %s ; want %s", subj, δB.Rev, tj.At)
}
if len(δB.ΔByRoot) != δrootsOK {
t.Errorf("%s: len(δB.ΔByRoot) != %d ; δroots=%v", subj, δrootsOK, δroots)
......@@ -1192,14 +791,14 @@ func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, treeRoot zodb.Oid,
}
// xverifyΔBTail_rebuild_TR verifies ΔBtail state after Track(keys) + rebuild.
func xverifyΔBTail_rebuild_TR(t *testing.T, δbtail *ΔBtail, tj *tTreeCommit, treeRoot zodb.Oid, xat map[zodb.Tid]string, keys setKey, trackSet PPTreeSubSet, trackNew, trackSetAfterRebuild PPTreeSubSet, vδTok ...map[Key]Δstring) {
func xverifyΔBTail_rebuild_TR(t *testing.T, δbtail *ΔBtail, tj *xbtreetest.Commit, treeRoot zodb.Oid, xat map[zodb.Tid]string, keys setKey, trackSet blib.PPTreeSubSet, trackNew, trackSetAfterRebuild blib.PPTreeSubSet, vδTok ...map[Key]Δstring) {
t.Helper()
ø := PPTreeSubSet{}
ø := blib.PPTreeSubSet{}
// Track(keys)
xtrackKeys(δbtail, tj, keys)
subj := fmt.Sprintf("@%s: after Track%v", xat[tj.at], keys)
subj := fmt.Sprintf("@%s: after Track%v", xat[tj.At], keys)
δbtail.assertTrack(t, subj, trackSet, trackNew)
δbtail.rebuildAll()
......@@ -1215,23 +814,23 @@ func xverifyΔBTail_rebuild_TR(t *testing.T, δbtail *ΔBtail, tj *tTreeCommit,
// assertΔTtail verifies state of ΔTtail that corresponds to treeRoot in δbtail.
// it also verifies that δbtail.vδBroots matches ΔTtail data.
func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit, treeRoot zodb.Oid, xat map[zodb.Tid]string, vδTok ...map[Key]Δstring) {
func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *xbtreetest.Commit, treeRoot zodb.Oid, xat map[zodb.Tid]string, vδTok ...map[Key]Δstring) {
t.Helper()
// XXX +KVAtTail, +lastRevOf
l := len(vδTok)
var vatOK []zodb.Tid
var vδTok_ []map[Key]Δstring
at2t := map[zodb.Tid]*tTreeCommit{tj.at: tj}
at2t := map[zodb.Tid]*xbtreetest.Commit{tj.At: tj}
t0 := tj
for i := 0; i<l; i++ {
// empty vδTok entries means they should be absent in vδT
if δTok := vδTok[l-i-1]; len(δTok) != 0 {
vatOK = append([]zodb.Tid{t0.at}, vatOK...)
vatOK = append([]zodb.Tid{t0.At}, vatOK...)
vδTok_ = append([]map[Key]Δstring{δTok}, vδTok_...)
}
t0 = t0.prev
at2t[t0.at] = t0
t0 = t0.Prev
at2t[t0.At] = t0
}
vδTok = vδTok_
δTtail, ok := δbtail.vδTbyRoot[treeRoot]
......@@ -1243,7 +842,7 @@ func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit,
l = len(vδToid)
var vat []zodb.Tid
var vδT []map[Key]Δstring
atPrev := t0.at
atPrev := t0.At
for _, δToid := range vδToid {
vat = append(vat, δToid.Rev)
δT := XGetδKV(at2t[atPrev], at2t[δToid.Rev], δToid.ΔKV) // {} k -> δ(ZBlk(oid).data)
......@@ -1290,11 +889,11 @@ func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit,
// xtrackKeys issues δbtail.Track requests for tree[keys].
// XXX place
func xtrackKeys(δbtail *ΔBtail, t *tTreeCommit, keys setKey) {
func xtrackKeys(δbtail *ΔBtail, t *xbtreetest.Commit, keys setKey) {
X := exc.Raiseif
head := δbtail.Head()
if head != t.at {
panicf("BUG: δbtail.head: %s ; t.at: %s", head, t.at)
if head != t.At {
panicf("BUG: δbtail.head: %s ; t.at: %s", head, t.At)
}
for k := range keys {
......@@ -1308,7 +907,7 @@ func xtrackKeys(δbtail *ΔBtail, t *tTreeCommit, keys setKey) {
// continues to be tracked and all keys migrate to holes in the
// tracking set. By aligning initial state to the same as after
// T1->ø, we test what will happen on ø->T2.
b := t.xkv.Get(k)
b := t.Xkv.Get(k)
err := δbtail.track(k, b.Path()); X(err)
}
}
......@@ -1317,19 +916,19 @@ func xtrackKeys(δbtail *ΔBtail, t *tTreeCommit, keys setKey) {
// XXX
// XXX kill
/*
func ___xverifyΔBTail_GetAt(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt ...*tTreeCommit) {
subj := vt[0].tree
func ___xverifyΔBTail_GetAt(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt ...*xbtreetest.Commit) {
subj := vt[0].Tree
for _, t := range vt[1:] {
subj += "→" + t.tree
subj += "→" + t.Tree
}
t.Run(fmt.Sprintf("Get/%s", subj), func(t *testing.T) {
// tid -> "at_i"
xat := map[zodb.Tid]string{}
for i := range vt {
xat[vt[i].at] = fmt.Sprintf("at%d", i)
xat[vt[i].At] = fmt.Sprintf("at%d", i)
fmt.Printf("@%s: %v\n", xat[vt[i].at], vt[i].xkv.Flatten())
fmt.Printf("@%s: %v\n", xat[vt[i].At], vt[i].Xkv.Flatten())
}
tkeys := allTestKeys(vt...)
......@@ -1350,19 +949,19 @@ func ___xverifyΔBTail_GetAt(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt ..
})
}
func xverifyΔBTail_GetAt1(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt []*tTreeCommit, xat map[zodb.Tid]string, keys setKey) {
func xverifyΔBTail_GetAt1(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt []*xbtreetest.Commit, xat map[zodb.Tid]string, keys setKey) {
X := exc.Raiseif
// t1->t2-> ... -> tn
δbtail := NewΔBtail(vt[0].at, db)
δbtail := NewΔBtail(vt[0].At, db)
for i := 1; i < len(vt); i++ {
_, err := δbtail.Update(vt[i].δZ); X(err)
_, err := δbtail.Update(vt[i].ΔZ); X(err)
}
// Track(keys)
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: vt[len(vt)-1].at}); X(err)
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: vt[len(vt)-1].At}); X(err)
xtree, err := zconn.Get(ctx, treeRoot); X(err)
ztree := xtree.(*Tree)
......@@ -1373,21 +972,21 @@ func xverifyΔBTail_GetAt1(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt []*t
// verify GetAt(k, @at) for all keys and @at
for i := 1; i < len(vt); i++ {
at := vt[i].at
at := vt[i].At
for _, k := range keys.SortedElements() {
vOid, ok, rev, revExact, err := δbtail.GetAt(ctx, ztree, k, at); X(err)
v := xzgetBlkDataAt(db, vOid, rev)
v_, ok_ := vt[i].xkv.Get(k).kv[k]
rev_, revExact_ := vt[i].at, false
v_, ok_ := vt[i].Xkv.Get(k).kv[k]
rev_, revExact_ := vt[i].At, false
for j := i-1; j >= 0; j-- {
v__ := vt[j].xkv.Get(k).kv[k]
v__ := vt[j].Xkv.Get(k).kv[k]
if v__ != v_ {
rev_ = vt[j+1].at
rev_ = vt[j+1].At
revExact_ = true
break
}
rev_ = vt[j].at
rev_ = vt[j].At
}
if v == "" { v = DEL }
......@@ -1436,245 +1035,21 @@ func ΔBTest(xtest interface{}) ΔBTestEntry {
}
// tTreeEnv is tree-based testing environment.
//
// It combines TreeSrv and client side access to ZODB with committed trees.
// It should be created it via tNewTreeEnv().
type tTreeEnv struct {
*testing.T
work string // working directory
treeSrv *TreeSrv
zstor zodb.IStorage
db *zodb.DB
// all committed trees
commitv []*tTreeCommit
}
// tTreeCommit represent test commit changing a tree.
type tTreeCommit struct {
tree string // the tree in topology-encoding
prev *tTreeCommit // previous commit
at zodb.Tid // commit revision
δZ *zodb.EventCommit // raw ZODB changes; δZ.tid == at
xkv RBucketSet // full tree state as of @at
δxkv map[Key]Δstring // full tree-diff against parent
blkDataTab map[zodb.Oid]string // full snapshot of all ZBlk data @at XXX -> zblkDataTab
// δzblkData map[zodb.Oid]Δstring // full diff for zblkData against parent XXX ?
}
// tNewTreeEnv creates new tTreeEnv.
func tNewTreeEnv(t *testing.T) *tTreeEnv {
X := exc.Raiseif
t.Helper()
tt := &tTreeEnv{T: t}
var err error
work := t.TempDir()
tt.treeSrv, err = StartTreeSrv(work + "/1.fs"); X(err)
t.Cleanup(func() {
err := tt.treeSrv.Close(); X(err)
})
tt.zstor, err = zodb.Open(context.Background(), tt.treeSrv.zurl, &zodb.OpenOptions{
ReadOnly: true,
}); X(err)
t.Cleanup(func() {
err := tt.zstor.Close(); X(err)
})
tt.db = zodb.NewDB(tt.zstor, &zodb.DBOptions{
// We need objects to be cached, because otherwise it is too
// slow to run the test for many testcases, especially
// xverifyΔBTail_rebuild.
CacheControl: &tZODBCacheEverything{},
})
t.Cleanup(func() {
err := tt.db.Close(); X(err)
})
head := tt.treeSrv.head
t1 := &tTreeCommit{
tree: "T/B:", // treegen.py creates the tree as initially empty
prev: nil,
at: head,
xkv: XGetTree(tt.db, head, tt.Root()),
blkDataTab: xGetBlkDataTab(tt.db, head),
δZ: nil,
δxkv: nil,
}
tt.commitv = []*tTreeCommit{t1}
return tt
}
// tZODBCacheEverything is workaround for ZODB/go not implementing real
// live cache for now: Objects get dropped on PDeactivate if cache
// control does not say we need the object to stay in the cache.
// XXX place
type tZODBCacheEverything struct{}
func (_ *tZODBCacheEverything) PCacheClassify(_ zodb.IPersistent) zodb.PCachePolicy {
return zodb.PCachePinObject | zodb.PCacheKeepState
}
// Root returns OID of root tree node.
func (t *tTreeEnv) Root() zodb.Oid {
return t.treeSrv.treeRoot
}
// Head returns most-recently committed tree.
func (t *tTreeEnv) Head() *tTreeCommit {
return t.commitv[len(t.commitv)-1]
}
// CommitTree calls t.treeSrv.Commit and returns tTreeCommit corresponding to committed transaction.
func (t *tTreeEnv) CommitTree(tree string) *tTreeCommit {
// TODO X = FatalIf
X := exc.Raiseif
defer exc.Contextf("commit %s", tree)
watchq := make(chan zodb.Event)
at0 := t.zstor.AddWatch(watchq)
defer t.zstor.DelWatch(watchq)
tid, err := t.treeSrv.Commit(tree); X(err)
if !(tid > at0) {
exc.Raisef("treegen -> %s ; want > %s", tid, at0)
}
zevent := <-watchq
δZ := zevent.(*zodb.EventCommit)
if δZ.Tid != tid {
exc.Raisef("treegen -> %s ; watchq -> %s", tid, δZ)
}
// load tree structure from the db
// if the tree does not exist yet - report its structure as empty
var xkv RBucketSet
if tree != DEL {
xkv = XGetTree(t.db, δZ.Tid, t.Root())
} else {
// empty tree with real treeRoot as oid even though the tree is
// deleted. Having real oid in the root tests that after deletion,
// root of the tree stays in the tracking set. We need root to stay
// in trackSet because e.g. in
//
// T1 -> ø -> T2
//
// where the tree is first deleted, then recreated, without root
// staying in trackSet after ->ø, treediff will notice nothing when
// it comes to ->T2.
xkv = RBucketSet{
&RBucket{
oid: zodb.InvalidOid,
parent: &RTree{
oid: t.Root(), // NOTE oid is not InvalidOid
parent: nil,
},
lo: KeyMin,
hi_: KeyMax,
kv: map[Key]string{},
},
}
}
ttree := &tTreeCommit{
tree: tree,
at: δZ.Tid,
δZ: δZ,
xkv: xkv,
blkDataTab: xGetBlkDataTab(t.db, δZ.Tid),
}
tprev := t.Head()
ttree.prev = tprev
ttree.δxkv = kvdiff(tprev.xkv.Flatten(), ttree.xkv.Flatten())
t.commitv = append(t.commitv, ttree)
return ttree
}
// xGetBlkDataTab loads all ZBlk from db@at.
//
// it returns {} oid -> blkdata.
func xGetBlkDataTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]string {
defer exc.Contextf("%s: @%s: get blkdatatab", db.Storage().URL(), at)
X := exc.Raiseif
blkDataTab := map[zodb.Oid]string{}
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)
xzroot, err := zconn.Get(ctx, 0); X(err)
zroot, ok := xzroot.(*zodb.Map)
if !ok {
exc.Raisef("root: expected %s, got %s", xzodb.TypeOf(zroot), xzodb.TypeOf(xzroot))
}
err = zroot.PActivate(ctx); X(err)
defer zroot.PDeactivate()
xzblkdir, ok := zroot.Data["treegen/values"]
if !ok {
exc.Raisef("root['treegen/values'] missing")
}
zblkdir, ok := xzblkdir.(*zodb.Map)
if !ok {
exc.Raisef("root['treegen/values']: expected %s, got %s", xzodb.TypeOf(zblkdir), xzodb.TypeOf(xzblkdir))
}
err = zblkdir.PActivate(ctx); X(err)
defer zblkdir.PDeactivate()
for k, xzblk := range zblkdir.Data {
zblk, ok := xzblk.(zodb.IPersistent)
if !ok {
exc.Raisef("root['treegen/values'][%q]: expected %s, got %s", k, xzodb.TypeOf(zblk), xzodb.TypeOf(xzblk))
}
oid := zblk.POid()
data := xzgetBlkData(ctx, zconn, oid)
blkDataTab[oid] = data
}
return blkDataTab
}
// xgetBlkData loads blk data for ZBlk<oid> @t.at
//
// For speed the load is done via preloaded t.blkDataTab instead of access to the DB.
func (t *tTreeCommit) xgetBlkData(oid zodb.Oid) string {
if oid == VDEL {
return DEL
}
data, ok := t.blkDataTab[oid]
if !ok {
exc.Raisef("getBlkData ZBlk<%s> @%s: no such ZBlk", oid, t.at)
}
return data
}
// testΔBTail verifies ΔBTail on sequence of tree topologies coming from testq.
func testΔBTail(t_ *testing.T, testq chan ΔBTestEntry) {
t := tNewTreeEnv(t_)
t := xbtreetest.NewT(t_)
var t0 *tTreeCommit
var t0 *xbtreetest.Commit
for test := range testq {
t1 := t.Head()
t2 := t.CommitTree(test.tree)
subj := fmt.Sprintf("%s -> %s", t1.tree, t2.tree)
subj := fmt.Sprintf("%s -> %s", t1.Tree, t2.Tree)
//t.Logf("\n\n\n**** %s ****\n\n", subj)
// KAdj
if kadjOK := test.kadjOK; kadjOK != nil {
t.Run(fmt.Sprintf("KAdj/%s→%s", t1.tree, t2.tree), func(t *testing.T) {
t.Run(fmt.Sprintf("KAdj/%s→%s", t1.Tree, t2.Tree), func(t *testing.T) {
kadj := KAdj(t1, t2)
if !reflect.DeepEqual(kadj, kadjOK) {
t.Fatalf("BUG: computed kadj is wrong:\nkadjOK: %v\nkadj : %v\n\n", kadjOK, kadj)
......@@ -1684,12 +1059,12 @@ func testΔBTail(t_ *testing.T, testq chan ΔBTestEntry) {
// ΔBTail.Update
if test.flags & ΔBTest_SkipUpdate == 0 {
xverifyΔBTail_Update(t.T, subj, t.db, t.Root(), t1,t2)
xverifyΔBTail_Update(t.T, subj, t.DB, t.Root(), t1,t2)
}
// ΔBTail.rebuild
if t0 != nil && (test.flags & ΔBTest_SkipRebuild == 0) {
xverifyΔBTail_rebuild(t.T, t.db, t.Root(), t0,t1,t2)
xverifyΔBTail_rebuild(t.T, t.DB, t.Root(), t0,t1,t2)
}
t0, t1 = t1, t2
......@@ -1859,7 +1234,7 @@ func TestΔBTail(t *testing.T) {
// * `ø -> Tree ...` (tree is created anew),
// * `... Tree -> ø` (tree is deleted), and
// * `Tree -> ø -> Tree` (tree is deleted and then recreated)
DEL,
xbtreetest.DEL,
// tree rotation
"T3/B2:b-B3:c,4:d",
......@@ -2019,7 +1394,7 @@ func TestΔBTailAllStructs(t *testing.T) {
nkeys := N(3, 5, 10)
// server to generate AllStructs(kv, ...)
sg, err := StartAllStructsSrv(); X(err)
sg, err := xbtreetest.StartAllStructsSrv(); X(err)
defer func() {
err := sg.Close(); X(err)
}()
......@@ -2144,7 +1519,7 @@ func TestΔBTailAllStructs(t *testing.T) {
func TestΔBtailForget(t_ *testing.T) {
t := tNewTreeEnv(t_)
t := xbtreetest.NewT(t_)
X := exc.Raiseif
t0 := t.CommitTree("T/B:")
......@@ -2152,51 +1527,49 @@ func TestΔBtailForget(t_ *testing.T) {
t2 := t.CommitTree("T2/B1:a-B2:b")
t3 := t.CommitTree("T/B2:b")
δbtail := NewΔBtail(t0.at, t.db)
_, err := δbtail.Update(t1.δZ); X(err)
_, err = δbtail.Update(t2.δZ); X(err)
δbtail := NewΔBtail(t0.At, t.DB)
_, err := δbtail.Update(t1.ΔZ); X(err)
_, err = δbtail.Update(t2.ΔZ); X(err)
// start tracking. everything becomes tracked because t1's T/B1:a has [-∞,∞) coverage
// By starting tracking after t2 we verify vδBroots update in both Update and rebuild
_0 := setKey{}; _0.Add(0)
xtrackKeys(δbtail, t2, _0)
_, err = δbtail.Update(t3.δZ); X(err)
_, err = δbtail.Update(t3.ΔZ); X(err)
xat := map[zodb.Tid]string{
t0.at: "at0",
t1.at: "at1",
t2.at: "at2",
t3.at: "at3",
}
assertΔTtail(t.T, "init", δbtail, t3, t.Root(), xat, t1.δxkv, t2.δxkv, t3.δxkv)
δbtail.ForgetPast(t0.at)
assertΔTtail(t.T, "forget ≤ at0", δbtail, t3, t.Root(), xat, t1.δxkv, t2.δxkv, t3.δxkv)
δbtail.ForgetPast(t1.at)
assertΔTtail(t.T, "forget ≤ at1", δbtail, t3, t.Root(), xat, t2.δxkv, t3.δxkv)
δbtail.ForgetPast(t3.at)
t0.At: "at0",
t1.At: "at1",
t2.At: "at2",
t3.At: "at3",
}
assertΔTtail(t.T, "init", δbtail, t3, t.Root(), xat, t1.Δxkv, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t0.At)
assertΔTtail(t.T, "forget ≤ at0", δbtail, t3, t.Root(), xat, t1.Δxkv, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t1.At)
assertΔTtail(t.T, "forget ≤ at1", δbtail, t3, t.Root(), xat, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t3.At)
assertΔTtail(t.T, "forget ≤ at3", δbtail, t3, t.Root(), xat, )
}
// ---- misc ----
func TestΔBtailClone(t_ *testing.T) {
// ΔBtail.Clone had bug that aliased klon data to orig
t := tNewTreeEnv(t_)
t := xbtreetest.NewT(t_)
X := exc.Raiseif
t0 := t.CommitTree("T2/B1:a-B2:b")
t1 := t.CommitTree("T2/B1:c-B2:d")
δbtail := NewΔBtail(t0.at, t.db)
_, err := δbtail.Update(t1.δZ); X(err)
δbtail := NewΔBtail(t0.At, t.DB)
_, err := δbtail.Update(t1.ΔZ); X(err)
_2 := setKey{}; _2.Add(2)
xtrackKeys(δbtail, t1, _2)
err = δbtail.rebuildAll(); X(err)
xat := map[zodb.Tid]string{
t0.at: "at0",
t1.at: "at1",
t0.At: "at0",
t1.At: "at1",
}
δkv1_1 := map[Key]Δstring{2:{"b","d"}}
......@@ -2205,8 +1578,8 @@ func TestΔBtailClone(t_ *testing.T) {
assertΔTtail(t.T, "klon @at1", δbklon, t1, t.Root(), xat, δkv1_1)
t2 := t.CommitTree("T/B1:b,2:a")
_, err = δbtail.Update(t2.δZ); X(err)
xat[t2.at] = "at2"
_, err = δbtail.Update(t2.ΔZ); X(err)
xat[t2.At] = "at2"
δkv1_2 := map[Key]Δstring{1:{"a","c"}, 2:{"b","d"}}
δkv2_2 := map[Key]Δstring{1:{"c","b"}, 2:{"d","a"}}
......@@ -2215,6 +1588,9 @@ func TestΔBtailClone(t_ *testing.T) {
}
// -------- misc --------
// IntSets generates all sets of integers in range [0,N)
func IntSets(N int) chan []int {
ch := make(chan []int)
......@@ -2258,89 +1634,12 @@ func TestIntSets(t *testing.T) {
}
// kvdiff returns difference in between kv1 and kv2.
var DEL = "ø" // DEL means deletion
type Δstring struct {
Old string
New string
}
func kvdiff(kv1, kv2 map[Key]string) map[Key]Δstring {
delta := map[Key]Δstring{}
keys := setKey{}
for k := range kv1 { keys.Add(k) }
for k := range kv2 { keys.Add(k) }
for k := range keys {
v1, ok := kv1[k]
if !ok { v1 = DEL }
v2, ok := kv2[k]
if !ok { v2 = DEL }
if v1 != v2 {
delta[k] = Δstring{v1,v2}
}
}
return delta
}
func TestKVDiff(t *testing.T) {
kv1 := map[Key]string{1:"a", 3:"c", 4:"d"}
kv2 := map[Key]string{1:"b", 4:"d", 5:"e"}
got := kvdiff(kv1, kv2)
want := map[Key]Δstring{1:{"a","b"}, 3:{"c",DEL}, 5:{DEL,"e"}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("error:\ngot: %v\nwant: %v", got, want)
}
}
// kvtxt returns string representation of {} kv.
func kvtxt(kv map[Key]string) string {
if len(kv) == 0 {
return "ø"
}
keyv := []Key{}
for k := range kv { keyv = append(keyv, k) }
sort.Slice(keyv, func(i,j int) bool { return keyv[i] < keyv[j] })
sv := []string{}
for _, k := range keyv {
v := kv[k]
if strings.ContainsAny(v, " \n\t,:") {
panicf("[%v]=%q: invalid value", k, v)
}
sv = append(sv, fmt.Sprintf("%v:%s", k, v))
}
return strings.Join(sv, ",")
}
func TestKVTxt(t *testing.T) {
kv := map[Key]string{3:"hello", 1:"zzz", 4:"world"}
got := kvtxt(kv)
want := "1:zzz,3:hello,4:world"
if got != want {
t.Fatalf("error:\ngot: %q\nwant: %q", got, want)
}
}
// Flatten converts xkv with bucket structure into regular dict.
func (xkv RBucketSet) Flatten() map[Key]string {
kv := make(map[Key]string)
for _, b := range xkv {
for k,v := range b.kv {
kv[k] = v
}
}
return kv
}
// allTestKeys returns all keys from vt + ∞.
func allTestKeys(vt ...*tTreeCommit) setKey {
func allTestKeys(vt ...*xbtreetest.Commit) setKey {
allKeys := setKey{}; allKeys.Add(KeyMax) // ∞ simulating ZBigFile.Size() query
for _, t := range vt {
for _, b := range t.xkv {
for k := range b.kv {
for _, b := range t.Xkv {
for k := range b.KV {
allKeys.Add(k)
}
}
......@@ -2359,11 +1658,6 @@ func sortedKeys(kv map[Key]Δstring) []Key {
return keyv
}
func (b *RBucket) String() string {
return fmt.Sprintf("%sB%s{%s}", KeyRange{b.lo, b.hi_}, b.oid, kvtxt(b.kv))
}
// XXX place
func tidvEqual(av, bv []zodb.Tid) bool {
if len(av) != len(bv) {
......@@ -2401,34 +1695,3 @@ func δTEqual(δa, δb map[Key]Δstring) bool {
}
return true
}
// ----------------------------------------
// ZBlk-related functions are imported at runtime by δbtail_x_test
var (
ZTreeGetBlkData func(context.Context, *Tree, Key) (string, bool, []Node, error)
ZGetBlkData func(context.Context, *zodb.Connection, zodb.Oid) (string, error)
)
// xzgetBlkData loads block data from ZBlk object specified by its oid.
func xzgetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) string {
X := exc.Raiseif
if zblkOid == VDEL {
return DEL
}
data, err := ZGetBlkData(ctx, zconn, zblkOid); X(err)
return string(data)
}
// xzgetBlkDataAt loads block data from ZBlk object specified by oid@at.
func xzgetBlkDataAt(db *zodb.DB, zblkOid zodb.Oid, at zodb.Tid) string {
X := exc.Raiseif
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)
return xzgetBlkData(ctx, zconn, zblkOid)
}
wcfs/internal/xbtree/δbtail_x_test.go
View file @ c7f1e3c9
......@@ -18,85 +18,7 @@
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree_test
// ZBlk-related part of δbtail_test
import (
"context"
"fmt"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
_ "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest/init"
)
type Tree = xbtree.Tree
type Node = xbtree.Node
type Key = xbtree.Key
type ZBlk = zdata.ZBlk
// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
// XXX +return blkRevMax and use it ?
func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
path = []Node{}
xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
path = append(path, node)
})
if err != nil {
return nil, false, nil, err
}
if ok {
zblk, ok = xzblk.(ZBlk)
if !ok {
return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk)) // XXX errctx
}
}
return zblk, ok, path, nil
}
func init() {
xbtree.ZTreeGetBlkData = ZTreeGetBlkData
xbtree.ZGetBlkData = ZGetBlkData
}
// ZTreeGetBlkData returns block data from block pointed to by ztree[k].
func ZTreeGetBlkData(ctx context.Context, ztree *Tree, k Key) (data string, ok bool, path []Node, err error) {
defer xerr.Contextf(&err, "@%s: tree<%s>: get blkdata from [%d]", ztree.PJar().At(), ztree.POid(), k)
zblk, ok, path, err := ztreeGetBlk(ctx, ztree, k)
if err != nil || !ok {
return "", ok, path, err
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", false, nil, err
}
return string(bdata), true, path, nil
}
// ZGetBlkData loads block data from ZBlk object specified by its oid.
func ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
xblk, err := zconn.Get(ctx, zblkOid)
if err != nil {
return "", err
}
zblk, ok := xblk.(ZBlk)
if !ok {
return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", err
}
return string(bdata), nil
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7