Skip to content

W
wendelin.core
Commit ef5e5183 authored by Kirill Smelkov's avatar Kirill Smelkov
4 Browse files
Options

X treediff ret += δtkeycov 

Hopefully rebuild will use this to correctly recompute trackSet
parent 1717400a
Hide whitespace changes
Inline Side-by-side
Showing with 87 additions and 51 deletions
wcfs/internal/xbtree/pptreesubset.go
View file @ ef5e5183
......@@ -54,10 +54,11 @@ const tracePPSet = false
type PPTreeSubSet map[zodb.Oid]*nodeInTree
// nodeInTree represents tracking information about a node.
// XXX -> treeNode ?
type nodeInTree struct {
parent zodb.Oid // parent node | InvalidOid for root
nchild int // number of direct children in PPTreeSubSet referring to this node
// XXX + [lo,hi) range this node is coming under in its parent
// XXX + [lo,hi) range this node is coming under in its parent XXX -> in its tree ?
}
......
wcfs/internal/xbtree/treediff.go
View file @ ef5e5183
......@@ -321,19 +321,20 @@ func (rs rangeSplit) String() string {
}
// treediff computes δT/δtrack for tree/trackSet specified by root in between old..new.
// treediff computes δT/δtrack/δtkeycov for tree/trackSet specified by root in between old..new.
//
// δtops is set of top nodes for changed subtrees.
// δZTC is connected(δZ/T) - connected closure for subset of δZ(old..new) that
// touches tracked nodes of T.
func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, trackSet PPTreeSubSet, zconnOld, zconnNew *zodb.Connection) (δT map[Key]ΔValue, δtrack *ΔPPTreeSubSet, err error) {
func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, trackSet PPTreeSubSet, zconnOld, zconnNew *zodb.Connection) (δT map[Key]ΔValue, δtrack *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
defer xerr.Contextf(&err, "treediff %s..%s %s", zconnOld.At(), zconnNew.At(), root)
δT = map[Key]ΔValue{}
δtrack = &ΔPPTreeSubSet{Del: PPTreeSubSet{}, Add: PPTreeSubSet{}, δnchildNonLeafs: map[zodb.Oid]int{}}
δtkeycov = &RangedKeySet{}
tracefDiff("\ntreediff %s δtops: %v δZTC: %v\n", root, δtops, δZTC)
defer tracefDiff("\n-> δT: %v\nδtrack: %v\n", δT, δtrack)
defer tracefDiff("\n-> δT: %v\nδtrack: %v\nδtkeycov: %v\n", δT, δtrack, δtkeycov)
δtrackv := []*ΔPPTreeSubSet{}
......@@ -342,12 +343,12 @@ func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, t
b, err2 := zgetNodeOrNil(ctx, zconnNew, top)
err := xerr.Merge(err1, err2)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
δtop, δtrackTop, err := diffX(ctx, a, b, δZTC, trackSet)
δtop, δtrackTop, δtkeycovTop, err := diffX(ctx, a, b, δZTC, trackSet)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
// FIXME -> merge (VDEL vs add)
......@@ -357,11 +358,13 @@ func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, t
// +k -> Tkextra -= k
debugfDiff("-> δtop: %v\n", δtop)
debugfDiff("-> δtrackTop: %v\n", δtrackTop)
debugfDiff("-> δtkeycovTop: %v\n", δtkeycovTop)
for k,δv := range δtop {
δT[k] = δv
}
δtrackv = append(δtrackv, δtrackTop)
δtkeycov.UnionInplace(δtkeycovTop)
}
// adjust trackSet by merge(δtrackTops)
......@@ -369,7 +372,7 @@ func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, t
δtrack.Update(δ)
}
return δT, δtrack, nil
return δT, δtrack, δtkeycov, nil
}
// diffX computes difference in between two revisions of a tree's subtree.
......@@ -383,7 +386,9 @@ func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, t
//
// δtrack is trackSet δ that needs to be applied to trackSet to keep it
// consistent with b (= a + δ).
func diffX(ctx context.Context, a, b Node, δZTC SetOid, trackSet PPTreeSubSet) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, err error) {
//
// δtkeycov represents how δtrack grows (always grows) tracking set key coverage.
func diffX(ctx context.Context, a, b Node, δZTC SetOid, trackSet PPTreeSubSet) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
if a==nil && b==nil {
panic("BUG: both a & b == nil") // XXX -> not a bug e.g. for `ø ø T` sequence?
}
......@@ -412,7 +417,7 @@ func diffX(ctx context.Context, a, b Node, δZTC SetOid, trackSet PPTreeSubSet)
panicf("BUG: a.oid != b.oid ; a: %s b: %s", a.POid(), b.POid())
}
if !((aT != nil && bT != nil) || (aB != nil && bB != nil)) {
return nil, nil, fmt.Errorf("object %s: type mutated %s -> %s", a.POid(),
return nil, nil, nil, fmt.Errorf("object %s: type mutated %s -> %s", a.POid(),
zodb.ClassOf(a), zodb.ClassOf(b))
}
}
......@@ -424,8 +429,9 @@ func diffX(ctx context.Context, a, b Node, δZTC SetOid, trackSet PPTreeSubSet)
δ, err := diffB(ctx, aB, bB)
if δ != nil {
δtrack = &ΔPPTreeSubSet{}
δtkeycov = &RangedKeySet{}
}
return δ, δtrack, err
return δ, δtrack, δtkeycov, err
}
}
......@@ -433,19 +439,21 @@ func diffX(ctx context.Context, a, b Node, δZTC SetOid, trackSet PPTreeSubSet)
//
// a, b point to top of subtrees @old and @new revisions.
// δZTC is connected set of objects covering δZT (objects changed in this tree in old..new).
func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, err error) {
func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
tracefDiff(" diffT %s %s\n", xidOf(A), xidOf(B))
defer xerr.Contextf(&err, "diffT %s %s", xidOf(A), xidOf(B))
δ = map[Key]ΔValue{}
δtrack = &ΔPPTreeSubSet{Del: PPTreeSubSet{}, Add: PPTreeSubSet{}, δnchildNonLeafs: map[zodb.Oid]int{}}
δtkeycov = &RangedKeySet{}
defer func() {
tracefDiff(" -> δ: %v\n", δ)
tracefDiff(" -> δtrack: %v\n", δtrack)
tracefDiff(" -> δtkeycov: %v\n", δtkeycov)
}()
if A == nil && B == nil {
return δ, δtrack, nil // ø changes
return δ, δtrack, δtkeycov, nil // ø changes
}
// assert A.oid == B.oid
......@@ -486,7 +494,8 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet)
}
// initial split ranges for A and B
// XXX maybe walk till a from root to get more precise initial range?
// FIXME walk till a from root to get more precise initial range
// XXX ^^^ get it from ABpath[-1].keyrange
prefix := ABpath[:len(ABpath)-1]
atop := &nodeInRange{prefix: prefix, lo: KeyMin, hi_: KeyMax, node: A} // [-∞, ∞)
btop := &nodeInRange{prefix: prefix, lo: KeyMin, hi_: KeyMax, node: B} // [-∞, ∞)
......@@ -535,15 +544,15 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet)
debugfDiff(" av: %s\n", Av)
debugfDiff(" bv: %s\n", Bv)
ra := pop(&Aq)
err = ra.node.PActivate(ctx); /*X*/if err != nil { return nil,nil, err }
err = ra.node.PActivate(ctx); /*X*/if err != nil { return nil,nil,nil, err }
defer ra.node.PDeactivate()
debugfDiff(" a: %s\n", ra)
switch a := ra.node.(type) {
case *Bucket:
// a is bucket -> δ-
δA, err := diffB(ctx, a, nil); /*X*/if err != nil { return nil,nil, err }
err = δMerge(δ, δA); /*X*/if err != nil { return nil,nil, err }
δA, err := diffB(ctx, a, nil); /*X*/if err != nil { return nil,nil,nil, err }
err = δMerge(δ, δA); /*X*/if err != nil { return nil,nil,nil, err }
δtrack.Del.AddPath(ra.Path())
// Bkqueue <- ra.range
......@@ -585,7 +594,7 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet)
if !ok {
break // bucket
}
err = bloT.PActivate(ctx); /*X*/if err != nil { return nil,nil, err }
err = bloT.PActivate(ctx); /*X*/if err != nil { return nil,nil,nil, err }
defer bloT.PDeactivate()
if len(bloT.Entryv()) == 0 {
......@@ -658,18 +667,18 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet)
for _, r := range Bkqueue.AllRanges() {
lo := r.lo
for {
b, err := Bv.GetToLeaf(ctx, lo); /*X*/if err != nil { return nil,nil, err }
b, err := Bv.GetToLeaf(ctx, lo); /*X*/if err != nil { return nil,nil,nil, err }
debugfDiff(" B k%d -> %s\n", lo, b)
// +bucket if that bucket is reached for the first time
if !b.done {
var δB map[Key]ΔValue
bbucket, ok := b.node.(*Bucket)
if ok { // !ok means ø tree
δB, err = diffB(ctx, nil, bbucket); /*X*/if err != nil { return nil,nil, err }
δB, err = diffB(ctx, nil, bbucket); /*X*/if err != nil { return nil,nil,nil, err }
}
// δ <- δB
err = δMerge(δ, δB); /*X*/if err != nil { return nil,nil, err }
err = δMerge(δ, δB); /*X*/if err != nil { return nil,nil,nil, err }
δtrack.Add.AddPath(b.Path())
// Akqueue <- δB
......@@ -695,18 +704,18 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet)
for _, r := range Akqueue.AllRanges() {
lo := r.lo
for {
a, err := Av.GetToLeaf(ctx, lo); /*X*/if err != nil { return nil,nil, err }
a, err := Av.GetToLeaf(ctx, lo); /*X*/if err != nil { return nil,nil,nil, err }
debugfDiff(" A k%d -> %s\n", lo, a)
// -bucket if that bucket is reached for the first time
if !a.done {
var δA map[Key]ΔValue
abucket, ok := a.node.(*Bucket)
if ok { // !ok means ø tree
δA, err = diffB(ctx, abucket, nil); /*X*/if err != nil { return nil,nil, err }
δA, err = diffB(ctx, abucket, nil); /*X*/if err != nil { return nil,nil,nil, err }
}
// δ <- δA
err = δMerge(δ, δA); /*X*/if err != nil { return nil,nil, err }
err = δMerge(δ, δA); /*X*/if err != nil { return nil,nil,nil, err }
δtrack.Del.AddPath(a.Path())
// Bkqueue <- a.range
......@@ -727,7 +736,7 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet)
Akqueue.Clear()
}
return δ, δtrack, nil
return δ, δtrack, δtkeycov, nil
}
......
wcfs/internal/xbtree/δbtail.go
View file @ ef5e5183
......@@ -37,8 +37,8 @@ import (
// TODO kill this after rebuild is finished
const XXX_killWhenRebuildWorks = true
const traceΔBtail = true
const debugΔBtail = true
const traceΔBtail = false
const debugΔBtail = false
// ΔBtail represents tail of revisional changes to BTrees.
......@@ -104,7 +104,7 @@ type ΔBtail struct {
// set of tracked nodes as of @head state.
trackSet PPTreeSubSet
// set of node that were requested to be tracked, but for which vδB was not yet rebuilt
// set of nodes that were requested to be tracked, but for which vδB was not yet rebuilt
trackNew PPTreeSubSet
}
......@@ -228,6 +228,8 @@ func (δBtail *ΔBtail) Track(key Key, keyPresent bool, nodePath []Node) error {
}
func (δBtail *ΔBtail) track(key Key, keyPresent bool, path []zodb.Oid) error {
// XXX locking
root := path[0]
δBtail.trackNew.AddPath(path)
......@@ -304,11 +306,14 @@ func (δBtail *ΔBtail) rebuild() (err error) {
for root, δtops := range δtopsByRoot {
// diff backwards curr -> prev
δT, δtrack, err := treediff(ctx, root, δtops, δZTC, trackNew, zconnCurr, zconnPrev)
δT, δtrack, δtkeycov, err := treediff(ctx, root, δtops, δZTC, trackNew, zconnCurr, zconnPrev)
if err != nil {
return err
}
// FIXME use δtkeycov to recompute track coverage
_ = δtkeycov
debugfΔBtail(" -> root<%s> δkv*: %v δtrack*: %v\n", root, δT, δtrack)
trackNew.ApplyΔ(δtrack)
......@@ -379,6 +384,15 @@ func (δBtail *ΔBtail) rebuild() (err error) {
//
// TODO optionally accept zconnOld/zconnNew from client
func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
δB, _, err := δBtail._Update(δZ)
return δB, err
}
// _Update serves Update and also returns δtkeycov from treediff for testing.
type _ΔTrackKeyCov struct {
ByRoot map[zodb.Oid]*RangedKeySet // {} root -> δtrackedKeys (always grow)
}
func (δBtail *ΔBtail) _Update(δZ *zodb.EventCommit) (_ ΔB, δTKeyCov _ΔTrackKeyCov, err error) {
headOld := δBtail.Head()
defer xerr.Contextf(&err, "ΔBtail update %s -> %s", headOld, δZ.Tid)
......@@ -386,6 +400,8 @@ func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
tracefΔBtail("trackSet: %v\n", δBtail.trackSet)
tracefΔBtail("trackNew: %v\n", δBtail.trackNew)
δTKeyCov = _ΔTrackKeyCov{ByRoot: make(map[zodb.Oid]*RangedKeySet)}
if XXX_killWhenRebuildWorks {
// XXX hack - until vvv is reenabled
δBtail.trackSet.UnionInplace(δBtail.trackNew)
......@@ -395,7 +411,7 @@ if XXX_killWhenRebuildWorks {
// update .trackSet and vδB from .trackNew
err = δBtail.rebuild()
if err != nil {
return ΔB{}, err
return ΔB{}, δTKeyCov, err
}
}
......@@ -409,7 +425,7 @@ if XXX_killWhenRebuildWorks {
// skip opening DB connections if there is no change to any tree node
if len(δtopsByRoot) == 0 {
return δB, nil
return δB, δTKeyCov, nil
}
// open ZODB connections corresponding to "old" and "new" states
......@@ -418,20 +434,20 @@ if XXX_killWhenRebuildWorks {
defer txn.Abort()
zconnOld, err := δBtail.db.Open(ctx, &zodb.ConnOptions{At: headOld})
if err != nil {
return ΔB{}, err
return ΔB{}, δTKeyCov, err
}
zconnNew, err := δBtail.db.Open(ctx, &zodb.ConnOptions{At: δZ.Tid})
if err != nil {
return ΔB{}, err
return ΔB{}, δTKeyCov, err
}
for root, δtops := range δtopsByRoot {
δT, δtrack, err := treediff(ctx, root, δtops, δZTC, δBtail.trackSet, zconnOld, zconnNew)
δT, δtrack, δtkeycov, err := treediff(ctx, root, δtops, δZTC, δBtail.trackSet, zconnOld, zconnNew)
if err != nil {
return ΔB{}, err
return ΔB{}, δTKeyCov, err
}
tracefΔBtail("\n-> root<%s> δkv: %v δtrack: %v\n", root, δT, δtrack)
tracefΔBtail("\n-> root<%s> δkv: %v δtrack: %v δtkeycov: %v\n", root, δT, δtrack, δtkeycov)
if len(δT) > 0 { // an object might be resaved without change
δB.ByRoot[root] = δT
......@@ -449,9 +465,10 @@ if XXX_killWhenRebuildWorks {
}
δBtail.trackSet.ApplyΔ(δtrack)
δTKeyCov.ByRoot[root] = δtkeycov
}
return δB, nil
return δB, δTKeyCov, nil
}
......
wcfs/internal/xbtree/δbtail_test.go
View file @ ef5e5183
......@@ -300,11 +300,19 @@ func (rbs RBucketSet) coverage() string {
return s
}
// trackSet returns what should be ΔBtree.trackSet for specified tracked key set.
// trackSet returns what should be ΔBtree.trackSet coverage for specified tracked key set.
func (rbs RBucketSet) trackSet(tracked SetKey) PPTreeSubSet {
trackSet := PPTreeSubSet{}
trackSet, _ := rbs.trackSetWithCov(tracked)
return trackSet
}
// trackSetWithCov returns what should be ΔBtree.trackSet and its key coverage for specified tracked key set.
func (rbs RBucketSet) trackSetWithCov(tracked SetKey) (trackSet PPTreeSubSet, keyCover *RangedKeySet) {
trackSet = PPTreeSubSet{}
keyCover = &RangedKeySet{}
for k := range tracked {
kb := rbs.Get(k)
keyCover.AddRange(KeyRange{kb.lo, kb.hi_})
// trackSet explicitly records only regular buckets.
// embedded buckets all have oid=zodb.InvalidOid and would lead to z
newNode := false
......@@ -345,7 +353,7 @@ func (rbs RBucketSet) trackSet(tracked SetKey) PPTreeSubSet {
p = p.parent
}
}
return trackSet
return trackSet, keyCover
}
......@@ -780,21 +788,18 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// verify δbtail.trackSet against @at1
// trackSet1 = xkv1[tracked1]
trackSet1 := xkv1.trackSet(initialTrackedKeys)
// if !reflect.DeepEqual(trackSet1, δbtail.trackSet) {
// badf("δbtail.trackSet1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, trackSet1)
// }
trackSet1, tkeyCov1 := xkv1.trackSetWithCov(initialTrackedKeys)
if !δbtail.trackSet.Equal(ø) {
badf("δbtail.trackSet1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, ø)
}
if !δbtail.trackNew.Equal(trackSet1) {
badf("δbtail.trackNew1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackNew, trackSet1)
}
// δbtail.assertTrack(t, "1", /*ø*/PPTreeSubSet{}, trackSet1)
// δbtail.assertTrack(t, "1", ø, trackSet1)
// δB <- δZ
δB, err := δbtail.Update(δZ); X(err)
δB, δTKeyCov, err := δbtail._Update(δZ); X(err)
if δB.Rev != δZ.Tid {
badf("δB: rev != δZ.Tid ; rev=%s δZ.Tid=%s", δB.Rev, δZ.Tid)
return
......@@ -802,17 +807,14 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// verify δbtail.trackSet against @at2
// trackSet2 = xkv2[tracked2] ( = xkv2[kadj[tracked1]]
trackSet2 := xkv2.trackSet(initialTrackedKeys.Union(kadjTrackedδZ))
// if !reflect.DeepEqual(trackSet2, δbtail.trackSet) {
// badf("δbtail.trackSet2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, trackSet2)
// }
trackSet2, tkeyCov2 := xkv2.trackSetWithCov(initialTrackedKeys.Union(kadjTrackedδZ))
if !δbtail.trackSet.Equal(trackSet2) {
badf("δbtail.trackSet2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, trackSet2)
}
if !δbtail.trackNew.Equal(ø) {
badf("δbtail.trackNew2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackNew, ø)
}
// δbtail.assertTrack(t, "2", trackSet2, /*ø*/PPTreeSubSet{})
// δbtail.assertTrack(t, "2", trackSet2, ø)
// assert δB.ByRoot == {treeRoot -> ...} if δTok != ø
......@@ -833,6 +835,13 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
return
}
// assert δtkeyCov == δ(tkeyCov1, tkeyCov2)
δtkeyCovOK := tkeyCov2.Difference(tkeyCov1)
δtkeyCov := δTKeyCov.ByRoot[treeRoot]
if !δtkeyCov.Equal(δtkeyCovOK) {
badf("δtkeyCov wrong:\nhave: %s\nwant: %s", δtkeyCov, δtkeyCovOK)
}
// δT <- δB
δToid := δB.ByRoot[treeRoot] // {} k -> δoid
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7