Commit 37c2e806 authored by Kirill Smelkov's avatar Kirill Smelkov

X wcfs: Teach treediff to compute not only δtrack (set of nodes), but also δ for track-key coverage

This will, hopefully, be used to fix rebuild.

* t2:
  .
  .
  .
  .
  .
  .
  .
  X treediff ret += δtkeycov
  .
  .
parents 2c5f6517 175f92c9
......@@ -54,10 +54,11 @@ const tracePPSet = false
type PPTreeSubSet map[zodb.Oid]*nodeInTree
// nodeInTree represents tracking information about a node.
// XXX -> treeNode ?
type nodeInTree struct {
parent zodb.Oid // parent node | InvalidOid for root
nchild int // number of direct children in PPTreeSubSet referring to this node
// XXX + [lo,hi) range this node is coming under in its parent
// XXX + [lo,hi) range this node is coming under in its parent XXX -> in its tree ?
}
......
......@@ -398,11 +398,13 @@ func (S RangedKeySet) String() string {
}
func (r KeyRange) String() string {
hi := r.hi_
if hi < KeyMax {
hi += 1
var shi string
if r.hi_ == KeyMax {
shi = kstr(r.hi_) // ∞
} else {
shi = fmt.Sprintf("%d", r.hi_+1)
}
return fmt.Sprintf("[%s,%s)", kstr(r.lo), kstr(hi))
return fmt.Sprintf("[%s,%s)", kstr(r.lo), shi)
}
......
This diff is collapsed.
......@@ -36,13 +36,6 @@ type SetOid = set.SetOid
const debug = false
func tracef(format string, argv ...interface{}) {
if debug {
fmt.Printf(format, argv...)
}
}
// nodePathToPath converts path from []Node to []Oid.
// XXX place = ?
func nodePathToPath(nodePath []Node) (path []zodb.Oid) {
......
......@@ -22,6 +22,7 @@ package xbtree
import (
"context"
"fmt"
"sort"
"strings"
......@@ -36,6 +37,10 @@ import (
// TODO kill this after rebuild is finished
const XXX_killWhenRebuildWorks = true
const traceΔBtail = false
const debugΔBtail = false
// ΔBtail represents tail of revisional changes to BTrees.
//
// It semantically consists of
......@@ -99,7 +104,7 @@ type ΔBtail struct {
// set of tracked nodes as of @head state.
trackSet PPTreeSubSet
// set of node that were requested to be tracked, but for which vδB was not yet rebuilt
// set of nodes that were requested to be tracked, but for which vδB was not yet rebuilt
trackNew PPTreeSubSet
}
......@@ -217,18 +222,19 @@ func (δBtail *ΔBtail) Track(key Key, keyPresent bool, nodePath []Node) error {
pathv := []string{}
for _, node := range nodePath { pathv = append(pathv, vnode(node)) }
tracef("Track [%v] %s\n", key, strings.Join(pathv, " -> "))
tracefΔBtail("\nTrack [%v] %s\n", key, strings.Join(pathv, " -> "))
return δBtail.track(key, keyPresent, path)
}
func (δBtail *ΔBtail) track(key Key, keyPresent bool, path []zodb.Oid) error {
// XXX locking
root := path[0]
δBtail.trackNew.AddPath(path)
// track is track of path[-1] (i.e. leaf)
// XXX hack - until rebuild is implemented
if XXX_killWhenRebuildWorks {
_, ok := δBtail.byRoot[root]
......@@ -248,6 +254,10 @@ func (δBtail *ΔBtail) rebuild() (err error) {
defer xerr.Context(&err, "ΔBtail rebuild")
// XXX locking
tracefΔBtail("\nRebuild @%s .. @%s\n", δBtail.Tail(), δBtail.Head())
tracefΔBtail("trackSet: %v\n", δBtail.trackSet)
tracefΔBtail("trackNew: %v\n", δBtail.trackNew)
trackNew := δBtail.trackNew
δBtail.trackNew = PPTreeSubSet{}
......@@ -272,11 +282,11 @@ func (δBtail *ΔBtail) rebuild() (err error) {
δZTC, δtopsByRoot := δZConnectTracked(δZ.Changev, trackNew)
tracef("\nrebuild @%s <- @%s\n", atPrev, δZ.Rev)
tracef("δZ:\t%v\n", δZ.Changev)
tracef("trackNew: %v\n", trackNew)
tracef("trackSet: %v\n", δBtail.trackSet) // XXX needed?
defer tracef("\n\n")
debugfΔBtail("\n rebuild @%s <- @%s\n", atPrev, δZ.Rev)
debugfΔBtail(" δZ:\t%v\n", δZ.Changev)
debugfΔBtail(" trackNew: %v\n", trackNew)
debugfΔBtail(" trackSet: %v\n", δBtail.trackSet) // XXX needed?
defer debugfΔBtail("\n\n")
// XXX len(δtopsByRoot) == 0 -> skip
......@@ -296,12 +306,15 @@ func (δBtail *ΔBtail) rebuild() (err error) {
for root, δtops := range δtopsByRoot {
// diff backwards curr -> prev
δT, δtrack, err := treediff(ctx, root, δtops, δZTC, trackNew, zconnCurr, zconnPrev)
δT, δtrack, δtkeycov, err := treediff(ctx, root, δtops, δZTC, trackNew, zconnCurr, zconnPrev)
if err != nil {
return err
}
tracef("-> root<%s> δkv*: %v δtrack*: %v\n", root, δT, δtrack)
// FIXME use δtkeycov to recompute track coverage
_ = δtkeycov
debugfΔBtail(" -> root<%s> δkv*: %v δtrack*: %v\n", root, δT, δtrack)
trackNew.ApplyΔ(δtrack)
vδtrack = append([]*ΔPPTreeSubSet{δtrack}, vδtrack...)
......@@ -371,9 +384,24 @@ func (δBtail *ΔBtail) rebuild() (err error) {
//
// TODO optionally accept zconnOld/zconnNew from client
func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
δB, _, err := δBtail._Update(δZ)
return δB, err
}
// _Update serves Update and also returns δtkeycov from treediff for testing.
type _ΔTrackKeyCov struct {
ByRoot map[zodb.Oid]*RangedKeySet // {} root -> δtrackedKeys (always grow)
}
func (δBtail *ΔBtail) _Update(δZ *zodb.EventCommit) (_ ΔB, δTKeyCov _ΔTrackKeyCov, err error) {
headOld := δBtail.Head()
defer xerr.Contextf(&err, "ΔBtail update %s -> %s", headOld, δZ.Tid)
tracefΔBtail("\nUpdate @%s -> @%s δZ: %v\n", δBtail.Head(), δZ.Tid, δZ.Changev)
tracefΔBtail("trackSet: %v\n", δBtail.trackSet)
tracefΔBtail("trackNew: %v\n", δBtail.trackNew)
δTKeyCov = _ΔTrackKeyCov{ByRoot: make(map[zodb.Oid]*RangedKeySet)}
if XXX_killWhenRebuildWorks {
// XXX hack - until vvv is reenabled
δBtail.trackSet.UnionInplace(δBtail.trackNew)
......@@ -383,14 +411,10 @@ if XXX_killWhenRebuildWorks {
// update .trackSet and vδB from .trackNew
err = δBtail.rebuild()
if err != nil {
return ΔB{}, err
return ΔB{}, δTKeyCov, err
}
}
tracef("Update @%s -> @%s\n", δBtail.Head(), δZ.Tid)
tracef("δZ:\t%v\n", δZ.Changev)
tracef("trackSet: %v\n", δBtail.trackSet)
// XXX dup wrt rebuild?
δBtail.δZtail.Append(δZ.Tid, δZ.Changev)
......@@ -401,7 +425,7 @@ if XXX_killWhenRebuildWorks {
// skip opening DB connections if there is no change to any tree node
if len(δtopsByRoot) == 0 {
return δB, nil
return δB, δTKeyCov, nil
}
// open ZODB connections corresponding to "old" and "new" states
......@@ -410,20 +434,20 @@ if XXX_killWhenRebuildWorks {
defer txn.Abort()
zconnOld, err := δBtail.db.Open(ctx, &zodb.ConnOptions{At: headOld})
if err != nil {
return ΔB{}, err
return ΔB{}, δTKeyCov, err
}
zconnNew, err := δBtail.db.Open(ctx, &zodb.ConnOptions{At: δZ.Tid})
if err != nil {
return ΔB{}, err
return ΔB{}, δTKeyCov, err
}
for root, δtops := range δtopsByRoot {
δT, δtrack, err := treediff(ctx, root, δtops, δZTC, δBtail.trackSet, zconnOld, zconnNew)
δT, δtrack, δtkeycov, err := treediff(ctx, root, δtops, δZTC, δBtail.trackSet, zconnOld, zconnNew)
if err != nil {
return ΔB{}, err
return ΔB{}, δTKeyCov, err
}
tracef("\n-> root<%s> δkv: %v δtrack: %v\n", root, δT, δtrack)
tracefΔBtail("\n-> root<%s> δkv: %v δtrack: %v δtkeycov: %v\n", root, δT, δtrack, δtkeycov)
if len(δT) > 0 { // an object might be resaved without change
δB.ByRoot[root] = δT
......@@ -441,9 +465,10 @@ if XXX_killWhenRebuildWorks {
}
δBtail.trackSet.ApplyΔ(δtrack)
δTKeyCov.ByRoot[root] = δtkeycov
}
return δB, nil
return δB, δTKeyCov, nil
}
......@@ -563,3 +588,16 @@ func (δBtail *ΔBtail) SliceByRootRev(root zodb.Oid, lo, hi zodb.Tid) /*readonl
func (δBtail *ΔBtail) ΔZtail() /*readonly*/*zodb.ΔTail {
return δBtail.δZtail
}
func tracefΔBtail(format string, argv ...interface{}) {
if traceΔBtail {
fmt.Printf(format, argv...)
}
}
func debugfΔBtail(format string, argv ...interface{}) {
if debugΔBtail {
fmt.Printf(format, argv...)
}
}
......@@ -300,11 +300,19 @@ func (rbs RBucketSet) coverage() string {
return s
}
// trackSet returns what should be ΔBtree.trackSet for specified tracked key set.
// trackSet returns what should be ΔBtree.trackSet coverage for specified tracked key set.
func (rbs RBucketSet) trackSet(tracked SetKey) PPTreeSubSet {
trackSet := PPTreeSubSet{}
trackSet, _ := rbs.trackSetWithCov(tracked)
return trackSet
}
// trackSetWithCov returns what should be ΔBtree.trackSet and its key coverage for specified tracked key set.
func (rbs RBucketSet) trackSetWithCov(tracked SetKey) (trackSet PPTreeSubSet, keyCover *RangedKeySet) {
trackSet = PPTreeSubSet{}
keyCover = &RangedKeySet{}
for k := range tracked {
kb := rbs.Get(k)
keyCover.AddRange(KeyRange{kb.lo, kb.hi_})
// trackSet explicitly records only regular buckets.
// embedded buckets all have oid=zodb.InvalidOid and would lead to z
newNode := false
......@@ -345,7 +353,7 @@ func (rbs RBucketSet) trackSet(tracked SetKey) PPTreeSubSet {
p = p.parent
}
}
return trackSet
return trackSet, keyCover
}
......@@ -666,7 +674,7 @@ func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb
// tracked state defined by initialTrackedKeys.
func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, at1,at2 zodb.Tid, xkv1,xkv2 RBucketSet, δZ *zodb.EventCommit, d12 map[Key]Δstring, initialTrackedKeys SetKey, kadj KAdjMatrix) {
X := exc.Raiseif
tracef("\n>>> Track=%s\n", initialTrackedKeys)
// t.Logf("\n>>> Track=%s\n", initialTrackedKeys)
var TrackedδZ SetKey = nil
var kadjTrackedδZ SetKey = nil
......@@ -780,21 +788,18 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// verify δbtail.trackSet against @at1
// trackSet1 = xkv1[tracked1]
trackSet1 := xkv1.trackSet(initialTrackedKeys)
// if !reflect.DeepEqual(trackSet1, δbtail.trackSet) {
// badf("δbtail.trackSet1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, trackSet1)
// }
trackSet1, tkeyCov1 := xkv1.trackSetWithCov(initialTrackedKeys)
if !δbtail.trackSet.Equal(ø) {
badf("δbtail.trackSet1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, ø)
}
if !δbtail.trackNew.Equal(trackSet1) {
badf("δbtail.trackNew1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackNew, trackSet1)
}
// δbtail.assertTrack(t, "1", /*ø*/PPTreeSubSet{}, trackSet1)
// δbtail.assertTrack(t, "1", ø, trackSet1)
// δB <- δZ
δB, err := δbtail.Update(δZ); X(err)
δB, δTKeyCov, err := δbtail._Update(δZ); X(err)
if δB.Rev != δZ.Tid {
badf("δB: rev != δZ.Tid ; rev=%s δZ.Tid=%s", δB.Rev, δZ.Tid)
return
......@@ -802,17 +807,14 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// verify δbtail.trackSet against @at2
// trackSet2 = xkv2[tracked2] ( = xkv2[kadj[tracked1]]
trackSet2 := xkv2.trackSet(initialTrackedKeys.Union(kadjTrackedδZ))
// if !reflect.DeepEqual(trackSet2, δbtail.trackSet) {
// badf("δbtail.trackSet2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, trackSet2)
// }
trackSet2, tkeyCov2 := xkv2.trackSetWithCov(initialTrackedKeys.Union(kadjTrackedδZ))
if !δbtail.trackSet.Equal(trackSet2) {
badf("δbtail.trackSet2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, trackSet2)
}
if !δbtail.trackNew.Equal(ø) {
badf("δbtail.trackNew2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackNew, ø)
}
// δbtail.assertTrack(t, "2", trackSet2, /*ø*/PPTreeSubSet{})
// δbtail.assertTrack(t, "2", trackSet2, ø)
// assert δB.ByRoot == {treeRoot -> ...} if δTok != ø
......@@ -833,6 +835,13 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
return
}
// assert δtkeycov == δ(tkeyCov1, tkeyCov2)
δtkeycovOK := tkeyCov2.Difference(tkeyCov1)
δtkeycov := δTKeyCov.ByRoot[treeRoot]
if !δtkeycov.Equal(δtkeycovOK) {
badf("δtkeycov wrong:\nhave: %s\nwant: %s", δtkeycov, δtkeycovOK)
}
// δT <- δB
δToid := δB.ByRoot[treeRoot] // {} k -> δoid
......@@ -1354,7 +1363,7 @@ func testΔBTail(t *testing.T, testq chan ΔBTestEntry) {
t2.prev = t1 // XXX ----//----
subj := fmt.Sprintf("%s -> %s", t1.tree, t2.tree)
tracef("\n\n\n**** %s ****\n\n", subj)
// t.Logf("\n\n\n**** %s ****\n\n", subj)
// KAdj
if kadjOK := test.kadjOK; kadjOK != nil {
......@@ -1938,12 +1947,7 @@ func sortedKeys(kv map[Key]Δstring) []Key {
}
func (b *RBucket) String() string {
// XXX dup wrt nodeInRange.String
hi := b.hi_
if hi < KeyMax {
hi += 1
}
return fmt.Sprintf("[%s,%s)B%s{%s}", kstr(b.lo), kstr(hi), b.oid, kvtxt(b.kv))
return fmt.Sprintf("%sB%s{%s}", KeyRange{b.lo, b.hi_}, b.oid, kvtxt(b.kv))
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment