Commit 52c72dbb authored by Kirill Smelkov's avatar Kirill Smelkov

X ΔBtail.rebuild started to work draftly

* t2: (50 commits)
  .
  X rebuild: Serial tests now started to PASS draftly
  .
  X rebuild: Don't return nil for empty ΔPPTreeSubSet - that leads to SIGSEGV
  X treediff: Fix BUG while computing AB coverage
  X ΔBtail.Clone had bug that it was aliasing klon and orig data
  X Fix rebuild with ø @at2
  X Fix bug in PPTreeSubSet.Difference  - it was always leaving root node alive
  .
  .
  X found why TestΔBTailAllStructs was not effective to find δtkeycov bugs
  .
  .
  X wcfs: assert that keycov only grow
  .
  X another bug in δtkeyconv computation
  .
  .
  .
  .
  ...
parents c144b4a4 8402f4c3
......@@ -136,6 +136,15 @@ func (a Set) Equal(b Set) bool {
return true
}
// Clone returns copy of the set.
func (orig Set) Clone() Set {
klon := make(Set, len(orig))
for v := range orig {
klon.Add(v)
}
return klon
}
// --------
func (s Set) SortedElements() []VALUE {
......
......@@ -138,6 +138,15 @@ func (a SetI64) Equal(b SetI64) bool {
return true
}
// Clone returns copy of the set.
func (orig SetI64) Clone() SetI64 {
klon := make(SetI64, len(orig))
for v := range orig {
klon.Add(v)
}
return klon
}
// --------
func (s SetI64) SortedElements() []int64 {
......
......@@ -138,6 +138,15 @@ func (a SetOid) Equal(b SetOid) bool {
return true
}
// Clone returns copy of the set.
func (orig SetOid) Clone() SetOid {
klon := make(SetOid, len(orig))
for v := range orig {
klon.Add(v)
}
return klon
}
// --------
func (s SetOid) SortedElements() []_Oid {
......
......@@ -27,6 +27,7 @@ import (
)
const tracePPSet = false
const debugPPSet = false
// PPTreeSubSet represents PP-connected subset of tree node objects.
//
......@@ -58,9 +59,13 @@ type PPTreeSubSet map[zodb.Oid]*nodeInTree
type nodeInTree struct {
parent zodb.Oid // parent node | InvalidOid for root
nchild int // number of direct children in PPTreeSubSet referring to this node
// XXX + [lo,hi) range this node is coming under in its parent XXX -> in its tree ?
}
// Has returns whether node is in the set.
func (S PPTreeSubSet) Has(oid zodb.Oid) bool {
_, ok := S[oid]
return ok
}
// Path returns path leading to the node specified by oid.
//
......@@ -84,6 +89,8 @@ func (S PPTreeSubSet) Path(oid zodb.Oid) (path []zodb.Oid) {
}
// AddPath adds path to a node to the set.
//
// Note: embedded buckets (leaf node with InvalidOid) are removed from the path.
func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
S.verify()
defer S.verify()
......@@ -93,18 +100,11 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
panic("empty path")
}
// don't keep track of artificial empty tree
if l == 1 && path[0] == zodb.InvalidOid {
return
}
// don't explicitly keep track of embedded buckets - they all have
// InvalidOid, and thus, if kept in S, e.g. T/B1:a and another
// T/B2:b would lead to InvalidOid having multiple parents.
if l == 2 && path[1] == zodb.InvalidOid {
path = path[:1]
}
// normalize path: remove embedded bucket and check whether it was an
// artificial empty tree.
path = normPath(path)
// go through path and add nodes to the set
parent := zodb.InvalidOid
var pt *nodeInTree = nil
for _, oid := range path {
......@@ -132,11 +132,33 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
}
}
// ---- Union/Difference/Intersetctior ----
// normPath normalizes path.
//
// It removes embedded buckets and artificial empty trees.
// Returned slice is subslice of path and aliases its memory.
func normPath(path []zodb.Oid) []zodb.Oid {
l := len(path)
// don't keep track of artificial empty tree
if l == 1 && path[0] == zodb.InvalidOid {
return nil
}
// don't explicitly keep track of embedded buckets - they all have
// InvalidOid, and thus, if kept in S, e.g. T/B1:a and another
// T/B2:b would lead to InvalidOid having multiple parents.
if l == 2 && path[1] == zodb.InvalidOid {
return path[:1]
}
return path
}
// ---- Union/Difference/Intersection ----
// Union returns U = PP(A.leafs | B.leafs)
//
// In other words it returns sum of A and B.
// In other words it adds A and B nodes.
func (A PPTreeSubSet) Union(B PPTreeSubSet) PPTreeSubSet {
U := A.Clone()
U.UnionInplace(B)
......@@ -163,7 +185,7 @@ func (A PPTreeSubSet) UnionInplace(B PPTreeSubSet) {
// Difference returns D = PP(A.leafs \ B.leafs)
//
// In other words ... XXX
// In other words it removes B nodes from A while still maintaining A as PP-connected.
func (A PPTreeSubSet) Difference(B PPTreeSubSet) PPTreeSubSet {
D := A.Clone()
D.DifferenceInplace(B)
......@@ -189,7 +211,7 @@ func (A PPTreeSubSet) DifferenceInplace(B PPTreeSubSet) {
A.xDifferenceInplace(B)
}
// XXX Intersection
// TODO Intersection
func (A PPTreeSubSet) xUnionInplace(B PPTreeSubSet) {
if tracePPSet {
......@@ -269,10 +291,16 @@ func (A PPTreeSubSet) fixup(δnchild map[zodb.Oid]int) {
A.xfixup(+1, δnchild)
}
func (A PPTreeSubSet) xfixup(sign int, δnchild map[zodb.Oid]int) {
//fmt.Printf("\nfixup:\n")
//fmt.Printf(" ·: %s\n", A)
//fmt.Printf(" δ: %v\n", δnchild)
//defer fmt.Printf(" ->·: %s\n\n", A)
if debugPPSet {
ssign := "+"
if sign < 0 {
ssign = "-"
}
fmt.Printf("\n fixup:\n")
fmt.Printf(" ·: %s\n", A)
fmt.Printf(" %sδ: %v\n", ssign, δnchild)
defer fmt.Printf(" ->·: %s\n\n", A)
}
gcq := []zodb.Oid{}
for oid, δnc := range δnchild {
......@@ -304,7 +332,7 @@ func (S PPTreeSubSet) gc1(oid zodb.Oid) {
for oid != zodb.InvalidOid {
t := S[oid]
t.nchild--
if t.nchild > 0 || /* root node */t.parent == zodb.InvalidOid {
if t.nchild > 0 {
break
}
delete(S, oid)
......@@ -395,6 +423,11 @@ func (A PPTreeSubSet) Equal(B PPTreeSubSet) bool {
return true
}
// Empty returns whether set is empty.
func (S PPTreeSubSet) Empty() bool {
return len(S) == 0
}
func (t nodeInTree) String() string {
return fmt.Sprintf("{p%s c%d}", t.parent, t.nchild)
}
......@@ -440,13 +473,22 @@ func (t nodeInTree) String() string {
// - δ.Add, and
// - xfixup(+1, δnchildNonLeafs)
//
// produce correctly PP-connected set.
// produces correctly PP-connected set.
type ΔPPTreeSubSet struct {
Del PPTreeSubSet
Add PPTreeSubSet
δnchildNonLeafs map[zodb.Oid]int
}
// NewΔPPTreeSubSet creates new empty ΔPPTreeSubSet.
func NewΔPPTreeSubSet() *ΔPPTreeSubSet {
return &ΔPPTreeSubSet{
Del: PPTreeSubSet{},
Add: PPTreeSubSet{},
δnchildNonLeafs: map[zodb.Oid]int{},
}
}
// Update updates δ to be combination of δ+δ2.
func (δ *ΔPPTreeSubSet) Update(δ2 *ΔPPTreeSubSet) {
δ.Del.UnionInplace(δ2.Del)
......
......@@ -20,6 +20,7 @@
package xbtree
import (
"strings"
"testing"
"lab.nexedi.com/kirr/neo/go/zodb"
......@@ -79,18 +80,43 @@ func TestPPTreeSubSetOps(t *testing.T) {
S{a:{ø,1}, b:{a,0}}, // B
S{a:{ø,1}, b:{a,0}}, // U
S{}), // D
E(
S{a:{ø,1}, b:{a,1}, c:{b,0}}, // A
S{a:{ø,1}, b:{a,1}, c:{b,0}}, // B (=A)
S{a:{ø,1}, b:{a,1}, c:{b,0}}, // U (=A)
S{}), // D
}
// assert1 asserts that result of op(A,B) == resOK.
assert1 := func(op string, A, B, res, resOK S) {
t.Helper()
if res.Equal(resOK) {
return
}
op1 := op[0:1]
t.Errorf("%s:\n A: %s\n B: %s\n ->%s: %s\n ok%s: %s\n",
strings.Title(op), A, B, op1, res, strings.ToUpper(op1), resOK)
}
for _, tt := range testv {
U := tt.A.Union(tt.B)
D := tt.A.Difference(tt.B)
Uab := tt.A.Union(tt.B)
Uba := tt.B.Union(tt.A)
Dab := tt.A.Difference(tt.B)
if !U.Equal(tt.Union) {
t.Errorf("Union:\n A: %s\n B: %s\n ->u: %s\n okU: %s\n", tt.A, tt.B, U, tt.Union)
}
if !D.Equal(tt.Difference) {
t.Errorf("Difference:\n A: %s\n B: %s\n ->d: %s\n okD: %s\n", tt.A, tt.B, D, tt.Difference)
}
assert1("union", tt.A, tt.B, Uab, tt.Union)
assert1("union", tt.B, tt.A, Uba, tt.Union)
assert1("difference", tt.A, tt.B, Dab, tt.Difference)
Uaa := tt.A.Union(tt.A)
Ubb := tt.B.Union(tt.B)
Daa := tt.A.Difference(tt.A)
Dbb := tt.B.Difference(tt.B)
assert1("union", tt.A, tt.A, Uaa, tt.A)
assert1("union", tt.B, tt.B, Ubb, tt.B)
assert1("difference", tt.A, tt.A, Daa, S{})
assert1("difference", tt.B, tt.B, Dbb, S{})
// XXX also verify U/D properties like (A+B)\B + (A+B)\A + (A^B) == (A+B) ?
}
......
......@@ -20,7 +20,7 @@
# See https://www.nexedi.com/licensing for rationale and options.
"""Program treegen provides infrastructure to generate ZODB BTree states.
It is used as helper for ΔBTree tests.
It is used as helper for ΔBtail tests.
The following subcommands are provided:
......@@ -319,6 +319,9 @@ def AllStructs(kv1txt, kv2txt, maxdepth, maxsplit, n, seed=None):
t1structv.insert(0, t1struct0)
t2structv.insert(0, t2struct0)
# XXX rework allstructs to accept only 1 kv and emit n structs for that one kv only
# -> iterate through the pairs/triplets in the caller (TestΔBTailAllStructs)
# emit topologies for tree1->tree2 and tree1<-tree2 transitions for all
# combinations of tree1 and tree2.
t12travel = list(bitravel2Way(t1structv, t2structv))
......
......@@ -159,7 +159,7 @@ func δZConnectTracked(δZv []zodb.Oid, T PPTreeSubSet) (δZTC SetOid, δtopsByR
// nodeInRange represents a Node coming under [lo, hi_] key range in its tree.
type nodeInRange struct {
prefix []zodb.Oid // path to this node goes via this objects
lo, hi_ Key // [lo, hi_] NOTE _not_ hi) not to overflow at ∞ XXX -> Range
lo, hi_ Key // [lo, hi_] NOTE _not_ hi) not to overflow at ∞ XXX -> keycov KeyRange?
node Node
done bool // whether this node was already taken into account while computing diff
}
......@@ -330,7 +330,7 @@ func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, t
defer xerr.Contextf(&err, "treediff %s..%s %s", zconnOld.At(), zconnNew.At(), root)
δT = map[Key]ΔValue{}
δtrack = &ΔPPTreeSubSet{Del: PPTreeSubSet{}, Add: PPTreeSubSet{}, δnchildNonLeafs: map[zodb.Oid]int{}}
δtrack = NewΔPPTreeSubSet()
δtkeycov = &RangedKeySet{}
tracefDiff("\ntreediff %s δtops: %v δZTC: %v\n", root, δtops, δZTC)
......@@ -429,7 +429,7 @@ func diffX(ctx context.Context, a, b Node, δZTC SetOid, trackSet PPTreeSubSet)
var δtrack *ΔPPTreeSubSet
δ, err := diffB(ctx, aB, bB)
if δ != nil {
δtrack = &ΔPPTreeSubSet{}
δtrack = NewΔPPTreeSubSet()
δtkeycov = &RangedKeySet{}
}
return δ, δtrack, δtkeycov, err
......@@ -445,7 +445,7 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet)
defer xerr.Contextf(&err, "diffT %s %s", xidOf(A), xidOf(B))
δ = map[Key]ΔValue{}
δtrack = &ΔPPTreeSubSet{Del: PPTreeSubSet{}, Add: PPTreeSubSet{}, δnchildNonLeafs: map[zodb.Oid]int{}}
δtrack = NewΔPPTreeSubSet()
δtkeycov = &RangedKeySet{}
defer func() {
tracefDiff(" -> δ: %v\n", δ)
......@@ -487,6 +487,9 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet)
ABcov:
for i := len(ABpath)-2; i >= 0; i-- {
xparent, err := node.PJar().Get(ctx, ABpath[i]); /*X*/if err != nil { return nil,nil,nil, err }
err = xparent.PActivate(ctx); /*X*/if err != nil { return nil,nil,nil, err}
defer xparent.PDeactivate()
parent := xparent.(*Tree) // must succeed
// find node in parent children and constrain ABlo/ABhi accordingly
entryv := parent.Entryv()
......@@ -512,7 +515,14 @@ ABcov:
}
}
panicf("BUG: T%s points to T%s as parent in trackSet, but not found in T%s children", node.POid(), parent.POid(), parent.POid())
emsg := fmt.Sprintf("BUG: T%s points to T%s as parent in trackSet, but not found in T%s children\n", node.POid(), parent.POid(), parent.POid())
children := []string{}
for _, entry := range entryv {
children = append(children, vnode(entry.Child()))
}
emsg += fmt.Sprintf("T%s children: %v\n", parent.POid(), children)
emsg += fmt.Sprintf("trackSet: %s\n", trackSet)
panic(emsg)
}
......@@ -565,10 +575,11 @@ ABcov:
}
}
// {} oid -> parent for all nodes in Bv: current and previously expanded - up till top B
BtrackSet := PPTreeSubSet{}
// {} oid -> nodeInRange for all nodes we've came through in Bv:
// current and previously expanded - up till top B.
BnodeIdx := map[zodb.Oid]*nodeInRange{}
if !Bempty {
BtrackSet.AddPath(ABpath)
BnodeIdx[ABoid] = btop
}
// δtkeycov will be = BAdd \ ADel
......@@ -579,7 +590,8 @@ ABcov:
// by default a node contributes to δ-
// a node ac does not contribute to δ- and can be skipped, if:
// - ac is not tracked, or
// - ac ∉ δZTC && ∃ bc from B: ac.oid == bc.oid (ac+ac.children were not changed, and ac stays in the tree)
// - ac ∉ δZTC && ∃ bc from B: ac.oid == bc.oid && ac.keycov == bc.keycov
// (ac+ac.children were not changed, ac stays in the tree with the same key range coverage)
Aq := []*nodeInRange{atop} // queue for A nodes that contribute to δ-
for len(Aq) > 0 {
debugfDiff("\n")
......@@ -606,7 +618,7 @@ ABcov:
ra.done = true
case *Tree:
// empty tree - only queue holes covered by it
// empty tree - queue holes covered by it
if len(a.Entryv()) == 0 {
ar := KeyRange{ra.lo, ra.hi_}
δtrack.Del.AddPath(ra.Path())
......@@ -632,7 +644,7 @@ ABcov:
// ( this does not give exact answer but should be a reasonable heuristic;
// the diff is the same if heuristic does not work and we
// look into and load more nodes to compute δ )
_, found := BtrackSet[acOid]
bc, found := BnodeIdx[acOid]
if !found {
for {
blo := Bv.Get(ac.lo)
......@@ -652,11 +664,12 @@ ABcov:
}
bchildren := Bv.Expand(blo)
for _, bc := range bchildren {
bcOid := bc.node.POid()
BtrackSet.AddPath(bc.Path())
if acOid == bcOid {
for _, bc_ := range bchildren {
bc_Oid := bc_.node.POid()
BnodeIdx[bc_Oid] = bc_
if acOid == bc_Oid {
found = true
bc = bc_
}
}
if found {
......@@ -665,29 +678,23 @@ ABcov:
}
}
if found {
// ac can be skipped
// adjust trackSet since path to the node could have changed
apath := trackSet.Path(acOid)
bpath := BtrackSet.Path(acOid)
if !pathEqual(apath, bpath) {
δtrack.Del.AddPath(apath)
δtrack.Add.AddPath(bpath)
if nc := at.nchild; nc != 0 {
δtrack.δnchildNonLeafs[acOid] = nc
// XXX debugDiff δtrack [) ...
} else {
// adjust δtkeycov only if it was leaf bucket
ar := KeyRange{ac.lo, ac.hi_}
bc := Bv.Get(ac.lo)
br := KeyRange{bc.lo, bc.hi_}
δtkeycovADel.AddRange(ar)
δtkeycovBAdd.AddRange(br)
debugfDiff(" δtrack - %s %v KKK\n", ar, apath)
debugfDiff(" δtrack + %s %v KKK\n", br, bpath)
// ac can be skipped if key coverage stays the same
ar := KeyRange{ac.lo, ac.hi_}
br := KeyRange{bc.lo, bc.hi_}
if ar == br {
// adjust trackSet since path to the node could have changed
apath := ac.Path()
bpath := bc.Path()
if !pathEqual(apath, bpath) {
δtrack.Del.AddPath(apath)
δtrack.Add.AddPath(bpath)
if nc := at.nchild; nc != 0 {
δtrack.δnchildNonLeafs[acOid] = nc
}
}
}
continue
continue
}
}
}
......@@ -751,11 +758,10 @@ ABcov:
b.done = true
}
// stop if r coverage is complete
// continue with next right bucket until r coverage is complete
if r.hi_ <= b.hi_ {
break
}
// continue with next right bucket
lo = b.hi_ + 1
}
}
......
This diff is collapsed.
This diff is collapsed.
......@@ -160,7 +160,7 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zb
// XXX blk = ∞ from beginning ?
blk = xbtree.KeyMax
}
err := δFtail.δBtail.Track(blk, zblk != nil, path)
err := δFtail.δBtail.Track(blk, path)
if err != nil {
panic(err) // XXX -> error? errctx
}
......@@ -233,7 +233,7 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
δF := ΔF{Rev: δB.Rev, ByFile: make(map[*ZBigFile]*ΔFile)}
// take btree changes into account
for root, δt := range δB.ByRoot {
for root, δt := range δB.ΔByRoot {
files := δFtail.fileIdx[root]
if len(files) == 0 {
panicf("BUG: ΔFtail: root<%s> -> ø files", root)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment