Skip to content

W
wendelin.core
Commit 11c95175 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

.

parent 042c5eeb
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 296 additions and 193 deletions
wcfs/internal/xbtree/blib/pptreesubset.go
View file @ 11c95175
......@@ -102,7 +102,7 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
// normalize path: remove embedded bucket and check whether it was an
// artificial empty tree.
path = normPath(path)
path = NormPath(path)
// go through path and add nodes to the set
parent := zodb.InvalidOid
......@@ -132,11 +132,11 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
}
}
// normPath normalizes path.
// NormPath normalizes path.
//
// It removes embedded buckets and artificial empty trees.
// Returned slice is subslice of path and aliases its memory.
func normPath(path []zodb.Oid) []zodb.Oid {
func NormPath(path []zodb.Oid) []zodb.Oid {
l := len(path)
// don't keep track of artificial empty tree
......
wcfs/internal/xbtree/treediff.go
View file @ 11c95175
......@@ -80,6 +80,7 @@ import (
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
)
const traceDiff = false
......@@ -113,7 +114,7 @@ func (δv ΔValue) String() string {
// for example for e.g. t₀->t₁->b₂ if δZ/T={t₀ b₂} -> δZ/TC=δZ/T+{t₁}
//
// δtopsByRoot = {} root -> {top changed nodes in that tree}
func δZConnectTracked(δZv []zodb.Oid, T PPTreeSubSet) (δZTC setOid, δtopsByRoot map[zodb.Oid]setOid) {
func δZConnectTracked(δZv []zodb.Oid, T blib.PPTreeSubSet) (δZTC setOid, δtopsByRoot map[zodb.Oid]setOid) {
δZ := setOid{}; for _, δ := range δZv { δZ.Add(δ) }
δZTC = setOid{}
δtopsByRoot = map[zodb.Oid]setOid{}
......@@ -338,18 +339,18 @@ func (rs rangeSplit) String() string {
// δtops is set of top nodes for changed subtrees.
// δZTC is connected(δZ/T) - connected closure for subset of δZ(old..new) that
// touches tracked nodes of T.
func treediff(ctx context.Context, root zodb.Oid, δtops setOid, δZTC setOid, trackSet PPTreeSubSet, zconnOld, zconnNew *zodb.Connection) (δT map[Key]ΔValue, δtrack *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
func treediff(ctx context.Context, root zodb.Oid, δtops setOid, δZTC setOid, trackSet blib.PPTreeSubSet, zconnOld, zconnNew *zodb.Connection) (δT map[Key]ΔValue, δtrack *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, err error) {
defer xerr.Contextf(&err, "treediff %s..%s %s", zconnOld.At(), zconnNew.At(), root)
δT = map[Key]ΔValue{}
δtrack = NewΔPPTreeSubSet()
δtkeycov = &RangedKeySet{}
δtrack = blib.NewΔPPTreeSubSet()
δtkeycov = &blib.RangedKeySet{}
tracefDiff("\ntreediff %s δtops: %v δZTC: %v\n", root, δtops, δZTC)
tracefDiff(" trackSet: %v\n", trackSet)
defer tracefDiff("\n-> δT: %v\nδtrack: %v\nδtkeycov: %v\n", δT, δtrack, δtkeycov)
δtrackv := []*ΔPPTreeSubSet{}
δtrackv := []*blib.ΔPPTreeSubSet{}
for top := range δtops { // XXX -> sorted?
a, err1 := zgetNodeOrNil(ctx, zconnOld, top)
......@@ -401,7 +402,7 @@ func treediff(ctx context.Context, root zodb.Oid, δtops setOid, δZTC setOid, t
// consistent with b (= a + δ).
//
// δtkeycov represents how δtrack grows (always grows) tracking set key coverage.
func diffX(ctx context.Context, a, b Node, δZTC setOid, trackSet PPTreeSubSet) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
func diffX(ctx context.Context, a, b Node, δZTC setOid, trackSet blib.PPTreeSubSet) (δ map[Key]ΔValue, δtrack *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, err error) {
if a==nil && b==nil {
panic("BUG: both a & b == nil") // XXX -> not a bug e.g. for `ø ø T` sequence?
}
......@@ -438,11 +439,11 @@ func diffX(ctx context.Context, a, b Node, δZTC setOid, trackSet PPTreeSubSet)
if isT {
return diffT(ctx, aT, bT, δZTC, trackSet)
} else {
var δtrack *ΔPPTreeSubSet
var δtrack *blib.ΔPPTreeSubSet
δ, err := diffB(ctx, aB, bB)
if δ != nil {
δtrack = NewΔPPTreeSubSet()
δtkeycov = &RangedKeySet{}
δtrack = blib.NewΔPPTreeSubSet()
δtkeycov = &blib.RangedKeySet{}
}
return δ, δtrack, δtkeycov, err
}
......@@ -452,13 +453,13 @@ func diffX(ctx context.Context, a, b Node, δZTC setOid, trackSet PPTreeSubSet)
//
// a, b point to top of subtrees @old and @new revisions.
// δZTC is connected set of objects covering δZT (objects changed in this tree in old..new).
func diffT(ctx context.Context, A, B *Tree, δZTC setOid, trackSet PPTreeSubSet) (δ map[Key]ΔValue, δtrack *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
func diffT(ctx context.Context, A, B *Tree, δZTC setOid, trackSet blib.PPTreeSubSet) (δ map[Key]ΔValue, δtrack *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, err error) {
tracefDiff(" diffT %s %s\n", xidOf(A), xidOf(B))
defer xerr.Contextf(&err, "diffT %s %s", xidOf(A), xidOf(B))
δ = map[Key]ΔValue{}
δtrack = NewΔPPTreeSubSet()
δtkeycov = &RangedKeySet{}
δtrack = blib.NewΔPPTreeSubSet()
δtkeycov = &blib.RangedKeySet{}
defer func() {
tracefDiff(" -> δ: %v\n", δ)
tracefDiff(" -> δtrack: %v\n", δtrack)
......@@ -564,22 +565,22 @@ ABcov:
Bv := rangeSplit{btop} // nodes expanded from B
// for phase 2:
Akqueue := &RangedKeySet{} // queue for keys in A to be processed for δ-
Bkqueue := &RangedKeySet{} // ----//---- in B for δ+
Akdone := &RangedKeySet{} // already processed keys in A
Bkdone := &RangedKeySet{} // ----//---- in B
Aktodo := func(r KeyRange) {
Akqueue := &blib.RangedKeySet{} // queue for keys in A to be processed for δ-
Bkqueue := &blib.RangedKeySet{} // ----//---- in B for δ+
Akdone := &blib.RangedKeySet{} // already processed keys in A
Bkdone := &blib.RangedKeySet{} // ----//---- in B
Aktodo := func(r blib.KeyRange) {
if !Akdone.HasRange(r) {
δtodo := &RangedKeySet{}
δtodo := &blib.RangedKeySet{}
δtodo.AddRange(r)
δtodo.DifferenceInplace(Akdone)
debugfDiff(" Akq <- %s\n", δtodo)
Akqueue.UnionInplace(δtodo)
}
}
Bktodo := func(r KeyRange) {
Bktodo := func(r blib.KeyRange) {
if !Bkdone.HasRange(r) {
δtodo := &RangedKeySet{}
δtodo := &blib.RangedKeySet{}
δtodo.AddRange(r)
δtodo.DifferenceInplace(Bkdone)
debugfDiff(" Bkq <- %s\n", δtodo)
......@@ -595,8 +596,8 @@ ABcov:
}
// δtkeycov will be = BAdd \ ADel
δtkeycovADel := &RangedKeySet{}
δtkeycovBAdd := &RangedKeySet{}
δtkeycovADel := &blib.RangedKeySet{}
δtkeycovBAdd := &blib.RangedKeySet{}
// phase 1: expand A top->down driven by δZTC.
// by default a node contributes to δ-
......@@ -620,7 +621,7 @@ ABcov:
// a is bucket -> δ-
δA, err := diffB(ctx, a, nil); /*X*/if err != nil { return nil,nil,nil, err }
err = δMerge(δ, δA); /*X*/if err != nil { return nil,nil,nil, err }
ar := KeyRange{ra.lo, ra.hi_}
ar := blib.KeyRange{ra.lo, ra.hi_}
δtrack.Del.AddPath(ra.Path())
δtkeycovADel.AddRange(ar)
debugfDiff(" δtrack - %s %v\n", ar, ra.Path())
......@@ -632,7 +633,7 @@ ABcov:
case *Tree:
// empty tree - queue holes covered by it
if len(a.Entryv()) == 0 {
ar := KeyRange{ra.lo, ra.hi_}
ar := blib.KeyRange{ra.lo, ra.hi_}
δtrack.Del.AddPath(ra.Path())
δtkeycovADel.AddRange(ar)
debugfDiff(" δtrack - %s %v\n", ar, ra.Path())
......@@ -691,8 +692,8 @@ ABcov:
}
if found {
// ac can be skipped if key coverage stays the same
ar := KeyRange{ac.lo, ac.hi_}
br := KeyRange{bc.lo, bc.hi_}
ar := blib.KeyRange{ac.lo, ac.hi_}
br := blib.KeyRange{bc.lo, bc.hi_}
if ar == br {
// adjust trackSet since path to the node could have changed
apath := ac.Path()
......@@ -744,7 +745,7 @@ ABcov:
}
for _, r := range Bkqueue.AllRanges() {
lo := r.lo
lo := r.Lo
for {
b, err := Bv.GetToLeaf(ctx, lo); /*X*/if err != nil { return nil,nil,nil, err }
debugfDiff(" B k%d -> %s\n", lo, b)
......@@ -758,7 +759,7 @@ ABcov:
// δ <- δB
err = δMerge(δ, δB); /*X*/if err != nil { return nil,nil,nil, err }
br := KeyRange{b.lo, b.hi_}
br := blib.KeyRange{b.lo, b.hi_}
δtrack.Add.AddPath(b.Path())
δtkeycovBAdd.AddRange(br)
debugfDiff(" δtrack + %s %v\n", br, b.Path())
......@@ -771,7 +772,7 @@ ABcov:
}
// continue with next right bucket until r coverage is complete
if r.hi_ <= b.hi_ {
if r.Hi_ <= b.hi_ {
break
}
lo = b.hi_ + 1
......@@ -782,7 +783,7 @@ ABcov:
debugfDiff("\n")
debugfDiff(" Akq: %s\n", Akqueue)
for _, r := range Akqueue.AllRanges() {
lo := r.lo
lo := r.Lo
for {
a, err := Av.GetToLeaf(ctx, lo); /*X*/if err != nil { return nil,nil,nil, err }
debugfDiff(" A k%d -> %s\n", lo, a)
......@@ -798,7 +799,7 @@ ABcov:
err = δMerge(δ, δA); /*X*/if err != nil { return nil,nil,nil, err }
δtrack.Del.AddPath(a.Path())
// NOTE adjust δtkeycovADel only if a was originally tracked
ar := KeyRange{a.lo, a.hi_}
ar := blib.KeyRange{a.lo, a.hi_}
_, tracked := trackSet[a.node.POid()]
if tracked {
δtkeycovADel.AddRange(ar)
......@@ -815,7 +816,7 @@ ABcov:
}
// continue with next right bucket until r coverage is complete
if r.hi_ <= a.hi_ {
if r.Hi_ <= a.hi_ {
break
}
lo = a.hi_ + 1
......@@ -1009,7 +1010,7 @@ func xidOf(obj zodb.IPersistent) string {
func (rn *nodeInRange) String() string {
done := " "; if rn.done { done = "*" }
return fmt.Sprintf("%s%s%s", done, KeyRange{rn.lo, rn.hi_}, vnode(rn.node))
return fmt.Sprintf("%s%s%s", done, blib.KeyRange{rn.lo, rn.hi_}, vnode(rn.node))
}
// push pushes element to node stack.
......
wcfs/internal/xbtree/xbtree.go
View file @ 11c95175
......@@ -5,10 +5,8 @@ package xbtree
import (
"fmt"
"math"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/neo/go/zodb/btree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
......
wcfs/internal/xbtree/xbtreetest/init/init.go 0 → 100644
View file @ 11c95175
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package xbtreetest/init (ex imported from package A) should be imported in
// addition to xbtreetest (from package A_test) to initialize xbtreetest at runtime.
package init
// ZBlk-related part of δbtail_test
import (
"context"
"fmt"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
)
type Tree = xbtree.Tree
type Node = xbtree.Node
type Key = xbtree.Key
type ZBlk = zdata.ZBlk
// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
// XXX +return blkRevMax and use it ?
func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
path = []Node{}
xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
path = append(path, node)
})
if err != nil {
return nil, false, nil, err
}
if ok {
zblk, ok = xzblk.(ZBlk)
if !ok {
return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk)) // XXX errctx
}
}
return zblk, ok, path, nil
}
func init() {
xbtree.ZTreeGetBlkData = ZTreeGetBlkData
xbtree.ZGetBlkData = ZGetBlkData
}
// ZTreeGetBlkData returns block data from block pointed to by ztree[k].
func ZTreeGetBlkData(ctx context.Context, ztree *Tree, k Key) (data string, ok bool, path []Node, err error) {
defer xerr.Contextf(&err, "@%s: tree<%s>: get blkdata from [%d]", ztree.PJar().At(), ztree.POid(), k)
zblk, ok, path, err := ztreeGetBlk(ctx, ztree, k)
if err != nil || !ok {
return "", ok, path, err
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", false, nil, err
}
return string(bdata), true, path, nil
}
// ZGetBlkData loads block data from ZBlk object specified by its oid.
func ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
xblk, err := zconn.Get(ctx, zblkOid)
if err != nil {
return "", err
}
zblk, ok := xblk.(ZBlk)
if !ok {
return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", err
}
return string(bdata), nil
}
wcfs/internal/xbtree/xbtreetest/rtree.go
View file @ 11c95175
......@@ -31,27 +31,27 @@ import (
// RTree represents Tree node covering [lo, hi_] key range in its parent tree.
// XXX actually no coverage here -> kill? -> change to just `path []zodb.Oid` in RBucket?
type RTree struct {
oid zodb.Oid
parent *RTree
Oid zodb.Oid
Parent *RTree
// XXX +children?
}
// RBucket represents Bucket node covering [lo, hi_] key range in its Tree.
// NOTE it is not [lo,hi) but [lo,hi_] instead to avoid overflow at KeyMax.
type RBucket struct {
oid zodb.Oid
parent *RTree
lo, hi_ Key // XXX -> KeyRange ?
kv map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
Oid zodb.Oid
Parent *RTree
Keycov blib.KeyRange
KV map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
}
// Path returns path to this bucket from tree root.
func (rb *RBucket) Path() []zodb.Oid {
path := []zodb.Oid{rb.oid}
p := rb.parent
path := []zodb.Oid{rb.Oid}
p := rb.Parent
for p != nil {
path = append([]zodb.Oid{p.oid}, path...)
p = p.parent
path = append([]zodb.Oid{p.Oid}, path...)
p = p.Parent
}
return path
}
......@@ -63,15 +63,15 @@ type RBucketSet []*RBucket // k↑
// Get returns RBucket which covers key k.
func (rbs RBucketSet) Get(k Key) *RBucket {
i := sort.Search(len(rbs), func(i int) bool {
return k <= rbs[i].hi_
return k <= rbs[i].Keycov.Hi_
})
if i == len(rbs) {
panicf("BUG: key %v not covered; coverage: %s", k, rbs.coverage())
}
rb := rbs[i]
if !(rb.lo <= k && k <= rb.hi_) {
panicf("BUG: get(%v) -> [%v, %v]; coverage: %s", k, rb.lo, rb.hi_, rbs.coverage())
if !(rb.Keycov.Lo <= k && k <= rb.Keycov.Hi_) {
panicf("BUG: get(%v) -> %s; coverage: %s", k, rb.Keycov, rbs.coverage())
}
return rb
......@@ -87,7 +87,7 @@ func (rbs RBucketSet) coverage() string {
if s != "" {
s += " "
}
s += fmt.Sprintf("[%v, %v]", rb.lo, rb.hi_)
s += fmt.Sprintf("%s", rb.Keycov)
}
return s
}
......@@ -96,7 +96,7 @@ func (rbs RBucketSet) coverage() string {
func (xkv RBucketSet) Flatten() map[Key]string {
kv := make(map[Key]string)
for _, b := range xkv {
for k,v := range b.kv {
for k,v := range b.KV {
kv[k] = v
}
}
......@@ -104,5 +104,5 @@ func (xkv RBucketSet) Flatten() map[Key]string {
}
func (b *RBucket) String() string {
return fmt.Sprintf("%sB%s{%s}", blib.KeyRange{b.lo, b.hi_}, b.oid, kvtxt(b.kv))
return fmt.Sprintf("%sB%s{%s}", b.Keycov, b.Oid, kvtxt(b.KV))
}
wcfs/internal/xbtree/xbtreetest/treeenv.go
View file @ 11c95175
......@@ -18,7 +18,7 @@
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// TreeEnv + friends
// T + friends
import (
"context"
......@@ -28,14 +28,15 @@ import (
"lab.nexedi.com/kirr/neo/go/transaction"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
)
// tTreeEnv is tree-based testing environment.
// T is tree-based testing environment.
//
// It combines TreeSrv and client side access to ZODB with committed trees.
// It should be created it via tNewTreeEnv().
type tTreeEnv struct {
// It should be created it via NewT().
type T struct {
*testing.T
work string // working directory
......@@ -44,13 +45,13 @@ type tTreeEnv struct {
db *zodb.DB
// all committed trees
commitv []*tTreeCommit
commitv []*Commit
}
// tTreeCommit represent test commit changing a tree.
type tTreeCommit struct {
// Commit represent test commit changing a tree.
type Commit struct {
tree string // the tree in topology-encoding
prev *tTreeCommit // previous commit
prev *Commit // previous commit
at zodb.Tid // commit revision
δZ *zodb.EventCommit // raw ZODB changes; δZ.tid == at
xkv RBucketSet // full tree state as of @at
......@@ -59,12 +60,12 @@ type tTreeCommit struct {
// δzblkData map[zodb.Oid]Δstring // full diff for zblkData against parent XXX ?
}
// tNewTreeEnv creates new tTreeEnv.
func tNewTreeEnv(t *testing.T) *tTreeEnv {
// NewT creates new T.
func NewT(t *testing.T) *T {
X := exc.Raiseif
t.Helper()
tt := &tTreeEnv{T: t}
tt := &T{T: t}
var err error
work := t.TempDir()
......@@ -91,7 +92,7 @@ func tNewTreeEnv(t *testing.T) *tTreeEnv {
})
head := tt.treeSrv.head
t1 := &tTreeCommit{
t1 := &Commit{
tree: "T/B:", // treegen.py creates the tree as initially empty
prev: nil,
at: head,
......@@ -100,7 +101,7 @@ func tNewTreeEnv(t *testing.T) *tTreeEnv {
δZ: nil,
δxkv: nil,
}
tt.commitv = []*tTreeCommit{t1}
tt.commitv = []*Commit{t1}
return tt
}
......@@ -115,17 +116,18 @@ func (_ *tZODBCacheEverything) PCacheClassify(_ zodb.IPersistent) zodb.PCachePol
}
// Root returns OID of root tree node.
func (t *tTreeEnv) Root() zodb.Oid {
func (t *T) Root() zodb.Oid {
return t.treeSrv.treeRoot
}
// Head returns most-recently committed tree.
func (t *tTreeEnv) Head() *tTreeCommit {
func (t *T) Head() *Commit {
return t.commitv[len(t.commitv)-1]
}
// CommitTree calls t.treeSrv.Commit and returns tTreeCommit corresponding to committed transaction.
func (t *tTreeEnv) CommitTree(tree string) *tTreeCommit {
// CommitTree calls t.treeSrv.Commit and returns Commit corresponding to committed transaction.
// XXX naming -> Commit ?
func (t *T) CommitTree(tree string) *Commit {
// TODO X = FatalIf
X := exc.Raiseif
defer exc.Contextf("commit %s", tree)
......@@ -164,19 +166,18 @@ func (t *tTreeEnv) CommitTree(tree string) *tTreeCommit {
// it comes to ->T2.
xkv = RBucketSet{
&RBucket{
oid: zodb.InvalidOid,
parent: &RTree{
oid: t.Root(), // NOTE oid is not InvalidOid
parent: nil,
Oid: zodb.InvalidOid,
Parent: &RTree{
Oid: t.Root(), // NOTE oid is not InvalidOid
Parent: nil,
},
lo: KeyMin,
hi_: KeyMax,
kv: map[Key]string{},
Keycov: blib.KeyRange{KeyMin, KeyMax},
KV: map[Key]string{},
},
}
}
ttree := &tTreeCommit{
ttree := &Commit{
tree: tree,
at: δZ.Tid,
δZ: δZ,
......@@ -244,7 +245,7 @@ func xGetBlkDataTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]string {
// xgetBlkData loads blk data for ZBlk<oid> @t.at
//
// For speed the load is done via preloaded t.blkDataTab instead of access to the DB.
func (t *tTreeCommit) xgetBlkData(oid zodb.Oid) string {
func (t *Commit) xgetBlkData(oid zodb.Oid) string {
if oid == VDEL {
return DEL
}
......@@ -282,15 +283,14 @@ func xGetTree(db *zodb.DB, at zodb.Tid, root zodb.Oid) RBucketSet {
})
if len(rbucketv) == 0 { // empty tree -> [-∞,∞){}
etree := &RTree{
oid: root,
parent: nil,
Oid: root,
Parent: nil,
}
ebucket := &RBucket{
oid: zodb.InvalidOid,
parent: etree,
lo: KeyMin,
hi_: KeyMax,
kv: map[Key]string{},
Oid: zodb.InvalidOid,
Parent: etree,
Keycov: blib.KeyRange{KeyMin, KeyMax},
KV: map[Key]string{},
}
rbucketv = RBucketSet{ebucket}
}
......@@ -307,7 +307,7 @@ func _xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, rparent *RTree, bv
err := ztree.PActivate(ctx); X(err)
defer ztree.PDeactivate()
rtree := &RTree{oid: ztree.POid(), parent: rparent}
rtree := &RTree{Oid: ztree.POid(), Parent: rparent}
// [i].Key ≤ [i].Child.*.Key < [i+1].Key i ∈ [0, len([]))
//
......@@ -346,7 +346,12 @@ func _xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, rparent *RTree, bv
}
b := &RBucket{oid: zbucket.POid(), parent: rtree, lo: xlo, hi_: xhi_, kv: bkv}
b := &RBucket{
Oid: zbucket.POid(),
Parent: rtree,
Keycov: blib.KeyRange{xlo, xhi_},
KV: bkv,
}
bvisit(b)
}
}
......
wcfs/internal/xbtree/xbtreetest/xbtreetest.go
View file @ 11c95175
......@@ -23,24 +23,27 @@ package xbtreetest
import (
"fmt"
"math"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/neo/go/zodb/btree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
// "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
)
// XXX dup from xbtree (to avoid import cycle)
type Tree = btree.LOBTree
type Bucket = btree.LOBucket
// XXX instead of generics
type Tree = blib.Tree
type Bucket = blib.Bucket
type Node = blib.Node
type TreeEntry = blib.TreeEntry
type BucketEntry = blib.BucketEntry
type Key = blib.Key
const KeyMax = blib.KeyMax
const KeyMin = blib.KeyMin
type Key = int64
const KeyMax Key = math.MaxInt64
const KeyMin Key = math.MinInt64
type setKey = set.I64
// XXX dup from xbtree (to avoid import cycle)
const VDEL = zodb.InvalidOid
......
wcfs/internal/xbtree/xbtreetest/zdata.go 0 → 100644
View file @ 11c95175
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// access to ZBlk data
import (
"context"
"lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/neo/go/transaction"
"lab.nexedi.com/kirr/neo/go/zodb"
_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
)
// ZBlk-related functions are imported at runtime by package xbtreetest/init
var (
ZTreeGetBlkData func(context.Context, *Tree, Key) (string, bool, []Node, error)
ZGetBlkData func(context.Context, *zodb.Connection, zodb.Oid) (string, error)
)
func zassertInitDone() {
if ZTreeGetBlkData == nil {
panic("xbtreetest/zdata not initialized -> import xbtreetest/init to fix")
}
}
// xzgetBlkData loads block data from ZBlk object specified by its oid.
func xzgetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) string {
zassertInitDone()
X := exc.Raiseif
if zblkOid == VDEL {
return DEL
}
data, err := ZGetBlkData(ctx, zconn, zblkOid); X(err)
return string(data)
}
// xzgetBlkDataAt loads block data from ZBlk object specified by oid@at.
func xzgetBlkDataAt(db *zodb.DB, zblkOid zodb.Oid, at zodb.Tid) string {
zassertInitDone()
X := exc.Raiseif
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)
return xzgetBlkData(ctx, zconn, zblkOid)
}
wcfs/internal/xbtree/δbtail.go
View file @ 11c95175
......@@ -30,6 +30,7 @@ import (
"lab.nexedi.com/kirr/neo/go/transaction"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xtail"
)
......@@ -96,7 +97,7 @@ type ΔBtail struct {
// For this set all vδT are fully computed.
// The set of nodes that were requested to be tracked, but were not yet
// taken into account, is kept in ΔTtail.trackNew & co.
trackSet PPTreeSubSet
trackSet blib.PPTreeSubSet
// set of trees for which .trackNew is non-empty
trackNewRoots setOid
......@@ -114,7 +115,7 @@ type ΔTtail struct {
// set of nodes that were requested to be tracked in this tree, but for
// which vδT was not yet rebuilt
trackNew PPTreeSubSet
trackNew blib.PPTreeSubSet
// XXX + trackNewKeys RangedKeySet
// {}k/v @tail for keys that are changed in (tail, head].
......@@ -158,7 +159,7 @@ func NewΔBtail(at0 zodb.Tid, db *zodb.DB) *ΔBtail {
δZtail: zodb.NewΔTail(at0),
vδBroots: nil,
vδTbyRoot: map[zodb.Oid]*ΔTtail{},
trackSet: PPTreeSubSet{},
trackSet: blib.PPTreeSubSet{},
trackNewRoots: setOid{},
db: db,
}
......@@ -167,7 +168,7 @@ func NewΔBtail(at0 zodb.Tid, db *zodb.DB) *ΔBtail {
// newΔTtail creates new empty ΔTtail object.
func newΔTtail() *ΔTtail {
return &ΔTtail{
trackNew: PPTreeSubSet{},
trackNew: blib.PPTreeSubSet{},
KVAtTail: make(map[Key]Value),
lastRevOf: make(map[Key]zodb.Tid),
}
......@@ -271,7 +272,7 @@ func (δBtail *ΔBtail) track(key Key, path []zodb.Oid) error {
// empty artificial tree. We need to do the normalization because we
// later check whether leaf path[-1] ∈ trackSet and without
// normalization path[-1] can be InvalidOid.
path = normPath(path)
path = blib.NormPath(path)
if len(path) == 0 {
return nil // empty tree
}
......@@ -334,7 +335,7 @@ func (δBtail *ΔBtail) rebuildAll() (err error) {
// - set of revisions for which new entries in .vδT have been created.
//
// XXX place
func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet PPTreeSubSet, δrevSet setTid, err error) {
func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet blib.PPTreeSubSet, δrevSet setTid, err error) {
defer xerr.Context(&err, "ΔTtail rebuild")
// XXX locking
......@@ -342,7 +343,7 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
tracefΔBtail("trackNew: %v\n", δTtail.trackNew)
trackNew := δTtail.trackNew
δTtail.trackNew = PPTreeSubSet{}
δTtail.trackNew = blib.PPTreeSubSet{}
if len(trackNew) == 0 {
return nil, nil, nil
......@@ -353,7 +354,7 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
// go backwards and merge vδT <- treediff(lo..hi/trackNew)
vδZ := δZtail.Data()
for {
δtkeycov := &RangedKeySet{} // all keys coming into tracking set during this lo<-hi scan
δtkeycov := &blib.RangedKeySet{} // all keys coming into tracking set during this lo<-hi scan
trackNewCur := trackNew.Clone() // trackNew adjusted as of when going to i<- entry
for i := len(vδZ)-1; i>=0; i-- {
δZ := vδZ[i]
......@@ -411,7 +412,7 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
// widenTrackNew widens trackNew to cover δtkeycov.
// XXX -> widenTrackSet?
func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid, at zodb.Tid, db *zodb.DB) (err error) {
func widenTrackNew(trackNew blib.PPTreeSubSet, δtkeycov *blib.RangedKeySet, root zodb.Oid, at zodb.Tid, db *zodb.DB) (err error) {
// XXX errctx, debug
defer xerr.Contextf(&err, "widenTrackNew tree<%s> @%s +%s", root, at, δtkeycov)
......@@ -430,13 +431,13 @@ func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid
top := &nodeInRange{prefix: nil, lo: KeyMin, hi_: KeyMax, node: tree}
V := rangeSplit{top}
for _, r := range δtkeycov.AllRanges() {
lo := r.lo
lo := r.Lo
for {
b, err := V.GetToLeaf(ctx, lo); /*X*/ if err != nil { return err }
trackNew.AddPath(b.Path())
// continue with next right bucket until r coverage is complete
if r.hi_ <= b.hi_ {
if r.Hi_ <= b.hi_ {
break
}
lo = b.hi_ + 1
......@@ -450,7 +451,7 @@ func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid
//
// δtrackNew/δtkeycov represents how trackNew changes when going through `atPrev <- δZ.Rev` .
// newRevEntry indicates whether δZ.Rev was not there before in .vδT and new corresponding δT entry was created.
func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew PPTreeSubSet, db *zodb.DB) (δtrackNew *ΔPPTreeSubSet, δtkeycov *RangedKeySet, newRevEntry bool, err error) {
func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew blib.PPTreeSubSet, db *zodb.DB) (δtrackNew *blib.ΔPPTreeSubSet, δtkeycov *blib.RangedKeySet, newRevEntry bool, err error) {
defer xerr.Contextf(&err, "rebuild1 %s<-%s", atPrev, δZ.Rev)
debugfΔBtail("\n rebuild1 @%s <- @%s\n", atPrev, δZ.Rev)
......@@ -467,7 +468,7 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
// skip opening DB connections if there is no change to this tree
if len(δtopsByRoot) == 0 {
return NewΔPPTreeSubSet(), &RangedKeySet{}, false, nil
return blib.NewΔPPTreeSubSet(), &blib.RangedKeySet{}, false, nil
}
if len(δtopsByRoot) != 1 {
......@@ -560,7 +561,7 @@ func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
// δtkeycov1 != ø -> rebuild δTtail with trackNew ~= δtkeycov1
if !δT1.δtkeycov1.Empty() && δBtail.δZtail.Len() > 1 {
trackNew := PPTreeSubSet{}
trackNew := blib.PPTreeSubSet{}
err := widenTrackNew(trackNew, δT1.δtkeycov1, root, δBtail.Head(), δBtail.db)
if err != nil {
return ΔB{}, err
......@@ -611,8 +612,8 @@ type _ΔBUpdate1 struct {
ByRoot map[zodb.Oid]*_ΔTUpdate1
}
type _ΔTUpdate1 struct {
δtkeycov1 *RangedKeySet // {} root -> δtrackedKeys after first treediff (always grow)
δtrack *ΔPPTreeSubSet // XXX kill (not used)
δtkeycov1 *blib.RangedKeySet // {} root -> δtrackedKeys after first treediff (always grow)
δtrack *blib.ΔPPTreeSubSet // XXX kill (not used)
}
func (δBtail *ΔBtail) _Update1(δZ *zodb.EventCommit) (δB1 _ΔBUpdate1, err error) {
headOld := δBtail.Head()
......
wcfs/internal/xbtree/δbtail_test.go
View file @ 11c95175
This diff is collapsed.
wcfs/internal/xbtree/δbtail_x_test.go
View file @ 11c95175
......@@ -18,85 +18,7 @@
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree_test
// ZBlk-related part of δbtail_test
import (
"context"
"fmt"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
_ "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest/init"
)
type Tree = xbtree.Tree
type Node = xbtree.Node
type Key = xbtree.Key
type ZBlk = zdata.ZBlk
// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
// XXX +return blkRevMax and use it ?
func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
path = []Node{}
xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
path = append(path, node)
})
if err != nil {
return nil, false, nil, err
}
if ok {
zblk, ok = xzblk.(ZBlk)
if !ok {
return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk)) // XXX errctx
}
}
return zblk, ok, path, nil
}
func init() {
xbtree.ZTreeGetBlkData = ZTreeGetBlkData
xbtree.ZGetBlkData = ZGetBlkData
}
// ZTreeGetBlkData returns block data from block pointed to by ztree[k].
func ZTreeGetBlkData(ctx context.Context, ztree *Tree, k Key) (data string, ok bool, path []Node, err error) {
defer xerr.Contextf(&err, "@%s: tree<%s>: get blkdata from [%d]", ztree.PJar().At(), ztree.POid(), k)
zblk, ok, path, err := ztreeGetBlk(ctx, ztree, k)
if err != nil || !ok {
return "", ok, path, err
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", false, nil, err
}
return string(bdata), true, path, nil
}
// ZGetBlkData loads block data from ZBlk object specified by its oid.
func ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
xblk, err := zconn.Get(ctx, zblkOid)
if err != nil {
return "", err
}
zblk, ok := xblk.(ZBlk)
if !ok {
return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", err
}
return string(bdata), nil
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7