Skip to content

W
wendelin.core
Commit f01f7677 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

.

parent 5aac8b0d
Hide whitespace changes
Inline Side-by-side
Showing with 103 additions and 63 deletions
wcfs/internal/xbtree/pptreesubset.go
View file @ f01f7677
......@@ -395,6 +395,11 @@ func (A PPTreeSubSet) Equal(B PPTreeSubSet) bool {
return true
}
// Empty returns whether set is empty.
func (S PPTreeSubSet) Empty() bool {
return len(S) == 0
}
func (t nodeInTree) String() string {
return fmt.Sprintf("{p%s c%d}", t.parent, t.nchild)
}
......
wcfs/internal/xbtree/δbtail.go
View file @ f01f7677
......@@ -89,16 +89,12 @@ const debugΔBtail = false
// See also zodb.ΔTail
// XXX naming -> ΔBTail ?
type ΔBtail struct {
// raw ZODB changes; Kept to rebuild .byRoot after new Track.
// raw ZODB changes; Kept to rebuild .vδTbyRoot after new Track.
// includes all changed objects, not only tracked ones.
δZtail *zodb.ΔTail
vδBroots []ΔBroots // [] (rev, roots changed in this rev)
byRoot map[zodb.Oid]*ΔTtail // {} root -> [] k/v change history; only for keys ∈ tracked subset XXX -> vδByRoot ?
// handle to make connections to access database.
// TODO allow client to optionally provide zconnOld/zconnNew on e.g. Update()
db *zodb.DB // to open connections to load new/old tree|buckets
vδBroots []ΔBroots // [] (rev, roots changed in this rev)
vδTbyRoot map[zodb.Oid]*ΔTtail // {} root -> [] k/v change history; only for keys ∈ tracked subset XXX -> byRoot?
// set of tracked nodes as of @head state.
// For this set all vδT are fully computed.
......@@ -106,8 +102,12 @@ type ΔBtail struct {
// taken into account, is kept in ΔTtail.trackNew & co.
trackSet PPTreeSubSet
// set of trees for which trackNew is non-empty
// set of trees for which .trackNew is non-empty
trackNewRoots SetOid
// handle to make connections to access database.
// TODO allow client to optionally provide zconnOld/zconnNew on e.g. Update()
db *zodb.DB // to open connections to load new/old tree|buckets
}
// ΔTtail represent tail of revisional changes to one BTree.
......@@ -128,7 +128,7 @@ type ΔTtail struct {
lastRevOf map[Key]zodb.Tid // {} key -> last
}
// ΔBroots represents roots-only part of ΔB - a change in BTrees space.
// ΔBroots represents roots-only part of ΔB.
//
// It describes which trees were changed, but does not provide δkv details for changed trees.
type ΔBroots struct {
......@@ -138,8 +138,8 @@ type ΔBroots struct {
// ΔB represents a change in BTrees space.
type ΔB struct {
Rev zodb.Tid
ByRoot map[zodb.Oid]map[Key]ΔValue // {} root -> {}(key, δvalue) XXX -> ΔByRoot ?
Rev zodb.Tid
ΔByRoot map[zodb.Oid]map[Key]ΔValue // {} root -> {}(key, δvalue) XXX -> ByRoot?
}
// ΔTree describes changes to one BTree in one revision.
......@@ -160,7 +160,8 @@ type ΔTree struct {
func NewΔBtail(at0 zodb.Tid, db *zodb.DB) *ΔBtail {
return &ΔBtail{
δZtail: zodb.NewΔTail(at0),
byRoot: map[zodb.Oid]*ΔTtail{},
vδBroots: nil,
vδTbyRoot: map[zodb.Oid]*ΔTtail{},
trackSet: PPTreeSubSet{},
trackNewRoots: SetOid{},
db: db,
......@@ -186,13 +187,12 @@ func (orig *ΔBtail) clone() *ΔBtail {
klon.δZtail.Append(δZ.Rev, δZ.Changev)
}
// trackSet, trackNewRoots
klon.trackSet = orig.trackSet.Clone()
klon.trackNewRoots = orig.trackNewRoots.Clone()
// vδBroots
klon.vδBroots = append(klon.vδBroots, orig.vδBroots...)
// byRoot
klon.byRoot = make(map[zodb.Oid]*ΔTtail, len(orig.byRoot))
for root, origΔTtail := range orig.byRoot {
// vδTbyRoot
klon.vδTbyRoot = make(map[zodb.Oid]*ΔTtail, len(orig.vδTbyRoot))
for root, origΔTtail := range orig.vδTbyRoot {
klonΔTtail := &ΔTtail{}
klonΔTtail.vδT = append(klonΔTtail.vδT, origΔTtail.vδT...)
klonΔTtail.trackNew = origΔTtail.trackNew.Clone()
......@@ -204,9 +204,13 @@ func (orig *ΔBtail) clone() *ΔBtail {
for k, rev := range origΔTtail.lastRevOf {
klonΔTtail.lastRevOf[k] = rev
}
klon.byRoot[root] = klonΔTtail
klon.vδTbyRoot[root] = klonΔTtail
}
// trackSet, trackNewRoots
klon.trackSet = orig.trackSet.Clone()
klon.trackNewRoots = orig.trackNewRoots.Clone()
return klon
}
......@@ -216,12 +220,6 @@ func (δBtail *ΔBtail) Tail() zodb.Tid { return δBtail.δZtail.Tail() }
// XXX SliceByRev?
// ForgetPast XXX
func (δBtail *ΔBtail) ForgetPast(revCut zodb.Tid) {
// TODO go throough vδBroots till revcut -> find which trees to trim -> trim ΔTtails.
panic("TODO")
}
// Track adds tree path to tracked set.
//
// path[0] signifies tree root.
......@@ -252,10 +250,10 @@ func (δBtail *ΔBtail) track(key Key, keyPresent bool, path []zodb.Oid) error {
// XXX locking
root := path[0]
δTtail, ok := δBtail.byRoot[root]
δTtail, ok := δBtail.vδTbyRoot[root]
if !ok {
δTtail = newΔTtail()
δBtail.byRoot[root] = δTtail
δBtail.vδTbyRoot[root] = δTtail
}
δBtail.trackNewRoots.Add(root)
......@@ -276,8 +274,8 @@ func (δBtail *ΔBtail) rebuildAll() (err error) {
trackNewRoots := δBtail.trackNewRoots
for root := range trackNewRoots {
δTtail := δBtail.byRoot[root] // must be there
δtrackSet, err := δTtail.rebuild()
δTtail := δBtail.vδTbyRoot[root] // must be there
δtrackSet, err := δTtail.rebuild(δBtail.δZtail, δBtail.db)
if err != nil {
return err
}
......@@ -293,11 +291,11 @@ func (δBtail *ΔBtail) rebuildAll() (err error) {
// It returns set of nodes that must be added to ΔBtail.trackSet to account for
// keys that becomes tracked. Note: this set is potentially wider compared to .trackNew.
// XXX place
func (δTtail *ΔTtail) rebuild() (δtrackSet PPTreeSubSet, err error) {
func (δTtail *ΔTtail) rebuild(δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet PPTreeSubSet, err error) {
defer xerr.Context(&err, "ΔTtail rebuild")
// XXX locking
tracefΔBtail("\nRebuild @%s .. @%s\n", δBtail.Tail(), δBtail.Head())
tracefΔBtail("\nRebuild @%s .. @%s\n", δZtail.Tail(), δZtail.Head())
tracefΔBtail("trackNew: %v\n", δTtail.trackNew)
trackNew := δTtail.trackNew
......@@ -308,7 +306,7 @@ func (δTtail *ΔTtail) rebuild() (δtrackSet PPTreeSubSet, err error) {
}
// go backwards and merge vδT <- treediff(lo..hi/trackNew)
vδZ := δBtail.δZtail.Data()
vδZ := δZtail.Data()
vδtrack := []*ΔPPTreeSubSet{}
for i := len(vδZ)-1; i>=0; i-- {
δZ := vδZ[i]
......@@ -319,7 +317,7 @@ func (δTtail *ΔTtail) rebuild() (δtrackSet PPTreeSubSet, err error) {
if i > 0 {
atPrev = vδZ[i-1].Rev
} else {
atPrev = δBtail.δZtail.Tail()
atPrev = δZtail.Tail()
}
δZTC, δtopsByRoot := δZConnectTracked(δZ.Changev, trackNew)
......@@ -336,11 +334,11 @@ func (δTtail *ΔTtail) rebuild() (δtrackSet PPTreeSubSet, err error) {
txn, ctx := transaction.New(context.TODO()) // XXX
defer txn.Abort() // XXX -> into func() or don't use defer
zconnPrev, err := δBtail.db.Open(ctx, &zodb.ConnOptions{At: atPrev})
zconnPrev, err := db.Open(ctx, &zodb.ConnOptions{At: atPrev})
if err != nil {
return nil, err
}
zconnCurr, err := δBtail.db.Open(ctx, &zodb.ConnOptions{At: δZ.Rev})
zconnCurr, err := db.Open(ctx, &zodb.ConnOptions{At: δZ.Rev})
if err != nil {
return nil, err
}
......@@ -365,11 +363,11 @@ func (δTtail *ΔTtail) rebuild() (δtrackSet PPTreeSubSet, err error) {
}
/* XXX kill
δTtail, ok := δBtail.byRoot[root]
δTtail, ok := δBtail.vδTbyRoot[root]
if !ok {
// this root was not tracked before -> create δTtail for it with empty changes
δTtail = newΔTtail()
δBtail.byRoot[root] = δTtail
δBtail.vδTbyRoot[root] = δTtail
}
*/
......@@ -434,7 +432,7 @@ func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
// _Update serves Update and also returns δtkeycov from treediff for testing.
type _ΔTrackKeyCov struct {
ByRoot map[zodb.Oid]*RangedKeySet // {} root -> δtrackedKeys (always grow)
ByRoot map[zodb.Oid]*RangedKeySet // {} root -> δtrackedKeys (always grow) XXX δtkeycovByRoot?
}
func (δBtail *ΔBtail) _Update(δZ *zodb.EventCommit) (_ ΔB, δTKeyCov _ΔTrackKeyCov, err error) {
headOld := δBtail.Head()
......@@ -443,8 +441,8 @@ func (δBtail *ΔBtail) _Update(δZ *zodb.EventCommit) (_ ΔB, δTKeyCov _ΔTrac
tracefΔBtail("\nUpdate @%s -> @%s δZ: %v\n", δBtail.Head(), δZ.Tid, δZ.Changev)
tracefΔBtail("trackSet: %v\n", δBtail.trackSet)
for _, root := range δBtail.trackNewRoots.SortedElements() {
δTtail := δBtail.byRoot[root]
tracefΔBtail("[%s].trackNew: %v\n", δTtail.trackNew)
δTtail := δBtail.vδTbyRoot[root]
tracefΔBtail("[%s].trackNew: %v\n", root, δTtail.trackNew)
}
δTKeyCov = _ΔTrackKeyCov{ByRoot: make(map[zodb.Oid]*RangedKeySet)}
......@@ -452,7 +450,7 @@ func (δBtail *ΔBtail) _Update(δZ *zodb.EventCommit) (_ ΔB, δTKeyCov _ΔTrac
if XXX_killWhenRebuildWorks {
// XXX hack - until vvv is reenabled
for root := range δBtail.trackNewRoots {
δTtail := δBtail.byRoot[root] // must succeed
δTtail := δBtail.vδTbyRoot[root] // must succeed
δBtail.trackSet.UnionInplace(δTtail.trackNew)
δTtail.trackNew = PPTreeSubSet{}
}
......@@ -472,7 +470,7 @@ if XXX_killWhenRebuildWorks {
δZTC, δtopsByRoot := δZConnectTracked(δZ.Changev, δBtail.trackSet)
δB := ΔB{Rev: δZ.Tid, ByRoot: make(map[zodb.Oid]map[Key]ΔValue)}
δB := ΔB{Rev: δZ.Tid, ΔByRoot: make(map[zodb.Oid]map[Key]ΔValue)}
// skip opening DB connections if there is no change to any tree node
if len(δtopsByRoot) == 0 {
......@@ -501,12 +499,12 @@ if XXX_killWhenRebuildWorks {
tracefΔBtail("\n-> root<%s> δkv: %v δtrack: %v δtkeycov: %v\n", root, δT, δtrack, δtkeycov)
if len(δT) > 0 { // an object might be resaved without change
δB.ByRoot[root] = δT
δTtail, ok := δBtail.byRoot[root]
δB.ΔByRoot[root] = δT
δTtail, ok := δBtail.vδTbyRoot[root]
if !ok {
// this root was not tracked before -> create δTtail for it with empty changes
δTtail = newΔTtail()
δBtail.byRoot[root] = δTtail
δBtail.vδTbyRoot[root] = δTtail
}
δTtail.vδT = append(δTtail.vδT, ΔTree{Rev: δZ.Tid, ΔKV: δT})
......@@ -525,7 +523,8 @@ if XXX_killWhenRebuildWorks {
func (δBtail *ΔBtail) ForgetPast(revCut zodb.Tid) {
δBtail.δZtail.ForgetPast(revCut) // XXX stub
// XXX clean vδT
// TODO go throough vδBroots till revcut -> find which trees to trim -> trim ΔTtails.
panic("TODO")
}
......@@ -555,7 +554,7 @@ func (δBtail *ΔBtail) GetAt(ctx context.Context, root *Tree, key Key, at zodb.
// XXX dirty -> rebuild
// XXX -> index lastXXXOf(key) | linear scan ↓ looking for change <= at
δTtail := δBtail.byRoot[root.POid()]
δTtail := δBtail.vδTbyRoot[root.POid()]
if δTtail == nil {
panicf("δBtail: root<%s> not tracked", root.POid())
}
......@@ -624,7 +623,7 @@ func (δBtail *ΔBtail) GetAt(ctx context.Context, root *Tree, key Key, at zodb.
func (δBtail *ΔBtail) SliceByRootRev(root zodb.Oid, lo, hi zodb.Tid) /*readonly*/[]ΔTree {
xtail.AssertSlice(δBtail, lo, hi)
// XXX locking
δTtail, ok := δBtail.byRoot[root]
δTtail, ok := δBtail.vδTbyRoot[root]
if !ok {
return []ΔTree{}
}
......
wcfs/internal/xbtree/δbtail_test.go
View file @ f01f7677
......@@ -690,7 +690,7 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
badv = append(badv, fmt.Sprintf(format, argv...))
}
defer func() {
if badv != nil {
if badv != nil || t.Failed() {
emsg := fmt.Sprintf("%s ; tracked=%v :\n\n", subj, initialTrackedKeys)
emsg += fmt.Sprintf("d12: %v\nδTok: %v\nδT: %v\n\n", d12, δTok, δT)
emsg += fmt.Sprintf("δZ: %v\n", δZset)
......@@ -789,13 +789,15 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// verify δbtail.trackSet against @at1
// trackSet1 = xkv1[tracked1]
trackSet1, tkeyCov1 := xkv1.trackSetWithCov(initialTrackedKeys)
/*
if !δbtail.trackSet.Equal(ø) {
badf("δbtail.trackSet1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, ø)
}
if !δbtail.trackNew.Equal(trackSet1) {
badf("δbtail.trackNew1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackNew, trackSet1)
}
// δbtail.assertTrack(t, "1", ø, trackSet1)
*/
δbtail.assertTrack(t, "1", ø, trackSet1)
// δB <- δZ
......@@ -808,13 +810,15 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// verify δbtail.trackSet against @at2
// trackSet2 = xkv2[tracked2] ( = xkv2[kadj[tracked1]]
trackSet2, tkeyCov2 := xkv2.trackSetWithCov(initialTrackedKeys.Union(kadjTrackedδZ))
/*
if !δbtail.trackSet.Equal(trackSet2) {
badf("δbtail.trackSet2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, trackSet2)
}
if !δbtail.trackNew.Equal(ø) {
badf("δbtail.trackNew2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackNew, ø)
}
// δbtail.assertTrack(t, "2", trackSet2, ø)
*/
δbtail.assertTrack(t, "2", trackSet2, ø)
// assert δB.ByRoot == {treeRoot -> ...} if δTok != ø
......@@ -824,13 +828,13 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
rootsOK.Add(treeRoot)
}
roots := SetOid{}
for root := range δB.ByRoot {
for root := range δB.ΔByRoot {
roots.Add(root)
}
if !reflect.DeepEqual(roots, rootsOK) {
badf("δB: roots != rootsOK ; roots=%v rootsOK=%v", roots, rootsOK)
}
_, inδB := δB.ByRoot[treeRoot]
_, inδB := δB.ΔByRoot[treeRoot]
if !inδB {
return
}
......@@ -844,7 +848,7 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// δT <- δB
δToid := δB.ByRoot[treeRoot] // {} k -> δoid
δToid := δB.ΔByRoot[treeRoot] // {} k -> δoid
δT = XGetδKV(db, at1,at2, δToid) // {} k -> δ(ZBlk(oid).data)
// δT must be subset of d12.
......@@ -883,20 +887,52 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// assertTrack verifies that trackSet == trackSetOK.
// XXX place
func assertTrack(t *testing.T, subj string, trackSet, trackSetOK PPTreeSubSet) {
// XXX inline into δbtail.assertTrack?
func _assertTrack(t *testing.T, subj string, trackSet, trackSetOK PPTreeSubSet) {
t.Helper()
eq := trackSet.Equal(trackSetOK)
if !eq {
if !trackSet.Equal(trackSetOK) {
t.Errorf("%s:\n\thave: %v\n\twant: %v", subj, trackSet, trackSetOK)
}
}
// assertTrack verifies state of .trackSet and .trackNew.
// assertTrack verifies state of .trackSet and ΔTtail.trackNew.
// it assumes that only one tree root is being tracked.
// XXX place
func (δbtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK, trackNewOK PPTreeSubSet) {
func (δBtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK PPTreeSubSet, trackNewOK PPTreeSubSet) {
t.Helper()
assertTrack(t, subj + ": trackSet", δbtail.trackSet, trackSetOK)
assertTrack(t, subj + ": trackNew", δbtail.trackNew, trackNewOK)
_assertTrack(t, subj + ": trackSet", δBtail.trackSet, trackSetOK)
roots := SetOid{}
for root := range δBtail.vδTbyRoot {
roots.Add(root)
}
nrootsOK := 1
if trackSetOK.Empty() && trackNewOK.Empty() {
nrootsOK = 0
}
if len(roots) != nrootsOK {
t.Errorf("%s: len(vδTbyRoot) != %d ; roots=%v", subj, nrootsOK, roots)
return
}
if nrootsOK == 0 {
return
}
root := roots.Elements()[0]
δTtail := δBtail.vδTbyRoot[root]
trackNewRootsOK := SetOid{}
if !trackNewOK.Empty() {
trackNewRootsOK.Add(root)
}
if !δBtail.trackNewRoots.Equal(trackNewRootsOK) {
t.Errorf("%s: trackNewRoots:\n\thave: %v\n\twant: %v", subj, δBtail.trackNewRoots, trackNewRootsOK)
}
_assertTrack(t, subj + ": vδT.trackNew", δTtail.trackNew, trackNewOK)
}
// xverifyΔBTail_rebuild verifies δBtail.rebuild during t0->t1->t2 transition.
......@@ -1079,7 +1115,7 @@ func xverifyΔBTail_rebuild_TR(t *testing.T, db *zodb.DB, δbtail *ΔBtail, tj *
δbtail.assertTrack(t, fmt.Sprintf("@%s: after Track%v", xat[tj.at], keys), trackSet, trackNew)
// XXX vδB=[ø]
δbtail.rebuild()
δbtail.rebuildAll()
δbtail.assertTrack(t, fmt.Sprintf("@%s: after Track%v + rebuild", xat[tj.at], keys), trackSetAfterRebuild, ø)
// XXX assert vδB=[δ1/T(keys)]
......
wcfs/internal/zdata/δftail.go
View file @ f01f7677
......@@ -233,7 +233,7 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
δF := ΔF{Rev: δB.Rev, ByFile: make(map[*ZBigFile]*ΔFile)}
// take btree changes into account
for root, δt := range δB.ByRoot {
for root, δt := range δB.ΔByRoot {
files := δFtail.fileIdx[root]
if len(files) == 0 {
panicf("BUG: ΔFtail: root<%s> -> ø files", root)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7