Commit d0fe680a authored by Kirill Smelkov's avatar Kirill Smelkov

X δbtail += ForgetPast

parent a8177067
......@@ -21,9 +21,11 @@ package set
//go:generate ./gen-set set I64 int64 zset_i64.go
//go:generate ./gen-set set Oid _Oid zset_oid.go
//go:generate ./gen-set set Tid _Tid zset_tid.go
import (
"lab.nexedi.com/kirr/neo/go/zodb"
)
type _Oid = zodb.Oid
type _Tid = zodb.Tid
// Code generated by gen-set Tid _Tid; DO NOT EDIT.
// Copyright (C) 2015-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package set
import (
"fmt"
"sort"
"strings"
)
// SetTid is a set of _Tid.
type SetTid map[_Tid]struct{}
// Add adds v to the set.
func (s SetTid) Add(v _Tid) {
s[v] = struct{}{}
}
// Del removes v from the set.
// it is noop if v was not in the set.
func (s SetTid) Del(v _Tid) {
delete(s, v)
}
// Has checks whether the set contains v.
func (s SetTid) Has(v _Tid) bool {
_, ok := s[v]
return ok
}
// Update adds t values to s.
func (s SetTid) Update(t SetTid) {
for v := range t {
s.Add(v)
}
}
// Elements returns all elements of set as slice.
func (s SetTid) Elements() []_Tid {
ev := make([]_Tid, len(s))
i := 0
for e := range s {
ev[i] = e
i++
}
return ev
}
// Union returns s ∪ t
func (s SetTid) Union(t SetTid) SetTid {
// l = max(len(s), len(t))
l := len(s)
if lt := len(t); lt > l {
l = lt
}
u := make(SetTid, l)
for v := range s {
u.Add(v)
}
for v := range t {
u.Add(v)
}
return u
}
// Intersection returns s ∩ t
func (s SetTid) Intersection(t SetTid) SetTid {
i := SetTid{}
for v := range s {
if t.Has(v) {
i.Add(v)
}
}
return i
}
// Difference returns s\t.
func (s SetTid) Difference(t SetTid) SetTid {
d := SetTid{}
for v := range s {
if !t.Has(v) {
d.Add(v)
}
}
return d
}
// SymmetricDifference returns s Δ t.
func (s SetTid) SymmetricDifference(t SetTid) SetTid {
d := SetTid{}
for v := range s {
if !t.Has(v) {
d.Add(v)
}
}
for v := range t {
if !s.Has(v) {
d.Add(v)
}
}
return d
}
// Equal returns whether a == b.
func (a SetTid) Equal(b SetTid) bool {
if len(a) != len(b) {
return false
}
for v := range a {
_, ok := b[v]
if !ok {
return false
}
}
return true
}
// Clone returns copy of the set.
func (orig SetTid) Clone() SetTid {
klon := make(SetTid, len(orig))
for v := range orig {
klon.Add(v)
}
return klon
}
// --------
func (s SetTid) SortedElements() []_Tid {
ev := s.Elements()
sort.Slice(ev, func(i, j int) bool {
return ev[i] < ev[j]
})
return ev
}
func (s SetTid) String() string {
ev := s.SortedElements()
strv := make([]string, len(ev))
for i, v := range ev {
strv[i] = fmt.Sprintf("%v", v)
}
return "{" + strings.Join(strv, " ") + "}"
}
......@@ -32,6 +32,7 @@ const VDEL = zodb.InvalidOid
type SetKey = set.SetI64
type SetOid = set.SetOid
type SetTid = set.SetTid
......
......@@ -89,7 +89,7 @@ type ΔBtail struct {
// includes all changed objects, not only tracked ones.
δZtail *zodb.ΔTail
vδBroots []ΔBroots // [] (rev, roots changed in this rev)
vδBroots []ΔBroots // [] (rev, roots changed in this rev)
vδTbyRoot map[zodb.Oid]*ΔTtail // {} root -> [] k/v change history; only for keys ∈ tracked subset XXX -> byRoot?
// set of tracked nodes as of @head state.
......@@ -314,11 +314,12 @@ func (δBtail *ΔBtail) rebuildAll() (err error) {
for root := range trackNewRoots {
δTtail := δBtail.vδTbyRoot[root] // must be there
δtrackSet, err := δTtail.rebuild(root, δBtail.δZtail, δBtail.db)
δtrackSet, δrevSet, err := δTtail.rebuild(root, δBtail.δZtail, δBtail.db)
if err != nil {
return err
}
δBtail.trackSet.UnionInplace(δtrackSet)
δBtail.vδBroots_Update(root, δrevSet)
}
δBtail.trackNewRoots = SetOid{}
......@@ -327,10 +328,14 @@ func (δBtail *ΔBtail) rebuildAll() (err error) {
// rebuild rebuilds ΔTtail taking trackNew requests into account.
//
// It returns set of nodes that must be added to ΔBtail.trackSet to account for
// It returns:
//
// - set of nodes that must be added to ΔBtail.trackSet to account for
// keys that becomes tracked. Note: this set is potentially wider compared to what was in .trackNew.
// - set of revisions for which new entries in .vδT have been created.
//
// XXX place
func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet PPTreeSubSet, err error) {
func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet PPTreeSubSet, δrevSet SetTid, err error) {
defer xerr.Context(&err, "ΔTtail rebuild")
// XXX locking
......@@ -341,9 +346,11 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
δTtail.trackNew = PPTreeSubSet{}
if len(trackNew) == 0 {
return nil, nil
return nil, nil, nil
}
δrevSet = SetTid{}
// go backwards and merge vδT <- treediff(lo..hi/trackNew)
vδZ := δZtail.Data()
for {
......@@ -361,13 +368,16 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
atPrev = δZtail.Tail()
}
δtrackNew, δtkeycov_, err := δTtail.rebuild1(atPrev, δZ, trackNewCur, db)
δtrackNew, δtkeycov_, newRevEntry, err := δTtail.rebuild1(atPrev, δZ, trackNewCur, db)
if err != nil {
return nil, err
return nil, nil, err
}
trackNewCur.ApplyΔ(δtrackNew)
δtkeycov.UnionInplace(δtkeycov_)
if newRevEntry {
δrevSet.Add(δZ.Rev)
}
// XXX update .KVAtTail, .lastRevOf
}
......@@ -393,11 +403,11 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
err := widenTrackNew(trackNew, δtkeycov, root, δZtail.Head(), db)
if err != nil {
return nil, err
return nil, nil, err
}
}
return trackNew, nil
return trackNew, δrevSet, nil
}
// widenTrackNew widens trackNew to cover δtkeycov.
......@@ -440,7 +450,8 @@ func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid
// rebuild1 rebuilds δT for single δZ.
//
// δtrackNew/δtkeycov represents how trackNew changes when going through `atPrev <- δZ.Rev` .
func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew PPTreeSubSet, db *zodb.DB) (δtrackNew *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
// newRevEntry indicates whether δZ.Rev was not there before in .vδT and new corresponding δT entry was created.
func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew PPTreeSubSet, db *zodb.DB) (δtrackNew *ΔPPTreeSubSet, δtkeycov *RangedKeySet, newRevEntry bool, err error) {
defer xerr.Contextf(&err, "rebuild1 %s<-%s", atPrev, δZ.Rev)
debugfΔBtail("\n rebuild1 @%s <- @%s\n", atPrev, δZ.Rev)
......@@ -449,6 +460,7 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
defer func() {
debugfΔBtail("-> δtrackNew: %v\n", δtrackNew)
debugfΔBtail("-> δtkeycov: %v\n", δtkeycov)
debugfΔBtail("-> newRevEntry: %v\n", newRevEntry)
debugfΔBtail("\n\n")
}()
......@@ -456,7 +468,7 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
// skip opening DB connections if there is no change to this tree
if len(δtopsByRoot) == 0 {
return NewΔPPTreeSubSet(), &RangedKeySet{}, nil
return NewΔPPTreeSubSet(), &RangedKeySet{}, false, nil
}
if len(δtopsByRoot) != 1 {
......@@ -476,23 +488,23 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
zconnPrev, err := db.Open(ctx, &zodb.ConnOptions{At: atPrev})
if err != nil {
return nil, nil, err
return nil, nil, false, err
}
zconnCurr, err := db.Open(ctx, &zodb.ConnOptions{At: δZ.Rev})
if err != nil {
return nil, nil, err
return nil, nil, false, err
}
// diff backwards curr -> prev
δT, δtrack, δtkeycov, err := treediff(ctx, root, δtops, δZTC, trackNew, zconnCurr, zconnPrev)
if err != nil {
return nil, nil, err
return nil, nil, false, err
}
debugfΔBtail(" -> root<%s> δkv*: %v δtrack*: %v δtkeycov*: %v\n", root, δT, δtrack, δtkeycov)
if len(δT) == 0 { // an object might be resaved without change
return δtrack, δtkeycov, nil
return δtrack, δtkeycov, false, nil
}
// δTtail.vδT <- merge δT*
......@@ -501,6 +513,7 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
return δZ.Rev <= δTtail.vδT[k].Rev
})
if j == l || δTtail.vδT[j].Rev != δZ.Rev {
newRevEntry = true
δTcurr := ΔTree{Rev: δZ.Rev, ΔKV: map[Key]ΔValue{}}
// insert(@j, δTcurr)
δTtail.vδT = append(δTtail.vδT[:j],
......@@ -524,7 +537,7 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
}
// XXX update .KVAtTail, .lastRevOf (here?)
return δtrack, δtkeycov, nil
return δtrack, δtkeycov, newRevEntry, nil
}
// Update updates δB with per-object-level ZODB changes.
......@@ -558,12 +571,12 @@ func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
// XXX vvv we can skip computing diff for HEAD~..HEAD (we just
// computed it in _Update1)? (or not - trackNew is as of @head ?)
δtrackSet, err := δTtail.rebuild(root, δBtail.δZtail, δBtail.db)
δtrackSet, δrevSet, err := δTtail.rebuild(root, δBtail.δZtail, δBtail.db)
if err != nil {
return ΔB{}, err
}
δBtail.trackSet.UnionInplace(δtrackSet)
δBtail.vδBroots_Update(root, δrevSet)
}
// build δB. Even if δT=ø after _Update1, but δtkeycov1 != ø, above
......@@ -580,6 +593,14 @@ func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
// XXX rebuild KVAtTail
// XXX rebuild lastRevOf
}
// vδBroots += δB
δroots := SetOid{}
for root := range δB.ΔByRoot {
δroots.Add(root)
}
δBtail.vδBroots = append(δBtail.vδBroots, ΔBroots{Rev: δB.Rev, ΔRoots: δroots})
return δB, err
}
......@@ -592,7 +613,7 @@ type _ΔBUpdate1 struct {
}
type _ΔTUpdate1 struct {
δtkeycov1 *RangedKeySet // {} root -> δtrackedKeys after first treediff (always grow)
δtrack *ΔPPTreeSubSet
δtrack *ΔPPTreeSubSet // XXX kill (not used)
}
func (δBtail *ΔBtail) _Update1(δZ *zodb.EventCommit) (δB1 _ΔBUpdate1, err error) {
headOld := δBtail.Head()
......@@ -657,11 +678,69 @@ func (δBtail *ΔBtail) _Update1(δZ *zodb.EventCommit) (δB1 _ΔBUpdate1, err e
return δB1, nil
}
// vδBroots_Update updates .vδBroots to remember that ΔTtail for root has
// changed entries with δrevSet revisions.
//
// XXX place TODO δrevSet -> []rev↑
func (δBtail *ΔBtail) vδBroots_Update(root zodb.Oid, δrevSet SetTid) {
// XXX locking
for rev := range δrevSet {
l := len(δBtail.vδBroots)
j := sort.Search(l, func(k int) bool {
return rev <= δBtail.vδBroots[k].Rev
})
if j == l || δBtail.vδBroots[j].Rev != rev {
δBroots := ΔBroots{Rev: rev, ΔRoots: SetOid{}}
// insert(@j, δBroots)
δBtail.vδBroots = append(δBtail.vδBroots[:j],
append([]ΔBroots{δBroots},
δBtail.vδBroots[j:]...)...)
}
δBroots := δBtail.vδBroots[j]
δBroots.ΔRoots.Add(root)
}
}
// ForgetPast forgets history entries with revision ≤ revCut.
func (δBtail *ΔBtail) ForgetPast(revCut zodb.Tid) {
// XXX locking
δBtail.δZtail.ForgetPast(revCut) // XXX stub
// TODO go through vδBroots till revcut -> find which trees to trim -> trim ΔTtails.
panic("TODO")
// go through vδBroots till revcut -> find which trees to trim -> trim ΔTtails.
totrim := SetOid{} // roots whose ΔTtail has changes ≤ revCut
icut := 0
for ; icut < len(δBtail.vδBroots); icut++ {
δBroots := δBtail.vδBroots[icut]
if δBroots.Rev > revCut {
break
}
totrim.Update(δBroots.ΔRoots)
}
// vδBroots[:icut] should be forgotten
δBtail.vδBroots = append([]ΔBroots(nil), δBtail.vδBroots[icut:]...)
// trim roots
for root := range totrim {
δTtail := δBtail.vδTbyRoot[root] // must be present
δTtail.forgetPast(revCut)
}
}
func (δTtail *ΔTtail) forgetPast(revCut zodb.Tid) {
// XXX KVAtTail, lastRevOf
icut := 0
for ; icut < len(δTtail.vδT); icut++ {
if δTtail.vδT[icut].Rev > revCut {
break
}
}
// vδT[:icut] should be forgotten
δTtail.vδT = append([]ΔTree(nil), δTtail.vδT[icut:]...)
}
......
......@@ -1214,6 +1214,7 @@ func xverifyΔBTail_rebuild_TR(t *testing.T, δbtail *ΔBtail, tj *tTreeCommit,
}
// assertΔTtail verifies state of ΔTtail that corresponds to treeRoot in δbtail.
// it also verifies that δbtail.vδBroots matches ΔTtail data.
func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit, treeRoot zodb.Oid, xat map[zodb.Tid]string, vδTok ...map[Key]Δstring) {
t.Helper()
// XXX +KVAtTail, +lastRevOf
......@@ -1250,16 +1251,40 @@ func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit,
atPrev = δToid.Rev
}
if !(tidvEqual(vat, vatOK) && vδTEqual(vδT, vδTok)) {
var vatδB []zodb.Tid // δbtail.vδBroots/treeRoot
for _, δBroots := range δbtail.vδBroots {
if δBroots.ΔRoots.Has(treeRoot) {
vatδB = append(vatδB, δBroots.Rev)
}
}
tok := tidvEqual(vat, vatOK) && vδTEqual(vδT, vδTok)
bok := tidvEqual(vatδB, vatOK)
if !(tok && bok) {
emsg := fmt.Sprintf("%s: vδT:\n", subj)
have := ""
for i := 0; i<len(vδT); i++ {
have += fmt.Sprintf("\n\t@%s: %v", xat[vat[i]], vδT[i])
}
emsg += fmt.Sprintf("have: %s\n", have)
if !tok {
want := ""
for i := 0; i<len(vδTok); i++ {
want += fmt.Sprintf("\n\t@%s: %v", xat[vatOK[i]], vδTok[i])
}
t.Errorf("%s: vδT:\nhave: %v\nwant: %v", subj, have, want)
emsg += fmt.Sprintf("want: %s\n", want)
}
if !bok {
δbroots := ""
for i := 0; i<len(vatδB); i++ {
δbroots += fmt.Sprintf("\n\t@%s", xat[vatδB[i]])
}
emsg += fmt.Sprintf("δbroots: %s\n", δbroots)
}
t.Error(emsg)
}
}
......@@ -2117,6 +2142,41 @@ func TestΔBTailAllStructs(t *testing.T) {
}
func TestΔBtailForget(t_ *testing.T) {
t := tNewTreeEnv(t_)
X := exc.Raiseif
t0 := t.CommitTree("T/B:")
t1 := t.CommitTree("T/B1:a")
t2 := t.CommitTree("T2/B1:a-B2:b")
t3 := t.CommitTree("T/B2:b")
δbtail := NewΔBtail(t0.at, t.db)
_, err := δbtail.Update(t1.δZ); X(err)
_, err = δbtail.Update(t2.δZ); X(err)
// start tracking. everything becomes tracked because t1's T/B1:a has [-∞,∞) coverage
// By starting tracking after t2 we verify vδBroots update in both Update and rebuild
_0 := SetKey{}; _0.Add(0)
xtrackKeys(δbtail, t2, _0)
_, err = δbtail.Update(t3.δZ); X(err)
xat := map[zodb.Tid]string{
t0.at: "at0",
t1.at: "at1",
t2.at: "at2",
t3.at: "at3",
}
assertΔTtail(t.T, "init", δbtail, t3, t.Root(), xat, t1.δxkv, t2.δxkv, t3.δxkv)
δbtail.ForgetPast(t0.at)
assertΔTtail(t.T, "forget ≤ at0", δbtail, t3, t.Root(), xat, t1.δxkv, t2.δxkv, t3.δxkv)
δbtail.ForgetPast(t1.at)
assertΔTtail(t.T, "forget ≤ at1", δbtail, t3, t.Root(), xat, t2.δxkv, t3.δxkv)
δbtail.ForgetPast(t3.at)
assertΔTtail(t.T, "forget ≤ at3", δbtail, t3, t.Root(), xat, )
}
// ---- misc ----
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment