X δbtail += ForgetPast

wcfs/internal/set/set.go

@@ -21,9 +21,11 @@ package set
 
 //go:generate ./gen-set set I64 int64 zset_i64.go
 //go:generate ./gen-set set Oid _Oid  zset_oid.go
+//go:generate ./gen-set set Tid _Tid  zset_tid.go
 
 import (
 	"lab.nexedi.com/kirr/neo/go/zodb"
 )
 
 type _Oid = zodb.Oid
+type _Tid = zodb.Tid

wcfs/internal/set/zset_tid.go

+// Code generated by gen-set Tid _Tid; DO NOT EDIT.
+
+// Copyright (C) 2015-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package set
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// SetTid is a set of _Tid.
+type SetTid map[_Tid]struct{}
+
+// Add adds v to the set.
+func (s SetTid) Add(v _Tid) {
+	s[v] = struct{}{}
+}
+
+// Del removes v from the set.
+// it is noop if v was not in the set.
+func (s SetTid) Del(v _Tid) {
+	delete(s, v)
+}
+
+// Has checks whether the set contains v.
+func (s SetTid) Has(v _Tid) bool {
+	_, ok := s[v]
+	return ok
+}
+
+// Update adds t values to s.
+func (s SetTid) Update(t SetTid) {
+	for v := range t {
+		s.Add(v)
+	}
+}
+
+// Elements returns all elements of set as slice.
+func (s SetTid) Elements() []_Tid {
+	ev := make([]_Tid, len(s))
+	i := 0
+	for e := range s {
+		ev[i] = e
+		i++
+	}
+	return ev
+}
+
+// Union returns s ∪ t
+func (s SetTid) Union(t SetTid) SetTid {
+	// l = max(len(s), len(t))
+	l := len(s)
+	if lt := len(t); lt > l {
+		l = lt
+	}
+
+	u := make(SetTid, l)
+
+	for v := range s {
+		u.Add(v)
+	}
+	for v := range t {
+		u.Add(v)
+	}
+	return u
+}
+
+// Intersection returns s ∩ t
+func (s SetTid) Intersection(t SetTid) SetTid {
+	i := SetTid{}
+	for v := range s {
+		if t.Has(v) {
+			i.Add(v)
+		}
+	}
+	return i
+}
+
+// Difference returns s\t.
+func (s SetTid) Difference(t SetTid) SetTid {
+	d := SetTid{}
+	for v := range s {
+		if !t.Has(v) {
+			d.Add(v)
+		}
+	}
+	return d
+}
+
+// SymmetricDifference returns s Δ t.
+func (s SetTid) SymmetricDifference(t SetTid) SetTid {
+	d := SetTid{}
+	for v := range s {
+		if !t.Has(v) {
+			d.Add(v)
+		}
+	}
+	for v := range t {
+		if !s.Has(v) {
+			d.Add(v)
+		}
+	}
+	return d
+}
+
+// Equal returns whether a == b.
+func (a SetTid) Equal(b SetTid) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for v := range a {
+		_, ok := b[v]
+		if !ok {
+			return false
+		}
+	}
+
+	return true
+}
+
+// Clone returns copy of the set.
+func (orig SetTid) Clone() SetTid {
+	klon := make(SetTid, len(orig))
+	for v := range orig {
+		klon.Add(v)
+	}
+	return klon
+}
+
+// --------
+
+func (s SetTid) SortedElements() []_Tid {
+	ev := s.Elements()
+	sort.Slice(ev, func(i, j int) bool {
+		return ev[i] < ev[j]
+	})
+	return ev
+}
+
+func (s SetTid) String() string {
+	ev := s.SortedElements()
+	strv := make([]string, len(ev))
+	for i, v := range ev {
+		strv[i] = fmt.Sprintf("%v", v)
+	}
+	return "{" + strings.Join(strv, " ") + "}"
+}

wcfs/internal/xbtree/xbtree.go

@@ -32,6 +32,7 @@ const VDEL  = zodb.InvalidOid
 
 type SetKey = set.SetI64
 type SetOid = set.SetOid
+type SetTid = set.SetTid
 
 
 

wcfs/internal/xbtree/δbtail.go

@@ -89,7 +89,7 @@ type ΔBtail struct {
 	// includes all changed objects, not only tracked ones.
 	δZtail *zodb.ΔTail
 
-	vδBroots  []ΔBroots            // [] (rev, roots changed in this rev)
+	vδBroots  []ΔBroots            // [] (rev↑, roots changed in this rev)
 	vδTbyRoot map[zodb.Oid]*ΔTtail // {} root -> [] k/v change history; only for keys ∈ tracked subset	XXX -> byRoot?
 
 	// set of tracked nodes as of @head state.
@@ -314,11 +314,12 @@ func (δBtail *ΔBtail) rebuildAll() (err error) {
 
 	for root := range trackNewRoots {
 		δTtail := δBtail.vδTbyRoot[root] // must be there
-		δtrackSet, err := δTtail.rebuild(root, δBtail.δZtail, δBtail.db)
+		δtrackSet, δrevSet, err := δTtail.rebuild(root, δBtail.δZtail, δBtail.db)
 		if err != nil {
 			return err
 		}
 		δBtail.trackSet.UnionInplace(δtrackSet)
+		δBtail.vδBroots_Update(root, δrevSet)
 	}
 
 	δBtail.trackNewRoots = SetOid{}
@@ -327,10 +328,14 @@ func (δBtail *ΔBtail) rebuildAll() (err error) {
 
 // rebuild rebuilds ΔTtail taking trackNew requests into account.
 //
-// It returns set of nodes that must be added to ΔBtail.trackSet to account for
-// keys that becomes tracked. Note: this set is potentially wider compared to what was in .trackNew.
+// It returns:
+//
+// - set of nodes that must be added to ΔBtail.trackSet to account for
+//   keys that becomes tracked. Note: this set is potentially wider compared to what was in .trackNew.
+// - set of revisions for which new entries in .vδT have been created.
+//
 // XXX place
-func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet PPTreeSubSet, err error) {
+func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet PPTreeSubSet, δrevSet SetTid, err error) {
 	defer xerr.Context(&err, "ΔTtail rebuild")
 	// XXX locking
 
@@ -341,9 +346,11 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
 	δTtail.trackNew = PPTreeSubSet{}
 
 	if len(trackNew) == 0 {
-		return nil, nil
+		return nil, nil, nil
 	}
 
+	δrevSet = SetTid{}
+
 	// go backwards and merge vδT <- treediff(lo..hi/trackNew)
 	vδZ := δZtail.Data()
 	for {
@@ -361,13 +368,16 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
 				atPrev = δZtail.Tail()
 			}
 
-			δtrackNew, δtkeycov_, err := δTtail.rebuild1(atPrev, δZ, trackNewCur, db)
+			δtrackNew, δtkeycov_, newRevEntry, err := δTtail.rebuild1(atPrev, δZ, trackNewCur, db)
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 
 			trackNewCur.ApplyΔ(δtrackNew)
 			δtkeycov.UnionInplace(δtkeycov_)
+			if newRevEntry {
+				δrevSet.Add(δZ.Rev)
+			}
 
 			// XXX update .KVAtTail, .lastRevOf
 		}
@@ -393,11 +403,11 @@ func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB
 
 		err := widenTrackNew(trackNew, δtkeycov, root, δZtail.Head(), db)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 	}
 
-	return trackNew, nil
+	return trackNew, δrevSet, nil
 }
 
 // widenTrackNew widens trackNew to cover δtkeycov.
@@ -440,15 +450,17 @@ func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid
 // rebuild1 rebuilds δT for single δZ.
 //
 // δtrackNew/δtkeycov represents how trackNew changes when going through `atPrev <- δZ.Rev` .
-func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew PPTreeSubSet, db *zodb.DB) (δtrackNew *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
+// newRevEntry indicates whether δZ.Rev was not there before in .vδT and new corresponding δT entry was created.
+func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew PPTreeSubSet, db *zodb.DB) (δtrackNew *ΔPPTreeSubSet, δtkeycov *RangedKeySet, newRevEntry bool, err error) {
 	defer xerr.Contextf(&err, "rebuild1 %s<-%s", atPrev, δZ.Rev)
 
 	debugfΔBtail("\n  rebuild1 @%s <- @%s\n", atPrev, δZ.Rev)
 	debugfΔBtail("  δZ:\t%v\n", δZ.Changev)
 	debugfΔBtail("  trackNew:  %v\n", trackNew)
 	defer func() {
-		debugfΔBtail("-> δtrackNew: %v\n", δtrackNew)
-		debugfΔBtail("-> δtkeycov:  %v\n", δtkeycov)
+		debugfΔBtail("-> δtrackNew:   %v\n", δtrackNew)
+		debugfΔBtail("-> δtkeycov:    %v\n", δtkeycov)
+		debugfΔBtail("-> newRevEntry: %v\n", newRevEntry)
 		debugfΔBtail("\n\n")
 	}()
 
@@ -456,7 +468,7 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
 
 	// skip opening DB connections if there is no change to this tree
 	if len(δtopsByRoot) == 0 {
-		return NewΔPPTreeSubSet(), &RangedKeySet{}, nil
+		return NewΔPPTreeSubSet(), &RangedKeySet{}, false, nil
 	}
 
 	if len(δtopsByRoot) != 1 {
@@ -476,23 +488,23 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
 
 	zconnPrev, err := db.Open(ctx, &zodb.ConnOptions{At: atPrev})
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, false, err
 	}
 	zconnCurr, err := db.Open(ctx, &zodb.ConnOptions{At: δZ.Rev})
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, false, err
 	}
 
 	// diff backwards curr -> prev
 	δT, δtrack, δtkeycov, err := treediff(ctx, root, δtops, δZTC, trackNew, zconnCurr, zconnPrev)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, false, err
 	}
 
 	debugfΔBtail("  -> root<%s>  δkv*: %v  δtrack*: %v  δtkeycov*: %v\n", root, δT, δtrack, δtkeycov)
 
 	if len(δT) == 0 { // an object might be resaved without change
-		return δtrack, δtkeycov, nil
+		return δtrack, δtkeycov, false, nil
 	}
 
 	// δTtail.vδT <- merge δT*
@@ -501,6 +513,7 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
 		return δZ.Rev <= δTtail.vδT[k].Rev
 	})
 	if j == l || δTtail.vδT[j].Rev != δZ.Rev {
+		newRevEntry = true
 		δTcurr := ΔTree{Rev: δZ.Rev, ΔKV: map[Key]ΔValue{}}
 		// insert(@j, δTcurr)
 		δTtail.vδT = append(δTtail.vδT[:j],
@@ -524,7 +537,7 @@ func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew
 	}
 
 	// XXX update .KVAtTail, .lastRevOf	(here?)
-	return δtrack, δtkeycov, nil
+	return δtrack, δtkeycov, newRevEntry, nil
 }
 
 // Update updates δB with per-object-level ZODB changes.
@@ -558,12 +571,12 @@ func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
 
 			// XXX vvv we can skip computing diff for HEAD~..HEAD (we just
 			//     computed it in _Update1)? (or not - trackNew is as of @head ?)
-			δtrackSet, err := δTtail.rebuild(root, δBtail.δZtail, δBtail.db)
+			δtrackSet, δrevSet, err := δTtail.rebuild(root, δBtail.δZtail, δBtail.db)
 			if err != nil {
 				return ΔB{}, err
 			}
-
 			δBtail.trackSet.UnionInplace(δtrackSet)
+			δBtail.vδBroots_Update(root, δrevSet)
 		}
 
 		// build δB. Even if δT=ø after _Update1, but δtkeycov1 != ø, above
@@ -580,6 +593,14 @@ func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
 		// XXX rebuild KVAtTail
 		// XXX rebuild lastRevOf
 	}
+
+	// vδBroots += δB
+	δroots := SetOid{}
+	for root := range δB.ΔByRoot {
+		δroots.Add(root)
+	}
+	δBtail.vδBroots = append(δBtail.vδBroots, ΔBroots{Rev: δB.Rev, ΔRoots: δroots})
+
 	return δB, err
 }
 
@@ -592,7 +613,7 @@ type _ΔBUpdate1 struct {
 }
 type _ΔTUpdate1 struct {
 	δtkeycov1 *RangedKeySet   // {} root -> δtrackedKeys after first treediff (always grow)
-	δtrack    *ΔPPTreeSubSet
+	δtrack    *ΔPPTreeSubSet  // XXX kill (not used)
 }
 func (δBtail *ΔBtail) _Update1(δZ *zodb.EventCommit) (δB1 _ΔBUpdate1, err error) {
 	headOld := δBtail.Head()
@@ -657,11 +678,69 @@ func (δBtail *ΔBtail) _Update1(δZ *zodb.EventCommit) (δB1 _ΔBUpdate1, err e
 	return δB1, nil
 }
 
+// vδBroots_Update updates .vδBroots to remember that ΔTtail for root has
+// changed entries with δrevSet revisions.
+//
+// XXX place  TODO δrevSet -> []rev↑
+func (δBtail *ΔBtail) vδBroots_Update(root zodb.Oid, δrevSet SetTid) {
+	// XXX locking
+	for rev := range δrevSet {
+		l := len(δBtail.vδBroots)
+		j := sort.Search(l, func(k int) bool {
+			return rev <= δBtail.vδBroots[k].Rev
+		})
+		if j == l || δBtail.vδBroots[j].Rev != rev {
+			δBroots := ΔBroots{Rev: rev, ΔRoots: SetOid{}}
+			// insert(@j, δBroots)
+			δBtail.vδBroots = append(δBtail.vδBroots[:j],
+						 append([]ΔBroots{δBroots},
+						 δBtail.vδBroots[j:]...)...)
+		}
+		δBroots := δBtail.vδBroots[j]
+		δBroots.ΔRoots.Add(root)
+	}
+}
 
+// ForgetPast forgets history entries with revision ≤ revCut.
 func (δBtail *ΔBtail) ForgetPast(revCut zodb.Tid) {
+	// XXX locking
+
 	δBtail.δZtail.ForgetPast(revCut) // XXX stub
-	// TODO go through vδBroots till revcut -> find which trees to trim -> trim ΔTtails.
-	panic("TODO")
+
+	// go through vδBroots till revcut -> find which trees to trim -> trim ΔTtails.
+
+	totrim := SetOid{} // roots whose ΔTtail has changes ≤ revCut
+	icut := 0
+	for ; icut < len(δBtail.vδBroots); icut++ {
+		δBroots := δBtail.vδBroots[icut]
+		if δBroots.Rev > revCut {
+			break
+		}
+		totrim.Update(δBroots.ΔRoots)
+	}
+
+	// vδBroots[:icut] should be forgotten
+	δBtail.vδBroots = append([]ΔBroots(nil), δBtail.vδBroots[icut:]...)
+
+	// trim roots
+	for root := range totrim {
+		δTtail := δBtail.vδTbyRoot[root] // must be present
+		δTtail.forgetPast(revCut)
+	}
+}
+
+func (δTtail *ΔTtail) forgetPast(revCut zodb.Tid) {
+	// XXX KVAtTail, lastRevOf
+
+	icut := 0
+	for ; icut < len(δTtail.vδT); icut++ {
+		if δTtail.vδT[icut].Rev > revCut {
+			break
+		}
+	}
+
+	// vδT[:icut] should be forgotten
+	δTtail.vδT = append([]ΔTree(nil), δTtail.vδT[icut:]...)
 }
 
 

wcfs/internal/xbtree/δbtail_test.go

@@ -1214,6 +1214,7 @@ func xverifyΔBTail_rebuild_TR(t *testing.T, δbtail *ΔBtail, tj *tTreeCommit,
 }
 
 // assertΔTtail verifies state of ΔTtail that corresponds to treeRoot in δbtail.
+// it also verifies that δbtail.vδBroots matches ΔTtail data.
 func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit, treeRoot zodb.Oid, xat map[zodb.Tid]string, vδTok ...map[Key]Δstring) {
 	t.Helper()
 	// XXX +KVAtTail, +lastRevOf
@@ -1250,16 +1251,40 @@ func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit,
 		atPrev = δToid.Rev
 	}
 
-	if !(tidvEqual(vat, vatOK) && vδTEqual(vδT, vδTok)) {
+	var vatδB []zodb.Tid // δbtail.vδBroots/treeRoot
+	for _, δBroots := range δbtail.vδBroots {
+		if δBroots.ΔRoots.Has(treeRoot) {
+			vatδB = append(vatδB, δBroots.Rev)
+		}
+	}
+
+	tok := tidvEqual(vat,   vatOK) && vδTEqual(vδT, vδTok)
+	bok := tidvEqual(vatδB, vatOK)
+	if !(tok && bok) {
+		emsg := fmt.Sprintf("%s: vδT:\n", subj)
 		have := ""
 		for i := 0; i<len(vδT); i++ {
 			have += fmt.Sprintf("\n\t@%s: %v", xat[vat[i]], vδT[i])
 		}
-		want := ""
-		for i := 0; i<len(vδTok); i++ {
-			want += fmt.Sprintf("\n\t@%s: %v", xat[vatOK[i]], vδTok[i])
+		emsg += fmt.Sprintf("have: %s\n", have)
+
+		if !tok {
+			want := ""
+			for i := 0; i<len(vδTok); i++ {
+				want += fmt.Sprintf("\n\t@%s: %v", xat[vatOK[i]], vδTok[i])
+			}
+			emsg += fmt.Sprintf("want: %s\n", want)
+		}
+
+		if !bok {
+			δbroots := ""
+			for i := 0; i<len(vatδB); i++ {
+				δbroots += fmt.Sprintf("\n\t@%s", xat[vatδB[i]])
+			}
+			emsg += fmt.Sprintf("δbroots: %s\n", δbroots)
 		}
-		t.Errorf("%s: vδT:\nhave: %v\nwant: %v", subj, have, want)
+
+		t.Error(emsg)
 	}
 }
 
@@ -2117,6 +2142,41 @@ func TestΔBTailAllStructs(t *testing.T) {
 }
 
 
+func TestΔBtailForget(t_ *testing.T) {
+	t := tNewTreeEnv(t_)
+	X := exc.Raiseif
+
+	t0 := t.CommitTree("T/B:")
+	t1 := t.CommitTree("T/B1:a")
+	t2 := t.CommitTree("T2/B1:a-B2:b")
+	t3 := t.CommitTree("T/B2:b")
+
+	δbtail := NewΔBtail(t0.at, t.db)
+	_, err := δbtail.Update(t1.δZ);  X(err)
+	_, err  = δbtail.Update(t2.δZ);  X(err)
+
+	// start tracking. everything becomes tracked because t1's T/B1:a has [-∞,∞) coverage
+	// By starting tracking after t2 we verify vδBroots update in both Update and rebuild
+	_0 := SetKey{}; _0.Add(0)
+	xtrackKeys(δbtail, t2, _0)
+
+	_, err  = δbtail.Update(t3.δZ);  X(err)
+
+	xat := map[zodb.Tid]string{
+		t0.at: "at0",
+		t1.at: "at1",
+		t2.at: "at2",
+		t3.at: "at3",
+	}
+	assertΔTtail(t.T, "init",         δbtail, t3, t.Root(), xat, t1.δxkv, t2.δxkv, t3.δxkv)
+	δbtail.ForgetPast(t0.at)
+	assertΔTtail(t.T, "forget ≤ at0", δbtail, t3, t.Root(), xat, t1.δxkv, t2.δxkv, t3.δxkv)
+	δbtail.ForgetPast(t1.at)
+	assertΔTtail(t.T, "forget ≤ at1", δbtail, t3, t.Root(), xat,          t2.δxkv, t3.δxkv)
+	δbtail.ForgetPast(t3.at)
+	assertΔTtail(t.T, "forget ≤ at3", δbtail, t3, t.Root(), xat,                          )
+}
+
 
 // ---- misc ----