ΔFtail concurrency

See changes in δftail.go for overview.

* t2+ΔFtail-concurrency: (39 commits)
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  X zdata: Switch SliceByFileRev not to clone Zinblk
  .
  .
  .
  .
  .
  ...

wcfs/internal/xbtree/xbtreetest/treeenv.go

@@ -137,31 +137,55 @@ func (t *T) Head() *Commit {
 
 // XGetCommit finds and returns Commit created with revision at.
 func (t *T) XGetCommit(at zodb.Tid) *Commit {
+	commit, _, _ := t.getCommit(at)
+	if commit == nil {
+		panicf("no commit corresponding to @%s", at)
+	}
+	return commit
+}
+
+func (t *T) getCommit(at zodb.Tid) (commit, cprev, cnext *Commit) {
 	l := len(t.commitv)
 	i := sort.Search(l, func(i int) bool {
 		return at <= t.commitv[i].At
 	})
-	var commit *Commit
 	if i < l {
 		commit = t.commitv[i]
 		if commit.At != at {
+			cnext = commit
 			commit = nil
+		} else if i+1 < l {
+			cnext = t.commitv[i+1]
 		}
 	}
-	if commit == nil {
-		panicf("no commit corresponding to @%s", at)
+	if i > 0 {
+		cprev = t.commitv[i-1]
 	}
-	if commit.idx != i {
+	if commit != nil && commit.idx != i {
 		panicf("BUG: commit.idx (%d) != i (%d)", commit.idx, i)
 	}
-	return commit
+	return commit, cprev, cnext
 }
 
 // AtSymb returns symbolic representation of at, for example "at3".
 //
-// at must correspond to a Commit.
+// at should correspond to a Commit.
 func (t *T) AtSymb(at zodb.Tid) string {
-	return t.XGetCommit(at).AtSymb()
+	commit, cprev, cnext := t.getCommit(at)
+	if commit != nil {
+		return commit.AtSymb()
+	}
+
+	// at does not correspond to commit - return something like ~at2<xxxx>at3
+	s := "~"
+	if cprev != nil {
+		s += cprev.AtSymb() + "<"
+	}
+	s += at.String()
+	if cnext != nil {
+		s += ">" + cnext.AtSymb()
+	}
+	return s
 }
 
 // AtSymb returns symbolic representation of c.At, for example "at3".

wcfs/internal/xbtree/δbtail.go

@@ -69,7 +69,9 @@ package xbtree
 // Concurrency
 //
 // In order to allow multiple Track and queries requests to be served in
-// parallel ΔBtail employs special organization of vδT rebuild process:
+// parallel ΔBtail employs special organization of vδT rebuild process where
+// complexity of concurrency is reduced to math on merging updates to vδT and
+// trackSet, and on key range lookup:
 //
 // 1. vδT is managed under read-copy-update (RCU) discipline: before making
 //    any vδT change the mutator atomically clones whole vδT and applies its
@@ -117,13 +119,21 @@ package xbtree
 //
 //	vδT/(T₁∪T₂) = vδT/T₁ | vδT/T₂
 //
-//    i.e. vδT computed for tracked set being union of T₁ and T₂ is the same
-//    as merge of vδT computed for tracked set T₁ and vδT computed for tracked
-//    set T₂.
+//      ( i.e. vδT computed for tracked set being union of T₁ and T₂ is the
+//        same as merge of vδT computed for tracked set T₁ and vδT computed
+//	  for tracked set T₂ )
 //
-//    this merge property allows to run computation for δ(vδT) independently
-//    and with ΔBtail unlocked, which in turn enables running several
-//    Track/queries in parallel.
+//    and that
+//
+//	trackSet | (δPP₁|δPP₂) = (trackSet|δPP₁) | (trackSet|δPP₂)
+//
+//	( i.e. tracking set updated for union of δPP₁ and δPP₂ is the same
+//	  as union of tracking set updated with δPP₁ and tracking set updated
+//	  with δPP₂ )
+//
+//    these merge properties allow to run computation for δ(vδT) and δ(trackSet)
+//    independently and with ΔBtail unlocked, which in turn enables running
+//    several Track/queries in parallel.
 //
 // 4. while vδT rebuild is being run, krebuildJobs keeps corresponding keycov
 //    entry to indicate in-progress rebuild. Should a query need vδT for keys
@@ -247,9 +257,9 @@ type _ΔTtail struct {
 	vδT []ΔTree
 
 	// set of keys that were requested to be tracked in this tree,
-	// but for which vδT rebuild was not yet started
+	// but for which vδT rebuild was not yet started	as of @head
 	ktrackNew blib.RangedKeySet        // {keycov}
-	// set of nodes corresponding to ktrackNew
+	// set of nodes corresponding to ktrackNew		as of @head
 	trackNew blib.PPTreeSubSet         // PP{nodes}
 
 	// set of keys(nodes) for which rebuild is in progress
@@ -672,13 +682,13 @@ func (δTtail *_ΔTtail) __rebuild(root zodb.Oid, δBtail *ΔBtail, releaseLock
 //
 // TODO optionally accept zconnOld/zconnNew from client
 func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
+	headOld := δBtail.Head()
+	defer xerr.Contextf(&err, "ΔBtail.Update %s -> %s", headOld, δZ.Tid)
+
 	δBtail.mu.Lock()
 	defer δBtail.mu.Unlock()
 	// TODO verify that there is no in-progress readers/writers
 
-	headOld := δBtail.Head()
-	defer xerr.Contextf(&err, "ΔBtail.Update %s -> %s", headOld, δZ.Tid)
-
 	δB1, err := δBtail._Update1(δZ)
 
 	δB := ΔB{Rev: δZ.Tid, ByRoot: make(map[zodb.Oid]map[Key]ΔValue)}
@@ -997,7 +1007,7 @@ func (δBtail *ΔBtail) GetAt(root zodb.Oid, key Key, at zodb.Tid) (value Value,
 // Only tracked keys are guaranteed to be present.
 //
 // Note: contrary to regular go slicing, low is exclusive while high is inclusive.
-func (δBtail *ΔBtail) SliceByRootRev(root zodb.Oid, lo, hi zodb.Tid) (/*readonly*/vδT []ΔTree) {
+func (δBtail *ΔBtail) SliceByRootRev(root zodb.Oid, lo, hi zodb.Tid) (/*readonly*/vδT []ΔTree, err error) {
 	xtail.AssertSlice(δBtail, lo, hi)
 
 	if traceΔBtail {
@@ -1008,22 +1018,22 @@ func (δBtail *ΔBtail) SliceByRootRev(root zodb.Oid, lo, hi zodb.Tid) (/*readon
 	}
 
 	// retrieve vδT snapshot that is rebuilt to take all previous Track requests into account
-	vδT, err := δBtail.vδTSnapForTracked(root)
+	vδT, err = δBtail.vδTSnapForTracked(root)
 	if err != nil {
-		panic(err) // XXX
+		return nil, err
 	}
 	debugfΔBtail("  vδT: %v\n", vδT)
 
 	l := len(vδT)
 	if l == 0 {
-		return nil
+		return nil, nil
 	}
 
 	// find max j : [j].rev ≤ hi		linear scan -> TODO binary search
 	j := l - 1
 	for ; j >= 0 && vδT[j].Rev > hi; j-- {}
 	if j < 0 {
-		return nil // ø
+		return nil, nil // ø
 	}
 
 	// find max i : [i].rev > lo		linear scan -> TODO binary search
@@ -1035,7 +1045,7 @@ func (δBtail *ΔBtail) SliceByRootRev(root zodb.Oid, lo, hi zodb.Tid) (/*readon
 	// modified via RCU: i.e. _ΔTtail.rebuild clones vδT before modifying it.
 	// This way the data we return to caller will stay unchanged even if
 	// rebuild is running simultaneously.
-	return vδT[i:j+1]
+	return vδT[i:j+1], nil
 }
 
 

wcfs/internal/xbtree/δbtail_test.go

@@ -1246,11 +1246,11 @@ func TestΔBtailSliceByRootRev(t_ *testing.T) {
 		t.Errorf("%s:\nhave: %s\nwant: %s", subj, have, want)
 	}
 
-	s00 := δbtail.SliceByRootRev(t.Root(), t0.At, t0.At)
-	s01 := δbtail.SliceByRootRev(t.Root(), t0.At, t1.At)
-	s02 := δbtail.SliceByRootRev(t.Root(), t0.At, t2.At)
-	s12 := δbtail.SliceByRootRev(t.Root(), t1.At, t2.At)
-	s22 := δbtail.SliceByRootRev(t.Root(), t2.At, t2.At)
+	s00, err := δbtail.SliceByRootRev(t.Root(), t0.At, t0.At);  X(err)
+	s01, err := δbtail.SliceByRootRev(t.Root(), t0.At, t1.At);  X(err)
+	s02, err := δbtail.SliceByRootRev(t.Root(), t0.At, t2.At);  X(err)
+	s12, err := δbtail.SliceByRootRev(t.Root(), t1.At, t2.At);  X(err)
+	s22, err := δbtail.SliceByRootRev(t.Root(), t2.At, t2.At);  X(err)
 
 	vδT := δttail.vδT
 	assertvδT("t2.vδT",  vδT,        ΔT{t1.At, δ{2:{f,g}}},   ΔT{t2.At, δ{2:{g,h}}})
@@ -1286,11 +1286,11 @@ func TestΔBtailSliceByRootRev(t_ *testing.T) {
 	trackKeys(δbtail, t2, _1)
 	err = δbtail._rebuildAll();  X(err)
 
-	s00_ := δbtail.SliceByRootRev(t.Root(), t0.At, t0.At)
-	s01_ := δbtail.SliceByRootRev(t.Root(), t0.At, t1.At)
-	s02_ := δbtail.SliceByRootRev(t.Root(), t0.At, t2.At)
-	s12_ := δbtail.SliceByRootRev(t.Root(), t1.At, t2.At)
-	s22_ := δbtail.SliceByRootRev(t.Root(), t2.At, t2.At)
+	s00_, err := δbtail.SliceByRootRev(t.Root(), t0.At, t0.At);  X(err)
+	s01_, err := δbtail.SliceByRootRev(t.Root(), t0.At, t1.At);  X(err)
+	s02_, err := δbtail.SliceByRootRev(t.Root(), t0.At, t2.At);  X(err)
+	s12_, err := δbtail.SliceByRootRev(t.Root(), t1.At, t2.At);  X(err)
+	s22_, err := δbtail.SliceByRootRev(t.Root(), t2.At, t2.At);  X(err)
 
 	vδT = δttail.vδT
 	assertvδT("t12.vδT",   vδT,     ΔT{t1.At, δ{1:{a,b},2:{f,g}}},  ΔT{t2.At, δ{1:{b,c},2:{g,h}}})

wcfs/internal/zdata/δftail.go

@@ -58,12 +58,44 @@ package zdata
 // tracked and is thus easy to maintain. It also can be maintained only in
 // ΔFtail because ΔBtail and ΔZtail does not "know" anything about ZBigFile.
 //
-// XXX concurrency
+//
+// Concurrency
+//
+// In order to allow multiple Track and queries requests to be served in
+// parallel, ΔFtail bases its concurrency promise on ΔBtail guarantees +
+// snapshot-style access for vδE and ztrackInBlk in queries:
+//
+// 1. Track calls ΔBtail.Track and quickly updates .byFile, .byRoot and
+//    _RootTrack indices under a lock.
+//
+// 2. BlkRevAt queries ΔBtail.GetAt and then combines retrieved information
+//    about zblk with vδE and δZ.
+//
+// 3. SliceByFileRev queries ΔBtail.SliceByRootRev and then merges retrieved
+//    vδT data with vδZ, vδE and ztrackInBlk.
+//
+// 4. In queries vδE is retrieved/built in snapshot style similarly to how vδT
+//    is built in ΔBtail. Note that vδE needs to be built only the first time,
+//    and does not need to be further rebuilt, so the logic in ΔFtail is simpler
+//    compared to ΔBtail.
+//
+// 5. for ztrackInBlk - that is used by SliceByFileRev query - an atomic
+//    snapshot is retrieved for objects of interest. This allows to hold
+//    δFtail.mu lock for relatively brief time without blocking other parallel
+//    Track/queries requests for long.
+//
+// Combined this organization allows non-overlapping queries/track-requests
+// to run simultaneously. This property is essential to WCFS because otherwise
+// WCFS would not be able to serve several non-overlapping READ requests to one
+// file in parallel.
+//
+// See also "Concurrency" in ΔBtail organization for more details.
 
 import (
 	"context"
 	"fmt"
 	"sort"
+	"sync"
 
 	"lab.nexedi.com/kirr/go123/xerr"
 	"lab.nexedi.com/kirr/neo/go/transaction"
@@ -116,26 +148,52 @@ type setOid = set.Oid
 //		.rev↑
 //		{}blk | EPOCH
 //
-// XXX concurrent use.
-//
 // See also zodb.ΔTail and xbtree.ΔBtail
+//
+//
+// Concurrency
+//
+// ΔFtail is safe to use in single-writer / multiple-readers mode. That is at
+// any time there should be either only sole writer, or, potentially several
+// simultaneous readers. The table below classifies operations:
+//
+//	Writers:  Update, ForgetPast
+//	Readers:  Track + all queries (SliceByRev, SliceByFileRev, BlkRevAt)
+//
+// Note that, in particular, it is correct to run multiple Track and queries
+// requests simultaneously.
 type ΔFtail struct {
 	// ΔFtail merges ΔBtail with history of ZBlk
 	δBtail      *xbtree.ΔBtail
-	byFile      map[zodb.Oid]*_ΔFileTail // file -> vδf tail
-	filesByRoot map[zodb.Oid]setOid      // tree-root -> {} ZBigFile<oid>    as of @head
+
+	// mu protects ΔFtail data _and_ all _ΔFileTail/_RootTrack data for all files and roots.
+	//
+	// NOTE: even though this lock is global it is used only for brief periods of time. In
+	// particular working with retrieved vδE and Zinblk snapshot does not need to hold the lock.
+	mu sync.Mutex
+
+	byFile map[zodb.Oid]*_ΔFileTail // file -> vδf tail
+	byRoot map[zodb.Oid]*_RootTrack // tree-root -> ({foid}, Zinblk)   as of @head
 
 	// set of files, which are newly tracked and for which byFile[foid].vδE was not yet rebuilt
-	trackNew setOid // {}foid
+	ftrackNew setOid // {}foid
 
-	// set of tracked ZBlk objects reverse-mapped to trees and block numbers
-	trackSetZBlk  map[zodb.Oid]*zblkTrack // zblk -> {} root -> {}blk    as of @head
+	// set of tracked ZBlk objects mapped to tree roots  as of @head
+	ztrackInRoot map[zodb.Oid]setOid // {} zblk -> {}root
+}
+
+// _RootTrack represents tracking information about one particular tree as of @head.
+type _RootTrack struct {
+	ftrackSet   setOid              // {}foid		which ZBigFiles refer to this tree
+	ztrackInBlk map[zodb.Oid]setI64 // {} zblk -> {}blk	which blocks map to zblk
 }
 
 // _ΔFileTail represents tail of revisional changes to one file.
 type _ΔFileTail struct {
 	root zodb.Oid      // .blktab as of @head
 	vδE  []_ΔFileEpoch // epochs (changes to ZBigFile object itself)  ; nil if not yet rebuilt
+
+	rebuildJob *_RebuildJob // !nil if vδE rebuild is currently in-progress
 }
 
 // _ΔFileEpoch represent a change to ZBigFile object.
@@ -146,13 +204,14 @@ type _ΔFileEpoch struct {
 	oldBlkSize int64    // .blksize was oldBlkSize                  ; -1   if ZBigFile deleted
 	newBlkSize int64    // .blksize was changed to newBlkSize       ; ----//----
 
-	// snapshot of trackSetZBlk for this file right before this epoch
-	oldTrackSetZBlk map[zodb.Oid]setI64 // {} zblk -> {}blk
+	// snapshot of ztrackInBlk for this file right before this epoch
+	oldZinblk map[zodb.Oid]setI64 // {} zblk -> {}blk
 }
 
-// zblkTrack keeps information in which root/blocks ZBlk is present as of @head.
-type zblkTrack struct {
-	inroot map[zodb.Oid]setI64  // {} root -> {}blk
+// _RebuildJob represents currently in-progress vδE rebuilding job.
+type _RebuildJob struct {
+	ready chan struct{} // closed when job completes
+	err   error
 }
 
 
@@ -180,11 +239,11 @@ type ΔFile struct {
 // ZODB when needed.
 func NewΔFtail(at0 zodb.Tid, db *zodb.DB) *ΔFtail {
 	return &ΔFtail{
-		δBtail:        xbtree.NewΔBtail(at0, db),
-		byFile:        map[zodb.Oid]*_ΔFileTail{},
-		filesByRoot:   map[zodb.Oid]setOid{},
-		trackNew:      setOid{},
-		trackSetZBlk:  map[zodb.Oid]*zblkTrack{},
+		δBtail:       xbtree.NewΔBtail(at0, db),
+		byFile:       map[zodb.Oid]*_ΔFileTail{},
+		byRoot:       map[zodb.Oid]*_RootTrack{},
+		ftrackNew:    setOid{},
+		ztrackInRoot: map[zodb.Oid]setOid{},
 	}
 }
 
@@ -205,8 +264,6 @@ func (δFtail *ΔFtail) Tail() zodb.Tid { return δFtail.δBtail.Tail() }
 //
 // Objects in path and zblk must be with .PJar().At() == .head
 func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, blkcov btree.LKeyRange, zblk ZBlk) {
-	// XXX locking
-
 	head := δFtail.Head()
 
 	fileAt := file.PJar().At()
@@ -226,18 +283,25 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, bl
 
 	rootObj := path[0].(*btree.LOBTree)
 	root    := rootObj.POid()
-	files, ok := δFtail.filesByRoot[root]
+
+	δFtail.mu.Lock()
+	defer δFtail.mu.Unlock()
+
+	rt, ok := δFtail.byRoot[root]
 	if !ok {
-		files = setOid{}
-		δFtail.filesByRoot[root] = files
+		rt = &_RootTrack{
+			ftrackSet:   setOid{},
+			ztrackInBlk: map[zodb.Oid]setI64{},
+		}
+		δFtail.byRoot[root] = rt
 	}
-	files.Add(foid)
+	rt.ftrackSet.Add(foid)
 
 	δftail, ok := δFtail.byFile[foid]
 	if !ok {
 		δftail = &_ΔFileTail{root: root, vδE: nil /*will need to be rebuilt to past till tail*/}
 		δFtail.byFile[foid] = δftail
-		δFtail.trackNew.Add(foid)
+		δFtail.ftrackNew.Add(foid)
 	}
 	if δftail.root != root {
 		// .root can change during epochs, but in between them it must be stable
@@ -248,109 +312,96 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, bl
 	// associate zblk with root, if it was not hole
 	if zblk != nil {
 		zoid := zblk.POid()
-		zt, ok := δFtail.trackSetZBlk[zoid]
+
+		inroot, ok := δFtail.ztrackInRoot[zoid]
 		if !ok {
-			zt = &zblkTrack{}
-			δFtail.trackSetZBlk[zoid] = zt
+			inroot = make(setOid, 1)
+			δFtail.ztrackInRoot[zoid] = inroot
 		}
+		inroot.Add(root)
 
-		inblk, ok := zt.inroot[root]
+		inblk, ok := rt.ztrackInBlk[zoid]
 		if !ok {
 			inblk = make(setI64, 1)
-			if zt.inroot == nil {
-				zt.inroot = make(map[zodb.Oid]setI64)
-			}
-			zt.inroot[root] = inblk
+			rt.ztrackInBlk[zoid] = inblk
 		}
 		inblk.Add(blk)
 	}
 }
 
-// rebuildAll rebuilds vδE for all files from trackNew requests.
-func (δFtail *ΔFtail) rebuildAll() (err error) {
-	defer xerr.Contextf(&err, "ΔFtail rebuildAll")
-	// XXX locking
+// vδEForFile returns vδE and current root for specified file.
+//
+// It builds vδE for that file if there is such need.
+// The only case when vδE actually needs to be built is when the file just started to be tracked.
+func (δFtail *ΔFtail) vδEForFile(foid zodb.Oid) (vδE []_ΔFileEpoch, headRoot zodb.Oid, err error) {
+	δFtail.mu.Lock() // TODO verify that there is no in-progress writers
+	defer δFtail.mu.Unlock()
 
-	δBtail := δFtail.δBtail
-	δZtail := δBtail.ΔZtail()
-	db     := δBtail.DB()
-	for foid := range δFtail.trackNew {
-		δFtail.trackNew.Del(foid)
-		δftail := δFtail.byFile[foid]
-		err := δftail.rebuild1(foid, δZtail, db)
-		if err != nil {
-			return err
+	δftail := δFtail.byFile[foid]
+	root := δftail.root
+	vδE = δftail.vδE
+	if vδE != nil {
+		return vδE, root, nil
+	}
+
+	// vδE needs to be built
+	job := δftail.rebuildJob
+
+	// rebuild is currently in-progress -> wait for corresponding job to complete
+	if job != nil {
+		δFtail.mu.Unlock()
+		<-job.ready
+		if job.err == nil {
+			δFtail.mu.Lock()
+			vδE = δftail.vδE
 		}
+		return vδE, root, job.err
 	}
 
-	return nil
-}
+	// we become responsible to build vδE
+	// release the lock while building to allow simultaneous access to other files
+	job = &_RebuildJob{ready: make(chan struct{})}
+	δftail.rebuildJob = job
+	δFtail.ftrackNew.Del(foid)
+	δBtail := δFtail.δBtail
 
-// rebuild1IfNeeded rebuilds vδE if there is such need.
-//
-// it returns corresponding δftail for convenience.
-// the only case when vδE actually needs to be rebuilt is when the file just started to be tracked.
-func (δFtail *ΔFtail) rebuild1IfNeeded(foid zodb.Oid) (_ *_ΔFileTail, err error) {
-	// XXX locking
+	δFtail.mu.Unlock()
+	vδE, err = vδEBuild(foid, δBtail.ΔZtail(), δBtail.DB())
+	δFtail.mu.Lock()
 
-	δftail := δFtail.byFile[foid]
-	if δftail.vδE != nil {
-		err = nil
+	if err == nil {
+		δftail.vδE = vδE
 	} else {
-		δFtail.trackNew.Del(foid)
-		δBtail := δFtail.δBtail
-		err = δftail.rebuild1(foid, δBtail.ΔZtail(), δBtail.DB())
+		δFtail.ftrackNew.Add(foid)
 	}
-	return δftail, err
-}
 
-// rebuild1 rebuilds vδE.
-func (δftail *_ΔFileTail) rebuild1(foid zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (err error) {
-	defer xerr.Contextf(&err, "file<%s>: rebuild", foid)
+	δftail.rebuildJob = nil
+	job.err = err
+	close(job.ready)
 
-	// XXX locking
-	if δftail.vδE != nil {
-		panic("rebuild1: vδE != nil")
-	}
-
-	vδE := []_ΔFileEpoch{}
-	vδZ := δZtail.Data()
-	atPrev := δZtail.Tail()
-	for i := 0; i < len(vδZ); i++ {
-		δZ := vδZ[i]
+	return vδE, root, err
+}
 
-		fchanged := false
-		for _, oid := range δZ.Changev {
-			if oid == foid {
-				fchanged = true
-				break
-			}
-		}
-		if !fchanged {
-			continue
-		}
+// _rebuildAll rebuilds vδE for all files from ftrackNew requests.
+//
+// must be called with δFtail.mu locked.
+func (δFtail *ΔFtail) _rebuildAll() (err error) {
+	defer xerr.Contextf(&err, "ΔFtail rebuildAll")
 
-		δ, err := zfilediff(db, foid, atPrev, δZ.Rev)
+	δBtail := δFtail.δBtail
+	δZtail := δBtail.ΔZtail()
+	db     := δBtail.DB()
+	for foid := range δFtail.ftrackNew {
+		δFtail.ftrackNew.Del(foid)
+		δftail := δFtail.byFile[foid]
+		// no need to set δftail.rebuildJob - we are under lock
+		δftail.vδE, err = vδEBuild(foid, δZtail, db)
 		if err != nil {
+			δFtail.ftrackNew.Add(foid)
 			return err
 		}
-
-		if δ != nil {
-			δE := _ΔFileEpoch{
-				Rev:             δZ.Rev,
-				oldRoot:         δ.blktabOld,
-				newRoot:         δ.blktabNew,
-				oldBlkSize:      δ.blksizeOld,
-				newBlkSize:      δ.blksizeNew,
-				oldTrackSetZBlk: nil, // nothing was tracked
-			}
-			vδE = append(vδE, δE)
-		}
-
-		atPrev = δZ.Rev
 	}
 
-	δftail.vδE = vδE
 	return nil
 }
 
@@ -360,17 +411,19 @@ func (δftail *_ΔFileTail) rebuild1(foid zodb.Oid, δZtail *zodb.ΔTail, db *zo
 //
 // δZ should include all objects changed by ZODB transaction.
 func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit) (_ ΔF, err error) {
-	defer xerr.Contextf(&err, "ΔFtail update %s -> %s", δFtail.Head(), δZ.Tid)
+	headOld := δFtail.Head()
+	defer xerr.Contextf(&err, "ΔFtail update %s -> %s", headOld, δZ.Tid)
 
-	// XXX locking
+	δFtail.mu.Lock()
+	defer δFtail.mu.Unlock()
+	// TODO verify that there is no in-progress readers/writers
 
 	// rebuild vδE for newly tracked files
-	err = δFtail.rebuildAll()
+	err = δFtail._rebuildAll()
 	if err != nil {
 		return ΔF{}, err
 	}
 
-	headOld := δFtail.Head()
 	δB, err := δFtail.δBtail.Update(δZ)
 	if err != nil {
 		return ΔF{}, err
@@ -394,23 +447,31 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit) (_ ΔF, err error) {
 		if δ != nil {
 			δzfile[oid] = δ
 			δE := _ΔFileEpoch{
-				Rev:             δZ.Tid,
-				oldRoot:         δ.blktabOld,
-				newRoot:         δ.blktabNew,
-				oldBlkSize:      δ.blksizeOld,
-				newBlkSize:      δ.blksizeNew,
-				oldTrackSetZBlk: map[zodb.Oid]setI64{},
+				Rev:        δZ.Tid,
+				oldRoot:    δ.blktabOld,
+				newRoot:    δ.blktabNew,
+				oldBlkSize: δ.blksizeOld,
+				newBlkSize: δ.blksizeNew,
+				oldZinblk:  map[zodb.Oid]setI64{},
 			}
 
-			for oid, zt := range δFtail.trackSetZBlk {
-				inblk, ok := zt.inroot[δftail.root]
-				if ok {
-					δE.oldTrackSetZBlk[oid] = inblk
-					delete(zt.inroot, δftail.root)
+			rt, ok := δFtail.byRoot[δftail.root]
+			if ok {
+				for zoid, inblk := range rt.ztrackInBlk {
+					δE.oldZinblk[zoid] = inblk.Clone()
+					inroot, ok := δFtail.ztrackInRoot[zoid]
+					if ok {
+						inroot.Del(δftail.root)
+						if len(inroot) == 0 {
+							delete(δFtail.ztrackInRoot, zoid)
+						}
+					}
 				}
 			}
 
 			δftail.root = δE.newRoot
+			// NOTE no need to clone vδE: we are writer, vδE is never returned to
+			// outside, append does not invalidate previous vδE retrievals.
 			δftail.vδE  = append(δftail.vδE, δE)
 		}
 	}
@@ -419,11 +480,15 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit) (_ ΔF, err error) {
 	//fmt.Printf("δB.ByRoot: %v\n", δB.ByRoot)
 	for root, δt := range δB.ByRoot {
 		//fmt.Printf("root: %v   δt: %v\n", root, δt)
-		files := δFtail.filesByRoot[root]
-		// NOTE files might be empty e.g. if a zfile was tracked, then
+		rt, ok := δFtail.byRoot[root]
+		// NOTE rt might be nil e.g. if a zfile was tracked, then
 		// deleted, but the tree referenced by zfile.blktab is still
 		// not-deleted, remains tracked and is changed.
-		for file := range files {
+		if !ok {
+			continue
+		}
+
+		for file := range rt.ftrackSet {
 			δfile, ok := δF.ByFile[file]
 			if !ok {
 				δfile = &ΔFile{Rev: δF.Rev, Blocks: make(setI64)}
@@ -440,31 +505,34 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit) (_ ΔF, err error) {
 			δfile.Size = true
 		}
 
-		// update trackSetZBlk according to btree changes
+		// update ztrackInBlk according to btree changes
 		for blk, δzblk := range δt {
 			if δzblk.Old != xbtree.VDEL {
-				ztOld, ok := δFtail.trackSetZBlk[δzblk.Old]
+				inblk, ok := rt.ztrackInBlk[δzblk.Old]
 				if ok {
-					inblk, ok := ztOld.inroot[root]
-					if ok {
-						inblk.Del(blk)
+					inblk.Del(blk)
+					if len(inblk) == 0 {
+						delete(rt.ztrackInBlk, δzblk.Old)
+						inroot := δFtail.ztrackInRoot[δzblk.Old]
+						inroot.Del(root)
+						if len(inroot) == 0 {
+							delete(δFtail.ztrackInRoot, δzblk.Old)
+						}
 					}
 				}
 			}
 
 			if δzblk.New != xbtree.VDEL {
-				ztNew, ok := δFtail.trackSetZBlk[δzblk.New]
-				if !ok {
-					ztNew = &zblkTrack{}
-					δFtail.trackSetZBlk[δzblk.New] = ztNew
-				}
-				inblk, ok := ztNew.inroot[root]
+				inblk, ok := rt.ztrackInBlk[δzblk.New]
 				if !ok {
 					inblk = make(setI64, 1)
-					if ztNew.inroot == nil {
-						ztNew.inroot = make(map[zodb.Oid]setI64)
+					rt.ztrackInBlk[δzblk.New] = inblk
+					inroot, ok := δFtail.ztrackInRoot[δzblk.New]
+					if !ok {
+						inroot = make(setOid, 1)
+						δFtail.ztrackInRoot[δzblk.New] = inroot
 					}
-					ztNew.inroot[root] = inblk
+					inroot.Add(root)
 				}
 				inblk.Add(blk)
 			}
@@ -473,18 +541,19 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit) (_ ΔF, err error) {
 
 	// take zblk changes into account
 	for _, oid := range δZ.Changev {
-		zt, ok := δFtail.trackSetZBlk[oid]
+		inroot, ok := δFtail.ztrackInRoot[oid]
 		if !ok {
 			continue // not tracked
 		}
 
-		for root, inblk := range zt.inroot {
-			if len(inblk) == 0 {
+		for root := range inroot {
+			rt := δFtail.byRoot[root] // must be there
+			inblk, ok := rt.ztrackInBlk[oid]
+			if !ok || len(inblk) == 0 {
 				continue
 			}
 			//fmt.Printf("root: %v   inblk: %v\n", root, inblk)
-			files := δFtail.filesByRoot[root]
-			for file := range files {
+			for file := range rt.ftrackSet {
 				δfile, ok := δF.ByFile[file]
 				if !ok {
 					δfile = &ΔFile{Rev: δF.Rev, Blocks: make(setI64)}
@@ -509,23 +578,36 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit) (_ ΔF, err error) {
 
 		//fmt.Printf("δZBigFile: %v\n", δ)
 
-		// update .filesByRoot
+		// update .byRoot
 		if δ.blktabOld != xbtree.VDEL {
-			files, ok := δFtail.filesByRoot[δ.blktabOld]
+			rt, ok := δFtail.byRoot[δ.blktabOld]
 			if ok {
-				files.Del(foid)
-				if len(files) == 0 {
-					delete(δFtail.filesByRoot, δ.blktabOld)
+				rt.ftrackSet.Del(foid)
+				if len(rt.ftrackSet) == 0 {
+					delete(δFtail.byRoot, δ.blktabOld)
+					// Zinroot -= δ.blktabNew
+					for zoid := range rt.ztrackInBlk {
+						inroot, ok := δFtail.ztrackInRoot[zoid]
+						if ok {
+							inroot.Del(δ.blktabOld)
+							if len(inroot) == 0 {
+								delete(δFtail.ztrackInRoot, zoid)
+							}
+						}
+					}
 				}
 			}
 		}
 		if δ.blktabNew != xbtree.VDEL {
-			files, ok := δFtail.filesByRoot[δ.blktabNew]
+			rt, ok := δFtail.byRoot[δ.blktabNew]
 			if !ok {
-				files = setOid{}
-				δFtail.filesByRoot[δ.blktabNew] = files
+				rt = &_RootTrack{
+					ftrackSet:   setOid{},
+					ztrackInBlk: map[zodb.Oid]setI64{},
+				}
+				δFtail.byRoot[δ.blktabNew] = rt
 			}
-			files.Add(foid)
+			rt.ftrackSet.Add(foid)
 		}
 	}
 
@@ -536,19 +618,20 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit) (_ ΔF, err error) {
 
 // ForgetPast discards all δFtail entries with rev ≤ revCut.
 func (δFtail *ΔFtail) ForgetPast(revCut zodb.Tid) {
-	δFtail.δBtail.ForgetPast(revCut)
+	δFtail.mu.Lock()
+	defer δFtail.mu.Unlock()
+	// TODO verify that there is no in-progress readers/writers
 
-	// XXX locking
+	δFtail.δBtail.ForgetPast(revCut)
 
 	// TODO keep index which file changed epoch where (similarly to ΔBtail),
 	// and, instead of scanning all files, trim vδE only on files that is really necessary.
 	for _, δftail := range δFtail.byFile {
-		δftail.forgetPast(revCut)
+		δftail._forgetPast(revCut)
 	}
 }
 
-func (δftail *_ΔFileTail) forgetPast(revCut zodb.Tid) {
-	// XXX locking
+func (δftail *_ΔFileTail) _forgetPast(revCut zodb.Tid) {
 	icut := 0
 	for ; icut < len(δftail.vδE); icut++ {
 		if δftail.vδE[icut].Rev > revCut {
@@ -567,6 +650,18 @@ func (δftail *_ΔFileTail) forgetPast(revCut zodb.Tid) {
 // TODO if needed
 // func (δFtail *ΔFtail) SliceByRev(lo, hi zodb.Tid) /*readonly*/ []ΔF
 
+// _ZinblkOverlay is used by SliceByFileRev.
+// It combines read-only Zinblk base with read-write adjustment.
+// It provides the following operations:
+//
+//	- Get(zblk) -> {blk},
+//	- AddBlk(zblk, blk),
+//	- DelBlk(zblk, blk)
+type _ZinblkOverlay struct {
+	Base map[zodb.Oid]setI64 // taken from _RootTrack.ztrackInBlk or _ΔFileEpoch.oldZinblk
+	Adj  map[zodb.Oid]setI64 // adjustment over base; blk<0 represents whiteout
+}
+
 // SliceByFileRev returns history of file changes in (lo, hi] range.
 //
 // it must be called with the following condition:
@@ -578,12 +673,18 @@ func (δftail *_ΔFileTail) forgetPast(revCut zodb.Tid) {
 // Only tracked blocks are guaranteed to be present.
 //
 // Note: contrary to regular go slicing, low is exclusive while high is inclusive.
-func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*readonly*/[]*ΔFile {
+func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) (/*readonly*/[]*ΔFile, error) {
 	foid := zfile.POid()
 	//fmt.Printf("\nslice f<%s> (@%s,@%s]\n", foid, lo, hi)
-	xtail.AssertSlice(δFtail, lo, hi)
+	vδf, err := δFtail._SliceByFileRev(foid, lo, hi)
+	if err != nil {
+		err = fmt.Errorf("slice f<%s> (@%s,@%s]: %e", foid, lo, hi, err)
+	}
+	return vδf, err
+}
 
-	// XXX locking
+func (δFtail *ΔFtail) _SliceByFileRev(foid zodb.Oid, lo, hi zodb.Tid) (/*readonly*/[]*ΔFile, error) {
+	xtail.AssertSlice(δFtail, lo, hi)
 
 	// query .δBtail.SliceByRootRev(file.blktab, lo, hi) +
 	// merge δZBlk history with that.
@@ -602,9 +703,9 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 	//     δFile	────────o───────o──────x─────x────────────────────────
 
 
-	δftail, err := δFtail.rebuild1IfNeeded(foid)
+	vδE, headRoot, err := δFtail.vδEForFile(foid)
 	if err != nil {
-		panic(err) // XXX
+		return nil, err
 	}
 
 	var vδf []*ΔFile
@@ -617,7 +718,7 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 				return δfTail
 			}
 			if !(tail < δfTail.Rev) {
-				panic("tail not ↓")
+				panic("BUG: tail not ↓")
 			}
 		}
 
@@ -630,7 +731,6 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 	iz := len(vδZ) - 1
 
 	// find epoch that covers hi
-	vδE := δftail.vδE
 	le := len(vδE)
 	ie := sort.Search(le, func(i int) bool {
 		return hi < vδE[i].Rev
@@ -658,32 +758,64 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 		var head zodb.Tid // head] of current epoch coverage
 		// state of Zinblk as we are scanning ← current epoch
 		// initially corresponds to head of the epoch (= @head for latest epoch)
-		Zinblk := map[zodb.Oid]setI64{} // zblk -> which #blk refers to it
+		Zinblk := _ZinblkOverlay{}	// zblk -> which #blk refers to it
 		var ZinblkAt zodb.Tid           // Zinblk covers [ZinblkAt,<next δT>)
 		if ie+1 == le {
 			// head
-			root = δftail.root
+			root = headRoot
 			head = δFtail.Head()
-			for zblk, zt := range δFtail.trackSetZBlk {
-				inblk, ok := zt.inroot[root]
-				if ok {
-					Zinblk[zblk] = inblk.Clone()
+
+			// take atomic Zinblk snapshot that covers vδZ
+			//
+			// - the reason we take atomic snapshot is because simultaneous Track
+			//   requests might change Zinblk concurrently, and without snapshotting
+			//   this might result in changes to a block being not uniformly present in
+			//   the returned vδf (some revision indicates change to that block, while
+			//   another one - where the block is too actually changed - does not
+			//   indicate change to that block).
+			//
+			// - the reason we limit snapshot to vδZ is to reduce amount of under-lock
+			//   copying, because original Zinblk is potentially very large.
+			//
+			// NOTE the other approach could be to keep blocks in _RootTrack.Zinblk with
+			// serial (!= zodb serial), and work with that _RootTrack.Zinblk snapshot by
+			// ignoring all blocks with serial > serial of snapshot view. Do not kill
+			// _ZinblkOverlay yet because we keep this  approach in mind for the future.
+			ZinblkSnap := map[zodb.Oid]setI64{}
+			δZAllOid := setOid{}
+			for _, δZ := range vδZ {
+				for _, oid := range δZ.Changev {
+					δZAllOid.Add(oid)
+				}
+			}
+			δFtail.mu.Lock()
+			rt, ok := δFtail.byRoot[root]
+			if ok {
+				for oid := range δZAllOid {
+					inblk, ok := rt.ztrackInBlk[oid]
+					if ok {
+						ZinblkSnap[oid] = inblk.Clone()
+					}
 				}
 			}
+			δFtail.mu.Unlock()
+
+			Zinblk.Base = ZinblkSnap
 		} else {
 			δE := vδE[ie+1]
 			root = δE.oldRoot
 			head = δE.Rev - 1 // TODO better set to exact revision coming before δE.Rev
-			for zblk, inblk := range δE.oldTrackSetZBlk {
-				Zinblk[zblk] = inblk.Clone()
-			}
+			Zinblk.Base = δE.oldZinblk
 		}
 		//fmt.Printf("Zinblk: %v\n", Zinblk)
 
 		// vδT for current epoch
 		var vδT []xbtree.ΔTree
 		if root != xbtree.VDEL {
-			vδT = δFtail.δBtail.SliceByRootRev(root, epoch, head) // NOTE @head, not hi
+			vδT, err = δFtail.δBtail.SliceByRootRev(root, epoch, head) // NOTE @head, not hi
+			if err != nil {
+				return nil, err
+			}
 		}
 		it := len(vδT) - 1
 		if it >= 0 {
@@ -707,33 +839,25 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 		// also all present @at2, @at3 and @at4 - because @at2 both 0
 		// and 1 are changed in the same tracked bucket. Note that
 		// changes to 2 should not be present at all.
-		ZinblkAdj := map[zodb.Oid]setI64{}
+		Zinblk.Adj = map[zodb.Oid]setI64{}
 		for _, δT := range vδT {
 			for blk, δzblk := range δT.KV {
 				if δzblk.Old != xbtree.VDEL {
-					inblk, ok := ZinblkAdj[δzblk.Old]
+					inblk, ok := Zinblk.Adj[δzblk.Old]
 					if ok {
 						inblk.Del(blk)
 					}
 				}
 				if δzblk.New != xbtree.VDEL {
-					inblk, ok := ZinblkAdj[δzblk.New]
+					inblk, ok := Zinblk.Adj[δzblk.New]
 					if !ok {
 						inblk = setI64{}
-						ZinblkAdj[δzblk.New] = inblk
+						Zinblk.Adj[δzblk.New] = inblk
 					}
 					inblk.Add(blk)
 				}
 			}
 		}
-		for zblk, inblkAdj := range ZinblkAdj {
-			inblk, ok := Zinblk[zblk]
-			if !ok {
-				Zinblk[zblk] = inblkAdj
-			} else {
-				inblk.Update(inblkAdj)
-			}
-		}
 
 		// merge vδZ and vδT of current epoch
 		for ((iz >= 0 && vδZ[iz].Rev > epoch) || it >= 0) {
@@ -744,7 +868,7 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 				if ZinblkAt <= δZ.Rev {
 					//fmt.Printf("δZ @%s\n", δZ.Rev)
 					for _, oid := range δZ.Changev {
-						inblk, ok := Zinblk[oid]
+						inblk, ok := Zinblk.Get_(oid)
 						if ok && len(inblk) != 0 {
 							δf := vδfTail(δZ.Rev)
 							δf.Blocks.Update(inblk)
@@ -762,18 +886,10 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 				for blk, δzblk := range δT.KV {
 					// apply in reverse as we go ←
 					if δzblk.New != xbtree.VDEL {
-						inblk, ok := Zinblk[δzblk.New]
-						if ok {
-							inblk.Del(blk)
-						}
+						Zinblk.DelBlk(δzblk.New, blk)
 					}
 					if δzblk.Old != xbtree.VDEL {
-						inblk, ok := Zinblk[δzblk.Old]
-						if !ok {
-							inblk = setI64{}
-							Zinblk[δzblk.Old] = inblk
-						}
-						inblk.Add(blk)
+						Zinblk.AddBlk(δzblk.Old, blk)
 					}
 
 					if δT.Rev <= hi {
@@ -810,9 +926,70 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 		vδf[i], vδf[j] = vδf[j], vδf[i]
 	}
 
-	return vδf
+	return vδf, nil
+}
+
+// ZinblkOverlay
+
+// Get_ returns set(blk) for o[zoid].
+func (o *_ZinblkOverlay) Get_(zoid zodb.Oid) (inblk /*readonly*/setI64, ok bool) {
+	base, bok := o.Base[zoid]
+	adj, aok  := o.Adj[zoid]
+	if !aok {
+		return base, bok
+	}
+
+	// combine base + adj
+	if bok {
+		inblk = base.Clone()
+	} else {
+		inblk = make(setI64, len(adj))
+	}
+	for blk := range adj {
+		if blk < 0 { // whiteout
+			inblk.Del(flipsign(blk))
+		} else {
+			inblk.Add(blk)
+		}
+	}
+	if len(inblk) == 0 {
+		return nil, false
+	}
+	return inblk, true
+}
+
+// DelBlk removes blk from o[zoid].
+func (o *_ZinblkOverlay) DelBlk(zoid zodb.Oid, blk int64) {
+	if blk < 0 {
+		panic("blk < 0")
+	}
+	o._AddBlk(zoid, flipsign(blk))
+}
+
+// AddBlk adds blk to o[zoid].
+func (o *_ZinblkOverlay) AddBlk(zoid zodb.Oid, blk int64) {
+	if blk < 0 {
+		panic("blk < 0")
+	}
+	o._AddBlk(zoid, blk)
+}
+
+func (o *_ZinblkOverlay) _AddBlk(zoid zodb.Oid, blk int64) {
+	adj, ok := o.Adj[zoid]
+	if !ok {
+		adj = make(setI64, 1)
+		o.Adj[zoid] = adj
+	}
+	adj.Add(blk)
+	adj.Del(flipsign(blk))
+}
+
+// flipsign returns x with sign bit flipped.
+func flipsign(x int64) int64 {
+	return int64(uint64(x) ^ (1<<63))
 }
 
+
 // BlkRevAt returns last revision that changed file[blk] as of @at database state.
 //
 // if exact=False - what is returned is only an upper bound for last block revision.
@@ -820,14 +997,7 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 // zfile must be any checkout from (tail, head]
 // at must ∈ (tail, head]
 // blk must be tracked
-func (δFtail *ΔFtail) BlkRevAt(ctx context.Context, zfile *ZBigFile, blk int64, at zodb.Tid) (_ zodb.Tid, exact bool) {
-	rev, exact, err := δFtail._BlkRevAt(ctx, zfile, blk, at)
-	if err != nil {
-		panic(err) // XXX
-	}
-	return rev, exact
-}
-func (δFtail *ΔFtail) _BlkRevAt(ctx context.Context, zfile *ZBigFile, blk int64, at zodb.Tid) (_ zodb.Tid, exact bool, err error) {
+func (δFtail *ΔFtail) BlkRevAt(ctx context.Context, zfile *ZBigFile, blk int64, at zodb.Tid) (_ zodb.Tid, exact bool, err error) {
 	foid := zfile.POid()
 	defer xerr.Contextf(&err, "blkrev f<%s> #%d @%s", foid, blk, at)
 
@@ -847,15 +1017,12 @@ func (δFtail *ΔFtail) _BlkRevAt(ctx context.Context, zfile *ZBigFile, blk int6
 		panicf("zconn.at out of bounds: zconn.at: @%s,  (tail, head] = (@%s, @%s]", zconnAt, tail, head)
 	}
 
-	// XXX locking
-
-	δftail, err := δFtail.rebuild1IfNeeded(foid)
+	vδE, headRoot, err := δFtail.vδEForFile(foid)
 	if err != nil {
 		return zodb.InvalidTid, false, err
 	}
 
 	// find epoch that covers at and associated blktab root/object
-	vδE := δftail.vδE
 	//fmt.Printf("  vδE: %v\n", vδE)
 	l := len(vδE)
 	i := sort.Search(l, func(i int) bool {
@@ -867,7 +1034,7 @@ func (δFtail *ΔFtail) _BlkRevAt(ctx context.Context, zfile *ZBigFile, blk int6
 	// root
 	var root zodb.Oid
 	if i == l {
-		root = δftail.root
+		root = headRoot
 	} else {
 		root = vδE[i].oldRoot
 	}
@@ -877,7 +1044,7 @@ func (δFtail *ΔFtail) _BlkRevAt(ctx context.Context, zfile *ZBigFile, blk int6
 	i--
 	if i < 0 {
 		// i<0 - first epoch (no explicit start) - use δFtail.tail as lo
-		epoch = δFtail.Tail()
+		epoch = tail
 	} else {
 		epoch = vδE[i].Rev
 	}
@@ -944,8 +1111,51 @@ func (δFtail *ΔFtail) _BlkRevAt(ctx context.Context, zfile *ZBigFile, blk int6
 }
 
 
-// ----------------------------------------
+// ---- vδEBuild (vδE rebuild core) ----
 
+// vδEBuild builds vδE for file from vδZ.
+func vδEBuild(foid zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (vδE []_ΔFileEpoch, err error) {
+	defer xerr.Contextf(&err, "file<%s>: build vδE", foid)
+
+	vδE  = []_ΔFileEpoch{}
+	vδZ := δZtail.Data()
+	atPrev := δZtail.Tail()
+	for i := 0; i < len(vδZ); i++ {
+		δZ := vδZ[i]
+
+		fchanged := false
+		for _, oid := range δZ.Changev {
+			if oid == foid {
+				fchanged = true
+				break
+			}
+		}
+		if !fchanged {
+			continue
+		}
+
+		δ, err := zfilediff(db, foid, atPrev, δZ.Rev)
+		if err != nil {
+			return nil, err
+		}
+
+		if δ != nil {
+			δE := _ΔFileEpoch{
+				Rev:        δZ.Rev,
+				oldRoot:    δ.blktabOld,
+				newRoot:    δ.blktabNew,
+				oldBlkSize: δ.blksizeOld,
+				newBlkSize: δ.blksizeNew,
+				oldZinblk:  nil, // nothing was tracked
+			}
+			vδE = append(vδE, δE)
+		}
+
+		atPrev = δZ.Rev
+	}
+
+	return vδE, nil
+}
 
 // zfilediff returns direct difference for ZBigFile<foid> old..new .
 type _ΔZBigFile struct {
@@ -1006,7 +1216,6 @@ func diffF(ctx context.Context, a, b *ZBigFile) (δ *_ΔZBigFile, err error) {
 	return δ, nil
 }
 
-
 // zgetFileOrNil returns ZBigFile corresponding to zconn.Get(oid) .
 // if the file does not exist, (nil, ok) is returned.
 func zgetFileOrNil(ctx context.Context, zconn *zodb.Connection, oid zodb.Oid) (zfile *ZBigFile, err error) {

wcfs/internal/zdata/δftail_test.go

@@ -242,12 +242,12 @@ func testΔFtail(t_ *testing.T, testq chan ΔFTestEntry) {
 	// update vδf + co for t1
 	vδf = append(vδf, &ΔFile{Rev: t1.At, Epoch: true})
 	vδE = append(vδE, _ΔFileEpoch{
-			Rev:             t1.At,
-			oldRoot:         zodb.InvalidOid,
-			newRoot:         t.Root(),
-			oldBlkSize:      -1,
-			newBlkSize:      blksize,
-			oldTrackSetZBlk: nil,
+			Rev:        t1.At,
+			oldRoot:    zodb.InvalidOid,
+			newRoot:    t.Root(),
+			oldBlkSize: -1,
+			newBlkSize: blksize,
+			oldZinblk:  nil,
 	})
 	epochv = append(epochv, t1.At)
 	for blk, zblk := range δt1 {
@@ -305,7 +305,11 @@ func testΔFtail(t_ *testing.T, testq chan ΔFTestEntry) {
 			for blk, zblk := range test.δblkTab {
 				zprev, ok := blkTab[blk]
 				if ok {
-					delete(Zinblk[zprev], blk)
+					inblk := Zinblk[zprev]
+					inblk.Del(blk)
+					if len(inblk) == 0 {
+						delete(Zinblk, zprev)
+					}
 				} else {
 					zprev = ø
 				}
@@ -423,12 +427,12 @@ func testΔFtail(t_ *testing.T, testq chan ΔFTestEntry) {
 				δE.oldBlkSize = -1
 				δE.newBlkSize = blksize
 			}
-			oldTrackSetZBlk := map[zodb.Oid]setI64{}
+			oldZinblk := map[zodb.Oid]setI64{}
 			for zblk, inblk := range ZinblkPrev {
 				oid, _ := commit.XGetBlkByName(zblk)
-				oldTrackSetZBlk[oid] = inblk
+				oldZinblk[oid] = inblk
 			}
-			δE.oldTrackSetZBlk = oldTrackSetZBlk
+			δE.oldZinblk = oldZinblk
 			vδE = append(vδE, δE)
 		}
 
@@ -445,26 +449,60 @@ func testΔFtail(t_ *testing.T, testq chan ΔFTestEntry) {
 			retrackAll()
 		}
 
-		// verify δFtail.trackSetZBlk
-		trackZinblk := map[string]setI64{}
-		for oid, zt := range δFtail.trackSetZBlk {
-			zblki := commit.ZBlkTab[oid]
-			for root, blocks := range zt.inroot {
-				if root != t.Root() {
-					t.Errorf(".trackSetZBlk: zblk %s points to unexpected blktab %s", zblki.Name, t.Root())
-					continue
-				}
+		// verify byRoot
+		trackRfiles  := map[zodb.Oid]setOid{}
+		for root, rt := range δFtail.byRoot {
+			trackRfiles[root] = rt.ftrackSet
+		}
+		filesOK := setOid{}
+		if !delfile {
+			filesOK.Add(foid)
+		}
+		RfilesOK := map[zodb.Oid]setOid{}
+		if len(filesOK) != 0 {
+			RfilesOK[t.Root()] = filesOK
+		}
+		if !reflect.DeepEqual(trackRfiles, RfilesOK) {
+			t.Errorf("Rfiles:\nhave: %v\nwant: %v", trackRfiles, RfilesOK)
+		}
 
-				inblk, ok := trackZinblk[zblki.Name]
-				if !ok {
-					inblk = setI64{}
-					trackZinblk[zblki.Name] = inblk
+		// verify Zinroot
+		trackZinroot := map[string]setOid{}
+		for zoid, inroot := range δFtail.ztrackInRoot {
+			zblki := commit.ZBlkTab[zoid]
+			trackZinroot[zblki.Name] = inroot.Clone() // XXX clone needed?
+		}
+		Zinroot := map[string]setOid{}
+		for zblk := range Zinblk {
+			inroot := setOid{}; inroot.Add(t.Root())
+			Zinroot[zblk] = inroot
+		}
+		if !reflect.DeepEqual(trackZinroot, Zinroot) {
+			t.Errorf("Zinroot:\nhave: %v\nwant: %v", trackZinroot, Zinroot)
+		}
+
+		// verify Zinblk
+		trackZinblk := map[string]setI64{}
+		switch {
+		case len(δFtail.byRoot) == 0:
+			// ok
+
+		case len(δFtail.byRoot) == 1:
+			rt, ok := δFtail.byRoot[t.Root()]
+			if !ok {
+				t.Errorf(".byRoot points to unexpected blktab")
+			} else {
+				for zoid, inblk := range rt.ztrackInBlk {
+					zblki := commit.ZBlkTab[zoid]
+					trackZinblk[zblki.Name] = inblk.Clone()	// XXX clone needed?
 				}
-				inblk.Update(blocks)
 			}
+
+		default:
+			t.Errorf("len(.byRoot) != (0,1)  ; byRoot: %v", δFtail.byRoot)
 		}
 		if !reflect.DeepEqual(trackZinblk, Zinblk) {
-			t.Errorf(".trackSetZBlk:\n~have: %v\n want: %v", trackZinblk, Zinblk)
+			t.Errorf("Zinblk:\nhave: %v\nwant: %v", trackZinblk, Zinblk)
 		}
 
 		// ForgetPast configured threshold
@@ -485,16 +523,6 @@ func testΔFtail(t_ *testing.T, testq chan ΔFTestEntry) {
 			vδE = vδE[icut:]
 		}
 
-		// verify δFtail.filesByRoot
-		filesByRootOK := map[zodb.Oid]setOid{}
-		if !delfile {
-			__ := setOid{}; __.Add(foid)
-			filesByRootOK[t.Root()] = __
-		}
-		if !reflect.DeepEqual(δFtail.filesByRoot, filesByRootOK) {
-			t.Errorf("filesByRoot:\nhave: %v\nwant: %v", δFtail.filesByRoot, filesByRootOK)
-		}
-
 		// verify δftail.root
 		δftail := δFtail.byFile[foid]
 		rootOK := t.Root()
@@ -523,7 +551,7 @@ func testΔFtail(t_ *testing.T, testq chan ΔFTestEntry) {
 				hi := vδf[k].Rev
 
 				vδf_ok := vδf[j:k+1] // [j,k]
-				vδf_ := δFtail.SliceByFileRev(zfile, lo, hi)
+				vδf_, err := δFtail.SliceByFileRev(zfile, lo, hi);  X(err)
 				if !reflect.DeepEqual(vδf_, vδf_ok) {
 					t.Errorf("slice (@%s,@%s]:\nhave: %v\nwant: %v", t.AtSymb(lo), t.AtSymb(hi), t.vδfstr(vδf_), t.vδfstr(vδf_ok))
 				}
@@ -547,7 +575,7 @@ func testΔFtail(t_ *testing.T, testq chan ΔFTestEntry) {
 			at := vδf[j].Rev
 			blkRev := blkRevAt[at]
 			for _, blk := range blkv {
-				rev, exact := δFtail.BlkRevAt(ctx, zfile, blk, at)
+				rev, exact, err := δFtail.BlkRevAt(ctx, zfile, blk, at);  X(err)
 				revOK, ok := blkRev[blk]
 				if !ok {
 					k := len(epochv) - 1
@@ -626,7 +654,7 @@ func TestΔFtailSliceUntrackedUniform(t_ *testing.T) {
 	// (at1, at4]  -> changes to both 0 and 1, because they both are changed in the same bucket @at2
 	lo := t1.At
 	hi := t4.At
-	vδf := δFtail.SliceByFileRev(zfile, lo, hi)
+	vδf, err := δFtail.SliceByFileRev(zfile, lo, hi);  X(err)
 	vδf_ok := []*ΔFile{
 		&ΔFile{Rev: t2.At, Blocks: b(0,1), Size: true},
 		&ΔFile{Rev: t3.At, Blocks: b(0,1), Size: false},
@@ -639,7 +667,7 @@ func TestΔFtailSliceUntrackedUniform(t_ *testing.T) {
 
 	// (at2, at4]  -> changes to only 0, because there is no change to 2 via blktab
 	lo = t2.At
-	vδf = δFtail.SliceByFileRev(zfile, lo, hi)
+	vδf, err = δFtail.SliceByFileRev(zfile, lo, hi);  X(err)
 	vδf_ok = []*ΔFile{
 		&ΔFile{Rev: t3.At, Blocks: b(0),   Size: false},
 	}
@@ -649,7 +677,7 @@ func TestΔFtailSliceUntrackedUniform(t_ *testing.T) {
 
 	// (at3, at4]  -> changes to only 0, ----/----
 	lo = t3.At
-	vδf = δFtail.SliceByFileRev(zfile, lo, hi)
+	vδf, err = δFtail.SliceByFileRev(zfile, lo, hi);  X(err)
 	vδf_ok = []*ΔFile(nil)
 	if !reflect.DeepEqual(vδf, vδf_ok) {
 		t.Errorf("slice (@%s,@%s]:\nhave: %v\nwant: %v", t.AtSymb(lo), t.AtSymb(hi), t.vδfstr(vδf), t.vδfstr(vδf_ok))

wcfs/wcfs.go

@@ -1085,22 +1085,29 @@ func (f *BigFile) invalidateBlk(ctx context.Context, blk int64) (err error) {
 	//
 	// if we have the data - preserve it under @revX/bigfile/file[blk].
 	if int64(len(blkdata)) == blksize {
-		func() {
+		err := func() error {
 			// store retrieved data back to OS cache for file @<rev>/file[blk]
 			δFtail := f.head.bfdir.δFtail
-			blkrev, _ := δFtail.BlkRevAt(ctx, f.zfile, blk, f.head.zconn.At())
+			blkrev, _, err := δFtail.BlkRevAt(ctx, f.zfile, blk, f.head.zconn.At())
+			if err != nil {
+				return err
+			}
 			frev, funlock, err := groot.lockRevFile(blkrev, f.zfile.POid())
 			if err != nil {
-				log.Errorf("BUG: %s: invalidate blk #%d: %s (ignoring, but reading @revX/bigfile will be slow)", f.path(), blk, err)
-				return
+				return fmt.Errorf("BUG: %s", err)
 			}
 			defer funlock()
 
 			st := fsconn.FileNotifyStoreCache(frev.Inode(), off, blkdata)
 			if st != fuse.OK {
-				log.Errorf("BUG: %s: invalidate blk #%d: %s: store cache: %s (ignoring, but reading @revX/bigfile will be slow)", f.path(), blk, frev.path(), st)
+				return fmt.Errorf("BUG: %s: store cache: %s", frev.path(), st)
 			}
+
+			return nil
 		}()
+		if err != nil {
+			log.Errorf("%s: invalidate blk #%d: %s (ignoring, but reading @revX/bigfile will be slow)", f.path(), blk, err)
+		}
 	}
 
 	// invalidate file/head/data[blk] in OS file cache.
@@ -1566,7 +1573,11 @@ func (f *BigFile) readPinWatchers(ctx context.Context, blk int64, treepath []btr
 			// we'll relock atMu again and recheck blkrev vs w.at after.
 			w.atMu.RUnlock()
 
-			blkrev, _ = δFtail.BlkRevAt(ctx, f.zfile, blk, f.head.zconn.At())
+			var err error
+			blkrev, _, err = δFtail.BlkRevAt(ctx, f.zfile, blk, f.head.zconn.At())
+			if err != nil {
+				panic(err)	// XXX
+			}
 			blkrevRough = false
 
 			w.atMu.RLock()
@@ -1582,8 +1593,11 @@ func (f *BigFile) readPinWatchers(ctx context.Context, blk int64, treepath []btr
 		// and most of them would be on different w.at - cache of the file will
 		// be lost. Via pinning to particular block revision, we make sure the
 		// revision to pin is the same on all clients, and so file cache is shared.
-		pinrev, _ := δFtail.BlkRevAt(ctx, w.file.zfile, blk, w.at) // XXX move into go?
+		pinrev, _, err := δFtail.BlkRevAt(ctx, w.file.zfile, blk, w.at) // XXX move into go?
 		// XXX ^^^ w.file vs f ?
+		if err != nil {
+			panic(err)	// XXX
+		}
 
 		//fmt.Printf("S: read #%d: watch @%s: pin -> @%s\n", blk, w.at, pinrev)
 		wg.Go(func(ctx context.Context) error {
@@ -1728,7 +1742,11 @@ func (wlink *WatchLink) setupWatch(ctx context.Context, foid zodb.Oid, at zodb.T
 	toPin := map[int64]zodb.Tid{} // blk -> @rev
 
 	δFtail := bfdir.δFtail
-	for _, δfile := range δFtail.SliceByFileRev(f.zfile, at, headAt) { // XXX locking δFtail
+	vδf, err := δFtail.SliceByFileRev(f.zfile, at, headAt)	// XXX locking δFtail
+	if err != nil {
+		panic(err)	// XXX
+	}
+	for _, δfile := range vδf {
 		if δfile.Epoch {
 			// file epochs are currently forbidden (see watcher), so the only
 			// case when we could see an epoch here is creation of
@@ -1764,7 +1782,10 @@ func (wlink *WatchLink) setupWatch(ctx context.Context, foid zodb.Oid, at zodb.T
 				continue
 			}
 
-			toPin[blk], _ = δFtail.BlkRevAt(ctx, f.zfile, blk, at)  // XXX err
+			toPin[blk], _, err = δFtail.BlkRevAt(ctx, f.zfile, blk, at)
+			if err != nil {
+				panic(err)	// XXX
+			}
 		}
 	}