X zdata: Switch SliceByFileRev not to clone Zinblk

Zinblk is potentially unbounded large.
Should also help further steps on concurrency.

wcfs/internal/xbtree/xbtreetest/treeenv.go

@@ -137,31 +137,55 @@ func (t *T) Head() *Commit {
 
 // XGetCommit finds and returns Commit created with revision at.
 func (t *T) XGetCommit(at zodb.Tid) *Commit {
+	commit, _, _ := t.getCommit(at)
+	if commit == nil {
+		panicf("no commit corresponding to @%s", at)
+	}
+	return commit
+}
+
+func (t *T) getCommit(at zodb.Tid) (commit, cprev, cnext *Commit) {
 	l := len(t.commitv)
 	i := sort.Search(l, func(i int) bool {
 		return at <= t.commitv[i].At
 	})
-	var commit *Commit
 	if i < l {
 		commit = t.commitv[i]
 		if commit.At != at {
+			cnext = commit
 			commit = nil
+		} else if i+1 < l {
+			cnext = t.commitv[i+1]
 		}
 	}
-	if commit == nil {
-		panicf("no commit corresponding to @%s", at)
+	if i > 0 {
+		cprev = t.commitv[i-1]
 	}
-	if commit.idx != i {
+	if commit != nil && commit.idx != i {
 		panicf("BUG: commit.idx (%d) != i (%d)", commit.idx, i)
 	}
-	return commit
+	return commit, cprev, cnext
 }
 
 // AtSymb returns symbolic representation of at, for example "at3".
 //
-// at must correspond to a Commit.
+// at should correspond to a Commit.
 func (t *T) AtSymb(at zodb.Tid) string {
-	return t.XGetCommit(at).AtSymb()
+	commit, cprev, cnext := t.getCommit(at)
+	if commit != nil {
+		return commit.AtSymb()
+	}
+
+	// at does not correspond to commit - return something like ~at2<xxxx>at3
+	s := "~"
+	if cprev != nil {
+		s += cprev.AtSymb() + "<"
+	}
+	s += at.String()
+	if cnext != nil {
+		s += ">" + cnext.AtSymb()
+	}
+	return s
 }
 
 // AtSymb returns symbolic representation of c.At, for example "at3".

wcfs/internal/zdata/δftail.go

@@ -180,7 +180,7 @@ type _ΔFileEpoch struct {
 type _RootTrack struct {
 	// XXX -> ftrackSet ?
 	files  setOid              // {}foid		which ZBigFiles refer to this tree
-	// XXX -> ztrackInBlk ?
+	// XXX -> ztrackInBlk ?  trackZInblk ?
 	Zinblk map[zodb.Oid]setI64 // {} zblk -> {}blk	which blocks map to zblk
 }
 
@@ -618,6 +618,18 @@ func (δftail *_ΔFileTail) _forgetPast(revCut zodb.Tid) {
 // TODO if needed
 // func (δFtail *ΔFtail) SliceByRev(lo, hi zodb.Tid) /*readonly*/ []ΔF
 
+// _ZinblkOverlay is used by SliceByFileRev.
+// It combiness read-only Zinblk base with read-write adjustment.
+// It provides the following operations:
+//
+//	- Get(zblk) -> {blk},
+//	- AddBlk(zblk, blk),
+//	- DelBlk(zblk, blk)
+type _ZinblkOverlay struct {
+	Base map[zodb.Oid]setI64 // taken from _RootTrack.Zinblk or _ΔFileEpoch.oldZinblk
+	Adj  map[zodb.Oid]setI64 // blk<0 reresents whitout
+}
+
 // SliceByFileRev returns history of file changes in (lo, hi] range.
 //
 // it must be called with the following condition:
@@ -706,9 +718,8 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 		var head zodb.Tid // head] of current epoch coverage
 		// state of Zinblk as we are scanning ← current epoch
 		// initially corresponds to head of the epoch (= @head for latest epoch)
-		//
-		// XXX -> ZinblkOverlay(.base = Zinblk@head w/o Clone; .adj = δZinblk)
-		Zinblk := map[zodb.Oid]setI64{} // zblk -> which #blk refers to it
+//		Zinblk := map[zodb.Oid]setI64{} // zblk -> which #blk refers to it
+		Zinblk := _ZinblkOverlay{}	// zblk -> which #blk refers to it
 		var ZinblkAt zodb.Tid           // Zinblk covers [ZinblkAt,<next δT>)
 		if ie+1 == le {
 			// head
@@ -716,19 +727,19 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 			head = δFtail.Head()
 			rt, ok := δFtail.byRoot[root]		// XXX locking
 			if ok {
-				for zblk, inblk := range rt.Zinblk {	// XXX -> no clone
-					Zinblk[zblk] = inblk.Clone()
-				}
+				Zinblk.Base = rt.Zinblk
+//				for zblk, inblk := range rt.Zinblk {	// XXX -> no clone
+//					Zinblk[zblk] = inblk.Clone()
+//				}
 			}
 		} else {
 			δE := vδE[ie+1]
 			root = δE.oldRoot
 			head = δE.Rev - 1 // TODO better set to exact revision coming before δE.Rev
-			for zblk, inblk := range δE.oldZinblk {
-				Zinblk[zblk] = inblk.Clone()	// XXX Clone needed ? -> just use oldTrackSetZBlk directly
-				// XXX -> just use Zinblk = δE.oldTrackSetZBlk directly (without full clone)
-				// XXX no -> vvv we adjust Zinblk	-> thinking
-			}
+			Zinblk.Base = δE.oldZinblk
+//			for zblk, inblk := range δE.oldZinblk {
+//				Zinblk[zblk] = inblk.Clone()	// XXX -> no Clone needed
+//			}
 		}
 		//fmt.Printf("Zinblk: %v\n", Zinblk)
 
@@ -759,25 +770,28 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 		// also all present @at2, @at3 and @at4 - because @at2 both 0
 		// and 1 are changed in the same tracked bucket. Note that
 		// changes to 2 should not be present at all.
-		ZinblkAdj := map[zodb.Oid]setI64{}
+		Zinblk.Adj = map[zodb.Oid]setI64{}
 		for _, δT := range vδT {
 			for blk, δzblk := range δT.KV {
+				// XXX VDEL -> whiteout?
 				if δzblk.Old != xbtree.VDEL {
-					inblk, ok := ZinblkAdj[δzblk.Old]
+					inblk, ok := Zinblk.Adj[δzblk.Old]
 					if ok {
 						inblk.Del(blk)
 					}
 				}
+				// XXX VDEL -> whiteout?
 				if δzblk.New != xbtree.VDEL {
-					inblk, ok := ZinblkAdj[δzblk.New]
+					inblk, ok := Zinblk.Adj[δzblk.New]
 					if !ok {
 						inblk = setI64{}
-						ZinblkAdj[δzblk.New] = inblk
+						Zinblk.Adj[δzblk.New] = inblk
 					}
 					inblk.Add(blk)
 				}
 			}
 		}
+		/*
 		for zblk, inblkAdj := range ZinblkAdj {
 			inblk, ok := Zinblk[zblk]
 			if !ok {
@@ -786,6 +800,7 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 				inblk.Update(inblkAdj)
 			}
 		}
+		*/
 
 		// merge vδZ and vδT of current epoch
 		for ((iz >= 0 && vδZ[iz].Rev > epoch) || it >= 0) {
@@ -796,7 +811,7 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 				if ZinblkAt <= δZ.Rev {
 					//fmt.Printf("δZ @%s\n", δZ.Rev)
 					for _, oid := range δZ.Changev {
-						inblk, ok := Zinblk[oid]
+						inblk, ok := Zinblk.Get_(oid)
 						if ok && len(inblk) != 0 {
 							δf := vδfTail(δZ.Rev)
 							δf.Blocks.Update(inblk)
@@ -814,18 +829,20 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 				for blk, δzblk := range δT.KV {
 					// apply in reverse as we go ←
 					if δzblk.New != xbtree.VDEL {
-						inblk, ok := Zinblk[δzblk.New]
-						if ok {
-							inblk.Del(blk)
-						}
+						Zinblk.DelBlk(δzblk.New, blk)
+//						inblk, ok := Zinblk[δzblk.New]	// XXX
+//						if ok {
+//							inblk.Del(blk)
+//						}
 					}
 					if δzblk.Old != xbtree.VDEL {
-						inblk, ok := Zinblk[δzblk.Old]
-						if !ok {
-							inblk = setI64{}
-							Zinblk[δzblk.Old] = inblk
-						}
-						inblk.Add(blk)
+						Zinblk.AddBlk(δzblk.Old, blk)
+//						inblk, ok := Zinblk[δzblk.Old]	// XXX
+//						if !ok {
+//							inblk = setI64{}
+//							Zinblk[δzblk.Old] = inblk
+//						}
+//						inblk.Add(blk)
 					}
 
 					if δT.Rev <= hi {
@@ -865,6 +882,64 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
 	return vδf
 }
 
+// ZinblkOverlay	XXX move closer to end?
+
+func (o *_ZinblkOverlay) Get_(zoid zodb.Oid) (inblk /*readonly*/setI64, ok bool) {
+	base, bok := o.Base[zoid]
+	adj, aok  := o.Adj[zoid]
+	if !aok {
+		return base, bok
+	}
+
+	// combine base + adj
+	if bok {
+		inblk = base.Clone()
+	} else {
+		inblk = make(setI64, len(adj))
+	}
+	for blk := range adj {
+		if blk < 0 { // whiteout
+			inblk.Del(flipsign(blk))
+		} else {
+			inblk.Add(blk)
+		}
+	}
+	if len(inblk) == 0 {
+		return nil, false
+	}
+	return inblk, true
+}
+
+func (o *_ZinblkOverlay) DelBlk(zoid zodb.Oid, blk int64) {
+	if blk < 0 {
+		panic("blk < 0")
+	}
+	o._AddBlk(zoid, flipsign(blk))
+}
+
+func (o *_ZinblkOverlay) AddBlk(zoid zodb.Oid, blk int64) {
+	if blk < 0 {
+		panic("blk < 0")
+	}
+	o._AddBlk(zoid, blk)
+}
+
+func (o *_ZinblkOverlay) _AddBlk(zoid zodb.Oid, blk int64) {
+	adj, ok := o.Adj[zoid]
+	if !ok {
+		adj = make(setI64, 1)
+		o.Adj[zoid] = adj
+	}
+	adj.Add(blk)
+	adj.Del(flipsign(blk))
+}
+
+// flipsign returns x with sign bit flipped.
+func flipsign(x int64) int64 {
+	return int64(uint64(x) ^ (1<<63))
+}
+
+
 // BlkRevAt returns last revision that changed file[blk] as of @at database state.
 //
 // if exact=False - what is returned is only an upper bound for last block revision.