.

wcfs/internal/xbtree/xbtreetest/kvdiff.go

@@ -26,13 +26,13 @@ import (
 	"strings"
 )
 
-// kvdiff returns difference in between kv1 and kv2.
+// KVDiff returns difference in between kv1 and kv2.
 const DEL = "ø" // DEL means deletion
 type Δstring struct {
 	Old string
 	New string
 }
-func kvdiff(kv1, kv2 map[Key]string) map[Key]Δstring {
+func KVDiff(kv1, kv2 map[Key]string) map[Key]Δstring {
 	delta := map[Key]Δstring{}
 	keys := setKey{}
 	for k := range kv1 { keys.Add(k) }
@@ -51,8 +51,8 @@ func kvdiff(kv1, kv2 map[Key]string) map[Key]Δstring {
 	return delta
 }
 
-// kvtxt returns string representation of {} kv.
-func kvtxt(kv map[Key]string) string {
+// KVTxt returns string representation of {} kv.
+func KVTxt(kv map[Key]string) string {
 	if len(kv) == 0 {
 		return "ø"
 	}

wcfs/internal/xbtree/xbtreetest/kvdiff_test.go

@@ -27,7 +27,7 @@ import (
 func TestKVDiff(t *testing.T) {
 	kv1 := map[Key]string{1:"a", 3:"c", 4:"d"}
 	kv2 := map[Key]string{1:"b",        4:"d", 5:"e"}
-	got  := kvdiff(kv1, kv2)
+	got  := KVDiff(kv1, kv2)
 	want := map[Key]Δstring{1:{"a","b"}, 3:{"c",DEL}, 5:{DEL,"e"}}
 	if !reflect.DeepEqual(got, want) {
 		t.Fatalf("error:\ngot:  %v\nwant: %v", got, want)

wcfs/internal/xbtree/xbtreetest/rtree.go

@@ -104,5 +104,5 @@ func (xkv RBucketSet) Flatten() map[Key]string {
 }
 
 func (b *RBucket) String() string {
-	return fmt.Sprintf("%sB%s{%s}", b.Keycov, b.Oid, kvtxt(b.KV))
+	return fmt.Sprintf("%sB%s{%s}", b.Keycov, b.Oid, KVTxt(b.KV))
 }

wcfs/internal/xbtree/xbtreetest/treeenv.go

@@ -35,7 +35,7 @@ import (
 // T is tree-based testing environment.
 //
 // It combines TreeSrv and client side access to ZODB with committed trees.
-// It should be created it via NewT().
+// It should be created it NewT().
 type T struct {
 	*testing.T
 
@@ -56,7 +56,7 @@ type Commit struct {
 	ΔZ          *zodb.EventCommit   // raw ZODB changes; δZ.tid == at
 	Xkv         RBucketSet          // full tree state as of @at
 	Δxkv        map[Key]Δstring     // full tree-diff against parent
-	zblkDataTab map[zodb.Oid]string // full snapshot of all ZBlk data @at
+	ZBlkDataTab map[zodb.Oid]string // full snapshot of all ZBlk data @at
 //	δzblkData  map[zodb.Oid]Δstring // full diff for zblkData against parent XXX ?
 }
 
@@ -97,7 +97,7 @@ func NewT(t *testing.T) *T {
 		Prev:        nil,
 		At:          head,
 		Xkv:         xGetTree(tt.DB, head, tt.Root()),
-		zblkDataTab: xGetBlkDataTab(tt.DB, head),
+		ZBlkDataTab: xGetBlkDataTab(tt.DB, head),
 		ΔZ:          nil,
 		Δxkv:        nil,
 	}
@@ -182,12 +182,12 @@ func (t *T) CommitTree(tree string) *Commit {
 		At:          δZ.Tid,
 		ΔZ:          δZ,
 		Xkv:         xkv,
-		zblkDataTab: xGetBlkDataTab(t.DB, δZ.Tid),
+		ZBlkDataTab: xGetBlkDataTab(t.DB, δZ.Tid),
 	}
 
 	tprev := t.Head()
 	ttree.Prev = tprev
-	ttree.Δxkv = kvdiff(tprev.Xkv.Flatten(), ttree.Xkv.Flatten())
+	ttree.Δxkv = KVDiff(tprev.Xkv.Flatten(), ttree.Xkv.Flatten())
 
 	t.commitv = append(t.commitv, ttree)
 
@@ -249,7 +249,7 @@ func (t *Commit) XGetBlkData(oid zodb.Oid) string {
 	if oid == VDEL {
 		return DEL
 	}
-	data, ok := t.zblkDataTab[oid]
+	data, ok := t.ZBlkDataTab[oid]
 	if !ok {
 		exc.Raisef("getBlkData ZBlk<%s> @%s: no such ZBlk", oid, t.At)
 	}

wcfs/internal/xbtree/xbtreetest/treegen.go

@@ -69,8 +69,8 @@ type AllStructsSrv struct {
 	*TreeGenSrv
 }
 
-// StartTreeGenSrv spawns `treegen ...` server.
-func StartTreeGenSrv(argv ...string) (_ *TreeGenSrv, hello string, err error) {
+// startTreeGenSrv spawns `treegen ...` server.
+func startTreeGenSrv(argv ...string) (_ *TreeGenSrv, hello string, err error) {
 	defer xerr.Contextf(&err, "treesrv %v: start", argv)
 
 	// spawn `treegen ...`
@@ -125,7 +125,7 @@ func (tg *TreeGenSrv) Close() (err error) {
 // StartTreeSrv spawns `treegen trees` server.
 func StartTreeSrv(zurl string) (_ *TreeSrv, err error) {
 	defer xerr.Contextf(&err, "tree.srv %s: start", zurl)
-	tgSrv, hello, err := StartTreeGenSrv("trees", zurl)
+	tgSrv, hello, err := startTreeGenSrv("trees", zurl)
 	if err != nil {
 		return nil, err
 	}
@@ -160,7 +160,7 @@ func StartTreeSrv(zurl string) (_ *TreeSrv, err error) {
 func StartAllStructsSrv() (_ *AllStructsSrv, err error) {
 	defer xerr.Context(&err, "allstructs.srv: start")
 
-	tgSrv, hello, err := StartTreeGenSrv("allstructs")
+	tgSrv, hello, err := startTreeGenSrv("allstructs")
 	if err != nil {
 		return nil, err
 	}
@@ -209,7 +209,7 @@ func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
 
 // AllStructs returns response from `treegen allstructs`
 func (tg *AllStructsSrv) AllStructs(kv map[Key]string, maxdepth, maxsplit, n int, seed int64) (_ []string, err error) {
-	req := fmt.Sprintf("%d %d %d/%d %s", maxdepth, maxsplit, n, seed, kvtxt(kv))
+	req := fmt.Sprintf("%d %d %d/%d %s", maxdepth, maxsplit, n, seed, KVTxt(kv))
 	defer xerr.Contextf(&err, "allstructs.srv: %s ", req)
 
 	_, err = io.WriteString(tg.pyin, req + "\n")

wcfs/internal/xbtree/xbtreetest/treegen.py

@@ -20,7 +20,7 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 """Program treegen provides infrastructure to generate ZODB BTree states.
 
-It is used as helper for ΔBtail tests.
+It is used as helper for ΔBtail and ΔFtail tests.
 
 The following subcommands are provided:
 
@@ -108,6 +108,9 @@ session example:
     T3/T-T/B1:a,2:b-B3:c
     # ----
 
+
+XXX ΔFtail support
+
 --------
 
 (*) 300-500ms, see https://github.com/pypa/setuptools/issues/510.
@@ -127,7 +130,7 @@ import random
 import six
 
 from wendelin.wcfs.internal import xbtree, xbtree_test
-from wendelin.bigfile.file_zodb import ZBlk
+from wendelin.bigfile.file_zodb import ZBlk, ZBigFile
 from zodbtools.util import storageFromURL, ashex
 
 from persistent import CHANGED
@@ -197,6 +200,8 @@ def TreesSrv(zstor, r):
     defer(zctx.close)
 
     ztree = zctx.root['treegen/tree'] = LOBTree()
+    zfile = zctx.root['treegen/file'] = ZBigFile(blksize=4) # for ΔFtail tests
+    zfile.blktab = ztree
     head = commit('treegen/tree: init')
     xprint("tree.srv start @%s root=%s" % (ashex(head), ashex(ztree._p_oid)))
     treetxtPrev = zctx.ztreetxt(ztree)
@@ -210,6 +215,22 @@ def TreesSrv(zstor, r):
             xprint("%s" % ashex(head))
             continue
 
+        # t... D... commands to natively commit updates to tree and values
+        if treetxt.startswith('t'):
+            t, D = treetxt.split()
+            assert D.startswith('D')
+            kv = kvDecode(t[1:], zctx.vdecode)
+            zv = kvDecode(D[1:], lambda vtxt: vtxt)
+            zdataTab = zctx.root['treegen/values']
+            patch(ztree,    diff(ztree, kv), kv)
+            patch(zdataTab, diff(zdataTab, zv), zv) # XXX v->ZBlk(v)
+
+            head = commit(subj)
+            xprint("%s" % ashex(head))
+            continue
+
+        # everything else is considerd to be a tree topology
+
         # mark tree as changed if the same topology is requested twice.
         # this ensures we can actually make a non-empty commit
         if treetxt == treetxtPrev:

wcfs/internal/xbtree/δbtail.go

@@ -84,7 +84,6 @@ const debugΔBtail = false
 // XXX -> multiple readers / single writer?
 //
 // See also zodb.ΔTail
-// XXX naming -> ΔBTail ?
 type ΔBtail struct {
 	// raw ZODB changes; Kept to rebuild .vδTbyRoot after new Track.
 	// includes all changed objects, not only tracked ones.

wcfs/internal/zdata/δftail_test.go

@@ -18,24 +18,38 @@
 // See https://www.nexedi.com/licensing for rationale and options.
 
 package zdata
+// tests for δftail.go
+//
+// This are the main tests for ΔFtail functionality.
+// XXX overview
+// XXX we assume that ΔBtail works correctly (this is covered by ΔBtail tests)
+// XXX -> no need to exercise many different topologies and tracking sets.
 
 import (
+	"context"
+	"fmt"
 	"testing"
 
+	"lab.nexedi.com/kirr/go123/exc"
+	"lab.nexedi.com/kirr/neo/go/transaction"
 	"lab.nexedi.com/kirr/neo/go/zodb"
 
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest"
 )
 
 type setStr = set.Str
 
 // ΔFTestEntry represents one entry in ΔFtail tests.
 type ΔFTestEntry struct {
-	δblkTab  map[int64]string  // change in tree part {} #blk -> ZBlk<oid>
-	δblkData setStr            // change to ZBlk objects
+	δblkTab  map[int64]string  // changes in tree part  {} #blk -> ZBlk<oid>
+	δblkData setStr            // changes to ZBlk objects
 }
 
-func TestΔFtail(t *testing.T) {
+func TestΔFtail(t_ *testing.T) {
+	t := xbtreetest.NewT(t_)
+	X := exc.Raiseif
+
 	// δT is shorthand to create δblkTab.
 	type δT = map[int64]string
 
@@ -56,14 +70,49 @@ func TestΔFtail(t *testing.T) {
 		{δT{2:c},		δD(a,b)},
 	}
 
-	vδf    := []ΔFile{}           // (rev↑, {}blk)		XXX +.Size?
-	blkTab := map[int64]string{}  // #blk -> ZBlk<oid>
-	Zinblk := map[string]setI64{} // ZBlk<oid> -> which #blk refer to it
-	for _, test := range testv {
+	δftail := NewΔFtail(t.Head().At, t.DB)
+
+	// load zfile via root['treegen/file']
+	txn, ctx := transaction.New(context.Background())
+	defer txn.Abort()
+	zconn, err := t.DB.Open(ctx, &zodb.ConnOptions{At: t.Head().At});  X(err)
+	xzroot, err := zconn.Get(ctx, 0);  X(err)
+	zroot := xzroot.(*zodb.Map)
+	err = zroot.PActivate(ctx);  X(err)
+	defer zroot.PDeactivate()
+	zfile := zroot.Data["treegen/file"].(*ZBigFile)
+	err = zfile.PActivate(ctx);  X(err)
+	defer zfile.PDeactivate()
+	if treeOid := zfile.blktab.POid(); treeOid != t.Root() {
+		t.Fatalf("BUG: zfile.blktab (%s)  !=  treeroot (%s)", treeOid, t.Root())
+	}
+
+	// track zfile[-∞,∞) from the beginning
+	// this should make ΔFtail to see all zfile changes
+	size, path, err := zfile.Size(ctx);  X(err)
+	δftail.Track(zfile, /*blk*/-1, path, /*zblk*/nil)
+	if size != 0 {
+		t.Fatalf("BUG: zfile is not initially empty:  size=%d", size)
+	}
+
+	// data built via applying changes from testv
+	vδf     := []ΔFile{}           // (rev↑, {}blk)		XXX +.Size?
+	blkTab  := map[int64]string{}  // #blk -> ZBlk<oid>
+	blkData := map[string]string{} // ZBlk<oid> -> data
+	Zinblk  := map[string]setI64{} // ZBlk<oid> -> which #blk refer to it
+
+	// initialize blkData from root['treegen/values']
+	for /*oid*/_, zblk := range t.Head().ZBlkDataTab {
+		// treegen initializes values[x] = ZBlk(x)
+		blkData[zblk] = zblk
+	}
+
+
+	for i, test := range testv {
 		δf := setI64{}
 
+		// rebuild blkTab/Zinblk
 		for blk, zblk := range test.δblkTab {
-			// rebuild blkTab/Zinblk
 			zprev, ok := blkTab[blk]
 			if ok {
 				delete(Zinblk[zprev], blk)
@@ -87,8 +136,16 @@ func TestΔFtail(t *testing.T) {
 			}
 		}
 
-		// update δf due to change in ZBlk data
+		// rebuild blkData
 		for zblk := range test.δblkData {
+			data, ok := blkData[zblk]               // e.g. a -> a2
+			if !ok {
+				t.Fatalf("BUG: blk %s not in blkData\nblkData: %v", zblk, blkData)
+			}
+			data = fmt.Sprintf("%s%d", data[:1], i) // e.g. a4
+			blkData[zblk] = data
+
+			// update δf due to change in ZBlk data
 			for blk := range Zinblk[zblk] {
 				δf.Add(blk)
 			}
@@ -99,8 +156,23 @@ func TestΔFtail(t *testing.T) {
 				Blocks: δf,
 				Size:   false/*XXX*/,
 		})
+
+		// commit updated blkTab + blkData
+		tdTxt := fmt.Sprintf("t%s D%s", xbtreetest.KVTxt(blkTab), blkDataTxt(blkData))
+		commit := t.CommitTree(tdTxt)
+		fmt.Printf("@%s  %s\n", commit.At, commit.Tree)
+
+		// XXX assert 
 	}
 }
 
 // XXX TestΔFtailRandom(t *testing.T) {
 //}
+
+
+// blkDataTxt returns string representation of {} blkData.
+//
+// it is similar to xbtreetest.KVTxt but uses string instead of Key for keys.
+func blkDataTxt(dataTab map[string]string) string {
+	panic("TODO")
+}

wcfs/internal/zdata/δftail_x_test.go

+// Copyright (C) 2021  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package zdata_test
+
+import (
+	_ "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest/init"
+)