.

wcfs/set.go.in

@@ -56,3 +56,53 @@ func (s Set) Elements() []VALUE {
 	}
 	return ev
 }
+
+// Union returns s ∪ t
+func (s Set) Union(t Set) Set {
+	u := Set{}
+	for v := range s {
+		u.Add(v)
+	}
+	for v := range t {
+		u.Add(v)
+	}
+	return u
+}
+
+// Intersection returns s ∩ t
+func (s Set) Intersection(t Set) Set {
+	i := Set{}
+	for v := range s {
+		if t.Has(v) {
+			i.Add(v)
+		}
+	}
+	return i
+}
+
+// Difference returns s\t.
+func (s Set) Difference(t Set) Set {
+	d := Set{}
+	for v := range s {
+		if !t.Has(v) {
+			d.Add(v)
+		}
+	}
+	return d
+}
+
+// SymmetricDifference returns s Δ t.
+func (s Set) SymmetricDifference(t Set) Set {
+	d := Set{}
+	for v := range s {
+		if !t.Has(v) {
+			d.Add(v)
+		}
+	}
+	for v := range t {
+		if !s.Has(v) {
+			d.Add(v)
+		}
+	}
+	return d
+}

wcfs/zset_bigfile.go

@@ -58,3 +58,53 @@ func (s SetBigFile) Elements() []*BigFile {
 	}
 	return ev
 }
+
+// Union returns s ∪ t
+func (s SetBigFile) Union(t SetBigFile) SetBigFile {
+	u := SetBigFile{}
+	for v := range s {
+		u.Add(v)
+	}
+	for v := range t {
+		u.Add(v)
+	}
+	return u
+}
+
+// Intersection returns s ∩ t
+func (s SetBigFile) Intersection(t SetBigFile) SetBigFile {
+	i := SetBigFile{}
+	for v := range s {
+		if t.Has(v) {
+			i.Add(v)
+		}
+	}
+	return i
+}
+
+// Difference returns s\t.
+func (s SetBigFile) Difference(t SetBigFile) SetBigFile {
+	d := SetBigFile{}
+	for v := range s {
+		if !t.Has(v) {
+			d.Add(v)
+		}
+	}
+	return d
+}
+
+// SymmetricDifference returns s Δ t.
+func (s SetBigFile) SymmetricDifference(t SetBigFile) SetBigFile {
+	d := SetBigFile{}
+	for v := range s {
+		if !t.Has(v) {
+			d.Add(v)
+		}
+	}
+	for v := range t {
+		if !s.Has(v) {
+			d.Add(v)
+		}
+	}
+	return d
+}

wcfs/zset_i64.go

@@ -58,3 +58,53 @@ func (s SetI64) Elements() []int64 {
 	}
 	return ev
 }
+
+// Union returns s ∪ t
+func (s SetI64) Union(t SetI64) SetI64 {
+	u := SetI64{}
+	for v := range s {
+		u.Add(v)
+	}
+	for v := range t {
+		u.Add(v)
+	}
+	return u
+}
+
+// Intersection returns s ∩ t
+func (s SetI64) Intersection(t SetI64) SetI64 {
+	i := SetI64{}
+	for v := range s {
+		if t.Has(v) {
+			i.Add(v)
+		}
+	}
+	return i
+}
+
+// Difference returns s\t.
+func (s SetI64) Difference(t SetI64) SetI64 {
+	d := SetI64{}
+	for v := range s {
+		if !t.Has(v) {
+			d.Add(v)
+		}
+	}
+	return d
+}
+
+// SymmetricDifference returns s Δ t.
+func (s SetI64) SymmetricDifference(t SetI64) SetI64 {
+	d := SetI64{}
+	for v := range s {
+		if !t.Has(v) {
+			d.Add(v)
+		}
+	}
+	for v := range t {
+		if !s.Has(v) {
+			d.Add(v)
+		}
+	}
+	return d
+}

wcfs/zset_oid.go

@@ -58,3 +58,53 @@ func (s SetOid) Elements() []Oid {
 	}
 	return ev
 }
+
+// Union returns s ∪ t
+func (s SetOid) Union(t SetOid) SetOid {
+	u := SetOid{}
+	for v := range s {
+		u.Add(v)
+	}
+	for v := range t {
+		u.Add(v)
+	}
+	return u
+}
+
+// Intersection returns s ∩ t
+func (s SetOid) Intersection(t SetOid) SetOid {
+	i := SetOid{}
+	for v := range s {
+		if t.Has(v) {
+			i.Add(v)
+		}
+	}
+	return i
+}
+
+// Difference returns s\t.
+func (s SetOid) Difference(t SetOid) SetOid {
+	d := SetOid{}
+	for v := range s {
+		if !t.Has(v) {
+			d.Add(v)
+		}
+	}
+	return d
+}
+
+// SymmetricDifference returns s Δ t.
+func (s SetOid) SymmetricDifference(t SetOid) SetOid {
+	d := SetOid{}
+	for v := range s {
+		if !t.Has(v) {
+			d.Add(v)
+		}
+	}
+	for v := range t {
+		if !s.Has(v) {
+			d.Add(v)
+		}
+	}
+	return d
+}

wcfs/δbtail.go

@@ -161,6 +161,7 @@ type ΔValue struct {
 // XXX -> multiple readers / single writer?
 //
 // See also zodb.ΔTail
+// XXX naming -> ΔBTail ?
 type ΔBtail struct {
 	// raw ZODB changes; Kept to rebuild δBtail/byRoot after new Track.
 	// includes all changed objects, not only tracked ones.
@@ -494,6 +495,12 @@ func (tidx trackIndex) AddPath(path []zodb.Oid) {
 	}
 }
 
+// rebuild rebuilds ΔBtree taking trackNew requests into account. XXX
+// XXX place
+func (δBtail *ΔBtail) rebuild() {
+	panic("TODO")
+}
+
 // Update updates δB with per-object level ZODB changes.
 //
 // Only those objects from δZ that belong to tracked set are guaranteed to be

wcfs/δbtail_test.go

@@ -424,6 +424,28 @@ func XGetTree(db *zodb.DB, at zodb.Tid, root zodb.Oid) RBucketSet {
 	return rbucketv
 }
 
+// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
+// XXX naming, place
+// XXX +return blkRevMax and use it
+func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
+	path = []Node{}
+	xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
+		path = append(path, node)
+	})
+	if err != nil {
+		return nil, false, nil, err
+	}
+
+	if ok {
+		zblk, ok = xzblk.(ZBlk)
+		if !ok {
+			return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", typeOf(xzblk))  // XXX errctx
+		}
+	}
+
+	return zblk, ok, path, nil
+}
+
 // xwalkDFS walks ztree in depth-first order emitting bvisit callback on visited bucket nodes.
 func xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, bvisit func(*RBucket)) {
 	_xwalkDFS(ctx, lo, hi_, ztree, /*rparent*/nil, bvisit)
@@ -522,20 +544,67 @@ func xzgetBlkDataAt(db *zodb.DB, zblkOid zodb.Oid, at zodb.Tid) string {
 	return xzgetBlkData(ctx, zconn, zblkOid)
 }
 
+// KAdj computes adjacency matrix for t1 -> t2 transition.
+// KAdj itself is verified by testΔBTail on entries with .kadjOK set.
+func KAdj(t1, t2 *tTreeCommit) (kadj map[Key]SetKey) {
+	// kadj = {} k -> adjacent keys.
+	// if k is tracked -> changes to adjacents must be in Update(t1->t2).
+	kadj = map[Key]SetKey{}
+	for k := range allTestKeys(t1, t2) {
+		adj1 := SetKey{}
+		adj2 := SetKey{}
+
+		q1 := []Key{k}
+		q2 := []Key{k}
+
+		//tracef("\n")
+		for len(q1) > 0 || len(q2) > 0 {
+			//tracef("q1: %v\n", q1)
+			//tracef("q2: %v\n", q2)
+			if l1 := len(q1); l1 > 0 {
+				k1 := q1[l1-1];  q1 = q1[:l1-1]
+				if !adj1.Has(k1) {
+					for k_ := range t1.xkv.Get(k1).kv {
+						adj1.Add(k_)
+						for k__ := range t2.xkv.Get(k_).kv {
+							if !adj2.Has(k__) {
+								q2 = append(q2, k__)
+							}
+						}
+					}
+				}
+			}
+
+			if l2 := len(q2); l2 > 0 {
+				k2 := q2[l2-1];  q2 = q2[:l2-1]
+				if !adj2.Has(k2) {
+					for k_ := range t2.xkv.Get(k2).kv {
+						adj2.Add(k_)
+						for k__ := range t1.xkv.Get(k_).kv {
+							if !adj1.Has(k__) {
+								q1 = append(q1, k__)
+							}
+						}
+					}
+				}
+			}
+		}
+
+		adj := SetKey{}; adj.Add(k); adj.Update(adj1); adj.Update(adj2)
+		kadj[k] = adj
+	}
+
+	return kadj
+}
+
 
 // xverifyΔBTail_Update verifies how ΔBTail handles ZODB update for a tree with changes in between t1->t2.
-//
-// it is known that @at1 and @at2 the tree has xkv1 and xkv2 values correspondingly.	XXX kill
-// it is known that for at1->at2 ZODB-level change is δZ.				XXX kill
-//
-// kadjOK may be optionally provided. if kadjOK != nil computed adjacency
-// matrix is verified against it.
-func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1, t2 *tTreeCommit, kadjOK map[Key]SetKey) {
+func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1, t2 *tTreeCommit) {
 	// verify transition at1->at2 for all initial states of tracked {keys} from kv1 + kv2 + ∞
 	allKeys := allTestKeys(t1, t2)
-	allKeyv := allKeys.SortedKeys()
-
+	allKeyv := allKeys.SortedElements()
 
+/*
 	xkv1 := t1.xkv
 	xkv2 := t2.xkv
 
@@ -588,7 +657,9 @@ func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb
 	if kadjOK != nil && !reflect.DeepEqual(kadj, kadjOK) {
 		t.Errorf("%s: BUG: computed kadj is wrong:\nkadjOK: %v\nkadj  : %v\n\n", subj, kadjOK, kadj)
 	}
+*/
 
+	kadj12 := KAdj(t1, t2)
 
 	// verify at1->at2 for all combination of initial tracked keys.
 	for kidx := range IntSets(len(allKeyv)) {
@@ -597,7 +668,7 @@ func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb
 			keys.Add(allKeyv[idx])
 		}
 
-		xverifyΔBTail_Update1(t, subj, db, treeRoot, t1.at,t2.at, xkv1,xkv2, t2.δZ, keys, kadj)
+		xverifyΔBTail_Update1(t, subj, db, treeRoot, t1.at,t2.at, t1.xkv,t2.xkv, t2.δZ, keys, kadj12)
 	}
 }
 
@@ -642,17 +713,13 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 
 	kadjTracked = SetKey{} // kadj[Tracked]  (all keys adjacent to tracked keys)
 	for k := range initialTrackedKeys {
-		path := []Node{}
-		_, ok, err := ztree.VGet(ctx, k, func(node Node) {
-			path = append(path, node)
-		}); X(err)
-
+		_, ok, path, err := ztreeGetBlk(ctx, ztree, k);  X(err)
 		err = δbtail.Track(k, ok, path); X(err)
 
 		kadjTracked.Update(kadj[k])
 	}
 
-	// assert Tracked ∈ kadj[Tracked]  except ∞
+	// assert Tracked ∈ kadj[Tracked]  except ∞   XXX -> initialTrackedKeys.Difference(kadjTracked)
 	trackedNotInKadj := SetKey{}
 	for k := range initialTrackedKeys {
 		if !kadjTracked.Has(k) {
@@ -771,10 +838,101 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
 	}
 }
 
-// xverifyΔBtail_Get verifies δBtail.Get on series of vt ZODB changes.
+// xverifyΔBTail_rebuild verifies δBtail.rebuild during t0->t1->t2 transition.
+//
+// t0->t1 exercises from-scratch rebuild,
+// t1->t2 further exercises incremental rebuild.
+func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1, t2 *tTreeCommit) {
+	return
+	tAllKeys := allTestKeys(t0, t1, t2)
+	tAllKeyv := tAllKeys.SortedElements()
+
+	// tid -> "at_i"
+	xat := map[zodb.Tid]string{
+		t0.at: "at0",
+		t1.at: "at1",
+		t2.at: "at2",
+	}
+
+	// XXX move prints under first t.Run ?
+	fmt.Printf("@%s:  %v\n", xat[t0.at], xkvFlatten(t0.xkv))
+	fmt.Printf("@%s:  %v\n", xat[t1.at], xkvFlatten(t1.xkv))
+	fmt.Printf("@%s:  %v\n", xat[t2.at], xkvFlatten(t2.xkv))
+
+	// verify t0 -> t1 Track(keys1) -> t2 Track(keys2)
+	// for all combinations of keys1 and keys2
+	for k1idx := range IntSets(len(tAllKeyv)) {
+		keys1 := SetKey{}
+		for _, idx1 := range k1idx {
+			keys1.Add(tAllKeyv[idx1])
+		}
+
+		t.Run(fmt.Sprintf("%s→%s;track=%s;rebuild", t0.tree, t1.tree, keys1), func(t *testing.T) {
+			δbtail := NewΔBtail(t0.at, db)
+
+			xverifyΔBTail_rebuild1(t, db, δbtail, t0, t1, treeRoot,
+				keys1,
+				/*trackIdx=*/ /*XXX temp*/nil /*Tadj(t1, keys1)*/,
+				/*vδB=*/      /*XXX temp*/nil /*[δ1/Tadj(keys1)*/)
+
+/* TODO reenable
+			// tRestKeys2 = tAllKeys - keys1
+			tRestKeys2 := tAllKeys.Difference(keys1)
+			tRestKeyv2 := tRestKeys2.SortedElements()
+			for k2idx := range IntSets(len(tRestKeyv2)) {
+				keys2 := SetKey{}
+				for _, idx2 := range k2idx {
+					keys2.Add(tRestKeyv2[idx2])
+				}
+
+				δbtail_ = δbtail.Clone()
+				// XXX t.Run
+				xverifyΔBTail_rebuild1(t, δbtail_, t1, t2, keys,
+					trackIdx==Tadj(t2,Tadj(t1,keys1)) v Tadj(t2,keys2),
+					vδΒ=[δ1/keys1+keys2, δ2/keys1+keys2])
+			}
+*/
+		})
+	}
+}
+
+// xverifyΔBTail_rebuild1 verifies ΔBTree state after Update(ti->tj), Track(keys), rebuild
+func xverifyΔBTail_rebuild1(t *testing.T, db *zodb.DB, δbtail *ΔBtail, ti, tj *tTreeCommit, treeRoot zodb.Oid, keys SetKey, trackIdxOK trackIndex, vδBok []ΔB) {
+	X := exc.Raiseif
+
+	// XXX assert trackIdx=ø, newTrackIdx=ø, vδB=[]
+
+	// Update ati -> atj
+	δB, err := δbtail.Update(tj.δZ);  X(err)
+	// XXX assert δB.Rev = tj.at; δB = δ(ti,tj)/initially tracked
+	// XXX assert trackIdx=ø, newTrackIdx=ø, vδB=[ø]
+	_ = δB
+
+	// Track(keys)
+	txn, ctx := transaction.New(context.Background())
+	defer txn.Abort()
+	zconn, err := db.Open(ctx, &zodb.ConnOptions{At: tj.at});  X(err)
+	xtree, err := zconn.Get(ctx, treeRoot);	 X(err)
+	ztree := xtree.(*Tree)
+	for k := range keys {
+		_, ok, path, err := ztreeGetBlk(ctx, ztree, k);  X(err)
+		err = δbtail.Track(k, ok, path);  X(err)
+	}
+
+	// XXX assert trackIdx=ø, newTrackIdx=T(keys), vδB=[ø]
+
+	δbtail.rebuild()
+
+	// XXX assert trackIdx=T(keys), newTrackIdx=ø, vδB=[δ1/T(keys)]
+
+	// XXX verify Get
+	// XXX verify SliceByRootRev
+}
+
+// xverifyΔBTail_Get verifies δBtail.Get on series of vt ZODB changes.
 // XXX
+// XXX kill
 func xverifyΔBTail_Get(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt ...*tTreeCommit) {
-	return
 	subj := vt[0].tree
 	for _, t := range vt[1:] {
 		subj += "→" + t.tree
@@ -790,7 +948,7 @@ func xverifyΔBTail_Get(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt ...*tTr
 		}
 
 		tkeys := allTestKeys(vt...)
-		tkeyv := tkeys.SortedKeys()
+		tkeyv := tkeys.SortedElements()
 
 		// verify  t1->t2-> ... ->tn Track(keys) Get(keys, @at)
 		// for all combinations of tracked keys and at
@@ -824,18 +982,14 @@ func xverifyΔBTail_Get1(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt []*tTr
 	ztree := xtree.(*Tree)
 
 	for k := range keys {
-		path := []Node{}
-		_, ok, err := ztree.VGet(ctx, k, func(node Node) {
-			path = append(path, node)
-		}); X(err)
-
+		_, ok, path, err := ztreeGetBlk(ctx, ztree, k);  X(err)
 		err = δbtail.Track(k, ok, path);  X(err)
 	}
 
 	// verify Get(k, @at) for all keys and @at
 	for i := 1; i < len(vt); i++ {
 		at := vt[i].at
-		for _, k := range keys.SortedKeys() {
+		for _, k := range keys.SortedElements() {
 			vOid, ok, rev, revExact, err := δbtail.Get(ctx, ztree, k, at);  X(err)
 			v := xzgetBlkDataAt(db, vOid, rev)
 
@@ -862,7 +1016,6 @@ func xverifyΔBTail_Get1(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt []*tTr
 			}
 		}
 	}
-
 }
 
 
@@ -962,12 +1115,28 @@ func testΔBTail(t *testing.T, testq chan ΔBTestEntry) {
 	for test := range testq {
 		t2 := XCommitTree(test.tree)
 
-		// ΔBTail.Update
 		subj := fmt.Sprintf("%s -> %s", t1.tree, t2.tree)
 		tracef("\n\n\n**** %s ****\n\n", subj)
-		xverifyΔBTail_Update(t, subj, db, tg.treeRoot, t1,t2, test.kadjOK)
 
-		// ΔBTail.Get
+		// kadj
+		if kadjOK := test.kadjOK; kadjOK != nil {
+			t.Run(fmt.Sprintf("kadj/%s→%s", t1.tree, t2.tree), func(t *testing.T) {
+				kadj := KAdj(t1, t2)
+				if !reflect.DeepEqual(kadj, kadjOK) {
+					t.Fatalf("BUG: computed kadj is wrong:\nkadjOK: %v\nkadj  : %v\n\n", kadjOK, kadj)
+				}
+			})
+		}
+
+		// ΔBTail.Update
+		xverifyΔBTail_Update(t, subj, db, tg.treeRoot, t1,t2)
+
+		// ΔBTail.rebuild
+		if t0 != nil {
+			xverifyΔBTail_rebuild(t, db, tg.treeRoot, t0,t1,t2)
+		}
+
+/* XXX kill
 		xverifyΔBTail_Get(t, db, tg.treeRoot, t2)
 		xverifyΔBTail_Get(t, db, tg.treeRoot, t1, t2)
 		if t0 != nil {
@@ -975,6 +1144,7 @@ func testΔBTail(t *testing.T, testq chan ΔBTestEntry) {
 		}
 
 		// XXX ΔBTail.SliceByRootRev
+*/
 
 		t0, t1 = t1, t2
 	}
@@ -1433,7 +1603,7 @@ func allTestKeys(vt ...*tTreeCommit) SetKey {
 }
 
 // easies debugging / makes error output stable from run to run.
-func (ks SetKey) SortedKeys() []Key {
+func (ks SetKey) SortedElements() []Key {
 	keyv := ks.Elements()
 	sort.Slice(keyv, func(i, j int) bool {
 		return keyv[i] < keyv[j]
@@ -1442,13 +1612,13 @@ func (ks SetKey) SortedKeys() []Key {
 }
 func (ks SetKey) String() string {
 	strv := []string{}
-	for _, k := range ks.SortedKeys() {
+	for _, k := range ks.SortedElements() {
 		strv = append(strv, fmt.Sprintf("%d", k))
 	}
 	return "{" + strings.Join(strv, " ") + "}"
 }
 
-func (os SetOid) SortedKeys() []Oid {
+func (os SetOid) SortedElements() []Oid {
 	oidv := os.Elements()
 	sort.Slice(oidv, func(i, j int) bool {
 		return oidv[i] < oidv[j]
@@ -1457,7 +1627,7 @@ func (os SetOid) SortedKeys() []Oid {
 }
 func (os SetOid) String() string {
 	strv := []string{}
-	for _, oid := range os.SortedKeys() {
+	for _, oid := range os.SortedElements() {
 		strv = append(strv, fmt.Sprintf("%s", oid))
 	}
 	return "{" + strings.Join(strv, " ") + "}"