X First round of preparation steps for ΔFtail tests

* t2: (29 commits)
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  Revert "."
  .
  X Unexprt ΔBroots
  .
  X Unexport SetXXX from packages API
  .
  ...

wcfs/TODO.txt

-- doc: notes on how things are organized in wendelin.core 2
-
-
-wcfs:
-
-- SIGSEGV is used only to track writes

wcfs/gen-δtail

-#!/bin/bash -e
-# δtail.go.in -> specialized with concrete types
-# gen-δtail KIND ID out
-
-# Copyright (C) 2018-2021  Nexedi SA and Contributors.
-#                          Kirill Smelkov <kirr@nexedi.com>
-#
-# This program is free software: you can Use, Study, Modify and Redistribute
-# it under the terms of the GNU General Public License version 3, or (at your
-# option) any later version, as published by the Free Software Foundation.
-#
-# You can also Link and Combine this program with other software covered by
-# the terms of any of the Free Software licenses or any of the Open Source
-# Initiative approved licenses and Convey the resulting work. Corresponding
-# source of such a combination shall include the source code for all other
-# software used.
-#
-# This program is distributed WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-#
-# See COPYING file for full licensing terms.
-# See https://www.nexedi.com/licensing for rationale and options.
-
-# TODO kill this after finishing ΔFtail (zodb/δtail.go generic is not used here)
-
-KIND=$1
-ID=$2
-out=$3
-
-zodb=lab.nexedi.com/kirr/neo/go/zodb
-zdir=`go list -f '{{.Dir}}' $zodb`
-zrev=`git -C $zdir describe --always`
-
-echo "// Code generated by gen-δtail $KIND $ID; DO NOT EDIT." >$out
-echo "// (from $zodb @ $zrev)" >>$out
-echo >>$out
-
-$zdir/δtail.go.cat-generic | sed		\
-	-e "s/PACKAGE/main/g"			\
-	-e "s/ID/$ID/g"				\
-	-e "s/ΔTail/ΔTail${KIND}/g"		\
-	-e "s/δRevEntry/δRevEntry${KIND}/g"	\
-	>>$out

wcfs/gogccrash

-#!/bin/bash -e
-# tries to minimize reproducer for https://github.com/golang/go/issues/41303
-
-export GOTRACEBACK=crash
-ulimit -c unlimited
-
-go test -c
-
-cwd=$(pwd)
-
-# runtest1
-function runtest1() {
-	for i in `seq 1000`; do
-		echo -e "\n>>> #$i"
-		#GOGC=1 $cwd/wcfs.test -test.v -test.run 'TestZBlk|TestΔBTail|TestΔBTreeAllStructs' || break
-		#GOGC=1 $cwd/wcfs.test -test.v -test.run 'TestZBlk|TestΔBTail' || break
-		#GOGC=0 $cwd/wcfs.test -test.v -test.run 'TestZBlk|TestΔBTail' || break
-		GOGC=0 $cwd/wcfs.test -test.v -test.count=100 -test.run 'TestZBlk|TestΔBTail' || break
-	done
-}
-
-nwork=2
-rm -rf BUG
-for n in `seq $nwork`; do
-	workdir=BUG/$n
-	mkdir -p $workdir
-	ln -s $cwd/testdata $workdir
-	ln -s $cwd/testprog $workdir
-	(cd $workdir && runtest1 >>log 2>&1) &
-done
-
-wait -n
-echo "done, crashes:"
-find BUG -name "*core*"
-
-echo
-kill `jobs -p`
-wait

wcfs/internal/set/gen-set

@@ -21,10 +21,9 @@
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
 
-PACKAGE=$1
-KIND=$2
-VALUE=$3
-out=$4
+KIND=$1
+VALUE=$2
+out=$3
 
 input=$(dirname $0)/set.go.in
 
@@ -32,7 +31,6 @@ echo "// Code generated by gen-set $KIND $VALUE; DO NOT EDIT." >$out
 echo >>$out
 
 sed	\
-	-e "s/PACKAGE/$PACKAGE/g"	\
 	-e "s/VALUE/$VALUE/g"		\
-	-e "s/Set/Set${KIND}/g"		\
+	-e "s/Set/${KIND}/g"		\
 	$input >>$out

wcfs/internal/set/set.go

@@ -19,9 +19,10 @@
 
 package set
 
-//go:generate ./gen-set set I64 int64 zset_i64.go
-//go:generate ./gen-set set Oid _Oid  zset_oid.go
-//go:generate ./gen-set set Tid _Tid  zset_tid.go
+//go:generate ./gen-set I64 int64  zset_i64.go
+//go:generate ./gen-set Str string zset_str.go
+//go:generate ./gen-set Oid _Oid   zset_oid.go
+//go:generate ./gen-set Tid _Tid   zset_tid.go
 
 import (
 	"lab.nexedi.com/kirr/neo/go/zodb"

wcfs/internal/set/set.go.in

@@ -17,7 +17,7 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package PACKAGE
+package set
 
 import (
 	"fmt"

wcfs/internal/set/zset_i64.go

@@ -27,35 +27,35 @@ import (
 	"strings"
 )
 
-// SetI64 is a set of int64.
-type SetI64 map[int64]struct{}
+// I64 is a set of int64.
+type I64 map[int64]struct{}
 
 // Add adds v to the set.
-func (s SetI64) Add(v int64) {
+func (s I64) Add(v int64) {
 	s[v] = struct{}{}
 }
 
 // Del removes v from the set.
 // it is noop if v was not in the set.
-func (s SetI64) Del(v int64) {
+func (s I64) Del(v int64) {
 	delete(s, v)
 }
 
 // Has checks whether the set contains v.
-func (s SetI64) Has(v int64) bool {
+func (s I64) Has(v int64) bool {
 	_, ok := s[v]
 	return ok
 }
 
 // Update adds t values to s.
-func (s SetI64) Update(t SetI64) {
+func (s I64) Update(t I64) {
 	for v := range t {
 		s.Add(v)
 	}
 }
 
 // Elements returns all elements of set as slice.
-func (s SetI64) Elements() []int64 {
+func (s I64) Elements() []int64 {
 	ev := make([]int64, len(s))
 	i := 0
 	for e := range s {
@@ -66,14 +66,14 @@ func (s SetI64) Elements() []int64 {
 }
 
 // Union returns s ∪ t
-func (s SetI64) Union(t SetI64) SetI64 {
+func (s I64) Union(t I64) I64 {
 	// l = max(len(s), len(t))
 	l := len(s)
 	if lt := len(t); lt > l {
 		l = lt
 	}
 
-	u := make(SetI64, l)
+	u := make(I64, l)
 
 	for v := range s {
 		u.Add(v)
@@ -85,8 +85,8 @@ func (s SetI64) Union(t SetI64) SetI64 {
 }
 
 // Intersection returns s ∩ t
-func (s SetI64) Intersection(t SetI64) SetI64 {
-	i := SetI64{}
+func (s I64) Intersection(t I64) I64 {
+	i := I64{}
 	for v := range s {
 		if t.Has(v) {
 			i.Add(v)
@@ -96,8 +96,8 @@ func (s SetI64) Intersection(t SetI64) SetI64 {
 }
 
 // Difference returns s\t.
-func (s SetI64) Difference(t SetI64) SetI64 {
-	d := SetI64{}
+func (s I64) Difference(t I64) I64 {
+	d := I64{}
 	for v := range s {
 		if !t.Has(v) {
 			d.Add(v)
@@ -107,8 +107,8 @@ func (s SetI64) Difference(t SetI64) SetI64 {
 }
 
 // SymmetricDifference returns s Δ t.
-func (s SetI64) SymmetricDifference(t SetI64) SetI64 {
-	d := SetI64{}
+func (s I64) SymmetricDifference(t I64) I64 {
+	d := I64{}
 	for v := range s {
 		if !t.Has(v) {
 			d.Add(v)
@@ -123,7 +123,7 @@ func (s SetI64) SymmetricDifference(t SetI64) SetI64 {
 }
 
 // Equal returns whether a == b.
-func (a SetI64) Equal(b SetI64) bool {
+func (a I64) Equal(b I64) bool {
 	if len(a) != len(b) {
 		return false
 	}
@@ -139,8 +139,8 @@ func (a SetI64) Equal(b SetI64) bool {
 }
 
 // Clone returns copy of the set.
-func (orig SetI64) Clone() SetI64 {
-	klon := make(SetI64, len(orig))
+func (orig I64) Clone() I64 {
+	klon := make(I64, len(orig))
 	for v := range orig {
 		klon.Add(v)
 	}
@@ -149,7 +149,7 @@ func (orig SetI64) Clone() SetI64 {
 
 // --------
 
-func (s SetI64) SortedElements() []int64 {
+func (s I64) SortedElements() []int64 {
 	ev := s.Elements()
 	sort.Slice(ev, func(i, j int) bool {
 		return ev[i] < ev[j]
@@ -157,7 +157,7 @@ func (s SetI64) SortedElements() []int64 {
 	return ev
 }
 
-func (s SetI64) String() string {
+func (s I64) String() string {
 	ev := s.SortedElements()
 	strv := make([]string, len(ev))
 	for i, v := range ev {

wcfs/internal/set/zset_oid.go

@@ -27,35 +27,35 @@ import (
 	"strings"
 )
 
-// SetOid is a set of _Oid.
-type SetOid map[_Oid]struct{}
+// Oid is a set of _Oid.
+type Oid map[_Oid]struct{}
 
 // Add adds v to the set.
-func (s SetOid) Add(v _Oid) {
+func (s Oid) Add(v _Oid) {
 	s[v] = struct{}{}
 }
 
 // Del removes v from the set.
 // it is noop if v was not in the set.
-func (s SetOid) Del(v _Oid) {
+func (s Oid) Del(v _Oid) {
 	delete(s, v)
 }
 
 // Has checks whether the set contains v.
-func (s SetOid) Has(v _Oid) bool {
+func (s Oid) Has(v _Oid) bool {
 	_, ok := s[v]
 	return ok
 }
 
 // Update adds t values to s.
-func (s SetOid) Update(t SetOid) {
+func (s Oid) Update(t Oid) {
 	for v := range t {
 		s.Add(v)
 	}
 }
 
 // Elements returns all elements of set as slice.
-func (s SetOid) Elements() []_Oid {
+func (s Oid) Elements() []_Oid {
 	ev := make([]_Oid, len(s))
 	i := 0
 	for e := range s {
@@ -66,14 +66,14 @@ func (s SetOid) Elements() []_Oid {
 }
 
 // Union returns s ∪ t
-func (s SetOid) Union(t SetOid) SetOid {
+func (s Oid) Union(t Oid) Oid {
 	// l = max(len(s), len(t))
 	l := len(s)
 	if lt := len(t); lt > l {
 		l = lt
 	}
 
-	u := make(SetOid, l)
+	u := make(Oid, l)
 
 	for v := range s {
 		u.Add(v)
@@ -85,8 +85,8 @@ func (s SetOid) Union(t SetOid) SetOid {
 }
 
 // Intersection returns s ∩ t
-func (s SetOid) Intersection(t SetOid) SetOid {
-	i := SetOid{}
+func (s Oid) Intersection(t Oid) Oid {
+	i := Oid{}
 	for v := range s {
 		if t.Has(v) {
 			i.Add(v)
@@ -96,8 +96,8 @@ func (s SetOid) Intersection(t SetOid) SetOid {
 }
 
 // Difference returns s\t.
-func (s SetOid) Difference(t SetOid) SetOid {
-	d := SetOid{}
+func (s Oid) Difference(t Oid) Oid {
+	d := Oid{}
 	for v := range s {
 		if !t.Has(v) {
 			d.Add(v)
@@ -107,8 +107,8 @@ func (s SetOid) Difference(t SetOid) SetOid {
 }
 
 // SymmetricDifference returns s Δ t.
-func (s SetOid) SymmetricDifference(t SetOid) SetOid {
-	d := SetOid{}
+func (s Oid) SymmetricDifference(t Oid) Oid {
+	d := Oid{}
 	for v := range s {
 		if !t.Has(v) {
 			d.Add(v)
@@ -123,7 +123,7 @@ func (s SetOid) SymmetricDifference(t SetOid) SetOid {
 }
 
 // Equal returns whether a == b.
-func (a SetOid) Equal(b SetOid) bool {
+func (a Oid) Equal(b Oid) bool {
 	if len(a) != len(b) {
 		return false
 	}
@@ -139,8 +139,8 @@ func (a SetOid) Equal(b SetOid) bool {
 }
 
 // Clone returns copy of the set.
-func (orig SetOid) Clone() SetOid {
-	klon := make(SetOid, len(orig))
+func (orig Oid) Clone() Oid {
+	klon := make(Oid, len(orig))
 	for v := range orig {
 		klon.Add(v)
 	}
@@ -149,7 +149,7 @@ func (orig SetOid) Clone() SetOid {
 
 // --------
 
-func (s SetOid) SortedElements() []_Oid {
+func (s Oid) SortedElements() []_Oid {
 	ev := s.Elements()
 	sort.Slice(ev, func(i, j int) bool {
 		return ev[i] < ev[j]
@@ -157,7 +157,7 @@ func (s SetOid) SortedElements() []_Oid {
 	return ev
 }
 
-func (s SetOid) String() string {
+func (s Oid) String() string {
 	ev := s.SortedElements()
 	strv := make([]string, len(ev))
 	for i, v := range ev {

wcfs/internal/zdata/zset_bigfile.go → wcfs/internal/set/zset_str.go

-// Code generated by gen-set ZBigFile *ZBigFile; DO NOT EDIT.
+// Code generated by gen-set Str string; DO NOT EDIT.
 
 // Copyright (C) 2015-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
@@ -19,38 +19,44 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package zdata
+package set
 
-// SetZBigFile is a set of *ZBigFile.
-type SetZBigFile map[*ZBigFile]struct{}
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// Str is a set of string.
+type Str map[string]struct{}
 
 // Add adds v to the set.
-func (s SetZBigFile) Add(v *ZBigFile) {
+func (s Str) Add(v string) {
 	s[v] = struct{}{}
 }
 
 // Del removes v from the set.
 // it is noop if v was not in the set.
-func (s SetZBigFile) Del(v *ZBigFile) {
+func (s Str) Del(v string) {
 	delete(s, v)
 }
 
 // Has checks whether the set contains v.
-func (s SetZBigFile) Has(v *ZBigFile) bool {
+func (s Str) Has(v string) bool {
 	_, ok := s[v]
 	return ok
 }
 
 // Update adds t values to s.
-func (s SetZBigFile) Update(t SetZBigFile) {
+func (s Str) Update(t Str) {
 	for v := range t {
 		s.Add(v)
 	}
 }
 
 // Elements returns all elements of set as slice.
-func (s SetZBigFile) Elements() []*ZBigFile {
-	ev := make([]*ZBigFile, len(s))
+func (s Str) Elements() []string {
+	ev := make([]string, len(s))
 	i := 0
 	for e := range s {
 		ev[i] = e
@@ -60,14 +66,14 @@ func (s SetZBigFile) Elements() []*ZBigFile {
 }
 
 // Union returns s ∪ t
-func (s SetZBigFile) Union(t SetZBigFile) SetZBigFile {
+func (s Str) Union(t Str) Str {
 	// l = max(len(s), len(t))
 	l := len(s)
 	if lt := len(t); lt > l {
 		l = lt
 	}
 
-	u := make(SetZBigFile, l)
+	u := make(Str, l)
 
 	for v := range s {
 		u.Add(v)
@@ -79,8 +85,8 @@ func (s SetZBigFile) Union(t SetZBigFile) SetZBigFile {
 }
 
 // Intersection returns s ∩ t
-func (s SetZBigFile) Intersection(t SetZBigFile) SetZBigFile {
-	i := SetZBigFile{}
+func (s Str) Intersection(t Str) Str {
+	i := Str{}
 	for v := range s {
 		if t.Has(v) {
 			i.Add(v)
@@ -90,8 +96,8 @@ func (s SetZBigFile) Intersection(t SetZBigFile) SetZBigFile {
 }
 
 // Difference returns s\t.
-func (s SetZBigFile) Difference(t SetZBigFile) SetZBigFile {
-	d := SetZBigFile{}
+func (s Str) Difference(t Str) Str {
+	d := Str{}
 	for v := range s {
 		if !t.Has(v) {
 			d.Add(v)
@@ -101,8 +107,8 @@ func (s SetZBigFile) Difference(t SetZBigFile) SetZBigFile {
 }
 
 // SymmetricDifference returns s Δ t.
-func (s SetZBigFile) SymmetricDifference(t SetZBigFile) SetZBigFile {
-	d := SetZBigFile{}
+func (s Str) SymmetricDifference(t Str) Str {
+	d := Str{}
 	for v := range s {
 		if !t.Has(v) {
 			d.Add(v)
@@ -117,7 +123,7 @@ func (s SetZBigFile) SymmetricDifference(t SetZBigFile) SetZBigFile {
 }
 
 // Equal returns whether a == b.
-func (a SetZBigFile) Equal(b SetZBigFile) bool {
+func (a Str) Equal(b Str) bool {
 	if len(a) != len(b) {
 		return false
 	}
@@ -131,3 +137,31 @@ func (a SetZBigFile) Equal(b SetZBigFile) bool {
 
 	return true
 }
+
+// Clone returns copy of the set.
+func (orig Str) Clone() Str {
+	klon := make(Str, len(orig))
+	for v := range orig {
+		klon.Add(v)
+	}
+	return klon
+}
+
+// --------
+
+func (s Str) SortedElements() []string {
+	ev := s.Elements()
+	sort.Slice(ev, func(i, j int) bool {
+		return ev[i] < ev[j]
+	})
+	return ev
+}
+
+func (s Str) String() string {
+	ev := s.SortedElements()
+	strv := make([]string, len(ev))
+	for i, v := range ev {
+		strv[i] = fmt.Sprintf("%v", v)
+	}
+	return "{" + strings.Join(strv, " ") + "}"
+}

wcfs/internal/set/zset_tid.go

@@ -27,35 +27,35 @@ import (
 	"strings"
 )
 
-// SetTid is a set of _Tid.
-type SetTid map[_Tid]struct{}
+// Tid is a set of _Tid.
+type Tid map[_Tid]struct{}
 
 // Add adds v to the set.
-func (s SetTid) Add(v _Tid) {
+func (s Tid) Add(v _Tid) {
 	s[v] = struct{}{}
 }
 
 // Del removes v from the set.
 // it is noop if v was not in the set.
-func (s SetTid) Del(v _Tid) {
+func (s Tid) Del(v _Tid) {
 	delete(s, v)
 }
 
 // Has checks whether the set contains v.
-func (s SetTid) Has(v _Tid) bool {
+func (s Tid) Has(v _Tid) bool {
 	_, ok := s[v]
 	return ok
 }
 
 // Update adds t values to s.
-func (s SetTid) Update(t SetTid) {
+func (s Tid) Update(t Tid) {
 	for v := range t {
 		s.Add(v)
 	}
 }
 
 // Elements returns all elements of set as slice.
-func (s SetTid) Elements() []_Tid {
+func (s Tid) Elements() []_Tid {
 	ev := make([]_Tid, len(s))
 	i := 0
 	for e := range s {
@@ -66,14 +66,14 @@ func (s SetTid) Elements() []_Tid {
 }
 
 // Union returns s ∪ t
-func (s SetTid) Union(t SetTid) SetTid {
+func (s Tid) Union(t Tid) Tid {
 	// l = max(len(s), len(t))
 	l := len(s)
 	if lt := len(t); lt > l {
 		l = lt
 	}
 
-	u := make(SetTid, l)
+	u := make(Tid, l)
 
 	for v := range s {
 		u.Add(v)
@@ -85,8 +85,8 @@ func (s SetTid) Union(t SetTid) SetTid {
 }
 
 // Intersection returns s ∩ t
-func (s SetTid) Intersection(t SetTid) SetTid {
-	i := SetTid{}
+func (s Tid) Intersection(t Tid) Tid {
+	i := Tid{}
 	for v := range s {
 		if t.Has(v) {
 			i.Add(v)
@@ -96,8 +96,8 @@ func (s SetTid) Intersection(t SetTid) SetTid {
 }
 
 // Difference returns s\t.
-func (s SetTid) Difference(t SetTid) SetTid {
-	d := SetTid{}
+func (s Tid) Difference(t Tid) Tid {
+	d := Tid{}
 	for v := range s {
 		if !t.Has(v) {
 			d.Add(v)
@@ -107,8 +107,8 @@ func (s SetTid) Difference(t SetTid) SetTid {
 }
 
 // SymmetricDifference returns s Δ t.
-func (s SetTid) SymmetricDifference(t SetTid) SetTid {
-	d := SetTid{}
+func (s Tid) SymmetricDifference(t Tid) Tid {
+	d := Tid{}
 	for v := range s {
 		if !t.Has(v) {
 			d.Add(v)
@@ -123,7 +123,7 @@ func (s SetTid) SymmetricDifference(t SetTid) SetTid {
 }
 
 // Equal returns whether a == b.
-func (a SetTid) Equal(b SetTid) bool {
+func (a Tid) Equal(b Tid) bool {
 	if len(a) != len(b) {
 		return false
 	}
@@ -139,8 +139,8 @@ func (a SetTid) Equal(b SetTid) bool {
 }
 
 // Clone returns copy of the set.
-func (orig SetTid) Clone() SetTid {
-	klon := make(SetTid, len(orig))
+func (orig Tid) Clone() Tid {
+	klon := make(Tid, len(orig))
 	for v := range orig {
 		klon.Add(v)
 	}
@@ -149,7 +149,7 @@ func (orig SetTid) Clone() SetTid {
 
 // --------
 
-func (s SetTid) SortedElements() []_Tid {
+func (s Tid) SortedElements() []_Tid {
 	ev := s.Elements()
 	sort.Slice(ev, func(i, j int) bool {
 		return ev[i] < ev[j]
@@ -157,7 +157,7 @@ func (s SetTid) SortedElements() []_Tid {
 	return ev
 }
 
-func (s SetTid) String() string {
+func (s Tid) String() string {
 	ev := s.SortedElements()
 	strv := make([]string, len(ev))
 	for i, v := range ev {

wcfs/internal/xbtree/blib/blib.go

+// Copyright (C) 2021  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package blib provides utilities related to BTrees.
+package blib
+
+import (
+	"fmt"
+	"math"
+
+	"lab.nexedi.com/kirr/neo/go/zodb/btree"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
+)
+
+// XXX instead of generics
+type Tree   = btree.LOBTree
+type Bucket = btree.LOBucket
+type Node   = btree.LONode
+type TreeEntry   = btree.LOEntry
+type BucketEntry = btree.LOBucketEntry
+
+type Key    = int64
+const KeyMax Key = math.MaxInt64
+const KeyMin Key = math.MinInt64
+
+type setOid = set.Oid
+
+
+// kstr formats key as string.
+func kstr(k Key) string {
+	if k == KeyMin {
+		return "-∞"
+	}
+	if k == KeyMax {
+		return "∞"
+	}
+	return fmt.Sprintf("%d", k)
+}
+
+
+func panicf(format string, argv ...interface{}) {
+	panic(fmt.Sprintf(format, argv...))
+}

wcfs/internal/xbtree/pptreesubset.go → wcfs/internal/xbtree/blib/pptreesubset.go

@@ -17,7 +17,7 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package xbtree
+package blib
 // PP-connected subset of tree nodes.
 
 import (
@@ -61,6 +61,16 @@ type nodeInTree struct {
 	nchild  int      // number of direct children in PPTreeSubSet referring to this node
 }
 
+// Parent returns parent of this node.
+func (n *nodeInTree) Parent() zodb.Oid {
+	return n.parent
+}
+
+// NChild returns number of children of this node in the tree subset.
+func (n *nodeInTree) NChild() int {
+	return n.nchild
+}
+
 // Has returns whether node is in the set.
 func (S PPTreeSubSet) Has(oid zodb.Oid) bool {
 	_, ok := S[oid]
@@ -102,7 +112,7 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
 
 	// normalize path: remove embedded bucket and check whether it was an
 	// artificial empty tree.
-	path = normPath(path)
+	path = NormPath(path)
 
 	// go through path and add nodes to the set
 	parent := zodb.InvalidOid
@@ -132,11 +142,11 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
 	}
 }
 
-// normPath normalizes path.
+// NormPath normalizes path.
 //
 // It removes embedded buckets and artificial empty trees.
 // Returned slice is subslice of path and aliases its memory.
-func normPath(path []zodb.Oid) []zodb.Oid {
+func NormPath(path []zodb.Oid) []zodb.Oid {
 	l := len(path)
 
 	// don't keep track of artificial empty tree
@@ -363,12 +373,12 @@ func (S PPTreeSubSet) verify() {
 	}()
 
 	// recompute {} oid -> children  and verify .nchild against it
-	children := make(map[zodb.Oid]SetOid, len(S))
+	children := make(map[zodb.Oid]setOid, len(S))
 	for oid, t := range S {
 		if t.parent != zodb.InvalidOid {
 			cc, ok := children[t.parent]
 			if !ok {
-				cc = make(SetOid, 1)
+				cc = make(setOid, 1)
 				children[t.parent] = cc
 			}
 			cc.Add(oid)
@@ -464,20 +474,20 @@ func (t nodeInTree) String() string {
 //     The second component with "-22" builds from leaf, but the first
 //     component with "-43" builds from non-leaf node.
 //
-//         δnchildNonLeafs = {43: +1}
+//         ΔnchildNonLeafs = {43: +1}
 //
 //     Only complete result of applying all
 //
-//         - xfixup(-1, δnchildNonLeafs)
+//         - xfixup(-1, ΔnchildNonLeafs)
 //         - δ.Del,
 //         - δ.Add, and
-//         - xfixup(+1, δnchildNonLeafs)
+//         - xfixup(+1, ΔnchildNonLeafs)
 //
 //     produces correctly PP-connected set.
 type ΔPPTreeSubSet struct {
 	Del PPTreeSubSet
 	Add PPTreeSubSet
-	δnchildNonLeafs map[zodb.Oid]int
+	ΔnchildNonLeafs map[zodb.Oid]int
 }
 
 // NewΔPPTreeSubSet creates new empty ΔPPTreeSubSet.
@@ -485,7 +495,7 @@ func NewΔPPTreeSubSet() *ΔPPTreeSubSet {
 	return &ΔPPTreeSubSet{
 		Del: PPTreeSubSet{},
 		Add: PPTreeSubSet{},
-		δnchildNonLeafs: map[zodb.Oid]int{},
+		ΔnchildNonLeafs: map[zodb.Oid]int{},
 	}
 }
 
@@ -493,15 +503,15 @@ func NewΔPPTreeSubSet() *ΔPPTreeSubSet {
 func (δ *ΔPPTreeSubSet) Update(δ2 *ΔPPTreeSubSet) {
 	δ.Del.UnionInplace(δ2.Del)
 	δ.Add.UnionInplace(δ2.Add)
-	for oid, δnc := range δ2.δnchildNonLeafs {
-		δ.δnchildNonLeafs[oid] += δnc
+	for oid, δnc := range δ2.ΔnchildNonLeafs {
+		δ.ΔnchildNonLeafs[oid] += δnc
 	}
 }
 
 // Reverse changes δ=diff(A->B) to δ'=diff(A<-B).
 func (δ *ΔPPTreeSubSet) Reverse() {
 	δ.Del, δ.Add = δ.Add, δ.Del
-	// δnchildNonLeafs stays the same
+	// ΔnchildNonLeafs stays the same
 }
 
 
@@ -514,7 +524,7 @@ func (S PPTreeSubSet) ApplyΔ(δ *ΔPPTreeSubSet) {
 		fmt.Printf("  A: %s\n", S)
 		fmt.Printf("  -: %s\n", δ.Del)
 		fmt.Printf("  +: %s\n", δ.Add)
-		fmt.Printf("  x: %v\n", δ.δnchildNonLeafs)
+		fmt.Printf("  x: %v\n", δ.ΔnchildNonLeafs)
 		defer fmt.Printf("\n->B: %s\n", S)
 	}
 
@@ -523,8 +533,8 @@ func (S PPTreeSubSet) ApplyΔ(δ *ΔPPTreeSubSet) {
 	δ.Add.verify()
 	defer S.verify()
 
-	S.xfixup(-1, δ.δnchildNonLeafs)
+	S.xfixup(-1, δ.ΔnchildNonLeafs)
 	S.xDifferenceInplace(δ.Del)
 	S.xUnionInplace(δ.Add)
-	S.xfixup(+1, δ.δnchildNonLeafs)
+	S.xfixup(+1, δ.ΔnchildNonLeafs)
 }

wcfs/internal/xbtree/pptreesubset_test.go → wcfs/internal/xbtree/blib/pptreesubset_test.go

@@ -17,7 +17,7 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package xbtree
+package blib
 
 import (
 	"strings"

wcfs/internal/xbtree/rangeset.go → wcfs/internal/xbtree/blib/rangeset.go

@@ -17,7 +17,7 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package xbtree
+package blib
 // set of [lo,hi) Key ranges.
 
 import (
@@ -30,8 +30,8 @@ const debugRangeSet = false
 
 // KeyRange represents [lo,hi) Key range.
 type KeyRange struct {
-	lo  Key
-	hi_ Key // NOTE _not_ hi) to avoid overflow at ∞;  hi = hi_ + 1
+	Lo  Key
+	Hi_ Key // NOTE _not_ hi) to avoid overflow at ∞;  hi = hi_ + 1
 }
 
 // RangedKeySet is set of Keys with adjacent keys coalesced into Ranges.
@@ -47,23 +47,23 @@ type RangedKeySet struct {
 
 // Has returns whether key k belongs to the range.
 func (r *KeyRange) Has(k Key) bool {
-	return (r.lo <= k && k <= r.hi_)
+	return (r.Lo <= k && k <= r.Hi_)
 }
 
 
 // Add adds key k to the set.
 func (S *RangedKeySet) Add(k Key) {
-	S.AddRange(KeyRange{lo: k, hi_: k})
+	S.AddRange(KeyRange{Lo: k, Hi_: k})
 }
 
 // Del removes key k from the set.
 func (S *RangedKeySet) Del(k Key) {
-	S.DelRange(KeyRange{lo: k, hi_: k})
+	S.DelRange(KeyRange{Lo: k, Hi_: k})
 }
 
 // Has returns whether key k belongs to the set.
 func (S *RangedKeySet) Has(k Key) bool {
-	return S.HasRange(KeyRange{lo: k, hi_: k})
+	return S.HasRange(KeyRange{Lo: k, Hi_: k})
 }
 
 
@@ -79,10 +79,10 @@ func (S *RangedKeySet) AddRange(r KeyRange) {
 	S.verify()
 	defer S.verify()
 
-	// find first ilo: r.lo < [ilo].hi
+	// find first ilo: r.Lo < [ilo].hi
 	l := len(S.rangev)
 	ilo := sort.Search(l, func(i int) bool {
-		 return r.lo <= S.rangev[i].hi_
+		 return r.Lo <= S.rangev[i].Hi_
 	})
 	debugfRSet("\tilo: %d\n", ilo)
 
@@ -92,58 +92,58 @@ func (S *RangedKeySet) AddRange(r KeyRange) {
 		debugfRSet("\tappend %s\t-> %s\n", r, S)
 	}
 
-	// find last jhi: [jhi].lo < r.hi
+	// find last jhi: [jhi].Lo < r.hi
 	jhi := ilo
 	for ;; jhi++ {
 		if jhi == l {
 			break
 		}
-		if S.rangev[jhi].lo <= r.hi_ {
+		if S.rangev[jhi].Lo <= r.Hi_ {
 			continue
 		}
 		break
 	}
 	debugfRSet("\tjhi: %d\n", jhi)
 
-	// entries in [ilo:jhi) ∈ [r.lo,r.hi) and should be merged into one
+	// entries in [ilo:jhi) ∈ [r.Lo,r.hi) and should be merged into one
 	if (jhi - ilo) > 1 {
-		lo  := S.rangev[ilo].lo
-		hi_ := S.rangev[jhi-1].hi_
+		lo  := S.rangev[ilo].Lo
+		hi_ := S.rangev[jhi-1].Hi_
 		vReplaceSlice(&S.rangev, ilo,jhi, KeyRange{lo,hi_})
 		debugfRSet("\tmerge S[%d:%d]\t-> %s\n", ilo, jhi, S)
 	}
 	jhi = -1 // no longer valid
 
 	// if [r.lo,r.hi) was outside of any entry - create new entry
-	if r.hi_ < S.rangev[ilo].lo {
+	if r.Hi_ < S.rangev[ilo].Lo {
 		vInsert(&S.rangev, ilo, r)
 		debugfRSet("\tinsert %s\t-> %s\n", r, S)
 	}
 
 	// now we have covered entries merged as needed into [ilo]
 	// extend this entry if r coverage is wider
-	if r.lo < S.rangev[ilo].lo {
-		S.rangev[ilo].lo = r.lo
+	if r.Lo < S.rangev[ilo].Lo {
+		S.rangev[ilo].Lo = r.Lo
 		debugfRSet("\textend left\t-> %s\n", S)
 	}
-	if r.hi_ > S.rangev[ilo].hi_ {
-		S.rangev[ilo].hi_ = r.hi_
+	if r.Hi_ > S.rangev[ilo].Hi_ {
+		S.rangev[ilo].Hi_ = r.Hi_
 		debugfRSet("\textend right\t-> %s\n", S)
 	}
 
 	// and check if we should merge it with right/left neighbours
 	if ilo+1 < len(S.rangev) { // right
-		if S.rangev[ilo].hi_+1 == S.rangev[ilo+1].lo {
+		if S.rangev[ilo].Hi_+1 == S.rangev[ilo+1].Lo {
 			vReplaceSlice(&S.rangev, ilo,ilo+2,
-				KeyRange{S.rangev[ilo].lo, S.rangev[ilo+1].hi_})
+				KeyRange{S.rangev[ilo].Lo, S.rangev[ilo+1].Hi_})
 			debugfRSet("\tmerge right\t-> %s\n", S)
 		}
 	}
 
 	if ilo > 0 { // left
-		if S.rangev[ilo-1].hi_+1 == S.rangev[ilo].lo {
+		if S.rangev[ilo-1].Hi_+1 == S.rangev[ilo].Lo {
 			vReplaceSlice(&S.rangev, ilo-1,ilo+1,
-				KeyRange{S.rangev[ilo-1].lo, S.rangev[ilo].hi_})
+				KeyRange{S.rangev[ilo-1].Lo, S.rangev[ilo].Hi_})
 			debugfRSet("\tmerge left\t-> %s\n", S)
 		}
 	}
@@ -163,10 +163,10 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
 	S.verify()
 	defer S.verify()
 
-	// find first ilo: r.lo < [ilo].hi
+	// find first ilo: r.Lo < [ilo].hi
 	l := len(S.rangev)
 	ilo := sort.Search(l, func(i int) bool {
-		 return r.lo <= S.rangev[i].hi_
+		 return r.Lo <= S.rangev[i].Hi_
 	})
 	debugfRSet("\tilo: %d\n", ilo)
 
@@ -175,13 +175,13 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
 		return
 	}
 
-	// find last jhi: [jhi].lo < r.hi
+	// find last jhi: [jhi].Lo < r.hi
 	jhi := ilo
 	for ;; jhi++ {
 		if jhi == l {
 			break
 		}
-		if S.rangev[jhi].lo <= r.hi_ {
+		if S.rangev[jhi].Lo <= r.Hi_ {
 			continue
 		}
 		break
@@ -196,19 +196,19 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
 	// [ilo+1:jhi-1] should be deleted
 	// [ilo] and [jhi-1] overlap with [r.lo,r.hi) - they should be deleted, or shrinked,
 	// or split+shrinked if ilo==jhi-1 and r is inside [ilo]
-	if jhi-ilo == 1 && S.rangev[ilo].lo < r.lo && r.hi_ < S.rangev[ilo].hi_ {
+	if jhi-ilo == 1 && S.rangev[ilo].Lo < r.Lo && r.Hi_ < S.rangev[ilo].Hi_ {
 		x := S.rangev[ilo]
 		vInsert(&S.rangev, ilo, x)
 		jhi++
 		debugfRSet("\tpresplit copy %s\t-> %s\n", x, S)
 	}
-	if S.rangev[ilo].lo < r.lo { // shrink left
-		S.rangev[ilo] = KeyRange{S.rangev[ilo].lo, r.lo-1}
+	if S.rangev[ilo].Lo < r.Lo { // shrink left
+		S.rangev[ilo] = KeyRange{S.rangev[ilo].Lo, r.Lo-1}
 		ilo++
 		debugfRSet("\tshrink [%d] left\t-> %s\n", ilo, S)
 	}
-	if r.hi_ < S.rangev[jhi-1].hi_ { // shrink right
-		S.rangev[jhi-1] = KeyRange{r.hi_+1, S.rangev[jhi-1].hi_}
+	if r.Hi_ < S.rangev[jhi-1].Hi_ { // shrink right
+		S.rangev[jhi-1] = KeyRange{r.Hi_+1, S.rangev[jhi-1].Hi_}
 		jhi--
 		debugfRSet("\tshrink [%d] right\t-> %s\n", jhi-1, S)
 	}
@@ -237,7 +237,7 @@ func (S *RangedKeySet) HasRange(r KeyRange) (yes bool) {
 	// find first ilo: r.lo < [ilo].hi
 	l := len(S.rangev)
 	ilo := sort.Search(l, func(i int) bool {
-		 return r.lo <= S.rangev[i].hi_
+		 return r.Lo <= S.rangev[i].Hi_
 	})
 	debugfRSet("\tilo: %d\n", ilo)
 
@@ -246,7 +246,7 @@ func (S *RangedKeySet) HasRange(r KeyRange) (yes bool) {
 	}
 
 	// all keys from r are in S if r ∈ [ilo]
-	return (S.rangev[ilo].lo <= r.lo && r.hi_ <= S.rangev[ilo].hi_)
+	return (S.rangev[ilo].Lo <= r.Lo && r.Hi_ <= S.rangev[ilo].Hi_)
 }
 
 
@@ -317,13 +317,13 @@ func (S *RangedKeySet) verify() {
 	hi_Prev := KeyMin
 	for i, r := range S.rangev {
 		hiPrev := hi_Prev + 1
-		if i > 0 && !(hiPrev < r.lo) { // NOTE not ≤ - adjacent ranges must be merged
+		if i > 0 && !(hiPrev < r.Lo) { // NOTE not ≤ - adjacent ranges must be merged
 			badf("[%d]: !(hiPrev < r.lo)", i)
 		}
-		if !(r.lo <= r.hi_) {
+		if !(r.Lo <= r.Hi_) {
 			badf("[%d]: !(r.lo <= r.hi_)", i)
 		}
-		hi_Prev = r.hi_
+		hi_Prev = r.Hi_
 	}
 }
 
@@ -378,12 +378,12 @@ func (S RangedKeySet) String() string {
 
 func (r KeyRange) String() string {
 	var shi string
-	if r.hi_ == KeyMax {
-		shi = kstr(r.hi_) // ∞
+	if r.Hi_ == KeyMax {
+		shi = kstr(r.Hi_) // ∞
 	} else {
-		shi = fmt.Sprintf("%d", r.hi_+1)
+		shi = fmt.Sprintf("%d", r.Hi_+1)
 	}
-	return fmt.Sprintf("[%s,%s)", kstr(r.lo), shi)
+	return fmt.Sprintf("[%s,%s)", kstr(r.Lo), shi)
 }
 
 

wcfs/internal/xbtree/rangeset_test.go → wcfs/internal/xbtree/blib/rangeset_test.go

@@ -17,7 +17,7 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package xbtree
+package blib
 
 import (
 	"testing"

wcfs/internal/xbtree/xbtree.go

@@ -5,34 +5,33 @@ package xbtree
 
 import (
 	"fmt"
-	"math"
 
 	"lab.nexedi.com/kirr/neo/go/zodb"
-	"lab.nexedi.com/kirr/neo/go/zodb/btree"
 
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
 )
 
 
 // XXX instead of generics
-type Tree   = btree.LOBTree
-type Bucket = btree.LOBucket
-type Node   = btree.LONode
-type TreeEntry   = btree.LOEntry
-type BucketEntry = btree.LOBucketEntry
+type Tree   = blib.Tree
+type Bucket = blib.Bucket
+type Node   = blib.Node
+type TreeEntry   = blib.TreeEntry
+type BucketEntry = blib.BucketEntry
 
-type Key    = int64
-const KeyMax Key = math.MaxInt64
-const KeyMin Key = math.MinInt64
+type Key    = blib.Key
+const KeyMax = blib.KeyMax
+const KeyMin = blib.KeyMin
 
 // value is assumed to be persistent reference.
 // deletion is represented as VDEL.
 type Value  = zodb.Oid
 const VDEL  = zodb.InvalidOid
 
-type SetKey = set.SetI64
-type SetOid = set.SetOid
-type SetTid = set.SetTid
+type setKey = set.I64
+type setOid = set.Oid
+type setTid = set.Tid
 
 
 
@@ -74,3 +73,7 @@ func kstr(k Key) string {
 	}
 	return fmt.Sprintf("%d", k)
 }
+
+func panicf(format string, argv ...interface{}) {
+	panic(fmt.Sprintf(format, argv...))
+}

wcfs/internal/xbtree/xbtreetest/init/init.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package xbtreetest/init (ex imported from package A) should be imported in
+// addition to xbtreetest (from package A_test) to initialize xbtreetest at runtime.
+package init
+
+// ZBlk-related part of δbtail_test
+
+import (
+	"context"
+	"fmt"
+
+	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
+)
+
+type Tree = xbtreetest.Tree
+type Node = xbtreetest.Node
+type Key  = xbtreetest.Key
+
+type ZBlk = zdata.ZBlk
+
+// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
+// XXX +return blkRevMax and use it ?
+func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
+	path = []Node{}
+	xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
+		path = append(path, node)
+	})
+	if err != nil {
+		return nil, false, nil, err
+	}
+
+	if ok {
+		zblk, ok = xzblk.(ZBlk)
+		if !ok {
+			return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk))  // XXX errctx
+		}
+	}
+
+	return zblk, ok, path, nil
+}
+
+
+func init() {
+	xbtreetest.ZTreeGetBlkData = _ZTreeGetBlkData
+	xbtreetest.ZGetBlkData     = _ZGetBlkData
+}
+
+// _ZTreeGetBlkData returns block data from block pointed to by ztree[k].
+func _ZTreeGetBlkData(ctx context.Context, ztree *Tree, k Key) (data string, ok bool, path []Node, err error) {
+	defer xerr.Contextf(&err, "@%s: tree<%s>: get blkdata from [%d]", ztree.PJar().At(), ztree.POid(), k)
+
+	zblk, ok, path, err := ztreeGetBlk(ctx, ztree, k)
+	if err != nil || !ok {
+		return "", ok, path, err
+	}
+
+	bdata, _, err := zblk.LoadBlkData(ctx)
+	if err != nil {
+		return "", false, nil, err
+	}
+
+	return string(bdata), true, path, nil
+}
+
+// _ZGetBlkData loads block data from ZBlk object specified by its oid.
+func _ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
+	defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
+
+	xblk, err := zconn.Get(ctx, zblkOid)
+	if err != nil {
+		return "", err
+	}
+	zblk, ok := xblk.(ZBlk)
+	if !ok {
+		return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
+	}
+	bdata, _, err := zblk.LoadBlkData(ctx)
+	if err != nil {
+		return "", err
+	}
+	return string(bdata), nil
+}

wcfs/internal/xbtree/xbtreetest/kvdiff.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+// kvdiff + friends
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// kvdiff returns difference in between kv1 and kv2.
+const DEL = "ø" // DEL means deletion
+type Δstring struct {
+	Old string
+	New string
+}
+func kvdiff(kv1, kv2 map[Key]string) map[Key]Δstring {
+	delta := map[Key]Δstring{}
+	keys := setKey{}
+	for k := range kv1 { keys.Add(k) }
+	for k := range kv2 { keys.Add(k) }
+
+	for k := range keys {
+		v1, ok := kv1[k]
+		if !ok { v1 = DEL }
+		v2, ok := kv2[k]
+		if !ok { v2 = DEL }
+		if v1 != v2 {
+			delta[k] = Δstring{v1,v2}
+		}
+	}
+
+	return delta
+}
+
+// kvtxt returns string representation of {} kv.
+func kvtxt(kv map[Key]string) string {
+	if len(kv) == 0 {
+		return "ø"
+	}
+
+	keyv := []Key{}
+	for k := range kv { keyv = append(keyv, k) }
+	sort.Slice(keyv, func(i,j int) bool { return keyv[i] < keyv[j] })
+
+	sv := []string{}
+	for _, k := range keyv {
+		v := kv[k]
+		if strings.ContainsAny(v, " \n\t,:") {
+			panicf("[%v]=%q: invalid value", k, v)
+		}
+		sv = append(sv, fmt.Sprintf("%v:%s", k, v))
+	}
+
+	return strings.Join(sv, ",")
+}

wcfs/internal/xbtree/misc.go → wcfs/internal/xbtree/xbtreetest/kvdiff_test.go

-// Copyright (C) 2018-2021  Nexedi SA and Contributors.
+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -17,12 +17,28 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package xbtree
+package xbtreetest
 
 import (
-	"fmt"
+	"reflect"
+	"testing"
 )
 
-func panicf(format string, argv ...interface{}) {
-	panic(fmt.Sprintf(format, argv...))
+func TestKVDiff(t *testing.T) {
+	kv1 := map[Key]string{1:"a", 3:"c", 4:"d"}
+	kv2 := map[Key]string{1:"b",        4:"d", 5:"e"}
+	got  := kvdiff(kv1, kv2)
+	want := map[Key]Δstring{1:{"a","b"}, 3:{"c",DEL}, 5:{DEL,"e"}}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("error:\ngot:  %v\nwant: %v", got, want)
+	}
+}
+
+func TestKVTxt(t *testing.T) {
+	kv := map[Key]string{3:"hello", 1:"zzz", 4:"world"}
+	got  := kvtxt(kv)
+	want := "1:zzz,3:hello,4:world"
+	if got != want {
+		t.Fatalf("error:\ngot:  %q\nwant: %q", got, want)
+	}
 }

wcfs/internal/xbtree/xbtreetest/rtree.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+
+import (
+	"fmt"
+	"sort"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
+)
+
+// RTree represents Tree node covering [lo, hi_] key range in its parent tree.
+// XXX actually no coverage here -> kill? -> change to just `path []zodb.Oid` in RBucket?
+type RTree struct {
+	Oid    zodb.Oid
+	Parent *RTree
+	// XXX +children?
+}
+
+// RBucket represents Bucket node covering [lo, hi_] key range in its Tree.
+// NOTE it is not [lo,hi) but [lo,hi_] instead to avoid overflow at KeyMax.
+type RBucket struct {
+	Oid     zodb.Oid
+	Parent  *RTree
+	Keycov  blib.KeyRange
+	KV      map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
+}
+
+// Path returns path to this bucket from tree root.
+func (rb *RBucket) Path() []zodb.Oid {
+	path := []zodb.Oid{rb.Oid}
+	p := rb.Parent
+	for p != nil {
+		path = append([]zodb.Oid{p.Oid}, path...)
+		p = p.Parent
+	}
+	return path
+}
+
+// RBucketSet represents set of buckets covering whole [-∞,∞) range.
+type RBucketSet []*RBucket // k↑
+
+
+// Get returns RBucket which covers key k.
+func (rbs RBucketSet) Get(k Key) *RBucket {
+	i := sort.Search(len(rbs), func(i int) bool {
+		return k <= rbs[i].Keycov.Hi_
+	})
+	if i == len(rbs) {
+		panicf("BUG: key %v not covered;  coverage: %s", k, rbs.coverage())
+	}
+
+	rb := rbs[i]
+	if !rb.Keycov.Has(k) {
+		panicf("BUG: get(%v) -> %s;  coverage: %s", k, rb.Keycov, rbs.coverage())
+	}
+
+	return rb
+}
+
+// coverage returns string representation of rbs coverage structure.
+func (rbs RBucketSet) coverage() string {
+	if len(rbs) == 0 {
+		return "ø"
+	}
+	s := ""
+	for _, rb := range rbs {
+		if s != "" {
+			s += " "
+		}
+		s += fmt.Sprintf("%s", rb.Keycov)
+	}
+	return s
+}
+
+// Flatten converts xkv with bucket structure into regular dict.
+func (xkv RBucketSet) Flatten() map[Key]string {
+	kv := make(map[Key]string)
+	for _, b := range xkv {
+		for k,v := range b.KV {
+			kv[k] = v
+		}
+	}
+	return kv
+}
+
+func (b *RBucket) String() string {
+	return fmt.Sprintf("%sB%s{%s}", b.Keycov, b.Oid, kvtxt(b.KV))
+}

wcfs/internal/xbtree/xbtreetest/treegen.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+// treegen.go provides functionality:
+//
+//   - to commit a particular BTree topology into ZODB, and
+//   - to generate set of random tree topologies that all correspond to particular {k->v} dict.
+//
+// treegen.py is used as helper for both tasks.
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"lab.nexedi.com/kirr/go123/my"
+	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+
+// TreeGenSrv represents connection to running `treegen ...` server.
+type TreeGenSrv struct {
+	argv      []string
+	pysrv     *exec.Cmd      // spawned `treegen ...`
+	pyin      io.WriteCloser // input to pysrv
+	pyoutRaw  io.ReadCloser  // output from pysrv
+	pyout     *bufio.Reader  // buffered ^^^
+}
+
+// TreeSrv represents connection to running `treegen trees` server.
+//
+// Create it with StartTreeSrv(zurl).
+// - Commit(treeTopology) -> tid
+type TreeSrv struct {
+	*TreeGenSrv
+	zurl	  string
+	treeRoot  zodb.Oid       // oid of the tree treegen works on
+	head      zodb.Tid       // last made commit
+}
+
+// AllStructsSrv represents connection to running `treegen allstructs` server.
+//
+// Create it with StartAllStructsSrv().
+// - AllStructs(maxdepth, maxsplit, n, seed, kv1, kv2)
+type AllStructsSrv struct {
+	*TreeGenSrv
+}
+
+// StartTreeGenSrv spawns `treegen ...` server.
+func StartTreeGenSrv(argv ...string) (_ *TreeGenSrv, hello string, err error) {
+	defer xerr.Contextf(&err, "treesrv %v: start", argv)
+
+	// spawn `treegen ...`
+	tg := &TreeGenSrv{argv: argv}
+	tg.pysrv = exec.Command(filepath.Dir(my.File())+"/treegen.py", argv...)
+	tg.pyin, err = tg.pysrv.StdinPipe()
+	if err != nil {
+		return nil, "", err
+	}
+	tg.pyoutRaw, err = tg.pysrv.StdoutPipe()
+	if err != nil {
+		return nil, "", err
+	}
+	tg.pyout = bufio.NewReader(tg.pyoutRaw)
+	tg.pysrv.Stderr = os.Stderr // no redirection
+
+	err = tg.pysrv.Start()
+	if err != nil {
+		return nil, "", err
+	}
+
+	// wait for hello message and return it
+	defer func() {
+		if err != nil {
+			tg.Close() // ignore error
+		}
+	}()
+	defer xerr.Context(&err, "handshake")
+	hello, err = tg.pyout.ReadString('\n')
+	if err != nil {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		return nil, "", err
+	}
+	hello = strings.TrimSuffix(hello, "\n")
+
+	return tg, hello, nil
+}
+
+// Close shutdowns treegen server.
+func (tg *TreeGenSrv) Close() (err error) {
+	defer xerr.Contextf(&err, "treegen %v: close", tg.argv)
+
+	err1 := tg.pyin.Close()
+	err2 := tg.pyoutRaw.Close()
+	err3 := tg.pysrv.Wait()
+	return xerr.Merge(err1, err2, err3)
+}
+
+
+// StartTreeSrv spawns `treegen trees` server.
+func StartTreeSrv(zurl string) (_ *TreeSrv, err error) {
+	defer xerr.Contextf(&err, "tree.srv %s: start", zurl)
+	tgSrv, hello, err := StartTreeGenSrv("trees", zurl)
+	if err != nil {
+		return nil, err
+	}
+
+	tg := &TreeSrv{TreeGenSrv: tgSrv, zurl: zurl}
+	defer func() {
+		if err != nil {
+			tgSrv.Close() // ignore error
+		}
+	}()
+
+	// tree.srv start @<at> tree=<root>
+	defer xerr.Contextf(&err, "invalid hello %q", hello)
+	startRe := regexp.MustCompile(`^tree.srv start @([^ ]+) root=([^ ]+)$`)
+	m := startRe.FindStringSubmatch(hello)
+	if m == nil {
+		return nil, fmt.Errorf("unexpected format")
+	}
+	tg.head, err = zodb.ParseTid(m[1]) // <at>
+	if err != nil {
+		return nil, fmt.Errorf("tid: %s", err)
+	}
+	tg.treeRoot, err = zodb.ParseOid(m[2]) // <root>
+	if err != nil {
+		return nil, fmt.Errorf("root: %s", err)
+	}
+
+	return tg, nil
+}
+
+// StartAllStructsSrv spawns `treegen allstructs` server.
+func StartAllStructsSrv() (_ *AllStructsSrv, err error) {
+	defer xerr.Context(&err, "allstructs.srv: start")
+
+	tgSrv, hello, err := StartTreeGenSrv("allstructs")
+	if err != nil {
+		return nil, err
+	}
+
+	sg := &AllStructsSrv{TreeGenSrv: tgSrv}
+	defer func() {
+		if err != nil {
+			tgSrv.Close() // ignore error
+		}
+	}()
+
+	defer xerr.Contextf(&err, "invalid hello %q", hello)
+	if hello != "# allstructs.srv start" {
+		return nil, fmt.Errorf("unexpected format")
+	}
+
+	return sg, nil
+}
+
+
+// Commit creates new commit with underlying tree changed to specified tree topology.
+func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
+	defer xerr.Contextf(&err, "tree.srv %s: commit %s", tg.zurl, tree)
+
+	_, err = io.WriteString(tg.pyin, tree + "\n")
+	if err != nil {
+		return zodb.InvalidTid, err
+	}
+
+	reply, err := tg.pyout.ReadString('\n')
+	if err != nil {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		return zodb.InvalidTid, err
+	}
+	reply = strings.TrimSuffix(reply, "\n")
+
+	tid, err := zodb.ParseTid(reply)
+	if err != nil {
+		return zodb.InvalidTid, fmt.Errorf("invalid reply: %s", err)
+	}
+	tg.head = tid
+	return tid, nil
+}
+
+// AllStructs returns response from `treegen allstructs`
+func (tg *AllStructsSrv) AllStructs(kv map[Key]string, maxdepth, maxsplit, n int, seed int64) (_ []string, err error) {
+	req := fmt.Sprintf("%d %d %d/%d %s", maxdepth, maxsplit, n, seed, kvtxt(kv))
+	defer xerr.Contextf(&err, "allstructs.srv: %s ", req)
+
+	_, err = io.WriteString(tg.pyin, req + "\n")
+	if err != nil {
+		return nil, err
+	}
+
+	structv := []string{}
+	for {
+		reply, err := tg.pyout.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			return nil, err
+		}
+		reply = strings.TrimSuffix(reply, "\n")
+
+		if reply == "# ----" {
+			return structv, nil // end of response
+		}
+		if strings.HasPrefix(reply, "#") {
+			continue // comment
+		}
+
+		structv = append(structv, reply)
+	}
+}

wcfs/internal/xbtree/xbtreetest/xbtreetest.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package xbtreetest provides infrastructure for testing LOBTree with ZBlk values.
+// XXX -> treetest?
+package xbtreetest
+
+import (
+	"fmt"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
+)
+
+// XXX instead of generics
+type Tree   = blib.Tree
+type Bucket = blib.Bucket
+type Node   = blib.Node
+type TreeEntry   = blib.TreeEntry
+type BucketEntry = blib.BucketEntry
+
+type Key    = blib.Key
+const KeyMax = blib.KeyMax
+const KeyMin = blib.KeyMin
+
+type setKey = set.I64
+
+// XXX dup from xbtree  (to avoid import cycle)
+const VDEL  = zodb.InvalidOid
+
+
+func panicf(format string, argv ...interface{}) {
+	panic(fmt.Sprintf(format, argv...))
+}

wcfs/internal/xbtree/xbtreetest/zdata.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+// access to ZBlk data
+
+import (
+	"context"
+
+	"lab.nexedi.com/kirr/go123/exc"
+
+	"lab.nexedi.com/kirr/neo/go/transaction"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
+)
+
+// ZBlk-related functions are imported at runtime by package xbtreetest/init
+var (
+	ZTreeGetBlkData func(context.Context, *Tree, Key) (string, bool, []Node, error)
+	ZGetBlkData     func(context.Context, *zodb.Connection, zodb.Oid) (string, error)
+)
+
+func zassertInitDone() {
+	if ZTreeGetBlkData == nil {
+		panic("xbtreetest/zdata not initialized -> import xbtreetest/init to fix")
+	}
+}
+
+// xzgetBlkData loads block data from ZBlk object specified by its oid.
+func xzgetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) string {
+	zassertInitDone()
+	X := exc.Raiseif
+
+	if zblkOid == VDEL {
+		return DEL
+	}
+
+	data, err := ZGetBlkData(ctx, zconn, zblkOid); X(err)
+	return string(data)
+}
+
+// xzgetBlkDataAt loads block data from ZBlk object specified by oid@at.
+func xzgetBlkDataAt(db *zodb.DB, zblkOid zodb.Oid, at zodb.Tid) string {
+	zassertInitDone()
+	X := exc.Raiseif
+
+	txn, ctx := transaction.New(context.Background())
+	defer txn.Abort()
+
+	zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at});  X(err)
+	return xzgetBlkData(ctx, zconn, zblkOid)
+}

wcfs/internal/xbtree/δbtail_x_test.go

@@ -18,85 +18,7 @@
 // See https://www.nexedi.com/licensing for rationale and options.
 
 package xbtree_test
-// ZBlk-related part of δbtail_test
 
 import (
-	"context"
-	"fmt"
-
-	"lab.nexedi.com/kirr/go123/xerr"
-	"lab.nexedi.com/kirr/neo/go/zodb"
-
-	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree"
-	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
-	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
+	_ "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest/init"
 )
-
-type Tree = xbtree.Tree
-type Node = xbtree.Node
-type Key  = xbtree.Key
-
-type ZBlk = zdata.ZBlk
-
-// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
-// XXX +return blkRevMax and use it ?
-func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
-	path = []Node{}
-	xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
-		path = append(path, node)
-	})
-	if err != nil {
-		return nil, false, nil, err
-	}
-
-	if ok {
-		zblk, ok = xzblk.(ZBlk)
-		if !ok {
-			return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk))  // XXX errctx
-		}
-	}
-
-	return zblk, ok, path, nil
-}
-
-
-func init() {
-	xbtree.ZTreeGetBlkData = ZTreeGetBlkData
-	xbtree.ZGetBlkData     = ZGetBlkData
-}
-
-// ZTreeGetBlkData returns block data from block pointed to by ztree[k].
-func ZTreeGetBlkData(ctx context.Context, ztree *Tree, k Key) (data string, ok bool, path []Node, err error) {
-	defer xerr.Contextf(&err, "@%s: tree<%s>: get blkdata from [%d]", ztree.PJar().At(), ztree.POid(), k)
-
-	zblk, ok, path, err := ztreeGetBlk(ctx, ztree, k)
-	if err != nil || !ok {
-		return "", ok, path, err
-	}
-
-	bdata, _, err := zblk.LoadBlkData(ctx)
-	if err != nil {
-		return "", false, nil, err
-	}
-
-	return string(bdata), true, path, nil
-}
-
-// ZGetBlkData loads block data from ZBlk object specified by its oid.
-func ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
-	defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
-
-	xblk, err := zconn.Get(ctx, zblkOid)
-	if err != nil {
-		return "", err
-	}
-	zblk, ok := xblk.(ZBlk)
-	if !ok {
-		return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
-	}
-	bdata, _, err := zblk.LoadBlkData(ctx)
-	if err != nil {
-		return "", err
-	}
-	return string(bdata), nil
-}

wcfs/internal/zdata/zblk.go

@@ -60,7 +60,6 @@ import (
 // ZBlk is the interface that every ZBlk* block implements.
 type ZBlk interface {
 	zodb.IPersistent
-	_ZBlkInΔFtail
 
 	// LoadBlkData loads from database and returns data block stored by this ZBlk.
 	//
@@ -79,7 +78,6 @@ var _ ZBlk = (*ZBlk1)(nil)
 // ZBlk0 mimics ZBlk0 from python.
 type ZBlk0 struct {
 	zodb.Persistent
-	zblkInΔFtail
 
 	// NOTE py source uses bytes(buf) but on python2 it still results in str
 	blkdata string
@@ -157,7 +155,6 @@ func (zd *zDataState) PySetState(pystate interface{}) error {
 // ZBlk1 mimics ZBlk1 from python.
 type ZBlk1 struct {
 	zodb.Persistent
-	zblkInΔFtail
 
 	chunktab *btree.IOBTree // {} offset -> ZData(chunk)
 }
@@ -468,7 +465,7 @@ func (bf *ZBigFile) LoadBlk(ctx context.Context, blk int64) (_ []byte, treePath
 
 // Size returns whole file size.
 //
-// it also returns BTree path scaned to obtain the size.
+// it also returns BTree path scanned to obtain the size.
 func (bf *ZBigFile) Size(ctx context.Context) (_ int64, treePath []btree.LONode, err error) {
 	defer xerr.Contextf(&err, "bigfile %s: size", bf.POid())
 

wcfs/internal/zdata/zblk_test.go

@@ -35,8 +35,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const K = 1024
-
 // TestZBlk verifies that ZBlk* and ZBigFile saved by Python can be read correctly by Go.
 // TODO also test with data saved by Python3.
 func TestZBlk(t *testing.T) {

wcfs/internal/zdata/δftail.go

@@ -19,13 +19,9 @@
 
 package zdata
 
-//go:generate ../set/gen-set zdata ZBigFile *ZBigFile zset_bigfile.go
-
 import (
 	"context"
 	"fmt"
-	"runtime"
-	"sync"
 
 	"lab.nexedi.com/kirr/go123/xerr"
 	"lab.nexedi.com/kirr/neo/go/zodb"
@@ -37,7 +33,8 @@ import (
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
 )
 
-type SetI64 = set.SetI64
+type setI64 = set.I64
+type setOid = set.Oid
 
 // ΔFtail represents tail of revisional changes to files.
 //
@@ -78,8 +75,12 @@ type SetI64 = set.SetI64
 type ΔFtail struct {
 	// ΔFtail merges ΔBtail with history of ZBlk
 	δBtail  *xbtree.ΔBtail
-	fileIdx map[zodb.Oid]SetZBigFile // tree-root -> {} ZBigFile as of @head
+	fileIdx map[zodb.Oid]setOid // tree-root -> {} ZBigFile<oid> as of @head
 
+	trackSetZFile setOid                  // set of tracked ZBigFiles	as of @head
+	trackSetZBlk  map[zodb.Oid]*zblkTrack // zblk -> {} root -> {}blk	as of @head
+// XXX kill
+///*
 	// XXX don't need vδF - everything is reconstructed at runtime from .δBtail.vδT
 	//     this way we also don't need to keep up updating vδF from vδT on its rebuild during.
 	// data with δF changes. Actual for part of tracked set that was taken
@@ -88,40 +89,29 @@ type ΔFtail struct {
 
 	// tracked ZBlk that are not yet taken into account in current vδF.
 	// grows on new track requests; flushes on queries and update.
-	trackNew map[*ZBigFile]map[zodb.Oid]*zblkInΔFtail // {} file -> {} oid -> zblk
+	trackNew map[zodb.Oid]map[zodb.Oid]*zblkTrack // {} foid -> {} zoid -> zblk
+//*/
+}
+
+// zblkTrack keeps information in which root/blocks ZBlk is present as of @head.
+type zblkTrack struct {
+//	inroot map[zodb.Oid]setI64  // {} root -> {}blk		XXX later switch to this
+	infile map[zodb.Oid]setI64  // {} foid -> {}blk
 }
 
 // ΔF represents a change in files space.
 type ΔF struct {
 	Rev    zodb.Tid
-	ByFile map[*ZBigFile]*ΔFile // file -> δfile
+	ByFile map[zodb.Oid]*ΔFile // foid -> δfile
 }
 
 // ΔFile represents a change to one file.
 type ΔFile struct {
 	Rev    zodb.Tid
-	Blocks SetI64 // changed blocks
-	Size   bool   // whether file size changed
+	Blocks setI64 // changed blocks			XXX -> ΔBlocks ?
+	Size   bool   // whether file size changed	XXX -> ΔSize?
 }
 
-// zblkInΔFtail is ΔFtail-related volatile data embedded into ZBlk*.
-//
-// The data is preserved even when ZBlk comes to ghost state, but is lost if
-// ZBlk is garbage collected. The data is transient - it is _not_ included into
-// persistent state.
-type zblkInΔFtail struct {
-	mu sync.Mutex // to support multiple concurrent loaders
-
-	// XXX change vvv to intree_parent? {} Bucket -> set(#blk)
-	// (this is uniform with building in-RAM reverse child->parents relation for
-	//  tree nodes and for tree_root->file)
-
-	// with which files/blocks this ZBlk is associated with as of @head state
-	infile map[*ZBigFile]SetI64 // {} file -> set(#blk)
-}
-
-type _ZBlkInΔFtail interface { inΔFtail() *zblkInΔFtail }
-func (z *zblkInΔFtail) inΔFtail() *zblkInΔFtail { return z }
 
 // NewΔFtail creates new ΔFtail object.
 //
@@ -132,9 +122,11 @@ func (z *zblkInΔFtail) inΔFtail() *zblkInΔFtail { return z }
 // ZODB when needed.
 func NewΔFtail(at0 zodb.Tid, db *zodb.DB) *ΔFtail {
 	return &ΔFtail{
-		δBtail:   xbtree.NewΔBtail(at0, db),
-		fileIdx:  make(map[zodb.Oid]SetZBigFile),
-		trackNew: make(map[*ZBigFile]map[zodb.Oid]*zblkInΔFtail),
+		δBtail:        xbtree.NewΔBtail(at0, db),
+		fileIdx:       map[zodb.Oid]setOid{},
+		trackSetZFile: setOid{},
+		trackSetZBlk:  map[zodb.Oid]*zblkTrack{},
+		trackNew:      map[zodb.Oid]map[zodb.Oid]*zblkTrack{},
 	}
 }
 
@@ -156,6 +148,9 @@ func (δFtail *ΔFtail) Tail() zodb.Tid { return δFtail.δBtail.Tail() }
 //
 // A root can be associated with several files (each provided on different Track call).
 func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zblk ZBlk) {
+	// XXX locking
+
+	foid := file.POid()
 	if blk == -1 {
 		// XXX blk = ∞ from beginning ?
 		blk = xbtree.KeyMax
@@ -164,38 +159,44 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zb
 	if err != nil {
 		panic(err) // XXX -> error? errctx
 	}
+
 	root := path[0].(*btree.LOBTree)
 	files, ok := δFtail.fileIdx[root.POid()]
 	if !ok {
-		files = SetZBigFile{}
+		files = setOid{}
 		δFtail.fileIdx[root.POid()] = files
 	}
-	files.Add(file)
+	files.Add(foid)
+
+	δFtail.trackSetZFile.Add(foid)
 
 	// associate zblk with file, if it was not hole
 	if zblk != nil {
-		z := zblk.inΔFtail()
-		z.mu.Lock()
-		blocks, ok := z.infile[file]
+		zoid := zblk.POid()
+		zt, ok := δFtail.trackSetZBlk[zoid]
+		if !ok {
+			zt = &zblkTrack{}
+			δFtail.trackSetZBlk[zoid] = zt
+		}
+
+		blocks, ok := zt.infile[foid]
 		if !ok {
-			blocks = make(SetI64, 1)
-			if z.infile == nil {
-				z.infile = make(map[*ZBigFile]SetI64)
+			blocks = make(setI64, 1)
+			if zt.infile == nil {
+				zt.infile = make(map[zodb.Oid]setI64)
 			}
-			z.infile[file] = blocks
+			zt.infile[foid] = blocks
 		}
 		blocks.Add(blk)
-		z.mu.Unlock()
 
-		// XXX locking
 		if !ok {
 			// zblk was not associated with this file
-			zt := δFtail.trackNew[file]
-			if zt == nil {
-				zt = make(map[zodb.Oid]*zblkInΔFtail, 1)
-				δFtail.trackNew[file] = zt
+			ft := δFtail.trackNew[foid]
+			if ft == nil {
+				ft = make(map[zodb.Oid]*zblkTrack, 1)
+				δFtail.trackNew[foid] = ft
 			}
-			zt[zblk.POid()] = z
+			ft[zoid] = zt
 		}
 	}
 
@@ -224,13 +225,14 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
 	// XXX δFtail.update() first?
 
 	// XXX verify zhead.At() == δFtail.Head()
+	// XXX locking
 
 	δB, err := δFtail.δBtail.Update(δZ)
 	if err != nil {
 		return ΔF{}, err
 	}
 
-	δF := ΔF{Rev: δB.Rev, ByFile: make(map[*ZBigFile]*ΔFile)}
+	δF := ΔF{Rev: δB.Rev, ByFile: make(map[zodb.Oid]*ΔFile)}
 
 	// take btree changes into account
 	for root, δt := range δB.ΔByRoot {
@@ -241,7 +243,7 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
 		for file := range files {
 			δfile, ok := δF.ByFile[file]
 			if !ok {
-				δfile = &ΔFile{Rev: δF.Rev, Blocks: make(SetI64)}
+				δfile = &ΔFile{Rev: δF.Rev, Blocks: make(setI64)}
 				δF.ByFile[file] = δfile
 			}
 			for blk /*, zblk*/ := range δt {
@@ -260,42 +262,29 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
 
 	// take zblk changes into account
 	for _, oid := range δZ.Changev {
-		// XXX cache lock/unlock
-		obj := zhead.Cache().Get(oid)
-		if obj == nil {
-			//fmt.Printf("%s: not in cache\n", oid)
-			continue // nothing to do - see invariant
+		if δFtail.trackSetZFile.Has(oid) {
+			// TODO check that .blksize and .blktab (it is only
+			// persistent reference) do not change.
+
+			return ΔF{}, fmt.Errorf("ZBigFile<%s> changed @%s", oid, δZ.Tid)
 		}
 
-		//fmt.Printf("%s:     in cache (%s)\n", oid, typeOf(obj))
-
-		switch obj := obj.(type) {
-		case ZBlk:	// ZBlk*
-			// z.infile locking: since we write-locked head.zheadMu
-			// - no other fuse reads are running, and thus no one
-			// is mutating z.infile. XXX recheck
-			z := obj.inΔFtail()
-			for file, blocks := range z.infile {
-				δfile, ok := δF.ByFile[file]
-				if !ok {
-					δfile = &ΔFile{Rev: δF.Rev, Blocks: make(SetI64)}
-					δF.ByFile[file] = δfile
-				}
+		zt, ok := δFtail.trackSetZBlk[oid]
+		if !ok {
+			continue // not tracked
+		}
 
-				δfile.Blocks.Update(blocks)
+		for foid, blocks := range zt.infile {
+			δfile, ok := δF.ByFile[foid]
+			if !ok {
+				δfile = &ΔFile{Rev: δF.Rev, Blocks: make(setI64)}
+				δF.ByFile[foid] = δfile
 			}
 
-			// XXX update z.infile according to btree changes
-
-		case *ZBigFile:
-			// TODO check that .blksize and .blktab (it is only
-			// persistent reference) do not change.
-
-			return ΔF{}, fmt.Errorf("ZBigFile<%s> changed @%s", oid, δZ.Tid)
+			δfile.Blocks.Update(blocks)
 		}
 
-		// make sure obj won't be garbage-collected until we finish handling it.
-		runtime.KeepAlive(obj)
+		// XXX update zt.infile according to btree changes
 	}
 
 	δFtail.vδF = append(δFtail.vδF, δF)
@@ -313,15 +302,16 @@ func (δFtail *ΔFtail) update(file *ZBigFile) {
 	}
 
 	// let's see if we need to rebuild .vδF due to not-yet processed track requests
+	foid := file.POid()
 
 	// XXX locking
 	// XXX dumb
-	zt, dirty := δFtail.trackNew[file]
+	zt, dirty := δFtail.trackNew[foid]
 	if !dirty {
 		return
 	}
 
-	delete(δFtail.trackNew, file)
+	delete(δFtail.trackNew, foid)
 	// XXX unlock here
 
 	for i, δZ := range δFtail.δBtail.ΔZtail().Data() {
@@ -335,14 +325,14 @@ func (δFtail *ΔFtail) update(file *ZBigFile) {
 			}
 
 			// XXX locking
-			// XXX -> func δF.δfile(file) ?
-			δfile, ok := δF.ByFile[file]
+			// XXX -> func δF.δfile(foid) ?
+			δfile, ok := δF.ByFile[foid]
 			if !ok {
-				δfile = &ΔFile{Rev: δF.Rev, Blocks: make(SetI64)}
-				δF.ByFile[file] = δfile
+				δfile = &ΔFile{Rev: δF.Rev, Blocks: make(setI64)}
+				δF.ByFile[foid] = δfile
 			}
 
-			δfile.Blocks.Update(z.infile[file])
+			δfile.Blocks.Update(z.infile[foid])
 		}
 	}
 }
@@ -353,10 +343,10 @@ func (δFtail *ΔFtail) ForgetPast(revCut zodb.Tid) {
 }
 
 // XXX don't need
-func (δFtail *ΔFtail) SliceByRev(lo, hi zodb.Tid) /*readonly*/ []ΔF {
-	xtail.AssertSlice(δFtail, lo, hi)
-	panic("TODO")
-}
+//func (δFtail *ΔFtail) SliceByRev(lo, hi zodb.Tid) /*readonly*/ []ΔF {
+//	xtail.AssertSlice(δFtail, lo, hi)
+//	panic("TODO")
+//}
 
 // SliceByFileRev returns history of file changes in (lo, hi] range.
 //
@@ -398,9 +388,10 @@ func (δFtail *ΔFtail) SliceByFileRev(file *ZBigFile, lo, hi zodb.Tid) /*readon
 	vδF = vδF[i:j+1]
 
 	// filter found changed to have only file-related bits
+	foid := file.POid()
 	var vδfile []*ΔFile
 	for _, δF := range vδF {
-		δfile, ok := δF.ByFile[file]
+		δfile, ok := δF.ByFile[foid]
 		if ok {
 			vδfile = append(vδfile, δfile)
 		}

wcfs/internal/zdata/δftail_test.go

+// Copyright (C) 2019-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package zdata
+
+import (
+	"testing"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
+)
+
+type setStr = set.Str
+
+// ΔFTestEntry represents one entry in ΔFtail tests.
+type ΔFTestEntry struct {
+	δblkTab  map[int64]string  // change in tree part {} #blk -> ZBlk<oid>
+	δblkData setStr            // change to ZBlk objects
+}
+
+func TestΔFtail(t *testing.T) {
+	// δT is shorthand to create δblkTab.
+	type δT = map[int64]string
+
+	// δD is shorthand to create δblkData.
+	δD := func(zblkv ...string) setStr {
+		δ := setStr{}
+		for _, zblk := range zblkv {
+			δ.Add(zblk)
+		}
+		return δ
+	}
+
+	const a, b, c, ø = "a", "b", "c", "ø"
+
+	testv := []ΔFTestEntry{
+		{δT{1:a,2:b,3:ø},	δD(a)},
+		{δT{},			δD(c)},
+		{δT{2:c},		δD(a,b)},
+	}
+
+	vδf    := []ΔFile{}           // (rev↑, {}blk)		XXX +.Size?
+	blkTab := map[int64]string{}  // #blk -> ZBlk<oid>
+	Zinblk := map[string]setI64{} // ZBlk<oid> -> which #blk refer to it
+	for _, test := range testv {
+		δf := setI64{}
+
+		for blk, zblk := range test.δblkTab {
+			// rebuild blkTab/Zinblk
+			zprev, ok := blkTab[blk]
+			if ok {
+				delete(Zinblk[zprev], blk)
+			} else {
+				zprev = ø
+			}
+
+			if zblk != ø {
+				blkTab[blk] = zblk
+				inblk, ok := Zinblk[zblk]
+				if !ok {
+					inblk = setI64{}
+					Zinblk[zblk] = inblk
+				}
+				inblk.Add(blk)
+			}
+
+			// update δf due to change in blkTab
+			if zblk != zprev {
+				δf.Add(blk)
+			}
+		}
+
+		// update δf due to change in ZBlk data
+		for zblk := range test.δblkData {
+			for blk := range Zinblk[zblk] {
+				δf.Add(blk)
+			}
+		}
+
+		vδf = append(vδf, ΔFile{
+				Rev:    zodb.InvalidTid, // XXX will be set after treegen commit
+				Blocks: δf,
+				Size:   false/*XXX*/,
+		})
+	}
+}
+
+// XXX TestΔFtailRandom(t *testing.T) {
+//}