Merge branch 't+btree' into t

* t+btree:
  .
  .
  .
  .
  .
  .
  .
  X Start reworking BTree to provide keycov on visit callback

go/zodb/btree/btree.go.in

@@ -65,7 +65,7 @@ type BTree struct {
 
 // Entry is one BTree node entry.
 //
-// It contains key and child, who is either BTree or Bucket.
+// It contains key and child, which is either BTree or Bucket.
 //
 // Key limits child's keys - see BTree.Entryv for details.
 type Entry struct {
@@ -102,6 +102,15 @@ type BucketEntry struct {
 	value interface{}
 }
 
+// KeyRange represents [lo,hi) key range.
+type KeyRange struct {
+	Lo  KEY
+	Hi_ KEY // NOTE _not_ hi) to avoid overflow at ∞;  hi = hi_ + 1
+}
+
+const _KeyMin KEY = math.Min<Key>
+const _KeyMax KEY = math.Max<Key>
+
 // ---- access []entry ----
 
 // Key returns BTree entry key.
@@ -123,7 +132,7 @@ func (e *Entry) Child() Node { return e.child }
 // Children of all entries are guaranteed to be of the same kind - either all BTree, or all Bucket.
 //
 // The caller must not modify returned array.
-func (t *BTree) Entryv() []Entry {
+func (t *BTree) Entryv() /*readonly*/ []Entry {
 	return t.data
 }
 
@@ -134,7 +143,7 @@ func (e *BucketEntry) Key() KEY { return e.key }
 func (e *BucketEntry) Value() interface{} { return e.value }
 
 // Entryv returns entries of a Bucket node.
-func (b *Bucket) Entryv() []BucketEntry {
+func (b *Bucket) Entryv() /*readonly*/ []BucketEntry {
 	ev := make([]BucketEntry, len(b.keys))
 	for i, k := range b.keys {
 		ev[i] = BucketEntry{k, b.values[i]}
@@ -168,36 +177,54 @@ func (t *BTree) Get(ctx context.Context, key KEY) (_ interface{}, _ bool, err er
 // VGet is like Get but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *BTree) VGet(ctx context.Context, key KEY, visit func(node Node)) (_ interface{}, _ bool, err error) {
+func (t *BTree) VGet(ctx context.Context, key KEY, visit func(node Node, keycov KeyRange)) (_ interface{}, _ bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): get %v", t.POid(), key)
 	err = t.PActivate(ctx)
 	if err != nil {
 		return nil, false, err
 	}
 
+	keycov := KeyRange{Lo: _KeyMin, Hi_: _KeyMax}
 	if visit != nil {
-		visit(t)
-	}
-
-	if len(t.data) == 0 {
-		// empty btree
-		t.PDeactivate()
-		return nil, false, nil
+		visit(t, keycov)
 	}
 
 	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
+			t.PDeactivate()
+			return nil, false, nil
+		}
+
 		// search i: K(i) ≤ k < K(i+1)	; K(0) = -∞, K(len) = +∞
-		i := sort.Search(len(t.data), func(i int) bool {
+		i := sort.Search(l, func(i int) bool {
 			j := i + 1
 			if j == len(t.data) {
 				return true // [len].key = +∞
 			}
 			return key < t.data[j].key
 		})
-
-		// FIXME panic index out of range (empty T without children;
-		// logically incorrect, but Restructure generated it once)
+		// i < l
 		child := t.data[i].child
+
+		// shorten global keycov by local [lo,hi) for this child
+		lo := _KeyMin
+		if i > 0 {
+			lo = t.data[i].key
+		}
+		i++
+		hi_ := _KeyMax
+		if i < l {
+			hi_ = t.data[i].key
+		}
+		if hi_ != _KeyMax {
+			hi_--
+		}
+		keycov.Lo  = kmax(keycov.Lo, lo)
+		keycov.Hi_ = kmin(keycov.Hi_, hi_)
+
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -207,7 +234,7 @@ func (t *BTree) VGet(ctx context.Context, key KEY, visit func(node Node)) (_ int
 		// XXX verify child keys are in valid range according to parent
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		switch child := child.(type) {
@@ -250,26 +277,40 @@ func (t *BTree) MinKey(ctx context.Context) (_ KEY, ok bool, err error) {
 // VMinKey is like MinKey but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *BTree) VMinKey(ctx context.Context, visit func(node Node)) (_ KEY, ok bool, err error) {
+func (t *BTree) VMinKey(ctx context.Context, visit func(node Node, keycov KeyRange)) (_ KEY, ok bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): minkey", t.POid())
 	err = t.PActivate(ctx)
 	if err != nil {
 		return 0, false, err
 	}
 
+	keycov := KeyRange{Lo: _KeyMin, Hi_: _KeyMax}
 	if visit != nil {
-		visit(t)
-	}
-
-	if len(t.data) == 0 {
-		// empty btree
-		t.PDeactivate()
-		return 0, false, nil
+		visit(t, keycov)
 	}
 
 	// NOTE -> can also use t.firstbucket
 	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
+			t.PDeactivate()
+			return 0, false, nil
+		}
+
 		child := t.data[0].child
+
+		// shorten global keycov by local hi) for this child
+		hi_ := _KeyMax
+		if 1 < l {
+			hi_ = t.data[1].key
+		}
+		if hi_ != _KeyMax {
+			hi_--
+		}
+		// keycov.Lo stays -∞
+		keycov.Hi_ = kmin(keycov.Hi_, hi_)
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -277,7 +318,7 @@ func (t *BTree) VMinKey(ctx context.Context, visit func(node Node)) (_ KEY, ok b
 		}
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		// XXX verify child keys are in valid range according to parent
@@ -305,26 +346,36 @@ func (t *BTree) MaxKey(ctx context.Context) (_ KEY, _ bool, err error) {
 // VMaxKey is like MaxKey but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *BTree) VMaxKey(ctx context.Context, visit func(node Node)) (_ KEY, _ bool, err error) {
+func (t *BTree) VMaxKey(ctx context.Context, visit func(node Node, keycov KeyRange)) (_ KEY, _ bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): maxkey", t.POid())
 	err = t.PActivate(ctx)
 	if err != nil {
 		return 0, false, err
 	}
 
+	keycov := KeyRange{Lo: _KeyMin, Hi_: _KeyMax}
 	if visit != nil {
-		visit(t)
-	}
-
-	l := len(t.data)
-	if l == 0 {
-		// empty btree
-		t.PDeactivate()
-		return 0, false, nil
+		visit(t, keycov)
 	}
 
 	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
+			t.PDeactivate()
+			return 0, false, nil
+		}
+
 		child := t.data[l-1].child
+
+		// shorten global keycov by local [lo for this chile
+		lo := _KeyMin
+		if l-1 > 0 {
+			lo = t.data[l-1].key
+		}
+		keycov.Lo  = kmax(keycov.Lo, lo)
+		// keycov.Hi_ stays ∞
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -332,7 +383,7 @@ func (t *BTree) VMaxKey(ctx context.Context, visit func(node Node)) (_ KEY, _ bo
 		}
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		// XXX verify child keys are in valid range according to parent
@@ -591,7 +642,7 @@ func (bt *btreeState) PySetState(pystate interface{}) (err error) {
 	var kprev int64
 	var childrenKind int // 1 - BTree, 2 - Bucket
 	for i, idx := 0, 0; i < n; i++ {
-		key := int64(math.Min<Key>) // KEY(-∞)   (qualifies for ≤)
+		key := int64(_KeyMin) // KEY(-∞)   (qualifies for ≤)
 		if i > 0 {
 			// key[0] is unused and not saved
 			key, ok = t[idx].(int64) // XXX Xint
@@ -646,3 +697,60 @@ func init() {
 	zodb.RegisterClass("BTrees.BTree.BTree",  t(BTree{}),  t(btreeState{}))
 	zodb.RegisterClass("BTrees.BTree.Bucket", t(Bucket{}), t(bucketState{}))
 }
+
+
+// ---- misc ----
+
+// Has returns whether key k belongs to the range.
+func (r *KeyRange) Has(k KEY) bool {
+	return (r.Lo <= k && k <= r.Hi_)
+}
+
+// Empty returns whether key range is empty.
+func (r *KeyRange) Empty() bool {
+	hi := r.Hi_
+	if hi == _KeyMax {
+		// [x,∞] cannot be empty because max x is ∞ and [∞,∞] has one element: ∞
+		return false
+	}
+	hi++ // no overflow
+	return r.Lo >= hi
+}
+
+func (r KeyRange) String() string {
+	var shi string
+	if r.Hi_ == _KeyMax {
+		shi = kstr(r.Hi_) // ∞
+	} else {
+		shi = fmt.Sprintf("%d", r.Hi_+1)
+	}
+	return fmt.Sprintf("[%s,%s)", kstr(r.Lo), shi)
+}
+
+
+func kmin(a, b KEY) KEY {
+	if a < b {
+		return a
+	} else {
+		return b
+	}
+}
+
+func kmax(a, b KEY) KEY {
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+// kstr formats key as string.
+func kstr(k KEY) string {
+	if k == _KeyMin {
+		return "-∞"
+	}
+	if k == _KeyMax {
+		return "∞"
+	}
+	return fmt.Sprintf("%d", k)
+}

go/zodb/btree/btree_test.go

@@ -23,6 +23,7 @@ package btree
 
 import (
 	"context"
+	"reflect"
 	"testing"
 
 	"lab.nexedi.com/kirr/go123/exc"
@@ -98,6 +99,12 @@ func (b *bucketWrap) MaxKey(ctx context.Context) (k int64, ok bool, err error) {
 	return
 }
 
+// tVisit is information about one visit call.
+type tVisit struct {
+	node   zodb.Oid
+	keycov LKeyRange
+}
+
 func TestBTree(t *testing.T) {
 	X := exc.Raiseif
 	ctx := context.Background()
@@ -105,6 +112,10 @@ func TestBTree(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		err := stor.Close(); X(err)
+	}()
+
 	db := zodb.NewDB(stor, &zodb.DBOptions{})
 	defer func() {
 		err := db.Close(); X(err)
@@ -118,8 +129,6 @@ func TestBTree(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// XXX close db/stor
-
 	// go through small test Buckets/BTrees and verify that Get(key) is as expected.
 	for _, tt := range smallTestv {
 		xobj, err := conn.Get(ctx, tt.oid)
@@ -262,4 +271,39 @@ func TestBTree(t *testing.T) {
 	// XXX verify FirstBucket / Next ?
 
 	verifyFirstBucket(B3)
+
+	// verify nodes/keycov visited through VGet/V{Min,Max}Key
+	xBv, err := conn.Get(ctx, Bv_oid);  X(err)
+	Bv, ok := xBv.(*LOBTree)
+	if !ok {
+		t.Fatalf("Bv: %v; got %T;  want LOBTree", Bv_oid, xBv)
+	}
+
+	for k, visitOK := range Bvdict {
+		visit := []tVisit{}
+		_, _, err := Bv.VGet(ctx, k, func(node LONode, keycov LKeyRange) {
+			visit = append(visit, tVisit{node.POid(), keycov})
+		}); X(err)
+		if !reflect.DeepEqual(visit, visitOK) {
+			t.Errorf("VGet(%d): visit:\nhave: %v\nwant: %v", k, visit, visitOK)
+		}
+	}
+
+	visitMinOK := Bvdict[Bv_kmin]
+	visitMaxOK := Bvdict[Bv_kmax]
+	visitMin := []tVisit{}
+	visitMax := []tVisit{}
+	_, _, err = Bv.VMinKey(ctx, func(node LONode, keycov LKeyRange) {
+		visitMin = append(visitMin, tVisit{node.POid(), keycov})
+	}); X(err)
+	_, _, err = Bv.VMaxKey(ctx, func(node LONode, keycov LKeyRange) {
+		visitMax = append(visitMax, tVisit{node.POid(), keycov})
+	}); X(err)
+
+	if !reflect.DeepEqual(visitMin, visitMinOK) {
+		t.Errorf("VMinKey(): visit:\nhave: %v\nwant: %v", visitMin, visitMinOK)
+	}
+	if !reflect.DeepEqual(visitMax, visitMaxOK) {
+		t.Errorf("VMaxKey(): visit:\nhave: %v\nwant: %v", visitMax, visitMaxOK)
+	}
 }

go/zodb/btree/gen-btree

@@ -28,6 +28,9 @@ out=$3
 kind=${KIND,,}	# IO -> io
 Key=${KEY^}
 
+KEYKIND=${KIND:0:1}	# IO -> I
+keykind=${KEYKIND,,}	# I -> i
+
 input=$(dirname $0)/btree.go.in
 
 echo "// Code generated by gen-btree; DO NOT EDIT." >$out
@@ -45,4 +48,10 @@ sed	\
 	-e "s/\bBucketEntry\b/${KIND}BucketEntry/g"	\
 	-e "s/\bbtreeState\b/${kind}btreeState/g"	\
 	-e "s/\bbucketState\b/${kind}bucketState/g"	\
+	-e "s/\b_KeyMin\b/_${KEYKIND}KeyMin/g"		\
+	-e "s/\b_KeyMax\b/_${KEYKIND}KeyMax/g"		\
+	-e "s/\bKeyRange\b/${KEYKIND}KeyRange/g"	\
+	-e "s/\bkmin\b/${keykind}kmin/g"		\
+	-e "s/\bkmax\b/${keykind}kmax/g"		\
+	-e "s/\bkstr\b/${keykind}kstr/g"		\
 	$input >>$out

go/zodb/btree/testdata/gen-testdata

@@ -22,6 +22,7 @@
 
 from ZODB.DB import DB
 from BTrees.LOBTree import LOBucket, LOBTree
+from BTrees.check import check as bcheck
 from ZODB.utils import u64
 from zodbtools.test.gen_testdata import run_with_zodb4py2_compat
 import os, os.path, transaction
@@ -52,14 +53,37 @@ def main2():
 
     root['B3'] = B3 = LOBTree(dict([(_, _) for _ in range(10000)]))
 
+
+    # T4/T2-T/B1-B2-T7,9/B5-B8-B10  (to verify VGet->visit)
+    # TODO use xbtree.py:Restructure after gimport works through modules and
+    #      xbtree.py is moved from wcfs to zodb/go.
+    v1 = LOBucket([(1,"a")])
+    v2 = LOBucket([(2,"b")])
+    v5 = LOBucket([(5,"c")])
+    v8 = LOBucket([(8,"d")])
+    v9 = LOBucket([(9,"e")])
+    T2, T79, T, T4 = LOBTree(), LOBTree(), LOBTree(), LOBTree()
+    T2.__setstate__ (((v1, 2, v2),         v1)) # (child, key, child, ...), firstbucket
+    T79.__setstate__(((v5, 7, v8, 9, v9),  v5))
+    T.__setstate__  (((T79,),              v5))
+    T4.__setstate__ (((T2, 4, T),          v1))
+    root['Bv'] = Bv = T4
+
     transaction.commit()
 
+    bcheck(Bv)
+    assert Bv[1] == "a"
+    assert Bv[2] == "b"
+    assert Bv[5] == "c"
+    assert Bv[8] == "d"
+    assert Bv[9] == "e"
+
+
     with open("ztestdata_expect_test.go", "w") as f:
         def emit(v):
             print >>f, v
         emit("// Code generated by %s; DO NOT EDIT." % __file__)
         emit("package btree\n")
-        #emit("import \"lab.nexedi.com/kirr/neo/go/zodb\"\n")
 
         def emititems(b):
             s = "testEntry{oid: %s, kind: %s, itemv: []kv{" \
@@ -84,6 +108,28 @@ def main2():
         emit("\nconst B3_oid = %s" % u64(B3._p_oid))
         emit("const B3_maxkey = %d" % B3.maxKey())
 
+        emit("\nconst Bv_oid = %s" % u64(Bv._p_oid))
+        emit("const Bv_kmin = %d"  % Bv.minKey())
+        emit("const Bv_kmax = %d"  % Bv.maxKey())
+        emit("var Bvdict = map[int64][]tVisit{")
+        noo = "_LKeyMin"
+        oo  = "_LKeyMax"
+        def emitVisit(key, *visitv): # visitv = [](node, lo,hi)
+            vstr = []
+            for node, lo,hi in visitv:
+                if isinstance(hi, str):
+                    hi_ = hi    # oo or noo
+                else:
+                    hi_ = hi-1
+                vstr.append("{%d, LKeyRange{%s, %s}}" % (u64(node._p_oid), lo, hi_))
+            emit("\t%d: []tVisit{%s}," % (key, ", ".join(vstr)))
+
+        emitVisit(1, (T4, noo,oo), (T2, noo,4), (v1, noo,2))
+        emitVisit(2, (T4, noo,oo), (T2, noo,4), (v2, 2,4))
+        emitVisit(5, (T4, noo,oo), (T,  4,oo),  (T79, 4,oo), (v5, 4,7))
+        emitVisit(8, (T4, noo,oo), (T,  4,oo),  (T79, 4,oo), (v8, 7,9))
+        emitVisit(9, (T4, noo,oo), (T,  4,oo),  (T79, 4,oo), (v9, 9,oo))
+        emit("}")
 
     conn.close()
     db.close()

go/zodb/btree/ziobtree.go

@@ -67,7 +67,7 @@ type IOBTree struct {
 
 // IOEntry is one IOBTree node entry.
 //
-// It contains key and child, who is either IOBTree or IOBucket.
+// It contains key and child, which is either IOBTree or IOBucket.
 //
 // Key limits child's keys - see IOBTree.Entryv for details.
 type IOEntry struct {
@@ -104,6 +104,15 @@ type IOBucketEntry struct {
 	value interface{}
 }
 
+// IKeyRange represents [lo,hi) key range.
+type IKeyRange struct {
+	Lo  int32
+	Hi_ int32 // NOTE _not_ hi) to avoid overflow at ∞;  hi = hi_ + 1
+}
+
+const _IKeyMin int32 = math.MinInt32
+const _IKeyMax int32 = math.MaxInt32
+
 // ---- access []entry ----
 
 // Key returns IOBTree entry key.
@@ -125,7 +134,7 @@ func (e *IOEntry) Child() IONode { return e.child }
 // Children of all entries are guaranteed to be of the same kind - either all IOBTree, or all IOBucket.
 //
 // The caller must not modify returned array.
-func (t *IOBTree) Entryv() []IOEntry {
+func (t *IOBTree) Entryv() /*readonly*/ []IOEntry {
 	return t.data
 }
 
@@ -136,7 +145,7 @@ func (e *IOBucketEntry) Key() int32 { return e.key }
 func (e *IOBucketEntry) Value() interface{} { return e.value }
 
 // Entryv returns entries of a IOBucket node.
-func (b *IOBucket) Entryv() []IOBucketEntry {
+func (b *IOBucket) Entryv() /*readonly*/ []IOBucketEntry {
 	ev := make([]IOBucketEntry, len(b.keys))
 	for i, k := range b.keys {
 		ev[i] = IOBucketEntry{k, b.values[i]}
@@ -170,36 +179,54 @@ func (t *IOBTree) Get(ctx context.Context, key int32) (_ interface{}, _ bool, er
 // VGet is like Get but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *IOBTree) VGet(ctx context.Context, key int32, visit func(node IONode)) (_ interface{}, _ bool, err error) {
+func (t *IOBTree) VGet(ctx context.Context, key int32, visit func(node IONode, keycov IKeyRange)) (_ interface{}, _ bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): get %v", t.POid(), key)
 	err = t.PActivate(ctx)
 	if err != nil {
 		return nil, false, err
 	}
 
+	keycov := IKeyRange{Lo: _IKeyMin, Hi_: _IKeyMax}
 	if visit != nil {
-		visit(t)
-	}
-
-	if len(t.data) == 0 {
-		// empty btree
-		t.PDeactivate()
-		return nil, false, nil
+		visit(t, keycov)
 	}
 
 	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
+			t.PDeactivate()
+			return nil, false, nil
+		}
+
 		// search i: K(i) ≤ k < K(i+1)	; K(0) = -∞, K(len) = +∞
-		i := sort.Search(len(t.data), func(i int) bool {
+		i := sort.Search(l, func(i int) bool {
 			j := i + 1
 			if j == len(t.data) {
 				return true // [len].key = +∞
 			}
 			return key < t.data[j].key
 		})
-
-		// FIXME panic index out of range (empty T without children;
-		// logically incorrect, but Restructure generated it once)
+		// i < l
 		child := t.data[i].child
+
+		// shorten global keycov by local [lo,hi) for this child
+		lo := _IKeyMin
+		if i > 0 {
+			lo = t.data[i].key
+		}
+		i++
+		hi_ := _IKeyMax
+		if i < l {
+			hi_ = t.data[i].key
+		}
+		if hi_ != _IKeyMax {
+			hi_--
+		}
+		keycov.Lo  = ikmax(keycov.Lo, lo)
+		keycov.Hi_ = ikmin(keycov.Hi_, hi_)
+
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -209,7 +236,7 @@ func (t *IOBTree) VGet(ctx context.Context, key int32, visit func(node IONode))
 		// XXX verify child keys are in valid range according to parent
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		switch child := child.(type) {
@@ -252,26 +279,40 @@ func (t *IOBTree) MinKey(ctx context.Context) (_ int32, ok bool, err error) {
 // VMinKey is like MinKey but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *IOBTree) VMinKey(ctx context.Context, visit func(node IONode)) (_ int32, ok bool, err error) {
+func (t *IOBTree) VMinKey(ctx context.Context, visit func(node IONode, keycov IKeyRange)) (_ int32, ok bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): minkey", t.POid())
 	err = t.PActivate(ctx)
 	if err != nil {
 		return 0, false, err
 	}
 
+	keycov := IKeyRange{Lo: _IKeyMin, Hi_: _IKeyMax}
 	if visit != nil {
-		visit(t)
-	}
-
-	if len(t.data) == 0 {
-		// empty btree
-		t.PDeactivate()
-		return 0, false, nil
+		visit(t, keycov)
 	}
 
 	// NOTE -> can also use t.firstbucket
 	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
+			t.PDeactivate()
+			return 0, false, nil
+		}
+
 		child := t.data[0].child
+
+		// shorten global keycov by local hi) for this child
+		hi_ := _IKeyMax
+		if 1 < l {
+			hi_ = t.data[1].key
+		}
+		if hi_ != _IKeyMax {
+			hi_--
+		}
+		// keycov.Lo stays -∞
+		keycov.Hi_ = ikmin(keycov.Hi_, hi_)
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -279,7 +320,7 @@ func (t *IOBTree) VMinKey(ctx context.Context, visit func(node IONode)) (_ int32
 		}
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		// XXX verify child keys are in valid range according to parent
@@ -307,26 +348,36 @@ func (t *IOBTree) MaxKey(ctx context.Context) (_ int32, _ bool, err error) {
 // VMaxKey is like MaxKey but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *IOBTree) VMaxKey(ctx context.Context, visit func(node IONode)) (_ int32, _ bool, err error) {
+func (t *IOBTree) VMaxKey(ctx context.Context, visit func(node IONode, keycov IKeyRange)) (_ int32, _ bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): maxkey", t.POid())
 	err = t.PActivate(ctx)
 	if err != nil {
 		return 0, false, err
 	}
 
+	keycov := IKeyRange{Lo: _IKeyMin, Hi_: _IKeyMax}
 	if visit != nil {
-		visit(t)
-	}
-
-	l := len(t.data)
-	if l == 0 {
-		// empty btree
-		t.PDeactivate()
-		return 0, false, nil
+		visit(t, keycov)
 	}
 
 	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
+			t.PDeactivate()
+			return 0, false, nil
+		}
+
 		child := t.data[l-1].child
+
+		// shorten global keycov by local [lo for this chile
+		lo := _IKeyMin
+		if l-1 > 0 {
+			lo = t.data[l-1].key
+		}
+		keycov.Lo  = ikmax(keycov.Lo, lo)
+		// keycov.Hi_ stays ∞
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -334,7 +385,7 @@ func (t *IOBTree) VMaxKey(ctx context.Context, visit func(node IONode)) (_ int32
 		}
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		// XXX verify child keys are in valid range according to parent
@@ -593,7 +644,7 @@ func (bt *iobtreeState) PySetState(pystate interface{}) (err error) {
 	var kprev int64
 	var childrenKind int // 1 - IOBTree, 2 - IOBucket
 	for i, idx := 0, 0; i < n; i++ {
-		key := int64(math.MinInt32) // int32(-∞)   (qualifies for ≤)
+		key := int64(_IKeyMin) // int32(-∞)   (qualifies for ≤)
 		if i > 0 {
 			// key[0] is unused and not saved
 			key, ok = t[idx].(int64) // XXX Xint
@@ -648,3 +699,60 @@ func init() {
 	zodb.RegisterClass("BTrees.IOBTree.IOBTree",  t(IOBTree{}),  t(iobtreeState{}))
 	zodb.RegisterClass("BTrees.IOBTree.IOBucket", t(IOBucket{}), t(iobucketState{}))
 }
+
+
+// ---- misc ----
+
+// Has returns whether key k belongs to the range.
+func (r *IKeyRange) Has(k int32) bool {
+	return (r.Lo <= k && k <= r.Hi_)
+}
+
+// Empty returns whether key range is empty.
+func (r *IKeyRange) Empty() bool {
+	hi := r.Hi_
+	if hi == _IKeyMax {
+		// [x,∞] cannot be empty because max x is ∞ and [∞,∞] has one element: ∞
+		return false
+	}
+	hi++ // no overflow
+	return r.Lo >= hi
+}
+
+func (r IKeyRange) String() string {
+	var shi string
+	if r.Hi_ == _IKeyMax {
+		shi = ikstr(r.Hi_) // ∞
+	} else {
+		shi = fmt.Sprintf("%d", r.Hi_+1)
+	}
+	return fmt.Sprintf("[%s,%s)", ikstr(r.Lo), shi)
+}
+
+
+func ikmin(a, b int32) int32 {
+	if a < b {
+		return a
+	} else {
+		return b
+	}
+}
+
+func ikmax(a, b int32) int32 {
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+// ikstr formats key as string.
+func ikstr(k int32) string {
+	if k == _IKeyMin {
+		return "-∞"
+	}
+	if k == _IKeyMax {
+		return "∞"
+	}
+	return fmt.Sprintf("%d", k)
+}

go/zodb/btree/zlobtree.go

@@ -67,7 +67,7 @@ type LOBTree struct {
 
 // LOEntry is one LOBTree node entry.
 //
-// It contains key and child, who is either LOBTree or LOBucket.
+// It contains key and child, which is either LOBTree or LOBucket.
 //
 // Key limits child's keys - see LOBTree.Entryv for details.
 type LOEntry struct {
@@ -104,6 +104,15 @@ type LOBucketEntry struct {
 	value interface{}
 }
 
+// LKeyRange represents [lo,hi) key range.
+type LKeyRange struct {
+	Lo  int64
+	Hi_ int64 // NOTE _not_ hi) to avoid overflow at ∞;  hi = hi_ + 1
+}
+
+const _LKeyMin int64 = math.MinInt64
+const _LKeyMax int64 = math.MaxInt64
+
 // ---- access []entry ----
 
 // Key returns LOBTree entry key.
@@ -125,7 +134,7 @@ func (e *LOEntry) Child() LONode { return e.child }
 // Children of all entries are guaranteed to be of the same kind - either all LOBTree, or all LOBucket.
 //
 // The caller must not modify returned array.
-func (t *LOBTree) Entryv() []LOEntry {
+func (t *LOBTree) Entryv() /*readonly*/ []LOEntry {
 	return t.data
 }
 
@@ -136,7 +145,7 @@ func (e *LOBucketEntry) Key() int64 { return e.key }
 func (e *LOBucketEntry) Value() interface{} { return e.value }
 
 // Entryv returns entries of a LOBucket node.
-func (b *LOBucket) Entryv() []LOBucketEntry {
+func (b *LOBucket) Entryv() /*readonly*/ []LOBucketEntry {
 	ev := make([]LOBucketEntry, len(b.keys))
 	for i, k := range b.keys {
 		ev[i] = LOBucketEntry{k, b.values[i]}
@@ -170,36 +179,54 @@ func (t *LOBTree) Get(ctx context.Context, key int64) (_ interface{}, _ bool, er
 // VGet is like Get but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *LOBTree) VGet(ctx context.Context, key int64, visit func(node LONode)) (_ interface{}, _ bool, err error) {
+func (t *LOBTree) VGet(ctx context.Context, key int64, visit func(node LONode, keycov LKeyRange)) (_ interface{}, _ bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): get %v", t.POid(), key)
 	err = t.PActivate(ctx)
 	if err != nil {
 		return nil, false, err
 	}
 
+	keycov := LKeyRange{Lo: _LKeyMin, Hi_: _LKeyMax}
 	if visit != nil {
-		visit(t)
-	}
-
-	if len(t.data) == 0 {
-		// empty btree
-		t.PDeactivate()
-		return nil, false, nil
+		visit(t, keycov)
 	}
 
 	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
+			t.PDeactivate()
+			return nil, false, nil
+		}
+
 		// search i: K(i) ≤ k < K(i+1)	; K(0) = -∞, K(len) = +∞
-		i := sort.Search(len(t.data), func(i int) bool {
+		i := sort.Search(l, func(i int) bool {
 			j := i + 1
 			if j == len(t.data) {
 				return true // [len].key = +∞
 			}
 			return key < t.data[j].key
 		})
-
-		// FIXME panic index out of range (empty T without children;
-		// logically incorrect, but Restructure generated it once)
+		// i < l
 		child := t.data[i].child
+
+		// shorten global keycov by local [lo,hi) for this child
+		lo := _LKeyMin
+		if i > 0 {
+			lo = t.data[i].key
+		}
+		i++
+		hi_ := _LKeyMax
+		if i < l {
+			hi_ = t.data[i].key
+		}
+		if hi_ != _LKeyMax {
+			hi_--
+		}
+		keycov.Lo  = lkmax(keycov.Lo, lo)
+		keycov.Hi_ = lkmin(keycov.Hi_, hi_)
+
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -209,7 +236,7 @@ func (t *LOBTree) VGet(ctx context.Context, key int64, visit func(node LONode))
 		// XXX verify child keys are in valid range according to parent
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		switch child := child.(type) {
@@ -252,26 +279,40 @@ func (t *LOBTree) MinKey(ctx context.Context) (_ int64, ok bool, err error) {
 // VMinKey is like MinKey but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *LOBTree) VMinKey(ctx context.Context, visit func(node LONode)) (_ int64, ok bool, err error) {
+func (t *LOBTree) VMinKey(ctx context.Context, visit func(node LONode, keycov LKeyRange)) (_ int64, ok bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): minkey", t.POid())
 	err = t.PActivate(ctx)
 	if err != nil {
 		return 0, false, err
 	}
 
+	keycov := LKeyRange{Lo: _LKeyMin, Hi_: _LKeyMax}
 	if visit != nil {
-		visit(t)
-	}
-
-	if len(t.data) == 0 {
-		// empty btree
-		t.PDeactivate()
-		return 0, false, nil
+		visit(t, keycov)
 	}
 
 	// NOTE -> can also use t.firstbucket
 	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
+			t.PDeactivate()
+			return 0, false, nil
+		}
+
 		child := t.data[0].child
+
+		// shorten global keycov by local hi) for this child
+		hi_ := _LKeyMax
+		if 1 < l {
+			hi_ = t.data[1].key
+		}
+		if hi_ != _LKeyMax {
+			hi_--
+		}
+		// keycov.Lo stays -∞
+		keycov.Hi_ = lkmin(keycov.Hi_, hi_)
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -279,7 +320,7 @@ func (t *LOBTree) VMinKey(ctx context.Context, visit func(node LONode)) (_ int64
 		}
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		// XXX verify child keys are in valid range according to parent
@@ -307,26 +348,36 @@ func (t *LOBTree) MaxKey(ctx context.Context) (_ int64, _ bool, err error) {
 // VMaxKey is like MaxKey but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *LOBTree) VMaxKey(ctx context.Context, visit func(node LONode)) (_ int64, _ bool, err error) {
+func (t *LOBTree) VMaxKey(ctx context.Context, visit func(node LONode, keycov LKeyRange)) (_ int64, _ bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): maxkey", t.POid())
 	err = t.PActivate(ctx)
 	if err != nil {
 		return 0, false, err
 	}
 
+	keycov := LKeyRange{Lo: _LKeyMin, Hi_: _LKeyMax}
 	if visit != nil {
-		visit(t)
-	}
-
-	l := len(t.data)
-	if l == 0 {
-		// empty btree
-		t.PDeactivate()
-		return 0, false, nil
+		visit(t, keycov)
 	}
 
 	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
+			t.PDeactivate()
+			return 0, false, nil
+		}
+
 		child := t.data[l-1].child
+
+		// shorten global keycov by local [lo for this chile
+		lo := _LKeyMin
+		if l-1 > 0 {
+			lo = t.data[l-1].key
+		}
+		keycov.Lo  = lkmax(keycov.Lo, lo)
+		// keycov.Hi_ stays ∞
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -334,7 +385,7 @@ func (t *LOBTree) VMaxKey(ctx context.Context, visit func(node LONode)) (_ int64
 		}
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		// XXX verify child keys are in valid range according to parent
@@ -593,7 +644,7 @@ func (bt *lobtreeState) PySetState(pystate interface{}) (err error) {
 	var kprev int64
 	var childrenKind int // 1 - LOBTree, 2 - LOBucket
 	for i, idx := 0, 0; i < n; i++ {
-		key := int64(math.MinInt64) // int64(-∞)   (qualifies for ≤)
+		key := int64(_LKeyMin) // int64(-∞)   (qualifies for ≤)
 		if i > 0 {
 			// key[0] is unused and not saved
 			key, ok = t[idx].(int64) // XXX Xint
@@ -648,3 +699,60 @@ func init() {
 	zodb.RegisterClass("BTrees.LOBTree.LOBTree",  t(LOBTree{}),  t(lobtreeState{}))
 	zodb.RegisterClass("BTrees.LOBTree.LOBucket", t(LOBucket{}), t(lobucketState{}))
 }
+
+
+// ---- misc ----
+
+// Has returns whether key k belongs to the range.
+func (r *LKeyRange) Has(k int64) bool {
+	return (r.Lo <= k && k <= r.Hi_)
+}
+
+// Empty returns whether key range is empty.
+func (r *LKeyRange) Empty() bool {
+	hi := r.Hi_
+	if hi == _LKeyMax {
+		// [x,∞] cannot be empty because max x is ∞ and [∞,∞] has one element: ∞
+		return false
+	}
+	hi++ // no overflow
+	return r.Lo >= hi
+}
+
+func (r LKeyRange) String() string {
+	var shi string
+	if r.Hi_ == _LKeyMax {
+		shi = lkstr(r.Hi_) // ∞
+	} else {
+		shi = fmt.Sprintf("%d", r.Hi_+1)
+	}
+	return fmt.Sprintf("[%s,%s)", lkstr(r.Lo), shi)
+}
+
+
+func lkmin(a, b int64) int64 {
+	if a < b {
+		return a
+	} else {
+		return b
+	}
+}
+
+func lkmax(a, b int64) int64 {
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+// lkstr formats key as string.
+func lkstr(k int64) string {
+	if k == _LKeyMin {
+		return "-∞"
+	}
+	if k == _LKeyMax {
+		return "∞"
+	}
+	return fmt.Sprintf("%d", k)
+}

go/zodb/btree/ztestdata_expect_test.go

@@ -3,13 +3,24 @@ package btree
 
 
 var smallTestv = [...]testEntry{
-	testEntry{oid: 6, kind: kindBucket, itemv: []kv{}},
-	testEntry{oid: 3, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
+	testEntry{oid: 7, kind: kindBucket, itemv: []kv{}},
+	testEntry{oid: 4, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
 	testEntry{oid: 1, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, "hello"}, }},
-	testEntry{oid: 2, kind: kindBTree, itemv: []kv{}},
-	testEntry{oid: 7, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
-	testEntry{oid: 4, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, "world"}, }},
+	testEntry{oid: 3, kind: kindBTree, itemv: []kv{}},
+	testEntry{oid: 8, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
+	testEntry{oid: 5, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, "world"}, }},
 }
 
-const B3_oid = 5
+const B3_oid = 6
 const B3_maxkey = 9999
+
+const Bv_oid = 2
+const Bv_kmin = 1
+const Bv_kmax = 9
+var Bvdict = map[int64][]tVisit{
+	1: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {344, LKeyRange{_LKeyMin, 1}}},
+	2: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {349, LKeyRange{2, 3}}},
+	5: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {346, LKeyRange{4, 6}}},
+	8: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {347, LKeyRange{7, 8}}},
+	9: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {348, LKeyRange{9, _LKeyMax}}},
+}