.

wcfs/gen-set

 #!/bin/bash -e
 # set.go.in -> specialized with concrete types
-# gen-set KIND VALUE out
+# gen-set package KIND VALUE out
 
 # Copyright (C) 2018-2019  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
@@ -21,9 +21,10 @@
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
 
-KIND=$1
-VALUE=$2
-out=$3
+PACKAGE=$1
+KIND=$2
+VALUE=$3
+out=$4
 
 input=$(dirname $0)/set.go.in
 
@@ -31,6 +32,7 @@ echo "// Code generated by gen-set $KIND $VALUE; DO NOT EDIT." >$out
 echo >>$out
 
 sed	\
+	-e "s/PACKAGE/$PACKAGE/g"	\
 	-e "s/VALUE/$VALUE/g"		\
 	-e "s/Set/Set${KIND}/g"		\
 	$input >>$out

wcfs/internal/δbtree/zset_tree.go

+// Code generated by gen-set Tree *Tree; DO NOT EDIT.
+
+// Copyright (C) 2015-2019  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package δbtree
+
+// SetTree is a set of *Tree.
+type SetTree map[*Tree]struct{}
+
+// Add adds v to the set.
+func (s SetTree) Add(v *Tree) {
+	s[v] = struct{}{}
+}
+
+// Has checks whether the set contains v.
+func (s SetTree) Has(v *Tree) bool {
+	_, ok := s[v]
+	return ok
+}
+
+// Update adds t values to s.
+func (s SetTree) Update(t SetTree) {
+	for v := range t {
+		s.Add(v)
+	}
+}
+
+// Elements returns all elements of set as slice.
+func (s SetTree) Elements() []*Tree {
+	ev := make([]*Tree, len(s))
+	i := 0
+	for e := range s {
+		ev[i] = e
+		i++
+	}
+	return ev
+}

wcfs/internal/δbtree/δbtree.go

+// Copyright (C) 2019  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package δbtree provides BTree diffing.
+package δbtree
+
+//go:generate ../../gen-set δbtree Tree *Tree zset_tree.go
+
+import (
+	"fmt"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+	"lab.nexedi.com/kirr/neo/go/zodb/btree"
+)
+
+type Tree   = btree.LOBTree
+type Bucket = btree.LOBucket
+
+// PathMap is a residency index which maps BTree|Bucket to top tree element.
+//
+// PathMap is not safe for concurrent access.
+type PathMap struct {
+	residencyIdx map[zodb.Oid]SetTree // oid -> {} roots
+}
+
+// Add adds path to collection.
+//
+// path[0] signifies a root.
+// All path elemens must be Tree except last one which must be Bucket.
+func (m *PathMap) Add(path []zodb.IPersistent) {
+	l := len(path)
+	if l == 0 {
+		panic("empty path")
+	}
+
+	var root *Tree
+	for i, node := range path {
+		switch node := node.(type) {
+		default:
+			panic(fmt.Sprintf("path elements must be Tree|Bucket, not %T", node))
+
+		case *Tree:
+			if i == 0 {
+				root = node
+			}
+			// XXX check only not last
+
+		case *Bucket:
+			// XXX check only last
+
+		}
+	}
+
+	for _, node := range path {
+		nodeRoots, ok := m.residencyIdx[node.POid()]
+		if !ok {
+			nodeRoots = make(SetTree)
+			m.residencyIdx[node.POid()] = nodeRoots
+		}
+		nodeRoots.Add(root)
+	}
+}
+
+// Invalidates returns which trees are invalidated by changed objects.
+//
+// returned are: roots of changed trees.
+//
+// XXX stub.
+func (m *PathMap) Invalidates(changev []zodb.Oid) SetTree {
+	toinvalidate := make(SetTree)
+
+	for _, oid := range changev {
+		roots, ok := m.residencyIdx[oid]
+		if !ok {
+			continue // we don't know this oid
+		}
+
+		toinvalidate.Update(roots)
+	}
+
+	return toinvalidate
+}

wcfs/set.go

@@ -21,4 +21,4 @@
 
 package main
 
-//go:generate ./gen-set I64 int64 zset_i64.go
+//go:generate ./gen-set main I64 int64 zset_i64.go

wcfs/set.go.in

@@ -17,7 +17,7 @@
 // See COPYING file for full licensing terms.
 // See https://www.nexedi.com/licensing for rationale and options.
 
-package main
+package PACKAGE
 
 // Set is a set of VALUE.
 type Set map[VALUE]struct{}

wcfs/wcfs.go

@@ -403,6 +403,8 @@ import (
 	"github.com/hanwen/go-fuse/fuse"
 	"github.com/hanwen/go-fuse/fuse/nodefs"
 	"github.com/pkg/errors"
+
+	"./internal/δbtree"
 )
 
 // Root represents root of wcfs filesystem.
@@ -439,6 +441,7 @@ type Head struct {
 	zconn   *ZConn       // for head/ zwatcher resyncs head.zconn; others only read zconn objects.
 
 	// XXX move zconn's current transaction to Head here?
+
 }
 
 // /head/watch				- served by Watch.
@@ -456,6 +459,9 @@ type BigFileDir struct {
 	// {} oid -> <bigfileX>
 	mu      sync.Mutex
 	fileTab map[zodb.Oid]*BigFile
+
+	// residency index for btree|bucket -> which file
+	btreeMap *δbtree.PathMap
 }
 
 // /(head|<rev>)/bigfile/<bigfileX>	- served by BigFile.
@@ -491,8 +497,9 @@ type BigFile struct {
 type blkLoadState struct {
 	ready chan struct{}
 
-	blkdata []byte
-	err     error
+	blkdata  []byte
+	treepath []zodb.IPersistent // LOBTree LOBTree ... LOBucket
+	err      error
 }
 
 // ----------------------------------------
@@ -1150,8 +1157,9 @@ func (f *BigFile) readBlk(ctx context.Context, blk int64, dest []byte) error {
 	// noone was loading - we became reponsible to load this block
 
 	zbf := f.zbf
-	blkdata, err := zbf.LoadBlk(ctx, blk)	// XXX -> +blkrevmax1
+	blkdata, treepath, err := zbf.LoadBlk(ctx, blk)	// XXX -> +blkrevmax1
 	loading.blkdata = blkdata
+	loading.treepath = treepath
 	loading.err = err
 	close(loading.ready)
 

wcfs/zblk.go

@@ -440,37 +440,43 @@ func (bf *zBigFileState) PySetState(pystate interface{}) (err error) {
 
 // LoadBlk loads data for file block #blk.
 //
+// it also returns BTree path in .blktab for loaded block.
+//
 // XXX better load into user-provided buf?
-func (bf *ZBigFile) LoadBlk(ctx context.Context, blk int64) (_ []byte, err error) {
+func (bf *ZBigFile) LoadBlk(ctx context.Context, blk int64) (_ []byte, treePath []zodb.IPersistent, err error) {
 	defer xerr.Contextf(&err, "bigfile %s: loadblk %d", bf.POid(), blk)
 
 	err = bf.PActivate(ctx)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	defer bf.PDeactivate()
 
-	xzblk, ok, err := bf.blktab.Get(ctx, blk)
+	xzblk, ok, treev, bucket, err := bf.blktab.GetTo(ctx, blk)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
+	for _, tree := range treev {
+		treePath = append(treePath, tree)
+	}
+	treePath = append(treePath, bucket)
 	if !ok {
-		return make([]byte, bf.blksize), nil
+		return make([]byte, bf.blksize), treePath, nil
 	}
 
 	zblk, ok := xzblk.(zBlk)
 	if !ok {
-		return nil, fmt.Errorf("expect ZBlk*; got %s", typeOf(xzblk))
+		return nil, nil, fmt.Errorf("expect ZBlk*; got %s", typeOf(xzblk))
 	}
 
 	blkdata, err := zblk.loadBlkData(ctx)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	l := int64(len(blkdata))
 	if l > bf.blksize {
-		return nil, fmt.Errorf("invalid blk: size = %d (> blksize = %d)", l, bf.blksize)
+		return nil, nil, fmt.Errorf("invalid blk: size = %d (> blksize = %d)", l, bf.blksize)
 	}
 
 	// append trailing \0 to data to reach .blksize
@@ -483,7 +489,7 @@ func (bf *ZBigFile) LoadBlk(ctx context.Context, blk int64) (_ []byte, err error
 	zblk.bindZFile(bf, blk)
 
 	//log.Printf("ZBigFile.loadblk(%d) -> %dB", blk, len(blkdata))
-	return blkdata, nil
+	return blkdata, treePath, nil
 }
 
 // Size returns whole file size.