.

wcfs/internal/xbtree/zset_object.go

+// Code generated by gen-set Object interface{}; DO NOT EDIT.
+
+// Copyright (C) 2015-2019  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtree
+
+// SetObject is a set of interface{}.
+type SetObject map[interface{}]struct{}
+
+// Add adds v to the set.
+func (s SetObject) Add(v interface{}) {
+	s[v] = struct{}{}
+}
+
+// XXX + Del(v) ?
+
+// Has checks whether the set contains v.
+func (s SetObject) Has(v interface{}) bool {
+	_, ok := s[v]
+	return ok
+}
+
+// Update adds t values to s.
+func (s SetObject) Update(t SetObject) {
+	for v := range t {
+		s.Add(v)
+	}
+}
+
+// Elements returns all elements of set as slice.
+func (s SetObject) Elements() []interface{} {
+	ev := make([]interface{}, len(s))
+	i := 0
+	for e := range s {
+		ev[i] = e
+		i++
+	}
+	return ev
+}

wcfs/internal/xbtree/zset_tree.go

+// Code generated by gen-set Tree *Tree; DO NOT EDIT.
+
+// Copyright (C) 2015-2019  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtree
+
+// SetTree is a set of *Tree.
+type SetTree map[*Tree]struct{}
+
+// Add adds v to the set.
+func (s SetTree) Add(v *Tree) {
+	s[v] = struct{}{}
+}
+
+// XXX + Del(v) ?
+
+// Has checks whether the set contains v.
+func (s SetTree) Has(v *Tree) bool {
+	_, ok := s[v]
+	return ok
+}
+
+// Update adds t values to s.
+func (s SetTree) Update(t SetTree) {
+	for v := range t {
+		s.Add(v)
+	}
+}
+
+// Elements returns all elements of set as slice.
+func (s SetTree) Elements() []*Tree {
+	ev := make([]*Tree, len(s))
+	i := 0
+	for e := range s {
+		ev[i] = e
+		i++
+	}
+	return ev
+}

wcfs/internal/xbtree/δbtail.go

@@ -69,8 +69,9 @@ package xbtree
 // . -k(blk) -> invalidate #blk
 // . +k(blk) -> invalidate #blk (e.g. if blk was previously read as hole)
 
-//go:generate ../../gen-set δbtree Tree *Tree zset_tree.go
-//go:generate ../../gen-set δbtree Object interface{} zset_object.go
+//go:generate ../../gen-set xbtree Tree *Tree zset_tree.go
+// //go:generate ../../gen-set xbtree Oid zodb.Oid zset_oid.go
+//go:generate ../../gen-set xbtree Object interface{} zset_object.go
 
 import (
 	"lab.nexedi.com/kirr/neo/go/zodb"
@@ -106,9 +107,14 @@ type ΔTail struct {
 	// XXX or ask client provide db on every call?
 	db     *zodb.DB    // to open connections to load new/old tree|buckets
 
-	// root index: BTree|Bucket -> top tree element.
-	// XXX represent root by oid instead of live object?
-	rootIdx map[zodb.Oid]SetTree // oid -> {} roots
+	// tracked index: BTree|Bucket -> top tree element.
+	trackIdx map[zodb.Oid]SetTree // oid -> {} roots		XXX root -> oid?
+
+	// tracked objects that are not yet taken into in current δBtail
+	trackNew map[zodb.Oid]struct{}	// XXX SetOid
+
+	// index for LastRevOf queries
+	lastRevOf map[zodb.Oid]map[Key]zodb.Tid // {} root -> {} key -> last
 }
 
 // ΔRevEntry represents information about BTrees changes in one revision.
@@ -119,7 +125,7 @@ type ΔRevEntry struct {
 
 // ΔTree represents changes to one tree.
 type ΔTree struct {
-	Root XXX	// XXX -> Oid?
+	Root *Tree	// XXX -> Oid?
 	Keyv []Key
 }
 
@@ -129,14 +135,15 @@ type ΔTree struct {
 // Initial coverage is (at₀, at₀].
 func NewΔTail(at0 zodb.Tid) *ΔTail {
 	return &ΔTail{
-		δZtail: zodb.NewΔTail(at0),
-		root:   make(map[zodb.Oid]SetTree),
+		δZtail:   zodb.NewΔTail(at0),
+		trackIdx: make(map[zodb.Oid]SetTree),
 	}
 }
 
 // Head is similar to zodb.ΔTail.Head .		XXX ok?
 func (δb *ΔTail) Head() zodb.Tid {
-	return δb.head
+	//return δb.head
+	return δb.δZtail.Head()
 }
 
 // Tail is similar to zodb.ΔTail.Tail .		XXX ok?
@@ -149,7 +156,7 @@ func (δb *ΔTail) Tail() zodb.Tid {
 
 // Track adds tree path to tracked set.
 //
-// XXX tailv is rebuild to also include keys corresponding to added nodes.
+// XXX δBtail is rebuild to also include keys corresponding to added nodes.
 //
 // XXX path is at @at state.
 // XXX objects in path must be with .PJar().At() == .head
@@ -159,8 +166,11 @@ func (δb *ΔTail) Track(path []Node) {	// XXX Tree|Bucket; path[0] = root
 	root = path[0]
 	for obj in path:
 		if not obj in trackIdx:
-			.new += obj		# XXX .4rebuild += ... ?
+			.trackNew += obj
 		.trackIdx[obj] += root
+
+	XXX update diff	XXX here? or as separate step?
+	XXX update lastRevOf
 */
 }
 
@@ -175,8 +185,9 @@ func (δb *ΔTail) Track(path []Node) {	// XXX Tree|Bucket; path[0] = root
 func (δB *ΔTail) Update(δZ *zodb.EventCommit) {
 /*
 	.δZtail += δZ
-	.update()
 */
+	δB.update()
+}
 
 // update brings .δBtail up to date by recomputing diff XXX and taking new
 // entries in .δZtail into account.
@@ -204,15 +215,24 @@ func (δB *ΔTail) update() {
 			...
 
 
-	.δBtail <- [] of (root, []key)
+	.δBtail <- (δZ.rev, [](root, []key))
+	XXX rebuilf lastRevOf
 */
 }
 
 // XXX -> Get(root, key, at) -> (valueOid, rev) ?
-func (δB *ΔTail) LastRevOf(root Tree, key Key, at zodb.Tid) zodb.Tid {
-	panic("TODO") // XXX return 0 as stub?
+func (δB *ΔTail) LastRevOf(root Tree, key Key, at zodb.Tid) (_ zodb.Tid, exact bool) {
+/*
+	lastRevOf = δB.lastRevOf[root]
+	rev = lastRevOf[key]
+	while rev > at:
+		rev = scan δB.δBtail ↓ looking for change(key)	# XXX linear scan
+	return rev
+
+*/
+	return 0, false // XXX stub (saying take only ZBlk.rev into account)
 }
 
 func (δB *ΔTail) ForgetPast(revCut zodb.Tid) {
-	panic("TODO")
+	δB.δZtail.ForgetPast(revCut) // XXX stub
 }

wcfs/internal/δbtree/zset_object.go

@@ -29,6 +29,8 @@ func (s SetObject) Add(v interface{}) {
 	s[v] = struct{}{}
 }
 
+// XXX + Del(v) ?
+
 // Has checks whether the set contains v.
 func (s SetObject) Has(v interface{}) bool {
 	_, ok := s[v]

wcfs/internal/δbtree/zset_tree.go

@@ -29,6 +29,8 @@ func (s SetTree) Add(v *Tree) {
 	s[v] = struct{}{}
 }
 
+// XXX + Del(v) ?
+
 // Has checks whether the set contains v.
 func (s SetTree) Has(v *Tree) bool {
 	_, ok := s[v]

wcfs/misc.go

@@ -20,6 +20,8 @@
 package main
 // misc utilities
 
+//go:generate ./gen-set main BigFile *BigFile zset_bigfile.go
+
 import (
 	"context"
 	"fmt"
@@ -40,6 +42,9 @@ import (
 
 	"lab.nexedi.com/kirr/neo/go/transaction"
 	"lab.nexedi.com/kirr/neo/go/zodb"
+	"lab.nexedi.com/kirr/neo/go/zodb/btree"
+
+	"./internal/xbtree"
 )
 
 // ---- FUSE ----
@@ -663,6 +668,26 @@ func parseWatch(msg string) (oid zodb.Oid, at zodb.Tid, err error) {
 	return oid, at, nil
 }
 
+// ---- btree.ΔTail + root -> BigFile ----
+
+// ΔFTail is btree.ΔTail but additionally tracks tree-root -> file relation.
+type ΔFTail struct {
+	*xbtree.ΔTail
+	fileIdx map[*btree.LOBTree]SetBigFile	// root -> {} BigFile	XXX root -> oid?
+}
+
+func (δf *ΔFTail) Track(file *BigFile, path []btree.LONode) {
+	δf.ΔTail.Track(path)
+	root := path[0].(*btree.LOBTree)
+	files, ok := δf.fileIdx[root]
+	if !ok {
+		files = SetBigFile{}
+		δf.fileIdx[root] = files
+	}
+	files.Add(file)
+}
+
+
 // ---- make df happy (else it complains "function not supported") ----
 
 func (root *Root) StatFs() *fuse.StatfsOut {

wcfs/wcfs.go

@@ -428,8 +428,6 @@ import (
 	"github.com/hanwen/go-fuse/fuse"
 	"github.com/hanwen/go-fuse/fuse/nodefs"
 	"github.com/pkg/errors"
-
-	"./internal/δbtree"
 )
 
 // Root represents root of wcfs filesystem.
@@ -501,12 +499,10 @@ type BigFileDir struct {
 	fileMu  sync.Mutex
 	fileTab map[zodb.Oid]*BigFile
 
-	// visited BTree nodes of all BigFiles
-	// -> which file + ordering for toposort on δbtree
-	//
+	// δ of tracked BTree nodes of all BigFiles + -> which file
 	// (used only for head/, not revX/)
-	indexMu     sync.Mutex
-	indexLooked *δbtree.PathSet	// XXX naming
+	δFmu   sync.Mutex	// XXX -> ΔFTail?
+	δFtail *ΔFTail
 }
 
 // /(head|<rev>)/bigfile/<bigfileX>	- served by BigFile.
@@ -525,8 +521,8 @@ type BigFile struct {
 	size    int64    // zfile.Size()
 	rev     zodb.Tid // last revision that modified zfile data
 
-	// tail change history of this file.
-	δtail *ΔTailI64 // [](rev↑, []#blk)
+//	// tail change history of this file.
+//	δtail *ΔTailI64 // [](rev↑, []#blk)
 
 	// inflight loadings of ZBigFile from ZODB.
 	// successful load results are kept here until blkdata is put into OS pagecache.
@@ -871,7 +867,7 @@ retry:
 		}
 
 		file.size = size
-		bfdir.indexLooked.Add(file, treePath)
+		bfdir.δFtail.Track(file, treePath)
 
 		file.rev = zhead.At()
 	}
@@ -1155,9 +1151,9 @@ func (f *BigFile) updateWatchers(ctx context.Context, blk int64, treepath []btre
 
 	// update δbtree index
 	bfdir := f.head.bfdir
-	bfdir.indexMu.Lock()		// XXX locking correct?
-	bfdir.indexLooked.Add(f, treepath)
-	bfdir.indexMu.Unlock()
+	bfdir.δFmu.Lock()		// XXX locking correct?
+	bfdir.δFtail.Track(f, treepath)
+	bfdir.δFmu.Unlock()
 
 	blkrevmax, _ := f.δtail.LastRevOf(blk, f.zfile.PJar().At())	// XXX = f.head.zconn.At()
 	blkrevmax = tidmin(blkrevmax, pathRevMax)
@@ -1623,10 +1619,10 @@ func (root *Root) lookup(name string, fctx *fuse.Context) (_ *Head, err error) {
 	}
 
 	bfdir := &BigFileDir{
-		fsNode:      newFSNode(&fsOptions{Sticky: false}),	// XXX + BigFileDir.OnForget()
-		head:        revDir,
-		fileTab:     make(map[zodb.Oid]*BigFile),
-		indexLooked: nil, // δbtree index not needed/used for @revX/
+		fsNode:  newFSNode(&fsOptions{Sticky: false}),	// XXX + BigFileDir.OnForget()
+		head:    revDir,
+		fileTab: make(map[zodb.Oid]*BigFile),
+		δFtail:  nil, // δFtail not needed/used for @revX/
 	}
 	revDir.bfdir = bfdir
 
@@ -1705,11 +1701,11 @@ func (head *Head) bigopen(ctx context.Context, oid zodb.Oid) (_ *BigFile, err er
 		loading: make(map[int64]*blkLoadState),
 	}
 
-	// only head/ needs δbtree index.
+	// only head/ needs δFtail.
 	if head.rev == 0 {
-		head.bfdir.indexMu.Lock()	// XXX locking ok?
-		head.bfdir.indexLooked.Add(f, treePath)
-		head.bfdir.indexMu.Unlock()
+		head.bfdir.δFmu.Lock()	// XXX locking ok?
+		head.bfdir.δFtail.Track(f, treePath)
+		head.bfdir.δFmu.Unlock()
 	}
 
 	return f, nil
@@ -1881,10 +1877,10 @@ func main() {
 	}
 
 	bfdir := &BigFileDir{
-		fsNode:      newFSNode(fSticky),
-		head:        head,
-		fileTab:     make(map[zodb.Oid]*BigFile),
-		indexLooked: δbtree.NewPathSet(),
+		fsNode:   newFSNode(fSticky),
+		head:     head,
+		fileTab:  make(map[zodb.Oid]*BigFile),
+		δFtail:   NewΔFtail(zhead.At()),
 	}
 	head.bfdir = bfdir
 

wcfs/zset_bigfile.go

+// Code generated by gen-set BigFile *BigFile; DO NOT EDIT.
+
+// Copyright (C) 2015-2019  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package main
+
+// SetBigFile is a set of *BigFile.
+type SetBigFile map[*BigFile]struct{}
+
+// Add adds v to the set.
+func (s SetBigFile) Add(v *BigFile) {
+	s[v] = struct{}{}
+}
+
+// XXX + Del(v) ?
+
+// Has checks whether the set contains v.
+func (s SetBigFile) Has(v *BigFile) bool {
+	_, ok := s[v]
+	return ok
+}
+
+// Update adds t values to s.
+func (s SetBigFile) Update(t SetBigFile) {
+	for v := range t {
+		s.Add(v)
+	}
+}
+
+// Elements returns all elements of set as slice.
+func (s SetBigFile) Elements() []*BigFile {
+	ev := make([]*BigFile, len(s))
+	i := 0
+	for e := range s {
+		ev[i] = e
+		i++
+	}
+	return ev
+}

wcfs/zset_i64.go

@@ -29,6 +29,8 @@ func (s SetI64) Add(v int64) {
 	s[v] = struct{}{}
 }
 
+// XXX + Del(v) ?
+
 // Has checks whether the set contains v.
 func (s SetI64) Has(v int64) bool {
 	_, ok := s[v]

wcfs/zδtail_i64.go

 // Code generated by gen-δtail I64 int64; DO NOT EDIT.
-// (from lab.nexedi.com/kirr/neo/go/zodb @ v1.9-146-g4224b580)
+// (from lab.nexedi.com/kirr/neo/go/zodb @ v1.9-2242-g1b838adf)
 
 // Copyright (C) 2018-2019  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>