.

wcfs/gen-set

+#!/bin/bash -e
+# set.go.in -> specialized with concrete types
+# gen-set KIND VALUE out
+
+# Copyright (C) 2018  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+KIND=$1
+VALUE=$2
+out=$3
+
+input=$(dirname $0)/set.go.in
+
+echo "// Code generated by gen-set $KIND $VALUE; DO NOT EDIT." >$out
+echo >>$out
+
+sed	\
+	-e "s/VALUE/$VALUE/g"		\
+	-e "s/Set/${KIND}Set/g"		\
+	$input >>$out

wcfs/gen-δtail

@@ -27,7 +27,7 @@ out=$3
 
 input=$(dirname $0)/δtail.go.in
 
-echo "// Code generated by gen-δtail; DO NOT EDIT." >$out
+echo "// Code generated by gen-δtail $KIND $ID; DO NOT EDIT." >$out
 echo >>$out
 
 sed	\

wcfs/set.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// XXX -> internal/set/ ?
+
+package main
+
+//go:generate ./gen-set I64 int64 zset_i64.go

wcfs/set.go.in

+// Copyright (C) 2015-2018  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package main
+
+// Set is a set of VALUE.
+type Set map[VALUE]struct{}
+
+// Add adds v to the set.
+func (s Set) Add(v VALUE) {
+	s[VALUE] = struct{}{}
+}
+
+// Has checks whether the set contains v.
+func (s Set) Has(v VALUE) bool {
+	_, ok := s[v]
+	return ok
+}
+
+// Elements returns all elements of set as slice.
+func (s Set) Elements() []VALUE {
+	ev := make([]VALUE, len(s))
+	i := 0
+	for e := range s {
+		ev[i] = e
+		i++
+	}
+	return ev
+}

wcfs/wcfs.go

@@ -633,7 +633,7 @@ func (r *Root) zhandle1(zevent zodb.WatchEvent) {
 		case *LOBucket:
 			// XXX -> δBTree
 
-		case *ZBlkXXXAny:	// ZBlk0, ZBlk1
+		case zBlk:	// ZBlk0, ZBlk1
 			fileinv := XXX(obj.file)
 			fileinv.blkv += obj.blk		// XXX or better obj.blkv ?
 

wcfs/zblk.go

@@ -56,23 +56,62 @@ import (
 	"./internal/pycompat"
 )
 
-// zBlkLoader is the interface that every ZBlk* block implements internally for
-// loading its data.
-type zBlkLoader interface {
+// zBlk is the interface that every ZBlk* block implements internally.
+type zBlk interface {
 	// loadBlkData loads from database and returns data block stored by this ZBlk.
 	//
 	// If returned data size is less than the block size of containing ZBigFile,
 	// the block trailing is assumed to be trailing \0.
 	loadBlkData(ctx context.Context) ([]byte, error)
+
+	// bindZFile associates ZBlk as being used by zfile to store block #blk.
+	//
+	// A ZBlk may be bound to several blocks inside one file, and to
+	// several files.
+	//
+	// XXX the information is preserved even when ZBlk comes to ghost
+	// state, but is lost if ZBlk is garbage collected.
+	//
+	// XXX concurrent access
+	//
+	// XXX link to overview.
+	bindZFile(zfile *ZBigFile, blk int64)
+
+	// XXX unbindZFile
+	// XXX zfile -> bind map for it
 }
 
 // module of Wendelin ZODB py objects
 const zwendelin = "wendelin.bigfile.file_zodb"
 
+// ---- zBlkBase ----
+
+// zBlkBase provides common functionality to implement ZBlk* -> zfile, #blk binding.
+//
+// The data stored by zBlkBase is transient - it is _not_ included into
+// persistent state.
+type zBlkBase struct {
+	mu      sync.Mutex
+	filetab map[*ZBigFile]SetI64 // {} zfile -> set(#blk)
+}
+
+func (zb *zBlkBase) bindZfile(zfile *ZBigFile, blk int64) {
+	zb.mu.Lock()
+	defer zb.mu.Unlock()
+
+	filemap, ok := zb.filetab[zfile]
+	if !ok {
+		filemap = I64Set{}
+		zb.filetab[zfile] = filemap
+	}
+	filemap.Add(blk)
+}
+
 // ---- ZBlk0 ----
 
 // ZBlk0 mimics ZBlk0 from python.
 type ZBlk0 struct {
+	zBlkBase
 	zodb.Persistent
 
 	// XXX py source uses bytes(buf) but on python2 it still results in str
@@ -112,6 +151,7 @@ func (zb *ZBlk0) loadBlkData(ctx context.Context) ([]byte, error) {
 
 // ZData mimics ZData from python.
 type ZData struct {
+	zBlkBase
 	zodb.Persistent
 
 	// XXX py source uses bytes(buf) but on python2 it still results in str
@@ -390,7 +430,7 @@ func (bf *ZBigFile) LoadBlk(ctx context.Context, blk int64) (_ []byte, err error
 		return make([]byte, bf.blksize), nil
 	}
 
-	zblk, ok := xzblk.(zBlkLoader)
+	zblk, ok := xzblk.(zBlk)
 	if !ok {
 		return nil, fmt.Errorf("expect ZBlk*; got %s", typeOf(xzblk))
 	}
@@ -412,7 +452,9 @@ func (bf *ZBigFile) LoadBlk(ctx context.Context, blk int64) (_ []byte, err error
 		blkdata = d
 	}
 
-	log.Printf("ZBigFile.loadblk(%d) -> %dB", blk, len(blkdata))
+	zblk.bindZFile(bf, blk)
+
+	//log.Printf("ZBigFile.loadblk(%d) -> %dB", blk, len(blkdata))
 	return blkdata, nil
 }
 

wcfs/zset_i64.go

+// Code generated by gen-set I64 int64; DO NOT EDIT.
+
+// Copyright (C) 2015-2018  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package main
+
+// I64Set is a set of int64.
+type I64Set map[int64]struct{}
+
+// Add adds v to the set.
+func (s I64Set) Add(v int64) {
+	s[int64] = struct{}{}
+}
+
+// Has checks whether the set contains v.
+func (s I64Set) Has(v int64) bool {
+	_, ok := s[v]
+	return ok
+}
+
+// Elements returns all elements of set as slice.
+func (s I64Set) Elements() []int64 {
+	ev := make([]int64, len(s))
+	i := 0
+	for e := range s {
+		ev[i] = e
+		i++
+	}
+	return ev
+}

wcfs/zδtail_i64.go

-// Code generated by gen-δtail; DO NOT EDIT.
+// Code generated by gen-δtail I64 int64; DO NOT EDIT.
 
 // Copyright (C) 2018  Nexedi SA and Contributors.
 //                     Kirill Smelkov <kirr@nexedi.com>