wcfs: Initial implementation of basic filesystem

Provide filesystem view of in-ZODB ZBigFiles, but do not implement support for
invalidations nor isolation protocol yet. In particular, because ZODB
invalidations are not yet handled, the filesystem does not update its data in
accordance with ZODB updates, and instead provides stale data view that
corresponds to the state of ZODB at the time when wcfs was mounted.

The main parts of this patch are:

- wcfs/wcfs.go is filesystem implementation itself together with overview.
- wcfs/__init__.py is python wrapper to spawn and interoperate with that filesystem.
- wcfs/wcfs_test.py is tests.

Some preliminary history:

fe7efb94    X start of wcfs
878b2787    X draft loading
d58c71e8    X don't overalign end by 1 blksize if end is already aligned
29c9f13d    X readBlk: Fix thinko in already case
59552328    X wcfs: Care to disable OS polling on us
c00d94c7    X workaround lack of exception chaining on Python2 with xdefer
0398e23d    X bytearray turned out to be copying data
7a837040    X print wcfs.py py-level traceback on SIGBUS (e.g. wcfs.go aborting due to bug/panic)
661b871f    X make sure tests don't get stuck even if wcfs gets killed -9 ...
2c043d29    X More effort to unmount failed wcfs.go
1ccc4478    X Use `with gil` + regular py code instead of PyGILState_Ensure/PyGILState_Release/PyRun_SimpleString
5dc9c791    X wcfs: Kill xdefer
91e9eba8    X wcfs: test: Register tFile to tDB early
a7138fef    X wcfs: mkdir /tmp/wcfs with sticky bit
1eec76d0    X wcfs: try to set sticky for /tmp/wcfs even if the directory already exists
c2c35851    X wcfs: tests: Factor-out waiting for a general condition to become true into waitfor
78f36993    X wcfs: test: Fix thinko in getting /sys/fs/fuse/connection/<X> for wcfs
bc9eb16f    X wcfs: tests: Don't use testmntpt everywhere
6dec74e7    X wcfs: tests: Split tDB into -> tDB + tWCFS
3a6bd764    X wcfs: tests: Run `fusermount -u` the second time if we had to kill wcfs
112720f3    X wcfs: tests: Print which files are still opened on wcfs if `fusermount -u` fails
bb40185b    X wcfs: Take $WENDELIN_CORE_WCFS_OPTIONS into account not only from under join
03a9ef33    X wcfs: Remove credentials from zurl when computing wcfs mountpoint
68ee5bdc    X wcfs: lsof tweaks
21671879    X wcfs: Teach entrypoint frontend to handle subcommands: serve, status, stop
b0642b80    X wcfs: Switch mountpoints from /tmp/wcfs/* to /dev/shm/*
b0ca031f    X wcfs: Teach join/serve to start successfully even after unclean wcfs shutdown
5bfa8cf8    X wcfs: Add start to spawn a Server that can be later stopped  (draft)
5fcec261    X wcfs: Run fusermount and friends with /bin:/usr/bin always on path
669d7a20    fixup! X wcfs: Run fusermount and friends with /bin:/usr/bin always on path
6b22f8c4    X wcfs: Teach start to start successfully even after unclean wcfs shutdown
15389db0    X wcfs: Tune _fuse_unmount to include `fusermount -u` error message into raised exception
153c002a    X wcfs: _fuse_unmount: Try first `kill -TERM` before `kill -QUIT` wcfs
3244f3a6    X wcfs: lsof +D misbehaves - don't use it
a126e709    X wcfs: Put client log into its own logger
ac303d1e    X wcfs: tests: -v  ->  show only wcfs.py logs verbosely
d671a9e9    X wcfs: Give more time to stop wcfs server

conftest.py

@@ -41,3 +41,20 @@ def transaction_reset():
     transaction.manager.clearSynchs()
     yield
     # nothing to run after test
+
+
+# enable log_cli on no-capture
+# (output during a test is a mixture of print and log)
+def pytest_configure(config):
+    if config.option.capture == "no":
+        config.inicfg['log_cli'] = "true"
+        assert config.getini("log_cli") is True
+        # -v   -> verbose wcfs.py logs
+        if config.option.verbose > 0:
+            import logging
+            wcfslog = logging.getLogger('wcfs')
+            wcfslog.setLevel(logging.INFO)
+        # -vv  -> verbose *.py logs
+        # XXX + $WENDELIN_CORE_WCFS_OPTIONS="-d -alsologtostderr -v=1" ?
+        if config.option.verbose > 1:
+            config.inicfg['log_cli_level'] = "INFO"

setup.py

@@ -18,6 +18,7 @@
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
 from golang.pyx.build import setup, DSO as _DSO, Extension as _PyGoExt, build_ext as _build_ext
+from setuptools_dso import Extension
 from setuptools import Command, find_packages
 from setuptools.command.build_py import build_py as _build_py
 from pkg_resources import working_set, EntryPoint
@@ -73,8 +74,12 @@ def _with_defaults(what, *argv, **kw):
         ccdefault.append('-std=gnu++11')    # not c++11 since we use typeof
 
     # DSOs are not yet annotated for visibility
+    # XXX pyext besides _bigfile.so also cannot do this because PyMODINIT_FUNC
+    # does not include export in it. TODO reenable for _bigfile.so
+    """
     if what != _DSO:
         ccdefault.append('-fvisibility=hidden')  # by default symbols not visible outside DSO
+    """
 
     _ = kw.get('extra_compile_args', [])[:]
     _[0:0] = ccdefault
@@ -313,6 +318,15 @@ setup(
                         define_macros   = [('_GNU_SOURCE',None)],
                         language        = 'c',
                         dsos = ['wendelin.bigfile.libvirtmem']),
+
+                    PyGoExt('wendelin.wcfs.internal.wcfs_test',
+                        ['wcfs/internal/wcfs_test.pyx']),
+
+                    Extension('wendelin.wcfs.internal.io',
+                        ['wcfs/internal/io.pyx']),
+
+                    Extension('wendelin.wcfs.internal.mm',
+                        ['wcfs/internal/mm.pyx']),
                   ],
 
     package_dir = {'wendelin': ''},

wcfs/go.mod

@@ -3,9 +3,16 @@ module lab.nexedi.com/nexedi/wendelin.core/wcfs
 go 1.14
 
 require (
+	github.com/golang/glog v1.0.0
+	github.com/hanwen/go-fuse/v2 v2.1.0 // replaced to -> kirr/go-fuse@y/nodefs-cancel
 	github.com/johncgriffin/overflow v0.0.0-20211019200055-46fa312c352c
 	github.com/kisielk/og-rek v1.1.1-0.20210310094122-8def3d024dac
+	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.7.0
 	lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408
 	lab.nexedi.com/kirr/neo/go v0.0.0-20211004111643-c74a5a3cd0d0
 )
+
+// we use kirr/go-fuse@y/nodefs-cancel
+// see https://github.com/hanwen/go-fuse/pull/343 for details
+replace github.com/hanwen/go-fuse/v2 v2.1.0 => lab.nexedi.com/kirr/go-fuse/v2 v2.0.0-20210910085851-e6ee85fd0a1e

wcfs/go.sum

@@ -27,6 +27,8 @@ github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52 h1:0NmERxogGT
 github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -60,6 +62,7 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
 github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
@@ -170,6 +173,12 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+lab.nexedi.com/kirr/go-fuse/v2 v2.0.0-20210215102255-f0cbba3ef97e h1:6eJe/VaiivudiUEc2R324WN5djDycNvM1IkSQrC9idk=
+lab.nexedi.com/kirr/go-fuse/v2 v2.0.0-20210215102255-f0cbba3ef97e/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc=
+lab.nexedi.com/kirr/go-fuse/v2 v2.0.0-20210610115330-7e0334c3e76a h1:1Kagc2s/E5p2iqLLyGBtJRDssIa0uPbUKEEjKUZKSM4=
+lab.nexedi.com/kirr/go-fuse/v2 v2.0.0-20210610115330-7e0334c3e76a/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc=
+lab.nexedi.com/kirr/go-fuse/v2 v2.0.0-20210910085851-e6ee85fd0a1e h1:QP8PhLssUs3SEoM+UfQLxfDke7uQtyte4FNu6cw00L4=
+lab.nexedi.com/kirr/go-fuse/v2 v2.0.0-20210910085851-e6ee85fd0a1e/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc=
 lab.nexedi.com/kirr/go123 v0.0.0-20210128150852-c20e95f0f789/go.mod h1:1wkWl3WhmutZiho+wsE7ymOKvRkN7hV3YZtL0f0gXTo=
 lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408 h1:H7YpNUDfTSvvRpKivUMrL9C09tQssQ6brEoX6K/OxOw=
 lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408/go.mod h1:pwDpdCuvtz0QxisDzV/z9eUb9zc/rMQec520h4i8VWQ=

wcfs/internal/.gitignore

+/io.c
+/mm.c
+/wcfs_test.cpp

wcfs/internal/io.pyx

+# Copyright (C) 2019-2021  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+# cython: language_level=2
+
+"""Package io complements IO facility provided by Python."""
+
+from posix.unistd cimport pread
+from cpython.exc cimport PyErr_SetFromErrno
+
+# readat calls pread to read from fd@off into buf.
+def readat(int fd, size_t off, unsigned char[::1] buf not None):    # -> n
+    cdef void   *dest = &buf[0]
+    cdef size_t  size = buf.shape[0]
+    cdef ssize_t n
+
+    with nogil:
+        n = pread(fd, dest, size, off)
+
+    if n < 0:
+        PyErr_SetFromErrno(OSError)
+
+    return n

wcfs/internal/mm.pyx

+# Copyright (C) 2019-2021  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+# cython: language_level=2
+
+"""Package mm provides access to OS memory management interfaces."""
+
+from posix cimport mman
+from cpython.exc cimport PyErr_SetFromErrno
+
+cdef extern from "<sys/user.h>":
+    cpdef enum:
+        PAGE_SIZE
+
+from posix.types cimport off_t
+
+# map_ro memory-maps fd[offset +size) as read-only.
+# The mapping is created with MAP_SHARED.
+def map_ro(int fd, off_t offset, size_t size):
+    cdef void *addr
+
+    addr = mman.mmap(NULL, size, mman.PROT_READ, mman.MAP_SHARED, fd, offset)
+    if addr == mman.MAP_FAILED:
+        PyErr_SetFromErrno(OSError)
+
+    return <unsigned char[:size:1]>addr
+
+# unmap unmaps memory covered by mem.
+def unmap(const unsigned char[::1] mem not None):
+    cdef const void *addr = &mem[0]
+    cdef size_t      size = mem.shape[0]
+
+    cdef err = mman.munmap(<void *>addr, size)
+    if err:
+        PyErr_SetFromErrno(OSError)
+
+    return

wcfs/internal/wcfs_test.pyx

+# Copyright (C) 2019-2021  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+# cython: language_level=2
+# distutils: language=c++
+
+"""Module wcfs_test.pyx complements wcfs_test.py with things that cannot be
+implemented in Python."""
+
+from posix.signal cimport sigaction, sigaction_t, siginfo_t, SA_SIGINFO
+from libc.signal cimport SIGBUS
+from libc.stdlib cimport abort
+from libc.string cimport strlen
+from posix.unistd cimport write, sleep
+
+from cpython.exc cimport PyErr_SetFromErrno
+
+from golang cimport panic
+
+# ---- signal handling ----
+# TODO -> golang.signal ?
+
+# install_sigbus_trap installs SIGBUS handler that prints python-level
+# traceback before aborting.
+#
+# Such handler is useful, because when wcfs.go bugs/panics while handling file
+# access from wcfs.py, wcfs.py receives SIGBUS signal and by default aborts.
+def install_sigbus_trap():
+    cdef sigaction_t act
+    act.sa_sigaction = on_sigbus
+    act.sa_flags     = SA_SIGINFO
+
+    cdef int err = sigaction(SIGBUS, &act, NULL)
+    if err:
+        PyErr_SetFromErrno(OSError)
+
+cdef void on_sigbus(int sig, siginfo_t *si, void *_uc) nogil:
+    # - wait a bit to give time for other threads to complete their exception dumps
+    #   (e.g. getting "Transport endpoint is not connected" after wcfs.go dying)
+    # - dump py-level traceback and abort.
+    # TODO turn SIGBUS into python-level exception? (see sigpanic in Go how to do).
+
+    writeerr("\nC: SIGBUS received; giving time to other threads "
+             "to dump their exceptions (if any) ...\n")
+    with gil:
+        pass
+    sleep(1)
+
+    writeerr("\nC: SIGBUS'ed thread traceback:\n")
+    with gil:
+        import traceback
+        traceback.print_stack()
+    writeerr("-> SIGBUS\n");
+    # FIXME nothing is printed if pytest stdout/stderr capture is on (no -s given)
+    abort()
+
+# writeerr writes msg to stderr without depending on stdio buffering and locking.
+cdef void writeerr(const char *msg) nogil:
+    xwrite(2, msg)
+
+# xwrite writes msg to fd without depending on stdio buffering and locking.
+cdef void xwrite(int fd, const char *msg) nogil:
+    cdef ssize_t n, left = strlen(msg)
+    while left > 0:
+        n = write(fd, msg, left)
+        if n == -1:
+            panic("write: failed")
+        left -= n
+        msg  += n

wcfs/internal/xzodb/xzodb.go

@@ -21,8 +21,12 @@
 package xzodb
 
 import (
+	"context"
 	"fmt"
 
+	"lab.nexedi.com/kirr/go123/xcontext"
+
+	"lab.nexedi.com/kirr/neo/go/transaction"
 	"lab.nexedi.com/kirr/neo/go/zodb"
 )
 
@@ -42,3 +46,37 @@ func TypeOf(obj interface{}) string {
 		return fmt.Sprintf("%T", obj)
 	}
 }
+
+// ZConn is zodb.Connection + associated read-only transaction under which
+// objects of the connection are accessed.
+type ZConn struct {
+	*zodb.Connection
+
+	// read-only transaction under which we access zodb.Connection data.
+	TxnCtx context.Context // XXX -> better directly store txn
+}
+
+// ZOpen opens new connection to ZODB database + associated read-only transaction.
+func ZOpen(ctx context.Context, zdb *zodb.DB, zopt *zodb.ConnOptions) (_ *ZConn, err error) {
+	// create new read-only transaction
+	txn, txnCtx := transaction.New(context.Background())
+	defer func() {
+		if err != nil {
+			txn.Abort()
+		}
+	}()
+
+	// XXX better ctx = transaction.PutIntoContext(ctx, txn)
+	ctx, cancel := xcontext.Merge(ctx, txnCtx)
+	defer cancel()
+
+	zconn, err := zdb.Open(ctx, zopt)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ZConn{
+		Connection: zconn,
+		TxnCtx:     txnCtx,
+	}, nil
+}

wcfs/misc.go

+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package main
+// misc utilities
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sync/atomic"
+	"syscall"
+
+	log "github.com/golang/glog"
+
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"github.com/hanwen/go-fuse/v2/fuse/nodefs"
+	"github.com/pkg/errors"
+)
+
+// ---- FUSE ----
+
+// eInvalError is the error wrapper signifying that underlying error is about "invalid argument".
+// err2LogStatus converts such errors into EINVAL return code + logs as warning.
+type eInvalError struct {
+	err error
+}
+
+func (e *eInvalError) Error() string {
+	return "invalid argument: " + e.err.Error()
+}
+
+// don't propagate eInvalError.Cause -> e.err
+
+func eINVAL(err error) *eInvalError {
+	return &eInvalError{err}
+}
+
+func eINVALf(format string, argv ...interface{}) *eInvalError {
+	return eINVAL(fmt.Errorf(format, argv...))
+}
+
+// err2LogStatus converts an error into FUSE status code and logs it appropriately.
+//
+// the error is logged because otherwise, if e.g. returning EINVAL or EIO
+// codes, there is no more detail except the error code itself.
+func err2LogStatus(err error) fuse.Status {
+	// no error
+	if err == nil {
+		return fuse.OK
+	}
+
+	// direct usage of error code - don't log
+	ecode, iscode := err.(syscall.Errno)
+	if iscode {
+		return fuse.Status(ecode)
+	}
+
+	// handling canceled -> EINTR, don't log
+	e := errors.Cause(err)
+	switch e {
+	case context.Canceled:
+		return fuse.EINTR
+	case io.ErrClosedPipe:
+		return fuse.Status(syscall.ECONNRESET)
+	}
+
+	// otherwise log as warnings EINVAL and as errors everything else
+	switch e.(type) {
+	case *eInvalError:
+		log.WarningDepth(1, err)
+		return fuse.EINVAL
+
+	default:
+		log.ErrorDepth(1, err)
+		return fuse.EIO
+	}
+}
+
+
+// fsNode should be used instead of nodefs.DefaultNode in wcfs.
+//
+// nodefs.DefaultNode.Open returns ENOSYS. This is convenient for filesystems
+// that have no dynamic files at all. But for filesystems, where there are some
+// dynamic files - i.e. nodes which do need to support Open, returning ENOSYS
+// from any single node will make the kernel think that the filesystem does not
+// support Open at all.
+//
+// In wcfs we have dynamic files (e.g. upcoming /head/watch) and this way we have to
+// avoid returning ENOSYS on nodes, that do not need file handles.
+//
+// fsNode is like nodefs.defaultNode, but by default Open returns to kernel
+// fh=0 and FOPEN_KEEP_CACHE - similarly how openless case is handled there.
+//
+// fsNode behaviour can be additionally controlled via fsOptions.
+//
+// fsNode should be created via newFSNode.
+type fsNode struct {
+	nodefs.Node
+
+	opt *fsOptions
+
+	// cache for path
+	// we don't use hardlinks / don't want to pay locks + traversal price every time.
+	xpath atomic.Value
+}
+
+func (n *fsNode) Open(flags uint32, _ *fuse.Context) (nodefs.File, fuse.Status) {
+	return &nodefs.WithFlags{
+		File:      nil,
+		FuseFlags: fuse.FOPEN_KEEP_CACHE,
+	}, fuse.OK
+}
+
+// fsOptions allows to tune fsNode behaviour.
+type fsOptions struct {
+	// Sticky nodes are not removed from inode tree on FORGET.
+	// Correspondingly OnForget is never called on a sticky node.
+	Sticky bool
+}
+
+var fSticky = &fsOptions{Sticky: true} // frequently used shortcut
+
+func (n *fsNode) Deletable() bool {
+	return !n.opt.Sticky
+}
+
+func newFSNode(opt *fsOptions) fsNode { // NOTE not pointer
+	return fsNode{
+		Node: nodefs.NewDefaultNode(),
+		opt:  opt,
+	}
+}
+
+// path returns node path in its filesystem.
+func (n *fsNode) path() string {
+	xpath := n.xpath.Load()
+	if xpath != nil {
+		return xpath.(string)
+	}
+
+	// slow part - let's construct the path and remember it
+	path := ""
+	inode := n.Inode()
+	for {
+		var name string
+		inode, name = inode.Parent()
+		if inode == nil {
+			break
+		}
+
+		path = "/" + name + path
+	}
+
+	n.xpath.Store(path)
+	return path
+}
+
+
+// NewStaticFile creates nodefs.Node for file with static data.
+//
+// Created file is sticky.
+func NewStaticFile(data []byte) *SmallFile {
+	return newSmallFile(func(_ *fuse.Context) ([]byte, error) {
+		return data, nil
+	}, fuse.FOPEN_KEEP_CACHE /*see ^^^*/)
+}
+
+// SmallFile is a nodefs.Node for file with potentially dynamic, but always small, data.
+type SmallFile struct {
+	fsNode
+	fuseFlags uint32 // fuse.FOPEN_*
+
+	// readData gives whole file data
+	readData func(fctx *fuse.Context) ([]byte, error)
+}
+
+func newSmallFile(readData func(*fuse.Context) ([]byte, error), fuseFlags uint32) *SmallFile {
+	return &SmallFile{
+		fsNode:    newFSNode(&fsOptions{Sticky: true}),
+		fuseFlags: fuseFlags,
+		readData:  readData,
+	}
+}
+
+// NewSmallFile creates nodefs.Node for file with dynamic, but always small, data.
+//
+// Created file is sticky.
+func NewSmallFile(readData func(*fuse.Context) ([]byte, error)) *SmallFile {
+	return newSmallFile(readData, fuse.FOPEN_DIRECT_IO)
+}
+
+func (f *SmallFile) Open(flags uint32, _ *fuse.Context) (nodefs.File, fuse.Status) {
+	return &nodefs.WithFlags{
+		File:      nil,
+		FuseFlags: f.fuseFlags,
+	}, fuse.OK
+}
+
+func (f *SmallFile) GetAttr(out *fuse.Attr, _ nodefs.File, fctx *fuse.Context) fuse.Status {
+	data, err := f.readData(fctx)
+	if err != nil {
+		return err2LogStatus(err)
+	}
+	out.Size = uint64(len(data))
+	out.Mode = fuse.S_IFREG | 0644
+	return fuse.OK
+}
+
+func (f *SmallFile) Read(_ nodefs.File, dest []byte, off int64, fctx *fuse.Context) (fuse.ReadResult, fuse.Status) {
+	data, err := f.readData(fctx)
+	if err != nil {
+		return nil, err2LogStatus(err)
+	}
+	l := int64(len(data))
+	end := off + l
+	if end > l {
+		end = l
+	}
+
+	return fuse.ReadResultData(data[off:end]), fuse.OK
+}
+
+
+// mkdir adds child to parent as directory.
+//
+// Note: parent must be already in the filesystem tree - i.e. associated
+// with Inode. if not - nodefs will panic in Inode.NewChild on nil dereference.
+func mkdir(parent nodefs.Node, name string, child nodefs.Node) {
+	parent.Inode().NewChild(name, true, child)
+}
+
+// mkfile adds child to parent as file.
+//
+// Note: parent must be already in the filesystem tree (see mkdir for details).
+func mkfile(parent nodefs.Node, name string, child nodefs.Node) {
+	parent.Inode().NewChild(name, false, child)
+}
+
+
+// mount is like nodefs.MountRoot but allows to pass in full fuse.MountOptions.
+func mount(mntpt string, root nodefs.Node, opts *fuse.MountOptions) (*fuse.Server, *nodefs.FileSystemConnector, error) {
+	nodefsOpts := nodefs.NewOptions()
+	nodefsOpts.Debug = opts.Debug
+
+	return nodefs.Mount(mntpt, root, opts, nodefsOpts)
+}
+
+
+// ---- make df happy (else it complains "function not supported") ----
+
+func (root *Root) StatFs() *fuse.StatfsOut {
+	return &fuse.StatfsOut{
+		// filesystem sizes (don't try to estimate)
+		Blocks: 0,
+		Bfree:  0,
+		Bavail: 0,
+
+		// do we need to count files?
+		Files: 0,
+		Ffree: 0,
+
+		// block size
+		Bsize:	2*1024*1024, // "optimal transfer block size" XXX better get from root?
+		Frsize: 2*1024*1024, // "fragment size"
+
+		NameLen: 255, // XXX ok? /proc uses the same
+	}
+}
+
+// ---- misc ----
+
+func panicf(format string, argv ...interface{}) {
+	panic(fmt.Sprintf(format, argv...))
+}