Merge branch 't2' into t

* t2:
  *: Cosmetics
  .
  .
  lib/zodb: Mark test_zconn_at as xfail on plain ZODB4
  .
  .
  wcfs: Server.stop: Make sure to remove mount entry even if we had to use FUSE abort
  tests: Don't leak WCFS log files
  tests: Remove test NEO database after test run is over
  nxdtest: Don't run test.go for multiple GOMAXPROCS
  wcfs: Make sure to remove mountpoint directory on Server.stop
  nxdtest: Run WCFS-related tests in verbose mode on testnodes
  setup: Fix egg_info after addition of δbtail.go

.nxdtest

@@ -3,11 +3,6 @@
 
 storv = ['fs', 'zeo', 'neo'] # storage backends to test against
 
-# some bugs are only likely to trigger when there is only 1 or 2 main OS thread(s) in wcfs
-# GOMAXPROCS='' means use `nproc`
-gonprocv = ['1', '2', '']           # GOMAXPROCS=... to test against
-
-
 # test.t & friends unit-test core of UVMM and are Go-, Python- and ZODB-storage independent.
 # we don't run test.vg* because there is currently no valgrind on SlapOS.
 for kind in ['t', 'fault', 'asan', 'tsan']:
@@ -34,20 +29,30 @@ for stor in storv:
 
 
 # test.go unit-tests Go bits in wcfs.
-for nproc in gonprocv:
-    TestCase('test.go:%s' % nproc, ['make', 'test.go'],
-             envadj={'GOMAXPROCS': nproc})
+TestCase('test.go', ['make', 'test.go'])
 
 # test.wcfs/<stor> runs unit tests for WCFS
 # test.py/<stor>-wcfs runs unit- and functional- tests for wendelin.core in wcfs mode.
 for stor in storv:
-    envdb = {'WENDELIN_CORE_TEST_DB': '<%s>' % stor}
+    env = {
+        'WENDELIN_CORE_TEST_DB': '<%s>' % stor,
+        # run in verbose mode and don't capture output in the early days of WCFS
+        #
+        # non-capture is needed because if WCFS gets stuck somehow on testnode, we
+        # still want to see some test output instead of full silence and pytest
+        # being killed after 4 hours of timeout.
+        'PYTEST_ADDOPTS': '-vs',
+    }
+
+    # some bugs are only likely to trigger when there is only 1 or 2 main OS thread(s) in wcfs
+    # GOMAXPROCS='' means use `nproc`
+    gonprocv = ['1', '2', '']           # GOMAXPROCS=... to test against
 
     for nproc in gonprocv:
         TestCase('test.wcfs/%s:%s' % (stor, nproc), ['make', 'test.wcfs'],
-                 envadj=dict(GOMAXPROCS=nproc, **envdb), summaryf=PyTest.summary)
+                 envadj=dict(GOMAXPROCS=nproc, **env), summaryf=PyTest.summary)
 
     for nproc in gonprocv:
         TestCase('test.py/%s-wcfs:%s' % (stor, nproc), ['make', 'test.py'],
-                 envadj=dict(WENDELIN_CORE_VIRTMEM='r:wcfs+w:uvmm', GOMAXPROCS=nproc, **envdb),
+                 envadj=dict(WENDELIN_CORE_VIRTMEM='r:wcfs+w:uvmm', GOMAXPROCS=nproc, **env),
                  summaryf=PyTest.summary)

CHANGELOG.rst

 Wendelin.core change history
 ============================
 
-0.14 aka 2.0.0.dev1 (2020-XX-YY)
---------------------------------
+2.0.alpha1 (2021-11-16)
+-----------------------
 
 This is a major release that speeds up pagefault handling and reduces
 wendelin.core RAM consumption dramatically:
@@ -20,11 +20,41 @@ cache and at the same time get bigfile view isolated from other's changes.
 By default wendelin.core python client continues to provide full ACID semantics as
 before.
 
+In addition to being significantly more efficient, WCFS also fixes
+data-corruption bugs that were discovered__ in how Wendelin.core 1 handles
+invalidations on BTree topology change.
+
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/8c32c9f6
+
 Please see wcfs.go__ for description of the new filesystem.
 
-.. XXX correct link -> nexedi, sha1 - fix.
+__ https://lab.nexedi.com/nexedi/wendelin.core/blob/master/wcfs/wcfs.go
+
+Major steps: 1__, 2__, 3__, 4__, 5__, 6__, 7__, 8__, 9__, 10__, 11__, 12__,
+13__, 14__, 15__, 16__, 17__, 18__, 19__, 20__, 21__.
+
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/2c152d41?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/e3f2ee2d?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/0e829874?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/a8595565?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/b87edcfe?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/1f2cd49d?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/27df5a3b?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/80153aa5?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/2ab4be93?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/f980471f?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/4430de41?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/6f0cdaff?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/10f7153a?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/fae045cc?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/23362204?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/ceadfcc7?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/1dba3a9a?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/1f866c00?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/e11edc70?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/986cf86e?expanded=1
+__ https://lab.nexedi.com/nexedi/wendelin.core/commit/c5e18c74?expanded=1
 
-__ https://lab.nexedi.com/kirr/wendelin.core/blob/t/wcfs/wcfs.go
 
 0.13 (2019-06-18)
 -----------------

Makefile

@@ -20,7 +20,7 @@
 all	:
 
 PYTHON	?= python
-PYTEST	?= $(PYTHON) -m pytest -vs
+PYTEST	?= $(PYTHON) -m pytest
 PYBENCH ?= $(PYTHON) -m golang.cmd.pybench
 VALGRIND?= valgrind
 GO	?= go

README.rst

@@ -81,13 +81,17 @@ limitations and things that need to be improved:
 Thus
 
 - we are currently working on improved wendelin.core design and implementation,
-  which will use kernel virtual memory manager (instead of one implemented__ in__
+  which uses kernel virtual memory manager (complemented by one implemented__ in__
   userspace__) with arrays backend presented to kernel via FUSE as virtual
   filesystem implemented in Go.
 
+  As of 2021 November `this filesystem`__ reached its alpha state and is staged
+  to be tried for real.
+
 __  https://lab.nexedi.com/nexedi/wendelin.core/blob/master/include/wendelin/bigfile/virtmem.h
 __  https://lab.nexedi.com/nexedi/wendelin.core/blob/master/bigfile/virtmem.c
 __  https://lab.nexedi.com/nexedi/wendelin.core/blob/master/bigfile/pagefault.c
+__  https://lab.nexedi.com/nexedi/wendelin.core/blob/master/wcfs/wcfs.go
 
 In parallel we will also:
 

conftest.py

@@ -24,7 +24,7 @@ import pytest
 import transaction
 from golang import func, defer
 from functools import partial
-import gc
+import os, gc
 
 # reset transaction synchronizers before every test run.
 #
@@ -46,9 +46,24 @@ def transaction_reset():
     # nothing to run after test
 
 
+# prepend_env prepends prefix + ' ' to environment variable var.
+def prepend_env(var, prefix):
+    v = os.environ.get(var, '')
+    if v != '':
+        v = ' ' + v
+    v = prefix + v
+    os.environ[var] = v
+
+
 # enable log_cli on no-capture
 # (output during a test is a mixture of print and log)
 def pytest_configure(config):
+    # put WCFS log into stderr instead of to many files in /tmp/wcfs.*.log
+    # this way we don't leak those files and include relevant information in test output
+    #
+    # TODO put WCFS logs into dedicated dir without -v?
+    prepend_env('WENDELIN_CORE_WCFS_OPTIONS', '-logtostderr')
+
     if config.option.capture == "no":
         config.inicfg['log_cli'] = "true"
         assert config.getini("log_cli") is True

lib/testing.py

@@ -292,14 +292,17 @@ class TestDB_NEO(TestDB_Base):
     def __init__(self, dburi):
         super(TestDB_NEO, self).__init__(dburi)
         from neo.tests.functional import NEOCluster
-        self.cluster = NEOCluster(['1'], adapter='SQLite')
+        self.NEOCluster = NEOCluster
 
     def setup(self):
+        self.tmpd = mkdtemp('', 'testdb_neo.')
+        self.cluster = self.NEOCluster(['1'], temp_dir=self.tmpd, adapter='SQLite')
         self.cluster.start()
         self.cluster.expectClusterRunning()
 
     def _teardown(self):
         self.cluster.stop()
+        rmtree(self.tmpd)
 
     def getZODBStorage(self):
         return self.cluster.getZODBStorage()

lib/tests/test_zodb.py

@@ -17,7 +17,7 @@
 #
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
-from wendelin.lib.zodb import LivePersistent, deactivate_btree, dbclose, zconn_at, zstor_2zurl
+from wendelin.lib.zodb import LivePersistent, deactivate_btree, dbclose, zconn_at, zstor_2zurl, zmajor, _zhasNXDPatch
 from wendelin.lib.testing import getTestDB
 from wendelin.lib import testing
 from persistent import Persistent, UPTODATE, GHOST, CHANGED
@@ -238,6 +238,9 @@ def test_deactivate_btree():
 # verify that zconn_at gives correct answer.
 @func
 def test_zconn_at():
+    if zmajor == 4 and not _zhasNXDPatch('conn:MVCC-via-loadBefore-only'):
+        pytest.xfail(reason="zconn_at needs https://lab.nexedi.com/nexedi/ZODB/merge_requests/1 to work on ZODB4")
+
     stor = testdb.getZODBStorage()
     defer(stor.close)
     db  = DB(stor)

lib/zodb.py

@@ -182,12 +182,17 @@ def before2at(before): # -> at
 
 # _zassertHasNXDPatch asserts that ZODB is patched with specified Nexedi-provided patch.
 def _zassertHasNXDPatch(patch, details_link):
-    nxd_patches = getattr(ZODB, 'nxd_patches', set())
-    if patch not in nxd_patches:
+    if not _zhasNXDPatch(patch):
         raise AssertionError(
             "ZODB%s is not patched with required Nexedi patch %r\n\tSee %s for details" %
             (zmajor, patch, details_link))
 
+# _zhasNXDPatch returns whether ZODB is patched with specified Nexedi-provided patch.
+def _zhasNXDPatch(patch):
+    nxd_patches = getattr(ZODB, 'nxd_patches', set())
+    return (patch in nxd_patches)
+
+
 # _zversion returns ZODB version object
 def _zversion():
     dzodb3 = pkg_resources.working_set.find(pkg_resources.Requirement.parse('ZODB3'))

setup.py

@@ -246,11 +246,11 @@ def git_lsfiles(dirname):
     # FIXME dirname is currently ignored
     # XXX non-ascii names, etc
     for _ in runcmd(['git', 'ls-files']).splitlines():
-        yield _.decode('utf8')  # XXX utf8 hardcoded
+        yield _
     # XXX recursive submodules
     for _ in runcmd(['git', 'submodule', 'foreach', '--quiet',  \
                 r'git ls-files | sed "s|\(.*\)\$|$path/\1|g"']).splitlines():
-        yield _.decode('utf8')  # XXX utf8 hardcoded
+        yield _
 
 # if we are in git checkout - inject git_lsfiles to setuptools.file_finders
 # entry-point on the fly.
@@ -290,7 +290,7 @@ libwcfs_h = [
 
 setup(
     name        = 'wendelin.core',
-    version     = '0.14',   # XXX aka 2.0.0.dev1
+    version     = '2.0.alpha1',
     description = 'Out-of-core NumPy arrays',
     long_description = '%s\n----\n\n%s' % (
                             readfile('README.rst'), readfile('CHANGELOG.rst')),

wcfs/__init__.py

@@ -417,6 +417,24 @@ def __stop(wcsrv, ctx, _onstuck):
 
     # unmount and wait for wcfs to exit
     # kill wcfs and abort FUSE connection if clean unmount fails
+    # at the end make sure mount entry and mountpoint directory are removed
+
+    def _():
+        # when stop runs:
+        # - wcsrv could be already `fusermount -u`'ed from outside
+        # - the mountpoint could be also already removed from outside
+        _rmdir_ifexists(wcsrv.mountpoint)
+    defer(_)
+
+    def _():
+        # second unmount, if first unmount failed and we had to abort FUSE connection
+        # -z (lazy) because this one has to succeed, but there could be still
+        # client file descriptors left pointing to the mounted filesystem.
+        if _is_mountpoint(wcsrv.mountpoint):
+            log.warn("-> unmount -z ...")
+            _fuse_unmount(wcsrv.mountpoint, "-z")
+    defer(_)
+
     def _():
         if wcsrv._fuseabort is not None:
             wcsrv._fuseabort.close()
@@ -521,10 +539,20 @@ def _mkdir_p(path, mode=0o777): # -> created(bool)
         return False
     return True
 
+# rmdir if path exists.
+def _rmdir_ifexists(path):
+    try:
+        os.rmdir(path)
+    except OSError as e:
+        if e.errno != ENOENT:
+            raise
+
 # _fuse_unmount calls `fusermount -u` + logs details if unmount failed.
+#
+# Additional options to fusermount can be passed via optv.
 @func
-def _fuse_unmount(mntpt):
-    ret, out = _sysproccallout(["fusermount", "-u", mntpt])
+def _fuse_unmount(mntpt, *optv):
+    ret, out = _sysproccallout(["fusermount", "-u"] + list(optv) + [mntpt])
     if ret != 0:
         # unmount failed, usually due to "device is busy".
         # Log which files are still opened and reraise
@@ -543,7 +571,10 @@ def _fuse_unmount(mntpt):
         defer(_)
 
         out = out.rstrip() # kill trailing \n\n
-        emsg = "fuse_unmount %s: failed: %s" % (mntpt, out)
+        opts = ' '.join(optv)
+        if opts != '':
+            opts += ' '
+        emsg = "fuse_unmount %s%s: failed: %s" % (opts, mntpt, out)
         log.warn(emsg)
         raise RuntimeError("%s\n(more details logged)" % emsg)
 
@@ -713,6 +744,7 @@ def main():
     elif cmd == "stop":
         mntpt = _mntpt_4zurl(zurl)
         _fuse_unmount(mntpt)
+        _rmdir_ifexists(mntpt)
 
     else:
         print("wcfs: unknown command %s" % qq(cmd), file=sys.stderr)

wcfs/wcfs_test.py

@@ -53,7 +53,7 @@ from pytest import raises, fail
 from wendelin.wcfs.internal import io, mm
 from wendelin.wcfs.internal.wcfs_test import _tWCFS, read_exfault_nogil, SegmentationFault, install_sigbus_trap, fadvise_dontneed
 from wendelin.wcfs.client._wcfs import _tpywlinkwrite as _twlinkwrite
-from wendelin.wcfs import _is_mountpoint as is_mountpoint, _procwait as procwait, _ready as ready
+from wendelin.wcfs import _is_mountpoint as is_mountpoint, _procwait as procwait, _ready as ready, _rmdir_ifexists as rmdir_ifexists
 
 
 # setup:
@@ -106,8 +106,7 @@ def teardown_function(f):
     mounted = is_mountpoint(testmntpt)
     if mounted:
         fuse_unmount(testmntpt)
-    if os.path.exists(testmntpt):
-        os.rmdir(testmntpt)
+    rmdir_ifexists(testmntpt)
     with raises(KeyError):
         procmounts_lookup_wcfs(testzurl)
 
@@ -384,7 +383,6 @@ class tWCFS(_tWCFS):
             if is_mountpoint(t.wc.mountpoint):
                 fuse_unmount(t.wc.mountpoint)
             assert not is_mountpoint(t.wc.mountpoint)
-            os.rmdir(t.wc.mountpoint)
         defer(_)
         def _():
             def onstuck():