Merge branch 't' into t2

* t:
  .
  X wcfs: Fix ZSync to close wconn on zdb.close, even if zconn stays alive
  X lib/zodb: Connection += onShutdownCallback
  .
  X wcfs: lsof +D misbehaves - don't use it
  X wcfs: _fuse_unmount: Try first `kill -TERM` before `kill -QUIT` wcfs
  X wcfs: Tune _fuse_unmount to include `fusermount -u` error message into raised exception
  X wcfs: Teach start to start successfully even after unclean wcfs shutdown
  fixup! X wcfs: Run fusermount and friends with /bin:/usr/bin always on path
  X wcfs: Run fusermount and friends with /bin:/usr/bin always on path
  X wcfs: Add start to spawn a Server that can be later stopped  (draft)

conftest.py

@@ -46,8 +46,18 @@ def transaction_reset():
     # nothing to run after test
 
 
-# Before pytest exits, teardown WCFS(s) that we automatically spawned during
-# test runs in bigfile/bigarray/...
+# enable log_cli on no-capture
+# (output during a test is a mixture of print and log)
+def pytest_configure(config):
+    if config.option.capture == "no":
+        config.inicfg['log_cli'] = "true"
+        assert config.getini("log_cli") is True
+        if config.option.verbose:
+            config.inicfg['log_cli_level'] = "INFO"
+
+
+# Before pytest exits, teardown WCFS server(s) that we automatically spawned
+# during test runs in bigfile/bigarray/...
 #
 # If we do not do this, spawned wcfs servers are left running _and_ connected
 # by stdout to nxdtest input - which makes nxdtest to wait for them to exit.
@@ -58,18 +68,13 @@ def pytest_unconfigure(config):
     gc.collect()
 
     from wendelin import wcfs
-    for wc in wcfs._wcstarted:
-        if wc._proc.poll() is not None:
-            continue # this wcfs server already exited
-
+    for wc in wcfs._wcautostarted:
         # NOTE: defer instead of direct call - to call all wc.close if there
         # was multiple wc spawned, and proceeding till the end even if any
         # particular call raises exception.
-        defer(partial(_wcclose, wc))
+        defer(partial(_wcclose_and_stop, wc))
 
-
-def _wcclose(wc):
-    from wendelin.wcfs.wcfs_test import tWCFS
-    print("# unmount/stop wcfs pid%d @ %s" % (wc._proc.pid, wc.mountpoint))
-    twc = tWCFS(wc=wc)
-    twc.close()
+@func
+def _wcclose_and_stop(wc):
+    defer(wc._wcsrv.stop)
+    defer(wc.close)

lib/tests/test_zodb.py

@@ -28,6 +28,7 @@ from BTrees.IOBTree import IOBTree
 import transaction
 from transaction import TransactionManager
 from golang import defer, func
+import weakref, gc
 from pytest import raises
 import pytest; xfail = pytest.mark.xfail
 
@@ -354,6 +355,40 @@ def test_zodb_onresync():
     conn.close()
 
 
+# verify that ZODB.Connection.onShutdownCallback works
+@func
+def test_zodb_onshutdown():
+    stor = testdb.getZODBStorage()
+    defer(stor.close)
+    db  = DB(stor)
+
+    class T:
+        def __init__(t):
+            t.nshutdown = 0
+        def on_connection_shutdown(t):
+            t.nshutdown += 1
+
+    t1 = T()
+    t2 = T()
+
+    # conn1 stays alive outside of db.pool
+    conn1 = db.open()
+    conn1.onShutdownCallback(t1)
+
+    # conn2 stays alive inside db.pool
+    conn2 = db.open()
+    conn2.onShutdownCallback(t2)
+    conn2.close()
+
+    assert t1.nshutdown == 0
+    assert t2.nshutdown == 0
+
+    # db.close triggers conn1 and conn2 shutdown
+    db.close()
+    assert t1.nshutdown == 1
+    assert t2.nshutdown == 1
+
+
 # test that zurl does not change from one open to another storage open.
 def test_zurlstable():
     if not isinstance(testdb, (testing.TestDB_FileStorage, testing.TestDB_ZEO, testing.TestDB_NEO)):

lib/zodb.py

@@ -296,6 +296,30 @@ else:
     raise AssertionError("ZODB3 is not supported anymore")
 
 
+# patch for ZODB.Connection to support callback on after database is closed
+ZODB.Connection.Connection._onShutdownCallbacks = None
+def Connection_onShutdownCallback(self, f):
+    if self._onShutdownCallbacks is None:
+        # NOTE WeakSet does not work for bound methods - they are always created
+        # anew for each obj.method access, and thus will go away almost immediately
+        self._onShutdownCallbacks = WeakSet()
+    self._onShutdownCallbacks.add(f)
+
+assert not hasattr(ZODB.Connection.Connection, 'onShutdownCallback')
+ZODB.Connection.Connection.onShutdownCallback = Connection_onShutdownCallback
+
+_orig_DB_close = ZODB.DB.close
+def _ZDB_close(self):
+    # the same code for ZODB3/4/5
+    @self._connectionMap
+    def _(conn):
+        if conn._onShutdownCallbacks:
+            for f in conn._onShutdownCallbacks:
+                f.on_connection_shutdown()
+    _orig_DB_close(self)
+
+ZODB.DB.close = _ZDB_close
+
 
 # zstor_2zurl converts a ZODB storage to URL to access it.
 def zstor_2zurl(zstor):

wcfs/client/_wczsync.pyx

@@ -58,7 +58,7 @@ cdef wcfs.PyConn pywconnOf(zconn):
     zconn._wcfs_wconn = wconn
 
     # keep wconn view of the database in sync with zconn
-    # wconn and wc (= wconn.wc) will be closed when zconn is garbage-collected
+    # wconn and wc (= wconn.wc) will be closed when zconn is garbage-collected or shutdown via DB.close
     _ZSync(zconn, wconn)
 
     return wconn
@@ -66,8 +66,8 @@ cdef wcfs.PyConn pywconnOf(zconn):
 
 # _ZSync keeps wconn in sync with zconn.
 #
-# wconn will be closed once zconn is destroyed (not closed, which returns it
-# back into DB pool).
+# wconn will be closed once zconn is garbage-collected (not closed, which
+# returns it back into DB pool), or once zconn.db is closed.
 #
 # _ZSync cares itself to stay alive as long as zconn stays alive.
 _zsyncReg   = {} # id(zsync) -> zsync   (protected by GIL)
@@ -79,8 +79,12 @@ class _ZSync:
         #print('ZSync: setup %r <-> %r' % (wconn, zconn))
         assert zconn.opened
         zsync.wconn     = wconn
+        # notify us on zconn GC
         zsync.zconn_ref = weakref.ref(zconn, zsync.on_zconn_dealloc)
+        # notify us on zconn.db.close
+        zconn.onShutdownCallback(zsync)
 
+        # notify us when zconn changes its view of the database
         # NOTE zconn.onOpenCallback is not enough: zconn.at can change even
         # without zconn.close/zconn.open, e.g.:
         # zconn = DB.open(transaction_manager=tm)
@@ -98,8 +102,14 @@ class _ZSync:
         if 1: # = `with gil:`  (GIL already held in python code)
             _zsyncReg[id(zsync)] = zsync
 
-    # .zconn dealloc -> wconn.close; release zsync
-    def on_zconn_dealloc(zsync, _):
+    # _release1 closes .wconn and releases zsync once.
+    def _release1(zsync):
+        # unregister zsync from being kept alive
+        if 1: # = `with gil:`  (see note in __init__)
+            _ = _zsyncReg.pop(id(zsync), None)
+        if _ is None:
+            return # another call already done/is simultaneously doing release1
+
         #print('ZSync: sched break %r <-> .' % (zsync.wconn,))
         # schedule wconn.close() + wconn.wc.close()
         _zsync_wclose_wg.add(1)
@@ -112,9 +122,13 @@ class _ZSync:
         _zsync_releaseq.send(zsync.wconn)
         """
 
-        # unregister zsync from being kept alive
-        if 1: # = `with gil:`  (see note in __init__)
-            del _zsyncReg[id(zsync)]
+    # .zconn dealloc -> wconn.close; release zsync.
+    def on_zconn_dealloc(zsync, _):
+        zsync._release1()
+
+    # DB.close -> wconn.close; release zsync.
+    def on_connection_shutdown(zsync):
+        zsync._release1()
 
     # DB resyncs .zconn onto new database view.
     # -> resync .wconn to updated database view of ZODB connection.

wcfs/client/_wczsync_test.py

@@ -39,12 +39,10 @@ def setup_module():
 def teardown_module():
     testdb.teardown()
 
-# verify that ZSync keeps wconn in sync wrt zconn.
-@func
-def test_zsync():
-    zstor = testdb.getZODBStorage()
-    defer(zstor.close)
 
+# _zsync_setup setups up DB, zconn and wconn _ZSync'ed to zconn.
+@func
+def _zsync_setup(zstor): # -> (db, zconn, wconn)
     zurl = zstor_2zurl(zstor)
 
     # create new DB that we'll precisely control
@@ -53,7 +51,6 @@ def test_zsync():
     at0 = zconn_at(zconn)
     # create wconn
     wc = wcfs.join(zurl)
-    wc_njoin0 = wc._njoin
     wconn = wc.connect(at0)
     assert wconn.at() == at0
     # setup ZSync for wconn <-> zconn; don't keep zsync explicitly referenced
@@ -61,8 +58,63 @@ def test_zsync():
     _ZSync(zconn, wconn)
 
     assert wconn.at() == at0
+    return db, zconn, wconn
+
+
+# verify that ZSync closes wconn when db is closed.
+@func
+def test_zsync_db_close():
+    zstor = testdb.getZODBStorage()
+    defer(zstor.close)
+
+    db, zconn, wconn = _zsync_setup(zstor)
+    defer(wconn.close)
+
+    # close db -> ZSync should close wconn and wc even though zconn stays referenced
+    wc_njoin0 = wconn.wc._njoin
+    db.close()
+    _zsync_wclose_wg.wait()
+    # NOTE db and zconn are still alive - not GC'ed
+    with raises(error, match=": connection closed"):
+        wconn.open(p64(0))
+    assert wconn.wc._njoin == (wc_njoin0 - 1)
+
+
+# verify that ZSync closes wconn when zconn is garbage-collected.
+@func
+def test_zsync_zconn_gc():
+    zstor = testdb.getZODBStorage()
+    defer(zstor.close)
+
+    db, zconn, wconn = _zsync_setup(zstor)
+    defer(wconn.close)
+
+    # del zconn -> zconn should disappear and ZSync should close wconn and wc
+    zconn_weak = weakref.ref(zconn)
+    assert zconn_weak() is not None
+    wc_njoin0 = wconn.wc._njoin
+    del zconn
+    # NOTE db stays alive and not closed
+    gc.collect()
+    assert zconn_weak() is None
+    _zsync_wclose_wg.wait()
+    with raises(error, match=": connection closed"):
+        wconn.open(p64(0))
+    assert wconn.wc._njoin == (wc_njoin0 - 1)
+
+
+# verify that ZSync keeps wconn in sync wrt zconn.
+@func
+def test_zsync_resync():
+    zstor = testdb.getZODBStorage()
+    defer(zstor.close)
+
+    db, zconn, wconn = _zsync_setup(zstor)
+    defer(db.close)
 
     # commit something - ZSync should resync wconn to updated db state
+    at0 = zconn_at(zconn)
+    assert wconn.at() == at0
     root = zconn.root()
     root['tzync'] = 1
     transaction.commit()
@@ -97,13 +149,3 @@ def test_zsync():
     assert zconn_weak() is zconn
     assert zconn_at(zconn) == at2
     assert wconn.at() == at2
-
-    # close db -> zconn should disappear and ZSync should close wconn and wc
-    del zconn
-    db.close()
-    gc.collect()
-    assert zconn_weak() is None
-    _zsync_wclose_wg.wait()
-    with raises(error, match=": connection closed"):
-        wconn.open(p64(0))
-    assert wc._njoin == wc_njoin0 - 1

wcfs/client/client_test.py

@@ -29,7 +29,8 @@ from __future__ import print_function, absolute_import
 
 from golang import func, defer, error, b
 from wendelin.bigfile.file_zodb import ZBigFile
-from wendelin.wcfs.wcfs_test import tDB, tAt, timeout, waitfor_, eprint
+from wendelin.wcfs.wcfs_test import tDB, tAt, timeout, eprint
+from wendelin.wcfs import _waitfor_ as waitfor_
 from wendelin.wcfs import wcfs_test
 from wendelin.wcfs.internal.wcfs_test import read_mustfault
 from wendelin.wcfs.internal import mm

wcfs/internal/wcfs_test.pyx

@@ -53,9 +53,8 @@ cdef class _tWCFS:
     # but pin handler is failing one way or another - select will wake-up
     # but, if _abort_ontimeout uses GIL, won't continue to run trying to lock
     # GIL -> deadlock.
-    def _abort_ontimeout(_tWCFS t, double dt, pychan nogilready not None):
+    def _abort_ontimeout(_tWCFS t, int fdabort, double dt, pychan nogilready not None):
         cdef chan[double] timeoutch = time.after(dt)
-        cdef int fdabort = t._wcfuseabort.fileno()
         emsg1 = "\nC: test timed out after %.1fs\n" % (dt / time.second)
         cdef char *_emsg1 = emsg1
         with nogil: