*: Use defer for dbclose & friends

For tests this makes sure that if one test fails, it won't make following
tests fail just because the next test will fail trying to lock test database.

For regular code (demo_zbigarray.py) this is also a good thing to do -
to always close the database irregardless of whether an exception was
raised before program reached end of main.

Pygolang becomes regular - not test only - dependency. Being regular
dependency is currently required only by demo_zbigarray.py, but it will
be also used in upcoming wcfs, so adding pygolang into wendelin.core
dependencies aligns with the plan.

dbclose now uses defer almost everywhere - there are still few places in
tests, where one test function is opening/closing test database multiple
times - those were not (yet ?) converted.

bigarray/tests/test_arrayzodb.py

@@ -27,6 +27,7 @@ import transaction
 from transaction import TransactionManager
 from ZODB.POSException import ConflictError
 from numpy import dtype, uint8, all, array_equal, arange
+from golang import defer, func
 from threading import Thread
 from six.moves import _thread
 
@@ -240,12 +241,14 @@ def test_zbigarray_order():
 
 # the same as test_bigfile_filezodb_vs_conn_migration but explicitly for ZBigArray
 # ( NOTE this test is almost dup of test_zbigarray_vs_conn_migration() )
+@func
 def test_zbigarray_vs_conn_migration():
     root01 = testdb.dbopen()
     conn01 = root01._p_jar
     db     = conn01.db()
     conn01.close()
     del root01
+    defer(db.close)
 
     c12_1 = NotifyChannel()   # T11 -> T21
     c21_1 = NotifyChannel()   # T21 -> T11
@@ -342,12 +345,14 @@ def test_zbigarray_vs_conn_migration():
     c21_2 = NotifyChannel()   # T22 -> T12
 
     # open, abort
+    @func
     def T12():
         tell, wait = c12_2.tell, c21_2.wait
 
         wait('T2-conn22-opened')
 
         conn12 = db.open()
+        defer(conn12.close)
 
         tell('T1-conn12-opened')
         wait('T2-zarray2-modified')
@@ -357,10 +362,9 @@ def test_zbigarray_vs_conn_migration():
         tell('T1-txn-aborted')
         wait('T2-txn-committed')
 
-        conn12.close()
-
 
     # open, modify, commit
+    @func
     def T22():
         tell, wait = c21_2.tell, c12_2.wait
 
@@ -369,6 +373,7 @@ def test_zbigarray_vs_conn_migration():
         assert _thread.get_ident() != t11_ident
 
         conn22 = db.open()
+        defer(conn22.close)
         assert conn22 is conn01
         tell('T2-conn22-opened')
 
@@ -388,8 +393,6 @@ def test_zbigarray_vs_conn_migration():
 
         tell('T2-txn-committed')
 
-        conn22.close()
-
 
     t12, t22 = Thread(target=T12), Thread(target=T22)
     t12.start(); t22.start()
@@ -401,6 +404,7 @@ def test_zbigarray_vs_conn_migration():
 
     # now verify that zarray2 changed to 22 state, i.e. T22 was really committed
     conn03 = db.open()
+    defer(conn03.close)
     # NOTE top of connection stack is conn22(=conn01), becase it has most # of
     # active objectd
     assert conn03 is conn01
@@ -410,23 +414,25 @@ def test_zbigarray_vs_conn_migration():
     assert a03[0] == 22
 
     del a03
-    dbclose(root03)
 
 
 # underlying ZBigFile/ZBigFileH should properly handle 'invalidate' messages from DB
 # ( NOTE this test is almost dup of test_zbigarray_vs_cache_invalidation() )
+@func
 def test_zbigarray_vs_cache_invalidation():
     root = testdb.dbopen()
     conn = root._p_jar
     db   = conn.db()
     conn.close()
     del root, conn
+    defer(db.close)
 
     tm1 = TransactionManager()
     tm2 = TransactionManager()
 
     conn1 = db.open(transaction_manager=tm1)
     root1 = conn1.root()
+    defer(conn1.close)
 
     # setup zarray
     root1['zarray3'] = a1 = ZBigArray((10,), uint8)
@@ -440,6 +446,7 @@ def test_zbigarray_vs_cache_invalidation():
     # read zarray in conn2
     conn2 = db.open(transaction_manager=tm2)
     root2 = conn2.root()
+    defer(conn2.close)
 
     a2 = root2['zarray3']
     assert a2[0:1] == [1]   # read data in conn2 + make sure read correctly
@@ -465,25 +472,26 @@ def test_zbigarray_vs_cache_invalidation():
     # data from tm1 should propagate -> ZODB -> ram pages for _ZBigFileH in conn2
     assert a2[0] == 2
 
-    conn2.close()
     del conn2, root2
-    dbclose(root1)
 
 
 # verify that conflicts on array content are handled properly
 # ( NOTE this test is almost dup of test_bigfile_filezodb_vs_conflicts() )
+@func
 def test_zbigarray_vs_conflicts():
     root = testdb.dbopen()
     conn = root._p_jar
     db   = conn.db()
     conn.close()
     del root, conn
+    defer(db.close)
 
     tm1 = TransactionManager()
     tm2 = TransactionManager()
 
     conn1 = db.open(transaction_manager=tm1)
     root1 = conn1.root()
+    defer(conn1.close)
 
     # setup zarray
     root1['zarray3a'] = a1 = ZBigArray((10,), uint8)
@@ -496,6 +504,7 @@ def test_zbigarray_vs_conflicts():
     # read zarray in conn2
     conn2 = db.open(transaction_manager=tm2)
     root2 = conn2.root()
+    defer(conn2.close)
 
     a2 = root2['zarray3a']
     assert a2[0:1] == [1]   # read data in conn2 + make sure read correctly
@@ -525,24 +534,24 @@ def test_zbigarray_vs_conflicts():
 
     assert a1[0:1] == [13]  # re-read in conn1  XXX -> [0] == 13
 
-    conn2.close()
-    dbclose(root1)
-
 
 # verify that conflicts on array metadata are handled properly
 # ( NOTE this test is close to test_zbigarray_vs_conflicts() )
+@func
 def test_zbigarray_vs_conflicts_metadata():
     root = testdb.dbopen()
     conn = root._p_jar
     db   = conn.db()
     conn.close()
     del root, conn
+    defer(db.close)
 
     tm1 = TransactionManager()
     tm2 = TransactionManager()
 
     conn1 = db.open(transaction_manager=tm1)
     root1 = conn1.root()
+    defer(conn1.close)
 
     # setup zarray
     root1['zarray3b'] = a1 = ZBigArray((10,), uint8)
@@ -555,6 +564,7 @@ def test_zbigarray_vs_conflicts_metadata():
     # read zarray in conn2
     conn2 = db.open(transaction_manager=tm2)
     root2 = conn2.root()
+    defer(conn2.close)
 
     a2 = root2['zarray3b']
     assert a2[0:1] == [1]   # read data in conn2 + make sure read correctly
@@ -584,17 +594,16 @@ def test_zbigarray_vs_conflicts_metadata():
 
     assert len(a1) == 13    # re-read in conn1
 
-    conn2.close()
-    dbclose(root1)
-
 
 # verify how ZBigArray behaves when plain properties are changed / invalidated
+@func
 def test_zbigarray_invalidate_shape():
     root = testdb.dbopen()
     conn = root._p_jar
     db   = conn.db()
     conn.close()
     del root, conn
+    defer(db.close)
 
     print
     tm1 = TransactionManager()
@@ -602,6 +611,7 @@ def test_zbigarray_invalidate_shape():
 
     conn1 = db.open(transaction_manager=tm1)
     root1 = conn1.root()
+    defer(conn1.close)
 
     # setup zarray
     root1['zarray4'] = a1 = ZBigArray((10,), uint8)
@@ -614,6 +624,7 @@ def test_zbigarray_invalidate_shape():
     # read zarray in conn2
     conn2 = db.open(transaction_manager=tm2)
     root2 = conn2.root()
+    defer(conn2.close)
 
     a2 = root2['zarray4']
     assert a2[0:1] == [1]   # read data in conn2 + make sure read correctly
@@ -632,6 +643,4 @@ def test_zbigarray_invalidate_shape():
     assert a2[10:11] == [123]   # XXX -> [10] = 123  after BigArray can
 
 
-    conn2.close()
     del conn2, root2, a2
-    dbclose(root1)

bigfile/tests/bench_1filezodb.py

 # Wendelin.bigfile | benchmarks for zodb backend
-# Copyright (C) 2014-2015  Nexedi SA and Contributors.
+# Copyright (C) 2014-2019  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -24,6 +24,7 @@ from wendelin.lib.mem import memset
 from wendelin.lib.testing import getTestDB, Adler32, nulladler32_bysize, ffadler32_bysize
 from wendelin.lib.zodb import dbclose
 import transaction
+from pygolang import defer, func
 
 testdb = None
 from wendelin.bigfile.tests.bench_0virtmem import filesize, blksize # to get comparable timings
@@ -51,8 +52,10 @@ def teardown_module():
 # NOTE runs before _writeff
 def bench_bigz_readhole():  _bench_bigz_hash(Adler32,   nulladler32)
 
+@func
 def bench_bigz_writeff():
     root = testdb.dbopen()
+    defer(lambda: dbclose(root))
     f   = root['zfile']
     fh  = f.fileh_open()    # TODO + ram
     vma = fh.mmap(0, blen)  # XXX assumes blksize == pagesize
@@ -63,11 +66,12 @@ def bench_bigz_writeff():
     del vma # TODO vma.close()
     del fh  # TODO fh.close()
     del f   # XXX  f.close() ?
-    dbclose(root)
 
 
+@func
 def _bench_bigz_hash(hasher, expect):
     root = testdb.dbopen()
+    defer(lambda: dbclose(root))
     f   = root['zfile']
     fh  = f.fileh_open()    # TODO + ram
     vma = fh.mmap(0, blen)  # XXX assumes blksize == pagesize
@@ -78,7 +82,6 @@ def _bench_bigz_hash(hasher, expect):
     del vma # vma.close()
     del fh  # fh.close()
     del f   # f.close()
-    dbclose(root)
     assert h.digest() == expect
 
 

bigfile/tests/test_filezodb.py

@@ -27,6 +27,7 @@ import transaction
 from transaction import TransactionManager
 from ZODB.POSException import ConflictError
 from numpy import ndarray, array_equal, uint32, zeros, arange
+from golang import defer, func
 from threading import Thread
 from six.moves import _thread
 from six import b
@@ -74,6 +75,7 @@ def cacheInfo(db):
 def kkey(klass):
     return '%s.%s' % (klass.__module__, klass.__name__)
 
+@func
 def test_livepersistent():
     root = dbopen()
     transaction.commit()    # set root._p_jar
@@ -152,10 +154,12 @@ def test_livepersistent():
 
     conn1 = db.open(transaction_manager=tm1)
     root1 = conn1.root()
+    defer(lambda: dbclose(root1))
     lp1 = root1['live']
 
     conn2 = db.open(transaction_manager=tm2)
     root2 = conn2.root()
+    defer(conn2.close)
     lp2 = root2['live']
 
     # 2 connections are setup running in parallel with initial obj state as ghost
@@ -202,9 +206,7 @@ def test_livepersistent():
     assert a == 1
 
 
-    conn2.close()
     del conn2, root2
-    dbclose(root1)
 
 
 
@@ -349,12 +351,14 @@ def test_bigfile_filezodb():
 # connection can migrate between threads handling requests.
 # verify _ZBigFileH properly adjusts.
 # ( NOTE this test is almost dupped at test_zbigarray_vs_conn_migration() )
+@func
 def test_bigfile_filezodb_vs_conn_migration():
     root01 = dbopen()
     conn01 = root01._p_jar
     db     = conn01.db()
     conn01.close()
     del root01
+    defer(db.close)
 
     c12_1 = NotifyChannel()   # T11 -> T21
     c21_1 = NotifyChannel()   # T21 -> T11
@@ -533,6 +537,7 @@ def test_bigfile_filezodb_vs_conn_migration():
 
     # now verify that zfile2 changed to 22 state, i.e. T22 was really committed
     conn03 = db.open()
+    defer(conn03.close)
     # NOTE top of connection stack is conn22(=conn01), because it has most # of
     # active objects
     assert conn03 is conn01
@@ -545,23 +550,25 @@ def test_bigfile_filezodb_vs_conn_migration():
     assert Blk(vma03, 0)[0] == 22
 
     del vma03, fh03, f03
-    dbclose(root03)
 
 
 # ZBlk should properly handle 'invalidate' messages from DB
 # ( NOTE this test is almost dupped at test_zbigarray_vs_cache_invalidation() )
+@func
 def test_bigfile_filezodb_vs_cache_invalidation():
     root = dbopen()
     conn = root._p_jar
     db   = conn.db()
     conn.close()
     del root, conn
+    defer(db.close)
 
     tm1 = TransactionManager()
     tm2 = TransactionManager()
 
     conn1 = db.open(transaction_manager=tm1)
     root1 = conn1.root()
+    defer(conn1.close)
 
     # setup zfile with fileh view to it
     root1['zfile3'] = f1 = ZBigFile(blksize)
@@ -579,6 +586,7 @@ def test_bigfile_filezodb_vs_cache_invalidation():
     # read zfile and setup fileh for it in conn2
     conn2 = db.open(transaction_manager=tm2)
     root2 = conn2.root()
+    defer(conn2.close)
 
     f2 = root2['zfile3']
     fh2 = f2.fileh_open()
@@ -606,25 +614,26 @@ def test_bigfile_filezodb_vs_cache_invalidation():
     # data from tm1 should propagate -> ZODB -> ram pages for _ZBigFileH in conn2
     assert Blk(vma2, 0)[0] == 2
 
-    conn2.close()
     del conn2, root2
-    dbclose(root1)
 
 
 # verify that conflicts on ZBlk are handled properly
 # ( NOTE this test is almost dupped at test_zbigarray_vs_conflicts() )
+@func
 def test_bigfile_filezodb_vs_conflicts():
     root = dbopen()
     conn = root._p_jar
     db   = conn.db()
     conn.close()
     del root, conn
+    defer(db.close)
 
     tm1 = TransactionManager()
     tm2 = TransactionManager()
 
     conn1 = db.open(transaction_manager=tm1)
     root1 = conn1.root()
+    defer(conn1.close)
 
     # setup zfile with fileh view to it
     root1['zfile3a'] = f1 = ZBigFile(blksize)
@@ -641,6 +650,7 @@ def test_bigfile_filezodb_vs_conflicts():
     # read zfile and setup fileh for it in conn2
     conn2 = db.open(transaction_manager=tm2)
     root2 = conn2.root()
+    defer(conn2.close)
 
     f2 = root2['zfile3a']
     fh2 = f2.fileh_open()
@@ -672,16 +682,15 @@ def test_bigfile_filezodb_vs_conflicts():
 
     assert Blk(vma1, 0)[0] == 13    # re-read in conn1
 
-    conn2.close()
-    dbclose(root1)
-
 
 
 # verify that fileh are garbage-collected after user free them
+@func
 def test_bigfile_filezodb_fileh_gc():
     root1= dbopen()
     conn1= root1._p_jar
     db   = conn1.db()
+    defer(db.close)
     root1['zfile4'] = f1 = ZBigFile(blksize)
     transaction.commit()
 
@@ -696,6 +705,7 @@ def test_bigfile_filezodb_fileh_gc():
 
     conn2 = db.open()
     root2 = conn2.root()
+    defer(conn2.close)
     f2 = root2['zfile4']
 
     fh2  = f2.fileh_open()
@@ -705,12 +715,13 @@ def test_bigfile_filezodb_fileh_gc():
     assert wfh1() is None   # fh1 should be gone
 
     del vma2, fh2, f2
-    dbclose(root2)
 
 
 # verify how zblk format change works
+@func
 def test_bigfile_filezodb_fmt_change():
     root = dbopen()
+    defer(lambda: dbclose(root))
     root['zfile5'] = f = ZBigFile(blksize)
     transaction.commit()
 
@@ -743,8 +754,6 @@ def test_bigfile_filezodb_fmt_change():
     finally:
         file_zodb.ZBlk_fmt_write = fmt_write_save
 
-    dbclose(root)
-
 
 # test that ZData are reused for changed chunks in ZBlk1 format
 def test_bigfile_zblk1_zdata_reuse():
@@ -756,8 +765,10 @@ def test_bigfile_zblk1_zdata_reuse():
     finally:
         file_zodb.ZBlk_fmt_write = fmt_write_save
 
+@func
 def _test_bigfile_zblk1_zdata_reuse():
     root = dbopen()
+    defer(lambda: dbclose(root))
     root['zfile6'] = f = ZBigFile(blksize)
     transaction.commit()
 
@@ -791,5 +802,3 @@ def _test_bigfile_zblk1_zdata_reuse():
     assert len(zdata_v1) == len(zdata_v2)
     for i in range(len(zdata_v1)):
         assert zdata_v1[i] is zdata_v2[i]
-
-    dbclose(root)

demo/demo_zbigarray.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2014-2017  Nexedi SA and Contributors.
+# Copyright (C) 2014-2019  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -37,6 +37,7 @@ from __future__ import print_function
 from wendelin.bigarray.array_zodb import ZBigArray
 from wendelin.lib.zodb import dbopen, dbclose
 import transaction
+from golang import defer, func
 
 from numpy import float64, dtype, cumsum, sin
 import psutil
@@ -96,7 +97,7 @@ options:
 """ % sys.argv[0], file=sys.stderr)
     sys.exit(1)
 
-
+@func
 def main():
     worksize = None
     optv, argv = getopt.getopt(sys.argv[1:], '', ['worksize='])
@@ -120,6 +121,7 @@ def main():
     print('I: RAM:  %.2fGB' % (float(ram_nbytes) / GB))
 
     root = dbopen(dburi)
+    defer(lambda: dbclose(root))
 
     if act == 'gen':
         if worksize is None:
@@ -145,8 +147,6 @@ def main():
     m = p.memory_info()
     print('VIRT: %i MB\tRSS: %iMB' % (m.vms//MB, m.rss//MB))
 
-    dbclose(root)
-
 
 if __name__ == '__main__':
     main()

lib/tests/test_zodb.py

 # Wendelin.core.bigfile | Tests for ZODB utilities
-# Copyright (C) 2014-2016  Nexedi SA and Contributors.
+# Copyright (C) 2014-2019  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -22,6 +22,7 @@ from wendelin.lib.testing import getTestDB
 from persistent import Persistent, UPTODATE, GHOST
 from BTrees.IOBTree import IOBTree
 import transaction
+from golang import defer, func
 import gc
 
 testdb = None
@@ -45,8 +46,11 @@ class XInt(Persistent):
 def objscachedv(jar):
     return [obj for oid, obj in jar._cache.lru_items()]
 
+@func
 def test_deactivate_btree():
     root = dbopen()
+    defer(lambda: dbclose(root))
+
     # init btree with many leaf nodes
     leafv = []
     root['btree'] = B = IOBTree()
@@ -81,5 +85,3 @@ def test_deactivate_btree():
         for obj in [B] + leafv:
             assert obj._p_state == GHOST
             assert obj not in cached
-
-    dbclose(root)

setup.py

 # Wendelin.core | pythonic package setup
-# Copyright (C) 2014-2015  Nexedi SA and Contributors.
+# Copyright (C) 2014-2019  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -243,13 +243,15 @@ setup(
                    #   specify either to use e.g. ZODB3.10 or ZODB4 )
                    'ZODB3 >= 3.10',
 
+                   'pygolang >= 0.0.2', # defer, sync.WaitGroup, ...
+
                    'six',       # compat py2/py3
 
                    'psutil',    # demo_zbigarray
                   ],
 
     extras_require = {
-                   'test': ['pytest', 'pygolang >= 0.0.0.dev4'],
+                   'test': ['pytest'],
     },
 
     cmdclass    = {'build_ext':     build_ext,