Commit 5c8340d2 authored by Kirill Smelkov's avatar Kirill Smelkov

*: Use defer for dbclose & friends

For tests this makes sure that if one test fails, it won't make following
tests fail just because the next test will fail trying to lock test database.

For regular code (demo_zbigarray.py) this is also a good thing to do -
to always close the database irregardless of whether an exception was
raised before program reached end of main.

Pygolang becomes regular - not test only - dependency. Being regular
dependency is currently required only by demo_zbigarray.py, but it will
be also used in upcoming wcfs, so adding pygolang into wendelin.core
dependencies aligns with the plan.

dbclose now uses defer almost everywhere - there are still few places in
tests, where one test function is opening/closing test database multiple
times - those were not (yet ?) converted.
parent b12e319e
...@@ -27,6 +27,7 @@ import transaction ...@@ -27,6 +27,7 @@ import transaction
from transaction import TransactionManager from transaction import TransactionManager
from ZODB.POSException import ConflictError from ZODB.POSException import ConflictError
from numpy import dtype, uint8, all, array_equal, arange from numpy import dtype, uint8, all, array_equal, arange
from golang import defer, func
from threading import Thread from threading import Thread
from six.moves import _thread from six.moves import _thread
...@@ -240,12 +241,14 @@ def test_zbigarray_order(): ...@@ -240,12 +241,14 @@ def test_zbigarray_order():
# the same as test_bigfile_filezodb_vs_conn_migration but explicitly for ZBigArray # the same as test_bigfile_filezodb_vs_conn_migration but explicitly for ZBigArray
# ( NOTE this test is almost dup of test_zbigarray_vs_conn_migration() ) # ( NOTE this test is almost dup of test_zbigarray_vs_conn_migration() )
@func
def test_zbigarray_vs_conn_migration(): def test_zbigarray_vs_conn_migration():
root01 = testdb.dbopen() root01 = testdb.dbopen()
conn01 = root01._p_jar conn01 = root01._p_jar
db = conn01.db() db = conn01.db()
conn01.close() conn01.close()
del root01 del root01
defer(db.close)
c12_1 = NotifyChannel() # T11 -> T21 c12_1 = NotifyChannel() # T11 -> T21
c21_1 = NotifyChannel() # T21 -> T11 c21_1 = NotifyChannel() # T21 -> T11
...@@ -342,12 +345,14 @@ def test_zbigarray_vs_conn_migration(): ...@@ -342,12 +345,14 @@ def test_zbigarray_vs_conn_migration():
c21_2 = NotifyChannel() # T22 -> T12 c21_2 = NotifyChannel() # T22 -> T12
# open, abort # open, abort
@func
def T12(): def T12():
tell, wait = c12_2.tell, c21_2.wait tell, wait = c12_2.tell, c21_2.wait
wait('T2-conn22-opened') wait('T2-conn22-opened')
conn12 = db.open() conn12 = db.open()
defer(conn12.close)
tell('T1-conn12-opened') tell('T1-conn12-opened')
wait('T2-zarray2-modified') wait('T2-zarray2-modified')
...@@ -357,10 +362,9 @@ def test_zbigarray_vs_conn_migration(): ...@@ -357,10 +362,9 @@ def test_zbigarray_vs_conn_migration():
tell('T1-txn-aborted') tell('T1-txn-aborted')
wait('T2-txn-committed') wait('T2-txn-committed')
conn12.close()
# open, modify, commit # open, modify, commit
@func
def T22(): def T22():
tell, wait = c21_2.tell, c12_2.wait tell, wait = c21_2.tell, c12_2.wait
...@@ -369,6 +373,7 @@ def test_zbigarray_vs_conn_migration(): ...@@ -369,6 +373,7 @@ def test_zbigarray_vs_conn_migration():
assert _thread.get_ident() != t11_ident assert _thread.get_ident() != t11_ident
conn22 = db.open() conn22 = db.open()
defer(conn22.close)
assert conn22 is conn01 assert conn22 is conn01
tell('T2-conn22-opened') tell('T2-conn22-opened')
...@@ -388,8 +393,6 @@ def test_zbigarray_vs_conn_migration(): ...@@ -388,8 +393,6 @@ def test_zbigarray_vs_conn_migration():
tell('T2-txn-committed') tell('T2-txn-committed')
conn22.close()
t12, t22 = Thread(target=T12), Thread(target=T22) t12, t22 = Thread(target=T12), Thread(target=T22)
t12.start(); t22.start() t12.start(); t22.start()
...@@ -401,6 +404,7 @@ def test_zbigarray_vs_conn_migration(): ...@@ -401,6 +404,7 @@ def test_zbigarray_vs_conn_migration():
# now verify that zarray2 changed to 22 state, i.e. T22 was really committed # now verify that zarray2 changed to 22 state, i.e. T22 was really committed
conn03 = db.open() conn03 = db.open()
defer(conn03.close)
# NOTE top of connection stack is conn22(=conn01), becase it has most # of # NOTE top of connection stack is conn22(=conn01), becase it has most # of
# active objectd # active objectd
assert conn03 is conn01 assert conn03 is conn01
...@@ -410,23 +414,25 @@ def test_zbigarray_vs_conn_migration(): ...@@ -410,23 +414,25 @@ def test_zbigarray_vs_conn_migration():
assert a03[0] == 22 assert a03[0] == 22
del a03 del a03
dbclose(root03)
# underlying ZBigFile/ZBigFileH should properly handle 'invalidate' messages from DB # underlying ZBigFile/ZBigFileH should properly handle 'invalidate' messages from DB
# ( NOTE this test is almost dup of test_zbigarray_vs_cache_invalidation() ) # ( NOTE this test is almost dup of test_zbigarray_vs_cache_invalidation() )
@func
def test_zbigarray_vs_cache_invalidation(): def test_zbigarray_vs_cache_invalidation():
root = testdb.dbopen() root = testdb.dbopen()
conn = root._p_jar conn = root._p_jar
db = conn.db() db = conn.db()
conn.close() conn.close()
del root, conn del root, conn
defer(db.close)
tm1 = TransactionManager() tm1 = TransactionManager()
tm2 = TransactionManager() tm2 = TransactionManager()
conn1 = db.open(transaction_manager=tm1) conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root() root1 = conn1.root()
defer(conn1.close)
# setup zarray # setup zarray
root1['zarray3'] = a1 = ZBigArray((10,), uint8) root1['zarray3'] = a1 = ZBigArray((10,), uint8)
...@@ -440,6 +446,7 @@ def test_zbigarray_vs_cache_invalidation(): ...@@ -440,6 +446,7 @@ def test_zbigarray_vs_cache_invalidation():
# read zarray in conn2 # read zarray in conn2
conn2 = db.open(transaction_manager=tm2) conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root() root2 = conn2.root()
defer(conn2.close)
a2 = root2['zarray3'] a2 = root2['zarray3']
assert a2[0:1] == [1] # read data in conn2 + make sure read correctly assert a2[0:1] == [1] # read data in conn2 + make sure read correctly
...@@ -465,25 +472,26 @@ def test_zbigarray_vs_cache_invalidation(): ...@@ -465,25 +472,26 @@ def test_zbigarray_vs_cache_invalidation():
# data from tm1 should propagate -> ZODB -> ram pages for _ZBigFileH in conn2 # data from tm1 should propagate -> ZODB -> ram pages for _ZBigFileH in conn2
assert a2[0] == 2 assert a2[0] == 2
conn2.close()
del conn2, root2 del conn2, root2
dbclose(root1)
# verify that conflicts on array content are handled properly # verify that conflicts on array content are handled properly
# ( NOTE this test is almost dup of test_bigfile_filezodb_vs_conflicts() ) # ( NOTE this test is almost dup of test_bigfile_filezodb_vs_conflicts() )
@func
def test_zbigarray_vs_conflicts(): def test_zbigarray_vs_conflicts():
root = testdb.dbopen() root = testdb.dbopen()
conn = root._p_jar conn = root._p_jar
db = conn.db() db = conn.db()
conn.close() conn.close()
del root, conn del root, conn
defer(db.close)
tm1 = TransactionManager() tm1 = TransactionManager()
tm2 = TransactionManager() tm2 = TransactionManager()
conn1 = db.open(transaction_manager=tm1) conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root() root1 = conn1.root()
defer(conn1.close)
# setup zarray # setup zarray
root1['zarray3a'] = a1 = ZBigArray((10,), uint8) root1['zarray3a'] = a1 = ZBigArray((10,), uint8)
...@@ -496,6 +504,7 @@ def test_zbigarray_vs_conflicts(): ...@@ -496,6 +504,7 @@ def test_zbigarray_vs_conflicts():
# read zarray in conn2 # read zarray in conn2
conn2 = db.open(transaction_manager=tm2) conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root() root2 = conn2.root()
defer(conn2.close)
a2 = root2['zarray3a'] a2 = root2['zarray3a']
assert a2[0:1] == [1] # read data in conn2 + make sure read correctly assert a2[0:1] == [1] # read data in conn2 + make sure read correctly
...@@ -525,24 +534,24 @@ def test_zbigarray_vs_conflicts(): ...@@ -525,24 +534,24 @@ def test_zbigarray_vs_conflicts():
assert a1[0:1] == [13] # re-read in conn1 XXX -> [0] == 13 assert a1[0:1] == [13] # re-read in conn1 XXX -> [0] == 13
conn2.close()
dbclose(root1)
# verify that conflicts on array metadata are handled properly # verify that conflicts on array metadata are handled properly
# ( NOTE this test is close to test_zbigarray_vs_conflicts() ) # ( NOTE this test is close to test_zbigarray_vs_conflicts() )
@func
def test_zbigarray_vs_conflicts_metadata(): def test_zbigarray_vs_conflicts_metadata():
root = testdb.dbopen() root = testdb.dbopen()
conn = root._p_jar conn = root._p_jar
db = conn.db() db = conn.db()
conn.close() conn.close()
del root, conn del root, conn
defer(db.close)
tm1 = TransactionManager() tm1 = TransactionManager()
tm2 = TransactionManager() tm2 = TransactionManager()
conn1 = db.open(transaction_manager=tm1) conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root() root1 = conn1.root()
defer(conn1.close)
# setup zarray # setup zarray
root1['zarray3b'] = a1 = ZBigArray((10,), uint8) root1['zarray3b'] = a1 = ZBigArray((10,), uint8)
...@@ -555,6 +564,7 @@ def test_zbigarray_vs_conflicts_metadata(): ...@@ -555,6 +564,7 @@ def test_zbigarray_vs_conflicts_metadata():
# read zarray in conn2 # read zarray in conn2
conn2 = db.open(transaction_manager=tm2) conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root() root2 = conn2.root()
defer(conn2.close)
a2 = root2['zarray3b'] a2 = root2['zarray3b']
assert a2[0:1] == [1] # read data in conn2 + make sure read correctly assert a2[0:1] == [1] # read data in conn2 + make sure read correctly
...@@ -584,17 +594,16 @@ def test_zbigarray_vs_conflicts_metadata(): ...@@ -584,17 +594,16 @@ def test_zbigarray_vs_conflicts_metadata():
assert len(a1) == 13 # re-read in conn1 assert len(a1) == 13 # re-read in conn1
conn2.close()
dbclose(root1)
# verify how ZBigArray behaves when plain properties are changed / invalidated # verify how ZBigArray behaves when plain properties are changed / invalidated
@func
def test_zbigarray_invalidate_shape(): def test_zbigarray_invalidate_shape():
root = testdb.dbopen() root = testdb.dbopen()
conn = root._p_jar conn = root._p_jar
db = conn.db() db = conn.db()
conn.close() conn.close()
del root, conn del root, conn
defer(db.close)
print print
tm1 = TransactionManager() tm1 = TransactionManager()
...@@ -602,6 +611,7 @@ def test_zbigarray_invalidate_shape(): ...@@ -602,6 +611,7 @@ def test_zbigarray_invalidate_shape():
conn1 = db.open(transaction_manager=tm1) conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root() root1 = conn1.root()
defer(conn1.close)
# setup zarray # setup zarray
root1['zarray4'] = a1 = ZBigArray((10,), uint8) root1['zarray4'] = a1 = ZBigArray((10,), uint8)
...@@ -614,6 +624,7 @@ def test_zbigarray_invalidate_shape(): ...@@ -614,6 +624,7 @@ def test_zbigarray_invalidate_shape():
# read zarray in conn2 # read zarray in conn2
conn2 = db.open(transaction_manager=tm2) conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root() root2 = conn2.root()
defer(conn2.close)
a2 = root2['zarray4'] a2 = root2['zarray4']
assert a2[0:1] == [1] # read data in conn2 + make sure read correctly assert a2[0:1] == [1] # read data in conn2 + make sure read correctly
...@@ -632,6 +643,4 @@ def test_zbigarray_invalidate_shape(): ...@@ -632,6 +643,4 @@ def test_zbigarray_invalidate_shape():
assert a2[10:11] == [123] # XXX -> [10] = 123 after BigArray can assert a2[10:11] == [123] # XXX -> [10] = 123 after BigArray can
conn2.close()
del conn2, root2, a2 del conn2, root2, a2
dbclose(root1)
# Wendelin.bigfile | benchmarks for zodb backend # Wendelin.bigfile | benchmarks for zodb backend
# Copyright (C) 2014-2015 Nexedi SA and Contributors. # Copyright (C) 2014-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -24,6 +24,7 @@ from wendelin.lib.mem import memset ...@@ -24,6 +24,7 @@ from wendelin.lib.mem import memset
from wendelin.lib.testing import getTestDB, Adler32, nulladler32_bysize, ffadler32_bysize from wendelin.lib.testing import getTestDB, Adler32, nulladler32_bysize, ffadler32_bysize
from wendelin.lib.zodb import dbclose from wendelin.lib.zodb import dbclose
import transaction import transaction
from pygolang import defer, func
testdb = None testdb = None
from wendelin.bigfile.tests.bench_0virtmem import filesize, blksize # to get comparable timings from wendelin.bigfile.tests.bench_0virtmem import filesize, blksize # to get comparable timings
...@@ -51,8 +52,10 @@ def teardown_module(): ...@@ -51,8 +52,10 @@ def teardown_module():
# NOTE runs before _writeff # NOTE runs before _writeff
def bench_bigz_readhole(): _bench_bigz_hash(Adler32, nulladler32) def bench_bigz_readhole(): _bench_bigz_hash(Adler32, nulladler32)
@func
def bench_bigz_writeff(): def bench_bigz_writeff():
root = testdb.dbopen() root = testdb.dbopen()
defer(lambda: dbclose(root))
f = root['zfile'] f = root['zfile']
fh = f.fileh_open() # TODO + ram fh = f.fileh_open() # TODO + ram
vma = fh.mmap(0, blen) # XXX assumes blksize == pagesize vma = fh.mmap(0, blen) # XXX assumes blksize == pagesize
...@@ -63,11 +66,12 @@ def bench_bigz_writeff(): ...@@ -63,11 +66,12 @@ def bench_bigz_writeff():
del vma # TODO vma.close() del vma # TODO vma.close()
del fh # TODO fh.close() del fh # TODO fh.close()
del f # XXX f.close() ? del f # XXX f.close() ?
dbclose(root)
@func
def _bench_bigz_hash(hasher, expect): def _bench_bigz_hash(hasher, expect):
root = testdb.dbopen() root = testdb.dbopen()
defer(lambda: dbclose(root))
f = root['zfile'] f = root['zfile']
fh = f.fileh_open() # TODO + ram fh = f.fileh_open() # TODO + ram
vma = fh.mmap(0, blen) # XXX assumes blksize == pagesize vma = fh.mmap(0, blen) # XXX assumes blksize == pagesize
...@@ -78,7 +82,6 @@ def _bench_bigz_hash(hasher, expect): ...@@ -78,7 +82,6 @@ def _bench_bigz_hash(hasher, expect):
del vma # vma.close() del vma # vma.close()
del fh # fh.close() del fh # fh.close()
del f # f.close() del f # f.close()
dbclose(root)
assert h.digest() == expect assert h.digest() == expect
......
...@@ -27,6 +27,7 @@ import transaction ...@@ -27,6 +27,7 @@ import transaction
from transaction import TransactionManager from transaction import TransactionManager
from ZODB.POSException import ConflictError from ZODB.POSException import ConflictError
from numpy import ndarray, array_equal, uint32, zeros, arange from numpy import ndarray, array_equal, uint32, zeros, arange
from golang import defer, func
from threading import Thread from threading import Thread
from six.moves import _thread from six.moves import _thread
from six import b from six import b
...@@ -74,6 +75,7 @@ def cacheInfo(db): ...@@ -74,6 +75,7 @@ def cacheInfo(db):
def kkey(klass): def kkey(klass):
return '%s.%s' % (klass.__module__, klass.__name__) return '%s.%s' % (klass.__module__, klass.__name__)
@func
def test_livepersistent(): def test_livepersistent():
root = dbopen() root = dbopen()
transaction.commit() # set root._p_jar transaction.commit() # set root._p_jar
...@@ -152,10 +154,12 @@ def test_livepersistent(): ...@@ -152,10 +154,12 @@ def test_livepersistent():
conn1 = db.open(transaction_manager=tm1) conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root() root1 = conn1.root()
defer(lambda: dbclose(root1))
lp1 = root1['live'] lp1 = root1['live']
conn2 = db.open(transaction_manager=tm2) conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root() root2 = conn2.root()
defer(conn2.close)
lp2 = root2['live'] lp2 = root2['live']
# 2 connections are setup running in parallel with initial obj state as ghost # 2 connections are setup running in parallel with initial obj state as ghost
...@@ -202,9 +206,7 @@ def test_livepersistent(): ...@@ -202,9 +206,7 @@ def test_livepersistent():
assert a == 1 assert a == 1
conn2.close()
del conn2, root2 del conn2, root2
dbclose(root1)
...@@ -349,12 +351,14 @@ def test_bigfile_filezodb(): ...@@ -349,12 +351,14 @@ def test_bigfile_filezodb():
# connection can migrate between threads handling requests. # connection can migrate between threads handling requests.
# verify _ZBigFileH properly adjusts. # verify _ZBigFileH properly adjusts.
# ( NOTE this test is almost dupped at test_zbigarray_vs_conn_migration() ) # ( NOTE this test is almost dupped at test_zbigarray_vs_conn_migration() )
@func
def test_bigfile_filezodb_vs_conn_migration(): def test_bigfile_filezodb_vs_conn_migration():
root01 = dbopen() root01 = dbopen()
conn01 = root01._p_jar conn01 = root01._p_jar
db = conn01.db() db = conn01.db()
conn01.close() conn01.close()
del root01 del root01
defer(db.close)
c12_1 = NotifyChannel() # T11 -> T21 c12_1 = NotifyChannel() # T11 -> T21
c21_1 = NotifyChannel() # T21 -> T11 c21_1 = NotifyChannel() # T21 -> T11
...@@ -533,6 +537,7 @@ def test_bigfile_filezodb_vs_conn_migration(): ...@@ -533,6 +537,7 @@ def test_bigfile_filezodb_vs_conn_migration():
# now verify that zfile2 changed to 22 state, i.e. T22 was really committed # now verify that zfile2 changed to 22 state, i.e. T22 was really committed
conn03 = db.open() conn03 = db.open()
defer(conn03.close)
# NOTE top of connection stack is conn22(=conn01), because it has most # of # NOTE top of connection stack is conn22(=conn01), because it has most # of
# active objects # active objects
assert conn03 is conn01 assert conn03 is conn01
...@@ -545,23 +550,25 @@ def test_bigfile_filezodb_vs_conn_migration(): ...@@ -545,23 +550,25 @@ def test_bigfile_filezodb_vs_conn_migration():
assert Blk(vma03, 0)[0] == 22 assert Blk(vma03, 0)[0] == 22
del vma03, fh03, f03 del vma03, fh03, f03
dbclose(root03)
# ZBlk should properly handle 'invalidate' messages from DB # ZBlk should properly handle 'invalidate' messages from DB
# ( NOTE this test is almost dupped at test_zbigarray_vs_cache_invalidation() ) # ( NOTE this test is almost dupped at test_zbigarray_vs_cache_invalidation() )
@func
def test_bigfile_filezodb_vs_cache_invalidation(): def test_bigfile_filezodb_vs_cache_invalidation():
root = dbopen() root = dbopen()
conn = root._p_jar conn = root._p_jar
db = conn.db() db = conn.db()
conn.close() conn.close()
del root, conn del root, conn
defer(db.close)
tm1 = TransactionManager() tm1 = TransactionManager()
tm2 = TransactionManager() tm2 = TransactionManager()
conn1 = db.open(transaction_manager=tm1) conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root() root1 = conn1.root()
defer(conn1.close)
# setup zfile with fileh view to it # setup zfile with fileh view to it
root1['zfile3'] = f1 = ZBigFile(blksize) root1['zfile3'] = f1 = ZBigFile(blksize)
...@@ -579,6 +586,7 @@ def test_bigfile_filezodb_vs_cache_invalidation(): ...@@ -579,6 +586,7 @@ def test_bigfile_filezodb_vs_cache_invalidation():
# read zfile and setup fileh for it in conn2 # read zfile and setup fileh for it in conn2
conn2 = db.open(transaction_manager=tm2) conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root() root2 = conn2.root()
defer(conn2.close)
f2 = root2['zfile3'] f2 = root2['zfile3']
fh2 = f2.fileh_open() fh2 = f2.fileh_open()
...@@ -606,25 +614,26 @@ def test_bigfile_filezodb_vs_cache_invalidation(): ...@@ -606,25 +614,26 @@ def test_bigfile_filezodb_vs_cache_invalidation():
# data from tm1 should propagate -> ZODB -> ram pages for _ZBigFileH in conn2 # data from tm1 should propagate -> ZODB -> ram pages for _ZBigFileH in conn2
assert Blk(vma2, 0)[0] == 2 assert Blk(vma2, 0)[0] == 2
conn2.close()
del conn2, root2 del conn2, root2
dbclose(root1)
# verify that conflicts on ZBlk are handled properly # verify that conflicts on ZBlk are handled properly
# ( NOTE this test is almost dupped at test_zbigarray_vs_conflicts() ) # ( NOTE this test is almost dupped at test_zbigarray_vs_conflicts() )
@func
def test_bigfile_filezodb_vs_conflicts(): def test_bigfile_filezodb_vs_conflicts():
root = dbopen() root = dbopen()
conn = root._p_jar conn = root._p_jar
db = conn.db() db = conn.db()
conn.close() conn.close()
del root, conn del root, conn
defer(db.close)
tm1 = TransactionManager() tm1 = TransactionManager()
tm2 = TransactionManager() tm2 = TransactionManager()
conn1 = db.open(transaction_manager=tm1) conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root() root1 = conn1.root()
defer(conn1.close)
# setup zfile with fileh view to it # setup zfile with fileh view to it
root1['zfile3a'] = f1 = ZBigFile(blksize) root1['zfile3a'] = f1 = ZBigFile(blksize)
...@@ -641,6 +650,7 @@ def test_bigfile_filezodb_vs_conflicts(): ...@@ -641,6 +650,7 @@ def test_bigfile_filezodb_vs_conflicts():
# read zfile and setup fileh for it in conn2 # read zfile and setup fileh for it in conn2
conn2 = db.open(transaction_manager=tm2) conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root() root2 = conn2.root()
defer(conn2.close)
f2 = root2['zfile3a'] f2 = root2['zfile3a']
fh2 = f2.fileh_open() fh2 = f2.fileh_open()
...@@ -672,16 +682,15 @@ def test_bigfile_filezodb_vs_conflicts(): ...@@ -672,16 +682,15 @@ def test_bigfile_filezodb_vs_conflicts():
assert Blk(vma1, 0)[0] == 13 # re-read in conn1 assert Blk(vma1, 0)[0] == 13 # re-read in conn1
conn2.close()
dbclose(root1)
# verify that fileh are garbage-collected after user free them # verify that fileh are garbage-collected after user free them
@func
def test_bigfile_filezodb_fileh_gc(): def test_bigfile_filezodb_fileh_gc():
root1= dbopen() root1= dbopen()
conn1= root1._p_jar conn1= root1._p_jar
db = conn1.db() db = conn1.db()
defer(db.close)
root1['zfile4'] = f1 = ZBigFile(blksize) root1['zfile4'] = f1 = ZBigFile(blksize)
transaction.commit() transaction.commit()
...@@ -696,6 +705,7 @@ def test_bigfile_filezodb_fileh_gc(): ...@@ -696,6 +705,7 @@ def test_bigfile_filezodb_fileh_gc():
conn2 = db.open() conn2 = db.open()
root2 = conn2.root() root2 = conn2.root()
defer(conn2.close)
f2 = root2['zfile4'] f2 = root2['zfile4']
fh2 = f2.fileh_open() fh2 = f2.fileh_open()
...@@ -705,12 +715,13 @@ def test_bigfile_filezodb_fileh_gc(): ...@@ -705,12 +715,13 @@ def test_bigfile_filezodb_fileh_gc():
assert wfh1() is None # fh1 should be gone assert wfh1() is None # fh1 should be gone
del vma2, fh2, f2 del vma2, fh2, f2
dbclose(root2)
# verify how zblk format change works # verify how zblk format change works
@func
def test_bigfile_filezodb_fmt_change(): def test_bigfile_filezodb_fmt_change():
root = dbopen() root = dbopen()
defer(lambda: dbclose(root))
root['zfile5'] = f = ZBigFile(blksize) root['zfile5'] = f = ZBigFile(blksize)
transaction.commit() transaction.commit()
...@@ -743,8 +754,6 @@ def test_bigfile_filezodb_fmt_change(): ...@@ -743,8 +754,6 @@ def test_bigfile_filezodb_fmt_change():
finally: finally:
file_zodb.ZBlk_fmt_write = fmt_write_save file_zodb.ZBlk_fmt_write = fmt_write_save
dbclose(root)
# test that ZData are reused for changed chunks in ZBlk1 format # test that ZData are reused for changed chunks in ZBlk1 format
def test_bigfile_zblk1_zdata_reuse(): def test_bigfile_zblk1_zdata_reuse():
...@@ -756,8 +765,10 @@ def test_bigfile_zblk1_zdata_reuse(): ...@@ -756,8 +765,10 @@ def test_bigfile_zblk1_zdata_reuse():
finally: finally:
file_zodb.ZBlk_fmt_write = fmt_write_save file_zodb.ZBlk_fmt_write = fmt_write_save
@func
def _test_bigfile_zblk1_zdata_reuse(): def _test_bigfile_zblk1_zdata_reuse():
root = dbopen() root = dbopen()
defer(lambda: dbclose(root))
root['zfile6'] = f = ZBigFile(blksize) root['zfile6'] = f = ZBigFile(blksize)
transaction.commit() transaction.commit()
...@@ -791,5 +802,3 @@ def _test_bigfile_zblk1_zdata_reuse(): ...@@ -791,5 +802,3 @@ def _test_bigfile_zblk1_zdata_reuse():
assert len(zdata_v1) == len(zdata_v2) assert len(zdata_v1) == len(zdata_v2)
for i in range(len(zdata_v1)): for i in range(len(zdata_v1)):
assert zdata_v1[i] is zdata_v2[i] assert zdata_v1[i] is zdata_v2[i]
dbclose(root)
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2014-2017 Nexedi SA and Contributors. # Copyright (C) 2014-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -37,6 +37,7 @@ from __future__ import print_function ...@@ -37,6 +37,7 @@ from __future__ import print_function
from wendelin.bigarray.array_zodb import ZBigArray from wendelin.bigarray.array_zodb import ZBigArray
from wendelin.lib.zodb import dbopen, dbclose from wendelin.lib.zodb import dbopen, dbclose
import transaction import transaction
from golang import defer, func
from numpy import float64, dtype, cumsum, sin from numpy import float64, dtype, cumsum, sin
import psutil import psutil
...@@ -96,7 +97,7 @@ options: ...@@ -96,7 +97,7 @@ options:
""" % sys.argv[0], file=sys.stderr) """ % sys.argv[0], file=sys.stderr)
sys.exit(1) sys.exit(1)
@func
def main(): def main():
worksize = None worksize = None
optv, argv = getopt.getopt(sys.argv[1:], '', ['worksize=']) optv, argv = getopt.getopt(sys.argv[1:], '', ['worksize='])
...@@ -120,6 +121,7 @@ def main(): ...@@ -120,6 +121,7 @@ def main():
print('I: RAM: %.2fGB' % (float(ram_nbytes) / GB)) print('I: RAM: %.2fGB' % (float(ram_nbytes) / GB))
root = dbopen(dburi) root = dbopen(dburi)
defer(lambda: dbclose(root))
if act == 'gen': if act == 'gen':
if worksize is None: if worksize is None:
...@@ -145,8 +147,6 @@ def main(): ...@@ -145,8 +147,6 @@ def main():
m = p.memory_info() m = p.memory_info()
print('VIRT: %i MB\tRSS: %iMB' % (m.vms//MB, m.rss//MB)) print('VIRT: %i MB\tRSS: %iMB' % (m.vms//MB, m.rss//MB))
dbclose(root)
if __name__ == '__main__': if __name__ == '__main__':
main() main()
# Wendelin.core.bigfile | Tests for ZODB utilities # Wendelin.core.bigfile | Tests for ZODB utilities
# Copyright (C) 2014-2016 Nexedi SA and Contributors. # Copyright (C) 2014-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -22,6 +22,7 @@ from wendelin.lib.testing import getTestDB ...@@ -22,6 +22,7 @@ from wendelin.lib.testing import getTestDB
from persistent import Persistent, UPTODATE, GHOST from persistent import Persistent, UPTODATE, GHOST
from BTrees.IOBTree import IOBTree from BTrees.IOBTree import IOBTree
import transaction import transaction
from golang import defer, func
import gc import gc
testdb = None testdb = None
...@@ -45,8 +46,11 @@ class XInt(Persistent): ...@@ -45,8 +46,11 @@ class XInt(Persistent):
def objscachedv(jar): def objscachedv(jar):
return [obj for oid, obj in jar._cache.lru_items()] return [obj for oid, obj in jar._cache.lru_items()]
@func
def test_deactivate_btree(): def test_deactivate_btree():
root = dbopen() root = dbopen()
defer(lambda: dbclose(root))
# init btree with many leaf nodes # init btree with many leaf nodes
leafv = [] leafv = []
root['btree'] = B = IOBTree() root['btree'] = B = IOBTree()
...@@ -81,5 +85,3 @@ def test_deactivate_btree(): ...@@ -81,5 +85,3 @@ def test_deactivate_btree():
for obj in [B] + leafv: for obj in [B] + leafv:
assert obj._p_state == GHOST assert obj._p_state == GHOST
assert obj not in cached assert obj not in cached
dbclose(root)
# Wendelin.core | pythonic package setup # Wendelin.core | pythonic package setup
# Copyright (C) 2014-2015 Nexedi SA and Contributors. # Copyright (C) 2014-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -243,13 +243,15 @@ setup( ...@@ -243,13 +243,15 @@ setup(
# specify either to use e.g. ZODB3.10 or ZODB4 ) # specify either to use e.g. ZODB3.10 or ZODB4 )
'ZODB3 >= 3.10', 'ZODB3 >= 3.10',
'pygolang >= 0.0.2', # defer, sync.WaitGroup, ...
'six', # compat py2/py3 'six', # compat py2/py3
'psutil', # demo_zbigarray 'psutil', # demo_zbigarray
], ],
extras_require = { extras_require = {
'test': ['pytest', 'pygolang >= 0.0.0.dev4'], 'test': ['pytest'],
}, },
cmdclass = {'build_ext': build_ext, cmdclass = {'build_ext': build_ext,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment