Skip to content

W
wendelin.core
Commit 5c8340d2 authored by Kirill Smelkov's avatar Kirill Smelkov
2 Browse files
Options

*: Use defer for dbclose & friends 

For tests this makes sure that if one test fails, it won't make following
tests fail just because the next test will fail trying to lock test database.

For regular code (demo_zbigarray.py) this is also a good thing to do -
to always close the database irregardless of whether an exception was
raised before program reached end of main.

Pygolang becomes regular - not test only - dependency. Being regular
dependency is currently required only by demo_zbigarray.py, but it will
be also used in upcoming wcfs, so adding pygolang into wendelin.core
dependencies aligns with the plan.

dbclose now uses defer almost everywhere - there are still few places in
tests, where one test function is opening/closing test database multiple
times - those were not (yet ?) converted.
parent b12e319e
Hide whitespace changes
Inline Side-by-side
Showing with 65 additions and 40 deletions
bigarray/tests/test_arrayzodb.py
View file @ 5c8340d2
......@@ -27,6 +27,7 @@ import transaction
from transaction import TransactionManager
from ZODB.POSException import ConflictError
from numpy import dtype, uint8, all, array_equal, arange
from golang import defer, func
from threading import Thread
from six.moves import _thread
......@@ -240,12 +241,14 @@ def test_zbigarray_order():
# the same as test_bigfile_filezodb_vs_conn_migration but explicitly for ZBigArray
# ( NOTE this test is almost dup of test_zbigarray_vs_conn_migration() )
@func
def test_zbigarray_vs_conn_migration():
root01 = testdb.dbopen()
conn01 = root01._p_jar
db = conn01.db()
conn01.close()
del root01
defer(db.close)
c12_1 = NotifyChannel() # T11 -> T21
c21_1 = NotifyChannel() # T21 -> T11
......@@ -342,12 +345,14 @@ def test_zbigarray_vs_conn_migration():
c21_2 = NotifyChannel() # T22 -> T12
# open, abort
@func
def T12():
tell, wait = c12_2.tell, c21_2.wait
wait('T2-conn22-opened')
conn12 = db.open()
defer(conn12.close)
tell('T1-conn12-opened')
wait('T2-zarray2-modified')
......@@ -357,10 +362,9 @@ def test_zbigarray_vs_conn_migration():
tell('T1-txn-aborted')
wait('T2-txn-committed')
conn12.close()
# open, modify, commit
@func
def T22():
tell, wait = c21_2.tell, c12_2.wait
......@@ -369,6 +373,7 @@ def test_zbigarray_vs_conn_migration():
assert _thread.get_ident() != t11_ident
conn22 = db.open()
defer(conn22.close)
assert conn22 is conn01
tell('T2-conn22-opened')
......@@ -388,8 +393,6 @@ def test_zbigarray_vs_conn_migration():
tell('T2-txn-committed')
conn22.close()
t12, t22 = Thread(target=T12), Thread(target=T22)
t12.start(); t22.start()
......@@ -401,6 +404,7 @@ def test_zbigarray_vs_conn_migration():
# now verify that zarray2 changed to 22 state, i.e. T22 was really committed
conn03 = db.open()
defer(conn03.close)
# NOTE top of connection stack is conn22(=conn01), becase it has most # of
# active objectd
assert conn03 is conn01
......@@ -410,23 +414,25 @@ def test_zbigarray_vs_conn_migration():
assert a03[0] == 22
del a03
dbclose(root03)
# underlying ZBigFile/ZBigFileH should properly handle 'invalidate' messages from DB
# ( NOTE this test is almost dup of test_zbigarray_vs_cache_invalidation() )
@func
def test_zbigarray_vs_cache_invalidation():
root = testdb.dbopen()
conn = root._p_jar
db = conn.db()
conn.close()
del root, conn
defer(db.close)
tm1 = TransactionManager()
tm2 = TransactionManager()
conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root()
defer(conn1.close)
# setup zarray
root1['zarray3'] = a1 = ZBigArray((10,), uint8)
......@@ -440,6 +446,7 @@ def test_zbigarray_vs_cache_invalidation():
# read zarray in conn2
conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root()
defer(conn2.close)
a2 = root2['zarray3']
assert a2[0:1] == [1] # read data in conn2 + make sure read correctly
......@@ -465,25 +472,26 @@ def test_zbigarray_vs_cache_invalidation():
# data from tm1 should propagate -> ZODB -> ram pages for _ZBigFileH in conn2
assert a2[0] == 2
conn2.close()
del conn2, root2
dbclose(root1)
# verify that conflicts on array content are handled properly
# ( NOTE this test is almost dup of test_bigfile_filezodb_vs_conflicts() )
@func
def test_zbigarray_vs_conflicts():
root = testdb.dbopen()
conn = root._p_jar
db = conn.db()
conn.close()
del root, conn
defer(db.close)
tm1 = TransactionManager()
tm2 = TransactionManager()
conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root()
defer(conn1.close)
# setup zarray
root1['zarray3a'] = a1 = ZBigArray((10,), uint8)
......@@ -496,6 +504,7 @@ def test_zbigarray_vs_conflicts():
# read zarray in conn2
conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root()
defer(conn2.close)
a2 = root2['zarray3a']
assert a2[0:1] == [1] # read data in conn2 + make sure read correctly
......@@ -525,24 +534,24 @@ def test_zbigarray_vs_conflicts():
assert a1[0:1] == [13] # re-read in conn1 XXX -> [0] == 13
conn2.close()
dbclose(root1)
# verify that conflicts on array metadata are handled properly
# ( NOTE this test is close to test_zbigarray_vs_conflicts() )
@func
def test_zbigarray_vs_conflicts_metadata():
root = testdb.dbopen()
conn = root._p_jar
db = conn.db()
conn.close()
del root, conn
defer(db.close)
tm1 = TransactionManager()
tm2 = TransactionManager()
conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root()
defer(conn1.close)
# setup zarray
root1['zarray3b'] = a1 = ZBigArray((10,), uint8)
......@@ -555,6 +564,7 @@ def test_zbigarray_vs_conflicts_metadata():
# read zarray in conn2
conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root()
defer(conn2.close)
a2 = root2['zarray3b']
assert a2[0:1] == [1] # read data in conn2 + make sure read correctly
......@@ -584,17 +594,16 @@ def test_zbigarray_vs_conflicts_metadata():
assert len(a1) == 13 # re-read in conn1
conn2.close()
dbclose(root1)
# verify how ZBigArray behaves when plain properties are changed / invalidated
@func
def test_zbigarray_invalidate_shape():
root = testdb.dbopen()
conn = root._p_jar
db = conn.db()
conn.close()
del root, conn
defer(db.close)
print
tm1 = TransactionManager()
......@@ -602,6 +611,7 @@ def test_zbigarray_invalidate_shape():
conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root()
defer(conn1.close)
# setup zarray
root1['zarray4'] = a1 = ZBigArray((10,), uint8)
......@@ -614,6 +624,7 @@ def test_zbigarray_invalidate_shape():
# read zarray in conn2
conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root()
defer(conn2.close)
a2 = root2['zarray4']
assert a2[0:1] == [1] # read data in conn2 + make sure read correctly
......@@ -632,6 +643,4 @@ def test_zbigarray_invalidate_shape():
assert a2[10:11] == [123] # XXX -> [10] = 123 after BigArray can
conn2.close()
del conn2, root2, a2
dbclose(root1)
bigfile/tests/bench_1filezodb.py
View file @ 5c8340d2
# Wendelin.bigfile | benchmarks for zodb backend
# Copyright (C) 2014-2015 Nexedi SA and Contributors.
# Copyright (C) 2014-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -24,6 +24,7 @@ from wendelin.lib.mem import memset
from wendelin.lib.testing import getTestDB, Adler32, nulladler32_bysize, ffadler32_bysize
from wendelin.lib.zodb import dbclose
import transaction
from pygolang import defer, func
testdb = None
from wendelin.bigfile.tests.bench_0virtmem import filesize, blksize # to get comparable timings
......@@ -51,8 +52,10 @@ def teardown_module():
# NOTE runs before _writeff
def bench_bigz_readhole(): _bench_bigz_hash(Adler32, nulladler32)
@func
def bench_bigz_writeff():
root = testdb.dbopen()
defer(lambda: dbclose(root))
f = root['zfile']
fh = f.fileh_open() # TODO + ram
vma = fh.mmap(0, blen) # XXX assumes blksize == pagesize
......@@ -63,11 +66,12 @@ def bench_bigz_writeff():
del vma # TODO vma.close()
del fh # TODO fh.close()
del f # XXX f.close() ?
dbclose(root)
@func
def _bench_bigz_hash(hasher, expect):
root = testdb.dbopen()
defer(lambda: dbclose(root))
f = root['zfile']
fh = f.fileh_open() # TODO + ram
vma = fh.mmap(0, blen) # XXX assumes blksize == pagesize
......@@ -78,7 +82,6 @@ def _bench_bigz_hash(hasher, expect):
del vma # vma.close()
del fh # fh.close()
del f # f.close()
dbclose(root)
assert h.digest() == expect
......
bigfile/tests/test_filezodb.py
View file @ 5c8340d2
......@@ -27,6 +27,7 @@ import transaction
from transaction import TransactionManager
from ZODB.POSException import ConflictError
from numpy import ndarray, array_equal, uint32, zeros, arange
from golang import defer, func
from threading import Thread
from six.moves import _thread
from six import b
......@@ -74,6 +75,7 @@ def cacheInfo(db):
def kkey(klass):
return '%s.%s' % (klass.__module__, klass.__name__)
@func
def test_livepersistent():
root = dbopen()
transaction.commit() # set root._p_jar
......@@ -152,10 +154,12 @@ def test_livepersistent():
conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root()
defer(lambda: dbclose(root1))
lp1 = root1['live']
conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root()
defer(conn2.close)
lp2 = root2['live']
# 2 connections are setup running in parallel with initial obj state as ghost
......@@ -202,9 +206,7 @@ def test_livepersistent():
assert a == 1
conn2.close()
del conn2, root2
dbclose(root1)
......@@ -349,12 +351,14 @@ def test_bigfile_filezodb():
# connection can migrate between threads handling requests.
# verify _ZBigFileH properly adjusts.
# ( NOTE this test is almost dupped at test_zbigarray_vs_conn_migration() )
@func
def test_bigfile_filezodb_vs_conn_migration():
root01 = dbopen()
conn01 = root01._p_jar
db = conn01.db()
conn01.close()
del root01
defer(db.close)
c12_1 = NotifyChannel() # T11 -> T21
c21_1 = NotifyChannel() # T21 -> T11
......@@ -533,6 +537,7 @@ def test_bigfile_filezodb_vs_conn_migration():
# now verify that zfile2 changed to 22 state, i.e. T22 was really committed
conn03 = db.open()
defer(conn03.close)
# NOTE top of connection stack is conn22(=conn01), because it has most # of
# active objects
assert conn03 is conn01
......@@ -545,23 +550,25 @@ def test_bigfile_filezodb_vs_conn_migration():
assert Blk(vma03, 0)[0] == 22
del vma03, fh03, f03
dbclose(root03)
# ZBlk should properly handle 'invalidate' messages from DB
# ( NOTE this test is almost dupped at test_zbigarray_vs_cache_invalidation() )
@func
def test_bigfile_filezodb_vs_cache_invalidation():
root = dbopen()
conn = root._p_jar
db = conn.db()
conn.close()
del root, conn
defer(db.close)
tm1 = TransactionManager()
tm2 = TransactionManager()
conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root()
defer(conn1.close)
# setup zfile with fileh view to it
root1['zfile3'] = f1 = ZBigFile(blksize)
......@@ -579,6 +586,7 @@ def test_bigfile_filezodb_vs_cache_invalidation():
# read zfile and setup fileh for it in conn2
conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root()
defer(conn2.close)
f2 = root2['zfile3']
fh2 = f2.fileh_open()
......@@ -606,25 +614,26 @@ def test_bigfile_filezodb_vs_cache_invalidation():
# data from tm1 should propagate -> ZODB -> ram pages for _ZBigFileH in conn2
assert Blk(vma2, 0)[0] == 2
conn2.close()
del conn2, root2
dbclose(root1)
# verify that conflicts on ZBlk are handled properly
# ( NOTE this test is almost dupped at test_zbigarray_vs_conflicts() )
@func
def test_bigfile_filezodb_vs_conflicts():
root = dbopen()
conn = root._p_jar
db = conn.db()
conn.close()
del root, conn
defer(db.close)
tm1 = TransactionManager()
tm2 = TransactionManager()
conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root()
defer(conn1.close)
# setup zfile with fileh view to it
root1['zfile3a'] = f1 = ZBigFile(blksize)
......@@ -641,6 +650,7 @@ def test_bigfile_filezodb_vs_conflicts():
# read zfile and setup fileh for it in conn2
conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root()
defer(conn2.close)
f2 = root2['zfile3a']
fh2 = f2.fileh_open()
......@@ -672,16 +682,15 @@ def test_bigfile_filezodb_vs_conflicts():
assert Blk(vma1, 0)[0] == 13 # re-read in conn1
conn2.close()
dbclose(root1)
# verify that fileh are garbage-collected after user free them
@func
def test_bigfile_filezodb_fileh_gc():
root1= dbopen()
conn1= root1._p_jar
db = conn1.db()
defer(db.close)
root1['zfile4'] = f1 = ZBigFile(blksize)
transaction.commit()
......@@ -696,6 +705,7 @@ def test_bigfile_filezodb_fileh_gc():
conn2 = db.open()
root2 = conn2.root()
defer(conn2.close)
f2 = root2['zfile4']
fh2 = f2.fileh_open()
......@@ -705,12 +715,13 @@ def test_bigfile_filezodb_fileh_gc():
assert wfh1() is None # fh1 should be gone
del vma2, fh2, f2
dbclose(root2)
# verify how zblk format change works
@func
def test_bigfile_filezodb_fmt_change():
root = dbopen()
defer(lambda: dbclose(root))
root['zfile5'] = f = ZBigFile(blksize)
transaction.commit()
......@@ -743,8 +754,6 @@ def test_bigfile_filezodb_fmt_change():
finally:
file_zodb.ZBlk_fmt_write = fmt_write_save
dbclose(root)
# test that ZData are reused for changed chunks in ZBlk1 format
def test_bigfile_zblk1_zdata_reuse():
......@@ -756,8 +765,10 @@ def test_bigfile_zblk1_zdata_reuse():
finally:
file_zodb.ZBlk_fmt_write = fmt_write_save
@func
def _test_bigfile_zblk1_zdata_reuse():
root = dbopen()
defer(lambda: dbclose(root))
root['zfile6'] = f = ZBigFile(blksize)
transaction.commit()
......@@ -791,5 +802,3 @@ def _test_bigfile_zblk1_zdata_reuse():
assert len(zdata_v1) == len(zdata_v2)
for i in range(len(zdata_v1)):
assert zdata_v1[i] is zdata_v2[i]
dbclose(root)
demo/demo_zbigarray.py
View file @ 5c8340d2
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2017 Nexedi SA and Contributors.
# Copyright (C) 2014-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -37,6 +37,7 @@ from __future__ import print_function
from wendelin.bigarray.array_zodb import ZBigArray
from wendelin.lib.zodb import dbopen, dbclose
import transaction
from golang import defer, func
from numpy import float64, dtype, cumsum, sin
import psutil
......@@ -96,7 +97,7 @@ options:
""" % sys.argv[0], file=sys.stderr)
sys.exit(1)
@func
def main():
worksize = None
optv, argv = getopt.getopt(sys.argv[1:], '', ['worksize='])
......@@ -120,6 +121,7 @@ def main():
print('I: RAM: %.2fGB' % (float(ram_nbytes) / GB))
root = dbopen(dburi)
defer(lambda: dbclose(root))
if act == 'gen':
if worksize is None:
......@@ -145,8 +147,6 @@ def main():
m = p.memory_info()
print('VIRT: %i MB\tRSS: %iMB' % (m.vms//MB, m.rss//MB))
dbclose(root)
if __name__ == '__main__':
main()
lib/tests/test_zodb.py
View file @ 5c8340d2
# Wendelin.core.bigfile | Tests for ZODB utilities
# Copyright (C) 2014-2016 Nexedi SA and Contributors.
# Copyright (C) 2014-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -22,6 +22,7 @@ from wendelin.lib.testing import getTestDB
from persistent import Persistent, UPTODATE, GHOST
from BTrees.IOBTree import IOBTree
import transaction
from golang import defer, func
import gc
testdb = None
......@@ -45,8 +46,11 @@ class XInt(Persistent):
def objscachedv(jar):
return [obj for oid, obj in jar._cache.lru_items()]
@func
def test_deactivate_btree():
root = dbopen()
defer(lambda: dbclose(root))
# init btree with many leaf nodes
leafv = []
root['btree'] = B = IOBTree()
......@@ -81,5 +85,3 @@ def test_deactivate_btree():
for obj in [B] + leafv:
assert obj._p_state == GHOST
assert obj not in cached
dbclose(root)
setup.py
View file @ 5c8340d2
# Wendelin.core | pythonic package setup
# Copyright (C) 2014-2015 Nexedi SA and Contributors.
# Copyright (C) 2014-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -243,13 +243,15 @@ setup(
# specify either to use e.g. ZODB3.10 or ZODB4 )
'ZODB3 >= 3.10',
'pygolang >= 0.0.2', # defer, sync.WaitGroup, ...
'six', # compat py2/py3
'psutil', # demo_zbigarray
],
extras_require = {
'test': ['pytest', 'pygolang >= 0.0.0.dev4'],
'test': ['pytest'],
},
cmdclass = {'build_ext': build_ext,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7