test_zodb.py 12 KB
Newer Older
1
# Wendelin.core.bigfile | Tests for ZODB utilities and critical properties of ZODB itself
Kirill Smelkov's avatar
Kirill Smelkov committed
2
# Copyright (C) 2014-2021  Nexedi SA and Contributors.
3 4 5 6 7 8 9
#                          Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
10 11 12 13
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
14 15 16 17 18
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
19
# See https://www.nexedi.com/licensing for rationale and options.
20
from wendelin.lib.zodb import LivePersistent, deactivate_btree, dbclose, zconn_at, zstor_2zurl, zmajor, _zhasNXDPatch
21
from wendelin.lib.testing import getTestDB
22
from wendelin.lib import testing
23
from persistent import Persistent, UPTODATE, GHOST, CHANGED
24
from ZODB import DB, POSException
25 26
from ZODB.FileStorage import FileStorage
from ZODB.MappingStorage import MappingStorage
27 28
from BTrees.IOBTree import IOBTree
import transaction
29
from transaction import TransactionManager
30
from golang import defer, func
31 32
from pytest import raises
import pytest; xfail = pytest.mark.xfail
33

34 35
from wendelin.lib.tests.testprog import zopenrace, zloadrace

36 37 38 39 40 41 42 43 44 45 46 47 48
testdb = None

def dbopen():
    return testdb.dbopen()

def setup_module():
    global testdb
    testdb = getTestDB()
    testdb.setup()

def teardown_module():
    testdb.teardown()

49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
# like db.cacheDetail(), but {} instead of []
def cacheInfo(db):
    return dict(db.cacheDetail())

# key for cacheInfo() result
def kkey(klass):
    return '%s.%s' % (klass.__module__, klass.__name__)

@func
def test_livepersistent():
    root = dbopen()
    transaction.commit()    # set root._p_jar
    db = root._p_jar.db()

    # ~~~ test `obj initially created` case
    root['live'] = lp = LivePersistent()
    assert lp._p_jar   is None          # connection does not know about it yet
    assert lp._p_state == UPTODATE      # object initially created in uptodate

    # should not be in cache yet & thus should stay after gc
    db.cacheMinimize()
    assert lp._p_jar   is None
    assert lp._p_state == UPTODATE
    ci = cacheInfo(db)
    assert kkey(LivePersistent) not in ci

    # should be registered to connection & cache after commit
    transaction.commit()
    assert lp._p_jar   is not None
    assert lp._p_state == UPTODATE
    ci = cacheInfo(db)
    assert ci[kkey(LivePersistent)] == 1

    # should stay that way after cache gc
    db.cacheMinimize()
    assert lp._p_jar   is not None
    assert lp._p_state == UPTODATE
    ci = cacheInfo(db)
    assert ci[kkey(LivePersistent)] == 1


    # ~~~ reopen & test `obj loaded from db` case
    dbclose(root)
    del root, db, lp

    root = dbopen()
    db = root._p_jar.db()

    # known to connection & cache & GHOST
    # right after first loading from DB
    lp = root['live']
    assert lp._p_jar   is not None
    assert lp._p_state is GHOST
    ci = cacheInfo(db)
    assert ci[kkey(LivePersistent)] == 1

    # should be UPTODATE for sure after read access
    getattr(lp, 'attr', None)
    assert lp._p_jar   is not None
    assert lp._p_state is UPTODATE
    ci = cacheInfo(db)
    assert ci[kkey(LivePersistent)] == 1

    # does not go back to ghost on cache gc
    db.cacheMinimize()
    assert lp._p_jar   is not None
    assert lp._p_state == UPTODATE
    ci = cacheInfo(db)
    assert ci[kkey(LivePersistent)] == 1

    # ok
    dbclose(root)
    del root, db, lp


    # demo that upon cache invalidation LivePersistent can go back to ghost
    root = dbopen()
    conn = root._p_jar
    db   = conn.db()
    conn.close()
    del root, conn

    tm1 = TransactionManager()
    tm2 = TransactionManager()

    conn1 = db.open(transaction_manager=tm1)
    root1 = conn1.root()
    defer(lambda: dbclose(root1))
    lp1 = root1['live']

    conn2 = db.open(transaction_manager=tm2)
    root2 = conn2.root()
    defer(conn2.close)
    lp2 = root2['live']

    # 2 connections are setup running in parallel with initial obj state as ghost
    assert lp1._p_jar   is conn1
    assert lp2._p_jar   is conn2

    assert lp1._p_state is GHOST
    assert lp2._p_state is GHOST

    # conn1: modify  ghost -> changed
    lp1.attr = 1

    assert lp1._p_state is CHANGED
    assert lp2._p_state is GHOST

    # conn2: read    ghost -> uptodate
    assert getattr(lp1, 'attr', None) == 1
    assert getattr(lp2, 'attr', None) is None

    assert lp1._p_state is CHANGED
    assert lp2._p_state is UPTODATE

    # conn1: commit  changed -> uptodate; conn2 untouched
    tm1.commit()

    assert lp1._p_state is UPTODATE
    assert lp2._p_state is UPTODATE

    assert getattr(lp1, 'attr', None) == 1
    assert getattr(lp2, 'attr', None) is None

    # conn2: commit  (nothing changed - just transaction boundary)
    #                 uptodate -> ghost (invalidation)
    tm2.commit()

    assert lp1._p_state is UPTODATE
    assert lp2._p_state is GHOST

    assert getattr(lp1, 'attr', None) == 1

    # conn2: after reading, the state is again uptodate + changes from conn1 are here
    a = getattr(lp2, 'attr', None)
    assert lp2._p_state is UPTODATE
    assert a == 1

    del conn2, root2

189 190 191 192 193 194 195 196

class XInt(Persistent):
    def __init__(self, i):
        self.i = i

def objscachedv(jar):
    return [obj for oid, obj in jar._cache.lru_items()]

197
@func
198 199
def test_deactivate_btree():
    root = dbopen()
200 201
    defer(lambda: dbclose(root))

202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
    # init btree with many leaf nodes
    leafv = []
    root['btree'] = B = IOBTree()
    for i in range(10000):
        B[i] = xi = XInt(i)
        leafv.append(xi)
    transaction.commit()

    for npass in range(2):
        # access all elements making them live
        for _ in B.values():
            _._p_activate()

        # now B or/and some leaf nodes should be up-to-date and in cache
        cached = objscachedv(root._p_jar)
        nlive = 0
        for obj in [B] + leafv:
            if obj._p_state == UPTODATE:
                assert obj in cached
                nlive += 1
        assert nlive > 0

        # check how deactivate_btree() works dependently from initially BTree state
        if npass == 0:
            B._p_activate()
        else:
            B._p_deactivate()

        # after btree deactivation B & all leaf nodes should be in ghost state and not in cache
        deactivate_btree(B)
        cached = objscachedv(root._p_jar)
        for obj in [B] + leafv:
            assert obj._p_state == GHOST
            assert obj not in cached
236 237 238 239 240


# verify that zconn_at gives correct answer.
@func
def test_zconn_at():
241 242 243
    if zmajor == 4 and not _zhasNXDPatch('conn:MVCC-via-loadBefore-only'):
        pytest.xfail(reason="zconn_at needs https://lab.nexedi.com/nexedi/ZODB/merge_requests/1 to work on ZODB4")

244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
    stor = testdb.getZODBStorage()
    defer(stor.close)
    db  = DB(stor)

    zsync(stor)
    at0 = stor.lastTransaction()

    # open connection, it must be viewing the database @at0
    tm1 = TransactionManager()
    conn1 = db.open(transaction_manager=tm1)
    assert zconn_at(conn1) == at0

    # open another simultaneous connection
    tm2 = TransactionManager()
    conn2 = db.open(transaction_manager=tm2)
    assert zconn_at(conn2) == at0

    # commit in conn1
    root1 = conn1.root()
    root1['z'] = 1
    tm1.commit()
    zsync(stor)
    at1 = stor.lastTransaction()

    # after commit conn1 view is updated; conn2 view stays @at0
    assert zconn_at(conn1) == at1
    assert zconn_at(conn2) == at0

    # reopen conn1 -> view @at1
    conn1.close()
    with raises(POSException.ConnectionStateError):
        zconn_at(conn1)
    assert zconn_at(conn2) == at0
    conn1_ = db.open(transaction_manager=tm1)
    assert conn1_ is conn1   # returned from DB pool
    assert zconn_at(conn1) == at1
    assert zconn_at(conn2) == at0
    conn1.close()

    # commit empty transaction - view stays in sync with storage head
    conn1_ = db.open(transaction_manager=tm1)
    assert conn1_ is conn1   # from DB pool
    assert zconn_at(conn1) == at1
    assert zconn_at(conn2) == at0
    tm1.commit()
    zsync(stor)
    at1_ = stor.lastTransaction()

    assert zconn_at(conn1) == at1_
    assert zconn_at(conn2) == at0


Kirill Smelkov's avatar
Kirill Smelkov committed
296
    # reopen conn2 -> view updated to @at1_
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
    conn2.close()
    conn2_ = db.open(transaction_manager=tm1)
    assert conn2_ is conn2  # from DB pool
    assert zconn_at(conn1) == at1_
    assert zconn_at(conn2) == at1_

    conn1.close()
    conn2.close()


    # verify with historic connection @at0
    tm_old = TransactionManager()
    defer(tm_old.abort)
    conn_at0 = db.open(transaction_manager=tm_old, at=at0)
    assert conn_at0 is not conn1
    assert conn_at0 is not conn2
    assert zconn_at(conn_at0) == at0


316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
# verify that ZODB.Connection.onResyncCallback works
@func
def test_zodb_onresync():
    stor = testdb.getZODBStorage()
    defer(stor.close)
    db  = DB(stor)

    class T:
        def __init__(t):
            t.nresync = 0
        def on_connection_resync(t):
            t.nresync += 1

    t = T()

    conn = db.open()
    conn.onResyncCallback(t)
    assert t.nresync == 0

    # abort makes conn to enter new transaction
    transaction.abort()
    assert t.nresync == 1

    # close/reopen -> new transaction
    conn.close()
    assert t.nresync == 1
    conn_ = db.open()
    assert conn_ is conn
    assert t.nresync == 2

    # commit -> new transaction
    root = conn.root()
    root['r'] = 1
    assert t.nresync == 2
    transaction.commit()
    assert t.nresync == 3
    transaction.commit()
    assert t.nresync == 4
    transaction.commit()
    assert t.nresync == 5

    conn.close()


360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393
# verify that ZODB.Connection.onShutdownCallback works
@func
def test_zodb_onshutdown():
    stor = testdb.getZODBStorage()
    defer(stor.close)
    db  = DB(stor)

    class T:
        def __init__(t):
            t.nshutdown = 0
        def on_connection_shutdown(t):
            t.nshutdown += 1

    t1 = T()
    t2 = T()

    # conn1 stays alive outside of db.pool
    conn1 = db.open()
    conn1.onShutdownCallback(t1)

    # conn2 stays alive inside db.pool
    conn2 = db.open()
    conn2.onShutdownCallback(t2)
    conn2.close()

    assert t1.nshutdown == 0
    assert t2.nshutdown == 0

    # db.close triggers conn1 and conn2 shutdown
    db.close()
    assert t1.nshutdown == 1
    assert t2.nshutdown == 1


394 395
# test that zurl does not change from one open to another storage open.
def test_zurlstable():
396 397
    if not isinstance(testdb, (testing.TestDB_FileStorage, testing.TestDB_ZEO, testing.TestDB_NEO)):
        pytest.xfail(reason="zstor_2zurl is TODO for %r" % testdb)
398 399 400 401 402 403 404 405 406 407 408
    zurl0 = None
    for i in range(10):
        zstor = testdb.getZODBStorage()
        zurl  = zstor_2zurl(zstor)
        zstor.close()
        if i == 0:
            zurl0 = zurl
        else:
            assert zurl == zurl0


409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430
# ---- tests for critical properties of ZODB ----

# verify race in between Connection.open and invalidations.
def test_zodb_zopenrace_basic():
    # exercises mostly logic inside ZODB around ZODB.Connection
    zopenrace.test(MappingStorage())
def test_zodb_zopenrace():
    # exercises ZODB.Connection + particular storage implementation
    zopenrace.main()

# verify race in between loading and invalidations.
def test_zodb_zloadrace():
    # skip testing with FileStorage - in ZODB/py opening simultaneous read-write
    # connections to the same file is not supported and will raise LockError.
    _ = testdb.getZODBStorage()
    _.close()
    if isinstance(_, FileStorage):
        pytest.skip("skipping on FileStorage")

    zloadrace.main()


431 432 433 434 435 436 437 438
# ---- misc ----

# zsync syncs ZODB storage.
# it is noop, if zstor does not support syncing (i.e. FileStorage has no .sync())
def zsync(zstor):
    sync = getattr(zstor, 'sync', None)
    if sync is not None:
        sync()