Commit c02776e9 authored by Kirill Smelkov's avatar Kirill Smelkov

bigfile/zodb: Factor-out LivePersistent into -> lib/zodb

It was from long-ago marked as "XXX move to common place".
parent 8c32c9f6
...@@ -29,7 +29,8 @@ inherits most of the properties and specifics from its parents. ...@@ -29,7 +29,8 @@ inherits most of the properties and specifics from its parents.
""" """
from wendelin.bigarray import BigArray from wendelin.bigarray import BigArray
from wendelin.bigfile.file_zodb import ZBigFile, LivePersistent from wendelin.bigfile.file_zodb import ZBigFile
from wendelin.lib.zodb import LivePersistent
import inspect import inspect
......
...@@ -18,9 +18,10 @@ ...@@ -18,9 +18,10 @@
# See COPYING file for full licensing terms. # See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options. # See https://www.nexedi.com/licensing for rationale and options.
from wendelin.bigarray.array_zodb import ZBigArray from wendelin.bigarray.array_zodb import ZBigArray
from wendelin.bigfile.tests.test_filezodb import kkey, cacheInfo, ram_reclaim_all from wendelin.bigfile.tests.test_filezodb import ram_reclaim_all
from wendelin.bigfile.tests.test_thread import NotifyChannel from wendelin.bigfile.tests.test_thread import NotifyChannel
from wendelin.lib.zodb import dbclose from wendelin.lib.zodb import dbclose
from wendelin.lib.tests.test_zodb import cacheInfo, kkey
from wendelin.lib.testing import getTestDB from wendelin.lib.testing import getTestDB
from persistent import UPTODATE from persistent import UPTODATE
import transaction import transaction
......
...@@ -133,7 +133,7 @@ will be our future approach after we teach NEO about object deduplication. ...@@ -133,7 +133,7 @@ will be our future approach after we teach NEO about object deduplication.
from wendelin.bigfile import BigFile, WRITEOUT_STORE, WRITEOUT_MARKSTORED from wendelin.bigfile import BigFile, WRITEOUT_STORE, WRITEOUT_MARKSTORED
from wendelin.lib.mem import bzero, memcpy from wendelin.lib.mem import bzero, memcpy
from wendelin.lib.zodb import deactivate_btree from wendelin.lib.zodb import LivePersistent, deactivate_btree
from transaction.interfaces import IDataManager, ISynchronizer from transaction.interfaces import IDataManager, ISynchronizer
from persistent import Persistent, PickleCache, GHOST from persistent import Persistent, PickleCache, GHOST
...@@ -482,33 +482,6 @@ class _ZBigFile(BigFile): ...@@ -482,33 +482,6 @@ class _ZBigFile(BigFile):
# Persistent that never goes to ghost state, if it was ever uptodate.
#
# NOTE
#
# On invalidation LivePersistent still goes to ghost state, because
# invalidation cannot be ignored, i.e. they indicate the object has been
# changed externally.
#
# Invalidation can happen only at transaction boundary, so during the course of
# transaction LivePersistent is guaranteed to stay uptodate.
#
# XXX move to common place?
class LivePersistent(Persistent):
# don't allow us to go to ghost
#
# NOTE we can't use STICKY as that state is assumed as
# short-lived-temporary by ZODB and is changed back to UPTODATE by
# persistent code. In fact ZODB says: STICKY is UPTODATE+keep in memory.
def _p_deactivate(self):
# just returning here won't allow Persistent._p_deactivate() run and
# thus we'll stay in non-ghost state.
return
# NOTE _p_invalidate() is triggered on invalidations. We do not override it.
# ZBigFile implements BigFile backend with data stored in ZODB. # ZBigFile implements BigFile backend with data stored in ZODB.
# #
# NOTE Can't inherit from Persistent and BigFile at the same time - both are C # NOTE Can't inherit from Persistent and BigFile at the same time - both are C
......
...@@ -17,12 +17,13 @@ ...@@ -17,12 +17,13 @@
# #
# See COPYING file for full licensing terms. # See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options. # See https://www.nexedi.com/licensing for rationale and options.
from wendelin.bigfile.file_zodb import LivePersistent, ZBigFile, ZBlk_fmt_registry from wendelin.bigfile.file_zodb import ZBigFile, ZBlk_fmt_registry
from wendelin.bigfile import file_zodb, ram_reclaim from wendelin.bigfile import file_zodb, ram_reclaim
from wendelin.bigfile.tests.test_thread import NotifyChannel from wendelin.bigfile.tests.test_thread import NotifyChannel
from wendelin.lib.zodb import dbclose from wendelin.lib.zodb import LivePersistent, dbclose
from wendelin.lib.tests.test_zodb import cacheInfo, kkey
from wendelin.lib.testing import getTestDB from wendelin.lib.testing import getTestDB
from persistent import UPTODATE, GHOST, CHANGED from persistent import UPTODATE
import transaction import transaction
from transaction import TransactionManager from transaction import TransactionManager
from ZODB.POSException import ConflictError from ZODB.POSException import ConflictError
...@@ -68,149 +69,6 @@ def ram_reclaim_all(): ...@@ -68,149 +69,6 @@ def ram_reclaim_all():
return reclaimed return reclaimed
# like db.cacheDetail(), but {} instead of []
def cacheInfo(db):
return dict(db.cacheDetail())
# key for cacheInfo() result
def kkey(klass):
return '%s.%s' % (klass.__module__, klass.__name__)
@func
def test_livepersistent():
root = dbopen()
transaction.commit() # set root._p_jar
db = root._p_jar.db()
# ~~~ test `obj initially created` case
root['live'] = lp = LivePersistent()
assert lp._p_jar is None # connection does not know about it yet
assert lp._p_state == UPTODATE # object initially created in uptodate
# should not be in cache yet & thus should stay after gc
db.cacheMinimize()
assert lp._p_jar is None
assert lp._p_state == UPTODATE
ci = cacheInfo(db)
assert kkey(LivePersistent) not in ci
# should be registered to connection & cache after commit
transaction.commit()
assert lp._p_jar is not None
assert lp._p_state == UPTODATE
ci = cacheInfo(db)
assert ci[kkey(LivePersistent)] == 1
# should stay that way after cache gc
db.cacheMinimize()
assert lp._p_jar is not None
assert lp._p_state == UPTODATE
ci = cacheInfo(db)
assert ci[kkey(LivePersistent)] == 1
# ~~~ reopen & test `obj loaded from db` case
dbclose(root)
del root, db, lp
root = dbopen()
db = root._p_jar.db()
# known to connection & cache & GHOST
# right after first loading from DB
lp = root['live']
assert lp._p_jar is not None
assert lp._p_state is GHOST
ci = cacheInfo(db)
assert ci[kkey(LivePersistent)] == 1
# should be UPTODATE for sure after read access
getattr(lp, 'attr', None)
assert lp._p_jar is not None
assert lp._p_state is UPTODATE
ci = cacheInfo(db)
assert ci[kkey(LivePersistent)] == 1
# does not go back to ghost on cache gc
db.cacheMinimize()
assert lp._p_jar is not None
assert lp._p_state == UPTODATE
ci = cacheInfo(db)
assert ci[kkey(LivePersistent)] == 1
# ok
dbclose(root)
del root, db, lp
# demo that upon cache invalidation LivePersistent can go back to ghost
root = dbopen()
conn = root._p_jar
db = conn.db()
conn.close()
del root, conn
tm1 = TransactionManager()
tm2 = TransactionManager()
conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root()
defer(lambda: dbclose(root1))
lp1 = root1['live']
conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root()
defer(conn2.close)
lp2 = root2['live']
# 2 connections are setup running in parallel with initial obj state as ghost
assert lp1._p_jar is conn1
assert lp2._p_jar is conn2
assert lp1._p_state is GHOST
assert lp2._p_state is GHOST
# conn1: modify ghost -> changed
lp1.attr = 1
assert lp1._p_state is CHANGED
assert lp2._p_state is GHOST
# conn2: read ghost -> uptodate
assert getattr(lp1, 'attr', None) == 1
assert getattr(lp2, 'attr', None) is None
assert lp1._p_state is CHANGED
assert lp2._p_state is UPTODATE
# conn1: commit changed -> uptodate; conn2 untouched
tm1.commit()
assert lp1._p_state is UPTODATE
assert lp2._p_state is UPTODATE
assert getattr(lp1, 'attr', None) == 1
assert getattr(lp2, 'attr', None) is None
# conn2: commit (nothing changed - just transaction boundary)
# uptodate -> ghost (invalidation)
tm2.commit()
assert lp1._p_state is UPTODATE
assert lp2._p_state is GHOST
assert getattr(lp1, 'attr', None) == 1
# conn2: after reading, the state is again uptodate + changes from conn1 are here
a = getattr(lp2, 'attr', None)
assert lp2._p_state is UPTODATE
assert a == 1
del conn2, root2
# i'th memory block as u32 ndarray # i'th memory block as u32 ndarray
blksize32 = blksize // 4 blksize32 = blksize // 4
def Blk(vma, i): def Blk(vma, i):
......
...@@ -17,11 +17,12 @@ ...@@ -17,11 +17,12 @@
# #
# See COPYING file for full licensing terms. # See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options. # See https://www.nexedi.com/licensing for rationale and options.
from wendelin.lib.zodb import deactivate_btree, dbclose from wendelin.lib.zodb import LivePersistent, deactivate_btree, dbclose
from wendelin.lib.testing import getTestDB from wendelin.lib.testing import getTestDB
from persistent import Persistent, UPTODATE, GHOST from persistent import Persistent, UPTODATE, GHOST, CHANGED
from BTrees.IOBTree import IOBTree from BTrees.IOBTree import IOBTree
import transaction import transaction
from transaction import TransactionManager
from golang import defer, func from golang import defer, func
import gc import gc
...@@ -38,6 +39,146 @@ def setup_module(): ...@@ -38,6 +39,146 @@ def setup_module():
def teardown_module(): def teardown_module():
testdb.teardown() testdb.teardown()
# like db.cacheDetail(), but {} instead of []
def cacheInfo(db):
return dict(db.cacheDetail())
# key for cacheInfo() result
def kkey(klass):
return '%s.%s' % (klass.__module__, klass.__name__)
@func
def test_livepersistent():
root = dbopen()
transaction.commit() # set root._p_jar
db = root._p_jar.db()
# ~~~ test `obj initially created` case
root['live'] = lp = LivePersistent()
assert lp._p_jar is None # connection does not know about it yet
assert lp._p_state == UPTODATE # object initially created in uptodate
# should not be in cache yet & thus should stay after gc
db.cacheMinimize()
assert lp._p_jar is None
assert lp._p_state == UPTODATE
ci = cacheInfo(db)
assert kkey(LivePersistent) not in ci
# should be registered to connection & cache after commit
transaction.commit()
assert lp._p_jar is not None
assert lp._p_state == UPTODATE
ci = cacheInfo(db)
assert ci[kkey(LivePersistent)] == 1
# should stay that way after cache gc
db.cacheMinimize()
assert lp._p_jar is not None
assert lp._p_state == UPTODATE
ci = cacheInfo(db)
assert ci[kkey(LivePersistent)] == 1
# ~~~ reopen & test `obj loaded from db` case
dbclose(root)
del root, db, lp
root = dbopen()
db = root._p_jar.db()
# known to connection & cache & GHOST
# right after first loading from DB
lp = root['live']
assert lp._p_jar is not None
assert lp._p_state is GHOST
ci = cacheInfo(db)
assert ci[kkey(LivePersistent)] == 1
# should be UPTODATE for sure after read access
getattr(lp, 'attr', None)
assert lp._p_jar is not None
assert lp._p_state is UPTODATE
ci = cacheInfo(db)
assert ci[kkey(LivePersistent)] == 1
# does not go back to ghost on cache gc
db.cacheMinimize()
assert lp._p_jar is not None
assert lp._p_state == UPTODATE
ci = cacheInfo(db)
assert ci[kkey(LivePersistent)] == 1
# ok
dbclose(root)
del root, db, lp
# demo that upon cache invalidation LivePersistent can go back to ghost
root = dbopen()
conn = root._p_jar
db = conn.db()
conn.close()
del root, conn
tm1 = TransactionManager()
tm2 = TransactionManager()
conn1 = db.open(transaction_manager=tm1)
root1 = conn1.root()
defer(lambda: dbclose(root1))
lp1 = root1['live']
conn2 = db.open(transaction_manager=tm2)
root2 = conn2.root()
defer(conn2.close)
lp2 = root2['live']
# 2 connections are setup running in parallel with initial obj state as ghost
assert lp1._p_jar is conn1
assert lp2._p_jar is conn2
assert lp1._p_state is GHOST
assert lp2._p_state is GHOST
# conn1: modify ghost -> changed
lp1.attr = 1
assert lp1._p_state is CHANGED
assert lp2._p_state is GHOST
# conn2: read ghost -> uptodate
assert getattr(lp1, 'attr', None) == 1
assert getattr(lp2, 'attr', None) is None
assert lp1._p_state is CHANGED
assert lp2._p_state is UPTODATE
# conn1: commit changed -> uptodate; conn2 untouched
tm1.commit()
assert lp1._p_state is UPTODATE
assert lp2._p_state is UPTODATE
assert getattr(lp1, 'attr', None) == 1
assert getattr(lp2, 'attr', None) is None
# conn2: commit (nothing changed - just transaction boundary)
# uptodate -> ghost (invalidation)
tm2.commit()
assert lp1._p_state is UPTODATE
assert lp2._p_state is GHOST
assert getattr(lp1, 'attr', None) == 1
# conn2: after reading, the state is again uptodate + changes from conn1 are here
a = getattr(lp2, 'attr', None)
assert lp2._p_state is UPTODATE
assert a == 1
del conn2, root2
class XInt(Persistent): class XInt(Persistent):
def __init__(self, i): def __init__(self, i):
......
...@@ -79,6 +79,31 @@ def dbclose(root): ...@@ -79,6 +79,31 @@ def dbclose(root):
stor.close() stor.close()
# LivePersistent is Persistent that never goes to ghost state, if it was ever uptodate.
#
# NOTE
#
# On invalidation LivePersistent still goes to ghost state, because
# invalidation cannot be ignored, i.e. they indicate the object has been
# changed externally.
#
# Invalidation can happen only at transaction boundary, so during the course of
# transaction LivePersistent is guaranteed to stay uptodate.
class LivePersistent(Persistent):
# don't allow us to go to ghost
#
# NOTE we can't use STICKY as that state is assumed as
# short-lived-temporary by ZODB and is changed back to UPTODATE by
# persistent code. In fact ZODB says: STICKY is UPTODATE+keep in memory.
def _p_deactivate(self):
# just returning here won't allow Persistent._p_deactivate() run and
# thus we'll stay in non-ghost state.
return
# NOTE _p_invalidate() is triggered on invalidations. We do not override it.
# deactivate a btree, including all internal buckets and leaf nodes # deactivate a btree, including all internal buckets and leaf nodes
def deactivate_btree(btree): def deactivate_btree(btree):
# first activate btree, to make sure its first bucket is loaded at all. # first activate btree, to make sure its first bucket is loaded at all.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment