Commit 805bf36f authored by Kirill Smelkov's avatar Kirill Smelkov

loadAt -> loadBeforeEx

@d-maurer suggests[1]:

    The ZODB logic relating to historical data (including MVCC) was largely
    centered around before. You have changed this to at - requiring wide spread
    modifications. I would much prefer to keep the before centered approach...

    (https://github.com/zopefoundation/ZODB/pull/323#pullrequestreview-650963363)

So let's change "at"-based logic to "before"-based logic and rename the new
method from loadAt to loadBeforeEx.
parent 55261f31
......@@ -59,7 +59,8 @@ class BaseStorage(UndoLogCompatible):
If it stores multiple revisions, it should implement
loadSerial()
loadAt()
loadBefore()
loadBeforeEx()
Each storage will have two locks that are accessed via lock
acquire and release methods bound to the instance. (Yuck.)
......@@ -267,7 +268,7 @@ class BaseStorage(UndoLogCompatible):
raise POSException.Unsupported(
"Retrieval of historical revisions is not supported")
# do not provide loadAt/loadBefore here in BaseStorage - if child forgets
# do not provide loadBeforeEx/loadBefore here in BaseStorage - if child forgets
# to override it - storage will always return "no data" instead of failing.
def copyTransactionsFrom(self, other, verbose=0):
......
......@@ -726,7 +726,8 @@ class DB(object):
- `before`: like `at`, but opens the readonly state before the
tid or datetime.
"""
# `at` is normalized to `before`.
# `at` is normalized to `before`, since we use storage.loadBeforeEx
# as the underlying implementation of both.
before = getTID(at, before)
if (before is not None and
before > self.lastTransaction() and
......
......@@ -218,34 +218,27 @@ class DemoStorage(ConflictResolvingStorage):
# still want load for old clients (e.g. zeo servers)
load = load_current
def loadAt(self, oid, at):
data, serial = ZODB.utils.loadAt(self.changes, oid, at)
def loadBeforeEx(self, oid, before):
data, serial = ZODB.utils.loadBeforeEx(self.changes, oid, before)
if (data is not None) or (serial != ZODB.utils.z64):
# object is present in changes either as data or deletion record.
return data, serial
# object is not present in changes at all - use base
return ZODB.utils.loadAt(self.base, oid, at)
return ZODB.utils.loadBeforeEx(self.base, oid, before)
def loadBefore(self, oid, before):
warnings.warn("loadBefore is deprecated - use loadAt instead",
warnings.warn("loadBefore is deprecated - use loadBeforeEx instead",
DeprecationWarning, stacklevel=2)
p64 = ZODB.utils.p64
u64 = ZODB.utils.u64
if before in (maxtid, ZODB.utils.z64):
at = before
else:
at = p64(u64(before)-1)
data, serial = self.loadAt(oid, at)
data, serial = self.loadBeforeEx(oid, before)
# find out next_serial.
# it is ok to use dumb/slow implementation since loadBefore should not
# be used and is provided only for backward compatibility.
next_serial = maxtid
while 1:
_, s = self.loadAt(oid, p64(u64(next_serial)-1))
_, s = self.loadBeforeEx(oid, next_serial)
assert s >= serial
if s == serial:
# found - next_serial is serial of the next data record
......
......@@ -58,7 +58,7 @@ from ZODB.interfaces import IStorageCurrentRecordIteration
from ZODB.interfaces import IStorageIteration
from ZODB.interfaces import IStorageRestoreable
from ZODB.interfaces import IStorageUndoable
from ZODB.interfaces import IStorageLoadAt
from ZODB.interfaces import IStorageLoadBeforeEx
from ZODB.POSException import ConflictError
from ZODB.POSException import MultipleUndoErrors
from ZODB.POSException import POSKeyError
......@@ -135,7 +135,7 @@ class TempFormatter(FileStorageFormatter):
IStorageCurrentRecordIteration,
IExternalGC,
IStorage,
IStorageLoadAt,
IStorageLoadBeforeEx,
)
class FileStorage(
FileStorageFormatter,
......@@ -562,8 +562,8 @@ class FileStorage(
else:
return self._loadBack_impl(oid, h.back)[0]
def loadAt(self, oid, at):
"""loadAt implements IStorageLoadAt."""
def loadBeforeEx(self, oid, before):
"""loadBeforeEx implements IStorageLoadBeforeEx."""
with self._files.get() as _file:
try:
pos = self._lookup_pos(oid)
......@@ -573,14 +573,14 @@ class FileStorage(
while 1:
h = self._read_data_header(pos, oid, _file)
if h.tid <= at:
if h.tid < before:
break
pos = h.prev
if not pos:
# object not yet created as of at
# object not yet created as of <before
return None, z64
# h is the most recent DataHeader with .tid <= at
# h is the most recent DataHeader with .tid < before
if h.plen:
# regular data record
return _file.read(h.plen), h.tid
......@@ -594,7 +594,7 @@ class FileStorage(
return None, h.tid
def loadBefore(self, oid, tid):
warnings.warn("loadBefore is deprecated - use loadAt instead",
warnings.warn("loadBefore is deprecated - use loadBeforeEx instead",
DeprecationWarning, stacklevel=2)
with self._files.get() as _file:
pos = self._lookup_pos(oid)
......
......@@ -31,7 +31,7 @@ import zope.interface
@zope.interface.implementer(
ZODB.interfaces.IStorage,
ZODB.interfaces.IStorageIteration,
ZODB.interfaces.IStorageLoadAt,
ZODB.interfaces.IStorageLoadBeforeEx,
)
class MappingStorage(object):
"""In-memory storage implementation
......@@ -150,15 +150,16 @@ class MappingStorage(object):
load = ZODB.utils.load_current
# ZODB.interfaces.IStorageLoadAt
# ZODB.interfaces.IStorageLoadBeforeEx
@ZODB.utils.locked(opened)
def loadAt(self, oid, at):
def loadBeforeEx(self, oid, before):
z64 = ZODB.utils.z64
tid_data = self._data.get(oid)
if not tid_data:
return None, z64
if at == z64:
if before == z64:
return None, z64
at = ZODB.utils.p64(ZODB.utils.u64(before)-1)
tids_at = tid_data.keys(None, at)
if not tids_at:
return None, z64
......@@ -168,7 +169,7 @@ class MappingStorage(object):
# ZODB.interfaces.IStorage
@ZODB.utils.locked(opened)
def loadBefore(self, oid, tid):
warnings.warn("loadBefore is deprecated - use loadAt instead",
warnings.warn("loadBefore is deprecated - use loadBeforeEx instead",
DeprecationWarning, stacklevel=2)
tid_data = self._data.get(oid)
if tid_data:
......
......@@ -866,8 +866,7 @@ class BlobStorage(BlobStorageMixin):
for oid in self.fshelper.getOIDsForSerial(serial_id):
# we want to find the serial id of the previous revision
# of this blob object.
at_before = utils.p64(utils.u64(serial_id)-1)
_, serial_before = utils.loadAt(self, oid, at_before)
_, serial_before = utils.loadBeforeEx(self, oid, serial_id)
if serial_before == utils.z64:
# There was no previous revision of this blob
......
......@@ -36,7 +36,7 @@ development continues on a "development" head.
A database can be opened historically ``at`` or ``before`` a given transaction
serial or datetime. Here's a simple example. It should work with any storage
that supports ``loadAt`` or ``loadBefore``.
that supports ``loadBeforeEx`` or ``loadBefore``.
We'll begin our example with a fairly standard set up. We
......@@ -138,9 +138,10 @@ root.
>>> historical_conn.root()['first']['count']
0
In fact, ``at`` arguments are translated into ``before`` values.
When you look at a connection's ``before`` attribute, it is normalized into a
``before`` serial, no matter what you pass into ``db.open``.
In fact, ``at`` arguments are translated into ``before`` values because the
underlying mechanism is a storage's loadBeforeEx method. When you look at a
connection's ``before`` attribute, it is normalized into a ``before`` serial,
no matter what you pass into ``db.open``.
>>> print(conn.before)
None
......
......@@ -711,7 +711,7 @@ class IStorage(Interface):
"""Load the object data written before a transaction id
( This method is deprecated and kept for backward-compatibility.
Please use loadAt instead. )
Please use loadBeforeEx instead. )
If there isn't data before the object before the given
transaction, then None is returned, otherwise three values are
......@@ -889,20 +889,20 @@ class IPrefetchStorage(IStorage):
more than once.
"""
class IStorageLoadAt(Interface):
class IStorageLoadBeforeEx(Interface):
def loadAt(oid, at): # -> (data, serial)
"""Load object data as observed at given database state.
def loadBeforeEx(oid, before): # -> (data, serial)
"""Load object data as observed before given database state.
loadAt returns data for object with given object ID as observed by
database state ≤ at. Two values are returned:
loadBeforeEx returns data for object with given object ID as observed by
most recent database transaction with ID < before. Two values are returned:
- The data record,
- The transaction ID of the data record.
If the object does not exist, or is deleted as of `at` database state,
loadAt returns data=None, and serial indicates transaction ID of the
most recent deletion done in transaction with ID ≤ at, or null tid if
If the object does not exist, or is deleted as of requested database state,
loadBeforeEx returns data=None, and serial indicates transaction ID of the
most recent deletion done in transaction with ID < before, or null tid if
there is no such deletion.
Note: no POSKeyError is raised even if object id is not in the storage.
......
......@@ -10,7 +10,7 @@ also simplifies the implementation of the DB and Connection classes.
import zope.interface
from . import interfaces, serialize, POSException
from .utils import p64, u64, z64, maxtid, Lock, loadAt, oid_repr, tid_repr
from .utils import p64, u64, z64, maxtid, Lock, loadBeforeEx, oid_repr, tid_repr
class Base(object):
......@@ -99,7 +99,7 @@ class MVCCAdapterInstance(Base):
'checkCurrentSerialInTransaction', 'tpc_abort',
)
_start = None # Transaction start time (before)
_start = None # Transaction start time
_ltid = b'' # Last storage transaction id
def __init__(self, base):
......@@ -151,16 +151,15 @@ class MVCCAdapterInstance(Base):
def load(self, oid):
assert self._start is not None
at = p64(u64(self._start)-1)
data, serial = loadAt(self._storage, oid, at)
data, serial = loadBeforeEx(self._storage, oid, self._start)
if data is None:
# raise POSKeyError if object does not exist at all
# TODO raise POSKeyError always and switch to raising ReadOnlyError only when
# actually detecting that load is being affected by simultaneous pack (see below).
if serial == z64:
# XXX second call to loadAt - it will become unneeded once we
# XXX second call to loadBeforeEx - it will become unneeded once we
# switch to raising POSKeyError.
_, serial_exists = loadAt(self._storage, oid, maxtid)
_, serial_exists = loadBeforeEx(self._storage, oid, maxtid)
if serial_exists == z64:
# object does not exist at all
raise POSException.POSKeyError(oid)
......@@ -272,8 +271,7 @@ class HistoricalStorageAdapter(Base):
new_oid = pack = store = read_only_writer
def load(self, oid, version=''):
at = p64(u64(self._before)-1)
data, serial = loadAt(self._storage, oid, at)
data, serial = loadBeforeEx(self._storage, oid, self._before)
if data is None:
raise POSException.POSKeyError(oid)
return data, serial
......
......@@ -16,7 +16,6 @@ A create_storage function is provided that creates a storage.
>>> transaction.commit()
>>> oid0 = conn.root()[0]._p_oid
>>> oid1 = conn.root()[1]._p_oid
>>> atLive = conn.root()._p_serial
>>> del conn.root()[0]
>>> del conn.root()[1]
>>> transaction.commit()
......@@ -67,10 +66,10 @@ Now if we try to load data for the objects, we get a POSKeyError:
We can still get the data if we load before the time we deleted.
>>> from ZODB.utils import loadAt, z64
>>> loadAt(storage, oid0, atLive) == (p0, s0)
>>> from ZODB.utils import loadBeforeEx, z64
>>> loadBeforeEx(storage, oid0, conn.root()._p_serial) == (p0, s0)
True
>>> loadAt(storage, oid1, atLive) == (p1, s1)
>>> loadBeforeEx(storage, oid1, conn.root()._p_serial) == (p1, s1)
True
>>> with open(storage.loadBlob(oid1, s1)) as fp: fp.read()
'some data'
......@@ -94,10 +93,10 @@ gone:
...
POSKeyError: ...
>>> loadAt(storage, oid0, atLive) == (None, z64)
>>> loadBeforeEx(storage, oid0, conn.root()._p_serial) == (None, z64)
True
>>> loadAt(storage, oid1, atLive) == (None, z64)
>>> loadBeforeEx(storage, oid1, conn.root()._p_serial) == (None, z64)
True
>>> storage.loadBlob(oid1, s1) # doctest: +ELLIPSIS
......
......@@ -46,7 +46,7 @@ class MVCCMappingStorage(MappingStorage):
inst.new_oid = self.new_oid
inst.pack = self.pack
inst.loadBefore = self.loadBefore
inst.loadAt = self.loadAt
inst.loadBeforeEx = self.loadBeforeEx
inst._ltid = self._ltid
inst._main_lock = self._lock
return inst
......
......@@ -39,13 +39,13 @@ class HexStorage(object):
setattr(self, name, v)
zope.interface.directlyProvides(self, zope.interface.providedBy(base))
if hasattr(base, 'loadAt') and 'loadAt' not in self.copied_methods:
def loadAt(oid, at):
data, serial = self.base.loadAt(oid, at)
if hasattr(base, 'loadBeforeEx') and 'loadBeforeEx' not in self.copied_methods:
def loadBeforeEx(oid, before):
data, serial = self.base.loadBeforeEx(oid, before)
if data is not None:
data = unhexlify(data[2:])
return data, serial
self.loadAt = loadAt
self.loadBeforeEx = loadBeforeEx
def __getattr__(self, name):
return getattr(self.base, name)
......@@ -137,7 +137,7 @@ class ServerHexStorage(HexStorage):
"""
copied_methods = HexStorage.copied_methods + (
'load', 'loadAt', 'loadBefore', 'loadSerial', 'store', 'restore',
'load', 'loadBeforeEx', 'loadBefore', 'loadSerial', 'store', 'restore',
'iterator', 'storeBlob', 'restoreBlob', 'record_iternext',
)
......
......@@ -1314,7 +1314,7 @@ class StubStorage(object):
raise TypeError('StubStorage does not support versions.')
return self._data[oid]
def loadAt(self, oid, at):
def loadBeforeEx(self, oid, before):
try:
data, serial = self._transdata[oid]
except KeyError:
......@@ -1322,7 +1322,7 @@ class StubStorage(object):
return data, serial
def loadBefore(self, oid, tid):
warnings.warn("loadBefore is deprecated - use loadAt instead",
warnings.warn("loadBefore is deprecated - use loadBeforeEx instead",
DeprecationWarning, stacklevel=2)
return self._data[oid] + (None, )
......
......@@ -24,7 +24,7 @@ import warnings
from ZODB.BaseStorage import BaseStorage
from ZODB import POSException
from ZODB.utils import p64, u64, z64
from ZODB.utils import z64
from ZODB.tests import StorageTestBase
from ZODB.tests import BasicStorage, MTStorage, Synchronization
......@@ -106,7 +106,7 @@ class MinimalMemoryStorage(BaseStorage, object):
self._ltid = self._tid
def loadBefore(self, the_oid, the_tid):
warnings.warn("loadBefore is deprecated - use loadAt instead",
warnings.warn("loadBefore is deprecated - use loadBeforeEx instead",
DeprecationWarning, stacklevel=2)
return self._loadBefore(the_oid, the_tid)
......@@ -132,9 +132,9 @@ class MinimalMemoryStorage(BaseStorage, object):
return self._index[(the_oid, tid)], tid, end_tid
def loadAt(self, oid, at):
def loadBeforeEx(self, oid, before):
try:
r = self._loadBefore(oid, p64(u64(at)+1))
r = self._loadBefore(oid, before)
except KeyError:
return None, z64
if r is None:
......
......@@ -35,7 +35,7 @@ MinimalMemoryStorage that implements MVCC support, but not much else.
***IMPORTANT***: The MVCC approach has changed since these tests were
originally written. The new approach is much simpler because we no
longer call load to get the current state of an object. We call
loadAt instead, having gotten a transaction time at the start of a
loadBeforeEx instead, having gotten a transaction time at the start of a
transaction. As a result, the rhythm of the tests is a little odd,
because we no longer need to probe a complex dance that doesn't exist any more.
......@@ -290,7 +290,7 @@ first connection's state for b "old".
Now deactivate "b" in the first connection, and (re)fetch it. The first
connection should still see 1, due to MVCC, but to get this old state
TmpStore needs to handle the loadAt() or loadBefore() methods.
TmpStore needs to handle the loadBeforeEx() or loadBefore() methods.
>>> r1["b"]._p_deactivate()
......@@ -322,7 +322,7 @@ why ZODB no-longer calls load. :)
Rather than add all the complexity of ZEO to these tests, the
MinimalMemoryStorage has a hook. We'll write a subclass that will
deliver an invalidation when it loads (or loadAt's) an object.
deliver an invalidation when it loads (or loadBeforeEx's) an object.
The hook allows us to test the Connection code.
>>> class TestStorage(MinimalMemoryStorage):
......
......@@ -383,41 +383,36 @@ def load_current(storage, oid, version=''):
some time in the future.
"""
assert not version
data, serial = loadAt(storage, oid, maxtid)
data, serial = loadBeforeEx(storage, oid, maxtid)
if data is None:
raise ZODB.POSException.POSKeyError(oid)
return data, serial
_loadAtWarned = set() # of storage class
def loadAt(storage, oid, at):
"""loadAt provides IStorageLoadAt semantic for all storages.
_loadBeforeExWarned = set() # of storage class
def loadBeforeEx(storage, oid, before):
"""loadBeforeEx provides IStorageLoadBeforeEx semantic for all storages.
Storages that do not implement loadAt are served via loadBefore.
Storages that do not implement loadBeforeEx are served via loadBefore.
"""
load_at = getattr(storage, 'loadAt', None)
if load_at is not None:
return load_at(oid, at)
loadBeforeEx = getattr(storage, 'loadBeforeEx', None)
if loadBeforeEx is not None:
return loadBeforeEx(oid, before)
# storage does not provide IStorageLoadAt - warn + fall back to loadBefore
if type(storage) not in _loadAtWarned:
# there is potential race around _loadAtWarned access, but due to the
# storage does not provide IStorageLoadBeforeEx - warn + fall back to loadBefore
if type(storage) not in _loadBeforeExWarned:
# there is potential race around _loadBeforeExWarned access, but due to the
# GIL this race cannot result in that set corruption, and can only lead
# to us emitting the warning twice instead of just once.
# -> do not spend CPU on lock and just ignore it.
warnings.warn(
"FIXME %s does not provide loadAt - emulating it via loadBefore, but ...\n"
"FIXME %s does not provide loadBeforeEx - emulating it via loadBefore, but ...\n"
"\t... 1) access will be potentially slower, and\n"
"\t... 2) not full semantic of loadAt could be provided.\n"
"\t... 2) not full semantic of loadBeforeEx could be provided.\n"
"\t... this can lead to data corruption.\n"
"\t... -> please see https://github.com/zopefoundation/ZODB/issues/318 for details." %
type(storage), DeprecationWarning)
_loadAtWarned.add(type(storage))
if at == maxtid:
before = at
else:
before = p64(u64(at)+1)
_loadBeforeExWarned.add(type(storage))
try:
r = storage.loadBefore(oid, before)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment