Skip to content

N
neoppod
Commit edde1fe3 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

Rewrite storage cache of client 

- Stop using a list of (in)validated tid (hence removal of RevisionIndex),
  because it can't work in all cases and would even cause memory leaks.
  For example, this bug could lead to ConflictError with a single client.
  Fixit it also requires that database backends always return the next serial.
- Several performance improvements. The most important one is when the latest
  version of an object is cached: it inherits the access counter of the
  previous one (for the same oid), which gets in turn its counter reset.
- Do not waste CPU evaluating the real size taken by an entry in memory.
  Just use 'len' on the value (which is always a pickle data, i.e. a string).

git-svn-id: https://svn.erp5.org/repos/neo/trunk@2711 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent 240c94b3
Show whitespace changes
Inline Side-by-side
Showing with 250 additions and 604 deletions
neo/client/app.py
View file @ edde1fe3
...@@ -42,12 +42,11 @@ from neo.client.handlers import storage, master ...@@ -42,12 +42,11 @@ from neo.client.handlers import storage, master
from neo.lib.dispatcher import Dispatcher, ForgottenPacket from neo.lib.dispatcher import Dispatcher, ForgottenPacket
from neo.client.poll import ThreadedPoll, psThreadedPoll from neo.client.poll import ThreadedPoll, psThreadedPoll
from neo.client.iterator import Iterator from neo.client.iterator import Iterator
from neo.client.mq import MQ from neo.client.cache import ClientCache
from neo.client.pool import ConnectionPool from neo.client.pool import ConnectionPool
from neo.lib.util import u64, parseMasterList from neo.lib.util import u64, parseMasterList
from neo.lib.profiling import profiler_decorator, PROFILING_ENABLED from neo.lib.profiling import profiler_decorator, PROFILING_ENABLED
from neo.lib.debug import register as registerLiveDebugger from neo.lib.debug import register as registerLiveDebugger
from neo.client.mq_index import RevisionIndex
from neo.client.container import ThreadContainer, TransactionContainer from neo.client.container import ThreadContainer, TransactionContainer
if PROFILING_ENABLED: if PROFILING_ENABLED:
...@@ -93,9 +92,7 @@ class Application(object): ...@@ -93,9 +92,7 @@ class Application(object):
# no self-assigned UUID, primary master will supply us one # no self-assigned UUID, primary master will supply us one
self.uuid = None self.uuid = None
self.mq_cache = MQ() self._cache = ClientCache()
self.cache_revision_index = RevisionIndex()
self.mq_cache.addIndex(self.cache_revision_index)
self.new_oid_list = [] self.new_oid_list = []
self.last_oid = '\0' * 8 self.last_oid = '\0' * 8
self.storage_event_handler = storage.StorageEventHandler(self) self.storage_event_handler = storage.StorageEventHandler(self)
...@@ -432,20 +429,17 @@ class Application(object): ...@@ -432,20 +429,17 @@ class Application(object):
self._load_lock_acquire() self._load_lock_acquire()
try: try:
try: result = self._loadFromCache(oid, serial, tid)
return self._loadFromCache(oid, serial, tid) if not result:
except KeyError: result = self._loadFromStorage(oid, serial, tid)
pass
data, start_serial, end_serial = self._loadFromStorage(oid, serial,
tid)
self._cache_lock_acquire() self._cache_lock_acquire()
try: try:
self.mq_cache[(oid, start_serial)] = data, end_serial self._cache.store(oid, *result)
finally: finally:
self._cache_lock_release() self._cache_lock_release()
if data == '': if result[0] == '':
raise NEOStorageCreationUndoneError(dump(oid)) raise NEOStorageCreationUndoneError(dump(oid))
return data, start_serial, end_serial return result
finally: finally:
self._load_lock_release() self._load_lock_release()
...@@ -479,24 +473,17 @@ class Application(object): ...@@ -479,24 +473,17 @@ class Application(object):
return data, tid, next_tid return data, tid, next_tid
@profiler_decorator @profiler_decorator
def _loadFromCache(self, oid, at_tid, before_tid): def _loadFromCache(self, oid, at_tid=None, before_tid=None):
""" """
Load from local cache, raising KeyError if not found. Load from local cache, return None if not found.
""" """
self._cache_lock_acquire() self._cache_lock_acquire()
try: try:
if at_tid is not None: if at_tid:
tid = at_tid result = self._cache.load(oid, at_tid + '*')
elif before_tid is not None: assert not result or result[1] == at_tid
tid = self.cache_revision_index.getSerialBefore(oid, return result
before_tid) return self._cache.load(oid, before_tid)
else:
tid = self.cache_revision_index.getLatestSerial(oid)
if tid is None:
raise KeyError
# Raises KeyError on miss
data, next_tid = self.mq_cache[(oid, tid)]
return (data, tid, next_tid)
finally: finally:
self._cache_lock_release() self._cache_lock_release()
...@@ -808,14 +795,7 @@ class Application(object): ...@@ -808,14 +795,7 @@ class Application(object):
# Update cache # Update cache
self._cache_lock_acquire() self._cache_lock_acquire()
try: try:
mq_cache = self.mq_cache cache = self._cache
update = mq_cache.update
def updateNextSerial(value):
data, next_tid = value
assert next_tid is None, (dump(oid), dump(base_tid),
dump(next_tid))
return (data, tid)
get_baseTID = txn_context['object_base_serial_dict'].get
for oid, data in txn_context['data_dict'].iteritems(): for oid, data in txn_context['data_dict'].iteritems():
if data is None: if data is None:
# this is just a remain of # this is just a remain of
...@@ -823,16 +803,10 @@ class Application(object): ...@@ -823,16 +803,10 @@ class Application(object):
# was modified). # was modified).
continue continue
# Update ex-latest value in cache # Update ex-latest value in cache
base_tid = get_baseTID(oid) cache.invalidate(oid, tid)
try: if data:
update((oid, base_tid), updateNextSerial)
except KeyError:
pass
if data == '':
self.cache_revision_index.invalidate([oid], tid)
else:
# Store in cache with no next_tid # Store in cache with no next_tid
mq_cache[(oid, tid)] = (data, None) cache.store(oid, data, tid, None)
finally: finally:
self._cache_lock_release() self._cache_lock_release()
txn_container.delete(transaction) txn_container.delete(transaction)
...@@ -1105,7 +1079,7 @@ class Application(object): ...@@ -1105,7 +1079,7 @@ class Application(object):
# by a pack), so don't bother invalidating on other clients. # by a pack), so don't bother invalidating on other clients.
self._cache_lock_acquire() self._cache_lock_acquire()
try: try:
self.mq_cache.clear() self._cache.clear()
finally: finally:
self._cache_lock_release() self._cache_lock_release()
......
neo/client/cache.py
View file @ edde1fe3
############################################################################## ##############################################################################
# #
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved. # Copyright (c) 2011 Nexedi SARL and Contributors. All Rights Reserved.
# Yoshinori Okuji <yo@nexedi.com> # Julien Muchembled <jm@nexedi.com>
# #
# WARNING: This program as such is intended to be used by professional # WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential # programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs # consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial # End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software # guarantees and support are strongly advised to contract a Free Software
# Service Company # Service Company
# #
# This program is Free Software; you can redistribute it and/or # This program is Free Software; you can redistribute it and/or
...@@ -22,135 +22,36 @@ ...@@ -22,135 +22,36 @@
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
# #
############################################################################## ##############################################################################
""" import math
Multi-Queue Cache Algorithm.
"""
from math import log class CacheItem(object):
class MQIndex(object): __slots__ = ('oid', 'tid', 'next_tid', 'data',
""" 'counter', 'level', 'expire',
Virtual base class for MQ cache external indexes. 'prev', 'next')
"""
def clear(self):
"""
Called when MQ is cleared.
"""
raise NotImplementedError
def remove(self, key):
"""
Called when key's value is removed from cache, and key is pushed to
history buffer.
"""
raise NotImplementedError
def add(self, key):
"""
Called when key is added into cache.
It is either a new key, or a know key comming back from history
buffer.
"""
raise NotImplementedError
class Element(object):
"""
This class defines an element of a FIFO buffer.
"""
pass
class FIFO(object):
"""
This class implements a FIFO buffer.
"""
def __init__(self):
self.head = None
self.tail = None
self._len = 0
self.prev = None
self.data = None
self.next = None
self.level = None
self.counter = None
self.value = None
self.element = None
self.key = None
self.expire_time = None
def __len__(self):
return self._len
def append(self):
element = Element()
element.next = None
element.prev = self.tail
if self.tail is not None:
self.tail.next = element
self.tail = element
if self.head is None:
self.head = element
self._len += 1
return element
def shift(self):
element = self.head
if element is None:
return None
del self[element]
del element.next
del element.prev
return element
def __delitem__(self, element):
if element.next is None:
self.tail = element.prev
else:
element.next.prev = element.prev
if element.prev is None:
self.head = element.next
else:
element.prev.next = element.next
self._len -= 1 def __repr__(self):
s = ''
class Data(object): for attr in self.__slots__:
""" try:
Data for each element in a FIFO buffer. value = getattr(self, attr)
""" if value:
if attr in ('prev', 'next'):
s += ' %s=<...>' % attr
continue
elif attr == 'data':
value = '...'
s += ' %s=%r' % (attr, value)
except AttributeError:
pass pass
return '<%s%s>' % (self.__class__.__name__, s)
def sizeof(o): class ClientCache(object):
"""This function returns the estimated size of an object.""" """In-memory pickle cache based on Multi-Queue cache algorithm
if isinstance(o, tuple):
return sum((sizeof(s) for s in o))
elif o is None:
# XXX: unknown size (arch pointer ?)
return 0
else:
return len(o)+16
class MQ(object):
"""
This class manages cached data by a variant of Multi-Queue.
This class caches various sizes of objects. Here are some considerations:
- Expired objects are not really deleted immediately. But if GC is invoked too often,
it degrades the performance significantly.
- If large objects are cached, the number of cached objects decreases. This might affect
the cache hit ratio. It might be better to tweak a buffer level according to the size of
an object.
- Stored values must be strings.
- The size calculation is not accurate.
Quick description of Multi-Queue algorithm: Quick description of Multi-Queue algorithm:
- There are multiple "regular" queues, plus a history queue - There are multiple "regular" queues, plus a history queue
...@@ -159,224 +60,179 @@ class MQ(object): ...@@ -159,224 +60,179 @@ class MQ(object):
longer lifespan) longer lifespan)
-> The more often an object is accessed, the higher lifespan it will -> The more often an object is accessed, the higher lifespan it will
have have
- Upon a cache miss, the oldest entry of first non-empty queue is
transfered to history queue
- Upon cache or history hit, object frequency is increased and object - Upon cache or history hit, object frequency is increased and object
might get moved to longer-lived queue might get moved to longer-lived queue
- Each access "ages" objects in cache, and an aging object is moved to - Each access "ages" objects in cache, and an aging object is moved to
shorter-lived queue as it ages without being accessed shorter-lived queue as it ages without being accessed, or in the
history queue if it's really too old.
""" """
def __init__(self, life_time=10000, buffer_levels=9, __slots__ = ('_life_time', '_max_history_size', '_max_size',
max_history_size=100000, max_size=20*1024*1024): '_queue_list', '_oid_dict', '_time', '_size', '_history_size')
self._index_list = []
def __init__(self, life_time=10000, max_history_size=100000,
max_size=20*1024*1024):
self._life_time = life_time self._life_time = life_time
self._buffer_levels = buffer_levels
self._max_history_size = max_history_size self._max_history_size = max_history_size
self._max_size = max_size self._max_size = max_size
self.clear() self.clear()
def addIndex(self, index, reindex=True):
"""
Add an index ot this cache.
index
Object implementing methods from MQIndex class.
reindex (True)
If True, give all existing keys as new to index.
"""
if reindex:
# Index existing entries
add = index.add
for key in self._data:
add(key)
self._index_list.append(index)
def _mapIndexes(self, method_id, args=(), kw={}):
for index in self._index_list:
getattr(index, method_id)(*args, **kw)
def clear(self): def clear(self):
self._history_buffer = FIFO() """Reset cache"""
self._cache_buffers = [] self._queue_list = [None] # first is history
for level in range(self._buffer_levels): self._oid_dict = {}
self._cache_buffers.append(FIFO())
self._data = {}
self._time = 0 self._time = 0
self._size = 0 self._size = 0
self._mapIndexes('clear') self._history_size = 0
def has_key(self, key): def _iterQueue(self, level):
if key in self._data: """for debugging purpose"""
data = self._data[key] if level < len(self._queue_list):
if data.level >= 0: item = head = self._queue_list[level]
return 1 if item:
return 0 while 1:
yield item
__contains__ = has_key item = item.next
if item is head:
def fetch(self, key): break
"""
Fetch a value associated with the key.
"""
data = self._data[key]
if data.level >= 0:
value = data.value
self._size -= sizeof(value)
self.store(key, value)
return value
raise KeyError(key)
__getitem__ = fetch
def get(self, key, d=None):
try:
return self.fetch(key)
except KeyError:
return d
def _evict(self, key):
"""
Evict an element to the history buffer.
"""
self._mapIndexes('remove', (key, ))
data = self._data[key]
self._size -= sizeof(data.value)
del self._cache_buffers[data.level][data.element]
element = self._history_buffer.append()
data.level = -1
data.element = element
del data.value
del data.expire_time
element.data = data
if len(self._history_buffer) > self._max_history_size:
element = self._history_buffer.shift()
del self._data[element.data.key]
def store(self, key, value):
cache_buffers = self._cache_buffers
def _add(self, item):
level = item.level
try: try:
data = self._data[key] head = self._queue_list[level]
except KeyError: except IndexError:
counter = 1 assert len(self._queue_list) == level
self._mapIndexes('add', (key, )) self._queue_list.append(item)
item.prev = item.next = item
else: else:
level, element, counter = data.level, data.element, data.counter + 1 if head:
if level >= 0: item.prev = tail = head.prev
del cache_buffers[level][element] tail.next = head.prev = item
item.next = head
else: else:
del self._history_buffer[element] self._queue_list[level] = item
self._mapIndexes('add', (key, )) item.prev = item.next = item
if level:
item.expire = self._time + self._life_time
else:
self._size -= len(item.data)
item.data = None
if self._history_size < self._max_history_size:
self._history_size += 1
else:
self._remove(head)
item_list = self._oid_dict[head.oid]
item_list.remove(head)
if not item_list:
del self._oid_dict[head.oid]
def _remove(self, item):
level = item.level
if level is not None:
item.level = level - 1
next = item.next
if next is item:
self._queue_list[level] = next = None
else:
item.prev.next = next
next.prev = item.prev
if self._queue_list[level] is item:
self._queue_list[level] = next
return next
def _fetched(self, item, _log=math.log):
self._remove(item)
item.counter = counter = item.counter + 1
# XXX It might be better to adjust the level according to the object # XXX It might be better to adjust the level according to the object
# size. # size.
level = min(int(log(counter, 2)), self._buffer_levels - 1) item.level = 1 + int(_log(counter, 2))
element = cache_buffers[level].append() self._add(item)
data = Data()
data.key = key self._time = time = self._time + 1
data.expire_time = self._time + self._life_time for head in self._queue_list[1:]:
data.level = level if head and head.expire < time:
data.element = element self._remove(head)
data.value = value self._add(head)
data.counter = counter break
element.data = data
self._data[key] = data
self._size += sizeof(value)
del value
self._time += 1
# Expire old elements. def _load(self, oid, before_tid=None):
time = self._time item_list = self._oid_dict.get(oid)
for level, cache_buffer in enumerate(cache_buffers): if item_list:
head = cache_buffer.head if before_tid:
if head is not None and head.data.expire_time < time: for item in reversed(item_list):
del cache_buffer[head] if item.tid < before_tid:
data = head.data next_tid = item.next_tid
if level > 0: if next_tid and next_tid < before_tid:
new_level = level - 1 break
element = cache_buffers[new_level].append() return item
element.data = data
data.expire_time = time + self._life_time
data.level = new_level
data.element = element
else: else:
self._evict(data.key) item = item_list[-1]
if not item.next_tid:
# Limit the size. return item
def load(self, oid, before_tid=None):
"""Return a revision of oid that was current before given tid"""
item = self._load(oid, before_tid)
if item:
data = item.data
if data is not None:
self._fetched(item)
return data, item.tid, item.next_tid
def store(self, oid, data, tid, next_tid):
"""Store a new data record in the cache"""
size = len(data)
max_size = self._max_size max_size = self._max_size
if self._size > max_size: if size < max_size:
for cache_buffer in cache_buffers: item = self._load(oid, next_tid)
while self._size > max_size: if item:
element = cache_buffer.head assert not (item.data or item.level)
if element is None: assert item.tid == tid and item.next_tid == next_tid
self._history_size -= 1
else:
item = CacheItem()
item.oid = oid
item.tid = tid
item.next_tid = next_tid
item.counter = 0
item.level = None
try:
item_list = self._oid_dict[oid]
except KeyError:
self._oid_dict[oid] = [item]
else:
if next_tid:
for i, x in enumerate(item_list):
if tid < x.tid:
break break
self._evict(element.data.key) item_list.insert(i, item)
else:
if item_list:
prev = item_list[-1]
item.counter = prev.counter
prev.counter = 0
if prev.level > 1:
self._fetched(prev)
item_list.append(item)
item.data = data
self._fetched(item)
self._size += size
if max_size < self._size:
for head in self._queue_list[1:]:
while head:
next = self._remove(head)
head.level = 0
self._add(head)
if self._size <= max_size: if self._size <= max_size:
break
__setitem__ = store
def invalidate(self, key):
if key in self._data:
data = self._data[key]
if data.level >= 0:
del self._cache_buffers[data.level][data.element]
self._evict(key)
return return
raise KeyError, "%s was not found in the cache" % (key, ) head = next
__delitem__ = invalidate
def update(self, key, callback):
"""
Update entry without changing its level.
"""
data = self._data[key]
if data.level < 0:
raise KeyError(key)
data.value = callback(data.value)
# Here is a test.
if __name__ == '__main__':
import hotshot, hotshot.stats
def test():
cache = MQ(life_time=100, buffer_levels=9, max_history_size=10000,
max_size=2*1024*1024)
cache[0] = 'foo'
assert cache[0] == 'foo', '0 should be present'
del cache[0]
assert cache.get(0) is None, '0 should not be present'
for i in xrange(10000): def invalidate(self, oid, tid):
assert cache.get(i) is None, '%d should not be present' % i """Mark data record as being valid only up to given tid"""
try:
for i in xrange(10000): item = self._oid_dict[oid][-1]
cache[i] = str(i) except KeyError:
assert cache.get(i) == str(i), '%d does not exist' % i pass
else:
for i in xrange(10000 - 100 - 1): if item.next_tid is None:
assert cache.get(i) is None, '%d should not be present' % i item.next_tid = tid
for i in xrange(10):
cache[i] = str(i)
for _ in xrange(1000):
for i in xrange(10):
assert cache.get(i) == str(i), '%d does not exist' % i
for i in xrange(10, 500):
cache[i] = str(i)
for i in xrange(10):
assert cache.get(i) == str(i), '%d does not exist' % i
prof = hotshot.Profile("mq.prof")
prof.runcall(test)
prof.close()
stats = hotshot.stats.load("mq.prof")
stats.strip_dirs()
stats.sort_stats('time', 'calls')
stats.print_stats(20)
neo/client/handlers/master.py
View file @ edde1fe3
...@@ -121,11 +121,12 @@ class PrimaryNotificationsHandler(BaseHandler): ...@@ -121,11 +121,12 @@ class PrimaryNotificationsHandler(BaseHandler):
app = self.app app = self.app
app._cache_lock_acquire() app._cache_lock_acquire()
try: try:
# ZODB required a dict with oid as key, so create it invalidate = app._cache.invalidate
app.cache_revision_index.invalidate(oid_list, tid) for oid in oid_list:
invalidate(oid, tid)
db = app.getDB() db = app.getDB()
if db is not None: if db is not None:
db.invalidate(tid, dict.fromkeys(oid_list, tid)) db.invalidate(tid, oid_list)
finally: finally:
app._cache_lock_release() app._cache_lock_release()
......
neo/client/mq_index.py deleted 100644 → 0
View file @ 240c94b3
#
# Copyright (C) 2010-2011 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
from neo.client.mq import MQIndex
class RevisionIndex(MQIndex):
"""
This cache index allows accessing a specific revision of a cached object.
It requires cache key to be a 2-tuple, composed of oid and revision.
Note: it is expected that rather few revisions are held in cache, with few
lookups for old revisions, so they are held in a simple sorted list
Note2: all methods here must be called with cache lock acquired.
"""
def __init__(self):
# key: oid
# value: tid list, from highest to lowest
self._oid_dict = {}
# key: oid
# value: tid list, from lowest to highest
self._invalidated = {}
def clear(self):
self._oid_dict.clear()
self._invalidated.clear()
def remove(self, key):
oid_dict = self._oid_dict
oid, tid = key
tid_list = oid_dict[oid]
tid_list.remove(tid)
if not tid_list:
# No more serial known for this object, drop entirely
del oid_dict[oid]
self._invalidated.pop(oid, None)
def add(self, key):
oid_dict = self._oid_dict
oid, tid = key
try:
serial_list = oid_dict[oid]
except KeyError:
serial_list = oid_dict[oid] = []
else:
assert tid not in serial_list
if not(serial_list) or tid > serial_list[0]:
serial_list.insert(0, tid)
else:
serial_list.insert(0, tid)
serial_list.sort(reverse=True)
invalidated = self._invalidated
try:
tid_list = invalidated[oid]
except KeyError:
pass
else:
try:
tid_list.remove(tid)
except ValueError:
pass
else:
if not tid_list:
del invalidated[oid]
def invalidate(self, oid_list, tid):
"""
Mark object invalidated by given transaction.
Must be called with increasing TID values (which is standard for
ZODB).
"""
invalidated = self._invalidated
oid_dict = self._oid_dict
for oid in (x for x in oid_list if x in oid_dict):
try:
tid_list = invalidated[oid]
except KeyError:
tid_list = invalidated[oid] = []
assert not tid_list or tid > tid_list[-1], (dump(oid), dump(tid),
dump(tid_list[-1]))
tid_list.append(tid)
def getSerialBefore(self, oid, tid):
"""
Get the first tid in cache which value is lower that given tid.
"""
# WARNING: return-intensive to save on indentation
oid_list = self._oid_dict.get(oid)
if oid_list is None:
# Unknown oid
return None
for result in oid_list:
if result < tid:
# Candidate found
break
else:
# No candidate in cache.
return None
# Check if there is a chance that an intermediate revision would
# exist, while missing from cache.
try:
inv_tid_list = self._invalidated[oid]
except KeyError:
return result
# Remember: inv_tid_list is sorted in ascending order.
for inv_tid in inv_tid_list:
if tid < inv_tid:
# We don't care about invalidations past requested TID.
break
elif result < inv_tid < tid:
# An invalidation was received between candidate revision,
# and before requested TID: there is a matching revision we
# don't know of, so we cannot answer.
return None
return result
def getLatestSerial(self, oid):
"""
Get the latest tid for given object.
"""
result = self._oid_dict.get(oid)
if result is not None:
result = result[0]
try:
tid_list = self._invalidated[oid]
except KeyError:
pass
else:
if result < tid_list[-1]:
# An invalidation happened from a transaction later than
# our most recent view of this object, so we cannot answer.
result = None
return result
def getSerialList(self, oid):
"""
Get the list of all serials cache knows about for given object.
"""
return self._oid_dict.get(oid, [])[:]
neo/storage/database/btree.py
View file @ edde1fe3
...@@ -275,37 +275,22 @@ class BTreeDatabaseManager(DatabaseManager): ...@@ -275,37 +275,22 @@ class BTreeDatabaseManager(DatabaseManager):
def _getObject(self, oid, tid=None, before_tid=None): def _getObject(self, oid, tid=None, before_tid=None):
tserial = self._obj.get(oid) tserial = self._obj.get(oid)
if tserial is None: if tserial is not None:
result = None
else:
if tid is None: if tid is None:
if before_tid is None:
try: try:
if before_tid is None:
tid = tserial.maxKey() tid = tserial.maxKey()
except ValueError:
tid = None
else: else:
before_tid -= 1 tid = tserial.maxKey(before_tid - 1)
try:
tid = tserial.maxKey(before_tid)
except ValueError: except ValueError:
tid = None return
if tid is None: result = tserial.get(tid)
result = None
else:
result = tserial.get(tid, None)
if result: if result:
compression, checksum, data, value_serial = result
if before_tid is None:
next_serial = None
else:
try: try:
next_serial = tserial.minKey(tid + 1) next_serial = tserial.minKey(tid + 1)
except ValueError: except ValueError:
next_serial = None next_serial = None
result = (tid, next_serial, compression, checksum, data, return (tid, next_serial) + result
value_serial)
return result
def doSetPartitionTable(self, ptid, cell_list, reset): def doSetPartitionTable(self, ptid, cell_list, reset):
pt = self._pt pt = self._pt
......
neo/storage/database/manager.py
View file @ edde1fe3
...@@ -255,6 +255,8 @@ class DatabaseManager(object): ...@@ -255,6 +255,8 @@ class DatabaseManager(object):
else: else:
serial, next_serial, compression, checksum, data, data_serial = \ serial, next_serial, compression, checksum, data, data_serial = \
result result
assert before_tid is None or next_serial is None or \
before_tid <= next_serial
if data is None and resolve_data: if data is None and resolve_data:
try: try:
_, compression, checksum, data = self._getObjectData(oid, _, compression, checksum, data = self._getObjectData(oid,
......
neo/storage/database/mysqldb.py
View file @ edde1fe3
...@@ -335,50 +335,30 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -335,50 +335,30 @@ class MySQLDatabaseManager(DatabaseManager):
def _getObject(self, oid, tid=None, before_tid=None): def _getObject(self, oid, tid=None, before_tid=None):
q = self.query q = self.query
partition = self._getPartition(oid) partition = self._getPartition(oid)
if tid is not None: sql = """SELECT serial, compression, checksum, value, value_serial
r = q("""SELECT serial, compression, checksum, value, value_serial
FROM obj
WHERE partition = %d AND oid = %d AND serial = %d""" \
% (partition, oid, tid))
try:
serial, compression, checksum, data, value_serial = r[0]
next_serial = None
except IndexError:
return None
elif before_tid is not None:
r = q("""SELECT serial, compression, checksum, value, value_serial
FROM obj FROM obj
WHERE partition = %d WHERE partition = %d
AND oid = %d AND serial < %d AND oid = %d""" % (partition, oid)
ORDER BY serial DESC LIMIT 1""" \ if tid is not None:
% (partition, oid, before_tid)) sql += ' AND serial = %d' % tid
elif before_tid is not None:
sql += ' AND serial < %d ORDER BY serial DESC LIMIT 1' % before_tid
else:
# XXX I want to express "HAVING serial = MAX(serial)", but
# MySQL does not use an index for a HAVING clause!
sql += ' ORDER BY serial DESC LIMIT 1'
r = q(sql)
try: try:
serial, compression, checksum, data, value_serial = r[0] serial, compression, checksum, data, value_serial = r[0]
except IndexError: except IndexError:
return None return None
r = q("""SELECT serial FROM obj_short r = q("""SELECT serial FROM obj_short
WHERE partition = %d WHERE partition = %d AND oid = %d AND serial > %d
AND oid = %d AND serial >= %d ORDER BY serial LIMIT 1""" % (partition, oid, serial))
ORDER BY serial LIMIT 1""" \
% (partition, oid, before_tid))
try: try:
next_serial = r[0][0] next_serial = r[0][0]
except IndexError: except IndexError:
next_serial = None next_serial = None
else:
# XXX I want to express "HAVING serial = MAX(serial)", but
# MySQL does not use an index for a HAVING clause!
r = q("""SELECT serial, compression, checksum, value, value_serial
FROM obj
WHERE partition = %d AND oid = %d
ORDER BY serial DESC LIMIT 1""" \
% (partition, oid))
try:
serial, compression, checksum, data, value_serial = r[0]
next_serial = None
except IndexError:
return None
return serial, next_serial, compression, checksum, data, value_serial return serial, next_serial, compression, checksum, data, value_serial
def doSetPartitionTable(self, ptid, cell_list, reset): def doSetPartitionTable(self, ptid, cell_list, reset):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7