Commit 5e966f01 authored by Ivan Tyagov's avatar Ivan Tyagov Committed by Julien Muchembled

Add Cache Bag implementation (multi level caching).

Fix tests (DMS) to use by default Cache Bags.
Fix some dead code and clean up.
Move cache initialization away from DMS into Cache Factory.
parent 62b62342
...@@ -74,20 +74,15 @@ class CachedConvertableMixin: ...@@ -74,20 +74,15 @@ class CachedConvertableMixin:
def _getCacheFactory(self): def _getCacheFactory(self):
""" """
""" """
# XXX: is this really needed ?
if self.getOriginalDocument() is None: if self.getOriginalDocument() is None:
return None return None
portal = self.getPortalObject() portal = self.getPortalObject()
cache_tool = portal.portal_caches cache_factory_name = portal.portal_preferences.getPreferredConversionCacheFactory('document_cache_factory')
preference_tool = portal.portal_preferences if cache_factory_name is not None:
cache_factory_name = preference_tool.getPreferredConversionCacheFactory('document_cache_factory') return getattr(portal.portal_caches, cache_factory_name, None)
cache_factory = cache_tool.getRamCacheRoot().get(cache_factory_name)
#XXX This conditional statement should be remove as soon as
#Broadcasting will be enable among all zeo clients.
#Interaction which update portal_caches should interact with all nodes.
if cache_factory is None and getattr(cache_tool, cache_factory_name, None) is not None:
#ram_cache_root is not up to date for current node
cache_tool.updateCache()
return cache_tool.getRamCacheRoot().get(cache_factory_name)
security.declareProtected(Permissions.AccessContentsInformation, security.declareProtected(Permissions.AccessContentsInformation,
'generateCacheId') 'generateCacheId')
...@@ -167,23 +162,18 @@ class CachedConvertableMixin: ...@@ -167,23 +162,18 @@ class CachedConvertableMixin:
self.temp_conversion_data = {} self.temp_conversion_data = {}
self.temp_conversion_data[cache_id] = stored_data_dict self.temp_conversion_data[cache_id] = stored_data_dict
return return
cache_duration = cache_factory.cache_duration
# The purpose of this transaction cache is to help calls # The purpose of this transaction cache is to help calls
# to the same cache value in the same transaction. # to the same cache value in the same transaction.
tv = getTransactionalVariable() tv = getTransactionalVariable()
tv[cache_id] = stored_data_dict tv[cache_id] = stored_data_dict
for cache_plugin in cache_factory.getCachePluginList(): cache_factory.set(cache_id, stored_data_dict)
cache_plugin.set(cache_id, DEFAULT_CACHE_SCOPE,
stored_data_dict, cache_duration=cache_duration)
security.declareProtected(Permissions.View, '_getConversionDataDict') security.declareProtected(Permissions.View, '_getConversionDataDict')
def _getConversionDataDict(self, **kw): def _getConversionDataDict(self, **kw):
""" """
""" """
cache_id = self._getCacheKey(**kw) cache_id = self._getCacheKey(**kw)
cache_factory = self._getCacheFactory()
if cache_factory is None:
return getattr(aq_base(self), 'temp_conversion_data', {})[cache_id]
# The purpose of this cache is to help calls to the same cache value # The purpose of this cache is to help calls to the same cache value
# in the same transaction. # in the same transaction.
tv = getTransactionalVariable() tv = getTransactionalVariable()
...@@ -191,26 +181,28 @@ class CachedConvertableMixin: ...@@ -191,26 +181,28 @@ class CachedConvertableMixin:
return tv[cache_id] return tv[cache_id]
except KeyError: except KeyError:
pass pass
for cache_plugin in cache_factory.getCachePluginList():
cache_entry = cache_plugin.get(cache_id, DEFAULT_CACHE_SCOPE) # get preferred cache factory or cache bag
if cache_entry is not None: cache_factory = self._getCacheFactory()
data_dict = cache_entry.getValue() if cache_factory is not None:
if data_dict: data_dict = cache_factory.get(cache_id, None)
if isinstance(data_dict, tuple): if data_dict:
# Backward compatibility: if cached value is a tuple if isinstance(data_dict, tuple):
# as it was before refactoring # Backward compatibility: if cached value is a tuple
# http://svn.erp5.org?rev=35216&view=rev # as it was before refactoring
# raise a KeyError to invalidate this cache entry and force # http://svn.erp5.org?rev=35216&view=rev
# calculation of a new conversion # raise a KeyError to invalidate this cache entry and force
raise KeyError('Old cache conversion format,'\ # calculation of a new conversion
'cache entry invalidated for key:%r' % cache_id) raise KeyError('Old cache conversion format,'\
content_md5 = data_dict['content_md5'] 'cache entry invalidated for key:%r' % cache_id)
if content_md5 != self.getContentMd5(): content_md5 = data_dict['content_md5']
raise KeyError, 'Conversion cache key is compromised for %r' % cache_id if content_md5 != self.getContentMd5():
# Fill transactional cache in order to help raise KeyError, 'Conversion cache key is compromised for %r' % cache_id
# querying real cache during same transaction # Fill transactional cache in order to help
tv[cache_id] = data_dict # querying real cache during same transaction
return data_dict tv[cache_id] = data_dict
return data_dict
raise KeyError, 'Conversion cache key does not exists for %r' % cache_id raise KeyError, 'Conversion cache key does not exists for %r' % cache_id
security.declareProtected(Permissions.View, 'getConversion') security.declareProtected(Permissions.View, 'getConversion')
......
...@@ -133,9 +133,14 @@ class TestDocumentMixin(ERP5TypeTestCase): ...@@ -133,9 +133,14 @@ class TestDocumentMixin(ERP5TypeTestCase):
preference_list = self.portal.portal_preferences.contentValues( preference_list = self.portal.portal_preferences.contentValues(
portal_type=portal_type) portal_type=portal_type)
if not preference_list: if not preference_list:
# create a Cache Bag for tests
cache_bag = self.portal.portal_caches.newContent(portal_type = 'Cache Bag')
cache_bag.cache_duration = 36000
cache_plugin = cache_bag.newContent(portal_type='Ram Cache')
cache_plugin.cache_expire_check_interval = 54000
preference = self.portal.portal_preferences.newContent(title="Default System Preference", preference = self.portal.portal_preferences.newContent(title="Default System Preference",
# use local RAM based cache as some tests need it # use local RAM based cache as some tests need it
preferred_conversion_cache_factory = 'erp5_content_long', preferred_conversion_cache_factory = cache_bag.getId(),
portal_type=portal_type) portal_type=portal_type)
else: else:
preference = preference_list[0] preference = preference_list[0]
......
...@@ -44,11 +44,14 @@ class TestDocumentWithFlare(TestDocument): ...@@ -44,11 +44,14 @@ class TestDocumentWithFlare(TestDocument):
def setSystemPreference(self): def setSystemPreference(self):
system_preference = TestDocument.setSystemPreference(self) system_preference = TestDocument.setSystemPreference(self)
memcached = _getPersistentMemcachedServerDict() memcached = _getPersistentMemcachedServerDict()
system_preference.setPreferredConversionCacheFactory('dms_cache_factory') # create a Cache Bag for tests
cache_bag = self.portal.portal_caches.newContent(portal_type = 'Cache Bag')
cache_bag.cache_duration = 15768000
cache_plugin = cache_bag.newContent(portal_type='Distributed Ram Cache')
system_preference.setPreferredConversionCacheFactory(cache_bag.getId())
persistent_memcached_plugin = self.portal.portal_memcached.persistent_memcached_plugin persistent_memcached_plugin = self.portal.portal_memcached.persistent_memcached_plugin
persistent_memcached_plugin.setUrlString('%s:%s' %(memcached['hostname'], memcached['port'])) persistent_memcached_plugin.setUrlString('%s:%s' %(memcached['hostname'], memcached['port']))
self.portal.portal_caches.dms_cache_factory.persistent_cache_plugin.setSpecialiseValue(persistent_memcached_plugin) cache_plugin.setSpecialiseValue(persistent_memcached_plugin)
def test_suite(): def test_suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
......
...@@ -68,7 +68,6 @@ class TestDocumentWithPreConversion(TestDocument): ...@@ -68,7 +68,6 @@ class TestDocumentWithPreConversion(TestDocument):
""" """
Test pre converion only happens on proper documents. Test pre converion only happens on proper documents.
""" """
print "da"
image = self.portal.image_module.newContent(portal_type='Image', image = self.portal.image_module.newContent(portal_type='Image',
reference='Embedded-XXX', reference='Embedded-XXX',
version='001', version='001',
......
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from AccessControl import ClassSecurityInfo
from Products.ERP5Type import Permissions
from Products.ERP5Type.Cache import DEFAULT_CACHE_SCOPE
from Products.ERP5Type.Core.CacheFactory import CacheFactory
class CacheBag(CacheFactory):
"""
CacheBag is a special type of CacheFactory that allows multi level caching
in different backends describe by CachePlugin.
CacheBag 1
- Cache Plugin 1 (priority 0)
- Cache Plugin 2 (priority 1)
"""
meta_type = 'ERP5 Cache Bag'
portal_type = 'Cache Bag'
security = ClassSecurityInfo()
security.declareProtected(Permissions.AccessContentsInformation, 'get')
def get(self, cache_id, default=None):
"""
Get value or return default.
"""
ram_cache_factory_plugin_list = self.getRamCacheFactoryPluginList()
for cache_plugin in ram_cache_factory_plugin_list:
data_dict = cache_plugin.get(cache_id, DEFAULT_CACHE_SCOPE, default)
if data_dict is not None:
value = data_dict.getValue()
if ram_cache_factory_plugin_list.index(cache_plugin)>0:
# update first plugin as it's the one to be used
cache_duration = self.getRamCacheFactory().cache_duration
ram_cache_factory_plugin_list[0].set(cache_id, DEFAULT_CACHE_SCOPE, value, cache_duration)
return value
return default
security.declareProtected(Permissions.AccessContentsInformation, 'set')
def set(self, cache_id, value):
"""
Set value.
"""
cache_duration = self.getRamCacheFactory().cache_duration
ram_cache_factory_plugin_list = self.getRamCacheFactoryPluginList()
# set only in first plugin in sequence
ram_cache_factory_plugin_list[0].set(cache_id, DEFAULT_CACHE_SCOPE, value, cache_duration)
...@@ -71,8 +71,16 @@ class CacheFactory(XMLObject): ...@@ -71,8 +71,16 @@ class CacheFactory(XMLObject):
security.declareProtected(Permissions.AccessContentsInformation, 'getRamCacheFactory') security.declareProtected(Permissions.AccessContentsInformation, 'getRamCacheFactory')
def getRamCacheFactory(self): def getRamCacheFactory(self):
""" Return RAM based cache factory """ """ Return RAM based cache factory """
erp5_site_id = self.getPortalObject().getId() cache_factory_name = self.getId()
return CachingMethod.factories[erp5_site_id][self.cache_scope] cache_tool = self.portal_caches
cache_factory = CachingMethod.factories.get(cache_factory_name)
#XXX This conditional statement should be remove as soon as
#Broadcasting will be enable among all zeo clients.
#Interaction which update portal_caches should interact with all nodes.
if cache_factory is None and getattr(cache_tool, cache_factory_name, None) is not None:
#ram_cache_root is not up to date for current node
cache_tool.updateCache()
return CachingMethod.factories[cache_factory_name]
security.declareProtected(Permissions.AccessContentsInformation, 'getRamCacheFactoryPluginList') security.declareProtected(Permissions.AccessContentsInformation, 'getRamCacheFactoryPluginList')
def getRamCacheFactoryPluginList(self): def getRamCacheFactoryPluginList(self):
......
...@@ -67,7 +67,7 @@ class CacheTool(BaseTool): ...@@ -67,7 +67,7 @@ class CacheTool(BaseTool):
def getCacheFactoryList(self): def getCacheFactoryList(self):
""" Return available cache factories """ """ Return available cache factories """
rd = {} rd = {}
for cf in self.objectValues('ERP5 Cache Factory'): for cf in self.objectValues(['ERP5 Cache Factory', 'ERP5 Cache Bag']):
cache_scope = cf.getId() cache_scope = cf.getId()
rd[cache_scope] = {} rd[cache_scope] = {}
rd[cache_scope]['cache_plugins'] = [] rd[cache_scope]['cache_plugins'] = []
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment