Add Cache Bag implementation (multi level caching).

Fix tests (DMS) to use by default Cache Bags.
Fix some dead code and clean up.
Move cache initialization away from DMS into Cache Factory.

product/ERP5/mixin/cached_convertable.py

@@ -74,20 +74,15 @@ class CachedConvertableMixin:
   def _getCacheFactory(self):
     """
     """
+    # XXX: is this really needed ?
     if self.getOriginalDocument() is None:
       return None
+
     portal = self.getPortalObject()
-    cache_tool = portal.portal_caches
-    preference_tool = portal.portal_preferences
-    cache_factory_name = preference_tool.getPreferredConversionCacheFactory('document_cache_factory')
-    cache_factory = cache_tool.getRamCacheRoot().get(cache_factory_name)
-    #XXX This conditional statement should be remove as soon as
-    #Broadcasting will be enable among all zeo clients.
-    #Interaction which update portal_caches should interact with all nodes.
-    if cache_factory is None and getattr(cache_tool, cache_factory_name, None) is not None:
-      #ram_cache_root is not up to date for current node
-      cache_tool.updateCache()
-    return cache_tool.getRamCacheRoot().get(cache_factory_name)
+    cache_factory_name = portal.portal_preferences.getPreferredConversionCacheFactory('document_cache_factory')
+    if cache_factory_name is not None:
+      return getattr(portal.portal_caches, cache_factory_name, None)
+
 
   security.declareProtected(Permissions.AccessContentsInformation,
                                                              'generateCacheId')
@@ -167,23 +162,18 @@ class CachedConvertableMixin:
         self.temp_conversion_data = {}
       self.temp_conversion_data[cache_id] = stored_data_dict
       return
-    cache_duration = cache_factory.cache_duration
     # The purpose of this transaction cache is to help calls
     # to the same cache value in the same transaction.
     tv = getTransactionalVariable()
     tv[cache_id] = stored_data_dict
-    for cache_plugin in cache_factory.getCachePluginList():
-      cache_plugin.set(cache_id, DEFAULT_CACHE_SCOPE,
-                       stored_data_dict, cache_duration=cache_duration)
+    cache_factory.set(cache_id, stored_data_dict)
 
   security.declareProtected(Permissions.View, '_getConversionDataDict')
   def _getConversionDataDict(self, **kw):
     """
     """
     cache_id = self._getCacheKey(**kw)
-    cache_factory = self._getCacheFactory()
-    if cache_factory is None:
-      return getattr(aq_base(self), 'temp_conversion_data', {})[cache_id]
+
     # The purpose of this cache is to help calls to the same cache value
     # in the same transaction.
     tv = getTransactionalVariable()
@@ -191,26 +181,28 @@ class CachedConvertableMixin:
       return tv[cache_id]
     except KeyError:
       pass
-    for cache_plugin in cache_factory.getCachePluginList():
-      cache_entry = cache_plugin.get(cache_id, DEFAULT_CACHE_SCOPE)
-      if cache_entry is not None:
-        data_dict = cache_entry.getValue()
-        if data_dict:
-          if isinstance(data_dict, tuple):
-            # Backward compatibility: if cached value is a tuple
-            # as it was before refactoring
-            # http://svn.erp5.org?rev=35216&view=rev
-            # raise a KeyError to invalidate this cache entry and force
-            # calculation of a new conversion
-            raise KeyError('Old cache conversion format,'\
-                               'cache entry invalidated for key:%r' % cache_id)
-          content_md5 = data_dict['content_md5']
-          if content_md5 != self.getContentMd5():
-            raise KeyError, 'Conversion cache key is compromised for %r' % cache_id
-          # Fill transactional cache in order to help
-          # querying real cache during same transaction
-          tv[cache_id] = data_dict
-          return data_dict
+
+    # get preferred cache factory or cache bag
+    cache_factory = self._getCacheFactory()
+    if cache_factory is not None:
+      data_dict = cache_factory.get(cache_id, None)
+      if data_dict:
+        if isinstance(data_dict, tuple):
+          # Backward compatibility: if cached value is a tuple
+          # as it was before refactoring
+          # http://svn.erp5.org?rev=35216&view=rev
+          # raise a KeyError to invalidate this cache entry and force
+          # calculation of a new conversion
+          raise KeyError('Old cache conversion format,'\
+                       'cache entry invalidated for key:%r' % cache_id)
+        content_md5 = data_dict['content_md5']
+        if content_md5 != self.getContentMd5():
+          raise KeyError, 'Conversion cache key is compromised for %r' % cache_id
+        # Fill transactional cache in order to help
+        # querying real cache during same transaction
+        tv[cache_id] = data_dict
+        return data_dict
+
     raise KeyError, 'Conversion cache key does not exists for %r' % cache_id
 
   security.declareProtected(Permissions.View, 'getConversion')

product/ERP5OOo/tests/testDms.py

@@ -133,9 +133,14 @@ class TestDocumentMixin(ERP5TypeTestCase):
     preference_list = self.portal.portal_preferences.contentValues(
                                                        portal_type=portal_type)
     if not preference_list:
+      # create a Cache Bag for tests
+      cache_bag = self.portal.portal_caches.newContent(portal_type = 'Cache Bag')
+      cache_bag.cache_duration = 36000
+      cache_plugin = cache_bag.newContent(portal_type='Ram Cache')
+      cache_plugin.cache_expire_check_interval = 54000
       preference = self.portal.portal_preferences.newContent(title="Default System Preference",
                                                              # use local RAM based cache as some tests need it
-                                                             preferred_conversion_cache_factory = 'erp5_content_long',
+                                                             preferred_conversion_cache_factory = cache_bag.getId(),
                                                              portal_type=portal_type)
     else:
       preference = preference_list[0]

product/ERP5OOo/tests/testDmsWithFlare.py

@@ -44,11 +44,14 @@ class TestDocumentWithFlare(TestDocument):
   def setSystemPreference(self):
     system_preference = TestDocument.setSystemPreference(self)
     memcached = _getPersistentMemcachedServerDict()
-    system_preference.setPreferredConversionCacheFactory('dms_cache_factory')
+    # create a Cache Bag for tests
+    cache_bag = self.portal.portal_caches.newContent(portal_type = 'Cache Bag')
+    cache_bag.cache_duration = 15768000
+    cache_plugin = cache_bag.newContent(portal_type='Distributed Ram Cache')
+    system_preference.setPreferredConversionCacheFactory(cache_bag.getId())
     persistent_memcached_plugin = self.portal.portal_memcached.persistent_memcached_plugin
     persistent_memcached_plugin.setUrlString('%s:%s' %(memcached['hostname'], memcached['port']))
-    self.portal.portal_caches.dms_cache_factory.persistent_cache_plugin.setSpecialiseValue(persistent_memcached_plugin)
-
+    cache_plugin.setSpecialiseValue(persistent_memcached_plugin)
 
 def test_suite():
   suite = unittest.TestSuite()

product/ERP5OOo/tests/testDmsWithPreConversion.py

@@ -68,7 +68,6 @@ class TestDocumentWithPreConversion(TestDocument):
     """
       Test pre converion only happens on proper documents.
     """
-    print "da"
     image = self.portal.image_module.newContent(portal_type='Image',
                                                reference='Embedded-XXX',
                                                version='001',

product/ERP5Type/Core/CacheBag.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
+#                     Ivan Tyagov <ivan@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+
+from AccessControl import ClassSecurityInfo
+from Products.ERP5Type import Permissions
+from Products.ERP5Type.Cache import DEFAULT_CACHE_SCOPE
+from Products.ERP5Type.Core.CacheFactory import CacheFactory
+
+class CacheBag(CacheFactory):
+  """
+  CacheBag is a special type of CacheFactory that allows multi level caching
+  in different backends describe by CachePlugin.
+
+  CacheBag 1
+     - Cache Plugin 1 (priority 0)
+     - Cache Plugin 2 (priority 1)
+  """
+
+  meta_type = 'ERP5 Cache Bag'
+  portal_type = 'Cache Bag'
+
+  security = ClassSecurityInfo()
+
+  security.declareProtected(Permissions.AccessContentsInformation, 'get')
+  def get(self, cache_id, default=None):
+    """
+      Get value or return default.
+    """
+    ram_cache_factory_plugin_list = self.getRamCacheFactoryPluginList()
+
+    for cache_plugin in ram_cache_factory_plugin_list:
+      data_dict = cache_plugin.get(cache_id, DEFAULT_CACHE_SCOPE, default)
+      if data_dict is not None:
+        value = data_dict.getValue()
+        if ram_cache_factory_plugin_list.index(cache_plugin)>0:
+          # update first plugin as it's the one to be used
+          cache_duration = self.getRamCacheFactory().cache_duration
+          ram_cache_factory_plugin_list[0].set(cache_id, DEFAULT_CACHE_SCOPE, value, cache_duration)
+        return value
+    return default
+
+  security.declareProtected(Permissions.AccessContentsInformation, 'set')
+  def set(self, cache_id, value):
+    """
+      Set value.
+    """
+    cache_duration = self.getRamCacheFactory().cache_duration
+    ram_cache_factory_plugin_list = self.getRamCacheFactoryPluginList()
+    # set only in first plugin in sequence
+    ram_cache_factory_plugin_list[0].set(cache_id, DEFAULT_CACHE_SCOPE, value, cache_duration)

product/ERP5Type/Core/CacheFactory.py

@@ -71,8 +71,16 @@ class CacheFactory(XMLObject):
   security.declareProtected(Permissions.AccessContentsInformation, 'getRamCacheFactory')
   def getRamCacheFactory(self):
     """ Return RAM based cache factory """
-    erp5_site_id = self.getPortalObject().getId()
-    return CachingMethod.factories[erp5_site_id][self.cache_scope]
+    cache_factory_name = self.getId()
+    cache_tool = self.portal_caches
+    cache_factory = CachingMethod.factories.get(cache_factory_name)
+    #XXX This conditional statement should be remove as soon as
+    #Broadcasting will be enable among all zeo clients.
+    #Interaction which update portal_caches should interact with all nodes.
+    if cache_factory is None and getattr(cache_tool, cache_factory_name, None) is not None:
+      #ram_cache_root is not up to date for current node
+      cache_tool.updateCache()
+    return CachingMethod.factories[cache_factory_name]
 
   security.declareProtected(Permissions.AccessContentsInformation, 'getRamCacheFactoryPluginList')
   def getRamCacheFactoryPluginList(self):

product/ERP5Type/Tool/CacheTool.py

@@ -67,7 +67,7 @@ class CacheTool(BaseTool):
   def getCacheFactoryList(self):
     """ Return available cache factories """
     rd = {}
-    for cf in self.objectValues('ERP5 Cache Factory'):
+    for cf in self.objectValues(['ERP5 Cache Factory', 'ERP5 Cache Bag']):
       cache_scope = cf.getId()
       rd[cache_scope] = {}
       rd[cache_scope]['cache_plugins'] = []