Commit 5512c515 authored by Sebastien Robin's avatar Sebastien Robin

several bug fixes and improvement

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@10634 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 60d49e77
...@@ -48,17 +48,34 @@ class DistributedRamCache(BaseCache): ...@@ -48,17 +48,34 @@ class DistributedRamCache(BaseCache):
def __init__(self, params): def __init__(self, params):
self._servers = params.get('server', '') self._servers = params.get('server', '')
self._debugLevel = params.get('debugLevel', 7) self._debugLevel = params.get('debugLevel', 7)
self._cache = memcache.Client(self._servers.split('\n'), self._debugLevel)
self._last_cache_conn_creation_time = time() self._last_cache_conn_creation_time = time()
BaseCache.__init__(self) BaseCache.__init__(self)
def getCacheStorage(self): def getCacheStorage(self):
## if we use one connection object this causes "MemCached: while expecting 'STORED', got unexpected response 'END'" ## if we use one connection object this causes "MemCached: while expecting 'STORED', got unexpected response 'END'"
## messages in log files and thus sometimes can block the thread. For the moment we create ## messages in log files and thus sometimes can block the thread. For the moment we create
## a new conn object for every memcache access which in turns cmeans another socket. ## a new conn object for every memcache access which in turns means another socket.
## See addiionaly expireOldCacheEntries() comments for one or many connections. ## See addiionaly expireOldCacheEntries() comments for one or many connections.
self._cache = memcache.Client(self._servers.split('\n'), debug=self._debugLevel) try:
return self._cache from Products.ERP5Type.Utils import get_request
request = get_request()
except ImportError:
request = None
if request:
## Zope/ERP5 environment
memcache_conn = request.get('_erp5_memcache_connection', None)
if not memcache_conn:
## we have not memcache_conn for this request
memcache_conn = memcache.Client(self._servers.split('\n'), debug=self._debugLevel)
request.set('_erp5_memcache_connection', memcache_conn)
return memcache_conn
else:
## we have memcache_conn for this request
return memcache_conn
else:
## run from unit tests
return memcache.Client(self._servers.split('\n'), debug=self._debugLevel)
def checkAndFixCacheId(self, cache_id, scope): def checkAndFixCacheId(self, cache_id, scope):
## memcached doesn't support namespaces (cache scopes) so to "emmulate" ## memcached doesn't support namespaces (cache scopes) so to "emmulate"
......
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
""" """
Local RAM based cache plugin. Local RAM based cache plugin.
""" """
......
...@@ -114,10 +114,32 @@ class SQLCache(BaseCache): ...@@ -114,10 +114,32 @@ class SQLCache(BaseCache):
def getCacheStorage(self): def getCacheStorage(self):
""" """
Return current DB connection or create a new one. Return current DB connection or create a new one for his thread.
See http://sourceforge.net/docman/display_doc.php?docid=32071&group_id=22307 See http://sourceforge.net/docman/display_doc.php?docid=32071&group_id=22307
especially threadsafety part why we create every time a new MySQL db connection object. especially threadsafety part why we create every time a new MySQL db connection object.
""" """
try:
from Products.ERP5Type.Utils import get_request
request = get_request()
except ImportError:
request = None
if request:
## Zope/ERP5 environment
dbConn = request.get('_erp5_dbcache_connection', None)
if not dbConn:
## we have not dbConn for this request
dbConn = MySQLdb.connect(host=self._db_server, \
user=self._db_user,\
passwd=self._db_passwd, \
db=self._db_name)
request.set('_erp5_dbcache_connection', dbConn)
return dbConn
else:
## we have already dbConn for this request
return dbConn
else:
## run from unit tests
dbConn = MySQLdb.connect(host=self._db_server, \ dbConn = MySQLdb.connect(host=self._db_server, \
user=self._db_user,\ user=self._db_user,\
passwd=self._db_passwd, \ passwd=self._db_passwd, \
...@@ -162,7 +184,6 @@ class SQLCache(BaseCache): ...@@ -162,7 +184,6 @@ class SQLCache(BaseCache):
now = time.time() now = time.time()
if forceCheck or (now > (self._last_cache_expire_check_at + self.cache_expire_check_interval)): if forceCheck or (now > (self._last_cache_expire_check_at + self.cache_expire_check_interval)):
## time to check for expired cache items ## time to check for expired cache items
#print "EXPIRE", self, self.cache_expire_check_interval
self._last_cache_expire_check_at = now self._last_cache_expire_check_at = now
my_query = self.delete_expired_keys_sql %(self._db_cache_table_name, now) my_query = self.delete_expired_keys_sql %(self._db_cache_table_name, now)
self.execSQLQuery(my_query) self.execSQLQuery(my_query)
......
...@@ -38,13 +38,6 @@ from Products.ERP5Cache.CachePlugins.RamCache import RamCache ...@@ -38,13 +38,6 @@ from Products.ERP5Cache.CachePlugins.RamCache import RamCache
from Products.ERP5Cache.CachePlugins.DistributedRamCache import DistributedRamCache from Products.ERP5Cache.CachePlugins.DistributedRamCache import DistributedRamCache
from Products.ERP5Cache.CachePlugins.SQLCache import SQLCache from Products.ERP5Cache.CachePlugins.SQLCache import SQLCache
##try:
## from Products.TimerService import getTimerService
##except ImportError:
## def getTimerService(self):
## pass
class CacheTool(BaseTool): class CacheTool(BaseTool):
""" Caches tool wrapper for ERP5 """ """ Caches tool wrapper for ERP5 """
...@@ -184,62 +177,12 @@ class CacheTool(BaseTool): ...@@ -184,62 +177,12 @@ class CacheTool(BaseTool):
if REQUEST: if REQUEST:
self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache factory %s cleared.' %cache_factory_id) self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache factory %s cleared.' %cache_factory_id)
security.declareProtected(Permissions.ModifyPortalContent, 'clearCacheFactoryScope')
def clearCacheFactoryScope(self, cache_factory_id, scope, REQUEST=None):
""" Clear only cache factory. """
ram_cache_root = self.getRamCacheRoot()
if ram_cache_root.has_key(cache_factory_id):
ram_cache_root[cache_factory_id].clearCacheForScope(scope)
if REQUEST:
self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache factory scope %s cleared.' %cache_factory_id)
# Timer - checks for cache expiration triggered by Zope's TimerService
## def isSubscribed(self):
## """
## return True, if we are subscribed to TimerService.
## Otherwise return False.
## """
## service = getTimerService(self)
## if not service:
## LOG('AlarmTool', INFO, 'TimerService not available')
## return False
##
## path = '/'.join(self.getPhysicalPath())
## if path in service.lisSubscriptions():
## return True
## return False
##
## security.declareProtected(Permissions.ManageProperties, 'subscribe')
## def subscribe(self):
## """
## Subscribe to the global Timer Service.
## """
## service = getTimerService(self)
## if not service:
## LOG('AlarmTool', INFO, 'TimerService not available')
## return
## service.subscribe(self)
## return "Subscribed to Timer Service"
##
## security.declareProtected(Permissions.ManageProperties, 'unsubscribe')
## def unsubscribe(self):
## """
## Unsubscribe from the global Timer Service.
## """
## service = getTimerService(self)
## if not service:
## LOG('AlarmTool', INFO, 'TimerService not available')
## return
## service.unsubscribe(self)
## return "Usubscribed from Timer Service"
##
## def manage_beforeDelete(self, item, container):
## self.unsubscribe()
## BaseTool.inheritedAttribute('manage_beforeDelete')(self, item, container)
##
## def manage_afterAdd(self, item, container):
## self.subscribe()
## BaseTool.inheritedAttribute('manage_afterAdd')(self, item, container)
##
## security.declarePrivate('process_timer')
## def process_timer(self, interval, tick, prev="", next=""):
## """
## This method is called by TimerService in the interval given
## in zope.conf. The Default is every 5 seconds. This method will
## try to expire cache entries.
## """
## ram_cache_root = self.getRamCacheRoot()
## for cf_id, cf_obj in ram_cache_root.items():
## cf_obj.expire()
...@@ -30,6 +30,7 @@ from AccessControl import ClassSecurityInfo ...@@ -30,6 +30,7 @@ from AccessControl import ClassSecurityInfo
from Products.CMFCore import CMFCorePermissions from Products.CMFCore import CMFCorePermissions
from Products.ERP5Type.XMLObject import XMLObject from Products.ERP5Type.XMLObject import XMLObject
from Products.ERP5Type import PropertySheet from Products.ERP5Type import PropertySheet
from Products.ERP5.PropertySheet.SortIndex import SortIndex
from Products.ERP5Cache.PropertySheet.BaseCachePlugin import BaseCachePlugin from Products.ERP5Cache.PropertySheet.BaseCachePlugin import BaseCachePlugin
from Products.ERP5Cache.PropertySheet.DistributedRamCachePlugin import DistributedRamCachePlugin from Products.ERP5Cache.PropertySheet.DistributedRamCachePlugin import DistributedRamCachePlugin
...@@ -57,5 +58,6 @@ class DistributedRamCachePlugin(XMLObject): ...@@ -57,5 +58,6 @@ class DistributedRamCachePlugin(XMLObject):
, PropertySheet.SimpleItem , PropertySheet.SimpleItem
, PropertySheet.Folder , PropertySheet.Folder
, BaseCachePlugin , BaseCachePlugin
, SortIndex
, DistributedRamCachePlugin , DistributedRamCachePlugin
) )
...@@ -31,6 +31,7 @@ from Products.CMFCore import CMFCorePermissions ...@@ -31,6 +31,7 @@ from Products.CMFCore import CMFCorePermissions
from Products.ERP5Type.Base import Base from Products.ERP5Type.Base import Base
from Products.ERP5Type.XMLObject import XMLObject from Products.ERP5Type.XMLObject import XMLObject
from Products.ERP5Type import PropertySheet from Products.ERP5Type import PropertySheet
from Products.ERP5.PropertySheet.SortIndex import SortIndex
from Products.ERP5Cache.PropertySheet.BaseCachePlugin import BaseCachePlugin from Products.ERP5Cache.PropertySheet.BaseCachePlugin import BaseCachePlugin
from Products.ERP5Cache.PropertySheet.SQLCachePlugin import SQLCachePlugin from Products.ERP5Cache.PropertySheet.SQLCachePlugin import SQLCachePlugin
...@@ -58,5 +59,6 @@ class SQLCachePlugin(XMLObject): ...@@ -58,5 +59,6 @@ class SQLCachePlugin(XMLObject):
, PropertySheet.SimpleItem , PropertySheet.SimpleItem
, PropertySheet.Folder , PropertySheet.Folder
, BaseCachePlugin , BaseCachePlugin
, SortIndex
, SQLCachePlugin , SQLCachePlugin
) )
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE Project SYSTEM "Project-3.7.dtd"> <!DOCTYPE Project SYSTEM "Project-3.7.dtd">
<!-- Project file for project ERP5Cache --> <!-- Project file for project ERP5Cache -->
<!-- Saved: 2006-10-03, 20:15:31 --> <!-- Saved: 2006-10-06, 19:59:47 -->
<!-- Copyright (C) 2006 Ivan Tyagov, ivan.tyagov@brmtec.com --> <!-- Copyright (C) 2006 Ivan Tyagov, ivan.tyagov@brmtec.com -->
<Project version="3.7"> <Project version="3.7">
<ProgLanguage mixed="0">Python</ProgLanguage> <ProgLanguage mixed="0">Python</ProgLanguage>
...@@ -99,6 +99,10 @@ ...@@ -99,6 +99,10 @@
<Dir>dtml</Dir> <Dir>dtml</Dir>
<Name>cache_tool_configure.dtml</Name> <Name>cache_tool_configure.dtml</Name>
</Source> </Source>
<Source>
<Dir>tests</Dir>
<Name>testCacheTool.py</Name>
</Source>
</Sources> </Sources>
<Forms> <Forms>
</Forms> </Forms>
......
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE Tasks SYSTEM "Tasks-3.7.dtd"> <!DOCTYPE Tasks SYSTEM "Tasks-3.7.dtd">
<!-- Tasks file for project ERP5Cache --> <!-- Tasks file for project ERP5Cache -->
<!-- Saved: 2006-10-05, 17:54:47 --> <!-- Saved: 2006-10-06, 20:07:54 -->
<Tasks version="3.7"> <Tasks version="3.7">
<Task priority="1" completed="0"> <Task priority="1" completed="0">
<Description>TODO: move result file path generation to runBenchmark.</Description> <Description>TODO: move result file path generation to runBenchmark.</Description>
...@@ -38,35 +38,13 @@ ...@@ -38,35 +38,13 @@
</Task> </Task>
<Task priority="1" completed="0"> <Task priority="1" completed="0">
<Description>TODO: Based on above data we can have a different invalidation policy</Description> <Description>TODO: Based on above data we can have a different invalidation policy</Description>
<Created>2006-09-28, 13:08:57</Created> <Created>2006-10-06, 20:00:38</Created>
<Resource> <Resource>
<Filename> <Filename>
<Dir>CachePlugins</Dir> <Dir>CachePlugins</Dir>
<Name>BaseCache.py</Name> <Name>BaseCache.py</Name>
</Filename> </Filename>
<Linenumber>19</Linenumber> <Linenumber>47</Linenumber>
</Resource>
</Task>
<Task priority="1" completed="0">
<Description>TODO: make check not always but each 100 or n calls</Description>
<Created>2006-09-28, 13:08:57</Created>
<Resource>
<Filename>
<Dir>CachePlugins</Dir>
<Name>BaseCache.py</Name>
</Filename>
<Linenumber>64</Linenumber>
</Resource>
</Task>
<Task priority="1" completed="0">
<Description>TODO: check how to avoid problems with memcache whe using one connection to</Description>
<Created>2006-10-05, 16:42:13</Created>
<Resource>
<Filename>
<Dir>CachePlugins</Dir>
<Name>DistributedRamCache.py</Name>
</Filename>
<Linenumber>25</Linenumber>
</Resource> </Resource>
</Task> </Task>
</Tasks> </Tasks>
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import random import random
import unittest import unittest
import time import time
...@@ -14,15 +42,15 @@ class Foo: ...@@ -14,15 +42,15 @@ class Foo:
class TestRamCache(unittest.TestCase): class TestRamCache(unittest.TestCase):
def setUp(self): def setUp(self):
self.cache_plugins = (#RamCache(), self.cache_plugins = (RamCache(),
DistributedRamCache({'servers': '127.0.0.1:11211', DistributedRamCache({'servers': '127.0.0.1:11211',
'debugLevel': 7,}), 'debugLevel': 7,}),
#SQLCache( {'server': '', SQLCache( {'server': '',
# 'user': '', 'user': '',
# 'passwd': '', 'passwd': '',
# 'db': 'test', 'db': 'test',
# 'cache_table_name': 'cache', 'cache_table_name': 'cache',
# }), }),
) )
def testScope(self): def testScope(self):
...@@ -77,13 +105,13 @@ class TestRamCache(unittest.TestCase): ...@@ -77,13 +105,13 @@ class TestRamCache(unittest.TestCase):
def testSetGet(self): def testSetGet(self):
""" set value to cache and then get it back """ """ set value to cache and then get it back """
for cache_plugin in self.cache_plugins: for cache_plugin in self.cache_plugins:
self.generaltestSetGet(cache_plugin, 1000) self.generaltestSetGet(cache_plugin, 100)
def testExpire(self):
""" Check expired by setting a key, wit for its timeout and check if in cache"""
for cache_plugin in self.cache_plugins:
self.generalExpire(cache_plugin, 2)
## def testExpire(self):
## """ Check expired by setting a key, wit for its timeout and check if in cache"""
## for cache_plugin in self.cache_plugins:
## self.generalExpire(cache_plugin, 2)
## pass
def generalExpire(self, cache_plugin, iterations): def generalExpire(self, cache_plugin, iterations):
print "TESTING (expire): ", cache_plugin print "TESTING (expire): ", cache_plugin
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment