Commit 48501ac2 authored by Vincent Pelletier's avatar Vincent Pelletier

Add MemcachedTool.

This tools offers a dictionnary interface to memcached.
As it depends on python-memcached and should not fail silently if the site administrator does not have this python module, this tool is disabled by default.
You can enable it runing "touch Products/ERP5Type/USE_MEMCACHED_TOOL"


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@11618 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 3f918866
##############################################################################
#
# Copyright (c) 2006 Nexedi SARL and Contributors. All Rights Reserved.
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Products.ERP5Type import allowMemcachedTool
from Products.ERP5Type.Tool.BaseTool import BaseTool
from Products.ERP5Type import Permissions, _dtmldir
from AccessControl import ClassSecurityInfo
from Globals import DTMLFile
MEMCACHED_TOOL_MODIFIED_FLAG_PROPERTY_ID = '_v_memcached_edited'
if allowMemcachedTool():
import memcache
from Shared.DC.ZRDB.TM import TM
from Products.PythonScripts.Utility import allow_class
MARKER = tuple()
UPDATE_ACTION = 'update'
DELETE_ACTION = 'delete'
class MemcachedDict(TM):
"""
Present memcached similarly to a dictionnary (not all method are
available).
Uses transactions to only update memcached at commit time.
No conflict generation/resolution : last edit wins.
TODO:
- prove that concurency handling in event queuing is not needed
- make picklable ?
"""
def __init__(self, server='127.0.0.1:11211'):
"""
Initialise properties :
memcached_connection
Connection to memcached.
local_cache
Dictionnary used as a connection cache with duration limited to
transaction length.
scheduled_action_dict
Each key in this dictionnary must be handled at transaction commit.
Value gives the action to take :
UPDATE_ACTION
Take value from local cache and send it to memcached.
DELETE_ACTION
Send a delete order to memcached.
"""
self.local_cache = {}
self.scheduled_action_dict = {}
self.memcached_connection = memcache.Client((server, ))
def __del__(self):
"""
Close connection before deleting object.
"""
self.memcached_connection.disconnect_all()
def _finish(self, *ignored):
"""
Actually modifies the values in memcached.
This avoids multiple accesses to memcached during the transaction.
Invalidate all local cache to make sure changes donc by other zopes
would not be ignored.
"""
for key, value in self.local_cache.iteritems():
if getattr(value, MEMCACHED_TOOL_MODIFIED_FLAG_PROPERTY_ID, None):
delattr(value, MEMCACHED_TOOL_MODIFIED_FLAG_PROPERTY_ID)
self.scheduled_action_dict[key] = UPDATE_ACTION
for key, action in self.scheduled_action_dict.iteritems():
if action is UPDATE_ACTION:
self.memcached_connection.set(key, self.local_cache[key], 0)
elif action is DELETE_ACTION:
self.memcached_connection.delete(key, 0)
self.scheduled_action_dict.clear()
self.local_cache.clear()
def _abort(self, *ignored):
"""
Cleanup the action dict and invalidate local cache.
"""
self.local_cache.clear()
self.scheduled_action_dict.clear()
def __getitem__(self, key):
"""
Get an item from local cache, otherwise from memcached.
Note that because of memcached python client limitations, it never
raises KeyError.
"""
# We need to register in this function too to be able to flush cache at
# transaction end.
self._register()
result = self.local_cache.get(key, MARKER)
if result is MARKER:
result = self.memcached_connection.get(key)
self.local_cache[key] = result
return result
def __setitem__(self, key, value):
"""
Set an item to local cache and schedule update of memcached.
"""
self._register()
self.scheduled_action_dict[key] = UPDATE_ACTION
self.local_cache[key] = value
def __delitem__(self, key):
"""
Delete an item from local cache and schedule deletion in memcached.
Never raises KeyError because action is delayed.
"""
self._register()
self.scheduled_action_dict[key] = DELETE_ACTION
if key in self.local_cache:
del self.local_cache[key]
def set(self, key, val, time=0):
"""
Set an item to local cache and schedule update of memcached.
"""
return self.__setitem__(key, value)
def get(self, key, default=MARKER):
"""
Get an item from local cache, otherwise from memcached.
Note that because __getitem__ never raises error, 'default' will never
be used (None will be returned instead).
"""
return self.__getitem__(key)
class SharedDict:
"""
Class to make possible for multiple "users" to store data in the same
dictionnary without risking to overwrite other's data.
Each "user" of the dictionnary must get an instance of this class.
TODO:
- handle persistence ?
"""
def __init__(self, dictionary, prefix):
"""
dictionary
Instance of dictionnary to share.
prefix
Prefix used by the "user" owning an instance of this class.
"""
self._v_dictionary = dictionary
self.prefix = prefix
def _prefixKey(self, key):
"""
Prefix key with self.prefix .
"""
return '%s_%s' % (self.prefix, key)
def __getitem__(self, key):
"""
Get item from memcached.
"""
return self._v_dictionary.__getitem__(self._prefixKey(key))
def __setitem__(self, key, value):
"""
Put item in memcached.
"""
self._v_dictionary.__setitem__(self._prefixKey(key), value)
def __delitem__(self, key):
"""
Delete item from memcached.
"""
self._v_dictionary.__delitem__(self._prefixKey(key))
# These are the method names called by zope
__guarded_setitem__ = __setitem__
__guarded_getitem__ = __getitem__
__guarded_delitem__ = __delitem__
def get(self, key, default=MARKER):
"""
Get item from memcached.
"""
return self.__getitem__(key)
def set(self, key, value):
"""
Put item in memcached.
"""
self.__setitem__(key, value)
allow_class(SharedDict)
class MemcachedTool(BaseTool):
"""
Memcached interface available as a tool.
"""
id = "portal_memcached"
meta_type = "ERP5 Memcached Tool"
portal_type = "Memcached Tool"
security = ClassSecurityInfo()
manage_options = ({'label': 'Configure',
'action': 'memcached_tool_configure',
},) + BaseTool.manage_options
memcached_tool_configure = DTMLFile('memcached_tool_configure', _dtmldir)
def _getMemcachedDict(self):
"""
Return used memcached dict.
Create it if does not exist.
"""
dictionary = getattr(self, '_v_memcached_dict', None)
if dictionary is None:
dictionary = MemcachedDict(self.getServerAddress())
setattr(self, '_v_memcached_dict', dictionary)
return dictionary
security.declareProtected(Permissions.AccessContentsInformation, 'getMemcacheDict')
def getMemcachedDict(self, key_prefix):
"""
Returns an object which can be used as a dict and which gets from/stores
to memcached server.
key_prefix
Mendatory argument allowing different tool users from sharing the same
dictionnary key namespace.
"""
return SharedDict(dictionary=self._getMemcachedDict(), prefix=key_prefix)
security.declareProtected(Permissions.ModifyPortalContent, 'setServerAddress')
def setServerAddress(self, value):
"""
Upon server address change, force next access to memcached dict to
reconnect to new ip.
This is safe in multi zope environment, since we modify self which is
persistent. Then all zopes will have to reload this tool instance, and
loose their volatile properties in the process, which will force them
to reconnect to memcached.
"""
self.server_address = value
setattr(self, '_v_memcached_dict', None)
security.declareProtected(Permissions.AccessContentsInformation, 'getServerAddress')
def getServerAddress(self):
"""
Return server address.
Defaults to 127.0.0.1:11211 if not set.
"""
return getattr(self, 'server_address', '127.0.0.1:11211')
else:
class MemcachedTool(BaseTool):
"""
Dummy MemcachedTool placeholder.
"""
id = "portal_memcached"
meta_type = "ERP5 Memcached Tool"
portal_type = "Memcached Tool"
security = ClassSecurityInfo()
manage_options = ({'label': 'Configure',
'action': 'memcached_tool_configure',
},) + BaseTool.manage_options
def getMemcachedDict(self, *args, **kw):
"""
if this function is called and memcachedtool is disabled, fail loudly
with a meaningfull message.
"""
from Products.ERP5Type import memcached_tool_enable_path
raise NotImplementedError, 'MemcachedTool is disabled. You should ask the'\
'server administrator to enable it by creating the file %s' % (memcached_tool_enable_path, )
memcached_tool_configure = getMemcachedDict = setServerAddress = getMemcachedDict
...@@ -50,13 +50,17 @@ import Products.Localizer # So that we make sure Globals.get_request is availabl ...@@ -50,13 +50,17 @@ import Products.Localizer # So that we make sure Globals.get_request is availabl
# read permissions for zope - this prevents security holes in # read permissions for zope - this prevents security holes in
# production environment # production environment
class_tool_security_path = '%s%s%s' % (product_path, os.sep, 'ALLOW_CLASS_TOOL') class_tool_security_path = '%s%s%s' % (product_path, os.sep, 'ALLOW_CLASS_TOOL')
memcached_tool_enable_path = '%s%s%s' % (product_path, os.sep, 'USE_MEMCACHED_TOOL')
def allowClassTool(): def allowClassTool():
return os.access(class_tool_security_path, os.F_OK) return os.access(class_tool_security_path, os.F_OK)
def allowMemcachedTool():
return os.access(memcached_tool_enable_path, os.F_OK)
def initialize( context ): def initialize( context ):
# Import Product Components # Import Product Components
from Tool import ClassTool, CacheTool from Tool import ClassTool, CacheTool, MemcachedTool
import Document import Document
import Base, XMLObject import Base, XMLObject
from ERP5Type import ERP5TypeInformation from ERP5Type import ERP5TypeInformation
...@@ -65,7 +69,8 @@ def initialize( context ): ...@@ -65,7 +69,8 @@ def initialize( context ):
content_constructors = () content_constructors = ()
content_classes = ( Base.Base, XMLObject.XMLObject, ) content_classes = ( Base.Base, XMLObject.XMLObject, )
portal_tools = ( ClassTool.ClassTool, portal_tools = ( ClassTool.ClassTool,
CacheTool.CacheTool, ) CacheTool.CacheTool,
MemcachedTool.MemcachedTool, )
# Do initialization step # Do initialization step
initializeProduct(context, this_module, globals(), initializeProduct(context, this_module, globals(),
document_module = Document, document_module = Document,
......
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<h3>Memcached server configuration</h3>
<p>
Memcached Tool provides the programer with an interface to a memcached server.
This interface is similar to a dictionnary.
The server to connet to can be configured below.
Programer can get an interface in his scripts using :
</p>
<pre>portal_memcached.getMemcachedDict(key_prefix=something)</pre>
<p>
<i>something</i> : Must be a string uniquely identifying the usage of memcached, similar to <i>CachingMethod</i>'s id parameter. This allows multiple memcached users to share the same connection to the same memcached server without overwriting other's values.
</p>
<form action="setServerAddress" method="POST">
<input type="text" name="value" value="<dtml-var expr="context.getServerAddress()">">
<br/>
<input type="submit" value="Update server address"/>
</form>
<dtml-var manage_page_footer>
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment