add a first version for client part, handler for storage is missing

and cache management must be complete


git-svn-id: https://svn.erp5.org/repos/neo/branches/prototype3@42 71dcc9de-d417-0410-9af5-da40c76e7ee4

neo/client/NEOStorage.py

+
+from ZODB import BaseStorage, ConflictResolution, POSException
+from ZODB.utils import p64, u64, cp, z64
+
+class NEOStorageError(POSException.StorageError):
+  pass
+
+class NEOStorageConflictError(NEOStorageError):  
+  pass
+
+class NEOStorageNotFoundError(NEOStorageError):
+  pass
+
+class NEOStorage(BaseStorage.BaseStorage,
+                 ConflictResolution.ConflictResolvingStorage):
+  """Wrapper class for neoclient."""
+  
+  def __init__(self, master_addr, master_port, read_only=False, **kw):
+    self._is_read_only = read_only
+    from neo.client.app import Application # here to prevent recursive import
+    self.app = Application(master_addr, master_port)
+
+  def load(self, oid, version=None):
+    try:
+      self.app.load(oid)
+    except NEOStorageNotFoundError:
+      raise POSException.POSKeyError (oid)
+          
+  def close(self):
+    self.app.close()
+
+  def cleanup(self):
+    self.app.cleanup()
+  
+  def lastSerial(self):
+    self.app.lastSerial()
+  
+  def lastTransaction(self):
+    self.app.lastTransaction()
+
+  def new_oid(self):
+    if self._is_read_only:
+      raise POSException.ReadOnlyError()
+    self.app.new_oid()
+
+  def tpc_begin(self, transaction, tid=None, status=' '):
+    if self._is_read_only:
+      raise POSException.ReadOnlyError()
+    self.app.tpc_begin(transaction, tid, status)
+
+  def tpc_vote(self, transaction):
+    if self._is_read_only:
+      raise POSException.ReadOnlyError()
+    self.app.tpc_vote(transaction)
+  
+  def tpc_abort(self, transaction):
+    if self._is_read_only:
+      raise POSException.ReadOnlyError()
+    self.app.tpc_abort(transaction)
+  
+  def tpc_finish(self, transaction, f=None):
+    self.app.tpc_finish(transaction, f)
+
+  def store(self, oid, serial, data, version, transaction):
+    if self._is_read_only:
+      raise POSException.ReadOnlyError()
+    try:
+      self.app.store(oid, serial, data, version, transaction)
+    except NEOStorageConflictError:
+      new_data = self.app.tryToResolveConflict(oid, self.app.tid, serial, data)
+      if new_data is not None:
+        # try again after conflict resolution
+        self.store(oid, serial, new_data, version, transaction)
+      else:
+        raise POSException.ConflictError(oid=oid,
+                                         serials=(self.app.tid, serial), data=data)
+
+          
+  def _clear_temp(self):
+    self.app._clear_temp()
+
+  # mutliple revisions
+  def loadSerial(self, oid, serial):
+    try:
+      self.app.loadSerial(oid,serial)
+    except NEOStorageNotFoundError:
+      raise POSException.POSKeyError (oid, serial)
+
+  def loadBefore(self, oid, tid):
+    try:
+      self.app.loadBefore(self, oid, tid)
+    except NEOStorageNotFoundError:
+      raise POSException.POSKeyError (oid, tid)
+  
+  def iterator(self, start=None, stop=None):
+    self.app.iterator(start, stop)
+
+  # undo  
+  def undo(self, transaction_id, txn):
+    if self._is_read_only:
+      raise POSException.ReadOnlyError()
+    self.app.undo(transaction_id, txn)
+
+  def undoInfo(self, first=0, last=-20, specification=None):
+    # XXX is it needed ?
+    if self._is_read_only:
+      raise POSException.ReadOnlyError()
+    self.app.undoInfo(first, last, specification)
+    
+  def undoLog(self, first, last, filter):
+    if self._is_read_only:
+      raise POSException.ReadOnlyError()
+    self.app.undoLog(first, last, filter)
+    
+
+

neo/client/master.py

+import logging
+
+from neo.handler import EventHandler
+from neo.connection import ClientConnection
+from neo.protocol import Packet, MASTER_NODE_TYPE, STORAGE_NODE_TYPE, CLIENT_NODE_TYPE, \
+     INVALID_UUID
+from neo.node import MasterNode, StorageNode, ClientNode
+from neo.pt import PartitionTable
+
+class MasterEventHandler(EventHandler):
+    """This class deals with events for a master."""
+
+    def __init__(self, app):
+        self.app = app
+        EventHandler.__init__(self)
+
+    def handleNotReady(self, conn, packet, message):
+        if isinstance(conn, ClientConnection):
+            app = self.app
+            app.node_not_ready = 1
+        else:
+            self.handleUnexpectedPacket(conn, packet)
+
+    def handleAcceptNodeIdentification(self, conn, packet, node_type,
+                                       uuid, ip_address, port,
+                                       num_partitions, num_replicas):
+        if isinstance(conn, ClientConnection):
+            app = self.app
+            node = app.nm.getNodeByServer(conn.getAddress())
+            if node_type != MASTER_NODE_TYPE:
+                # The peer is not a master node!
+                logging.error('%s:%d is not a master node', ip_address, port)
+                app.nm.remove(node)
+                conn.close()
+                return
+            if conn.getAddress() != (ip_address, port):
+                # The server address is different! Then why was
+                # the connection successful?
+                logging.error('%s:%d is waiting for %s:%d', 
+                              conn.getAddress()[0], conn.getAddress()[1], ip_address, port)
+                app.nm.remove(node)
+                conn.close()
+                return
+
+            conn.setUUID(uuid)
+            node.setUUID(uuid)
+
+            # Create partition table if necessary
+            if app.pt is None:
+                app.pt = PartitionTable(num_partitions, num_replicas)
+            
+            # Ask a primary master.
+            msg_id = conn.getNextId()
+            conn.addPacket(Packet().askPrimaryMaster(msg_id))
+            conn.expectMessage(msg_id)
+        else:
+            self.handleUnexpectedPacket(conn, packet)
+
+    def handleAnswerPrimaryMaster(self, conn, packet, primary_uuid, known_master_list):
+        if isinstance(conn, ClientConnection):
+            app = self.app
+            # Register new master nodes.
+            for ip_address, port, uuid in known_master_list:
+                addr = (ip_address, port)
+                n = app.nm.getNodeByServer(addr)
+                if n is None:
+                    n = MasterNode(server = addr)
+                    app.nm.add(n)
+                if uuid != INVALID_UUID:
+                    # If I don't know the UUID yet, believe what the peer
+                    # told me at the moment.
+                    if n.getUUID() is None:
+                        n.setUUID(uuid)
+
+            if primary_uuid != INVALID_UUID:
+                # The primary master is defined.                
+                if app.primary_master_node is not None \
+                        and app.primary_master_node.getUUID() != primary_uuid:
+                    # There are multiple primary master nodes. This is
+                    # dangerous.
+                    raise ElectionFailure, 'multiple primary master nodes'
+                primary_node = app.nm.getNodeByUUID(primary_uuid)
+                if primary_node is None:
+                    # I don't know such a node. Probably this information
+                    # is old. So ignore it.
+                    pass
+                else:
+                    if primary_node.getUUID() == primary_uuid:
+                        # Whatever the situation is, I trust this master.
+                        app.primary_master_node = primary_node
+        else:
+            self.handleUnexpectedPacket(conn, packet)
+
+    def handleSendPartitionTable(self, conn, packet, row_list):
+        if isinstance(conn, ClientConnection):
+            app = self.app
+            for offset, node in row_list:
+                app.pt.setRow(offset, row)                
+        else:
+            self.handleUnexpectedPacket(conn, packet)
+
+    def handleNotifyNodeInformation(self, conn, packet, node_list):
+        app = self.app
+        for node_type, ip_address, port, uuid, state in node_list:
+            # Register new nodes.
+            addr = (ip_address, port)
+            if app.server == addr:
+                # This is self.
+                continue
+            else:
+                n = app.app.nm.getNodeByServer(addr)
+                if n is None:
+                    if node_type == MASTER_NODE:
+                        n = MasterNode(server = addr)
+                    elif node_typ == STORAGE_NODE:
+                        n = StorageNode(server = addr)
+                    elif node_typ == CLIENT_NODE:
+                        n = ClientNode(server = addr)
+                    else:
+                        continue
+                    app.app.nm.add(n)
+
+                if uuid != INVALID_UUID:
+                    # If I don't know the UUID yet, believe what the peer
+                    # told me at the moment.
+                    if n.getUUID() is None:
+                        n.setUUID(uuid)
+        
+    def handleNotifyPartitionChanges(self, conn, packet, cell_list):
+        if isinstance(conn, ClientConnection):
+            app = self.app
+            for cell in cell_list:
+                app.pt.addNode(cell)
+        else:
+            self.handleUnexpectedPacket(conn, packet)
+
+    def handleAnswerNewTID(self, conn, packet, tid):
+        if isinstance(conn, ClientConnection):
+            app = self.app
+            app.tid = tid
+        else:
+            self.handleUnexpectedPacket(conn, packet)
+
+    def handleNotifyTransactionFinished(self, conn, packet, tid):
+        if isinstance(conn, ClientConnection):
+            app = self.app
+            if tid != app.tid:                
+                # what's this ?
+                raise 
+            else:
+                app.txn_finished = 1
+        else:
+            self.handleUnexpectedPacket(conn, packet)
+
+    def handleInvalidateObjects(self, conn, packet, oid_list):
+        raise NotImplementedError('this method must be overridden')
+
+    def handleAnswerNewOIDList(self, conn, packet, oid_list):
+        if isinstance(conn, ClientConnection):
+            app = self.app
+            app.new_oid_list = oid_list
+            app.new_oid_list.reverse()
+        else:
+            self.handleUnexpectedPacket(conn, packet)

neo/client/mq.py

+##############################################################################
+#
+# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
+#                    Yoshinori Okuji <yo@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+"""
+Multi-Queue Cache Algorithm.
+"""
+
+from math import log
+
+class Element:
+  """
+    This class defines an element of a FIFO buffer.
+  """
+  pass
+      
+class FIFO:
+  """
+    This class implements a FIFO buffer.
+  """
+  
+  def __init__(self):
+    self._head = None
+    self._tail = None
+    self._len = 0
+    
+  def __len__(self):
+    return self._len
+    
+  def append(self):
+    element = Element()
+    element.next = None
+    element.prev = self._tail
+    if self._tail is not None:
+      self._tail.next = element
+    self._tail = element
+    if self._head is None:
+      self._head = element
+    self._len += 1
+    return element
+
+  def head(self):
+    return self._head
+    
+  def tail(self):
+    return self._tail
+          
+  def shift(self):
+    element = self._head
+    if element is None:
+      return None
+    del self[element]
+    return element
+    
+  def __delitem__(self, element):
+    if element.next is None:
+      self._tail = element.prev
+    else:
+      element.next.prev = element.prev
+      
+    if element.prev is None:
+      self._head = element.next
+    else:
+      element.prev.next = element.next
+      
+    self._len -= 1      
+    
+class Data:
+  """
+    Data for each element in a FIFO buffer.
+  """
+  pass
+      
+class MQ:
+  """
+    This class manages cached data by a variant of Multi-Queue.
+    
+    This class caches various sizes of objects. Here are some considerations:
+    
+    - Expired objects are not really deleted immediately. But if GC is invoked too often,
+      it degrades the performance significantly.
+      
+    - If large objects are cached, the number of cached objects decreases. This might affect
+      the cache hit ratio. It might be better to tweak a buffer level according to the size of
+      an object.
+      
+    - Stored values must be strings.
+    
+    - The size calculation is not accurate.
+  """
+  
+  def __init__(self, life_time=10000, buffer_levels=9, max_history_size=100000, max_size=20*1024*1024):
+    self._history_buffer = FIFO()
+    self._cache_buffers = []
+    for level in range(buffer_levels):
+      self._cache_buffers.append(FIFO())
+    self._data = {}
+    self._time = 0
+    self._life_time = life_time
+    self._buffer_levels = buffer_levels
+    self._max_history_size = max_history_size
+    self._max_size = max_size
+    self._size = 0
+    
+  def has_key(self, id):
+    if id in self._data:
+      data = self._data[id]
+      if data.level >= 0:
+        return 1
+    return 0
+    
+  __contains__ = has_key
+  
+  def fetch(self, id):
+    """
+      Fetch a value associated with the id.
+    """
+    if id in self._data:
+      data = self._data[id]
+      if data.level >= 0:
+        del self._cache_buffers[data.level][data.element]
+        value = data.value
+        self._size -= len(value) # XXX inaccurate
+        self.store(id, value)
+        return value
+    raise KeyError, "%s was not found in the cache" % id
+  
+  __getitem__ = fetch
+  
+  def get(self, id, d=None):
+    try:
+      return self.fetch(id)
+    except KeyError:
+      return d
+  
+  def _evict(self, id):
+    """
+      Evict an element to the history buffer.
+    """
+    data = self._data[id]
+    self._size -= len(data.value) # XXX inaccurate
+    del self._cache_buffers[data.level][data.element]
+    element = self._history_buffer.append()
+    data.level = -1
+    data.element = element
+    delattr(data, 'value')
+    delattr(data, 'expire_time')
+    element.data = data
+    if len(self._history_buffer) > self._max_history_size:
+      element = self._history_buffer.shift()
+      del self._data[element.data.id]
+      
+  def store(self, id, value):
+    if id in self._data:
+      data = self._data[id]
+      level, element, counter = data.level, data.element, data.counter + 1
+      if level >= 0:
+        del self._cache_buffers[level][element]
+      else:
+        del self._history_buffer[element]
+    else:
+      counter = 1
+      
+    # XXX It might be better to adjust the level according to the object size.
+    level = int(log(counter, 2))
+    if level >= self._buffer_levels:
+      level = self._buffer_levels - 1
+    element = self._cache_buffers[level].append()
+    data = Data()
+    data.id = id
+    data.expire_time = self._time + self._life_time
+    data.level = level
+    data.element = element
+    data.value = value
+    data.counter = counter
+    element.data = data
+    self._data[id] = data
+    self._size += len(value) # XXX inaccurate
+    
+    self._time += 1
+
+    # Expire old elements.
+    for level in range(self._buffer_levels):
+      cache_buffer = self._cache_buffers[level]
+      head = cache_buffer.head()
+      if head is not None and head.data.expire_time < self._time:
+        del cache_buffer[head]
+        data = head.data
+        if level > 0:
+          new_level = level - 1
+          element = cache_buffer[new_level].append()
+          element.data = data
+          data.expire_time = self._time + self._life_time
+          data.level = new_level
+          data.element = element
+        else:
+          self._evict(data.id)
+        
+    # Limit the size.
+    size = self._size
+    max_size = self._max_size
+    if size > max_size:
+      for cache_buffer in self._cache_buffers:
+        while size > max_size:
+          element = cache_buffer.shift()
+          if element is None:
+            break
+          data = element.data
+          del self._data[data.id]
+          size -= len(data.value) # XXX inaccurate
+        if size <= max_size:
+          break
+    self._size = size
+    
+  __setitem__ = store
+  
+  def invalidate(self, id):
+    if id in self._data:
+      data = self._data[id]
+      if data.level >= 0:
+        del self._cache_buffers[data.level][data.element]
+        self._evict(id)
+        return
+    raise KeyError, "%s was not found in the cache" % id
+
+  __delitem__ = invalidate
+  
+  
+# Here is a test.
+if __name__ == '__main__':
+  cache = MQ()
+  cache[1] = "1"
+  cache[2] = "2"
+  assert cache.get(1) == "1", 'cannot get 1'
+  assert cache.get(2) == "2", 'cannot get 2'
+  assert cache.get(3) == None, 'can get 3!'
+  del cache[1]
+  assert cache.get(1) == None, 'can get 1!'