Commit 112d91e3 authored by Yoshinori Okuji's avatar Yoshinori Okuji

Implement an operation handler for storage node. This does not support replication yet.

git-svn-id: https://svn.erp5.org/repos/neo/branches/prototype3@67 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent f5b79552
...@@ -16,3 +16,5 @@ TODO ...@@ -16,3 +16,5 @@ TODO
- Stopping packet processing by returning a boolean value from - Stopping packet processing by returning a boolean value from
a handler, otherwise too tricky to exchange a handler with another a handler, otherwise too tricky to exchange a handler with another
- Replication.
...@@ -3,6 +3,7 @@ import MySQLdb ...@@ -3,6 +3,7 @@ import MySQLdb
import os import os
from time import time from time import time
from struct import pack, unpack from struct import pack, unpack
from collections import deque
from neo.config import ConfigurationManager from neo.config import ConfigurationManager
from neo.protocol import Packet, ProtocolError, \ from neo.protocol import Packet, ProtocolError, \
...@@ -240,8 +241,21 @@ class Application(object): ...@@ -240,8 +241,21 @@ class Application(object):
# for locking objects against load operations. # for locking objects against load operations.
self.load_lock_dict = {} self.load_lock_dict = {}
# This is a queue of events used to delay operations due to locks.
self.event_queue = deque()
while 1: while 1:
em.poll(1) em.poll(1)
def queueEvent(self, callable, *args, **kwargs):
self.event_queue.append((callable, args, kwargs))
def executeQueuedEvents(self):
l = len(self.event_queue)
p = self.event_queue.popleft
for i in xrange(l):
callable, args, kwargs = p()
callable(*args, **kwargs)
def getPartition(self, oid_or_tid): def getPartition(self, oid_or_tid):
return unpack('!Q', oid_or_tid)[0] % self.num_partitions return unpack('!Q', oid_or_tid)[0] % self.num_partitions
...@@ -74,13 +74,14 @@ class DatabaseManager(object): ...@@ -74,13 +74,14 @@ class DatabaseManager(object):
raise NotImplementedError('this method must be overridden') raise NotImplementedError('this method must be overridden')
def getObject(self, oid, tid = None, before_tid = None): def getObject(self, oid, tid = None, before_tid = None):
"""Return a tuple of a serial, a compression """Return a tuple of a serial, next serial, a compression
specification, a checksum, and object data, if a given object specification, a checksum, and object data, if a given object
ID is present. Otherwise, return None. If tid is None and ID is present. Otherwise, return None. If tid is None and
before_tid is None, the latest revision is taken. If tid is before_tid is None, the latest revision is taken. If tid is
specified, the given revision is taken. If tid is not specified, specified, the given revision is taken. If tid is not specified,
but before_tid is specified, the latest revision before the but before_tid is specified, the latest revision before the
given revision is taken.""" given revision is taken. The next serial is a serial right after
before_tid, if specified. Otherwise, it is None."""
raise NotImplementedError('this method must be overridden') raise NotImplementedError('this method must be overridden')
def changePartitionTable(self, ptid, cell_list): def changePartitionTable(self, ptid, cell_list):
...@@ -125,8 +126,14 @@ class DatabaseManager(object): ...@@ -125,8 +126,14 @@ class DatabaseManager(object):
raise NotImplementedError('this method must be overridden') raise NotImplementedError('this method must be overridden')
def getObjectHistory(self, oid, length = 1): def getObjectHistory(self, oid, length = 1):
"""Return a list of serials for a given object ID. The length """Return a list of serials and sizes for a given object ID.
specifies the maximum size of such a list. The first serial The length specifies the maximum size of such a list. The first serial
must be the last serial, and the list must be sorted in descending must be the last serial, and the list must be sorted in descending
order. If there is no such object ID in a database, return None.""" order. If there is no such object ID in a database, return None."""
raise NotImplementedError('this method must be overridden') raise NotImplementedError('this method must be overridden')
def getTIDList(self, offset, length, num_partitions, partition_list):
"""Return a list of TIDs in descending order from an offset,
at most the specified length. The list of partitions are passed
to filter out non-applicable TIDs."""
raise NotImplementedError('this method must be overridden')
...@@ -181,9 +181,20 @@ class StorageEventHandler(EventHandler): ...@@ -181,9 +181,20 @@ class StorageEventHandler(EventHandler):
def handleUnlockInformation(self, conn, packet, tid): def handleUnlockInformation(self, conn, packet, tid):
raise NotImplementedError('this method must be overridden') raise NotImplementedError('this method must be overridden')
def handleAskObject(self, conn, packet, oid, serial, tid):
self.handleUnexpectedPacket(conn, packet)
def handleAskTIDs(self, conn, packet, first, last):
self.handleUnexpectedPacket(conn, packet)
def handleAskObjectHistory(self, conn, packet, oid, length):
self.handleUnexpectedPacket(conn, packet)
def handleAskStoreTransaction(self, conn, packet, tid, user, desc,
ext, oid_list):
self.handleUnexpectedPacket(conn, packet)
def handleAskStoreObject(self, conn, packet, msg_id, oid, serial, def handleAskStoreObject(self, conn, packet, msg_id, oid, serial,
compression, data, checksum, tid): compression, data, checksum, tid):
self.handleUnexpectedPacket(conn, packet) self.handleUnexpectedPacket(conn, packet)
def handleAskObject(self, conn, packet, oid, serial, tid):
self.handleUnexpectedPacket(conn, packet)
...@@ -267,12 +267,29 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -267,12 +267,29 @@ class MySQLDatabaseManager(DatabaseManager):
r = q("""SELECT serial, compression, checksum, data FROM obj r = q("""SELECT serial, compression, checksum, data FROM obj
WHERE oid = '%s' AND serial = '%s'""" \ WHERE oid = '%s' AND serial = '%s'""" \
% (oid, tid)) % (oid, tid))
try:
serial, compression, checksum, data = r[0]
next_serial = None
except IndexError:
return None
elif before_tid is not None: elif before_tid is not None:
before_tid = e(before_tid) before_tid = e(before_tid)
r = q("""SELECT serial, compression, checksum, data FROM obj r = q("""SELECT serial, compression, checksum, data FROM obj
WHERE oid = '%s' AND serial < '%s' WHERE oid = '%s' AND serial < '%s'
ORDER BY serial DESC LIMIT 1""" \ ORDER BY serial DESC LIMIT 1""" \
% (oid, before_tid)) % (oid, before_tid))
try:
serial, compression, checksum, data = r[0]
r = q("""SELECT serial FROM obj
WHERE oid = '%s' AND serial > '%s'
ORDER BY serial LIMIT 1""" \
% (oid, before_tid))
try:
next_serial = r[0][0]
except IndexError:
next_serial = None
except IndexError:
return None
else: else:
# XXX I want to express "HAVING serial = MAX(serial)", but # XXX I want to express "HAVING serial = MAX(serial)", but
# MySQL does not use an index for a HAVING clause! # MySQL does not use an index for a HAVING clause!
...@@ -280,10 +297,13 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -280,10 +297,13 @@ class MySQLDatabaseManager(DatabaseManager):
WHERE oid = '%s' ORDER BY serial DESC LIMIT 1""" \ WHERE oid = '%s' ORDER BY serial DESC LIMIT 1""" \
% oid) % oid)
try: try:
return r[0] serial, compression, checksum, data = r[0]
next_serial = None
except IndexError: except IndexError:
return None return None
return serial, next_serial, compression, checksum, data
def doSetPartitionTable(self, ptid, cell_list, reset): def doSetPartitionTable(self, ptid, cell_list, reset):
q = self.query q = self.query
e = self.escape e = self.escape
...@@ -403,9 +423,17 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -403,9 +423,17 @@ class MySQLDatabaseManager(DatabaseManager):
q = self.query q = self.query
e = self.escape e = self.escape
oid = e(oid) oid = e(oid)
r = q("""SELECT serial FROM obj WHERE oid = '%s' r = q("""SELECT serial, LENGTH(data) FROM obj WHERE oid = '%s'
ORDER BY serial DESC LIMIT %d""" \ ORDER BY serial DESC LIMIT %d""" \
% (oid, length)) % (oid, length))
if r: if r:
return [t[0] for t in r] return r
return None return None
def getTIDList(self, offset, length, num_partitions, partition_list):
q = self.query
e = self.escape
r = q("""SELECT tid FROM trans WHERE MOD(tid,%d) in (%s)
ORDER BY tid DESC LIMIT %d""" \
% (offset, num_partitions, ','.join(partition_list), length))
return [t[0] for t in r]
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment