Skip to content

N
neoppod
Commit a4d15b91 authored by Yoshinori Okuji's avatar Yoshinori Okuji
Browse files
Options

Fix arguments to handleAnswerTIDs. Change the MySQL database to use BIGINT... 

Fix arguments to handleAnswerTIDs. Change the MySQL database to use BIGINT rather than BINARY for oid, serial and tid.

git-svn-id: https://svn.erp5.org/repos/neo/branches/prototype3@165 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent 99d73db1
Hide whitespace changes
Inline Side-by-side
Showing with 62 additions and 52 deletions
neo/handler.py
View file @ a4d15b91
......@@ -238,7 +238,7 @@ class EventHandler(object):
self.handleUnexpectedPacket(conn, packet)
def handleAnswerTIDs(self, conn, packet, tid_list):
self.handleUnexpectedPacket(conn, conn, packet, packet)
self.handleUnexpectedPacket(conn, packet)
def handleAskTransactionInformation(self, conn, packet, tid):
self.handleUnexpectedPacket(conn, packet)
......
neo/storage/mysqldb.py
View file @ a4d15b91
......@@ -4,12 +4,19 @@ from MySQLdb.constants.CR import SERVER_GONE_ERROR, SERVER_LOST
import logging
from array import array
import string
from struct import pack, unpack
from neo.storage.database import DatabaseManager
from neo.exception import DatabaseFailure
from neo.util import dump
from neo.protocol import DISCARDED_STATE
def p64(n):
return pack('!Q', n)
def u64(s):
return unpack('!Q', s)[0]
class MySQLDatabaseManager(DatabaseManager):
"""This class manages a database on MySQL."""
......@@ -54,7 +61,7 @@ class MySQLDatabaseManager(DatabaseManager):
conn = self.conn
try:
printable_char_list = []
for c in query.split('\n', 1)[0]:
for c in query.split('\n', 1)[0][:70]:
if c not in string.printable or c in '\t\x0b\x0c\r':
c = '\\x%02x' % ord(c)
printable_char_list.append(c)
......@@ -109,7 +116,7 @@ class MySQLDatabaseManager(DatabaseManager):
# The table "trans" stores information on committed transactions.
q("""CREATE TABLE IF NOT EXISTS trans (
tid BINARY(8) NOT NULL PRIMARY KEY,
tid BIGINT UNSIGNED NOT NULL PRIMARY KEY,
oids MEDIUMBLOB NOT NULL,
user BLOB NOT NULL,
description BLOB NOT NULL,
......@@ -118,8 +125,8 @@ class MySQLDatabaseManager(DatabaseManager):
# The table "obj" stores committed object data.
q("""CREATE TABLE IF NOT EXISTS obj (
oid BINARY(8) NOT NULL,
serial BINARY(8) NOT NULL,
oid BIGINT UNSIGNED NOT NULL,
serial BIGINT UNSIGNED NOT NULL,
compression TINYINT UNSIGNED NOT NULL,
checksum INT UNSIGNED NOT NULL,
value MEDIUMBLOB NOT NULL,
......@@ -128,7 +135,7 @@ class MySQLDatabaseManager(DatabaseManager):
# The table "ttrans" stores information on uncommitted transactions.
q("""CREATE TABLE IF NOT EXISTS ttrans (
tid BINARY(8) NOT NULL,
tid BIGINT UNSIGNED NOT NULL,
oids MEDIUMBLOB NOT NULL,
user BLOB NOT NULL,
description BLOB NOT NULL,
......@@ -137,8 +144,8 @@ class MySQLDatabaseManager(DatabaseManager):
# The table "tobj" stores uncommitted object data.
q("""CREATE TABLE IF NOT EXISTS tobj (
oid BINARY(8) NOT NULL,
serial BINARY(8) NOT NULL,
oid BIGINT UNSIGNED NOT NULL,
serial BIGINT UNSIGNED NOT NULL,
compression TINYINT UNSIGNED NOT NULL,
checksum INT UNSIGNED NOT NULL,
value MEDIUMBLOB NOT NULL
......@@ -224,6 +231,8 @@ class MySQLDatabaseManager(DatabaseManager):
if loid is None or (tmp_loid is not None and loid < tmp_loid):
loid = tmp_loid
self.commit()
if loid is not None:
loid = p64(loid)
return loid
def getLastTID(self, all = True):
......@@ -247,6 +256,8 @@ class MySQLDatabaseManager(DatabaseManager):
if ltid is None or (tmp_serial is not None and ltid < tmp_serial):
ltid = tmp_serial
self.commit()
if ltid is not None:
ltid = p64(ltid)
return ltid
def getUnfinishedTIDList(self):
......@@ -254,22 +265,21 @@ class MySQLDatabaseManager(DatabaseManager):
tid_set = set()
self.begin()
r = q("""SELECT tid FROM ttrans""")
tid_set.update((t[0] for t in r))
tid_set.update((p64(t[0]) for t in r))
r = q("""SELECT serial FROM tobj""")
self.commit()
tid_set.update((t[0] for t in r))
tid_set.update((p64(t[0]) for t in r))
return list(tid_set)
def objectPresent(self, oid, tid, all = True):
q = self.query
e = self.escape
oid = e(oid)
tid = e(tid)
oid = u64(oid)
tid = u64(tid)
self.begin()
r = q("""SELECT oid FROM obj WHERE oid = '%s' AND serial = '%s'""" \
r = q("""SELECT oid FROM obj WHERE oid = %d AND serial = %d""" \
% (oid, tid))
if not r and all:
r = q("""SELECT oid FROM tobj WHERE oid = '%s' AND serial = '%s'""" \
r = q("""SELECT oid FROM tobj WHERE oid = %d AND serial = %d""" \
% (oid, tid))
self.commit()
if r:
......@@ -278,12 +288,11 @@ class MySQLDatabaseManager(DatabaseManager):
def getObject(self, oid, tid = None, before_tid = None):
q = self.query
e = self.escape
oid = e(oid)
oid = u64(oid)
if tid is not None:
tid = e(tid)
tid = u64(tid)
r = q("""SELECT serial, compression, checksum, value FROM obj
WHERE oid = '%s' AND serial = '%s'""" \
WHERE oid = %d AND serial = %d""" \
% (oid, tid))
try:
serial, compression, checksum, data = r[0]
......@@ -291,15 +300,15 @@ class MySQLDatabaseManager(DatabaseManager):
except IndexError:
return None
elif before_tid is not None:
before_tid = e(before_tid)
before_tid = u64(before_tid)
r = q("""SELECT serial, compression, checksum, value FROM obj
WHERE oid = '%s' AND serial < '%s'
WHERE oid = %d AND serial < %d
ORDER BY serial DESC LIMIT 1""" \
% (oid, before_tid))
try:
serial, compression, checksum, data = r[0]
r = q("""SELECT serial FROM obj
WHERE oid = '%s' AND serial > '%s'
WHERE oid = %d AND serial > %d
ORDER BY serial LIMIT 1""" \
% (oid, before_tid))
try:
......@@ -312,7 +321,7 @@ class MySQLDatabaseManager(DatabaseManager):
# XXX I want to express "HAVING serial = MAX(serial)", but
# MySQL does not use an index for a HAVING clause!
r = q("""SELECT serial, compression, checksum, value FROM obj
WHERE oid = '%s' ORDER BY serial DESC LIMIT 1""" \
WHERE oid = %d ORDER BY serial DESC LIMIT 1""" \
% oid)
try:
serial, compression, checksum, data = r[0]
......@@ -320,6 +329,10 @@ class MySQLDatabaseManager(DatabaseManager):
except IndexError:
return None
if serial is not None:
serial = p64(serial)
if next_serial is not None:
next_serial = p64(next_serial)
return serial, next_serial, compression, checksum, data
def doSetPartitionTable(self, ptid, cell_list, reset):
......@@ -365,7 +378,7 @@ class MySQLDatabaseManager(DatabaseManager):
def storeTransaction(self, tid, object_list, transaction):
q = self.query
e = self.escape
tid = e(tid)
tid = u64(tid)
self.begin()
try:
# XXX it might be more efficient to insert multiple objects
......@@ -375,9 +388,9 @@ class MySQLDatabaseManager(DatabaseManager):
# tobj has no index, so inserting one by one should not be
# significantly different from inserting many at a time.
for oid, compression, checksum, data in object_list:
oid = e(oid)
oid = u64(oid)
data = e(data)
q("""INSERT INTO tobj VALUES ('%s', '%s', %d, %d, '%s')""" \
q("""INSERT INTO tobj VALUES (%d, %d, %d, %d, '%s')""" \
% (oid, tid, compression, checksum, data))
if transaction is not None:
oid_list, user, desc, ext = transaction
......@@ -385,7 +398,7 @@ class MySQLDatabaseManager(DatabaseManager):
user = e(user)
desc = e(desc)
ext = e(ext)
q("""INSERT INTO ttrans VALUES ('%s', '%s', '%s', '%s', '%s')""" \
q("""INSERT INTO ttrans VALUES (%d, '%s', '%s', '%s', '%s')""" \
% (tid, oids, user, desc, ext))
except:
self.rollback()
......@@ -394,16 +407,15 @@ class MySQLDatabaseManager(DatabaseManager):
def finishTransaction(self, tid):
q = self.query
e = self.escape
tid = e(tid)
tid = u64(tid)
self.begin()
try:
q("""INSERT INTO obj SELECT * FROM tobj WHERE tobj.serial = '%s'""" \
q("""INSERT INTO obj SELECT * FROM tobj WHERE tobj.serial = %d""" \
% tid)
q("""DELETE FROM tobj WHERE serial = '%s'""" % tid)
q("""INSERT INTO trans SELECT * FROM ttrans WHERE ttrans.tid = '%s'""" \
q("""DELETE FROM tobj WHERE serial = %d""" % tid)
q("""INSERT INTO trans SELECT * FROM ttrans WHERE ttrans.tid = %d""" \
% tid)
q("""DELETE FROM ttrans WHERE tid = '%s'""" % tid)
q("""DELETE FROM ttrans WHERE tid = %d""" % tid)
except:
self.rollback()
raise
......@@ -411,16 +423,15 @@ class MySQLDatabaseManager(DatabaseManager):
def deleteTransaction(self, tid, all = False):
q = self.query
e = self.escape
tid = e(tid)
tid = u64(tid)
self.begin()
try:
q("""DELETE FROM tobj WHERE serial = '%s'""" % tid)
q("""DELETE FROM ttrans WHERE tid = '%s'""" % tid)
q("""DELETE FROM tobj WHERE serial = %d""" % tid)
q("""DELETE FROM ttrans WHERE tid = %d""" % tid)
if all:
# Note that this can be very slow.
q("""DELETE FROM obj WHERE serial = '%s'""" % tid)
q("""DELETE FROM trans WHERE tid = '%s'""" % tid)
q("""DELETE FROM obj WHERE serial = %d""" % tid)
q("""DELETE FROM trans WHERE tid = %d""" % tid)
except:
self.rollback()
raise
......@@ -428,21 +439,20 @@ class MySQLDatabaseManager(DatabaseManager):
def getTransaction(self, tid, all = False):
q = self.query
e = self.escape
tid = e(tid)
tid = u64(tid)
self.begin()
r = q("""SELECT oids, user, description, ext FROM trans
WHERE tid = '%s'""" \
WHERE tid = %d""" \
% tid)
if not r and all:
r = q("""SELECT oids, user, description, ext FROM ttrans
WHERE tid = '%s'""" \
WHERE tid = %d""" \
% tid)
self.commit()
if r:
oids, user, desc, ext = r[0]
if (len(oids) % 8) != 0 or len(oids) == 0:
raise DatabaseFailure('invalid oids for tid %s' % dump(tid))
raise DatabaseFailure('invalid oids for tid %x' % tid)
oid_list = []
for i in xrange(0, len(oids), 8):
oid_list.append(oids[i:i+8])
......@@ -451,19 +461,19 @@ class MySQLDatabaseManager(DatabaseManager):
def getObjectHistory(self, oid, length = 1):
q = self.query
e = self.escape
oid = e(oid)
r = q("""SELECT serial, LENGTH(value) FROM obj WHERE oid = '%s'
oid = u64(oid)
r = q("""SELECT serial, LENGTH(value) FROM obj WHERE oid = %d
ORDER BY serial DESC LIMIT %d""" \
% (oid, length))
if r:
return r
return [(p64(serial), length) for serial, length in r]
return None
def getTIDList(self, offset, length, num_partitions, partition_list):
q = self.query
e = self.escape
r = q("""SELECT tid FROM trans WHERE MOD(tid,%d) in (%s)
ORDER BY tid DESC LIMIT %d""" \
% (offset, num_partitions, ','.join(partition_list), length))
return [t[0] for t in r]
ORDER BY tid DESC LIMIT %d,%d""" \
% (num_partitions,
','.join([str(p) for p in partition_list]),
offset, length))
return [p64(t[0]) for t in r]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7