Commit 68401e70 authored by Julien Muchembled's avatar Julien Muchembled

mysql: use fewer queries to fill (t)obj when storing a transaction

parent b70b4689
...@@ -417,6 +417,9 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -417,6 +417,9 @@ class MySQLDatabaseManager(DatabaseManager):
obj_table = 'obj' obj_table = 'obj'
trans_table = 'trans' trans_table = 'trans'
q = self.query q = self.query
sql = ["REPLACE INTO %s VALUES " % obj_table]
values_max = self._max_allowed_packet - len(sql[0])
values_size = 0
for oid, data_id, value_serial in object_list: for oid, data_id, value_serial in object_list:
oid = u64(oid) oid = u64(oid)
partition = self._getPartition(oid) partition = self._getPartition(oid)
...@@ -429,8 +432,20 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -429,8 +432,20 @@ class MySQLDatabaseManager(DatabaseManager):
self.holdData(data_id) self.holdData(data_id)
else: else:
value_serial = 'NULL' value_serial = 'NULL'
q("REPLACE INTO %s VALUES (%d, %d, %d, %s, %s)" % (obj_table, value = "(%s,%s,%s,%s,%s)," % (
partition, oid, tid, data_id or 'NULL', value_serial)) partition, oid, tid, data_id or 'NULL', value_serial)
values_size += len(value)
# actually: max_values < values_size + EXTRA - len(final comma)
# (test_max_allowed_packet checks that EXTRA == 2)
if values_max <= values_size:
sql[-1] = sql[-1][:-1] # remove final comma
q(''.join(sql))
del sql[1:]
values_size = len(value)
sql.append(value)
if values_size:
sql[-1] = value[:-1] # remove final comma
q(''.join(sql))
if transaction: if transaction:
oid_list, user, desc, ext, packed, ttid = transaction oid_list, user, desc, ext, packed, ttid = transaction
partition = self._getPartition(tid) partition = self._getPartition(tid)
......
...@@ -18,6 +18,7 @@ import unittest ...@@ -18,6 +18,7 @@ import unittest
import MySQLdb import MySQLdb
from mock import Mock from mock import Mock
from neo.lib.exception import DatabaseFailure from neo.lib.exception import DatabaseFailure
from neo.lib.util import p64
from .testStorageDBTests import StorageDBTests from .testStorageDBTests import StorageDBTests
from neo.storage.database.mysqldb import MySQLDatabaseManager from neo.storage.database.mysqldb import MySQLDatabaseManager
...@@ -109,6 +110,15 @@ class StorageMySQLdbTests(StorageDBTests): ...@@ -109,6 +110,15 @@ class StorageMySQLdbTests(StorageDBTests):
size, = query_list size, = query_list
max_allowed = self.db.__class__._max_allowed_packet max_allowed = self.db.__class__._max_allowed_packet
self.assertTrue(max_allowed - 1024 < size <= max_allowed, size) self.assertTrue(max_allowed - 1024 < size <= max_allowed, size)
# Check storeTransaction
for count, max_allowed_packet in (7, 64), (6, 65), (1, 215):
self.db._max_allowed_packet = max_allowed_packet
del query_list[:]
self.db.storeTransaction(p64(0),
((p64(1<<i),0,None) for i in xrange(10)), None)
self.assertEqual(max(query_list), max_allowed_packet)
self.assertEqual(len(query_list), count)
class StorageMySQLdbTokuDBTests(StorageMySQLdbTests): class StorageMySQLdbTokuDBTests(StorageMySQLdbTests):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment