mysql: use fewer queries to fill (t)obj when storing a transaction

neo/storage/database/mysqldb.py

@@ -417,6 +417,9 @@ class MySQLDatabaseManager(DatabaseManager):
             obj_table = 'obj'
             trans_table = 'trans'
         q = self.query
+        sql = ["REPLACE INTO %s VALUES " % obj_table]
+        values_max = self._max_allowed_packet - len(sql[0])
+        values_size = 0
         for oid, data_id, value_serial in object_list:
             oid = u64(oid)
             partition = self._getPartition(oid)
@@ -429,8 +432,20 @@ class MySQLDatabaseManager(DatabaseManager):
                     self.holdData(data_id)
             else:
                 value_serial = 'NULL'
-            q("REPLACE INTO %s VALUES (%d, %d, %d, %s, %s)" % (obj_table,
-                partition, oid, tid, data_id or 'NULL', value_serial))
+            value = "(%s,%s,%s,%s,%s)," % (
+                partition, oid, tid, data_id or 'NULL', value_serial)
+            values_size += len(value)
+            # actually: max_values < values_size + EXTRA - len(final comma)
+            # (test_max_allowed_packet checks that EXTRA == 2)
+            if values_max <= values_size:
+                sql[-1] = sql[-1][:-1] # remove final comma
+                q(''.join(sql))
+                del sql[1:]
+                values_size = len(value)
+            sql.append(value)
+        if values_size:
+            sql[-1] = value[:-1] # remove final comma
+            q(''.join(sql))
         if transaction:
             oid_list, user, desc, ext, packed, ttid = transaction
             partition = self._getPartition(tid)

neo/tests/storage/testStorageMySQL.py

@@ -18,6 +18,7 @@ import unittest
 import MySQLdb
 from mock import Mock
 from neo.lib.exception import DatabaseFailure
+from neo.lib.util import p64
 from .testStorageDBTests import StorageDBTests
 from neo.storage.database.mysqldb import MySQLDatabaseManager
 
@@ -109,6 +110,15 @@ class StorageMySQLdbTests(StorageDBTests):
         size, = query_list
         max_allowed = self.db.__class__._max_allowed_packet
         self.assertTrue(max_allowed - 1024 < size <= max_allowed, size)
+        # Check storeTransaction
+        for count, max_allowed_packet in (7, 64), (6, 65), (1, 215):
+            self.db._max_allowed_packet = max_allowed_packet
+            del query_list[:]
+            self.db.storeTransaction(p64(0),
+                ((p64(1<<i),0,None) for i in xrange(10)), None)
+            self.assertEqual(max(query_list), max_allowed_packet)
+            self.assertEqual(len(query_list), count)
+
 
 class StorageMySQLdbTokuDBTests(StorageMySQLdbTests):