Commit 233a50cc authored by Julien Muchembled's avatar Julien Muchembled

storage: do not commit too often when importing many small transactions

The only purpose of committing during import is to not restart from the
beginning in case of interruption.
parent 6ca0ab70
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import cPickle as pickle import cPickle as pickle, time
from bisect import bisect, insort from bisect import bisect, insort
from collections import defaultdict from collections import defaultdict
from ConfigParser import SafeConfigParser from ConfigParser import SafeConfigParser
...@@ -147,6 +147,8 @@ class ZODBIterator(object): ...@@ -147,6 +147,8 @@ class ZODBIterator(object):
class ImporterDatabaseManager(DatabaseManager): class ImporterDatabaseManager(DatabaseManager):
"""Proxy that transparently imports data from a ZODB storage """Proxy that transparently imports data from a ZODB storage
""" """
_last_commit = 0
def __init__(self, *args, **kw): def __init__(self, *args, **kw):
super(ImporterDatabaseManager, self).__init__(*args, **kw) super(ImporterDatabaseManager, self).__init__(*args, **kw)
self.db._connect() self.db._connect()
...@@ -166,13 +168,17 @@ class ImporterDatabaseManager(DatabaseManager): ...@@ -166,13 +168,17 @@ class ImporterDatabaseManager(DatabaseManager):
self.compress = main.get('compress', 1) self.compress = main.get('compress', 1)
self.db = buildDatabaseManager(main['adapter'], self.db = buildDatabaseManager(main['adapter'],
(main['database'], main['wait'])) (main['database'], main['wait']))
for x in """commit query erase getConfiguration _setConfiguration for x in """query erase getConfiguration _setConfiguration
getPartitionTable changePartitionTable getUnfinishedTIDList getPartitionTable changePartitionTable getUnfinishedTIDList
dropUnfinishedData storeTransaction finishTransaction dropUnfinishedData storeTransaction finishTransaction
storeData storeData
""".split(): """.split():
setattr(self, x, getattr(self.db, x)) setattr(self, x, getattr(self.db, x))
def commit(self):
self.db.commit()
self._last_commit = time.time()
def setNumPartitions(self, num_partitions): def setNumPartitions(self, num_partitions):
self.db.setNumPartitions(num_partitions) self.db.setNumPartitions(num_partitions)
try: try:
...@@ -230,6 +236,7 @@ class ImporterDatabaseManager(DatabaseManager): ...@@ -230,6 +236,7 @@ class ImporterDatabaseManager(DatabaseManager):
pickle.dumps(txn.extension), False, tid), False) pickle.dumps(txn.extension), False, tid), False)
logging.debug("TXN %s imported (user=%r, desc=%r, len(oid)=%s)", logging.debug("TXN %s imported (user=%r, desc=%r, len(oid)=%s)",
util.dump(tid), txn.user, txn.description, len(oid_list)) util.dump(tid), txn.user, txn.description, len(oid_list))
if self._last_commit + 1 < time.time():
self.commit() self.commit()
self.zodb_tid = u64(tid) self.zodb_tid = u64(tid)
if self.compress: if self.compress:
...@@ -275,6 +282,7 @@ class ImporterDatabaseManager(DatabaseManager): ...@@ -275,6 +282,7 @@ class ImporterDatabaseManager(DatabaseManager):
z.next() z.next()
except StopIteration: except StopIteration:
del zodb_list[0] del zodb_list[0]
self._last_commit = 0
finish() finish()
logging.warning("All data are imported. You should change" logging.warning("All data are imported. You should change"
" your configuration to use the native backend and restart.") " your configuration to use the native backend and restart.")
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment