Commit 9502dd02 by Julien Muchembled

New populate() method for threaded tests

Storage.importFrom is changed for resumable imports.
1 parent 0a511052
...@@ -232,10 +232,10 @@ class Storage(BaseStorage.BaseStorage, ...@@ -232,10 +232,10 @@ class Storage(BaseStorage.BaseStorage,
return self.app.importFrom(source, None, None, return self.app.importFrom(source, None, None,
self.tryToResolveConflict) self.tryToResolveConflict)
def importFrom(self, source, start=None, stop=None): def importFrom(self, source, start=None, stop=None, preindex=None):
""" Allow import only a part of the source storage """ """ Allow import only a part of the source storage """
return self.app.importFrom(source, start, stop, return self.app.importFrom(source, start, stop,
self.tryToResolveConflict) self.tryToResolveConflict, preindex)
def restore(self, oid, serial, data, version, prev_txn, transaction): def restore(self, oid, serial, data, version, prev_txn, transaction):
raise NotImplementedError raise NotImplementedError
......
...@@ -1000,18 +1000,20 @@ class Application(object): ...@@ -1000,18 +1000,20 @@ class Application(object):
return result return result
@profiler_decorator @profiler_decorator
def importFrom(self, source, start, stop, tryToResolveConflict): def importFrom(self, source, start, stop, tryToResolveConflict,
serials = {} preindex=None):
if preindex is None:
preindex = {}
transaction_iter = source.iterator(start, stop) transaction_iter = source.iterator(start, stop)
for transaction in transaction_iter: for transaction in transaction_iter:
tid = transaction.tid tid = transaction.tid
self.tpc_begin(transaction, tid, transaction.status) self.tpc_begin(transaction, tid, transaction.status)
for r in transaction: for r in transaction:
oid = r.oid oid = r.oid
pre = serials.get(oid, None) pre = preindex.get(oid)
# TODO: bypass conflict resolution, locks... # TODO: bypass conflict resolution, locks...
self.store(oid, pre, r.data, r.version, transaction) self.store(oid, pre, r.data, r.version, transaction)
serials[oid] = tid preindex[oid] = tid
conflicted = self.tpc_vote(transaction, tryToResolveConflict) conflicted = self.tpc_vote(transaction, tryToResolveConflict)
assert not conflicted, conflicted assert not conflicted, conflicted
real_tid = self.tpc_finish(transaction, tryToResolveConflict) real_tid = self.tpc_finish(transaction, tryToResolveConflict)
......
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import math, os, random, sys import math, os, random, sys, time
from cStringIO import StringIO from cStringIO import StringIO
from ZODB.utils import p64 from persistent.TimeStamp import TimeStamp
from ZODB.utils import p64, newTid
from ZODB.BaseStorage import TransactionRecord from ZODB.BaseStorage import TransactionRecord
from ZODB.FileStorage import FileStorage from ZODB.FileStorage import FileStorage
...@@ -96,8 +97,10 @@ class DummyZODB(object): ...@@ -96,8 +97,10 @@ class DummyZODB(object):
size = 0 size = 0
def iterator(storage, *args): def iterator(storage, *args):
args = ' ', '', '', {} args = ' ', '', '', {}
tid = None
for i in xrange(1, transaction_count+1): for i in xrange(1, transaction_count+1):
t = dummy_transaction(p64(i), *args) tid = newTid(tid)
t = dummy_transaction(tid, *args)
storage.size += t.size storage.size += t.size
yield t yield t
def getSize(self): def getSize(self):
......
...@@ -655,6 +655,15 @@ class NEOCluster(object): ...@@ -655,6 +655,15 @@ class NEOCluster(object):
self.client.setPoll(True) self.client.setPoll(True)
return Storage.Storage(None, self.name, _app=self.client, **kw) return Storage.Storage(None, self.name, _app=self.client, **kw)
def populate(self, dummy_zodb=None):
if dummy_zodb is None:
from ..stat_zodb import PROD1
dummy_zodb = PROD1()
importFrom = self.getZODBStorage().importFrom
preindex = {}
as_storage = dummy_zodb.as_storage
return lambda count: importFrom(as_storage(count), preindex=preindex)
def getTransaction(self): def getTransaction(self):
txn = transaction.TransactionManager() txn = transaction.TransactionManager()
return txn, self.db.open(transaction_manager=txn) return txn, self.db.open(transaction_manager=txn)
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!