Commit 9502dd02 by Julien Muchembled

New populate() method for threaded tests

Storage.importFrom is changed for resumable imports.
1 parent 0a511052
......@@ -232,10 +232,10 @@ class Storage(BaseStorage.BaseStorage,
return self.app.importFrom(source, None, None,
self.tryToResolveConflict)
def importFrom(self, source, start=None, stop=None):
def importFrom(self, source, start=None, stop=None, preindex=None):
""" Allow import only a part of the source storage """
return self.app.importFrom(source, start, stop,
self.tryToResolveConflict)
self.tryToResolveConflict, preindex)
def restore(self, oid, serial, data, version, prev_txn, transaction):
raise NotImplementedError
......
......@@ -1000,18 +1000,20 @@ class Application(object):
return result
@profiler_decorator
def importFrom(self, source, start, stop, tryToResolveConflict):
serials = {}
def importFrom(self, source, start, stop, tryToResolveConflict,
preindex=None):
if preindex is None:
preindex = {}
transaction_iter = source.iterator(start, stop)
for transaction in transaction_iter:
tid = transaction.tid
self.tpc_begin(transaction, tid, transaction.status)
for r in transaction:
oid = r.oid
pre = serials.get(oid, None)
pre = preindex.get(oid)
# TODO: bypass conflict resolution, locks...
self.store(oid, pre, r.data, r.version, transaction)
serials[oid] = tid
preindex[oid] = tid
conflicted = self.tpc_vote(transaction, tryToResolveConflict)
assert not conflicted, conflicted
real_tid = self.tpc_finish(transaction, tryToResolveConflict)
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import math, os, random, sys
import math, os, random, sys, time
from cStringIO import StringIO
from ZODB.utils import p64
from persistent.TimeStamp import TimeStamp
from ZODB.utils import p64, newTid
from ZODB.BaseStorage import TransactionRecord
from ZODB.FileStorage import FileStorage
......@@ -96,8 +97,10 @@ class DummyZODB(object):
size = 0
def iterator(storage, *args):
args = ' ', '', '', {}
tid = None
for i in xrange(1, transaction_count+1):
t = dummy_transaction(p64(i), *args)
tid = newTid(tid)
t = dummy_transaction(tid, *args)
storage.size += t.size
yield t
def getSize(self):
......
......@@ -655,6 +655,15 @@ class NEOCluster(object):
self.client.setPoll(True)
return Storage.Storage(None, self.name, _app=self.client, **kw)
def populate(self, dummy_zodb=None):
if dummy_zodb is None:
from ..stat_zodb import PROD1
dummy_zodb = PROD1()
importFrom = self.getZODBStorage().importFrom
preindex = {}
as_storage = dummy_zodb.as_storage
return lambda count: importFrom(as_storage(count), preindex=preindex)
def getTransaction(self):
txn = transaction.TransactionManager()
return txn, self.db.open(transaction_manager=txn)
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!