Commit de4c9e1d authored by Julien Muchembled's avatar Julien Muchembled

New 'Importer' storage backend

parent 53df0e1a
......@@ -33,7 +33,9 @@ ZODB API is fully implemented except:
(full implementation is considered)
- blobs: not implemented (not considered yet)
There is a simple way to convert FileStorage to NEO and back again.
Any ZODB like FileStorage can be converted to NEO instanteously,
which means the database is operational before all data are imported.
There's also a tool to convert back to FileStorage.
See also http://www.neoppod.org/links for more detailed information about
features related to scalability.
......@@ -88,7 +90,7 @@ e. Clients can connect when the cluster is in RUNNING state::
$ neoctl -a <admin> print cluster
RUNNING
f. See `neomigrate` command to import an existing FileStorage database,
f. See `importer.conf` file to import an existing database,
or `neoctl` command for more administrative tasks.
Alternatively, you can use `neosimple` command to quickly setup a cluster for
......
# This file describes how to use the NEO Importer storage backend,
# which is the recommended way to import the data of an existing Zope database
# into a NEO cluster.
#
# Note that the 'neomigrate' command is another way to migrate to NEO but:
# - the database is unusable as long as migration is not finished;
# - it can not merge bases at mountpoints;
# - it can not resume an interrupted migration;
# - it does not preserve metadata that are specific to "undo" transactions
# (which would then behave like normal transactions);
# - it only supports conversion from FileStorage;
# - it is slower.
# The only advantage of 'neomigrate' over Importer is that data can be imported
# directly to a NEO cluster with replicas or several storage nodes.
# Importer backend can only be used with a single storage node.
#
# Here is how to proceed once this file ready:
# 1. Restart ZODB clients to connect to new NEO cluster (not started yet).
# 2. Start NEO cluster and use 'neoctl -a <admin> start' command.
# 3. Your Zope applications should work and background migration of data
# started automatically. The only downtime depends on how fast you are to do
# the first 2 steps. The only limitations in this mode are that:
# - pack is not supported
# - IStorage.history() is not implemented (this is used for example by
# history tab in Zope Management Interface)
# 4. A warning message reporting that "All data are imported"
# is emitted to storage node's log when the migration is finished.
# This can take days with big databases.
# The following steps can be scheduled any time after migration is over,
# at your convenience:
# 5. Change NEO configuration to stop using Importer backend.
# 6. Stop clients.
# 7. Restart NEO cluster.
# 8. Start clients. This was the second, very short, downtime.
# 9. Optionally, add storage nodes and balance partitions.
#
# Data are migrated even if your ZODB clients are stopped.
# The first section describes the destination NEO storage.
# See neo.conf for description of parameters.
[neo]
# Once migration is finished, restart NEO storage to use the below directly
# (instead of adapter=Importer & database=/path_to_this_file).
adapter=MySQL
database=neo
# The other sections are for source databases.
[root]
# Example with FileStorage but this can be anything else.
# ZEO is possible but less efficient: ZEO servers must be stopped
# if NEO opens FileStorage DBs directly.
storage=
<filestorage>
path /path/to/root.fs
</filestorage>
# (leading spaces indicate value continuation)
# This file can stop here if your source DB is not splitted.
# Otherwise, you need to describe mountpoints and other ZODB.
# OID mapping can't be changed once the NEO cluster is started with this
# configuration.
# Mountpoints for this ZODB (see the use case at the end of this file).
# <section_name>=<oid>
foo=421
bar=666
# Following sections must define 'oid' parameter.
[foo]
# Any reference to oid 421 in 'root' is changed to point to oid 123 of 'foo'.
# Of course, original oid 421 in 'root' will become unreferenced.
oid=123
storage=
<filestorage>
path /path/to/foo.fs
</filestorage>
baz=1000
[bar]
oid=4567
storage=
<filestorage>
path /path/to/bar.fs
</filestorage>
[baz]
oid=2000
storage=
<filestorage>
path /path/to/baz.fs
</filestorage>
## Case of several databases linked with MountedObject objects
#
# MountedObject is provided by ZODBMountPoint Zope product.
# It relies on IAcquirer and ITraversable to fetch the real object.
#
# Given the following multi-base:
# - in 'foo', /a/b is the real object
# - in 'root', /c/d is a MountedObject object configured to point to
# /a/b in 'foo'
#
# So you need to get the oid of /a/b in 'foo':
# unrestrictedTraverse("/a/b")._p_oid
# which equals to 123 in the above example
#
# And that of /c/d in 'root':
# unrestrictedTraverse("/c").__dict__["d"]._p_oid -> 421
# The way to retrieve the mount point depends on the container type.
# For a BTreeFolder2, it would be: c._tree["d"]._p_oid
......@@ -2,6 +2,9 @@
# must be identical for all nodes in a given cluster.
# This file is optionnal: parameters can be given at the command line.
# See SafeConfigParser at https://docs.python.org/2/library/configparser.html
# for more information about the syntax.
# Common parameters.
[DEFAULT]
# The cluster name
......
......@@ -122,6 +122,11 @@ class Storage(BaseStorage.BaseStorage,
raise POSException.POSKeyError(oid)
def loadBefore(self, oid, tid):
# XXX: FileStorage return an empty string for a deleted object
# but it may cause EOFError exceptions in ZODB.Connection
# and it makes impossible to store empty values.
# We think it's wrong behaviour and raise POSKeyError instead.
# Or maybe we should return None?
try:
return self.app.load(oid, None, tid)
except NEOStorageDoesNotExistError:
......
......@@ -46,9 +46,12 @@ def main(args=None):
from ZODB.FileStorage import FileStorage
from neo.client.Storage import Storage as NEOStorage
if os.path.exists(source):
print("NOTE: NEO does not implement IStorageRestoreable interface,"
" which means that undo information is not preserved: conflict"
" resolution could happen when undoing an old transaction.")
print("WARNING: This is not the recommended way to import data to NEO:"
" you should use Imported backend instead.\n"
"NEO also does not implement IStorageRestoreable interface,"
" which means that undo information is not preserved when using"
" this tool: conflict resolution could happen when undoing an"
" old transaction.")
src = FileStorage(file_name=source, read_only=True)
dst = NEOStorage(master_nodes=destination, name=cluster,
logfile=options.logfile)
......
......@@ -65,6 +65,7 @@ UNIT_TEST_MODULES = [
'neo.tests.client.testConnectionPool',
# light functional tests
'neo.tests.threaded.test',
'neo.tests.threaded.testImporter',
'neo.tests.threaded.testReplication',
]
......
......@@ -289,7 +289,8 @@ class Application(object):
"""Handle everything, including replications and transactions."""
logging.info('doing operation')
_poll = self._poll
poll = self._poll
_poll = self.em._poll
isIdle = self.em.isIdle
handler = master.MasterOperationHandler(self)
......@@ -301,14 +302,18 @@ class Application(object):
self.task_queue = task_queue = deque()
try:
self.dm.doOperation(self)
while True:
while task_queue and isIdle():
while task_queue:
try:
task_queue[-1].next()
while isIdle():
if task_queue[-1].next():
_poll(0)
task_queue.rotate()
break
except StopIteration:
task_queue.pop()
_poll()
poll()
finally:
del self.task_queue
# XXX: Although no handled exception should happen between
......
......@@ -20,6 +20,7 @@ from neo.lib.exception import DatabaseFailure
from .manager import DatabaseManager
DATABASE_MANAGER_DICT = {
'Importer': 'importer.ImporterDatabaseManager',
'MySQL': 'mysqldb.MySQLDatabaseManager',
'SQLite': 'sqlite.SQLiteDatabaseManager',
}
......
This diff is collapsed.
......@@ -81,6 +81,9 @@ class DatabaseManager(object):
"""
raise NotImplementedError
def doOperation(self, app):
pass
def commit(self):
pass
......@@ -198,6 +201,14 @@ class DatabaseManager(object):
node, and a cell state."""
raise NotImplementedError
def getLastTID(self, max_tid):
"""Return greatest tid in trans table that is <= given 'max_tid'
Required only to import a DB using Importer backend.
max_tid must be in unpacked format.
"""
raise NotImplementedError
def _getLastIDs(self, all=True):
raise NotImplementedError
......@@ -228,6 +239,19 @@ class DatabaseManager(object):
r = self.getObject(oid)
return r and r[0]
def _getNextTID(self, partition, oid, tid):
"""
partition (int)
Must be the result of (oid % self.getPartition(oid))
oid (int)
Identifier of object to retrieve.
tid (int)
Exact serial to retrieve.
If tid is the last revision of oid, None is returned.
"""
raise NotImplementedError
def _getObject(self, oid, tid=None, before_tid=None):
"""
oid (int)
......
......@@ -16,9 +16,9 @@
from binascii import a2b_hex
import MySQLdb
from MySQLdb import IntegrityError, OperationalError
from MySQLdb import DataError, IntegrityError, OperationalError
from MySQLdb.constants.CR import SERVER_GONE_ERROR, SERVER_LOST
from MySQLdb.constants.ER import DUP_ENTRY
from MySQLdb.constants.ER import DATA_TOO_LONG, DUP_ENTRY
from array import array
from hashlib import sha1
import re
......@@ -237,16 +237,27 @@ class MySQLDatabaseManager(DatabaseManager):
q = self.query
e = self.escape
self._config[key] = value
key = e(str(key))
k = e(str(key))
if value is None:
q("DELETE FROM config WHERE name = '%s'" % key)
else:
value = e(str(value))
q("REPLACE INTO config VALUES ('%s', '%s')" % (key, value))
q("DELETE FROM config WHERE name = '%s'" % k)
return
value = str(value)
sql = "REPLACE INTO config VALUES ('%s', '%s')" % (k, e(value))
try:
q(sql)
except DataError, (code, _):
if code != DATA_TOO_LONG or len(value) < 256 or key != "zodb":
raise
q("ALTER TABLE config MODIFY value VARBINARY(%s) NULL" % len(value))
q(sql)
def getPartitionTable(self):
return self.query("SELECT * FROM pt")
def getLastTID(self, max_tid):
return self.query("SELECT MAX(tid) FROM trans WHERE tid<=%s"
% max_tid)[0][0]
def _getLastIDs(self, all=True):
p64 = util.p64
q = self.query
......@@ -290,6 +301,12 @@ class MySQLDatabaseManager(DatabaseManager):
% (self._getPartition(oid), oid))
return util.p64(r[0][0]) if r else None
def _getNextTID(self, *args): # partition, oid, tid
r = self.query("SELECT tid FROM obj"
" WHERE partition=%d AND oid=%d AND tid>%d"
" ORDER BY tid LIMIT 1" % args)
return r[0][0] if r else None
def _getObject(self, oid, tid=None, before_tid=None):
q = self.query
partition = self._getPartition(oid)
......@@ -309,10 +326,8 @@ class MySQLDatabaseManager(DatabaseManager):
serial, compression, checksum, data, value_serial = r[0]
except IndexError:
return None
r = q("SELECT tid FROM obj WHERE partition=%d AND oid=%d AND tid>%d"
" ORDER BY tid LIMIT 1" % (partition, oid, serial))
return (serial, r[0][0] if r else None, compression, checksum, data,
value_serial)
return (serial, self._getNextTID(partition, oid, serial),
compression, checksum, data, value_serial)
def changePartitionTable(self, ptid, cell_list, reset=False):
offset_list = []
......
......@@ -208,6 +208,10 @@ class SQLiteDatabaseManager(DatabaseManager):
def getPartitionTable(self):
return self.query("SELECT * FROM pt")
def getLastTID(self, max_tid):
return self.query("SELECT MAX(tid) FROM trans WHERE tid<=?",
(max_tid,)).next()[0]
def _getLastIDs(self, all=True):
p64 = util.p64
q = self.query
......@@ -252,6 +256,12 @@ class SQLiteDatabaseManager(DatabaseManager):
(self._getPartition(oid), oid)).fetchone()
return r and util.p64(r[0])
def _getNextTID(self, *args): # partition, oid, tid
r = self.query("""SELECT tid FROM obj
WHERE partition=? AND oid=? AND tid>?
ORDER BY tid LIMIT 1""", args).fetchone()
return r and r[0]
def _getObject(self, oid, tid=None, before_tid=None):
q = self.query
partition = self._getPartition(oid)
......@@ -269,14 +279,11 @@ class SQLiteDatabaseManager(DatabaseManager):
serial, compression, checksum, data, value_serial = r.fetchone()
except TypeError:
return None
r = q("""SELECT tid FROM obj
WHERE partition=? AND oid=? AND tid>?
ORDER BY tid LIMIT 1""",
(partition, oid, serial)).fetchone()
if checksum:
checksum = str(checksum)
data = str(data)
return serial, r and r[0], compression, checksum, data, value_serial
return (serial, self._getNextTID(partition, oid, serial),
compression, checksum, data, value_serial)
def changePartitionTable(self, ptid, cell_list, reset=False):
q = self.query
......
#
# Copyright (C) 2014 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os, stat, time
from persistent import Persistent
from persistent.TimeStamp import TimeStamp
from BTrees.OOBTree import OOBTree
class Inode(OOBTree):
data = None
def __init__(self, up=None, mode=stat.S_IFDIR):
self[os.pardir] = self if up is None else up
self.mode = mode
self.mtime = time.time()
def __getstate__(self):
return Persistent.__getstate__(self), OOBTree.__getstate__(self)
def __setstate__(self, state):
Persistent.__setstate__(self, state[0])
OOBTree.__setstate__(self, state[1])
def edit(self, data=None, mtime=None):
fmt = stat.S_IFMT(self.mode)
if data is None:
assert fmt == stat.S_IFDIR, oct(fmt)
else:
assert fmt == stat.S_IFREG or fmt == stat.S_IFLNK, oct(fmt)
if self.data != data:
self.data = data
if self.mtime != mtime:
self.mtime = mtime or time.time()
def root(self):
try:
self = self[os.pardir]
except KeyError:
return self
return self.root()
def traverse(self, path, followlinks=True):
path = iter(path.split(os.sep) if isinstance(path, basestring) and path
else path)
for d in path:
if not d:
return self.root().traverse(path, followlinks)
if d != os.curdir:
d = self[d]
if followlinks and stat.S_ISLNK(d.mode):
d = self.traverse(d.data, True)
return d.traverse(path, followlinks)
return self
def inodeFromFs(self, path):
s = os.lstat(path)
mode = s.st_mode
name = os.path.basename(path)
try:
i = self[name]
assert stat.S_IFMT(i.mode) == stat.S_IFMT(mode)
changed = False
except KeyError:
i = self[name] = self.__class__(self, mode)
changed = True
i.edit(open(path).read() if stat.S_ISREG(mode) else
os.readlink(p) if stat.S_ISLNK(mode) else
None, s.st_mtime)
return changed or i._p_changed
def treeFromFs(self, path, yield_interval=None, filter=None):
prefix_len = len(path) + len(os.sep)
n = 0
for dirpath, dirnames, filenames in os.walk(path):
inodeFromFs = self.traverse(dirpath[prefix_len:]).inodeFromFs
for names in dirnames, filenames:
skipped = []
for j, name in enumerate(names):
p = os.path.join(dirpath, name)
if filter and not filter(p[prefix_len:]):
skipped.append(j)
elif inodeFromFs(p):
n += 1
if n == yield_interval:
n = 0
yield self
while skipped:
del names[skipped.pop()]
if n:
yield self
def walk(self):
s = [(None, self)]
while s:
top, self = s.pop()
dirs = []
nondirs = []
for name, inode in self.iteritems():
if name != os.pardir:
(dirs if stat.S_ISDIR(inode.mode) else nondirs).append(name)
yield top or os.curdir, dirs, nondirs
for name in dirs:
s.append((os.path.join(top, name) if top else name, self[name]))
......@@ -30,6 +30,7 @@ import tempfile
import traceback
import threading
import psutil
from ConfigParser import SafeConfigParser
import neo.scripts
from neo.neoctl.neoctl import NeoCTL, NotReadyException
......@@ -238,7 +239,7 @@ class NEOCluster(object):
cleanup_on_delete=False, temp_dir=None, clear_databases=True,
adapter=os.getenv('NEO_TESTS_ADAPTER'),
address_type=ADDRESS_TYPE, bind_ip=None, logger=True,
):
importer=None):
if not adapter:
adapter = 'MySQL'
self.adapter = adapter
......@@ -263,6 +264,21 @@ class NEOCluster(object):
self.local_ip = local_ip = bind_ip or \
IP_VERSION_FORMAT_DICT[self.address_type]
self.setupDB(clear_databases)
if importer:
cfg = SafeConfigParser()
cfg.add_section("neo")
cfg.set("neo", "adapter", adapter)
cfg.set("neo", "database", self.db_template(*db_list))
for name, zodb in importer:
cfg.add_section(name)
for x in zodb.iteritems():
cfg.set(name, *x)
importer_conf = os.path.join(temp_dir, 'importer.cfg')
with open(importer_conf, 'w') as f:
cfg.write(f)
adapter = "Importer"
self.db_template = str
db_list = importer_conf,
self.process_dict = {}
self.temp_dir = temp_dir
self.port_allocator = PortAllocator()
......
......@@ -22,6 +22,7 @@ import socket
from struct import pack
from neo.neoctl.neoctl import NeoCTL
from neo.lib.util import makeChecksum, u64
from ZODB.FileStorage import FileStorage
from ZODB.POSException import ConflictError
from ZODB.tests.StorageTestBase import zodb_pickle
......@@ -168,9 +169,7 @@ class ClientTests(NEOFunctionalTest):
name = os.path.join(self.getTempDirectory(), 'data.fs')
if reset and os.path.exists(name):
os.remove(name)
storage = FileStorage(file_name=name)
db = ZODB.DB(storage=storage)
return (db, storage)
return FileStorage(file_name=name)
def __populate(self, db, tree_size=TREE_SIZE):
if isinstance(db.storage, FileStorage):
......@@ -204,7 +203,8 @@ class ClientTests(NEOFunctionalTest):
def testImport(self):
# source database
dfs_db, dfs_storage = self.__getDataFS()
dfs_storage = self.__getDataFS()
dfs_db = ZODB.DB(dfs_storage)
self.__populate(dfs_db)
# create a neo storage
......@@ -213,28 +213,48 @@ class ClientTests(NEOFunctionalTest):
# copy data fs to neo
neo_storage.copyTransactionsFrom(dfs_storage, verbose=0)
dfs_db.close()
# check neo content
(neo_db, neo_conn) = self.neo.getZODBConnection()
self.__checkTree(neo_conn.root()['trees'])
def __dump(self, storage):
return {u64(t.tid): [(u64(o.oid), o.data_txn and u64(o.data_txn),
None if o.data is None else makeChecksum(o.data))
for o in t]
for t in storage.iterator()}
def testExport(self):
# create a neo storage
self.neo.start()
(neo_db, neo_conn) = self.neo.getZODBConnection()
self.__populate(neo_db)
dump = self.__dump(neo_db.storage)
# copy neo to data fs
dfs_db, dfs_storage = self.__getDataFS(reset=True)
dfs_storage = self.__getDataFS(reset=True)
neo_storage = self.neo.getZODBStorage()
dfs_storage.copyTransactionsFrom(neo_storage)
# check data fs content
conn = dfs_db.open()
root = conn.root()
dfs_db = ZODB.DB(dfs_storage)
root = dfs_db.open().root()
self.__checkTree(root['trees'])
dfs_db.close()
self.neo.stop()
self.neo = NEOCluster(db_list=['test_neo1'], partitions=3,
importer=[("root", {
"storage": "<filestorage>\npath %s\n</filestorage>"
% dfs_storage.getName()})],
temp_dir=self.getTempDirectory())
self.neo.start()
neo_db, neo_conn = self.neo.getZODBConnection()
self.__checkTree(neo_conn.root()['trees'])
self.assertEqual(dump, self.__dump(neo_db.storage))
def testLockTimeout(self):
""" Hold a lock on an object to block a second transaction """
......
......@@ -19,6 +19,7 @@
import os, random, socket, sys, tempfile, threading, time, types, weakref
import traceback
from collections import deque
from ConfigParser import SafeConfigParser
from contextlib import contextmanager
from itertools import count
from functools import wraps
......@@ -545,7 +546,8 @@ class NEOCluster(object):
def __init__(self, master_count=1, partitions=1, replicas=0, upstream=None,
adapter=os.getenv('NEO_TESTS_ADAPTER', 'SQLite'),
storage_count=None, db_list=None, clear_databases=True,
db_user=DB_USER, db_password='', compress=True):
db_user=DB_USER, db_password='', compress=True,
importer=None):
self.name = 'neo_%s' % self._allocate('name',
lambda: random.randint(0, 100))
master_list = [MasterApplication.newAddress()
......@@ -573,6 +575,19 @@ class NEOCluster(object):
db = os.path.join(getTempDirectory(), '%s.sqlite')
else:
assert False, adapter
if importer:
cfg = SafeConfigParser()
cfg.add_section("neo")
cfg.set("neo", "adapter", adapter)
cfg.set("neo", "database", db % tuple(db_list))
for name, zodb in importer:
cfg.add_section(name)
for x in zodb.iteritems():
cfg.set(name, *x)
db = os.path.join(getTempDirectory(), '%s.conf')
with open(db % tuple(db_list), "w") as f:
cfg.write(f)
kw["getAdapter"] = "Importer"
self.storage_list = [StorageApplication(getDatabase=db % x, **kw)
for x in db_list]
self.admin_list = [AdminApplication(**kw)]
......
#
# Copyright (C) 2014 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from collections import deque
from itertools import islice, izip_longest
import os, time, unittest
import neo, transaction, ZODB
from neo.lib import logging
from neo.lib.util import u64
from ..fs2zodb import Inode
from .. import getTempDirectory
from . import NEOCluster, NEOThreadedTest
from ZODB.FileStorage import FileStorage
class ImporterTests(NEOThreadedTest):
def test(self):
importer = []
fs_dir = os.path.join(getTempDirectory(), self.id())
os.mkdir(fs_dir)
src_root, = neo.__path__
fs_list = "root", "client", "master", "tests"
def root_filter(name):
if not name.endswith(".pyc"):
i = name.find(os.sep)
return i < 0 or name[:i] not in fs_list
def sub_filter(name):
return lambda n: n[-4:] != '.pyc' and \
n.split(os.sep, 1)[0] in (name, "scripts")
conn_list = []
iter_list = []
# Setup several FileStorage databases.
for i, name in enumerate(fs_list):
fs_path = os.path.join(fs_dir, name + ".fs")
c = ZODB.DB(FileStorage(fs_path)).open()
r = c.root()["neo"] = Inode()
transaction.commit()
conn_list.append(c)
iter_list.append(r.treeFromFs(src_root, 10,
sub_filter(name) if i else root_filter))
importer.append((name, {
"storage": "<filestorage>\npath %s\n</filestorage>" % fs_path
}))
# Populate FileStorage databases.
for iter_list in izip_longest(*iter_list):
for i in iter_list:
if i:
transaction.commit()
del iter_list
# Get oids of mount points and close.
for (name, cfg), c in zip(importer, conn_list):
r = c.root()["neo"]
if name == "root":
for name in fs_list[1:]:
cfg[name] = str(u64(r[name]._p_oid))
else:
cfg["oid"] = str(u64(r[name]._p_oid))
c.db().close()
#del importer[0][1][importer.pop()[0]]
# Start NEO cluster with transparent import of a multi-base ZODB.
cluster = NEOCluster(compress=False, importer=importer)
try:
# Suspend import for a while, so that import
# is finished in the middle of the below 'for' loop.
# Use a slightly different main loop for storage so that it
# does not import data too fast and we test read/write access
# by the client during the import.
dm = cluster.storage.dm
def doOperation(app):
del dm.doOperation
try:
while True:
if app.task_queue:
app.task_queue[-1].next()
app._poll()
except StopIteration:
app.task_queue.pop()
dm.doOperation = doOperation
cluster.start()
t, c = cluster.getTransaction()
r = c.root()["neo"]
i = r.walk()
next(islice(i, 9, None))
dm.doOperation(cluster.storage) # resume
deque(i, maxlen=0)
last_import = None
for i, r in enumerate(r.treeFromFs(src_root, 10)):
t.commit()
if cluster.storage.dm._import:
last_import = i
self.assertTrue(last_import and not cluster.storage.dm._import)
i = len(src_root) + 1
self.assertEqual(sorted(r.walk()), sorted(
(x[i:] or '.', sorted(y), sorted(z))
for x, y, z in os.walk(src_root)))
t.commit()
finally:
cluster.stop()
if __name__ == "__main__":
unittest.main()
......@@ -23,13 +23,16 @@ if not os.path.exists('mock.py'):
raise EnvironmentError("MD5 checksum mismatch downloading 'mock.py'")
open('mock.py', 'w').write(mock_py)
zodb_require = ['ZODB3>=3.10', 'ZODB3<3.11dev']
extras_require = {
'admin': [],
'client': ['ZODB3>=3.10', 'ZODB3<3.11dev'],
'client': zodb_require,
'ctl': [],
'master': [],
'storage-sqlite': [],
'storage-mysqldb': ['MySQL-python'],
'storage-importer': zodb_require,
}
extras_require['tests'] = ['zope.testing', 'psutil>=2',
'neoppod[%s]' % ', '.join(extras_require)]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment