Commit 1316c225 authored by Julien Muchembled's avatar Julien Muchembled

Fix a few issues with ZODB5

In the Importer storage backend, the repickler code never really worked with
ZODB 5 (use of protocol > 1), and now the test does not pass anymore.

The other issues caused by ZODB commit 12ee41c47310156027a674932df34b60de86ba36
are fixed:

  TypeError: list indices must be integers, not binary

  ValueError: unsupported pickle protocol: 3

Although not necessary as long as we don't support Python 3,
this commit also replaces `str` by `bytes` in a few places.
parent b6989a0e
......@@ -14,6 +14,8 @@
# directly to a NEO cluster with replicas or several storage nodes.
# Importer backend can only be used with a single storage node.
# WARNING: Merging several DB only works if they were only used with ZODB < 5.
# Here is how to proceed once this file is ready:
# 1. Restart ZODB clients to connect to new NEO cluster (not started yet).
# 2. Start NEO cluster (use 'neoctl -a <admin> start' command if necessary).
......@@ -44,7 +44,7 @@ def patch():
# <patch>
serial = self._storage.tpc_finish(transaction, callback)
if serial is not None:
assert isinstance(serial, str), repr(serial)
assert isinstance(serial, bytes), repr(serial)
for oid_iterator in (self._modified, self._creating):
for oid in oid_iterator:
obj = self._cache.get(oid, None)
......@@ -272,7 +272,8 @@ class Application(ThreadedApplication):
def _askStorageForRead(self, object_id, packet, askStorage=None):
cp = self.cp
pt =
if type(object_id) is str:
# BBB: On Py2, it can be a subclass of bytes (binary from zodbpickle).
if isinstance(object_id, bytes):
object_id = pt.getPartition(object_id)
if askStorage is None:
askStorage = self._askStorage
......@@ -117,7 +117,7 @@ class Transaction(object):
if uuid_list:
del self.data_dict[oid]
if type(data) is str:
if type(data) is bytes:
size = len(data)
self.data_size -= size
size += self.cache_size
......@@ -115,7 +115,7 @@ def add64(packed, offset):
def dump(s):
"""Dump a binary string in hex."""
if s is not None:
if isinstance(s, str):
if isinstance(s, bytes):
return b2a_hex(s)
return repr(s)
......@@ -146,7 +146,7 @@ class Repickler(pickle.Unpickler):
args = self.stack[k+1:]
self.stack[k:] = self._obj(klass, *args),
del dispatch[pickle.NEWOBJ] # ZODB has never used protocol 2
del dispatch[pickle.NEWOBJ] # ZODB < 5 has never used protocol 2
def find_class(self, args):
......@@ -221,7 +221,7 @@ class ZODB(object):
oid = u64(obj[0])
# If this oid pointed to a mount point, drop 2nd item because
# it's probably different than the real class of the new oid.
elif isinstance(obj, str):
elif isinstance(obj, bytes):
oid = u64(obj)
raise NotImplementedError(
......@@ -232,7 +232,7 @@ class ZODB(object):
if not self.shift_oid:
return obj # common case for root db
oid = p64(oid + self.shift_oid)
return oid if isinstance(obj, str) else (oid, obj[1])
return oid if isinstance(obj, bytes) else (oid, obj[1])
self.repickle = Repickler(map_oid)
return self.repickle(data)
......@@ -29,7 +29,10 @@ import MySQLdb
import transaction
from cStringIO import StringIO
from cPickle import Unpickler
from ZODB._compat import Unpickler
except ImportError:
from cPickle import Unpickler
from functools import wraps
from inspect import isclass
from .mock import Mock
......@@ -19,12 +19,14 @@ from cStringIO import StringIO
from itertools import islice, izip_longest
import os, shutil, unittest
import neo, transaction, ZODB
from neo.client.exception import NEOPrimaryMasterLost
from neo.lib import logging
from neo.lib.util import u64
from import Repickler
from ..fs2zodb import Inode
from .. import getTempDirectory
from .. import expectedFailure, getTempDirectory
from . import NEOCluster, NEOThreadedTest
from ZODB import serialize
from ZODB.FileStorage import FileStorage
......@@ -232,6 +234,10 @@ class ImporterTests(NEOThreadedTest):
for x, y, z in os.walk(src_root)))
if getattr(serialize, '_protocol', 1) > 1:
# XXX: With ZODB5, we should at least keep a working test that does not
# merge several DB.
test = expectedFailure(NEOPrimaryMasterLost)(test)
if __name__ == "__main__":
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment