Added support for wrapper storages that transform pickle data. (I

thought this support already existed, but found out I was wrong when I
tried to make it work. :/) Applications for this include compression
and encryption.  An example wrapper storage implementation,
ZODB.tests.hexstorage, was included for testing.

It is important that storage implementations not assume that
storages contain pickles.  Renamed IStorageDB to IStorageWrapper and
expanded it to provide methods for transforming and untransforming
data records.  Storages implementations should use these methods to
get pickle data from stored records.

src/ZEO/StorageServer.py

@@ -778,6 +778,7 @@ class StorageServerDB:
     def invalidateCache(self):
         self.server._invalidateCache(self.storage_id)
 
+    transform_record_data = untransform_record_data = lambda self, data: data
 
 class StorageServer:
 

src/ZEO/tests/IterationTests.py

@@ -35,7 +35,12 @@ class IterationTests:
 
     def checkIteratorExhaustionStorage(self):
         # Test the storage's garbage collection mechanism.
+        self._dostore()
         iterator = self._storage.iterator()
+
+        # At this point, a wrapping iterator might not have called the CS
+        # iterator yet. We'll consume one item to make sure this happens.
+        iterator.next()
         self.assertEquals(1, len(self._storage._iterator_ids))
         iid = list(self._storage._iterator_ids)[0]
         self.assertEquals([], list(iterator))
@@ -47,18 +52,25 @@ class IterationTests:
     def checkIteratorGCSpanTransactions(self):
         # Keep a hard reference to the iterator so it won't be automatically
         # garbage collected at the transaction boundary.
+        self._dostore()
         iterator = self._storage.iterator()
         self._dostore()
         # As the iterator was not garbage collected, we can still use it. (We
         # don't see the transaction we just wrote being picked up, because
         # iterators only see the state from the point in time when they were
         # created.)
-        self.assertEquals([], list(iterator))
+        self.assert_(list(iterator))
 
     def checkIteratorGCStorageCommitting(self):
         # We want the iterator to be garbage-collected, so we don't keep any
         # hard references to it. The storage tracks its ID, though.
-        self._storage.iterator()
+
+        # The odd little jig we do below arises from the fact that the
+        # CS iterator may not be constructed right away if the CS is wrapped.
+        # We need to actually do some iteration to get the iterator created.
+        # We do a store to make sure the iterator isn't exhausted right away.
+        self._dostore()
+        self._storage.iterator().next()
 
         self.assertEquals(1, len(self._storage._iterator_ids))
         iid = list(self._storage._iterator_ids)[0]
@@ -70,8 +82,15 @@ class IterationTests:
         self.assertRaises(KeyError, self._storage._server.iterator_next, iid)
 
     def checkIteratorGCStorageTPCAborting(self):
-        self._storage.iterator()
+        # The odd little jig we do below arises from the fact that the
+        # CS iterator may not be constructed right away if the CS is wrapped.
+        # We need to actually do some iteration to get the iterator created.
+        # We do a store to make sure the iterator isn't exhausted right away.
+        self._dostore()
+        self._storage.iterator().next()
+
         iid = list(self._storage._iterator_ids)[0]
+
         t = transaction.Transaction()
         self._storage.tpc_begin(t)
         self._storage.tpc_abort(t)
@@ -79,7 +98,14 @@ class IterationTests:
         self.assertRaises(KeyError, self._storage._server.iterator_next, iid)
 
     def checkIteratorGCStorageDisconnect(self):
-        self._storage.iterator()
+
+        # The odd little jig we do below arises from the fact that the
+        # CS iterator may not be constructed right away if the CS is wrapped.
+        # We need to actually do some iteration to get the iterator created.
+        # We do a store to make sure the iterator isn't exhausted right away.
+        self._dostore()
+        self._storage.iterator().next()
+
         iid = list(self._storage._iterator_ids)[0]
         t = transaction.Transaction()
         self._storage.tpc_begin(t)

src/ZEO/tests/testZEO.py

@@ -47,6 +47,7 @@ import ZEO.tests.ConnectionTests
 import ZEO.zrpc.connection
 import ZODB
 import ZODB.blob
+import ZODB.tests.hexstorage
 import ZODB.tests.testblob
 import ZODB.tests.util
 import ZODB.utils
@@ -59,6 +60,7 @@ class DummyDB:
         pass
     def invalidateCache(*unused):
         pass
+    transform_record_data = untransform_record_data = lambda self, v: v
 
 
 class CreativeGetState(persistent.Persistent):
@@ -94,7 +96,8 @@ class MiscZEOTests:
 
     def checkZEOInvalidation(self):
         addr = self._storage._addr
-        storage2 = ClientStorage(addr, wait=1, min_disconnect_poll=0.1)
+        storage2 = self._wrap_client(
+            ClientStorage(addr, wait=1, min_disconnect_poll=0.1))
         try:
             oid = self._storage.new_oid()
             ob = MinPO('first')
@@ -196,13 +199,16 @@ class GenericTests(
             self.blob_cache_dir = tempfile.mkdtemp(
                 'blob_cache',
                 dir=os.path.abspath(os.getcwd()))
-        self._storage = ClientStorage(
+        self._storage = self._wrap_client(ClientStorage(
             zport, '1', cache_size=20000000,
             min_disconnect_poll=0.5, wait=1,
             wait_timeout=60, blob_dir=self.blob_cache_dir,
-            shared_blob_dir=self.shared_blob_dir)
+            shared_blob_dir=self.shared_blob_dir))
         self._storage.registerDB(DummyDB())
 
+    def _wrap_client(self, client):
+        return client
+
     def tearDown(self):
         self._storage.close()
         for server in self._servers:
@@ -317,6 +323,16 @@ class FileStorageTests(FullGenericTests):
         </filestorage>
         """
 
+    _expected_interfaces = (
+        ('ZODB.interfaces', 'IStorageRestoreable'),
+        ('ZODB.interfaces', 'IStorageIteration'),
+        ('ZODB.interfaces', 'IStorageUndoable'),
+        ('ZODB.interfaces', 'IStorageCurrentRecordIteration'),
+        ('ZODB.interfaces', 'IExternalGC'),
+        ('ZODB.interfaces', 'IStorage'),
+        ('zope.interface', 'Interface'),
+        )
+
     def checkInterfaceFromRemoteStorage(self):
         # ClientStorage itself doesn't implement IStorageIteration, but the
         # FileStorage on the other end does, and thus the ClientStorage
@@ -326,18 +342,48 @@ class FileStorageTests(FullGenericTests):
         self.failUnless(ZODB.interfaces.IStorageIteration.providedBy(
             self._storage))
         # This is communicated using ClientStorage's _info object:
-        self.assertEquals(
-            (('ZODB.interfaces', 'IStorageRestoreable'),
-             ('ZODB.interfaces', 'IStorageIteration'),
-             ('ZODB.interfaces', 'IStorageUndoable'),
-             ('ZODB.interfaces', 'IStorageCurrentRecordIteration'),
-             ('ZODB.interfaces', 'IExternalGC'),
-             ('ZODB.interfaces', 'IStorage'),
-             ('zope.interface', 'Interface'),
-             ),
+        self.assertEquals(self._expected_interfaces,
             self._storage._info['interfaces']
             )
 
+class FileStorageHexTests(FileStorageTests):
+    _expected_interfaces = (
+        ('ZODB.interfaces', 'IStorageRestoreable'),
+        ('ZODB.interfaces', 'IStorageIteration'),
+        ('ZODB.interfaces', 'IStorageUndoable'),
+        ('ZODB.interfaces', 'IStorageCurrentRecordIteration'),
+        ('ZODB.interfaces', 'IExternalGC'),
+        ('ZODB.interfaces', 'IStorage'),
+        ('ZODB.interfaces', 'IStorageWrapper'),
+        ('zope.interface', 'Interface'),
+        )
+
+    def getConfig(self):
+        return """\
+        %import ZODB.tests
+        <hexstorage>
+        <filestorage 1>
+        path Data.fs
+        </filestorage>
+        </hexstorage>
+        """
+
+class FileStorageClientHexTests(FileStorageHexTests):
+
+    def getConfig(self):
+        return """\
+        %import ZODB.tests
+        <serverhexstorage>
+        <filestorage 1>
+        path Data.fs
+        </filestorage>
+        </serverhexstorage>
+        """
+
+    def _wrap_client(self, client):
+        return ZODB.tests.hexstorage.HexStorage(client)
+
+
 
 class MappingStorageTests(GenericTests):
     """ZEO backed by a Mapping storage."""
@@ -1405,7 +1451,8 @@ def quick_close_doesnt_kill_server():
 
 slow_test_classes = [
     BlobAdaptedFileStorageTests, BlobWritableCacheTests,
-    DemoStorageTests, FileStorageTests, MappingStorageTests,
+    MappingStorageTests, DemoStorageTests,
+    FileStorageTests, FileStorageHexTests, FileStorageClientHexTests,
     ]
 
 quick_test_classes = [

src/ZODB/ConflictResolution.py

@@ -53,6 +53,7 @@ def find_global(*args):
 
 def state(self, oid, serial, prfactory, p=''):
     p = p or self.loadSerial(oid, serial)
+    p = self._crs_untransform_record_data(p)
     file = StringIO(p)
     unpickler = Unpickler(file)
     unpickler.find_global = find_global
@@ -80,13 +81,13 @@ class IPersistentReference(zope.interface.Interface):
 
     def __cmp__(other):
         '''if other is equivalent reference, return 0; else raise ValueError.
-        
+
         Equivalent in this case means that oid and database_name are the same.
 
         If either is a weak reference, we only support `is` equivalence, and
         otherwise raise a ValueError even if the datbase_names and oids are
         the same, rather than guess at the correct semantics.
-        
+
         It is impossible to sort reliably, since the actual persistent
         class may have its own comparison, and we have no idea what it is.
         We assert that it is reasonably safe to assume that an object is
@@ -135,7 +136,7 @@ class PersistentReference(object):
 
     def __cmp__(self, other):
         if self is other or (
-            isinstance(other, PersistentReference) and 
+            isinstance(other, PersistentReference) and
             self.oid == other.oid and
             self.database_name == other.database_name and
             not self.weak and
@@ -179,6 +180,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
     # class_tuple, old, committed, newstate = ('',''), 0, 0, 0
     try:
         prfactory = PersistentReferenceFactory()
+        newpickle = self._crs_untransform_record_data(newpickle)
         file = StringIO(newpickle)
         unpickler = Unpickler(file)
         unpickler.find_global = find_global
@@ -194,7 +196,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
             newargs = ()
 
         if klass in _unresolvable:
-            return None
+            raise ConflictError
 
         newstate = unpickler.load()
         inst = klass.__new__(klass, *newargs)
@@ -203,7 +205,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
             resolve = inst._p_resolveConflict
         except AttributeError:
             _unresolvable[klass] = 1
-            return None
+            raise ConflictError
 
         old = state(self, oid, oldSerial, prfactory)
         committed = state(self, oid, committedSerial, prfactory, committedData)
@@ -215,9 +217,9 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
         pickler.inst_persistent_id = persistent_id
         pickler.dump(meta)
         pickler.dump(resolved)
-        return file.getvalue(1)
+        return self._crs_transform_record_data(file.getvalue(1))
     except (ConflictError, BadClassName):
-        return None
+        pass
     except:
         # If anything else went wrong, catch it here and avoid passing an
         # arbitrary exception back to the client.  The error here will mask
@@ -225,9 +227,19 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
         # ConflictError, but not necessarily from other errors.  But log
         # the error so that any problems can be fixed.
         logger.error("Unexpected error", exc_info=True)
-        return None
 
-class ConflictResolvingStorage:
+    raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
+                        data=newpickle)
+
+class ConflictResolvingStorage(object):
     "Mix-in class that provides conflict resolution handling for storages"
 
     tryToResolveConflict = tryToResolveConflict
+
+    _crs_transform_record_data = _crs_untransform_record_data = (
+        lambda self, o: o)
+
+    def registerDB(self, wrapper):
+        self._crs_untransform_record_data = wrapper.untransform_record_data
+        self._crs_transform_record_data = wrapper.transform_record_data
+        super(ConflictResolvingStorage, self).registerDB(wrapper)

src/ZODB/DB.py

@@ -707,6 +707,8 @@ class DB(object):
         """
         self._connectionMap(lambda c: c.invalidateCache())
 
+    transform_record_data = untransform_record_data = lambda self, data: data
+
     def objectCount(self):
         return len(self.storage)
 

src/ZODB/FileStorage/FileStorage.py

@@ -476,14 +476,8 @@ class FileStorage(
                 committed_tid = h.tid
 
                 if oldserial != committed_tid:
-                    rdata = self.tryToResolveConflict(oid, committed_tid,
+                    data = self.tryToResolveConflict(oid, committed_tid,
                                                      oldserial, data)
-                    if rdata is None:
-                        raise POSException.ConflictError(
-                            oid=oid, serials=(committed_tid, oldserial),
-                            data=data)
-                    else:
-                        data = rdata
 
             pos = self._pos
             here = pos + self._tfile.tell() + self._thl
@@ -843,10 +837,11 @@ class FileStorage(
         except KeyError:
             # couldn't find oid; what's the real explanation for this?
             raise UndoError("_loadBack() failed for %s", oid)
-        data = self.tryToResolveConflict(oid, ctid, tid, bdata, cdata)
-
-        if data:
+        try:
+            data = self.tryToResolveConflict(oid, ctid, tid, bdata, cdata)
             return data, 0, ipos
+        except POSException.ConflictError:
+            pass
 
         raise UndoError("Some data were modified by a later transaction", oid)
 
@@ -961,7 +956,7 @@ class FileStorage(
                     except ZODB.POSException.POSKeyError:
                         pass # It was removed, so no need to copy data
                     else:
-                        if ZODB.blob.is_blob_record(up):
+                        if self.is_blob_record(up):
                             # We're undoing a blob modification operation.
                             # We have to copy the blob data
                             tmp = ZODB.utils.mktemp(dir=self.fshelper.temp_dir)
@@ -1768,7 +1763,7 @@ class FileIterator(FileStorageFormatter):
 
     def next(self):
         if self._file is None:
-            raise ZODB.interfaces.StorageStopIteration()
+            raise StopIteration()
 
         pos = self._pos
         while True:
@@ -1857,7 +1852,7 @@ class FileIterator(FileStorageFormatter):
             return result
 
         self.close()
-        raise ZODB.interfaces.StorageStopIteration()
+        raise StopIteration()
 
 
 class TransactionRecord(BaseStorage.TransactionRecord):
@@ -1916,7 +1911,7 @@ class TransactionRecordIterator(FileStorageFormatter):
 
             return Record(h.oid, h.tid, data, prev_txn, pos)
 
-        raise ZODB.interfaces.StorageStopIteration()
+        raise StopIteration()
 
 
 class Record(BaseStorage.DataRecord):

src/ZODB/FileStorage/fspack.py

@@ -30,7 +30,6 @@ from ZODB.utils import p64, u64, z64
 
 import logging
 import os
-import ZODB.blob
 import ZODB.fsIndex
 import ZODB.POSException
 
@@ -502,7 +501,7 @@ class FileStoragePacker(FileStorageFormatter):
                         data = self._file.read(h.plen)
                     else:
                         data = self.fetchDataViaBackpointer(h.oid, h.back)
-                    if data and ZODB.blob.is_blob_record(data):
+                    if data and self._storage.is_blob_record(data):
                         # We need to remove the blob record. Maybe we
                         # need to remove oid:
 

src/ZODB/blob.py

@@ -617,29 +617,24 @@ class BlobStorageMixin(object):
         """
         self.dirty_oids = []
 
-    def copyTransactionsFrom(self, other):
-        for trans in other.iterator():
-            self.tpc_begin(trans, trans.tid, trans.status)
-            for record in trans:
-                blobfilename = None
-                if is_blob_record(record.data):
-                    try:
-                        blobfilename = other.loadBlob(record.oid, record.tid)
-                    except POSKeyError:
-                        pass
-                if blobfilename is not None:
-                    fd, name = tempfile.mkstemp(
-                        suffix='.tmp', dir=self.fshelper.temp_dir)
-                    os.close(fd)
-                    utils.cp(open(blobfilename, 'rb'), open(name, 'wb'))
-                    self.restoreBlob(record.oid, record.tid, record.data,
-                                     name, record.data_txn, trans)
-                else:
-                    self.restore(record.oid, record.tid, record.data,
-                                 '', record.data_txn, trans)
+    def registerDB(self, db):
+        self.__untransform_record_data = db.untransform_record_data
+        try:
+            m = super(BlobStorageMixin, self).registerDB
+        except AttributeError:
+            pass
+        else:
+            m(db)
+
+    def __untransform_record_data(self, record):
+        return record
 
-            self.tpc_vote(trans)
-            self.tpc_finish(trans)
+    def is_blob_record(self, record):
+        if record:
+            return is_blob_record(self.__untransform_record_data(record))
+
+    def copyTransactionsFrom(self, other):
+        copyTransactionsFromTo(other, self)
 
     def loadBlob(self, oid, serial):
         """Return the filename where the blob file can be found.
@@ -943,3 +938,27 @@ def is_blob_record(record):
             pass
 
     return False
+
+def copyTransactionsFromTo(source, destination):
+    for trans in source.iterator():
+        destination.tpc_begin(trans, trans.tid, trans.status)
+        for record in trans:
+            blobfilename = None
+            if is_blob_record(record.data):
+                try:
+                    blobfilename = source.loadBlob(record.oid, record.tid)
+                except POSKeyError:
+                    pass
+            if blobfilename is not None:
+                fd, name = tempfile.mkstemp(
+                    suffix='.tmp', dir=destination.fshelper.temp_dir)
+                os.close(fd)
+                utils.cp(open(blobfilename, 'rb'), open(name, 'wb'))
+                destination.restoreBlob(record.oid, record.tid, record.data,
+                                 name, record.data_txn, trans)
+            else:
+                destination.restore(record.oid, record.tid, record.data,
+                             '', record.data_txn, trans)
+
+        destination.tpc_vote(trans)
+        destination.tpc_finish(trans)

src/ZODB/interfaces.py

@@ -287,23 +287,32 @@ class IConnection(Interface):
         """
 
 
-class IStorageDB(Interface):
-    """Database interface exposed to storages
+class IStorageWrapper(Interface):
+    """Storage wrapper interface
 
-    This interface provides 2 facilities:
+    This interface provides 3 facilities:
 
     - Out-of-band invalidation support
 
-      A storage can notify it's database of object invalidations that
+      A storage can notify it's wrapper of object invalidations that
       don't occur due to direct operations on the storage.  Currently
       this is only used by ZEO client storages to pass invalidation
       messages sent from a server.
 
-    - Record-reference extraction.
+    - Record-reference extraction
 
       The references method can be used to extract referenced object
       IDs from a database record.  This can be used by storages to
-      provide more advanced garbage collection.
+      provide more advanced garbage collection.  A wrapper storage
+      that transforms data will provide a references method that
+      untransforms data passed to it and then pass the data to the
+      layer above it.
+
+    - Record transformation
+
+      A storage wrapper may transform data, for example for
+      compression or encryption.  Methods are provided to transform or
+      untransform data.
 
     This interface may be implemented by storage adapters or other
     intermediaries.  For example, a storage adapter that provides
@@ -337,6 +346,16 @@ class IStorageDB(Interface):
         be created and returned.
         """
 
+    def transform_record_data(data):
+        """Return transformed data
+        """
+
+    def untransform_record_data(data):
+        """Return untransformed data
+        """
+
+IStorageDB = IStorageWrapper # for backward compatibility
+
 
 class IDatabase(IStorageDB):
     """ZODB DB.
@@ -595,12 +614,18 @@ class IStorage(Interface):
         revisions.
         """
 
-    def registerDB(db):
-        """Register an IStorageDB.
+    def registerDB(wrapper):
+        """Register a storage wrapper IStorageWrapper.
+
+        The passed object is a wrapper object that provides an upcall
+        interface to support composition.
 
         Note that, for historical reasons, an implementation may
         require a second argument, however, if required, the None will
         be passed as the second argument.
+
+        Also, for historical reasons, this is called registerDB rather
+        than register_wrapper.
         """
 
     def sortKey():

src/ZODB/tests/BasicStorage.py

@@ -49,7 +49,7 @@ class BasicStorage:
         self.assertRaises(
             POSException.StorageTransactionError,
             self._storage.store,
-            ZERO, 1, 2, '', transaction.Transaction())
+            ZERO, 1, '2', '', transaction.Transaction())
         self._storage.tpc_abort(t)
 
     def checkSerialIsNoneForInitialRevision(self):

src/ZODB/tests/component.xml

+<component>
+  <sectiontype
+     name="hexstorage"
+     datatype="ZODB.tests.hexstorage.ZConfigHex"
+     implements="ZODB.storage"
+     >
+    <section type="ZODB.storage" name="*" attribute="base" required="yes" />
+  </sectiontype>
+  <sectiontype
+     name="serverhexstorage"
+     datatype="ZODB.tests.hexstorage.ZConfigServerHex"
+     implements="ZODB.storage"
+     >
+    <section type="ZODB.storage" name="*" attribute="base" required="yes" />
+  </sectiontype>
+</component>

src/ZODB/tests/hexstorage.py

+##############################################################################
+#
+# Copyright (c) 2010 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import ZODB.blob
+import ZODB.interfaces
+import zope.interface
+
+class HexStorage(object):
+
+    zope.interface.implements(ZODB.interfaces.IStorageWrapper)
+
+    copied_methods = (
+            'close', 'getName', 'getSize', 'history', 'isReadOnly',
+            'lastTransaction', 'new_oid', 'sortKey',
+            'tpc_abort', 'tpc_begin', 'tpc_finish', 'tpc_vote',
+            'loadBlob', 'openCommittedBlobFile', 'temporaryDirectory',
+            'supportsUndo', 'undo', 'undoLog', 'undoInfo',
+            )
+
+    def __init__(self, base):
+        self.base = base
+        base.registerDB(self)
+
+        for name in self.copied_methods:
+            v = getattr(base, name, None)
+            if v is not None:
+                setattr(self, name, v)
+
+        zope.interface.directlyProvides(self, zope.interface.providedBy(base))
+
+    def __getattr__(self, name):
+        return getattr(self.base, name)
+
+    def __len__(self):
+        return len(self.base)
+
+    def load(self, oid, version=''):
+        data, serial = self.base.load(oid, version)
+        return data[2:].decode('hex'), serial
+
+    def loadBefore(self, oid, tid):
+        r = self.base.loadBefore(oid, tid)
+        if r is not None:
+            data, serial, after = r
+            return data[2:].decode('hex'), serial, after
+        else:
+            return r
+
+    def loadSerial(self, oid, serial):
+        return self.base.loadSerial(oid, serial)[2:].decode('hex')
+
+    def pack(self, pack_time, referencesf, gc=True):
+        def refs(p, oids=None):
+            return referencesf(p[2:].decode('hex'), oids)
+        return self.base.pack(pack_time, refs, gc)
+
+    def registerDB(self, db):
+        self.db = db
+        self._db_transform = db.transform_record_data
+        self._db_untransform = db.untransform_record_data
+
+    _db_transform = _db_untransform = lambda self, data: data
+
+    def store(self, oid, serial, data, version, transaction):
+        return self.base.store(
+            oid, serial, '.h'+data.encode('hex'), version, transaction)
+
+    def restore(self, oid, serial, data, version, prev_txn, transaction):
+        return self.base.restore(
+            oid, serial, data and ('.h'+data.encode('hex')), version, prev_txn,
+            transaction)
+
+    def iterator(self, start=None, stop=None):
+        for t in self.base.iterator(start, stop):
+            yield Transaction(self, t)
+
+    def storeBlob(self, oid, oldserial, data, blobfilename, version,
+                  transaction):
+        return self.base.storeBlob(oid, oldserial, '.h'+data.encode('hex'),
+                                   blobfilename, version, transaction)
+
+    def restoreBlob(self, oid, serial, data, blobfilename, prev_txn,
+                    transaction):
+        return self.base.restoreBlob(oid, serial,
+                                     data and ('.h'+data.encode('hex')),
+                                     blobfilename, prev_txn, transaction)
+
+    def invalidateCache(self):
+        return self.db.invalidateCache()
+
+    def invalidate(self, transaction_id, oids, version=''):
+        return self.db.invalidate(transaction_id, oids, version)
+
+    def references(self, record, oids=None):
+        return self.db.references(record[2:].decode('hex'), oids)
+
+    def transform_record_data(self, data):
+        return '.h'+self._db_transform(data).encode('hex')
+
+    def untransform_record_data(self, data):
+        return self._db_untransform(data[2:].decode('hex'))
+
+    def record_iternext(self, next=None):
+        oid, tid, data, next = self.base.record_iternext(next)
+        return oid, tid, data[2:].decode('hex'), next
+
+    def copyTransactionsFrom(self, other):
+        ZODB.blob.copyTransactionsFromTo(other, self)
+
+class ServerHexStorage(HexStorage):
+    """Use on ZEO storage server when Hex is used on client
+
+    Don't do conversion as part of load/store, but provide
+    pickle decoding.
+    """
+
+    copied_methods = HexStorage.copied_methods + (
+        'load', 'loadBefore', 'loadSerial', 'store', 'restore',
+        'iterator', 'storeBlob', 'restoreBlob', 'record_iternext',
+        )
+
+class Transaction(object):
+
+    def __init__(self, store, trans):
+        self.__store = store
+        self.__trans = trans
+
+    def __iter__(self):
+        for r in self.__trans:
+            if r.data:
+                r.data = self.__store.untransform_record_data(r.data)
+            yield r
+
+    def __getattr__(self, name):
+        return getattr(self.__trans, name)
+
+class ZConfigHex:
+
+    _factory = HexStorage
+
+    def __init__(self, config):
+        self.config = config
+        self.name = config.getSectionName()
+
+    def open(self):
+        base = self.config.base.open()
+        return self._factory(base)
+
+class ZConfigServerHex(ZConfigHex):
+
+    _factory = ServerHexStorage

src/ZODB/tests/testDemoStorage.py

@@ -16,6 +16,7 @@ import random
 import transaction
 from ZODB.DB import DB
 from zope.testing import doctest
+import ZODB.tests.hexstorage
 import ZODB.tests.util
 import ZODB.utils
 import ZODB.DemoStorage
@@ -78,6 +79,13 @@ class DemoStorageTests(
     checkUndoZombie = checkLoadBeforeUndo
 
 
+class DemoStorageHexTests(DemoStorageTests):
+
+    def setUp(self):
+        StorageTestBase.StorageTestBase.setUp(self)
+        self._storage = ZODB.tests.hexstorage.HexStorage(
+            ZODB.DemoStorage.DemoStorage())
+
 class DemoStorageWrappedBase(DemoStorageTests):
 
     def setUp(self):
@@ -111,6 +119,11 @@ class DemoStorageWrappedAroundFileStorage(DemoStorageWrappedBase):
         from ZODB.FileStorage import FileStorage
         return FileStorage('FileStorageTests.fs')
 
+class DemoStorageWrappedAroundHexMappingStorage(DemoStorageWrappedBase):
+
+    def _makeBaseStorage(self):
+        from ZODB.MappingStorage import MappingStorage
+        return ZODB.tests.hexstorage.HexStorage(MappingStorage())
 
 
 def setUp(test):
@@ -232,8 +245,11 @@ def test_suite():
             ),
         ))
     suite.addTest(unittest.makeSuite(DemoStorageTests, 'check'))
+    suite.addTest(unittest.makeSuite(DemoStorageHexTests, 'check'))
     suite.addTest(unittest.makeSuite(DemoStorageWrappedAroundFileStorage,
                                      'check'))
     suite.addTest(unittest.makeSuite(DemoStorageWrappedAroundMappingStorage,
                                      'check'))
+    suite.addTest(unittest.makeSuite(DemoStorageWrappedAroundHexMappingStorage,
+                                     'check'))
     return suite

src/ZODB/tests/testFileStorage.py

@@ -15,6 +15,7 @@ import cPickle
 import os, unittest
 import transaction
 import ZODB.FileStorage
+import ZODB.tests.hexstorage
 import ZODB.tests.testblob
 import ZODB.tests.util
 import zope.testing.setupstack
@@ -279,13 +280,29 @@ class FileStorageTests(
             else:
                 self.assertNotEqual(next_oid, None)
 
+class FileStorageHexTests(FileStorageTests):
+
+    def open(self, **kwargs):
+        self._storage = ZODB.tests.hexstorage.HexStorage(
+            ZODB.FileStorage.FileStorage('FileStorageTests.fs',**kwargs))
+
+
 class FileStorageTestsWithBlobsEnabled(FileStorageTests):
 
     def open(self, **kwargs):
         if 'blob_dir' not in kwargs:
             kwargs = kwargs.copy()
             kwargs['blob_dir'] = 'blobs'
-        return FileStorageTests.open(self, **kwargs)
+        FileStorageTests.open(self, **kwargs)
+
+class FileStorageHexTestsWithBlobsEnabled(FileStorageTests):
+
+    def open(self, **kwargs):
+        if 'blob_dir' not in kwargs:
+            kwargs = kwargs.copy()
+            kwargs['blob_dir'] = 'blobs'
+        FileStorageTests.open(self, **kwargs)
+        self._storage = ZODB.tests.hexstorage.HexStorage(self._storage)
 
 class FileStorageRecoveryTest(
     StorageTestBase.StorageTestBase,
@@ -304,6 +321,15 @@ class FileStorageRecoveryTest(
     def new_dest(self):
         return ZODB.FileStorage.FileStorage('Dest.fs')
 
+class FileStorageHexRecoveryTest(FileStorageRecoveryTest):
+
+    def setUp(self):
+        StorageTestBase.StorageTestBase.setUp(self)
+        self._storage = ZODB.tests.hexstorage.HexStorage(
+            ZODB.FileStorage.FileStorage("Source.fs", create=True))
+        self._dst = ZODB.tests.hexstorage.HexStorage(
+            ZODB.FileStorage.FileStorage("Dest.fs", create=True))
+
 
 class FileStorageNoRestore(ZODB.FileStorage.FileStorage):
 
@@ -634,10 +660,14 @@ def test_suite():
     from zope.testing import doctest
 
     suite = unittest.TestSuite()
-    for klass in [FileStorageTests, Corruption.FileStorageCorruptTests,
-                  FileStorageRecoveryTest, FileStorageNoRestoreRecoveryTest,
-                  FileStorageTestsWithBlobsEnabled, AnalyzeDotPyTest,
-                  ]:
+    for klass in [
+        FileStorageTests, FileStorageHexTests,
+        Corruption.FileStorageCorruptTests,
+        FileStorageRecoveryTest, FileStorageHexRecoveryTest,
+        FileStorageNoRestoreRecoveryTest,
+        FileStorageTestsWithBlobsEnabled, FileStorageHexTestsWithBlobsEnabled,
+        AnalyzeDotPyTest,
+        ]:
         suite.addTest(unittest.makeSuite(klass, "check"))
     suite.addTest(doctest.DocTestSuite(
         setUp=zope.testing.setupstack.setUpDirectory,
@@ -649,9 +679,18 @@ def test_suite():
         test_blob_storage_recovery=True,
         test_packing=True,
         ))
+    suite.addTest(ZODB.tests.testblob.storage_reusable_suite(
+        'BlobFileHexStorage',
+        lambda name, blob_dir:
+        ZODB.tests.hexstorage.HexStorage(
+            ZODB.FileStorage.FileStorage('%s.fs' % name, blob_dir=blob_dir)),
+        test_blob_storage_recovery=True,
+        test_packing=True,
+        ))
     suite.addTest(PackableStorage.IExternalGC_suite(
         lambda : ZODB.FileStorage.FileStorage(
             'data.fs', blob_dir='blobs', pack_gc=False)))
+    suite.layer = ZODB.tests.util.MininalTestLayer('testFileStorage')
     return suite
 
 if __name__=='__main__':