Collapsed the Blob package into a single module (and a single test

module).

src/ZEO/ClientStorage.py

@@ -40,7 +40,7 @@ import ZODB.lock_file
 from ZODB import POSException
 from ZODB import utils
 from ZODB.loglevels import BLATHER
-from ZODB.Blobs.interfaces import IBlobStorage
+from ZODB.interfaces import IBlobStorage
 from persistent.TimeStamp import TimeStamp
 
 logger = logging.getLogger('ZEO.ClientStorage')
@@ -324,8 +324,8 @@ class ClientStorage(object):
         if blob_dir is not None:
             # Avoid doing this import unless we need it, as it
             # currently requires pywin32 on Windows.
-            import ZODB.Blobs.Blob 
-            self.fshelper = ZODB.Blobs.Blob.FilesystemHelper(blob_dir)
+            import ZODB.blob 
+            self.fshelper = ZODB.blob.FilesystemHelper(blob_dir)
             self.fshelper.create()
             self.fshelper.checkSecure()
         else:

src/ZEO/tests/testZEO.py

@@ -460,8 +460,7 @@ class CommonBlobTests:
 
     def checkStoreBlob(self):
         from ZODB.utils import oid_repr, tid_repr
-        from ZODB.Blobs.Blob import Blob
-        from ZODB.Blobs.BlobStorage import BLOB_SUFFIX
+        from ZODB.blob import Blob, BLOB_SUFFIX
         from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \
              handle_serials
         import transaction
@@ -494,7 +493,7 @@ class CommonBlobTests:
         self.assertEqual(somedata, open(filename).read())
 
     def checkLoadBlob(self):
-        from ZODB.Blobs.Blob import Blob
+        from ZODB.blob import Blob
         from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \
              handle_serials
         import transaction
@@ -534,8 +533,7 @@ class BlobAdaptedFileStorageTests(GenericTests, CommonBlobTests):
 
     def checkStoreAndLoadBlob(self):
         from ZODB.utils import oid_repr, tid_repr
-        from ZODB.Blobs.Blob import Blob
-        from ZODB.Blobs.BlobStorage import BLOB_SUFFIX
+        from ZODB.blob import Blob, BLOB_SUFFIX
         from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \
              handle_serials
         import transaction

src/ZODB/Blobs/TODO.txt

-Production
-
-    - Ensure we detect and replay a failed txn involving blobs forward or
-      backward at startup.
-
-Far future
-
-      More options for blob directory structures (e.g. dirstorages
-      bushy/chunky/lawn/flat).
-
-      Make the ClientStorage support minimizing the blob cache. (Idea: LRU
-      principle via mstat access time and a size-based threshold) currently).
-
-      Make blobs able to efficiently consume existing files from the filesystem
-
-Savepoint support
-=================
-
- - A savepoint represents the whole state of the data at a certain point in
-   time
-
- - Need special storage for blob savepointing (in the spirit of tmpstorage) 
-
- - What belongs to the state of the data?
-
-   - Data contained in files at that point in time
-
-   - File handles are complex because they might be referred to from various
-     places. We would have to introduce an abstraction layer to allow
-     switching them around... 
-
-     Simpler solution: :

src/ZODB/Blobs/__init__.py

-"""The ZODB Blob package."""

src/ZODB/Blobs/concept.txt

-
-Goal: Handle storage and retrieval of binary large objects efficiently,
-      transactionally, and transparently.
-
-Measure:
-
-    -   Don't block ZServer on uploads and downloads
-
-    -   Don't hold BLOBS in memory or cache if not necessary (LRU caches tend
-        to break if we split BLOBs in lot of small objects. Size-based caches
-        tend to break on single large objects)
-
-    -   Transparent for other systems, support normal ZODB operations.
-    
-Comments:
-
-    - Cache: BLOBs could be cached in a seperate "BLOB" space, e.g. in
-      single files
-
-    - Be storage independent?
-
-    - Memory efficiency: Storge.load() currently holds all data of an
-      object in a string.
-
-Steps:
-
-    - simple aspects:
-        
-        - blobs should be known by zodb 
-        
-            - storages, esp. clientstorage must be able to recognize blobs 
-            
-                - to avoid putting blob data into the client cache.
-
-            - blob data mustn't end up in the object cache
-
-        - blob object and blob data need to be handled separately 
-
-        - blob data on client is stored in temporary files
-
-    - complicated aspects
-
-        - temporary files holding blob data could server as a
-          separated cache for blob data
-
-        - storage / zodb api change
-        
-Restrictions:
-
-    - a particular BLOB instance can't be open for read _and_ write at
-      the same time
-
-        -   Allowed: N readers, no writers; 1 writer, no readers
-
-        -   Reason: 
-
-    - a writable filehandle opened via a BLOB's 'open' method has a
-      lifetime tied to the transaction in which the 'open' method was
-      called.  We do this in order to prevent changes to blob data
-      from "bleeding over" between transactions.
-
-- Data has been committed? -> File(name) for commited data available
-
-- .open("r") on fresh loaded blob returns committed data
-
-- first .open("w") -> new empty file for uncommitted data
-
-- .open("a") or .open("r+"), we copy existing data into file for
-  uncommitted data
-
-- if uncommitted data exists, subsequent .open("*") will use the
-  uncommitted data
-
-- if opened for writing, the object is marked as changed
-  (optimiziation possible)
-
-- connections want to recognize blobs on transaction boundaries
-
-

src/ZODB/Blobs/exceptions.py

-
-class BlobError(Exception):
-    pass

src/ZODB/Blobs/interfaces.py

-##############################################################################
-#
-# Copyright (c) 2005-2007 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-"""Blob-related interfaces
-
-"""
-
-from zope.interface import Interface
-
-
-class IBlob(Interface):
-    """A BLOB supports efficient handling of large data within ZODB."""
-
-    def open(mode):
-        """Returns a file(-like) object for handling the blob data.
-
-        mode: Mode to open the file with. Possible values: r,w,r+,a
-        """
-
-    def openDetached(class_=file):
-        """Returns a file(-like) object in read mode that can be used
-        outside of transaction boundaries.
-
-        The file handle returned by this method is read-only and at the
-        beginning of the file. 
-
-        The handle is not attached to the blob and can be used outside of a
-        transaction.
-
-        Optionally the class that should be used to open the file can be
-        specified. This can be used to e.g. use Zope's FileStreamIterator.
-        """
-
-    def consumeFile(filename):
-        """Will replace the current data of the blob with the file given under
-        filename.
-
-        This method uses link-like semantics internally and has the requirement
-        that the file that is to be consumed lives on the same volume (or
-        mount/share) as the blob directory.
-
-        The blob must not be opened for reading or writing when consuming a 
-        file.
-        """
-
-
-class IBlobStorage(Interface):
-    """A storage supporting BLOBs."""
-
-    def storeBlob(oid, oldserial, data, blob, version, transaction):
-        """Stores data that has a BLOB attached."""
-
-    def loadBlob(oid, serial):
-        """Return the filename of the Blob data for this OID and serial.
-
-        Returns a filename or None if no Blob data is connected with this OID. 
-
-        Raises POSKeyError if the blobfile cannot be found.
-        """
-
-    def temporaryDirectory():
-        """Return a directory that should be used for uncommitted blob data.
-
-        If Blobs use this, then commits can be performed with a simple rename.
-        """

src/ZODB/Blobs/tests/__init__.py

-# python package

src/ZODB/Blobs/tests/test_config.py

-##############################################################################
-#
-# Copyright (c) 2004-2006 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-import tempfile, shutil, unittest
-import os
-
-from ZODB.tests.testConfig import ConfigTestBase
-from ZConfig import ConfigurationSyntaxError
-
-
-class BlobConfigTestBase(ConfigTestBase):
-
-    def setUp(self):
-        super(BlobConfigTestBase, self).setUp()
-
-        self.blob_dir = tempfile.mkdtemp()
-
-    def tearDown(self):
-        super(BlobConfigTestBase, self).tearDown()
-
-        shutil.rmtree(self.blob_dir)
-
-
-class ZODBBlobConfigTest(BlobConfigTestBase):
-
-    def test_map_config1(self):
-        self._test(
-            """
-            <zodb>
-              <blobstorage>
-                blob-dir %s
-                <mappingstorage/>
-              </blobstorage>
-            </zodb>
-            """ % self.blob_dir)
-
-    def test_file_config1(self):
-        path = tempfile.mktemp()
-        self._test(
-            """
-            <zodb>
-              <blobstorage>
-                blob-dir %s
-                <filestorage>
-                  path %s
-                </filestorage>
-              </blobstorage>
-            </zodb>
-            """ %(self.blob_dir, path))
-        os.unlink(path)
-        os.unlink(path+".index")
-        os.unlink(path+".tmp")
-
-    def test_blob_dir_needed(self):
-        self.assertRaises(ConfigurationSyntaxError,
-                          self._test,
-                          """
-                          <zodb>
-                            <blobstorage>
-                              <mappingstorage/>
-                            </blobstorage>
-                          </zodb>
-                          """)
-
-
-def test_suite():
-    suite = unittest.TestSuite()
-    suite.addTest(unittest.makeSuite(ZODBBlobConfigTest))
-
-    return suite
-
-if __name__ == '__main__':
-    unittest.main(defaultTest = 'test_suite')

src/ZODB/Blobs/tests/test_doctests.py

-##############################################################################
-#
-# Copyright (c) 2004 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-from zope.testing import doctest
-import ZODB.tests.util
-
-def test_suite():
-    return doctest.DocFileSuite(
-        "basic.txt",  "connection.txt", "transaction.txt",
-        "packing.txt", "importexport.txt", "consume.txt",
-        setUp=ZODB.tests.util.setUp,
-        tearDown=ZODB.tests.util.tearDown,
-        )

src/ZODB/Connection.py

@@ -29,7 +29,7 @@ from persistent import PickleCache
 # interfaces
 from persistent.interfaces import IPersistentDataManager
 from ZODB.interfaces import IConnection
-from ZODB.Blobs.interfaces import IBlob, IBlobStorage
+from ZODB.interfaces import IBlob, IBlobStorage
 from transaction.interfaces import ISavepointDataManager
 from transaction.interfaces import IDataManagerSavepoint
 from transaction.interfaces import ISynchronizer

src/ZODB/ExportImport.py

@@ -22,7 +22,7 @@ import logging
 
 from ZODB.POSException import ExportError, POSKeyError
 from ZODB.utils import p64, u64, cp, mktemp
-from ZODB.Blobs.interfaces import IBlobStorage
+from ZODB.interfaces import IBlobStorage
 from ZODB.serialize import referencesf
 
 logger = logging.getLogger('ZODB.ExportImport')

src/ZODB/config.py

@@ -137,7 +137,7 @@ class FileStorage(BaseConfig):
 class BlobStorage(BaseConfig):
 
     def open(self):
-        from ZODB.Blobs.BlobStorage import BlobStorage
+        from ZODB.blob import BlobStorage
         base = self.config.base.open()
         return BlobStorage(self.config.blob_dir, base)
 

src/ZODB/interfaces.py

@@ -897,3 +897,62 @@ class IStorageCurrentRecordIteration(IStorage):
             ...         break
         
         """
+
+class IBlob(Interface):
+    """A BLOB supports efficient handling of large data within ZODB."""
+
+    def open(mode):
+        """Returns a file(-like) object for handling the blob data.
+
+        mode: Mode to open the file with. Possible values: r,w,r+,a
+        """
+
+    def openDetached(class_=file):
+        """Returns a file(-like) object in read mode that can be used
+        outside of transaction boundaries.
+
+        The file handle returned by this method is read-only and at the
+        beginning of the file. 
+
+        The handle is not attached to the blob and can be used outside of a
+        transaction.
+
+        Optionally the class that should be used to open the file can be
+        specified. This can be used to e.g. use Zope's FileStreamIterator.
+        """
+
+    def consumeFile(filename):
+        """Will replace the current data of the blob with the file given under
+        filename.
+
+        This method uses link-like semantics internally and has the requirement
+        that the file that is to be consumed lives on the same volume (or
+        mount/share) as the blob directory.
+
+        The blob must not be opened for reading or writing when consuming a 
+        file.
+        """
+
+
+class IBlobStorage(Interface):
+    """A storage supporting BLOBs."""
+
+    def storeBlob(oid, oldserial, data, blob, version, transaction):
+        """Stores data that has a BLOB attached."""
+
+    def loadBlob(oid, serial):
+        """Return the filename of the Blob data for this OID and serial.
+
+        Returns a filename or None if no Blob data is connected with this OID. 
+
+        Raises POSKeyError if the blobfile cannot be found.
+        """
+
+    def temporaryDirectory():
+        """Return a directory that should be used for uncommitted blob data.
+
+        If Blobs use this, then commits can be performed with a simple rename.
+        """
+
+class BlobError(Exception):
+    pass

src/ZODB/Blobs/tests/basic.txt → src/ZODB/tests/blob_basic.txt

@@ -17,12 +17,12 @@ ZODB Blob support
 
 You create a blob like this:
 
-    >>> from ZODB.Blobs.Blob import Blob
+    >>> from ZODB.blob import Blob
     >>> myblob = Blob()
 
 A blob implements the IBlob interface:
 
-    >>> from ZODB.Blobs.interfaces import IBlob
+    >>> from ZODB.interfaces import IBlob
     >>> IBlob.providedBy(myblob)
     True
 

src/ZODB/Blobs/tests/connection.txt → src/ZODB/tests/blob_connection.txt

@@ -17,8 +17,8 @@ Connection support for Blobs tests
 
 Connections handle Blobs specially. To demonstrate that, we first need a Blob with some data:
 
-    >>> from ZODB.Blobs.interfaces import IBlob
-    >>> from ZODB.Blobs.Blob import Blob
+    >>> from ZODB.interfaces import IBlob
+    >>> from ZODB.blob import Blob
     >>> import transaction
     >>> blob = Blob()
     >>> data = blob.open("w")
@@ -28,7 +28,7 @@ Connections handle Blobs specially. To demonstrate that, we first need a Blob wi
 We also need a database with a blob supporting storage:
 
     >>> from ZODB.MappingStorage import MappingStorage
-    >>> from ZODB.Blobs.BlobStorage import BlobStorage
+    >>> from ZODB.blob import BlobStorage
     >>> from ZODB.DB import DB
     >>> from tempfile import mkdtemp
     >>> base_storage = MappingStorage("test")

src/ZODB/Blobs/tests/consume.txt → src/ZODB/tests/blob_consume.txt

@@ -15,7 +15,7 @@ The file *must* be closed before giving it to consumeFile:
 
 Now, let's consume this file in a blob by specifying it's name::
 
-    >>> from ZODB.Blobs.Blob import Blob
+    >>> from ZODB.blob import Blob
     >>> blob = Blob()
     >>> blob.consumeFile('to_import')
 

src/ZODB/Blobs/tests/importexport.txt → src/ZODB/tests/blob_importexport.txt

@@ -18,8 +18,7 @@ Import/export support for blob data
 Set up:
 
     >>> from ZODB.FileStorage import FileStorage
-    >>> from ZODB.Blobs.BlobStorage import BlobStorage
-    >>> from ZODB.Blobs.Blob import Blob
+    >>> from ZODB.blob import Blob, BlobStorage
     >>> from ZODB.DB import DB
     >>> from persistent.mapping import PersistentMapping
     >>> import shutil

src/ZODB/Blobs/tests/packing.txt → src/ZODB/tests/blob_packing.txt

@@ -20,8 +20,7 @@ Set up:
     >>> from ZODB.FileStorage import FileStorage
     >>> from ZODB.MappingStorage import MappingStorage
     >>> from ZODB.serialize import referencesf
-    >>> from ZODB.Blobs.BlobStorage import BlobStorage
-    >>> from ZODB.Blobs.Blob import Blob
+    >>> from ZODB.blob import Blob, BlobStorage
     >>> from ZODB import utils
     >>> from ZODB.DB import DB
     >>> import shutil

src/ZODB/Blobs/tests/transaction.txt → src/ZODB/tests/blob_transaction.txt

@@ -18,7 +18,7 @@ Transaction support for Blobs
 We need a database with a blob supporting storage::
 
     >>> from ZODB.MappingStorage import MappingStorage
-    >>> from ZODB.Blobs.BlobStorage import BlobStorage
+    >>> from ZODB.blob import Blob, BlobStorage
     >>> from ZODB.DB import DB
     >>> import transaction
     >>> import tempfile
@@ -29,7 +29,6 @@ We need a database with a blob supporting storage::
     >>> database = DB(blob_storage)
     >>> connection1 = database.open()
     >>> root1 = connection1.root()
-    >>> from ZODB.Blobs.Blob import Blob
 
 Putting a Blob into a Connection works like any other Persistent object::
 
@@ -176,7 +175,7 @@ connections should result in a write conflict error::
     >>> tm2.get().commit()
     Traceback (most recent call last):
         ...
-    ConflictError: database conflict error (oid 0x01, class ZODB.Blobs.Blob.Blob)
+    ConflictError: database conflict error (oid 0x01, class ZODB.blob.Blob)
 
 After the conflict, the winning transaction's result is visible on both
 connections::
@@ -233,7 +232,7 @@ We do not support non-optimistic savepoints::
     >>> savepoint = transaction.savepoint()             # doctest: +ELLIPSIS
     Traceback (most recent call last):
         ...
-    TypeError: ('Savepoints unsupported', <ZODB.Blobs.Blob.BlobDataManager instance at 0x...>)
+    TypeError: ('Savepoints unsupported', <ZODB.blob.BlobDataManager instance at 0x...>)
     >>> transaction.abort()
 
 Reading Blobs outside of a transaction

src/ZODB/Blobs/tests/test_undo.py → src/ZODB/tests/testblob.py

@@ -11,19 +11,74 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
-import unittest
-import tempfile
-import os
-import shutil
-import base64
 
+import base64, os, shutil, tempfile, unittest
+from zope.testing import doctest
+import ZODB.tests.util
+
+from ZODB import utils
 from ZODB.FileStorage import FileStorage
-from ZODB.Blobs.BlobStorage import BlobStorage
-from ZODB.Blobs.Blob import Blob
+from ZODB.blob import Blob, BlobStorage
 from ZODB.DB import DB
 import transaction
-from ZODB.Blobs.Blob import Blob
-from ZODB import utils
+
+from ZODB.tests.testConfig import ConfigTestBase
+from ZConfig import ConfigurationSyntaxError
+
+class BlobConfigTestBase(ConfigTestBase):
+
+    def setUp(self):
+        super(BlobConfigTestBase, self).setUp()
+
+        self.blob_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        super(BlobConfigTestBase, self).tearDown()
+
+        shutil.rmtree(self.blob_dir)
+
+
+class ZODBBlobConfigTest(BlobConfigTestBase):
+
+    def test_map_config1(self):
+        self._test(
+            """
+            <zodb>
+              <blobstorage>
+                blob-dir %s
+                <mappingstorage/>
+              </blobstorage>
+            </zodb>
+            """ % self.blob_dir)
+
+    def test_file_config1(self):
+        path = tempfile.mktemp()
+        self._test(
+            """
+            <zodb>
+              <blobstorage>
+                blob-dir %s
+                <filestorage>
+                  path %s
+                </filestorage>
+              </blobstorage>
+            </zodb>
+            """ %(self.blob_dir, path))
+        os.unlink(path)
+        os.unlink(path+".index")
+        os.unlink(path+".tmp")
+
+    def test_blob_dir_needed(self):
+        self.assertRaises(ConfigurationSyntaxError,
+                          self._test,
+                          """
+                          <zodb>
+                            <blobstorage>
+                              <mappingstorage/>
+                            </blobstorage>
+                          </zodb>
+                          """)
+
 
 class BlobUndoTests(unittest.TestCase):
 
@@ -212,9 +267,19 @@ class BlobUndoTests(unittest.TestCase):
 
 def test_suite():
     suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(ZODBBlobConfigTest))
+    suite.addTest(doctest.DocFileSuite(
+        "blob_basic.txt",  "blob_connection.txt", "blob_transaction.txt",
+        "blob_packing.txt", "blob_importexport.txt", "blob_consume.txt",
+        setUp=ZODB.tests.util.setUp,
+        tearDown=ZODB.tests.util.tearDown,
+        ))
     suite.addTest(unittest.makeSuite(BlobUndoTests))
 
     return suite
 
 if __name__ == '__main__':
     unittest.main(defaultTest = 'test_suite')
+
+
+