Commit 1213f8a6 authored by Christian Theune's avatar Christian Theune

Implemented new bushy layout for blob directories including backwards

compatibility for the old `lawn` layout.
parent de31996b
......@@ -8,6 +8,11 @@ Change History
New Features
------------
- Changed layout strategy for the blob directory to a bushy approach (8 levels
deep, at most ~256 entries per directory level, one directory for each
blob). Old directories are automatically detected and will be handled with
the old strategy.
- Versions are no-longer supported.
- ZEO cache files can be larger than 4G. Note that older ZEO cache
......
......@@ -855,9 +855,7 @@ class ClientStorage(object):
def _storeBlob_shared(self, oid, serial, data, filename, txn):
# First, move the blob into the blob directory
dir = self.fshelper.getPathForOID(oid)
if not os.path.exists(dir):
os.mkdir(dir)
self.fshelper.getPathForOID(oid, create=True)
fd, target = self.fshelper.blob_mkstemp(oid, serial)
os.close(fd)
......@@ -924,14 +922,7 @@ class ClientStorage(object):
raise POSException.POSKeyError("No blob file", oid, serial)
# First, we'll create the directory for this oid, if it doesn't exist.
targetpath = self.fshelper.getPathForOID(oid)
if not os.path.exists(targetpath):
try:
os.makedirs(targetpath, 0700)
except OSError:
# We might have lost a race. If so, the directory
# must exist now
assert os.path.exists(targetpath)
targetpath = self.fshelper.getPathForOID(oid, create=True)
# OK, it's not here and we (or someone) needs to get it. We
# want to avoid getting it multiple times. We want to avoid
......@@ -1118,19 +1109,15 @@ class ClientStorage(object):
assert s == tid, (s, tid)
self._cache.store(oid, s, None, data)
if self.fshelper is not None:
blobs = self._tbuf.blobs
while blobs:
oid, blobfilename = blobs.pop()
targetpath = self.fshelper.getPathForOID(oid)
if not os.path.exists(targetpath):
os.makedirs(targetpath, 0700)
targetpath = self.fshelper.getPathForOID(oid, create=True)
rename_or_copy_blob(blobfilename,
self.fshelper.getBlobFilename(oid, tid),
)
self._tbuf.clear()
def undo(self, trans_id, txn):
......
......@@ -515,8 +515,7 @@ class CommonBlobTests:
self._storage.tpc_abort(t)
raise
self.assert_(not os.path.exists(tfname))
filename = os.path.join(self.blobdir, oid_repr(oid),
tid_repr(revid) + BLOB_SUFFIX)
filename = self._storage.fshelper.getBlobFilename(oid, revid)
self.assert_(os.path.exists(filename))
self.assertEqual(somedata, open(filename).read())
......@@ -631,16 +630,14 @@ class BlobAdaptedFileStorageTests(GenericTests, CommonBlobTests):
d2 = somedata.read(8096)
self.assertEqual(d1, d2)
# The file should have been copied to the server:
filename = os.path.join(self.blobdir, oid_repr(oid),
tid_repr(revid) + BLOB_SUFFIX)
# The file should be in the cache ...
filename = self._storage.fshelper.getBlobFilename(oid, revid)
check_data(filename)
# It should also be in the cache:
filename = os.path.join(self.blob_cache_dir, oid_repr(oid),
tid_repr(revid) + BLOB_SUFFIX)
check_data(filename)
# ... and on the server
server_filename = filename.replace(self.blob_cache_dir, self.blobdir)
self.assert_(server_filename.startswith(self.blobdir))
check_data(server_filename)
# If we remove it from the cache and call loadBlob, it should
# come back. We can do this in many threads. We'll instrument
......
This diff is collapsed.
======================
Blob directory layouts
======================
The internal structure of the blob directories is governed by so called
`layouts`. The current default layout is called `bushy`.
The original blob implementation used a layout that we now call `lawn` and
which is still available for backwards compatibility.
Layouts implement two methods: one for computing a relative path for an
OID and one for turning a relative path back into an OID.
Our terminology is roughly the same as used in `DirectoryStorage`.
The `bushy` layout
==================
The bushy layout splits the OID into the 8 byte parts, reverses them and
creates one directory level for each part, named by the hexlified
representation of the byte value. This results in 8 levels of directories, the
leaf directories being used for the revisions of the blobs and at most 256
entries per directory level:
>>> from ZODB.blob import BushyLayout
>>> bushy = BushyLayout()
>>> bushy.oid_to_path('\x00\x00\x00\x00\x00\x00\x00\x00')
'0x00/0x00/0x00/0x00/0x00/0x00/0x00/0x00'
>>> bushy.oid_to_path('\x00\x00\x00\x00\x00\x00\x00\x01')
'0x01/0x00/0x00/0x00/0x00/0x00/0x00/0x00'
>>> bushy.path_to_oid('0x01/0x00/0x00/0x00/0x00/0x00/0x00/0x00')
'\x00\x00\x00\x00\x00\x00\x00\x01'
Paths that do not represent an OID will cause a ValueError:
>>> bushy.path_to_oid('tmp')
Traceback (most recent call last):
ValueError: Not a valid OID path: tmp
The `lawn` layout
=================
The lawn layout creates on directory for each blob named by the blob's hex
representation of its OID. This has some limitations on various file systems
like performance penalties or the inability to store more than a given number
of blobs at the same time (e.g. 32k on ext3).
>>> from ZODB.blob import LawnLayout
>>> lawn = LawnLayout()
>>> lawn.oid_to_path('\x00\x00\x00\x00\x00\x00\x00\x00')
'0x00'
>>> lawn.oid_to_path('\x00\x00\x00\x00\x00\x00\x00\x01')
'0x01'
>>> lawn.path_to_oid('0x01')
'\x00\x00\x00\x00\x00\x00\x00\x01'
Paths that do not represent an OID will cause a ValueError:
>>> lawn.path_to_oid('tmp')
Traceback (most recent call last):
ValueError: Not a valid OID path: tmp
Auto-detecting the layout of a directory
========================================
To allow easier migration, we provide an auto-detection feature that analyses a
blob directory and decides for a strategy to use. In general it prefers to
choose the `bushy` layout, except if it determines that the directory has
already been used to create a lawn structure.
>>> from ZODB.blob import auto_layout_select
1. Non-existing directories will trigger a bushy layout:
>>> import tempfile
>>> import shutil
>>> d = tempfile.mkdtemp()
>>> shutil.rmtree(d)
>>> auto_layout_select(d)
'bushy'
2. Empty directories will trigger a bushy layout too:
>>> d = tempfile.mkdtemp()
>>> auto_layout_select(d)
'bushy'
3. If the directory contains a marker for the strategy it will be used:
>>> from ZODB.blob import LAYOUT_MARKER
>>> import os.path
>>> open(os.path.join(d, LAYOUT_MARKER), 'wb').write('bushy')
>>> auto_layout_select(d)
'bushy'
>>> open(os.path.join(d, LAYOUT_MARKER), 'wb').write('lawn')
>>> auto_layout_select(d)
'lawn'
>>> shutil.rmtree(d)
4. If the directory does not contain a marker but other files, we assume that
it was created with an earlier version of the blob implementation and uses our
`lawn` layout:
>>> d = tempfile.mkdtemp()
>>> open(os.path.join(d, '0x0101'), 'wb').write('foo')
>>> auto_layout_select(d)
'lawn'
>>> shutil.rmtree(d)
Directory layout markers
========================
When the file system helper (FSH) is asked to create the directory structure,
it will leave a marker with the choosen layout if no marker exists yet:
>>> from ZODB.blob import FilesystemHelper
>>> d = tempfile.mkdtemp()
>>> blobs = os.path.join(d, 'blobs')
>>> fsh = FilesystemHelper(blobs)
>>> fsh.layout_name
'bushy'
>>> fsh.create()
>>> open(os.path.join(blobs, LAYOUT_MARKER), 'rb').read()
'bushy'
If the FSH finds a marker, then it verifies whether its content matches the
strategy that was chosen. It will raise an exception if we try to work with a
directory that has a different marker than the chosen strategy:
>>> fsh = FilesystemHelper(blobs, 'lawn')
>>> fsh.layout_name
'lawn'
>>> fsh.create() # doctest: +ELLIPSIS
Traceback (most recent call last):
ValueError: Directory layout `lawn` selected for blob directory /.../blobs/, but marker found for layout `bushy`
>>> shutil.rmtree(blobs)
This function interacts with the automatic detection in the way, that an
unmarked directory will be marked the first time when it is auto-guessed and
the marker will be used in the future:
>>> import ZODB.FileStorage
>>> from ZODB.blob import BlobStorage
>>> datafs = os.path.join(d, 'data.fs')
>>> base_storage = ZODB.FileStorage.FileStorage(datafs)
>>> os.mkdir(blobs)
>>> open(os.path.join(blobs, 'foo'), 'wb').write('foo')
>>> blob_storage = BlobStorage(blobs, base_storage)
>>> blob_storage.fshelper.layout_name
'lawn'
>>> open(os.path.join(blobs, LAYOUT_MARKER), 'rb').read()
'lawn'
>>> blob_storage = BlobStorage(blobs, base_storage, layout='bushy') # doctest: +ELLIPSIS
Traceback (most recent call last):
ValueError: Directory layout `bushy` selected for blob directory /.../blobs/, but marker found for layout `lawn`
>>> shutil.rmtree(d)
......@@ -32,7 +32,7 @@ First, we need a datatabase with blob support::
>>> from ZODB.DB import DB
>>> from tempfile import mkdtemp
>>> import os.path
>>> base_storage = MappingStorage("test")
>>> base_storage = MappingStorage('test')
>>> blob_dir = mkdtemp()
>>> blob_storage = BlobStorage(blob_dir, base_storage)
>>> database = DB(blob_storage)
......
......@@ -322,9 +322,9 @@ clean up dirty files:
>>> base_storage = DummyBaseStorage()
>>> blob_dir2 = mkdtemp()
>>> blob_storage2 = BlobStorage(blob_dir2, base_storage)
>>> committed_blob_dir = os.path.join(blob_dir2, '0')
>>> committed_blob_file = os.path.join(committed_blob_dir, '0.blob')
>>> os.mkdir(committed_blob_dir)
>>> committed_blob_dir = blob_storage2.fshelper.getPathForOID(0)
>>> os.makedirs(committed_blob_dir)
>>> committed_blob_file = blob_storage2.fshelper.getBlobFilename(0, 0)
>>> open(os.path.join(committed_blob_file), 'w').write('foo')
>>> os.path.exists(committed_blob_file)
True
......
......@@ -105,7 +105,6 @@ class BlobUndoTests(unittest.TestCase):
self.here = os.getcwd()
os.chdir(self.test_dir)
self.storagefile = 'Data.fs'
os.mkdir('blobs')
self.blob_dir = 'blobs'
def tearDown(self):
......@@ -483,7 +482,7 @@ def loadblob_tmpstore():
We can access the blob correctly:
>>> tmpstore.loadBlob(blob_oid, tid) # doctest: +ELLIPSIS
'.../0x01/0x...blob'
'.../0x01/0x00/0x00/0x00/0x00/0x00/0x00/0x00/0x...blob'
Clean up:
......@@ -503,7 +502,7 @@ def test_suite():
suite.addTest(doctest.DocFileSuite(
"blob_basic.txt", "blob_connection.txt", "blob_transaction.txt",
"blob_packing.txt", "blob_importexport.txt", "blob_consume.txt",
"blob_tempdir.txt",
"blob_tempdir.txt", "blob_layout.txt",
setUp=ZODB.tests.util.setUp,
tearDown=ZODB.tests.util.tearDown,
))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment