Commit e3570c83 authored by Kirill Smelkov's avatar Kirill Smelkov

fixup! ZBigFile: Add ZBlk format option 'h' (heuristic) (3)

Use type objects instead of strings to refer to formats because with
strings scattered here and there it is very easy to cause misbehaviour
due to e.g. a typo. With referering to objects by their name, a type is
automatically caught either at runtime, or statically via e.g. pyflakes.
parent f845ad4f
...@@ -481,15 +481,19 @@ class ZBlk1(ZBlkBase): ...@@ -481,15 +481,19 @@ class ZBlk1(ZBlkBase):
# backward compatibility (early versions wrote ZBlk0 named as ZBlk) # backward compatibility (early versions wrote ZBlk0 named as ZBlk)
ZBlk = ZBlk0 ZBlk = ZBlk0
# _ZBlk_auto indicates to heuristically select ZBlk format
_ZBlk_auto = object()
# format-name -> blk format type # format-name -> blk format type
ZBlk_fmt_registry = { ZBlk_fmt_registry = {
'ZBlk0': ZBlk0, 'ZBlk0': ZBlk0,
'ZBlk1': ZBlk1, 'ZBlk1': ZBlk1,
'auto': _ZBlk_auto,
} }
# format for updated blocks # format for updated blocks
ZBlk_fmt_write = os.environ.get('WENDELIN_CORE_ZBLK_FMT', 'auto') ZBlk_fmt_write = os.environ.get('WENDELIN_CORE_ZBLK_FMT', 'auto')
if ZBlk_fmt_write != "auto" and ZBlk_fmt_write not in ZBlk_fmt_registry: if ZBlk_fmt_write not in ZBlk_fmt_registry:
raise RuntimeError('E: Unknown ZBlk format %r' % ZBlk_fmt_write) raise RuntimeError('E: Unknown ZBlk format %r' % ZBlk_fmt_write)
...@@ -551,13 +555,12 @@ class ZBigFile(LivePersistent): ...@@ -551,13 +555,12 @@ class ZBigFile(LivePersistent):
# store data dirty page -> ZODB obj # store data dirty page -> ZODB obj
def storeblk(self, blk, buf): def storeblk(self, blk, buf):
zblk = self.blktab.get(blk) zblk = self.blktab.get(blk)
zblk_fmt = ZBlk_fmt_write zblk_type_write = ZBlk_fmt_registry[ZBlk_fmt_write]
if zblk_fmt == "auto": # apply heuristic if zblk_type_write is _ZBlk_auto: # apply heuristic
zblk_fmt = self._zblk_fmt_heuristic(zblk, blk, buf) zblk_type_write = self._zblk_fmt_heuristic(zblk, blk, buf)
self._setzblk(blk, zblk, buf, zblk_fmt) self._setzblk(blk, zblk, buf, zblk_type_write)
def _setzblk(self, blk, zblk, buf, zblk_fmt): # helper def _setzblk(self, blk, zblk, buf, zblk_type_write): # helper
zblk_type_write = ZBlk_fmt_registry[zblk_fmt or ZBlk_fmt_write]
# if zblk was absent or of different type - we (re-)create it anew # if zblk was absent or of different type - we (re-)create it anew
if zblk is None or \ if zblk is None or \
type(zblk) is not zblk_type_write: type(zblk) is not zblk_type_write:
...@@ -590,14 +593,14 @@ class ZBigFile(LivePersistent): ...@@ -590,14 +593,14 @@ class ZBigFile(LivePersistent):
previous_blk = blk - 1 previous_blk = blk - 1
previous_zblk = self.blktab.get(previous_blk) previous_zblk = self.blktab.get(previous_blk)
if previous_zblk is not None: if previous_zblk is not None:
self._setzblk(previous_blk, previous_zblk, previous_zblk.loadblkdata(), "ZBlk0") self._setzblk(previous_blk, previous_zblk, previous_zblk.loadblkdata(), ZBlk0)
return "ZBlk1" return ZBlk1
else: # it's changing else: # it's changing
# kirr: "to support sporadic small changes over initial big fillup [...] # kirr: "to support sporadic small changes over initial big fillup [...]
# we could introduce e.g. a ZBlkδ object, which would refer to base # we could introduce e.g. a ZBlkδ object, which would refer to base
# underlying ZBlk object and add "patch" information on top of that [...]." # underlying ZBlk object and add "patch" information on top of that [...]."
# See https://lab.nexedi.com/nexedi/wendelin.core/merge_requests/20#note_196084 # See https://lab.nexedi.com/nexedi/wendelin.core/merge_requests/20#note_196084
return 'ZBlk1' return ZBlk1
# invalidate data .blktab[blk] invalidated -> invalidate page # invalidate data .blktab[blk] invalidated -> invalidate page
......
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
# #
# See COPYING file for full licensing terms. # See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options. # See https://www.nexedi.com/licensing for rationale and options.
from wendelin.bigfile.file_zodb import ZBigFile, ZBlk_fmt_registry from wendelin.bigfile.file_zodb import ZBigFile, ZBlk_fmt_registry, _ZBlk_auto
from wendelin.bigfile import file_zodb, ram_reclaim from wendelin.bigfile import file_zodb, ram_reclaim
from wendelin.bigfile.tests.test_thread import NotifyChannel from wendelin.bigfile.tests.test_thread import NotifyChannel
from wendelin.lib.zodb import LivePersistent, dbclose from wendelin.lib.zodb import LivePersistent, dbclose
...@@ -631,6 +631,8 @@ def test_bigfile_filezodb_fmt_change(): ...@@ -631,6 +631,8 @@ def test_bigfile_filezodb_fmt_change():
for dst_fmt, dst_type in ZBlk_fmt_registry.items(): for dst_fmt, dst_type in ZBlk_fmt_registry.items():
if src_fmt == dst_fmt: if src_fmt == dst_fmt:
continue # skip checking e.g. ZBlk0 -> ZBlk0 continue # skip checking e.g. ZBlk0 -> ZBlk0
if src_type is _ZBlk_auto or dst_type is _ZBlk_auto:
continue # skip checking e.g. * -> auto
file_zodb.ZBlk_fmt_write = src_fmt file_zodb.ZBlk_fmt_write = src_fmt
struct.pack_into('p', vma, 0, b(src_fmt)) struct.pack_into('p', vma, 0, b(src_fmt))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment