ZBigFile: Allow explicitly overridding global ZBlk fmt

With this patch it's possible now to explicitly set the ZBlk format of a
single ZBigFile. Before this patch it was only possible to set the ZBlk
format globally, for all ZBigFile. This new feature is useful if we have a
project where we would like to have mixed ZBlk formats, e.g. some
ZBigFile with ZBlk0 and some ZBigFile with ZBlk1.

bigfile/file_zodb.py

@@ -509,17 +509,25 @@ class ZBigFile(LivePersistent):
     # NOTE Live: don't allow us to go to ghost state not to loose ._v_file
     #      which DataManager _ZBigFileH refers to.
 
-    def __init__(self, blksize):
+    def __init__(self, blksize, zblk_fmt=""):
         LivePersistent.__init__(self)
-        self.__setstate__((blksize, LOBTree()))     # NOTE L enough for blk_t
+        self.__setstate__((blksize, LOBTree(), zblk_fmt))     # NOTE L enough for blk_t
+        self.zblk_fmt = zblk_fmt  # Evoke check if zblk_fmt is valid
 
 
-    # state is (.blksize, .blktab)
+    # state is (.blksize, .blktab, .zblk_fmt)
     def __getstate__(self):
-        return (self.blksize, self.blktab)
+        return (self.blksize, self.blktab, self.zblk_fmt)
 
     def __setstate__(self, state):
-        self.blksize, self.blktab = state
+        state_length = len(state)
+        if state_length == 2:  # BBB
+            self.__setstate__(tuple(state) + ("",))
+        elif state_length == 3:
+            # NOTE set _zblk_fmt instead of zblk_fmt to avoid check => ↑ performance
+            self.blksize, self.blktab, self._zblk_fmt = state
+        else:
+            raise RuntimeError("E: Unexpected state length: %s" % state)
         self._v_file = _ZBigFile._new(self, self.blksize)
         self._v_filehset = WeakSet()
 
@@ -546,7 +554,7 @@ class ZBigFile(LivePersistent):
     # store data    dirty page -> ZODB obj
     def storeblk(self, blk, buf):
         zblk = self.blktab.get(blk)
-        zblk_type_write = ZBlk_fmt_registry[ZBlk_fmt_write]
+        zblk_type_write = ZBlk_fmt_registry[self.zblk_fmt or ZBlk_fmt_write]
         # if zblk was absent or of different type - we (re-)create it anew
         if zblk is None  or \
            type(zblk) is not zblk_type_write:
@@ -605,6 +613,19 @@ class ZBigFile(LivePersistent):
         return fileh
 
 
+    # zblk_fmt is a property to implement check if user declared zblk_fmt
+    # is valid.
+    @property
+    def zblk_fmt(self):
+        return self._zblk_fmt
+
+    @zblk_fmt.setter
+    def zblk_fmt(self, zblk_fmt):
+        if zblk_fmt and zblk_fmt not in ZBlk_fmt_registry:
+            raise RuntimeError('E: Unknown ZBlk format %r' % zblk_fmt)
+        self._zblk_fmt = zblk_fmt
+
+
     # _default_use_wcfs returns whether default virtmem setting is to use wcfs or not.
     @staticmethod
     def _default_use_wcfs():

bigfile/tests/test_filezodb.py

@@ -690,3 +690,29 @@ def test_bigfile_zblk1_zdata_reuse():
     assert len(zdata_v1) == len(zdata_v2)
     for i in range(len(zdata_v1)):
         assert zdata_v1[i] is zdata_v2[i]
+
+
+# test that explicitly setting the ZBlk format works
+@func
+def test_bigfile_set_zblk_fmt():
+    # ensure ZBlk_fmt_write is ZBlk0 during this test,
+    # so that we can be sure the global default is overridden
+    # by an explicitly set value during ZBigFile initialization
+    fmt_write_save = file_zodb.ZBlk_fmt_write
+    file_zodb.ZBlk_fmt_write = 'ZBlk0'
+    def _():
+        file_zodb.ZBlk_fmt_write = fmt_write_save
+    defer(_)
+
+    root = dbopen()
+    defer(lambda: dbclose(root))
+    root['zfile7'] = f = ZBigFile(blksize, zblk_fmt="ZBlk1")
+    transaction.commit()
+
+    fh  = f.fileh_open()
+    vma = fh.mmap(0, blen)
+
+    Blk(vma, 0)[:] = 1
+    transaction.commit()
+
+    assert type(f.blktab[0]) is file_zodb.ZBlk1

wcfs/internal/pycompat/pycompat.go

@@ -3,7 +3,10 @@
 package pycompat
 
 import (
+	"fmt"
 	"math/big"
+
+	pickle "github.com/kisielk/og-rek"
 )
 
 // Int64 tries to convert unpickled Python value to int64.
@@ -23,3 +26,21 @@ func Int64(xv interface{}) (v int64, ok bool) {
 
         return 0, false
 }
+
+
+// Xstrbytes verifies and extacts str|bytes from unpickled value.
+func Xstrbytes(x interface{}) (string, error) {
+	var s string
+	switch x := x.(type) {
+	default:
+		return "", fmt.Errorf("expect str|bytes; got %T", x)
+
+	case string:
+		s = x
+
+	case pickle.Bytes:
+		s = string(x)
+	}
+
+	return s, nil
+}

wcfs/internal/zdata/zblk.go

@@ -354,9 +354,10 @@ func (zb *ZBlk1) LoadBlkData(ctx context.Context) (_ []byte, _ zodb.Tid, err err
 type ZBigFile struct {
 	zodb.Persistent
 
-	// state: (.blksize, .blktab)
+	// state: (.blksize, .blktab, .zblk_fmt)
 	blksize  int64
 	blktab   *btree.LOBTree // {}  blk -> ZBlk*(blkdata)
+	zblk_fmt string
 }
 
 type zBigFileState ZBigFile // hide state methods from public API
@@ -365,21 +366,25 @@ type zBigFileState ZBigFile // hide state methods from public API
 func (bf *zBigFileState) DropState() {
 	bf.blksize  = 0
 	bf.blktab   = nil
+	bf.zblk_fmt = ""
 }
 
 // PyGetState implements zodb.PyStateful.
 func (bf *zBigFileState) PyGetState() interface{} {
-	return pickle.Tuple{bf.blksize, bf.blktab}
+	return pickle.Tuple{bf.blksize, bf.blktab, bf.zblk_fmt}
 }
 
 // PySetState implements zodb.PyStateful.
 func (bf *zBigFileState) PySetState(pystate interface{}) (err error) {
 	t, ok := pystate.(pickle.Tuple)
 	if !ok {
-		return fmt.Errorf("expect [2](); got %s", xzodb.TypeOf(pystate))
+		return fmt.Errorf("expect [2|3](); got %s", xzodb.TypeOf(pystate))
 	}
-	if len(t) != 2 {
-		return fmt.Errorf("expect [2](); got [%d]()", len(t))
+	// BBB: we either accept data before adding zblk_fmt to state
+	// (lent==2) or data after adding zblk_fmt to state (lent==3).
+	lent := len(t)
+	if lent != 2 && lent != 3 {
+		return fmt.Errorf("expect [2|3](); got [%d]()", len(t))
 	}
 
 	blksize, ok := pycompat.Int64(t[0])
@@ -397,6 +402,15 @@ func (bf *zBigFileState) PySetState(pystate interface{}) (err error) {
 
 	bf.blksize = blksize
 	bf.blktab  = blktab
+
+	if lent != 2 {
+		zblk_fmt, err := pycompat.Xstrbytes(t[2])
+		if err != nil {
+			return fmt.Errorf("zblk_fmt: expect str; got %s", xzodb.TypeOf(t[2]))
+		}
+		bf.zblk_fmt = zblk_fmt
+	}
+
 	return nil
 }