Commit 543041a3 authored by Kirill Smelkov's avatar Kirill Smelkov

go/zodb/fs1: Fix access to whiteout

A data record with len(data)=0 and backpointer=0 is considered by
FileStorage/py as "no data":

https://github.com/zopefoundation/ZODB/blob/5.6.0-15-g22d1405d4/src/ZODB/FileStorage/FileStorage.py#L576-L582

Even though currently it is not possible to create such data record via
FileStorage(py).deleteObject (becase it raises POSKeyError if there is
no previous object revision), being able to use such data records is
semantically useful in overlayed DemoStorage settings, where δ part
marks an object that exists only in base with delete record whiteout.

It is also generally meaningfull to be able to create "delete" record
even if object was not previously existing: "deleteObject" is actually
similar to "store" (and so should be better named as "storeDelete"). If
one wants to store deletion, there should not be a reason to reject it,
because deleteObject already has seatbelt in the form of oldserial, and
if the user calls deleteObject(oid, oldserial=z64), he/she is already
telling that "I know that this object does not exist in this storage
(oldserial=z64), but still please create a deletion record for it". Once
again this is useful in overlayed DemoStorage settings described above.

For the reference, such whiteout deletion records pass ZODB/scripts/fstest just fine.

Even though FileStorage/py loads such data records just fine, on
FileStorage/go side it was not the case - DataHeader.LoadBackRef, even
with backpointer=0, was verifying that backpointer to be valid and
failing seeing it might overlap with current transaction:

    === RUN   TestLoadWhiteout
    2021/03/17 06:40:58 index load: open testdata/whiteout.fs.index: no such file or directory
    2021/03/17 06:40:58 testdata/whiteout.fs: index rebuild...
        filestorage_test.go:398: load 0000000000000017:0000000000000001: bad err:
            have: testdata/whiteout.fs: load 0000000000000017:0000000000000001: testdata/whiteout.fs: data record @27: check: backpointer (0) overlaps with txn (4)
            want: testdata/whiteout.fs: load 0000000000000017:0000000000000001: 0000000000000001: object was not yet created

It was a thinko: backPos==0 was already kind of handled in LoadBackRef,
but only in one verification case. -> Fix all checks not to trigger when
seeing backPos=0. DataHeader.LoadBack - the caller of LoadBackRef -
already handles returned backPos=0 as "no data".
parent a708663d
// Copyright (C) 2017-2020 Nexedi SA and Contributors. // Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -367,3 +367,37 @@ func TestOpenRecovery(t *testing.T) { ...@@ -367,3 +367,37 @@ func TestOpenRecovery(t *testing.T) {
}) })
} }
} }
// TestLoadWhiteout verifies access to whiteout data record.
//
// Whiteout is data record with deletion when object was not previously there.
// It has both len(data)=0 and backpointer=0.
//
// TODO merge into regular tests on testdata/1.fs when
// FileStorage/py.deleteObject allows to create whiteouts instead of raising
// POSKeyError.
func TestLoadWhiteout(t *testing.T) {
fs, _ := xfsopen(t, "testdata/whiteout.fs")
defer exc.XRun(fs.Close)
xid := zodb.Xid{At: zodb.Tid(0x17), Oid: zodb.Oid(1)}
buf, serial, err := fs.Load(context.Background(), xid)
errOk := &zodb.OpError{
URL: fs.URL(),
Op: "load",
Args: xid,
Err: &zodb.NoDataError{Oid: xid.Oid, DeletedAt: xid.At},
}
if !reflect.DeepEqual(err, errOk) {
t.Errorf("load %s: bad err:\nhave: %v\nwant: %v", xid, err, errOk)
}
if buf != nil {
t.Errorf("load %s: buf != nil", xid)
}
if serial != 0 {
t.Errorf("load %s: bad serial %s ; want 0", xid, serial)
}
}
// Copyright (C) 2017-2020 Nexedi SA and Contributors. // Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -570,7 +570,10 @@ func (dh *DataHeader) LoadBackRef(r io.ReaderAt) (backPos int64, err error) { ...@@ -570,7 +570,10 @@ func (dh *DataHeader) LoadBackRef(r io.ReaderAt) (backPos int64, err error) {
} }
backPos = int64(binary.BigEndian.Uint64(dh.workMem[0:])) backPos = int64(binary.BigEndian.Uint64(dh.workMem[0:]))
if !(backPos == 0 || backPos >= dataValidFrom) { if backPos == 0 {
return 0, nil // deletion
}
if backPos < dataValidFrom {
return 0, checkErr(r, dh, "invalid backpointer: %v", backPos) return 0, checkErr(r, dh, "invalid backpointer: %v", backPos)
} }
if backPos + DataHeaderSize > dh.TxnPos - 8 { if backPos + DataHeaderSize > dh.TxnPos - 8 {
......
...@@ -29,6 +29,10 @@ package fs1tools ...@@ -29,6 +29,10 @@ package fs1tools
//go:generate sh -c "python2 -c 'from ZODB.FileStorage import fsdump; fsdump.main()' ../testdata/empty.fs >testdata/empty.fsdump.ok" //go:generate sh -c "python2 -c 'from ZODB.FileStorage import fsdump; fsdump.main()' ../testdata/empty.fs >testdata/empty.fsdump.ok"
//go:generate sh -c "python2 -c 'from ZODB.FileStorage.fsdump import Dumper; import sys; d = Dumper(sys.argv[1]); d.dump()' ../testdata/empty.fs >testdata/empty.fsdumpv.ok" //go:generate sh -c "python2 -c 'from ZODB.FileStorage.fsdump import Dumper; import sys; d = Dumper(sys.argv[1]); d.dump()' ../testdata/empty.fs >testdata/empty.fsdumpv.ok"
//go:generate sh -c "python2 -m ZODB.scripts.fstail -n 1000000 ../testdata/whiteout.fs >testdata/whiteout.fstail.ok"
//go:generate sh -c "python2 -c 'from ZODB.FileStorage import fsdump; fsdump.main()' ../testdata/whiteout.fs >testdata/whiteout.fsdump.ok"
//go:generate sh -c "python2 -c 'from ZODB.FileStorage.fsdump import Dumper; import sys; d = Dumper(sys.argv[1]); d.dump()' ../testdata/whiteout.fs >testdata/whiteout.fsdumpv.ok"
import ( import (
"bytes" "bytes"
"fmt" "fmt"
...@@ -49,7 +53,7 @@ func loadFile(t *testing.T, path string) string { ...@@ -49,7 +53,7 @@ func loadFile(t *testing.T, path string) string {
} }
func testDump(t *testing.T, dir fs1.IterDir, newd func() Dumper) { func testDump(t *testing.T, dir fs1.IterDir, newd func() Dumper) {
testv := []string{"1", "empty"} testv := []string{"1", "empty", "whiteout"}
for _, tt := range testv { for _, tt := range testv {
t.Run("db=" + tt, func(t *testing.T) { t.Run("db=" + tt, func(t *testing.T) {
d := newd() d := newd()
......
Trans #00000 tid=0000000000000017 time=1900-01-01 00:00:00.000000 offset=27
status=' ' user='' description=''
data #00000 oid=0000000000000001 class=undo or abort of object creation
************************************************************
file identifier: 'FS21'
============================================================
offset: 4
end pos: 77
transaction id: 0000000000000017
trec len: 73
status: ' '
user: ''
description: ''
len(extra): 0
------------------------------------------------------------
offset: 27
oid: 0000000000000001
revid: 0000000000000017
previous record offset: 0
transaction offset: 4
len(data): 0
backpointer: 0
redundant trec len: 73
1900-01-01 00:00:00.000000: hash=e3e0ff6c686f2fa02266e744f6629d592d432061
user='' description='' length=73 offset=4 (+23)
...@@ -21,6 +21,7 @@ ...@@ -21,6 +21,7 @@
"""generate reference fs1 database and index for tests""" """generate reference fs1 database and index for tests"""
from ZODB.FileStorage import FileStorage from ZODB.FileStorage import FileStorage
from ZODB.FileStorage.FileStorage import FILESTORAGE_MAGIC, TxnHeader, DataHeader, TRANS_HDR_LEN
from ZODB import DB from ZODB import DB
from ZODB.Connection import TransactionMetaData from ZODB.Connection import TransactionMetaData
from zodbtools.test.gen_testdata import gen_testdb, precommit, run_with_zodb4py2_compat from zodbtools.test.gen_testdata import gen_testdb, precommit, run_with_zodb4py2_compat
...@@ -174,5 +175,36 @@ def main(): ...@@ -174,5 +175,36 @@ def main():
remove(voted+".lock") remove(voted+".lock")
# prepare file with whiteout (deletion of previously non-existing object)
whiteout = "testdata/whiteout.fs"
# as of 20210317 FileStorage.deleteObject verifies that object exists
# -> prepare magic/transaction/data records manually
with open(whiteout, "wb") as f:
oid = p64(1)
tid = p64(0x17)
# file header
f.write(FILESTORAGE_MAGIC)
tpos = f.tell()
# data record (see FileStorage.deleteObject)
dh = DataHeader(oid, tid, 0, tpos, 0, 0)
drec = dh.asString() + p64(0)
# emit txn header (see FileStorage.tpc_vote)
tlen = TRANS_HDR_LEN + 0 + 0 + 0 + len(drec) # empty u,d,e
th = TxnHeader(tid, tlen, ' ', 0, 0, 0)
th.user = b''
th.descr = b''
th.ext = b''
f.write(th.asString())
# emit data record
f.write(drec)
# emit txn tail
f.write(p64(tlen))
if __name__ == '__main__': if __name__ == '__main__':
main() main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment