go/zodb/fs1: Test FileStorage on all py2/py3 ZODB kinds of data we care about

Previously we were testing FileStorage/go only with data generated by
python and pickle protocol=2. However even on py2 there are more pickle
protocols that are in use, and also there is python3.

-> Modernize py/gen-testdata to use run_with_all_zodb_pickle_kinds
   that was recently added as part of nexedi/zodbtools@f9d36ba7
   and generate test data with both python2 and python3. It is handy to
   use py2py3-venv(*) to prepare python environment to do that.

   Adjust tests on Go side to verify how FileStorage handles all generated zkinds.

py2_pickle1, py2_pickle2 and py2_pickle3 are handled well.
Tests for py3_pickle3 currently fail and so are marked with "xfail".

We will fix tests for py3_pickle3 in follow-up patches.

Old testdata are not yet removed because e.g. fs1tools and zodbdump
tests depend on them. We will remove old fs1 testdata after adjusting
tests in dependent packages step-by-step.

(*) see nexedi/zodbtools@fac2f190

go/zodb/storage/fs1/filestorage_test.go

-// Copyright (C) 2017-2021  Nexedi SA and Contributors.
+// Copyright (C) 2017-2024  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -33,6 +33,17 @@ import (
 	"lab.nexedi.com/kirr/go123/exc"
 )
 
+// ztestdataReg maintains registry of all entries under testdata/ .
+var ztestdataReg = xtesting.ZTestDataRegistry[_TestDataOK]{}
+type ZTestData   = xtesting.ZTestData[_TestDataOK]
+
+// _TestDataOK describes expected data for one entry under testdata/ .
+type _TestDataOK struct {
+	_1fs_indexTopPos int64	      // topPos of 1fs.index
+	_1fs_indexEntryv []indexEntry // index content
+	_1fs_dbEntryv    []dbEntry    // database content
+}
+
 // one database transaction record
 type dbEntry struct {
 	Header TxnHeader
@@ -73,21 +84,33 @@ func xfsopenopt(t testing.TB, path string, opt *zodb.DriverOptions) (*FileStorag
 }
 
 func TestEmptyDB(t *testing.T) {
-	fs, _ := xfsopen(t, "testdata/empty.fs")
+	ztestdataReg.RunWithEach(t, _TestEmptyDB)
+}
+func _TestEmptyDB(t *testing.T, z *ZTestData) {
+	if z.Kind == "py3_pickle3" {
+		t.Skip("xfail")
+	}
+	fs, _ := xfsopen(t, z.Path("empty.fs"))
 	defer exc.XRun(fs.Close)
 
 	xtesting.DrvTestEmptyDB(t, fs)
 }
 
 func TestLoad(t *testing.T) {
-	fs, _ := xfsopen(t, "testdata/1.fs")
+	ztestdataReg.RunWithEach(t, _TestLoad)
+}
+func _TestLoad(t *testing.T, z *ZTestData) {
+	if z.Kind == "py3_pickle3" {
+		t.Skip("xfail")
+	}
+	fs, _ := xfsopen(t, z.Path("1.fs"))
 	defer exc.XRun(fs.Close)
 
 	// NOTE don't use xtesting.LoadDBHistory here - it is itself tested
 	// with the assumption that fs1.Load and fs1.Iterate work correctly.
 	// Use what testdata generator gave use with what to expect.
 	txnv := []xtesting.Txn{}
-	for _, dbe := range _1fs_dbEntryv {
+	for _, dbe := range z.Misc._1fs_dbEntryv {
 		txn := xtesting.Txn{Header: &zodb.TxnInfo{
 			Tid: dbe.Header.Tid,
 
@@ -221,12 +244,20 @@ func testIterate(t *testing.T, fs *FileStorage, tidMin, tidMax zodb.Tid, expectv
 
 // TODO -> xtesting
 func TestIterate(t *testing.T) {
-	fs, _ := xfsopen(t, "testdata/1.fs")
+	ztestdataReg.RunWithEach(t, _TestIterate)
+}
+func _TestIterate(t *testing.T, z *ZTestData) {
+	if z.Kind == "py3_pickle3" {
+		t.Skip("xfail")
+	}
+	zz := z.Misc
+
+	fs, _ := xfsopen(t, z.Path("1.fs"))
 	defer exc.XRun(fs.Close)
 
 	// all []tids in test database
 	tidv := []zodb.Tid{}
-	for _, dbe := range _1fs_dbEntryv {
+	for _, dbe := range zz._1fs_dbEntryv {
 		tidv = append(tidv, dbe.Header.Tid)
 	}
 
@@ -248,17 +279,23 @@ func TestIterate(t *testing.T) {
 			}
 
 			//fmt.Printf("%d%+d .. %d%+d\t -> %d steps\n", i, ii-1, j, jj-1, nsteps)
-			testIterate(t, fs, tmin, tmax, _1fs_dbEntryv[i + ii/2:][:nsteps])
+			testIterate(t, fs, tmin, tmax, zz._1fs_dbEntryv[i + ii/2:][:nsteps])
 		}}
 	}}
 
 	// also check 0..tidMax
-	testIterate(t, fs, 0, zodb.TidMax, _1fs_dbEntryv[:])
+	testIterate(t, fs, 0, zodb.TidMax, zz._1fs_dbEntryv[:])
 }
 
 // TODO -> xtesting
 func BenchmarkIterate(b *testing.B) {
-	fs, _ := xfsopen(b, "testdata/1.fs")
+	ztestdataReg.BenchWithEach(b, _BenchmarkIterate)
+}
+func _BenchmarkIterate(b *testing.B, z *ZTestData) {
+	if z.Kind == "py3_pickle3" {
+		b.Skip("xfail")
+	}
+	fs, _ := xfsopen(b, z.Path("1.fs"))
 	defer exc.XRun(fs.Close)
 
 	ctx := context.Background()
@@ -307,12 +344,19 @@ func TestWatch(t *testing.T) {
 // TestOpenRecovery verifies how Open handles data file with not-finished voted
 // transaction in the end.
 func TestOpenRecovery(t *testing.T) {
+	ztestdataReg.RunWithEach(t, _TestOpenRecovery)
+}
+func _TestOpenRecovery(t *testing.T, z *ZTestData) {
+	if z.Kind == "py3_pickle3" {
+		t.Skip("xfail")
+	}
 	X := exc.Raiseif
-	main, err := ioutil.ReadFile("testdata/1.fs"); X(err)
-	index, err := ioutil.ReadFile("testdata/1.fs.index"); X(err)
-	headOk := _1fs_dbEntryv[len(_1fs_dbEntryv)-1].Header.Tid
-	topPos := int64(_1fs_indexTopPos)
-	voteTail, err := ioutil.ReadFile("testdata/1voted.tail"); X(err)
+	zz := z.Misc
+	main, err := ioutil.ReadFile(z.Path("1.fs")); X(err)
+	index, err := ioutil.ReadFile(z.Path("1.fs.index")); X(err)
+	headOk := zz._1fs_dbEntryv[len(zz._1fs_dbEntryv)-1].Header.Tid
+	topPos := int64(zz._1fs_indexTopPos)
+	voteTail, err := ioutil.ReadFile(z.Path("1voted.tail")); X(err)
 
 	workdir := xworkdir(t)
 	ctx := context.Background()
@@ -381,7 +425,13 @@ func TestOpenRecovery(t *testing.T) {
 // FileStorage/py.deleteObject allows to create whiteouts instead of raising
 // POSKeyError.
 func TestLoadWhiteout(t *testing.T) {
-	fs, _ := xfsopen(t, "testdata/whiteout.fs")
+	ztestdataReg.RunWithEach(t, _TestLoadWhiteout)
+}
+func _TestLoadWhiteout(t *testing.T, z *ZTestData) {
+	if z.Kind == "py3_pickle3" {
+		t.Skip("xfail")
+	}
+	fs, _ := xfsopen(t, z.Path("whiteout.fs"))
 	defer exc.XRun(fs.Close)
 
 	xid := zodb.Xid{At: zodb.Tid(0x17), Oid: zodb.Oid(1)}

go/zodb/storage/fs1/index_test.go

-// Copyright (C) 2017-2019  Nexedi SA and Contributors.
+// Copyright (C) 2017-2024  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -19,7 +19,8 @@
 
 package fs1
 
-//go:generate ./py/gen-testdata
+//go:generate python2 py/gen-testdata
+//go:generate python3 py/gen-testdata
 
 import (
 	"context"
@@ -183,61 +184,80 @@ func TestIndexSaveLoad(t *testing.T) {
 	// {0xb000000000000000, 0x7fffffffffffffff}, // will cause 'entry position too large'
 }
 
-var _1fs_index = func() *Index {
+func (z *_TestDataOK) _1fs_index() *Index {
 	idx := IndexNew()
-	idx.TopPos = _1fs_indexTopPos
-	setIndex(idx, _1fs_indexEntryv[:])
+	idx.TopPos = z._1fs_indexTopPos
+	setIndex(idx, z._1fs_indexEntryv)
 	return idx
-}()
+}
 
 // test that we can correctly load index data as saved by zodb/py
 func TestIndexLoadFromPy(t *testing.T) {
-	fsiPy, err := LoadIndexFile("testdata/1.fs.index")
+	ztestdataReg.RunWithEach(t, _TestIndexLoadFromPy)
+}
+func _TestIndexLoadFromPy(t *testing.T, z *ZTestData) {
+	if z.Kind == "py3_pickle3" {
+		t.Skip("xfail")
+	}
+	fsiPy, err := LoadIndexFile(z.Path("1.fs.index"))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	checkIndexEqual(t, "index load", fsiPy, _1fs_index)
+	checkIndexEqual(t, "index load", fsiPy, z.Misc._1fs_index())
 }
 
 // test zodb/py can read index data as saved by us
 func TestIndexSaveToPy(t *testing.T) {
-	xtesting.NeedPy(t, "ZODB")
-	workdir := xworkdir(t)
+	ztestdataReg.RunWithEach(t, _TestIndexSaveToPy)
+}
+func _TestIndexSaveToPy(t *testing.T, z *ZTestData) {
+	xtesting.WithEachPy(t, func(t *testing.T) {
+		if strings.HasSuffix(t.Name(), "/py3") {
+			t.Skip("xfail")
+		}
+		xtesting.NeedPy(t, "ZODB")
+		workdir := xworkdir(t)
 
-	err := _1fs_index.SaveFile(workdir + "/1.fs.index")
-	if err != nil {
-		t.Fatal(err)
-	}
+		_1fs_index := z.Misc._1fs_index()
 
-	// now ask python part to compare testdata and saved-by-us index
-	cmd := exec.Command("./py/indexcmp", "testdata/1.fs.index", workdir+"/1.fs.index")
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	err = cmd.Run()
-	if err != nil {
-		t.Fatalf("zodb/py read/compare index: %v", err)
-	}
+		err := _1fs_index.SaveFile(workdir + "/1.fs.index")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// now ask python part to compare testdata and saved-by-us index
+		cmd := exec.Command("./py/indexcmp", z.Path("1.fs.index"), workdir+"/1.fs.index")
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		err = cmd.Run()
+		if err != nil {
+			t.Fatalf("zodb/py read/compare index: %v", err)
+		}
+	})
 }
 
 func TestIndexBuildVerify(t *testing.T) {
-	index, err := BuildIndexForFile(context.Background(), "testdata/1.fs", nil)
+	ztestdataReg.RunWithEach(t, _TestIndexBuildVerify)
+}
+func _TestIndexBuildVerify(t *testing.T, z *ZTestData) {
+	index, err := BuildIndexForFile(context.Background(), z.Path("1.fs"), nil)
 	if err != nil {
 		t.Fatalf("index build: %v", err)
 	}
 
-	if !index.Equal(_1fs_index) {
+	if !index.Equal(z.Misc._1fs_index()) {
 		t.Fatal("computed index differ from expected")
 	}
 
-	_, err = index.VerifyForFile(context.Background(), "testdata/1.fs", -1, nil)
+	_, err = index.VerifyForFile(context.Background(), z.Path("1.fs"), -1, nil)
 	if err != nil {
 		t.Fatalf("index verify: %v", err)
 	}
 
 	pos0, _ := index.Get(0)
 	index.Set(0, pos0+1)
-	_, err = index.VerifyForFile(context.Background(), "testdata/1.fs", -1, nil)
+	_, err = index.VerifyForFile(context.Background(), z.Path("1.fs"), -1, nil)
 	if err == nil {
 		t.Fatalf("index verify: expected error after tweak")
 	}
@@ -245,9 +265,15 @@ func TestIndexBuildVerify(t *testing.T) {
 
 
 func BenchmarkIndexLoad(b *testing.B) {
+	ztestdataReg.BenchWithEach(b, _BenchmarkIndexLoad)
+}
+func _BenchmarkIndexLoad(b *testing.B, z *ZTestData) {
+	if z.Kind == "py3_pickle3" {
+		b.Skip("xfail")
+	}
 	// FIXME small testdata/1.fs is not representative for benchmarks
 	for i := 0; i < b.N; i++ {
-		_, err := LoadIndexFile("testdata/1.fs.index")
+		_, err := LoadIndexFile(z.Path("1.fs.index"))
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -255,8 +281,14 @@ func BenchmarkIndexLoad(b *testing.B) {
 }
 
 func BenchmarkIndexSave(b *testing.B) {
+	ztestdataReg.BenchWithEach(b, _BenchmarkIndexSave)
+}
+func _BenchmarkIndexSave(b *testing.B, z *ZTestData) {
+	if z.Kind == "py3_pickle3" {
+		b.Skip("xfail")
+	}
 	// FIXME small testdata/1.fs is not representative for benchmarks
-	index, err := LoadIndexFile("testdata/1.fs.index")
+	index, err := LoadIndexFile(z.Path("1.fs.index"))
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -273,8 +305,14 @@ func BenchmarkIndexSave(b *testing.B) {
 }
 
 func BenchmarkIndexGet(b *testing.B) {
+	ztestdataReg.BenchWithEach(b, _BenchmarkIndexGet)
+}
+func _BenchmarkIndexGet(b *testing.B, z *ZTestData) {
+	if z.Kind == "py3_pickle3" {
+		b.Skip("xfail")
+	}
 	// FIXME small testdata/1.fs is not representative for benchmarks
-	fsi, err := LoadIndexFile("testdata/1.fs.index")
+	fsi, err := LoadIndexFile(z.Path("1.fs.index"))
 	if err != nil {
 		b.Fatal(err)
 	}

go/zodb/storage/fs1/py/gen-testdata

-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2017-2021  Nexedi SA and Contributors.
+# Copyright (C) 2017-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -20,13 +20,16 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 """generate reference fs1 database and index for tests"""
 
+from __future__ import print_function
+
 from ZODB.FileStorage import FileStorage
 from ZODB.FileStorage.FileStorage import FILESTORAGE_MAGIC, TxnHeader, DataHeader, TRANS_HDR_LEN
 from ZODB import DB
 from ZODB.Connection import TransactionMetaData
-from zodbtools.test.gen_testdata import gen_testdb, precommit, run_with_zodb4py2_compat
-from os import stat, remove
-from shutil import copyfile
+from zodbtools.test.gen_testdata import gen_testdb, precommit, run_with_all_zodb_pickle_kinds, current_zkind
+from os import stat, remove, makedirs
+from os.path import exists, relpath
+from shutil import copyfile, rmtree
 from golang.gcompat import qq
 
 import struct
@@ -41,29 +44,44 @@ def unpack64(packed):
 def hex64(packed):
     return '0x%016x' % unpack64(packed)
 
+# bchar returns bytes corresponding to bytes_data[i].
+# it works as bytes_data[i] on py2, but on py3, contrary to builtin behaviour, also returns bytes instead of int.
+def bchar(bytes_data, i): # -> bytes
+    if i >= len(bytes_data):
+        raise IndexError('index out of range')
+    return bytes_data[i:i+1]
 
-def main():
-    outfs = "testdata/1.fs"
+
+def main2():
+    zkind = current_zkind()
+    prefix = "testdata/%s" % zkind
+    if exists(prefix):
+        rmtree(prefix)
+    makedirs(prefix)
+
+    outfs = "%s/1.fs" % prefix
     gen_testdb(outfs)
 
     # dump to go what to expect
     stor = FileStorage(outfs, read_only=True)
-    with open("ztestdata_expect_test.go", "w") as f:
+    with open("ztestdata_expect_%s_test.go" % zkind, "w") as f:
         def emit(v):
-            print >>f, v
-        emit("// Code generated by %s; DO NOT EDIT." % __file__)
+            print(v, file=f)
+        emit("// Code generated by %s; DO NOT EDIT." % relpath(__file__))
         emit("package fs1\n")
         emit("import \"lab.nexedi.com/kirr/neo/go/zodb\"\n")
 
+        emit("func init() {")
+
         # index
-        emit("const _1fs_indexTopPos = %i" % stor._pos)
-        emit("var _1fs_indexEntryv = [...]indexEntry{")
+        emit("\tconst _1fs_indexTopPos = %i" % stor._pos)
+        emit("\tvar _1fs_indexEntryv = []indexEntry{")
         for k, v in stor._index.iteritems():
-            emit("\t{%8i, %8i}," % (unpack64(k), v))
-        emit("}")
+            emit("\t\t{%8i, %8i}," % (unpack64(k), v))
+        emit("\t}")
 
         # database records
-        emit("\nvar _1fs_dbEntryv = [...]dbEntry{")
+        emit("\n\tvar _1fs_dbEntryv = []dbEntry{")
         txnLenPrev = 0
         for txn in stor.iterator():     # txn is TransactionRecord
             # txn.extension is already depickled dict - we want to put raw data from file
@@ -76,26 +94,26 @@ def main():
             # fs1/go keeps in RAM whole txn length, not len-8 as it is on disk
             txnLen = th.tlen + 8
 
-            emit("\t{")
+            emit("\t\t{")
 
             # -> TxnHeader
-            emit("\t\tTxnHeader{")
-            emit("\t\t\tPos:\t  %i," % txn._tpos)
-            emit("\t\t\tLenPrev:  %i," % txnLenPrev)
-            emit("\t\t\tLen:\t  %i," % txnLen)
-            emit("\t\t\tTxnInfo:\tzodb.TxnInfo{")
-            emit("\t\t\t\tTid:\t%s," % hex64(txn.tid))
-            emit("\t\t\t\tStatus:\t'%s'," % txn.status)
-            emit("\t\t\t\tUser:\t\t[]byte(%s)," % qq(txn.user))
-            emit("\t\t\t\tDescription:\t[]byte(%s)," % qq(txn.description))
-            emit("\t\t\t\tExtension:\t[]byte(%s)," % qq(th.ext))
+            emit("\t\t\tTxnHeader{")
+            emit("\t\t\t\tPos:\t  %i," % txn._tpos)
+            emit("\t\t\t\tLenPrev:  %i," % txnLenPrev)
+            emit("\t\t\t\tLen:\t  %i," % txnLen)
+            emit("\t\t\t\tTxnInfo:\tzodb.TxnInfo{")
+            emit("\t\t\t\t\tTid:\t%s," % hex64(txn.tid))
+            emit("\t\t\t\t\tStatus:\t'%s'," % txn.status)
+            emit("\t\t\t\t\tUser:\t\t[]byte(%s)," % qq(txn.user))
+            emit("\t\t\t\t\tDescription:\t[]byte(%s)," % qq(txn.description))
+            emit("\t\t\t\t\tExtension:\t[]byte(%s)," % qq(th.ext))
+            emit("\t\t\t\t},")
             emit("\t\t\t},")
-            emit("\t\t},")
 
             txnLenPrev = txnLen
 
             # -> DataHeader + payload
-            emit("\n\t\t[]txnEntry{")
+            emit("\n\t\t\t[]txnEntry{")
 
             for drec in txn:            # drec is itemof(TransactionRecordIterator) = Record
                 # same as with txn - not everything is possible to get via
@@ -105,17 +123,17 @@ def main():
                 assert dh.tid == drec.tid
                 assert dh.tloc == txn._tpos
 
-                emit("\t\t\t{")
-                emit("\t\t\t\tDataHeader{")
-                emit("\t\t\t\t\tPos:\t%i," % drec.pos)
-                emit("\t\t\t\t\tOid:\t%i," % unpack64(drec.oid))
-                emit("\t\t\t\t\tTid:\t%s," % hex64(drec.tid))
+                emit("\t\t\t\t{")
+                emit("\t\t\t\t\tDataHeader{")
+                emit("\t\t\t\t\t\tPos:\t%i," % drec.pos)
+                emit("\t\t\t\t\t\tOid:\t%i," % unpack64(drec.oid))
+                emit("\t\t\t\t\t\tTid:\t%s," % hex64(drec.tid))
 
-                emit("\t\t\t\t\tPrevRevPos:\t%i," % dh.prev)
-                emit("\t\t\t\t\tTxnPos:\t%i," % txn._tpos)
+                emit("\t\t\t\t\t\tPrevRevPos:\t%i," % dh.prev)
+                emit("\t\t\t\t\t\tTxnPos:\t%i," % txn._tpos)
                 assert drec.version == ''
-                emit("\t\t\t\t\tDataLen:\t%i," % dh.plen)
-                emit("\t\t\t\t},")
+                emit("\t\t\t\t\t\tDataLen:\t%i," % dh.plen)
+                emit("\t\t\t\t\t},")
                 plen = dh.plen
                 if plen == 0:
                     rawdata = p64(dh.back) # back-pointer or 0 (= delete)
@@ -130,19 +148,22 @@ def main():
                     data = "/* same as ^^^ */ sameAsRaw"
                     datatid = "/* no copy */ 0"
 
-                emit("\t\t\t\t[]byte(%s)," % qq(rawdata))
-                emit("\t\t\t\t%s," % data)
-                emit("\t\t\t\t%s," % datatid)
-                emit("\t\t\t},")
+                emit("\t\t\t\t\t[]byte(%s)," % qq(rawdata))
+                emit("\t\t\t\t\t%s," % data)
+                emit("\t\t\t\t\t%s," % datatid)
+                emit("\t\t\t\t},")
 
+            emit("\t\t\t},")
             emit("\t\t},")
-            emit("\t},")
+        emit("\t}")
+
+        emit("\n\tztestdataReg.Register(%s, %s, &_TestDataOK{_1fs_indexTopPos, _1fs_indexEntryv, _1fs_dbEntryv})" % (qq(zkind), qq(prefix)))
         emit("}")
     stor.close()
 
 
     # prepare file with voted (not fully committed) tail
-    voted = "testdata/1voted.fs"
+    voted = "%s/1voted.fs" % prefix
     copyfile(outfs, voted)
     def _():
         vstor = FileStorage(voted)
@@ -154,19 +175,19 @@ def main():
         txn = precommit(u"author", u"description", {'aaa': 'bbb'})
         txn_stormeta = TransactionMetaData(txn.user, txn.description, txn.extension)
         vstor.tpc_begin(txn_stormeta)
-        vstor.store(vroot._p_oid, vroot._p_serial, '000 data 000', '', txn_stormeta)
+        vstor.store(vroot._p_oid, vroot._p_serial, b'000 data 000', '', txn_stormeta)
         vstor.tpc_vote(txn_stormeta)
         # NO tpc_finish here so that status remain 'c' (voted) instead of ' ' (committed)
-    run_with_zodb4py2_compat(_)
+    _()
 
     st = stat(outfs)
     l  = st.st_size
     vf = open(voted, 'rb')
     vf.seek(l)
     voted_tail = vf.read()
-    assert voted_tail[-1+8+8+1] == 'c'  # voted, not finished (' ')
+    assert bchar(voted_tail, -1+8+8+1) == b'c'  # voted, not finished (' ')
 
-    with open("testdata/1voted.tail", "wb") as vt:
+    with open("%s/1voted.tail" % prefix, "wb") as vt:
         vt.write(voted_tail)
 
     remove(voted)
@@ -176,7 +197,7 @@ def main():
 
 
     # prepare file with whiteout (deletion of previously non-existing object)
-    whiteout = "testdata/whiteout.fs"
+    whiteout = "%s/whiteout.fs" % prefix
     # as of 20210317 FileStorage.deleteObject verifies that object exists
     # -> prepare magic/transaction/data records manually
     with open(whiteout, "wb") as f:
@@ -206,5 +227,14 @@ def main():
         f.write(p64(tlen))
 
 
+    # prepare empty.fs
+    empty = "%s/empty.fs" % prefix
+    with open(empty, "wb") as f:
+        f.write(FILESTORAGE_MAGIC)
+
+
+def main():
+    run_with_all_zodb_pickle_kinds(main2)
+
 if __name__ == '__main__':
     main()

go/zodb/storage/fs1/py/indexcmp

-#!/usr/bin/env python2
-# Copyright (C) 2017  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+#!/usr/bin/env python
+# Copyright (C) 2017-2024  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -19,6 +19,8 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 """compare two ZODB FileStorage v1 index files"""
 
+from __future__ import print_function
+
 from ZODB.fsIndex import fsIndex
 import sys
 
@@ -31,9 +33,9 @@ def main():
     topPos1, fsi1 = d1["pos"], d1["index"]
     topPos2, fsi2 = d2["pos"], d2["index"]
 
-    #print topPos1, topPos2
-    #print fsi1.items()
-    #print fsi2.items()
+    #print(topPos1, topPos2)
+    #print(fsi1.items())
+    #print(fsi2.items())
     equal = (topPos1 == topPos2 and fsi1.items() == fsi2.items())
     sys.exit(int(not equal))
 

go/zodb/storage/fs1/testdata/.gitignore

-/*.lock
-/*.tmp
-/*.tr[0-9]
+*.lock
+*.tmp
+*.tr[0-9]
+*.old

go/zodb/storage/fs1/testdata/py2_pickle1/empty.fs

+FS21
\ No newline at end of file

go/zodb/storage/fs1/testdata/py2_pickle2/empty.fs

+FS21
\ No newline at end of file

go/zodb/storage/fs1/testdata/py2_pickle3/empty.fs

+FS21
\ No newline at end of file

go/zodb/storage/fs1/testdata/py3_pickle3/empty.fs

+FS30
\ No newline at end of file