go/zodb/zodbtools: Test it on all py2/py3 ZODB kinds of data we care about

Like with FileStorage and fs1tools previously we were testing zodbtools/go only
with data generated by python2 and pickle protocol=2. However even on py2 there
are more pickle protocols that are in use, and also there is python3.

-> Modernize testdata generation to automatically pick up all ZODB kinds
   from recently-updated FileStorage/go testdata.

   Adjust tests on Go side to verify how zodbtools/go handle all generated zkinds.

All py2_pickle1, py2_pickle2, py2_pickle3 and py3_pickle3 are handled well out of the box.

go/zodb/zodbtools/dump_test.go

-// Copyright (C) 2016-2019  Nexedi SA and Contributors.
+// Copyright (C) 2016-2024  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -19,17 +19,16 @@
 
 package zodbtools
 
-//go:generate sh -c "python2 -m zodbtools.zodb dump ../../zodb/storage/fs1/testdata/1.fs >testdata/1.zdump.pyok"
-//go:generate sh -c "python2 -m zodbtools.zodb dump ../../zodb/storage/fs1/testdata/empty.fs >testdata/empty.zdump.pyok"
+//go:generate ./gen-testdata
 
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"io/ioutil"
 	"regexp"
 	"testing"
 
+	"lab.nexedi.com/kirr/neo/go/internal/xtesting"
 	"lab.nexedi.com/kirr/neo/go/zodb"
 	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
 
@@ -37,6 +36,11 @@ import (
 	"lab.nexedi.com/kirr/go123/exc"
 )
 
+// ztestdataReg maintains registry of all entries under testdata/ .
+var  ztestdataReg = xtesting.ZTestDataRegistry[struct{}]{}
+type ZTestData    = xtesting.ZTestData[struct{}]
+
+
 // loadZdumpPy loads a zdump file and normalizes escaped strings to the way go
 // would escape them.
 func loadZdumpPy(t *testing.T, path string) string {
@@ -66,8 +70,8 @@ func loadZdumpPy(t *testing.T, path string) string {
 	return string(dump)
 }
 
-func withTestdataFs(t testing.TB, db string, f func(zstor zodb.IStorage)) {
-	zstor, err := zodb.Open(context.Background(), fmt.Sprintf("../../zodb/storage/fs1/testdata/%s.fs", db), &zodb.OpenOptions{ReadOnly: true})
+func withTestdataFs(t testing.TB, z *ZTestData, db string, f func(zstor zodb.IStorage)) {
+	zstor, err := zodb.Open(context.Background(), "../../zodb/storage/fs1/"+z.Path(db+".fs"), &zodb.OpenOptions{ReadOnly: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -78,10 +82,13 @@ func withTestdataFs(t testing.TB, db string, f func(zstor zodb.IStorage)) {
 }
 
 func TestZodbDump(t *testing.T) {
+	ztestdataReg.RunWithEach(t, _TestZodbDump)
+}
+func _TestZodbDump(t *testing.T, z *ZTestData) {
 	testv := []string{"1", "empty"}
 	for _, tt := range testv {
 		t.Run("db=" + tt, func(t *testing.T) {
-			withTestdataFs(t, tt, func(zstor zodb.IStorage) {
+			withTestdataFs(t, z, tt, func(zstor zodb.IStorage) {
 				buf := bytes.Buffer{}
 
 				err := Dump(context.Background(), &buf, zstor, 0, zodb.TidMax, false)
@@ -89,7 +96,7 @@ func TestZodbDump(t *testing.T) {
 					t.Fatal(err)
 				}
 
-				dumpOk := loadZdumpPy(t, fmt.Sprintf("testdata/%s.zdump.pyok", tt))
+				dumpOk := loadZdumpPy(t, z.Path(tt+".zdump.pyok"))
 
 				if dumpOk != buf.String() {
 					t.Errorf("dump different:\n%v", diff.Diff(dumpOk, buf.String()))
@@ -101,8 +108,11 @@ func TestZodbDump(t *testing.T) {
 
 
 func BenchmarkZodbDump(b *testing.B) {
+	ztestdataReg.BenchWithEach(b, _BenchmarkZodbDump)
+}
+func _BenchmarkZodbDump(b *testing.B, z *ZTestData) {
 	// FIXME small testdata/1.fs is not representative for benchmarking
-	withTestdataFs(b, "1", func(zstor zodb.IStorage) {
+	withTestdataFs(b, z, "1", func(zstor zodb.IStorage) {
 		b.ResetTimer()
 
 		for i := 0; i < b.N; i++ {

go/zodb/zodbtools/gen-testdata

+#!/bin/bash -e
+# generate files in testdata/
+#
+# Copyright (C) 2016-2024  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+rm -rf testdata ztestdata_test.go
+
+emit() {
+	echo -e "$@" >>ztestdata_test.go
+}
+
+die() {
+	echo "$@" 1>&2
+	exit 1
+}
+
+emit "// Code generated by gen-testdata; DO NOT EDIT."
+emit "package zodbtools"
+emit "func init() {"
+
+for zin in ../../zodb/storage/fs1/testdata/*/ ; do
+	zkind=`basename $zin`	# py2_pickle3
+
+	# `zodb dump` output should be the same for py2 and py3
+	# unfortunately FileStorage/py rejects to open a database created by
+	# different major version of python
+	py=(${zkind//_/ })	# (py2 pickle3)
+	py=${py[0]}		# py2
+	python=${py/py/python}	# python2
+
+	out=testdata/$zkind
+	mkdir -p "$out"
+
+	emit "\tztestdataReg.Register(\"$zkind\", \"$out\", nil)"
+
+	$python -m zodbtools.zodb dump $zin/1.fs     >$out/1.zdump.pyok
+	$python -m zodbtools.zodb dump $zin/empty.fs >$out/empty.zdump.pyok
+done
+
+emit "}"

go/zodb/zodbtools/ztestdata_test.go

+// Code generated by gen-testdata; DO NOT EDIT.
+package zodbtools
+func init() {
+	ztestdataReg.Register("py2_pickle1", "testdata/py2_pickle1", nil)
+	ztestdataReg.Register("py2_pickle2", "testdata/py2_pickle2", nil)
+	ztestdataReg.Register("py2_pickle3", "testdata/py2_pickle3", nil)
+	ztestdataReg.Register("py3_pickle3", "testdata/py3_pickle3", nil)
+}