go/zodb: Test PyData on all py2/py3 kinds of data we care about

Previously we were testing PyData only with data generated by python2
and pickle protocol=2. However even on py2 there are more pickle
protocols that are in use, and also there is python3.

-> Modernize py/pydata-gen-testdata to use run_with_all_zodb_pickle_kinds
   that was recently added as part of nexedi/zodbtools@f9d36ba7
   and generate test data with both python2 and python3. It is handy to
   use py2py3-venv(*) to prepare python environment to do that.

   Adjust tests on Go side to verify how PyData handles all generated
   zkinds.

py2_pickle1, py2_pickle2 and py2_pickle3 are handled well.
Tests for py3_pickle3 currently fail and so are marked with "xfail".

We will fix tests for py3_pickle3 in the next patch.

(*)  see nexedi/zodbtools@fac2f190

go/zodb/py/pydata-gen-testdata

-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2017-2021  Nexedi SA and Contributors.
+# Copyright (C) 2017-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -20,19 +20,31 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 """generate reference pickle objects encoding for tests"""
 
+from __future__ import print_function
+
 from ZODB import serialize
-from zodbtools.test.gen_testdata import run_with_zodb4py2_compat
+from zodbtools.test.gen_testdata import run_with_all_zodb_pickle_kinds, current_zkind
 from golang.gcompat import qq
+from os.path import relpath
+
+from six import PY3
+if PY3:
+    from importlib import reload
+else:
+    reload  # available as builtin
 
 def main2():
     # import ZODB.tests at runtime after ZODB.X._protocol is patched
+    # reload the module each time because SerializerTestCase uses make_pickle to initialize class-level attributes
     from ZODB.tests import testSerialize
+    reload(testSerialize)
 
     # dump to go what to expect
-    with open("ztestdata_pydata_test.go", "w") as f:
+    zkind = current_zkind()
+    with open("ztestdata_pydata_%s_test.go" % zkind, "w") as f:
         def emit(v):
-            print >>f, v
-        emit("// Code generated by %s; DO NOT EDIT." % __file__)
+            print(v, file=f)
+        emit("// Code generated by %s; DO NOT EDIT." % relpath(__file__))
         emit("package zodb")
 
         # [] of pickle
@@ -46,17 +58,19 @@ def main2():
 
         r = serialize.ObjectReader(factory=testSerialize._factory)
 
-        emit("\nvar _PyData_ClassName_Testv = [...]_PyDataClassName_TestEntry{")
+        emit("func init() {")
+        emit("\t_PyDataClassName_TestDataRegistry[%s] = []_PyDataClassName_TestEntry{" % qq(zkind))
         for test in testv:
-            emit("\t{")
-            emit("\t\t%s," % qq(test))
-            emit("\t\t%s," % qq(r.getClassName(test)))
-            emit("\t},")
-        emit('\t{"aaa", "?.?"},') # invalid
+            emit("\t\t{")
+            emit("\t\t\t%s," % qq(test))
+            emit("\t\t\t%s," % qq(r.getClassName(test)))
+            emit("\t\t},")
+        emit('\t\t{"aaa", "?.?"},') # invalid
+        emit("\t}")
         emit("}")
 
 def main():
-    run_with_zodb4py2_compat(main2)
+    run_with_all_zodb_pickle_kinds(main2)
 
 if __name__ == '__main__':
     main()

go/zodb/pydata_test.go

-// Copyright (C) 2016-2017  Nexedi SA and Contributors.
+// Copyright (C) 2016-2024  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -19,10 +19,13 @@
 
 package zodb
 
-//go:generate ./py/pydata-gen-testdata
+//go:generate python2 py/pydata-gen-testdata
+//go:generate python3 py/pydata-gen-testdata
 
 import (
 	"testing"
+
+	"lab.nexedi.com/kirr/neo/go/internal/xmaps"
 )
 
 type _PyDataClassName_TestEntry struct {
@@ -30,7 +33,20 @@ type _PyDataClassName_TestEntry struct {
 	className string
 }
 
+var _PyDataClassName_TestDataRegistry = map[/*zkind*/string][]_PyDataClassName_TestEntry{}
+
 func TestPyClassName(t *testing.T) {
+	for _, zkind := range xmaps.SortedKeys(_PyDataClassName_TestDataRegistry) {
+		t.Run(zkind, func(t *testing.T) {
+			if zkind == "py3_pickle3" {
+				t.Skip("xfail")
+			}
+			_TestPyClassName(t, _PyDataClassName_TestDataRegistry[zkind])
+		})
+	}
+}
+
+func _TestPyClassName(t *testing.T, _PyData_ClassName_Testv []_PyDataClassName_TestEntry) {
 	for _, tt := range _PyData_ClassName_Testv {
 		className := PyData(tt.pydata).ClassName()
 		if className != tt.className {

go/zodb/ztestdata_pydata_py2_pickle1_test.go

+// Code generated by py/pydata-gen-testdata; DO NOT EDIT.
+package zodb
+func init() {
+	_PyDataClassName_TestDataRegistry["py2_pickle1"] = []_PyDataClassName_TestEntry{
+		{
+			"((U\x18ZODB.tests.testSerializeq\x01U\x13ClassWithoutNewargsq\x02tNtq\x03.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"((U\x18ZODB.tests.testSerializeq\x01U\x10ClassWithNewargsq\x02t(K\x01tq\x03tq\x04.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{
+			"cZODB.tests.testSerialize\nClassWithoutNewargs\nq\x01.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"(cZODB.tests.testSerialize\nClassWithNewargs\nq\x01(K\x01tq\x02tq\x03.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{"aaa", "?.?"},
+	}
+}

go/zodb/ztestdata_pydata_py2_pickle2_test.go

+// Code generated by py/pydata-gen-testdata; DO NOT EDIT.
+package zodb
+func init() {
+	_PyDataClassName_TestDataRegistry["py2_pickle2"] = []_PyDataClassName_TestEntry{
+		{
+			"\x80\x02U\x18ZODB.tests.testSerializeq\x01U\x13ClassWithoutNewargsq\x02\x86N\x86q\x03.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"\x80\x02U\x18ZODB.tests.testSerializeq\x01U\x10ClassWithNewargsq\x02\x86K\x01\x85q\x03\x86q\x04.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{
+			"\x80\x02cZODB.tests.testSerialize\nClassWithoutNewargs\nq\x01.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"\x80\x02cZODB.tests.testSerialize\nClassWithNewargs\nq\x01K\x01\x85q\x02\x86q\x03.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{"aaa", "?.?"},
+	}
+}

go/zodb/ztestdata_pydata_py2_pickle3_test.go

+// Code generated by py/pydata-gen-testdata; DO NOT EDIT.
+package zodb
+func init() {
+	_PyDataClassName_TestDataRegistry["py2_pickle3"] = []_PyDataClassName_TestEntry{
+		{
+			"\x80\x03U\x18ZODB.tests.testSerializeq\x01U\x13ClassWithoutNewargsq\x02\x86N\x86q\x03.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"\x80\x03U\x18ZODB.tests.testSerializeq\x01U\x10ClassWithNewargsq\x02\x86K\x01\x85q\x03\x86q\x04.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{
+			"\x80\x03cZODB.tests.testSerialize\nClassWithoutNewargs\nq\x01.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"\x80\x03cZODB.tests.testSerialize\nClassWithNewargs\nq\x01K\x01\x85q\x02\x86q\x03.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{"aaa", "?.?"},
+	}
+}

go/zodb/ztestdata_pydata_py3_pickle3_test.go

+// Code generated by py/pydata-gen-testdata; DO NOT EDIT.
+package zodb
+func init() {
+	_PyDataClassName_TestDataRegistry["py3_pickle3"] = []_PyDataClassName_TestEntry{
+		{
+			"\x80\x03X\x18\x00\x00\x00ZODB.tests.testSerializeq\x00X\x13\x00\x00\x00ClassWithoutNewargsq\x01\x86q\x02N\x86q\x03.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"\x80\x03X\x18\x00\x00\x00ZODB.tests.testSerializeq\x00X\x10\x00\x00\x00ClassWithNewargsq\x01\x86q\x02K\x01\x85q\x03\x86q\x04.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{
+			"\x80\x03cZODB.tests.testSerialize\nClassWithoutNewargs\nq\x00.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"\x80\x03cZODB.tests.testSerialize\nClassWithNewargs\nq\x00K\x01\x85q\x01\x86q\x02.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{"aaa", "?.?"},
+	}
+}

go/zodb/ztestdata_pydata_test.go

-// Code generated by ./py/pydata-gen-testdata; DO NOT EDIT.
-package zodb
-
-var _PyData_ClassName_Testv = [...]_PyDataClassName_TestEntry{
-	{
-		"\x80\x02U\x18ZODB.tests.testSerializeq\x01U\x13ClassWithoutNewargsq\x02\x86N\x86q\x03.",
-		"ZODB.tests.testSerialize.ClassWithoutNewargs",
-	},
-	{
-		"\x80\x02U\x18ZODB.tests.testSerializeq\x01U\x10ClassWithNewargsq\x02\x86K\x01\x85q\x03\x86q\x04.",
-		"ZODB.tests.testSerialize.ClassWithNewargs",
-	},
-	{
-		"\x80\x02cZODB.tests.testSerialize\nClassWithoutNewargs\nq\x01.",
-		"ZODB.tests.testSerialize.ClassWithoutNewargs",
-	},
-	{
-		"\x80\x02cZODB.tests.testSerialize\nClassWithNewargs\nq\x01K\x01\x85q\x02\x86q\x03.",
-		"ZODB.tests.testSerialize.ClassWithNewargs",
-	},
-	{"aaa", "?.?"},
-}