go/zodb: Minimal serialization compatibility with ZODB/py

ZODB/py serializes data using python pickles. Basically every serialized
object has two parts: class description and object state. Here we
start by providing minimal functionality to extract class-name from
serialized data.

The library used for pickle decoding (and in later patches encoding) is

	github.com/kisielk/og-rek

It was audited by me for security flaws to some extent.

Contrary to Python pickle module it does not run arbitrary code on
decoding.

go/zodb/py/pydata-gen-testdata

+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+# Copyright (C) 2017  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""generate reference pickle objects encoding for tests"""
+
+from ZODB.tests import testSerialize
+from ZODB import serialize
+from zodbtools.util import escapeqq
+
+def main():
+    # dump to go what to expect
+    with open("ztestdata_pydata_test.go", "w") as f:
+        def emit(v):
+            print >>f, v
+        emit("// Code generated by %s; DO NOT EDIT." % __file__)
+        emit("package zodb")
+
+        # [] of pickle
+        t = testSerialize.SerializerTestCase
+        testv = [
+            t.old_style_without_newargs,
+            t.old_style_with_newargs,
+            t.new_style_without_newargs,
+            t.new_style_with_newargs,
+        ]
+
+        r = serialize.ObjectReader(factory=testSerialize._factory)
+
+        emit("\nvar _PyData_ClassName_Testv = [...]_PyDataClassName_TestEntry{")
+        for test in testv:
+            emit("\t{")
+            emit("\t\t%s," % escapeqq(test))
+            emit("\t\t%s," % escapeqq(r.getClassName(test)))
+            emit("\t},")
+        emit('\t{"aaa", "?.?"},') # invalid
+        emit("}")
+
+if __name__ == '__main__':
+    main()

go/zodb/pydata.go

+// Copyright (C) 2016-2017  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package zodb
+// serialization compatibility with ZODB/py
+
+import (
+	"bytes"
+	"fmt"
+
+	pickle "github.com/kisielk/og-rek"
+)
+
+// PyData represents data stored into ZODB by Python applications.
+//
+// The format is based on python pickles. Basically every serialized object has
+// two parts: class description and object state. See
+//
+//	https://github.com/zopefoundation/ZODB/blob/a89485c1/src/ZODB/serialize.py
+//
+// for format description.
+type PyData []byte
+
+// ClassName returns fully-qualified python class name used for object type.
+//
+// The format is "module.class".
+// If pickle decoding fails - "?.?" is returned.
+func (d PyData) ClassName() string {
+	// see ObjectReader.getClassName & get_pickle_metadata in zodb/py
+	p := pickle.NewDecoder(bytes.NewReader([]byte(d)))
+	xklass, err := p.Decode()
+	if err != nil {
+		return "?.?"
+	}
+
+	if t, ok := xklass.(pickle.Tuple); ok {
+		if len(t) != 2 { // (klass, args)
+			return "?.?"
+		}
+		xklass = t[0]
+		if t, ok := xklass.(pickle.Tuple); ok {
+			// py: "old style reference"
+			if len(t) != 2 {
+				return "?.?" // (modname, classname)
+			}
+			return fmt.Sprintf("%s.%s", t...)
+		}
+	}
+
+	if klass, ok := xklass.(pickle.Class); ok {
+		return klass.Module + "." + klass.Name
+	}
+
+	return "?.?"
+}

go/zodb/pydata_test.go

+// Copyright (C) 2016-2017  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package zodb
+
+//go:generate ./py/pydata-gen-testdata
+
+import (
+	"testing"
+)
+
+type _PyDataClassName_TestEntry struct {
+	pydata    string
+	className string
+}
+
+func TestPyClassName(t *testing.T) {
+	for _, tt := range _PyData_ClassName_Testv {
+		className := PyData(tt.pydata).ClassName()
+		if className != tt.className {
+			t.Errorf("class name for %q:\nhave: %q\nwant: %q",
+				tt.pydata, className, tt.className)
+		}
+	}
+}

go/zodb/ztestdata_pydata_test.go

+// Code generated by ./py/pydata-gen-testdata; DO NOT EDIT.
+package zodb
+
+var _PyData_ClassName_Testv = [...]_PyDataClassName_TestEntry{
+	{
+		"\x80\x02U\x18ZODB.tests.testSerializeq\x01U\x13ClassWithoutNewargsq\x02\x86N\x86q\x03.",
+		"ZODB.tests.testSerialize.ClassWithoutNewargs",
+	},
+	{
+		"\x80\x02U\x18ZODB.tests.testSerializeq\x01U\x10ClassWithNewargsq\x02\x86K\x01\x85q\x03\x86q\x04.",
+		"ZODB.tests.testSerialize.ClassWithNewargs",
+	},
+	{
+		"\x80\x02cZODB.tests.testSerialize\nClassWithoutNewargs\nq\x01.",
+		"ZODB.tests.testSerialize.ClassWithoutNewargs",
+	},
+	{
+		"\x80\x02cZODB.tests.testSerialize\nClassWithNewargs\nq\x01K\x01\x85q\x02\x86q\x03.",
+		"ZODB.tests.testSerialize.ClassWithNewargs",
+	},
+	{"aaa", "?.?"},
+}