Commit dfd4fb73 authored by Kirill Smelkov's avatar Kirill Smelkov

go/zodb: Minimal serialization compatibility with ZODB/py

ZODB/py serializes data using python pickles. Basically every serialized
object has two parts: class description and object state. Here we
start by providing minimal functionality to extract class-name from
serialized data.

The library used for pickle decoding (and in later patches encoding) is

	github.com/kisielk/og-rek

It was audited by me for security flaws to some extent.

Contrary to Python pickle module it does not run arbitrary code on
decoding.
parent bac6c953
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# Copyright (C) 2017 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""generate reference pickle objects encoding for tests"""
from ZODB.tests import testSerialize
from ZODB import serialize
from zodbtools.util import escapeqq
def main():
# dump to go what to expect
with open("ztestdata_pydata_test.go", "w") as f:
def emit(v):
print >>f, v
emit("// Code generated by %s; DO NOT EDIT." % __file__)
emit("package zodb")
# [] of pickle
t = testSerialize.SerializerTestCase
testv = [
t.old_style_without_newargs,
t.old_style_with_newargs,
t.new_style_without_newargs,
t.new_style_with_newargs,
]
r = serialize.ObjectReader(factory=testSerialize._factory)
emit("\nvar _PyData_ClassName_Testv = [...]_PyDataClassName_TestEntry{")
for test in testv:
emit("\t{")
emit("\t\t%s," % escapeqq(test))
emit("\t\t%s," % escapeqq(r.getClassName(test)))
emit("\t},")
emit('\t{"aaa", "?.?"},') # invalid
emit("}")
if __name__ == '__main__':
main()
// Copyright (C) 2016-2017 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package zodb
// serialization compatibility with ZODB/py
import (
"bytes"
"fmt"
pickle "github.com/kisielk/og-rek"
)
// PyData represents data stored into ZODB by Python applications.
//
// The format is based on python pickles. Basically every serialized object has
// two parts: class description and object state. See
//
// https://github.com/zopefoundation/ZODB/blob/a89485c1/src/ZODB/serialize.py
//
// for format description.
type PyData []byte
// ClassName returns fully-qualified python class name used for object type.
//
// The format is "module.class".
// If pickle decoding fails - "?.?" is returned.
func (d PyData) ClassName() string {
// see ObjectReader.getClassName & get_pickle_metadata in zodb/py
p := pickle.NewDecoder(bytes.NewReader([]byte(d)))
xklass, err := p.Decode()
if err != nil {
return "?.?"
}
if t, ok := xklass.(pickle.Tuple); ok {
if len(t) != 2 { // (klass, args)
return "?.?"
}
xklass = t[0]
if t, ok := xklass.(pickle.Tuple); ok {
// py: "old style reference"
if len(t) != 2 {
return "?.?" // (modname, classname)
}
return fmt.Sprintf("%s.%s", t...)
}
}
if klass, ok := xklass.(pickle.Class); ok {
return klass.Module + "." + klass.Name
}
return "?.?"
}
// Copyright (C) 2016-2017 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package zodb
//go:generate ./py/pydata-gen-testdata
import (
"testing"
)
type _PyDataClassName_TestEntry struct {
pydata string
className string
}
func TestPyClassName(t *testing.T) {
for _, tt := range _PyData_ClassName_Testv {
className := PyData(tt.pydata).ClassName()
if className != tt.className {
t.Errorf("class name for %q:\nhave: %q\nwant: %q",
tt.pydata, className, tt.className)
}
}
}
// Code generated by ./py/pydata-gen-testdata; DO NOT EDIT.
package zodb
var _PyData_ClassName_Testv = [...]_PyDataClassName_TestEntry{
{
"\x80\x02U\x18ZODB.tests.testSerializeq\x01U\x13ClassWithoutNewargsq\x02\x86N\x86q\x03.",
"ZODB.tests.testSerialize.ClassWithoutNewargs",
},
{
"\x80\x02U\x18ZODB.tests.testSerializeq\x01U\x10ClassWithNewargsq\x02\x86K\x01\x85q\x03\x86q\x04.",
"ZODB.tests.testSerialize.ClassWithNewargs",
},
{
"\x80\x02cZODB.tests.testSerialize\nClassWithoutNewargs\nq\x01.",
"ZODB.tests.testSerialize.ClassWithoutNewargs",
},
{
"\x80\x02cZODB.tests.testSerialize\nClassWithNewargs\nq\x01K\x01\x85q\x02\x86q\x03.",
"ZODB.tests.testSerialize.ClassWithNewargs",
},
{"aaa", "?.?"},
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment