go/zodb: PyStateful persistency support

As promised in 354e0e51 (go/zodb: Persistent - the base type to
implement IPersistent objects) add support to persistency machinery to
set object state from python pickles serialized by ZODB/py.

Persistent references are not yet handled.

As promised add some very minimal persistent tests.

go/zodb/connection.go

@@ -22,6 +22,7 @@ package zodb
 
 import (
 	"context"
+	"fmt"
 
 	"lab.nexedi.com/kirr/go123/mem"
 )
@@ -32,6 +33,15 @@ type Connection struct {
 	at   Tid                     // current view of database; stable inside a transaction.
 }
 
+// wrongClassError is the error cause returned when ZODB object's class is not what was expected.
+type wrongClassError struct {
+	want, have string
+}
+
+func (e *wrongClassError) Error() string {
+	return fmt.Sprintf("wrong class: want %q; have %q", e.want, e.have)
+}
+
 // load loads object specified by oid.
 func (conn *Connection) load(ctx context.Context, oid Oid) (_ *mem.Buf, serial Tid, _ error) {
 	return conn.stor.Load(ctx, Xid{Oid: oid, At: conn.at})

go/zodb/persistent.go

@@ -34,7 +34,7 @@ import (
 // representation of database objects.
 //
 // To use - a class needs to embed Persistent and register itself additionally
-// providing Ghostable and Stateful methods. For example:
+// providing Ghostable and (Py)Stateful methods. For example:
 //
 //	type MyObject struct {
 //		Persistent
@@ -69,7 +69,7 @@ type Persistent struct {
 	refcnt int32
 
 	// Persistent is the base for the instance.
-	// instance, via its state type, is additionally Ghostable and Stateful.
+	// instance, via its state type, is additionally Ghostable and (Stateful | PyStateful).
 	instance IPersistent
 	loading  *loadState
 }
@@ -178,6 +178,10 @@ func (obj *Persistent) PActivate(ctx context.Context) (err error) {
 			err = istate.SetState(state)
 			xerr.Context(&err, "setstate")
 
+		case PyStateful:
+			err = pySetState(istate, obj.zclass.class, state)
+			xerr.Context(&err, "pysetstate")
+
 		default:
 			panic(obj.badf("activate: !stateful instance"))
 		}
@@ -242,7 +246,7 @@ func (obj *Persistent) PInvalidate() {
 
 // istate returns .instance casted to corresponding stateType.
 //
-// returns: Ghostable + Stateful.
+// returns: Ghostable + (Stateful | PyStateful).
 func (obj *Persistent) istate() Ghostable {
 	xstateful := reflect.ValueOf(obj.instance).Convert(reflect.PtrTo(obj.zclass.stateType))
 	return xstateful.Interface().(Ghostable)
@@ -282,6 +286,7 @@ var rIPersistent = reflect.TypeOf((*IPersistent)(nil)).Elem() // typeof(IPersist
 var rPersistent  = reflect.TypeOf(Persistent{})               // typeof(Persistent)
 var rGhostable   = reflect.TypeOf((*Ghostable)(nil)).Elem()   // typeof(Ghostable)
 var rStateful    = reflect.TypeOf((*Stateful)(nil)).Elem()    // typeof(Stateful)
+var rPyStateful  = reflect.TypeOf((*PyStateful)(nil)).Elem()  // typeof(PyStateful)
 
 // RegisterClass registers ZODB class to correspond to Go type.
 //
@@ -295,7 +300,7 @@ var rStateful    = reflect.TypeOf((*Stateful)(nil)).Elem()    // typeof(Stateful
 // typ must embed Persistent; *typ must implement IPersistent.
 //
 // typ must be convertible to stateType; stateType must implement Ghostable and
-// Stateful(*).
+// either Stateful or PyStateful(*).
 //
 // RegisterClass must be called from global init().
 //
@@ -333,8 +338,9 @@ func RegisterClass(class string, typ, stateType reflect.Type) {
 	}
 
 	stateful := ptstate.Implements(rStateful)
-	if !stateful {
-		badf("%q does not implement Stateful", ptstate)
+	pystateful := ptstate.Implements(rPyStateful)
+	if !(stateful || pystateful) {
+		badf("%q does not implement any of Stateful or PyStateful", ptstate)
 	}
 
 	zc := &zclass{class: class, typ: typ, stateType: stateType}

go/zodb/persistent_test.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package zodb
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"lab.nexedi.com/kirr/go123/mem"
+	"github.com/stretchr/testify/require"
+)
+
+// test Persistent type.
+type MyObject struct {
+	Persistent
+
+	value string
+}
+
+type myObjectState MyObject
+
+func (o *myObjectState) DropState() {
+	o.value = ""
+}
+
+func (o *myObjectState) PySetState(pystate interface{}) error {
+	s, ok := pystate.(string)
+	if !ok {
+		return fmt.Errorf("myobject: setstate: want str; got %T", pystate)
+	}
+
+	o.value = s
+	return nil
+}
+
+func init() {
+	t := reflect.TypeOf
+	RegisterClass("t.zodb.MyObject", t(MyObject{}), t(myObjectState{}))
+}
+
+func TestPersistent(t *testing.T) {
+	assert := require.New(t)
+
+	// checkObj verifies current state of persistent object.
+	checkObj := func(obj IPersistent, jar *Connection, oid Oid, serial Tid, state ObjectState, refcnt int32, loading *loadState) {
+		t.Helper()
+		xbase := reflect.ValueOf(obj).Elem().FieldByName("Persistent")
+		pbase := xbase.Addr().Interface().(*Persistent)
+
+		badf := func(format string, argv ...interface{}) {
+			t.Helper()
+			msg := fmt.Sprintf(format, argv...)
+			t.Fatalf("%#v: %s", obj, msg)
+		}
+
+		zc := pbase.zclass
+		//zc.class
+		if typ := reflect.TypeOf(obj).Elem(); typ != zc.typ {
+			badf("invalid zclass: .typ = %s  ; want %s", zc.typ, typ)
+		}
+		//zc.stateType
+
+		if pbase.jar != jar {
+			badf("invalid jar")
+		}
+		if pbase.oid != oid {
+			badf("invalid oid: %s  ; want %s", pbase.oid, oid)
+		}
+		if pbase.serial != serial {
+			badf("invalid serial: %s  ; want %s", pbase.serial, serial)
+		}
+		if pbase.state != state {
+			badf("invalid state: %s  ; want %s", pbase.state, state)
+		}
+		if pbase.refcnt != refcnt {
+			badf("invalid refcnt: %s  ; want %s", pbase.refcnt, refcnt)
+		}
+		if pbase.instance != obj {
+			badf("base.instance != obj")
+		}
+		// XXX loading too?
+	}
+
+	// unknown type -> Broken
+	xobj := newGhost("t.unknown", 10, nil)
+	b, ok := xobj.(*Broken)
+	if !ok {
+		t.Fatalf("unknown -> %T;  want Broken", xobj)
+	}
+
+	checkObj(b, nil, 10, InvalidTid, GHOST, 0, nil)
+	assert.Equal(b.class, "t.unknown")
+	assert.Equal(b.state, (*mem.Buf)(nil))
+
+
+	// t.zodb.MyObject -> *MyObject
+	xobj = newGhost("t.zodb.MyObject", 11, nil)
+	obj, ok := xobj.(*MyObject)
+	if !ok {
+		t.Fatalf("unknown -> %T;  want Broken", xobj)
+	}
+
+	checkObj(obj, nil, 11, InvalidTid, GHOST, 0, nil)
+
+	assert.Equal(zclassOf(obj), "t.zodb.MyObject")
+
+
+	// TODO activate	- jar has to load, state changes
+	// TODO activate again	- refcnt++
+	// TODO deactivate	- refcnt--
+	// TODO deactivate	- state dropped
+}

go/zodb/pydata.go

@@ -58,6 +58,33 @@ func (d PyData) ClassName() string {
 	return pyclassPath(klass)
 }
 
+// decode decodes raw ZODB python data into Python class and state.
+func (d PyData) decode() (pyclass pickle.Class, pystate interface{}, err error) {
+	defer xerr.Context(&err, "pydata: decode")
+
+	p := pickle.NewDecoderWithConfig(
+		bytes.NewReader([]byte(d)),
+		&pickle.DecoderConfig{/* TODO: handle persistent references */},
+	)
+
+	xklass, err := p.Decode()
+	if err != nil {
+		return pickle.Class{}, nil, fmt.Errorf("class description: %s", err)
+	}
+
+	klass, err := xpyclass(xklass)
+	if err != nil {
+		return pickle.Class{}, nil, fmt.Errorf("class description: %s", err)
+	}
+
+	state, err := p.Decode()
+	if err != nil {
+		return pickle.Class{}, nil, fmt.Errorf("object state: %s", err)
+	}
+
+	return klass, state, nil
+}
+
 // xpyclass verifies and extracts py class from unpickled value.
 //
 // it normalizes py class that has just been decoded from a serialized ZODB

go/zodb/zodb.go

@@ -99,6 +99,19 @@
 // multiple goroutines simultaneously.
 //
 //
+// Python data
+//
+// To maintain database data compatibility with ZODB/py, ZODB/go provides
+// first class support for Python data. At storage-level PyData provides way to
+// treat raw data record content as serialized by ZODB/py, and at application
+// level types that are registered with state type providing PyStateful (see
+// RegisterClass) are automatically (de)serialized as Python pickles(*).
+//
+// --------
+//
+// (*) for pickle support package github.com/kisielk/og-rek is used.
+//
+//
 // Storage drivers
 //
 // To implement a ZODB storage one need to provide IStorageDriver interface and

go/zodb/zodbpy.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package zodb
+// Support for python objects/data in ZODB.
+
+import (
+	"lab.nexedi.com/kirr/go123/mem"
+)
+
+// PyStateful is the interface describing in-RAM object whose data state can be
+// exchanged as Python data.
+type PyStateful interface {
+	// PySetState should set state of the in-RAM object from Python data.
+	//
+	// It is analog of __setstate__() in Python.
+	//
+	// The error returned does not need to have object/setstate prefix -
+	// persistent machinery is adding such prefix automatically.
+	PySetState(pystate interface{}) error
+
+	// PyGetState should return state of the in-RAM object as Python data.
+	// Analog of __getstate__() in Python.
+	//PyGetState() interface{}	TODO
+}
+
+// pySetState decodes raw state as zodb/py serialized stream, and sets decoded
+// state on PyStateful obj.
+//
+// It is an error if decoded state has python class not as specified.
+func pySetState(obj PyStateful, objClass string, state *mem.Buf) error {
+	pyclass, pystate, err := PyData(state.Data).decode()
+	if err != nil {
+		return err
+	}
+
+	class := pyclassPath(pyclass)
+
+	if class != objClass {
+		// complain that pyclass changed
+		// (both ref and object data use pyclass so it indeed can be different)
+		return &wrongClassError{want: objClass, have: class}
+	}
+
+	return obj.PySetState(pystate)
+}
+
+// TODO pyGetState