Add support for persistent references (#37)

* Add docstrings to types exposed in public API

I was refreshing my knowled of ogórek and added them along the way.

* Add support for persistent references

Python has the mechanism for objects in one pickle to reference objects
in another pickle. This mechanism is primarily used in ZODB object
database for one object to "persistently" reference another object(*).

I needed a way to load and understand objects from a ZODB database in
Go, and hence comes this patch for ogórek that teaches it to understand
such persistent references.

Like it is already with Call and Class we add a dedicated Ref type to
represent persistent references in pickle stream.

(*) in ZODB when one object is changed, there is no need to resave
unchanged object that is referencing it because its reference stays
valid and pointing to changed object: the "persistent ID" in ZODB is
only object ID and which revision of the referenced object it points to
is implicitly follows by current transaction number view of the database.

encode.go

@@ -7,6 +7,7 @@ import (
 	"math"
 	"math/big"
 	"reflect"
+	"strings"
 )
 
 type TypeError struct {
@@ -299,6 +300,21 @@ func (e *Encoder) encodeClass(v *Class) error {
 	return err
 }
 
+func (e *Encoder) encodeRef(v *Ref) error {
+	if pids, ok := v.Pid.(string); ok && !strings.Contains(pids, "\n") {
+		_, err := fmt.Fprintf(e.w, "%c%s\n", opPersid, pids)
+		return err
+	} else {
+		// XXX we can use opBinpersid only if .protocol >= 1
+		err := e.encode(reflectValueOf(v.Pid))
+		if err != nil {
+			return err
+		}
+		_, err = e.w.Write([]byte{opBinpersid})
+		return err
+	}
+}
+
 func (e *Encoder) encodeStruct(st reflect.Value) error {
 
 	typ := st.Type()
@@ -312,6 +328,8 @@ func (e *Encoder) encodeStruct(st reflect.Value) error {
 		return e.encodeCall(&v)
 	case Class:
 		return e.encodeClass(&v)
+	case Ref:
+		return e.encodeRef(&v)
 	case big.Int:
 		return e.encodeLong(&v)
 	}

ogorek.go

@@ -90,6 +90,7 @@ var ErrInvalidPickleVersion = errors.New("invalid pickle version")
 var errNoMarker = errors.New("no marker in stack")
 var errStackUnderflow = errors.New("pickle: stack underflow")
 
+// OpcodeError is the error that Decode returns when it sees unknown pickle opcode.
 type OpcodeError struct {
 	Key byte
 	Pos int
@@ -468,16 +469,43 @@ func (d *Decoder) loadNone() error {
 	return nil
 }
 
+
+// Ref represents Python's persistent reference.
+//
+// Such references are used when one pickle somehow references another pickle
+// in e.g. a database.
+//
+// See https://docs.python.org/3/library/pickle.html#pickle-persistent for details.
+type Ref struct {
+	// persistent ID of referenced object.
+	//
+	// used to be string for protocol 0, but "upgraded" to be arbitrary
+	// object for later protocols.
+	Pid interface{}
+}
+
 // Push a persistent object id
 func (d *Decoder) loadPersid() error {
-	return errNotImplemented
+	pid, err := d.readLine()
+	if err != nil {
+		return err
+	}
+
+	d.push(Ref{Pid: string(pid)})
+	return nil
 }
 
 // Push a persistent object id from items on the stack
 func (d *Decoder) loadBinPersid() error {
-	return errNotImplemented
+	pid, err := d.pop()
+	if err != nil {
+		return err
+	}
+	d.push(Ref{Pid: pid})
+	return nil
 }
 
+// Call represents Python's call.
 type Call struct {
 	Callable Class
 	Args     Tuple
@@ -645,6 +673,7 @@ func (d *Decoder) build() error {
 	return errNotImplemented
 }
 
+// Class represents a Python class.
 type Class struct {
 	Module, Name string
 }

ogorek_test.go

@@ -62,6 +62,9 @@ var tests = []struct {
 	{"dict with strings", "(dp0\nS'a'\np1\nS'1'\np2\nsS'b'\np3\nS'2'\np4\ns.", map[interface{}]interface{}{"a": "1", "b": "2"}},
 	{"GLOBAL and REDUCE opcodes", "cfoo\nbar\nS'bing'\n\x85R.", Call{Callable: Class{Module: "foo", Name: "bar"}, Args: []interface{}{"bing"}}},
 	{"LONG_BINPUT opcode", "(lr0000I17\na.", []interface{}{int64(17)}},
+	{"persistent ref", "Pabc\n.", Ref{"abc"}},
+	{"bin persistent ref", "\x80\x01U\x05def\ngQ.", Ref{"def\ng"}},
+	{"bin persistent ref to !string", "\x80\x01(I1\nI2\ntQ.", Ref{Tuple{int64(1), int64(2)}}},
 	{"graphite message1", string(graphitePickle1), []interface{}{map[interface{}]interface{}{"values": []interface{}{float64(473), float64(497), float64(540), float64(1497), float64(1808), float64(1890), float64(2013), float64(1821), float64(1847), float64(2176), float64(2156), float64(1250), float64(2055), float64(1570), None{}, None{}}, "start": int64(1383782400), "step": int64(86400), "end": int64(1385164800), "name": "ZZZZ.UUUUUUUU.CCCCCCCC.MMMMMMMM.XXXXXXXXX.TTT"}}},
 	{"graphite message2", string(graphitePickle2), []interface{}{map[interface{}]interface{}{"values": []interface{}{float64(473), float64(497), float64(540), float64(1497), float64(1808), float64(1890), float64(2013), float64(1821), float64(1847), float64(2176), float64(2156), float64(1250), float64(2055), float64(1570), None{}, None{}}, "start": int64(1383782400), "step": int64(86400), "end": int64(1385164800), "name": "user.login.area.machine.metric.minute"}}},
 	{"graphite message3", string(graphitePickle3), []interface{}{map[interface{}]interface{}{"intervals": []interface{}{}, "metric_path": "carbon.agents", "isLeaf": false}, map[interface{}]interface{}{"intervals": []interface{}{}, "metric_path": "carbon.aggregator", "isLeaf": false}, map[interface{}]interface{}{"intervals": []interface{}{}, "metric_path": "carbon.relays", "isLeaf": false}}},