Commit f98f54b1 authored by Kirill Smelkov's avatar Kirill Smelkov

encoder/decoder: Teach ogórek about tuple

It is true that tuples and lists are very similar, and when e.g. a
python function accepts tuple as input, and then uses only indexes
access, it can be substituted with list.

However some functions do `isinstance(..., tuple)` etc, and e.g. for a
go program which wants to produce pickle for such python programs there
is currently no way to generate pickle with tuple opcodes.

So to solve this teach ogórek about tuples:

- introduce simple Tuple type which has []interface{} as underlying
- when decoding tuple opcodes produce this Tuple instead of
  []interface{} used previously.
- when encoding encode Tuple with tuple opcodes.

Otherwise Tuple can be seen and behaves like a regular go []interface{}
slice. In fact the only difference is that runtime type of Tuple is
different from runtime type of []interface{} but otherwise both values
are the same and can be casted to each other if/when needed freely.

NOTE opReduce decoder is adjusted to always require tuple, not list,
because with e.g. cPickle:

    "c__main__\nf\n]R." -> TypeError('argument list must be a tuple', ...)
    "c__main__\nf\n)R." -> ok

( the first one uses empty list - "]", and the second one empty tuple - ")" )

So it is ok and compatible to always require args to be tuple for
reduce.
parent 4fd6be93
...@@ -53,6 +53,8 @@ func (e *Encoder) encode(rv reflect.Value) error { ...@@ -53,6 +53,8 @@ func (e *Encoder) encode(rv reflect.Value) error {
case reflect.Array, reflect.Slice: case reflect.Array, reflect.Slice:
if rv.Type().Elem().Kind() == reflect.Uint8 { if rv.Type().Elem().Kind() == reflect.Uint8 {
return e.encodeBytes(rv.Bytes()) return e.encodeBytes(rv.Bytes())
} else if _, ok := rv.Interface().(Tuple); ok {
return e.encodeTuple(rv.Interface().(Tuple))
} else { } else {
return e.encodeArray(rv) return e.encodeArray(rv)
} }
...@@ -92,6 +94,36 @@ func (e *Encoder) encode(rv reflect.Value) error { ...@@ -92,6 +94,36 @@ func (e *Encoder) encode(rv reflect.Value) error {
return nil return nil
} }
func (e *Encoder) encodeTuple(t Tuple) error {
l := len(t)
switch l {
case 0:
_, err := e.w.Write([]byte{opEmptyTuple})
return err
// TODO this are protocol 2 opcodes - check e.protocol before using them
//case 1:
//case 2:
//case 3:
}
_, err := e.w.Write([]byte{opMark})
if err != nil {
return err
}
for i := 0; i < l; i++ {
err = e.encode(reflectValueOf(t[i]))
if err != nil {
return err
}
}
_, err = e.w.Write([]byte{opTuple})
return err
}
func (e *Encoder) encodeArray(arr reflect.Value) error { func (e *Encoder) encodeArray(arr reflect.Value) error {
l := arr.Len() l := arr.Len()
...@@ -250,7 +282,7 @@ func (e *Encoder) encodeCall(v *Call) error { ...@@ -250,7 +282,7 @@ func (e *Encoder) encodeCall(v *Call) error {
if err != nil { if err != nil {
return err return err
} }
err = e.encode(reflectValueOf(v.Args)) err = e.encodeTuple(v.Args)
if err != nil { if err != nil {
return err return err
} }
......
...@@ -104,6 +104,9 @@ type mark struct{} ...@@ -104,6 +104,9 @@ type mark struct{}
// None is a representation of Python's None. // None is a representation of Python's None.
type None struct{} type None struct{}
// Tuple is a representation of Python's tuple.
type Tuple []interface{}
// Decoder is a decoder for pickle streams. // Decoder is a decoder for pickle streams.
type Decoder struct { type Decoder struct {
r *bufio.Reader r *bufio.Reader
...@@ -227,7 +230,7 @@ loop: ...@@ -227,7 +230,7 @@ loop:
case opTuple3: case opTuple3:
err = d.loadTuple3() err = d.loadTuple3()
case opEmptyTuple: case opEmptyTuple:
d.push([]interface{}{}) d.push(Tuple{})
case opSetitems: case opSetitems:
err = d.loadSetItems() err = d.loadSetItems()
case opBinfloat: case opBinfloat:
...@@ -469,7 +472,7 @@ func (d *Decoder) loadBinPersid() error { ...@@ -469,7 +472,7 @@ func (d *Decoder) loadBinPersid() error {
type Call struct { type Call struct {
Callable Class Callable Class
Args []interface{} Args Tuple
} }
func (d *Decoder) reduce() error { func (d *Decoder) reduce() error {
...@@ -478,7 +481,7 @@ func (d *Decoder) reduce() error { ...@@ -478,7 +481,7 @@ func (d *Decoder) reduce() error {
} }
xargs := d.xpop() xargs := d.xpop()
xclass := d.xpop() xclass := d.xpop()
args, ok := xargs.([]interface{}) args, ok := xargs.(Tuple)
if !ok { if !ok {
return fmt.Errorf("pickle: reduce: invalid args: %T", xargs) return fmt.Errorf("pickle: reduce: invalid args: %T", xargs)
} }
...@@ -767,7 +770,7 @@ func (d *Decoder) loadTuple() error { ...@@ -767,7 +770,7 @@ func (d *Decoder) loadTuple() error {
return err return err
} }
v := append([]interface{}{}, d.stack[k+1:]...) v := append(Tuple{}, d.stack[k+1:]...)
d.stack = append(d.stack[:k], v) d.stack = append(d.stack[:k], v)
return nil return nil
} }
...@@ -777,7 +780,7 @@ func (d *Decoder) loadTuple1() error { ...@@ -777,7 +780,7 @@ func (d *Decoder) loadTuple1() error {
return errStackUnderflow return errStackUnderflow
} }
k := len(d.stack) - 1 k := len(d.stack) - 1
v := append([]interface{}{}, d.stack[k:]...) v := append(Tuple{}, d.stack[k:]...)
d.stack = append(d.stack[:k], v) d.stack = append(d.stack[:k], v)
return nil return nil
} }
...@@ -787,7 +790,7 @@ func (d *Decoder) loadTuple2() error { ...@@ -787,7 +790,7 @@ func (d *Decoder) loadTuple2() error {
return errStackUnderflow return errStackUnderflow
} }
k := len(d.stack) - 2 k := len(d.stack) - 2
v := append([]interface{}{}, d.stack[k:]...) v := append(Tuple{}, d.stack[k:]...)
d.stack = append(d.stack[:k], v) d.stack = append(d.stack[:k], v)
return nil return nil
} }
...@@ -797,7 +800,7 @@ func (d *Decoder) loadTuple3() error { ...@@ -797,7 +800,7 @@ func (d *Decoder) loadTuple3() error {
return errStackUnderflow return errStackUnderflow
} }
k := len(d.stack) - 3 k := len(d.stack) - 3
v := append([]interface{}{}, d.stack[k:]...) v := append(Tuple{}, d.stack[k:]...)
d.stack = append(d.stack[:k], v) d.stack = append(d.stack[:k], v)
return nil return nil
} }
......
...@@ -47,13 +47,13 @@ func TestDecode(t *testing.T) { ...@@ -47,13 +47,13 @@ func TestDecode(t *testing.T) {
{"float", "F1.23\n.", float64(1.23)}, {"float", "F1.23\n.", float64(1.23)},
{"long", "L12321231232131231231L\n.", bigInt("12321231232131231231")}, {"long", "L12321231232131231231L\n.", bigInt("12321231232131231231")},
{"None", "N.", None{}}, {"None", "N.", None{}},
{"empty tuple", "(t.", []interface{}{}}, {"empty tuple", "(t.", Tuple{}},
{"tuple of two ints", "(I1\nI2\ntp0\n.", []interface{}{int64(1), int64(2)}}, {"tuple of two ints", "(I1\nI2\ntp0\n.", Tuple{int64(1), int64(2)}},
{"nested tuples", "((I1\nI2\ntp0\n(I3\nI4\ntp1\ntp2\n.", {"nested tuples", "((I1\nI2\ntp0\n(I3\nI4\ntp1\ntp2\n.",
[]interface{}{[]interface{}{int64(1), int64(2)}, []interface{}{int64(3), int64(4)}}}, Tuple{Tuple{int64(1), int64(2)}, Tuple{int64(3), int64(4)}}},
{"tuple with top 1 items from stack", "I0\n\x85.", []interface{}{int64(0)}}, {"tuple with top 1 items from stack", "I0\n\x85.", Tuple{int64(0)}},
{"tuple with top 2 items from stack", "I0\nI1\n\x86.", []interface{}{int64(0), int64(1)}}, {"tuple with top 2 items from stack", "I0\nI1\n\x86.", Tuple{int64(0), int64(1)}},
{"tuple with top 3 items from stack", "I0\nI1\nI2\n\x87.", []interface{}{int64(0), int64(1), int64(2)}}, {"tuple with top 3 items from stack", "I0\nI1\nI2\n\x87.", Tuple{int64(0), int64(1), int64(2)}},
{"empty list", "(lp0\n.", []interface{}{}}, {"empty list", "(lp0\n.", []interface{}{}},
{"list of numbers", "(lp0\nI1\naI2\naI3\naI4\na.", []interface{}{int64(1), int64(2), int64(3), int64(4)}}, {"list of numbers", "(lp0\nI1\naI2\naI3\naI4\na.", []interface{}{int64(1), int64(2), int64(3), int64(4)}},
{"string", "S'abc'\np0\n.", string("abc")}, {"string", "S'abc'\np0\n.", string("abc")},
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment