go/neo/proto: MessagePack support for messages (draft)

This patch adds proto.Encoding('M') and teaches it to encode/decode
messages via MessagePack by the rules defined in

	nexedi/neoppod@9d0bf97a
	( nexedi/neoppod!11 )

It only adds support for messages serialization, without changing
proto.go to match changes in e.g. enums reordering, and without adding
support for MessagePack at link layer in neonet.

M-encoding was only tested for NEO/go-NEO/go, and was not yet tested for
NEO/go-NEO/py interoperation. There will be likely small mistakes
discovered on my side that should be hopefully easy to fix step by step
once we get to that phase.

go/neo/proto/msgpack.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package proto
+// runtime glue for msgpack support
+
+import (
+	"fmt"
+
+	"github.com/tinylib/msgp/msgp"
+	"lab.nexedi.com/kirr/neo/go/neo/internal/msgpack"
+)
+
+
+// mstructDecodeError represents decode error when decoder was expecting
+// tuple<nfield> for structure named path.
+type mstructDecodeError struct {
+	path string     // "Type.field.field"
+	op   msgpack.Op // op we got
+	opOk msgpack.Op // op expected
+}
+
+func (e *mstructDecodeError) Error() string {
+	return fmt.Sprintf("decode: M: struct %s: got opcode %02x; expect %02x", e.path, e.op, e.opOk)
+}
+
+
+// mdecodeErr is called to normilize error when msgp.ReadXXX returns err when decoding path.
+func mdecodeErr(path string, err error) error {
+	if err == msgp.ErrShortBytes {
+		return ErrDecodeOverflow
+	}
+	return &mdecodeError{path, err}
+}
+type mdecodeError struct {
+	path string // "Type.field.field"
+	err  error
+}
+func (e *mdecodeError) Error() string {
+	return fmt.Sprintf("decode: M: %s: %s", e.path, e.err)
+}
+
+// mOpError represents decode error when decoder faces unexpected operation.
+type mOpError struct {
+	op        msgpack.Op   // op we got
+	expectedv []msgpack.Op // expected was any op from expectedv
+}
+func (e *mOpError) Error() string {
+	return fmt.Sprintf("got opcode %02x;  expected any from %02x", e.op, e.expectedv)
+}
+func mdecodeOpErr(path string, op msgpack.Op, expectedv ...msgpack.Op) error {
+	return mdecodeErr(path+"/op", &mOpError{op, expectedv})
+}
+
+// mLen8Error represents decode error when decoder faces unexpected length in Bin8.
+type mLen8Error struct {
+	l, lOk byte // len we got and expected
+}
+func (e *mLen8Error) Error() string {
+	return fmt.Sprintf("expected length %d;  got %d", e.lOk, e.l)
+}
+func mdecodeLen8Err(path string, l, lOk uint8) error {
+	return mdecodeErr(path+"/len", &mLen8Error{l, lOk})
+}
+
+
+func mdecodeEnumTypeErr(path string, enumType, enumTypeOk byte) error {
+	return mdecodeErr(path+"/enumType",
+		fmt.Errorf("expected %d;  got %d", enumTypeOk, enumType))
+}
+
+func mdecodeEnumValueErr(path string, v byte) error {
+	return mdecodeErr(path, fmt.Errorf("invalid enum payload %02x", v))
+}

go/neo/proto/proto.go

@@ -129,6 +129,10 @@ type Msg interface {
 	neoMsgEncodeN(buf []byte)
 	neoMsgDecodeN(data []byte) (nread int, err error)
 
+	// M encoding (via MessagePack)
+	neoMsgEncodedLenM() int
+	neoMsgEncodeM(buf []byte)
+	neoMsgDecodeM(data []byte) (nread int, err error)
 }
 
 // Encoding represents messages encoding.
@@ -139,6 +143,7 @@ func (e Encoding) MsgEncodedLen(msg Msg) int {
 	switch e {
 	default:   panic("bug")
 	case 'N':  return msg.neoMsgEncodedLenN()
+	case 'M':  return msg.neoMsgEncodedLenM()
 	}
 }
 
@@ -149,6 +154,7 @@ func (e Encoding) MsgEncode(msg Msg, buf []byte) {
 	switch e {
 	default:   panic("bug")
 	case 'N':  msg.neoMsgEncodeN(buf)
+	case 'M':  msg.neoMsgEncodeM(buf)
 	}
 }
 
@@ -157,6 +163,7 @@ func (e Encoding) MsgDecode(msg Msg, data []byte) (nread int, err error) {
 	switch e {
 	default:   panic("bug")
 	case 'N':  return msg.neoMsgDecodeN(data)
+	case 'M':  return msg.neoMsgDecodeM(data)
 	}
 }
 
@@ -166,6 +173,7 @@ var ErrDecodeOverflow = errors.New("decode: buffer overflow")
 
 // ---- messages ----
 
+//neo:proto enum
 type ErrorCode uint32
 const (
 	ACK ErrorCode = iota
@@ -183,6 +191,7 @@ const (
 	INCOMPLETE_TRANSACTION
 )
 
+//neo:proto enum
 type ClusterState int8
 const (
 	// The cluster is initially in the RECOVERING state, and it goes back to
@@ -216,6 +225,7 @@ const (
 	STOPPING_BACKUP
 )
 
+//neo:proto enum
 type NodeType int8
 const (
 	MASTER NodeType = iota
@@ -224,6 +234,7 @@ const (
 	ADMIN
 )
 
+//neo:proto enum
 type NodeState int8
 const (
 	UNKNOWN NodeState = iota //short: U     // XXX tag prefix name ?
@@ -232,6 +243,7 @@ const (
 	PENDING                  //short: P
 )
 
+//neo:proto enum
 type CellState int8
 const (
 	// Write-only cell. Last transactions are missing because storage is/was down

go/neo/proto/proto_test.go

@@ -164,12 +164,19 @@ func TestMsgMarshal(t *testing.T) {
 	var testv = []struct {
 		msg      Msg
 		encodedN string // []byte
+		encodedM string // []byte
 	}{
 		// empty
-		{&Ping{}, ""},
+		{&Ping{},
+			"",
+			"\x90",
+		},
 
-		// uint32, string
-		{&Error{Code: 0x01020304, Message: "hello"}, "\x01\x02\x03\x04\x00\x00\x00\x05hello"},
+		// uint32(N)/enum(M), string
+		{&Error{Code: 0x00000045, Message: "hello"},
+			"\x00\x00\x00\x45\x00\x00\x00\x05hello",
+			hex("92") + hex("d40045") + "\xc4\x05hello",
+		},
 
 		// Oid, Tid, bool, Checksum, []byte
 		{&StoreObject{
@@ -185,7 +192,18 @@ func TestMsgMarshal(t *testing.T) {
 			hex("01020304050607080a0b0c0d0e0f010200") +
 			hex("0102030405060708090a0b0c0d0e0f1011121314") +
 			hex("0000000b") + "hello world" +
-			hex("0a0b0c0d0e0f01030a0b0c0d0e0f0104")},
+			hex("0a0b0c0d0e0f01030a0b0c0d0e0f0104"),
+
+			// M
+			hex("97") +
+			hex("c408") + hex("0102030405060708") +
+			hex("c408") + hex("0a0b0c0d0e0f0102") +
+			hex("c2") +
+			hex("c414") + hex("0102030405060708090a0b0c0d0e0f1011121314") +
+			hex("c40b") + "hello world" +
+			hex("c408") + hex("0a0b0c0d0e0f0103") +
+			hex("c408") + hex("0a0b0c0d0e0f0104"),
+		},
 
 		// PTid, [] (of [] of {UUID, CellState})
 		{&AnswerPartitionTable{
@@ -205,6 +223,15 @@ func TestMsgMarshal(t *testing.T) {
 				hex("000000020000000b010000001100") +
 				hex("000000010000000b02") +
 				hex("000000030000000b030000000f040000001701"),
+
+			// M
+			hex("93") +
+			hex("cf0102030405060708") +
+			hex("22") +
+			hex("93") +
+				hex("91"+"92"+"920bd40401"+"9211d40400") +
+				hex("91"+"91"+"920bd40402") +
+				hex("91"+"93"+"920bd40403"+"920fd40404"+"9217d40401"),
 		},
 
 		// map[Oid]struct {Tid,Tid,bool}
@@ -226,6 +253,14 @@ func TestMsgMarshal(t *testing.T) {
 				u64(2) + u64(7) + u64(1) + hex("01") +
 				u64(5) + u64(4) + u64(3) + hex("01") +
 				u64(8) + u64(7) + u64(1) + hex("00"),
+
+			// M
+			hex("91") +
+			hex("84") +
+				hex("c408")+u64(1) + hex("93") + hex("c408")+u64(1) + hex("c408")+u64(0) + hex("c2") +
+				hex("c408")+u64(2) + hex("93") + hex("c408")+u64(7) + hex("c408")+u64(1) + hex("c3") +
+				hex("c408")+u64(5) + hex("93") + hex("c408")+u64(4) + hex("c408")+u64(3) + hex("c3") +
+				hex("c408")+u64(8) + hex("93") + hex("c408")+u64(7) + hex("c408")+u64(1) + hex("c2"),
 		},
 
 		// map[uint32]UUID + trailing ...
@@ -247,12 +282,28 @@ func TestMsgMarshal(t *testing.T) {
 				u32(4) + u32(17) +
 				u32(7) + u32(3) +
 			u64(23) + u64(128),
+
+			// M
+			hex("93") +
+			hex("84") +
+				hex("01" + "07") +
+				hex("02" + "09") +
+				hex("04" + "11") +
+				hex("07" + "03") +
+			hex("c408") + u64(23) +
+			hex("c408") + u64(128),
 		},
 
 		// uint32, []uint32
 		{&PartitionCorrupted{7, []NodeUUID{1, 3, 9, 4}},
 			// N
 			u32(7) + u32(4) + u32(1) + u32(3) + u32(9) + u32(4),
+
+			// M
+			hex("92") +
+			hex("07") +
+			hex("94") +
+				hex("01030904"),
 		},
 
 		// uint32, Address, string, IdTime
@@ -264,6 +315,16 @@ func TestMsgMarshal(t *testing.T) {
 			hex("3fbf9add1091c895") +
 			u32(2) + u32(5)+"room1" + u32(7)+"rack234" +
 			u32(3) + u32(3)+u32(4)+u32(5),
+
+			// M
+			hex("97") +
+			hex("d40202") +
+			hex("11") +
+			hex("92") + hex("c409")+"localhost" + hex("cd")+u16(7777) +
+			hex("c406")+"myname" +
+			hex("cb" + "3fbf9add1091c895") +
+			hex("92") + hex("c405")+"room1" + hex("c407")+"rack234" +
+			hex("93") + hex("030405"),
 		},
 
 		// IdTime, empty Address, int32
@@ -273,12 +334,26 @@ func TestMsgMarshal(t *testing.T) {
 			hex("41d66b15517b469d") + u32(1) +
 				u8(2) + u32(0) /* <- ø Address */ + hex("e0000001") + u8(2) +
 				hex("41d66b15517b3d04"),
+			// M
+			hex("92") +
+			hex("cb" + "41d66b15517b469d") +
+			hex("91") +
+				hex("95") +
+					hex("d40202") +
+					hex("92" + "c400"+"" + "00") +
+					hex("d2" + "e0000001") +
+					hex("d40302") +
+					hex("cb" + "41d66b15517b3d04"),
 		},
 
 		// empty IdTime
 		{&NotifyNodeInformation{IdTimeNone, []NodeInfo{}},
 			// N
 			hex("ffffffffffffffff") + hex("00000000"),
+			// M
+			hex("92") +
+			hex("cb" + "fff0000000000000") + // XXX nan/-inf not handled yet
+			hex("90"),
 		},
 
 		// TODO we need tests for:
@@ -288,6 +363,7 @@ func TestMsgMarshal(t *testing.T) {
 
 	for _, tt := range testv {
 		testMsgMarshal(t, 'N', tt.msg, tt.encodedN)
+		testMsgMarshal(t, 'M', tt.msg, tt.encodedM)
 	}
 }
 
@@ -295,11 +371,14 @@ func TestMsgMarshal(t *testing.T) {
 // this way we additionally lightly check encode / decode overflow behaviour for all types.
 func TestMsgMarshalAllOverflowLightly(t *testing.T) {
 	for _, typ := range msgTypeRegistry {
-		for _, enc := range []Encoding{'N'} {
+		for _, enc := range []Encoding{'N', 'M'} {
 			// zero-value for a type
 			msg := reflect.New(typ).Interface().(Msg)
 			l := enc.MsgEncodedLen(msg)
 			zerol := make([]byte, l)
+			if enc != 'N' { // M-encoding of zero-value is not all zeros
+				enc.MsgEncode(msg, zerol)
+			}
 			// decoding will turn nil slice & map into empty allocated ones.
 			// we need it so that reflect.DeepEqual works for msg encode/decode comparison
 			n, err := enc.MsgDecode(msg, zerol)
@@ -313,6 +392,7 @@ func TestMsgMarshalAllOverflowLightly(t *testing.T) {
 }
 
 // Verify overflow handling on decodeN len checks
+// TODO + M-variants with big len too?
 func TestMsgDecodeLenOverflowN(t *testing.T) {
 	enc := Encoding('N')