go/neo/proto: MessagePack support for messages (draft)

This patch adds proto.Encoding('M') and teaches it to encode/decode
messages via MessagePack by the rules defined in

	nexedi/neoppod@9d0bf97a
	( nexedi/neoppod!11 )

It only adds support for messages serialization, without changing
proto.go to match changes in e.g. enums reordering, and without adding
support for MessagePack at link layer in neonet.

M-encoding was only tested for NEO/go-NEO/go, and was not yet tested for
NEO/go-NEO/py interoperation. There will be likely small mistakes
discovered on my side that should be hopefully easy to fix step by step
once we get to that phase.

go/neo/proto/msgpack.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package proto
+// runtime glue for msgpack support
+
+import (
+	"fmt"
+
+	"github.com/tinylib/msgp/msgp"
+	"lab.nexedi.com/kirr/neo/go/neo/internal/msgpack"
+)
+
+
+// mstructDecodeError represents decode error when decoder was expecting
+// tuple<nfield> for structure named path.
+type mstructDecodeError struct {
+	path string     // "Type.field.field"
+	op   msgpack.Op // op we got
+	opOk msgpack.Op // op expected
+}
+
+func (e *mstructDecodeError) Error() string {
+	return fmt.Sprintf("decode: M: struct %s: got opcode %02x; expect %02x", e.path, e.op, e.opOk)
+}
+
+
+// mdecodeErr is called to normilize error when msgp.ReadXXX returns err when decoding path.
+func mdecodeErr(path string, err error) error {
+	if err == msgp.ErrShortBytes {
+		return ErrDecodeOverflow
+	}
+	return &mdecodeError{path, err}
+}
+type mdecodeError struct {
+	path string // "Type.field.field"
+	err  error
+}
+func (e *mdecodeError) Error() string {
+	return fmt.Sprintf("decode: M: %s: %s", e.path, e.err)
+}
+
+// mOpError represents decode error when decoder faces unexpected operation.
+type mOpError struct {
+	op        msgpack.Op   // op we got
+	expectedv []msgpack.Op // expected was any op from expectedv
+}
+func (e *mOpError) Error() string {
+	return fmt.Sprintf("got opcode %02x;  expected any from %02x", e.op, e.expectedv)
+}
+func mdecodeOpErr(path string, op msgpack.Op, expectedv ...msgpack.Op) error {
+	return mdecodeErr(path+"/op", &mOpError{op, expectedv})
+}
+
+// mLen8Error represents decode error when decoder faces unexpected length in Bin8.
+type mLen8Error struct {
+	l, lOk byte // len we got and expected
+}
+func (e *mLen8Error) Error() string {
+	return fmt.Sprintf("expected length %d;  got %d", e.lOk, e.l)
+}
+func mdecodeLen8Err(path string, l, lOk uint8) error {
+	return mdecodeErr(path+"/len", &mLen8Error{l, lOk})
+}
+
+
+func mdecodeEnumTypeErr(path string, enumType, enumTypeOk byte) error {
+	return mdecodeErr(path+"/enumType",
+		fmt.Errorf("expected %d;  got %d", enumTypeOk, enumType))
+}
+
+func mdecodeEnumValueErr(path string, v byte) error {
+	return mdecodeErr(path, fmt.Errorf("invalid enum payload %02x", v))
+}

go/neo/proto/proto.go

@@ -129,6 +129,10 @@ type Msg interface {
 	neoMsgEncodeN(buf []byte)
 	neoMsgDecodeN(data []byte) (nread int, err error)
 
+	// M encoding (via MessagePack)
+	neoMsgEncodedLenM() int
+	neoMsgEncodeM(buf []byte)
+	neoMsgDecodeM(data []byte) (nread int, err error)
 }
 
 // Encoding represents messages encoding.
@@ -139,6 +143,7 @@ func (e Encoding) MsgEncodedLen(msg Msg) int {
 	switch e {
 	default:   panic("bug")
 	case 'N':  return msg.neoMsgEncodedLenN()
+	case 'M':  return msg.neoMsgEncodedLenM()
 	}
 }
 
@@ -149,6 +154,7 @@ func (e Encoding) MsgEncode(msg Msg, buf []byte) {
 	switch e {
 	default:   panic("bug")
 	case 'N':  msg.neoMsgEncodeN(buf)
+	case 'M':  msg.neoMsgEncodeM(buf)
 	}
 }
 
@@ -157,6 +163,7 @@ func (e Encoding) MsgDecode(msg Msg, data []byte) (nread int, err error) {
 	switch e {
 	default:   panic("bug")
 	case 'N':  return msg.neoMsgDecodeN(data)
+	case 'M':  return msg.neoMsgDecodeM(data)
 	}
 }
 
@@ -166,6 +173,7 @@ var ErrDecodeOverflow = errors.New("decode: buffer overflow")
 
 // ---- messages ----
 
+//neo:proto enum
 type ErrorCode uint32
 const (
 	ACK ErrorCode = iota
@@ -183,6 +191,7 @@ const (
 	INCOMPLETE_TRANSACTION
 )
 
+//neo:proto enum
 type ClusterState int8
 const (
 	// The cluster is initially in the RECOVERING state, and it goes back to
@@ -216,6 +225,7 @@ const (
 	STOPPING_BACKUP
 )
 
+//neo:proto enum
 type NodeType int8
 const (
 	MASTER NodeType = iota
@@ -224,6 +234,7 @@ const (
 	ADMIN
 )
 
+//neo:proto enum
 type NodeState int8
 const (
 	UNKNOWN NodeState = iota //short: U     // XXX tag prefix name ?
@@ -232,6 +243,7 @@ const (
 	PENDING                  //short: P
 )
 
+//neo:proto enum
 type CellState int8
 const (
 	// Write-only cell. Last transactions are missing because storage is/was down

go/neo/proto/proto_test.go

@@ -164,12 +164,19 @@ func TestMsgMarshal(t *testing.T) {
 	var testv = []struct {
 		msg      Msg
 		encodedN string // []byte
+		encodedM string // []byte
 	}{
 		// empty
-		{&Ping{}, ""},
+		{&Ping{},
+			"",
+			"\x90",
+		},
 
-		// uint32, string
-		{&Error{Code: 0x01020304, Message: "hello"}, "\x01\x02\x03\x04\x00\x00\x00\x05hello"},
+		// uint32(N)/enum(M), string
+		{&Error{Code: 0x00000045, Message: "hello"},
+			"\x00\x00\x00\x45\x00\x00\x00\x05hello",
+			hex("92") + hex("d40045") + "\xc4\x05hello",
+		},
 
 		// Oid, Tid, bool, Checksum, []byte
 		{&StoreObject{
@@ -185,7 +192,18 @@ func TestMsgMarshal(t *testing.T) {
 			hex("01020304050607080a0b0c0d0e0f010200") +
 			hex("0102030405060708090a0b0c0d0e0f1011121314") +
 			hex("0000000b") + "hello world" +
-			hex("0a0b0c0d0e0f01030a0b0c0d0e0f0104")},
+			hex("0a0b0c0d0e0f01030a0b0c0d0e0f0104"),
+
+			// M
+			hex("97") +
+			hex("c408") + hex("0102030405060708") +
+			hex("c408") + hex("0a0b0c0d0e0f0102") +
+			hex("c2") +
+			hex("c414") + hex("0102030405060708090a0b0c0d0e0f1011121314") +
+			hex("c40b") + "hello world" +
+			hex("c408") + hex("0a0b0c0d0e0f0103") +
+			hex("c408") + hex("0a0b0c0d0e0f0104"),
+		},
 
 		// PTid, [] (of [] of {UUID, CellState})
 		{&AnswerPartitionTable{
@@ -205,6 +223,15 @@ func TestMsgMarshal(t *testing.T) {
 				hex("000000020000000b010000001100") +
 				hex("000000010000000b02") +
 				hex("000000030000000b030000000f040000001701"),
+
+			// M
+			hex("93") +
+			hex("cf0102030405060708") +
+			hex("22") +
+			hex("93") +
+				hex("91"+"92"+"920bd40401"+"9211d40400") +
+				hex("91"+"91"+"920bd40402") +
+				hex("91"+"93"+"920bd40403"+"920fd40404"+"9217d40401"),
 		},
 
 		// map[Oid]struct {Tid,Tid,bool}
@@ -226,6 +253,14 @@ func TestMsgMarshal(t *testing.T) {
 				u64(2) + u64(7) + u64(1) + hex("01") +
 				u64(5) + u64(4) + u64(3) + hex("01") +
 				u64(8) + u64(7) + u64(1) + hex("00"),
+
+			// M
+			hex("91") +
+			hex("84") +
+				hex("c408")+u64(1) + hex("93") + hex("c408")+u64(1) + hex("c408")+u64(0) + hex("c2") +
+				hex("c408")+u64(2) + hex("93") + hex("c408")+u64(7) + hex("c408")+u64(1) + hex("c3") +
+				hex("c408")+u64(5) + hex("93") + hex("c408")+u64(4) + hex("c408")+u64(3) + hex("c3") +
+				hex("c408")+u64(8) + hex("93") + hex("c408")+u64(7) + hex("c408")+u64(1) + hex("c2"),
 		},
 
 		// map[uint32]UUID + trailing ...
@@ -247,12 +282,28 @@ func TestMsgMarshal(t *testing.T) {
 				u32(4) + u32(17) +
 				u32(7) + u32(3) +
 			u64(23) + u64(128),
+
+			// M
+			hex("93") +
+			hex("84") +
+				hex("01" + "07") +
+				hex("02" + "09") +
+				hex("04" + "11") +
+				hex("07" + "03") +
+			hex("c408") + u64(23) +
+			hex("c408") + u64(128),
 		},
 
 		// uint32, []uint32
 		{&PartitionCorrupted{7, []NodeUUID{1, 3, 9, 4}},
 			// N
 			u32(7) + u32(4) + u32(1) + u32(3) + u32(9) + u32(4),
+
+			// M
+			hex("92") +
+			hex("07") +
+			hex("94") +
+				hex("01030904"),
 		},
 
 		// uint32, Address, string, IdTime
@@ -264,6 +315,16 @@ func TestMsgMarshal(t *testing.T) {
 			hex("3fbf9add1091c895") +
 			u32(2) + u32(5)+"room1" + u32(7)+"rack234" +
 			u32(3) + u32(3)+u32(4)+u32(5),
+
+			// M
+			hex("97") +
+			hex("d40202") +
+			hex("11") +
+			hex("92") + hex("c409")+"localhost" + hex("cd")+u16(7777) +
+			hex("c406")+"myname" +
+			hex("cb" + "3fbf9add1091c895") +
+			hex("92") + hex("c405")+"room1" + hex("c407")+"rack234" +
+			hex("93") + hex("030405"),
 		},
 
 		// IdTime, empty Address, int32
@@ -273,12 +334,26 @@ func TestMsgMarshal(t *testing.T) {
 			hex("41d66b15517b469d") + u32(1) +
 				u8(2) + u32(0) /* <- ø Address */ + hex("e0000001") + u8(2) +
 				hex("41d66b15517b3d04"),
+			// M
+			hex("92") +
+			hex("cb" + "41d66b15517b469d") +
+			hex("91") +
+				hex("95") +
+					hex("d40202") +
+					hex("92" + "c400"+"" + "00") +
+					hex("d2" + "e0000001") +
+					hex("d40302") +
+					hex("cb" + "41d66b15517b3d04"),
 		},
 
 		// empty IdTime
 		{&NotifyNodeInformation{IdTimeNone, []NodeInfo{}},
 			// N
 			hex("ffffffffffffffff") + hex("00000000"),
+			// M
+			hex("92") +
+			hex("cb" + "fff0000000000000") + // XXX nan/-inf not handled yet
+			hex("90"),
 		},
 
 		// TODO we need tests for:
@@ -288,6 +363,7 @@ func TestMsgMarshal(t *testing.T) {
 
 	for _, tt := range testv {
 		testMsgMarshal(t, 'N', tt.msg, tt.encodedN)
+		testMsgMarshal(t, 'M', tt.msg, tt.encodedM)
 	}
 }
 
@@ -295,11 +371,14 @@ func TestMsgMarshal(t *testing.T) {
 // this way we additionally lightly check encode / decode overflow behaviour for all types.
 func TestMsgMarshalAllOverflowLightly(t *testing.T) {
 	for _, typ := range msgTypeRegistry {
-		for _, enc := range []Encoding{'N'} {
+		for _, enc := range []Encoding{'N', 'M'} {
 			// zero-value for a type
 			msg := reflect.New(typ).Interface().(Msg)
 			l := enc.MsgEncodedLen(msg)
 			zerol := make([]byte, l)
+			if enc != 'N' { // M-encoding of zero-value is not all zeros
+				enc.MsgEncode(msg, zerol)
+			}
 			// decoding will turn nil slice & map into empty allocated ones.
 			// we need it so that reflect.DeepEqual works for msg encode/decode comparison
 			n, err := enc.MsgDecode(msg, zerol)
@@ -313,6 +392,7 @@ func TestMsgMarshalAllOverflowLightly(t *testing.T) {
 }
 
 // Verify overflow handling on decodeN len checks
+// TODO + M-variants with big len too?
 func TestMsgDecodeLenOverflowN(t *testing.T) {
 	enc := Encoding('N')
 

go/neo/proto/protogen.go

@@ -27,7 +27,7 @@ For every type the following methods are generated in accordance with neo.Msg in
 
 	neoMsgCode() uint16
 
-	; E stands for 'N' encoding
+	; E stands for 'N' and 'M' encodings
 	neoMsgEncodedLen<E>() int
 	neoMsgEncode<E>(buf []byte)
 	neoMsgDecode<E>(data []byte) (nread int, err error)
@@ -80,6 +80,8 @@ import (
 	"os"
 	"sort"
 	"strings"
+
+	"lab.nexedi.com/kirr/neo/go/neo/internal/msgpack"
 )
 
 // parsed & typechecked input
@@ -126,6 +128,10 @@ var zodbOid types.Type
 var neo_customCodecN *types.Interface // type of neo.customCodecN
 var memBuf           types.Type       // type of mem.Buf
 
+// registry of enums
+var enumRegistry = map[types.Type]int{} // type -> enum type serial
+
+
 // bytes.Buffer + bell & whistles
 type Buffer struct {
 	bytes.Buffer
@@ -188,6 +194,7 @@ func loadPkg(pkgPath string, sources ...string) *types.Package {
 type Annotation struct {
 	typeonly bool
 	answer   bool
+	enum     bool
 }
 
 // parse checks doc for specific comment annotations and, if present, loads them.
@@ -218,6 +225,12 @@ func (a *Annotation) parse(doc *ast.CommentGroup) {
 			}
 			a.answer = true
 
+		case "enum":
+			if a.enum {
+				log.Fatalf("%v: duplicate `enum`", cpos)
+			}
+			a.enum = true
+
 		default:
 			log.Fatalf("%v: unknown neo:proto directive %q", cpos, arg)
 		}
@@ -303,6 +316,9 @@ import (
 	"reflect"
 	"sort"
 
+	"github.com/tinylib/msgp/msgp"
+	"lab.nexedi.com/kirr/neo/go/neo/internal/msgpack"
+
 	"lab.nexedi.com/kirr/go123/mem"
 	"lab.nexedi.com/kirr/neo/go/zodb"
 )`)
@@ -312,6 +328,7 @@ import (
 	// go over message types declaration and generate marshal code for them
 	buf.emit("// messages marshalling\n")
 	msgSerial := 0
+	enumSerial := 0
 	for _, decl := range f.Decls {
 		// we look for types (which can be only under GenDecl)
 		gendecl, ok := decl.(*ast.GenDecl)
@@ -332,16 +349,21 @@ import (
 			typespec := spec.(*ast.TypeSpec) // must be because tok = TYPE
 			typename := typespec.Name.Name
 
-			// we are only interested in struct types
-			if _, ok := typespec.Type.(*ast.StructType); !ok {
-				continue
-			}
-
 			// `//neo:proto ...` annotation for this particular type
 			specAnnotation := declAnnotation // inheriting from decl
 			specAnnotation.parse(typespec.Doc)
 
-			// type only -> don't generate message interface for it
+			// remember enum types
+			if specAnnotation.enum {
+				typ := typeInfo.Defs[typespec.Name].Type()
+				enumRegistry[typ]= enumSerial
+				enumSerial++
+			}
+
+			// messages are only struct types without typeonly annotation
+			if _, ok := typespec.Type.(*ast.StructType); !ok {
+				continue
+			}
 			if specAnnotation.typeonly {
 				continue
 			}
@@ -366,6 +388,11 @@ import (
 			buf.WriteString(generateCodecCode(typespec, &encoderN{}))
 			buf.WriteString(generateCodecCode(typespec, &decoderN{}))
 
+			// TODO keep all M routines separate from N for code locality ?
+			buf.WriteString(generateCodecCode(typespec, &sizerM{}))
+			buf.WriteString(generateCodecCode(typespec, &encoderM{}))
+			buf.WriteString(generateCodecCode(typespec, &decoderM{}))
+
 			msgTypeRegistry[msgCode] = typename
 			msgSerial++
 		}
@@ -431,6 +458,11 @@ func typeSizeFixed(encoding byte, typ types.Type) (wireSize int, ok bool) {
 	var size SymSize
 
 	switch encoding {
+	case 'M':
+		s := &sizerM{}
+		codegenType("x", typ, nil, s)
+		size = s.size
+
 	case 'N':
 		s := &sizerN{}
 		codegenType("x", typ, nil, s)
@@ -493,6 +525,7 @@ type CodeGenCustomize interface {
 
 // X reports encoding=X
 type N struct{};  func (_ *N) encoding() byte { return 'N' }
+type M struct{};  func (_ *M) encoding() byte { return 'M' }
 
 // common part of codegenerators
 type commonCodeGen struct {
@@ -526,6 +559,12 @@ func (c *commonCodeGen) var_(varname string) string {
 	return varname
 }
 
+// pathName returns name representing path or assignto.
+func (c *commonCodeGen) pathName(path string) string {
+	// Type, p.f1.f2 -> Type.f1.f2
+	return strings.Join(append([]string{c.typeName}, strings.Split(path, ".")[1:]...), ".")
+}
+
 // symbolic size
 // consists of numeric & symbolic expression parts
 // size is num + expr1 + expr2 + ...
@@ -631,6 +670,7 @@ type sizerCommon struct {
 	size SymSize // currently accumulated size
 }
 type sizerN struct { sizerCommon; N }
+type sizerM struct { sizerCommon; M }
 
 // encoderX generates code to X-encode a message.
 //
@@ -652,6 +692,7 @@ type encoderCommon struct {
 	n int // current write position in data
 }
 type encoderN struct { encoderCommon; N }
+type encoderM struct { encoderCommon; M }
 
 // decoderX generates code to X-decode a message.
 //
@@ -686,12 +727,17 @@ type decoderCommon struct {
 	overflow OverflowCheck
 }
 type decoderN struct { decoderCommon; N }
+type decoderM struct { decoderCommon; M }
 
 
 var _ CodeGenerator = (*sizerN)(nil)
 var _ CodeGenerator = (*encoderN)(nil)
 var _ CodeGenerator = (*decoderN)(nil)
 
+var _ CodeGenerator = (*sizerM)(nil)
+var _ CodeGenerator = (*encoderM)(nil)
+var _ CodeGenerator = (*decoderM)(nil)
+
 
 func (s *sizerCommon) generatedCode() string {
 	code := Buffer{}
@@ -849,7 +895,7 @@ func (d *decoderCommon) generatedCode() string {
 
 	// `goto overflow` is not used only for empty structs
 	// NOTE for >0 check actual X in StdSizes{X} does not particularly matter
-	if (&types.StdSizes{8, 8}).Sizeof(d.typ) > 0 {
+	if (&types.StdSizes{8, 8}).Sizeof(d.typ) > 0 || d.enc != 'N' {
 		code.emit("\noverflow:")
 		code.emit("return 0, ErrDecodeOverflow")
 	}
@@ -897,6 +943,228 @@ func (d *decoderN) genBasic(assignto string, typ *types.Basic, userType types.Ty
 	d.overflow.Add(basic.wireSize)
 }
 
+// M: emit code to size/encode/decode basic fixed type
+func (s *sizerM) genBasic(path string, typ *types.Basic, userType types.Type) {
+	// upath casts path into basic type if needed
+	// e.g. p.x -> int32(p.x)  if p.x is custom type with underlying int32
+	upath := path
+	if userType.Underlying() != userType {
+		upath = fmt.Sprintf("%s(%s)", typ.Name(), upath)
+	}
+
+	// zodb.Tid and zodb.Oid are encoded as [8]bin
+	if userType == zodbTid || userType == zodbOid {
+		 s.size.Add(1+1+8) // mbin8 + 8 + [8]data
+		 return
+	}
+
+	// enums are encoded as extensions
+	if _, isEnum := enumRegistry[userType]; isEnum {
+		s.size.Add(1+1+1) // fixext1 enumType value
+		return
+	}
+
+	switch typ.Kind() {
+	case types.Bool:	s.size.Add(1) // mfalse|mtrue
+	case types.Int8:	s.size.AddExpr("msgpack.Int8Size(%s)",   upath)
+	case types.Int16:	s.size.AddExpr("msgpack.Int16Size(%s)",  upath)
+	case types.Int32:	s.size.AddExpr("msgpack.Int32Size(%s)",  upath)
+	case types.Int64:	s.size.AddExpr("msgpack.Int64Size(%s)",  upath)
+
+	case types.Uint8:	s.size.AddExpr("msgpack.Uint8Size(%s)",  upath)
+	case types.Uint16:	s.size.AddExpr("msgpack.Uint16Size(%s)", upath)
+	case types.Uint32:	s.size.AddExpr("msgpack.Uint32Size(%s)", upath)
+	case types.Uint64:	s.size.AddExpr("msgpack.Uint64Size(%s)", upath)
+
+	case types.Float64:	s.size.Add(1+8) // mfloat64 + <value64>
+	}
+}
+
+func (e *encoderM) genBasic(path string, typ *types.Basic, userType types.Type) {
+	// upath casts path into basic type if needed
+	// e.g. p.x -> int32(p.x)  if p.x is custom type with underlying int32
+	upath := path
+	if userType.Underlying() != userType {
+		upath = fmt.Sprintf("%s(%s)", typ.Name(), upath)
+	}
+
+	// zodb.Tid and zodb.Oid are encoded as [8]bin
+	if userType == zodbTid || userType == zodbOid {
+		e.emit("data[%v] = byte(msgpack.Bin8)", e.n);  e.n++
+		e.emit("data[%v] = 8",     e.n);               e.n++
+		e.emit("binary.BigEndian.PutUint64(data[%v:], uint64(%s))", e.n, path)
+		e.n += 8
+		return
+	}
+
+	// enums are encoded as `fixext1 enumType fixint<value>`
+	if enum, ok := enumRegistry[userType]; ok {
+		e.emit("data[%v] = byte(msgpack.FixExt1)", e.n);  e.n++
+		e.emit("data[%v] = %d", e.n, enum);               e.n++
+		e.emit("if !(0 <= %s && %s <= 0x7f) {", path, path) // mposfixint
+		e.emit(`  panic("%s: invalid %s enum value)")`, path, typeName(userType))
+		e.emit("}")
+		e.emit("data[%v] = byte(%s)", e.n, path);  e.n++
+		return
+	}
+
+	// mputint emits mput<kind>int<size>(path)
+	mputint := func(kind string, size int) {
+		KI := "I" // I or <Kind>i
+		if kind != "" {
+			KI = strings.ToUpper(kind) + "i"
+		}
+		e.emit("{")
+		e.emit("n := msgpack.Put%snt%d(data[%v:], %s)", KI, size, e.n, upath)
+		e.emit("data = data[%v+n:]", e.n)
+		e.emit("}")
+		e.n = 0
+	}
+
+	switch typ.Kind() {
+	case types.Bool:
+		e.emit("data[%v] = byte(msgpack.Bool(%s))", e.n, path)
+		e.n += 1
+
+	case types.Int8:	mputint("", 8)
+	case types.Int16:	mputint("", 16)
+	case types.Int32:	mputint("", 32)
+	case types.Int64:	mputint("", 64)
+
+	case types.Uint8:	mputint("u", 8)
+	case types.Uint16:	mputint("u", 16)
+	case types.Uint32:	mputint("u", 32)
+	case types.Uint64:	mputint("u", 64)
+
+	case types.Float64:
+		// mfloat64 f64
+		e.emit("data[%v] = byte(msgpack.Float64)", e.n);         e.n++
+		e.emit("float64_neoEncode(data[%v:], %s)", e.n, upath);  e.n += 8
+	}
+}
+
+// decoder expects <op>
+func (d *decoderM) expectOp(assignto string, op string) {
+	d.emit("if op := msgpack.Op(data[%v]); op != %s {", d.n, op);  d.n++
+	d.emit("  return 0, mdecodeOpErr(%q, op, %s)", d.pathName(assignto), op)
+	d.emit("}")
+	d.overflow.Add(1)
+}
+// decoder expects mbin8 l
+func (d *decoderM) expectBin8Fix(assignto string, l int) {
+	d.expectOp(assignto, "msgpack.Bin8")
+	d.emit("if l := data[%v]; l != %d {", d.n, l);  d.n++
+	d.emit("  return 0, mdecodeLen8Err(%q, l, %d)", d.pathName(assignto), l)
+	d.emit("}")
+	d.overflow.Add(1)
+}
+// decoder expects mfixext1 <enumType>
+func (d *decoderM) expectEnum(assignto string, enumType int) {
+	d.expectOp(assignto, "msgpack.FixExt1")
+	d.emit("if enumType := data[%v]; enumType != %d {", d.n, enumType);  d.n++
+	d.emit("  return 0, mdecodeEnumTypeErr(%q, enumType, %d)", d.pathName(assignto), enumType)
+	d.emit("}")
+	d.overflow.Add(1)
+}
+
+func (d *decoderM) genBasic(assignto string, typ *types.Basic, userType types.Type) {
+	// zodb.Tid and zodb.Oid are encoded as [8]bin
+	if userType == zodbTid || userType == zodbOid {
+		d.expectBin8Fix(assignto, 8)
+		d.emit("%s= %s(binary.BigEndian.Uint64(data[%v:]))", assignto, typeName(userType), d.n)
+		d.n += 8
+		d.overflow.Add(8)
+
+		return
+	}
+
+	// enums are encoded as `fixext1 enumType fixint<value>`
+	if enum, ok := enumRegistry[userType]; ok {
+		d.expectEnum(assignto, enum)
+		d.emit("{")
+		d.emit("v := data[%v]", d.n);  d.n++
+		d.emit("if !(0 <= v && v <= 0x7f) {") // mposfixint
+		d.emit("  return 0, mdecodeEnumValueErr(%q, v)", d.pathName(assignto))
+		d.emit("}")
+		d.emit("%s= %s(v)", assignto, typeName(userType))
+		d.emit("}")
+		d.overflow.Add(1)
+		return
+	}
+
+	// v represents basic decoded value casted to user type if needed
+	v := "v"
+	if userType.Underlying() != userType {
+		v = fmt.Sprintf("%s(v)", typeName(userType))
+	}
+
+	// mgetint emits assignto = mget<kind>int<size>()
+	mgetint := func(kind string, size int) {
+		// we are going to go into msgp - flush previously queued
+		// overflow checks; put place for next overflow check after
+		// msgp is done.
+		d.overflowCheck()
+		d.resetPos()
+		defer d.overflowCheck()
+
+		KI := "I" // I or <Kind>i
+		if kind != "" {
+			KI = strings.ToUpper(kind) + "i"
+		}
+		d.emit("{")
+		d.emit("v, tail, err := msgp.Read%snt%dBytes(data)", KI, size)
+		d.emit("if err != nil {")
+		d.emit("  return 0, mdecodeErr(%q, err)", d.pathName(assignto))
+		d.emit("}")
+		d.emit("%s= %s", assignto, v)
+		d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
+		d.emit("data = tail")
+		d.emit("}")
+	}
+
+	// mgetfloat emits mgetfloat<size>
+	mgetfloat := func(size int) {
+		// delving into msgp - flush/prepare next site for overflow check
+		d.overflowCheck()
+		d.resetPos()
+		defer d.overflowCheck()
+
+		d.emit("{")
+		d.emit("v, tail, err := msgp.ReadFloat%dBytes(data)", size)
+		d.emit("if err != nil {")
+		d.emit("  return 0, mdecodeErr(%q, err)", d.pathName(assignto))
+		d.emit("}")
+		d.emit("%s= %s", assignto, v)
+		d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
+		d.emit("data = tail")
+		d.emit("}")
+	}
+
+	switch typ.Kind() {
+	case types.Bool:
+		d.emit("switch op := msgpack.Op(data[%v]); op {", d.n)
+		d.emit("default: return 0, mdecodeOpErr(%q, op, msgpack.True, msgpack.False)", d.pathName(assignto))
+		d.emit("case msgpack.True:  %s = true", assignto)
+		d.emit("case msgpack.False: %s = false", assignto)
+		d.emit("}")
+
+		d.n++
+		d.overflow.Add(1)
+
+	case types.Int8:	mgetint("", 8)
+	case types.Int16:	mgetint("", 16)
+	case types.Int32:	mgetint("", 32)
+	case types.Int64:	mgetint("", 64)
+
+	case types.Uint8:	mgetint("u", 8)
+	case types.Uint16:	mgetint("u", 16)
+	case types.Uint32:	mgetint("u", 32)
+	case types.Uint64:	mgetint("u", 64)
+
+	case types.Float64:	mgetfloat(64)
+	}
+}
+
 // emit code to size/encode/decode array with sizeof(elem)==1
 // [len(A)]byte
 func (s *sizerN) genArray1(path string, typ *types.Array) {
@@ -915,6 +1183,39 @@ func (d *decoderN) genArray1(assignto string, typ *types.Array) {
 	d.overflow.Add(typLen)
 }
 
+
+// binX+lenX
+// [len(A)]byte
+func (s *sizerM) genArray1(path string, typ *types.Array) {
+	l := int(typ.Len())
+	s.size.Add(msgpack.BinHeadSize(l))
+	s.size.Add(l)
+}
+
+func (e *encoderM) genArray1(path string, typ *types.Array) {
+	l := int(typ.Len())
+	if l > 0xff {
+		panic("TODO: array1 with > 255 elements")
+	}
+	e.emit("data[%v] = byte(msgpack.Bin8)", e.n);  e.n++
+	e.emit("data[%v] = %d", e.n, l);               e.n++
+	e.emit("copy(data[%v:], %v[:])", e.n, path)
+	e.n += l
+}
+
+func (d *decoderM) genArray1(assignto string, typ *types.Array) {
+	l := int(typ.Len())
+	if l > 0xff {
+		panic("TODO: array1 with > 255 elements")
+	}
+
+	d.expectBin8Fix(assignto, l)
+	d.emit("copy(%v[:], data[%v:%v])", assignto, d.n, d.n+l)
+	d.n += l
+	d.overflow.Add(l)
+}
+
+
 // emit code to size/encode/decode string or []byte
 // len	u32
 // [len]byte
@@ -963,15 +1264,71 @@ func (d *decoderN) genSlice1(assignto string, typ types.Type) {
 	d.emit("}")
 }
 
+// bin8+len8|bin16+len16|bin32+len32
+// [len]byte
+func (s *sizerM) genSlice1(path string, typ types.Type) {
+	s.size.AddExpr("msgpack.BinHeadSize(len(%s))", path)
+	s.size.AddExpr("len(%s)", path)
+}
+func (e *encoderM) genSlice1(path string, typ types.Type) {
+	e.emit("{")
+	e.emit("l := len(%s)", path)
+	e.emit("n := msgpack.PutBinHead(data[%v:], l)", e.n)
+	e.emit("data = data[%v+n:]", e.n)
+	e.emit("copy(data, %v)", path)
+	e.emit("data = data[l:]")
+	e.emit("}")
+	e.n = 0
+}
+func (d *decoderM) genSlice1(assignto string, typ types.Type) {
+	// -> msgp: flush queued overflow checks; put place for next overflow
+	// checks after msgp is done.
+	d.overflowCheck()
+	d.resetPos()
+	defer d.overflowCheck()
+
+	d.emit("{")
+	d.emit("b, tail, err := msgp.ReadBytesZC(data)")
+	d.emit("if err != nil {")
+	d.emit("  return 0, mdecodeErr(%q, err)", d.pathName(assignto))
+	d.emit("}")
+
+	switch t := typ.(type) {
+	case *types.Basic:
+		if t.Kind() != types.String {
+			log.Panicf("bad basic type in slice1: %v", t)
+		}
+		d.emit("%v= string(b)", assignto)
+
+	case *types.Slice:
+		// TODO eventually do not copy, but reference data from original
+		d.emit("%v= make(%v, len(b))", assignto, typeName(typ))
+		d.emit("copy(%v, b)", assignto)
+
+	default:
+		log.Panicf("bad type in slice1: %v", typ)
+	}
+
+	d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
+	d.emit("data = tail")
+	d.emit("}")
+}
+
 // emit code to size/encode/decode mem.Buf
 // same as slice1 but buffer is allocated via mem.BufAlloc
 func (s *sizerN) genBuf(path string) {
 	s.genSlice1(path+".XData()", nil /* typ unused */)
 }
+func (s *sizerM) genBuf(path string) {
+	s.genSlice1(path+".XData()", nil /* typ unused */)
+}
 
 func (e *encoderN) genBuf(path string) {
 	e.genSlice1(path+".XData()", nil /* typ unused */)
 }
+func (e *encoderM) genBuf(path string) {
+	e.genSlice1(path+".XData()", nil /* typ unused */)
+}
 
 func (d *decoderN) genBuf(assignto string) {
 	d.emit("{")
@@ -990,6 +1347,29 @@ func (d *decoderN) genBuf(assignto string) {
 	d.emit("}")
 }
 
+func (d *decoderM) genBuf(assignto string) {
+	// -> msgp: flush queued overflow checks; put place for next overflow
+	// checks after msgp is done.
+	d.overflowCheck()
+	d.resetPos()
+	defer d.overflowCheck()
+
+	d.emit("{")
+	d.emit("b, tail, err := msgp.ReadBytesZC(data)")
+	d.emit("if err != nil {")
+	d.emit("  return 0, mdecodeErr(%q, err)", d.pathName(assignto))
+	d.emit("}")
+
+	// TODO eventually do not copy but reference original
+	d.emit("%v= mem.BufAlloc(len(b))", assignto)
+	d.emit("copy(%v.Data, b)", assignto)
+
+	d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
+	d.emit("data = tail")
+	d.emit("}")
+}
+
+
 // emit code to size/encode/decode slice
 // len	u32
 // [len]item
@@ -1087,6 +1467,44 @@ func (d *decoderCommon) genSliceCommon(xd CodeGenCustomize, assignto string, typ
 	d.emit("}")
 }
 
+// fixarray|array16+YYYY|array32+ZZZZ
+// [len]item
+func (s *sizerM) genSliceHead(path string, typ *types.Slice, obj types.Object) {
+	s.size.AddExpr("msgpack.ArrayHeadSize(len(%s))", path)
+}
+func (s *sizerM) genSlice(path string, typ *types.Slice, obj types.Object) {
+	s.genSliceCommon(s, path, typ, obj)
+}
+
+func (e *encoderM) genSliceHead(path string, typ *types.Slice, obj types.Object) {
+	e.emit("l := len(%s)", path)
+	e.emit("n := msgpack.PutArrayHead(data[%v:], l)", e.n)
+	e.emit("data = data[%v+n:]", e.n)
+	e.n = 0
+}
+func (e *encoderM) genSlice(path string, typ *types.Slice, obj types.Object) {
+	e.genSliceCommon(e, path, typ, obj)
+}
+
+func (d *decoderM) genSliceHead(assignto string, typ *types.Slice, obj types.Object) {
+	// -> msgp: flush queued overflow checks; put place for next overflow
+	// checks after msgp is done.
+	d.overflowCheck()
+	d.resetPos()
+	defer d.overflowCheck()
+
+	d.emit("l, tail, err := msgp.ReadArrayHeaderBytes(data)")
+	d.emit("if err != nil {")
+	d.emit("  return 0, mdecodeErr(%q, err)", d.pathName(assignto))
+	d.emit("}")
+
+	d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
+	d.emit("data = tail")
+}
+func (d *decoderM) genSlice(assignto string, typ *types.Slice, obj types.Object) {
+	d.genSliceCommon(d, assignto, typ, obj)
+}
+
 // generate code to encode/decode map
 // len  u32
 // [len](key, value)
@@ -1208,6 +1626,44 @@ func (d *decoderCommon) genMapCommon(xd CodeGenCustomize, assignto string, typ *
 	d.emit("}")
 }
 
+// fixmap|map16+YYYY|map32+ZZZZ
+// [len]key/value
+func (s *sizerM) genMapHead(path string, typ *types.Map, obj types.Object) {
+	s.size.AddExpr("msgpack.MapHeadSize(len(%s))", path)
+}
+func (s *sizerM) genMap(path string, typ *types.Map, obj types.Object) {
+	s.genMapCommon(s, path, typ, obj)
+}
+
+func (e *encoderM) genMapHead(path string, typ *types.Map, obj types.Object) {
+	e.emit("l := len(%s)", path)
+	e.emit("n := msgpack.PutMapHead(data[%v:], l)", e.n)
+	e.emit("data = data[%v+n:]", e.n)
+	e.n = 0
+}
+func (e *encoderM) genMap(path string, typ *types.Map, obj types.Object) {
+	e.genMapCommon(e, path, typ, obj)
+}
+
+func (d *decoderM) genMapHead(assignto string, typ *types.Map, obj types.Object) {
+	// -> msgp: flush queued overflow checks; put place for next overflow
+	// checks after msgp is done.
+	d.overflowCheck()
+	d.resetPos()
+	defer d.overflowCheck()
+
+	d.emit("l, tail, err := msgp.ReadMapHeaderBytes(data)")
+	d.emit("if err != nil {")
+	d.emit("  return 0, mdecodeErr(%q, err)", d.pathName(assignto))
+	d.emit("}")
+
+	d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
+	d.emit("data = tail")
+}
+func (d *decoderM) genMap(assignto string, typ *types.Map, obj types.Object) {
+	d.genMapCommon(d, assignto, typ, obj)
+}
+
 // emit code to size/encode/decode custom type
 func (s *sizerN) genCustomN(path string) {
 	s.size.AddExpr("%s.neoEncodedLenN()", path)
@@ -1246,6 +1702,35 @@ func (s *sizerN)   genStructHead(path string, typ *types.Struct, userType types.
 func (e *encoderN) genStructHead(path string, typ *types.Struct, userType types.Type) {}
 func (d *decoderN) genStructHead(path string, typ *types.Struct, userType types.Type) {}
 
+// M: array<nfields>
+func (s *sizerM) genStructHead(path string, typ *types.Struct, userType types.Type) {
+	s.size.Add(1) // mfixarray|marray16|marray32
+	if typ.NumFields() > 0x0f {
+		panic("TODO: struct with > 15 elements")
+	}
+}
+
+func (e *encoderM) genStructHead(path string, typ *types.Struct, userType types.Type) {
+	if typ.NumFields() > 0x0f {
+		panic("TODO: struct with > 15 elements")
+	}
+	e.emit("data[%v] = byte(msgpack.FixArray_4 | %d)", e.n, typ.NumFields())
+	e.n += 1
+}
+
+func (d *decoderM) genStructHead(path string, typ *types.Struct, userType types.Type) {
+	if typ.NumFields() > 0x0f {
+		panic("TODO: struct with > 15 elements")
+	}
+
+	d.emit("if op, opOk := msgpack.Op(data[%v]), msgpack.FixArray_4 | %d ; op != opOk {", d.n, typ.NumFields())
+	d.emit("return 0, &mstructDecodeError{%q, op, opOk}", d.pathName(path))
+	d.emit("}")
+
+	d.n += 1
+	d.overflow.Add(1)
+}
+
 
 // top-level driver for emitting size/encode/decode code for a type
 //
@@ -1274,7 +1759,7 @@ func codegenType(path string, typ types.Type, obj types.Object, codegen CodeGene
 			break
 		}
 
-		_, ok := basicTypesN[u.Kind()] // ok to check N to see if supported for any encoding
+		_, ok := basicTypesN[u.Kind()] // ok to check N to see if supported for both N and M
 		if !ok {
 			log.Fatalf("%v: %v: basic type %v not supported", pos(obj), obj.Name(), u)
 		}