proto/msgpack: Fix {en,de}coding INVALID_{TID,OID}

In pre-msgpack protocol an 'INVALID_{TID,OID}' was always
decoded as 'None' in NEO/py [1]. But in msgpack protocol
this isn't true anymore. An `INVALID_TID` is now decoded
as an `INVALID_TID`. And this then leads to errors later [2].
We fix this by encoding 'INVALID_{TID,OID}' to NIL on the
wire and by decoding NIL to 'INVALID_{TID,OID}'.

---

[1] https://lab.nexedi.com/nexedi/neoppod/-/blob/6332112cba979dfd29b40fe9f98d097911fde696/neo/lib/protocol.py#L579-583
[2] With SQLite backend we can see the following exception:

Traceback (most recent call last):
  File "/home/levin/neo/neo/tests/functional/__init__.py", line 192, in start
    self.run()
  File "/home/levin/neo/neo/tests/functional/__init__.py", line 288, in run
    getattr(neo.scripts,  self.command).main()
  File "/home/levin/neo/neo/scripts/neostorage.py", line 32, in main
    app.run()
  File "/home/levin/neo/neo/storage/app.py", line 196, in run
    self._run()
  File "/home/levin/neo/neo/storage/app.py", line 227, in _run
    self.doOperation()
  File "/home/levin/neo/neo/storage/app.py", line 301, in doOperation
    poll()
  File "/home/levin/neo/neo/storage/app.py", line 145, in _poll
    self.em.poll(1)
  File "/home/levin/neo/neo/lib/event.py", line 160, in poll
    to_process.process()
  File "/home/levin/neo/neo/lib/connection.py", line 508, in process
    self._handlers.handle(self, self._queue.pop(0))
  File "/home/levin/neo/neo/lib/connection.py", line 93, in handle
    self._handle(connection, packet)
  File "/home/levin/neo/neo/lib/connection.py", line 108, in _handle
    pending[0][1].packetReceived(connection, packet)
  File "/home/levin/neo/neo/lib/handler.py", line 125, in packetReceived
    self.dispatch(*args)
  File "/home/levin/neo/neo/lib/handler.py", line 75, in dispatch
    method(conn, *args, **kw)
  File "/home/levin/neo/neo/storage/handlers/client.py", line 67, in askObject
    o = app.dm.getObject(oid, at, before)
  File "/home/levin/neo/neo/storage/database/manager.py", line 484, in getObject
    before_tid and u64(before_tid))
  File "/home/levin/neo/neo/storage/database/sqlite.py", line 336, in _getObject
    r = q(sql + ' AND tid=?', (partition, oid, tid))
OverflowError: Python int too large to convert to SQLite INTEGER

go/neo/internal/msgpack/msgpack.go

@@ -292,12 +292,31 @@ func PutMapHead(data []byte, l int) (n int) {
 // optional float64. It is a guess, because the function
 // doesn't flag corrupt or empty data.
 func OptionalFloat64Size(data []byte) uint64 {
-	if len(data) > 0 && data[0] == byte(Nil) {
-		return 1
 	// valid float must be encoded in 9 byte
 	// 1:   msgpack.Float64 header
 	// 2-9: <value64>
+	return OptionalValueSize(data, 9)
+}
+
+// TidOrOidSize returns guessed size of encoded optional
+// TID or OID. It is a guess, because the function
+// doesn't flag corrupt or empty data.
+func TidOrOidSize(data []byte) uint64 {
+	// valid {TID,OID} must be encoded in 10 byte
+	// 1:    msgpack.Bin8 header
+	// 2:    msgpack.Len8
+	// 3-10: binary.BigEndian.Uint64
+	return OptionalValueSize(data, 10)
+}
+
+// OptionalValueSize returns guessed size of encoded
+// optional value. It is a guess, because the function
+// doesn't flag corrupt or empty data.
+func OptionalValueSize(data []byte, valuesize uint64) (n uint64) {
+	if len(data) >= 1 && data[0] == byte(Nil) {
+		n = 1
 	} else {
-		return 9
+		n = valuesize
 	}
+	return
 }

go/neo/internal/msgpack/msgpack_test.go

@@ -203,3 +203,19 @@ func TestOptionalFloat64Size(t *testing.T) {
 	// anything else => error is raised later at overflow check
 	ts(9, []byte {1, 2})
 }
+
+func TestTidOrOidSize(t *testing.T) {
+	ts := func (wanted int, value []byte) {
+		v := TidOrOidSize(value)
+		if v != uint64(wanted) {
+			t.Errorf("want: %v; got: %v", wanted, v)
+		}
+	}
+
+	// Overflow
+	ts(10, []byte {})
+	// INVALID_{TID,OID}
+	ts(1, []byte {byte(Nil)})
+	// anything else
+	ts(10, []byte {1, 2})
+}

go/neo/proto/proto_test.go

@@ -205,6 +205,34 @@ func TestMsgMarshal(t *testing.T) {
 			hex("c408") + hex("0a0b0c0d0e0f0104"),
 		},
 
+		// Invalid Oid, Invalid Tid, bool, Checksum, []byte
+		{&StoreObject{
+			Oid:         INVALID_OID,
+			Serial:      INVALID_TID,
+			Compression: 0,
+			Checksum:    Checksum{},
+			Data:        []byte("hello world"),
+			DataSerial:  INVALID_TID,
+			Tid:         INVALID_TID,
+			},
+
+			// N
+			hex("ffffffffffffffffffffffffffffffff00") +
+			hex("0000000000000000000000000000000000000000") +
+			hex("0000000b") + "hello world" +
+			hex("ffffffffffffffffffffffffffffffff"),
+
+			// M
+			hex("97") +
+			hex("c0") +
+			hex("c0") +
+			hex("00") +
+			hex("c414") + hex("0000000000000000000000000000000000000000") +
+			hex("c40b") + "hello world" +
+			hex("c0") +
+			hex("c0"),
+		},
+
 		// PTid, [] (of [] of {NodeID, CellState})
 		{&AnswerPartitionTable{
 			PTid:        0x0102030405060708,

go/neo/proto/protogen.go

@@ -131,6 +131,13 @@ var memBuf           types.Type       // type of mem.Buf
 // registry of enums
 var enumRegistry = map[types.Type]int{} // type -> enum type serial
 
+// Define INVALID_ID because encoding behaves different depending
+// on if we have an INVALID_{TID,OID} or not.
+//
+// NOTE This assumes that INVALID_OID == INVALID_TID
+//
+// XXX Duplication wrt proto.go
+const INVALID_ID uint64 = 1<<64 - 1
 
 // bytes.Buffer + bell & whistles
 type Buffer struct {
@@ -960,9 +967,14 @@ func (s *sizerM) genBasic(path string, typ *types.Basic, userType types.Type) {
 		upath = fmt.Sprintf("%s(%s)", typ.Name(), upath)
 	}
 
-	// zodb.Tid and zodb.Oid are encoded as [8]bin	XXX or nil for INVALID_{TID_OID}
+	// zodb.Tid and zodb.Oid are encoded as [8]bin or nil for INVALID_{TID_OID}
 	if userType == zodbTid || userType == zodbOid {
-		 s.size.Add(1+1+8) // mbin8 + 8 + [8]data
+		// INVALID_{TID,OID} must be NIL on the wire
+		s.emit("if uint64(%s) == %v {", path, INVALID_ID)
+		s.emit("%v += 1 // mnil", s.var_("size"))
+		s.emit("} else {")
+		s.emit("%v += 1+1+8 // mbin8 + 8 + [8]data", s.var_("size"))
+		s.emit("}")
 		return
 	}
 
@@ -1006,12 +1018,25 @@ func (e *encoderM) genBasic(path string, typ *types.Basic, userType types.Type)
 		upath = fmt.Sprintf("%s(%s)", typ.Name(), upath)
 	}
 
-	// zodb.Tid and zodb.Oid are encoded as [8]bin	XXX or nil
+	// optional emits mputnil(path) for optval and putval in any other case
+	optional := func(optval string, putval func()) {
+		e.emit("if %s == %v {", upath, optval)
+		e.emit("data[%v] = byte(msgpack.Nil)", e.n)
+		e.emit("data = data[%v:]", e.n + 1)
+		e.emit("} else {")
+		putval()
+		e.resetPos()
+		e.emit("}")
+	}
+
+	// zodb.Tid and zodb.Oid are encoded as [8]bin or nil
 	if userType == zodbTid || userType == zodbOid {
+		optional(fmt.Sprintf("%v", INVALID_ID), func() {
 			e.emit("data[%v] = byte(msgpack.Bin8)", e.n);  e.n++
 			e.emit("data[%v] = 8",     e.n);               e.n++
 			e.emit("binary.BigEndian.PutUint64(data[%v:], uint64(%s))", e.n, path)
 			e.n += 8
+		})
 		return
 	}
 
@@ -1050,13 +1075,7 @@ func (e *encoderM) genBasic(path string, typ *types.Basic, userType types.Type)
 	// infinite -1, see
 	// https://lab.nexedi.com/kirr/neo/-/blob/1ad088c8/go/neo/proto/proto.go#L352-357
 	if typeName(userType) == "IdTime" {
-		e.emit("if %s == float64(IdTimeNone) {", upath)
-		e.emit("data[%v] = byte(msgpack.Nil)", e.n) // mnil
-		e.emit("data = data[%v:]", e.n + 1)
-		e.emit("} else {")
-		mputfloat64()
-		e.resetPos()
-		e.emit("}")
+		optional("float64(IdTimeNone)", mputfloat64)
 		return
 	}
 
@@ -1081,23 +1100,27 @@ func (e *encoderM) genBasic(path string, typ *types.Basic, userType types.Type)
 }
 
 // decoder expects <op>
-func (d *decoderM) expectOp(assignto string, op string) {
+func (d *decoderM) expectOp(assignto string, op string, addOverflow bool) {
 	d.emit("if op := msgpack.Op(data[%v]); op != %s {", d.n, op);  d.n++
 	d.emit("  return 0, mdecodeOpErr(%q, op, %s)", d.pathName(assignto), op)
 	d.emit("}")
+	if addOverflow {
 		d.overflow.Add(1)
+	}
 }
 // decoder expects mbin8 l
-func (d *decoderM) expectBin8Fix(assignto string, l int) {
-	d.expectOp(assignto, "msgpack.Bin8")
+func (d *decoderM) expectBin8Fix(assignto string, l int, addOverflow bool) {
+	d.expectOp(assignto, "msgpack.Bin8", addOverflow)
 	d.emit("if l := data[%v]; l != %d {", d.n, l);  d.n++
 	d.emit("  return 0, mdecodeLen8Err(%q, l, %d)", d.pathName(assignto), l)
 	d.emit("}")
+	if addOverflow {
 		d.overflow.Add(1)
+	}
 }
 // decoder expects mfixext1 <enumType>
 func (d *decoderM) expectEnum(assignto string, enumType int) {
-	d.expectOp(assignto, "msgpack.FixExt1")
+	d.expectOp(assignto, "msgpack.FixExt1", true)
 	d.emit("if enumType := data[%v]; enumType != %d {", d.n, enumType);  d.n++
 	d.emit("  return 0, mdecodeEnumTypeErr(%q, enumType, %d)", d.pathName(assignto), enumType)
 	d.emit("}")
@@ -1105,13 +1128,26 @@ func (d *decoderM) expectEnum(assignto string, enumType int) {
 }
 
 func (d *decoderM) genBasic(assignto string, typ *types.Basic, userType types.Type) {
-	// zodb.Tid and zodb.Oid are encoded as [8]bin
+	// zodb.Tid and zodb.Oid are encoded as [8]bin or Nil
 	if userType == zodbTid || userType == zodbOid {
-		d.expectBin8Fix(assignto, 8)
+		d.resetPos()
+		d.overflowCheck()
+		defer d.overflowCheck()
+
+		// Size depends on if we have an INVALID_{TID,OID}
+		// which is encoded as NIL or if we have a valid
+		// {TID,OID} which is encdoed as Bin8Fix.
+		d.overflow.AddExpr("msgpack.TidOrOidSize(data)")
+		d.emit("if data[%v] == byte(msgpack.Nil) {", d.n)
+		d.emit("%s = %s(%v)", assignto, typeName(userType), INVALID_ID)
+		d.n += 1
+		d.resetPos()
+		d.emit("} else {")
+		d.expectBin8Fix(assignto, 8, false)
 		d.emit("%s= %s(binary.BigEndian.Uint64(data[%v:]))", assignto, typeName(userType), d.n)
 		d.n += 8
-		d.overflow.Add(8)
-
+		d.resetPos()
+		d.emit("}")
 		return
 	}
 
@@ -1270,7 +1306,7 @@ func (d *decoderM) genArray1(assignto string, typ *types.Array) {
 		panic("TODO: array1 with > 255 elements")
 	}
 
-	d.expectBin8Fix(assignto, l)
+	d.expectBin8Fix(assignto, l, true)
 	d.emit("copy(%v[:], data[%v:%v])", assignto, d.n, d.n+l)
 	d.n += l
 	d.overflow.Add(l)