proto/msgpack: Fix encoding INVALID_{TID,OID}

In pre-msgpack protocol an 'INVALID_{TID,OID}' was always
decoded as 'None' in NEO/py [1]. But in msgpack protocol
this isn't true anymore. An `INVALID_TID` is now decoded
as an `INVALID_TID`. And this then leads to errors later [2].
Instead of restoring NEO/py to a pre-msgpack behaviour, we
fix this on NEO/go side by encoding 'INVALID_{TID,OID}' to NIL
on the wire. This is good as it makes the messages we send
smaller.

[1] https://lab.nexedi.com/nexedi/neoppod/-/blob/6332112cba979dfd29b40fe9f98d097911fde696/neo/lib/protocol.py#L579-583
[2] With SQLite backend we can see the following exception:

Traceback (most recent call last):
  File "/home/levin/neo/neo/tests/functional/__init__.py", line 192, in start
    self.run()
  File "/home/levin/neo/neo/tests/functional/__init__.py", line 288, in run
    getattr(neo.scripts,  self.command).main()
  File "/home/levin/neo/neo/scripts/neostorage.py", line 32, in main
    app.run()
  File "/home/levin/neo/neo/storage/app.py", line 196, in run
    self._run()
  File "/home/levin/neo/neo/storage/app.py", line 227, in _run
    self.doOperation()
  File "/home/levin/neo/neo/storage/app.py", line 301, in doOperation
    poll()
  File "/home/levin/neo/neo/storage/app.py", line 145, in _poll
    self.em.poll(1)
  File "/home/levin/neo/neo/lib/event.py", line 160, in poll
    to_process.process()
  File "/home/levin/neo/neo/lib/connection.py", line 508, in process
    self._handlers.handle(self, self._queue.pop(0))
  File "/home/levin/neo/neo/lib/connection.py", line 93, in handle
    self._handle(connection, packet)
  File "/home/levin/neo/neo/lib/connection.py", line 108, in _handle
    pending[0][1].packetReceived(connection, packet)
  File "/home/levin/neo/neo/lib/handler.py", line 125, in packetReceived
    self.dispatch(*args)
  File "/home/levin/neo/neo/lib/handler.py", line 75, in dispatch
    method(conn, *args, **kw)
  File "/home/levin/neo/neo/storage/handlers/client.py", line 67, in askObject
    o = app.dm.getObject(oid, at, before)
  File "/home/levin/neo/neo/storage/database/manager.py", line 484, in getObject
    before_tid and u64(before_tid))
  File "/home/levin/neo/neo/storage/database/sqlite.py", line 336, in _getObject
    r = q(sql + ' AND tid=?', (partition, oid, tid))
OverflowError: Python int too large to convert to SQLite INTEGER

go/neo/internal/msgpack/msgpack.go

@@ -33,6 +33,8 @@ const (
 	FixMap_4   Op  = 0b1000_0000 // 1000_XXXX
 	FixArray_4 Op  = 0b1001_0000 // 1001_XXXX
 
+	Nil      Op  = 0xc0
+
 	False	 Op  = 0xc2
 	True	 Op  = 0xc3
 

go/neo/proto/protogen.go

@@ -131,6 +131,13 @@ var memBuf           types.Type       // type of mem.Buf
 // registry of enums
 var enumRegistry = map[types.Type]int{} // type -> enum type serial
 
+// Define INVALID_TID because encoding behaves different depending
+// on if we have an INVALID_TID or not.
+//
+// NOTE This assumes that INVALID_OID == INVALID_TID
+//
+// XXX Duplication wrt proto.go
+const INVALID_ID uint64 = 1<<64 - 1
 
 // bytes.Buffer + bell & whistles
 type Buffer struct {
@@ -961,10 +968,15 @@ func (s *sizerM) genBasic(path string, typ *types.Basic, userType types.Type) {
 		upath = fmt.Sprintf("%s(%s)", typ.Name(), upath)
 	}
 
-	// zodb.Tid and zodb.Oid are encoded as [8]bin	XXX or nil for INVALID_{TID_OID}
+	// zodb.Tid and zodb.Oid are encoded as [8]bin or nil for INVALID_{TID_OID}
 	if userType == zodbTid || userType == zodbOid {
-		 s.size.Add(1+1+8) // mbin8 + 8 + [8]data
-		 return
+		// INVALID_{TID,OID} must be NIL on the wire
+		s.emit("if uint64(%s) == %v {", path, INVALID_ID)
+		s.emit("%v += 1 // mnil", s.var_("size"))
+		s.emit("} else {")
+		s.emit("%v += 1+1+8 // mbin8 + 8 + [8]data", s.var_("size"))
+		s.emit("}")
+		return
 	}
 
 	// enums are encoded as extensions
@@ -997,12 +1009,18 @@ func (e *encoderM) genBasic(path string, typ *types.Basic, userType types.Type)
 		upath = fmt.Sprintf("%s(%s)", typ.Name(), upath)
 	}
 
-	// zodb.Tid and zodb.Oid are encoded as [8]bin	XXX or nil
+	// zodb.Tid and zodb.Oid are encoded as [8]bin or nil
 	if userType == zodbTid || userType == zodbOid {
+		e.emit("if %s == %v {", path, INVALID_ID) // INVALID_{TID,OID} =>
+		e.emit("data[%v] = msgpack.Nil", e.n)     // mnil
+		e.emit("data = data[%v:]", e.n + 1)
+		e.emit("} else {")
 		e.emit("data[%v] = byte(msgpack.Bin8)", e.n);  e.n++
 		e.emit("data[%v] = 8",     e.n);               e.n++
 		e.emit("binary.BigEndian.PutUint64(data[%v:], uint64(%s))", e.n, path)
 		e.n += 8
+		e.resetPos()
+		e.emit("}")
 		return
 	}
 
@@ -1456,6 +1474,17 @@ func (e *encoderCommon) genSliceCommon(xe CodeGenCustomize, path string, typ *ty
 	e.emit("}")
 }
 
+// data = data[n:]
+// n    = 0
+//
+// XXX duplication wrt decoderCommon.resetPost
+func (e *encoderCommon) resetPos() {
+	if e.n != 0 {
+		e.emit("data = data[%v:]", e.n)
+		e.n = 0
+	}
+}
+
 func (d *decoderN) genSliceHead(assignto string, typ *types.Slice, obj types.Object) {
 	d.genBasic("l:", types.Typ[types.Uint32], nil)
 }