Commit 37dd83e9 authored by Levin Zimmermann's avatar Levin Zimmermann

proto/msgpack: Fix encoding INVALID_{TID,OID}

In pre-msgpack protocol an 'INVALID_{TID,OID}' was always
decoded as 'None' in NEO/py [1]. But in msgpack protocol
this isn't true anymore. An `INVALID_TID` is now decoded
as an `INVALID_TID`. And this then leads to errors later [2].
Instead of restoring NEO/py to a pre-msgpack behaviour, we
fix this on NEO/go side by encoding 'INVALID_{TID,OID}' to NIL
on the wire. This is good as it makes the messages we send
smaller.

[1] https://lab.nexedi.com/nexedi/neoppod/-/blob/6332112cba979dfd29b40fe9f98d097911fde696/neo/lib/protocol.py#L579-583
[2] With SQLite backend we can see the following exception:

Traceback (most recent call last):
  File "/home/levin/neo/neo/tests/functional/__init__.py", line 192, in start
    self.run()
  File "/home/levin/neo/neo/tests/functional/__init__.py", line 288, in run
    getattr(neo.scripts,  self.command).main()
  File "/home/levin/neo/neo/scripts/neostorage.py", line 32, in main
    app.run()
  File "/home/levin/neo/neo/storage/app.py", line 196, in run
    self._run()
  File "/home/levin/neo/neo/storage/app.py", line 227, in _run
    self.doOperation()
  File "/home/levin/neo/neo/storage/app.py", line 301, in doOperation
    poll()
  File "/home/levin/neo/neo/storage/app.py", line 145, in _poll
    self.em.poll(1)
  File "/home/levin/neo/neo/lib/event.py", line 160, in poll
    to_process.process()
  File "/home/levin/neo/neo/lib/connection.py", line 508, in process
    self._handlers.handle(self, self._queue.pop(0))
  File "/home/levin/neo/neo/lib/connection.py", line 93, in handle
    self._handle(connection, packet)
  File "/home/levin/neo/neo/lib/connection.py", line 108, in _handle
    pending[0][1].packetReceived(connection, packet)
  File "/home/levin/neo/neo/lib/handler.py", line 125, in packetReceived
    self.dispatch(*args)
  File "/home/levin/neo/neo/lib/handler.py", line 75, in dispatch
    method(conn, *args, **kw)
  File "/home/levin/neo/neo/storage/handlers/client.py", line 67, in askObject
    o = app.dm.getObject(oid, at, before)
  File "/home/levin/neo/neo/storage/database/manager.py", line 484, in getObject
    before_tid and u64(before_tid))
  File "/home/levin/neo/neo/storage/database/sqlite.py", line 336, in _getObject
    r = q(sql + ' AND tid=?', (partition, oid, tid))
OverflowError: Python int too large to convert to SQLite INTEGER
parent b44769cf
...@@ -33,6 +33,8 @@ const ( ...@@ -33,6 +33,8 @@ const (
FixMap_4 Op = 0b1000_0000 // 1000_XXXX FixMap_4 Op = 0b1000_0000 // 1000_XXXX
FixArray_4 Op = 0b1001_0000 // 1001_XXXX FixArray_4 Op = 0b1001_0000 // 1001_XXXX
Nil Op = 0xc0
False Op = 0xc2 False Op = 0xc2
True Op = 0xc3 True Op = 0xc3
......
...@@ -131,6 +131,13 @@ var memBuf types.Type // type of mem.Buf ...@@ -131,6 +131,13 @@ var memBuf types.Type // type of mem.Buf
// registry of enums // registry of enums
var enumRegistry = map[types.Type]int{} // type -> enum type serial var enumRegistry = map[types.Type]int{} // type -> enum type serial
// Define INVALID_TID because encoding behaves different depending
// on if we have an INVALID_TID or not.
//
// NOTE This assumes that INVALID_OID == INVALID_TID
//
// XXX Duplication wrt proto.go
const INVALID_ID uint64 = 1<<64 - 1
// bytes.Buffer + bell & whistles // bytes.Buffer + bell & whistles
type Buffer struct { type Buffer struct {
...@@ -961,10 +968,15 @@ func (s *sizerM) genBasic(path string, typ *types.Basic, userType types.Type) { ...@@ -961,10 +968,15 @@ func (s *sizerM) genBasic(path string, typ *types.Basic, userType types.Type) {
upath = fmt.Sprintf("%s(%s)", typ.Name(), upath) upath = fmt.Sprintf("%s(%s)", typ.Name(), upath)
} }
// zodb.Tid and zodb.Oid are encoded as [8]bin XXX or nil for INVALID_{TID_OID} // zodb.Tid and zodb.Oid are encoded as [8]bin or nil for INVALID_{TID_OID}
if userType == zodbTid || userType == zodbOid { if userType == zodbTid || userType == zodbOid {
s.size.Add(1+1+8) // mbin8 + 8 + [8]data // INVALID_{TID,OID} must be NIL on the wire
return s.emit("if uint64(%s) == %v {", path, INVALID_ID)
s.emit("%v += 1 // mnil", s.var_("size"))
s.emit("} else {")
s.emit("%v += 1+1+8 // mbin8 + 8 + [8]data", s.var_("size"))
s.emit("}")
return
} }
// enums are encoded as extensions // enums are encoded as extensions
...@@ -997,12 +1009,18 @@ func (e *encoderM) genBasic(path string, typ *types.Basic, userType types.Type) ...@@ -997,12 +1009,18 @@ func (e *encoderM) genBasic(path string, typ *types.Basic, userType types.Type)
upath = fmt.Sprintf("%s(%s)", typ.Name(), upath) upath = fmt.Sprintf("%s(%s)", typ.Name(), upath)
} }
// zodb.Tid and zodb.Oid are encoded as [8]bin XXX or nil // zodb.Tid and zodb.Oid are encoded as [8]bin or nil
if userType == zodbTid || userType == zodbOid { if userType == zodbTid || userType == zodbOid {
e.emit("if %s == %v {", path, INVALID_ID) // INVALID_{TID,OID} =>
e.emit("data[%v] = msgpack.Nil", e.n) // mnil
e.emit("data = data[%v:]", e.n + 1)
e.emit("} else {")
e.emit("data[%v] = byte(msgpack.Bin8)", e.n); e.n++ e.emit("data[%v] = byte(msgpack.Bin8)", e.n); e.n++
e.emit("data[%v] = 8", e.n); e.n++ e.emit("data[%v] = 8", e.n); e.n++
e.emit("binary.BigEndian.PutUint64(data[%v:], uint64(%s))", e.n, path) e.emit("binary.BigEndian.PutUint64(data[%v:], uint64(%s))", e.n, path)
e.n += 8 e.n += 8
e.resetPos()
e.emit("}")
return return
} }
...@@ -1456,6 +1474,17 @@ func (e *encoderCommon) genSliceCommon(xe CodeGenCustomize, path string, typ *ty ...@@ -1456,6 +1474,17 @@ func (e *encoderCommon) genSliceCommon(xe CodeGenCustomize, path string, typ *ty
e.emit("}") e.emit("}")
} }
// data = data[n:]
// n = 0
//
// XXX duplication wrt decoderCommon.resetPost
func (e *encoderCommon) resetPos() {
if e.n != 0 {
e.emit("data = data[%v:]", e.n)
e.n = 0
}
}
func (d *decoderN) genSliceHead(assignto string, typ *types.Slice, obj types.Object) { func (d *decoderN) genSliceHead(assignto string, typ *types.Slice, obj types.Object) {
d.genBasic("l:", types.Typ[types.Uint32], nil) d.genBasic("l:", types.Typ[types.Uint32], nil)
} }
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment