Commit 487a3033 authored by Levin Zimmermann's avatar Levin Zimmermann

proto: Update 'Compression' to int to support different compression algorithms

With nexedi/neoppod@fd80cc30 NEO/py added support to encode the compression
algorithm with the 'Compression' parameter. Before this, compression could
only be true (= with compression) or false (= without compression). Now
the absence of compression is encoded with 0. Any other number than 0
encodes a compression algorithm. The mapping is currently:

	1 = zlib

In the future, 2 could mean zstd [1].

[1] https://github.com/facebook/zstd/issues/1134
parent d8634701
...@@ -381,7 +381,10 @@ func (c *Client) Load(ctx context.Context, xid zodb.Xid) (buf *mem.Buf, serial z ...@@ -381,7 +381,10 @@ func (c *Client) Load(ctx context.Context, xid zodb.Xid) (buf *mem.Buf, serial z
} }
} }
if resp.Compression { if resp.Compression != 0 {
if resp.Compression != 1 {
return nil, 0, fmt.Errorf("unsupported compression algorithm: %v", resp.Compression)
}
buf2 := &mem.Buf{Data: nil} buf2 := &mem.Buf{Data: nil}
udata, err := xzlib.Decompress(buf.Data) udata, err := xzlib.Decompress(buf.Data)
buf.Release() buf.Release()
......
...@@ -257,7 +257,7 @@ func _TestMasterStorage(t0 *tEnv) { ...@@ -257,7 +257,7 @@ func _TestMasterStorage(t0 *tEnv) {
Oid: xid1.Oid, Oid: xid1.Oid,
Serial: serial1, Serial: serial1,
NextSerial: proto.INVALID_TID, NextSerial: proto.INVALID_TID,
Compression: false, Compression: 0,
Data: buf1, Data: buf1,
DataSerial: 0, // XXX DataSerial: 0, // XXX
Checksum: sha1.Sum(buf1.Data), Checksum: sha1.Sum(buf1.Data),
...@@ -292,7 +292,7 @@ func _TestMasterStorage(t0 *tEnv) { ...@@ -292,7 +292,7 @@ func _TestMasterStorage(t0 *tEnv) {
Oid: xid1prev.Oid, Oid: xid1prev.Oid,
Serial: serial1prev, Serial: serial1prev,
NextSerial: serial1, NextSerial: serial1,
Compression: false, Compression: 0,
Data: buf1prev, Data: buf1prev,
DataSerial: 0, // XXX DataSerial: 0, // XXX
Checksum: sha1.Sum(buf1prev.Data), Checksum: sha1.Sum(buf1prev.Data),
......
...@@ -336,6 +336,11 @@ func (a *Address) neoDecodeN(b []byte) (uint64, bool) { ...@@ -336,6 +336,11 @@ func (a *Address) neoDecodeN(b []byte) (uint64, bool) {
// Checksum is a SHA1 hash. // Checksum is a SHA1 hash.
type Checksum [20]byte type Checksum [20]byte
// Compression is an integer that tells the compression algorithm:
// 0 = no compression algorithm is used
// 1 = zlib is used
type Compression uint64
// PTid is Partition Table identifier. // PTid is Partition Table identifier.
// //
// Zero value means "invalid id" (<-> None in py.PPTID, nil in msgpack) // Zero value means "invalid id" (<-> None in py.PPTID, nil in msgpack)
...@@ -713,7 +718,7 @@ type AnswerRebaseObject struct { ...@@ -713,7 +718,7 @@ type AnswerRebaseObject struct {
Serial zodb.Tid Serial zodb.Tid
ConflictSerial zodb.Tid ConflictSerial zodb.Tid
// FIXME POption('data') // FIXME POption('data')
Compression bool Compression Compression
Checksum Checksum Checksum Checksum
Data *mem.Buf Data *mem.Buf
} }
...@@ -727,7 +732,7 @@ type AnswerRebaseObject struct { ...@@ -727,7 +732,7 @@ type AnswerRebaseObject struct {
type StoreObject struct { type StoreObject struct {
Oid zodb.Oid Oid zodb.Oid
Serial zodb.Tid Serial zodb.Tid
Compression bool Compression Compression
Checksum Checksum Checksum Checksum
Data []byte // TODO -> msg.Buf, separately (for writev) Data []byte // TODO -> msg.Buf, separately (for writev)
DataSerial zodb.Tid DataSerial zodb.Tid
...@@ -782,7 +787,7 @@ type AnswerObject struct { ...@@ -782,7 +787,7 @@ type AnswerObject struct {
Oid zodb.Oid Oid zodb.Oid
Serial zodb.Tid Serial zodb.Tid
NextSerial zodb.Tid NextSerial zodb.Tid
Compression bool Compression Compression
Checksum Checksum Checksum Checksum
Data *mem.Buf // TODO encode -> separately (for writev) Data *mem.Buf // TODO encode -> separately (for writev)
DataSerial zodb.Tid DataSerial zodb.Tid
...@@ -1217,7 +1222,7 @@ type AddTransaction struct { ...@@ -1217,7 +1222,7 @@ type AddTransaction struct {
type AddObject struct { type AddObject struct {
Oid zodb.Oid Oid zodb.Oid
Serial zodb.Tid Serial zodb.Tid
Compression bool Compression Compression
Checksum Checksum Checksum Checksum
Data *mem.Buf Data *mem.Buf
DataSerial zodb.Tid DataSerial zodb.Tid
......
...@@ -182,7 +182,7 @@ func TestMsgMarshal(t *testing.T) { ...@@ -182,7 +182,7 @@ func TestMsgMarshal(t *testing.T) {
{&StoreObject{ {&StoreObject{
Oid: 0x0102030405060708, Oid: 0x0102030405060708,
Serial: 0x0a0b0c0d0e0f0102, Serial: 0x0a0b0c0d0e0f0102,
Compression: false, Compression: 0,
Checksum: Checksum{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20}, // XXX simpler? Checksum: Checksum{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20}, // XXX simpler?
Data: []byte("hello world"), Data: []byte("hello world"),
DataSerial: 0x0a0b0c0d0e0f0103, DataSerial: 0x0a0b0c0d0e0f0103,
...@@ -190,6 +190,7 @@ func TestMsgMarshal(t *testing.T) { ...@@ -190,6 +190,7 @@ func TestMsgMarshal(t *testing.T) {
}, },
hex("01020304050607080a0b0c0d0e0f010200") + hex("01020304050607080a0b0c0d0e0f010200") +
hex("00000000000000") +
hex("0102030405060708090a0b0c0d0e0f1011121314") + hex("0102030405060708090a0b0c0d0e0f1011121314") +
hex("0000000b") + "hello world" + hex("0000000b") + "hello world" +
hex("0a0b0c0d0e0f01030a0b0c0d0e0f0104"), hex("0a0b0c0d0e0f01030a0b0c0d0e0f0104"),
...@@ -198,7 +199,7 @@ func TestMsgMarshal(t *testing.T) { ...@@ -198,7 +199,7 @@ func TestMsgMarshal(t *testing.T) {
hex("97") + hex("97") +
hex("c408") + hex("0102030405060708") + hex("c408") + hex("0102030405060708") +
hex("c408") + hex("0a0b0c0d0e0f0102") + hex("c408") + hex("0a0b0c0d0e0f0102") +
hex("c2") + hex("00") +
hex("c414") + hex("0102030405060708090a0b0c0d0e0f1011121314") + hex("c414") + hex("0102030405060708090a0b0c0d0e0f1011121314") +
hex("c40b") + "hello world" + hex("c40b") + "hello world" +
hex("c408") + hex("0a0b0c0d0e0f0103") + hex("c408") + hex("0a0b0c0d0e0f0103") +
......
This diff is collapsed.
...@@ -87,7 +87,7 @@ func (f *Backend) Load(ctx context.Context, xid zodb.Xid) (*proto.AnswerObject, ...@@ -87,7 +87,7 @@ func (f *Backend) Load(ctx context.Context, xid zodb.Xid) (*proto.AnswerObject,
Serial: serial, Serial: serial,
NextSerial: nextSerial, NextSerial: nextSerial,
Compression: false, Compression: 0,
Data: buf, Data: buf,
Checksum: xsha1.NEOSum(buf.Data), // XXX computing every time Checksum: xsha1.NEOSum(buf.Data), // XXX computing every time
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment