Commit 5a2b639c authored by Carlos Ramos Carreño's avatar Carlos Ramos Carreño

Parse `bytes` as str in Golang.

[In Go, strings can contain arbitrary bytes](https://go.dev/blog/strings)
(similar to the ones in Python 2), and there is no explicit `bytes`
class.
Thus, all the code that deals with arbitrary data in Go is currently
using the string type.

However, the [ogórek package](https://github.com/kisielk/og-rek) used
to load Python pickles in Go, decodes Python 3 `bytes` objects as a
special string type `ogórek.Bytes`, incompatible with APIs expecting
normal strings.
This causes errors such as the following:
```
--- FAIL: TestΔFtail (1.31s)
panic: @03f97a3b982b7bcc: get blkdata from obj 0000000000000002: ZBlk0(0000000000000002): loadBlkData: wendelin.bigfile.file_zodb.ZBlk0(0000000000000002): activate: pysetstate: expect str; got ogórek.Bytes [recovered]
        panic: file:///tmp/TestΔFtail493253177/001/1.fs: @03f97a3b982b7bcc: get blktab: @03f97a3b982b7bcc: get blkdata from obj 0000000000000002: ZBlk0(0000000000000002): loadBlkData: wendelin.bigfile.file_zodb.ZBlk0(0000000000000002): activate: pysetstate: expect str; got ogórek.Bytes [recovered]
        panic: file:///tmp/TestΔFtail493253177/001/1.fs: @03f97a3b982b7bcc: get blktab: @03f97a3b982b7bcc: get blkdata from obj 0000000000000002: ZBlk0(0000000000000002): loadBlkData: wendelin.bigfile.file_zodb.ZBlk0(0000000000000002): activate: pysetstate: expect str; got ogórek.Bytes
```

In order to fix that, we have copied the [`Xstrbytes` function from NEO](
https://lab.nexedi.com/kirr/neo/-/blob/f7776fc1689b0d62e582b132ecc017ab72dc3b23/go/zodb/internal/pickletools/pickletools.go#L49-64)
in the `pycompat.go` file, which accepts either a string or an
`ogórek.Bytes` object and returns the corresponding string.

We have used that function whenever a `bytes` object could be present.
parent eca099bc
...@@ -3,7 +3,9 @@ ...@@ -3,7 +3,9 @@
package pycompat package pycompat
import ( import (
"fmt"
"math/big" "math/big"
pickle "github.com/kisielk/og-rek"
) )
// Int64 tries to convert unpickled Python value to int64. // Int64 tries to convert unpickled Python value to int64.
...@@ -12,14 +14,31 @@ import ( ...@@ -12,14 +14,31 @@ import (
// //
// XXX + support for float? // XXX + support for float?
func Int64(xv interface{}) (v int64, ok bool) { func Int64(xv interface{}) (v int64, ok bool) {
switch v := xv.(type) { switch v := xv.(type) {
case int64: case int64:
return v, true return v, true
case *big.Int: case *big.Int:
if v.IsInt64() { if v.IsInt64() {
return v.Int64(), true return v.Int64(), true
} }
} }
return 0, false return 0, false
} }
// Xstrbytes verifies and extacts str|bytes from unpickled value.
func Xstrbytes(x interface{}) (string, error) {
var s string
switch x := x.(type) {
default:
return "", fmt.Errorf("expect str|bytes; got %T", x)
case string:
s = x
case pickle.Bytes:
s = string(x)
}
return s, nil
}
\ No newline at end of file
...@@ -31,6 +31,7 @@ import ( ...@@ -31,6 +31,7 @@ import (
"lab.nexedi.com/kirr/neo/go/zodb" "lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib" "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/pycompat"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb" "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
) )
...@@ -302,8 +303,8 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo { ...@@ -302,8 +303,8 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo {
defer zblkdir.PDeactivate() defer zblkdir.PDeactivate()
for xname, xzblk := range zblkdir.Data { for xname, xzblk := range zblkdir.Data {
name, ok := xname.(string) name, err := pycompat.Xstrbytes(xname)
if !ok { if err != nil {
exc.Raisef("root['treegen/values']: key [%q]: expected str, got %T", xname, xname) exc.Raisef("root['treegen/values']: key [%q]: expected str, got %T", xname, xname)
} }
......
...@@ -97,8 +97,8 @@ func (zb *zBlk0State) PyGetState() interface{} { ...@@ -97,8 +97,8 @@ func (zb *zBlk0State) PyGetState() interface{} {
// PySetState implements zodb.PyStateful. // PySetState implements zodb.PyStateful.
func (zb *zBlk0State) PySetState(pystate interface{}) error { func (zb *zBlk0State) PySetState(pystate interface{}) error {
blkdata, ok := pystate.(string) blkdata, err := pycompat.Xstrbytes(pystate)
if !ok { if err != nil {
return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate)) return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate))
} }
...@@ -143,8 +143,8 @@ func (zd *zDataState) PyGetState() interface{} { ...@@ -143,8 +143,8 @@ func (zd *zDataState) PyGetState() interface{} {
// PySetState implements zodb.PyStateful. // PySetState implements zodb.PyStateful.
func (zd *zDataState) PySetState(pystate interface{}) error { func (zd *zDataState) PySetState(pystate interface{}) error {
data, ok := pystate.(string) data, err := pycompat.Xstrbytes(pystate)
if !ok { if err != nil {
return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate)) return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate))
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment