wcfs: v↑ neo/go

This brings ZODB/go updates to support Python3 from
neo!8 that we will need to
support Python3(*). For now do not add any py3 support on wcfs side - just
update neo/go with preserving wcfs/py2 in working state.

Note that this ZODB/go update brings backward incompatible changes to
activate StrictUnicode and PyDict modes(+) on unpickling and also changes
API of zodb.Map to follow that PyDict mode. We thus need to update
go bits inside wcfs because else compilation and tests, expectedly,
start to break in the following ways

    # lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest
    internal/xbtree/xbtreetest/treeenv.go:292:24: zroot.Data undefined (type *zodb.Map has no field or method Data)

    # lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata [lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata.test]
    ./δftail_test.go:737:16: zroot.Data undefined (type *zodb.Map has no field or method Data)

    --- FAIL: TestZBlk (0.00s)
        --- FAIL: TestZBlk/py2_pickle1 (0.00s)
    panic: ZBlk0(0000000000000002): loadBlkData: wendelin.bigfile.file_zodb.ZBlk0(0000000000000002): activate: pysetstate: expect str; got ogórek.ByteString [recovered]
            panic: ZBlk0(0000000000000002): loadBlkData: wendelin.bigfile.file_zodb.ZBlk0(0000000000000002): activate: pysetstate: expect str; got ogórek.ByteString

    --- FAIL: TestΔFtail (0.98s)
    panic: root['treegen/values']: key ["a"]: expected str, got ogórek.ByteString [recovered]
            panic: file:///tmp/TestΔFtail4275282346/001/1.fs: @03fb8a25642d1e88: get blktab: root['treegen/values']: key ["a"]: expected str, got ogórek.ByteString [recovered]
            panic: file:///tmp/TestΔFtail4275282346/001/1.fs: @03fb8a25642d1e88: get blktab: root['treegen/values']: key ["a"]: expected str, got ogórek.ByteString

    --- FAIL: TestΔBTail (1.13s)
    panic: root['treegen/values']: key ["a"]: expected str, got ogórek.ByteString [recovered]
            panic: file:///tmp/TestΔBTail3238267558/001/1.fs: @03fb8a212cd06933: get blktab: root['treegen/values']: key ["a"]: expected str, got ogórek.ByteString [recovered]
            panic: file:///tmp/TestΔBTail3238267558/001/1.fs: @03fb8a212cd06933: get blktab: root['treegen/values']: key ["a"]: expected str, got ogórek.ByteString

The changes are minimal just to keep wcfs/py2 working ok. We will do more changes in the follow-up patches to support py3.

All the changes in go.mod and go.sum are produced automatically by sole `go get lab.nexedi.com/kirr/neo/go@t+ypy3`.

(*) see neo@efde5253...6235fb60 for full list of changes.
(+) see neo@078b6262, neo@1e87fe5b,
    https://github.com/kisielk/og-rek/pull/68 and https://github.com/kisielk/og-rek/pull/75 for details.

wcfs/go.mod

@@ -10,8 +10,8 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/shirou/gopsutil/v4 v4.24.8
 	github.com/stretchr/testify v1.9.0
-	lab.nexedi.com/kirr/go123 v0.0.0-20230822135329-95433de34faf
-	lab.nexedi.com/kirr/neo/go v0.0.0-20240918161556-efde5253dc81
+	lab.nexedi.com/kirr/go123 v0.0.0-20240626173136-48920809d24c
+	lab.nexedi.com/kirr/neo/go v0.0.0-20240924102820-6235fb602308
 )
 
 require (
@@ -20,19 +20,20 @@ require (
 	github.com/aristanetworks/gomap v0.0.0-20230726210543-f4e41046dced // indirect
 	github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
-	github.com/philhofer/fwd v1.1.1 // indirect
+	github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/shamaton/msgpack v1.2.1 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/someonegg/gocontainer v1.0.0 // indirect
-	github.com/tinylib/msgp v1.1.6 // indirect
+	github.com/tinylib/msgp v1.2.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 // indirect
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
 	golang.org/x/sys v0.24.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

wcfs/go.sum

@@ -29,6 +29,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -84,6 +86,8 @@ github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vyg
 github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
 github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
 github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 h1:jYi87L8j62qkXzaYHAQAhEapgukhenIMZRBKTNRLHJ4=
+github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -111,6 +115,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw=
 github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
+github.com/tinylib/msgp v1.2.0 h1:0uKB/662twsVBpYUPbokj4sTSKhWFKB7LopO2kWK8lY=
+github.com/tinylib/msgp v1.2.0/go.mod h1:2vIGs3lcUo8izAATNobrCHevYZC/LMsJtw4JPiYPHro=
 github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
@@ -119,6 +125,8 @@ github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+l
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 h1:lGdhQUN/cnWdSH3291CUuxSEqc+AsGTiDxPP3r2J0l4=
+go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -223,9 +231,13 @@ lab.nexedi.com/kirr/go-fuse/v2 v2.4.2-0.20231211215333-9f9ad4a1c7cc/go.mod h1:xK
 lab.nexedi.com/kirr/go123 v0.0.0-20211124154638-01e8697d1901/go.mod h1:pwDpdCuvtz0QxisDzV/z9eUb9zc/rMQec520h4i8VWQ=
 lab.nexedi.com/kirr/go123 v0.0.0-20230822135329-95433de34faf h1:UwAEraoydFHxDqudGYtxBpyXbt9SqInI657OKf9VMJc=
 lab.nexedi.com/kirr/go123 v0.0.0-20230822135329-95433de34faf/go.mod h1:pwDpdCuvtz0QxisDzV/z9eUb9zc/rMQec520h4i8VWQ=
+lab.nexedi.com/kirr/go123 v0.0.0-20240626173136-48920809d24c h1:GboU09uDjU09xORiCgCco5hPI4pmMOw6Qye2hJNal70=
+lab.nexedi.com/kirr/go123 v0.0.0-20240626173136-48920809d24c/go.mod h1:pwDpdCuvtz0QxisDzV/z9eUb9zc/rMQec520h4i8VWQ=
 lab.nexedi.com/kirr/neo/go v0.0.0-20240723085959-839ee634bd66 h1:xAQcab3p0CiJ36aLqNGB8kH6hpsalgOnEL9D5CZgoN0=
 lab.nexedi.com/kirr/neo/go v0.0.0-20240723085959-839ee634bd66/go.mod h1:0Wk1qKrdjMWr8njsuBIbgPMBNjPlNFozuBDe15Lqq6A=
 lab.nexedi.com/kirr/neo/go v0.0.0-20240806095154-6fb93a602cbe h1:E8qMqhZJlVtsNTzVxdj2zsDq8XJx/QOBODGhid9gqxM=
 lab.nexedi.com/kirr/neo/go v0.0.0-20240806095154-6fb93a602cbe/go.mod h1:0Wk1qKrdjMWr8njsuBIbgPMBNjPlNFozuBDe15Lqq6A=
 lab.nexedi.com/kirr/neo/go v0.0.0-20240918161556-efde5253dc81 h1:eleDgtjbfZ2jz89tyyJCQAjFC1d6lGtzYLREpCORfrI=
 lab.nexedi.com/kirr/neo/go v0.0.0-20240918161556-efde5253dc81/go.mod h1:YzPDKaJIPNRCvRrpLDHP54ggZbLKlH3lErGX8NILtCQ=
+lab.nexedi.com/kirr/neo/go v0.0.0-20240924102820-6235fb602308 h1:KMS9m8EbI05yKK/482Q1+DrzJyzD1pTSMHsZWwXczPA=
+lab.nexedi.com/kirr/neo/go v0.0.0-20240924102820-6235fb602308/go.mod h1:P7HdxcOKE4FUf2/KI7UyfjRmQ97YYcXaoYQog7rsJNs=

wcfs/internal/xbtree/xbtreetest/treeenv.go

-// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+// Copyright (C) 2020-2024  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -26,6 +26,7 @@ import (
 	"sort"
 	"testing"
 
+	pickle "github.com/kisielk/og-rek"
 	"lab.nexedi.com/kirr/go123/exc"
 	"lab.nexedi.com/kirr/neo/go/transaction"
 	"lab.nexedi.com/kirr/neo/go/zodb"
@@ -289,7 +290,7 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo {
 	err = zroot.PActivate(ctx); X(err)
 	defer zroot.PDeactivate()
 
-	xzblkdir, ok := zroot.Data["treegen/values"]
+	xzblkdir, ok := zroot.Get_("treegen/values")
 	if !ok {
 		exc.Raisef("root['treegen/values'] missing")
 	}
@@ -301,8 +302,8 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo {
 	err = zblkdir.PActivate(ctx); X(err)
 	defer zblkdir.PDeactivate()
 
-	for xname, xzblk := range zblkdir.Data {
-		name, ok := xname.(string)
+	zblkdir.Iter()(func(xname, xzblk any) bool {
+		name, ok := xname.(pickle.ByteString)
 		if !ok {
 			exc.Raisef("root['treegen/values']: key [%q]: expected str, got %T", xname, xname)
 		}
@@ -314,8 +315,10 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo {
 
 		oid := zblk.POid()
 		data := xzgetBlkData(ctx, zconn, oid)
-		blkTab[oid] = ZBlkInfo{name, data}
-	}
+		blkTab[oid] = ZBlkInfo{string(name), data}
+
+		return true
+	})
 
 	return blkTab
 }

wcfs/internal/zdata/zblk.go

@@ -79,7 +79,7 @@ type ZBlk0 struct {
 	zodb.Persistent
 
 	// NOTE py source uses bytes(buf) but on python2 it still results in str
-	blkdata string
+	blkdata pickle.ByteString
 }
 
 type zBlk0State ZBlk0 // hide state methods from public API
@@ -96,7 +96,7 @@ func (zb *zBlk0State) PyGetState() interface{} {
 
 // PySetState implements zodb.PyStateful.
 func (zb *zBlk0State) PySetState(pystate interface{}) error {
-	blkdata, ok := pystate.(string)
+	blkdata, ok := pystate.(pickle.ByteString)
 	if !ok {
 		return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate))
 	}
@@ -115,7 +115,7 @@ func (zb *ZBlk0) LoadBlkData(ctx context.Context) (_ []byte, _ zodb.Tid, err err
 	}
 	defer zb.PDeactivate()
 
-	return mem.Bytes(zb.blkdata), zb.PSerial(), nil
+	return mem.Bytes(string(zb.blkdata)), zb.PSerial(), nil
 }
 
 // ---- ZBlk1 ---
@@ -125,7 +125,7 @@ type ZData struct {
 	zodb.Persistent
 
 	// NOTE py source uses bytes(buf) but on python2 it still results in str
-	data string
+	data pickle.ByteString
 }
 
 type zDataState ZData // hide state methods from public API
@@ -142,7 +142,7 @@ func (zd *zDataState) PyGetState() interface{} {
 
 // PySetState implements zodb.PyStateful.
 func (zd *zDataState) PySetState(pystate interface{}) error {
-	data, ok := pystate.(string)
+	data, ok := pystate.(pickle.ByteString)
 	if !ok {
 		return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate))
 	}

wcfs/internal/zdata/δftail_test.go

-// Copyright (C) 2019-2022  Nexedi SA and Contributors.
+// Copyright (C) 2019-2024  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -734,7 +734,7 @@ func (t *T) XLoadZFile(ctx context.Context, zconn *zodb.Connection) (zfile *ZBig
 	xzroot, err := zconn.Get(ctx, 0);  X(err)
 	zroot := xzroot.(*zodb.Map)
 	err = zroot.PActivate(ctx);  X(err)
-	zfile = zroot.Data["treegen/file"].(*ZBigFile)
+	zfile = zroot.Get("treegen/file").(*ZBigFile)
 	zroot.PDeactivate()
 	err = zfile.PActivate(ctx);  X(err)
 	blksize    = zfile.blksize