wcfs: Add zdata package to load ZBlk/ZBigFile data

Add functionality to load objects from ZODB as saved by py wendelin.core.
Mostly straightforward code.
The main part is in zblk.go .

Contrary to python implementation, go can load ZBlk1's subobjects in
parallel, which, given scalable ZODB storage, can be significantly
faster compared to serially loading all ZData subobjects as py code
does.

TODO test wrt data saved by Python3.

Some preliminary history:

kirr/wendelin.core@878b2787 X draft loading
kirr/wendelin.core@bf9a7405 X No longer rely on ZODB cache invariant for invalidations
kirr/wendelin.core@0d62b05e X Adjust to btree.VGet & friends signature change to include keycov in visit callback
kirr/wendelin.core@b74dda09 X Start switching Track from Track(key) to Track(keycov)

wcfs/go.mod

 module lab.nexedi.com/nexedi/wendelin.core/wcfs
 
 go 1.14
+
+require (
+	github.com/johncgriffin/overflow v0.0.0-20211019200055-46fa312c352c
+	github.com/kisielk/og-rek v1.1.1-0.20210310094122-8def3d024dac
+	github.com/stretchr/testify v1.7.0
+	lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408
+	lab.nexedi.com/kirr/neo/go v0.0.0-20211004111643-c74a5a3cd0d0
+)

wcfs/go.sum

+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk=
+crawshaw.io/sqlite v0.3.2 h1:N6IzTjkiw9FItHAa0jp+ZKC6tuLzXqAYIv+ccIWos1I=
+crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7 h1:6ZJZdzkbvKb6HRXmZ12ICZ0IbqfR+0Cd2C0IutWHHIA=
+github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7/go.mod h1:ROY4muaTWpoeQAx/oUkvxe9zKCmgU5xDGXsfEbA+omc=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
+github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 h1:MZRmHqDBd0vxNwenEbKSQqRVT24d3C05ft8kduSwlqM=
+github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52 h1:0NmERxogGTU8hgzOhRKNoKivtBZkDW29GeuJtK9e0sc=
+github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gwenn/gosqlite v0.0.0-20201008200117-82e079acf5b6/go.mod h1:WBYs9HfQGOYDCz7rFwMk7aHkbTTB0cUkQe3pZQARvIg=
+github.com/gwenn/yacr v0.0.0-20200110180258-a66d8c42d0ff/go.mod h1:5SNcBGxZ5OaJAMJCSI/x3V7SGsvXqbwnwP/sHZLgYsw=
+github.com/gwenn/yacr v0.0.0-20200112083327-bbe82c1f4d60/go.mod h1:Ps/gikIXcn2rRmeP0HQ9EvUYJrfrjAi51Wg8acsrkP0=
+github.com/johncgriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:2n/HCxBM7oa5PNCPKIhV26EtJkaPXFfcVojPAT3ujTU=
+github.com/johncgriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:B9OPZOhZ3FIi6bu54lAgCMzXLh11Z7ilr3rOr/ClP+E=
+github.com/kisielk/og-rek v1.1.0 h1:u10TvQbPtrlY/6H4+BiFsBywwSVTGFsx0YOVtpx3IbI=
+github.com/kisielk/og-rek v1.1.0/go.mod h1:6ihsOSzSAxR/65S3Bn9zNihoEqRquhDQZ2c6I2+MG3c=
+github.com/kisielk/og-rek v1.1.1-0.20210310094122-8def3d024dac h1:si+TV++k+F5H30GL9a7lrRxvxzhdo8sD4rX0pKZHeNI=
+github.com/kisielk/og-rek v1.1.1-0.20210310094122-8def3d024dac/go.mod h1:6ihsOSzSAxR/65S3Bn9zNihoEqRquhDQZ2c6I2+MG3c=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
+github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/shamaton/msgpack v1.2.1 h1:40cwW7YAEdOIxcxIsUkAxSMUyYWZUyNiazI5AyiBntI=
+github.com/shamaton/msgpack v1.2.1/go.mod h1:ibiaNQRTCUISAYkkyOpaSCEBiCAxXe6u6Mu1sQ6945U=
+github.com/soheilhy/cmux v0.1.5-0.20210205191134-5ec6847320e5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
+github.com/someonegg/gocontainer v1.0.0 h1:9MMUFbQf7g+g9sMG4ggBHPDS1+Iz+wd9Ee/O4BNRdw0=
+github.com/someonegg/gocontainer v1.0.0/go.mod h1:zGJcXRK0ikzEYPFKTaFXi6UU/ulNuJypfADX4UQGtMw=
+github.com/someonegg/gox v1.0.0/go.mod h1:pngAcWxBFnyYM4oY+h9Rgv0WaikLkSfY5dBYuabUnBE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tinylib/msgp v1.1.5 h1:2gXmtWueD2HefZHQe1QOy9HVzmFrLOVvsXwXBQ0ayy0=
+github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=
+github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b h1:kHlr0tATeLRMEiZJu5CknOw/E8V6h69sXXQFGoPtjcc=
+golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc/examples v0.0.0-20210301210255-fc8f38cccf75/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+lab.nexedi.com/kirr/go123 v0.0.0-20210128150852-c20e95f0f789/go.mod h1:1wkWl3WhmutZiho+wsE7ymOKvRkN7hV3YZtL0f0gXTo=
+lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408 h1:H7YpNUDfTSvvRpKivUMrL9C09tQssQ6brEoX6K/OxOw=
+lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408/go.mod h1:pwDpdCuvtz0QxisDzV/z9eUb9zc/rMQec520h4i8VWQ=
+lab.nexedi.com/kirr/neo/go v0.0.0-20211004111643-c74a5a3cd0d0 h1:rmfVDj/IaTiMUFAXTKyW993f1G5IxKcZ1vtcrrqscpk=
+lab.nexedi.com/kirr/neo/go v0.0.0-20211004111643-c74a5a3cd0d0/go.mod h1:llI3hcJJMACe+rYuXUfS5dljjwIrlBMfJ1ZeRcey96A=

wcfs/internal/pycompat/pycompat.go

+// XXX dup from neo/go/zodb/internal/pickletools
+
+package pycompat
+
+import (
+	"math/big"
+)
+
+// Int64 tries to convert unpickled Python value to int64.
+//
+// (ogórek decodes python long as big.Int)
+//
+// XXX + support for float?
+func Int64(xv interface{}) (v int64, ok bool) {
+        switch v := xv.(type) {
+        case int64:
+                return v, true
+        case *big.Int:
+                if v.IsInt64() {
+                        return v.Int64(), true
+                }
+        }
+
+        return 0, false
+}

wcfs/internal/xzodb/xzodb.go

+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package xzodb compements package zodb.
+package xzodb
+
+import (
+	"fmt"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+
+// TypeOf returns string for object's type.
+//
+//	- for ZODB objects, it uses zodb.ClassOf, which in particular supports
+//	  printing zodb.Broken with details properly.
+//
+//	- for other objects, it uses %T.
+func TypeOf(obj interface{}) string {
+	switch obj := obj.(type) {
+	case zodb.IPersistent:
+		return zodb.ClassOf(obj)
+
+	default:
+		return fmt.Sprintf("%T", obj)
+	}
+}

wcfs/internal/zdata/misc.go

+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package zdata
+
+import (
+	"fmt"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+// tidmax returns latest revision.
+func tidmax(a, b zodb.Tid) zodb.Tid {
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+// tidmin returns earliest revision.
+func tidmin(a, b zodb.Tid) zodb.Tid {
+	if a < b {
+		return a
+	} else {
+		return b
+	}
+}
+
+func panicf(format string, argv ...interface{}) {
+	panic(fmt.Sprintf(format, argv...))
+}

wcfs/internal/zdata/testdata/.gitignore

+/*.lock
+/*.tmp
+/*.tr[0-9]

wcfs/internal/zdata/testdata/zblk_test_gen.py

+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+# Copyright (C) 2018-2021  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""zblk_test_gen.py generates test data for zblk_test.go"""
+
+from ZODB.DB import DB
+from ZODB.utils import u64
+from wendelin.bigfile.file_zodb import ZBlk0, ZBlk1, ZBigFile
+from BTrees.IOBTree import IOBTree, IOBucket
+from numpy import arange
+import os, os.path, transaction
+
+import zodbtools.test.gen_testdata  # to make time predictable
+from zodbtools.test.gen_testdata import run_with_zodb4py2_compat
+
+K = 1024
+
+def main():
+    run_with_zodb4py2_compat(main2)
+
+def main2():
+    outfs = "testdata/zblk.fs"
+    rm_f(outfs)
+    rm_f(outfs + ".index")
+    db = DB(outfs)
+    conn = db.open()
+    root = conn.root()
+
+    # one ZBigFile with two data blocks
+    root['zbigf'] = zf = ZBigFile(2*1024*K)
+    root['zblk0'] = z0 = ZBlk0()
+    root['zblk1'] = z1 = ZBlk1()    # also covers ZData
+
+    # zblk0 with small data
+    z0.setblkdata(brange32(16*K))
+    # zblk1 with head + large hole + tail.
+    # head and tail are 128K because if smaller, z1.chunktab would serialize without buckets.
+    z1ht = 128*K
+    z1.setblkdata(brange32(z1ht) + bzeros(zf.blksize - 2*z1ht) + breverse(brange32(z1ht)))
+    zf.blktab[1] = z0
+    zf.blktab[3] = z1
+
+    # make sure there is at least one bucket in z1's chunktab
+    #
+    # we need at least one bucket to verify how zblk.go handles chunktab bucket loading;
+    # more than 1 bucket just wastes space, however with only 1 bucket a tree
+    # is serialized to have leaf nodes directly in the tree state.
+    # -> verify for ==2 buckets.
+    assertIOBTreeHas2Buckets(z1.chunktab)
+
+    transaction.commit()
+
+    with open("ztestdata_zblk_test.go", "w") as f:
+        def emit(v):
+            print >>f, v
+        emit("// Code generated by %s; DO NOT EDIT." % __file__)
+        emit("package zdata\n")
+        emit('import "lab.nexedi.com/kirr/neo/go/zodb"\n')
+
+        emit("const zf_blksize = %d" % zf.blksize)
+        emit("const zf_size    = %d" % ((zf.blktab.maxKey()+1)*zf.blksize))
+        emit("const z0_oid     = zodb.Oid(%d)" % u64(z0._p_oid))
+        emit("const z1_oid     = zodb.Oid(%d)" % u64(z1._p_oid))
+        emit("const zf_oid     = zodb.Oid(%d)" % u64(zf._p_oid))
+        emit("const z0_rev     = zodb.Tid(0x%x)"  % u64(z0._p_serial))
+        emit("const z1_rev     = zodb.Tid(0x%x)"  % u64(z1._p_serial))
+        emit("const z0_len     = %d" % len(z0.loadblkdata()))
+        emit("const z1_htlen   = %d" % z1ht)
+
+
+    conn.close()
+    db.close()
+
+
+# brange32 returns bytes with big-endian uint32 sequence filling them.
+# returned bytes has len == size.
+def brange32(size):
+    # 0, 1, 2, ... as u32
+    return arange(0, size//4, dtype='>u4').tobytes()
+
+# bzeros returns bytes of requested size with 0 filling them.
+def bzeros(size):
+    return b'\0'*size
+
+# breverse returns bytes in the reverse order.
+def breverse(b):
+    assert isinstance(b, bytes)
+    _ = bytearray(b)
+    _.reverse()
+    return bytes(_)
+
+# assertIOBTreeHas2Buckets asserts that IOBTree has 2 buckets.
+def assertIOBTreeHas2Buckets(t):
+    assert isinstance(t, IOBTree)
+
+    # https://github.com/zopefoundation/BTrees/blob/4.5.0-1-gc8bf24e/BTrees/BTreeTemplate.c#L1087-L1109
+    _ = t.__getstate__()
+    assert len(_) == 2, (len(_), _)
+    assert len(_[0]) == 3, _[0]                   # (bucket0, key1, bucket1)
+    assert isinstance(_[0][0], IOBucket), _[0][0] # bucket0
+    assert isinstance(_[0][1], int),      _[0][2] # key1
+    assert isinstance(_[0][2], IOBucket), _[0][2] # bucket1
+    assert isinstance(_[1],    IOBucket), _[1]    # .firstbucket
+
+# rm_f is like `rm -f` in shell.
+def rm_f(path):
+    if os.path.exists(path):
+        os.remove(path)
+
+
+if __name__ == '__main__':
+    main()

wcfs/internal/zdata/zblk.go

+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package zdata provides access for wendelin.core in-ZODB data.
+//
+// ZBlk* + ZBigFile.
+package zdata
+
+// module: "wendelin.bigfile.file_zodb"
+//
+// ZBigFile
+//	.blksize	xint
+//	.blktab		LOBtree{}  blk -> ZBlk*(blkdata)
+//
+// ZBlk0 (aliased as ZBlk)
+//	str with trailing '\0' removed.
+//
+// ZBlk1
+//	.chunktab	IOBtree{}  offset -> ZData(chunk)
+//
+// ZData
+//	str (chunk)
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"sort"
+	"sync"
+	"syscall"
+
+	"github.com/johncgriffin/overflow"
+	pickle "github.com/kisielk/og-rek"
+	"lab.nexedi.com/kirr/go123/mem"
+	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/go123/xsync"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+	"lab.nexedi.com/kirr/neo/go/zodb/btree"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/pycompat"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
+)
+
+// ZBlk is the interface that every ZBlk* block implements.
+type ZBlk interface {
+	zodb.IPersistent
+
+	// LoadBlkData loads from database and returns data block stored by this ZBlk.
+	//
+	// If returned data size is less than the block size of containing ZBigFile,
+	// the block trailing is assumed to be trailing \0.
+	//
+	// returns data and revision of ZBlk.
+	LoadBlkData(ctx context.Context) (data []byte, rev zodb.Tid, _ error)
+}
+
+var _ ZBlk = (*ZBlk0)(nil)
+var _ ZBlk = (*ZBlk1)(nil)
+
+// ---- ZBlk0 ----
+
+// ZBlk0 mimics ZBlk0 from python.
+type ZBlk0 struct {
+	zodb.Persistent
+
+	// NOTE py source uses bytes(buf) but on python2 it still results in str
+	blkdata string
+}
+
+type zBlk0State ZBlk0 // hide state methods from public API
+
+// DropState implements zodb.Ghostable.
+func (zb *zBlk0State) DropState() {
+	zb.blkdata = ""
+}
+
+// PyGetState implements zodb.PyStateful.
+func (zb *zBlk0State) PyGetState() interface{} {
+	return zb.blkdata
+}
+
+// PySetState implements zodb.PyStateful.
+func (zb *zBlk0State) PySetState(pystate interface{}) error {
+	blkdata, ok := pystate.(string)
+	if !ok {
+		return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate))
+	}
+
+	zb.blkdata = blkdata
+	return nil
+}
+
+// LoadBlkData implements ZBlk.
+func (zb *ZBlk0) LoadBlkData(ctx context.Context) (_ []byte, _ zodb.Tid, err error) {
+	defer xerr.Contextf(&err, "ZBlk0(%s): loadBlkData", zb.POid())
+
+	err = zb.PActivate(ctx)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer zb.PDeactivate()
+
+	return mem.Bytes(zb.blkdata), zb.PSerial(), nil
+}
+
+// ---- ZBlk1 ---
+
+// ZData mimics ZData from python.
+type ZData struct {
+	zodb.Persistent
+
+	// NOTE py source uses bytes(buf) but on python2 it still results in str
+	data string
+}
+
+type zDataState ZData // hide state methods from public API
+
+// DropState implements zodb.Ghostable.
+func (zd *zDataState) DropState() {
+	zd.data = ""
+}
+
+// PyGetState implements zodb.PyStateful.
+func (zd *zDataState) PyGetState() interface{} {
+	return zd.data
+}
+
+// PySetState implements zodb.PyStateful.
+func (zd *zDataState) PySetState(pystate interface{}) error {
+	data, ok := pystate.(string)
+	if !ok {
+		return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate))
+	}
+
+	zd.data = data
+	return nil
+}
+
+// ZBlk1 mimics ZBlk1 from python.
+type ZBlk1 struct {
+	zodb.Persistent
+
+	chunktab *btree.IOBTree // {} offset -> ZData(chunk)
+}
+
+type zBlk1State ZBlk1 // hide state methods from public API
+
+// DropState implements zodb.Ghostable.
+func (zb *zBlk1State) DropState() {
+	zb.chunktab = nil
+}
+
+// PyGetState implements zodb.PyStateful.
+func (zb *zBlk1State) PyGetState() interface{} {
+	return zb.chunktab
+}
+
+// PySetState implements zodb.PyStateful.
+func (zb *zBlk1State) PySetState(pystate interface{}) error {
+	chunktab, ok := pystate.(*btree.IOBTree)
+	if !ok {
+		return fmt.Errorf("expect IOBTree; got %s", xzodb.TypeOf(pystate))
+	}
+
+	zb.chunktab = chunktab
+	return nil
+}
+
+// LoadBlkData implements ZBlk.
+func (zb *ZBlk1) LoadBlkData(ctx context.Context) (_ []byte, _ zodb.Tid, err error) {
+	defer xerr.Contextf(&err, "ZBlk1(%s): loadBlkData", zb.POid())
+
+	err = zb.PActivate(ctx)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer zb.PDeactivate()
+
+	// get to all ZData objects; activate them and build
+	//
+	//	{} offset -> ZData
+	//
+	// with all ZData being live.
+	var mu sync.Mutex
+	chunktab := make(map[int32]*ZData)
+
+	// on return deactivate all ZData objects loaded in chunktab
+	defer func() {
+		for _, zd := range chunktab {
+			zd.PDeactivate()
+		}
+	}()
+
+
+	wg := xsync.NewWorkGroup(ctx)
+
+	// loadZData loads 1 ZData object into chunktab and leaves it activated.
+	loadZData := func(ctx context.Context, offset int32, zd *ZData) (err error) {
+		err = zd.PActivate(ctx)
+		if err != nil {
+			return err
+		}
+		// no PDeactivate, zd remains live
+
+		//fmt.Printf("@%d -> zdata #%s (%d)\n", offset, zd.POid(), len(zd.data))
+		mu.Lock()
+		defer mu.Unlock()
+
+		_, already := chunktab[offset]
+		if already {
+			return fmt.Errorf("duplicate offset %d", offset)
+		}
+
+		chunktab[offset] = zd
+		return nil
+	}
+
+	// loadBucket loads all ZData objects from leaf BTree bucket.
+	loadBucket := func(ctx context.Context, b *btree.IOBucket) (err error) {
+		err = b.PActivate(ctx)
+		if err != nil {
+			return err
+		}
+		defer b.PDeactivate()
+
+		// go through all bucket key/v -> chunktab
+		defer xerr.Contextf(&err, "%s(%s)", xzodb.TypeOf(b), b.POid())
+
+		//fmt.Printf("\nbucket: %v\n\n", b.Entryv())
+		for i, e := range b.Entryv() {
+			zd, ok := e.Value().(*ZData)
+			if !ok {
+				return fmt.Errorf("[%d]: !ZData (%s)", i, xzodb.TypeOf(e.Value()))
+			}
+
+			offset := e.Key()
+			if offset < 0 {
+				return fmt.Errorf("[%d]: offset < 0 (%d)", i, offset)
+			}
+			wg.Go(func(ctx context.Context) error {
+				return loadZData(ctx, offset, zd)
+			})
+		}
+
+		return nil
+	}
+
+	// loadBTree spawns loading of all BTree children.
+	var loadBTree func(ctx context.Context, t *btree.IOBTree) error
+	loadBTree = func(ctx context.Context, t *btree.IOBTree) error {
+		err := t.PActivate(ctx)
+		if err != nil {
+			return err
+		}
+		defer t.PDeactivate()
+
+		//fmt.Printf("\nbtree: %v\n\n", t.Entryv())
+		for _, e := range t.Entryv() {
+			switch child := e.Child().(type) {
+			case *btree.IOBTree:
+				wg.Go(func(ctx context.Context) error {
+					return loadBTree(ctx, child)
+				})
+
+			case *btree.IOBucket:
+				wg.Go(func(ctx context.Context) error {
+					return loadBucket(ctx, child)
+				})
+
+			default:
+				panicf("IOBTree has %s child", xzodb.TypeOf(child))
+			}
+		}
+
+		return nil
+	}
+
+	wg.Go(func(ctx context.Context) error {
+		return loadBTree(ctx, zb.chunktab)
+	})
+
+	err = wg.Wait()
+	if err != nil {
+		return nil, 0, err
+	}
+
+	// empty .chunktab -> ø
+	if len(chunktab) == 0 {
+		return nil, 0, nil
+	}
+
+	// glue all chunks from chunktab
+	offv := make([]int32, 0, len(chunktab)) // ↑
+	for off := range chunktab {
+		offv = append(offv, off)
+	}
+	sort.Slice(offv, func(i, j int) bool {
+		return offv[i] < offv[j]
+	})
+
+	//fmt.Printf("#chunktab: %d\n", len(chunktab))
+	//fmt.Printf("offv: %v\n", offv)
+
+
+	// find out whole blk len via inspecting tail chunk
+	tailStart  := offv[len(offv)-1]
+	tailChunk  := chunktab[tailStart]
+	blklen, ok := overflow.Add32(tailStart, int32(len(tailChunk.data)))
+	if !ok {
+		return nil, 0, fmt.Errorf("invalid data: blklen overflow")
+	}
+
+	// whole buffer initialized as 0 + tail_chunk
+	blkdata := make([]byte, blklen)
+	copy(blkdata[tailStart:], tailChunk.data)
+
+	// go through all chunks besides tail and extract them
+	stop := int32(0)
+	for _, start := range offv[:len(offv)-1] {
+		chunk := chunktab[start]
+		if !(start >= stop) { // verify chunks don't overlap
+			return nil, 0, fmt.Errorf("invalid data: chunks overlap")
+		}
+		stop, ok = overflow.Add32(start, int32(len(chunk.data)))
+		if !(ok && stop <= blklen) {
+			return nil, 0, fmt.Errorf("invalid data: blkdata overrun")
+		}
+		copy(blkdata[start:], chunk.data)
+	}
+
+	return blkdata, zb.PSerial(), nil
+}
+
+
+// ----------------------------------------
+
+// ZBigFile mimics ZBigFile from python.
+type ZBigFile struct {
+	zodb.Persistent
+
+	// state: (.blksize, .blktab)
+	blksize int64
+	blktab  *btree.LOBTree // {}  blk -> ZBlk*(blkdata)
+}
+
+type zBigFileState ZBigFile // hide state methods from public API
+
+// DropState implements zodb.Ghostable.
+func (bf *zBigFileState) DropState() {
+	bf.blksize = 0
+	bf.blktab  = nil
+}
+
+// PyGetState implements zodb.PyStateful.
+func (bf *zBigFileState) PyGetState() interface{} {
+	return pickle.Tuple{bf.blksize, bf.blktab}
+}
+
+// PySetState implements zodb.PyStateful.
+func (bf *zBigFileState) PySetState(pystate interface{}) (err error) {
+	t, ok := pystate.(pickle.Tuple)
+	if !ok {
+		return fmt.Errorf("expect [2](); got %s", xzodb.TypeOf(pystate))
+	}
+	if len(t) != 2 {
+		return fmt.Errorf("expect [2](); got [%d]()", len(t))
+	}
+
+	blksize, ok := pycompat.Int64(t[0])
+	if !ok {
+		return fmt.Errorf("blksize: expect integer; got %s", xzodb.TypeOf(t[0]))
+	}
+	if blksize <= 0 {
+		return fmt.Errorf("blksize: must be > 0; got %d", blksize)
+	}
+
+	blktab, err := vBlktab(t[1])
+	if err != nil {
+		return err
+	}
+
+	bf.blksize = blksize
+	bf.blktab  = blktab
+	return nil
+}
+
+// BlkSize returns size of block used by this file.
+//
+// The file must be activated.
+func (bf *ZBigFile) BlkSize() int64 {
+	return bf.blksize
+}
+
+// LoadBlk loads data for file block #blk.
+//
+// it also returns:
+//
+//	- BTree path in .blktab to loaded block,
+//	- blocks covered by leaf node in the BTree path,
+//	- max(_.serial for _ in ZBlk(#blk), all BTree/Bucket that lead to ZBlk)
+//	  which provides a rough upper-bound estimate for file[blk] revision.
+//
+// TODO load into user-provided buf.
+func (bf *ZBigFile) LoadBlk(ctx context.Context, blk int64) (_ []byte, treePath []btree.LONode, blkCov btree.LKeyRange, zblk ZBlk, blkRevMax zodb.Tid, err error) {
+	defer xerr.Contextf(&err, "bigfile %s: loadblk %d", bf.POid(), blk)
+	kø := btree.LKeyRange{Lo: 0, Hi_: -1} // empty KeyRange
+
+	err = bf.PActivate(ctx)
+	if err != nil {
+		return nil, nil, kø, nil, 0, err
+	}
+	defer bf.PDeactivate()
+
+	blkRevMax = 0
+	xzblk, ok, err := bf.blktab.VGet(ctx, blk, func(node btree.LONode, keycov btree.LKeyRange) {
+		treePath = append(treePath, node)
+		blkCov = keycov // will be set last for leaf
+		blkRevMax = tidmax(blkRevMax, node.PSerial())
+	})
+	if err != nil {
+		return nil, nil, kø, nil, 0, err
+	}
+	if !ok {
+		return make([]byte, bf.blksize), treePath, blkCov, nil, blkRevMax, nil
+	}
+
+	zblk, err = vZBlk(xzblk)
+	if err != nil {
+		return nil, nil, kø, nil, 0, err
+	}
+
+	blkdata, zblkrev, err := zblk.LoadBlkData(ctx)
+	if err != nil {
+		return nil, nil, kø, nil, 0, err
+	}
+	blkRevMax = tidmax(blkRevMax, zblkrev)
+
+	l := int64(len(blkdata))
+	if l > bf.blksize {
+		return nil, nil, kø, nil, 0, fmt.Errorf("zblk %s: invalid blk: size = %d (> blksize = %d)", zblk.POid(), l, bf.blksize)
+	}
+
+	// append trailing \0 to data to reach .blksize
+	if l < bf.blksize {
+		d := make([]byte, bf.blksize)
+		copy(d, blkdata)
+		blkdata = d
+	}
+
+	return blkdata, treePath, blkCov, zblk, blkRevMax, nil
+}
+
+// Size returns whole file size.
+//
+// it also returns BTree path scanned to obtain the size.
+func (bf *ZBigFile) Size(ctx context.Context) (_ int64, treePath []btree.LONode, blkCov btree.LKeyRange, err error) {
+	defer xerr.Contextf(&err, "bigfile %s: size", bf.POid())
+	kø := btree.LKeyRange{Lo: 0, Hi_: -1} // empty KeyRange
+
+	err = bf.PActivate(ctx)
+	if err != nil {
+		return 0, nil, kø, err
+	}
+	defer bf.PDeactivate()
+
+	tailblk, ok, err := bf.blktab.VMaxKey(ctx, func(node btree.LONode, keycov btree.LKeyRange) {
+		treePath = append(treePath, node)
+		blkCov = keycov // will be set last for leaf
+	})
+	if err != nil {
+		return 0, nil, kø, err
+	}
+	if !ok {
+		return 0, treePath, blkCov, nil
+	}
+
+	size := (tailblk + 1) * bf.blksize
+	if size / bf.blksize != tailblk + 1 {
+		return 0, nil, kø, syscall.EFBIG // overflow
+	}
+
+	return size, treePath, blkCov, nil
+}
+
+// vZBlk checks and converts xzblk to a ZBlk object.
+func vZBlk(xzblk interface{}) (ZBlk, error) {
+	zblk, ok := xzblk.(ZBlk)
+	if !ok {
+		return nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk))
+	}
+	return zblk, nil
+}
+
+// vBlktab checks and converts xblktab to LOBTree object.
+func vBlktab(xblktab interface{}) (*btree.LOBTree, error) {
+	blktab, ok := xblktab.(*btree.LOBTree)
+	if !ok {
+		return nil, fmt.Errorf("blktab: expect LOBTree; got %s", xzodb.TypeOf(xblktab))
+	}
+	return blktab, nil
+}
+
+// ----------------------------------------
+
+// module of Wendelin ZODB py objects
+const zwendelin = "wendelin.bigfile.file_zodb"
+
+func init() {
+	t := reflect.TypeOf
+	zodb.RegisterClass(zwendelin + ".ZBlk0",    t(ZBlk0{}),    t(zBlk0State{}))
+	zodb.RegisterClass(zwendelin + ".ZBlk1",    t(ZBlk1{}),    t(zBlk1State{}))
+	zodb.RegisterClass(zwendelin + ".ZData",    t(ZData{}),    t(zDataState{}))
+	zodb.RegisterClass(zwendelin + ".ZBigFile", t(ZBigFile{}), t(zBigFileState{}))
+
+	// backward compatibility
+	zodb.RegisterClassAlias(zwendelin + ".ZBlk", zwendelin + ".ZBlk0")
+}

wcfs/internal/zdata/zblk_test.go

+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package zdata
+
+//go:generate ./testdata/zblk_test_gen.py
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"testing"
+
+	"lab.nexedi.com/kirr/go123/exc"
+	"lab.nexedi.com/kirr/neo/go/transaction"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestZBlk verifies that ZBlk* and ZBigFile saved by Python can be read correctly by Go.
+// TODO also test with data saved by Python3.
+func TestZBlk(t *testing.T) {
+	X := exc.Raiseif
+	assert := require.New(t)
+	ctx := context.Background()
+	stor, err := zodb.Open(ctx, "testdata/zblk.fs", &zodb.OpenOptions{ReadOnly: true}); X(err)
+	db := zodb.NewDB(stor, &zodb.DBOptions{})
+	defer func() {
+		err := db.Close();	X(err)
+		err  = stor.Close();	X(err)
+	}()
+
+	txn, ctx := transaction.New(ctx)
+	defer txn.Abort()
+
+	conn, err := db.Open(ctx, &zodb.ConnOptions{});	X(err)
+
+	xz0, err := conn.Get(ctx, z0_oid);	X(err)
+	xz1, err := conn.Get(ctx, z1_oid);	X(err)
+	xzf, err := conn.Get(ctx, zf_oid);	X(err)
+
+	z0, ok := xz0.(*ZBlk0)
+	if !ok {
+		t.Fatalf("z0: want ZBlk0; got %T", xz0)
+	}
+
+	z1, ok := xz1.(*ZBlk1)
+	if !ok {
+		t.Fatalf("z1: want ZBlk1; got %T", xz1)
+	}
+
+	zf, ok := xzf.(*ZBigFile)
+	if !ok {
+		t.Fatalf("zf: want ZBigFile; got %T", xzf)
+	}
+
+	xactivate := func(obj zodb.IPersistent) {
+		t.Helper()
+		err := obj.PActivate(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+
+	z0Data, z0Rev, err := z0.LoadBlkData(ctx);	X(err)
+	z0DataOK := brange32(z0_len)
+	assert.Equal(z0Data, z0DataOK,   "ZBlk0 data wrong")
+	assert.Equal(z0Rev,  z0_rev,	 "ZBlk0 rev wrong")
+
+	z1Data, z1Rev, err := z1.LoadBlkData(ctx);	X(err)
+	z1DataOK := make([]byte, zf_blksize)                                  // zeros
+	copy(z1DataOK[0:],                      brange32(z1_htlen))           // head
+	copy(z1DataOK[len(z1DataOK)-z1_htlen:], breverse(brange32(z1_htlen))) // tail
+	z1DataOK = bytes.TrimRight(z1DataOK, "\x00") // trailing 0 are not persisted
+	assert.Equal(z1Data, z1DataOK,   "ZBlk1 data wrong")
+	assert.Equal(z1Rev,  z1_rev,     "ZBlk1 rev wrong")
+
+
+	xactivate(zf)
+	if zf.blksize != zf_blksize {
+		t.Fatalf("zf: blksize=%d;  want %d", zf.blksize, zf_blksize)
+	}
+
+	z0_, ok, err := zf.blktab.Get(ctx, 1);	X(err)
+	if !(ok && z0_ == z0) {
+		t.Fatalf("zf: [0] -> %#v; want z0", z0_)
+	}
+
+	z1_, ok, err := zf.blktab.Get(ctx, 3);	X(err)
+	if !(ok && z1_ == z1) {
+		t.Fatalf("zf: [1] -> %#v; want z1", z1_)
+	}
+
+	size, _, _, err := zf.Size(ctx);	X(err)
+	assert.Equal(size, int64(zf_size),	"ZBigFile size wrong")
+
+
+	// LoadBlk
+	z0Data, _, _, _, _, err = zf.LoadBlk(ctx, 1);	X(err)
+	assert.Equal(len(z0Data), int(zf.blksize))
+	z0Data = bytes.TrimRight(z0Data, "\x00")
+	assert.Equal(z0Data, z0DataOK)
+
+	z1Data, _, _, _, _, err = zf.LoadBlk(ctx, 3);	X(err)
+	assert.Equal(len(z1Data), int(zf.blksize))
+	z1Data = bytes.TrimRight(z1Data, "\x00")
+	assert.Equal(z1Data, z1DataOK)
+}
+
+// TODO verify PyGetState vs PySetState
+
+// brange32 returns bytes with big-endian uint32 sequence filling them.
+// returned bytes has len == size.
+func brange32(size int) []byte {
+	data := make([]byte, size)
+	for i := 0; i < size / 4; i++ {
+		binary.BigEndian.PutUint32(data[i*4:], uint32(i))
+	}
+	return data
+}
+
+// breverse returns bytes in the reverse order.
+func breverse(b []byte) []byte {
+	r := make([]byte, len(b))
+	for i := range(b) {
+		r[i] = b[len(b)-i-1]
+	}
+	return r
+}

wcfs/internal/zdata/ztestdata_zblk_test.go

+// Code generated by ./testdata/zblk_test_gen.py; DO NOT EDIT.
+package zdata
+
+import "lab.nexedi.com/kirr/neo/go/zodb"
+
+const zf_blksize = 2097152
+const zf_size    = 8388608
+const z0_oid     = zodb.Oid(2)
+const z1_oid     = zodb.Oid(3)
+const zf_oid     = zodb.Oid(1)
+const z0_rev     = zodb.Tid(0x285cbac3851eb99)
+const z1_rev     = zodb.Tid(0x285cbac3851eb99)
+const z0_len     = 16384
+const z1_htlen   = 131072