Skip to content

W
wendelin.core
Commit 25072aaf authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

.

parent 98ad5c09
Show whitespace changes
Inline Side-by-side
Showing with 0 additions and 937 deletions
wcfs/go.mod
View file @ 25072aaf
...@@ -3,7 +3,6 @@ module lab.nexedi.com/nexedi/wendelin.core/wcfs ...@@ -3,7 +3,6 @@ module lab.nexedi.com/nexedi/wendelin.core/wcfs
go 1.14 go 1.14
require ( require (
github.com/johncgriffin/overflow v0.0.0-20170615021017-4d914c927216
github.com/kisielk/og-rek v1.0.1-0.20180928202415-8b25c4cefd6c github.com/kisielk/og-rek v1.0.1-0.20180928202415-8b25c4cefd6c
lab.nexedi.com/kirr/go123 v0.0.0-20200916121347-316617668e12 lab.nexedi.com/kirr/go123 v0.0.0-20200916121347-316617668e12
lab.nexedi.com/kirr/neo/go v0.0.0-20201012044742-28494187df87 lab.nexedi.com/kirr/neo/go v0.0.0-20201012044742-28494187df87
......
wcfs/go.sum
View file @ 25072aaf
...@@ -110,8 +110,6 @@ github.com/gwenn/yacr v0.0.0-20200110180258-a66d8c42d0ff/go.mod h1:5SNcBGxZ5OaJA ...@@ -110,8 +110,6 @@ github.com/gwenn/yacr v0.0.0-20200110180258-a66d8c42d0ff/go.mod h1:5SNcBGxZ5OaJA
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/johncgriffin/overflow v0.0.0-20170615021017-4d914c927216 h1:Sts1ngIYr+lJwFc6VfH2tOwqx7xu7y+sDljVRjNLGOo=
github.com/johncgriffin/overflow v0.0.0-20170615021017-4d914c927216/go.mod h1:B9OPZOhZ3FIi6bu54lAgCMzXLh11Z7ilr3rOr/ClP+E=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
......
wcfs/vendor/github.com/johncgriffin/overflow/README.md deleted 100644 → 0
View file @ 98ad5c09
[![Build Status](https://travis-ci.org/JohnCGriffin/overflow.png)](https://travis-ci.org/JohnCGriffin/overflow)
# overflow
Check for int/int8/int16/int64/int32 integer overflow in Golang arithmetic.
### Install
```
go get github.com/johncgriffin/overflow
```
Note that because Go has no template types, the majority of repetitive code is
generated by overflow_template.sh. If you have to change an
algorithm, change it there and regenerate the Go code via:
```
go generate
```
### Synopsis
```
package main
import "fmt"
import "math"
import "github.com/JohnCGriffin/overflow"
func main() {
addend := math.MaxInt64 - 5
for i := 0; i < 10; i++ {
sum, ok := overflow.Add(addend, i)
fmt.Printf("%v+%v -> (%v,%v)\n",
addend, i, sum, ok)
}
}
```
yields the output
```
9223372036854775802+0 -> (9223372036854775802,true)
9223372036854775802+1 -> (9223372036854775803,true)
9223372036854775802+2 -> (9223372036854775804,true)
9223372036854775802+3 -> (9223372036854775805,true)
9223372036854775802+4 -> (9223372036854775806,true)
9223372036854775802+5 -> (9223372036854775807,true)
9223372036854775802+6 -> (0,false)
9223372036854775802+7 -> (0,false)
9223372036854775802+8 -> (0,false)
9223372036854775802+9 -> (0,false)
```
For int, int64, and int32 types, provide Add, Add32, Add64, Sub, Sub32, Sub64, etc.
Unsigned types not covered at the moment, but such additions are welcome.
### Stay calm and panic
There's a good case to be made that a panic is an unidiomatic but proper response. Iff you
believe that there's no valid way to continue your program after math goes wayward, you can
use the easier Addp, Mulp, Subp, and Divp versions which return the normal result or panic.
wcfs/vendor/github.com/johncgriffin/overflow/overflow.go deleted 100644 → 0
View file @ 98ad5c09
/*Package overflow offers overflow-checked integer arithmetic operations
for int, int32, and int64. Each of the operations returns a
result,bool combination. This was prompted by the need to know when
to flow into higher precision types from the math.big library.
For instance, assuing a 64 bit machine:
10 + 20 -> 30
int(math.MaxInt64) + 1 -> -9223372036854775808
whereas
overflow.Add(10,20) -> (30, true)
overflow.Add(math.MaxInt64,1) -> (0, false)
Add, Sub, Mul, Div are for int. Add64, Add32, etc. are specifically sized.
If anybody wishes an unsigned version, submit a pull request for code
and new tests. */
package overflow
//go:generate ./overflow_template.sh
import "math"
func _is64Bit() bool {
maxU32 := uint(math.MaxUint32)
return ((maxU32 << 1) >> 1) == maxU32
}
/********** PARTIAL TEST COVERAGE FROM HERE DOWN *************
The only way that I could see to do this is a combination of
my normal 64 bit system and a GopherJS running on Node. My
understanding is that its ints are 32 bit.
So, FEEL FREE to carefully review the code visually.
*************************************************************/
// Unspecified size, i.e. normal signed int
// Add sums two ints, returning the result and a boolean status.
func Add(a, b int) (int, bool) {
if _is64Bit() {
r64, ok := Add64(int64(a), int64(b))
return int(r64), ok
}
r32, ok := Add32(int32(a), int32(b))
return int(r32), ok
}
// Sub returns the difference of two ints and a boolean status.
func Sub(a, b int) (int, bool) {
if _is64Bit() {
r64, ok := Sub64(int64(a), int64(b))
return int(r64), ok
}
r32, ok := Sub32(int32(a), int32(b))
return int(r32), ok
}
// Mul returns the product of two ints and a boolean status.
func Mul(a, b int) (int, bool) {
if _is64Bit() {
r64, ok := Mul64(int64(a), int64(b))
return int(r64), ok
}
r32, ok := Mul32(int32(a), int32(b))
return int(r32), ok
}
// Div returns the quotient of two ints and a boolean status
func Div(a, b int) (int, bool) {
if _is64Bit() {
r64, ok := Div64(int64(a), int64(b))
return int(r64), ok
}
r32, ok := Div32(int32(a), int32(b))
return int(r32), ok
}
// Quotient returns the quotient, remainder and status of two ints
func Quotient(a, b int) (int, int, bool) {
if _is64Bit() {
q64, r64, ok := Quotient64(int64(a), int64(b))
return int(q64), int(r64), ok
}
q32, r32, ok := Quotient32(int32(a), int32(b))
return int(q32), int(r32), ok
}
/************* Panic versions for int ****************/
// Addp returns the sum of two ints, panicking on overflow
func Addp(a, b int) int {
r, ok := Add(a, b)
if !ok {
panic("addition overflow")
}
return r
}
// Subp returns the difference of two ints, panicking on overflow.
func Subp(a, b int) int {
r, ok := Sub(a, b)
if !ok {
panic("subtraction overflow")
}
return r
}
// Mulp returns the product of two ints, panicking on overflow.
func Mulp(a, b int) int {
r, ok := Mul(a, b)
if !ok {
panic("multiplication overflow")
}
return r
}
// Divp returns the quotient of two ints, panicking on overflow.
func Divp(a, b int) int {
r, ok := Div(a, b)
if !ok {
panic("division failure")
}
return r
}
wcfs/vendor/github.com/johncgriffin/overflow/overflow_impl.go deleted 100644 → 0
View file @ 98ad5c09
package overflow
// This is generated code, created by overflow_template.sh executed
// by "go generate"
// Add8 performs + operation on two int8 operands
// returning a result and status
func Add8(a, b int8) (int8, bool) {
c := a + b
if (c > a) == (b > 0) {
return c, true
}
return c, false
}
// Add8p is the unchecked panicing version of Add8
func Add8p(a, b int8) int8 {
r, ok := Add8(a, b)
if !ok {
panic("addition overflow")
}
return r
}
// Sub8 performs - operation on two int8 operands
// returning a result and status
func Sub8(a, b int8) (int8, bool) {
c := a - b
if (c < a) == (b > 0) {
return c, true
}
return c, false
}
// Sub8p is the unchecked panicing version of Sub8
func Sub8p(a, b int8) int8 {
r, ok := Sub8(a, b)
if !ok {
panic("subtraction overflow")
}
return r
}
// Mul8 performs * operation on two int8 operands
// returning a result and status
func Mul8(a, b int8) (int8, bool) {
if a == 0 || b == 0 {
return 0, true
}
c := a * b
if (c < 0) == ((a < 0) != (b < 0)) {
if c/b == a {
return c, true
}
}
return c, false
}
// Mul8p is the unchecked panicing version of Mul8
func Mul8p(a, b int8) int8 {
r, ok := Mul8(a, b)
if !ok {
panic("multiplication overflow")
}
return r
}
// Div8 performs / operation on two int8 operands
// returning a result and status
func Div8(a, b int8) (int8, bool) {
q, _, ok := Quotient8(a, b)
return q, ok
}
// Div8p is the unchecked panicing version of Div8
func Div8p(a, b int8) int8 {
r, ok := Div8(a, b)
if !ok {
panic("division failure")
}
return r
}
// Quotient8 performs + operation on two int8 operands
// returning a quotient, a remainder and status
func Quotient8(a, b int8) (int8, int8, bool) {
if b == 0 {
return 0, 0, false
}
c := a / b
status := (c < 0) == ((a < 0) != (b < 0))
return c, a % b, status
}
// Add16 performs + operation on two int16 operands
// returning a result and status
func Add16(a, b int16) (int16, bool) {
c := a + b
if (c > a) == (b > 0) {
return c, true
}
return c, false
}
// Add16p is the unchecked panicing version of Add16
func Add16p(a, b int16) int16 {
r, ok := Add16(a, b)
if !ok {
panic("addition overflow")
}
return r
}
// Sub16 performs - operation on two int16 operands
// returning a result and status
func Sub16(a, b int16) (int16, bool) {
c := a - b
if (c < a) == (b > 0) {
return c, true
}
return c, false
}
// Sub16p is the unchecked panicing version of Sub16
func Sub16p(a, b int16) int16 {
r, ok := Sub16(a, b)
if !ok {
panic("subtraction overflow")
}
return r
}
// Mul16 performs * operation on two int16 operands
// returning a result and status
func Mul16(a, b int16) (int16, bool) {
if a == 0 || b == 0 {
return 0, true
}
c := a * b
if (c < 0) == ((a < 0) != (b < 0)) {
if c/b == a {
return c, true
}
}
return c, false
}
// Mul16p is the unchecked panicing version of Mul16
func Mul16p(a, b int16) int16 {
r, ok := Mul16(a, b)
if !ok {
panic("multiplication overflow")
}
return r
}
// Div16 performs / operation on two int16 operands
// returning a result and status
func Div16(a, b int16) (int16, bool) {
q, _, ok := Quotient16(a, b)
return q, ok
}
// Div16p is the unchecked panicing version of Div16
func Div16p(a, b int16) int16 {
r, ok := Div16(a, b)
if !ok {
panic("division failure")
}
return r
}
// Quotient16 performs + operation on two int16 operands
// returning a quotient, a remainder and status
func Quotient16(a, b int16) (int16, int16, bool) {
if b == 0 {
return 0, 0, false
}
c := a / b
status := (c < 0) == ((a < 0) != (b < 0))
return c, a % b, status
}
// Add32 performs + operation on two int32 operands
// returning a result and status
func Add32(a, b int32) (int32, bool) {
c := a + b
if (c > a) == (b > 0) {
return c, true
}
return c, false
}
// Add32p is the unchecked panicing version of Add32
func Add32p(a, b int32) int32 {
r, ok := Add32(a, b)
if !ok {
panic("addition overflow")
}
return r
}
// Sub32 performs - operation on two int32 operands
// returning a result and status
func Sub32(a, b int32) (int32, bool) {
c := a - b
if (c < a) == (b > 0) {
return c, true
}
return c, false
}
// Sub32p is the unchecked panicing version of Sub32
func Sub32p(a, b int32) int32 {
r, ok := Sub32(a, b)
if !ok {
panic("subtraction overflow")
}
return r
}
// Mul32 performs * operation on two int32 operands
// returning a result and status
func Mul32(a, b int32) (int32, bool) {
if a == 0 || b == 0 {
return 0, true
}
c := a * b
if (c < 0) == ((a < 0) != (b < 0)) {
if c/b == a {
return c, true
}
}
return c, false
}
// Mul32p is the unchecked panicing version of Mul32
func Mul32p(a, b int32) int32 {
r, ok := Mul32(a, b)
if !ok {
panic("multiplication overflow")
}
return r
}
// Div32 performs / operation on two int32 operands
// returning a result and status
func Div32(a, b int32) (int32, bool) {
q, _, ok := Quotient32(a, b)
return q, ok
}
// Div32p is the unchecked panicing version of Div32
func Div32p(a, b int32) int32 {
r, ok := Div32(a, b)
if !ok {
panic("division failure")
}
return r
}
// Quotient32 performs + operation on two int32 operands
// returning a quotient, a remainder and status
func Quotient32(a, b int32) (int32, int32, bool) {
if b == 0 {
return 0, 0, false
}
c := a / b
status := (c < 0) == ((a < 0) != (b < 0))
return c, a % b, status
}
// Add64 performs + operation on two int64 operands
// returning a result and status
func Add64(a, b int64) (int64, bool) {
c := a + b
if (c > a) == (b > 0) {
return c, true
}
return c, false
}
// Add64p is the unchecked panicing version of Add64
func Add64p(a, b int64) int64 {
r, ok := Add64(a, b)
if !ok {
panic("addition overflow")
}
return r
}
// Sub64 performs - operation on two int64 operands
// returning a result and status
func Sub64(a, b int64) (int64, bool) {
c := a - b
if (c < a) == (b > 0) {
return c, true
}
return c, false
}
// Sub64p is the unchecked panicing version of Sub64
func Sub64p(a, b int64) int64 {
r, ok := Sub64(a, b)
if !ok {
panic("subtraction overflow")
}
return r
}
// Mul64 performs * operation on two int64 operands
// returning a result and status
func Mul64(a, b int64) (int64, bool) {
if a == 0 || b == 0 {
return 0, true
}
c := a * b
if (c < 0) == ((a < 0) != (b < 0)) {
if c/b == a {
return c, true
}
}
return c, false
}
// Mul64p is the unchecked panicing version of Mul64
func Mul64p(a, b int64) int64 {
r, ok := Mul64(a, b)
if !ok {
panic("multiplication overflow")
}
return r
}
// Div64 performs / operation on two int64 operands
// returning a result and status
func Div64(a, b int64) (int64, bool) {
q, _, ok := Quotient64(a, b)
return q, ok
}
// Div64p is the unchecked panicing version of Div64
func Div64p(a, b int64) int64 {
r, ok := Div64(a, b)
if !ok {
panic("division failure")
}
return r
}
// Quotient64 performs + operation on two int64 operands
// returning a quotient, a remainder and status
func Quotient64(a, b int64) (int64, int64, bool) {
if b == 0 {
return 0, 0, false
}
c := a / b
status := (c < 0) == ((a < 0) != (b < 0))
return c, a % b, status
}
wcfs/vendor/github.com/johncgriffin/overflow/overflow_template.sh deleted 100644 → 0
View file @ 98ad5c09
#!/bin/sh
exec > overflow_impl.go
echo "package overflow
// This is generated code, created by overflow_template.sh executed
// by \"go generate\"
"
for SIZE in 8 16 32 64
do
echo "
// Add${SIZE} performs + operation on two int${SIZE} operands
// returning a result and status
func Add${SIZE}(a, b int${SIZE}) (int${SIZE}, bool) {
c := a + b
if (c > a) == (b > 0) {
return c, true
}
return c, false
}
// Add${SIZE}p is the unchecked panicing version of Add${SIZE}
func Add${SIZE}p(a, b int${SIZE}) int${SIZE} {
r, ok := Add${SIZE}(a, b)
if !ok {
panic(\"addition overflow\")
}
return r
}
// Sub${SIZE} performs - operation on two int${SIZE} operands
// returning a result and status
func Sub${SIZE}(a, b int${SIZE}) (int${SIZE}, bool) {
c := a - b
if (c < a) == (b > 0) {
return c, true
}
return c, false
}
// Sub${SIZE}p is the unchecked panicing version of Sub${SIZE}
func Sub${SIZE}p(a, b int${SIZE}) int${SIZE} {
r, ok := Sub${SIZE}(a, b)
if !ok {
panic(\"subtraction overflow\")
}
return r
}
// Mul${SIZE} performs * operation on two int${SIZE} operands
// returning a result and status
func Mul${SIZE}(a, b int${SIZE}) (int${SIZE}, bool) {
if a == 0 || b == 0 {
return 0, true
}
c := a * b
if (c < 0) == ((a < 0) != (b < 0)) {
if c/b == a {
return c, true
}
}
return c, false
}
// Mul${SIZE}p is the unchecked panicing version of Mul${SIZE}
func Mul${SIZE}p(a, b int${SIZE}) int${SIZE} {
r, ok := Mul${SIZE}(a, b)
if !ok {
panic(\"multiplication overflow\")
}
return r
}
// Div${SIZE} performs / operation on two int${SIZE} operands
// returning a result and status
func Div${SIZE}(a, b int${SIZE}) (int${SIZE}, bool) {
q, _, ok := Quotient${SIZE}(a, b)
return q, ok
}
// Div${SIZE}p is the unchecked panicing version of Div${SIZE}
func Div${SIZE}p(a, b int${SIZE}) int${SIZE} {
r, ok := Div${SIZE}(a, b)
if !ok {
panic(\"division failure\")
}
return r
}
// Quotient${SIZE} performs + operation on two int${SIZE} operands
// returning a quotient, a remainder and status
func Quotient${SIZE}(a, b int${SIZE}) (int${SIZE}, int${SIZE}, bool) {
if b == 0 {
return 0, 0, false
}
c := a / b
status := (c < 0) == ((a < 0) != (b < 0))
return c, a % b, status
}
"
done
wcfs/vendor/modules.txt
View file @ 25072aaf
...@@ -6,9 +6,6 @@ github.com/cznic/strutil ...@@ -6,9 +6,6 @@ github.com/cznic/strutil
github.com/fsnotify/fsnotify github.com/fsnotify/fsnotify
# github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b # github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
github.com/golang/glog github.com/golang/glog
# github.com/johncgriffin/overflow v0.0.0-20170615021017-4d914c927216
## explicit
github.com/johncgriffin/overflow
# github.com/kisielk/og-rek v1.0.1-0.20180928202415-8b25c4cefd6c # github.com/kisielk/og-rek v1.0.1-0.20180928202415-8b25c4cefd6c
## explicit ## explicit
github.com/kisielk/og-rek github.com/kisielk/og-rek
......
wcfs/zblk.go
View file @ 25072aaf
...@@ -29,28 +29,17 @@ package main ...@@ -29,28 +29,17 @@ package main
// //
// ZBlk0 (aliased as ZBlk) // ZBlk0 (aliased as ZBlk)
// str with trailing '\0' removed. // str with trailing '\0' removed.
//
// ZBlk1
// .chunktab IOBtree{} offset -> ZData(chunk)
//
// ZData
// str (chunk)
import ( import (
"context" "context"
"fmt" "fmt"
"reflect" "reflect"
"sort"
"sync"
"syscall" "syscall"
// "lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/xerr" "lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/go123/xsync"
"lab.nexedi.com/kirr/neo/go/zodb" "lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/neo/go/zodb/btree" "lab.nexedi.com/kirr/neo/go/zodb/btree"
pickle "github.com/kisielk/og-rek" pickle "github.com/kisielk/og-rek"
"github.com/johncgriffin/overflow"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/pycompat" "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/pycompat"
) )
...@@ -70,14 +59,12 @@ type zBlk interface { ...@@ -70,14 +59,12 @@ type zBlk interface {
} }
var _ zBlk = (*ZBlk0)(nil) var _ zBlk = (*ZBlk0)(nil)
var _ zBlk = (*ZBlk1)(nil)
// ---- ZBlk0 ---- // ---- ZBlk0 ----
// ZBlk0 mimics ZBlk0 from python. // ZBlk0 mimics ZBlk0 from python.
type ZBlk0 struct { type ZBlk0 struct {
// zblkInΔFtail
zodb.Persistent zodb.Persistent
// NOTE py source uses bytes(buf) but on python2 it still results in str // NOTE py source uses bytes(buf) but on python2 it still results in str
...@@ -113,233 +100,9 @@ func (zb *ZBlk0) loadBlkData(ctx context.Context) (_ []byte, _ zodb.Tid, err err ...@@ -113,233 +100,9 @@ func (zb *ZBlk0) loadBlkData(ctx context.Context) (_ []byte, _ zodb.Tid, err err
} }
defer zb.PDeactivate() defer zb.PDeactivate()
//return mem.Bytes(zb.blkdata), zb.PSerial(), nil // XXX -> []bytes(zb.blkdata)
return []byte(zb.blkdata), zb.PSerial(), nil return []byte(zb.blkdata), zb.PSerial(), nil
} }
// ---- ZBlk1 ---
// ZData mimics ZData from python.
type ZData struct {
zodb.Persistent
// NOTE py source uses bytes(buf) but on python2 it still results in str
data string
}
type zDataState ZData // hide state methods from public API
func (zd *zDataState) DropState() {
zd.data = ""
}
func (zd *zDataState) PyGetState() interface{} {
return zd.data
}
func (zd *zDataState) PySetState(pystate interface{}) error {
data, ok := pystate.(string)
if !ok {
return fmt.Errorf("expect str; got %s", typeOf(pystate))
}
zd.data = data
return nil
}
// ZBlk1 mimics ZBlk1 from python.
type ZBlk1 struct {
// zblkInΔFtail
zodb.Persistent
chunktab *btree.IOBTree // {} offset -> ZData(chunk)
}
type zBlk1State ZBlk1 // hide state methods from public API
func (zb *zBlk1State) DropState() {
zb.chunktab = nil
}
func (zb *zBlk1State) PyGetState() interface{} {
return zb.chunktab
}
func (zb *zBlk1State) PySetState(pystate interface{}) error {
chunktab, ok := pystate.(*btree.IOBTree)
if !ok {
return fmt.Errorf("expect IOBTree; got %s", typeOf(pystate))
}
zb.chunktab = chunktab
return nil
}
func (zb *ZBlk1) loadBlkData(ctx context.Context) (_ []byte, _ zodb.Tid, err error) {
defer xerr.Contextf(&err, "ZBlk1(%s): loadBlkData", zb.POid())
err = zb.PActivate(ctx)
if err != nil {
return nil, 0, err
}
defer zb.PDeactivate()
// get to all ZData objects; activate them and build
//
// {} offset -> ZData
//
// with all ZData being live.
var mu sync.Mutex
chunktab := make(map[int32]*ZData)
// on return deactivate all ZData objects loaded in chunktab
defer func() {
for _, zd := range chunktab {
zd.PDeactivate()
}
}()
wg := xsync.NewWorkGroup(ctx)
// loadZData loads 1 ZData object into chunktab and leaves it activated.
loadZData := func(ctx context.Context, offset int32, zd *ZData) (err error) {
err = zd.PActivate(ctx)
if err != nil {
return err
}
// no PDeactivate, zd remains live
//fmt.Printf("@%d -> zdata #%s (%d)\n", offset, zd.POid(), len(zd.data))
mu.Lock()
defer mu.Unlock()
_, already := chunktab[offset]
if already {
return fmt.Errorf("duplicate offset %d", offset)
}
chunktab[offset] = zd
return nil
}
// loadBucket loads all ZData objects from leaf BTree bucket.
loadBucket := func(ctx context.Context, b *btree.IOBucket) (err error) {
err = b.PActivate(ctx)
if err != nil {
return err
}
defer b.PDeactivate()
// go through all bucket key/v -> chunktab
defer xerr.Contextf(&err, "%s(%s)", typeOf(b), b.POid())
//fmt.Printf("\nbucket: %v\n\n", b.Entryv())
for i, e := range b.Entryv() {
zd, ok := e.Value().(*ZData)
if !ok {
return fmt.Errorf("[%d]: !ZData (%s)", i, typeOf(e.Value()))
}
offset := e.Key()
if offset < 0 {
return fmt.Errorf("[%d]: offset < 0 (%d)", i, offset)
}
wg.Go(func(ctx context.Context) error {
return loadZData(ctx, offset, zd)
})
}
return nil
}
// loadBTree spawns loading of all BTree children.
var loadBTree func(ctx context.Context, t *btree.IOBTree) error
loadBTree = func(ctx context.Context, t *btree.IOBTree) error {
err := t.PActivate(ctx)
if err != nil {
return err
}
defer t.PDeactivate()
//fmt.Printf("\nbtree: %v\n\n", t.Entryv())
for _, e := range t.Entryv() {
switch child := e.Child().(type) {
case *btree.IOBTree:
wg.Go(func(ctx context.Context) error {
return loadBTree(ctx, child)
})
case *btree.IOBucket:
wg.Go(func(ctx context.Context) error {
return loadBucket(ctx, child)
})
default:
panicf("IOBTree has %s child", typeOf(child))
}
}
return nil
}
wg.Go(func(ctx context.Context) error {
return loadBTree(ctx, zb.chunktab)
})
err = wg.Wait()
if err != nil {
return nil, 0, err
}
// empty .chunktab -> ø
if len(chunktab) == 0 {
return nil, 0, nil
}
// glue all chunks from chunktab
offv := make([]int32, 0, len(chunktab)) // ↑
for off := range(chunktab) {
offv = append(offv, off)
}
sort.Slice(offv, func(i, j int) bool {
return offv[i] < offv[j]
})
//fmt.Printf("#chunktab: %d\n", len(chunktab))
//fmt.Printf("offv: %v\n", offv)
// find out whole blk len via inspecting tail chunk
tailStart := offv[len(offv)-1]
tailChunk := chunktab[tailStart]
blklen, ok := overflow.Add32(tailStart, int32(len(tailChunk.data)))
if !ok {
return nil, 0, fmt.Errorf("invalid data: blklen overflow")
}
// whole buffer initialized as 0 + tail_chunk
blkdata := make([]byte, blklen)
copy(blkdata[tailStart:], tailChunk.data)
// go through all chunks besides tail and extract them
stop := int32(0)
for _, start := range offv[:len(offv)-1] {
chunk := chunktab[start]
if !(start >= stop) { // verify chunks don't overlap
return nil, 0, fmt.Errorf("invalid data: chunks overlap")
}
stop, ok = overflow.Add32(start, int32(len(chunk.data)))
if !(ok && stop <= blklen) {
return nil, 0, fmt.Errorf("invalid data: blkdata overrun")
}
copy(blkdata[start:], chunk.data)
}
return blkdata, zb.PSerial(), nil
}
// ---------------------------------------- // ----------------------------------------
// ZBigFile mimics ZBigFile from python. // ZBigFile mimics ZBigFile from python.
...@@ -488,10 +251,5 @@ const zwendelin = "wendelin.bigfile.file_zodb" ...@@ -488,10 +251,5 @@ const zwendelin = "wendelin.bigfile.file_zodb"
func init() { func init() {
t := reflect.TypeOf t := reflect.TypeOf
zodb.RegisterClass(zwendelin + ".ZBlk0", t(ZBlk0{}), t(zBlk0State{})) zodb.RegisterClass(zwendelin + ".ZBlk0", t(ZBlk0{}), t(zBlk0State{}))
zodb.RegisterClass(zwendelin + ".ZBlk1", t(ZBlk1{}), t(zBlk1State{}))
zodb.RegisterClass(zwendelin + ".ZData", t(ZData{}), t(zDataState{}))
zodb.RegisterClass(zwendelin + ".ZBigFile", t(ZBigFile{}), t(zBigFileState{})) zodb.RegisterClass(zwendelin + ".ZBigFile", t(ZBigFile{}), t(zBigFileState{}))
// backward compatibility
zodb.RegisterClassAlias(zwendelin + ".ZBlk", zwendelin + ".ZBlk0")
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7