Skip to content

W
wendelin.core
Commit f91982af authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

X First round of preparation steps for ΔFtail tests 

* t2: (29 commits)
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  .
  Revert "."
  .
  X Unexprt ΔBroots
  .
  X Unexport SetXXX from packages API
  .
  ...
parents d0fe680a c7f1e3c9
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1589 additions and 542 deletions
wcfs/TODO.txt deleted 100644 → 0
View file @ d0fe680a
- doc: notes on how things are organized in wendelin.core 2
wcfs:
- SIGSEGV is used only to track writes
wcfs/gen-δtail deleted 100755 → 0
View file @ d0fe680a
#!/bin/bash -e
# δtail.go.in -> specialized with concrete types
# gen-δtail KIND ID out
# Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
# TODO kill this after finishing ΔFtail (zodb/δtail.go generic is not used here)
KIND=$1
ID=$2
out=$3
zodb=lab.nexedi.com/kirr/neo/go/zodb
zdir=`go list -f '{{.Dir}}' $zodb`
zrev=`git -C $zdir describe --always`
echo "// Code generated by gen-δtail $KIND $ID; DO NOT EDIT." >$out
echo "// (from $zodb @ $zrev)" >>$out
echo >>$out
$zdir/δtail.go.cat-generic | sed \
-e "s/PACKAGE/main/g" \
-e "s/ID/$ID/g" \
-e "s/ΔTail/ΔTail${KIND}/g" \
-e "s/δRevEntry/δRevEntry${KIND}/g" \
>>$out
wcfs/gogccrash deleted 100755 → 0
View file @ d0fe680a
#!/bin/bash -e
# tries to minimize reproducer for https://github.com/golang/go/issues/41303
export GOTRACEBACK=crash
ulimit -c unlimited
go test -c
cwd=$(pwd)
# runtest1
function runtest1() {
for i in `seq 1000`; do
echo -e "\n>>> #$i"
#GOGC=1 $cwd/wcfs.test -test.v -test.run 'TestZBlk|TestΔBTail|TestΔBTreeAllStructs' || break
#GOGC=1 $cwd/wcfs.test -test.v -test.run 'TestZBlk|TestΔBTail' || break
#GOGC=0 $cwd/wcfs.test -test.v -test.run 'TestZBlk|TestΔBTail' || break
GOGC=0 $cwd/wcfs.test -test.v -test.count=100 -test.run 'TestZBlk|TestΔBTail' || break
done
}
nwork=2
rm -rf BUG
for n in `seq $nwork`; do
workdir=BUG/$n
mkdir -p $workdir
ln -s $cwd/testdata $workdir
ln -s $cwd/testprog $workdir
(cd $workdir && runtest1 >>log 2>&1) &
done
wait -n
echo "done, crashes:"
find BUG -name "*core*"
echo
kill `jobs -p`
wait
wcfs/internal/set/gen-set
View file @ f91982af
......@@ -21,10 +21,9 @@
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
PACKAGE=$1
KIND=$2
VALUE=$3
out=$4
KIND=$1
VALUE=$2
out=$3
input=$(dirname $0)/set.go.in
......@@ -32,7 +31,6 @@ echo "// Code generated by gen-set $KIND $VALUE; DO NOT EDIT." >$out
echo >>$out
sed \
-e "s/PACKAGE/$PACKAGE/g" \
-e "s/VALUE/$VALUE/g" \
-e "s/Set/Set${KIND}/g" \
-e "s/Set/${KIND}/g" \
$input >>$out
wcfs/internal/set/set.go
View file @ f91982af
......@@ -19,9 +19,10 @@
package set
//go:generate ./gen-set set I64 int64 zset_i64.go
//go:generate ./gen-set set Oid _Oid zset_oid.go
//go:generate ./gen-set set Tid _Tid zset_tid.go
//go:generate ./gen-set I64 int64 zset_i64.go
//go:generate ./gen-set Str string zset_str.go
//go:generate ./gen-set Oid _Oid zset_oid.go
//go:generate ./gen-set Tid _Tid zset_tid.go
import (
"lab.nexedi.com/kirr/neo/go/zodb"
......
wcfs/internal/set/set.go.in
View file @ f91982af
......@@ -17,7 +17,7 @@
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package PACKAGE
package set
import (
"fmt"
......
wcfs/internal/set/zset_i64.go
View file @ f91982af
......@@ -27,35 +27,35 @@ import (
"strings"
)
// SetI64 is a set of int64.
type SetI64 map[int64]struct{}
// I64 is a set of int64.
type I64 map[int64]struct{}
// Add adds v to the set.
func (s SetI64) Add(v int64) {
func (s I64) Add(v int64) {
s[v] = struct{}{}
}
// Del removes v from the set.
// it is noop if v was not in the set.
func (s SetI64) Del(v int64) {
func (s I64) Del(v int64) {
delete(s, v)
}
// Has checks whether the set contains v.
func (s SetI64) Has(v int64) bool {
func (s I64) Has(v int64) bool {
_, ok := s[v]
return ok
}
// Update adds t values to s.
func (s SetI64) Update(t SetI64) {
func (s I64) Update(t I64) {
for v := range t {
s.Add(v)
}
}
// Elements returns all elements of set as slice.
func (s SetI64) Elements() []int64 {
func (s I64) Elements() []int64 {
ev := make([]int64, len(s))
i := 0
for e := range s {
......@@ -66,14 +66,14 @@ func (s SetI64) Elements() []int64 {
}
// Union returns s ∪ t
func (s SetI64) Union(t SetI64) SetI64 {
func (s I64) Union(t I64) I64 {
// l = max(len(s), len(t))
l := len(s)
if lt := len(t); lt > l {
l = lt
}
u := make(SetI64, l)
u := make(I64, l)
for v := range s {
u.Add(v)
......@@ -85,8 +85,8 @@ func (s SetI64) Union(t SetI64) SetI64 {
}
// Intersection returns s ∩ t
func (s SetI64) Intersection(t SetI64) SetI64 {
i := SetI64{}
func (s I64) Intersection(t I64) I64 {
i := I64{}
for v := range s {
if t.Has(v) {
i.Add(v)
......@@ -96,8 +96,8 @@ func (s SetI64) Intersection(t SetI64) SetI64 {
}
// Difference returns s\t.
func (s SetI64) Difference(t SetI64) SetI64 {
d := SetI64{}
func (s I64) Difference(t I64) I64 {
d := I64{}
for v := range s {
if !t.Has(v) {
d.Add(v)
......@@ -107,8 +107,8 @@ func (s SetI64) Difference(t SetI64) SetI64 {
}
// SymmetricDifference returns s Δ t.
func (s SetI64) SymmetricDifference(t SetI64) SetI64 {
d := SetI64{}
func (s I64) SymmetricDifference(t I64) I64 {
d := I64{}
for v := range s {
if !t.Has(v) {
d.Add(v)
......@@ -123,7 +123,7 @@ func (s SetI64) SymmetricDifference(t SetI64) SetI64 {
}
// Equal returns whether a == b.
func (a SetI64) Equal(b SetI64) bool {
func (a I64) Equal(b I64) bool {
if len(a) != len(b) {
return false
}
......@@ -139,8 +139,8 @@ func (a SetI64) Equal(b SetI64) bool {
}
// Clone returns copy of the set.
func (orig SetI64) Clone() SetI64 {
klon := make(SetI64, len(orig))
func (orig I64) Clone() I64 {
klon := make(I64, len(orig))
for v := range orig {
klon.Add(v)
}
......@@ -149,7 +149,7 @@ func (orig SetI64) Clone() SetI64 {
// --------
func (s SetI64) SortedElements() []int64 {
func (s I64) SortedElements() []int64 {
ev := s.Elements()
sort.Slice(ev, func(i, j int) bool {
return ev[i] < ev[j]
......@@ -157,7 +157,7 @@ func (s SetI64) SortedElements() []int64 {
return ev
}
func (s SetI64) String() string {
func (s I64) String() string {
ev := s.SortedElements()
strv := make([]string, len(ev))
for i, v := range ev {
......
wcfs/internal/set/zset_oid.go
View file @ f91982af
......@@ -27,35 +27,35 @@ import (
"strings"
)
// SetOid is a set of _Oid.
type SetOid map[_Oid]struct{}
// Oid is a set of _Oid.
type Oid map[_Oid]struct{}
// Add adds v to the set.
func (s SetOid) Add(v _Oid) {
func (s Oid) Add(v _Oid) {
s[v] = struct{}{}
}
// Del removes v from the set.
// it is noop if v was not in the set.
func (s SetOid) Del(v _Oid) {
func (s Oid) Del(v _Oid) {
delete(s, v)
}
// Has checks whether the set contains v.
func (s SetOid) Has(v _Oid) bool {
func (s Oid) Has(v _Oid) bool {
_, ok := s[v]
return ok
}
// Update adds t values to s.
func (s SetOid) Update(t SetOid) {
func (s Oid) Update(t Oid) {
for v := range t {
s.Add(v)
}
}
// Elements returns all elements of set as slice.
func (s SetOid) Elements() []_Oid {
func (s Oid) Elements() []_Oid {
ev := make([]_Oid, len(s))
i := 0
for e := range s {
......@@ -66,14 +66,14 @@ func (s SetOid) Elements() []_Oid {
}
// Union returns s ∪ t
func (s SetOid) Union(t SetOid) SetOid {
func (s Oid) Union(t Oid) Oid {
// l = max(len(s), len(t))
l := len(s)
if lt := len(t); lt > l {
l = lt
}
u := make(SetOid, l)
u := make(Oid, l)
for v := range s {
u.Add(v)
......@@ -85,8 +85,8 @@ func (s SetOid) Union(t SetOid) SetOid {
}
// Intersection returns s ∩ t
func (s SetOid) Intersection(t SetOid) SetOid {
i := SetOid{}
func (s Oid) Intersection(t Oid) Oid {
i := Oid{}
for v := range s {
if t.Has(v) {
i.Add(v)
......@@ -96,8 +96,8 @@ func (s SetOid) Intersection(t SetOid) SetOid {
}
// Difference returns s\t.
func (s SetOid) Difference(t SetOid) SetOid {
d := SetOid{}
func (s Oid) Difference(t Oid) Oid {
d := Oid{}
for v := range s {
if !t.Has(v) {
d.Add(v)
......@@ -107,8 +107,8 @@ func (s SetOid) Difference(t SetOid) SetOid {
}
// SymmetricDifference returns s Δ t.
func (s SetOid) SymmetricDifference(t SetOid) SetOid {
d := SetOid{}
func (s Oid) SymmetricDifference(t Oid) Oid {
d := Oid{}
for v := range s {
if !t.Has(v) {
d.Add(v)
......@@ -123,7 +123,7 @@ func (s SetOid) SymmetricDifference(t SetOid) SetOid {
}
// Equal returns whether a == b.
func (a SetOid) Equal(b SetOid) bool {
func (a Oid) Equal(b Oid) bool {
if len(a) != len(b) {
return false
}
......@@ -139,8 +139,8 @@ func (a SetOid) Equal(b SetOid) bool {
}
// Clone returns copy of the set.
func (orig SetOid) Clone() SetOid {
klon := make(SetOid, len(orig))
func (orig Oid) Clone() Oid {
klon := make(Oid, len(orig))
for v := range orig {
klon.Add(v)
}
......@@ -149,7 +149,7 @@ func (orig SetOid) Clone() SetOid {
// --------
func (s SetOid) SortedElements() []_Oid {
func (s Oid) SortedElements() []_Oid {
ev := s.Elements()
sort.Slice(ev, func(i, j int) bool {
return ev[i] < ev[j]
......@@ -157,7 +157,7 @@ func (s SetOid) SortedElements() []_Oid {
return ev
}
func (s SetOid) String() string {
func (s Oid) String() string {
ev := s.SortedElements()
strv := make([]string, len(ev))
for i, v := range ev {
......
wcfs/internal/zdata/zset_bigfile.go → wcfs/internal/set/zset_str.go
View file @ f91982af
// Code generated by gen-set ZBigFile *ZBigFile; DO NOT EDIT.
// Code generated by gen-set Str string; DO NOT EDIT.
// Copyright (C) 2015-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
......@@ -19,38 +19,44 @@
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package zdata
package set
// SetZBigFile is a set of *ZBigFile.
type SetZBigFile map[*ZBigFile]struct{}
import (
"fmt"
"sort"
"strings"
)
// Str is a set of string.
type Str map[string]struct{}
// Add adds v to the set.
func (s SetZBigFile) Add(v *ZBigFile) {
func (s Str) Add(v string) {
s[v] = struct{}{}
}
// Del removes v from the set.
// it is noop if v was not in the set.
func (s SetZBigFile) Del(v *ZBigFile) {
func (s Str) Del(v string) {
delete(s, v)
}
// Has checks whether the set contains v.
func (s SetZBigFile) Has(v *ZBigFile) bool {
func (s Str) Has(v string) bool {
_, ok := s[v]
return ok
}
// Update adds t values to s.
func (s SetZBigFile) Update(t SetZBigFile) {
func (s Str) Update(t Str) {
for v := range t {
s.Add(v)
}
}
// Elements returns all elements of set as slice.
func (s SetZBigFile) Elements() []*ZBigFile {
ev := make([]*ZBigFile, len(s))
func (s Str) Elements() []string {
ev := make([]string, len(s))
i := 0
for e := range s {
ev[i] = e
......@@ -60,14 +66,14 @@ func (s SetZBigFile) Elements() []*ZBigFile {
}
// Union returns s ∪ t
func (s SetZBigFile) Union(t SetZBigFile) SetZBigFile {
func (s Str) Union(t Str) Str {
// l = max(len(s), len(t))
l := len(s)
if lt := len(t); lt > l {
l = lt
}
u := make(SetZBigFile, l)
u := make(Str, l)
for v := range s {
u.Add(v)
......@@ -79,8 +85,8 @@ func (s SetZBigFile) Union(t SetZBigFile) SetZBigFile {
}
// Intersection returns s ∩ t
func (s SetZBigFile) Intersection(t SetZBigFile) SetZBigFile {
i := SetZBigFile{}
func (s Str) Intersection(t Str) Str {
i := Str{}
for v := range s {
if t.Has(v) {
i.Add(v)
......@@ -90,8 +96,8 @@ func (s SetZBigFile) Intersection(t SetZBigFile) SetZBigFile {
}
// Difference returns s\t.
func (s SetZBigFile) Difference(t SetZBigFile) SetZBigFile {
d := SetZBigFile{}
func (s Str) Difference(t Str) Str {
d := Str{}
for v := range s {
if !t.Has(v) {
d.Add(v)
......@@ -101,8 +107,8 @@ func (s SetZBigFile) Difference(t SetZBigFile) SetZBigFile {
}
// SymmetricDifference returns s Δ t.
func (s SetZBigFile) SymmetricDifference(t SetZBigFile) SetZBigFile {
d := SetZBigFile{}
func (s Str) SymmetricDifference(t Str) Str {
d := Str{}
for v := range s {
if !t.Has(v) {
d.Add(v)
......@@ -117,7 +123,7 @@ func (s SetZBigFile) SymmetricDifference(t SetZBigFile) SetZBigFile {
}
// Equal returns whether a == b.
func (a SetZBigFile) Equal(b SetZBigFile) bool {
func (a Str) Equal(b Str) bool {
if len(a) != len(b) {
return false
}
......@@ -131,3 +137,31 @@ func (a SetZBigFile) Equal(b SetZBigFile) bool {
return true
}
// Clone returns copy of the set.
func (orig Str) Clone() Str {
klon := make(Str, len(orig))
for v := range orig {
klon.Add(v)
}
return klon
}
// --------
func (s Str) SortedElements() []string {
ev := s.Elements()
sort.Slice(ev, func(i, j int) bool {
return ev[i] < ev[j]
})
return ev
}
func (s Str) String() string {
ev := s.SortedElements()
strv := make([]string, len(ev))
for i, v := range ev {
strv[i] = fmt.Sprintf("%v", v)
}
return "{" + strings.Join(strv, " ") + "}"
}
wcfs/internal/set/zset_tid.go
View file @ f91982af
......@@ -27,35 +27,35 @@ import (
"strings"
)
// SetTid is a set of _Tid.
type SetTid map[_Tid]struct{}
// Tid is a set of _Tid.
type Tid map[_Tid]struct{}
// Add adds v to the set.
func (s SetTid) Add(v _Tid) {
func (s Tid) Add(v _Tid) {
s[v] = struct{}{}
}
// Del removes v from the set.
// it is noop if v was not in the set.
func (s SetTid) Del(v _Tid) {
func (s Tid) Del(v _Tid) {
delete(s, v)
}
// Has checks whether the set contains v.
func (s SetTid) Has(v _Tid) bool {
func (s Tid) Has(v _Tid) bool {
_, ok := s[v]
return ok
}
// Update adds t values to s.
func (s SetTid) Update(t SetTid) {
func (s Tid) Update(t Tid) {
for v := range t {
s.Add(v)
}
}
// Elements returns all elements of set as slice.
func (s SetTid) Elements() []_Tid {
func (s Tid) Elements() []_Tid {
ev := make([]_Tid, len(s))
i := 0
for e := range s {
......@@ -66,14 +66,14 @@ func (s SetTid) Elements() []_Tid {
}
// Union returns s ∪ t
func (s SetTid) Union(t SetTid) SetTid {
func (s Tid) Union(t Tid) Tid {
// l = max(len(s), len(t))
l := len(s)
if lt := len(t); lt > l {
l = lt
}
u := make(SetTid, l)
u := make(Tid, l)
for v := range s {
u.Add(v)
......@@ -85,8 +85,8 @@ func (s SetTid) Union(t SetTid) SetTid {
}
// Intersection returns s ∩ t
func (s SetTid) Intersection(t SetTid) SetTid {
i := SetTid{}
func (s Tid) Intersection(t Tid) Tid {
i := Tid{}
for v := range s {
if t.Has(v) {
i.Add(v)
......@@ -96,8 +96,8 @@ func (s SetTid) Intersection(t SetTid) SetTid {
}
// Difference returns s\t.
func (s SetTid) Difference(t SetTid) SetTid {
d := SetTid{}
func (s Tid) Difference(t Tid) Tid {
d := Tid{}
for v := range s {
if !t.Has(v) {
d.Add(v)
......@@ -107,8 +107,8 @@ func (s SetTid) Difference(t SetTid) SetTid {
}
// SymmetricDifference returns s Δ t.
func (s SetTid) SymmetricDifference(t SetTid) SetTid {
d := SetTid{}
func (s Tid) SymmetricDifference(t Tid) Tid {
d := Tid{}
for v := range s {
if !t.Has(v) {
d.Add(v)
......@@ -123,7 +123,7 @@ func (s SetTid) SymmetricDifference(t SetTid) SetTid {
}
// Equal returns whether a == b.
func (a SetTid) Equal(b SetTid) bool {
func (a Tid) Equal(b Tid) bool {
if len(a) != len(b) {
return false
}
......@@ -139,8 +139,8 @@ func (a SetTid) Equal(b SetTid) bool {
}
// Clone returns copy of the set.
func (orig SetTid) Clone() SetTid {
klon := make(SetTid, len(orig))
func (orig Tid) Clone() Tid {
klon := make(Tid, len(orig))
for v := range orig {
klon.Add(v)
}
......@@ -149,7 +149,7 @@ func (orig SetTid) Clone() SetTid {
// --------
func (s SetTid) SortedElements() []_Tid {
func (s Tid) SortedElements() []_Tid {
ev := s.Elements()
sort.Slice(ev, func(i, j int) bool {
return ev[i] < ev[j]
......@@ -157,7 +157,7 @@ func (s SetTid) SortedElements() []_Tid {
return ev
}
func (s SetTid) String() string {
func (s Tid) String() string {
ev := s.SortedElements()
strv := make([]string, len(ev))
for i, v := range ev {
......
wcfs/internal/xbtree/blib/blib.go 0 → 100644
View file @ f91982af
// Copyright (C) 2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package blib provides utilities related to BTrees.
package blib
import (
"fmt"
"math"
"lab.nexedi.com/kirr/neo/go/zodb/btree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
)
// XXX instead of generics
type Tree = btree.LOBTree
type Bucket = btree.LOBucket
type Node = btree.LONode
type TreeEntry = btree.LOEntry
type BucketEntry = btree.LOBucketEntry
type Key = int64
const KeyMax Key = math.MaxInt64
const KeyMin Key = math.MinInt64
type setOid = set.Oid
// kstr formats key as string.
func kstr(k Key) string {
if k == KeyMin {
return "-∞"
}
if k == KeyMax {
return "∞"
}
return fmt.Sprintf("%d", k)
}
func panicf(format string, argv ...interface{}) {
panic(fmt.Sprintf(format, argv...))
}
wcfs/internal/xbtree/pptreesubset.go → wcfs/internal/xbtree/blib/pptreesubset.go
View file @ f91982af
......@@ -17,7 +17,7 @@
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree
package blib
// PP-connected subset of tree nodes.
import (
......@@ -61,6 +61,16 @@ type nodeInTree struct {
nchild int // number of direct children in PPTreeSubSet referring to this node
}
// Parent returns parent of this node.
func (n *nodeInTree) Parent() zodb.Oid {
return n.parent
}
// NChild returns number of children of this node in the tree subset.
func (n *nodeInTree) NChild() int {
return n.nchild
}
// Has returns whether node is in the set.
func (S PPTreeSubSet) Has(oid zodb.Oid) bool {
_, ok := S[oid]
......@@ -102,7 +112,7 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
// normalize path: remove embedded bucket and check whether it was an
// artificial empty tree.
path = normPath(path)
path = NormPath(path)
// go through path and add nodes to the set
parent := zodb.InvalidOid
......@@ -132,11 +142,11 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
}
}
// normPath normalizes path.
// NormPath normalizes path.
//
// It removes embedded buckets and artificial empty trees.
// Returned slice is subslice of path and aliases its memory.
func normPath(path []zodb.Oid) []zodb.Oid {
func NormPath(path []zodb.Oid) []zodb.Oid {
l := len(path)
// don't keep track of artificial empty tree
......@@ -363,12 +373,12 @@ func (S PPTreeSubSet) verify() {
}()
// recompute {} oid -> children and verify .nchild against it
children := make(map[zodb.Oid]SetOid, len(S))
children := make(map[zodb.Oid]setOid, len(S))
for oid, t := range S {
if t.parent != zodb.InvalidOid {
cc, ok := children[t.parent]
if !ok {
cc = make(SetOid, 1)
cc = make(setOid, 1)
children[t.parent] = cc
}
cc.Add(oid)
......@@ -464,20 +474,20 @@ func (t nodeInTree) String() string {
// The second component with "-22" builds from leaf, but the first
// component with "-43" builds from non-leaf node.
//
// δnchildNonLeafs = {43: +1}
// ΔnchildNonLeafs = {43: +1}
//
// Only complete result of applying all
//
// - xfixup(-1, δnchildNonLeafs)
// - xfixup(-1, ΔnchildNonLeafs)
// - δ.Del,
// - δ.Add, and
// - xfixup(+1, δnchildNonLeafs)
// - xfixup(+1, ΔnchildNonLeafs)
//
// produces correctly PP-connected set.
type ΔPPTreeSubSet struct {
Del PPTreeSubSet
Add PPTreeSubSet
δnchildNonLeafs map[zodb.Oid]int
ΔnchildNonLeafs map[zodb.Oid]int
}
// NewΔPPTreeSubSet creates new empty ΔPPTreeSubSet.
......@@ -485,7 +495,7 @@ func NewΔPPTreeSubSet() *ΔPPTreeSubSet {
return &ΔPPTreeSubSet{
Del: PPTreeSubSet{},
Add: PPTreeSubSet{},
δnchildNonLeafs: map[zodb.Oid]int{},
ΔnchildNonLeafs: map[zodb.Oid]int{},
}
}
......@@ -493,15 +503,15 @@ func NewΔPPTreeSubSet() *ΔPPTreeSubSet {
func (δ *ΔPPTreeSubSet) Update(δ2 *ΔPPTreeSubSet) {
δ.Del.UnionInplace(δ2.Del)
δ.Add.UnionInplace(δ2.Add)
for oid, δnc := range δ2.δnchildNonLeafs {
δ.δnchildNonLeafs[oid] += δnc
for oid, δnc := range δ2.ΔnchildNonLeafs {
δ.ΔnchildNonLeafs[oid] += δnc
}
}
// Reverse changes δ=diff(A->B) to δ'=diff(A<-B).
func (δ *ΔPPTreeSubSet) Reverse() {
δ.Del, δ.Add = δ.Add, δ.Del
// δnchildNonLeafs stays the same
// ΔnchildNonLeafs stays the same
}
......@@ -514,7 +524,7 @@ func (S PPTreeSubSet) ApplyΔ(δ *ΔPPTreeSubSet) {
fmt.Printf(" A: %s\n", S)
fmt.Printf(" -: %s\n", δ.Del)
fmt.Printf(" +: %s\n", δ.Add)
fmt.Printf(" x: %v\n", δ.δnchildNonLeafs)
fmt.Printf(" x: %v\n", δ.ΔnchildNonLeafs)
defer fmt.Printf("\n->B: %s\n", S)
}
......@@ -523,8 +533,8 @@ func (S PPTreeSubSet) ApplyΔ(δ *ΔPPTreeSubSet) {
δ.Add.verify()
defer S.verify()
S.xfixup(-1, δ.δnchildNonLeafs)
S.xfixup(-1, δ.ΔnchildNonLeafs)
S.xDifferenceInplace(δ.Del)
S.xUnionInplace(δ.Add)
S.xfixup(+1, δ.δnchildNonLeafs)
S.xfixup(+1, δ.ΔnchildNonLeafs)
}
wcfs/internal/xbtree/pptreesubset_test.go → wcfs/internal/xbtree/blib/pptreesubset_test.go
View file @ f91982af
......@@ -17,7 +17,7 @@
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree
package blib
import (
"strings"
......
wcfs/internal/xbtree/rangeset.go → wcfs/internal/xbtree/blib/rangeset.go
View file @ f91982af
......@@ -17,7 +17,7 @@
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree
package blib
// set of [lo,hi) Key ranges.
import (
......@@ -30,8 +30,8 @@ const debugRangeSet = false
// KeyRange represents [lo,hi) Key range.
type KeyRange struct {
lo Key
hi_ Key // NOTE _not_ hi) to avoid overflow at ∞; hi = hi_ + 1
Lo Key
Hi_ Key // NOTE _not_ hi) to avoid overflow at ∞; hi = hi_ + 1
}
// RangedKeySet is set of Keys with adjacent keys coalesced into Ranges.
......@@ -47,23 +47,23 @@ type RangedKeySet struct {
// Has returns whether key k belongs to the range.
func (r *KeyRange) Has(k Key) bool {
return (r.lo <= k && k <= r.hi_)
return (r.Lo <= k && k <= r.Hi_)
}
// Add adds key k to the set.
func (S *RangedKeySet) Add(k Key) {
S.AddRange(KeyRange{lo: k, hi_: k})
S.AddRange(KeyRange{Lo: k, Hi_: k})
}
// Del removes key k from the set.
func (S *RangedKeySet) Del(k Key) {
S.DelRange(KeyRange{lo: k, hi_: k})
S.DelRange(KeyRange{Lo: k, Hi_: k})
}
// Has returns whether key k belongs to the set.
func (S *RangedKeySet) Has(k Key) bool {
return S.HasRange(KeyRange{lo: k, hi_: k})
return S.HasRange(KeyRange{Lo: k, Hi_: k})
}
......@@ -79,10 +79,10 @@ func (S *RangedKeySet) AddRange(r KeyRange) {
S.verify()
defer S.verify()
// find first ilo: r.lo < [ilo].hi
// find first ilo: r.Lo < [ilo].hi
l := len(S.rangev)
ilo := sort.Search(l, func(i int) bool {
return r.lo <= S.rangev[i].hi_
return r.Lo <= S.rangev[i].Hi_
})
debugfRSet("\tilo: %d\n", ilo)
......@@ -92,58 +92,58 @@ func (S *RangedKeySet) AddRange(r KeyRange) {
debugfRSet("\tappend %s\t-> %s\n", r, S)
}
// find last jhi: [jhi].lo < r.hi
// find last jhi: [jhi].Lo < r.hi
jhi := ilo
for ;; jhi++ {
if jhi == l {
break
}
if S.rangev[jhi].lo <= r.hi_ {
if S.rangev[jhi].Lo <= r.Hi_ {
continue
}
break
}
debugfRSet("\tjhi: %d\n", jhi)
// entries in [ilo:jhi) ∈ [r.lo,r.hi) and should be merged into one
// entries in [ilo:jhi) ∈ [r.Lo,r.hi) and should be merged into one
if (jhi - ilo) > 1 {
lo := S.rangev[ilo].lo
hi_ := S.rangev[jhi-1].hi_
lo := S.rangev[ilo].Lo
hi_ := S.rangev[jhi-1].Hi_
vReplaceSlice(&S.rangev, ilo,jhi, KeyRange{lo,hi_})
debugfRSet("\tmerge S[%d:%d]\t-> %s\n", ilo, jhi, S)
}
jhi = -1 // no longer valid
// if [r.lo,r.hi) was outside of any entry - create new entry
if r.hi_ < S.rangev[ilo].lo {
if r.Hi_ < S.rangev[ilo].Lo {
vInsert(&S.rangev, ilo, r)
debugfRSet("\tinsert %s\t-> %s\n", r, S)
}
// now we have covered entries merged as needed into [ilo]
// extend this entry if r coverage is wider
if r.lo < S.rangev[ilo].lo {
S.rangev[ilo].lo = r.lo
if r.Lo < S.rangev[ilo].Lo {
S.rangev[ilo].Lo = r.Lo
debugfRSet("\textend left\t-> %s\n", S)
}
if r.hi_ > S.rangev[ilo].hi_ {
S.rangev[ilo].hi_ = r.hi_
if r.Hi_ > S.rangev[ilo].Hi_ {
S.rangev[ilo].Hi_ = r.Hi_
debugfRSet("\textend right\t-> %s\n", S)
}
// and check if we should merge it with right/left neighbours
if ilo+1 < len(S.rangev) { // right
if S.rangev[ilo].hi_+1 == S.rangev[ilo+1].lo {
if S.rangev[ilo].Hi_+1 == S.rangev[ilo+1].Lo {
vReplaceSlice(&S.rangev, ilo,ilo+2,
KeyRange{S.rangev[ilo].lo, S.rangev[ilo+1].hi_})
KeyRange{S.rangev[ilo].Lo, S.rangev[ilo+1].Hi_})
debugfRSet("\tmerge right\t-> %s\n", S)
}
}
if ilo > 0 { // left
if S.rangev[ilo-1].hi_+1 == S.rangev[ilo].lo {
if S.rangev[ilo-1].Hi_+1 == S.rangev[ilo].Lo {
vReplaceSlice(&S.rangev, ilo-1,ilo+1,
KeyRange{S.rangev[ilo-1].lo, S.rangev[ilo].hi_})
KeyRange{S.rangev[ilo-1].Lo, S.rangev[ilo].Hi_})
debugfRSet("\tmerge left\t-> %s\n", S)
}
}
......@@ -163,10 +163,10 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
S.verify()
defer S.verify()
// find first ilo: r.lo < [ilo].hi
// find first ilo: r.Lo < [ilo].hi
l := len(S.rangev)
ilo := sort.Search(l, func(i int) bool {
return r.lo <= S.rangev[i].hi_
return r.Lo <= S.rangev[i].Hi_
})
debugfRSet("\tilo: %d\n", ilo)
......@@ -175,13 +175,13 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
return
}
// find last jhi: [jhi].lo < r.hi
// find last jhi: [jhi].Lo < r.hi
jhi := ilo
for ;; jhi++ {
if jhi == l {
break
}
if S.rangev[jhi].lo <= r.hi_ {
if S.rangev[jhi].Lo <= r.Hi_ {
continue
}
break
......@@ -196,19 +196,19 @@ func (S *RangedKeySet) DelRange(r KeyRange) {
// [ilo+1:jhi-1] should be deleted
// [ilo] and [jhi-1] overlap with [r.lo,r.hi) - they should be deleted, or shrinked,
// or split+shrinked if ilo==jhi-1 and r is inside [ilo]
if jhi-ilo == 1 && S.rangev[ilo].lo < r.lo && r.hi_ < S.rangev[ilo].hi_ {
if jhi-ilo == 1 && S.rangev[ilo].Lo < r.Lo && r.Hi_ < S.rangev[ilo].Hi_ {
x := S.rangev[ilo]
vInsert(&S.rangev, ilo, x)
jhi++
debugfRSet("\tpresplit copy %s\t-> %s\n", x, S)
}
if S.rangev[ilo].lo < r.lo { // shrink left
S.rangev[ilo] = KeyRange{S.rangev[ilo].lo, r.lo-1}
if S.rangev[ilo].Lo < r.Lo { // shrink left
S.rangev[ilo] = KeyRange{S.rangev[ilo].Lo, r.Lo-1}
ilo++
debugfRSet("\tshrink [%d] left\t-> %s\n", ilo, S)
}
if r.hi_ < S.rangev[jhi-1].hi_ { // shrink right
S.rangev[jhi-1] = KeyRange{r.hi_+1, S.rangev[jhi-1].hi_}
if r.Hi_ < S.rangev[jhi-1].Hi_ { // shrink right
S.rangev[jhi-1] = KeyRange{r.Hi_+1, S.rangev[jhi-1].Hi_}
jhi--
debugfRSet("\tshrink [%d] right\t-> %s\n", jhi-1, S)
}
......@@ -237,7 +237,7 @@ func (S *RangedKeySet) HasRange(r KeyRange) (yes bool) {
// find first ilo: r.lo < [ilo].hi
l := len(S.rangev)
ilo := sort.Search(l, func(i int) bool {
return r.lo <= S.rangev[i].hi_
return r.Lo <= S.rangev[i].Hi_
})
debugfRSet("\tilo: %d\n", ilo)
......@@ -246,7 +246,7 @@ func (S *RangedKeySet) HasRange(r KeyRange) (yes bool) {
}
// all keys from r are in S if r ∈ [ilo]
return (S.rangev[ilo].lo <= r.lo && r.hi_ <= S.rangev[ilo].hi_)
return (S.rangev[ilo].Lo <= r.Lo && r.Hi_ <= S.rangev[ilo].Hi_)
}
......@@ -317,13 +317,13 @@ func (S *RangedKeySet) verify() {
hi_Prev := KeyMin
for i, r := range S.rangev {
hiPrev := hi_Prev + 1
if i > 0 && !(hiPrev < r.lo) { // NOTE not ≤ - adjacent ranges must be merged
if i > 0 && !(hiPrev < r.Lo) { // NOTE not ≤ - adjacent ranges must be merged
badf("[%d]: !(hiPrev < r.lo)", i)
}
if !(r.lo <= r.hi_) {
if !(r.Lo <= r.Hi_) {
badf("[%d]: !(r.lo <= r.hi_)", i)
}
hi_Prev = r.hi_
hi_Prev = r.Hi_
}
}
......@@ -378,12 +378,12 @@ func (S RangedKeySet) String() string {
func (r KeyRange) String() string {
var shi string
if r.hi_ == KeyMax {
shi = kstr(r.hi_) // ∞
if r.Hi_ == KeyMax {
shi = kstr(r.Hi_) // ∞
} else {
shi = fmt.Sprintf("%d", r.hi_+1)
shi = fmt.Sprintf("%d", r.Hi_+1)
}
return fmt.Sprintf("[%s,%s)", kstr(r.lo), shi)
return fmt.Sprintf("[%s,%s)", kstr(r.Lo), shi)
}
......
wcfs/internal/xbtree/rangeset_test.go → wcfs/internal/xbtree/blib/rangeset_test.go
View file @ f91982af
......@@ -17,7 +17,7 @@
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree
package blib
import (
"testing"
......
wcfs/internal/xbtree/treediff.go
View file @ f91982af
This diff is collapsed.
wcfs/internal/xbtree/xbtree.go
View file @ f91982af
......@@ -5,34 +5,33 @@ package xbtree
import (
"fmt"
"math"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/neo/go/zodb/btree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
)
// XXX instead of generics
type Tree = btree.LOBTree
type Bucket = btree.LOBucket
type Node = btree.LONode
type TreeEntry = btree.LOEntry
type BucketEntry = btree.LOBucketEntry
type Tree = blib.Tree
type Bucket = blib.Bucket
type Node = blib.Node
type TreeEntry = blib.TreeEntry
type BucketEntry = blib.BucketEntry
type Key = int64
const KeyMax Key = math.MaxInt64
const KeyMin Key = math.MinInt64
type Key = blib.Key
const KeyMax = blib.KeyMax
const KeyMin = blib.KeyMin
// value is assumed to be persistent reference.
// deletion is represented as VDEL.
type Value = zodb.Oid
const VDEL = zodb.InvalidOid
type SetKey = set.SetI64
type SetOid = set.SetOid
type SetTid = set.SetTid
type setKey = set.I64
type setOid = set.Oid
type setTid = set.Tid
......@@ -74,3 +73,7 @@ func kstr(k Key) string {
}
return fmt.Sprintf("%d", k)
}
func panicf(format string, argv ...interface{}) {
panic(fmt.Sprintf(format, argv...))
}
wcfs/internal/xbtree/xbtreetest/init/init.go 0 → 100644
View file @ f91982af
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package xbtreetest/init (ex imported from package A) should be imported in
// addition to xbtreetest (from package A_test) to initialize xbtreetest at runtime.
package init
// ZBlk-related part of δbtail_test
import (
"context"
"fmt"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
)
type Tree = xbtreetest.Tree
type Node = xbtreetest.Node
type Key = xbtreetest.Key
type ZBlk = zdata.ZBlk
// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
// XXX +return blkRevMax and use it ?
func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
path = []Node{}
xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
path = append(path, node)
})
if err != nil {
return nil, false, nil, err
}
if ok {
zblk, ok = xzblk.(ZBlk)
if !ok {
return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk)) // XXX errctx
}
}
return zblk, ok, path, nil
}
func init() {
xbtreetest.ZTreeGetBlkData = _ZTreeGetBlkData
xbtreetest.ZGetBlkData = _ZGetBlkData
}
// _ZTreeGetBlkData returns block data from block pointed to by ztree[k].
func _ZTreeGetBlkData(ctx context.Context, ztree *Tree, k Key) (data string, ok bool, path []Node, err error) {
defer xerr.Contextf(&err, "@%s: tree<%s>: get blkdata from [%d]", ztree.PJar().At(), ztree.POid(), k)
zblk, ok, path, err := ztreeGetBlk(ctx, ztree, k)
if err != nil || !ok {
return "", ok, path, err
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", false, nil, err
}
return string(bdata), true, path, nil
}
// _ZGetBlkData loads block data from ZBlk object specified by its oid.
func _ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
xblk, err := zconn.Get(ctx, zblkOid)
if err != nil {
return "", err
}
zblk, ok := xblk.(ZBlk)
if !ok {
return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", err
}
return string(bdata), nil
}
wcfs/internal/xbtree/xbtreetest/kvdiff.go 0 → 100644
View file @ f91982af
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// kvdiff + friends
import (
"fmt"
"sort"
"strings"
)
// kvdiff returns difference in between kv1 and kv2.
const DEL = "ø" // DEL means deletion
type Δstring struct {
Old string
New string
}
func kvdiff(kv1, kv2 map[Key]string) map[Key]Δstring {
delta := map[Key]Δstring{}
keys := setKey{}
for k := range kv1 { keys.Add(k) }
for k := range kv2 { keys.Add(k) }
for k := range keys {
v1, ok := kv1[k]
if !ok { v1 = DEL }
v2, ok := kv2[k]
if !ok { v2 = DEL }
if v1 != v2 {
delta[k] = Δstring{v1,v2}
}
}
return delta
}
// kvtxt returns string representation of {} kv.
func kvtxt(kv map[Key]string) string {
if len(kv) == 0 {
return "ø"
}
keyv := []Key{}
for k := range kv { keyv = append(keyv, k) }
sort.Slice(keyv, func(i,j int) bool { return keyv[i] < keyv[j] })
sv := []string{}
for _, k := range keyv {
v := kv[k]
if strings.ContainsAny(v, " \n\t,:") {
panicf("[%v]=%q: invalid value", k, v)
}
sv = append(sv, fmt.Sprintf("%v:%s", k, v))
}
return strings.Join(sv, ",")
}
wcfs/internal/xbtree/misc.go → wcfs/internal/xbtree/xbtreetest/kvdiff_test.go
View file @ f91982af
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -17,12 +17,28 @@
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree
package xbtreetest
import (
"fmt"
"reflect"
"testing"
)
func panicf(format string, argv ...interface{}) {
panic(fmt.Sprintf(format, argv...))
func TestKVDiff(t *testing.T) {
kv1 := map[Key]string{1:"a", 3:"c", 4:"d"}
kv2 := map[Key]string{1:"b", 4:"d", 5:"e"}
got := kvdiff(kv1, kv2)
want := map[Key]Δstring{1:{"a","b"}, 3:{"c",DEL}, 5:{DEL,"e"}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("error:\ngot: %v\nwant: %v", got, want)
}
}
func TestKVTxt(t *testing.T) {
kv := map[Key]string{3:"hello", 1:"zzz", 4:"world"}
got := kvtxt(kv)
want := "1:zzz,3:hello,4:world"
if got != want {
t.Fatalf("error:\ngot: %q\nwant: %q", got, want)
}
}
wcfs/internal/xbtree/xbtreetest/rtree.go 0 → 100644
View file @ f91982af
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
import (
"fmt"
"sort"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
)
// RTree represents Tree node covering [lo, hi_] key range in its parent tree.
// XXX actually no coverage here -> kill? -> change to just `path []zodb.Oid` in RBucket?
type RTree struct {
Oid zodb.Oid
Parent *RTree
// XXX +children?
}
// RBucket represents Bucket node covering [lo, hi_] key range in its Tree.
// NOTE it is not [lo,hi) but [lo,hi_] instead to avoid overflow at KeyMax.
type RBucket struct {
Oid zodb.Oid
Parent *RTree
Keycov blib.KeyRange
KV map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
}
// Path returns path to this bucket from tree root.
func (rb *RBucket) Path() []zodb.Oid {
path := []zodb.Oid{rb.Oid}
p := rb.Parent
for p != nil {
path = append([]zodb.Oid{p.Oid}, path...)
p = p.Parent
}
return path
}
// RBucketSet represents set of buckets covering whole [-∞,∞) range.
type RBucketSet []*RBucket // k↑
// Get returns RBucket which covers key k.
func (rbs RBucketSet) Get(k Key) *RBucket {
i := sort.Search(len(rbs), func(i int) bool {
return k <= rbs[i].Keycov.Hi_
})
if i == len(rbs) {
panicf("BUG: key %v not covered; coverage: %s", k, rbs.coverage())
}
rb := rbs[i]
if !rb.Keycov.Has(k) {
panicf("BUG: get(%v) -> %s; coverage: %s", k, rb.Keycov, rbs.coverage())
}
return rb
}
// coverage returns string representation of rbs coverage structure.
func (rbs RBucketSet) coverage() string {
if len(rbs) == 0 {
return "ø"
}
s := ""
for _, rb := range rbs {
if s != "" {
s += " "
}
s += fmt.Sprintf("%s", rb.Keycov)
}
return s
}
// Flatten converts xkv with bucket structure into regular dict.
func (xkv RBucketSet) Flatten() map[Key]string {
kv := make(map[Key]string)
for _, b := range xkv {
for k,v := range b.KV {
kv[k] = v
}
}
return kv
}
func (b *RBucket) String() string {
return fmt.Sprintf("%sB%s{%s}", b.Keycov, b.Oid, kvtxt(b.KV))
}
wcfs/internal/xbtree/xbtreetest/treeenv.go 0 → 100644
View file @ f91982af
This diff is collapsed.
wcfs/internal/xbtree/xbtreetest/treegen.go 0 → 100644
View file @ f91982af
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// treegen.go provides functionality:
//
// - to commit a particular BTree topology into ZODB, and
// - to generate set of random tree topologies that all correspond to particular {k->v} dict.
//
// treegen.py is used as helper for both tasks.
import (
"bufio"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"lab.nexedi.com/kirr/go123/my"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
)
// TreeGenSrv represents connection to running `treegen ...` server.
type TreeGenSrv struct {
argv []string
pysrv *exec.Cmd // spawned `treegen ...`
pyin io.WriteCloser // input to pysrv
pyoutRaw io.ReadCloser // output from pysrv
pyout *bufio.Reader // buffered ^^^
}
// TreeSrv represents connection to running `treegen trees` server.
//
// Create it with StartTreeSrv(zurl).
// - Commit(treeTopology) -> tid
type TreeSrv struct {
*TreeGenSrv
zurl string
treeRoot zodb.Oid // oid of the tree treegen works on
head zodb.Tid // last made commit
}
// AllStructsSrv represents connection to running `treegen allstructs` server.
//
// Create it with StartAllStructsSrv().
// - AllStructs(maxdepth, maxsplit, n, seed, kv1, kv2)
type AllStructsSrv struct {
*TreeGenSrv
}
// StartTreeGenSrv spawns `treegen ...` server.
func StartTreeGenSrv(argv ...string) (_ *TreeGenSrv, hello string, err error) {
defer xerr.Contextf(&err, "treesrv %v: start", argv)
// spawn `treegen ...`
tg := &TreeGenSrv{argv: argv}
tg.pysrv = exec.Command(filepath.Dir(my.File())+"/treegen.py", argv...)
tg.pyin, err = tg.pysrv.StdinPipe()
if err != nil {
return nil, "", err
}
tg.pyoutRaw, err = tg.pysrv.StdoutPipe()
if err != nil {
return nil, "", err
}
tg.pyout = bufio.NewReader(tg.pyoutRaw)
tg.pysrv.Stderr = os.Stderr // no redirection
err = tg.pysrv.Start()
if err != nil {
return nil, "", err
}
// wait for hello message and return it
defer func() {
if err != nil {
tg.Close() // ignore error
}
}()
defer xerr.Context(&err, "handshake")
hello, err = tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, "", err
}
hello = strings.TrimSuffix(hello, "\n")
return tg, hello, nil
}
// Close shutdowns treegen server.
func (tg *TreeGenSrv) Close() (err error) {
defer xerr.Contextf(&err, "treegen %v: close", tg.argv)
err1 := tg.pyin.Close()
err2 := tg.pyoutRaw.Close()
err3 := tg.pysrv.Wait()
return xerr.Merge(err1, err2, err3)
}
// StartTreeSrv spawns `treegen trees` server.
func StartTreeSrv(zurl string) (_ *TreeSrv, err error) {
defer xerr.Contextf(&err, "tree.srv %s: start", zurl)
tgSrv, hello, err := StartTreeGenSrv("trees", zurl)
if err != nil {
return nil, err
}
tg := &TreeSrv{TreeGenSrv: tgSrv, zurl: zurl}
defer func() {
if err != nil {
tgSrv.Close() // ignore error
}
}()
// tree.srv start @<at> tree=<root>
defer xerr.Contextf(&err, "invalid hello %q", hello)
startRe := regexp.MustCompile(`^tree.srv start @([^ ]+) root=([^ ]+)$`)
m := startRe.FindStringSubmatch(hello)
if m == nil {
return nil, fmt.Errorf("unexpected format")
}
tg.head, err = zodb.ParseTid(m[1]) // <at>
if err != nil {
return nil, fmt.Errorf("tid: %s", err)
}
tg.treeRoot, err = zodb.ParseOid(m[2]) // <root>
if err != nil {
return nil, fmt.Errorf("root: %s", err)
}
return tg, nil
}
// StartAllStructsSrv spawns `treegen allstructs` server.
func StartAllStructsSrv() (_ *AllStructsSrv, err error) {
defer xerr.Context(&err, "allstructs.srv: start")
tgSrv, hello, err := StartTreeGenSrv("allstructs")
if err != nil {
return nil, err
}
sg := &AllStructsSrv{TreeGenSrv: tgSrv}
defer func() {
if err != nil {
tgSrv.Close() // ignore error
}
}()
defer xerr.Contextf(&err, "invalid hello %q", hello)
if hello != "# allstructs.srv start" {
return nil, fmt.Errorf("unexpected format")
}
return sg, nil
}
// Commit creates new commit with underlying tree changed to specified tree topology.
func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
defer xerr.Contextf(&err, "tree.srv %s: commit %s", tg.zurl, tree)
_, err = io.WriteString(tg.pyin, tree + "\n")
if err != nil {
return zodb.InvalidTid, err
}
reply, err := tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return zodb.InvalidTid, err
}
reply = strings.TrimSuffix(reply, "\n")
tid, err := zodb.ParseTid(reply)
if err != nil {
return zodb.InvalidTid, fmt.Errorf("invalid reply: %s", err)
}
tg.head = tid
return tid, nil
}
// AllStructs returns response from `treegen allstructs`
func (tg *AllStructsSrv) AllStructs(kv map[Key]string, maxdepth, maxsplit, n int, seed int64) (_ []string, err error) {
req := fmt.Sprintf("%d %d %d/%d %s", maxdepth, maxsplit, n, seed, kvtxt(kv))
defer xerr.Contextf(&err, "allstructs.srv: %s ", req)
_, err = io.WriteString(tg.pyin, req + "\n")
if err != nil {
return nil, err
}
structv := []string{}
for {
reply, err := tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err
}
reply = strings.TrimSuffix(reply, "\n")
if reply == "# ----" {
return structv, nil // end of response
}
if strings.HasPrefix(reply, "#") {
continue // comment
}
structv = append(structv, reply)
}
}
wcfs/internal/xbtree/xbtreetest/xbtreetest.go 0 → 100644
View file @ f91982af
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package xbtreetest provides infrastructure for testing LOBTree with ZBlk values.
// XXX -> treetest?
package xbtreetest
import (
"fmt"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
)
// XXX instead of generics
type Tree = blib.Tree
type Bucket = blib.Bucket
type Node = blib.Node
type TreeEntry = blib.TreeEntry
type BucketEntry = blib.BucketEntry
type Key = blib.Key
const KeyMax = blib.KeyMax
const KeyMin = blib.KeyMin
type setKey = set.I64
// XXX dup from xbtree (to avoid import cycle)
const VDEL = zodb.InvalidOid
func panicf(format string, argv ...interface{}) {
panic(fmt.Sprintf(format, argv...))
}
wcfs/internal/xbtree/xbtreetest/zdata.go 0 → 100644
View file @ f91982af
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// access to ZBlk data
import (
"context"
"lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/neo/go/transaction"
"lab.nexedi.com/kirr/neo/go/zodb"
_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
)
// ZBlk-related functions are imported at runtime by package xbtreetest/init
var (
ZTreeGetBlkData func(context.Context, *Tree, Key) (string, bool, []Node, error)
ZGetBlkData func(context.Context, *zodb.Connection, zodb.Oid) (string, error)
)
func zassertInitDone() {
if ZTreeGetBlkData == nil {
panic("xbtreetest/zdata not initialized -> import xbtreetest/init to fix")
}
}
// xzgetBlkData loads block data from ZBlk object specified by its oid.
func xzgetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) string {
zassertInitDone()
X := exc.Raiseif
if zblkOid == VDEL {
return DEL
}
data, err := ZGetBlkData(ctx, zconn, zblkOid); X(err)
return string(data)
}
// xzgetBlkDataAt loads block data from ZBlk object specified by oid@at.
func xzgetBlkDataAt(db *zodb.DB, zblkOid zodb.Oid, at zodb.Tid) string {
zassertInitDone()
X := exc.Raiseif
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)
return xzgetBlkData(ctx, zconn, zblkOid)
}
wcfs/internal/xbtree/δbtail.go
View file @ f91982af
This diff is collapsed.
wcfs/internal/xbtree/δbtail_test.go
View file @ f91982af
This diff is collapsed.
wcfs/internal/xbtree/δbtail_x_test.go
View file @ f91982af
......@@ -18,85 +18,7 @@
// See https://www.nexedi.com/licensing for rationale and options.
package xbtree_test
// ZBlk-related part of δbtail_test
import (
"context"
"fmt"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
_ "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest/init"
)
type Tree = xbtree.Tree
type Node = xbtree.Node
type Key = xbtree.Key
type ZBlk = zdata.ZBlk
// ztreeGetBlk returns ztree[k] and tree path that lead to this block.
// XXX +return blkRevMax and use it ?
func ztreeGetBlk(ctx context.Context, ztree *Tree, k Key) (zblk ZBlk, ok bool, path []Node, err error) {
path = []Node{}
xzblk, ok, err := ztree.VGet(ctx, k, func(node Node) {
path = append(path, node)
})
if err != nil {
return nil, false, nil, err
}
if ok {
zblk, ok = xzblk.(ZBlk)
if !ok {
return nil, false, nil, fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xzblk)) // XXX errctx
}
}
return zblk, ok, path, nil
}
func init() {
xbtree.ZTreeGetBlkData = ZTreeGetBlkData
xbtree.ZGetBlkData = ZGetBlkData
}
// ZTreeGetBlkData returns block data from block pointed to by ztree[k].
func ZTreeGetBlkData(ctx context.Context, ztree *Tree, k Key) (data string, ok bool, path []Node, err error) {
defer xerr.Contextf(&err, "@%s: tree<%s>: get blkdata from [%d]", ztree.PJar().At(), ztree.POid(), k)
zblk, ok, path, err := ztreeGetBlk(ctx, ztree, k)
if err != nil || !ok {
return "", ok, path, err
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", false, nil, err
}
return string(bdata), true, path, nil
}
// ZGetBlkData loads block data from ZBlk object specified by its oid.
func ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
xblk, err := zconn.Get(ctx, zblkOid)
if err != nil {
return "", err
}
zblk, ok := xblk.(ZBlk)
if !ok {
return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", err
}
return string(bdata), nil
}
wcfs/internal/zdata/zblk.go
View file @ f91982af
......@@ -60,7 +60,6 @@ import (
// ZBlk is the interface that every ZBlk* block implements.
type ZBlk interface {
zodb.IPersistent
_ZBlkInΔFtail
// LoadBlkData loads from database and returns data block stored by this ZBlk.
//
......@@ -79,7 +78,6 @@ var _ ZBlk = (*ZBlk1)(nil)
// ZBlk0 mimics ZBlk0 from python.
type ZBlk0 struct {
zodb.Persistent
zblkInΔFtail
// NOTE py source uses bytes(buf) but on python2 it still results in str
blkdata string
......@@ -157,7 +155,6 @@ func (zd *zDataState) PySetState(pystate interface{}) error {
// ZBlk1 mimics ZBlk1 from python.
type ZBlk1 struct {
zodb.Persistent
zblkInΔFtail
chunktab *btree.IOBTree // {} offset -> ZData(chunk)
}
......@@ -468,7 +465,7 @@ func (bf *ZBigFile) LoadBlk(ctx context.Context, blk int64) (_ []byte, treePath
// Size returns whole file size.
//
// it also returns BTree path scaned to obtain the size.
// it also returns BTree path scanned to obtain the size.
func (bf *ZBigFile) Size(ctx context.Context) (_ int64, treePath []btree.LONode, err error) {
defer xerr.Contextf(&err, "bigfile %s: size", bf.POid())
......
wcfs/internal/zdata/zblk_test.go
View file @ f91982af
......@@ -35,8 +35,6 @@ import (
"github.com/stretchr/testify/require"
)
const K = 1024
// TestZBlk verifies that ZBlk* and ZBigFile saved by Python can be read correctly by Go.
// TODO also test with data saved by Python3.
func TestZBlk(t *testing.T) {
......
wcfs/internal/zdata/δftail.go
View file @ f91982af
......@@ -19,13 +19,9 @@
package zdata
//go:generate ../set/gen-set zdata ZBigFile *ZBigFile zset_bigfile.go
import (
"context"
"fmt"
"runtime"
"sync"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
......@@ -37,7 +33,8 @@ import (
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
)
type SetI64 = set.SetI64
type setI64 = set.I64
type setOid = set.Oid
// ΔFtail represents tail of revisional changes to files.
//
......@@ -78,8 +75,12 @@ type SetI64 = set.SetI64
type ΔFtail struct {
// ΔFtail merges ΔBtail with history of ZBlk
δBtail *xbtree.ΔBtail
fileIdx map[zodb.Oid]SetZBigFile // tree-root -> {} ZBigFile as of @head
fileIdx map[zodb.Oid]setOid // tree-root -> {} ZBigFile<oid> as of @head
trackSetZFile setOid // set of tracked ZBigFiles as of @head
trackSetZBlk map[zodb.Oid]*zblkTrack // zblk -> {} root -> {}blk as of @head
// XXX kill
///*
// XXX don't need vδF - everything is reconstructed at runtime from .δBtail.vδT
// this way we also don't need to keep up updating vδF from vδT on its rebuild during.
// data with δF changes. Actual for part of tracked set that was taken
......@@ -88,40 +89,29 @@ type ΔFtail struct {
// tracked ZBlk that are not yet taken into account in current vδF.
// grows on new track requests; flushes on queries and update.
trackNew map[*ZBigFile]map[zodb.Oid]*zblkInΔFtail // {} file -> {} oid -> zblk
trackNew map[zodb.Oid]map[zodb.Oid]*zblkTrack // {} foid -> {} zoid -> zblk
//*/
}
// zblkTrack keeps information in which root/blocks ZBlk is present as of @head.
type zblkTrack struct {
// inroot map[zodb.Oid]setI64 // {} root -> {}blk XXX later switch to this
infile map[zodb.Oid]setI64 // {} foid -> {}blk
}
// ΔF represents a change in files space.
type ΔF struct {
Rev zodb.Tid
ByFile map[*ZBigFile]*ΔFile // file -> δfile
ByFile map[zodb.Oid]*ΔFile // foid -> δfile
}
// ΔFile represents a change to one file.
type ΔFile struct {
Rev zodb.Tid
Blocks SetI64 // changed blocks
Size bool // whether file size changed
Blocks setI64 // changed blocks XXX -> ΔBlocks ?
Size bool // whether file size changed XXX -> ΔSize?
}
// zblkInΔFtail is ΔFtail-related volatile data embedded into ZBlk*.
//
// The data is preserved even when ZBlk comes to ghost state, but is lost if
// ZBlk is garbage collected. The data is transient - it is _not_ included into
// persistent state.
type zblkInΔFtail struct {
mu sync.Mutex // to support multiple concurrent loaders
// XXX change vvv to intree_parent? {} Bucket -> set(#blk)
// (this is uniform with building in-RAM reverse child->parents relation for
// tree nodes and for tree_root->file)
// with which files/blocks this ZBlk is associated with as of @head state
infile map[*ZBigFile]SetI64 // {} file -> set(#blk)
}
type _ZBlkInΔFtail interface { inΔFtail() *zblkInΔFtail }
func (z *zblkInΔFtail) inΔFtail() *zblkInΔFtail { return z }
// NewΔFtail creates new ΔFtail object.
//
......@@ -132,9 +122,11 @@ func (z *zblkInΔFtail) inΔFtail() *zblkInΔFtail { return z }
// ZODB when needed.
func NewΔFtail(at0 zodb.Tid, db *zodb.DB) *ΔFtail {
return &ΔFtail{
δBtail: xbtree.NewΔBtail(at0, db),
fileIdx: make(map[zodb.Oid]SetZBigFile),
trackNew: make(map[*ZBigFile]map[zodb.Oid]*zblkInΔFtail),
δBtail: xbtree.NewΔBtail(at0, db),
fileIdx: map[zodb.Oid]setOid{},
trackSetZFile: setOid{},
trackSetZBlk: map[zodb.Oid]*zblkTrack{},
trackNew: map[zodb.Oid]map[zodb.Oid]*zblkTrack{},
}
}
......@@ -156,6 +148,9 @@ func (δFtail *ΔFtail) Tail() zodb.Tid { return δFtail.δBtail.Tail() }
//
// A root can be associated with several files (each provided on different Track call).
func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zblk ZBlk) {
// XXX locking
foid := file.POid()
if blk == -1 {
// XXX blk = ∞ from beginning ?
blk = xbtree.KeyMax
......@@ -164,38 +159,44 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zb
if err != nil {
panic(err) // XXX -> error? errctx
}
root := path[0].(*btree.LOBTree)
files, ok := δFtail.fileIdx[root.POid()]
if !ok {
files = SetZBigFile{}
files = setOid{}
δFtail.fileIdx[root.POid()] = files
}
files.Add(file)
files.Add(foid)
δFtail.trackSetZFile.Add(foid)
// associate zblk with file, if it was not hole
if zblk != nil {
z := zblk.inΔFtail()
z.mu.Lock()
blocks, ok := z.infile[file]
zoid := zblk.POid()
zt, ok := δFtail.trackSetZBlk[zoid]
if !ok {
zt = &zblkTrack{}
δFtail.trackSetZBlk[zoid] = zt
}
blocks, ok := zt.infile[foid]
if !ok {
blocks = make(SetI64, 1)
if z.infile == nil {
z.infile = make(map[*ZBigFile]SetI64)
blocks = make(setI64, 1)
if zt.infile == nil {
zt.infile = make(map[zodb.Oid]setI64)
}
z.infile[file] = blocks
zt.infile[foid] = blocks
}
blocks.Add(blk)
z.mu.Unlock()
// XXX locking
if !ok {
// zblk was not associated with this file
zt := δFtail.trackNew[file]
if zt == nil {
zt = make(map[zodb.Oid]*zblkInΔFtail, 1)
δFtail.trackNew[file] = zt
ft := δFtail.trackNew[foid]
if ft == nil {
ft = make(map[zodb.Oid]*zblkTrack, 1)
δFtail.trackNew[foid] = ft
}
zt[zblk.POid()] = z
ft[zoid] = zt
}
}
......@@ -224,13 +225,14 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
// XXX δFtail.update() first?
// XXX verify zhead.At() == δFtail.Head()
// XXX locking
δB, err := δFtail.δBtail.Update(δZ)
if err != nil {
return ΔF{}, err
}
δF := ΔF{Rev: δB.Rev, ByFile: make(map[*ZBigFile]*ΔFile)}
δF := ΔF{Rev: δB.Rev, ByFile: make(map[zodb.Oid]*ΔFile)}
// take btree changes into account
for root, δt := range δB.ΔByRoot {
......@@ -241,7 +243,7 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
for file := range files {
δfile, ok := δF.ByFile[file]
if !ok {
δfile = &ΔFile{Rev: δF.Rev, Blocks: make(SetI64)}
δfile = &ΔFile{Rev: δF.Rev, Blocks: make(setI64)}
δF.ByFile[file] = δfile
}
for blk /*, zblk*/ := range δt {
......@@ -260,42 +262,29 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
// take zblk changes into account
for _, oid := range δZ.Changev {
// XXX cache lock/unlock
obj := zhead.Cache().Get(oid)
if obj == nil {
//fmt.Printf("%s: not in cache\n", oid)
continue // nothing to do - see invariant
if δFtail.trackSetZFile.Has(oid) {
// TODO check that .blksize and .blktab (it is only
// persistent reference) do not change.
return ΔF{}, fmt.Errorf("ZBigFile<%s> changed @%s", oid, δZ.Tid)
}
//fmt.Printf("%s: in cache (%s)\n", oid, typeOf(obj))
switch obj := obj.(type) {
case ZBlk: // ZBlk*
// z.infile locking: since we write-locked head.zheadMu
// - no other fuse reads are running, and thus no one
// is mutating z.infile. XXX recheck
z := obj.inΔFtail()
for file, blocks := range z.infile {
δfile, ok := δF.ByFile[file]
if !ok {
δfile = &ΔFile{Rev: δF.Rev, Blocks: make(SetI64)}
δF.ByFile[file] = δfile
}
zt, ok := δFtail.trackSetZBlk[oid]
if !ok {
continue // not tracked
}
δfile.Blocks.Update(blocks)
for foid, blocks := range zt.infile {
δfile, ok := δF.ByFile[foid]
if !ok {
δfile = &ΔFile{Rev: δF.Rev, Blocks: make(setI64)}
δF.ByFile[foid] = δfile
}
// XXX update z.infile according to btree changes
case *ZBigFile:
// TODO check that .blksize and .blktab (it is only
// persistent reference) do not change.
return ΔF{}, fmt.Errorf("ZBigFile<%s> changed @%s", oid, δZ.Tid)
δfile.Blocks.Update(blocks)
}
// make sure obj won't be garbage-collected until we finish handling it.
runtime.KeepAlive(obj)
// XXX update zt.infile according to btree changes
}
δFtail.vδF = append(δFtail.vδF, δF)
......@@ -313,15 +302,16 @@ func (δFtail *ΔFtail) update(file *ZBigFile) {
}
// let's see if we need to rebuild .vδF due to not-yet processed track requests
foid := file.POid()
// XXX locking
// XXX dumb
zt, dirty := δFtail.trackNew[file]
zt, dirty := δFtail.trackNew[foid]
if !dirty {
return
}
delete(δFtail.trackNew, file)
delete(δFtail.trackNew, foid)
// XXX unlock here
for i, δZ := range δFtail.δBtail.ΔZtail().Data() {
......@@ -335,14 +325,14 @@ func (δFtail *ΔFtail) update(file *ZBigFile) {
}
// XXX locking
// XXX -> func δF.δfile(file) ?
δfile, ok := δF.ByFile[file]
// XXX -> func δF.δfile(foid) ?
δfile, ok := δF.ByFile[foid]
if !ok {
δfile = &ΔFile{Rev: δF.Rev, Blocks: make(SetI64)}
δF.ByFile[file] = δfile
δfile = &ΔFile{Rev: δF.Rev, Blocks: make(setI64)}
δF.ByFile[foid] = δfile
}
δfile.Blocks.Update(z.infile[file])
δfile.Blocks.Update(z.infile[foid])
}
}
}
......@@ -353,10 +343,10 @@ func (δFtail *ΔFtail) ForgetPast(revCut zodb.Tid) {
}
// XXX don't need
func (δFtail *ΔFtail) SliceByRev(lo, hi zodb.Tid) /*readonly*/ []ΔF {
xtail.AssertSlice(δFtail, lo, hi)
panic("TODO")
}
//func (δFtail *ΔFtail) SliceByRev(lo, hi zodb.Tid) /*readonly*/ []ΔF {
// xtail.AssertSlice(δFtail, lo, hi)
// panic("TODO")
//}
// SliceByFileRev returns history of file changes in (lo, hi] range.
//
......@@ -398,9 +388,10 @@ func (δFtail *ΔFtail) SliceByFileRev(file *ZBigFile, lo, hi zodb.Tid) /*readon
vδF = vδF[i:j+1]
// filter found changed to have only file-related bits
foid := file.POid()
var vδfile []*ΔFile
for _, δF := range vδF {
δfile, ok := δF.ByFile[file]
δfile, ok := δF.ByFile[foid]
if ok {
vδfile = append(vδfile, δfile)
}
......
wcfs/internal/zdata/δftail_test.go 0 → 100644
View file @ f91982af
// Copyright (C) 2019-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package zdata
import (
"testing"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
)
type setStr = set.Str
// ΔFTestEntry represents one entry in ΔFtail tests.
type ΔFTestEntry struct {
δblkTab map[int64]string // change in tree part {} #blk -> ZBlk<oid>
δblkData setStr // change to ZBlk objects
}
func TestΔFtail(t *testing.T) {
// δT is shorthand to create δblkTab.
type δT = map[int64]string
// δD is shorthand to create δblkData.
δD := func(zblkv ...string) setStr {
δ := setStr{}
for _, zblk := range zblkv {
δ.Add(zblk)
}
return δ
}
const a, b, c, ø = "a", "b", "c", "ø"
testv := []ΔFTestEntry{
{δT{1:a,2:b,3:ø}, δD(a)},
{δT{}, δD(c)},
{δT{2:c}, δD(a,b)},
}
vδf := []ΔFile{} // (rev↑, {}blk) XXX +.Size?
blkTab := map[int64]string{} // #blk -> ZBlk<oid>
Zinblk := map[string]setI64{} // ZBlk<oid> -> which #blk refer to it
for _, test := range testv {
δf := setI64{}
for blk, zblk := range test.δblkTab {
// rebuild blkTab/Zinblk
zprev, ok := blkTab[blk]
if ok {
delete(Zinblk[zprev], blk)
} else {
zprev = ø
}
if zblk != ø {
blkTab[blk] = zblk
inblk, ok := Zinblk[zblk]
if !ok {
inblk = setI64{}
Zinblk[zblk] = inblk
}
inblk.Add(blk)
}
// update δf due to change in blkTab
if zblk != zprev {
δf.Add(blk)
}
}
// update δf due to change in ZBlk data
for zblk := range test.δblkData {
for blk := range Zinblk[zblk] {
δf.Add(blk)
}
}
vδf = append(vδf, ΔFile{
Rev: zodb.InvalidTid, // XXX will be set after treegen commit
Blocks: δf,
Size: false/*XXX*/,
})
}
}
// XXX TestΔFtailRandom(t *testing.T) {
//}
wcfs/wcfs.go
View file @ f91982af
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7