Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
W
wendelin.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Commits
Open sidebar
Kirill Smelkov
wendelin.core
Commits
70a8eb7d
Commit
70a8eb7d
authored
Jul 01, 2021
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
a458126e
Changes
10
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
908 additions
and
27 deletions
+908
-27
wcfs/internal/xbtree/xbtreetest/kvdiff.go
wcfs/internal/xbtree/xbtreetest/kvdiff.go
+74
-0
wcfs/internal/xbtree/xbtreetest/kvdiff_test.go
wcfs/internal/xbtree/xbtreetest/kvdiff_test.go
+44
-0
wcfs/internal/xbtree/xbtreetest/rtree.go
wcfs/internal/xbtree/xbtreetest/rtree.go
+106
-0
wcfs/internal/xbtree/xbtreetest/treeenv.go
wcfs/internal/xbtree/xbtreetest/treeenv.go
+353
-0
wcfs/internal/xbtree/xbtreetest/treegen.go
wcfs/internal/xbtree/xbtreetest/treegen.go
+238
-0
wcfs/internal/xbtree/xbtreetest/treegen.py
wcfs/internal/xbtree/xbtreetest/treegen.py
+0
-0
wcfs/internal/xbtree/xbtreetest/xbtreetest.go
wcfs/internal/xbtree/xbtreetest/xbtreetest.go
+48
-0
wcfs/internal/xbtree/δbtail_test.go
wcfs/internal/xbtree/δbtail_test.go
+0
-0
wcfs/internal/zdata/δftail.go
wcfs/internal/zdata/δftail.go
+41
-23
wcfs/wcfs.go
wcfs/wcfs.go
+4
-4
No files found.
wcfs/internal/xbtree/xbtreetest/kvdiff.go
0 → 100644
View file @
70a8eb7d
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package
xbtreetest
// kvdiff + friends
import
(
"fmt"
"sort"
"strings"
)
// kvdiff returns difference in between kv1 and kv2.
const
DEL
=
"ø"
// DEL means deletion
type
Δstring
struct
{
Old
string
New
string
}
func
kvdiff
(
kv1
,
kv2
map
[
Key
]
string
)
map
[
Key
]
Δstring
{
delta
:=
map
[
Key
]
Δstring
{}
keys
:=
SetKey
{}
for
k
:=
range
kv1
{
keys
.
Add
(
k
)
}
for
k
:=
range
kv2
{
keys
.
Add
(
k
)
}
for
k
:=
range
keys
{
v1
,
ok
:=
kv1
[
k
]
if
!
ok
{
v1
=
DEL
}
v2
,
ok
:=
kv2
[
k
]
if
!
ok
{
v2
=
DEL
}
if
v1
!=
v2
{
delta
[
k
]
=
Δstring
{
v1
,
v2
}
}
}
return
delta
}
// kvtxt returns string representation of {} kv.
func
kvtxt
(
kv
map
[
Key
]
string
)
string
{
if
len
(
kv
)
==
0
{
return
"ø"
}
keyv
:=
[]
Key
{}
for
k
:=
range
kv
{
keyv
=
append
(
keyv
,
k
)
}
sort
.
Slice
(
keyv
,
func
(
i
,
j
int
)
bool
{
return
keyv
[
i
]
<
keyv
[
j
]
})
sv
:=
[]
string
{}
for
_
,
k
:=
range
keyv
{
v
:=
kv
[
k
]
if
strings
.
ContainsAny
(
v
,
"
\n\t
,:"
)
{
panicf
(
"[%v]=%q: invalid value"
,
k
,
v
)
}
sv
=
append
(
sv
,
fmt
.
Sprintf
(
"%v:%s"
,
k
,
v
))
}
return
strings
.
Join
(
sv
,
","
)
}
wcfs/internal/xbtree/xbtreetest/kvdiff_test.go
0 → 100644
View file @
70a8eb7d
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package
xbtreetest
import
(
"reflect"
"testing"
)
func
TestKVDiff
(
t
*
testing
.
T
)
{
kv1
:=
map
[
Key
]
string
{
1
:
"a"
,
3
:
"c"
,
4
:
"d"
}
kv2
:=
map
[
Key
]
string
{
1
:
"b"
,
4
:
"d"
,
5
:
"e"
}
got
:=
kvdiff
(
kv1
,
kv2
)
want
:=
map
[
Key
]
Δstring
{
1
:
{
"a"
,
"b"
},
3
:
{
"c"
,
DEL
},
5
:
{
DEL
,
"e"
}}
if
!
reflect
.
DeepEqual
(
got
,
want
)
{
t
.
Fatalf
(
"error:
\n
got: %v
\n
want: %v"
,
got
,
want
)
}
}
func
TestKVTxt
(
t
*
testing
.
T
)
{
kv
:=
map
[
Key
]
string
{
3
:
"hello"
,
1
:
"zzz"
,
4
:
"world"
}
got
:=
kvtxt
(
kv
)
want
:=
"1:zzz,3:hello,4:world"
if
got
!=
want
{
t
.
Fatalf
(
"error:
\n
got: %q
\n
want: %q"
,
got
,
want
)
}
}
wcfs/internal/xbtree/xbtreetest/rtree.go
0 → 100644
View file @
70a8eb7d
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package
xbtreetest
import
(
"fmt"
"sort"
"lab.nexedi.com/kirr/neo/go/zodb"
)
// RTree represents Tree node covering [lo, hi_] key range in its parent tree.
// XXX actually no coverage here -> kill? -> change to just `path []zodb.Oid` in RBucket?
type
RTree
struct
{
oid
zodb
.
Oid
parent
*
RTree
// XXX +children?
}
// RBucket represents Bucket node covering [lo, hi_] key range in its Tree.
// NOTE it is not [lo,hi) but [lo,hi_] instead to avoid overflow at KeyMax.
type
RBucket
struct
{
oid
zodb
.
Oid
parent
*
RTree
lo
,
hi_
Key
// XXX -> KeyRange ?
kv
map
[
Key
]
string
// bucket's k->v; values were ZBlk objects whose data is loaded instead.
}
// Path returns path to this bucket from tree root.
func
(
rb
*
RBucket
)
Path
()
[]
zodb
.
Oid
{
path
:=
[]
zodb
.
Oid
{
rb
.
oid
}
p
:=
rb
.
parent
for
p
!=
nil
{
path
=
append
([]
zodb
.
Oid
{
p
.
oid
},
path
...
)
p
=
p
.
parent
}
return
path
}
// RBucketSet represents set of buckets covering whole [-∞,∞) range.
type
RBucketSet
[]
*
RBucket
// k↑
// Get returns RBucket which covers key k.
func
(
rbs
RBucketSet
)
Get
(
k
Key
)
*
RBucket
{
i
:=
sort
.
Search
(
len
(
rbs
),
func
(
i
int
)
bool
{
return
k
<=
rbs
[
i
]
.
hi_
})
if
i
==
len
(
rbs
)
{
panicf
(
"BUG: key %v not covered; coverage: %s"
,
k
,
rbs
.
coverage
())
}
rb
:=
rbs
[
i
]
if
!
(
rb
.
lo
<=
k
&&
k
<=
rb
.
hi_
)
{
panicf
(
"BUG: get(%v) -> [%v, %v]; coverage: %s"
,
k
,
rb
.
lo
,
rb
.
hi_
,
rbs
.
coverage
())
}
return
rb
}
// coverage returns string representation of rbs coverage structure.
func
(
rbs
RBucketSet
)
coverage
()
string
{
if
len
(
rbs
)
==
0
{
return
"ø"
}
s
:=
""
for
_
,
rb
:=
range
rbs
{
if
s
!=
""
{
s
+=
" "
}
s
+=
fmt
.
Sprintf
(
"[%v, %v]"
,
rb
.
lo
,
rb
.
hi_
)
}
return
s
}
// Flatten converts xkv with bucket structure into regular dict.
func
(
xkv
RBucketSet
)
Flatten
()
map
[
Key
]
string
{
kv
:=
make
(
map
[
Key
]
string
)
for
_
,
b
:=
range
xkv
{
for
k
,
v
:=
range
b
.
kv
{
kv
[
k
]
=
v
}
}
return
kv
}
func
(
b
*
RBucket
)
String
()
string
{
return
fmt
.
Sprintf
(
"%sB%s{%s}"
,
KeyRange
{
b
.
lo
,
b
.
hi_
},
b
.
oid
,
kvtxt
(
b
.
kv
))
}
wcfs/internal/xbtree/xbtreetest/treeenv.go
0 → 100644
View file @
70a8eb7d
This diff is collapsed.
Click to expand it.
wcfs/internal/xbtree/xbtreetest/treegen.go
0 → 100644
View file @
70a8eb7d
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package
xbtreetest
// treegen.go provides functionality:
//
// - to commit a particular BTree topology into ZODB, and
// - to generate set of random tree topologies that all correspond to particular {k->v} dict.
//
// treegen.py is used as helper for both tasks.
import
(
"bufio"
"fmt"
"io"
"os"
"os/exec"
"regexp"
"strings"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
)
// TreeGenSrv represents connection to running `treegen ...` server.
type
TreeGenSrv
struct
{
argv
[]
string
pysrv
*
exec
.
Cmd
// spawned `treegen ...`
pyin
io
.
WriteCloser
// input to pysrv
pyoutRaw
io
.
ReadCloser
// output from pysrv
pyout
*
bufio
.
Reader
// buffered ^^^
}
// TreeSrv represents connection to running `treegen trees` server.
//
// Create it with StartTreeSrv(zurl).
// - Commit(treeTopology) -> tid
type
TreeSrv
struct
{
*
TreeGenSrv
zurl
string
treeRoot
zodb
.
Oid
// oid of the tree treegen works on
head
zodb
.
Tid
// last made commit
}
// AllStructsSrv represents connection to running `treegen allstructs` server.
//
// Create it with StartAllStructsSrv().
// - AllStructs(maxdepth, maxsplit, n, seed, kv1, kv2)
type
AllStructsSrv
struct
{
*
TreeGenSrv
}
// StartTreeGenSrv spawns `treegen ...` server.
func
StartTreeGenSrv
(
argv
...
string
)
(
_
*
TreeGenSrv
,
hello
string
,
err
error
)
{
defer
xerr
.
Contextf
(
&
err
,
"treesrv %v: start"
,
argv
)
// spawn `treegen ...`
tg
:=
&
TreeGenSrv
{
argv
:
argv
}
tg
.
pysrv
=
exec
.
Command
(
"./testprog/treegen.py"
,
argv
...
)
tg
.
pyin
,
err
=
tg
.
pysrv
.
StdinPipe
()
if
err
!=
nil
{
return
nil
,
""
,
err
}
tg
.
pyoutRaw
,
err
=
tg
.
pysrv
.
StdoutPipe
()
if
err
!=
nil
{
return
nil
,
""
,
err
}
tg
.
pyout
=
bufio
.
NewReader
(
tg
.
pyoutRaw
)
tg
.
pysrv
.
Stderr
=
os
.
Stderr
// no redirection
err
=
tg
.
pysrv
.
Start
()
if
err
!=
nil
{
return
nil
,
""
,
err
}
// wait for hello message and return it
defer
func
()
{
if
err
!=
nil
{
tg
.
Close
()
// ignore error
}
}()
defer
xerr
.
Context
(
&
err
,
"handshake"
)
hello
,
err
=
tg
.
pyout
.
ReadString
(
'\n'
)
if
err
!=
nil
{
if
err
==
io
.
EOF
{
err
=
io
.
ErrUnexpectedEOF
}
return
nil
,
""
,
err
}
hello
=
strings
.
TrimSuffix
(
hello
,
"
\n
"
)
return
tg
,
hello
,
nil
}
// Close shutdowns treegen server.
func
(
tg
*
TreeGenSrv
)
Close
()
(
err
error
)
{
defer
xerr
.
Contextf
(
&
err
,
"treegen %v: close"
,
tg
.
argv
)
err1
:=
tg
.
pyin
.
Close
()
err2
:=
tg
.
pyoutRaw
.
Close
()
err3
:=
tg
.
pysrv
.
Wait
()
return
xerr
.
Merge
(
err1
,
err2
,
err3
)
}
// StartTreeSrv spawns `treegen trees` server.
func
StartTreeSrv
(
zurl
string
)
(
_
*
TreeSrv
,
err
error
)
{
defer
xerr
.
Contextf
(
&
err
,
"tree.srv %s: start"
,
zurl
)
tgSrv
,
hello
,
err
:=
StartTreeGenSrv
(
"trees"
,
zurl
)
if
err
!=
nil
{
return
nil
,
err
}
tg
:=
&
TreeSrv
{
TreeGenSrv
:
tgSrv
,
zurl
:
zurl
}
defer
func
()
{
if
err
!=
nil
{
tgSrv
.
Close
()
// ignore error
}
}()
// tree.srv start @<at> tree=<root>
defer
xerr
.
Contextf
(
&
err
,
"invalid hello %q"
,
hello
)
startRe
:=
regexp
.
MustCompile
(
`^tree.srv start @([^ ]+) root=([^ ]+)$`
)
m
:=
startRe
.
FindStringSubmatch
(
hello
)
if
m
==
nil
{
return
nil
,
fmt
.
Errorf
(
"unexpected format"
)
}
tg
.
head
,
err
=
zodb
.
ParseTid
(
m
[
1
])
// <at>
if
err
!=
nil
{
return
nil
,
fmt
.
Errorf
(
"tid: %s"
,
err
)
}
tg
.
treeRoot
,
err
=
zodb
.
ParseOid
(
m
[
2
])
// <root>
if
err
!=
nil
{
return
nil
,
fmt
.
Errorf
(
"root: %s"
,
err
)
}
return
tg
,
nil
}
// StartAllStructsSrv spawns `treegen allstructs` server.
func
StartAllStructsSrv
()
(
_
*
AllStructsSrv
,
err
error
)
{
defer
xerr
.
Context
(
&
err
,
"allstructs.srv: start"
)
tgSrv
,
hello
,
err
:=
StartTreeGenSrv
(
"allstructs"
)
if
err
!=
nil
{
return
nil
,
err
}
sg
:=
&
AllStructsSrv
{
TreeGenSrv
:
tgSrv
}
defer
func
()
{
if
err
!=
nil
{
tgSrv
.
Close
()
// ignore error
}
}()
defer
xerr
.
Contextf
(
&
err
,
"invalid hello %q"
,
hello
)
if
hello
!=
"# allstructs.srv start"
{
return
nil
,
fmt
.
Errorf
(
"unexpected format"
)
}
return
sg
,
nil
}
// Commit creates new commit with underlying tree changed to specified tree topology.
func
(
tg
*
TreeSrv
)
Commit
(
tree
string
)
(
_
zodb
.
Tid
,
err
error
)
{
defer
xerr
.
Contextf
(
&
err
,
"tree.srv %s: commit %s"
,
tg
.
zurl
,
tree
)
_
,
err
=
io
.
WriteString
(
tg
.
pyin
,
tree
+
"
\n
"
)
if
err
!=
nil
{
return
zodb
.
InvalidTid
,
err
}
reply
,
err
:=
tg
.
pyout
.
ReadString
(
'\n'
)
if
err
!=
nil
{
if
err
==
io
.
EOF
{
err
=
io
.
ErrUnexpectedEOF
}
return
zodb
.
InvalidTid
,
err
}
reply
=
strings
.
TrimSuffix
(
reply
,
"
\n
"
)
tid
,
err
:=
zodb
.
ParseTid
(
reply
)
if
err
!=
nil
{
return
zodb
.
InvalidTid
,
fmt
.
Errorf
(
"invalid reply: %s"
,
err
)
}
tg
.
head
=
tid
return
tid
,
nil
}
// AllStructs returns response from `treegen allstructs`
func
(
tg
*
AllStructsSrv
)
AllStructs
(
kv
map
[
Key
]
string
,
maxdepth
,
maxsplit
,
n
int
,
seed
int64
)
(
_
[]
string
,
err
error
)
{
req
:=
fmt
.
Sprintf
(
"%d %d %d/%d %s"
,
maxdepth
,
maxsplit
,
n
,
seed
,
kvtxt
(
kv
))
defer
xerr
.
Contextf
(
&
err
,
"allstructs.srv: %s "
,
req
)
_
,
err
=
io
.
WriteString
(
tg
.
pyin
,
req
+
"
\n
"
)
if
err
!=
nil
{
return
nil
,
err
}
structv
:=
[]
string
{}
for
{
reply
,
err
:=
tg
.
pyout
.
ReadString
(
'\n'
)
if
err
!=
nil
{
if
err
==
io
.
EOF
{
err
=
io
.
ErrUnexpectedEOF
}
return
nil
,
err
}
reply
=
strings
.
TrimSuffix
(
reply
,
"
\n
"
)
if
reply
==
"# ----"
{
return
structv
,
nil
// end of response
}
if
strings
.
HasPrefix
(
reply
,
"#"
)
{
continue
// comment
}
structv
=
append
(
structv
,
reply
)
}
}
wcfs/internal/xbtree/
testprog
/treegen.py
→
wcfs/internal/xbtree/
xbtreetest
/treegen.py
View file @
70a8eb7d
File moved
wcfs/internal/xbtree/xbtreetest/xbtreetest.go
0 → 100644
View file @
70a8eb7d
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package xbtreetest provides infrastructure for testing LOBTree with ZBlk values.
package
xbtreetest
import
(
"fmt"
"math"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/neo/go/zodb/btree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
// "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree"
)
// XXX dup from xbtree (to avoid import cycle)
type
Tree
=
btree
.
LOBTree
type
Bucket
=
btree
.
LOBucket
type
Key
=
int64
const
KeyMax
Key
=
math
.
MaxInt64
const
KeyMin
Key
=
math
.
MinInt64
type
SetKey
=
set
.
SetI64
const
VDEL
=
zodb
.
InvalidOid
func
panicf
(
format
string
,
argv
...
interface
{})
{
panic
(
fmt
.
Sprintf
(
format
,
argv
...
))
}
wcfs/internal/xbtree/δbtail_test.go
View file @
70a8eb7d
This diff is collapsed.
Click to expand it.
wcfs/internal/zdata/δftail.go
View file @
70a8eb7d
...
...
@@ -19,6 +19,7 @@
package
zdata
// XXX kill
//go:generate ../set/gen-set zdata ZBigFile *ZBigFile zset_bigfile.go
import
(
...
...
@@ -38,6 +39,7 @@ import (
)
type
SetI64
=
set
.
SetI64
type
SetOid
=
set
.
SetOid
// ΔFtail represents tail of revisional changes to files.
//
...
...
@@ -78,7 +80,8 @@ type SetI64 = set.SetI64
type
ΔFtail
struct
{
// ΔFtail merges ΔBtail with history of ZBlk
δBtail
*
xbtree
.
ΔBtail
fileIdx
map
[
zodb
.
Oid
]
SetZBigFile
// tree-root -> {} ZBigFile as of @head
// fileIdx map[zodb.Oid]SetZBigFile // tree-root -> {} ZBigFile as of @head
fileIdx
map
[
zodb
.
Oid
]
SetOid
// tree-root -> {} ZBigFile<oid> as of @head
// XXX kill
///*
...
...
@@ -90,14 +93,16 @@ type ΔFtail struct {
// tracked ZBlk that are not yet taken into account in current vδF.
// grows on new track requests; flushes on queries and update.
trackNew
map
[
*
ZBigFile
]
map
[
zodb
.
Oid
]
*
zblkInΔFtail
// {} file -> {} oid -> zblk
// trackNew map[*ZBigFile]map[zodb.Oid]*zblkInΔFtail // {} file -> {} oid -> zblk
trackNew
map
[
zodb
.
Oid
]
map
[
zodb
.
Oid
]
*
zblkInΔFtail
// {} root -> {} oid -> zblk
//*/
}
// ΔF represents a change in files space.
type
ΔF
struct
{
Rev
zodb
.
Tid
ByFile
map
[
*
ZBigFile
]
*
ΔFile
// file -> δfile
// ByFile map[*ZBigFile]*ΔFile // file -> δfile
ByFile
map
[
zodb
.
Oid
]
*
ΔFile
// file<oid> -> δfile
}
// ΔFile represents a change to one file.
...
...
@@ -120,7 +125,8 @@ type zblkInΔFtail struct {
// tree nodes and for tree_root->file)
// with which files/blocks this ZBlk is associated with as of @head state
infile
map
[
*
ZBigFile
]
SetI64
// {} file -> set(#blk)
// infile map[*ZBigFile]SetI64 // {} file -> set(#blk)
inroot
map
[
zodb
.
Oid
]
SetI64
// {} root -> set(#blk)
}
type
_ZBlkInΔFtail
interface
{
inΔFtail
()
*
zblkInΔFtail
}
...
...
@@ -136,8 +142,10 @@ func (z *zblkInΔFtail) inΔFtail() *zblkInΔFtail { return z }
func
NewΔFtail
(
at0
zodb
.
Tid
,
db
*
zodb
.
DB
)
*
ΔFtail
{
return
&
ΔFtail
{
δBtail
:
xbtree
.
NewΔBtail
(
at0
,
db
),
fileIdx
:
make
(
map
[
zodb
.
Oid
]
SetZBigFile
),
trackNew
:
make
(
map
[
*
ZBigFile
]
map
[
zodb
.
Oid
]
*
zblkInΔFtail
),
// fileIdx: make(map[zodb.Oid]SetZBigFile),
fileIdx
:
make
(
map
[
zodb
.
Oid
]
SetOid
),
// trackNew: make(map[*ZBigFile]map[zodb.Oid]*zblkInΔFtail),
trackNew
:
make
(
map
[
zodb
.
Oid
]
map
[
zodb
.
Oid
]
*
zblkInΔFtail
),
}
}
...
...
@@ -159,6 +167,7 @@ func (δFtail *ΔFtail) Tail() zodb.Tid { return δFtail.δBtail.Tail() }
//
// A root can be associated with several files (each provided on different Track call).
func
(
δFtail
*
ΔFtail
)
Track
(
file
*
ZBigFile
,
blk
int64
,
path
[]
btree
.
LONode
,
zblk
ZBlk
)
{
fileOid
:=
file
.
POid
()
if
blk
==
-
1
{
// XXX blk = ∞ from beginning ?
blk
=
xbtree
.
KeyMax
...
...
@@ -168,24 +177,29 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zb
panic
(
err
)
// XXX -> error? errctx
}
root
:=
path
[
0
]
.
(
*
btree
.
LOBTree
)
files
,
ok
:=
δFtail
.
fileIdx
[
root
.
POid
()]
rootOid
:=
root
.
POid
()
files
,
ok
:=
δFtail
.
fileIdx
[
rootOid
]
if
!
ok
{
files
=
Set
ZBigFile
{}
δFtail
.
fileIdx
[
root
.
POid
()
]
=
files
files
=
Set
Oid
{}
δFtail
.
fileIdx
[
root
Oid
]
=
files
}
files
.
Add
(
file
)
files
.
Add
(
file
Oid
)
// associate zblk with file, if it was not hole
if
zblk
!=
nil
{
z
:=
zblk
.
inΔFtail
()
z
.
mu
.
Lock
()
blocks
,
ok
:=
z
.
infile
[
file
]
// blocks, ok := z.infile[file]
blocks
,
ok
:=
z
.
inroot
[
rootOid
]
if
!
ok
{
blocks
=
make
(
SetI64
,
1
)
if
z
.
infile
==
nil
{
z
.
infile
=
make
(
map
[
*
ZBigFile
]
SetI64
)
// if z.infile == nil {
// z.infile = make(map[*ZBigFile]SetI64)
if
z
.
inroot
==
nil
{
z
.
inroot
=
make
(
map
[
zodb
.
Oid
]
SetI64
)
}
z
.
infile
[
file
]
=
blocks
// z.infile[file] = blocks
z
.
inroot
[
rootOid
]
=
blocks
}
blocks
.
Add
(
blk
)
z
.
mu
.
Unlock
()
...
...
@@ -193,10 +207,10 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zb
// XXX locking
if
!
ok
{
// zblk was not associated with this file
zt
:=
δFtail
.
trackNew
[
file
]
zt
:=
δFtail
.
trackNew
[
file
Oid
]
if
zt
==
nil
{
zt
=
make
(
map
[
zodb
.
Oid
]
*
zblkInΔFtail
,
1
)
δFtail
.
trackNew
[
file
]
=
zt
δFtail
.
trackNew
[
file
Oid
]
=
zt
}
zt
[
zblk
.
POid
()]
=
z
}
...
...
@@ -233,7 +247,7 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
return
ΔF
{},
err
}
δF
:=
ΔF
{
Rev
:
δB
.
Rev
,
ByFile
:
make
(
map
[
*
ZBigFile
]
*
ΔFile
)}
δF
:=
ΔF
{
Rev
:
δB
.
Rev
,
ByFile
:
make
(
map
[
zodb
.
Oid
]
*
ΔFile
)}
// take btree changes into account
for
root
,
δt
:=
range
δB
.
ΔByRoot
{
...
...
@@ -315,16 +329,18 @@ func (δFtail *ΔFtail) update(file *ZBigFile) {
panic
(
"TODO"
)
}
fileOid
:=
file
.
POid
()
// let's see if we need to rebuild .vδF due to not-yet processed track requests
// XXX locking
// XXX dumb
zt
,
dirty
:=
δFtail
.
trackNew
[
file
]
zt
,
dirty
:=
δFtail
.
trackNew
[
file
Oid
]
if
!
dirty
{
return
}
delete
(
δFtail
.
trackNew
,
file
)
delete
(
δFtail
.
trackNew
,
file
Oid
)
// XXX unlock here
for
i
,
δZ
:=
range
δFtail
.
δBtail
.
ΔZtail
()
.
Data
()
{
...
...
@@ -339,13 +355,13 @@ func (δFtail *ΔFtail) update(file *ZBigFile) {
// XXX locking
// XXX -> func δF.δfile(file) ?
δfile
,
ok
:=
δF
.
ByFile
[
file
]
δfile
,
ok
:=
δF
.
ByFile
[
file
Oid
]
if
!
ok
{
δfile
=
&
ΔFile
{
Rev
:
δF
.
Rev
,
Blocks
:
make
(
SetI64
)}
δF
.
ByFile
[
file
]
=
δfile
δF
.
ByFile
[
file
Oid
]
=
δfile
}
δfile
.
Blocks
.
Update
(
z
.
infile
[
file
])
δfile
.
Blocks
.
Update
(
z
.
infile
[
file
Oid
])
}
}
}
...
...
@@ -376,6 +392,8 @@ func (δFtail *ΔFtail) SliceByFileRev(file *ZBigFile, lo, hi zodb.Tid) /*readon
// FIXME rework to just query .δBtail.SliceByRootRev(file.blktab, lo, hi) +
// merge δZBlk history with that.
foid
:=
file
.
POid
()
// XXX locking?
δFtail
.
update
(
file
)
...
...
@@ -403,7 +421,7 @@ func (δFtail *ΔFtail) SliceByFileRev(file *ZBigFile, lo, hi zodb.Tid) /*readon
// filter found changed to have only file-related bits
var
vδfile
[]
*
ΔFile
for
_
,
δF
:=
range
vδF
{
δfile
,
ok
:=
δF
.
ByFile
[
f
ile
]
δfile
,
ok
:=
δF
.
ByFile
[
f
oid
]
if
ok
{
vδfile
=
append
(
vδfile
,
δfile
)
}
...
...
wcfs/wcfs.go
View file @
70a8eb7d
...
...
@@ -882,7 +882,7 @@ retry:
if
log
.
V
(
2
)
{
// debug dump δF
log
.
Infof
(
"
\n\n
S: handleδZ: δF (#%d):
\n
"
,
len
(
δF
.
ByFile
))
for
zfile
,
δfile
:=
range
δF
.
ByFile
{
for
zfile
Oid
,
δfile
:=
range
δF
.
ByFile
{
blkv
:=
δfile
.
Blocks
.
Elements
()
sort
.
Slice
(
blkv
,
func
(
i
,
j
int
)
bool
{
return
blkv
[
i
]
<
blkv
[
j
]
...
...
@@ -891,19 +891,19 @@ retry:
if
δfile
.
Size
{
size
=
"S"
}
log
.
Infof
(
"S:
\t
- %s
\t
%s %v
\n
"
,
zfile
.
POid
()
,
size
,
blkv
)
log
.
Infof
(
"S:
\t
- %s
\t
%s %v
\n
"
,
zfile
Oid
,
size
,
blkv
)
}
log
.
Infof
(
"
\n\n
"
)
}
wg
:=
xsync
.
NewWorkGroup
(
ctx
)
for
zfile
,
δfile
:=
range
δF
.
ByFile
{
for
zfile
Oid
,
δfile
:=
range
δF
.
ByFile
{
// // XXX needed?
// // XXX even though δBtail is complete, not all ZBlk are present here
// file.δtail.Append(δF.Rev, δfile.Blocks.Elements())
// zfile was requested to be tracked -> it must be present in fileTab
file
:=
bfdir
.
fileTab
[
zfile
.
POid
()
]
file
:=
bfdir
.
fileTab
[
zfile
Oid
]
for
blk
:=
range
δfile
.
Blocks
{
blk
:=
blk
wg
.
Go
(
func
(
ctx
context
.
Context
)
error
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment