Skip to content

W
wendelin.core
Commit ef74aebc authored by Kirill Smelkov's avatar Kirill Smelkov
2 Browse files
Options

X ΔFtail: Keep reference to ZBigFile via Oid, not via *ZBigFile 

*ZBigFile is live object associated with particular ZODB Connection.
parent a458126e
Hide whitespace changes
Inline Side-by-side
Showing with 36 additions and 165 deletions
wcfs/internal/zdata/zset_bigfile.go deleted 100644 → 0
View file @ a458126e
// Code generated by gen-set ZBigFile *ZBigFile; DO NOT EDIT.
// Copyright (C) 2015-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package zdata
// SetZBigFile is a set of *ZBigFile.
type SetZBigFile map[*ZBigFile]struct{}
// Add adds v to the set.
func (s SetZBigFile) Add(v *ZBigFile) {
s[v] = struct{}{}
}
// Del removes v from the set.
// it is noop if v was not in the set.
func (s SetZBigFile) Del(v *ZBigFile) {
delete(s, v)
}
// Has checks whether the set contains v.
func (s SetZBigFile) Has(v *ZBigFile) bool {
_, ok := s[v]
return ok
}
// Update adds t values to s.
func (s SetZBigFile) Update(t SetZBigFile) {
for v := range t {
s.Add(v)
}
}
// Elements returns all elements of set as slice.
func (s SetZBigFile) Elements() []*ZBigFile {
ev := make([]*ZBigFile, len(s))
i := 0
for e := range s {
ev[i] = e
i++
}
return ev
}
// Union returns s ∪ t
func (s SetZBigFile) Union(t SetZBigFile) SetZBigFile {
// l = max(len(s), len(t))
l := len(s)
if lt := len(t); lt > l {
l = lt
}
u := make(SetZBigFile, l)
for v := range s {
u.Add(v)
}
for v := range t {
u.Add(v)
}
return u
}
// Intersection returns s ∩ t
func (s SetZBigFile) Intersection(t SetZBigFile) SetZBigFile {
i := SetZBigFile{}
for v := range s {
if t.Has(v) {
i.Add(v)
}
}
return i
}
// Difference returns s\t.
func (s SetZBigFile) Difference(t SetZBigFile) SetZBigFile {
d := SetZBigFile{}
for v := range s {
if !t.Has(v) {
d.Add(v)
}
}
return d
}
// SymmetricDifference returns s Δ t.
func (s SetZBigFile) SymmetricDifference(t SetZBigFile) SetZBigFile {
d := SetZBigFile{}
for v := range s {
if !t.Has(v) {
d.Add(v)
}
}
for v := range t {
if !s.Has(v) {
d.Add(v)
}
}
return d
}
// Equal returns whether a == b.
func (a SetZBigFile) Equal(b SetZBigFile) bool {
if len(a) != len(b) {
return false
}
for v := range a {
_, ok := b[v]
if !ok {
return false
}
}
return true
}
wcfs/internal/zdata/δftail.go
View file @ ef74aebc
......@@ -19,8 +19,6 @@
package zdata
//go:generate ../set/gen-set zdata ZBigFile *ZBigFile zset_bigfile.go
import (
"context"
"fmt"
......@@ -38,6 +36,7 @@ import (
)
type SetI64 = set.SetI64
type SetOid = set.SetOid
// ΔFtail represents tail of revisional changes to files.
//
......@@ -78,7 +77,7 @@ type SetI64 = set.SetI64
type ΔFtail struct {
// ΔFtail merges ΔBtail with history of ZBlk
δBtail *xbtree.ΔBtail
fileIdx map[zodb.Oid]SetZBigFile // tree-root -> {} ZBigFile as of @head
fileIdx map[zodb.Oid]SetOid // tree-root -> {} ZBigFile<oid> as of @head
// XXX kill
///*
......@@ -90,14 +89,14 @@ type ΔFtail struct {
// tracked ZBlk that are not yet taken into account in current vδF.
// grows on new track requests; flushes on queries and update.
trackNew map[*ZBigFile]map[zodb.Oid]*zblkInΔFtail // {} file -> {} oid -> zblk
trackNew map[zodb.Oid]map[zodb.Oid]*zblkInΔFtail // {} foid -> {} zoid -> zblk
//*/
}
// ΔF represents a change in files space.
type ΔF struct {
Rev zodb.Tid
ByFile map[*ZBigFile]*ΔFile // file -> δfile
ByFile map[zodb.Oid]*ΔFile // foid -> δfile
}
// ΔFile represents a change to one file.
......@@ -120,7 +119,7 @@ type zblkInΔFtail struct {
// tree nodes and for tree_root->file)
// with which files/blocks this ZBlk is associated with as of @head state
infile map[*ZBigFile]SetI64 // {} file -> set(#blk)
infile map[zodb.Oid]SetI64 // {} foid -> set(#blk)
}
type _ZBlkInΔFtail interface { inΔFtail() *zblkInΔFtail }
......@@ -136,8 +135,8 @@ func (z *zblkInΔFtail) inΔFtail() *zblkInΔFtail { return z }
func NewΔFtail(at0 zodb.Tid, db *zodb.DB) *ΔFtail {
return &ΔFtail{
δBtail: xbtree.NewΔBtail(at0, db),
fileIdx: make(map[zodb.Oid]SetZBigFile),
trackNew: make(map[*ZBigFile]map[zodb.Oid]*zblkInΔFtail),
fileIdx: make(map[zodb.Oid]SetOid),
trackNew: make(map[zodb.Oid]map[zodb.Oid]*zblkInΔFtail),
}
}
......@@ -159,6 +158,7 @@ func (δFtail *ΔFtail) Tail() zodb.Tid { return δFtail.δBtail.Tail() }
//
// A root can be associated with several files (each provided on different Track call).
func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zblk ZBlk) {
foid := file.POid()
if blk == -1 {
// XXX blk = ∞ from beginning ?
blk = xbtree.KeyMax
......@@ -170,22 +170,22 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zb
root := path[0].(*btree.LOBTree)
files, ok := δFtail.fileIdx[root.POid()]
if !ok {
files = SetZBigFile{}
files = SetOid{}
δFtail.fileIdx[root.POid()] = files
}
files.Add(file)
files.Add(foid)
// associate zblk with file, if it was not hole
if zblk != nil {
z := zblk.inΔFtail()
z.mu.Lock()
blocks, ok := z.infile[file]
blocks, ok := z.infile[foid]
if !ok {
blocks = make(SetI64, 1)
if z.infile == nil {
z.infile = make(map[*ZBigFile]SetI64)
z.infile = make(map[zodb.Oid]SetI64)
}
z.infile[file] = blocks
z.infile[foid] = blocks
}
blocks.Add(blk)
z.mu.Unlock()
......@@ -193,10 +193,10 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zb
// XXX locking
if !ok {
// zblk was not associated with this file
zt := δFtail.trackNew[file]
zt := δFtail.trackNew[foid]
if zt == nil {
zt = make(map[zodb.Oid]*zblkInΔFtail, 1)
δFtail.trackNew[file] = zt
δFtail.trackNew[foid] = zt
}
zt[zblk.POid()] = z
}
......@@ -233,7 +233,7 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
return ΔF{}, err
}
δF := ΔF{Rev: δB.Rev, ByFile: make(map[*ZBigFile]*ΔFile)}
δF := ΔF{Rev: δB.Rev, ByFile: make(map[zodb.Oid]*ΔFile)}
// take btree changes into account
for root, δt := range δB.ΔByRoot {
......@@ -316,15 +316,16 @@ func (δFtail *ΔFtail) update(file *ZBigFile) {
}
// let's see if we need to rebuild .vδF due to not-yet processed track requests
foid := file.POid()
// XXX locking
// XXX dumb
zt, dirty := δFtail.trackNew[file]
zt, dirty := δFtail.trackNew[foid]
if !dirty {
return
}
delete(δFtail.trackNew, file)
delete(δFtail.trackNew, foid)
// XXX unlock here
for i, δZ := range δFtail.δBtail.ΔZtail().Data() {
......@@ -338,14 +339,14 @@ func (δFtail *ΔFtail) update(file *ZBigFile) {
}
// XXX locking
// XXX -> func δF.δfile(file) ?
δfile, ok := δF.ByFile[file]
// XXX -> func δF.δfile(foid) ?
δfile, ok := δF.ByFile[foid]
if !ok {
δfile = &ΔFile{Rev: δF.Rev, Blocks: make(SetI64)}
δF.ByFile[file] = δfile
δF.ByFile[foid] = δfile
}
δfile.Blocks.Update(z.infile[file])
δfile.Blocks.Update(z.infile[foid])
}
}
}
......@@ -401,9 +402,10 @@ func (δFtail *ΔFtail) SliceByFileRev(file *ZBigFile, lo, hi zodb.Tid) /*readon
vδF = vδF[i:j+1]
// filter found changed to have only file-related bits
foid := file.POid()
var vδfile []*ΔFile
for _, δF := range vδF {
δfile, ok := δF.ByFile[file]
δfile, ok := δF.ByFile[foid]
if ok {
vδfile = append(vδfile, δfile)
}
......
wcfs/wcfs.go
View file @ ef74aebc
......@@ -882,7 +882,7 @@ retry:
if log.V(2) {
// debug dump δF
log.Infof("\n\nS: handleδZ: δF (#%d):\n", len(δF.ByFile))
for zfile, δfile := range δF.ByFile {
for foid, δfile := range δF.ByFile {
blkv := δfile.Blocks.Elements()
sort.Slice(blkv, func(i, j int) bool {
return blkv[i] < blkv[j]
......@@ -891,19 +891,19 @@ retry:
if δfile.Size {
size = "S"
}
log.Infof("S: \t- %s\t%s %v\n", zfile.POid(), size, blkv)
log.Infof("S: \t- %s\t%s %v\n", foid, size, blkv)
}
log.Infof("\n\n")
}
wg := xsync.NewWorkGroup(ctx)
for zfile, δfile := range δF.ByFile {
for foid, δfile := range δF.ByFile {
// // XXX needed?
// // XXX even though δBtail is complete, not all ZBlk are present here
// file.δtail.Append(δF.Rev, δfile.Blocks.Elements())
// zfile was requested to be tracked -> it must be present in fileTab
file := bfdir.fileTab[zfile.POid()]
// file was requested to be tracked -> it must be present in fileTab
file := bfdir.fileTab[foid]
for blk := range δfile.Blocks {
blk := blk
wg.Go(func(ctx context.Context) error {
......@@ -921,11 +921,11 @@ retry:
//
// do it after completing data invalidations.
wg = xsync.NewWorkGroup(ctx)
for zfile, δfile := range δF.ByFile {
for foid, δfile := range δF.ByFile {
if !δfile.Size {
continue
}
file := bfdir.fileTab[zfile.POid()] // must be present
file := bfdir.fileTab[foid] // must be present
wg.Go(func(ctx context.Context) error {
return file.invalidateAttr() // NOTE does not accept ctx
})
......@@ -950,13 +950,15 @@ retry:
// 2. restat invalidated ZBigFile
// NOTE no lock needed since .blksize and .size are constant during lifetime of one txn.
// XXX -> parallel
for zfile := range δF.ByFile {
for foid := range δF.ByFile {
file := bfdir.fileTab[foid] // must be present
zfile := file.zfile
size, sizePath, err := zfile.Size(ctx)
if err != nil {
return err
}
file := bfdir.fileTab[zfile.POid()] // must be present
file.size = size
bfdir.δFtail.Track(zfile, -1, sizePath, nil)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7