Skip to content

W
wendelin.core
Commit 0a94ba43 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

.

parent 33e0dfce
Show whitespace changes
Inline Side-by-side
Showing with 416 additions and 175 deletions
wcfs/gen-δtail 0 → 100755
View file @ 0a94ba43
#!/bin/bash -e
# δtail.go.in -> specialized with concrete types
# gen-δtail KIND ID out
# Copyright (C) 2018 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
KIND=$1
ID=$2
out=$3
input=$(dirname $0)/δtail.go.in
echo "// Code generated by gen-δtail; DO NOT EDIT." >$out
echo >>$out
sed \
-e "s/ID/$ID/g" \
-e "s/ΔTail/ΔTail${KIND}/g" \
-e "s/δRevEntry/δRevEntry${KIND}/g" \
$input >>$out
wcfs/zδtail_i64.go 0 → 100644
View file @ 0a94ba43
// Code generated by gen-δtail; DO NOT EDIT.
// Copyright (C) 2018 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package main
import (
"fmt"
"lab.nexedi.com/kirr/neo/go/zodb"
)
// ΔTailI64 represents tail of revisional changes.
//
// It semantically consists of
//
// [](rev↑, []id)
//
// where
//
// rev - is ZODB revision, and
// id - is an identifier of what has been changed(*)
//
// It provides operations to
//
// - append information to the tail about next revision,
// - forget information in the tail past specified revision, and
// - query the tail about what is last revision that changed an id.
//
// It is generally not safe to use ΔTailI64 from multiple goroutines simultaneously.
// It is safe to perform multiple simultaneous read-kind operations.
//
// (*) examples of id:
//
// oid - ZODB object identifier, when ΔTailI64 represents changes to ZODB objects,
// #blk - file block number, when ΔTailI64 represents changes to a file.
type ΔTailI64 struct {
tailv []δRevEntryI64
lastRevOf map[int64]zodb.Tid // index for LastRevOf queries
// TODO also add either tailv idx <-> rev index, or lastRevOf -> tailv idx
// (if linear back-scan of δRevEntryI64 starts eat cpu).
}
// δRevEntryI64 represents information of what have been changed in one revision.
type δRevEntryI64 struct {
rev zodb.Tid
changev []int64
}
// NewΔTailI64 creates new ΔTailI64 object.
func NewΔTailI64() *ΔTailI64 {
return &ΔTailI64{lastRevOf: make(map[int64]zodb.Tid)}
}
// XXX add way to extend coverage without appending changed data? (i.e. if a
// txn did not change file at all) -> but then it is simply .Append(rev, nil)?
// Append appends to δtail information about what have been changed in next revision.
//
// rev must be ↑.
func (δtail *ΔTailI64) Append(rev zodb.Tid, changev []int64) {
// check rev↑
// XXX better also check even when δtail is ø (after forget)
if l := len(δtail.tailv); l > 0 {
if revPrev := δtail.tailv[l-1].rev; revPrev >= rev {
panic(fmt.Sprintf("δtail.Append: rev not ↑: %s -> %s", revPrev, rev))
}
}
δtail.tailv = append(δtail.tailv, δRevEntryI64{rev, changev})
for _, id := range changev {
δtail.lastRevOf[id] = rev
}
}
// ForgetBefore discards all δtail entries with rev < revCut.
func (δtail *ΔTailI64) ForgetBefore(revCut zodb.Tid) {
icut := 0
for i, δ := range δtail.tailv {
rev := δ.rev
if rev >= revCut {
break
}
icut = i+1
// if forgotten revision was last for id, we have to update lastRevOf index
for _, id := range δ.changev {
if δtail.lastRevOf[id] == rev {
delete(δtail.lastRevOf, id)
}
}
}
// tailv = tailv[icut:] but without
// 1) growing underlying storage array indefinitely
// 2) keeping underlying storage after forget
l := len(δtail.tailv)-icut
tailv := make([]δRevEntryI64, l)
copy(tailv, δtail.tailv[icut:])
δtail.tailv = tailv
}
// LastRevOf tries to return what was the last revision that changed id as of at database state.
//
// Depending on current information in δtail it returns either exact result, or
// an upper-bound estimate for the last id revision, assuming id was changed ≤ at:
//
// 1) if δtail does not cover at, at is returned:
//
// # at ∉ [min(rev ∈ δtail), max(rev ∈ δtail)]
// LastRevOf(id, at) = at
//
// 2) if δtail has an entry corresponding to id change, it gives exactly the
// last revision that changed id:
//
// # at ∈ [min(rev ∈ δtail), max(rev ∈ δtail)]
// # ∃ rev ∈ δtail: rev changed id && rev ≤ at
// LastRevOf(id, at) = max(rev: rev changed id && rev ≤ at)
//
// 3) if δtail does not contain appropriate record with id - it returns δtail's
// lower bound as the estimate for the upper bound of the last id revision:
//
// # at ∈ [min(rev ∈ δtail), max(rev ∈ δtail)]
// # ∄ rev ∈ δtail: rev changed id && rev ≤ at
// LastRevOf(id, at) = min(rev ∈ δtail)
//
// On return exact indicates whether returned revision is exactly the last
// revision of id, or only an upper-bound estimate of it.
func (δtail *ΔTailI64) LastRevOf(id int64, at zodb.Tid) (_ zodb.Tid, exact bool) {
// check if we have no coverage at all
l := len(δtail.tailv)
if l == 0 {
return at, false
}
revMin := δtail.tailv[0].rev
revMax := δtail.tailv[l-1].rev
if !(revMin <= at && at <= revMax) {
return at, false
}
// we have the coverage
rev, ok := δtail.lastRevOf[id]
if !ok {
return δtail.tailv[0].rev, false
}
if rev <= at {
return rev, true
}
// what's in index is after at - scan tailv back to find appropriate entry
// XXX linear scan
for i := l - 1; i >= 0; i-- {
δ := δtail.tailv[i]
if δ.rev > at {
continue
}
for _, δid := range δ.changev {
if id == δid {
return δ.rev, true
}
}
}
// nothing found
return δtail.tailv[0].rev, false
}
wcfs/δtail.go
View file @ 0a94ba43
......@@ -22,166 +22,4 @@
package main
// δtail maintenance
import (
"fmt"
"lab.nexedi.com/kirr/neo/go/zodb"
)
type ID int64 // XXX -> template ?
// ΔTail represents tail of revisional changes.
//
// It semantically consists of
//
// [](rev↑, []id)
//
// where
//
// rev - is ZODB revision, and
// id - is an identifier of what has been changed(*)
//
// It provides operations to
//
// - append information to the tail about next revision,
// - forget information in the tail past specified revision, and
// - query the tail about what is last revision that changed an id.
//
// It is generally not safe to use ΔTail from multiple goroutines simultaneously.
// It is safe to perform multiple simultaneous read-kind operations.
//
// (*) examples of id:
//
// oid - ZODB object identifier, when ΔTail represents changes to ZODB objects,
// #blk - file block number, when ΔTail represents changes to a file.
type ΔTail struct {
tailv []δRevEntry
lastRevOf map[ID]zodb.Tid // index for LastRevOf queries
// TODO also add either tailv idx <-> rev index, or lastRevOf -> tailv idx
// (if linear back-scan of δRevEntry starts eat cpu).
}
// δRevEntry represents information of what have been changed in one revision.
type δRevEntry struct {
rev zodb.Tid
changev []ID
}
// NewΔTail creates new ΔTail object.
func NewΔTail() *ΔTail {
return &ΔTail{lastRevOf: make(map[ID]zodb.Tid)}
}
// Append appends to δtail information about what have been changed in next revision.
//
// rev must be ↑.
func (δtail *ΔTail) Append(rev zodb.Tid, changev []ID) {
// check rev↑
// XXX better also check even when δtail is ø (after forget)
if l := len(δtail.tailv); l > 0 {
if revPrev := δtail.tailv[l-1].rev; revPrev >= rev {
panic(fmt.Sprintf("δtail.Append: rev not ↑: %s -> %s", revPrev, rev))
}
}
δtail.tailv = append(δtail.tailv, δRevEntry{rev, changev})
for _, id := range changev {
δtail.lastRevOf[id] = rev
}
}
// ForgetBefore discards all δtail entries with rev < revCut.
func (δtail *ΔTail) ForgetBefore(revCut zodb.Tid) {
icut := 0
for i, δ := range δtail.tailv {
rev := δ.rev
if rev >= revCut {
break
}
icut = i+1
// if forgotten revision was last for id, we have to update lastRevOf index
for _, id := range δ.changev {
if δtail.lastRevOf[id] == rev {
delete(δtail.lastRevOf, id)
}
}
}
// tailv = tailv[icut:] but without
// 1) growing underlying storage array indefinitely
// 2) keeping underlying storage after forget
l := len(δtail.tailv)-icut
tailv := make([]δRevEntry, l)
copy(tailv, δtail.tailv[icut:])
δtail.tailv = tailv
}
// LastRevOf tries to return what was the last revision that changed id as of at database state.
//
// Depending on current information in δtail it returns either exact result, or
// an upper-bound estimate for the last id revision, assuming id was changed ≤ at:
//
// 1) if δtail does not cover at, at is returned:
//
// # at ∉ [min(rev ∈ δtail), max(rev ∈ δtail)]
// LastRevOf(id, at) = at
//
// 2) if δtail has an entry corresponding to id change, it gives exactly the
// last revision that changed id:
//
// # at ∈ [min(rev ∈ δtail), max(rev ∈ δtail)]
// # ∃ rev ∈ δtail: rev changed id && rev ≤ at
// LastRevOf(id, at) = max(rev: rev changed id && rev ≤ at)
//
// 3) if δtail does not contain appropriate record with id - it returns δtail's
// lower bound as the estimate for the upper bound of the last id revision:
//
// # at ∈ [min(rev ∈ δtail), max(rev ∈ δtail)]
// # ∄ rev ∈ δtail: rev changed id && rev ≤ at
// LastRevOf(id, at) = min(rev ∈ δtail)
//
// On return exact indicates whether returned revision is exactly the last
// revision of id, or only an upper-bound estimate of it.
func (δtail *ΔTail) LastRevOf(id ID, at zodb.Tid) (_ zodb.Tid, exact bool) {
// check if we have no coverage at all
l := len(δtail.tailv)
if l == 0 {
return at, false
}
revMin := δtail.tailv[0].rev
revMax := δtail.tailv[l-1].rev
if !(revMin <= at && at <= revMax) {
return at, false
}
// we have the coverage
rev, ok := δtail.lastRevOf[id]
if !ok {
return δtail.tailv[0].rev, false
}
if rev <= at {
return rev, true
}
// what's in index is after at - scan tailv back to find appropriate entry
// XXX linear scan
for i := l - 1; i >= 0; i-- {
δ := δtail.tailv[i]
if δ.rev > at {
continue
}
for _, δid := range δ.changev {
if id == δid {
return δ.rev, true
}
}
}
// nothing found
return δtail.tailv[0].rev, false
}
//go:generate ./gen-δtail I64 int64 zδtail_i64.go
wcfs/δtail.go.in 0 → 100644
View file @ 0a94ba43
// Copyright (C) 2018 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package main
import (
"fmt"
"lab.nexedi.com/kirr/neo/go/zodb"
)
// ΔTail represents tail of revisional changes.
//
// It semantically consists of
//
// [](rev↑, []id)
//
// where
//
// rev - is ZODB revision, and
// id - is an identifier of what has been changed(*)
//
// It provides operations to
//
// - append information to the tail about next revision,
// - forget information in the tail past specified revision, and
// - query the tail about what is last revision that changed an id.
//
// It is generally not safe to use ΔTail from multiple goroutines simultaneously.
// It is safe to perform multiple simultaneous read-kind operations.
//
// (*) examples of id:
//
// oid - ZODB object identifier, when ΔTail represents changes to ZODB objects,
// #blk - file block number, when ΔTail represents changes to a file.
type ΔTail struct {
tailv []δRevEntry
lastRevOf map[ID]zodb.Tid // index for LastRevOf queries
// TODO also add either tailv idx <-> rev index, or lastRevOf -> tailv idx
// (if linear back-scan of δRevEntry starts eat cpu).
}
// δRevEntry represents information of what have been changed in one revision.
type δRevEntry struct {
rev zodb.Tid
changev []ID
}
// NewΔTail creates new ΔTail object.
func NewΔTail() *ΔTail {
return &ΔTail{lastRevOf: make(map[ID]zodb.Tid)}
}
// XXX add way to extend coverage without appending changed data? (i.e. if a
// txn did not change file at all) -> but then it is simply .Append(rev, nil)?
// Append appends to δtail information about what have been changed in next revision.
//
// rev must be ↑.
func (δtail *ΔTail) Append(rev zodb.Tid, changev []ID) {
// check rev↑
// XXX better also check even when δtail is ø (after forget)
if l := len(δtail.tailv); l > 0 {
if revPrev := δtail.tailv[l-1].rev; revPrev >= rev {
panic(fmt.Sprintf("δtail.Append: rev not ↑: %s -> %s", revPrev, rev))
}
}
δtail.tailv = append(δtail.tailv, δRevEntry{rev, changev})
for _, id := range changev {
δtail.lastRevOf[id] = rev
}
}
// ForgetBefore discards all δtail entries with rev < revCut.
func (δtail *ΔTail) ForgetBefore(revCut zodb.Tid) {
icut := 0
for i, δ := range δtail.tailv {
rev := δ.rev
if rev >= revCut {
break
}
icut = i+1
// if forgotten revision was last for id, we have to update lastRevOf index
for _, id := range δ.changev {
if δtail.lastRevOf[id] == rev {
delete(δtail.lastRevOf, id)
}
}
}
// tailv = tailv[icut:] but without
// 1) growing underlying storage array indefinitely
// 2) keeping underlying storage after forget
l := len(δtail.tailv)-icut
tailv := make([]δRevEntry, l)
copy(tailv, δtail.tailv[icut:])
δtail.tailv = tailv
}
// LastRevOf tries to return what was the last revision that changed id as of at database state.
//
// Depending on current information in δtail it returns either exact result, or
// an upper-bound estimate for the last id revision, assuming id was changed ≤ at:
//
// 1) if δtail does not cover at, at is returned:
//
// # at ∉ [min(rev ∈ δtail), max(rev ∈ δtail)]
// LastRevOf(id, at) = at
//
// 2) if δtail has an entry corresponding to id change, it gives exactly the
// last revision that changed id:
//
// # at ∈ [min(rev ∈ δtail), max(rev ∈ δtail)]
// # ∃ rev ∈ δtail: rev changed id && rev ≤ at
// LastRevOf(id, at) = max(rev: rev changed id && rev ≤ at)
//
// 3) if δtail does not contain appropriate record with id - it returns δtail's
// lower bound as the estimate for the upper bound of the last id revision:
//
// # at ∈ [min(rev ∈ δtail), max(rev ∈ δtail)]
// # ∄ rev ∈ δtail: rev changed id && rev ≤ at
// LastRevOf(id, at) = min(rev ∈ δtail)
//
// On return exact indicates whether returned revision is exactly the last
// revision of id, or only an upper-bound estimate of it.
func (δtail *ΔTail) LastRevOf(id ID, at zodb.Tid) (_ zodb.Tid, exact bool) {
// check if we have no coverage at all
l := len(δtail.tailv)
if l == 0 {
return at, false
}
revMin := δtail.tailv[0].rev
revMax := δtail.tailv[l-1].rev
if !(revMin <= at && at <= revMax) {
return at, false
}
// we have the coverage
rev, ok := δtail.lastRevOf[id]
if !ok {
return δtail.tailv[0].rev, false
}
if rev <= at {
return rev, true
}
// what's in index is after at - scan tailv back to find appropriate entry
// XXX linear scan
for i := l - 1; i >= 0; i-- {
δ := δtail.tailv[i]
if δ.rev > at {
continue
}
for _, δid := range δ.changev {
if id == δid {
return δ.rev, true
}
}
}
// nothing found
return δtail.tailv[0].rev, false
}
wcfs/δtail_test.go
View file @ 0a94ba43
......@@ -25,27 +25,23 @@ import (
"testing"
"lab.nexedi.com/kirr/neo/go/zodb"
//"github.com/stretchr/testify/require"
)
func TestΔTail(t *testing.T) {
//assert := require.New(t)
δtail := NewΔTail()
δtail := NewΔTailI64()
// R is syntactic sugar to create 1 δRevEntry
R := func(rev zodb.Tid, changev ...ID) δRevEntry {
return δRevEntry{rev, changev}
R := func(rev zodb.Tid, changev ...int64) δRevEntryI64 {
return δRevEntryI64{rev, changev}
}
// δAppend is syntactic sugar for δtail.Append
δAppend := func(δ δRevEntry) {
δAppend := func(δ δRevEntryI64) {
δtail.Append(δ.rev, δ.changev)
}
// δCheck verifies that δtail state corresponds to provided tailv
δCheck := func(tailv ...δRevEntry) {
δCheck := func(tailv ...δRevEntryI64) {
t.Helper()
for i := 1; i < len(tailv); i++ {
......@@ -59,7 +55,7 @@ func TestΔTail(t *testing.T) {
}
// verify lastRevOf query / index
lastRevOf := make(map[ID]zodb.Tid)
lastRevOf := make(map[int64]zodb.Tid)
for _, δ := range tailv {
for _, id := range δ.changev {
idRev, exact := δtail.LastRevOf(id, δ.rev)
......@@ -79,7 +75,7 @@ func TestΔTail(t *testing.T) {
// δCheckLastUP verifies that δtail.LastRevOf(id, at) gives lastOk and exact=false.
// (we don't need to check for exact=true as those cases are covered in δCheck)
δCheckLastUP := func(id ID, at, lastOk zodb.Tid) {
δCheckLastUP := func(id int64, at, lastOk zodb.Tid) {
t.Helper()
last, exact := δtail.LastRevOf(id, at)
......@@ -160,7 +156,7 @@ func TestΔTail(t *testing.T) {
// access to whole underlying array from a slice.
}
func tailvEqual(a, b []δRevEntry) bool {
func tailvEqual(a, b []δRevEntryI64) bool {
// for empty one can be nil and another !nil [] = reflect.DeepEqual
// does not think those are equal.
return (len(a) == 0 && len(b) == 0) ||
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7