.

wcfs/gen-δtail

@@ -25,13 +25,17 @@ KIND=$1
 ID=$2
 out=$3
 
-input=$(dirname $0)/δtail.go.in
+zodb=lab.nexedi.com/kirr/neo/go/zodb
+zdir=`go list -f '{{.Dir}}' $zodb`
+zrev=`git -C $zdir describe --always`
 
 echo "// Code generated by gen-δtail $KIND $ID; DO NOT EDIT." >$out
+echo "// (from $zodb @ $zrev)" >>$out
 echo >>$out
 
-sed	\
-	-e "s/ID/$ID/g"		\
-	-e "s/ΔTail/ΔTail${KIND}/g"	\
+$zdir/δtail.go.cat-generic | sed		\
+	-e "s/PACKAGE/main/g"			\
+	-e "s/ID/$ID/g"				\
+	-e "s/ΔTail/ΔTail${KIND}/g"		\
 	-e "s/δRevEntry/δRevEntry${KIND}/g"	\
-	$input >>$out
+	>>$out

wcfs/zset_i64.go

 // Code generated by gen-set I64 int64; DO NOT EDIT.
 
-// Copyright (C) 2015-2018  Nexedi SA and Contributors.
+// Copyright (C) 2015-2019  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute

wcfs/zδtail_i64.go

 // Code generated by gen-δtail I64 int64; DO NOT EDIT.
+// (from lab.nexedi.com/kirr/neo/go/zodb @ v1.9-2078-gae3c4829)
 
-// Copyright (C) 2018  Nexedi SA and Contributors.
-//                     Kirill Smelkov <kirr@nexedi.com>
+// Copyright (C) 2018-2019  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
 // it under the terms of the GNU General Public License version 3, or (at your
@@ -21,18 +22,25 @@
 
 package main
 
+import "lab.nexedi.com/kirr/neo/go/zodb"
+
 import (
 	"fmt"
-
-	"lab.nexedi.com/kirr/neo/go/zodb"
 )
 
+// XXX do we really need ΔTailI64 to be exported from zodb?
+// (other users are low level caches + maybe ZEO/NEO -> zplumbing? but then import cycle)
+
 // ΔTailI64 represents tail of revisional changes.
 //
 // It semantically consists of
 //
 //	[](rev↑, []id)
 //
+// and index
+//
+//	{} id -> max(rev: rev changed id)
+//
 // where
 //
 //	rev - is ZODB revision, and
@@ -40,12 +48,12 @@ import (
 //
 // It provides operations to
 //
+//	- XXX Head
 //	- append information to the tail about next revision,
 //	- forget information in the tail past specified revision, and
 //	- query the tail about what is last revision that changed an id.
 //
-// It is generally not safe to use ΔTailI64 from multiple goroutines simultaneously.
-// It is safe to perform multiple simultaneous read-kind operations.
+// ΔTailI64 is safe to access for multiple-readers / single writer.
 //
 // (*) examples of id:
 //
@@ -56,7 +64,7 @@ type ΔTailI64 struct {
 
 	lastRevOf map[int64]zodb.Tid // index for LastRevOf queries
 	// TODO also add either tailv idx <-> rev index, or lastRevOf -> tailv idx
-	// (if linear back-scan of δRevEntryI64 starts eat cpu).
+	// (if linear back-scan of δRevEntryI64 starts to eat cpu).
 }
 
 // δRevEntryI64 represents information of what have been changed in one revision.
@@ -70,6 +78,11 @@ func NewΔTailI64() *ΔTailI64 {
 	return &ΔTailI64{lastRevOf: make(map[int64]zodb.Tid)}
 }
 
+// XXX + .Head() -> max(rev)	XXX or 0 if len(tailv) == 0?
+func (δtail *ΔTailI64) Head() zodb.Tid {
+	panic("TODO")
+}
+
 // XXX add way to extend coverage without appending changed data? (i.e. if a
 // txn did not change file at all) -> but then it is simply .Append(rev, nil)?
 

wcfs/δtail.go.in

-// Copyright (C) 2018-2019  Nexedi SA and Contributors.
-//                          Kirill Smelkov <kirr@nexedi.com>
-//
-// This program is free software: you can Use, Study, Modify and Redistribute
-// it under the terms of the GNU General Public License version 3, or (at your
-// option) any later version, as published by the Free Software Foundation.
-//
-// You can also Link and Combine this program with other software covered by
-// the terms of any of the Free Software licenses or any of the Open Source
-// Initiative approved licenses and Convey the resulting work. Corresponding
-// source of such a combination shall include the source code for all other
-// software used.
-//
-// This program is distributed WITHOUT ANY WARRANTY; without even the implied
-// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-//
-// See COPYING file for full licensing terms.
-// See https://www.nexedi.com/licensing for rationale and options.
-
-package main
-
-import (
-	"fmt"
-
-	"lab.nexedi.com/kirr/neo/go/zodb"
-)
-
-// ΔTail represents tail of revisional changes.
-//
-// It semantically consists of
-//
-//	[](rev↑, []id)
-//
-// and index
-//
-//	{} id -> max(rev: rev changed id)
-//
-// where
-//
-//	rev - is ZODB revision, and
-//	id  - is an identifier of what has been changed(*)
-//
-// It provides operations to
-//
-//	- append information to the tail about next revision,
-//	- forget information in the tail past specified revision, and
-//	- query the tail about what is last revision that changed an id.
-//
-// ΔTail is safe to access for multiple-readers / single writer.
-//
-// (*) examples of id:
-//
-//	oid  - ZODB object identifier, when ΔTail represents changes to ZODB objects,
-//	#blk - file block number, when ΔTail represents changes to a file.
-type ΔTail struct {
-	tailv []δRevEntry
-
-	lastRevOf map[ID]zodb.Tid // index for LastRevOf queries
-	// TODO also add either tailv idx <-> rev index, or lastRevOf -> tailv idx
-	// (if linear back-scan of δRevEntry starts to eat cpu).
-}
-
-// δRevEntry represents information of what have been changed in one revision.
-type δRevEntry struct {
-	rev     zodb.Tid
-	changev []ID
-}
-
-// NewΔTail creates new ΔTail object.
-func NewΔTail() *ΔTail {
-	return &ΔTail{lastRevOf: make(map[ID]zodb.Tid)}
-}
-
-// XXX + .Head() -> max(rev)	XXX or 0 if len(tailv) == 0?
-
-// XXX add way to extend coverage without appending changed data? (i.e. if a
-// txn did not change file at all) -> but then it is simply .Append(rev, nil)?
-
-// Append appends to δtail information about what have been changed in next revision.
-//
-// rev must be ↑.
-func (δtail *ΔTail) Append(rev zodb.Tid, changev []ID) {
-	// check rev↑
-	// XXX better also check even when δtail is ø (after forget)
-	if l := len(δtail.tailv); l > 0 {
-		if revPrev := δtail.tailv[l-1].rev; revPrev >= rev {
-			panic(fmt.Sprintf("δtail.Append: rev not ↑: %s -> %s", revPrev, rev))
-		}
-	}
-
-	δtail.tailv = append(δtail.tailv, δRevEntry{rev, changev})
-	for _, id := range changev {
-		δtail.lastRevOf[id] = rev
-	}
-}
-
-// ForgetBefore discards all δtail entries with rev < revCut.
-func (δtail *ΔTail) ForgetBefore(revCut zodb.Tid) {
-	icut := 0
-	for i, δ := range δtail.tailv {
-		rev := δ.rev
-		if rev >= revCut {
-			break
-		}
-		icut = i+1
-
-		// if forgotten revision was last for id, we have to update lastRevOf index
-		for _, id := range δ.changev {
-			if δtail.lastRevOf[id] == rev {
-				delete(δtail.lastRevOf, id)
-			}
-		}
-	}
-
-	// tailv = tailv[icut:]	  but without
-	// 1) growing underlying storage array indefinitely
-	// 2) keeping underlying storage after forget
-	l := len(δtail.tailv)-icut
-	tailv := make([]δRevEntry, l)
-	copy(tailv, δtail.tailv[icut:])
-	δtail.tailv = tailv
-}
-
-// LastRevOf tries to return what was the last revision that changed id as of at database state.
-//
-// Depending on current information in δtail it returns either exact result, or
-// an upper-bound estimate for the last id revision, assuming id was changed ≤ at:
-//
-// 1) if δtail does not cover at, at is returned:
-//
-//	# at ∉ [min(rev ∈ δtail), max(rev ∈ δtail)]
-//	LastRevOf(id, at) = at
-//
-// 2) if δtail has an entry corresponding to id change, it gives exactly the
-//    last revision that changed id:
-//
-//	# at ∈ [min(rev ∈ δtail), max(rev ∈ δtail)]
-//	# ∃ rev ∈ δtail: rev changed id && rev ≤ at
-//	LastRevOf(id, at) = max(rev: rev changed id && rev ≤ at)
-//
-// 3) if δtail does not contain appropriate record with id - it returns δtail's
-//    lower bound as the estimate for the upper bound of the last id revision:
-//
-//	# at ∈ [min(rev ∈ δtail), max(rev ∈ δtail)]
-//	# ∄ rev ∈ δtail: rev changed id && rev ≤ at
-//	LastRevOf(id, at) = min(rev ∈ δtail)
-//
-// On return exact indicates whether returned revision is exactly the last
-// revision of id, or only an upper-bound estimate of it.
-func (δtail *ΔTail) LastRevOf(id ID, at zodb.Tid) (_ zodb.Tid, exact bool) {
-	// check if we have no coverage at all
-	l := len(δtail.tailv)
-	if l == 0 {
-		return at, false
-	}
-	revMin := δtail.tailv[0].rev
-	revMax := δtail.tailv[l-1].rev
-	if !(revMin <= at && at <= revMax) {
-		return at, false
-	}
-
-	// we have the coverage
-	rev, ok := δtail.lastRevOf[id]
-	if !ok {
-		return δtail.tailv[0].rev, false
-	}
-
-	if rev <= at {
-		return rev, true
-	}
-
-	// what's in index is after at - scan tailv back to find appropriate entry
-	// XXX linear scan
-	for i := l - 1; i >= 0; i-- {
-		δ := δtail.tailv[i]
-		if δ.rev > at {
-			continue
-		}
-
-		for _, δid := range δ.changev {
-			if id == δid {
-				return δ.rev, true
-			}
-		}
-	}
-
-	// nothing found
-	return δtail.tailv[0].rev, false
-}

wcfs/δtail_test.go

-// Copyright (C) 2018-2019  Nexedi SA and Contributors.
-//                          Kirill Smelkov <kirr@nexedi.com>
-//
-// This program is free software: you can Use, Study, Modify and Redistribute
-// it under the terms of the GNU General Public License version 3, or (at your
-// option) any later version, as published by the Free Software Foundation.
-//
-// You can also Link and Combine this program with other software covered by
-// the terms of any of the Free Software licenses or any of the Open Source
-// Initiative approved licenses and Convey the resulting work. Corresponding
-// source of such a combination shall include the source code for all other
-// software used.
-//
-// This program is distributed WITHOUT ANY WARRANTY; without even the implied
-// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-//
-// See COPYING file for full licensing terms.
-// See https://www.nexedi.com/licensing for rationale and options.
-
-package main
-
-import (
-	"fmt"
-	"reflect"
-	"testing"
-
-	"lab.nexedi.com/kirr/neo/go/zodb"
-)
-
-func TestΔTail(t *testing.T) {
-	δtail := NewΔTailI64()
-
-	// R is syntactic sugar to create 1 δRevEntry
-	R := func(rev zodb.Tid, changev ...int64) δRevEntryI64 {
-		return δRevEntryI64{rev, changev}
-	}
-
-	// δAppend is syntactic sugar for δtail.Append
-	δAppend := func(δ δRevEntryI64) {
-		δtail.Append(δ.rev, δ.changev)
-	}
-
-	// δCheck verifies that δtail state corresponds to provided tailv
-	δCheck := func(tailv ...δRevEntryI64) {
-		t.Helper()
-
-		for i := 1; i < len(tailv); i++ {
-			if !(tailv[i-1].rev < tailv[i].rev) {
-				panic("test tailv: rev not ↑")
-			}
-		}
-
-		if !tailvEqual(δtail.tailv, tailv) {
-			t.Fatalf("tailv:\nhave: %v\nwant: %v", δtail.tailv, tailv)
-		}
-
-		// verify lastRevOf query / index
-		lastRevOf := make(map[int64]zodb.Tid)
-		for _, δ := range tailv {
-			for _, id := range δ.changev {
-				idRev, exact := δtail.LastRevOf(id, δ.rev)
-				if !(idRev == δ.rev && exact) {
-					t.Fatalf("LastRevOf(%v, at=%s) -> %s, %v  ; want %s, %v", id, δ.rev, idRev, exact, δ.rev, true)
-				}
-
-				lastRevOf[id] = δ.rev
-			}
-		}
-
-		if !reflect.DeepEqual(δtail.lastRevOf, lastRevOf) {
-			t.Fatalf("lastRevOf:\nhave: %v\nwant: %v", δtail.lastRevOf, lastRevOf)
-		}
-
-	}
-
-	// δCheckLastUP verifies that δtail.LastRevOf(id, at) gives lastOk and exact=false.
-	// (we don't need to check for exact=true as those cases are covered in δCheck)
-	δCheckLastUP := func(id int64, at, lastOk zodb.Tid) {
-		t.Helper()
-
-		last, exact := δtail.LastRevOf(id, at)
-		if !(last == lastOk && exact == false) {
-			t.Fatalf("LastRevOf(%v, at=%s) -> %s, %v  ; want %s, %v", id, at, last, exact, lastOk, false)
-		}
-	}
-
-
-	δCheck()
-	δCheckLastUP(4, 12, 12)	// δtail = ø
-
-	δAppend(R(10, 3,5))
-	δCheck(R(10, 3,5))
-
-	δCheckLastUP(3,  9,  9)	// at < δtail
-	δCheckLastUP(3, 12, 12)	// at > δtail
-	δCheckLastUP(4, 10, 10)	// id ∉ δtail
-
-	δAppend(R(11, 7))
-	δCheck(R(10, 3,5), R(11, 7))
-
-	δAppend(R(12, 7))
-	δCheck(R(10, 3,5), R(11, 7), R(12, 7))
-
-	δAppend(R(14, 3,8))
-	δCheck(R(10, 3,5), R(11, 7), R(12, 7), R(14, 3,8))
-
-	δCheckLastUP(8, 12, 10) // id ∈ δtail, but has no entry with rev ≤ at
-
-	δtail.ForgetBefore(10)
-	δCheck(R(10, 3,5), R(11, 7), R(12, 7), R(14, 3,8))
-
-	δtail.ForgetBefore(11)
-	δCheck(R(11, 7), R(12, 7), R(14, 3,8))
-
-	δtail.ForgetBefore(13)
-	δCheck(R(14, 3,8))
-
-	δtail.ForgetBefore(15)
-	δCheck()
-
-	// Append panics on non-↑ rev
-	δAppend(R(15, 1))
-	func() {
-		defer func() {
-			r := recover()
-			if r == nil {
-				t.Fatal("append non-↑: not panicked")
-			}
-			rev := zodb.Tid(15)
-			want := fmt.Sprintf("δtail.Append: rev not ↑: %s -> %s", rev, rev)
-			if r != want {
-				t.Fatalf("append non-↑:\nhave: %q\nwant: %q", r, want)
-			}
-		}()
-
-		δAppend(R(15, 1))
-	}()
-
-
-	// .tailv underlying storage is not kept after forget
-	δtail.ForgetBefore(16)
-
-	const N = 1E3
-	for rev, i := zodb.Tid(16), 0; i < N; i, rev = i+1, rev+1 {
-		δAppend(R(rev, 1))
-	}
-
-	capN := cap(δtail.tailv)
-	δtail.ForgetBefore(N)
-	if c := cap(δtail.tailv); !(c < capN/10) {
-		t.Fatalf("forget: tailv storage did not shrink: cap%v: %d -> cap: %d", N, capN, c)
-	}
-
-	// .tailv underlying storage does not grow indefinitely
-	// XXX cannot test as the growth here goes to left and we cannot get
-	// access to whole underlying array from a slice.
-}
-
-func tailvEqual(a, b []δRevEntryI64) bool {
-	// for empty one can be nil and another !nil [] = reflect.DeepEqual
-	// does not think those are equal.
-	return (len(a) == 0 && len(b) == 0) ||
-		reflect.DeepEqual(a, b)
-}