.

wcfs/wcfs.go

@@ -307,7 +307,7 @@ package main
 //
 //	or another upper bound if #blk ∉ δFtail:
 //
-//	  rev(blk) ≤ min(rev) is δFtail		; #blk ∉ δFtail
+//	  rev(blk) ≤ min(rev) in δFtail		; #blk ∉ δFtail
 //
 //
 //	below rev'(blk) is min(of the estimates found):
@@ -456,7 +456,6 @@ import (
 	"lab.nexedi.com/kirr/go123/xcontext"
 	"lab.nexedi.com/kirr/go123/xerr"
 
-//	"lab.nexedi.com/kirr/neo/go/transaction"
 	"lab.nexedi.com/kirr/neo/go/zodb"
 	"lab.nexedi.com/kirr/neo/go/zodb/btree"
 	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
@@ -476,7 +475,7 @@ type Root struct {
 	// ZODB DB handle for zstor.
 	// keeps cache of connections for both head/ and @<rev>/ accesses.
 	//
-	// only one connection is used for head/ and only one for each @<rev>.	XXX
+	// only one connection is used for head/ and only one for each @<rev>.
 	zdb *zodb.DB
 
 	// ZODB connection for head/

wcfs/δtail.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// XXX -> internal/δtail/ ?
+
+package main
+// δtail maintenance	XXX
+
+import (
+	"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+type ID int64	// XXX -> template
+
+// ΔTail represents tail of revisional changes.
+//
+// It semantically consists of
+//
+//	[] of (rev↑, []id)
+//
+// where
+//
+//	rev - is ZODB revision, and
+//	id  - is an identifier of what has been changed(*)
+//
+// It provides operations to
+//
+//	- append information to the tail about next revision,
+//	- forget information in the tail past specified revision, and
+//	- query the tail about what is last revision that changed an id.
+//
+// It is generally not safe to use ΔTail from multiple goroutines simultaneously.
+// It is safe to perform multiple simultaneous queries.
+//
+// (*) examples of id:
+//
+//	oid  - ZODB object identifier, when ΔTail represents changes to ZODB objects,
+//	#blk - file block number, when ΔTail represents changes to a file.
+type ΔTail struct {
+	tailv     []δRevEntry
+	lastRevOf map[ID]zodb.Tid
+	// TODO also add either tailv idx <-> rev index, or lastRevOf -> tailv idx
+	// (if linear back-scan of δRevEntry starts eat cpu).
+}
+
+// δRevEntry represents information of what have been changed in one revision.
+// XXX naming
+type δRevEntry struct {
+	rev     zodb.Tid
+	δv      []ID
+}
+
+// NewΔTail creates new ΔTail object.
+func NewΔTail() *ΔTail {
+	return &ΔTail{lastRevOf: make(map[ID]zodb.Tid)}
+}
+
+
+// Append appends to δtail information about what have been changed in next revision.
+//
+// rev must be ↑.
+func (δtail *ΔTail) Append(rev zodb.Tid, δv []ID) {
+	// XXX check rev↑
+	δtail.tailv = append(δtail.tailv, δRevEntry{rev, δv})
+	for _, id := range δv {
+		δtail.lastRevOf[id] = rev
+	}
+}
+
+// ForgetBefore discards all δtail entries with rev < revCut.
+func (δtail *ΔTail) ForgetBefore(revCut zodb.Tid) {
+	icut := 0
+	for i, δ := range δtail.tailv {
+		rev := δ.rev
+		if rev >= revCut {
+			break
+		}
+		icut = i+1
+
+		// if forgottent revision was last for id, we have to update lastRevOf index
+		for _, id := range δ.δv {
+			if δtail.lastRevOf[id] == rev {
+				delete(δtail.lastRevOf, id)
+			}
+		}
+	}
+
+	// tailv = tailv[icut:]	  but without growing underlying storage array indefinetely
+	copy(δtail.tailv, δtail.tailv[icut:])
+	δtail.tailv = δtail.tailv[:len(δtail.tailv)-icut]
+}
+
+// LastRevOf returns what was the last revision that changed id as of at database state.
+//
+// in other words
+//
+//	LastRevOf(id, at) = max(rev: rev changed id && rev ≤ at)
+//
+// XXX if δtail does not contain records with id -> what is returned?
+func (δtail *ΔTail) LastRevOf(id ID, at zodb.Tid) zodb.Tid {
+	rev, ok := δtail.lastRevOf[id]
+	if !ok {
+		panic("TODO")	// XXX
+	}
+
+	if rev <= at {
+		return rev
+	}
+
+	// what's in index is after at - scan tailv back to find appropriate entry
+	// XXX linear scan
+	for i := len(δtail.tailv) - 1; i >= 0; i-- {
+		δ := δtail.tailv[i]
+		if δ.rev > at {
+			continue
+		}
+
+		for _, δid := range δ.δv {
+			if id == δid {
+				return δ.rev
+			}
+		}
+	}
+
+	// nothing found
+	panic("TODO")	// XXX
+}

wcfs/δtail_test.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package main
+
+import (
+	"reflect"
+	"testing"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	//"github.com/stretchr/testify/require"
+)
+
+func TestΔTail(t *testing.T) {
+	//assert := require.New(t)
+
+	δtail := NewΔTail()
+
+	// R is syntic sugar to create 1 δRevEntry
+	R := func(rev zodb.Tid, δv ...ID) δRevEntry {
+		return δRevEntry{rev, δv}
+	}
+
+	// δAppend is syntatic sugar for δtail.Append
+	δAppend := func(δ δRevEntry) {
+		δtail.Append(δ.rev, δ.δv)
+	}
+
+	// δCheck verifies that δtail state corresponds to provided tailv
+	δCheck := func(tailv ...δRevEntry) {
+		t.Helper()
+
+		for i := 1; i < len(tailv); i++ {
+			if !(tailv[i-1].rev < tailv[i].rev) {
+				panic("test tailv: rev not ↑")
+			}
+		}
+
+		if !tailvEqual(δtail.tailv, tailv) {
+			t.Fatalf("tailv:\nhave: %v\nwant: %v", δtail.tailv, tailv)
+		}
+
+		// verify lastRevOf query / index
+		lastRevOf := make(map[ID]zodb.Tid)
+		for _, δ := range tailv {
+			for _, id := range δ.δv {
+				idRev := δtail.LastRevOf(id, δ.rev)
+				if idRev != δ.rev {
+					t.Fatalf("LastRevOf(%v, at=%s) -> %s  ; want %s", id, δ.rev, idRev, δ.rev)
+				}
+
+				lastRevOf[id] = δ.rev
+			}
+		}
+
+		if !reflect.DeepEqual(δtail.lastRevOf, lastRevOf) {
+			t.Fatalf("lastRevOf:\nhave: %v\nwant: %v", δtail.lastRevOf, lastRevOf)
+		}
+
+	}
+
+
+	δCheck()
+
+	δAppend(R(10, 3,5))
+	δCheck(R(10, 3,5))
+
+	δAppend(R(11, 7))
+	δCheck(R(10, 3,5), R(11, 7))
+
+	δAppend(R(12, 7))
+	δCheck(R(10, 3,5), R(11, 7), R(12, 7))
+
+	δAppend(R(14, 3,7))
+	δCheck(R(10, 3,5), R(11, 7), R(12, 7), R(14, 3,7))
+
+	δtail.ForgetBefore(10)
+	δCheck(R(10, 3,5), R(11, 7), R(12, 7), R(14, 3,7))
+
+	δtail.ForgetBefore(11)
+	δCheck(R(11, 7), R(12, 7), R(14, 3,7))
+
+	δtail.ForgetBefore(13)
+	δCheck(R(14, 3,7))
+
+	δtail.ForgetBefore(15)
+	δCheck()
+
+
+	// XXX .Append(rev not ↑ - panic)
+	// XXX edge cases
+	// XXX .tailv underlying storage does not grow indefinitely
+}
+
+func tailvEqual(a, b []δRevEntry) bool {
+	// for empty one can be nil and another !nil [] = reflect.DeepEqual
+	// does not think those are equal.
+	return (len(a) == 0 && len(b) == 0) ||
+		reflect.DeepEqual(a, b)
+}