.

go/xcommon/xbufio/seqbuf_ioat.go

@@ -60,6 +60,10 @@ func NewSeqReaderAtSize(r io.ReaderAt, size int) *SeqReaderAt {
 	return sb
 }
 
+// XXX not yet sure it is good idea
+func (sb *SeqReaderAt) IOUnwrap() interface{} {
+	return sb.r
+}
 
 // // XXX temp
 // func init() {

go/xcommon/xio/xio.go

@@ -48,6 +48,11 @@ func (l *LimitedWriter) Write(p []byte) (n int, err error) {
 
 func LimitWriter(w io.Writer, n int64) io.Writer { return &LimitedWriter{w, n} }
 
+// XXX not sure it is a good ide
+// XXX another option is IOName() - but that requires leaf packages to import xio
+type Wrapper interface {
+	IOUnwrap() interface{}
+}
 
 // Name returns a "filename" associated with io.Reader, io.Writer, net.Conn, ...
 func Name(f interface {}) string {
@@ -70,6 +75,8 @@ func Name(f interface {}) string {
 	// XXX SectionReader MultiReader TeeReader
 	// XXX bufio.Reader bufio.Writer bufio.Scanner
 
+	case Wrapper:		return Name(f.IOUnwrap())
+
 	// if name cannot be determined - let's provide full info
 	default:
 		return fmt.Sprintf("%#v", f)

go/zodb/storage/fs1/fs1tools/index.go

@@ -31,9 +31,9 @@ import (
 )
 
 // Reindex rebuilds index for FileStorage file @ path
-func Reindex(path string) error {
+func Reindex(ctx context.Context, path string) error {
 	// XXX lock path.lock ?
-	index, err := fs1.BuildIndexForFile(context.Background(), path)
+	index, err := fs1.BuildIndexForFile(ctx, path)
 	if err != nil {
 		return err
 	}
@@ -84,21 +84,47 @@ func reindexMain(argv []string) {
 // TODO verify-index -quick (only small sanity check)
 
 // VerifyIndexFor verifies that on-disk index for FileStorage file @ path is correct
-func VerifyIndexFor(path string) error {
-	panic("TODO")
-/*
-	// XXX open read-only
-	fs, err := fs1.Open(context.Background(), path)	// XXX , 0)
+func VerifyIndexFor(ctx context.Context, path string) (err error) {
+	// XXX lock path.lock ?
+	inverify := false
+	defer func() {
+		if inverify {
+			return // index.Verify provides all context in error
+		}
+		xerr.Contextf(&err, "%s: verify index", path)
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		err2 := f.Close()	// XXX vs inverify
+		err = xerr.First(err, err2)
+	}()
+
+	index, err := LoadIndexFile(path + ".index")
 	if err != nil {
-		return nil	// XXX err ctx
+		return err
+	}
+
+	fi, err := f.Stat()
+	if err != nil {
+		return err
+	}
+
+	topPos := fi.Size()	// XXX there might be last TxnInprogress transaction
+	if index.TopPos != topPos {
+		return fmt.Errorf("topPos mismatch: data=%v  index=%v", topPos, index.TopPos)
 	}
-	defer fs.Close()	// XXX err
 
-	err = fs.VerifyIndex(nil)
+	// XXX - better somehow not here?
+	fSeq := xbufio.NewSeqReaderAt(f)
+
+	inverify = true
+	err = index.Verify(ctx, fSeq)
 	return err
-	//fs.Index()
-	//fs.ComputeIndex
-*/
 }
 
 const verifyIdxSummary = "verify database index"

go/zodb/storage/fs1/index.go

@@ -555,7 +555,7 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru
 
 // VerifyTail checks index correctness against several newest transactions of FileStorage data in r.
 //
-// For ntxn transactions starting from index.TopPos backwards, it verifies
+// For (XXX max) ntxn transactions starting from index.TopPos backwards, it verifies
 // whether oid there have correct entries in the index.
 //
 // ntxn=-1 means data range to verify is till start of the file.
@@ -585,7 +585,9 @@ func (index *Index) VerifyTail(ctx context.Context, r io.ReaderAt, ntxn int) (oi
 
 		err := it.NextTxn(LoadNoStrings)
 		if err != nil {
-			err = okEOF(err)
+			if err == io.EOF {
+				break
+			}
 			return nil, err		// XXX err ctx
 		}
 
@@ -618,10 +620,11 @@ func (index *Index) VerifyTail(ctx context.Context, r io.ReaderAt, ntxn int) (oi
 		}
 	}
 
+	// TODO err = EOF -> merge from Verify
+
 	return oidChecked, nil
 }
 
-
 // Verify checks index correctness against FileStorage data in r.
 //
 // it verifies whether index is exactly the same as if it was build anew for
@@ -634,6 +637,8 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt) error {
 		return err
 	}
 
+	// XXX merge this into VerifyTail
+
 	// all oids from data were checked to be in index
 	// now verify that there is no extra oids in index
 	if len(oidChecked) == index.Len() {
@@ -656,3 +661,48 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt) error {
 
 	return nil
 }
+
+// XXX text
+func (index *Index) VerifyTailForFile(ctx context.Context, path string, ntxn int) (oidChecked map[zodb.Oid]struct{}, err error) {
+	err = index.verifyForFile(path, func(r io.ReaderAt) error {
+		oidChecked, err = index.VerifyTail(ctx, r, ntxn)
+		return err
+	})
+	return
+}
+
+// VerifyForFile verifies index correctness against FileStorage file @ path
+// XXX text
+func (index *Index) VerifyForFile(ctx context.Context, path string) error {
+	return index.verifyForFile(path, func(r io.ReaderAt) error {
+		return index.Verify(ctx, r)
+	})
+}
+
+// common driver for Verify*ForFile
+func (index *Index) verifyForFile(path string, check func(r io.ReaderAt) error) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		err2 := f.Close()
+		err = xerr.First(err, err2)
+	}()
+
+	fi, err := f.Stat()
+	if err != nil {
+		return err
+	}
+
+	topPos := fi.Size()	// XXX there might be last TxnInprogress transaction
+	if index.TopPos != topPos {
+		return indexCorrupt(f, "topPos mismatch: data=%v  index=%v", topPos, index.TopPos)
+	}
+
+	// use IO optimized for sequential access when verifying index
+	fSeq := xbufio.NewSeqReaderAt(f)
+
+	return check(fSeq)
+}

go/zodb/storage/fs1/index_test.go

@@ -220,15 +220,27 @@ func TestIndexSaveToPy(t *testing.T) {
 	}
 }
 
-func TestIndexBuild(t *testing.T) {
+func TestIndexBuildVerify(t *testing.T) {
 	index, err := BuildIndexForFile(context.Background(), "testdata/1.fs")
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("index build: %v", err)
 	}
 
 	if !index.Equal(_1fs_index) {
 		t.Fatal("computed index differ from expected")
 	}
+
+	err = index.VerifyForFile(context.Background(), "testdata/1.fs")
+	if err != nil {
+		t.Fatalf("index verify: %v", err)
+	}
+
+	pos0, _ := index.Get(0)
+	index.Set(0, pos0 + 1)
+	err = index.VerifyForFile(context.Background(), "testdata/1.fs")
+	if err == nil {
+		t.Fatalf("index verify: expected error after tweak")
+	}
 }