Commit ffde05a5 authored by Kirill Smelkov's avatar Kirill Smelkov

.

parent a69c7dee
No related merge requests found
......@@ -82,14 +82,14 @@ func reindexMain(argv []string) {
// ----------------------------------------
// VerifyIndexFor verifies that on-disk index for FileStorage file @ path is correct
func VerifyIndexFor(ctx context.Context, path string, ntxn int) (err error) {
func VerifyIndexFor(ctx context.Context, path string, ntxn int, progress func(*fs1.IndexVerifyProgress)) (err error) {
// XXX lock path.lock ?
index, err := fs1.LoadIndexFile(path + ".index")
if err != nil {
return err // XXX err ctx
}
_, err = index.VerifyForFile(context.Background(), path, ntxn)
_, err = index.VerifyForFile(context.Background(), path, ntxn, progress)
return err
}
......@@ -106,15 +106,18 @@ Verify FileStorage index
-checkonly <n> only check consistency by verifying against <n>
last transactions.
-quiet do not show intermediate progress.
-h --help this help text.
`)
}
func verifyIdxMain(argv []string) {
ntxn := -1
quiet := false
flags := flag.FlagSet{Usage: func() { verifyIdxUsage(os.Stderr) }}
flags.Init("", flag.ExitOnError)
flags.IntVar(&ntxn, "checkonly", ntxn, "check consistency only wrt last <n> transactions")
flags.BoolVar(&quiet, "quiet", quiet, "do not show intermediate progress")
flags.Parse(argv[1:])
argv = flags.Args()
......@@ -124,8 +127,33 @@ func verifyIdxMain(argv []string) {
}
storPath := argv[0]
err := VerifyIndexFor(context.Background(), storPath, ntxn)
// progress display
progress := func(p *fs1.IndexVerifyProgress) {
if p.TxnTotal == -1 {
bytesChecked := p.Index.TopPos - p.Iter.Txnh.Pos
bytesAll := p.Index.TopPos
fmt.Printf("Checked data bytes: %.1f%% (%d/%d); #txn: %d, #oid: %d\n",
100 * float64(bytesChecked) / float64(bytesAll), // XXX /0 ?
bytesChecked, bytesAll,
p.TxnChecked, len(p.OidChecked))
} else {
fmt.Printf("Checked data transactions: %.1f%% (%d/%d); #oid: %d\n",
100 * float64(p.TxnChecked) / float64(p.TxnTotal), // XXX /0 ?
p.TxnChecked, p.TxnTotal, len(p.OidChecked))
}
}
if quiet {
progress = nil
}
err := VerifyIndexFor(context.Background(), storPath, ntxn, progress)
if err != nil {
zt.Fatal(err)
}
if !quiet {
fmt.Println("OK")
}
}
......@@ -553,6 +553,15 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru
return &IndexCorruptError{DataFileName: xio.Name(r), Detail: fmt.Sprintf(format, argv...)}
}
// IndexVerifyProgress is data sent by Index.Verify to progress
type IndexVerifyProgress struct {
TxnTotal int // total # of transactions to verify; if = -1 -- whole data
TxnChecked int
Index *Index // index verification runs for
Iter *Iter // iterator thtough data
OidChecked map[zodb.Oid]struct{} // oid checked so far
}
// Verify checks index correctness against FileStorage data in r.
//
// For ntxn transactions starting from index.TopPos backwards, it verifies
......@@ -568,7 +577,7 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru
// Returned error is either:
// - of type *IndexCorruptError, when data in index was found not to match original data, or
// - any other error type representing e.g. IO error when reading original data or something else.
func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChecked map[zodb.Oid]struct{}, err error) {
func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progress func(*IndexVerifyProgress)) (oidChecked map[zodb.Oid]struct{}, err error) {
defer func() {
if _, ok := err.(*IndexCorruptError); ok {
return // leave it as is
......@@ -581,6 +590,14 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChe
wholeData := false
it := Iterate(r, index.TopPos, IterBackward)
pd := &IndexVerifyProgress{
TxnTotal: ntxn,
Index: index,
Iter: it,
OidChecked: oidChecked,
}
for i := 0; ntxn == -1 || i < ntxn; i++ {
// check ctx cancel once per transaction
select {
......@@ -625,6 +642,11 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChe
it.Datah.Oid, it.Datah.Pos, dataPos)
}
}
if progress != nil {
pd.TxnChecked = i
progress(pd)
}
}
// all oids from data were checked to be in index
......@@ -652,7 +674,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChe
// VerifyForFile checks index correctness against FileStorage data in file @ path
//
// See Verify for semantic description.
func (index *Index) VerifyForFile(ctx context.Context, path string, ntxn int) (oidChecked map[zodb.Oid]struct{}, err error) {
func (index *Index) VerifyForFile(ctx context.Context, path string, ntxn int, progress func (*IndexVerifyProgress)) (oidChecked map[zodb.Oid]struct{}, err error) {
f, err := os.Open(path)
if err != nil {
return nil, err
......@@ -676,5 +698,5 @@ func (index *Index) VerifyForFile(ctx context.Context, path string, ntxn int) (o
// use IO optimized for sequential access when verifying index
fSeq := xbufio.NewSeqReaderAt(f)
return index.Verify(ctx, fSeq, ntxn)
return index.Verify(ctx, fSeq, ntxn, progress)
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment