Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Levin Zimmermann
neoppod
Commits
ffde05a5
Commit
ffde05a5
authored
7 years ago
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
a69c7dee
No related merge requests found
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
56 additions
and
6 deletions
+56
-6
go/zodb/storage/fs1/fs1tools/index.go
go/zodb/storage/fs1/fs1tools/index.go
+31
-3
go/zodb/storage/fs1/index.go
go/zodb/storage/fs1/index.go
+25
-3
No files found.
go/zodb/storage/fs1/fs1tools/index.go
View file @
ffde05a5
...
...
@@ -82,14 +82,14 @@ func reindexMain(argv []string) {
// ----------------------------------------
// VerifyIndexFor verifies that on-disk index for FileStorage file @ path is correct
func
VerifyIndexFor
(
ctx
context
.
Context
,
path
string
,
ntxn
int
)
(
err
error
)
{
func
VerifyIndexFor
(
ctx
context
.
Context
,
path
string
,
ntxn
int
,
progress
func
(
*
fs1
.
IndexVerifyProgress
)
)
(
err
error
)
{
// XXX lock path.lock ?
index
,
err
:=
fs1
.
LoadIndexFile
(
path
+
".index"
)
if
err
!=
nil
{
return
err
// XXX err ctx
}
_
,
err
=
index
.
VerifyForFile
(
context
.
Background
(),
path
,
ntxn
)
_
,
err
=
index
.
VerifyForFile
(
context
.
Background
(),
path
,
ntxn
,
progress
)
return
err
}
...
...
@@ -106,15 +106,18 @@ Verify FileStorage index
-checkonly <n> only check consistency by verifying against <n>
last transactions.
-quiet do not show intermediate progress.
-h --help this help text.
`
)
}
func
verifyIdxMain
(
argv
[]
string
)
{
ntxn
:=
-
1
quiet
:=
false
flags
:=
flag
.
FlagSet
{
Usage
:
func
()
{
verifyIdxUsage
(
os
.
Stderr
)
}}
flags
.
Init
(
""
,
flag
.
ExitOnError
)
flags
.
IntVar
(
&
ntxn
,
"checkonly"
,
ntxn
,
"check consistency only wrt last <n> transactions"
)
flags
.
BoolVar
(
&
quiet
,
"quiet"
,
quiet
,
"do not show intermediate progress"
)
flags
.
Parse
(
argv
[
1
:
])
argv
=
flags
.
Args
()
...
...
@@ -124,8 +127,33 @@ func verifyIdxMain(argv []string) {
}
storPath
:=
argv
[
0
]
err
:=
VerifyIndexFor
(
context
.
Background
(),
storPath
,
ntxn
)
// progress display
progress
:=
func
(
p
*
fs1
.
IndexVerifyProgress
)
{
if
p
.
TxnTotal
==
-
1
{
bytesChecked
:=
p
.
Index
.
TopPos
-
p
.
Iter
.
Txnh
.
Pos
bytesAll
:=
p
.
Index
.
TopPos
fmt
.
Printf
(
"Checked data bytes: %.1f%% (%d/%d); #txn: %d, #oid: %d
\n
"
,
100
*
float64
(
bytesChecked
)
/
float64
(
bytesAll
),
// XXX /0 ?
bytesChecked
,
bytesAll
,
p
.
TxnChecked
,
len
(
p
.
OidChecked
))
}
else
{
fmt
.
Printf
(
"Checked data transactions: %.1f%% (%d/%d); #oid: %d
\n
"
,
100
*
float64
(
p
.
TxnChecked
)
/
float64
(
p
.
TxnTotal
),
// XXX /0 ?
p
.
TxnChecked
,
p
.
TxnTotal
,
len
(
p
.
OidChecked
))
}
}
if
quiet
{
progress
=
nil
}
err
:=
VerifyIndexFor
(
context
.
Background
(),
storPath
,
ntxn
,
progress
)
if
err
!=
nil
{
zt
.
Fatal
(
err
)
}
if
!
quiet
{
fmt
.
Println
(
"OK"
)
}
}
This diff is collapsed.
Click to expand it.
go/zodb/storage/fs1/index.go
View file @
ffde05a5
...
...
@@ -553,6 +553,15 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru
return
&
IndexCorruptError
{
DataFileName
:
xio
.
Name
(
r
),
Detail
:
fmt
.
Sprintf
(
format
,
argv
...
)}
}
// IndexVerifyProgress is data sent by Index.Verify to progress
type
IndexVerifyProgress
struct
{
TxnTotal
int
// total # of transactions to verify; if = -1 -- whole data
TxnChecked
int
Index
*
Index
// index verification runs for
Iter
*
Iter
// iterator thtough data
OidChecked
map
[
zodb
.
Oid
]
struct
{}
// oid checked so far
}
// Verify checks index correctness against FileStorage data in r.
//
// For ntxn transactions starting from index.TopPos backwards, it verifies
...
...
@@ -568,7 +577,7 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru
// Returned error is either:
// - of type *IndexCorruptError, when data in index was found not to match original data, or
// - any other error type representing e.g. IO error when reading original data or something else.
func
(
index
*
Index
)
Verify
(
ctx
context
.
Context
,
r
io
.
ReaderAt
,
ntxn
int
)
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
func
(
index
*
Index
)
Verify
(
ctx
context
.
Context
,
r
io
.
ReaderAt
,
ntxn
int
,
progress
func
(
*
IndexVerifyProgress
)
)
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
defer
func
()
{
if
_
,
ok
:=
err
.
(
*
IndexCorruptError
);
ok
{
return
// leave it as is
...
...
@@ -581,6 +590,14 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChe
wholeData
:=
false
it
:=
Iterate
(
r
,
index
.
TopPos
,
IterBackward
)
pd
:=
&
IndexVerifyProgress
{
TxnTotal
:
ntxn
,
Index
:
index
,
Iter
:
it
,
OidChecked
:
oidChecked
,
}
for
i
:=
0
;
ntxn
==
-
1
||
i
<
ntxn
;
i
++
{
// check ctx cancel once per transaction
select
{
...
...
@@ -625,6 +642,11 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChe
it
.
Datah
.
Oid
,
it
.
Datah
.
Pos
,
dataPos
)
}
}
if
progress
!=
nil
{
pd
.
TxnChecked
=
i
progress
(
pd
)
}
}
// all oids from data were checked to be in index
...
...
@@ -652,7 +674,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChe
// VerifyForFile checks index correctness against FileStorage data in file @ path
//
// See Verify for semantic description.
func
(
index
*
Index
)
VerifyForFile
(
ctx
context
.
Context
,
path
string
,
ntxn
int
)
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
func
(
index
*
Index
)
VerifyForFile
(
ctx
context
.
Context
,
path
string
,
ntxn
int
,
progress
func
(
*
IndexVerifyProgress
)
)
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
f
,
err
:=
os
.
Open
(
path
)
if
err
!=
nil
{
return
nil
,
err
...
...
@@ -676,5 +698,5 @@ func (index *Index) VerifyForFile(ctx context.Context, path string, ntxn int) (o
// use IO optimized for sequential access when verifying index
fSeq
:=
xbufio
.
NewSeqReaderAt
(
f
)
return
index
.
Verify
(
ctx
,
fSeq
,
ntxn
)
return
index
.
Verify
(
ctx
,
fSeq
,
ntxn
,
progress
)
}
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment