Commit 3ce22f28 authored by Kirill Smelkov's avatar Kirill Smelkov

zodbanalyze: Teach it to work with any ZODB storage

Analyze uses regular ZODB storage API: .iterator() & friends. This way
it should be possible apply it not only to FileStorage, but to other
type of storages as well - for example to NEO and ZEO.

Use zodbtools.util.storageFromURL to open a storage by knowing its URL.
Preserve support to directly apply zodbanalyze to FileStorage deltas.

/reviewed-on nexedi/zodbtools!7
parent 9dbe70f3
...@@ -8,7 +8,7 @@ scripts anymore. So we are here: ...@@ -8,7 +8,7 @@ scripts anymore. So we are here:
__ https://github.com/zopefoundation/ZODB/pull/128#issuecomment-260970932 __ https://github.com/zopefoundation/ZODB/pull/128#issuecomment-260970932
- `zodb analyze` - analyze FileStorage or repozo deltafs usage. - `zodb analyze` - analyze ZODB database or repozo deltafs usage.
- `zodb cmp` - compare content of two ZODB databases bit-to-bit. - `zodb cmp` - compare content of two ZODB databases bit-to-bit.
- `zodb commit` - commit new transaction into a ZODB database. - `zodb commit` - commit new transaction into a ZODB database.
- `zodb dump` - dump content of a ZODB database. - `zodb dump` - dump content of a ZODB database.
......
...@@ -9,9 +9,10 @@ import getopt ...@@ -9,9 +9,10 @@ import getopt
import anydbm as dbm import anydbm as dbm
import tempfile import tempfile
import shutil import shutil
from ZODB.FileStorage import FileIterator, FileStorage, packed_version from ZODB.FileStorage import FileIterator, packed_version
from ZODB.FileStorage.format import FileStorageFormatter from ZODB.FileStorage.format import FileStorageFormatter
from ZODB.utils import get_pickle_metadata from ZODB.utils import get_pickle_metadata
from zodbtools.util import storageFromURL
from golang import func, defer from golang import func, defer
class DeltaFileStorage( class DeltaFileStorage(
...@@ -166,7 +167,7 @@ def analyze(path, use_dbm, delta_fs): ...@@ -166,7 +167,7 @@ def analyze(path, use_dbm, delta_fs):
if delta_fs: if delta_fs:
fs = DeltaFileStorage(path, read_only=1) fs = DeltaFileStorage(path, read_only=1)
else: else:
fs = FileStorage(path, read_only=1) fs = storageFromURL(path, read_only=1)
defer(fs.close) defer(fs.close)
fsi = fs.iterator() fsi = fs.iterator()
report = Report(use_dbm, delta_fs) report = Report(use_dbm, delta_fs)
...@@ -229,20 +230,22 @@ def analyze_rec(report, record): ...@@ -229,20 +230,22 @@ def analyze_rec(report, record):
except Exception, err: except Exception, err:
print err print err
__doc__ = """%(program)s: Analyzer for FileStorage data or repozo deltafs __doc__ = """%(program)s: Analyzer for ZODB data or repozo deltafs
usage: %(program)s [options] /path/to/Data.fs (or /path/to/file.deltafs) usage: %(program)s [options] <storage>
<storage> is an URL (see 'zodb help zurl') or /path/to/file.deltafs(*)
Options: Options:
-h, --help this help screen -h, --help this help screen
-c, --csv output CSV -c, --csv output CSV
-d, --dbm use DBM as temporary storage to limit memory usage -d, --dbm use DBM as temporary storage to limit memory usage
(no meaning for deltafs case) (no meaning for deltafs case)
Note: (*) Note:
Input deltafs file should be uncompressed. Input deltafs file should be uncompressed.
""" """
summary = "analyze FileStorage or repozo deltafs usage" summary = "analyze ZODB database or repozo deltafs usage"
def usage(stream, msg=None): def usage(stream, msg=None):
if msg: if msg:
...@@ -269,15 +272,16 @@ def main(argv): ...@@ -269,15 +272,16 @@ def main(argv):
if opt in ('-h', '--help'): if opt in ('-h', '--help'):
usage(sys.stdout) usage(sys.stdout)
sys.exit() sys.exit()
header = open(path, 'rb').read(4) # try to see whether it is zurl or a path to file.deltafs
if header == packed_version: delta_fs = False
delta_fs = False if os.path.exists(path):
else: header = open(path, 'rb').read(4)
delta_fs = True if header != packed_version:
_orig_read_data_header = FileStorageFormatter._read_data_header delta_fs = True
def _read_data_header(self, pos, oid=None): _orig_read_data_header = FileStorageFormatter._read_data_header
h = _orig_read_data_header(self, pos, oid=oid) def _read_data_header(self, pos, oid=None):
h.tloc = self._tpos h = _orig_read_data_header(self, pos, oid=oid)
return h h.tloc = self._tpos
FileStorageFormatter._read_data_header = _read_data_header return h
FileStorageFormatter._read_data_header = _read_data_header
report(analyze(path, use_dbm, delta_fs), csv) report(analyze(path, use_dbm, delta_fs), csv)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment