Merge branch 'y/loadAt.8' into y/demo-delete

* y/loadAt.8: (35 commits)
  Make lint happy
  Let the year float.
  Configuring for pure-python
  Specify a PyPy2 version.
  Lint the code.
  Configuring for pure-python
  Fix TypeError for fsoids (#351)
  Fix deprecation warnings occurring on Python 3.10.
  fix more PY3 incompatibilities in `fsstats`
  fix Python 3 incompatibility for `fsstats`
  add `fsdump/fsstats` test
  fsdump/fsstats improvements
  Undeprecate loadBefore
  fixup! changes: Add draft entry for loadAt/loadBeforeEx/DemoStorage fix
  changes: Add draft entry for loadAt/loadBeforeEx/DemoStorage fix
  fixup! Handle NotImplementedError raised by loadBefore/loadBeforeEx as "interface not provided"
  *: Don't emit warnings on loadBefore
  Handle NotImplementedError raised by loadBefore/loadBeforeEx as "interface not provided"
  loadAt -> loadBeforeEx
  - add coverage combine step
  ...

.coveragerc

-[run]
-source = src/ZODB/
-parallel = true
-omit =
-	 src/ZODB/tests/*
-	 src/ZODB/scripts/tests/*
-
-[report]
-exclude_lines =
-    pragma: nocover
-    pragma: no cover
-    if __name__ == ['"]__main__['"]:
-    assert False
-    self.fail

.editorconfig

+# Generated from:
+# https://github.com/zopefoundation/meta/tree/master/config/pure-python
+#
+# EditorConfig Configuration file, for more details see:
+# http://EditorConfig.org
+# EditorConfig is a convention description, that could be interpreted
+# by multiple editors to enforce common coding conventions for specific
+# file types
+
+# top-most EditorConfig file:
+# Will ignore other EditorConfig files in Home directory or upper tree level.
+root = true
+
+
+[*]  # For All Files
+# Unix-style newlines with a newline ending every file
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+# Set default charset
+charset = utf-8
+# Indent style default
+indent_style = space
+# Max Line Length - a hard line wrap, should be disabled
+max_line_length = off
+
+[*.{py,cfg,ini}]
+# 4 space indentation
+indent_size = 4
+
+[*.{yml,zpt,pt,dtml,zcml}]
+# 2 space indentation
+indent_size = 2
+
+[{Makefile,.gitmodules}]
+# Tab indentation (no size specified, but view as 4 spaces)
+indent_style = tab
+indent_size = unset
+tab_width = unset

.github/workflows/tests.yml

+# Generated from:
+# https://github.com/zopefoundation/meta/tree/master/config/pure-python
+name: tests
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 12 * * 0'  # run once a week on Sunday
+  # Allow to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  build:
+    strategy:
+      # We want to see all failures:
+      fail-fast: false
+      matrix:
+        os:
+        - ubuntu
+        - windows
+        config:
+        # [Python version, tox env]
+        - ["3.9",   "lint"]
+        - ["2.7",   "py27"]
+        - ["3.5",   "py35"]
+        - ["3.6",   "py36"]
+        - ["3.7",   "py37"]
+        - ["3.8",   "py38"]
+        - ["3.9",   "py39"]
+        - ["3.10",  "py310"]
+        - ["pypy2", "pypy"]
+        - ["pypy3", "pypy3"]
+        - ["3.9",   "docs"]
+        - ["3.9",   "coverage"]
+        - ["3.8",   "py38-pure"]
+        exclude:
+          - { os: windows, config: ["3.9",   "lint"] }
+          - { os: windows, config: ["3.9",   "docs"] }
+          - { os: windows, config: ["3.9",   "coverage"] }
+          - { os: windows, config: ["pypy2", "pypy"] }
+
+    runs-on: ${{ matrix.os }}-latest
+    name: ${{ matrix.os }}-${{ matrix.config[1] }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.config[0] }}
+    - name: Pip cache
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ matrix.config[0] }}-${{ hashFiles('setup.*', 'tox.ini') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-${{ matrix.config[0] }}-
+          ${{ runner.os }}-pip-
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox
+    - name: Test
+      run: tox -e ${{ matrix.config[1] }}
+    - name: Coverage
+      if: matrix.config[1] == 'coverage'
+      run: |
+        pip install coveralls coverage-python-version
+        coveralls --service=github
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

-bin
-eggs
-develop-eggs
-parts
-.installed.cfg
-build
-doc/_build
-__pycache__
+# Generated from:
+# https://github.com/zopefoundation/meta/tree/master/config/pure-python
+*.dll
+*.egg-info/
+*.profraw
 *.pyc
+*.pyo
 *.so
-.tox
 .coverage
 .coverage.*
-nosetests.xml
-coverage.xml
-*.egg-info
-*.egg
-dist
-testing.log
 .eggs/
-.dir-locals.el
-htmlcov
-tmp
-*~
+.installed.cfg
+.mr.developer.cfg
+.tox/
+.vscode/
+__pycache__/
+bin/
+build/
+coverage.xml
+develop-eggs/
+develop/
+dist/
+docs/_build
+eggs/
+etc/
+lib/
+lib64
+log/
+parts/
+pyvenv.cfg
+var/

.meta.toml

+# Generated from:
+# https://github.com/zopefoundation/meta/tree/master/config/pure-python
+[meta]
+template = "pure-python"
+commit-id = "de499940b679dcda1c60b089f30134146da31e9a"
+
+[python]
+with-windows = true
+with-pypy = true
+with-future-python = false
+with-legacy-python = true
+with-docs = true
+with-sphinx-doctests = false
+
+[tox]
+use-flake8 = true
+testenv-setenv = [
+    "ZOPE_INTERFACE_STRICT_IRO=1",
+    ]
+additional-envlist = [
+    "py38-pure",
+    ]
+testenv-additional = [
+    "",
+    "[testenv:py38-pure]",
+    "basepython = python3.8",
+    "setenv =",
+    "    PURE_PYTHON = 1",
+    ]
+
+[coverage]
+fail-under = 80
+
+[flake8]
+additional-config = [
+    "# F401 imported but unused",
+    "per-file-ignores =",
+    "    src/ZODB/FileStorage/__init__.py: F401",
+    "    src/ZODB/__init__.py: F401",
+    ]
+
+[manifest]
+additional-rules = [
+    "include COPYING",
+    "recursive-include docs *.ico",
+    "recursive-include docs *.png",
+    "recursive-include docs *.svg",
+    "recursive-include src *.fs",
+    "recursive-include src *.rst",
+    "recursive-include src *.test",
+    "recursive-include src *.txt",
+    "recursive-include src *.xml",
+    ]
+
+[check-manifest]
+additional-ignores = [
+    "docs/_build/doctest/*/*/*/*",
+    "docs/_build/doctest/*/*/*",
+    "docs/_build/doctest/*/*",
+    "docs/_build/html/*/*/*/*",
+    "docs/_build/html/*/*/*",
+    "docs/_build/html/*/*",
+    ]
+
+[github-actions]
+additional-config = [
+    "- [\"3.8\",   \"py38-pure\"]",
+    ]
+additional-exclude = [
+    "- { os: windows, config: [\"pypy2\", \"pypy\"] }",
+    ]

.travis.yml

-language: python
-
-env:
-  global:
-    ZOPE_INTERFACE_STRICT_IRO: 1
-
-python:
-  - 2.7
-  - 3.5
-  - 3.6
-  - 3.7
-  - 3.8
-  - pypy
-  - pypy3
-
-jobs:
-  include:
-    # Special Linux builds
-    - name: "Python: 3.8, pure (no C extensions)"
-      python: 3.8
-      env: PURE_PYTHON=1
-
-install:
-    - pip install -U pip
-    # BBB use older setuptools on pypy, setuptools 50.0.0 is not compatible with pypy 7.1.1-beta0
-    - if [[ $TRAVIS_PYTHON_VERSION == pypy3* ]]; then pip install -U setuptools==49.6.0 zc.buildout; fi
-    - if [[ $TRAVIS_PYTHON_VERSION != pypy3* ]]; then pip install -U setuptools zc.buildout; fi
-    - buildout $BUILOUT_OPTIONS
-script:
-    - if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then bin/coverage run bin/coverage-test -v; fi
-    - if [[ $TRAVIS_PYTHON_VERSION == pypy* ]]; then bin/test -v; fi
-    - if [[ $TRAVIS_PYTHON_VERSION != pypy3* ]]; then pip install --upgrade --requirement doc/requirements.txt; fi
-    - if [[ $TRAVIS_PYTHON_VERSION != pypy3* ]]; then make -C doc html; fi
-    - if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then pip install coveralls; fi # install early enough to get into the cache
-after_success:
-  - if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then bin/coverage combine; fi
-  - if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then coveralls; fi
-notifications:
-    email: false
-cache:
-  directories:
-    - $HOME/.cache/pip
-    - eggs
-before_cache:
-    - rm -f $HOME/.cache/pip/log/debug.log

CHANGES.rst

@@ -2,11 +2,38 @@
  Change History
 ================
 
-5.6.1 (unreleased)
+5.7.0 (unreleased)
 ==================
 
+- Introduce a new ``loadBeforeEx`` interface that complements ``loadBefore``:
+  ``loadBeforeEx`` is simpler, provides better information for object delete
+  records and can be more efficiently implemented by many storages.
+  ``loadBeforeEx`` is used (and required) to fix a ``DemoStorage`` data corruption
+  in the presence of object delete records.
+  See `issue 318 <https://github.com/zopefoundation/ZODB/issues/318>`_
+  and `PR 323 <https://github.com/zopefoundation/ZODB/pull/323>`_
+  for details.
+
+- Fix ``TypeError: can't concat str to bytes`` when running fsoids.py script with Python 3.
+  See `issue 350 <https://github.com/zopefoundation/ZODB/issues/350>`_.
+
+- Readd transaction size information to ``fsdump`` output;
+  adapt `fsstats` to ``fsdump``'s exchanged order for ``size`` and ``class``
+  information in data records;
+  (fixes `#354 <https://github.com/zopefoundation/ZODB/issues/354>_).
+  Make ``fsdump`` callable via Python's ``-m`` command line option.
+
 - Fix UnboundLocalError when running fsoids.py script.
-  See `issue 268 <https://github.com/zopefoundation/ZODB/issues/285>`_.
+  See `issue 285 <https://github.com/zopefoundation/ZODB/issues/285>`_.
+
+- Rework ``fsrefs`` script to work significantly faster by optimizing how it does
+  IO. See `PR 340 <https://github.com/zopefoundation/ZODB/pull/340>`_.
+
+- Require Python 3 to build the documentation.
+
+- Fix deprecation warnings occurring on Python 3.10.
+
+- Add support for Python 3.9 and 3.10.
 
 
 5.6.0 (2020-06-11)

MANIFEST.in

+# Generated from:
+# https://github.com/zopefoundation/meta/tree/master/config/pure-python
 include *.rst
 include *.txt
-include *.py
-include *.ini
-exclude .coveragerc
-exclude .travis.yml
-exclude appveyor.yml
-exclude buildout.cfg
-include COPYING
-
-recursive-include doc *
-recursive-include src *
+include buildout.cfg
+include tox.ini
 
-global-exclude *.dll
-global-exclude *.pyc
-global-exclude *.pyo
-global-exclude *.so
-global-exclude *~
+recursive-include docs *.py
+recursive-include docs *.rst
+recursive-include docs *.txt
+recursive-include docs Makefile
 
+recursive-include src *.py
+include COPYING
+recursive-include docs *.ico
+recursive-include docs *.png
+recursive-include docs *.svg
+recursive-include src *.fs
+recursive-include src *.rst
+recursive-include src *.test
+recursive-include src *.txt
+recursive-include src *.xml

README.rst

@@ -41,7 +41,7 @@ ZODB is an ACID Transactional database.
 
 To learn more, visit: https://zodb-docs.readthedocs.io
 
-The github repository is: at https://github.com/zopefoundation/zodb
+The github repository is at https://github.com/zopefoundation/zodb
 
 If you're interested in contributing to ZODB itself, see the
 `developer notes

appveyor.yml

-environment:
-  matrix:
-    - python: 27
-    - python: 27-x64
-    - python: 35
-    - python: 35-x64
-    - python: 36
-    - python: 36-x64
-    - python: 37
-    - python: 37-x64
-    - python: 38
-    - python: 38-x64
-
-install:
-  - "SET PATH=C:\\Python%PYTHON%;c:\\Python%PYTHON%\\scripts;%PATH%"
-  - echo "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 > "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64\vcvars64.bat"
-  - python -m pip install -U pip setuptools wheel
-  - pip install -U -e .[test]
-
-build_script:
-  - python -m pip install -U wheel
-  - python -W ignore setup.py -q bdist_wheel
-
-test_script:
-  - zope-testrunner --test-path=src -vvv --auto-color

bootstrap.py

-##############################################################################
-#
-# Copyright (c) 2006 Zope Foundation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-"""Bootstrap a buildout-based project
-
-Simply run this script in a directory containing a buildout.cfg.
-The script accepts buildout command-line options, so you can
-use the -c option to specify an alternate configuration file.
-"""
-
-import os
-import shutil
-import sys
-import tempfile
-
-from optparse import OptionParser
-
-tmpeggs = tempfile.mkdtemp()
-
-usage = '''\
-[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
-
-Bootstraps a buildout-based project.
-
-Simply run this script in a directory containing a buildout.cfg, using the
-Python that you want bin/buildout to use.
-
-Note that by using --find-links to point to local resources, you can keep
-this script from going over the network.
-'''
-
-parser = OptionParser(usage=usage)
-parser.add_option("-v", "--version", help="use a specific zc.buildout version")
-
-parser.add_option("-t", "--accept-buildout-test-releases",
-                  dest='accept_buildout_test_releases',
-                  action="store_true", default=False,
-                  help=("Normally, if you do not specify a --version, the "
-                        "bootstrap script and buildout gets the newest "
-                        "*final* versions of zc.buildout and its recipes and "
-                        "extensions for you.  If you use this flag, "
-                        "bootstrap and buildout will get the newest releases "
-                        "even if they are alphas or betas."))
-parser.add_option("-c", "--config-file",
-                  help=("Specify the path to the buildout configuration "
-                        "file to be used."))
-parser.add_option("-f", "--find-links",
-                  help=("Specify a URL to search for buildout releases"))
-parser.add_option("--allow-site-packages",
-                  action="store_true", default=False,
-                  help=("Let bootstrap.py use existing site packages"))
-parser.add_option("--setuptools-version",
-                  help="use a specific setuptools version")
-
-
-options, args = parser.parse_args()
-
-######################################################################
-# load/install setuptools
-
-try:
-    if options.allow_site_packages:
-        import setuptools
-        import pkg_resources
-    from urllib.request import urlopen
-except ImportError:
-    from urllib2 import urlopen
-
-ez = {}
-exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
-
-if not options.allow_site_packages:
-    # ez_setup imports site, which adds site packages
-    # this will remove them from the path to ensure that incompatible versions
-    # of setuptools are not in the path
-    import site
-    # inside a virtualenv, there is no 'getsitepackages'.
-    # We can't remove these reliably
-    if hasattr(site, 'getsitepackages'):
-        for sitepackage_path in site.getsitepackages():
-            sys.path[:] = [x for x in sys.path if sitepackage_path not in x]
-
-setup_args = dict(to_dir=tmpeggs, download_delay=0)
-
-if options.setuptools_version is not None:
-    setup_args['version'] = options.setuptools_version
-
-ez['use_setuptools'](**setup_args)
-import setuptools
-import pkg_resources
-
-# This does not (always?) update the default working set.  We will
-# do it.
-for path in sys.path:
-    if path not in pkg_resources.working_set.entries:
-        pkg_resources.working_set.add_entry(path)
-
-######################################################################
-# Install buildout
-
-ws = pkg_resources.working_set
-
-cmd = [sys.executable, '-c',
-       'from setuptools.command.easy_install import main; main()',
-       '-mZqNxd', tmpeggs]
-
-find_links = os.environ.get(
-    'bootstrap-testing-find-links',
-    options.find_links or
-    ('http://downloads.buildout.org/'
-     if options.accept_buildout_test_releases else None)
-    )
-if find_links:
-    cmd.extend(['-f', find_links])
-
-setuptools_path = ws.find(
-    pkg_resources.Requirement.parse('setuptools')).location
-
-requirement = 'zc.buildout'
-version = options.version
-if version is None and not options.accept_buildout_test_releases:
-    # Figure out the most recent final version of zc.buildout.
-    import setuptools.package_index
-    _final_parts = '*final-', '*final'
-
-    def _final_version(parsed_version):
-        try:
-            return not parsed_version.is_prerelease
-        except AttributeError:
-            # Older setuptools
-            for part in parsed_version:
-                if (part[:1] == '*') and (part not in _final_parts):
-                    return False
-            return True
-
-    index = setuptools.package_index.PackageIndex(
-        search_path=[setuptools_path])
-    if find_links:
-        index.add_find_links((find_links,))
-    req = pkg_resources.Requirement.parse(requirement)
-    if index.obtain(req) is not None:
-        best = []
-        bestv = None
-        for dist in index[req.project_name]:
-            distv = dist.parsed_version
-            if _final_version(distv):
-                if bestv is None or distv > bestv:
-                    best = [dist]
-                    bestv = distv
-                elif distv == bestv:
-                    best.append(dist)
-        if best:
-            best.sort()
-            version = best[-1].version
-if version:
-    requirement = '=='.join((requirement, version))
-cmd.append(requirement)
-
-import subprocess
-if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
-    raise Exception(
-        "Failed to execute command:\n%s" % repr(cmd)[1:-1])
-
-######################################################################
-# Import and run buildout
-
-ws.add_entry(tmpeggs)
-ws.require(requirement)
-import zc.buildout.buildout
-
-if not [a for a in args if '=' not in a]:
-    args.append('bootstrap')
-
-# if -c was provided, we push it back into args for buildout' main function
-if options.config_file is not None:
-    args[0:0] = ['-c', options.config_file]
-
-zc.buildout.buildout.main(args)
-shutil.rmtree(tmpeggs)

buildout.cfg

@@ -9,6 +9,8 @@ parts =
 [versions]
 # Avoid breakage in 4.4.5:
 zope.testrunner = >= 4.4.6
+# sphinx_rtd_theme depends on docutils<0.17
+docutils = < 0.17
 
 [versions:python2]
 Sphinx = < 2

doc/conf.py → docs/conf.py

@@ -17,6 +17,7 @@
 import sys
 import os
 import pkg_resources
+import datetime
 
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
@@ -56,7 +57,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'ZODB'
-copyright = '2009-2020, Zope Foundation'
+copyright = '2009-{}, Zope Foundation'.format(datetime.date.today().year)
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

doc/index.rst → docs/index.rst

@@ -44,6 +44,19 @@ Learning more
 
 * `The ZODB Book (in progress) <http://zodb.readthedocs.org/en/latest/>`_
 
+
+What is the expansion of "ZODB"?
+================================
+
+The expansion of "ZODB" is the Z Object Database.  But, of course, we
+usually just use "ZODB".
+
+In the past, it was the Zope Object Database, because it was
+developed as part of the Zope project.  But ZODB doesn't depend on
+Zope in any way and is used in many projects that have nothing to do
+with Zope.
+
+
 Downloads
 =========
 

doc/requirements.txt → docs/requirements.txt

 Sphinx
-# pygments 2.6 stops the support for python2
-pygments<2.6
+# Silence dependabot claiming a security issue in older versions:
+pygments >= 2.7.4 ; python_version >= '3'
+# pygments 2.6 stopped supporting py2
+pygments <  2.6   ; python_version <  '3'
 docutils
 ZODB
 sphinxcontrib_zopeext

release.py

-#! /usr/bin/env python
-"""Update version numbers and release dates for the next release.
-
-usage: release.py version date
-
-version should be a string like "3.2.0c1"
-date should be a string like "23-Sep-2003"
-
-The following files are updated:
-    - setup.py
-    - NEWS.txt
-    - doc/guide/zodb.tex
-    - src/ZEO/__init__.py
-    - src/ZEO/version.txt
-    - src/ZODB/__init__.py
-"""
-
-import fileinput
-import os
-import re
-
-# In file filename, replace the first occurrence of regexp pat with
-# string repl.
-def replace(filename, pat, repl):
-    from sys import stderr as e # fileinput hijacks sys.stdout
-    foundone = False
-    for line in fileinput.input([filename], inplace=True, backup="~"):
-        if foundone:
-            print line,
-        else:
-            match = re.search(pat, line)
-            if match is not None:
-                foundone = True
-
-                new = re.sub(pat, repl, line)
-                print new,
-
-                print >> e, "In %s, replaced:" % filename
-                print >> e, "   ", repr(line)
-                print >> e, "   ", repr(new)
-
-            else:
-                print line,
-
-    if not foundone:
-        print >> e, "*" * 60, "Oops!"
-        print >> e, "    Failed to find %r in %r" % (pat, filename)
-
-# Nothing in our codebase cares about ZEO/version.txt.  Jeremy said
-# someone asked for it so that a shell script could read up the ZEO
-# version easily.
-# Before ZODB 3.4, the ZEO version was one smaller than the ZODB version;
-# e.g., ZEO 2.2.7 shipped with ZODB 3.2.7.  Now ZEO and ZODB share their
-# version number.
-def write_zeoversion(path, version):
-    with open(path, "w") as f:
-        print >> f, version
-
-def main(args):
-    version, date = args
-
-    replace("setup.py",
-            r'^VERSION = "\S+"$',
-            'VERSION = "%s"' % version)
-    replace("src/ZODB/__init__.py",
-            r'__version__ = "\S+"',
-            '__version__ = "%s"' % version)
-    replace("src/ZEO/__init__.py",
-            r'version = "\S+"',
-            'version = "%s"' % version)
-    write_zeoversion("src/ZEO/version.txt", version)
-    replace("NEWS.txt",
-            r"^Release date: .*",
-            "Release date: %s" % date)
-    replace("doc/guide/zodb.tex",
-            r"release{\S+}",
-            "release{%s}" % version)
-if __name__ == "__main__":
-    import sys
-    main(sys.argv[1:])

setup.cfg

+# Generated from:
+# https://github.com/zopefoundation/meta/tree/master/config/pure-python
 [bdist_wheel]
 universal = 1
+
+[flake8]
+doctests = 1
+# F401 imported but unused
+per-file-ignores =
+    src/ZODB/FileStorage/__init__.py: F401
+    src/ZODB/__init__.py: F401
+
+[check-manifest]
+ignore =
+    .editorconfig
+    .meta.toml
+    docs/_build/html/_sources/*
+    docs/_build/doctest/*/*/*/*
+    docs/_build/doctest/*/*/*
+    docs/_build/doctest/*/*
+    docs/_build/html/*/*/*/*
+    docs/_build/html/*/*/*
+    docs/_build/html/*/*

setup.py

@@ -13,7 +13,7 @@
 ##############################################################################
 from setuptools import setup, find_packages
 
-version = '5.6.1.dev0'
+version = '5.7.0.dev0'
 
 classifiers = """\
 Intended Audience :: Developers
@@ -26,6 +26,8 @@ Programming Language :: Python :: 3.5
 Programming Language :: Python :: 3.6
 Programming Language :: Python :: 3.7
 Programming Language :: Python :: 3.8
+Programming Language :: Python :: 3.9
+Programming Language :: Python :: 3.10
 Programming Language :: Python :: Implementation :: CPython
 Programming Language :: Python :: Implementation :: PyPy
 Topic :: Database
@@ -35,10 +37,12 @@ Operating System :: Unix
 Framework :: ZODB
 """
 
+
 def read(path):
     with open(path) as f:
         return f.read()
 
+
 long_description = read("README.rst") + "\n\n" + read("CHANGES.rst")
 
 tests_require = [
@@ -67,6 +71,13 @@ setup(
     tests_require=tests_require,
     extras_require={
         'test': tests_require,
+        'docs': [
+            'Sphinx',
+            'ZODB',
+            'j1m.sphinxautozconfig',
+            'sphinx_rtd_theme',
+            'sphinxcontrib_zopeext',
+        ]
     },
     install_requires=[
         'persistent >= 4.4.0',

src/ZODB/BaseStorage.py

@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import time
 import logging
-import sys
 from struct import pack as _structpack, unpack as _structunpack
 
 import zope.interface
@@ -35,6 +34,7 @@ from ._compat import py2_hasattr
 
 log = logging.getLogger("ZODB.BaseStorage")
 
+
 class BaseStorage(UndoLogCompatible):
     """Base class that supports storage implementations.
 
@@ -59,7 +59,8 @@ class BaseStorage(UndoLogCompatible):
 
     If it stores multiple revisions, it should implement
     loadSerial()
-    loadAt()
+    loadBefore()
+    loadBeforeEx()
 
     Each storage will have two locks that are accessed via lock
     acquire and release methods bound to the instance.  (Yuck.)
@@ -74,12 +75,12 @@ class BaseStorage(UndoLogCompatible):
     perhaps other things.  It is always held when load() is called, so
     presumably the load() implementation should also acquire the lock.
     """
-    _transaction=None # Transaction that is being committed
-    _tstatus=' '      # Transaction status, used for copying data
+    _transaction = None  # Transaction that is being committed
+    _tstatus = ' '      # Transaction status, used for copying data
     _is_read_only = False
 
     def __init__(self, name, base=None):
-        self.__name__= name
+        self.__name__ = name
         log.debug("create storage %s", self.__name__)
 
         # Allocate locks:
@@ -93,7 +94,7 @@ class BaseStorage(UndoLogCompatible):
         self._commit_lock_release = self._commit_lock.release
 
         t = time.time()
-        t = self._ts = TimeStamp(*(time.gmtime(t)[:5] + (t%60,)))
+        t = self._ts = TimeStamp(*(time.gmtime(t)[:5] + (t % 60,)))
         self._tid = t.raw()
 
         # ._oid is the highest oid in use (0 is always in use -- it's
@@ -267,8 +268,15 @@ class BaseStorage(UndoLogCompatible):
         raise POSException.Unsupported(
             "Retrieval of historical revisions is not supported")
 
-    # do not provide loadAt/loadBefore here in BaseStorage - if child forgets
-    # to override it - storage will always return "no data" instead of failing.
+    def loadBefore(self, oid, tid):
+        """Return most recent revision of oid before tid committed."""
+        raise NotImplementedError
+
+    def loadBeforeEx(self, oid, tid):
+        """Return most recent revision of oid before tid committed.
+           (see IStorageLoadBeforeEx).
+        """
+        raise NotImplementedError
 
     def copyTransactionsFrom(self, other, verbose=0):
         """Copy transactions from another storage.
@@ -278,6 +286,7 @@ class BaseStorage(UndoLogCompatible):
         """
         copy(other, self, verbose)
 
+
 def copy(source, dest, verbose=0):
     """Copy transactions from a source to a destination storage
 
@@ -286,7 +295,7 @@ def copy(source, dest, verbose=0):
     """
     _ts = None
     ok = 1
-    preindex = {};
+    preindex = {}
     preget = preindex.get
     # restore() is a new storage API method which has an identical
     # signature to store() except that it does not return anything.
@@ -309,7 +318,8 @@ def copy(source, dest, verbose=0):
         else:
             t = TimeStamp(tid)
             if t <= _ts:
-                if ok: print(('Time stamps out of order %s, %s' % (_ts, t)))
+                if ok:
+                    print(('Time stamps out of order %s, %s' % (_ts, t)))
                 ok = 0
                 _ts = t.laterThan(_ts)
                 tid = _ts.raw()
@@ -350,23 +360,24 @@ def checkCurrentSerialInTransaction(self, oid, serial, transaction):
         raise POSException.ReadConflictError(
             oid=oid, serials=(committed_tid, serial))
 
+
 BaseStorage.checkCurrentSerialInTransaction = checkCurrentSerialInTransaction
 
+
 @zope.interface.implementer(ZODB.interfaces.IStorageTransactionInformation)
 class TransactionRecord(TransactionMetaData):
     """Abstract base class for iterator protocol"""
 
-
     def __init__(self, tid, status, user, description, extension):
         self.tid = tid
         self.status = status
         TransactionMetaData.__init__(self, user, description, extension)
 
+
 @zope.interface.implementer(ZODB.interfaces.IStorageRecordInformation)
 class DataRecord(object):
     """Abstract base class for iterator protocol"""
 
-
     version = ''
 
     def __init__(self, oid, tid, data, prev):

src/ZODB/ConflictResolution.py

@@ -29,9 +29,11 @@ from pickle import PicklingError
 
 logger = logging.getLogger('ZODB.ConflictResolution')
 
+
 class BadClassName(Exception):
     pass
 
+
 class BadClass(object):
 
     def __init__(self, *args):
@@ -40,8 +42,11 @@ class BadClass(object):
     def __reduce__(self):
         raise BadClassName(*self.args)
 
+
 _class_cache = {}
 _class_cache_get = _class_cache.get
+
+
 def find_global(*args):
     cls = _class_cache_get(args, 0)
     if cls == 0:
@@ -61,13 +66,13 @@ def find_global(*args):
         # Not importable
         if (isinstance(args, tuple) and len(args) == 2 and
                 isinstance(args[0], six.string_types) and
-            isinstance(args[1], six.string_types)
-            ):
+                isinstance(args[1], six.string_types)):
             return BadClass(*args)
         else:
             raise BadClassName(*args)
     return cls
 
+
 def state(self, oid, serial, prfactory, p=''):
     p = p or self.loadSerial(oid, serial)
     p = self._crs_untransform_record_data(p)
@@ -77,6 +82,7 @@ def state(self, oid, serial, prfactory, p=''):
     unpickler.load()  # skip the class tuple
     return unpickler.load()
 
+
 class IPersistentReference(zope.interface.Interface):
     '''public contract for references to persistent objects from an object
     with conflicts.'''
@@ -114,10 +120,10 @@ class IPersistentReference(zope.interface.Interface):
         have two references to the same object that are spelled with different
         data (for instance, one with a class and one without).'''
 
+
 @zope.interface.implementer(IPersistentReference)
 class PersistentReference(object):
 
-
     weak = False
     oid = database_name = klass = None
 
@@ -211,6 +217,7 @@ class PersistentReference(object):
         elif isinstance(data, list) and data[0] == 'm':
             return data[1][2]
 
+
 class PersistentReferenceFactory(object):
 
     data = None
@@ -218,7 +225,8 @@ class PersistentReferenceFactory(object):
     def persistent_load(self, ref):
         if self.data is None:
             self.data = {}
-        key = tuple(ref) # lists are not hashable; formats are different enough
+        # lists are not hashable; formats are different enough
+        key = tuple(ref)
         # even after eliminating list/tuple distinction
         r = self.data.get(key, None)
         if r is None:
@@ -227,12 +235,16 @@ class PersistentReferenceFactory(object):
 
         return r
 
+
 def persistent_id(object):
     if getattr(object, '__class__', 0) is not PersistentReference:
         return None
     return object.data
 
+
 _unresolvable = {}
+
+
 def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
                          committedData=b''):
     # class_tuple, old, committed, newstate = ('',''), 0, 0, 0
@@ -264,7 +276,6 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
             _unresolvable[klass] = 1
             raise ConflictError
 
-
         oldData = self.loadSerial(oid, oldSerial)
         if not committedData:
             committedData = self.loadSerial(oid, committedSerial)
@@ -284,7 +295,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
         logger.debug(
             "Conflict resolution on %s failed with %s: %s",
             klass, e.__class__.__name__, str(e))
-    except:
+    except:  # noqa: E722 do not use bare 'except'
         # If anything else went wrong, catch it here and avoid passing an
         # arbitrary exception back to the client.  The error here will mask
         # the original ConflictError.  A client can recover from a
@@ -296,6 +307,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
     raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
                         data=newpickle)
 
+
 class ConflictResolvingStorage(object):
     "Mix-in class that provides conflict resolution handling for storages"
 

src/ZODB/Connection.py

@@ -15,7 +15,6 @@
 """
 from __future__ import print_function
 import logging
-import sys
 import tempfile
 import warnings
 import os
@@ -43,7 +42,6 @@ from ZODB import POSException
 from ZODB.POSException import InvalidObjectReference, ConnectionStateError
 from ZODB.POSException import ConflictError, ReadConflictError
 from ZODB.POSException import Unsupported, ReadOnlyHistoryError
-from ZODB.POSException import POSKeyError
 from ZODB.serialize import ObjectWriter, ObjectReader
 from ZODB.utils import p64, u64, z64, oid_repr, positive_id
 from ZODB import utils
@@ -56,7 +54,10 @@ from . import valuedoc
 
 global_reset_counter = 0
 
-noop = lambda : None
+
+def noop():
+    return None
+
 
 def resetCaches():
     """Causes all connection caches to be reset as connections are reopened.
@@ -259,6 +260,7 @@ class Connection(ExportImport, object):
             connection._cache.incrgc()
 
     __onCloseCallbacks = None
+
     def onCloseCallback(self, f):
         """Register a callable, f, to be called by close()."""
         if self.__onCloseCallbacks is None:
@@ -281,18 +283,18 @@ class Connection(ExportImport, object):
             for f in callbacks:
                 try:
                     f()
-                except: # except what?
+                except:  # noqa: E722 do not use bare 'except'
                     f = getattr(f, 'im_self', f)
                     self._log.exception("Close callback failed for %s", f)
 
         self._debug_info = ()
 
         if self.opened and self.transaction_manager is not None:
-            # transaction_manager could be None if one of the __onCloseCallbacks
-            # closed the DB already, .e.g, ZODB.connection() does this.
+            # transaction_manager could be None if one of the
+            # __onCloseCallbacks closed the DB already, .e.g, ZODB.connection()
+            # does this.
             self.transaction_manager.unregisterSynch(self)
 
-
         am = self._db._activity_monitor
         if am is not None:
             am.closedConnection(self)
@@ -322,7 +324,6 @@ class Connection(ExportImport, object):
         # We may have been reused by another thread at this point so
         # we can't manipulate or check the state of `self` any more.
 
-
     def db(self):
         """Returns a handle to the database this connection belongs to."""
         return self._db
@@ -545,8 +546,7 @@ class Connection(ExportImport, object):
                     ((self._savepoint_storage is None)
                      or (oid not in self._savepoint_storage.creating)
                      or self._savepoint_storage.creating[oid]
-                 )
-                ):
+                     )):
 
                 # obj is a new object
 
@@ -593,7 +593,7 @@ class Connection(ExportImport, object):
             # serial number for a newly created object
             try:
                 self._cache[oid] = obj
-            except:
+            except:  # noqa: E722 do not use bare 'except'
                 # Dang, I bet it's wrapped:
                 # TODO:  Deprecate, then remove, this.
                 if hasattr(obj, 'aq_base'):
@@ -664,7 +664,6 @@ class Connection(ExportImport, object):
                 del o._p_jar
                 del o._p_oid
 
-
     def tpc_vote(self, transaction):
         """Verify that a data manager can commit the transaction."""
         try:
@@ -769,7 +768,7 @@ class Connection(ExportImport, object):
                    % (className(obj), oid_repr(oid)))
             try:
                 raise ConnectionStateError(msg)
-            except:
+            except:  # noqa: E722 do not use bare 'except'
                 self._log.exception(msg)
                 raise
 
@@ -790,7 +789,7 @@ class Connection(ExportImport, object):
 
         except ConflictError:
             raise
-        except:
+        except:  # noqa: E722 do not use bare 'except'
             self._log.exception("Couldn't load state for %s %s",
                                 className(obj), oid_repr(oid))
             raise
@@ -847,7 +846,7 @@ class Connection(ExportImport, object):
         # fine everything. some on the lru list, some not
         everything = self._cache.cache_data
         # remove those items that are on the lru list
-        for k,v in items:
+        for k, v in items:
             del everything[k]
         # return a list of [ghosts....not recently used.....recently used]
         return list(everything.items()) + items
@@ -1102,6 +1101,7 @@ class Connection(ExportImport, object):
                     else:
                         yield ob._p_oid
 
+
 @implementer(IDataManagerSavepoint)
 class Savepoint(object):
 
@@ -1117,7 +1117,6 @@ class Savepoint(object):
 class TmpStore(object):
     """A storage-like thing to support savepoints."""
 
-
     def __init__(self, storage):
         self._storage = storage
         for method in (
@@ -1167,14 +1166,14 @@ class TmpStore(object):
         # commit logic
         assert version == ''
         self._file.seek(self.position)
-        l = len(data)
+        lenght = len(data)
         if serial is None:
             serial = z64
-        header = p64(len(oid)) + oid + serial + p64(l)
+        header = p64(len(oid)) + oid + serial + p64(lenght)
         self._file.write(header)
         self._file.write(data)
         self.index[oid] = self.position
-        self.position += l + len(header)
+        self.position += lenght + len(header)
         return serial
 
     def storeBlob(self, oid, serial, data, blobfilename, version,
@@ -1271,6 +1270,7 @@ class RootConvenience(object):
             names = names[:57].rsplit(' ', 1)[0] + ' ...'
         return "<root: %s>" % names
 
+
 large_object_message = """The %s
 object you're saving is large. (%s bytes.)
 
@@ -1291,6 +1291,7 @@ large-record-size option in a configuration file) to specify a larger
 size.
 """
 
+
 class overridable_property(object):
     """
     Same as property() with only a getter, except that setting a

src/ZODB/DB.py

@@ -41,6 +41,7 @@ from ZODB import valuedoc
 
 logger = logging.getLogger('ZODB.DB')
 
+
 class AbstractConnectionPool(object):
     """Manage a pool of connections.
 
@@ -111,7 +112,7 @@ class AbstractConnectionPool(object):
 
 class ConnectionPool(AbstractConnectionPool):
 
-    def __init__(self, size, timeout=1<<31):
+    def __init__(self, size, timeout=1 << 31):
         super(ConnectionPool, self).__init__(size, timeout)
 
         # A stack of connections available to hand out.  This is a subset
@@ -127,9 +128,8 @@ class ConnectionPool(AbstractConnectionPool):
     def _append(self, c):
         available = self.available
         cactive = c._cache.cache_non_ghost_count
-        if (available and
-            (available[-1][1]._cache.cache_non_ghost_count > cactive)
-            ):
+        if (available
+                and (available[-1][1]._cache.cache_non_ghost_count > cactive)):
             i = len(available) - 1
             while (i and
                    (available[i-1][1]._cache.cache_non_ghost_count > cactive)
@@ -244,7 +244,7 @@ class KeyedConnectionPool(AbstractConnectionPool):
 
     # see the comments in ConnectionPool for method descriptions.
 
-    def __init__(self, size, timeout=1<<31):
+    def __init__(self, size, timeout=1 << 31):
         super(KeyedConnectionPool, self).__init__(size, timeout)
         self.pools = {}
 
@@ -303,6 +303,7 @@ def toTimeStamp(dt):
     args = utc_struct[:5]+(utc_struct[5] + dt.microsecond/1000000.0,)
     return TimeStamp(*args)
 
+
 def getTID(at, before):
     if at is not None:
         if before is not None:
@@ -319,6 +320,7 @@ def getTID(at, before):
             before = TimeStamp(before).raw()
     return before
 
+
 @implementer(IDatabase)
 class DB(object):
     """The Object Database
@@ -348,7 +350,7 @@ class DB(object):
     def __init__(self,
                  storage,
                  pool_size=7,
-                 pool_timeout=1<<31,
+                 pool_timeout=1 << 31,
                  cache_size=400,
                  cache_size_bytes=0,
                  historical_pool_size=3,
@@ -358,7 +360,7 @@ class DB(object):
                  database_name='unnamed',
                  databases=None,
                  xrefs=True,
-                 large_record_size=1<<24,
+                 large_record_size=1 << 24,
                  **storage_args):
         """Create an object database.
 
@@ -425,10 +427,10 @@ class DB(object):
 
         # Setup storage
         if isinstance(storage, six.string_types):
-            from ZODB import FileStorage
+            from ZODB import FileStorage  # noqa: F401 import unused
             storage = ZODB.FileStorage.FileStorage(storage, **storage_args)
         elif storage is None:
-            from ZODB import MappingStorage
+            from ZODB import MappingStorage  # noqa: F401 import unused
             storage = ZODB.MappingStorage.MappingStorage(**storage_args)
         else:
             assert not storage_args
@@ -507,6 +509,7 @@ class DB(object):
         """
 
         detail = {}
+
         def f(con, detail=detail):
             for oid, ob in con._cache.items():
                 module = getattr(ob.__class__, '__module__', '')
@@ -570,7 +573,7 @@ class DB(object):
                     'rc': (rc(ob) - 3 - (ob._p_changed is not None)
                            if rc else False),
                     'state': ob._p_changed,
-                    #'references': con.references(oid),
+                    # 'references': con.references(oid),
                 })
 
         self._connectionMap(f)
@@ -581,6 +584,7 @@ class DB(object):
 
     def cacheLastGCTime(self):
         m = [0]
+
         def f(con, m=m):
             t = con._cache.cache_last_gc_time
             if t > m[0]:
@@ -598,6 +602,7 @@ class DB(object):
         """Return the total count of non-ghost objects in all object caches
         """
         m = [0]
+
         def f(con, m=m):
             m[0] += con._cache.cache_non_ghost_count
 
@@ -608,6 +613,7 @@ class DB(object):
         """Return non-ghost counts sizes for all connections.
         """
         m = []
+
         def f(con, m=m):
             m.append({'connection': repr(con),
                       'ngsize': con._cache.cache_non_ghost_count,
@@ -726,7 +732,8 @@ class DB(object):
           - `before`: like `at`, but opens the readonly state before the
             tid or datetime.
         """
-        # `at` is normalized to `before`.
+        # `at` is normalized to `before`, since we use storage.loadBeforeEx
+        # as the underlying implementation of both.
         before = getTID(at, before)
         if (before is not None and
             before > self.lastTransaction() and
@@ -772,7 +779,6 @@ class DB(object):
             self.pool.availableGC()
             self.historical_pool.availableGC()
 
-
         result.open(transaction_manager)
         return result
 
@@ -835,7 +841,7 @@ class DB(object):
         t -= days * 86400
         try:
             self.storage.pack(t, self.references)
-        except:
+        except:  # noqa: E722 do not use bare 'except'
             logger.exception("packing")
             raise
 
@@ -1028,9 +1034,11 @@ class ContextManager(object):
             self.tm.abort()
         self.conn.close()
 
+
 resource_counter_lock = utils.Lock()
 resource_counter = 0
 
+
 class TransactionalUndo(object):
 
     def __init__(self, db, tids):
@@ -1063,9 +1071,10 @@ class TransactionalUndo(object):
         # a new storage instance, and so we must close it to be sure
         # to reclaim resources in a timely manner.
         #
-        # Once the tpc_begin method has been called, the transaction manager will
-        # guarantee to call either `tpc_finish` or `tpc_abort`, so those are the only
-        # methods we need to be concerned about calling close() from.
+        # Once the tpc_begin method has been called, the transaction manager
+        # will guarantee to call either `tpc_finish` or `tpc_abort`, so those
+        # are the only methods we need to be concerned about calling close()
+        # from.
         db_mvcc_storage = self._db._mvcc_storage
         self._storage = getattr(
             db_mvcc_storage,
@@ -1116,7 +1125,10 @@ def connection(*args, **kw):
     """
     return DB(*args, **kw).open_then_close_db_when_connection_closes()
 
+
 _transaction_meta_data_text_variables = 'user_name', 'description'
+
+
 def _text_transaction_info(info):
     for d in info:
         for name in _transaction_meta_data_text_variables:

src/ZODB/DemoStorage.py

@@ -25,7 +25,6 @@ import os
 import random
 import weakref
 import tempfile
-import warnings
 import ZODB.BaseStorage
 import ZODB.blob
 import ZODB.interfaces
@@ -37,10 +36,11 @@ import zope.interface
 from .ConflictResolution import ConflictResolvingStorage
 from .utils import load_current, maxtid
 
+
 @zope.interface.implementer(
     ZODB.interfaces.IStorage,
     ZODB.interfaces.IStorageIteration,
-        )
+)
 class DemoStorage(ConflictResolvingStorage):
     """A storage that stores changes against a read-only base database
 
@@ -101,7 +101,6 @@ class DemoStorage(ConflictResolvingStorage):
         self.base = base
         self.close_base_on_close = close_base_on_close
 
-
         if changes is None:
             self._temporary_changes = True
             changes = ZODB.MappingStorage.MappingStorage()
@@ -135,12 +134,11 @@ class DemoStorage(ConflictResolvingStorage):
 
         self._copy_methods_from_changes(changes)
 
-        self._next_oid = random.randint(1, 1<<62)
+        self._next_oid = random.randint(1, 1 << 62)
 
     def _blobify(self):
         if (self._temporary_changes and
-            isinstance(self.changes, ZODB.MappingStorage.MappingStorage)
-            ):
+                isinstance(self.changes, ZODB.MappingStorage.MappingStorage)):
             blob_dir = tempfile.mkdtemp('.demoblobs')
             _temporary_blobdirs[
                 weakref.ref(self, cleanup_temporary_blobdir)
@@ -154,6 +152,7 @@ class DemoStorage(ConflictResolvingStorage):
         self.changes.cleanup()
 
     __opened = True
+
     def opened(self):
         return self.__opened
 
@@ -224,34 +223,24 @@ class DemoStorage(ConflictResolvingStorage):
     # still want load for old clients (e.g. zeo servers)
     load = load_current
 
-    def loadAt(self, oid, at):
-        data, serial = ZODB.utils.loadAt(self.changes, oid, at)
+    def loadBeforeEx(self, oid, before):
+        data, serial = ZODB.utils.loadBeforeEx(self.changes, oid, before)
         if (data is not None) or (serial != ZODB.utils.z64):
             # object is present in changes either as data or deletion record.
             return data, serial
 
         # object is not present in changes at all - use base
-        return ZODB.utils.loadAt(self.base, oid, at)
+        return ZODB.utils.loadBeforeEx(self.base, oid, before)
 
     def loadBefore(self, oid, before):
-        warnings.warn("loadBefore is deprecated - use loadAt instead",
-                DeprecationWarning, stacklevel=2)
-        p64 = ZODB.utils.p64
-        u64 = ZODB.utils.u64
-
-        if before in (maxtid, ZODB.utils.z64):
-            at = before
-        else:
-            at = p64(u64(before)-1)
-
-        data, serial = self.loadAt(oid, at)
+        data, serial = self.loadBeforeEx(oid, before)
 
         # find out next_serial.
         # it is ok to use dumb/slow implementation since loadBefore should not
         # be used and is provided only for backward compatibility.
         next_serial = maxtid
         while 1:
-            _, s = self.loadAt(oid, p64(u64(next_serial)-1))
+            _, s = self.loadBeforeEx(oid, next_serial)
             assert s >= serial
             if s == serial:
                 # found - next_serial is serial of the next data record
@@ -273,7 +262,6 @@ class DemoStorage(ConflictResolvingStorage):
         # regular data record
         return data, serial, next_serial
 
-
     def loadBlob(self, oid, serial):
         try:
             return self.changes.loadBlob(oid, serial)
@@ -313,7 +301,7 @@ class DemoStorage(ConflictResolvingStorage):
     def new_oid(self):
         with self._lock:
             while 1:
-                oid = ZODB.utils.p64(self._next_oid )
+                oid = ZODB.utils.p64(self._next_oid)
                 if oid not in self._issued_oids:
                     try:
                         load_current(self.changes, oid)
@@ -325,7 +313,7 @@ class DemoStorage(ConflictResolvingStorage):
                             self._issued_oids.add(oid)
                             return oid
 
-                self._next_oid = random.randint(1, 1<<62)
+                self._next_oid = random.randint(1, 1 << 62)
 
     def pack(self, t, referencesf, gc=None):
         if gc is None:
@@ -361,7 +349,7 @@ class DemoStorage(ConflictResolvingStorage):
                               close_base_on_close=False)
 
     def store(self, oid, serial, data, version, transaction):
-        assert version=='', "versions aren't supported"
+        assert version == '', "versions aren't supported"
         if transaction is not self._transaction:
             raise ZODB.POSException.StorageTransactionError(self, transaction)
 
@@ -418,7 +406,7 @@ class DemoStorage(ConflictResolvingStorage):
 
     def storeBlob(self, oid, oldserial, data, blobfilename, version,
                   transaction):
-        assert version=='', "versions aren't supported"
+        assert version == '', "versions aren't supported"
         if transaction is not self._transaction:
             raise ZODB.POSException.StorageTransactionError(self, transaction)
 
@@ -476,7 +464,7 @@ class DemoStorage(ConflictResolvingStorage):
                 "Unexpected resolved conflicts")
         return self._resolved
 
-    def tpc_finish(self, transaction, func = lambda tid: None):
+    def tpc_finish(self, transaction, func=lambda tid: None):
         with self._lock:
             if (transaction is not self._transaction):
                 raise ZODB.POSException.StorageTransactionError(
@@ -488,11 +476,14 @@ class DemoStorage(ConflictResolvingStorage):
             self._commit_lock.release()
         return tid
 
+
 _temporary_blobdirs = {}
+
+
 def cleanup_temporary_blobdir(
     ref,
     _temporary_blobdirs=_temporary_blobdirs,  # Make sure it stays around
-    ):
+):
     blob_dir = _temporary_blobdirs.pop(ref, None)
     if blob_dir and os.path.exists(blob_dir):
         ZODB.blob.remove_committed_dir(blob_dir)

src/ZODB/ExportImport.py

@@ -29,17 +29,17 @@ from ZODB._compat import PersistentPickler, Unpickler, BytesIO, _protocol
 
 logger = logging.getLogger('ZODB.ExportImport')
 
+
 class ExportImport(object):
 
     def exportFile(self, oid, f=None, bufsize=64 * 1024):
         if f is None:
             f = TemporaryFile(prefix="EXP")
         elif isinstance(f, six.string_types):
-            f = open(f,'w+b')
+            f = open(f, 'w+b')
         f.write(b'ZEXP')
         oids = [oid]
         done_oids = {}
-        done = done_oids.__contains__
         load = self._storage.load
         supports_blobs = IBlobStorage.providedBy(self._storage)
         while oids:
@@ -49,7 +49,7 @@ class ExportImport(object):
             done_oids[oid] = True
             try:
                 p, serial = load(oid)
-            except:
+            except:  # noqa: E722 do not use bare 'except'
                 logger.debug("broken reference for oid %s", repr(oid),
                              exc_info=True)
             else:
@@ -159,8 +159,7 @@ class ExportImport(object):
                 return_oid_list.append(oid)
 
             if (b'blob' in data and
-                isinstance(self._reader.getGhost(data), Blob)
-                ):
+                    isinstance(self._reader.getGhost(data), Blob)):
                 # Blob support
 
                 # Make sure we have a (redundant, overly) blob marker.
@@ -198,11 +197,14 @@ class ExportImport(object):
 export_end_marker = b'\377'*16
 blob_begin_marker = b'\000BLOBSTART'
 
+
 class Ghost(object):
     __slots__ = ("oid",)
+
     def __init__(self, oid):
         self.oid = oid
 
+
 def persistent_id(obj):
     if isinstance(obj, Ghost):
         return obj.oid

src/ZODB/FileStorage/format.py

@@ -90,9 +90,11 @@ from ZODB.POSException import POSKeyError
 from ZODB.utils import u64, oid_repr, as_bytes
 from ZODB._compat import PY3
 
+
 class CorruptedError(Exception):
     pass
 
+
 class CorruptedDataError(CorruptedError):
 
     def __init__(self, oid=None, buf=None, pos=None):
@@ -110,6 +112,7 @@ class CorruptedDataError(CorruptedError):
             msg += " at %d" % self.pos
         return msg
 
+
 # the struct formats for the headers
 TRANS_HDR = ">8sQcHHH"
 DATA_HDR = ">8s8sQQHQ"
@@ -121,6 +124,7 @@ assert struct.calcsize(DATA_HDR) == DATA_HDR_LEN
 
 logger = logging.getLogger('ZODB.FileStorage.format')
 
+
 class FileStorageFormatter(object):
     """Mixin class that can read and write the low-level format."""
 
@@ -211,7 +215,7 @@ class FileStorageFormatter(object):
         self.ltid = th.tid
         if th.status == "c":
             self.fail(pos, "transaction with checkpoint flag set")
-        if not th.status in " pu": # recognize " ", "p", and "u" as valid
+        if th.status not in " pu":  # recognize " ", "p", and "u" as valid
             self.fail(pos, "invalid transaction status: %r", th.status)
         if th.tlen < th.headerlen():
             self.fail(pos, "invalid transaction header: "
@@ -232,9 +236,11 @@ class FileStorageFormatter(object):
             if dh.plen:
                 self.fail(pos, "data record has back pointer and data")
 
+
 def DataHeaderFromString(s):
     return DataHeader(*struct.unpack(DATA_HDR, s))
 
+
 class DataHeader(object):
     """Header for a data record."""
 
@@ -259,12 +265,14 @@ class DataHeader(object):
     def recordlen(self):
         return DATA_HDR_LEN + (self.plen or 8)
 
+
 def TxnHeaderFromString(s):
     res = TxnHeader(*struct.unpack(TRANS_HDR, s))
     if PY3:
         res.status = res.status.decode('ascii')
     return res
 
+
 class TxnHeader(object):
     """Header for a transaction record."""
 

src/ZODB/FileStorage/fsdump.py

@@ -20,15 +20,18 @@ from ZODB.FileStorage.format import DATA_HDR, DATA_HDR_LEN
 from ZODB.TimeStamp import TimeStamp
 from ZODB.utils import u64, get_pickle_metadata
 
+
 def fsdump(path, file=None, with_offset=1):
     iter = FileIterator(path)
     for i, trans in enumerate(iter):
+        size = trans._tend - trans._tpos
         if with_offset:
-            print(("Trans #%05d tid=%016x time=%s offset=%d" %
-                  (i, u64(trans.tid), TimeStamp(trans.tid), trans._pos)), file=file)
+            print(("Trans #%05d tid=%016x size=%d time=%s offset=%d" %
+                   (i, u64(trans.tid), size,
+                    TimeStamp(trans.tid), trans._pos)), file=file)
         else:
-            print(("Trans #%05d tid=%016x time=%s" %
-                  (i, u64(trans.tid), TimeStamp(trans.tid))), file=file)
+            print(("Trans #%05d tid=%016x size=%d time=%s" %
+                   (i, u64(trans.tid), size, TimeStamp(trans.tid))), file=file)
         print(("    status=%r user=%r description=%r" %
                (trans.status, trans.user, trans.description)), file=file)
 
@@ -52,10 +55,12 @@ def fsdump(path, file=None, with_offset=1):
                    (j, u64(rec.oid), size, fullclass, bp)), file=file)
     iter.close()
 
+
 def fmt(p64):
     # Return a nicely formatted string for a packaged 64-bit value
     return "%016x" % u64(p64)
 
+
 class Dumper(object):
     """A very verbose dumper for debuggin FileStorage problems."""
 
@@ -85,13 +90,13 @@ class Dumper(object):
         print("transaction id: %s" % fmt(tid), file=self.dest)
         print("trec len: %d" % tlen, file=self.dest)
         print("status: %r" % status, file=self.dest)
-        user = descr = extra = ""
+        user = descr = ""
         if ul:
             user = self.file.read(ul)
         if dl:
             descr = self.file.read(dl)
         if el:
-            extra = self.file.read(el)
+            self.file.read(el)
         print("user: %r" % user, file=self.dest)
         print("description: %r" % descr, file=self.dest)
         print("len(extra): %d" % el, file=self.dest)
@@ -119,6 +124,11 @@ class Dumper(object):
             sbp = self.file.read(8)
             print("backpointer: %d" % u64(sbp), file=self.dest)
 
+
 def main():
     import sys
     fsdump(sys.argv[1])
+
+
+if __name__ == "__main__":
+    main()

src/ZODB/FileStorage/fsoids.py

@@ -18,10 +18,14 @@ from ZODB.serialize import get_refs
 from ZODB.TimeStamp import TimeStamp
 
 # Extract module.class string from pickle.
+
+
 def get_class(pickle):
     return "%s.%s" % get_pickle_metadata(pickle)
 
 # Shorten a string for display.
+
+
 def shorten(s, size=50):
     if len(s) <= size:
         return s
@@ -29,7 +33,12 @@ def shorten(s, size=50):
     navail = size - 5
     nleading = navail // 2
     ntrailing = size - nleading
-    return s[:nleading] + " ... " + s[-ntrailing:]
+    if isinstance(s, bytes):
+        sep = b" ... "
+    else:
+        sep = " ... "
+    return s[:nleading] + sep + s[-ntrailing:]
+
 
 class Tracer(object):
     """Trace all occurrences of a set of oids in a FileStorage.
@@ -80,7 +89,7 @@ class Tracer(object):
             self.oids[oid] = 0  # 0 revisions seen so far
 
     def _msg(self, oid, tid, *args):
-        self.msgs.append( (oid, tid, ' '.join(map(str, args))) )
+        self.msgs.append((oid, tid, ' '.join(map(str, args))))
         self._produced_msg = True
 
     def report(self):
@@ -94,7 +103,7 @@ class Tracer(object):
         NOT_SEEN = "this oid was not defined (no data record for it found)"
         for oid in oids:
             if oid not in oid2name:
-                msgs.append( (oid, None, NOT_SEEN) )
+                msgs.append((oid, None, NOT_SEEN))
 
         msgs.sort()  # oids are primary key, tids secondary
         current_oid = current_tid = None

src/ZODB/FileStorage/fspack.py

@@ -36,9 +36,11 @@ import ZODB.POSException
 
 logger = logging.getLogger(__name__)
 
+
 class PackError(ZODB.POSException.POSError):
     pass
 
+
 class PackCopier(FileStorageFormatter):
 
     def __init__(self, f, index, tindex):
@@ -144,6 +146,7 @@ class PackCopier(FileStorageFormatter):
         finally:
             self._file.seek(pos)
 
+
 class GC(FileStorageFormatter):
 
     def __init__(self, file, eof, packtime, gc, referencesf):
@@ -330,6 +333,7 @@ class GC(FileStorageFormatter):
         else:
             return []
 
+
 class FileStoragePacker(FileStorageFormatter):
 
     # path is the storage file path.
@@ -409,15 +413,15 @@ class FileStoragePacker(FileStorageFormatter):
             # try our best, but don't fail
             try:
                 self._tfile.close()
-            except:
+            except:  # noqa: E722 do not use bare 'except'
                 pass
             try:
                 self._file.close()
-            except:
+            except:  # noqa: E722 do not use bare 'except'
                 pass
             try:
                 os.remove(self._name + ".pack")
-            except:
+            except:  # noqa: E722 do not use bare 'except'
                 pass
             if self.blob_removed is not None:
                 self.blob_removed.close()
@@ -483,13 +487,12 @@ class FileStoragePacker(FileStorageFormatter):
             if self.locked:
                 self._commit_lock.release()
             raise  # don't succeed silently
-        except:
+        except:  # noqa: E722 do not use bare 'except'
             if self.locked:
                 self._commit_lock.release()
             raise
 
     def copyToPacktime(self):
-        offset = 0  # the amount of space freed by packing
         pos = self._metadata_size
         new_pos = pos
 
@@ -506,7 +509,6 @@ class FileStoragePacker(FileStorageFormatter):
                 self._tfile.seek(new_pos - 8)
                 self._tfile.write(p64(tlen))
 
-
             tlen = self._read_num(pos)
             if tlen != th.tlen:
                 self.fail(pos, "redundant transaction length does not "
@@ -546,8 +548,8 @@ class FileStoragePacker(FileStorageFormatter):
                         # record. There's a bug in ZEO blob support that causes
                         # duplicate data records.
                         rpos = self.gc.reachable.get(h.oid)
-                        is_dup = (rpos
-                                  and self._read_data_header(rpos).tid == h.tid)
+                        is_dup = (
+                            rpos and self._read_data_header(rpos).tid == h.tid)
                         if not is_dup:
                             if h.oid not in self.gc.reachable:
                                 self.blob_removed.write(
@@ -569,7 +571,6 @@ class FileStoragePacker(FileStorageFormatter):
                 s = th.asString()
                 new_tpos = self._tfile.tell()
                 self._tfile.write(s)
-                new_pos = new_tpos + len(s)
                 copy = 1
 
             if h.plen:
@@ -578,7 +579,6 @@ class FileStoragePacker(FileStorageFormatter):
                 data = self.fetchDataViaBackpointer(h.oid, h.back)
 
             self.writePackedDataRecord(h, data, new_tpos)
-            new_pos = self._tfile.tell()
 
         return new_tpos, pos
 

src/ZODB/FileStorage/interfaces.py

@@ -13,6 +13,7 @@
 ##############################################################################
 import zope.interface
 
+
 class IFileStoragePacker(zope.interface.Interface):
 
     def __call__(storage, referencesf, stop, gc):
@@ -58,6 +59,7 @@ class IFileStoragePacker(zope.interface.Interface):
           corresponding to the file records.
         """
 
+
 class IFileStorage(zope.interface.Interface):
 
     packer = zope.interface.Attribute(

src/ZODB/FileStorage/tests.py

@@ -33,6 +33,7 @@ checker = renormalizing.RENormalizing([
     (re.compile('data.fs:[0-9]+'), 'data.fs:<OFFSET>'),
 ])
 
+
 def pack_keep_old():
     """Should a copy of the database be kept?
 
@@ -106,6 +107,7 @@ directory for blobs is kept.)
     >>> db.close()
     """
 
+
 def pack_with_repeated_blob_records():
     """
     There is a bug in ZEO that causes duplicate bloc database records
@@ -144,6 +146,7 @@ def pack_with_repeated_blob_records():
     >>> db.close()
     """
 
+
 def _save_index():
     """
 
@@ -187,6 +190,7 @@ cleanup
 
     """
 
+
 def pack_disk_full_copyToPacktime():
     """Recover from a disk full situation by removing the `.pack` file
 
@@ -239,6 +243,7 @@ check the data we added
     >>> db.close()
     """
 
+
 def pack_disk_full_copyRest():
     """Recover from a disk full situation by removing the `.pack` file
 
@@ -307,6 +312,7 @@ check the data we added
     >>> db.close()
     """
 
+
 def test_suite():
     return unittest.TestSuite((
         doctest.DocFileSuite(

src/ZODB/MappingStorage.py

@@ -19,7 +19,6 @@ storage without distracting storage details.
 
 import BTrees
 import time
-import warnings
 import ZODB.BaseStorage
 import ZODB.interfaces
 import ZODB.POSException
@@ -31,8 +30,8 @@ import zope.interface
 @zope.interface.implementer(
     ZODB.interfaces.IStorage,
     ZODB.interfaces.IStorageIteration,
-        ZODB.interfaces.IStorageLoadAt,
-        )
+    ZODB.interfaces.IStorageLoadBeforeEx,
+)
 class MappingStorage(object):
     """In-memory storage implementation
 
@@ -52,7 +51,8 @@ class MappingStorage(object):
         """
         self.__name__ = name
         self._data = {}                               # {oid->{tid->pickle}}
-        self._transactions = BTrees.OOBTree.OOBTree() # {tid->TransactionRecord}
+        # {tid->TransactionRecord}
+        self._transactions = BTrees.OOBTree.OOBTree()
         self._ltid = ZODB.utils.z64
         self._last_pack = None
         self._lock = ZODB.utils.RLock()
@@ -119,13 +119,13 @@ class MappingStorage(object):
         tids.reverse()
         return [
             dict(
-                time = ZODB.TimeStamp.TimeStamp(tid).timeTime(),
-                tid = tid,
-                serial = tid,
-                user_name = self._transactions[tid].user,
-                description = self._transactions[tid].description,
-                extension = self._transactions[tid].extension,
-                size = len(tid_data[tid])
+                time=ZODB.TimeStamp.TimeStamp(tid).timeTime(),
+                tid=tid,
+                serial=tid,
+                user_name=self._transactions[tid].user,
+                description=self._transactions[tid].description,
+                extension=self._transactions[tid].extension,
+                size=len(tid_data[tid])
             )
             for tid in tids]
 
@@ -150,15 +150,16 @@ class MappingStorage(object):
 
     load = ZODB.utils.load_current
 
-    # ZODB.interfaces.IStorageLoadAt
+    # ZODB.interfaces.IStorageLoadBeforeEx
     @ZODB.utils.locked(opened)
-    def loadAt(self, oid, at):
+    def loadBeforeEx(self, oid, before):
         z64 = ZODB.utils.z64
         tid_data = self._data.get(oid)
         if not tid_data:
             return None, z64
-        if at == z64:
+        if before == z64:
             return None, z64
+        at = ZODB.utils.p64(ZODB.utils.u64(before)-1)
         tids_at = tid_data.keys(None, at)
         if not tids_at:
             return None, z64
@@ -168,8 +169,6 @@ class MappingStorage(object):
     # ZODB.interfaces.IStorage
     @ZODB.utils.locked(opened)
     def loadBefore(self, oid, tid):
-        warnings.warn("loadBefore is deprecated - use loadAt instead",
-                DeprecationWarning, stacklevel=2)
         tid_data = self._data.get(oid)
         if tid_data:
             before = ZODB.utils.u64(tid)
@@ -186,8 +185,8 @@ class MappingStorage(object):
         else:
             raise ZODB.POSException.POSKeyError(oid)
 
-
     # ZODB.interfaces.IStorage
+
     @ZODB.utils.locked(opened)
     def loadSerial(self, oid, serial):
         tid_data = self._data.get(oid)
@@ -211,7 +210,7 @@ class MappingStorage(object):
         if not self._data:
             return
 
-        stop = ZODB.TimeStamp.TimeStamp(*time.gmtime(t)[:5]+(t%60,)).raw()
+        stop = ZODB.TimeStamp.TimeStamp(*time.gmtime(t)[:5]+(t % 60,)).raw()
         if self._last_pack is not None and self._last_pack >= stop:
             if self._last_pack == stop:
                 return
@@ -317,7 +316,7 @@ class MappingStorage(object):
 
     # ZODB.interfaces.IStorage
     @ZODB.utils.locked(opened)
-    def tpc_finish(self, transaction, func = lambda tid: None):
+    def tpc_finish(self, transaction, func=lambda tid: None):
         if (transaction is not self._transaction):
             raise ZODB.POSException.StorageTransactionError(
                 "tpc_finish called with wrong transaction")
@@ -351,6 +350,7 @@ class MappingStorage(object):
             raise ZODB.POSException.StorageTransactionError(
                 "tpc_vote called with wrong transaction")
 
+
 class TransactionRecord(object):
 
     status = ' '
@@ -376,11 +376,11 @@ class TransactionRecord(object):
         del self.data[oid]
         return not self.data
 
+
 @zope.interface.implementer(ZODB.interfaces.IStorageRecordInformation)
 class DataRecord(object):
     """Abstract base class for iterator protocol"""
 
-
     version = ''
     data_txn = None
 
@@ -389,5 +389,6 @@ class DataRecord(object):
         self.tid = tid
         self.data = data
 
+
 def DB(*args, **kw):
     return ZODB.DB(MappingStorage(), *args, **kw)

src/ZODB/POSException.py

@@ -18,20 +18,26 @@ $Id$"""
 from ZODB.utils import oid_repr, readable_tid_repr
 
 # BBB: We moved the two transactions to the transaction package
-from transaction.interfaces import TransactionError, TransactionFailedError
+from transaction.interfaces import TransactionError  # noqa: F401 import unused
+from transaction.interfaces import TransactionFailedError  # noqa: F401
 
 import transaction.interfaces
 
+
 def _fmt_undo(oid, reason):
     s = reason and (": %s" % reason) or ""
     return "Undo error %s%s" % (oid_repr(oid), s)
 
+
 def _recon(class_, state):
     err = class_.__new__(class_)
     err.__setstate__(state)
     return err
+
+
 _recon.__no_side_effects__ = True
 
+
 class POSError(Exception):
     """Persistent object system error."""
 
@@ -49,9 +55,10 @@ class POSError(Exception):
         # the args would then get lost, leading to unprintable exceptions
         # and worse. Manually assign to args from the state to be sure
         # this doesn't happen.
-        super(POSError,self).__setstate__(state)
+        super(POSError, self).__setstate__(state)
         self.args = state['args']
 
+
 class POSKeyError(POSError, KeyError):
     """Key not found in database."""
 
@@ -143,6 +150,7 @@ class ConflictError(POSError, transaction.interfaces.TransientError):
     def get_serials(self):
         return self.serials
 
+
 class ReadConflictError(ConflictError):
     """Conflict detected when object was requested to stay unchanged.
 
@@ -156,16 +164,19 @@ class ReadConflictError(ConflictError):
       - object is found to be removed, and
       - there is possibility that database pack was running simultaneously.
     """
+
     def __init__(self, message=None, object=None, serials=None, **kw):
         if message is None:
             message = "database read conflict error"
         ConflictError.__init__(self, message=message, object=object,
                                serials=serials, **kw)
 
+
 class BTreesConflictError(ConflictError):
     """A special subclass for BTrees conflict errors."""
 
-    msgs = [# 0; i2 or i3 bucket split; positions are all -1
+    msgs = [
+        # 0; i2 or i3 bucket split; positions are all -1
         'Conflicting bucket split',
 
         # 1; keys the same, but i2 and i3 values differ, and both values
@@ -226,11 +237,14 @@ class BTreesConflictError(ConflictError):
                                                         self.p2,
                                                         self.p3,
                                                         self.reason)
+
     def __str__(self):
         return "BTrees conflict error at %d/%d/%d: %s" % (
             self.p1, self.p2, self.p3, self.msgs[self.reason])
 
-class DanglingReferenceError(POSError, transaction.interfaces.TransactionError):
+
+class DanglingReferenceError(
+        POSError, transaction.interfaces.TransactionError):
     """An object has a persistent reference to a missing object.
 
     If an object is stored and it has a reference to another object
@@ -258,9 +272,11 @@ class DanglingReferenceError(POSError, transaction.interfaces.TransactionError):
 class VersionError(POSError):
     """An error in handling versions occurred."""
 
+
 class VersionCommitError(VersionError):
     """An invalid combination of versions was used in a version commit."""
 
+
 class VersionLockError(VersionError, transaction.interfaces.TransactionError):
     """Modification to an object modified in an unsaved version.
 
@@ -269,6 +285,7 @@ class VersionLockError(VersionError, transaction.interfaces.TransactionError):
     """
 ############################################################################
 
+
 class UndoError(POSError):
     """An attempt was made to undo a non-undoable transaction."""
 
@@ -279,6 +296,7 @@ class UndoError(POSError):
     def __str__(self):
         return _fmt_undo(self._oid, self._reason)
 
+
 class MultipleUndoErrors(UndoError):
     """Several undo errors occurred during a single transaction."""
 
@@ -290,33 +308,43 @@ class MultipleUndoErrors(UndoError):
     def __str__(self):
         return "\n".join([_fmt_undo(*pair) for pair in self._errs])
 
+
 class StorageError(POSError):
     """Base class for storage based exceptions."""
 
+
 class StorageTransactionError(StorageError):
     """An operation was invoked for an invalid transaction or state."""
 
+
 class StorageSystemError(StorageError):
     """Panic! Internal storage error!"""
 
+
 class MountedStorageError(StorageError):
     """Unable to access mounted storage."""
 
+
 class ReadOnlyError(StorageError):
     """Unable to modify objects in a read-only storage."""
 
+
 class TransactionTooLargeError(StorageTransactionError):
     """The transaction exhausted some finite storage resource."""
 
+
 class ExportError(POSError):
     """An export file doesn't have the right format."""
 
+
 class Unsupported(POSError):
     """A feature was used that is not supported by the storage."""
 
+
 class ReadOnlyHistoryError(POSError):
     """Unable to add or modify objects in an historical connection."""
 
+
 class InvalidObjectReference(POSError):
     """An object contains an invalid reference to another object.
 
@@ -329,6 +357,7 @@ class InvalidObjectReference(POSError):
     TODO:  The exception ought to have a member that is the invalid object.
     """
 
+
 class ConnectionStateError(POSError):
     """A Connection isn't in the required state for an operation.
 

src/ZODB/__init__.py

@@ -12,6 +12,7 @@
 #
 ##############################################################################
 
+from ZODB.DB import DB, connection
 import sys
 
 from persistent import TimeStamp
@@ -24,5 +25,3 @@ sys.modules['ZODB.PersistentMapping'] = sys.modules['persistent.mapping']
 sys.modules['ZODB.PersistentList'] = sys.modules['persistent.list']
 
 del mapping, list, sys
-
-from ZODB.DB import DB, connection

src/ZODB/_compat.py

@@ -11,13 +11,13 @@
 # FOR A PARTICULAR PURPOSE
 #
 ##############################################################################
+from zodbpickle import binary  # noqa: F401 import unused
 import sys
 from six import PY3
 
 IS_JYTHON = sys.platform.startswith('java')
 
 _protocol = 3
-from zodbpickle import binary
 
 if not PY3:
     # Python 2.x
@@ -42,7 +42,8 @@ else:
     # http://bugs.python.org/issue6784
     import zodbpickle.pickle
     HIGHEST_PROTOCOL = 3
-    from _compat_pickle import IMPORT_MAPPING, NAME_MAPPING
+    from _compat_pickle import IMPORT_MAPPING  # noqa: F401 import unused
+    from _compat_pickle import NAME_MAPPING  # noqa: F401 import unused
 
     class Pickler(zodbpickle.pickle.Pickler):
         def __init__(self, f, protocol=None):
@@ -92,6 +93,7 @@ def PersistentPickler(persistent_id, *args, **kwargs):
     p.persistent_id = persistent_id
     return p
 
+
 def PersistentUnpickler(find_global, load_persistent, *args, **kwargs):
     """
     Returns a :class:`Unpickler` that will use the given `find_global` function
@@ -104,7 +106,8 @@ def PersistentUnpickler(find_global, load_persistent, *args, **kwargs):
     if find_global is not None:
         unpickler.find_global = find_global
         try:
-            unpickler.find_class = find_global # PyPy, zodbpickle, the non-c-accelerated version
+            # PyPy, zodbpickle, the non-c-accelerated version
+            unpickler.find_class = find_global
         except AttributeError:
             pass
     if load_persistent is not None:
@@ -118,7 +121,7 @@ try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
     # Python 3.x
-    from io import BytesIO
+    from io import BytesIO  # noqa: F401 import unused
 
 
 try:
@@ -126,14 +129,15 @@ try:
     from base64 import decodebytes, encodebytes
 except ImportError:
     # Python 2.x
-    from base64 import decodestring as decodebytes, encodestring as encodebytes
+    from base64 import decodestring as decodebytes  # noqa: F401 import unused
+    from base64 import encodestring as encodebytes  # noqa: F401 import unused
 
 
 # Python 3.x: ``hasattr()`` swallows only AttributeError.
 def py2_hasattr(obj, name):
     try:
         getattr(obj, name)
-    except:
+    except:  # noqa: E722 do not use bare 'except'
         return False
     return True
 
@@ -151,9 +155,10 @@ else:
 
 try:
     TEXT = unicode
-except NameError: #pragma NO COVER Py3k
+except NameError:  # pragma NO COVER Py3k
     TEXT = str
 
+
 def ascii_bytes(x):
     if isinstance(x, TEXT):
         x = x.encode('ascii')

src/ZODB/blob.py

@@ -35,7 +35,6 @@ from ZODB._compat import BytesIO
 from ZODB._compat import PersistentUnpickler
 from ZODB._compat import decodebytes
 from ZODB._compat import ascii_bytes
-from ZODB._compat import INT_TYPES
 from ZODB._compat import PY3
 
 
@@ -62,17 +61,18 @@ valid_modes = 'r', 'w', 'r+', 'a', 'c'
 # of a weakref when the weakref object dies at the same time
 # as the object it refers to. In other words, this doesn't work:
 #    self._ref = weakref.ref(self, lambda ref: ...)
-# because the function never gets called (https://bitbucket.org/pypy/pypy/issue/2030).
+# because the function never gets called
+# (https://bitbucket.org/pypy/pypy/issue/2030).
 # The Blob class used to use that pattern to clean up uncommitted
 # files; now we use this module-level global (but still keep a
 # reference in the Blob in case we need premature cleanup).
 _blob_close_refs = []
 
+
 @zope.interface.implementer(ZODB.interfaces.IBlob)
 class Blob(persistent.Persistent):
     """A BLOB supports efficient handling of large data within ZODB."""
 
-
     _p_blob_uncommitted = None  # Filename of the uncommitted (dirty) data
     _p_blob_committed = None  # Filename of the committed data
     _p_blob_ref = None  # weakreference to self; also in _blob_close_refs
@@ -143,8 +143,7 @@ class Blob(persistent.Persistent):
                     or
                     not self._p_blob_committed
                     or
-                self._p_blob_committed.endswith(SAVEPOINT_SUFFIX)
-                ):
+                    self._p_blob_committed.endswith(SAVEPOINT_SUFFIX)):
                 raise BlobError('Uncommitted changes')
             return self._p_jar._storage.openCommittedBlobFile(
                 self._p_oid, self._p_serial)
@@ -217,8 +216,7 @@ class Blob(persistent.Persistent):
                 or
                 not self._p_blob_committed
                 or
-            self._p_blob_committed.endswith(SAVEPOINT_SUFFIX)
-            ):
+                self._p_blob_committed.endswith(SAVEPOINT_SUFFIX)):
             raise BlobError('Uncommitted changes')
 
         result = self._p_blob_committed
@@ -254,7 +252,7 @@ class Blob(persistent.Persistent):
 
         try:
             rename_or_copy_blob(filename, target, chmod=False)
-        except:
+        except:  # noqa: E722 do not use bare 'except'
             # Recover from the failed consumption: First remove the file, it
             # might exist and mark the pointer to the uncommitted file.
             self._p_blob_uncommitted = None
@@ -317,6 +315,7 @@ class Blob(persistent.Persistent):
         self._p_blob_uncommitted = self._p_blob_ref = None
         return filename
 
+
 class BlobFile(file):
     """A BlobFile that holds a file handle to actual blob data.
 
@@ -348,8 +347,10 @@ class BlobFile(file):
         # prohibit it on all versions.
         raise TypeError("Pickling a BlobFile is not allowed")
 
+
 _pid = str(os.getpid())
 
+
 def log(msg, level=logging.INFO, subsys=_pid, exc_info=False):
     message = "(%s) %s" % (subsys, msg)
     logger.log(level, message, exc_info=exc_info)
@@ -394,8 +395,8 @@ class FilesystemHelper(object):
                 layout = layout_marker.read().strip()
             if layout != self.layout_name:
                 raise ValueError(
-                    "Directory layout `%s` selected for blob directory %s, but "
-                    "marker found for layout `%s`" %
+                    "Directory layout `%s` selected for blob directory %s, but"
+                    " marker found for layout `%s`" %
                     (self.layout_name, self.base_dir, layout))
 
     def isSecure(self, path):
@@ -541,6 +542,7 @@ class NoBlobsFileSystemHelper(object):
 class BlobStorageError(Exception):
     """The blob storage encountered an invalid state."""
 
+
 def auto_layout_select(path):
     # A heuristic to look at a path and determine which directory layout to
     # use.
@@ -618,8 +620,10 @@ class BushyLayout(object):
         filename = "%s%s" % (utils.tid_repr(tid), BLOB_SUFFIX)
         return os.path.join(oid_path, filename)
 
+
 LAYOUTS['bushy'] = BushyLayout()
 
+
 class LawnLayout(BushyLayout):
     """A shallow directory layout for blob directories.
 
@@ -640,8 +644,10 @@ class LawnLayout(BushyLayout):
         except (TypeError, binascii.Error):
             raise ValueError('Not a valid OID path: `%s`' % path)
 
+
 LAYOUTS['lawn'] = LawnLayout()
 
+
 class BlobStorageMixin(object):
     """A mix-in to help storages support blobs."""
 
@@ -738,7 +744,6 @@ class BlobStorage(BlobStorageMixin):
     """A wrapper/proxy storage to support blobs.
     """
 
-
     def __init__(self, base_directory, storage, layout='automatic'):
         assert not ZODB.interfaces.IBlobStorage.providedBy(storage)
         self.__storage = storage
@@ -780,8 +785,8 @@ class BlobStorage(BlobStorageMixin):
 
     def tpc_abort(self, *arg, **kw):
         # We need to override the base storage's abort instead of
-        # providing an _abort method because methods found on the proxied object
-        # aren't rebound to the proxy
+        # providing an _abort method because methods found on the proxied
+        # object aren't rebound to the proxy
         self.__storage.tpc_abort(*arg, **kw)
         self._blob_tpc_abort()
 
@@ -866,8 +871,7 @@ class BlobStorage(BlobStorageMixin):
             for oid in self.fshelper.getOIDsForSerial(serial_id):
                 # we want to find the serial id of the previous revision
                 # of this blob object.
-                at_before = utils.p64(utils.u64(serial_id)-1)
-                _, serial_before = utils.loadAt(self, oid, at_before)
+                _, serial_before = utils.loadBeforeEx(self, oid, serial_id)
 
                 if serial_before == utils.z64:
                     # There was no previous revision of this blob
@@ -904,7 +908,10 @@ class BlobStorage(BlobStorageMixin):
         res = BlobStorage(base_dir, s)
         return res
 
+
 copied = logging.getLogger('ZODB.blob.copied').debug
+
+
 def rename_or_copy_blob(f1, f2, chmod=True):
     """Try to rename f1 to f2, fallback to copy.
 
@@ -925,6 +932,7 @@ def rename_or_copy_blob(f1, f2, chmod=True):
     if chmod:
         set_not_writable(f2)
 
+
 if sys.platform == 'win32':
     # On Windows, you can't remove read-only files, so make the
     # file writable first.
@@ -951,6 +959,7 @@ def find_global_Blob(module, class_):
     if module == 'ZODB.blob' and class_ == 'Blob':
         return Blob
 
+
 def is_blob_record(record):
     """Check whether a database record is a blob record.
 
@@ -959,7 +968,8 @@ def is_blob_record(record):
 
     """
     if record and (b'ZODB.blob' in record):
-        unpickler = PersistentUnpickler(find_global_Blob, None, BytesIO(record))
+        unpickler = PersistentUnpickler(
+            find_global_Blob, None, BytesIO(record))
 
         try:
             return unpickler.load() is Blob
@@ -970,6 +980,7 @@ def is_blob_record(record):
 
     return False
 
+
 def copyTransactionsFromTo(source, destination):
     for trans in source.iterator():
         destination.tpc_begin(trans, trans.tid, trans.status)
@@ -1000,6 +1011,8 @@ def copyTransactionsFromTo(source, destination):
 
 NO_WRITE = ~ (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
 READ_PERMS = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
+
+
 def set_not_writable(path):
     perms = stat.S_IMODE(os.lstat(path).st_mode)
 

src/ZODB/broken.py

@@ -25,6 +25,7 @@ from ZODB._compat import NAME_MAPPING
 
 broken_cache = {}
 
+
 @zope.interface.implementer(ZODB.interfaces.IBroken)
 class Broken(object):
     """Broken object base class
@@ -99,7 +100,6 @@ class Broken(object):
          >>> broken_cache.clear()
        """
 
-
     __Broken_state__ = __Broken_initargs__ = None
 
     __name__ = 'broken object'
@@ -131,6 +131,7 @@ class Broken(object):
     def __setattr__(self, name, value):
         raise BrokenModified("Can't change broken objects")
 
+
 def find_global(modulename, globalname,
                 # These are *not* optimizations. Callers can override these.
                 Broken=Broken, type=type,
@@ -220,6 +221,7 @@ def find_global(modulename, globalname,
     broken_cache[(modulename, globalname)] = class_
     return class_
 
+
 def rebuild(modulename, globalname, *args):
     """Recreate a broken object, possibly recreating the missing class
 
@@ -257,10 +259,12 @@ def rebuild(modulename, globalname, *args):
     class_ = find_global(modulename, globalname)
     return class_.__new__(class_, *args)
 
+
 class BrokenModified(TypeError):
     """Attempt to modify a broken object
     """
 
+
 class PersistentBroken(Broken, persistent.Persistent):
     r"""Persistent broken objects
 
@@ -347,6 +351,7 @@ class PersistentBroken(Broken, persistent.Persistent):
     def __getnewargs__(self):
         return self.__Broken_newargs__
 
+
 def persistentBroken(class_):
     try:
         return class_.__dict__['__Broken_Persistent__']

src/ZODB/config.py

@@ -29,18 +29,21 @@ _db_schema = None
 s_schema_path = os.path.join(ZODB.__path__[0], "storage.xml")
 _s_schema = None
 
+
 def getDbSchema():
     global _db_schema
     if _db_schema is None:
         _db_schema = ZConfig.loadSchema(db_schema_path)
     return _db_schema
 
+
 def getStorageSchema():
     global _s_schema
     if _s_schema is None:
         _s_schema = ZConfig.loadSchema(s_schema_path)
     return _s_schema
 
+
 def databaseFromString(s):
     """Create a database from a database-configuration string.
 
@@ -56,6 +59,7 @@ def databaseFromString(s):
     """
     return databaseFromFile(StringIO(s))
 
+
 def databaseFromFile(f):
     """Create a database from a file object that provides configuration.
 
@@ -64,6 +68,7 @@ def databaseFromFile(f):
     config, handle = ZConfig.loadConfigFile(getDbSchema(), f)
     return databaseFromConfig(config.database)
 
+
 def databaseFromURL(url):
     """Load a database from URL (or file name) that provides configuration.
 
@@ -72,6 +77,7 @@ def databaseFromURL(url):
     config, handler = ZConfig.loadConfig(getDbSchema(), url)
     return databaseFromConfig(config.database)
 
+
 def databaseFromConfig(database_factories):
     databases = {}
     first = None
@@ -82,17 +88,20 @@ def databaseFromConfig(database_factories):
 
     return first
 
+
 def storageFromString(s):
     """Create a storage from a storage-configuration string.
     """
     return storageFromFile(StringIO(s))
 
+
 def storageFromFile(f):
     """Create a storage from a file object providing storage-configuration.
     """
     config, handle = ZConfig.loadConfigFile(getStorageSchema(), f)
     return storageFromConfig(config.storage)
 
+
 def storageFromURL(url):
     """\
     Create a storage from a URL (or file name) providing storage-configuration.
@@ -100,9 +109,11 @@ def storageFromURL(url):
     config, handler = ZConfig.loadConfig(getStorageSchema(), url)
     return storageFromConfig(config.storage)
 
+
 def storageFromConfig(section):
     return section.open()
 
+
 class BaseConfig(object):
     """Object representing a configured storage or database.
 
@@ -124,6 +135,7 @@ class BaseConfig(object):
         """Open and return the storage object."""
         raise NotImplementedError
 
+
 class ZODBDatabase(BaseConfig):
 
     def open(self, databases=None):
@@ -150,21 +162,23 @@ class ZODBDatabase(BaseConfig):
                 cache_size_bytes=section.cache_size_bytes,
                 historical_pool_size=section.historical_pool_size,
                 historical_cache_size=section.historical_cache_size,
-                historical_cache_size_bytes=section.historical_cache_size_bytes,
+                historical_cache_size_bytes=section.historical_cache_size_bytes,  # noqa: E501 line too long
                 historical_timeout=section.historical_timeout,
                 database_name=section.database_name or self.name or '',
                 databases=databases,
                 **options)
-        except:
+        except:  # noqa: E722 do not use bare 'except'
             storage.close()
             raise
 
+
 class MappingStorage(BaseConfig):
 
     def open(self):
         from ZODB.MappingStorage import MappingStorage
         return MappingStorage(self.config.name)
 
+
 class DemoStorage(BaseConfig):
 
     def open(self):
@@ -181,6 +195,7 @@ class DemoStorage(BaseConfig):
         from ZODB.DemoStorage import DemoStorage
         return DemoStorage(self.config.name, base=base, changes=changes)
 
+
 class FileStorage(BaseConfig):
 
     def open(self):
@@ -206,6 +221,7 @@ class FileStorage(BaseConfig):
 
         return FileStorage(config.path, **options)
 
+
 class BlobStorage(BaseConfig):
 
     def open(self):
@@ -225,7 +241,8 @@ class ZEOClient(BaseConfig):
         if self.config.blob_cache_size is not None:
             options['blob_cache_size'] = self.config.blob_cache_size
         if self.config.blob_cache_size_check is not None:
-            options['blob_cache_size_check'] = self.config.blob_cache_size_check
+            options['blob_cache_size_check'] = (
+                self.config.blob_cache_size_check)
         if self.config.client_label is not None:
             options['client_label'] = self.config.client_label
 
@@ -249,6 +266,7 @@ class ZEOClient(BaseConfig):
             realm=self.config.realm,
             **options)
 
+
 class BDBStorage(BaseConfig):
 
     def open(self):
@@ -261,12 +279,14 @@ class BDBStorage(BaseConfig):
             setattr(bconf, name, getattr(self.config, name))
         return storageclass(self.config.envdir, config=bconf)
 
+
 class BDBMinimalStorage(BDBStorage):
 
     def get_storageclass(self):
         import BDBStorage.BDBMinimalStorage
         return BDBStorage.BDBMinimalStorage.BDBMinimalStorage
 
+
 class BDBFullStorage(BDBStorage):
 
     def get_storageclass(self):

src/ZODB/conversionhack.py

@@ -14,21 +14,29 @@
 
 import persistent.mapping
 
+
 class fixer(object):
     def __of__(self, parent):
         def __setstate__(state, self=parent):
-            self._container=state
+            self._container = state
             del self.__setstate__
         return __setstate__
 
-fixer=fixer()
 
-class hack(object): pass
-hack=hack()
+fixer = fixer()
+
+
+class hack(object):
+    pass
+
+
+hack = hack()
+
 
 def __basicnew__():
-    r=persistent.mapping.PersistentMapping()
-    r.__setstate__=fixer
+    r = persistent.mapping.PersistentMapping()
+    r.__setstate__ = fixer
     return r
 
-hack.__basicnew__=__basicnew__
+
+hack.__basicnew__ = __basicnew__

src/ZODB/event.py

@@ -14,5 +14,5 @@
 try:
     from zope.event import notify
 except ImportError:
-    notify = lambda event: None
-
+    def notify(event):
+        return None

src/ZODB/fsIndex.py

@@ -55,17 +55,21 @@ from ZODB._compat import _protocol
 def num2str(n):
     return struct.pack(">Q", n)[2:]
 
+
 def str2num(s):
     return struct.unpack(">Q", b"\000\000" + s)[0]
 
+
 def prefix_plus_one(s):
     num = str2num(s)
     return num2str(num + 1)
 
+
 def prefix_minus_one(s):
     num = str2num(s)
     return num2str(num - 1)
 
+
 def ensure_bytes(s):
     # on Python 3 we might pickle bytes and unpickle unicode strings
     return s.encode('ascii') if not isinstance(s, bytes) else s
@@ -80,8 +84,8 @@ class fsIndex(object):
 
     def __getstate__(self):
         return dict(
-            state_version = 1,
-            _data = [(k, v.toString())
+            state_version=1,
+            _data=[(k, v.toString())
                    for (k, v) in six.iteritems(self._data)
                    ]
         )

src/ZODB/fsrecover.py

@@ -94,12 +94,15 @@ def die(mess='', show_docstring=False):
         print(__doc__ % sys.argv[0], file=sys.stderr)
     sys.exit(1)
 
+
 class ErrorFound(Exception):
     pass
 
+
 def error(mess, *args):
     raise ErrorFound(mess % args)
 
+
 def read_txn_header(f, pos, file_size, outp, ltid):
     # Read the transaction record
     f.seek(pos)
@@ -107,7 +110,7 @@ def read_txn_header(f, pos, file_size, outp, ltid):
     if len(h) < 23:
         raise EOFError
 
-    tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
+    tid, stl, status, ul, dl, el = unpack(">8s8scHHH", h)
     status = as_text(status)
     tl = u64(stl)
 
@@ -157,6 +160,7 @@ def read_txn_header(f, pos, file_size, outp, ltid):
 
     return pos, result, tid
 
+
 def truncate(f, pos, file_size, outp):
     """Copy data from pos to end of f to a .trNNN file."""
 
@@ -176,6 +180,7 @@ def truncate(f, pos, file_size, outp):
     f.seek(pos)
     tr.close()
 
+
 def copy(src, dst, n):
     while n:
         buf = src.read(8096)
@@ -186,6 +191,7 @@ def copy(src, dst, n):
         dst.write(buf)
         n -= len(buf)
 
+
 def scan(f, pos):
     """Return a potential transaction location following pos in f.
 
@@ -206,20 +212,21 @@ def scan(f, pos):
 
         s = 0
         while 1:
-            l = data.find(b".", s)
-            if l < 0:
+            l_ = data.find(b".", s)
+            if l_ < 0:
                 pos += len(data)
                 break
             # If we are less than 8 bytes from the end of the
             # string, we need to read more data.
-            s = l + 1
+            s = l_ + 1
             if s > len(data) - 8:
-                pos += l
+                pos += l_
                 break
             tl = u64(data[s:s+8])
             if tl < pos:
                 return pos + s + 8
 
+
 def iprogress(i):
     if i % 2:
         print(".", end=' ')
@@ -227,10 +234,12 @@ def iprogress(i):
         print((i/2) % 10, end=' ')
     sys.stdout.flush()
 
+
 def progress(p):
     for i in range(p):
         iprogress(i)
 
+
 def main():
     try:
         opts, args = getopt.getopt(sys.argv[1:], "fv:pP:")
@@ -256,6 +265,7 @@ def main():
 
     recover(inp, outp, verbose, partial, force, pack)
 
+
 def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
     print("Recovering", inp, "into", outp)
 
@@ -266,7 +276,7 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
     if f.read(4) != ZODB.FileStorage.packed_version:
         die("input is not a file storage")
 
-    f.seek(0,2)
+    f.seek(0, 2)
     file_size = f.tell()
 
     ofs = ZODB.FileStorage.FileStorage(outp, create=1)
@@ -332,11 +342,11 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
             for r in txn:
                 if verbose > 1:
                     if r.data is None:
-                        l = "bp"
+                        l_ = "bp"
                     else:
-                        l = len(r.data)
+                        l_ = len(r.data)
 
-                    print("%7d %s %s" % (u64(r.oid), l))
+                    print("%7d %s" % (u64(r.oid), l_))
                 ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,
                             txn)
                 nrec += 1
@@ -370,7 +380,6 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
                 prog1 = prog1 + 1
                 iprogress(prog1)
 
-
     bad = file_size - undone - ofs._pos
 
     print("\n%s bytes removed during recovery" % bad)
@@ -385,5 +394,6 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
     ofs.close()
     f.close()
 
+
 if __name__ == "__main__":
     main()

src/ZODB/fstools.py

@@ -100,6 +100,7 @@ class TxnHeader(object):
         tlen = u64(self._file.read(8))
         return TxnHeader(self._file, self._pos - (tlen + 8))
 
+
 class DataHeader(object):
     """Object representing a data record header.
 
@@ -138,6 +139,7 @@ class DataHeader(object):
             off += 8  # backpointer
         return off
 
+
 def prev_txn(f):
     """Return transaction located before current file position."""
     f.seek(-8, 1)

src/ZODB/historical_connections.rst

@@ -36,7 +36,7 @@ development continues on a "development" head.
 
 A database can be opened historically ``at`` or ``before`` a given transaction
 serial or datetime. Here's a simple example. It should work with any storage
-that supports ``loadAt`` or ``loadBefore``.
+that supports ``loadBeforeEx`` or ``loadBefore``.
 
 We'll begin our example with a fairly standard set up.  We
 
@@ -138,9 +138,10 @@ root.
     >>> historical_conn.root()['first']['count']
     0
 
-In fact, ``at`` arguments are translated into ``before`` values.
-When you look at a connection's ``before`` attribute, it is normalized into a
-``before`` serial, no matter what you pass into ``db.open``.
+In fact, ``at`` arguments are translated into ``before`` values because the
+underlying mechanism is a storage's loadBeforeEx method.  When you look at a
+connection's ``before`` attribute, it is normalized into a ``before`` serial,
+no matter what you pass into ``db.open``.
 
     >>> print(conn.before)
     None

src/ZODB/mvccadapter.py

@@ -10,7 +10,9 @@ also simplifies the implementation of the DB and Connection classes.
 import zope.interface
 
 from . import interfaces, serialize, POSException
-from .utils import p64, u64, z64, maxtid, Lock, loadAt, oid_repr, tid_repr
+from .utils import p64, u64, z64, maxtid, Lock, loadBeforeEx, oid_repr, \
+                   tid_repr
+
 
 class Base(object):
 
@@ -37,6 +39,7 @@ class Base(object):
     def __len__(self):
         return len(self._storage)
 
+
 class MVCCAdapter(Base):
 
     def __init__(self, storage):
@@ -63,6 +66,7 @@ class MVCCAdapter(Base):
             self._instances.remove(instance)
 
     closed = False
+
     def close(self):
         if not self.closed:
             self.closed = True
@@ -92,6 +96,7 @@ class MVCCAdapter(Base):
     def pack(self, pack_time, referencesf):
         return self._storage.pack(pack_time, referencesf)
 
+
 class MVCCAdapterInstance(Base):
 
     _copy_methods = Base._copy_methods + (
@@ -99,7 +104,7 @@ class MVCCAdapterInstance(Base):
         'checkCurrentSerialInTransaction', 'tpc_abort',
     )
 
-    _start = None # Transaction start time (before)
+    _start = None  # Transaction start time
     _ltid = b''   # Last storage transaction id
 
     def __init__(self, base):
@@ -107,7 +112,7 @@ class MVCCAdapterInstance(Base):
         Base.__init__(self, base._storage)
         self._lock = Lock()
         self._invalidations = set()
-        self._sync = getattr(self._storage, 'sync', lambda : None)
+        self._sync = getattr(self._storage, 'sync', lambda: None)
 
     def release(self):
         self._base._release(self)
@@ -151,16 +156,16 @@ class MVCCAdapterInstance(Base):
 
     def load(self, oid):
         assert self._start is not None
-        at = p64(u64(self._start)-1)
-        data, serial = loadAt(self._storage, oid, at)
+        data, serial = loadBeforeEx(self._storage, oid, self._start)
         if data is None:
             # raise POSKeyError if object does not exist at all
-            # TODO raise POSKeyError always and switch to raising ReadOnlyError only when
-            # actually detecting that load is being affected by simultaneous pack (see below).
+            # TODO raise POSKeyError always and switch to raising ReadOnlyError
+            # only when actually detecting that load is being affected by
+            # simultaneous pack (see below).
             if serial == z64:
-                # XXX second call to loadAt - it will become unneeded once we
-                # switch to raising POSKeyError.
-                _, serial_exists = loadAt(self._storage, oid, maxtid)
+                # XXX second call to loadBeforeEx - it will become unneeded
+                # once we switch to raising POSKeyError.
+                _, serial_exists = loadBeforeEx(self._storage, oid, maxtid)
                 if serial_exists == z64:
                     # object does not exist at all
                     raise POSException.POSKeyError(oid)
@@ -220,7 +225,7 @@ class MVCCAdapterInstance(Base):
             oid, serial, data, blobfilename, '', transaction)
         self._modified.add(oid)
 
-    def tpc_finish(self, transaction, func = lambda tid: None):
+    def tpc_finish(self, transaction, func=lambda tid: None):
         modified = self._modified
         self._modified = None
 
@@ -231,9 +236,11 @@ class MVCCAdapterInstance(Base):
 
         return self._storage.tpc_finish(transaction, invalidate_finish)
 
+
 def read_only_writer(self, *a, **kw):
     raise POSException.ReadOnlyError
 
+
 class HistoricalStorageAdapter(Base):
     """Adapt a storage to a historical storage
     """
@@ -272,8 +279,7 @@ class HistoricalStorageAdapter(Base):
     new_oid = pack = store = read_only_writer
 
     def load(self, oid, version=''):
-        at = p64(u64(self._before)-1)
-        data, serial = loadAt(self._storage, oid, at)
+        data, serial = loadBeforeEx(self._storage, oid, self._before)
         if data is None:
             raise POSException.POSKeyError(oid)
         return data, serial
@@ -309,7 +315,7 @@ class UndoAdapterInstance(Base):
         if result:
             self._undone.update(result)
 
-    def tpc_finish(self, transaction, func = lambda tid: None):
+    def tpc_finish(self, transaction, func=lambda tid: None):
 
         def invalidate_finish(tid):
             self._base._invalidate_finish(tid, self._undone, None)

src/ZODB/persistentclass.py

@@ -63,6 +63,7 @@ class _p_DataDescr(object):
     def __delete__(self, inst):
         raise AttributeError(self.__name__)
 
+
 class _p_oid_or_jar_Descr(_p_DataDescr):
     # Special descr for _p_oid and _p_jar that loads
     # state when set if both are set and _p_changed is None
@@ -79,10 +80,10 @@ class _p_oid_or_jar_Descr(_p_DataDescr):
         jar = get('_p_jar')
         if (jar is not None
                 and get('_p_oid') is not None
-            and get('_p_changed') is None
-            ):
+                and get('_p_changed') is None):
             jar.setstate(inst)
 
+
 class _p_ChangedDescr(object):
     # descriptor to handle special weird semantics of _p_changed
 
@@ -99,6 +100,7 @@ class _p_ChangedDescr(object):
     def __delete__(self, inst):
         inst._p_invalidate()
 
+
 class _p_MethodDescr(object):
     """Provide unassignable class attributes
     """
@@ -120,6 +122,7 @@ class _p_MethodDescr(object):
 
 special_class_descrs = '__dict__', '__weakref__'
 
+
 class PersistentMetaClass(type):
 
     _p_jar = _p_oid_or_jar_Descr('_p_jar')
@@ -148,7 +151,7 @@ class PersistentMetaClass(type):
             and
             (get('_p_oid') is not None)
             and
-            (get('_p_changed') == False)
+            (get('_p_changed') is False)
         ):
 
             self._p_changed = True
@@ -177,7 +180,6 @@ class PersistentMetaClass(type):
 
     _p_invalidate = _p_MethodDescr(_p_invalidate)
 
-
     def __getstate__(self):
         return (self.__bases__,
                 dict([(k, v) for (k, v) in self.__dict__.items()

src/ZODB/scripts/analyze.py

@@ -9,7 +9,6 @@ from ZODB.FileStorage import FileStorage
 from ZODB._compat import PersistentUnpickler, BytesIO
 
 
-
 class FakeError(Exception):
     def __init__(self, module, name):
         Exception.__init__(self)
@@ -41,9 +40,10 @@ class Report(object):
         self.FOIDS = 0
         self.FBYTES = 0
 
+
 def shorten(s, n):
-    l = len(s)
-    if l <= n:
+    length = len(s)
+    if length <= n:
         return s
     while len(s) + 3 > n:  # account for ...
         i = s.find(".")
@@ -52,9 +52,10 @@ def shorten(s, n):
             return s[-n:]
         else:
             s = s[i + 1:]
-            l = len(s)
+            length = len(s)
     return "..." + s
 
+
 def report(rep):
     print("Processed %d records in %d transactions" % (rep.OIDS, rep.TIDS))
     print("Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS))
@@ -76,8 +77,9 @@ def report(rep):
                       pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t]))
 
     print(fmt % ('='*46, '='*7, '='*9, '='*5, '='*7))
-    print("%46s %7d %9s %6s %6.2fk" % ('Total Transactions', rep.TIDS, ' ',
-        ' ', rep.DBYTES * 1.0 / rep.TIDS / 1024.0))
+    print("%46s %7d %9s %6s %6.2fk" % (
+        'Total Transactions', rep.TIDS, ' ', ' ',
+        rep.DBYTES * 1.0 / rep.TIDS / 1024.0))
     print(fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct,
                   rep.DBYTES * 1.0 / rep.OIDS))
 
@@ -89,6 +91,7 @@ def report(rep):
                       rep.FBYTES * 100.0 / rep.DBYTES,
                       rep.FBYTES * 1.0 / rep.FOIDS))
 
+
 def analyze(path):
     fs = FileStorage(path, read_only=1)
     fsi = fs.iterator()
@@ -97,11 +100,13 @@ def analyze(path):
         analyze_trans(report, txn)
     return report
 
+
 def analyze_trans(report, txn):
     report.TIDS += 1
     for rec in txn:
         analyze_rec(report, rec)
 
+
 def get_type(record):
     try:
         unpickled = FakeUnpickler(BytesIO(record.data)).load()
@@ -114,6 +119,7 @@ def get_type(record):
     else:
         return str(classinfo)
 
+
 def analyze_rec(report, record):
     oid = record.oid
     report.OIDS += 1
@@ -142,6 +148,7 @@ def analyze_rec(report, record):
     except Exception as err:
         print(err)
 
+
 if __name__ == "__main__":
     path = sys.argv[1]
     report(analyze(path))

src/ZODB/scripts/checkbtrees.py

@@ -19,6 +19,8 @@ oids_seen = {}
 
 # Append (obj, path) to L if and only if obj is a persistent object
 # and we haven't seen it before.
+
+
 def add_if_new_persistent(L, obj, path):
     global oids_seen
 
@@ -29,6 +31,7 @@ def add_if_new_persistent(L, obj, path):
             L.append((obj, path))
             oids_seen[oid] = 1
 
+
 def get_subobjects(obj):
     getattr(obj, '_', None)  # unghostify
     sub = []
@@ -55,19 +58,20 @@ def get_subobjects(obj):
     while 1:
         try:
             elt = obj[i]
-        except:
+        except:  # noqa: E722 do not use bare 'except'
             break
         sub.append(("[%d]" % i, elt))
         i += 1
 
     return sub
 
+
 def main(fname=None):
     if fname is None:
         import sys
         try:
             fname, = sys.argv[1:]
-        except:
+        except:  # noqa: E722 do not use bare 'except'
             print(__doc__)
             sys.exit(2)
 
@@ -116,5 +120,6 @@ def main(fname=None):
 
     print("total", len(fs._index), "found", found)
 
+
 if __name__ == "__main__":
     main()

src/ZODB/scripts/fsoids.py

@@ -43,9 +43,11 @@ import sys
 
 from ZODB.FileStorage.fsoids import Tracer
 
+
 def usage():
     print(__doc__)
 
+
 def main():
     import getopt
 
@@ -75,5 +77,6 @@ def main():
     c.run()
     c.report()
 
+
 if __name__ == "__main__":
     main()

src/ZODB/scripts/fsrefs.py

@@ -66,13 +66,16 @@ import traceback
 
 from ZODB.FileStorage import FileStorage
 from ZODB.TimeStamp import TimeStamp
-from ZODB.utils import u64, oid_repr, get_pickle_metadata, load_current
+from ZODB.utils import u64, p64, oid_repr, get_pickle_metadata, load_current
 from ZODB.serialize import get_refs
 from ZODB.POSException import POSKeyError
+from BTrees.QQBTree import QQBTree
 
 # There's a problem with oid.  'data' is its pickle, and 'serial' its
 # serial number.  'missing' is a list of (oid, class, reason) triples,
 # explaining what the problem(s) is(are).
+
+
 def report(oid, data, serial, missing):
     from_mod, from_class = get_pickle_metadata(data)
     if len(missing) > 1:
@@ -91,6 +94,7 @@ def report(oid, data, serial, missing):
         print("\toid %s %s: %r" % (oid_repr(oid), reason, description))
     print()
 
+
 def main(path=None):
     verbose = 0
     if path is None:
@@ -104,7 +108,6 @@ def main(path=None):
 
         path, = args
 
-
     fs = FileStorage(path, read_only=1)
 
     # Set of oids in the index that failed to load due to POSKeyError.
@@ -118,21 +121,36 @@ def main(path=None):
     # This does not include oids in undone.
     noload = {}
 
-    for oid in fs._index.keys():
+    # build {pos -> oid} index that is reverse to {oid -> pos} fs._index
+    # we'll need this to iterate objects in order of ascending file position to
+    # optimize disk IO.
+    pos2oid = QQBTree()  # pos -> u64(oid)
+    for oid, pos in fs._index.iteritems():
+        pos2oid[pos] = u64(oid)
+
+    # pass 1: load all objects listed in the index and remember those objects
+    # that are deleted or load with an error. Iterate objects in order of
+    # ascending file position to optimize disk IO.
+    for oid64 in pos2oid.itervalues():
+        oid = p64(oid64)
         try:
             data, serial = load_current(fs, oid)
         except (KeyboardInterrupt, SystemExit):
             raise
         except POSKeyError:
             undone[oid] = 1
-        except:
+        except:  # noqa: E722 do not use bare 'except'
             if verbose:
                 traceback.print_exc()
             noload[oid] = 1
 
+    # pass 2: go through all objects again and verify that their references do
+    # not point to problematic object set. Iterate objects in order of
+    # ascending file position to optimize disk IO.
     inactive = noload.copy()
     inactive.update(undone)
-    for oid in fs._index.keys():
+    for oid64 in pos2oid.itervalues():
+        oid = p64(oid64)
         if oid in inactive:
             continue
         data, serial = load_current(fs, oid)
@@ -150,5 +168,6 @@ def main(path=None):
         if missing:
             report(oid, data, serial, missing)
 
+
 if __name__ == "__main__":
     main()

src/ZODB/scripts/fsstats.py

@@ -6,8 +6,9 @@ import sys
 import six
 from six.moves import filter
 
-rx_txn = re.compile("tid=([0-9a-f]+).*size=(\d+)")
-rx_data = re.compile("oid=([0-9a-f]+) class=(\S+) size=(\d+)")
+rx_txn = re.compile(r"tid=([0-9a-f]+).*size=(\d+)")
+rx_data = re.compile(r"oid=([0-9a-f]+) size=(\d+) class=(\S+)")
+
 
 def sort_byhsize(seq, reverse=False):
     L = [(v.size(), k, v) for k, v in seq]
@@ -16,6 +17,7 @@ def sort_byhsize(seq, reverse=False):
         L.reverse()
     return [(k, v) for n, k, v in L]
 
+
 class Histogram(dict):
 
     def add(self, size):
@@ -31,8 +33,7 @@ class Histogram(dict):
     def median(self):
         # close enough?
         n = self.size() / 2
-        L = self.keys()
-        L.sort()
+        L = sorted(self.keys())
         L.reverse()
         while 1:
             k = L.pop()
@@ -50,11 +51,14 @@ class Histogram(dict):
         return mode
 
     def make_bins(self, binsize):
+        try:
             maxkey = max(six.iterkeys(self))
+        except ValueError:
+            maxkey = 0
         self.binsize = binsize
-        self.bins = [0] * (1 + maxkey / binsize)
+        self.bins = [0] * (1 + maxkey // binsize)
         for k, v in six.iteritems(self):
-            b = k / binsize
+            b = k // binsize
             self.bins[b] += v
 
     def report(self, name, binsize=50, usebins=False, gaps=True, skip=True):
@@ -88,9 +92,10 @@ class Histogram(dict):
             cum += n
             pc = 100 * cum / tot
             print("%6d %6d %3d%% %3d%% %s" % (
-                i * binsize, n, p, pc, "*" * (n / dot)))
+                i * binsize, n, p, pc, "*" * (n // dot)))
         print()
 
+
 def class_detail(class_size):
     # summary of classes
     fmt = "%5s %6s %6s %6s   %-50.50s"
@@ -104,10 +109,11 @@ def class_detail(class_size):
     # per class details
     for klass, h in sort_byhsize(six.iteritems(class_size), reverse=True):
         h.make_bins(50)
-        if len(filter(None, h.bins)) == 1:
+        if len(tuple(filter(None, h.bins))) == 1:
             continue
         h.report("Object size for %s" % klass, usebins=True)
 
+
 def revision_detail(lifetimes, classes):
     # Report per-class details for any object modified more than once
     for name, oids in six.iteritems(classes):
@@ -122,6 +128,7 @@ def revision_detail(lifetimes, classes):
         if keep:
             h.report("Number of revisions for %s" % name, binsize=10)
 
+
 def main(path=None):
     if path is None:
         path = sys.argv[1]
@@ -138,7 +145,7 @@ def main(path=None):
     objects = 0
     tid = None
 
-    f = open(path, "rb")
+    f = open(path, "r")
     for i, line in enumerate(f):
         if MAX and i > MAX:
             break
@@ -146,7 +153,7 @@ def main(path=None):
             m = rx_data.search(line)
             if not m:
                 continue
-            oid, klass, size = m.groups()
+            oid, size, klass = m.groups()
             size = int(size)
 
             obj_size.add(size)
@@ -178,6 +185,8 @@ def main(path=None):
             objects = 0
 
             txn_bytes.add(size)
+    if objects:
+        txn_objects.add(objects)
     f.close()
 
     print("Summary: %d txns, %d objects, %d revisions" % (
@@ -199,5 +208,6 @@ def main(path=None):
 
     class_detail(class_size)
 
+
 if __name__ == "__main__":
     main()

src/ZODB/scripts/fstail.py

@@ -25,6 +25,7 @@ try:
 except ImportError:
     from sha import sha as sha1
 
+
 def main(path, ntxn):
     with open(path, "rb") as f:
         f.seek(0, 2)
@@ -32,7 +33,6 @@ def main(path, ntxn):
         i = ntxn
         while th and i > 0:
             hash = sha1(th.get_raw_data()).digest()
-            l = len(str(th.get_timestamp())) + 1
             th.read_meta()
             print("%s: hash=%s" % (th.get_timestamp(),
                                    binascii.hexlify(hash).decode()))
@@ -42,6 +42,7 @@ def main(path, ntxn):
             th = th.prev_txn()
             i -= 1
 
+
 def Main():
     ntxn = 10
     opts, args = getopt.getopt(sys.argv[1:], "n:")
@@ -51,5 +52,6 @@ def Main():
             ntxn = int(v)
     main(path, ntxn)
 
+
 if __name__ == "__main__":
     Main()

src/ZODB/scripts/fstest.py

@@ -41,13 +41,16 @@ import struct
 import sys
 from ZODB._compat import FILESTORAGE_MAGIC
 
+
 class FormatError(ValueError):
     """There is a problem with the format of the FileStorage."""
 
+
 class Status(object):
     checkpoint = b'c'
     undone = b'u'
 
+
 packed_version = FILESTORAGE_MAGIC
 
 TREC_HDR_LEN = 23
@@ -55,6 +58,7 @@ DREC_HDR_LEN = 42
 
 VERBOSE = 0
 
+
 def hexify(s):
     r"""Format an 8-bit string as hex
 
@@ -64,17 +68,20 @@ def hexify(s):
     """
     return '0x' + binascii.hexlify(s).decode()
 
+
 def chatter(msg, level=1):
     if VERBOSE >= level:
         sys.stdout.write(msg)
 
+
 def U64(v):
     """Unpack an 8-byte string as a 64-bit long"""
-    h, l = struct.unpack(">II", v)
+    h, l_ = struct.unpack(">II", v)
     if h:
-        return (h << 32) + l
+        return (h << 32) + l_
     else:
-        return l
+        return l_
+
 
 def check(path):
     with open(path, 'rb') as file:
@@ -106,7 +113,7 @@ def check_trec(path, file, pos, ltid, file_size):
     used for generating error messages.
     """
 
-    h = file.read(TREC_HDR_LEN) #XXX must be bytes under Py3k
+    h = file.read(TREC_HDR_LEN)  # XXX must be bytes under Py3k
     if not h:
         return None, None
     if len(h) != TREC_HDR_LEN:
@@ -162,6 +169,7 @@ def check_trec(path, file, pos, ltid, file_size):
     pos = tend + 8
     return pos, tid
 
+
 def check_drec(path, file, pos, tpos, tid):
     """Check a data record for the current transaction record"""
 
@@ -170,7 +178,7 @@ def check_drec(path, file, pos, tpos, tid):
         raise FormatError("%s truncated at %s" % (path, pos))
     oid, serial, _prev, _tloc, vlen, _plen = (
         struct.unpack(">8s8s8s8sH8s", h))
-    prev = U64(_prev)
+    U64(_prev)
     tloc = U64(_tloc)
     plen = U64(_plen)
     dlen = DREC_HDR_LEN + (plen or 8)
@@ -178,7 +186,7 @@ def check_drec(path, file, pos, tpos, tid):
     if vlen:
         dlen = dlen + 16 + vlen
         file.seek(8, 1)
-        pv = U64(file.read(8))
+        U64(file.read(8))
         file.seek(vlen, 1)  # skip the version data
 
     if tloc != tpos:
@@ -195,9 +203,11 @@ def check_drec(path, file, pos, tpos, tid):
 
     return pos, oid
 
+
 def usage():
     sys.exit(__doc__)
 
+
 def main(args=None):
     if args is None:
         args = sys.argv[1:]
@@ -221,5 +231,6 @@ def main(args=None):
 
     chatter("no errors detected")
 
+
 if __name__ == "__main__":
     main()

src/ZODB/scripts/manual_tests/testfstest.py

@@ -6,12 +6,12 @@ Note:  To run this test script fstest.py must be on your PYTHONPATH.
 from cStringIO import StringIO
 import re
 import struct
-import unittest
 import ZODB.tests.util
 
 import fstest
 from fstest import FormatError, U64
 
+
 class TestCorruptedFS(ZODB.tests.util.TestCase):
 
     f = open('test-checker.fs', 'rb')
@@ -117,7 +117,7 @@ class TestCorruptedFS(ZODB.tests.util.TestCase):
         self._file.write(data)
         buf = self._datafs.read(tl - 8)
         self._file.write(buf[0])
-        assert tl <= 1<<16, "can't use this transaction for this test"
+        assert tl <= 1 << 16, "can't use this transaction for this test"
         self._file.write("\777\777")
         self._file.write(buf[3:])
         self.detectsError("invalid transaction header")
@@ -172,6 +172,3 @@ class TestCorruptedFS(ZODB.tests.util.TestCase):
         self._file.write("\000" * 4 + "\077" + "\000" * 3)
         self._file.write(data[32:])
         self.detectsError("record exceeds transaction")
-
-if __name__ == "__main__":
-    unittest.main()

src/ZODB/scripts/migrate.py

@@ -130,7 +130,7 @@ def main():
         elif opt in ('-v', '--verbose'):
             options.verbose += 1
         elif opt in ('-T', '--storage_types'):
-            print_types()
+            print('Unknown option.')
             sys.exit(0)
         elif opt in ('-S', '--stype'):
             options.stype = arg
@@ -247,16 +247,16 @@ def doit(srcdb, dstdb, options):
             t = TimeStamp(tid)
             if t <= ts:
                 if ok:
-                    print((
-                        'Time stamps are out of order %s, %s' % (ts, t)), file=sys.stderr)
+                    print('Time stamps are out of order %s, %s' % (ts, t),
+                          file=sys.stderr)
                     ok = False
                     ts = t.laterThan(ts)
                     tid = ts.raw()
                 else:
                     ts = t
                     if not ok:
-                        print((
-                            'Time stamps are back in order %s' % t), file=sys.stderr)
+                        print('Time stamps are back in order %s' % t,
+                              file=sys.stderr)
                         ok = True
         if verbose > 1:
             print(ts)

src/ZODB/scripts/migrateblobs.py

@@ -23,17 +23,17 @@ from ZODB.blob import FilesystemHelper
 from ZODB.utils import oid_repr
 
 
-def link_or_copy(f1, f2):
-    try:
-        os.link(f1, f2)
-    except OSError:
-        shutil.copy(f1, f2)
-
 # Check if we actually have link
 try:
     os.link
 except AttributeError:
     link_or_copy = shutil.copy
+else:
+    def link_or_copy(f1, f2):
+        try:
+            os.link(f1, f2)
+        except OSError:
+            shutil.copy(f1, f2)
 
 
 def migrate(source, dest, layout):