Add documentation for the unsigned modules.

And some other small tweaks to the documentation to make cross-references work and things generally look good.

BTrees/BTreeModuleTemplate.c

@@ -509,30 +509,30 @@ static char *search_keywords[] = {"min", "max",
 
 static struct PyMethodDef module_methods[] = {
   {"difference", (PyCFunction) difference_m,    METH_VARARGS,
-   "difference(o1, o2) -- "
+   "difference(o1, o2)\n"
    "compute the difference between o1 and o2"
   },
   {"union", (PyCFunction) union_m,      METH_VARARGS,
-   "union(o1, o2) -- compute the union of o1 and o2\n"
+   "union(o1, o2)\ncompute the union of o1 and o2\n"
   },
   {"intersection", (PyCFunction) intersection_m,        METH_VARARGS,
-   "intersection(o1, o2) -- "
+   "intersection(o1, o2)\n"
    "compute the intersection of o1 and o2"
   },
 #ifdef MERGE
   {"weightedUnion", (PyCFunction) wunion_m,     METH_VARARGS,
-   "weightedUnion(o1, o2 [, w1, w2]) -- compute the union of o1 and o2\n"
+   "weightedUnion(o1, o2 [, w1, w2])\ncompute the union of o1 and o2\n"
    "\nw1 and w2 are weights."
   },
   {"weightedIntersection", (PyCFunction) wintersection_m,       METH_VARARGS,
-   "weightedIntersection(o1, o2 [, w1, w2]) -- "
+   "weightedIntersection(o1, o2 [, w1, w2])\n"
    "compute the intersection of o1 and o2\n"
    "\nw1 and w2 are weights."
   },
 #endif
 #ifdef MULTI_INT_UNION
   {"multiunion", (PyCFunction) multiunion_m, METH_VARARGS,
-   "multiunion(seq) -- compute union of a sequence of integer sets.\n"
+   "multiunion(seq)\ncompute union of a sequence of integer sets.\n"
    "\n"
    "Each element of seq must be an integer set, or convertible to one\n"
    "via the set iteration protocol.  The union returned is an IISet."

BTrees/SetTemplate.c

@@ -37,7 +37,7 @@ _Set_update(Bucket *self, PyObject *seq)
 {
     int n=0, ind=0;
     PyObject *iter, *v;
- 
+
     iter = PyObject_GetIter(seq);
     if (iter == NULL)
         return -1;
@@ -177,49 +177,49 @@ set_setstate(Bucket *self, PyObject *args)
 
 static struct PyMethodDef Set_methods[] = {
     {"__getstate__", (PyCFunction) bucket_getstate, METH_VARARGS,
-     "__getstate__() -- Return the picklable state of the object"},
+     "__getstate__()\nReturn the picklable state of the object"},
 
     {"__setstate__", (PyCFunction) set_setstate, METH_VARARGS,
-     "__setstate__() -- Set the state of the object"},
+     "__setstate__()\nSet the state of the object"},
 
     {"keys", (PyCFunction) bucket_keys, METH_VARARGS | METH_KEYWORDS,
-     "keys() -- Return the keys"},
+     "keys()\nReturn the keys"},
 
     {"has_key", (PyCFunction) bucket_has_key, METH_O,
-     "has_key(key) -- Test whether the bucket contains the given key"},
+     "has_key(key)\nTest whether the bucket contains the given key"},
 
     {"clear", (PyCFunction) bucket_clear, METH_VARARGS,
-     "clear() -- Remove all of the items from the bucket"},
+     "clear()\nRemove all of the items from the bucket"},
 
     {"maxKey", (PyCFunction) Bucket_maxKey, METH_VARARGS,
-     "maxKey([key]) -- Find the maximum key\n\n"
+     "maxKey([key])\nFind the maximum key\n\n"
      "If an argument is given, find the maximum <= the argument"},
 
     {"minKey", (PyCFunction) Bucket_minKey, METH_VARARGS,
-     "minKey([key]) -- Find the minimum key\n\n"
+     "minKey([key])\nFind the minimum key\n\n"
      "If an argument is given, find the minimum >= the argument"},
 
 #ifdef PERSISTENT
     {"_p_resolveConflict",
      (PyCFunction) bucket__p_resolveConflict, METH_VARARGS,
-     "_p_resolveConflict() -- Reinitialize from a newly created copy"},
+     "_p_resolveConflict()\nReinitialize from a newly created copy"},
 
     {"_p_deactivate",
      (PyCFunction) bucket__p_deactivate, METH_VARARGS | METH_KEYWORDS,
-     "_p_deactivate() -- Reinitialize from a newly created copy"},
+     "_p_deactivate()\nReinitialize from a newly created copy"},
 #endif
 
     {"add", (PyCFunction)Set_insert, METH_VARARGS,
-     "add(id) -- Add a key to the set"},
+     "add(id)\nAdd a key to the set"},
 
     {"insert", (PyCFunction)Set_insert, METH_VARARGS,
-     "insert(id) -- Add a key to the set"},
+     "insert(id)\nAdd a key to the set"},
 
     {"update", (PyCFunction)Set_update,    METH_VARARGS,
-     "update(seq) -- Add the items from the given sequence to the set"},
+     "update(seq)\nAdd the items from the given sequence to the set"},
 
     {"remove",    (PyCFunction)Set_remove,    METH_VARARGS,
-     "remove(id) -- Remove an id from the set"},
+     "remove(id)\nRemove an id from the set"},
 
     {NULL, NULL}        /* sentinel */
 };

BTrees/__init__.py

@@ -19,8 +19,23 @@ import BTrees.Interfaces
 @zope.interface.implementer(BTrees.Interfaces.IBTreeFamily)
 class _Family(object):
     from BTrees import OOBTree as OO
+    _BITSIZE = 0
+    minint = maxint = maxuint = 0
+
+    def __str__(self):
+        return (
+            "BTree family using {} bits. "
+            "Supports signed integer values from {:,} to {:,} "
+            "and maximum unsigned integer value {:,}."
+        ).format(self._BITSIZE, self.minint, self.maxint, self.maxuint)
+
+    def __repr__(self):
+        return "<%s>" % (
+            self
+        )
 
 class _Family32(_Family):
+    _BITSIZE = 32
     from BTrees import OIBTree as OI
     from BTrees import OUBTree as OU
 
@@ -42,6 +57,7 @@ class _Family32(_Family):
         return _family32, ()
 
 class _Family64(_Family):
+    _BITSIZE = 64
     from BTrees import OLBTree as OI
     from BTrees import OQBTree as OU
 
@@ -70,8 +86,10 @@ def _family64():
     return family64
 _family64.__safe_for_unpickling__ = True
 
-
+#: 32-bit BTree family.
 family32 = _Family32()
+
+#: 64-bit BTree family.
 family64 = _Family64()
 
 for _family in family32, family64:

CHANGES.rst

@@ -17,8 +17,6 @@
   consistent between Python 2 and Python 3 and between 32-bit and
   64-bit variants.
 
-- Fix the value for ``BTrees.OIBTree`` when using the pure Python
-  implementation (PyPy and when ``PURE_PYTHON`` is in the environment).
 
 4.6.1 (2019-11-07)
 ==================

doc-requirements.txt

+sphinx >= 1.8
 pip>=8.1.2
 setuptools>=23.0.0
 repoze.sphinx.autointerface

docs/_static/custom.css

+/*
+   sphinx_rtd_theme pulls from wyrm, which applies white-space: nowrap to tables.
+   https://github.com/snide/wyrm/blob/fd41b56978f009e8c33cb26f384dd0dfaf430a7d/sass/wyrm_core/_table.sass#L144
+   That makes it hard to have a table with more than three columns without
+   pointless scrolling.
+*/
+.wrapped td, .wrapped th {
+    white-space: normal !important;
+}

docs/api.rst

@@ -6,108 +6,151 @@
 Protocol APIs
 =============
 
-.. automodule:: BTrees.Interfaces
+.. module:: BTrees.Interfaces
+
+.. autointerface:: ICollection
+.. autointerface:: IReadSequence
+.. autointerface:: IKeyed
+.. autointerface:: ISetMutable
+.. autointerface:: ISized
+.. autointerface:: IKeySequence
+.. autointerface:: IMinimalDictionary
+.. autointerface:: IDictionaryIsh
+.. autointerface:: IMerge
+.. autointerface:: IIMerge
+.. autointerface:: IMergeIntegerKey
+
+BTree Family APIs
+-----------------
+.. autointerface:: ISet
+.. autointerface:: ITreeSet
+.. autointerface:: IBTree
+.. autointerface:: IBTreeFamily
+
+There are two families defined:
+
+.. autodata:: BTrees.family32
+.. autodata:: BTrees.family64
+
+Module APIs
+-----------
+.. autointerface:: IBTreeModule
+.. autointerface:: IObjectObjectBTreeModule
+.. autointerface:: IIntegerObjectBTreeModule
+.. autointerface:: IObjectIntegerBTreeModule
+.. autointerface:: IIntegerIntegerBTreeModule
+.. autointerface:: IIntegerFloatBTreeModule
 
-  .. autointerface:: ICollection
-     :members:
-     :member-order: bysource
 
-  .. autointerface:: IReadSequence
-     :members:
-     :member-order: bysource
+Utilities
+=========
 
-  .. autointerface:: IKeyed
-     :members:
-     :member-order: bysource
+.. automodule:: BTrees.Length
 
-  .. autointerface:: ISetMutable
-     :members:
-     :member-order: bysource
+.. automodule:: BTrees.check
 
-  .. autointerface:: ISized
-     :members:
-     :member-order: bysource
 
-  .. autointerface:: IKeySequence
-     :members:
-     :member-order: bysource
+BTree Data Structure Variants
+=============================
 
-  .. autointerface:: IMinimalDictionary
-     :members:
-     :member-order: bysource
+Integer Keys
+------------
 
-  .. autointerface:: IDictionaryIsh
-     :members:
-     :member-order: bysource
+Float Values
+~~~~~~~~~~~~
+.. automodule:: BTrees.IFBTree
 
-  .. autointerface:: IMerge
-     :members:
-     :member-order: bysource
+Integer Values
+~~~~~~~~~~~~~~
+.. automodule:: BTrees.IIBTree
 
-  .. autointerface:: IIMerge
-     :members:
-     :member-order: bysource
+Object Values
+~~~~~~~~~~~~~
+.. automodule:: BTrees.IOBTree
 
-  .. autointerface:: IMergeIntegerKey
-     :members:
-     :member-order: bysource
+Unsigned Integer Values
+~~~~~~~~~~~~~~~~~~~~~~~
+.. automodule:: BTrees.IUBTree
 
-  BTree Family APIs
-  -----------------
+Long Integer Keys
+-----------------
 
-  .. autointerface:: ISet
-     :members:
-     :member-order: bysource
+Float Values
+~~~~~~~~~~~~
+.. automodule:: BTrees.LFBTree
 
-  .. autointerface:: ITreeSet
-     :members:
-     :member-order: bysource
+Long Integer Values
+~~~~~~~~~~~~~~~~~~~
+.. automodule:: BTrees.LLBTree
 
-  .. autointerface:: IBTree
-     :members:
-     :member-order: bysource
 
-  .. autointerface:: IBTreeFamily
-     :members:
-     :member-order: bysource
+Object Values
+~~~~~~~~~~~~~
+.. automodule:: BTrees.LOBTree
 
-  Module APIs
-  -----------
+Quad Unsigned Integer Values
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. automodule:: BTrees.LQBTree
 
-  .. autointerface:: IBTreeModule
-     :members:
-     :member-order: bysource
 
-  .. autointerface:: IObjectObjectBTreeModule
-     :members:
-     :member-order: bysource
+Object Keys
+-----------
 
-  .. autointerface:: IIntegerObjectBTreeModule
-     :members:
-     :member-order: bysource
+Integer Values
+~~~~~~~~~~~~~~
+.. automodule:: BTrees.OIBTree
 
-  .. autointerface:: IObjectIntegerBTreeModule
-     :members:
-     :member-order: bysource
+Long Integer Values
+~~~~~~~~~~~~~~~~~~~
+.. automodule:: BTrees.OLBTree
 
-  .. autointerface:: IIntegerIntegerBTreeModule
-     :members:
-     :member-order: bysource
+Object Values
+~~~~~~~~~~~~~
+.. automodule:: BTrees.OOBTree
 
-  .. autointerface:: IIntegerFloatBTreeModule
-     :members:
-     :member-order: bysource
+Quad Unsigned Integer Values
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. automodule:: BTrees.OQBTree
 
-Utilities
-=========
+Unsigned Integer Values
+~~~~~~~~~~~~~~~~~~~~~~~
+.. automodule:: BTrees.OUBTree
 
-.. automodule:: BTrees.Length
 
-  .. autoclass:: Length
-     :members:
+Quad Unsigned Integer Keys
+--------------------------
 
-     .. automethod:: __call__
+Float Values
+~~~~~~~~~~~~
+.. automodule:: BTrees.QFBTree
 
+Long Integer Values
+~~~~~~~~~~~~~~~~~~~
+.. automodule:: BTrees.QLBTree
 
-.. automodule:: BTrees.check
-    :members:
+Object Values
+~~~~~~~~~~~~~
+.. automodule:: BTrees.QOBTree
+
+Quad Unsigned Integer Values
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. automodule:: BTrees.QQBTree
+
+Unsigned Integer Keys
+---------------------
+
+Float Values
+~~~~~~~~~~~~
+.. automodule:: BTrees.UFBTree
+
+Integer Values
+~~~~~~~~~~~~~~
+.. automodule:: BTrees.UIBTree
+
+Object Values
+~~~~~~~~~~~~~
+.. automodule:: BTrees.UOBTree
+
+Unsigned Integer Values
+~~~~~~~~~~~~~~~~~~~~~~~
+.. automodule:: BTrees.UUBTree

docs/conf.py

@@ -12,6 +12,14 @@
 # serve to show the default.
 
 #import sys, os
+import sys
+import os
+import pkg_resources
+# We actually have slightly better results by documenting the
+# C objects right now.
+# os.environ['PURE_PYTHON'] = '1'
+sys.path.append(os.path.abspath('../'))
+rqmt = pkg_resources.require('BTrees')[0]
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -21,7 +29,7 @@
 # -- General configuration -----------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+needs_sphinx = '1.8'
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
@@ -34,6 +42,18 @@ extensions = [
     'repoze.sphinx.autointerface',
 ]
 
+# Sphinx 1.8+ prefers this to `autodoc_default_flags`. It's documented that
+# either True or None mean the same thing as just setting the flag, but
+# only None works in 1.8 (True works in 2.0)
+autodoc_default_options = {
+    'members': None,
+    'show-inheritance': None,
+    'special-members': None,
+    'undoc-members': None,
+}
+autodoc_member_order = 'groupwise'
+autoclass_content = 'both'
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -55,9 +75,9 @@ copyright = u'2012, Zope Foundation Contributors'
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '%s.%s' % tuple(map(int, rqmt.version.split('.')[:2]))
 # The full version, including alpha/beta/rc tags.
-release = '4.0.2dev'
+release = rqmt.version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -128,6 +148,9 @@ html_theme = 'sphinx_rtd_theme'
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
+html_css_files = [
+    'custom.css'
+]
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.

docs/overview.rst

@@ -14,40 +14,98 @@ important consequences -- see below).  Nodes are then only unpickled and
 brought into memory as they're accessed, so the entire tree doesn't have to
 occupy memory (unless you really are touching every single key).
 
-The BTrees package provides a large collection of related data structures.
-There are variants of the data structures specialized to integers, which
-are faster and use less memory.  There are five modules that handle the
-different variants.  The first two letters of the module name specify the
-types of the keys and values in mappings -- O for any object, I for 32-bit
-signed integer, and (new in ZODB 3.4) F for 32-bit C float.  For example,
-the :mod:`BTrees.IOBTree` module provides a mapping with integer keys and
-arbitrary objects as values.
-
-The four data structures provide by each module are a BTree, a Bucket, a
-TreeSet, and a Set.  The BTree and Bucket types are mappings and support
-all the usual mapping methods, e.g. :func:`~BTrees.Interfaces.ISetMutable.update` and :func:`~BTrees.Interfaces.IKeyed.keys`.  The
-TreeSet and Set types are similar to mappings but they have no values; they
-support the methods that make sense for a mapping with no keys, e.g.
-:func:`~BTrees.Interfaces.IKeyed.keys` but not :func:`~BTrees.Interfaces.IMinimalDictionary.items`.  The Bucket and Set types are the
-individual building blocks for BTrees and TreeSets, respectively.  A Bucket
-or Set can be used when you are sure that it will have few elements.  If
-the data structure will grow large, you should use a BTree or TreeSet. Like
-Python lists, Buckets and Sets are allocated in one contiguous piece, and
-insertions and deletions can take time proportional to the number of
-existing elements.  Also like Python lists, a Bucket or Set is a single
-object, and is pickled and unpickled in its entirety.  BTrees and TreeSets
-are multi-level tree structures with much better (logarithmic) worst- case
-time bounds, and the tree structure is built out of multiple objects, which
-ZODB can load individually as needed.
-
-The five modules are named :mod:`~BTrees.OOBTree`, :mod:`~BTrees.IOBTree`, :mod:`~BTrees.OIBTree`,
-:mod:`~BTrees.IIBTree`, and (new in ZODB 3.4) :mod:`~BTrees.IFBTree`.  The two letter
-prefixes are repeated in the data types names.  The :mod:`~BTrees.OOBTree`
-module defines the following types: :class:`~BTrees.OOBTree.OOBTree`, :class:`~BTrees.OOBTree.OOBucket`,
-:class:`~BTrees.OOBTree.OOSet`, and :class:`~BTrees.OOBTree.OOTreeSet`. Similarly, the other four modules
+Related Data Structures
+=======================
+
+The BTrees package provides a large collection of related data
+structures. The most general data structures store arbitrary ordered_
+objects. There are variants of the data structures specialized to
+numbers, which are faster and more memory efficient than those dealing
+with objects. There are several modules that handle the different
+variants. The first two letters of the module name specify the types
+of the keys and values in mappings. For example, the
+:mod:`BTrees.IOBTree` module provides a mapping with 32-bit integer
+keys and arbitrary objects as values.
+
+.. list-table:: Data Type Notation
+   :widths: auto
+   :class: wrapped
+   :header-rows: 1
+
+   * - Letter
+     - Mnemonic Device
+     - Data Type
+     - Notes
+   * - O
+     - "Object"
+     - Any sortable Python object
+     -
+   * - I
+     - "Integer"
+     - 32-bit signed integer
+     - Values from ``-2**31 - 1`` through ``2**31 - 1`` (about plus or
+       minus two billion)
+   * - L
+     - "Long integer"
+     - 64-bit signed integer
+     - Values from ``-2**63 - 1`` through
+       ``2**63 - 1`` (about plus or minus nine quintillion)
+   * - F
+     - "Float"
+     - 32-bit C-language ``float``
+     - New in ZODB 3.4
+   * - U
+     - "Unsigned"
+     - 32-bit unsigned integer
+     - (New in BTrees 4.7.0) Values from 0 through ``2**32`` (about four billion)
+   * - Q
+     - "Quad"
+     - 64-bit unsigned integer
+     - Values from 0 through ``2**64`` (about 18 quintillion) (New in BTrees 4.7.0)
+
+
+The four data structures provide by each module are a BTree, a Bucket,
+a TreeSet, and a Set. The BTree and Bucket types are mappings and
+support all the usual mapping methods, e.g.
+:func:`~BTrees.Interfaces.ISetMutable.update` and
+:func:`~BTrees.Interfaces.IKeyed.keys`. The TreeSet and Set types are
+similar to mappings but they have no values; they support the methods
+that make sense for a mapping with no keys, e.g.
+:func:`~BTrees.Interfaces.IKeyed.keys` but not
+:func:`~BTrees.Interfaces.IMinimalDictionary.items`. The Bucket and
+Set types are the individual building blocks for BTrees and TreeSets,
+respectively. A Bucket or Set can be used when you are sure that it
+will have few elements. If the data structure will grow large, you
+should use a BTree or TreeSet. Like Python lists, Buckets and Sets are
+allocated in one contiguous piece, and insertions and deletions can
+take time proportional to the number of existing elements. Also like
+Python lists, a Bucket or Set is a single object, and is pickled and
+unpickled in its entirety. BTrees and TreeSets are multi-level tree
+structures with much better (logarithmic) worst- case time bounds, and
+the tree structure is built out of multiple objects, which ZODB can
+load individually as needed.
+
+The two letter prefixes are repeated in the data types names. For
+example, the
+:mod:`BTrees.OOBTree` module defines the following types:
+:class:`BTrees.OOBTree.OOBTree`, :class:`BTrees.OOBTree.OOBucket`,
+:class:`BTrees.OOBTree.OOSet`, and
+:class:`BTrees.OOBTree.OOTreeSet`. Similarly, the other modules
 each define their own variants of those four types.
 
-The :func:`keys`, :func:`values`, and :func:`items` methods on BTree and
+For convenience, BTrees groups the most closely related data
+structures together into a "family" (defined by
+:class:`BTrees.Interfaces.IBTreeFamily`). :obj:`BTrees.family32`
+groups 32-bit data structures, while :obj:`BTrees.family64` contains
+64-bit data structures. It is a common practice for code that creates
+BTrees to be parameterized on the family so that the caller can choose
+the desired characteristics.
+
+
+Behaviour
+=========
+
+The `keys`, :func:`values`, and :func:`items` methods on BTree and
 TreeSet types do not materialize a list with all of the data.  Instead,
 they return lazy sequences that fetch data from the BTree as needed.  They
 also support optional arguments to specify the minimum and maximum values
@@ -144,6 +202,7 @@ may not be true in future releases. If the interfaces don't specify a behavior,
 then whether that behavior appears to work, and exactly happens if it does
 appear to work, are undefined and should not be relied on.
 
+.. _ordered:
 
 Total Ordering and Persistence
 ==============================
@@ -375,7 +434,8 @@ BTree node sizes
 
 BTrees (and TreeSets) are made up of a tree of Buckets (and Sets) and
 internal nodes.  There are maximum sizes of these notes configured for
-the various key and value types:
+the various key and value types (unsigned and quad unsigned follow
+integer and long, respectively):
 
 ======== ========== ========================== =============================
 Key Type Value Type Maximum Bucket or Set Size Maximum BTree or TreeSet Size