Support PURE_PYTHON at runtime.

Add failing tests and then fix them. Use a common function to DRY in
all the modules.

Fixes #78

Also use zope.testrunner in tox (and travis) because it allows passing
arguments to select what tests to run, see slow tests, etc.

Simplify setup.py to always build the C modules and avoid polluting
wheel caches (the same as zope.security and other projects that have
been worked on lately).

Bump the minimum 'persistent' requirement to one that actually works
in PURE_PYTHON mode (plus fix wheel metadata; otherwise we need
environment markers).

Test PURE_PYTHON in environments on Travis. Do some minor tuning to
eliminate redundant test cases and speed up test time.

.travis.yml

@@ -4,6 +4,9 @@ matrix:
     include:
         - os: linux
           python: 2.7
+        - os: linux
+          python: 2.7
+          env: PURE_PYTHON=1
         - os: linux
           python: 3.3
         - os: linux
@@ -12,10 +15,13 @@ matrix:
           python: 3.5
         - os: linux
           python: 3.6
+        - os: linux
+          python: 3.6
+          env: PURE_PYTHON=1
         - os: linux
           python: pypy
         - os: linux
-          python: pypy3.3-5.2-alpha1
+          python: pypy3
         # It's important to use 'macpython' builds to get the least
         # restrictive wheel tag. It's also important to avoid
         # 'homebrew 3' because it floats instead of being a specific version.
@@ -45,7 +51,7 @@ matrix:
           before_install:
             - if [[ $TRAVIS_TAG ]]; then bash .manylinux.sh; fi
             - exit 0
-  
+
 before_install:
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then git clone https://github.com/MacPython/terryfy; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then source terryfy/travis_tools.sh; fi
@@ -58,9 +64,9 @@ install:
     # (https://travis-ci.org/zopefoundation/BTrees/jobs/192340692) so
     # we install with pip manually.
     - pip install -U persistent
-    - pip install -e .[ZODB]
+    - pip install -e .[test,ZODB]
 script:
-    - python setup.py -q test -q
+    - zope-testrunner --test-path=. --auto-color --auto-progress
 notifications:
     email: false
 after_success:

BTrees/IFBTree.py

@@ -39,6 +39,7 @@ from ._base import union as _union
 from ._base import weightedIntersection as _weightedIntersection
 from ._base import weightedUnion as _weightedUnion
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 120
 _TREE_SIZE = 500
@@ -104,36 +105,7 @@ multiunionPy = _set_operation(_multiunion, IFSetPy)
 weightedUnionPy = _set_operation(_weightedUnion, IFSetPy)
 weightedIntersectionPy = _set_operation(_weightedIntersection, IFSetPy)
 
-try:
-    from ._IFBTree import IFBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    IFBucket = IFBucketPy
-    IFSet = IFSetPy
-    IFBTree = IFBTreePy
-    IFTreeSet = IFTreeSetPy
-    IFTreeIterator = IFTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    multiunion = multiunionPy
-    weightedUnion = weightedUnionPy
-    weightedIntersection = weightedIntersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._IFBTree import IFSet
-    from ._IFBTree import IFBTree
-    from ._IFBTree import IFTreeSet
-    from ._IFBTree import IFTreeIterator
-    from ._IFBTree import difference
-    from ._IFBTree import union
-    from ._IFBTree import intersection
-    from ._IFBTree import multiunion
-    from ._IFBTree import weightedUnion
-    from ._IFBTree import weightedIntersection
-
-Bucket = IFBucket
-Set = IFSet
-BTree = IFBTree
-TreeSet = IFTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/IIBTree.py

@@ -34,11 +34,12 @@ from ._base import intersection as _intersection
 from ._base import multiunion as _multiunion
 from ._base import set_operation as _set_operation
 from ._base import to_int as _to_key
-from ._base import to_int as _to_value
+_to_value = _to_key
 from ._base import union as _union
 from ._base import weightedIntersection as _weightedIntersection
 from ._base import weightedUnion as _weightedUnion
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 120
 _TREE_SIZE = 500
@@ -105,36 +106,7 @@ multiunionPy = _set_operation(_multiunion, IISetPy)
 weightedUnionPy = _set_operation(_weightedUnion, IISetPy)
 weightedIntersectionPy = _set_operation(_weightedIntersection, IISetPy)
 
-try:
-    from ._IIBTree import IIBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    IIBucket = IIBucketPy
-    IISet = IISetPy
-    IIBTree = IIBTreePy
-    IITreeSet = IITreeSetPy
-    IITreeIterator = IITreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    multiunion = multiunionPy
-    weightedUnion = weightedUnionPy
-    weightedIntersection = weightedIntersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._IIBTree import IISet
-    from ._IIBTree import IIBTree
-    from ._IIBTree import IITreeSet
-    from ._IIBTree import IITreeIterator
-    from ._IIBTree import difference
-    from ._IIBTree import union
-    from ._IIBTree import intersection
-    from ._IIBTree import multiunion
-    from ._IIBTree import weightedUnion
-    from ._IIBTree import weightedIntersection
-
-Bucket = IIBucket
-Set = IISet
-BTree = IIBTree
-TreeSet = IITreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/IOBTree.py

@@ -34,6 +34,7 @@ from ._base import to_int as _to_key
 from ._base import to_ob as _to_value
 from ._base import union as _union
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 60
 _TREE_SIZE = 500
@@ -87,32 +88,7 @@ unionPy = _set_operation(_union, IOSetPy)
 intersectionPy = _set_operation(_intersection, IOSetPy)
 multiunionPy = _set_operation(_multiunion, IOSetPy)
 
-try:
-    from ._IOBTree import IOBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    IOBucket = IOBucketPy
-    IOSet = IOSetPy
-    IOBTree = IOBTreePy
-    IOTreeSet = IOTreeSetPy
-    IOTreeIterator = IOTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    multiunion = multiunionPy
-else: #pragma NO COVER w/o C extensions
-    from ._IOBTree import IOSet
-    from ._IOBTree import IOBTree
-    from ._IOBTree import IOTreeSet
-    from ._IOBTree import IOTreeIterator
-    from ._IOBTree import difference
-    from ._IOBTree import union
-    from ._IOBTree import intersection
-    from ._IOBTree import multiunion
-
-Bucket = IOBucket
-Set = IOSet
-BTree = IOBTree
-TreeSet = IOTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/LFBTree.py

@@ -39,6 +39,7 @@ from ._base import union as _union
 from ._base import weightedIntersection as _weightedIntersection
 from ._base import weightedUnion as _weightedUnion
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 120
 _TREE_SIZE = 500
@@ -105,36 +106,7 @@ multiunionPy = _set_operation(_multiunion, LFSetPy)
 weightedUnionPy = _set_operation(_weightedUnion, LFSetPy)
 weightedIntersectionPy = _set_operation(_weightedIntersection, LFSetPy)
 
-try:
-    from ._LFBTree import LFBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    LFBucket = LFBucketPy
-    LFSet = LFSetPy
-    LFBTree = LFBTreePy
-    LFTreeSet = LFTreeSetPy
-    LFTreeIterator = LFTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    multiunion = multiunionPy
-    weightedUnion = weightedUnionPy
-    weightedIntersection = weightedIntersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._LFBTree import LFSet
-    from ._LFBTree import LFBTree
-    from ._LFBTree import LFTreeSet
-    from ._LFBTree import LFTreeIterator
-    from ._LFBTree import difference
-    from ._LFBTree import union
-    from ._LFBTree import intersection
-    from ._LFBTree import multiunion
-    from ._LFBTree import weightedUnion
-    from ._LFBTree import weightedIntersection
-
-Bucket = LFBucket
-Set = LFSet
-BTree = LFBTree
-TreeSet = LFTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/LLBTree.py

@@ -39,6 +39,7 @@ from ._base import union as _union
 from ._base import weightedIntersection as _weightedIntersection
 from ._base import weightedUnion as _weightedUnion
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 120
 _TREE_SIZE = 500
@@ -105,36 +106,7 @@ multiunionPy = _set_operation(_multiunion, LLSetPy)
 weightedUnionPy = _set_operation(_weightedUnion, LLSetPy)
 weightedIntersectionPy = _set_operation(_weightedIntersection, LLSetPy)
 
-try:
-    from ._LLBTree import LLBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    LLBucket = LLBucketPy
-    LLSet = LLSetPy
-    LLBTree = LLBTreePy
-    LLTreeSet = LLTreeSetPy
-    LLTreeIterator = LLTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    multiunion = multiunionPy
-    weightedUnion = weightedUnionPy
-    weightedIntersection = weightedIntersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._LLBTree import LLSet
-    from ._LLBTree import LLBTree
-    from ._LLBTree import LLTreeSet
-    from ._LLBTree import LLTreeIterator
-    from ._LLBTree import difference
-    from ._LLBTree import union
-    from ._LLBTree import intersection
-    from ._LLBTree import multiunion
-    from ._LLBTree import weightedUnion
-    from ._LLBTree import weightedIntersection
-
-Bucket = LLBucket
-Set = LLSet
-BTree = LLBTree
-TreeSet = LLTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/LOBTree.py

@@ -34,6 +34,7 @@ from ._base import to_long as _to_key
 from ._base import to_ob as _to_value
 from ._base import union as _union
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 60
 _TREE_SIZE = 500
@@ -88,32 +89,7 @@ unionPy = _set_operation(_union, LOSetPy)
 intersectionPy = _set_operation(_intersection, LOSetPy)
 multiunionPy = _set_operation(_multiunion, LOSetPy)
 
-try:
-    from ._LOBTree import LOBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    LOBucket = LOBucketPy
-    LOSet = LOSetPy
-    LOBTree = LOBTreePy
-    LOTreeSet = LOTreeSetPy
-    LOTreeIterator = LOTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    multiunion = multiunionPy
-else: #pragma NO COVER w/o C extensions
-    from ._LOBTree import LOSet
-    from ._LOBTree import LOBTree
-    from ._LOBTree import LOTreeSet
-    from ._LOBTree import LOTreeIterator
-    from ._LOBTree import difference
-    from ._LOBTree import union
-    from ._LOBTree import intersection
-    from ._LOBTree import multiunion
-
-Bucket = LOBucket
-Set = LOSet
-BTree = LOBTree
-TreeSet = LOTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/OIBTree.py

@@ -38,6 +38,7 @@ from ._base import union as _union
 from ._base import weightedIntersection as _weightedIntersection
 from ._base import weightedUnion as _weightedUnion
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 60
 _TREE_SIZE = 250
@@ -102,35 +103,7 @@ intersectionPy = _set_operation(_intersection, OISetPy)
 weightedUnionPy = _set_operation(_weightedUnion, OISetPy)
 weightedIntersectionPy = _set_operation(_weightedIntersection, OISetPy)
 
-try:
-    from ._OIBTree import OIBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    OIBucket = OIBucketPy
-    OISet = OISetPy
-    OIBTree = OIBTreePy
-    OITreeSet = OITreeSetPy
-    OITreeIterator = OITreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    weightedUnion = weightedUnionPy
-    weightedIntersection = weightedIntersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._OIBTree import OISet
-    from ._OIBTree import OIBTree
-    from ._OIBTree import OITreeSet
-    from ._OIBTree import OITreeIterator
-    from ._OIBTree import difference
-    from ._OIBTree import union
-    from ._OIBTree import intersection
-    from ._OIBTree import weightedUnion
-    from ._OIBTree import weightedIntersection
-
-
-Bucket = OIBucket
-Set = OISet
-BTree = OIBTree
-TreeSet = OITreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/OLBTree.py

@@ -38,6 +38,7 @@ from ._base import union as _union
 from ._base import weightedIntersection as _weightedIntersection
 from ._base import weightedUnion as _weightedUnion
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 60
 _TREE_SIZE = 250
@@ -103,34 +104,7 @@ intersectionPy = _set_operation(_intersection, OLSetPy)
 weightedUnionPy = _set_operation(_weightedUnion, OLSetPy)
 weightedIntersectionPy = _set_operation(_weightedIntersection, OLSetPy)
 
-try:
-    from ._OLBTree import OLBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    OLBucket = OLBucketPy
-    OLSet = OLSetPy
-    OLBTree = OLBTreePy
-    OLTreeSet = OLTreeSetPy
-    OLTreeIterator = OLTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    weightedUnion = weightedUnionPy
-    weightedIntersection = weightedIntersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._OLBTree import OLSet
-    from ._OLBTree import OLBTree
-    from ._OLBTree import OLTreeSet
-    from ._OLBTree import OLTreeIterator
-    from ._OLBTree import difference
-    from ._OLBTree import union
-    from ._OLBTree import intersection
-    from ._OLBTree import weightedUnion
-    from ._OLBTree import weightedIntersection
-
-Bucket = OLBucket
-Set = OLSet
-BTree = OLBTree
-TreeSet = OLTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/OOBTree.py

@@ -29,9 +29,10 @@ from ._base import difference as _difference
 from ._base import intersection as _intersection
 from ._base import set_operation as _set_operation
 from ._base import to_ob as _to_key
-from ._base import to_ob as _to_value
+_to_value = _to_key
 from ._base import union as _union
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 30
 _TREE_SIZE = 250
@@ -83,32 +84,7 @@ differencePy = _set_operation(_difference, OOSetPy)
 unionPy = _set_operation(_union, OOSetPy)
 intersectionPy = _set_operation(_intersection, OOSetPy)
 
-try:
-    from ._OOBTree import OOBucket
-except ImportError as e: #pragma NO COVER w/ C extensions
-    OOBucket = OOBucketPy
-    OOSet = OOSetPy
-    OOBTree = OOBTreePy
-    OOTreeSet = OOTreeSetPy
-    OOTreeIterator = OOTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._OOBTree import OOSet
-    from ._OOBTree import OOBTree
-    from ._OOBTree import OOTreeSet
-    from ._OOBTree import OOTreeIterator
-    from ._OOBTree import difference
-    from ._OOBTree import union
-    from ._OOBTree import intersection
-
-
-
-Bucket = OOBucket
-Set = OOSet
-BTree = OOBTree
-TreeSet = OOTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/_compat.py

@@ -11,16 +11,21 @@
 # FOR A PARTICULAR PURPOSE
 #
 ##############################################################################
+import os
 import sys
 
+PYPY = hasattr(sys, 'pypy_version_info')
+# We can and do build the C extensions on PyPy, but
+# as of Persistent 4.2.5 the persistent C extension is not
+# built on PyPy, so importing our C extension will fail anyway.
+PURE_PYTHON = os.environ.get('PURE_PYTHON', PYPY)
+
+
 if sys.version_info[0] < 3: #pragma NO COVER Python2
 
     PY2 = True
     PY3 = False
 
-    from StringIO import StringIO
-    BytesIO = StringIO
-
     int_types = int, long
     xrange = xrange
     def compare(x, y):
@@ -38,17 +43,11 @@ if sys.version_info[0] < 3: #pragma NO COVER Python2
     def _ascii(x):
         return bytes(x)
 
-    def _u(s, encoding='unicode_escape'):
-        return unicode(s, encoding)
-
 else: #pragma NO COVER Python3
 
     PY2 = False
     PY3 = True
 
-    from io import StringIO
-    from io import BytesIO
-
     int_types = int,
     xrange = range
 
@@ -67,7 +66,34 @@ else: #pragma NO COVER Python3
     def _ascii(x):
         return bytes(x, 'ascii')
 
-    def _u(s, encoding=None):
-        if encoding is None:
-            return s
-        return str(s, encoding)
+
+def import_c_extension(mod_globals):
+    import importlib
+    c_module = None
+    module_name = mod_globals['__name__']
+    assert module_name.startswith('BTrees.')
+    module_name = module_name.split('.')[1]
+    if not PURE_PYTHON:
+        try:
+            c_module = importlib.import_module('BTrees._' + module_name)
+        except ImportError:
+            pass
+    if c_module is not None:
+        new_values = dict(c_module.__dict__)
+        new_values.pop("__name__", None)
+        new_values.pop('__file__', None)
+        new_values.pop('__doc__', None)
+        mod_globals.update(new_values)
+    else:
+        # No C extension,
+        # make the Py versions available without that extension
+        for py in [k for k in mod_globals.keys() if k.endswith('Py')]:
+            mod_globals[py[:-2]] = mod_globals[py]
+
+    # Assign the global aliases
+    prefix = module_name[:2]
+    for name in ('Bucket', 'Set', 'BTree', 'TreeSet'):
+        mod_globals[name] = mod_globals[prefix + name]
+
+    # Cleanup
+    del mod_globals['import_c_extension']

BTrees/fsBTree.py

@@ -35,6 +35,8 @@ from ._base import set_operation as _set_operation
 from ._base import to_bytes as _to_bytes
 from ._base import union as _union
 from ._base import _fix_pickle
+from ._compat import import_c_extension
+
 
 _BUCKET_SIZE = 500
 _TREE_SIZE = 500
@@ -101,28 +103,7 @@ differencePy = _set_operation(_difference, fsSetPy)
 unionPy = _set_operation(_union, fsSetPy)
 intersectionPy = _set_operation(_intersection, fsSetPy)
 
-try:
-    from ._fsBTree import fsBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    fsBucket = fsBucketPy
-    fsSet = fsSetPy
-    fsBTree = fsBTreePy
-    fsTreeSet = fsTreeSetPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._fsBTree import fsSet
-    from ._fsBTree import fsBTree
-    from ._fsBTree import fsTreeSet
-    from ._fsBTree import difference
-    from ._fsBTree import union
-    from ._fsBTree import intersection
-
-Bucket = fsBucket
-Set = fsSet
-BTree = fsBTree
-TreeSet = fsTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/tests/common.py

@@ -11,34 +11,48 @@
 # FOR A PARTICULAR PURPOSE
 #
 ##############################################################################
+from __future__ import division
 
+import unittest
 import platform
 from unittest import skip
 
 
-def _skip_wo_ZODB(test_method):  # pragma: no COVER
-    try:
-        import ZODB  # noqa
-    except ImportError:  # skip this test if ZODB is not available
-        return skip("ZODB not available")(test_method)
-    else:
-        return test_method
-
-
-def _skip_under_Py3k(test_method):  # pragma: no COVER
-    try:
-        unicode
-    except NameError:  # skip this test
-        return skip("Python 3")(test_method)
-    else:
-        return test_method
-
+from BTrees._compat import PY3
+from BTrees._compat import PURE_PYTHON
+from BTrees._compat import PYPY
 
-def _skip_on_32_bits(test_method):  # pragma: no COVER
-    if platform.architecture()[0] == '32bit':
-        return skip("32-bit platform")(test_method)
+def _no_op(test_method):
     return test_method
 
+try:
+    __import__('ZODB')
+except ImportError:
+    _skip_wo_ZODB = skip('ZODB not available')
+else:
+    _skip_wo_ZODB = _no_op
+
+if PY3:
+    _skip_under_Py3k = skip("Not on Python 3")
+else:
+    _skip_under_Py3k = _no_op
+
+if platform.architecture()[0] == '32bit':
+    _skip_on_32_bits = skip("32-bit platform")
+else:
+    _skip_on_32_bits = _no_op
+
+if PURE_PYTHON:
+    skipOnPurePython = skip("Not on Pure Python")
+else:
+    skipOnPurePython = _no_op
+
+def _skip_if_pure_py_and_py_test(self):
+    if PURE_PYTHON and 'Py' in type(self).__name__:
+        # No need to run this again. The "C" tests will catch it.
+        # This relies on the fact that we always define tests in pairs,
+        # one normal/C and one with Py in the name for the Py test.
+        raise unittest.SkipTest("Redundant with the C test")
 
 class Base(object):
     # Tests common to all types: sets, buckets, and BTrees
@@ -51,6 +65,9 @@ class Base(object):
     def _makeOne(self):
         return self._getTargetClass()()
 
+    def setUp(self):
+        super(Base, self).setUp()
+        _skip_if_pure_py_and_py_test(self)
 
     def tearDown(self):
         if self.db is not None:
@@ -77,6 +94,38 @@ class Base(object):
         transaction.abort()
         root._p_jar.close()
 
+    def testPersistentSubclass(self):
+        # Can we subclass this and Persistent?
+        # https://github.com/zopefoundation/BTrees/issues/78
+        import persistent
+
+        class PersistentSubclass(persistent.Persistent):
+            pass
+
+        __traceback_info__ = self._getTargetClass(), persistent.Persistent
+        type('Subclass', (self._getTargetClass(), PersistentSubclass), {})
+
+    def testPurePython(self):
+        import importlib
+        kind = self._getTargetClass()
+        class_name = kind.__name__
+        module_name = kind.__module__
+        module = importlib.import_module(module_name)
+
+        # If we're in pure python mode, our target class module
+        # should not have an '_' in it (fix_pickle changes the name
+        # to remove the 'Py')
+
+        # If we're in the C extension mode, our target class
+        # module still doesn't have the _ in it, but we should be able to find
+        # a Py class that's different
+
+        self.assertNotIn('_', module_name)
+        self.assertIs(getattr(module, class_name), kind)
+
+        if not PURE_PYTHON and 'Py' not in type(self).__name__:
+            self.assertIsNot(getattr(module, class_name + 'Py'), kind)
+
     @_skip_wo_ZODB
     def testLoadAndStore(self):
         import transaction
@@ -1735,6 +1784,10 @@ class TypeTest(object):
 
 class I_SetsBase(object):
 
+    def setUp(self):
+        super(I_SetsBase, self).setUp()
+        _skip_if_pure_py_and_py_test(self)
+
     def testBadBadKeyAfterFirst(self):
         t = self._makeOne()
         self.assertRaises(TypeError, t.__class__, [1, ''])
@@ -1768,7 +1821,7 @@ SMALLEST_POSITIVE_65_BITS = LARGEST_64_BITS + 1
 LARGEST_NEGATIVE_65_BITS = SMALLEST_64_BITS - 1
 
 
-class TestLongIntSupport:
+class TestLongIntSupport(object):
 
     def getTwoValues(self):
         # Return two distinct values; these must compare as un-equal.
@@ -1885,6 +1938,9 @@ def makeBuilder(mapbuilder):
 #     intersection, union, difference - set to the type-correct versions
 class SetResult(object):
     def setUp(self):
+        super(SetResult, self).setUp()
+        _skip_if_pure_py_and_py_test(self)
+
         self.Akeys = [1,    3,    5, 6   ]
         self.Bkeys = [   2, 3, 4,    6, 7]
         self.As = [makeset(self.Akeys) for makeset in self.builders()]
@@ -2167,6 +2223,10 @@ def isaset(thing):
 #     mkbucket, mkbtree
 class MultiUnion(object):
 
+    def setUp(self):
+        super(MultiUnion, self).setUp()
+        _skip_if_pure_py_and_py_test(self)
+
     def testEmpty(self):
         self.assertEqual(len(self.multiunion([])), 0)
 
@@ -2193,6 +2253,10 @@ class MultiUnion(object):
 
     def testBigInput(self):
         N = 100000
+        if (PURE_PYTHON or 'Py' in type(self).__name__) and not PYPY:
+            # This is extremely slow in CPython implemented in Python,
+            # taking 20s or more on a 2015-era laptop
+            N = N // 10
         input = self.mkset(list(range(N)))
         output = self.multiunion([input] * 10)
         self.assertEqual(len(output), N)
@@ -2234,6 +2298,11 @@ class ConflictTestBase(object):
 
     storage = None
 
+    def setUp(self):
+        super(ConflictTestBase, self).setUp()
+        _skip_if_pure_py_and_py_test(self)
+
+
     def tearDown(self):
         import transaction
         transaction.abort()

BTrees/tests/testBTrees.py

@@ -11,11 +11,8 @@
 # FOR A PARTICULAR PURPOSE
 #
 ##############################################################################
-import sys
 import unittest
 
-python3 = sys.version_info >= (3, )
-
 from BTrees.tests.common import permutations
 
 
@@ -171,6 +168,8 @@ class DegenerateBTree(unittest.TestCase):
         #      RuntimeWarning: tp_compare didn't return -1 or -2 for exception
         #    warnings, possibly due to memory corruption after a BTree
         #    goes insane.
+        # On CPython in PURE_PYTHON mode, this is a *slow* test, taking 15+s
+        # on a 2015 laptop.
         from BTrees.check import check
         t, keys = self._build_degenerate_tree()
         for oneperm in permutations(keys):
@@ -196,7 +195,7 @@ LP294788_ids = {}
 
 class ToBeDeleted(object):
     def __init__(self, id):
-        assert type(id) is int #we don't want to store any object ref here
+        assert isinstance(id, int) #we don't want to store any object ref here
         self.id = id
 
         global LP294788_ids
@@ -257,8 +256,6 @@ class BugFixes(unittest.TestCase):
         import gc
         import random
         from BTrees.OOBTree import OOBTree
-        from .._compat import _u
-        from .._compat import xrange
 
         t = OOBTree()
 
@@ -269,12 +266,12 @@ class BugFixes(unittest.TestCase):
         # /// BTree keys are integers, value is an object
         LP294788_ids = {}
         ids = {}
-        for i in xrange(1024):
-            if trandom.random() > 0.1 or len(ids) == 0:
+        for i in range(1024):
+            if trandom.random() > 0.1 or not ids:
                 #add
                 id = None
                 while id is None or id in ids:
-                    id = trandom.randint(0,1000000)
+                    id = trandom.randint(0, 1000000)
 
                 ids[id] = 1
                 t[id] = ToBeDeleted(id)
@@ -304,15 +301,15 @@ class BugFixes(unittest.TestCase):
         # /// BTree keys are integers, value is a tuple having an object
         LP294788_ids = {}
         ids = {}
-        for i in xrange(1024):
-            if trandom.random() > 0.1 or len(ids) == 0:
+        for i in range(1024):
+            if trandom.random() > 0.1 or not ids:
                 #add
                 id = None
                 while id is None or id in ids:
-                    id = trandom.randint(0,1000000)
+                    id = trandom.randint(0, 1000000)
 
                 ids[id] = 1
-                t[id] = (id, ToBeDeleted(id), _u('somename'))
+                t[id] = (id, ToBeDeleted(id), u'somename')
             else:
                 #del
                 keys = list(ids.keys())
@@ -341,12 +338,12 @@ class BugFixes(unittest.TestCase):
         t = OOBTree()
         LP294788_ids = {}
         ids = {}
-        for i in xrange(1024):
-            if trandom.random() > 0.1 or len(ids) == 0:
+        for i in range(1024):
+            if trandom.random() > 0.1 or not ids:
                 #add
                 id = None
                 while id is None or id in ids:
-                    id = ToBeDeleted(trandom.randint(0,1000000))
+                    id = ToBeDeleted(trandom.randint(0, 1000000))
 
                 ids[id] = 1
                 t[id] = 1
@@ -361,7 +358,7 @@ class BugFixes(unittest.TestCase):
         for id in ids:
             del t[id]
         #release all refs
-        ids = obj = id = None
+        ids = id = None
 
         #to be on the safe side run a full GC
         gc.collect()
@@ -375,13 +372,13 @@ class BugFixes(unittest.TestCase):
         t = OOBTree()
         LP294788_ids = {}
         ids = {}
-        for i in xrange(1024):
-            if trandom.random() > 0.1 or len(ids) == 0:
+        for i in range(1024):
+            if trandom.random() > 0.1 or not ids:
                 #add
                 id = None
                 while id is None or id in ids:
-                    id = trandom.randint(0,1000000)
-                    id = (id, ToBeDeleted(id), _u('somename'))
+                    id = trandom.randint(0, 1000000)
+                    id = (id, ToBeDeleted(id), u'somename')
 
                 ids[id] = 1
                 t[id] = 1
@@ -396,7 +393,7 @@ class BugFixes(unittest.TestCase):
         for id in ids:
             del t[id]
         #release all refs
-        ids = id = obj = key = None
+        ids = id = key = None
 
         #to be on the safe side run a full GC
         gc.collect()
@@ -412,7 +409,7 @@ class BugFixes(unittest.TestCase):
 
 class DoesntLikeBeingCompared:
 
-    def __cmp__(self,other):
+    def __cmp__(self, other):
         raise ValueError('incomparable')
     __lt__ = __le__ = __eq__ = __ne__ = __ge__ = __gt__ = __cmp__
 
@@ -457,13 +454,11 @@ class FamilyTest(unittest.TestCase):
         # this next bit illustrates an, um, "interesting feature".  If
         # the characteristics change to match the 64 bit version, please
         # feel free to change.
-        try: s.insert(BTrees.family32.maxint + 1)
-        except (TypeError, OverflowError): pass
-        else: self.assert_(False)
+        with self.assertRaises((TypeError, OverflowError)):
+            s.insert(BTrees.family32.maxint + 1)
 
-        try: s.insert(BTrees.family32.minint - 1)
-        except (TypeError, OverflowError): pass
-        else: self.assert_(False)
+        with self.assertRaises((TypeError, OverflowError)):
+            s.insert(BTrees.family32.minint - 1)
         self.check_pickling(BTrees.family32)
 
     def test64(self):
@@ -499,12 +494,12 @@ class FamilyTest(unittest.TestCase):
         # unpickling, whether from the same unpickler or different
         # unpicklers.
         import pickle
-        from .._compat import BytesIO
+        from io import BytesIO
 
         s = pickle.dumps((family, family))
         (f1, f2) = pickle.loads(s)
-        self.assertTrue(f1 is family)
-        self.assertTrue(f2 is family)
+        self.assertIs(f1, family)
+        self.assertIs(f2, family)
 
         # Using a single memo across multiple pickles:
         sio = BytesIO()

BTrees/tests/testBTreesUnicode.py

@@ -29,9 +29,8 @@ class TestBTreesUnicode(unittest.TestCase):
         #setup an OOBTree with some unicode strings
         from BTrees.OOBTree import OOBTree
         from BTrees._compat import _bytes
-        from BTrees._compat import _u
 
-        self.s = _u(b'dreit\xe4gigen', 'latin1')
+        self.s = b'dreit\xe4gigen'.decode('latin1')
 
         self.data = [(b'alien', 1),
                      (b'k\xf6nnten', 2),
@@ -39,23 +38,22 @@ class TestBTreesUnicode(unittest.TestCase):
                      (b'future', 4),
                      (b'quick', 5),
                      (b'zerst\xf6rt', 6),
-                     (_u(b'dreit\xe4gigen','latin1'), 7),
+                     (b'dreit\xe4gigen'.decode('latin1'), 7),
                     ]
 
         self.tree = OOBTree()
         for k, v in self.data:
             if isinstance(k, _bytes):
-                k = _u(k, 'latin1')
+                k = k.decode('latin1')
             self.tree[k] = v
 
     @_skip_under_Py3k
     def testAllKeys(self):
         # check every item of the tree
-        from BTrees._compat import _u
         from BTrees._compat import _bytes
         for k, v in self.data:
             if isinstance(k, _bytes):
-                k = _u(k, encoding)
+                k = k.decode(encoding)
             self.assertTrue(k in self.tree)
             self.assertEqual(self.tree[k], v)
 

CHANGES.rst

@@ -11,7 +11,10 @@
   https://github.com/zopefoundation/BTrees/pull/55
 - Fix the possibility of a rare crash in the C extension when
   deallocating items. See https://github.com/zopefoundation/BTrees/issues/75
-
+- Respect the ``PURE_PYTHON`` environment variable at runtime even if
+  the C extensions are available. See
+  https://github.com/zopefoundation/BTrees/issues/78
+- Always attempt to build the C extensions, but make their success optional.
 
 4.4.1 (2017-01-24)
 ------------------

appveyor.yml

@@ -19,14 +19,14 @@ environment:
 install:
   - "SET PATH=C:\\Python%PYTHON%;c:\\Python%PYTHON%\\scripts;%PATH%"
   - echo "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 > "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64\vcvars64.bat"
-  - pip install -e .[ZODB]
+  - pip install -e .[test,ZODB]
 
 build_script:
   - pip install wheel
   - python -W ignore setup.py -q bdist_wheel
 
 test_script:
-  - python setup.py -q test -q
+  - zope-testrunner --test-path=.
 
 artifacts:
   - path: 'dist\*.whl'

setup.py

@@ -15,17 +15,50 @@
 __version__ = '4.4.1'
 
 import os
-import platform
-import sys
+
+from distutils.errors import CCompilerError
+from distutils.errors import DistutilsExecError
+from distutils.errors import DistutilsPlatformError
 
 from setuptools import Extension
 from setuptools import find_packages
 from setuptools import setup
+from setuptools.command.build_ext import build_ext
+
+
+def _read(fname):
+    here = os.path.abspath(os.path.dirname(__file__))
+    with open(os.path.join(here, fname)) as f:
+        return f.read()
+
+
+README = _read("README.rst") + '\n\n' + _read('CHANGES.rst')
+
 
-here = os.path.abspath(os.path.dirname(__file__))
-README = (open(os.path.join(here, 'README.rst')).read()
-          + '\n\n' +
-          open(os.path.join(here, 'CHANGES.rst')).read())
+class optional_build_ext(build_ext):
+    """This class subclasses build_ext and allows
+       the building of C extensions to fail.
+    """
+    def run(self):
+        try:
+            build_ext.run(self)
+        except DistutilsPlatformError as e:
+            self._unavailable(e)
+
+    def build_extension(self, ext):
+        try:
+            build_ext.build_extension(self, ext)
+        except (CCompilerError, DistutilsExecError, OSError) as e:
+            self._unavailable(e)
+
+    def _unavailable(self, e):
+        print('*' * 80)
+        print("""WARNING:
+        An optional code optimization (C extension) could not be compiled.
+        Optimizations for this package will not be available!""")
+        print()
+        print(e)
+        print('*' * 80)
 
 # Include directories for C extensions
 # Sniff the location of the headers in 'persistent' or fall back
@@ -49,6 +82,7 @@ class ModuleHeaderDir(object):
         path = resource_filename(self._require_spec, self._where)
         return os.path.abspath(path)
 
+
 include = [ModuleHeaderDir('persistent')]
 
 # Set up dependencies for the BTrees package
@@ -87,58 +121,47 @@ def BTreeExtension(family):
         kwargs["define_macros"] = [('EXCLUDE_INTSET_SUPPORT', None)]
     return Extension(name, sources, **kwargs)
 
-py_impl = getattr(platform, 'python_implementation', lambda: None)
-pure_python = os.environ.get('PURE_PYTHON', False)
-is_pypy = py_impl() == 'PyPy'
-is_jython = 'java' in sys.platform
 
-# Jython cannot build the C optimizations, while on PyPy they are
-# anti-optimizations (the C extension compatibility layer is known-slow,
-# and defeats JIT opportunities).
-if pure_python or is_pypy or is_jython:
-    ext_modules = []
-else:
+ext_modules = [BTreeExtension(family) for family in FAMILIES]
 
-    ext_modules = [BTreeExtension(family) for family in FAMILIES]
+REQUIRES = [
+    # 4.1.0 is the first version that PURE_PYTHON can run
+    # ZODB tests
+    'persistent >= 4.1.0',
+    'zope.interface',
+]
 
-if sys.version_info[0] >= 3:
-    REQUIRES = [
-        'persistent>=4.0.4',
-        'zope.interface',
-    ]
-else:
-    REQUIRES = [
-        'persistent',
-        'zope.interface',
-    ]
-TESTS_REQUIRE = REQUIRES + ['transaction']
+TESTS_REQUIRE = [
+    'transaction',
+    'zope.testrunner',
+]
 
 setup(name='BTrees',
       version=__version__,
       description='Scalable persistent object containers',
       long_description=README,
       classifiers=[
-        "Development Status :: 6 - Mature",
-        "License :: OSI Approved :: Zope Public License",
-        "Programming Language :: Python",
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
-        "Programming Language :: Python :: Implementation :: CPython",
-        "Programming Language :: Python :: Implementation :: PyPy",
-        "Framework :: ZODB",
-        "Topic :: Database",
-        "Topic :: Software Development :: Libraries :: Python Modules",
-        "Operating System :: Microsoft :: Windows",
-        "Operating System :: Unix",
-        ],
+          "Development Status :: 6 - Mature",
+          "License :: OSI Approved :: Zope Public License",
+          "Programming Language :: Python",
+          'Programming Language :: Python :: 2',
+          'Programming Language :: Python :: 2.7',
+          'Programming Language :: Python :: 3',
+          'Programming Language :: Python :: 3.3',
+          'Programming Language :: Python :: 3.4',
+          'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
+          "Programming Language :: Python :: Implementation :: CPython",
+          "Programming Language :: Python :: Implementation :: PyPy",
+          "Framework :: ZODB",
+          "Topic :: Database",
+          "Topic :: Software Development :: Libraries :: Python Modules",
+          "Operating System :: Microsoft :: Windows",
+          "Operating System :: Unix",
+      ],
       author="Zope Foundation",
       author_email="zodb-dev@zope.org",
-      url="http://packages.python.org/BTrees",
+      url="https://github.com/zopefoundation/BTrees",
       license="ZPL 2.1",
       platforms=["any"],
       packages=find_packages(),
@@ -147,14 +170,21 @@ setup(name='BTrees',
       ext_modules=ext_modules,
       setup_requires=['persistent'],
       extras_require={
-        'test': TESTS_REQUIRE,
-        'ZODB': ['ZODB'],
-        'testing': TESTS_REQUIRE + ['nose', 'coverage'],
-        'docs': ['Sphinx', 'repoze.sphinx.autointerface'],
+          'test': TESTS_REQUIRE,
+          'ZODB': [
+              'ZODB',
+          ],
+          'docs': [
+              'Sphinx',
+              'repoze.sphinx.autointerface',
+          ],
       },
       test_suite="BTrees.tests",
       tests_require=TESTS_REQUIRE,
       install_requires=REQUIRES,
+      cmdclass={
+          'build_ext': optional_build_ext,
+      },
       entry_points="""\
       """
-      )
+)

tox.ini

@@ -6,34 +6,23 @@ envlist =
     py27,py27-pure,pypy,py33,py34,py35,py35-pure,py36,pypy3,w_zodb,coverage,docs
 
 [testenv]
+usedevelop = true
 deps =
-    zope.interface
-    persistent
-    transaction
+    .[test]
 commands =
-    python setup.py -q test -q
+    zope-testrunner --test-path=. --auto-color --auto-progress []
 
 [testenv:py27-pure]
 basepython =
     python2.7
 setenv =
     PURE_PYTHON = 1
-    PIP_CACHE_DIR = {envdir}/.cache
-deps =
-    {[testenv]deps}
-commands =
-    python setup.py -q test -q
 
 [testenv:py35-pure]
 basepython =
     python3.5
 setenv =
     PURE_PYTHON = 1
-    PIP_CACHE_DIR = {envdir}/.cache
-deps =
-    {[testenv]deps}
-commands =
-    python setup.py -q test -q
 
 #[testenv:jython]
 #commands =
@@ -42,29 +31,19 @@ commands =
 [testenv:w_zodb]
 basepython =
     python2.7
-commands =
-    python setup.py -q test -q
 deps =
-    zope.interface
-    persistent
-    transaction
+	{[testenv]deps}
     ZODB
-    nose
-    coverage
-    nosexcover
 
 [testenv:coverage]
 basepython =
     python2.7
 commands =
-    nosetests --with-xunit --with-xcoverage --cover-package=BTrees
+    coverage run -m zope.testrunner --test-path=. --auto-color --auto-progress []
+    coverage report
 deps =
-    zope.interface
-    persistent
-    transaction
-    nose
+    {[testenv]deps}
     coverage
-    nosexcover
 
 [testenv:docs]
 basepython =
@@ -73,8 +52,4 @@ commands =
     sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html
     sphinx-build -b doctest -d docs/_build/doctrees docs docs/_build/doctest
 deps =
-    zope.interface
-    persistent
-    transaction
-    Sphinx
-    repoze.sphinx.autointerface
+    .[docs]