Merge pull request #80 from zopefoundation/issue78

Support PURE_PYTHON at runtime. 

.travis.yml

@@ -4,6 +4,9 @@ matrix:
     include:
         - os: linux
           python: 2.7
+        - os: linux
+          python: 2.7
+          env: PURE_PYTHON=1
         - os: linux
           python: 3.3
         - os: linux
@@ -12,10 +15,13 @@ matrix:
           python: 3.5
         - os: linux
           python: 3.6
+        - os: linux
+          python: 3.6
+          env: PURE_PYTHON=1
         - os: linux
           python: pypy
         - os: linux
-          python: pypy3.3-5.2-alpha1
+          python: pypy3
         # It's important to use 'macpython' builds to get the least
         # restrictive wheel tag. It's also important to avoid
         # 'homebrew 3' because it floats instead of being a specific version.
@@ -45,7 +51,7 @@ matrix:
           before_install:
             - if [[ $TRAVIS_TAG ]]; then bash .manylinux.sh; fi
             - exit 0
-  
+
 before_install:
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then git clone https://github.com/MacPython/terryfy; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then source terryfy/travis_tools.sh; fi
@@ -58,9 +64,9 @@ install:
     # (https://travis-ci.org/zopefoundation/BTrees/jobs/192340692) so
     # we install with pip manually.
     - pip install -U persistent
-    - pip install -e .[ZODB]
+    - pip install -e .[test,ZODB]
 script:
-    - python setup.py -q test -q
+    - zope-testrunner --test-path=. --auto-color --auto-progress
 notifications:
     email: false
 after_success:

BTrees/IFBTree.py

@@ -39,6 +39,7 @@ from ._base import union as _union
 from ._base import weightedIntersection as _weightedIntersection
 from ._base import weightedUnion as _weightedUnion
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 120
 _TREE_SIZE = 500
@@ -104,36 +105,7 @@ multiunionPy = _set_operation(_multiunion, IFSetPy)
 weightedUnionPy = _set_operation(_weightedUnion, IFSetPy)
 weightedIntersectionPy = _set_operation(_weightedIntersection, IFSetPy)
 
-try:
-    from ._IFBTree import IFBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    IFBucket = IFBucketPy
-    IFSet = IFSetPy
-    IFBTree = IFBTreePy
-    IFTreeSet = IFTreeSetPy
-    IFTreeIterator = IFTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    multiunion = multiunionPy
-    weightedUnion = weightedUnionPy
-    weightedIntersection = weightedIntersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._IFBTree import IFSet
-    from ._IFBTree import IFBTree
-    from ._IFBTree import IFTreeSet
-    from ._IFBTree import IFTreeIterator
-    from ._IFBTree import difference
-    from ._IFBTree import union
-    from ._IFBTree import intersection
-    from ._IFBTree import multiunion
-    from ._IFBTree import weightedUnion
-    from ._IFBTree import weightedIntersection
-
-Bucket = IFBucket
-Set = IFSet
-BTree = IFBTree
-TreeSet = IFTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/IIBTree.py

@@ -34,11 +34,12 @@ from ._base import intersection as _intersection
 from ._base import multiunion as _multiunion
 from ._base import set_operation as _set_operation
 from ._base import to_int as _to_key
-from ._base import to_int as _to_value
+_to_value = _to_key
 from ._base import union as _union
 from ._base import weightedIntersection as _weightedIntersection
 from ._base import weightedUnion as _weightedUnion
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 120
 _TREE_SIZE = 500
@@ -105,36 +106,7 @@ multiunionPy = _set_operation(_multiunion, IISetPy)
 weightedUnionPy = _set_operation(_weightedUnion, IISetPy)
 weightedIntersectionPy = _set_operation(_weightedIntersection, IISetPy)
 
-try:
-    from ._IIBTree import IIBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    IIBucket = IIBucketPy
-    IISet = IISetPy
-    IIBTree = IIBTreePy
-    IITreeSet = IITreeSetPy
-    IITreeIterator = IITreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    multiunion = multiunionPy
-    weightedUnion = weightedUnionPy
-    weightedIntersection = weightedIntersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._IIBTree import IISet
-    from ._IIBTree import IIBTree
-    from ._IIBTree import IITreeSet
-    from ._IIBTree import IITreeIterator
-    from ._IIBTree import difference
-    from ._IIBTree import union
-    from ._IIBTree import intersection
-    from ._IIBTree import multiunion
-    from ._IIBTree import weightedUnion
-    from ._IIBTree import weightedIntersection
-
-Bucket = IIBucket
-Set = IISet
-BTree = IIBTree
-TreeSet = IITreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/IOBTree.py

@@ -34,6 +34,7 @@ from ._base import to_int as _to_key
 from ._base import to_ob as _to_value
 from ._base import union as _union
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 60
 _TREE_SIZE = 500
@@ -87,32 +88,7 @@ unionPy = _set_operation(_union, IOSetPy)
 intersectionPy = _set_operation(_intersection, IOSetPy)
 multiunionPy = _set_operation(_multiunion, IOSetPy)
 
-try:
-    from ._IOBTree import IOBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    IOBucket = IOBucketPy
-    IOSet = IOSetPy
-    IOBTree = IOBTreePy
-    IOTreeSet = IOTreeSetPy
-    IOTreeIterator = IOTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    multiunion = multiunionPy
-else: #pragma NO COVER w/o C extensions
-    from ._IOBTree import IOSet
-    from ._IOBTree import IOBTree
-    from ._IOBTree import IOTreeSet
-    from ._IOBTree import IOTreeIterator
-    from ._IOBTree import difference
-    from ._IOBTree import union
-    from ._IOBTree import intersection
-    from ._IOBTree import multiunion
-
-Bucket = IOBucket
-Set = IOSet
-BTree = IOBTree
-TreeSet = IOTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/LFBTree.py

@@ -39,6 +39,7 @@ from ._base import union as _union
 from ._base import weightedIntersection as _weightedIntersection
 from ._base import weightedUnion as _weightedUnion
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 120
 _TREE_SIZE = 500
@@ -105,36 +106,7 @@ multiunionPy = _set_operation(_multiunion, LFSetPy)
 weightedUnionPy = _set_operation(_weightedUnion, LFSetPy)
 weightedIntersectionPy = _set_operation(_weightedIntersection, LFSetPy)
 
-try:
-    from ._LFBTree import LFBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    LFBucket = LFBucketPy
-    LFSet = LFSetPy
-    LFBTree = LFBTreePy
-    LFTreeSet = LFTreeSetPy
-    LFTreeIterator = LFTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    multiunion = multiunionPy
-    weightedUnion = weightedUnionPy
-    weightedIntersection = weightedIntersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._LFBTree import LFSet
-    from ._LFBTree import LFBTree
-    from ._LFBTree import LFTreeSet
-    from ._LFBTree import LFTreeIterator
-    from ._LFBTree import difference
-    from ._LFBTree import union
-    from ._LFBTree import intersection
-    from ._LFBTree import multiunion
-    from ._LFBTree import weightedUnion
-    from ._LFBTree import weightedIntersection
-
-Bucket = LFBucket
-Set = LFSet
-BTree = LFBTree
-TreeSet = LFTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/LLBTree.py

@@ -39,6 +39,7 @@ from ._base import union as _union
 from ._base import weightedIntersection as _weightedIntersection
 from ._base import weightedUnion as _weightedUnion
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 120
 _TREE_SIZE = 500
@@ -105,36 +106,7 @@ multiunionPy = _set_operation(_multiunion, LLSetPy)
 weightedUnionPy = _set_operation(_weightedUnion, LLSetPy)
 weightedIntersectionPy = _set_operation(_weightedIntersection, LLSetPy)
 
-try:
-    from ._LLBTree import LLBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    LLBucket = LLBucketPy
-    LLSet = LLSetPy
-    LLBTree = LLBTreePy
-    LLTreeSet = LLTreeSetPy
-    LLTreeIterator = LLTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    multiunion = multiunionPy
-    weightedUnion = weightedUnionPy
-    weightedIntersection = weightedIntersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._LLBTree import LLSet
-    from ._LLBTree import LLBTree
-    from ._LLBTree import LLTreeSet
-    from ._LLBTree import LLTreeIterator
-    from ._LLBTree import difference
-    from ._LLBTree import union
-    from ._LLBTree import intersection
-    from ._LLBTree import multiunion
-    from ._LLBTree import weightedUnion
-    from ._LLBTree import weightedIntersection
-
-Bucket = LLBucket
-Set = LLSet
-BTree = LLBTree
-TreeSet = LLTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/LOBTree.py

@@ -34,6 +34,7 @@ from ._base import to_long as _to_key
 from ._base import to_ob as _to_value
 from ._base import union as _union
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 60
 _TREE_SIZE = 500
@@ -88,32 +89,7 @@ unionPy = _set_operation(_union, LOSetPy)
 intersectionPy = _set_operation(_intersection, LOSetPy)
 multiunionPy = _set_operation(_multiunion, LOSetPy)
 
-try:
-    from ._LOBTree import LOBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    LOBucket = LOBucketPy
-    LOSet = LOSetPy
-    LOBTree = LOBTreePy
-    LOTreeSet = LOTreeSetPy
-    LOTreeIterator = LOTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    multiunion = multiunionPy
-else: #pragma NO COVER w/o C extensions
-    from ._LOBTree import LOSet
-    from ._LOBTree import LOBTree
-    from ._LOBTree import LOTreeSet
-    from ._LOBTree import LOTreeIterator
-    from ._LOBTree import difference
-    from ._LOBTree import union
-    from ._LOBTree import intersection
-    from ._LOBTree import multiunion
-
-Bucket = LOBucket
-Set = LOSet
-BTree = LOBTree
-TreeSet = LOTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/OIBTree.py

@@ -38,6 +38,7 @@ from ._base import union as _union
 from ._base import weightedIntersection as _weightedIntersection
 from ._base import weightedUnion as _weightedUnion
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 60
 _TREE_SIZE = 250
@@ -102,35 +103,7 @@ intersectionPy = _set_operation(_intersection, OISetPy)
 weightedUnionPy = _set_operation(_weightedUnion, OISetPy)
 weightedIntersectionPy = _set_operation(_weightedIntersection, OISetPy)
 
-try:
-    from ._OIBTree import OIBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    OIBucket = OIBucketPy
-    OISet = OISetPy
-    OIBTree = OIBTreePy
-    OITreeSet = OITreeSetPy
-    OITreeIterator = OITreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    weightedUnion = weightedUnionPy
-    weightedIntersection = weightedIntersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._OIBTree import OISet
-    from ._OIBTree import OIBTree
-    from ._OIBTree import OITreeSet
-    from ._OIBTree import OITreeIterator
-    from ._OIBTree import difference
-    from ._OIBTree import union
-    from ._OIBTree import intersection
-    from ._OIBTree import weightedUnion
-    from ._OIBTree import weightedIntersection
-
-
-Bucket = OIBucket
-Set = OISet
-BTree = OIBTree
-TreeSet = OITreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/OLBTree.py

@@ -38,6 +38,7 @@ from ._base import union as _union
 from ._base import weightedIntersection as _weightedIntersection
 from ._base import weightedUnion as _weightedUnion
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 60
 _TREE_SIZE = 250
@@ -103,34 +104,7 @@ intersectionPy = _set_operation(_intersection, OLSetPy)
 weightedUnionPy = _set_operation(_weightedUnion, OLSetPy)
 weightedIntersectionPy = _set_operation(_weightedIntersection, OLSetPy)
 
-try:
-    from ._OLBTree import OLBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    OLBucket = OLBucketPy
-    OLSet = OLSetPy
-    OLBTree = OLBTreePy
-    OLTreeSet = OLTreeSetPy
-    OLTreeIterator = OLTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-    weightedUnion = weightedUnionPy
-    weightedIntersection = weightedIntersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._OLBTree import OLSet
-    from ._OLBTree import OLBTree
-    from ._OLBTree import OLTreeSet
-    from ._OLBTree import OLTreeIterator
-    from ._OLBTree import difference
-    from ._OLBTree import union
-    from ._OLBTree import intersection
-    from ._OLBTree import weightedUnion
-    from ._OLBTree import weightedIntersection
-
-Bucket = OLBucket
-Set = OLSet
-BTree = OLBTree
-TreeSet = OLTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/OOBTree.py

@@ -29,9 +29,10 @@ from ._base import difference as _difference
 from ._base import intersection as _intersection
 from ._base import set_operation as _set_operation
 from ._base import to_ob as _to_key
-from ._base import to_ob as _to_value
+_to_value = _to_key
 from ._base import union as _union
 from ._base import _fix_pickle
+from ._compat import import_c_extension
 
 _BUCKET_SIZE = 30
 _TREE_SIZE = 250
@@ -83,32 +84,7 @@ differencePy = _set_operation(_difference, OOSetPy)
 unionPy = _set_operation(_union, OOSetPy)
 intersectionPy = _set_operation(_intersection, OOSetPy)
 
-try:
-    from ._OOBTree import OOBucket
-except ImportError as e: #pragma NO COVER w/ C extensions
-    OOBucket = OOBucketPy
-    OOSet = OOSetPy
-    OOBTree = OOBTreePy
-    OOTreeSet = OOTreeSetPy
-    OOTreeIterator = OOTreeIteratorPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._OOBTree import OOSet
-    from ._OOBTree import OOBTree
-    from ._OOBTree import OOTreeSet
-    from ._OOBTree import OOTreeIterator
-    from ._OOBTree import difference
-    from ._OOBTree import union
-    from ._OOBTree import intersection
-
-
-
-Bucket = OOBucket
-Set = OOSet
-BTree = OOBTree
-TreeSet = OOTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/_base.py

@@ -14,8 +14,7 @@
 """Python BTree implementation
 """
 
-from struct import pack
-from struct import unpack
+from struct import Struct
 from struct import error as struct_error
 
 from persistent import Persistent
@@ -1496,10 +1495,16 @@ def multiunion(set_type, seqs):
 def to_ob(self, v):
     return v
 
+def _packer_unpacker(struct_format):
+    s = Struct(struct_format)
+    return s.pack, s.unpack
+
+int_pack, int_unpack = _packer_unpacker('i')
+
 def to_int(self, v):
     try:
         # XXX Python 2.6 doesn't truncate, it spews a warning.
-        if not unpack("i", pack("i", v))[0] == v: #pragma: no cover
+        if not int_unpack(int_pack(v))[0] == v: #pragma: no cover
             raise TypeError('32-bit integer expected')
     except (struct_error,
             OverflowError, #PyPy
@@ -1508,17 +1513,22 @@ def to_int(self, v):
 
     return int(v)
 
+float_pack = _packer_unpacker('f')[0]
+
 def to_float(self, v):
     try:
-        pack("f", v)
+        float_pack(v)
     except struct_error:
         raise TypeError('float expected')
     return float(v)
 
+
+long_pack, long_unpack = _packer_unpacker('q')
+
 def to_long(self, v):
     try:
         # XXX Python 2.6 doesn't truncate, it spews a warning.
-        if not unpack("q", pack("q", v))[0] == v: #pragma: no cover
+        if not long_unpack(long_pack(v))[0] == v: #pragma: no cover
             if isinstance(v, int_types):
                 raise ValueError("Value out of range", v)
             raise TypeError('64-bit integer expected')

BTrees/_compat.py

@@ -11,16 +11,21 @@
 # FOR A PARTICULAR PURPOSE
 #
 ##############################################################################
+import os
 import sys
 
+PYPY = hasattr(sys, 'pypy_version_info')
+# We can and do build the C extensions on PyPy, but
+# as of Persistent 4.2.5 the persistent C extension is not
+# built on PyPy, so importing our C extension will fail anyway.
+PURE_PYTHON = os.environ.get('PURE_PYTHON', PYPY)
+
+
 if sys.version_info[0] < 3: #pragma NO COVER Python2
 
     PY2 = True
     PY3 = False
 
-    from StringIO import StringIO
-    BytesIO = StringIO
-
     int_types = int, long
     xrange = xrange
     def compare(x, y):
@@ -38,17 +43,11 @@ if sys.version_info[0] < 3: #pragma NO COVER Python2
     def _ascii(x):
         return bytes(x)
 
-    def _u(s, encoding='unicode_escape'):
-        return unicode(s, encoding)
-
 else: #pragma NO COVER Python3
 
     PY2 = False
     PY3 = True
 
-    from io import StringIO
-    from io import BytesIO
-
     int_types = int,
     xrange = range
 
@@ -67,7 +66,35 @@ else: #pragma NO COVER Python3
     def _ascii(x):
         return bytes(x, 'ascii')
 
-    def _u(s, encoding=None):
-        if encoding is None:
-            return s
-        return str(s, encoding)
+
+def import_c_extension(mod_globals):
+    import importlib
+    c_module = None
+    module_name = mod_globals['__name__']
+    assert module_name.startswith('BTrees.')
+    module_name = module_name.split('.')[1]
+    if not PURE_PYTHON:
+        try:
+            c_module = importlib.import_module('BTrees._' + module_name)
+        except ImportError:
+            pass
+    if c_module is not None:
+        new_values = dict(c_module.__dict__)
+        new_values.pop("__name__", None)
+        new_values.pop('__file__', None)
+        new_values.pop('__doc__', None)
+        mod_globals.update(new_values)
+    else:
+        # No C extension, make the Py versions available without that
+        # extension. The list comprehension both filters and prevents
+        # concurrent modification errors.
+        for py in [k for k in mod_globals if k.endswith('Py')]:
+            mod_globals[py[:-2]] = mod_globals[py]
+
+    # Assign the global aliases
+    prefix = module_name[:2]
+    for name in ('Bucket', 'Set', 'BTree', 'TreeSet'):
+        mod_globals[name] = mod_globals[prefix + name]
+
+    # Cleanup
+    del mod_globals['import_c_extension']

BTrees/fsBTree.py

@@ -35,6 +35,8 @@ from ._base import set_operation as _set_operation
 from ._base import to_bytes as _to_bytes
 from ._base import union as _union
 from ._base import _fix_pickle
+from ._compat import import_c_extension
+
 
 _BUCKET_SIZE = 500
 _TREE_SIZE = 500
@@ -101,28 +103,7 @@ differencePy = _set_operation(_difference, fsSetPy)
 unionPy = _set_operation(_union, fsSetPy)
 intersectionPy = _set_operation(_intersection, fsSetPy)
 
-try:
-    from ._fsBTree import fsBucket
-except ImportError: #pragma NO COVER w/ C extensions
-    fsBucket = fsBucketPy
-    fsSet = fsSetPy
-    fsBTree = fsBTreePy
-    fsTreeSet = fsTreeSetPy
-    difference = differencePy
-    union = unionPy
-    intersection = intersectionPy
-else: #pragma NO COVER w/o C extensions
-    from ._fsBTree import fsSet
-    from ._fsBTree import fsBTree
-    from ._fsBTree import fsTreeSet
-    from ._fsBTree import difference
-    from ._fsBTree import union
-    from ._fsBTree import intersection
-
-Bucket = fsBucket
-Set = fsSet
-BTree = fsBTree
-TreeSet = fsTreeSet
+import_c_extension(globals())
 
 _fix_pickle(globals(), __name__)
 

BTrees/tests/common.py

@@ -11,34 +11,48 @@
 # FOR A PARTICULAR PURPOSE
 #
 ##############################################################################
+from __future__ import division
 
+import unittest
 import platform
 from unittest import skip
 
 
-def _skip_wo_ZODB(test_method):  # pragma: no COVER
-    try:
-        import ZODB  # noqa
-    except ImportError:  # skip this test if ZODB is not available
-        return skip("ZODB not available")(test_method)
-    else:
-        return test_method
-
-
-def _skip_under_Py3k(test_method):  # pragma: no COVER
-    try:
-        unicode
-    except NameError:  # skip this test
-        return skip("Python 3")(test_method)
-    else:
-        return test_method
-
+from BTrees._compat import PY3
+from BTrees._compat import PURE_PYTHON
+from BTrees._compat import PYPY
 
-def _skip_on_32_bits(test_method):  # pragma: no COVER
-    if platform.architecture()[0] == '32bit':
-        return skip("32-bit platform")(test_method)
+def _no_op(test_method):
     return test_method
 
+try:
+    __import__('ZODB')
+except ImportError:
+    _skip_wo_ZODB = skip('ZODB not available')
+else:
+    _skip_wo_ZODB = _no_op
+
+if PY3:
+    _skip_under_Py3k = skip("Not on Python 3")
+else:
+    _skip_under_Py3k = _no_op
+
+if platform.architecture()[0] == '32bit':
+    _skip_on_32_bits = skip("32-bit platform")
+else:
+    _skip_on_32_bits = _no_op
+
+if PURE_PYTHON:
+    skipOnPurePython = skip("Not on Pure Python")
+else:
+    skipOnPurePython = _no_op
+
+def _skip_if_pure_py_and_py_test(self):
+    if PURE_PYTHON and 'Py' in type(self).__name__:
+        # No need to run this again. The "C" tests will catch it.
+        # This relies on the fact that we always define tests in pairs,
+        # one normal/C and one with Py in the name for the Py test.
+        raise unittest.SkipTest("Redundant with the C test")
 
 class Base(object):
     # Tests common to all types: sets, buckets, and BTrees
@@ -51,6 +65,9 @@ class Base(object):
     def _makeOne(self):
         return self._getTargetClass()()
 
+    def setUp(self):
+        super(Base, self).setUp()
+        _skip_if_pure_py_and_py_test(self)
 
     def tearDown(self):
         if self.db is not None:
@@ -77,6 +94,38 @@ class Base(object):
         transaction.abort()
         root._p_jar.close()
 
+    def testPersistentSubclass(self):
+        # Can we subclass this and Persistent?
+        # https://github.com/zopefoundation/BTrees/issues/78
+        import persistent
+
+        class PersistentSubclass(persistent.Persistent):
+            pass
+
+        __traceback_info__ = self._getTargetClass(), persistent.Persistent
+        type('Subclass', (self._getTargetClass(), PersistentSubclass), {})
+
+    def testPurePython(self):
+        import importlib
+        kind = self._getTargetClass()
+        class_name = kind.__name__
+        module_name = kind.__module__
+        module = importlib.import_module(module_name)
+
+        # If we're in pure python mode, our target class module
+        # should not have an '_' in it (fix_pickle changes the name
+        # to remove the 'Py')
+
+        # If we're in the C extension mode, our target class
+        # module still doesn't have the _ in it, but we should be able to find
+        # a Py class that's different
+
+        self.assertNotIn('_', module_name)
+        self.assertIs(getattr(module, class_name), kind)
+
+        if not PURE_PYTHON and 'Py' not in type(self).__name__:
+            self.assertIsNot(getattr(module, class_name + 'Py'), kind)
+
     @_skip_wo_ZODB
     def testLoadAndStore(self):
         import transaction
@@ -1735,6 +1784,10 @@ class TypeTest(object):
 
 class I_SetsBase(object):
 
+    def setUp(self):
+        super(I_SetsBase, self).setUp()
+        _skip_if_pure_py_and_py_test(self)
+
     def testBadBadKeyAfterFirst(self):
         t = self._makeOne()
         self.assertRaises(TypeError, t.__class__, [1, ''])
@@ -1768,7 +1821,7 @@ SMALLEST_POSITIVE_65_BITS = LARGEST_64_BITS + 1
 LARGEST_NEGATIVE_65_BITS = SMALLEST_64_BITS - 1
 
 
-class TestLongIntSupport:
+class TestLongIntSupport(object):
 
     def getTwoValues(self):
         # Return two distinct values; these must compare as un-equal.
@@ -1885,6 +1938,9 @@ def makeBuilder(mapbuilder):
 #     intersection, union, difference - set to the type-correct versions
 class SetResult(object):
     def setUp(self):
+        super(SetResult, self).setUp()
+        _skip_if_pure_py_and_py_test(self)
+
         self.Akeys = [1,    3,    5, 6   ]
         self.Bkeys = [   2, 3, 4,    6, 7]
         self.As = [makeset(self.Akeys) for makeset in self.builders()]
@@ -2167,6 +2223,10 @@ def isaset(thing):
 #     mkbucket, mkbtree
 class MultiUnion(object):
 
+    def setUp(self):
+        super(MultiUnion, self).setUp()
+        _skip_if_pure_py_and_py_test(self)
+
     def testEmpty(self):
         self.assertEqual(len(self.multiunion([])), 0)
 
@@ -2193,6 +2253,10 @@ class MultiUnion(object):
 
     def testBigInput(self):
         N = 100000
+        if (PURE_PYTHON or 'Py' in type(self).__name__) and not PYPY:
+            # This is extremely slow in CPython implemented in Python,
+            # taking 20s or more on a 2015-era laptop
+            N = N // 10
         input = self.mkset(list(range(N)))
         output = self.multiunion([input] * 10)
         self.assertEqual(len(output), N)
@@ -2234,6 +2298,10 @@ class ConflictTestBase(object):
 
     storage = None
 
+    def setUp(self):
+        super(ConflictTestBase, self).setUp()
+        _skip_if_pure_py_and_py_test(self)
+
     def tearDown(self):
         import transaction
         transaction.abort()

BTrees/tests/testBTrees.py

@@ -11,11 +11,8 @@
 # FOR A PARTICULAR PURPOSE
 #
 ##############################################################################
-import sys
 import unittest
 
-python3 = sys.version_info >= (3, )
-
 from BTrees.tests.common import permutations
 
 
@@ -171,6 +168,8 @@ class DegenerateBTree(unittest.TestCase):
         #      RuntimeWarning: tp_compare didn't return -1 or -2 for exception
         #    warnings, possibly due to memory corruption after a BTree
         #    goes insane.
+        # On CPython in PURE_PYTHON mode, this is a *slow* test, taking 15+s
+        # on a 2015 laptop.
         from BTrees.check import check
         t, keys = self._build_degenerate_tree()
         for oneperm in permutations(keys):
@@ -196,7 +195,7 @@ LP294788_ids = {}
 
 class ToBeDeleted(object):
     def __init__(self, id):
-        assert type(id) is int #we don't want to store any object ref here
+        assert isinstance(id, int) #we don't want to store any object ref here
         self.id = id
 
         global LP294788_ids
@@ -257,8 +256,6 @@ class BugFixes(unittest.TestCase):
         import gc
         import random
         from BTrees.OOBTree import OOBTree
-        from .._compat import _u
-        from .._compat import xrange
 
         t = OOBTree()
 
@@ -269,12 +266,12 @@ class BugFixes(unittest.TestCase):
         # /// BTree keys are integers, value is an object
         LP294788_ids = {}
         ids = {}
-        for i in xrange(1024):
-            if trandom.random() > 0.1 or len(ids) == 0:
+        for i in range(1024):
+            if trandom.random() > 0.1 or not ids:
                 #add
                 id = None
                 while id is None or id in ids:
-                    id = trandom.randint(0,1000000)
+                    id = trandom.randint(0, 1000000)
 
                 ids[id] = 1
                 t[id] = ToBeDeleted(id)
@@ -304,15 +301,15 @@ class BugFixes(unittest.TestCase):
         # /// BTree keys are integers, value is a tuple having an object
         LP294788_ids = {}
         ids = {}
-        for i in xrange(1024):
-            if trandom.random() > 0.1 or len(ids) == 0:
+        for i in range(1024):
+            if trandom.random() > 0.1 or not ids:
                 #add
                 id = None
                 while id is None or id in ids:
-                    id = trandom.randint(0,1000000)
+                    id = trandom.randint(0, 1000000)
 
                 ids[id] = 1
-                t[id] = (id, ToBeDeleted(id), _u('somename'))
+                t[id] = (id, ToBeDeleted(id), u'somename')
             else:
                 #del
                 keys = list(ids.keys())
@@ -341,12 +338,12 @@ class BugFixes(unittest.TestCase):
         t = OOBTree()
         LP294788_ids = {}
         ids = {}
-        for i in xrange(1024):
-            if trandom.random() > 0.1 or len(ids) == 0:
+        for i in range(1024):
+            if trandom.random() > 0.1 or not ids:
                 #add
                 id = None
                 while id is None or id in ids:
-                    id = ToBeDeleted(trandom.randint(0,1000000))
+                    id = ToBeDeleted(trandom.randint(0, 1000000))
 
                 ids[id] = 1
                 t[id] = 1
@@ -361,7 +358,7 @@ class BugFixes(unittest.TestCase):
         for id in ids:
             del t[id]
         #release all refs
-        ids = obj = id = None
+        ids = id = None
 
         #to be on the safe side run a full GC
         gc.collect()
@@ -375,13 +372,13 @@ class BugFixes(unittest.TestCase):
         t = OOBTree()
         LP294788_ids = {}
         ids = {}
-        for i in xrange(1024):
-            if trandom.random() > 0.1 or len(ids) == 0:
+        for i in range(1024):
+            if trandom.random() > 0.1 or not ids:
                 #add
                 id = None
                 while id is None or id in ids:
-                    id = trandom.randint(0,1000000)
-                    id = (id, ToBeDeleted(id), _u('somename'))
+                    id = trandom.randint(0, 1000000)
+                    id = (id, ToBeDeleted(id), u'somename')
 
                 ids[id] = 1
                 t[id] = 1
@@ -396,7 +393,7 @@ class BugFixes(unittest.TestCase):
         for id in ids:
             del t[id]
         #release all refs
-        ids = id = obj = key = None
+        ids = id = key = None
 
         #to be on the safe side run a full GC
         gc.collect()
@@ -412,7 +409,7 @@ class BugFixes(unittest.TestCase):
 
 class DoesntLikeBeingCompared:
 
-    def __cmp__(self,other):
+    def __cmp__(self, other):
         raise ValueError('incomparable')
     __lt__ = __le__ = __eq__ = __ne__ = __ge__ = __gt__ = __cmp__
 
@@ -457,13 +454,11 @@ class FamilyTest(unittest.TestCase):
         # this next bit illustrates an, um, "interesting feature".  If
         # the characteristics change to match the 64 bit version, please
         # feel free to change.
-        try: s.insert(BTrees.family32.maxint + 1)
-        except (TypeError, OverflowError): pass
-        else: self.assert_(False)
+        with self.assertRaises((TypeError, OverflowError)):
+            s.insert(BTrees.family32.maxint + 1)
 
-        try: s.insert(BTrees.family32.minint - 1)
-        except (TypeError, OverflowError): pass
-        else: self.assert_(False)
+        with self.assertRaises((TypeError, OverflowError)):
+            s.insert(BTrees.family32.minint - 1)
         self.check_pickling(BTrees.family32)
 
     def test64(self):
@@ -499,12 +494,12 @@ class FamilyTest(unittest.TestCase):
         # unpickling, whether from the same unpickler or different
         # unpicklers.
         import pickle
-        from .._compat import BytesIO
+        from io import BytesIO
 
         s = pickle.dumps((family, family))
         (f1, f2) = pickle.loads(s)
-        self.assertTrue(f1 is family)
-        self.assertTrue(f2 is family)
+        self.assertIs(f1, family)
+        self.assertIs(f2, family)
 
         # Using a single memo across multiple pickles:
         sio = BytesIO()

BTrees/tests/testBTreesUnicode.py

@@ -29,9 +29,8 @@ class TestBTreesUnicode(unittest.TestCase):
         #setup an OOBTree with some unicode strings
         from BTrees.OOBTree import OOBTree
         from BTrees._compat import _bytes
-        from BTrees._compat import _u
 
-        self.s = _u(b'dreit\xe4gigen', 'latin1')
+        self.s = b'dreit\xe4gigen'.decode('latin1')
 
         self.data = [(b'alien', 1),
                      (b'k\xf6nnten', 2),
@@ -39,23 +38,22 @@ class TestBTreesUnicode(unittest.TestCase):
                      (b'future', 4),
                      (b'quick', 5),
                      (b'zerst\xf6rt', 6),
-                     (_u(b'dreit\xe4gigen','latin1'), 7),
+                     (u'dreit\xe4gigen', 7),
                     ]
 
         self.tree = OOBTree()
         for k, v in self.data:
             if isinstance(k, _bytes):
-                k = _u(k, 'latin1')
+                k = k.decode('latin1')
             self.tree[k] = v
 
     @_skip_under_Py3k
     def testAllKeys(self):
         # check every item of the tree
-        from BTrees._compat import _u
         from BTrees._compat import _bytes
         for k, v in self.data:
             if isinstance(k, _bytes):
-                k = _u(k, encoding)
+                k = k.decode(encoding)
             self.assertTrue(k in self.tree)
             self.assertEqual(self.tree[k], v)
 

CHANGES.rst

@@ -11,7 +11,10 @@
   https://github.com/zopefoundation/BTrees/pull/55
 - Fix the possibility of a rare crash in the C extension when
   deallocating items. See https://github.com/zopefoundation/BTrees/issues/75
-
+- Respect the ``PURE_PYTHON`` environment variable at runtime even if
+  the C extensions are available. See
+  https://github.com/zopefoundation/BTrees/issues/78
+- Always attempt to build the C extensions, but make their success optional.
 
 4.4.1 (2017-01-24)
 ------------------

appveyor.yml

@@ -19,14 +19,14 @@ environment:
 install:
   - "SET PATH=C:\\Python%PYTHON%;c:\\Python%PYTHON%\\scripts;%PATH%"
   - echo "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 > "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64\vcvars64.bat"
-  - pip install -e .[ZODB]
+  - pip install -e .[test,ZODB]
 
 build_script:
   - pip install wheel
   - python -W ignore setup.py -q bdist_wheel
 
 test_script:
-  - python setup.py -q test -q
+  - zope-testrunner --test-path=.
 
 artifacts:
   - path: 'dist\*.whl'

setup.py

@@ -11,21 +11,60 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
-
+from __future__ import print_function
 __version__ = '4.4.1'
 
 import os
-import platform
 import sys
 
+from distutils.errors import CCompilerError
+from distutils.errors import DistutilsExecError
+from distutils.errors import DistutilsPlatformError
+
 from setuptools import Extension
 from setuptools import find_packages
 from setuptools import setup
-
-here = os.path.abspath(os.path.dirname(__file__))
-README = (open(os.path.join(here, 'README.rst')).read()
-          + '\n\n' +
-          open(os.path.join(here, 'CHANGES.rst')).read())
+from setuptools.command.build_ext import build_ext
+
+
+def _read(fname):
+    here = os.path.abspath(os.path.dirname(__file__))
+    with open(os.path.join(here, fname)) as f:
+        return f.read()
+
+
+README = _read("README.rst") + '\n\n' + _read('CHANGES.rst')
+
+
+class optional_build_ext(build_ext):
+    """This class subclasses build_ext and allows
+       the building of C extensions to fail.
+    """
+    def run(self):
+        try:
+            build_ext.run(self)
+        except DistutilsPlatformError as e:
+            self._unavailable(e)
+
+    def build_extension(self, ext):
+        try:
+            build_ext.build_extension(self, ext)
+        except (CCompilerError, DistutilsExecError, OSError) as e:
+            self._unavailable(e)
+
+    def _unavailable(self, e):
+        print('*' * 80)
+        print("""WARNING:
+        An optional code optimization (C extension) could not be compiled.
+        Optimizations for this package will not be available!""")
+        print()
+        print(e)
+        print('*' * 80)
+        if 'bdist_wheel' in sys.argv and not os.environ.get("PURE_PYTHON"):
+            # pip uses bdist_wheel by default, and hides the error output.
+            # Let this error percolate up so the user can see it.
+            # pip will then go ahead and run 'setup.py install' directly.
+            raise
 
 # Include directories for C extensions
 # Sniff the location of the headers in 'persistent' or fall back
@@ -49,6 +88,7 @@ class ModuleHeaderDir(object):
         path = resource_filename(self._require_spec, self._where)
         return os.path.abspath(path)
 
+
 include = [ModuleHeaderDir('persistent')]
 
 # Set up dependencies for the BTrees package
@@ -87,58 +127,47 @@ def BTreeExtension(family):
         kwargs["define_macros"] = [('EXCLUDE_INTSET_SUPPORT', None)]
     return Extension(name, sources, **kwargs)
 
-py_impl = getattr(platform, 'python_implementation', lambda: None)
-pure_python = os.environ.get('PURE_PYTHON', False)
-is_pypy = py_impl() == 'PyPy'
-is_jython = 'java' in sys.platform
 
-# Jython cannot build the C optimizations, while on PyPy they are
-# anti-optimizations (the C extension compatibility layer is known-slow,
-# and defeats JIT opportunities).
-if pure_python or is_pypy or is_jython:
-    ext_modules = []
-else:
+ext_modules = [BTreeExtension(family) for family in FAMILIES]
 
-    ext_modules = [BTreeExtension(family) for family in FAMILIES]
+REQUIRES = [
+    # 4.1.0 is the first version that PURE_PYTHON can run
+    # ZODB tests
+    'persistent >= 4.1.0',
+    'zope.interface',
+]
 
-if sys.version_info[0] >= 3:
-    REQUIRES = [
-        'persistent>=4.0.4',
-        'zope.interface',
-    ]
-else:
-    REQUIRES = [
-        'persistent',
-        'zope.interface',
-    ]
-TESTS_REQUIRE = REQUIRES + ['transaction']
+TESTS_REQUIRE = [
+    'transaction',
+    'zope.testrunner',
+]
 
 setup(name='BTrees',
       version=__version__,
       description='Scalable persistent object containers',
       long_description=README,
       classifiers=[
-        "Development Status :: 6 - Mature",
-        "License :: OSI Approved :: Zope Public License",
-        "Programming Language :: Python",
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
-        "Programming Language :: Python :: Implementation :: CPython",
-        "Programming Language :: Python :: Implementation :: PyPy",
-        "Framework :: ZODB",
-        "Topic :: Database",
-        "Topic :: Software Development :: Libraries :: Python Modules",
-        "Operating System :: Microsoft :: Windows",
-        "Operating System :: Unix",
-        ],
+          "Development Status :: 6 - Mature",
+          "License :: OSI Approved :: Zope Public License",
+          "Programming Language :: Python",
+          'Programming Language :: Python :: 2',
+          'Programming Language :: Python :: 2.7',
+          'Programming Language :: Python :: 3',
+          'Programming Language :: Python :: 3.3',
+          'Programming Language :: Python :: 3.4',
+          'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
+          "Programming Language :: Python :: Implementation :: CPython",
+          "Programming Language :: Python :: Implementation :: PyPy",
+          "Framework :: ZODB",
+          "Topic :: Database",
+          "Topic :: Software Development :: Libraries :: Python Modules",
+          "Operating System :: Microsoft :: Windows",
+          "Operating System :: Unix",
+      ],
       author="Zope Foundation",
       author_email="zodb-dev@zope.org",
-      url="http://packages.python.org/BTrees",
+      url="https://github.com/zopefoundation/BTrees",
       license="ZPL 2.1",
       platforms=["any"],
       packages=find_packages(),
@@ -147,14 +176,21 @@ setup(name='BTrees',
       ext_modules=ext_modules,
       setup_requires=['persistent'],
       extras_require={
-        'test': TESTS_REQUIRE,
-        'ZODB': ['ZODB'],
-        'testing': TESTS_REQUIRE + ['nose', 'coverage'],
-        'docs': ['Sphinx', 'repoze.sphinx.autointerface'],
+          'test': TESTS_REQUIRE,
+          'ZODB': [
+              'ZODB',
+          ],
+          'docs': [
+              'Sphinx',
+              'repoze.sphinx.autointerface',
+          ],
       },
       test_suite="BTrees.tests",
       tests_require=TESTS_REQUIRE,
       install_requires=REQUIRES,
+      cmdclass={
+          'build_ext': optional_build_ext,
+      },
       entry_points="""\
       """
-      )
+)

tox.ini

@@ -6,34 +6,23 @@ envlist =
     py27,py27-pure,pypy,py33,py34,py35,py35-pure,py36,pypy3,w_zodb,coverage,docs
 
 [testenv]
+usedevelop = true
 deps =
-    zope.interface
-    persistent
-    transaction
+    .[test]
 commands =
-    python setup.py -q test -q
+    zope-testrunner --test-path=. --auto-color --auto-progress []
 
 [testenv:py27-pure]
 basepython =
     python2.7
 setenv =
     PURE_PYTHON = 1
-    PIP_CACHE_DIR = {envdir}/.cache
-deps =
-    {[testenv]deps}
-commands =
-    python setup.py -q test -q
 
 [testenv:py35-pure]
 basepython =
     python3.5
 setenv =
     PURE_PYTHON = 1
-    PIP_CACHE_DIR = {envdir}/.cache
-deps =
-    {[testenv]deps}
-commands =
-    python setup.py -q test -q
 
 #[testenv:jython]
 #commands =
@@ -42,29 +31,19 @@ commands =
 [testenv:w_zodb]
 basepython =
     python2.7
-commands =
-    python setup.py -q test -q
 deps =
-    zope.interface
-    persistent
-    transaction
+    {[testenv]deps}
     ZODB
-    nose
-    coverage
-    nosexcover
 
 [testenv:coverage]
 basepython =
     python2.7
 commands =
-    nosetests --with-xunit --with-xcoverage --cover-package=BTrees
+    coverage run -m zope.testrunner --test-path=. --auto-color --auto-progress []
+    coverage report
 deps =
-    zope.interface
-    persistent
-    transaction
-    nose
+    {[testenv]deps}
     coverage
-    nosexcover
 
 [testenv:docs]
 basepython =
@@ -73,8 +52,4 @@ commands =
     sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html
     sphinx-build -b doctest -d docs/_build/doctrees docs docs/_build/doctest
 deps =
-    zope.interface
-    persistent
-    transaction
-    Sphinx
-    repoze.sphinx.autointerface
+    .[docs]