Fix #140 by turning OverflowError into TypeError.

Fingers crossed for 32-bit builds on Windows.

BTrees/BTreeModuleTemplate.c

@@ -82,9 +82,11 @@ longlong_handle_overflow(PY_LONG_LONG result, int overflow)
 {
     if (overflow)
     {
-        /* Python 3 tends to have an exception already set, Python 2 not so much */
-        if (!PyErr_Occurred())
-            PyErr_SetString(PyExc_OverflowError, "couldn't convert integer to C long long");
+        PyErr_Clear();
+        /* Python 3 tends to have an exception already set, Python 2 not so
+           much. We want to consistently raise a TypeError.
+        */
+        PyErr_SetString(PyExc_TypeError, "couldn't convert integer to C long long");
         return 0;
     }
     else if (result == -1 && PyErr_Occurred())
@@ -106,7 +108,7 @@ ulonglong_check(PyObject *ob)
         long tmp;
         tmp = PyInt_AS_LONG(ob);
         if (tmp < 0) {
-            PyErr_SetString(PyExc_OverflowError, "unsigned value less than 0");
+            PyErr_SetString(PyExc_TypeError, "unsigned value less than 0");
             return 0;
         }
         return 1;
@@ -166,7 +168,7 @@ ulonglong_convert(PyObject *ob, unsigned PY_LONG_LONG *value)
         long tmp;
         tmp = PyInt_AS_LONG(ob);
         if (tmp < 0) {
-            PyErr_SetString(PyExc_OverflowError, "unsigned value less than 0");
+            PyErr_SetString(PyExc_TypeError, "unsigned value less than 0");
             return 0;
         }
         (*value) = (unsigned PY_LONG_LONG)tmp;
@@ -182,7 +184,14 @@ ulonglong_convert(PyObject *ob, unsigned PY_LONG_LONG *value)
 
     val = PyLong_AsUnsignedLongLong(ob);
     if (val == (unsigned long long)-1 && PyErr_Occurred())
+    {
+        if (PyErr_ExceptionMatches(PyExc_OverflowError))
+        {
+            PyErr_Clear();
+            PyErr_SetString(PyExc_TypeError, "overflow error converting int to C long long");
+        }
         return 0;
+    }
     (*value) = val;
     return 1;
 }

BTrees/BTreeTemplate.c

@@ -1971,12 +1971,6 @@ BTree_getm(BTree *self, PyObject *args)
     return d;
 }
 
-static PyObject *
-BTree_has_key(BTree *self, PyObject *key)
-{
-    return _BTree_get(self, key, 1, _BGET_REPLACE_TYPE_ERROR);
-}
-
 static PyObject *
 BTree_setdefault(BTree *self, PyObject *args)
 {
@@ -2081,6 +2075,21 @@ BTree_contains(BTree *self, PyObject *key)
     return result;
 }
 
+static PyObject *
+BTree_has_key(BTree *self, PyObject *key)
+{
+    int result = -1;
+    result = BTree_contains(self, key);
+    if (result == -1) {
+        return NULL;
+    }
+
+    if (result)
+        Py_RETURN_TRUE;
+    Py_RETURN_FALSE;
+}
+
+
 static PyObject *
 BTree_addUnique(BTree *self, PyObject *args)
 {

BTrees/BucketTemplate.c

@@ -1367,11 +1367,6 @@ bucket_setstate(Bucket *self, PyObject *state)
     return Py_None;
 }
 
-static PyObject *
-bucket_has_key(Bucket *self, PyObject *key)
-{
-    return _bucket_get(self, key, 1);
-}
 
 static PyObject *
 bucket_setdefault(Bucket *self, PyObject *args)
@@ -1475,6 +1470,20 @@ bucket_contains(Bucket *self, PyObject *key)
     return result;
 }
 
+static PyObject *
+bucket_has_key(Bucket *self, PyObject *key)
+{
+    int result = -1;
+    result = bucket_contains(self, key);
+    if (result == -1) {
+        return NULL;
+    }
+
+    if (result)
+        Py_RETURN_TRUE;
+    Py_RETURN_FALSE;
+}
+
 /*
 ** bucket_getm
 **

BTrees/_datatypes.py

@@ -55,7 +55,7 @@ class DataType(object):
         """
         Convert *item* into the correct format and return it.
 
-        If this cannot be done, raise an appropriate exception.
+        If this cannot be done, raise a :exc:`TypeError`.
         """
         raise NotImplementedError
 
@@ -239,7 +239,7 @@ class _AbstractNativeDataType(KeyDataType):
             # PyPy can raise ValueError converting a negative number to a
             # unsigned value.
             if isinstance(item, int_types):
-                raise OverflowError("Value out of range", item)
+                raise TypeError("Value out of range", item)
             raise TypeError(self._error_description)
 
         return self._as_python_type(item)

BTrees/intkeymacros.h

@@ -23,9 +23,15 @@
 #define COPY_KEY_FROM_ARG(TARGET, ARG, STATUS)                    \
   if (INT_CHECK(ARG)) {                                           \
       long vcopy = INT_AS_LONG(ARG);                              \
-      if (PyErr_Occurred()) { (STATUS)=0; (TARGET)=0; }           \
+      if (PyErr_Occurred()) {                                     \
+        if (PyErr_ExceptionMatches(PyExc_OverflowError)) {        \
+            PyErr_Clear();                                        \
+            PyErr_SetString(PyExc_TypeError, "integer out of range"); \
+        }                                                         \
+        (STATUS)=0; (TARGET)=0;                                   \
+      }                                                           \
       else if ((int)vcopy != vcopy) {                             \
-        PyErr_SetString(PyExc_OverflowError, "integer out of range"); \
+        PyErr_SetString(PyExc_TypeError, "integer out of range"); \
         (STATUS)=0; (TARGET)=0;                                   \
       }                                                           \
       else TARGET = vcopy;                                        \
@@ -55,13 +61,19 @@
 #define COPY_KEY_FROM_ARG(TARGET, ARG, STATUS)                    \
   if (INT_CHECK(ARG)) {                                           \
       long vcopy = INT_AS_LONG(ARG);                     \
-      if (PyErr_Occurred()) { (STATUS)=0; (TARGET)=0; }           \
+      if (PyErr_Occurred()) {                                     \
+        if (PyErr_ExceptionMatches(PyExc_OverflowError)) {        \
+            PyErr_Clear();                                        \
+            PyErr_SetString(PyExc_TypeError, "integer out of range"); \
+        }                                                         \
+        (STATUS)=0; (TARGET)=0;                                   \
+      }                                                           \
       else if (vcopy < 0) {                                       \
-        PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned int"); \
+        PyErr_SetString(PyExc_TypeError, "can't convert negative value to unsigned int"); \
         (STATUS)=0; (TARGET)=0;                                   \
       }                                                           \
       else if ((unsigned int)vcopy != vcopy) {                     \
-        PyErr_SetString(PyExc_OverflowError, "integer out of range"); \
+        PyErr_SetString(PyExc_TypeError, "integer out of range"); \
         (STATUS)=0; (TARGET)=0;                                   \
       }                                                           \
       else TARGET = vcopy;                                        \

BTrees/intvaluemacros.h

@@ -31,7 +31,7 @@
       long vcopy = INT_AS_LONG(ARG);                            \
       if (PyErr_Occurred()) { (STATUS)=0; (TARGET)=0; }           \
       else if ((int)vcopy != vcopy) {                                  \
-        PyErr_SetString(PyExc_OverflowError, "integer out of range"); \
+        PyErr_SetString(PyExc_TypeError, "integer out of range"); \
         (STATUS)=0; (TARGET)=0;                                   \
       }                                                           \
       else TARGET = vcopy;                                        \
@@ -64,11 +64,11 @@
       long vcopy = INT_AS_LONG(ARG);                              \
       if (PyErr_Occurred()) { (STATUS)=0; (TARGET)=0; }           \
       else if (vcopy < 0) {                                       \
-        PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned int"); \
+        PyErr_SetString(PyExc_TypeError, "can't convert negative value to unsigned int"); \
         (STATUS)=0; (TARGET)=0;                                   \
-      }								  \
+      }                                                           \
       else if ((unsigned int)vcopy != vcopy) {                    \
-        PyErr_SetString(PyExc_OverflowError, "integer out of range"); \
+        PyErr_SetString(PyExc_TypeError, "integer out of range"); \
         (STATUS)=0; (TARGET)=0;                                   \
       }                                                           \
       else TARGET = vcopy;                                        \

BTrees/tests/common.py

@@ -449,6 +449,19 @@ class MappingBase(Base):
         self.assertEqual(self._makeOne().get(1), None)
         self.assertEqual(self._makeOne().get(1, 'foo'), 'foo')
 
+    def testGetReturnsDefaultWrongTypes(self):
+        self.assertIsNone(self._makeOne().get('abc'))
+        self.assertEqual(self._makeOne().get('abc', 'def'), 'def')
+
+    def testGetReturnsDefaultOverflowRanges(self):
+        too_big = 2 ** 64 + 1
+        self.assertIsNone(self._makeOne().get(too_big))
+        self.assertEqual(self._makeOne().get(too_big, 'def'), 'def')
+
+        too_small = -too_big
+        self.assertIsNone(self._makeOne().get(too_small))
+        self.assertEqual(self._makeOne().get(too_small, 'def'), 'def')
+
     def testSetItemGetItemWorks(self):
         t = self._makeOne()
         t[1] = 1
@@ -475,14 +488,24 @@ class MappingBase(Base):
         self.assertEqual(len(t), len(addl), len(t))
 
     def testHasKeyWorks(self):
-        from .._compat import PY2
         t = self._makeOne()
         t[1] = 1
-        if PY2:
-            self.assertTrue(t.has_key(1))
-        self.assertTrue(1 in t)
-        self.assertTrue(0 not in t)
-        self.assertTrue(2 not in t)
+        self.assertTrue(t.has_key(1))
+
+        self.assertIn(1, t)
+        self.assertNotIn(0, t)
+        self.assertNotIn(2, t)
+
+    def testHasKeyOverflowAndTypes(self):
+        t = self._makeOne()
+
+        too_big = 2 ** 64 + 1
+        too_small = -too_big
+        self.assertNotIn(too_big, t)
+        self.assertNotIn(too_small, t)
+        self.assertFalse(t.has_key(too_big))
+        self.assertFalse(t.has_key(too_small))
+        self.assertFalse(t.has_key('abc'))
 
     def testValuesWorks(self):
         t = self._makeOne()
@@ -1061,10 +1084,11 @@ class MappingBase(Base):
         # for values that are in range on most other platforms. And on Python 2,
         # PyInt_Check can fail with a TypeError starting at small values
         # like 2147483648. So we look for small longs and catch those errors
-        # even when we think we should be in range.
+        # even when we think we should be in range. In all cases, our code
+        # catches the unexpected error (OverflowError) and turns it into TypeError.
         long_is_32_bit = struct.calcsize('@l') < 8
-        in_range_errors = (OverflowError, TypeError) if long_is_32_bit else ()
-        out_of_range_errors = (OverflowError, TypeError) if long_is_32_bit and PY2 else (OverflowError,)
+        in_range_errors = TypeError
+        out_of_range_errors = TypeError
 
         def trial(i):
             i = int(i)
@@ -2143,11 +2167,11 @@ class TestLongIntKeys(TestLongIntSupport):
         o1, o2 = self.getTwoValues()
         t = self._makeOne()
         k1 = SMALLEST_POSITIVE_65_BITS if self.SUPPORTS_NEGATIVE_KEYS else 2**64 + 1
-        with self.assertRaises(OverflowError):
+        with self.assertRaises(TypeError):
             t[k1] = o1
 
         t = self._makeOne()
-        with self.assertRaises(OverflowError):
+        with self.assertRaises(TypeError):
             t[LARGEST_NEGATIVE_65_BITS] = o1
 
 class TestLongIntValues(TestLongIntSupport):
@@ -2175,11 +2199,11 @@ class TestLongIntValues(TestLongIntSupport):
         k1, k2 = self.getTwoKeys()
         t = self._makeOne()
         v1 = SMALLEST_POSITIVE_65_BITS if self.SUPPORTS_NEGATIVE_VALUES else 2**64 + 1
-        with self.assertRaises(OverflowError):
+        with self.assertRaises(TypeError):
             t[k1] = v1
 
         t = self._makeOne()
-        with self.assertRaises(OverflowError):
+        with self.assertRaises(TypeError):
             t[k1] = LARGEST_NEGATIVE_65_BITS
 
 

BTrees/tests/testBTrees.py

@@ -114,14 +114,14 @@ class DegenerateBTree(unittest.TestCase):
         t, keys = self._build_degenerate_tree()
         self.assertEqual(len(t), len(keys))
         self.assertEqual(list(t.keys()), keys)
-        # has_key actually returns the depth of a bucket.
-        self.assertEqual(t.has_key(1), 4)
-        self.assertEqual(t.has_key(3), 4)
-        self.assertEqual(t.has_key(5), 6)
-        self.assertEqual(t.has_key(7), 5)
-        self.assertEqual(t.has_key(11), 5)
+
+        self.assertTrue(t.has_key(1))
+        self.assertTrue(t.has_key(3))
+        self.assertTrue(t.has_key(5))
+        self.assertTrue(t.has_key(7))
+        self.assertTrue(t.has_key(11))
         for i in 0, 2, 4, 6, 8, 9, 10, 12:
-            self.assertTrue(i not in t)
+            self.assertNotIn(i, t)
 
     def _checkRanges(self, tree, keys):
         self.assertEqual(len(tree), len(keys))

BTrees/tests/test__datatypes.py

@@ -36,7 +36,7 @@ class TestDatatypes(unittest.TestCase):
             pass
 
     def test_to_int_w_overflow(self):
-        self.assertRaises(OverflowError, to_int, 2**64)
+        self.assertRaises(TypeError, to_int, 2**64)
 
     def test_to_int_w_invalid(self):
         self.assertRaises(TypeError, to_int, ())
@@ -60,7 +60,7 @@ class TestDatatypes(unittest.TestCase):
             pass
 
     def test_to_long_w_overflow(self):
-        self.assertRaises(OverflowError, to_long, 2**64)
+        self.assertRaises(TypeError, to_long, 2**64)
 
     def test_to_long_w_invalid(self):
         self.assertRaises(TypeError, to_long, ())

CHANGES.rst

@@ -5,7 +5,15 @@
 4.7.2 (unreleased)
 ==================
 
-- Nothing changed yet.
+- Fix more cases of C and Python inconsistency. The C implementation
+  now behaves like the Python implementation when it comes to integer
+  overflow for the integer keys for ``in``, ``get`` and ``has_key``.
+  Now they return False, the default value, and False, respectively in
+  both versions if the tested value would overflow or underflow.
+  Previously, the C implementation would raise ``OverflowError`` or
+  ``KeyError``, while the Python implementation functioned as
+  expected. See `issue 140
+  <https://github.com/zopefoundation/BTrees/issues/140>`_.
 
 
 4.7.1 (2020-03-22)

docs/overview.rst

@@ -168,10 +168,10 @@ exclusive of the range's endpoints.
    [1, 2, 3, 4]
    >>> [pair for pair in t.iteritems()]  # new in ZODB 3.3
    [(1, 'red'), (2, 'green'), (3, 'blue'), (4, 'spades')]
-   >>> t.has_key(4)  # returns a true value, but exactly what undefined
-   2
+   >>> t.has_key(4)  # returns a true value
+   True
    >>> t.has_key(5)
-   0
+   False
    >>> 4 in t  # new in ZODB 3.3
    True
    >>> 5 in t  # new in ZODB 3.3
@@ -256,10 +256,10 @@ example, lists supply a total ordering, and then
    >>> list(s.keys())           # note that the lists are in sorted order
    [[1], [2], [3]]
    >>> s.has_key([3])           # and [3] is in the set
-   1
+   True
    >>> L2[0] = 5                # horrible -- the set is insane now
    >>> s.has_key([3])           # for example, it's insane this way
-   0
+   False
    >>> s
    OOSet([[1], [5], [3]])
    >>>