bpo-34922: Fix integer overflow in the digest() and hexdigest() methods (GH-9751)

for the SHAKE algorithm in the hashlib module.

Lib/test/test_hashlib.py

@@ -230,6 +230,19 @@ class HashLibTestCase(unittest.TestCase):
                 self.assertIsInstance(h.digest(), bytes)
                 self.assertEqual(hexstr(h.digest()), h.hexdigest())
 
+    def test_digest_length_overflow(self):
+        # See issue #34922
+        large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10)
+        for cons in self.hash_constructors:
+            h = cons()
+            if h.name not in self.shakes:
+                continue
+            for digest in h.digest, h.hexdigest:
+                self.assertRaises(ValueError, digest, -10)
+                for length in large_sizes:
+                    with self.assertRaises((ValueError, OverflowError)):
+                        digest(length)
+
     def test_name_attribute(self):
         for cons in self.hash_constructors:
             h = cons()

Misc/NEWS.d/next/Library/2018-10-07-21-18-52.bpo-34922.37IdsA.rst

+Fixed integer overflow in the :meth:`~hashlib.shake.digest()` and
+:meth:`~hashlib.shake.hexdigest()` methods for the SHAKE algorithm
+in the :mod:`hashlib` module.

Modules/_sha3/sha3module.c

@@ -589,6 +589,10 @@ _SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex)
     int res;
     PyObject *result = NULL;
 
+    if (digestlen >= (1 << 29)) {
+        PyErr_SetString(PyExc_ValueError, "length is too large");
+        return NULL;
+    }
     /* ExtractLane needs at least SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE and
      * SHA3_LANESIZE extra space.
      */