Issue #10276: Fix the results of zlib.crc32() and zlib.adler32() on buffers

larger than 4GB.  Patch by Nadeem Vawda.

Lib/test/test_zlib.py

@@ -2,10 +2,16 @@ import unittest
 from test import support
 import binascii
 import random
+import sys
 from test.support import precisionbigmemtest, _1G, _4G
 
 zlib = support.import_module('zlib')
 
+try:
+    import mmap
+except ImportError:
+    mmap = None
+
 
 class ChecksumTestCase(unittest.TestCase):
     # checksum test cases
@@ -57,6 +63,28 @@ class ChecksumTestCase(unittest.TestCase):
         self.assertEqual(binascii.crc32(b'spam'), zlib.crc32(b'spam'))
 
 
+# Issue #10276 - check that inputs >=4GB are handled correctly.
+class ChecksumBigBufferTestCase(unittest.TestCase):
+
+    def setUp(self):
+        with open(support.TESTFN, "wb+") as f:
+            f.seek(_4G)
+            f.write(b"asdf")
+            f.flush()
+            self.mapping = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+
+    def tearDown(self):
+        self.mapping.close()
+        support.unlink(support.TESTFN)
+
+    @unittest.skipUnless(mmap, "mmap() is not available.")
+    @unittest.skipUnless(sys.maxsize > _4G, "Can't run on a 32-bit system.")
+    @unittest.skipUnless(support.is_resource_enabled("largefile"),
+                         "May use lots of disk space.")
+    def test_big_buffer(self):
+        self.assertEqual(zlib.crc32(self.mapping), 3058686908)
+        self.assertEqual(zlib.adler32(self.mapping), 82837919)
+
 
 class ExceptionTestCase(unittest.TestCase):
     # make sure we generate some expected errors
@@ -577,6 +605,7 @@ LAERTES
 def test_main():
     support.run_unittest(
         ChecksumTestCase,
+        ChecksumBigBufferTestCase,
         ExceptionTestCase,
         CompressTestCase,
         CompressObjectTestCase

Misc/NEWS

@@ -12,6 +12,12 @@ Core and Builtins
 
 - Check for NULL result in PyType_FromSpec.
 
+Library
+-------
+
+- Issue #10276: Fix the results of zlib.crc32() and zlib.adler32() on buffers
+  larger than 4GB.  Patch by Nadeem Vawda.
+
 
 What's New in Python 3.2?
 =========================

Modules/zlibmodule.c

@@ -945,8 +945,18 @@ PyZlib_adler32(PyObject *self, PyObject *args)
     /* Releasing the GIL for very small buffers is inefficient
        and may lower performance */
     if (pbuf.len > 1024*5) {
+        void *buf = pbuf.buf;
+        Py_ssize_t len = pbuf.len;
+
         Py_BEGIN_ALLOW_THREADS
-        adler32val = adler32(adler32val, pbuf.buf, pbuf.len);
+        /* Avoid truncation of length for very large buffers. adler32() takes
+           length as an unsigned int, which may be narrower than Py_ssize_t. */
+        while (len > (Py_ssize_t)UINT_MAX) {
+            adler32val = adler32(adler32val, buf, UINT_MAX);
+            buf += UINT_MAX;
+            len -= UINT_MAX;
+        }
+        adler32val = adler32(adler32val, buf, len);
         Py_END_ALLOW_THREADS
     } else {
         adler32val = adler32(adler32val, pbuf.buf, pbuf.len);
@@ -973,8 +983,18 @@ PyZlib_crc32(PyObject *self, PyObject *args)
     /* Releasing the GIL for very small buffers is inefficient
        and may lower performance */
     if (pbuf.len > 1024*5) {
+        void *buf = pbuf.buf;
+        Py_ssize_t len = pbuf.len;
+
         Py_BEGIN_ALLOW_THREADS
-        signed_val = crc32(crc32val, pbuf.buf, pbuf.len);
+        /* Avoid truncation of length for very large buffers. crc32() takes
+           length as an unsigned int, which may be narrower than Py_ssize_t. */
+        while (len > (Py_ssize_t)UINT_MAX) {
+            crc32val = crc32(crc32val, buf, UINT_MAX);
+            buf += UINT_MAX;
+            len -= UINT_MAX;
+        }
+        signed_val = crc32(crc32val, buf, len);
         Py_END_ALLOW_THREADS
     } else {
         signed_val = crc32(crc32val, pbuf.buf, pbuf.len);