Issue #13555: cPickle now supports files larger than 2 GiB.

Lib/test/pickletester.py

@@ -6,7 +6,8 @@ import cStringIO
 import pickletools
 import copy_reg
 
-from test.test_support import TestFailed, have_unicode, TESTFN
+from test.test_support import (TestFailed, have_unicode, TESTFN, _2G, _1M,
+                               precisionbigmemtest)
 
 # Tests that try a number of pickle protocols should have a
 #     for proto in protocols:
@@ -1280,3 +1281,31 @@ class AbstractPicklerUnpicklerObjectTests(unittest.TestCase):
         f.write(pickled2)
         f.seek(0)
         self.assertEqual(unpickler.load(), data2)
+
+class BigmemPickleTests(unittest.TestCase):
+
+    # Memory requirements: 1 byte per character for input strings, 1 byte
+    # for pickled data, 1 byte for unpickled strings, 1 byte for internal
+    # buffer and 1 byte of free space for resizing of internal buffer.
+
+    @precisionbigmemtest(size=_2G + 100*_1M, memuse=5)
+    def test_huge_strlist(self, size):
+        chunksize = 2**20
+        data = []
+        while size > chunksize:
+            data.append('x' * chunksize)
+            size -= chunksize
+            chunksize += 1
+        data.append('y' * size)
+
+        try:
+            for proto in protocols:
+                try:
+                    pickled = self.dumps(data, proto)
+                    res = self.loads(pickled)
+                    self.assertEqual(res, data)
+                finally:
+                    res = None
+                    pickled = None
+        finally:
+            data = None

Lib/test/test_cpickle.py

 import cPickle, unittest
 from cStringIO import StringIO
-from test.pickletester import AbstractPickleTests, AbstractPickleModuleTests
-from test.pickletester import AbstractPicklerUnpicklerObjectTests
+from test.pickletester import (AbstractPickleTests,
+                               AbstractPickleModuleTests,
+                               AbstractPicklerUnpicklerObjectTests,
+                               BigmemPickleTests)
 from test import test_support
 
 class cPickleTests(AbstractPickleTests, AbstractPickleModuleTests):
@@ -101,6 +103,16 @@ class cPicklePicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests):
     pickler_class = cPickle.Pickler
     unpickler_class = cPickle.Unpickler
 
+class cPickleBigmemPickleTests(BigmemPickleTests):
+
+    def dumps(self, arg, proto=0, fast=0):
+        # Ignore fast
+        return cPickle.dumps(arg, proto)
+
+    def loads(self, buf):
+        # Ignore fast
+        return cPickle.loads(buf)
+
 
 class Node(object):
     pass
@@ -133,6 +145,7 @@ def test_main():
         cPickleFastPicklerTests,
         cPickleDeepRecursive,
         cPicklePicklerUnpicklerObjectTests,
+        cPickleBigmemPickleTests,
     )
 
 if __name__ == "__main__":

Lib/test/test_pickle.py

@@ -3,10 +3,11 @@ from cStringIO import StringIO
 
 from test import test_support
 
-from test.pickletester import AbstractPickleTests
-from test.pickletester import AbstractPickleModuleTests
-from test.pickletester import AbstractPersistentPicklerTests
-from test.pickletester import AbstractPicklerUnpicklerObjectTests
+from test.pickletester import (AbstractPickleTests,
+                               AbstractPickleModuleTests,
+                               AbstractPersistentPicklerTests,
+                               AbstractPicklerUnpicklerObjectTests,
+                               BigmemPickleTests)
 
 class PickleTests(AbstractPickleTests, AbstractPickleModuleTests):
 
@@ -66,6 +67,16 @@ class PicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests):
     pickler_class = pickle.Pickler
     unpickler_class = pickle.Unpickler
 
+class PickleBigmemPickleTests(BigmemPickleTests):
+
+    def dumps(self, arg, proto=0, fast=0):
+        # Ignore fast
+        return pickle.dumps(arg, proto)
+
+    def loads(self, buf):
+        # Ignore fast
+        return pickle.loads(buf)
+
 
 def test_main():
     test_support.run_unittest(
@@ -73,6 +84,7 @@ def test_main():
         PicklerTests,
         PersPicklerTests,
         PicklerUnpicklerObjectTests,
+        PickleBigmemPickleTests,
     )
     test_support.run_doctest(pickle)
 

Misc/NEWS

@@ -202,6 +202,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #13555: cPickle now supports files larger than 2 GiB.
+
 - Issue #17052: unittest discovery should use self.testLoader.
 
 - Issue #4591: Uid and gid values larger than 2**31 are supported now.

Modules/cStringIO.c

@@ -210,11 +210,8 @@ IO_creadline(PyObject *self, char **output) {
     if (n < end) n++;
 
     len = n - start;
-    if (len > INT_MAX) {
-        PyErr_SetString(PyExc_OverflowError,
-                        "length too large");
-        return -1;
-    }
+    if (len > INT_MAX)
+        len = INT_MAX;
 
     *output=start;