Commit 6e73ff1a authored by Alexandre Vassalotti's avatar Alexandre Vassalotti

Issue #19881: Fix bad pickling of large bytes in cpickle.

parent c9a87e6b
...@@ -3,6 +3,7 @@ import io ...@@ -3,6 +3,7 @@ import io
import pickle import pickle
import pickletools import pickletools
import random import random
import struct
import sys import sys
import unittest import unittest
import weakref import weakref
...@@ -1611,9 +1612,9 @@ class BigmemPickleTests(unittest.TestCase): ...@@ -1611,9 +1612,9 @@ class BigmemPickleTests(unittest.TestCase):
data = 1 << (8 * size) data = 1 << (8 * size)
try: try:
for proto in protocols: for proto in protocols:
if proto < 2:
continue
with self.subTest(proto=proto): with self.subTest(proto=proto):
if proto < 2:
continue
with self.assertRaises((ValueError, OverflowError)): with self.assertRaises((ValueError, OverflowError)):
self.dumps(data, protocol=proto) self.dumps(data, protocol=proto)
finally: finally:
...@@ -1628,13 +1629,17 @@ class BigmemPickleTests(unittest.TestCase): ...@@ -1628,13 +1629,17 @@ class BigmemPickleTests(unittest.TestCase):
data = b"abcd" * (size // 4) data = b"abcd" * (size // 4)
try: try:
for proto in protocols: for proto in protocols:
if proto < 3:
continue
with self.subTest(proto=proto): with self.subTest(proto=proto):
if proto < 3:
continue
try: try:
pickled = self.dumps(data, protocol=proto) pickled = self.dumps(data, protocol=proto)
self.assertTrue(b"abcd" in pickled[:19]) header = (pickle.BINBYTES +
self.assertTrue(b"abcd" in pickled[-18:]) struct.pack("<I", len(data)))
data_start = pickled.index(data)
self.assertEqual(
header,
pickled[data_start-len(header):data_start])
finally: finally:
pickled = None pickled = None
finally: finally:
...@@ -1642,14 +1647,28 @@ class BigmemPickleTests(unittest.TestCase): ...@@ -1642,14 +1647,28 @@ class BigmemPickleTests(unittest.TestCase):
@bigmemtest(size=_4G, memuse=1 + 1, dry_run=False) @bigmemtest(size=_4G, memuse=1 + 1, dry_run=False)
def test_huge_bytes_64b(self, size): def test_huge_bytes_64b(self, size):
data = b"a" * size data = b"acbd" * (size // 4)
try: try:
for proto in protocols: for proto in protocols:
if proto < 3:
continue
with self.subTest(proto=proto): with self.subTest(proto=proto):
if proto < 3: if proto == 3:
# Protocol 3 does not support large bytes objects.
# Verify that we do not crash when processing one.
with self.assertRaises((ValueError, OverflowError)):
self.dumps(data, protocol=proto)
continue continue
with self.assertRaises((ValueError, OverflowError)): try:
self.dumps(data, protocol=proto) pickled = self.dumps(data, protocol=proto)
header = (pickle.BINBYTES8 +
struct.pack("<Q", len(data)))
data_start = pickled.index(data)
self.assertEqual(
header,
pickled[data_start-len(header):data_start])
finally:
pickled = None
finally: finally:
data = None data = None
...@@ -1661,11 +1680,19 @@ class BigmemPickleTests(unittest.TestCase): ...@@ -1661,11 +1680,19 @@ class BigmemPickleTests(unittest.TestCase):
data = "abcd" * (size // 4) data = "abcd" * (size // 4)
try: try:
for proto in protocols: for proto in protocols:
if proto == 0:
continue
with self.subTest(proto=proto): with self.subTest(proto=proto):
try: try:
pickled = self.dumps(data, protocol=proto) pickled = self.dumps(data, protocol=proto)
self.assertTrue(b"abcd" in pickled[:19]) header = (pickle.BINUNICODE +
self.assertTrue(b"abcd" in pickled[-18:]) struct.pack("<I", len(data)))
data_start = pickled.index(b'abcd')
self.assertEqual(
header,
pickled[data_start-len(header):data_start])
self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
pickled.index(b"abcd")), len(data))
finally: finally:
pickled = None pickled = None
finally: finally:
...@@ -1680,19 +1707,25 @@ class BigmemPickleTests(unittest.TestCase): ...@@ -1680,19 +1707,25 @@ class BigmemPickleTests(unittest.TestCase):
data = "abcd" * (size // 4) data = "abcd" * (size // 4)
try: try:
for proto in protocols: for proto in protocols:
if proto == 0:
continue
with self.subTest(proto=proto): with self.subTest(proto=proto):
if proto == 0:
continue
if proto < 4: if proto < 4:
with self.assertRaises((ValueError, OverflowError)): with self.assertRaises((ValueError, OverflowError)):
self.dumps(data, protocol=proto) self.dumps(data, protocol=proto)
else: continue
try: try:
pickled = self.dumps(data, protocol=proto) pickled = self.dumps(data, protocol=proto)
self.assertTrue(b"abcd" in pickled[:19]) header = (pickle.BINUNICODE8 +
self.assertTrue(b"abcd" in pickled[-18:]) struct.pack("<Q", len(data)))
finally: data_start = pickled.index(b'abcd')
pickled = None self.assertEqual(
header,
pickled[data_start-len(header):data_start])
self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
pickled.index(b"abcd")), len(data))
finally:
pickled = None
finally: finally:
data = None data = None
......
...@@ -18,11 +18,15 @@ Core and Builtins ...@@ -18,11 +18,15 @@ Core and Builtins
Library Library
------- -------
- Issue #19296: Silence compiler warning in dbm_open - Issue #19296: Silence compiler warning in dbm_open
- Issue #19839: Fix regression in bz2 module's handling of non-bzip2 data at - Issue #19839: Fix regression in bz2 module's handling of non-bzip2 data at
EOF, and analogous bug in lzma module. EOF, and analogous bug in lzma module.
- Issue #19881: Fix pickling bug where cpickle would emit bad pickle data for
large bytes string (i.e., with size greater than 2**32-1).
- Issue #19138: doctest's IGNORE_EXCEPTION_DETAIL now allows a match when - Issue #19138: doctest's IGNORE_EXCEPTION_DETAIL now allows a match when
no exception detail exists (no colon following the exception's name, or no exception detail exists (no colon following the exception's name, or
a colon does follow but no text follows the colon). a colon does follow but no text follows the colon).
......
...@@ -2027,7 +2027,7 @@ save_bytes(PicklerObject *self, PyObject *obj) ...@@ -2027,7 +2027,7 @@ save_bytes(PicklerObject *self, PyObject *obj)
else if (self->proto >= 4) { else if (self->proto >= 4) {
header[0] = BINBYTES8; header[0] = BINBYTES8;
_write_size64(header + 1, size); _write_size64(header + 1, size);
len = 8; len = 9;
} }
else { else {
PyErr_SetString(PyExc_OverflowError, PyErr_SetString(PyExc_OverflowError,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment