Skip to content

C
cpython
Commit 8dd05147 authored by Mark Dickinson's avatar Mark Dickinson
Browse files
Options

Issue #4842, patch 1/2: fix pickle in Python 3.x so that pickling with the 

'L' opcode always appends an 'L' on output, just as 2.x does.  When
unpickling, remove the trailing 'L' (if present) before passing the
result to PyLong_FromString.
parent 6dc43967
Hide whitespace changes
Inline Side-by-side
Showing with 155 additions and 128 deletions
Lib/pickle.py
View file @ 8dd05147
...@@ -470,7 +470,7 @@ class _Pickler: ...@@ -470,7 +470,7 @@ class _Pickler:
else: else:
self.write(LONG4 + pack("<i", n) + encoded) self.write(LONG4 + pack("<i", n) + encoded)
return return
self.write(LONG + repr(obj).encode("ascii") + b'\n') self.write(LONG + repr(obj).encode("ascii") + b'L\n')
dispatch[int] = save_long dispatch[int] = save_long
def save_float(self, obj, pack=struct.pack): def save_float(self, obj, pack=struct.pack):
...@@ -890,6 +890,8 @@ class _Unpickler: ...@@ -890,6 +890,8 @@ class _Unpickler:
def load_long(self): def load_long(self):
val = self.readline()[:-1].decode("ascii") val = self.readline()[:-1].decode("ascii")
if val and val[-1] == 'L':
val = val[:-1]
self.append(int(val, 0)) self.append(int(val, 0))
dispatch[LONG[0]] = load_long dispatch[LONG[0]] = load_long
......
Lib/pickletools.py
View file @ 8dd05147
...@@ -527,6 +527,8 @@ def read_decimalnl_long(f): ...@@ -527,6 +527,8 @@ def read_decimalnl_long(f):
""" """
s = read_stringnl(f, decode=False, stripquotes=False) s = read_stringnl(f, decode=False, stripquotes=False)
if s[-1:] == b'L':
s = s[:-1]
return int(s) return int(s)
...@@ -2052,39 +2054,39 @@ _dis_test = r""" ...@@ -2052,39 +2054,39 @@ _dis_test = r"""
1: l LIST (MARK at 0) 1: l LIST (MARK at 0)
2: p PUT 0 2: p PUT 0
5: L LONG 1 5: L LONG 1
8: a APPEND 9: a APPEND
9: L LONG 2 10: L LONG 2
12: a APPEND 14: a APPEND
13: ( MARK 15: ( MARK
14: L LONG 3 16: L LONG 3
17: L LONG 4 20: L LONG 4
20: t TUPLE (MARK at 13) 24: t TUPLE (MARK at 15)
21: p PUT 1 25: p PUT 1
24: a APPEND 28: a APPEND
25: ( MARK 29: ( MARK
26: d DICT (MARK at 25) 30: d DICT (MARK at 29)
27: p PUT 2 31: p PUT 2
30: c GLOBAL 'builtins bytes' 34: c GLOBAL 'builtins bytes'
46: p PUT 3 50: p PUT 3
49: ( MARK 53: ( MARK
50: ( MARK 54: ( MARK
51: l LIST (MARK at 50) 55: l LIST (MARK at 54)
52: p PUT 4 56: p PUT 4
55: L LONG 97 59: L LONG 97
59: a APPEND
60: L LONG 98
64: a APPEND 64: a APPEND
65: L LONG 99 65: L LONG 98
69: a APPEND 70: a APPEND
70: t TUPLE (MARK at 49) 71: L LONG 99
71: p PUT 5 76: a APPEND
74: R REDUCE 77: t TUPLE (MARK at 53)
75: p PUT 6 78: p PUT 5
78: V UNICODE 'def' 81: R REDUCE
83: p PUT 7 82: p PUT 6
86: s SETITEM 85: V UNICODE 'def'
87: a APPEND 90: p PUT 7
88: . STOP 93: s SETITEM
94: a APPEND
95: . STOP
highest protocol among opcodes = 0 highest protocol among opcodes = 0
Try again with a "binary" pickle. Try again with a "binary" pickle.
...@@ -2157,12 +2159,12 @@ highest protocol among opcodes = 0 ...@@ -2157,12 +2159,12 @@ highest protocol among opcodes = 0
92: V UNICODE 'value' 92: V UNICODE 'value'
99: p PUT 7 99: p PUT 7
102: L LONG 42 102: L LONG 42
106: s SETITEM 107: s SETITEM
107: b BUILD 108: b BUILD
108: a APPEND 109: a APPEND
109: g GET 5 110: g GET 5
112: a APPEND 113: a APPEND
113: . STOP 114: . STOP
highest protocol among opcodes = 0 highest protocol among opcodes = 0
>>> dis(pickle.dumps(x, 1)) >>> dis(pickle.dumps(x, 1))
......
Lib/test/pickletester.py
View file @ 8dd05147
...@@ -90,21 +90,21 @@ class use_metaclass(object, metaclass=metaclass): ...@@ -90,21 +90,21 @@ class use_metaclass(object, metaclass=metaclass):
# the object returned by create_data(). # the object returned by create_data().
DATA0 = ( DATA0 = (
b'(lp0\nL0\naL1\naF2.0\nac' b'(lp0\nL0L\naL1L\naF2.0\nac'
b'builtins\ncomplex\n' b'builtins\ncomplex\n'
b'p1\n(F3.0\nF0.0\ntp2\nRp' b'p1\n(F3.0\nF0.0\ntp2\nRp'
b'3\naL1\naL-1\naL255\naL-' b'3\naL1L\naL-1L\naL255L\naL-'
b'255\naL-256\naL65535\na' b'255L\naL-256L\naL65535L\na'
b'L-65535\naL-65536\naL2' b'L-65535L\naL-65536L\naL2'
b'147483647\naL-2147483' b'147483647L\naL-2147483'
b'647\naL-2147483648\na(' b'647L\naL-2147483648L\na('
b'Vabc\np4\ng4\nccopyreg' b'Vabc\np4\ng4\nccopyreg'
b'\n_reconstructor\np5\n(' b'\n_reconstructor\np5\n('
b'c__main__\nC\np6\ncbu' b'c__main__\nC\np6\ncbu'
b'iltins\nobject\np7\nNt' b'iltins\nobject\np7\nNt'
b'p8\nRp9\n(dp10\nVfoo\np1' b'p8\nRp9\n(dp10\nVfoo\np1'
b'1\nL1\nsVbar\np12\nL2\nsb' b'1\nL1L\nsVbar\np12\nL2L\nsb'
b'g9\ntp13\nag13\naL5\na.' b'g9\ntp13\nag13\naL5L\na.'
) )
# Disassembly of DATA0 # Disassembly of DATA0
...@@ -113,80 +113,80 @@ DATA0_DIS = """\ ...@@ -113,80 +113,80 @@ DATA0_DIS = """\
1: l LIST (MARK at 0) 1: l LIST (MARK at 0)
2: p PUT 0 2: p PUT 0
5: L LONG 0 5: L LONG 0
8: a APPEND 9: a APPEND
9: L LONG 1 10: L LONG 1
12: a APPEND 14: a APPEND
13: F FLOAT 2.0 15: F FLOAT 2.0
18: a APPEND 20: a APPEND
19: c GLOBAL 'builtins complex' 21: c GLOBAL 'builtins complex'
37: p PUT 1 39: p PUT 1
40: ( MARK 42: ( MARK
41: F FLOAT 3.0 43: F FLOAT 3.0
46: F FLOAT 0.0 48: F FLOAT 0.0
51: t TUPLE (MARK at 40) 53: t TUPLE (MARK at 42)
52: p PUT 2 54: p PUT 2
55: R REDUCE 57: R REDUCE
56: p PUT 3 58: p PUT 3
59: a APPEND 61: a APPEND
60: L LONG 1 62: L LONG 1
63: a APPEND 66: a APPEND
64: L LONG -1 67: L LONG -1
68: a APPEND 72: a APPEND
69: L LONG 255 73: L LONG 255
74: a APPEND 79: a APPEND
75: L LONG -255 80: L LONG -255
81: a APPEND 87: a APPEND
82: L LONG -256 88: L LONG -256
88: a APPEND 95: a APPEND
89: L LONG 65535 96: L LONG 65535
96: a APPEND 104: a APPEND
97: L LONG -65535 105: L LONG -65535
105: a APPEND
106: L LONG -65536
114: a APPEND 114: a APPEND
115: L LONG 2147483647 115: L LONG -65536
127: a APPEND 124: a APPEND
128: L LONG -2147483647 125: L LONG 2147483647
141: a APPEND 138: a APPEND
142: L LONG -2147483648 139: L LONG -2147483647
155: a APPEND 153: a APPEND
156: ( MARK 154: L LONG -2147483648
157: V UNICODE 'abc' 168: a APPEND
162: p PUT 4 169: ( MARK
165: g GET 4 170: V UNICODE 'abc'
168: c GLOBAL 'copyreg _reconstructor' 175: p PUT 4
192: p PUT 5 178: g GET 4
195: ( MARK 181: c GLOBAL 'copyreg _reconstructor'
196: c GLOBAL '__main__ C' 205: p PUT 5
208: p PUT 6 208: ( MARK
211: c GLOBAL 'builtins object' 209: c GLOBAL '__main__ C'
228: p PUT 7 221: p PUT 6
231: N NONE 224: c GLOBAL 'builtins object'
232: t TUPLE (MARK at 195) 241: p PUT 7
233: p PUT 8 244: N NONE
236: R REDUCE 245: t TUPLE (MARK at 208)
237: p PUT 9 246: p PUT 8
240: ( MARK 249: R REDUCE
241: d DICT (MARK at 240) 250: p PUT 9
242: p PUT 10 253: ( MARK
246: V UNICODE 'foo' 254: d DICT (MARK at 253)
251: p PUT 11 255: p PUT 10
255: L LONG 1 259: V UNICODE 'foo'
258: s SETITEM 264: p PUT 11
259: V UNICODE 'bar' 268: L LONG 1
264: p PUT 12 272: s SETITEM
268: L LONG 2 273: V UNICODE 'bar'
271: s SETITEM 278: p PUT 12
272: b BUILD 282: L LONG 2
273: g GET 9 286: s SETITEM
276: t TUPLE (MARK at 156) 287: b BUILD
277: p PUT 13 288: g GET 9
281: a APPEND 291: t TUPLE (MARK at 169)
282: g GET 13 292: p PUT 13
286: a APPEND 296: a APPEND
287: L LONG 5 297: g GET 13
290: a APPEND 301: a APPEND
291: . STOP 302: L LONG 5
306: a APPEND
307: . STOP
highest protocol among opcodes = 0 highest protocol among opcodes = 0
""" """
......
Misc/NEWS
View file @ 8dd05147
...@@ -134,6 +134,9 @@ Core and Builtins ...@@ -134,6 +134,9 @@ Core and Builtins
Library Library
------- -------
- Issue #4842: Always append a trailing 'L' when pickling longs using
pickle protocol 0. When reading, the 'L' is optional.
- Add the importlib package. - Add the importlib package.
- Issue #4301: Patch the logging module to add processName support, remove - Issue #4301: Patch the logging module to add processName support, remove
......
Modules/_pickle.c
View file @ 8dd05147
...@@ -846,8 +846,8 @@ save_int(PicklerObject *self, long x) ...@@ -846,8 +846,8 @@ save_int(PicklerObject *self, long x)
/* Text-mode pickle, or long too big to fit in the 4-byte /* Text-mode pickle, or long too big to fit in the 4-byte
* signed BININT format: store as a string. * signed BININT format: store as a string.
*/ */
pdata[0] = LONG; /* use LONG for consistence with pickle.py */ pdata[0] = LONG; /* use LONG for consistency with pickle.py */
PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ld\n", x); PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ldL\n", x);
if (pickler_write(self, pdata, strlen(pdata)) < 0) if (pickler_write(self, pdata, strlen(pdata)) < 0)
return -1; return -1;
} }
...@@ -977,8 +977,9 @@ save_long(PicklerObject *self, PyObject *obj) ...@@ -977,8 +977,9 @@ save_long(PicklerObject *self, PyObject *obj)
else { else {
char *string; char *string;
/* proto < 2: write the repr and newline. This is quadratic-time /* proto < 2: write the repr and newline. This is quadratic-time (in
(in the number of digits), in both directions. */ the number of digits), in both directions. We add a trailing 'L'
to the repr, for compatibility with Python 2.x. */
repr = PyObject_Repr(obj); repr = PyObject_Repr(obj);
if (repr == NULL) if (repr == NULL)
...@@ -990,7 +991,7 @@ save_long(PicklerObject *self, PyObject *obj) ...@@ -990,7 +991,7 @@ save_long(PicklerObject *self, PyObject *obj)
if (pickler_write(self, &long_op, 1) < 0 || if (pickler_write(self, &long_op, 1) < 0 ||
pickler_write(self, string, size) < 0 || pickler_write(self, string, size) < 0 ||
pickler_write(self, "\n", 1) < 0) pickler_write(self, "L\n", 2) < 0)
goto error; goto error;
} }
...@@ -2880,7 +2881,7 @@ static int ...@@ -2880,7 +2881,7 @@ static int
load_long(UnpicklerObject *self) load_long(UnpicklerObject *self)
{ {
PyObject *value; PyObject *value;
char *s; char *s, *ss;
Py_ssize_t len; Py_ssize_t len;
if ((len = unpickler_readline(self, &s)) < 0) if ((len = unpickler_readline(self, &s)) < 0)
...@@ -2888,8 +2889,27 @@ load_long(UnpicklerObject *self) ...@@ -2888,8 +2889,27 @@ load_long(UnpicklerObject *self)
if (len < 2) if (len < 2)
return bad_readline(); return bad_readline();
/* XXX: Should the base argument explicitly set to 10? */ /* s[len-2] will usually be 'L' (and s[len-1] is '\n'); we need to remove
if ((value = PyLong_FromString(s, NULL, 0)) == NULL) the 'L' before calling PyLong_FromString. In order to maintain
compatibility with Python 3.0.0, we don't actually *require*
the 'L' to be present. */
if (s[len-2] == 'L') {
ss = (char *)PyMem_Malloc(len-1);
if (ss == NULL) {
PyErr_NoMemory();
return -1;
}
strncpy(ss, s, len-2);
ss[len-2] = '\0';
/* XXX: Should the base argument explicitly set to 10? */
value = PyLong_FromString(ss, NULL, 0);
PyMem_Free(ss);
}
else {
value = PyLong_FromString(s, NULL, 0);
}
if (value == NULL)
return -1; return -1;
PDATA_PUSH(self->stack, value, -1); PDATA_PUSH(self->stack, value, -1);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7