Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
C
cpython
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Kirill Smelkov
cpython
Commits
cdc7a91d
Commit
cdc7a91d
authored
Feb 12, 2013
by
Serhiy Storchaka
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Issue #13555: cPickle now supports files larger than 2 GiB.
parent
da5c2a06
Changes
6
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
217 additions
and
130 deletions
+217
-130
Lib/test/pickletester.py
Lib/test/pickletester.py
+30
-1
Lib/test/test_cpickle.py
Lib/test/test_cpickle.py
+15
-2
Lib/test/test_pickle.py
Lib/test/test_pickle.py
+16
-4
Misc/NEWS
Misc/NEWS
+2
-0
Modules/cPickle.c
Modules/cPickle.c
+152
-118
Modules/cStringIO.c
Modules/cStringIO.c
+2
-5
No files found.
Lib/test/pickletester.py
View file @
cdc7a91d
...
...
@@ -6,7 +6,8 @@ import cStringIO
import
pickletools
import
copy_reg
from
test.test_support
import
TestFailed
,
have_unicode
,
TESTFN
from
test.test_support
import
(
TestFailed
,
have_unicode
,
TESTFN
,
_2G
,
_1M
,
precisionbigmemtest
)
# Tests that try a number of pickle protocols should have a
# for proto in protocols:
...
...
@@ -1280,3 +1281,31 @@ class AbstractPicklerUnpicklerObjectTests(unittest.TestCase):
f
.
write
(
pickled2
)
f
.
seek
(
0
)
self
.
assertEqual
(
unpickler
.
load
(),
data2
)
class
BigmemPickleTests
(
unittest
.
TestCase
):
# Memory requirements: 1 byte per character for input strings, 1 byte
# for pickled data, 1 byte for unpickled strings, 1 byte for internal
# buffer and 1 byte of free space for resizing of internal buffer.
@
precisionbigmemtest
(
size
=
_2G
+
100
*
_1M
,
memuse
=
5
)
def
test_huge_strlist
(
self
,
size
):
chunksize
=
2
**
20
data
=
[]
while
size
>
chunksize
:
data
.
append
(
'x'
*
chunksize
)
size
-=
chunksize
chunksize
+=
1
data
.
append
(
'y'
*
size
)
try
:
for
proto
in
protocols
:
try
:
pickled
=
self
.
dumps
(
data
,
proto
)
res
=
self
.
loads
(
pickled
)
self
.
assertEqual
(
res
,
data
)
finally
:
res
=
None
pickled
=
None
finally
:
data
=
None
Lib/test/test_cpickle.py
View file @
cdc7a91d
import
cPickle
,
unittest
from
cStringIO
import
StringIO
from
test.pickletester
import
AbstractPickleTests
,
AbstractPickleModuleTests
from
test.pickletester
import
AbstractPicklerUnpicklerObjectTests
from
test.pickletester
import
(
AbstractPickleTests
,
AbstractPickleModuleTests
,
AbstractPicklerUnpicklerObjectTests
,
BigmemPickleTests
)
from
test
import
test_support
class
cPickleTests
(
AbstractPickleTests
,
AbstractPickleModuleTests
):
...
...
@@ -101,6 +103,16 @@ class cPicklePicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests):
pickler_class
=
cPickle
.
Pickler
unpickler_class
=
cPickle
.
Unpickler
class
cPickleBigmemPickleTests
(
BigmemPickleTests
):
def
dumps
(
self
,
arg
,
proto
=
0
,
fast
=
0
):
# Ignore fast
return
cPickle
.
dumps
(
arg
,
proto
)
def
loads
(
self
,
buf
):
# Ignore fast
return
cPickle
.
loads
(
buf
)
class
Node
(
object
):
pass
...
...
@@ -133,6 +145,7 @@ def test_main():
cPickleFastPicklerTests
,
cPickleDeepRecursive
,
cPicklePicklerUnpicklerObjectTests
,
cPickleBigmemPickleTests
,
)
if
__name__
==
"__main__"
:
...
...
Lib/test/test_pickle.py
View file @
cdc7a91d
...
...
@@ -3,10 +3,11 @@ from cStringIO import StringIO
from
test
import
test_support
from
test.pickletester
import
AbstractPickleTests
from
test.pickletester
import
AbstractPickleModuleTests
from
test.pickletester
import
AbstractPersistentPicklerTests
from
test.pickletester
import
AbstractPicklerUnpicklerObjectTests
from
test.pickletester
import
(
AbstractPickleTests
,
AbstractPickleModuleTests
,
AbstractPersistentPicklerTests
,
AbstractPicklerUnpicklerObjectTests
,
BigmemPickleTests
)
class
PickleTests
(
AbstractPickleTests
,
AbstractPickleModuleTests
):
...
...
@@ -66,6 +67,16 @@ class PicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests):
pickler_class
=
pickle
.
Pickler
unpickler_class
=
pickle
.
Unpickler
class
PickleBigmemPickleTests
(
BigmemPickleTests
):
def
dumps
(
self
,
arg
,
proto
=
0
,
fast
=
0
):
# Ignore fast
return
pickle
.
dumps
(
arg
,
proto
)
def
loads
(
self
,
buf
):
# Ignore fast
return
pickle
.
loads
(
buf
)
def
test_main
():
test_support
.
run_unittest
(
...
...
@@ -73,6 +84,7 @@ def test_main():
PicklerTests
,
PersPicklerTests
,
PicklerUnpicklerObjectTests
,
PickleBigmemPickleTests
,
)
test_support
.
run_doctest
(
pickle
)
...
...
Misc/NEWS
View file @
cdc7a91d
...
...
@@ -202,6 +202,8 @@ Core and Builtins
Library
-------
- Issue #13555: cPickle now supports files larger than 2 GiB.
- Issue #17052: unittest discovery should use self.testLoader.
- Issue #4591: Uid and gid values larger than 2**31 are supported now.
...
...
Modules/cPickle.c
View file @
cdc7a91d
This diff is collapsed.
Click to expand it.
Modules/cStringIO.c
View file @
cdc7a91d
...
...
@@ -210,11 +210,8 @@ IO_creadline(PyObject *self, char **output) {
if
(
n
<
end
)
n
++
;
len
=
n
-
start
;
if
(
len
>
INT_MAX
)
{
PyErr_SetString
(
PyExc_OverflowError
,
"length too large"
);
return
-
1
;
}
if
(
len
>
INT_MAX
)
len
=
INT_MAX
;
*
output
=
start
;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment