Issue 2917: Merge the pickle and cPickle module.

Lib/pickletools.py

@@ -2079,11 +2079,12 @@ _dis_test = r"""
    70: t        TUPLE      (MARK at 49)
    71: p    PUT        5
    74: R    REDUCE
-   75: V    UNICODE    'def'
-   80: p    PUT        6
-   83: s    SETITEM
-   84: a    APPEND
-   85: .    STOP
+   75: p    PUT        6
+   78: V    UNICODE    'def'
+   83: p    PUT        7
+   86: s    SETITEM
+   87: a    APPEND
+   88: .    STOP
 highest protocol among opcodes = 0
 
 Try again with a "binary" pickle.
@@ -2115,11 +2116,12 @@ Try again with a "binary" pickle.
    49: t            TUPLE      (MARK at 37)
    50: q        BINPUT     5
    52: R        REDUCE
-   53: X        BINUNICODE 'def'
-   61: q        BINPUT     6
-   63: s        SETITEM
-   64: e        APPENDS    (MARK at 3)
-   65: .    STOP
+   53: q        BINPUT     6
+   55: X        BINUNICODE 'def'
+   63: q        BINPUT     7
+   65: s        SETITEM
+   66: e        APPENDS    (MARK at 3)
+   67: .    STOP
 highest protocol among opcodes = 1
 
 Exercise the INST/OBJ/BUILD family.

Lib/test/pickletester.py

@@ -362,7 +362,7 @@ def create_data():
     return x
 
 class AbstractPickleTests(unittest.TestCase):
-    # Subclass must define self.dumps, self.loads, self.error.
+    # Subclass must define self.dumps, self.loads.
 
     _testdata = create_data()
 
@@ -463,8 +463,9 @@ class AbstractPickleTests(unittest.TestCase):
             self.assertEqual(list(x[0].attr.keys()), [1])
             self.assert_(x[0].attr[1] is x)
 
-    def test_garyp(self):
-        self.assertRaises(self.error, self.loads, b'garyp')
+    def test_get(self):
+        self.assertRaises(KeyError, self.loads, b'g0\np0')
+        self.assertEquals(self.loads(b'((Kdtp0\nh\x00l.))'), [(100,), (100,)])
 
     def test_insecure_strings(self):
         # XXX Some of these tests are temporarily disabled
@@ -955,7 +956,7 @@ class AbstractPickleModuleTests(unittest.TestCase):
         f = open(TESTFN, "wb")
         try:
             f.close()
-            self.assertRaises(ValueError, self.module.dump, 123, f)
+            self.assertRaises(ValueError, pickle.dump, 123, f)
         finally:
             os.remove(TESTFN)
 
@@ -964,24 +965,24 @@ class AbstractPickleModuleTests(unittest.TestCase):
         f = open(TESTFN, "wb")
         try:
             f.close()
-            self.assertRaises(ValueError, self.module.dump, 123, f)
+            self.assertRaises(ValueError, pickle.dump, 123, f)
         finally:
             os.remove(TESTFN)
 
     def test_highest_protocol(self):
         # Of course this needs to be changed when HIGHEST_PROTOCOL changes.
-        self.assertEqual(self.module.HIGHEST_PROTOCOL, 3)
+        self.assertEqual(pickle.HIGHEST_PROTOCOL, 3)
 
     def test_callapi(self):
         from io import BytesIO
         f = BytesIO()
         # With and without keyword arguments
-        self.module.dump(123, f, -1)
-        self.module.dump(123, file=f, protocol=-1)
-        self.module.dumps(123, -1)
-        self.module.dumps(123, protocol=-1)
-        self.module.Pickler(f, -1)
-        self.module.Pickler(f, protocol=-1)
+        pickle.dump(123, f, -1)
+        pickle.dump(123, file=f, protocol=-1)
+        pickle.dumps(123, -1)
+        pickle.dumps(123, protocol=-1)
+        pickle.Pickler(f, -1)
+        pickle.Pickler(f, protocol=-1)
 
 class AbstractPersistentPicklerTests(unittest.TestCase):
 

Lib/test/test_pickle.py

@@ -7,37 +7,42 @@ from test.pickletester import AbstractPickleTests
 from test.pickletester import AbstractPickleModuleTests
 from test.pickletester import AbstractPersistentPicklerTests
 
-class PickleTests(AbstractPickleTests, AbstractPickleModuleTests):
+try:
+    import _pickle
+    has_c_implementation = True
+except ImportError:
+    has_c_implementation = False
 
-    module = pickle
-    error = KeyError
 
-    def dumps(self, arg, proto=None):
-        return pickle.dumps(arg, proto)
+class PickleTests(AbstractPickleModuleTests):
+    pass
 
-    def loads(self, buf):
-        return pickle.loads(buf)
 
-class PicklerTests(AbstractPickleTests):
+class PyPicklerTests(AbstractPickleTests):
 
-    error = KeyError
+    pickler = pickle._Pickler
+    unpickler = pickle._Unpickler
 
     def dumps(self, arg, proto=None):
         f = io.BytesIO()
-        p = pickle.Pickler(f, proto)
+        p = self.pickler(f, proto)
         p.dump(arg)
         f.seek(0)
         return bytes(f.read())
 
     def loads(self, buf):
         f = io.BytesIO(buf)
-        u = pickle.Unpickler(f)
+        u = self.unpickler(f)
         return u.load()
 
-class PersPicklerTests(AbstractPersistentPicklerTests):
+
+class PyPersPicklerTests(AbstractPersistentPicklerTests):
+
+    pickler = pickle._Pickler
+    unpickler = pickle._Unpickler
 
     def dumps(self, arg, proto=None):
-        class PersPickler(pickle.Pickler):
+        class PersPickler(self.pickler):
             def persistent_id(subself, obj):
                 return self.persistent_id(obj)
         f = io.BytesIO()
@@ -47,19 +52,29 @@ class PersPicklerTests(AbstractPersistentPicklerTests):
         return f.read()
 
     def loads(self, buf):
-        class PersUnpickler(pickle.Unpickler):
+        class PersUnpickler(self.unpickler):
             def persistent_load(subself, obj):
                 return self.persistent_load(obj)
         f = io.BytesIO(buf)
         u = PersUnpickler(f)
         return u.load()
 
+
+if has_c_implementation:
+    class CPicklerTests(PyPicklerTests):
+        pickler = _pickle.Pickler
+        unpickler = _pickle.Unpickler
+
+    class CPersPicklerTests(PyPersPicklerTests):
+        pickler = _pickle.Pickler
+        unpickler = _pickle.Unpickler
+
+
 def test_main():
-    support.run_unittest(
-        PickleTests,
-        PicklerTests,
-        PersPicklerTests
-    )
+    tests = [PickleTests, PyPicklerTests, PyPersPicklerTests]
+    if has_c_implementation:
+        tests.extend([CPicklerTests, CPersPicklerTests])
+    support.run_unittest(*tests)
     support.run_doctest(pickle)
 
 if __name__ == "__main__":

Lib/test/test_pickletools.py

@@ -12,8 +12,6 @@ class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests):
     def loads(self, buf):
         return pickle.loads(buf)
 
-    module = pickle
-    error = KeyError
 
 def test_main():
     support.run_unittest(OptimizedPickleTests)

Misc/NEWS

@@ -78,6 +78,10 @@ Extension Modules
 Library
 -------
 
+- The ``pickle`` module is now automatically use an optimized C
+  implementation of Pickler and Unpickler when available. The
+  ``cPickle`` module is no longer needed.
+
 - Removed the ``htmllib`` and ``sgmllib`` modules.
 
 - The deprecated ``SmartCookie`` and ``SimpleCookie`` classes have

setup.py

@@ -422,6 +422,8 @@ class PyBuildExt(build_ext):
         exts.append( Extension("_functools", ["_functoolsmodule.c"]) )
         # Memory-based IO accelerator modules
         exts.append( Extension("_bytesio", ["_bytesio.c"]) )
+        # C-optimized pickle replacement
+        exts.append( Extension("_pickle", ["_pickle.c"]) )
         # atexit
         exts.append( Extension("atexit", ["atexitmodule.c"]) )
         # _json speedups