[ZEO4] Use pickle protocol 3 on the wire.

( Upstream commit 1b21b3a8 )

--------

Kirr: wendelin.core 2 switched to using pickle protocol=3 on its go side
completely in neo@1bb0eb24
( neo!8 ). It all works with
ZODB5/ZEO5 on both py2 and py3, but without this backport ZEO/go driver
started to fail without being able to connect to ZEO4/py server:

    (neo) (z4-dev) (g.env) kirr@deca:~/src/neo/src/lab.nexedi.com/kirr/neo/go/zodb/storage/zeo$ go test -v -failfast
    ...
    === RUN   TestEmptyDB/msgpack=false/py2
    ------
    2025-02-28T13:34:59 INFO ZEO.runzeo (1979570) opening storage '1' using FileStorage
    ------
    2025-02-28T13:34:59 INFO ZEO.StorageServer StorageServer created RW with storages: 1:RW:/tmp/zeo1011468807/1.fs
    ------
    2025-02-28T13:34:59 INFO ZEO.zrpc (1979570) listening on /tmp/zeo1011468807/1.fs.zeosock
    ------
    2025-02-28T13:34:59 INFO ZEO.zrpc.Connection('S') () received handshake 'Z4'
    ------
    2025-02-28T13:34:59 ERROR ZEO.zrpc (1979570) can't decode message: '\x80\x03(K\x00K\x00X\x08\x00\x00\x00registerX\x01\x00\x...'
    ------
    2025-02-28T13:34:59 ERROR ZEO.zrpc.Connection('S') () Error caught in asyncore
    Traceback (most recent call last):
      File "/usr/lib/python2.7/asyncore.py", line 83, in read
        obj.handle_read_event()
      File "/usr/lib/python2.7/asyncore.py", line 449, in handle_read_event
        self.handle_read()
      File "/home/kirr/src/wendelin/z/ZEO4/src/ZEO/zrpc/smac.py", line 243, in handle_read
        self.message_input(msg)
      File "/home/kirr/src/wendelin/z/ZEO4/src/ZEO/zrpc/connection.py", line 421, in message_input
        msgid, async, name, args = self.decode(message)
      File "/home/kirr/src/wendelin/z/ZEO4/src/ZEO/zrpc/marshal.py", line 98, in server_decode
        return unpickler.load() # msgid, flags, name, args
    ValueError: unsupported pickle protocol: 3

As support for ZODB4 is scheduled to be phased out it is easier to make
ZEO4 backport instead of adding more runtime detection and compatibility
code to ZEO/go.

CHANGES.rst

 Changelog
 =========
 
+Nexedi-done backports
+---------------------
+
+- ZEO now uses pickle protocol 3 for both Python 2 and Python 3.
+  (Previously protocol 1 was used for Python 2.) This matches the
+  change in ZODB 5.4.0.
+
 
 4.3.1 (2016-11-18)
 ------------------

src/ZEO/ClientStorage.py

@@ -1302,10 +1302,7 @@ class ClientStorage(object):
         # setup tempfile to hold zeoVerify results and interim
         # invalidation results
         self._tfile = tempfile.TemporaryFile(suffix=".inv")
-        if PY3:
         self._pickler = Pickler(self._tfile, 3)
-        else:
-            self._pickler = Pickler(self._tfile, 1)
         self._pickler.fast = 1 # Don't use the memo
 
         if self._connection.peer_protocol_version < b'Z309':

src/ZEO/StorageServer.py

@@ -686,7 +686,7 @@ class ZEOStorage:
             pickler = Pickler(BytesIO(), 3)
         else:
             # The pure-python version requires at least one argument (PyPy)
-            pickler = Pickler(0)
+            pickler = Pickler(3)
         pickler.fast = 1
         try:
             pickler.dump(error)
@@ -1588,7 +1588,7 @@ class CommitLog:
 
     def __init__(self):
         self.file = tempfile.TemporaryFile(suffix=".comit-log")
-        self.pickler = Pickler(self.file, 1)
+        self.pickler = Pickler(self.file, 3)
         self.pickler.fast = 1
         self.stores = 0
 

src/ZEO/TransactionBuffer.py

@@ -64,7 +64,7 @@ class TransactionBuffer:
         self.blobs = []
         # It's safe to use a fast pickler because the only objects
         # stored are builtin types -- strings or None.
-        self.pickler = Pickler(self.file, 1)
+        self.pickler = Pickler(self.file, 3)
         self.pickler.fast = 1
 
     def close(self):

src/ZEO/_compat.py

@@ -21,7 +21,7 @@ PY32 = sys.version_info[:2] == (3, 2)
 PYPY = getattr(platform, 'python_implementation', lambda: None)() == 'PyPy'
 
 if PY3:
-    from pickle import Pickler, Unpickler as _Unpickler, dump, dumps, loads
+    from zodbpickle.pickle import Pickler, Unpickler as _Unpickler, dump, dumps, loads
     class Unpickler(_Unpickler):
         # Py3: Python 3 doesn't allow assignments to find_global,
         # instead, find_class can be overridden
@@ -33,8 +33,15 @@ if PY3:
                 return super(Unpickler, self).find_class(modulename, name)
             return self.find_global(modulename, name)
 else:
-    # Pickle support
-    from cPickle import Pickler, Unpickler, dump, dumps, loads
+    try:
+        import zodbpickle.fastpickle as cPickle
+    except ImportError:
+        import zodbpickle.pickle as cPickle
+    Pickler = cPickle.Pickler
+    Unpickler = cPickle.Unpickler
+    dump = cPickle.dump
+    dumps = cPickle.dumps
+    loads = cPickle.loads
 
 # String and Bytes IO
 from ZODB._compat import BytesIO

src/ZEO/zrpc/marshal.py

@@ -30,7 +30,6 @@ def encode(*args): # args: (msgid, flags, name, args)
     # being represented by \xij escapes in proto 0).
     # Undocumented:  cPickle.Pickler accepts a lone protocol argument;
     # pickle.py does not.
-    if PY3:
     # XXX: Py3: Needs optimization.
     f = BytesIO()
     pickler = Pickler(f, 3)
@@ -38,16 +37,6 @@ def encode(*args): # args: (msgid, flags, name, args)
     pickler.dump(args)
     res = f.getvalue()
     return res
-    else:
-        pickler = Pickler(1)
-        pickler.fast = 1
-        # Only CPython's cPickle supports dumping
-        # and returning in one operation:
-        #   return pickler.dump(args, 1)
-        # For PyPy we must return the value; fortunately this
-        # works the same on CPython and is no more expensive
-        pickler.dump(args)
-        return pickler.getvalue()
 
 
 
@@ -62,11 +51,11 @@ else:
     def fast_encode():
         # Only use in cases where you *know* the data contains only basic
         # Python objects
-        pickler = Pickler(1)
+        pickler = Pickler(3)
         pickler.fast = 1
         dump = pickler.dump
         def fast_encode(*args):
-            return dump(args, 1)
+            return dump(args, 3)
         return fast_encode
     fast_encode = fast_encode()