Commit 82c6df06 authored by Jérome Perrin's avatar Jérome Perrin

fixup! fixup! fixup! fixup! XMLExportImport: more support pickle protocol 3 🚧 ( repair python2 )

parent 51cb4518
############################################################################## ##############################################################################
# # coding: utf-8
# Copyright (c) 2008 Nexedi SA and Contributors. All Rights Reserved. # Copyright (c) 2008 Nexedi SA and Contributors. All Rights Reserved.
# TAHARA Yusei <yusei@nexedi.com> # TAHARA Yusei <yusei@nexedi.com>
# #
...@@ -26,16 +26,17 @@ ...@@ -26,16 +26,17 @@
# #
############################################################################## ##############################################################################
import base64
import unittest import unittest
import zodbpickle import zodbpickle
import zodbpickle.fastpickle as pickle import zodbpickle.fastpickle as pickle
import re import re
import xml.sax from io import BytesIO
from io import BytesIO, StringIO from six import StringIO
from Products.ERP5Type.XMLExportImport import importXML, ppml
from Products.ERP5Type.XMLExportImport import ppml
import six import six
class DummyClass: class DummyClass:
""" """
A dummy data class A dummy data class
...@@ -50,26 +51,37 @@ class XMLPickleTestCase(unittest.TestCase): ...@@ -50,26 +51,37 @@ class XMLPickleTestCase(unittest.TestCase):
def dump_to_xml(self, obj): def dump_to_xml(self, obj):
pickled_string = pickle.dumps(obj, protocol=self._pickle_protocol) pickled_string = pickle.dumps(obj, protocol=self._pickle_protocol)
f = BytesIO(pickled_string) f = BytesIO(pickled_string)
return str(ppml.ToXMLUnpickler(f).load()) xml = ppml.ToXMLUnpickler(f).load().__str__()
if six.PY2 and isinstance(xml, six.text_type):
def load_from_xml(self, xml_string): xml = xml.encode('utf-8')
output = StringIO() return xml
F=ppml.xmlPickler()
F.file = output def load_from_xml(self, xml_string, persistent_load=None):
F.binary = 1 # XML pickle actually only supports the case of binary = 1 assertEqual = self.assertEqual
class DummyJar:
content_handler = xml.sax.handler.ContentHandler() loaded = None
content_handler.startElement = F.unknown_starttag """follow interface expected by importXML"""
content_handler.endElement = F.unknown_endtag def importFile(self, file, clue):
content_handler.characters = F.handle_data assertEqual(clue, 'ignored')
xml.sax.parseString(xml_string, content_handler) assertEqual(file.read(4), b'ZEXP')
unpickler = pickle.Unpickler(file)
reconstructed_pickled_data = F._stack[0][0] if persistent_load:
return pickle.loads(reconstructed_pickled_data) unpickler.persistent_load = persistent_load
self.loaded = unpickler.load()
jar = DummyJar()
xml_string = '<?xml version="1.0"?>\n<ZopeData>%s</ZopeData>' % xml_string
importXML(jar, StringIO(xml_string), clue='ignored')
return jar.loaded
def dump_and_load(self, obj): def dump_and_load(self, obj):
return self.load_from_xml(self.dump_to_xml(obj)) return self.load_from_xml(self.dump_to_xml(obj))
def check_and_load(self, v):
reconstructed = self.dump_and_load(v)
self.assertEqual(reconstructed, v)
self.assertIs(type(reconstructed), type(v))
class TestXMLPickle(XMLPickleTestCase): class TestXMLPickle(XMLPickleTestCase):
...@@ -101,76 +113,95 @@ class TestXMLPickle(XMLPickleTestCase): ...@@ -101,76 +113,95 @@ class TestXMLPickle(XMLPickleTestCase):
self.assertIs(self.dump_and_load(False), False) self.assertIs(self.dump_and_load(False), False)
def test_int(self): def test_int(self):
def check_int(v): self.check_and_load(-0)
reconstructed = self.dump_and_load(v) self.check_and_load(1)
self.assertEqual(reconstructed, v) self.check_and_load(-1)
self.assertIs(type(reconstructed), int) self.check_and_load(0xff)
check_int(-0) self.check_and_load(0xff1)
check_int(1) self.check_and_load(0xffff)
check_int(-1) self.check_and_load(2**128)
check_int(0xff) # long4
check_int(0xff1) # https://github.com/python/cpython/blob/4d4a6f1b/Lib/test/pickletester.py#L2049-L2050
check_int(0xffff) self.check_and_load(12345678910111213141516178920 << (256*8))
check_int(0xffff1)
if six.PY2:
def test_long(self):
self.check_and_load(long(-0))
self.check_and_load(long(1))
self.check_and_load(long(-1))
self.check_and_load(long(0xff))
self.check_and_load(long(0xff1))
self.check_and_load(long(0xffff))
self.check_and_load(long(2**128))
self.check_and_load(12345678910111213141516178920 << (256*8))
def test_float(self): def test_float(self):
def check_float(v): self.check_and_load(-0.0)
reconstructed = self.dump_and_load(v) self.check_and_load(1.0)
self.assertEqual(reconstructed, v) self.check_and_load(-1.0)
self.assertIs(type(reconstructed), float) self.check_and_load(.33)
check_float(-0.0)
check_float(1.0)
check_float(-1.0)
check_float(.1 + .2)
def test_None(self): def test_None(self):
self.assertIs( self.assertIs(
self.dump_and_load(None), None) self.dump_and_load(None), None)
def test_bytes(self): def test_bytes(self):
self.assertEqual(self.dump_and_load(b"bytes"), b"bytes") self.check_and_load(b"bytes")
self.assertEqual(self.dump_and_load(b"long bytes" * 100), b"long bytes" * 100) self.check_and_load(b"long bytes" * 100)
self.assertEqual( self.check_and_load(zodbpickle.binary(b"bytes"))
self.dump_and_load(zodbpickle.binary(b"bytes")), self.check_and_load(zodbpickle.binary(b""))
zodbpickle.binary(b"bytes"))
self.assertIs(type(self.dump_and_load(zodbpickle.binary(b"bytes"))), zodbpickle.binary)
def test_unicode(self): def test_unicode(self): # BBB PY2
self.assertIs(type(self.dump_and_load(u"OK")), six.text_type) self.assertIs(type(self.dump_and_load(u"OK")), six.text_type)
self.assertEqual(self.dump_and_load(u"short"), u"short") self.check_and_load(u"short")
self.assertEqual(self.dump_and_load(u"unicode 👍"), u"unicode 👍") self.check_and_load(u"unicode 👍")
self.assertEqual(self.dump_and_load(u"long" * 100), u"long" * 100) self.check_and_load(u"long" * 100)
self.assertEqual(self.dump_and_load(u"long…" * 100), u"long…" * 100) self.check_and_load(u"long…" * 100)
self.assertEqual(self.dump_and_load(u">"), u">") self.check_and_load(u">")
self.assertEqual(self.dump_and_load(u"a\nb"), u"a\nb") self.check_and_load(u"a\nb")
self.check_and_load(u" with spaces ")
self.check_and_load(u"\twith\ttabs\t")
self.check_and_load(u"")
def test_str(self):
self.assertIs(type(self.dump_and_load("OK")), str)
self.check_and_load("short")
self.check_and_load("unicode 👍")
self.check_and_load("long" * 100)
self.check_and_load("long…" * 100)
self.check_and_load(">")
self.check_and_load("a\nb")
self.check_and_load(" with spaces ")
self.check_and_load("\twith\ttabs\t")
self.check_and_load("")
def test_dict(self): def test_dict(self):
self.assertEqual( self.check_and_load({'a': 1, 'b': 2})
self.dump_and_load({'a': 1, 'b': 2}), {'a': 1, 'b': 2}) self.check_and_load({'hé': 'ho'})
self.check_and_load(dict.fromkeys(range(3000)))
def test_tuple(self): def test_tuple(self):
self.assertEqual( self.check_and_load((1, ))
self.dump_and_load((1, )), (1, )) self.check_and_load((1, 'two'))
self.assertEqual( self.check_and_load((1, 'two', 3.0))
self.dump_and_load((1, 'two')), (1, 'two')) self.check_and_load(tuple([1] * 1000))
self.assertEqual( self.check_and_load(())
self.dump_and_load((1, 'two', 3.0)), (1, 'two', 3.0)) self.check_and_load(('hé',))
self.assertEqual( self.check_and_load(('hé', 'hé'))
self.dump_and_load(tuple([1] * 1000)), tuple([1] * 1000)) self.check_and_load(('hé', 'hé', 'hé'))
self.assertEqual( self.check_and_load(('hé', 'hé', 'hé', 'hé'))
self.dump_and_load(()), ())
def test_list(self): def test_list(self):
self.assertEqual( self.check_and_load([1])
self.dump_and_load([1]), [1]) self.check_and_load([])
self.assertEqual( self.check_and_load([1] * 1000)
self.dump_and_load([]), []) self.check_and_load(['hé'])
self.assertEqual(
self.dump_and_load([1] * 1000), [1] * 1000)
def test_set(self): def test_set(self):
self.assertEqual( self.check_and_load(set('abc'))
self.dump_and_load(set('abc')), set('abc')) self.check_and_load(set('hé'))
self.check_and_load(set([]))
def test_reference(self): def test_reference(self):
ref = [] ref = []
...@@ -178,6 +209,14 @@ class TestXMLPickle(XMLPickleTestCase): ...@@ -178,6 +209,14 @@ class TestXMLPickle(XMLPickleTestCase):
self.assertEqual(reconstructed, [ref, ref, ref]) self.assertEqual(reconstructed, [ref, ref, ref])
self.assertIs(reconstructed[0], reconstructed[1]) self.assertIs(reconstructed[0], reconstructed[1])
def test_reference_long(self):
# same as reference (which is using BINPUT/BINGET but with large data
# to use LONG_BINPUT/LONG_BINGET)
ref = [list() for _ in range(256)]
reconstructed = self.dump_and_load([ref, ref, ref])
self.assertEqual(reconstructed, [ref, ref, ref])
self.assertIs(reconstructed[0], reconstructed[1])
class TestXMLPickleStringEncoding(XMLPickleTestCase): class TestXMLPickleStringEncoding(XMLPickleTestCase):
def test_string_base64(self): def test_string_base64(self):
...@@ -214,19 +253,39 @@ class TestXMLPickleStringEncoding(XMLPickleTestCase): ...@@ -214,19 +253,39 @@ class TestXMLPickleStringEncoding(XMLPickleTestCase):
class TestXMLPickleStringHeuristics(XMLPickleTestCase): class TestXMLPickleStringHeuristics(XMLPickleTestCase):
"""Heuristics to map python2 str to unicode or bytes in business templates. """Heuristics to map python2 str to unicode or bytes in business templates.
""" """
def test_oid_base64(self):
# if it looks like an oid, it's bytes
self.assertEqual(
self.load_from_xml("""
<pickle><string encoding="base64">AAAAAAAAAAE=</string></pickle>
"""),
b"\x00\x00\x00\x00\x00\x00\x00\x01")
def test_bytes_base64(self): def test_bytes_base64(self):
# if it does not decode as utf-8 it's bytes # if it does not decode as utf-8, it's bytes
self.assertEqual( self.assertEqual(
self.load_from_xml(""" self.load_from_xml("""
<pickle><string encoding="base64">/wA=</string></pickle> <pickle><string encoding="base64">/wA=</string></pickle>
"""), """),
b"\xFF\x00") b"\xFF\x00")
def test_long_bytes_base64(self):
# if it does not decode as utf-8, it's bytes
long_bytes = b"\xFF\x00" * 256
self.assertEqual(
self.load_from_xml("""
<pickle><string encoding="base64">%s</string></pickle>
""" % base64.b64encode(long_bytes).decode()),
long_bytes)
def test_string_persistent_id_base64(self):
# persistent ids are loaded as bytes
persistent_ids = []
def persistent_load(oid):
persistent_ids.append(oid)
self.assertEqual(
self.load_from_xml("""
<pickle>
<persistent>
<string encoding="base64">AAAAAAAAAAE=</string>
</persistent>
</pickle>
""",
persistent_load=persistent_load),
None)
self.assertEqual(
persistent_ids,
[b'\x00\x00\x00\x00\x00\x00\x00\x01'])
...@@ -363,12 +363,6 @@ def exportXML(jar, oid, file=None): ...@@ -363,12 +363,6 @@ def exportXML(jar, oid, file=None):
p = getReorderedPickle(oid) p = getReorderedPickle(oid)
write(XMLrecord(oid, len(p), p, id_mapping)) write(XMLrecord(oid, len(p), p, id_mapping))
write('</ZopeData>\n') write('</ZopeData>\n')
if 0:
try:
print(file.getvalue())
except AttributeError:
pass
import pdb; pdb.set_trace()
return file return file
class zopedata: class zopedata:
...@@ -421,7 +415,6 @@ def importXML(jar, file, clue=''): ...@@ -421,7 +415,6 @@ def importXML(jar, file, clue=''):
F.end_handlers['record'] = save_record F.end_handlers['record'] = save_record
F.end_handlers['ZopeData'] = save_zopedata F.end_handlers['ZopeData'] = save_zopedata
F.start_handlers['ZopeData'] = start_zopedata F.start_handlers['ZopeData'] = start_zopedata
F.binary=1
F.file=outfile F.file=outfile
# <patch> # <patch>
# Our BTs XML files don't declare encoding but have accented chars in them # Our BTs XML files don't declare encoding but have accented chars in them
......
...@@ -24,8 +24,10 @@ import struct ...@@ -24,8 +24,10 @@ import struct
import six import six
if six.PY2: if six.PY2:
from base64 import encodestring as base64_encodebytes, decodestring as base64_decodebytes from base64 import encodestring as base64_encodebytes, decodestring as base64_decodebytes
from zodbpickle.pickle_2 import decode_long
else: else:
from base64 import encodebytes as base64_encodebytes, decodebytes as base64_decodebytes from base64 import encodebytes as base64_encodebytes, decodebytes as base64_decodebytes
from zodbpickle.pickle_3 import decode_long
import re import re
from marshal import loads as mloads from marshal import loads as mloads
from .xyap import NoBlanks from .xyap import NoBlanks
...@@ -201,12 +203,14 @@ class String(Scalar): ...@@ -201,12 +203,14 @@ class String(Scalar):
self.mapping.setImmutable(self.id, Immutable(value = result)) self.mapping.setImmutable(self.id, Immutable(value = result))
return '%s%s\n' % (' '*indent, result) return '%s%s\n' % (' '*indent, result)
class Unicode(String): class Unicode(String):
def tag_name(self): def tag_name(self):
if six.PY3: if six.PY3:
return 'string' return 'string'
return super(Unicode, self).tag_name() return super(Unicode, self).tag_name()
class Bytes(String): class Bytes(String):
pass pass
...@@ -226,7 +230,7 @@ class Wrapper(object): ...@@ -226,7 +230,7 @@ class Wrapper(object):
v=self._v v=self._v
i=' '*indent i=' '*indent
if isinstance(v, Scalar): if isinstance(v, Scalar):
return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name) return '%s<%s%s> %s </%s>\n' % (i, name, id, v.__str__()[:-1], name)
else: else:
v=v.__str__(indent+2) v=v.__str__(indent+2)
return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name) return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
...@@ -479,6 +483,18 @@ class MinimalMapping(IdentityMapping): ...@@ -479,6 +483,18 @@ class MinimalMapping(IdentityMapping):
def __str__(self, a): def __str__(self, a):
return "Error here" return "Error here"
class UnsupportedOpCode(AssertionError):
"""Error when encountering an opcode that is not supposed to be used
by this implementation.
"""
def unsupported_opcode(opcode):
def handler(self):
raise UnsupportedOpCode(opcode)
return handler
class ToXMLUnpickler(Unpickler): class ToXMLUnpickler(Unpickler):
def load(self, id_mapping=None): def load(self, id_mapping=None):
if id_mapping is None: if id_mapping is None:
...@@ -493,13 +509,6 @@ class ToXMLUnpickler(Unpickler): ...@@ -493,13 +509,6 @@ class ToXMLUnpickler(Unpickler):
def persistent_load(self, v): def persistent_load(self, v):
return Persistent(v, self.id_mapping) return Persistent(v, self.id_mapping)
def load_persid(self):
pid = self.readline()[:-1]
self.append(self.persistent_load(String(pid, self.id_mapping)))
if six.PY2:
dispatch[PERSID] = load_persid
dispatch[PERSID[0]] = load_persid
def load_binpersid(self): def load_binpersid(self):
pid = self.stack.pop() pid = self.stack.pop()
self.append(self.persistent_load(pid)) self.append(self.persistent_load(pid))
...@@ -513,12 +522,6 @@ class ToXMLUnpickler(Unpickler): ...@@ -513,12 +522,6 @@ class ToXMLUnpickler(Unpickler):
dispatch[NONE] = load_none dispatch[NONE] = load_none
dispatch[NONE[0]] = load_none dispatch[NONE[0]] = load_none
def load_int(self):
self.append(Int(int(self.readline()[:-1]), self.id_mapping))
if six.PY2:
dispatch[INT] = load_int
dispatch[INT[0]] = load_int
def load_binint(self): def load_binint(self):
self.append(Int(mloads(b'i' + self.read(4)), self.id_mapping)) self.append(Int(mloads(b'i' + self.read(4)), self.id_mapping))
if six.PY2: if six.PY2:
...@@ -537,17 +540,24 @@ class ToXMLUnpickler(Unpickler): ...@@ -537,17 +540,24 @@ class ToXMLUnpickler(Unpickler):
dispatch[BININT2] = load_binint2 dispatch[BININT2] = load_binint2
dispatch[BININT2[0]] = load_binint2 dispatch[BININT2[0]] = load_binint2
def load_long(self): def load_long1(self):
self.append(Long(long_(self.readline()[:-1], 0), self.id_mapping)) n = ord(self.read(1))
data = self.read(n)
self.append(Long(decode_long(data), self.id_mapping))
if six.PY2: if six.PY2:
dispatch[LONG] = load_long dispatch[LONG1] = load_long1
dispatch[LONG[0]] = load_long dispatch[LONG1[0]] = load_long1
def load_float(self): def load_long4(self):
self.append(Float(float(self.readline()[:-1]), self.id_mapping)) n = mloads(b'i' + self.read(4))
if n < 0:
# Corrupt or hostile pickle -- we never write one like this
raise UnpicklingError("LONG pickle has negative byte count");
data = self.read(n)
self.append(Long(decode_long(data), self.id_mapping))
if six.PY2: if six.PY2:
dispatch[FLOAT] = load_float dispatch[LONG4] = load_long4
dispatch[FLOAT[0]] = load_float dispatch[LONG4[0]] = load_long4
def load_true(self): def load_true(self):
self.append(Bool(True, self.id_mapping)) self.append(Bool(True, self.id_mapping))
...@@ -567,13 +577,6 @@ class ToXMLUnpickler(Unpickler): ...@@ -567,13 +577,6 @@ class ToXMLUnpickler(Unpickler):
dispatch[BINFLOAT] = load_binfloat dispatch[BINFLOAT] = load_binfloat
dispatch[BINFLOAT[0]] = load_binfloat dispatch[BINFLOAT[0]] = load_binfloat
def load_string(self):
self.append(String(ast.literal_eval(self.readline()[:-1],
{'__builtins__': {}}), self.id_mapping)) # Let's be careful
if six.PY2:
dispatch[STRING] = load_string
dispatch[STRING[0]] = load_string
def load_binstring(self): def load_binstring(self):
len = mloads(b'i' + self.read(4)) len = mloads(b'i' + self.read(4))
self.append(String(self.read(len), self.id_mapping)) self.append(String(self.read(len), self.id_mapping))
...@@ -581,14 +584,6 @@ class ToXMLUnpickler(Unpickler): ...@@ -581,14 +584,6 @@ class ToXMLUnpickler(Unpickler):
dispatch[BINSTRING] = load_binstring dispatch[BINSTRING] = load_binstring
dispatch[BINSTRING[0]] = load_binstring dispatch[BINSTRING[0]] = load_binstring
def load_unicode(self):
line = self.readline()
self.append(Unicode(six.text_type(ast.literal_eval(line[:-1],
{'__builtins__': {}})), self.id_mapping)) # Let's be careful
if six.PY2:
dispatch[UNICODE] = load_unicode
dispatch[UNICODE[0]] = load_unicode
def load_binunicode(self): def load_binunicode(self):
len = mloads(b'i' + self.read(4)) len = mloads(b'i' + self.read(4))
self.append(Unicode(six.text_type(self.read(len), 'utf-8'), self.id_mapping)) self.append(Unicode(six.text_type(self.read(len), 'utf-8'), self.id_mapping))
...@@ -619,8 +614,6 @@ class ToXMLUnpickler(Unpickler): ...@@ -619,8 +614,6 @@ class ToXMLUnpickler(Unpickler):
def load_tuple(self): def load_tuple(self):
k = self.marker() k = self.marker()
#LOG('load_tuple, k',0,k)
#LOG('load_tuple, stack[k+1:]',0,self.stack[k+1:])
self.stack[k:] = [Tuple(self.id_mapping, v=self.stack[k+1:])] self.stack[k:] = [Tuple(self.id_mapping, v=self.stack[k+1:])]
if six.PY2: if six.PY2:
dispatch[TUPLE] = load_tuple dispatch[TUPLE] = load_tuple
...@@ -742,12 +735,6 @@ class ToXMLUnpickler(Unpickler): ...@@ -742,12 +735,6 @@ class ToXMLUnpickler(Unpickler):
idprefix='' idprefix=''
def load_get(self):
self.append(Get(self.idprefix+self.readline()[:-1], self.id_mapping))
if six.PY2:
dispatch[GET] = load_get
dispatch[GET[0]] = load_get
def load_binget(self): def load_binget(self):
i = mloads(b'i' + self.read(1) + b'\000\000\000') i = mloads(b'i' + self.read(1) + b'\000\000\000')
self.append(Get(self.idprefix+repr(i), self.id_mapping)) self.append(Get(self.idprefix+repr(i), self.id_mapping))
...@@ -782,6 +769,11 @@ class ToXMLUnpickler(Unpickler): ...@@ -782,6 +769,11 @@ class ToXMLUnpickler(Unpickler):
dispatch[LONG_BINPUT] = load_long_binput dispatch[LONG_BINPUT] = load_long_binput
dispatch[LONG_BINPUT[0]] = load_long_binput dispatch[LONG_BINPUT[0]] = load_long_binput
for code in PERSID, INT, LONG, FLOAT, STRING, UNICODE, GET, PUT:
if six.PY2:
dispatch[code] = unsupported_opcode(code)
dispatch[code[0]] = unsupported_opcode(code)
class LogCall: class LogCall:
def __init__(self, func): def __init__(self, func):
self.func = func self.func = func
...@@ -809,7 +801,6 @@ def start_pickle(self, tag, attrs): ...@@ -809,7 +801,6 @@ def start_pickle(self, tag, attrs):
return [tag, attrs] return [tag, attrs]
def save_int(self, tag, data): def save_int(self, tag, data):
if self.binary:
v = int(name(self, tag, data)) v = int(name(self, tag, data))
if v >= 0: if v >= 0:
if v <= 0xff: if v <= 0xff:
...@@ -822,10 +813,7 @@ def save_int(self, tag, data): ...@@ -822,10 +813,7 @@ def save_int(self, tag, data):
return INT + name(self, tag, data) + b'\n' return INT + name(self, tag, data) + b'\n'
def save_float(self, tag, data): def save_float(self, tag, data):
if self.binary:
return BINFLOAT + struct.pack('>d', float(name(self, tag, data))) return BINFLOAT + struct.pack('>d', float(name(self, tag, data)))
else:
return FLOAT + name(self, tag, data) + b'\n'
def save_put(self, v, attrs): def save_put(self, v, attrs):
id = attrs.get('id', '') id = attrs.get('id', '')
...@@ -835,14 +823,11 @@ def save_put(self, v, attrs): ...@@ -835,14 +823,11 @@ def save_put(self, v, attrs):
id = id[prefix + 1:] id = id[prefix + 1:]
elif id[0] == 'i': elif id[0] == 'i':
id = id[1:] id = id[1:]
if self.binary:
id = int(id) id = int(id)
if id < 256: if id < 256:
id = BINPUT + six.int2byte(id) id = BINPUT + six.int2byte(id)
else: else:
id = LONG_BINPUT + struct.pack('<i', id) id = LONG_BINPUT + struct.pack('<i', id)
else:
id = PUT + repr(id).encode() + b'\n'
return v + id return v + id
return v return v
...@@ -852,20 +837,16 @@ def save_string(self, tag, data): ...@@ -852,20 +837,16 @@ def save_string(self, tag, data):
encoding = a.get('encoding', 'repr') # JPS: repr is default encoding encoding = a.get('encoding', 'repr') # JPS: repr is default encoding
if encoding != '': if encoding != '':
v = unconvert(encoding, v) v = unconvert(encoding, v)
if self.binary:
l = len(v) l = len(v)
if l < 256: if l < 256:
if encoding == 'base64': if encoding == 'base64':
# TODO: zope4py3 (all this is unfinished)
# We can be here for two reasons: # We can be here for two reasons:
# - the input was a string with \n or similar control characters # - the input was a string with \n or similar control characters
# that are not allowed in XML, so the str was exported as base64. # that are not allowed in XML, so the str was exported as base64.
# - the input was a _p_oid exported from python2, in that case # - the input was a persistent id exported from python2, in that case
# we want to get a zodbpickle.binary back # we want to get a zodbpickle.binary back
# XXX all this seems a bad idea, we need more context if we want if len(v) == 8 and self._stack[-1][0] in ('persistent', ):
# to have such heuristics # looks like a persistent id, assume it is a persistent_id -> bytes
if len(v) == 8:
# looks like a _p_oid, assume it is a persistent_id -> bytes
op = SHORT_BINBYTES op = SHORT_BINBYTES
else: else:
# if it's a valid UTF-8 string -> str # if it's a valid UTF-8 string -> str
...@@ -879,11 +860,9 @@ def save_string(self, tag, data): ...@@ -879,11 +860,9 @@ def save_string(self, tag, data):
# not valid utf-8 -> bytes # not valid utf-8 -> bytes
op = SHORT_BINBYTES op = SHORT_BINBYTES
else: else:
# XXX this branch seems wrong
op = SHORT_BINSTRING op = SHORT_BINSTRING
try: try:
v.decode('ascii') v.decode('ascii')
# XXX zope4py3 we could also create an unpickler with encoding utf-8 ?
except UnicodeDecodeError: except UnicodeDecodeError:
op = BINUNICODE if six.PY3 else BINSTRING op = BINUNICODE if six.PY3 else BINSTRING
v = op + struct.pack('<i', l) + v v = op + struct.pack('<i', l) + v
...@@ -891,17 +870,19 @@ def save_string(self, tag, data): ...@@ -891,17 +870,19 @@ def save_string(self, tag, data):
v = op + six.int2byte(l) + v v = op + six.int2byte(l) + v
else: else:
# TODO: zope4py3 see assumption above for SHORT_BINBYTES / SHORT_BINSTRING
# TODO no! check this more ...
# op = BINSTRING
if encoding == 'base64': if encoding == 'base64':
op = BINBYTES op = BINBYTES
# if it's a valid UTF-8 string -> str
try:
v.decode('utf-8')
op = BINUNICODE if six.PY3 else BINSTRING
except UnicodeDecodeError:
# not valid utf-8 -> bytes
pass
else: else:
op = BINSTRING if six.PY2 else BINUNICODE op = BINSTRING if six.PY2 else BINUNICODE
v = op + struct.pack('<i', l) + v v = op + struct.pack('<i', l) + v
else:
v = STRING + repr(v).encode() + b'\n'
return save_put(self, v, a) return save_put(self, v, a)
def save_bytes(self, tag, data): def save_bytes(self, tag, data):
...@@ -910,7 +891,6 @@ def save_bytes(self, tag, data): ...@@ -910,7 +891,6 @@ def save_bytes(self, tag, data):
encoding = a.get('encoding', 'repr') encoding = a.get('encoding', 'repr')
if encoding: if encoding:
v = unconvert(encoding, v) v = unconvert(encoding, v)
if self.binary:
l = len(v) l = len(v)
if l < 256: if l < 256:
op = SHORT_BINBYTES op = SHORT_BINBYTES
...@@ -918,14 +898,10 @@ def save_bytes(self, tag, data): ...@@ -918,14 +898,10 @@ def save_bytes(self, tag, data):
else: else:
op = BINBYTES op = BINBYTES
v = op + struct.pack('<i', l) + v v = op + struct.pack('<i', l) + v
else:
# XXX used ??? seems wrong
v = BYTES + repr(v) + '\n'
return save_put(self, v, a) return save_put(self, v, a)
def save_unicode(self, tag, data): def save_unicode(self, tag, data):
binary=self.binary
v=b'' v=b''
a=data[1] a=data[1]
if len(data)>2: if len(data)>2:
...@@ -934,11 +910,9 @@ def save_unicode(self, tag, data): ...@@ -934,11 +910,9 @@ def save_unicode(self, tag, data):
encoding=a.get('encoding','repr') # JPS: repr is default encoding encoding=a.get('encoding','repr') # JPS: repr is default encoding
if encoding != '': if encoding != '':
v=unconvert(encoding,v) v=unconvert(encoding,v)
if binary:
l=len(v) l=len(v)
s=mdumps(l)[1:] s=mdumps(l)[1:]
v=BINUNICODE+s+v v=BINUNICODE+s+v
else: v=UNICODE+"'"+v+"'\012"
return save_put(self, v, a) return save_put(self, v, a)
def save_tuple(self, tag, data): def save_tuple(self, tag, data):
...@@ -949,26 +923,16 @@ def save_tuple(self, tag, data): ...@@ -949,26 +923,16 @@ def save_tuple(self, tag, data):
def save_list(self, tag, data): def save_list(self, tag, data):
L = data[2:] L = data[2:]
if self.binary:
v = save_put(self, EMPTY_LIST, data[1]) v = save_put(self, EMPTY_LIST, data[1])
if L: if L:
v = v + MARK + b''.join(L) + APPENDS v = v + MARK + b''.join(L) + APPENDS
else:
v = save_put(self, MARK + LIST, data[1])
if L:
v = APPEND.join(L) + APPEND
return v return v
def save_dict(self, tag, data): def save_dict(self, tag, data):
D = data[2:] D = data[2:]
if self.binary:
v = save_put(self, EMPTY_DICT, data[1]) v = save_put(self, EMPTY_DICT, data[1])
if D: if D:
v = v + MARK + b''.join(D) + SETITEMS v = v + MARK + b''.join(D) + SETITEMS
else:
v = save_put(self, MARK + DICT, data[1])
if D:
v = v + SETITEM.join(D) + SETITEM
return v return v
def save_reference(self, tag, data): def save_reference(self, tag, data):
...@@ -977,14 +941,11 @@ def save_reference(self, tag, data): ...@@ -977,14 +941,11 @@ def save_reference(self, tag, data):
prefix = id.rfind('.') prefix = id.rfind('.')
if prefix >= 0: if prefix >= 0:
id = id[prefix + 1:] id = id[prefix + 1:]
if self.binary:
id = int(id) id = int(id)
if id < 256: if id < 256:
return BINGET + six.int2byte(id) return BINGET + six.int2byte(id)
else: else:
return LONG_BINGET + struct.pack('<i', id) return LONG_BINGET + struct.pack('<i', id)
else:
return GET + repr(id).encode() + b'\n'
def save_object(self, tag, data): def save_object(self, tag, data):
if len(data)==5: if len(data)==5:
...@@ -1012,10 +973,7 @@ def save_global(self, tag, data): ...@@ -1012,10 +973,7 @@ def save_global(self, tag, data):
def save_persis(self, tag, data): def save_persis(self, tag, data):
v = data[2] v = data[2]
if self.binary:
return v + BINPERSID return v + BINPERSID
else:
return PERSID + v
def save_pickle_start(self, tag, attrs): def save_pickle_start(self, tag, attrs):
return [tag, attrs] return [tag, attrs]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment