Commit 11aec7e1 authored by Jérome Perrin's avatar Jérome Perrin

WIP XMLExportImport: export business template as pickle protocol 1 on py3

( this seems impossible )
parent 2043c062
...@@ -344,3 +344,47 @@ class TestXMLPickleStringHeuristics(XMLPickleTestCase): ...@@ -344,3 +344,47 @@ class TestXMLPickleStringHeuristics(XMLPickleTestCase):
self.assertEqual( self.assertEqual(
persistent_ids, persistent_ids,
[b'\x00\x00\x00\x00\x00\x00\x00\x01']) [b'\x00\x00\x00\x00\x00\x00\x00\x01'])
def test_save_persistent_id_bytes_base64(self):
self._pickle_protocol = 1
p1 = DummyPersistentClass(1, b'\x00\x00\x00\x00\x00\x00\x00\x01')
def persistent_id(obj):
if isinstance(obj, DummyPersistentClass):
return obj._p_oid
xml = self.dump_to_xml(p1, persistent_id=persistent_id)
c14n_xml = lxml.etree.tostring(
lxml.etree.fromstring(xml),
method="c14n"
).decode("utf-8")
self.assertEqual(
c14n_xml,
"""\
<pickle>
<persistent> <string encoding="base64">AAAAAAAAAAE=</string> </persistent>
</pickle>""")
def test_save_persistent_id_bytes_base64_with_references(self):
self._pickle_protocol = 1
p1 = DummyPersistentClass(1, b'\x00\x00\x00\x00\x00\x00\x00\x01')
p2 = DummyPersistentClass(2, b'\x00\x00\x00\x00\x00\x00\x00\x02')
def persistent_id(obj):
if isinstance(obj, DummyPersistentClass):
return obj._p_oid
xml = self.dump_to_xml([p1, p2], persistent_id=persistent_id)
c14n_xml = lxml.etree.tostring(
lxml.etree.fromstring(xml),
method="c14n"
).decode("utf-8")
self.assertEqual(
c14n_xml,
"""\
<pickle>
<list>
<persistent> <string encoding="base64">AAAAAAAAAAE=</string> </persistent>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</list>
</pickle>""")
...@@ -67,7 +67,8 @@ MARSHALLER_NAMESPACE_URI = 'http://www.erp5.org/namespaces/marshaller' ...@@ -67,7 +67,8 @@ MARSHALLER_NAMESPACE_URI = 'http://www.erp5.org/namespaces/marshaller'
marshaller = Marshaller(namespace_uri=MARSHALLER_NAMESPACE_URI, marshaller = Marshaller(namespace_uri=MARSHALLER_NAMESPACE_URI,
as_tree=True).dumps as_tree=True).dumps
DEFAULT_PICKLE_PROTOCOL = 1 if six.PY2 else 3 DEFAULT_PICKLE_PROTOCOL = 1
HIGHEST_PICKLE_PROTOCOL = 1 if six.PY2 else 3
class OrderedPickler(Pickler): class OrderedPickler(Pickler):
...@@ -338,16 +339,23 @@ def exportXML(jar, oid, file=None, pickle_protocol=DEFAULT_PICKLE_PROTOCOL): ...@@ -338,16 +339,23 @@ def exportXML(jar, oid, file=None, pickle_protocol=DEFAULT_PICKLE_PROTOCOL):
load = jar._storage.load load = jar._storage.load
if 'version' in getfullargspec(load).args: # BBB: ZODB<5 (TmpStore) if 'version' in getfullargspec(load).args: # BBB: ZODB<5 (TmpStore)
load = partial(load, version='') load = partial(load, version='')
pickle_dict = {oid: None}
# BBB During the py2 -> py3 transition, we export business templates as
# protocol 1 even on py3 - but we need to use protocol 3 to use referencesf
# on py3, so we export once using protocol 3 and do a final export using protocol 1
pickle_dict = {
(oid, pickle_protocol): None,
(oid, HIGHEST_PICKLE_PROTOCOL): None,
}
max_cache = [1e7] # do not cache more than 10MB of pickle data max_cache = [1e7] # do not cache more than 10MB of pickle data
def getReorderedPickle(oid): def getReorderedPickle(oid, pickle_protocol):
p = pickle_dict.get(oid) p = pickle_dict.get((oid, pickle_protocol))
if p is None: if p is None:
p = load(oid)[0] p = load(oid)[0]
p = reorderPickle(jar, p, pickle_protocol)[1] p = reorderPickle(jar, p, pickle_protocol)[1]
if len(p) < max_cache[0]: if len(p) < max_cache[0]:
max_cache[0] -= len(p) max_cache[0] -= len(p)
pickle_dict[oid] = p pickle_dict[oid, pickle_protocol] = p
return p return p
# Sort records and initialize id_mapping # Sort records and initialize id_mapping
...@@ -355,9 +363,9 @@ def exportXML(jar, oid, file=None, pickle_protocol=DEFAULT_PICKLE_PROTOCOL): ...@@ -355,9 +363,9 @@ def exportXML(jar, oid, file=None, pickle_protocol=DEFAULT_PICKLE_PROTOCOL):
reordered_oid_list = [oid] reordered_oid_list = [oid]
for oid in reordered_oid_list: for oid in reordered_oid_list:
_mapOid(id_mapping, oid) _mapOid(id_mapping, oid)
for oid in referencesf(getReorderedPickle(oid)): for oid in referencesf(getReorderedPickle(oid, HIGHEST_PICKLE_PROTOCOL)):
if oid not in pickle_dict: if (oid, HIGHEST_PICKLE_PROTOCOL) not in pickle_dict:
pickle_dict[oid] = None pickle_dict[oid, HIGHEST_PICKLE_PROTOCOL] = None
reordered_oid_list.append(oid) reordered_oid_list.append(oid)
# Do real export # Do real export
......
...@@ -33,6 +33,7 @@ from marshal import loads as mloads ...@@ -33,6 +33,7 @@ from marshal import loads as mloads
from .xyap import NoBlanks from .xyap import NoBlanks
from .xyap import xyap from .xyap import xyap
from Products.ERP5Type.Utils import bytes2str, str2bytes, unicode2str from Products.ERP5Type.Utils import bytes2str, str2bytes, unicode2str
import lxml.etree
from marshal import dumps as mdumps from marshal import dumps as mdumps
...@@ -348,10 +349,52 @@ class Persistent(Wrapper): ...@@ -348,10 +349,52 @@ class Persistent(Wrapper):
return '%s<%s%s> %s </%s>\n' % (i, name, id, v.__str__(map_value=1)[:-1], name) return '%s<%s%s> %s </%s>\n' % (i, name, id, v.__str__(map_value=1)[:-1], name)
elif isinstance(v,Scalar): elif isinstance(v,Scalar):
return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name) return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name)
# BBB on py3 export bytes oids as <string> for stable business template XML
elif self._is_py3_p_oid_bytes_serialized_as_protocol_1() or self._is_py3_p_oid_bytes_serialized_with_references_as_protocol_1():
wrapped_s = v._subs[1]._subs[0]
print(wrapped_s.__str__())
if wrapped_s.encoding == 'base64':
wrapped_v = base64_decodebytes(wrapped_s._v.encode())
elif wrapped_s.encoding == 'cdata':
wrapped_v = lxml.etree.fromstring(wrapped_s.__str__()).xpath('./text()')[0].strip().encode()
else:
wrapped_v = wrapped_s._v.encode('latin1')
s = String(wrapped_v, self.mapping)
return Persistent(s, self.mapping).__str__(indent)
else: else:
v=v.__str__(indent+2) v=v.__str__(indent+2)
return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name) return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
def _is_py3_p_oid_bytes_serialized_as_protocol_1(self):
v = self._v
return six.PY3 and (
isinstance(v, Object)
and len(v._subs) == 2
and isinstance(v._subs[0], Klass)
and isinstance(v._subs[0]._v, Global)
and v._subs[0]._v.module == '_codecs'
and v._subs[0]._v.name == 'encode'
and isinstance(v._subs[1], Tuple)
and isinstance(v._subs[1]._subs[0], Unicode)
and isinstance(v._subs[1]._subs[1], Unicode)
and v._subs[1]._subs[1]._v == 'latin1'
and len(v._subs[1]._subs[0]._v.encode('latin1')) == 8
)
def _is_py3_p_oid_bytes_serialized_with_references_as_protocol_1(self):
# references, when more than one persistent object was exported
# this is saved with PUT and GET, so it becomes reference in the XML
v = self._v
return six.PY3 and (
isinstance(v, Object)
and len(v._subs) == 2
and isinstance(v._subs[0], Klass)
and isinstance(v._subs[0]._v, Reference)
and isinstance(v._subs[1], Tuple)
and isinstance(v._subs[1]._subs[0], Unicode)
and isinstance(v._subs[1]._subs[1], Reference)
and len(v._subs[1]._subs[0]._v.encode('latin1', 'ignore')) == 8
)
blanck_line_expression = re.compile('^ +$') blanck_line_expression = re.compile('^ +$')
class NoBlanks(object): class NoBlanks(object):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment