WIP XMLExportImport: export business template as pickle protocol 1 on py3

( this seems impossible )

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testXMLPickle.py

@@ -344,3 +344,47 @@ class TestXMLPickleStringHeuristics(XMLPickleTestCase):
     self.assertEqual(
       persistent_ids,
       [b'\x00\x00\x00\x00\x00\x00\x00\x01'])
+
+  def test_save_persistent_id_bytes_base64(self):
+    self._pickle_protocol = 1
+    p1 = DummyPersistentClass(1, b'\x00\x00\x00\x00\x00\x00\x00\x01')
+
+    def persistent_id(obj):
+      if isinstance(obj, DummyPersistentClass):
+        return obj._p_oid
+
+    xml = self.dump_to_xml(p1, persistent_id=persistent_id)
+    c14n_xml = lxml.etree.tostring(
+      lxml.etree.fromstring(xml),
+      method="c14n"
+    ).decode("utf-8")
+    self.assertEqual(
+      c14n_xml,
+      """\
+<pickle>
+  <persistent> <string encoding="base64">AAAAAAAAAAE=</string> </persistent>
+</pickle>""")
+
+  def test_save_persistent_id_bytes_base64_with_references(self):
+    self._pickle_protocol = 1
+    p1 = DummyPersistentClass(1, b'\x00\x00\x00\x00\x00\x00\x00\x01')
+    p2 = DummyPersistentClass(2, b'\x00\x00\x00\x00\x00\x00\x00\x02')
+
+    def persistent_id(obj):
+      if isinstance(obj, DummyPersistentClass):
+        return obj._p_oid
+
+    xml = self.dump_to_xml([p1, p2], persistent_id=persistent_id)
+    c14n_xml = lxml.etree.tostring(
+      lxml.etree.fromstring(xml),
+      method="c14n"
+    ).decode("utf-8")
+    self.assertEqual(
+      c14n_xml,
+      """\
+<pickle>
+  <list>
+    <persistent> <string encoding="base64">AAAAAAAAAAE=</string> </persistent>
+    <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+  </list>
+</pickle>""")

product/ERP5Type/XMLExportImport/__init__.py

@@ -67,7 +67,8 @@ MARSHALLER_NAMESPACE_URI = 'http://www.erp5.org/namespaces/marshaller'
 marshaller = Marshaller(namespace_uri=MARSHALLER_NAMESPACE_URI,
                                                             as_tree=True).dumps
 
-DEFAULT_PICKLE_PROTOCOL = 1 if six.PY2 else 3
+DEFAULT_PICKLE_PROTOCOL = 1
+HIGHEST_PICKLE_PROTOCOL = 1 if six.PY2 else 3
 
 
 class OrderedPickler(Pickler):
@@ -338,16 +339,23 @@ def exportXML(jar, oid, file=None, pickle_protocol=DEFAULT_PICKLE_PROTOCOL):
     load = jar._storage.load
     if 'version' in getfullargspec(load).args: # BBB: ZODB<5 (TmpStore)
         load = partial(load, version='')
-    pickle_dict = {oid: None}
+
+    # BBB During the py2 -> py3 transition, we export business templates as
+    # protocol 1 even on py3 - but we need to use protocol 3 to use referencesf
+    # on py3, so we export once using protocol 3 and do a final export using protocol 1
+    pickle_dict = {
+      (oid, pickle_protocol): None,
+      (oid, HIGHEST_PICKLE_PROTOCOL): None,
+    }
     max_cache = [1e7] # do not cache more than 10MB of pickle data
-    def getReorderedPickle(oid):
-        p = pickle_dict.get(oid)
+    def getReorderedPickle(oid, pickle_protocol):
+        p = pickle_dict.get((oid, pickle_protocol))
         if p is None:
             p = load(oid)[0]
             p = reorderPickle(jar, p, pickle_protocol)[1]
             if len(p) < max_cache[0]:
                 max_cache[0] -= len(p)
-                pickle_dict[oid] = p
+                pickle_dict[oid, pickle_protocol] = p
         return p
 
     # Sort records and initialize id_mapping
@@ -355,9 +363,9 @@ def exportXML(jar, oid, file=None, pickle_protocol=DEFAULT_PICKLE_PROTOCOL):
     reordered_oid_list = [oid]
     for oid in reordered_oid_list:
         _mapOid(id_mapping, oid)
-        for oid in referencesf(getReorderedPickle(oid)):
-            if oid not in pickle_dict:
-                pickle_dict[oid] = None
+        for oid in referencesf(getReorderedPickle(oid, HIGHEST_PICKLE_PROTOCOL)):
+            if (oid, HIGHEST_PICKLE_PROTOCOL) not in pickle_dict:
+                pickle_dict[oid, HIGHEST_PICKLE_PROTOCOL] = None
                 reordered_oid_list.append(oid)
 
     # Do real export

product/ERP5Type/XMLExportImport/ppml.py

@@ -33,6 +33,7 @@ from marshal import loads as mloads
 from .xyap import NoBlanks
 from .xyap import xyap
 from Products.ERP5Type.Utils import bytes2str, str2bytes, unicode2str
+import lxml.etree
 
 from marshal import dumps as mdumps
 
@@ -348,10 +349,52 @@ class Persistent(Wrapper):
             return '%s<%s%s> %s </%s>\n' % (i, name, id, v.__str__(map_value=1)[:-1], name)
         elif isinstance(v,Scalar):
             return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name)
+        # BBB on py3 export bytes oids as <string> for stable business template XML
+        elif self._is_py3_p_oid_bytes_serialized_as_protocol_1() or self._is_py3_p_oid_bytes_serialized_with_references_as_protocol_1():
+            wrapped_s = v._subs[1]._subs[0]
+            print(wrapped_s.__str__())
+            if wrapped_s.encoding == 'base64':
+                wrapped_v = base64_decodebytes(wrapped_s._v.encode())
+            elif wrapped_s.encoding == 'cdata':
+                wrapped_v = lxml.etree.fromstring(wrapped_s.__str__()).xpath('./text()')[0].strip().encode()
+            else:
+                wrapped_v = wrapped_s._v.encode('latin1')
+            s = String(wrapped_v, self.mapping)
+            return Persistent(s, self.mapping).__str__(indent)
         else:
             v=v.__str__(indent+2)
             return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
 
+    def _is_py3_p_oid_bytes_serialized_as_protocol_1(self):
+        v = self._v
+        return six.PY3 and (
+            isinstance(v, Object)
+            and len(v._subs) == 2
+            and isinstance(v._subs[0], Klass)
+            and isinstance(v._subs[0]._v, Global)
+            and v._subs[0]._v.module == '_codecs'
+            and v._subs[0]._v.name == 'encode'
+            and isinstance(v._subs[1], Tuple)
+            and isinstance(v._subs[1]._subs[0], Unicode)
+            and isinstance(v._subs[1]._subs[1], Unicode)
+            and v._subs[1]._subs[1]._v == 'latin1'
+            and len(v._subs[1]._subs[0]._v.encode('latin1')) == 8
+        )
+    def _is_py3_p_oid_bytes_serialized_with_references_as_protocol_1(self):
+        # references, when more than one persistent object was exported
+        # this is saved with PUT and GET, so it becomes reference in the XML
+        v = self._v
+        return six.PY3 and (
+            isinstance(v, Object)
+            and len(v._subs) == 2
+            and isinstance(v._subs[0], Klass)
+            and isinstance(v._subs[0]._v, Reference)
+            and isinstance(v._subs[1], Tuple)
+            and isinstance(v._subs[1]._subs[0], Unicode)
+            and isinstance(v._subs[1]._subs[1], Reference)
+            and len(v._subs[1]._subs[0]._v.encode('latin1', 'ignore')) == 8
+        )
+
 
 blanck_line_expression = re.compile('^ +$')
 class NoBlanks(object):