Commit 91172319 authored by Julien Muchembled's avatar Julien Muchembled

Fix and speed up XML export of objects

- Fix random failures in testBusinessTemplate on Zope 2.12
- Do not sort record per 'id' anymore because it was a noop:
  first value returned by reorderPickle is not the object but its state,
  so "getattr(o, 'id', None)" was always None
- By reducing the number of calls to reorderedPickle/XMLObject, exportXML
  function is now much faster.

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@40325 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent a9adcdb2
...@@ -154,17 +154,20 @@ def reorderPickle(jar, p): ...@@ -154,17 +154,20 @@ def reorderPickle(jar, p):
p = p.replace('_','',1) p = p.replace('_','',1)
return obj, p return obj, p
def _mapOid(id_mapping, oid):
idprefix = str(u64(oid))
id = id_mapping[idprefix]
old_aka = encodestring(oid)[:-1]
aka=encodestring(p64(long(id)))[:-1] # Rebuild oid based on mapped id
id_mapping.setConvertedAka(old_aka, aka)
return idprefix+'.', id, aka
def XMLrecord(oid, plen, p, id_mapping): def XMLrecord(oid, plen, p, id_mapping):
# Proceed as usual # Proceed as usual
q=ppml.ToXMLUnpickler q=ppml.ToXMLUnpickler
f=StringIO(p) f=StringIO(p)
u=q(f) u=q(f)
id=u64(oid) u.idprefix, id, aka = _mapOid(id_mapping, oid)
u.idprefix=str(id)+'.'
id = id_mapping[id]
old_aka = encodestring(oid)[:-1]
aka=encodestring(p64(long(id)))[:-1] # Rebuild oid based on mapped id
id_mapping.setConvertedAka(old_aka, aka)
p=u.load(id_mapping=id_mapping).__str__(4) p=u.load(id_mapping=id_mapping).__str__(4)
if f.tell() < plen: if f.tell() < plen:
p=p+u.load(id_mapping=id_mapping).__str__(4) p=p+u.load(id_mapping=id_mapping).__str__(4)
...@@ -174,68 +177,47 @@ def XMLrecord(oid, plen, p, id_mapping): ...@@ -174,68 +177,47 @@ def XMLrecord(oid, plen, p, id_mapping):
XMLExportImport.XMLrecord = XMLrecord XMLExportImport.XMLrecord = XMLrecord
def exportXML(jar, oid, file=None): def exportXML(jar, oid, file=None):
# XXX: For performance reasons, we should change XMLExportImport/ppml code # For performance reasons, exportXML does not use 'XMLrecord' anymore to map
# so that we can call reorderPickle and XMLrecord only once. # oids. This requires to initialize MinimalMapping.marked_reference before
# This means we should be able to do a real export immediately. # any string output, i.e. in ppml.Reference.__init__
# This would also fix random failures when DemoStorage is used, # This also fixed random failures when DemoStorage is used, because oids
# because oids can have values that have a shorter representation # can have values that have a shorter representation in 'repr' instead of
# in 'repr' instead of 'base64' (see ppml.convert) and ppml.String # 'base64' (see ppml.convert) and ppml.String does not support this.
# does not support this. load = jar._storage.load
pickle_dict = {oid: None}
if file is None: file=TemporaryFile() max_cache = [1e7] # do not cache more than 10MB of pickle data
elif type(file) is StringType: file=open(file,'w+b') def getReorderedPickle(oid):
id_mapping = ppml.MinimalMapping() p = pickle_dict[oid]
#id_mapping = ppml.IdentityMapping() if p is None:
write=file.write
write('<?xml version="1.0"?>\012<ZopeData>\012')
# Versions are ignored, but some 'load()' implementations require them # Versions are ignored, but some 'load()' implementations require them
# FIXME: remove 'version' when TmpStore.load() on ZODB stops asking for it. # FIXME: remove "''" when TmpStore.load() on ZODB stops asking for it.
version='' p = load(oid, '')[0]
ref=referencesf p = reorderPickle(jar, p)[1]
oids=[oid] if len(p) < max_cache[0]:
done_oids={} max_cache[0] -= len(p)
done=done_oids.has_key pickle_dict[oid] = p
load=jar._storage.load return p
original_oid = oid
reordered_pickle = [] # Sort records and initialize id_mapping
# Build mapping for refs id_mapping = ppml.MinimalMapping()
while oids: reordered_oid_list = [oid]
oid=oids[0] for oid in reordered_oid_list:
del oids[0] _mapOid(id_mapping, oid)
if done(oid): continue for oid in referencesf(getReorderedPickle(oid)):
done_oids[oid]=1 if oid not in pickle_dict:
try: p, serial = load(oid, version) pickle_dict[oid] = None
except: reordered_oid_list.append(oid)
# Ick, a broken reference
log.error('exportXML: could not load oid %r' % oid,
exc_info=True)
else:
o, p = reorderPickle(jar, p)
reordered_pickle.append((oid, o, p))
XMLrecord(oid,len(p),p, id_mapping)
# Determine new oids added to the list after reference calculation
old_oids = tuple(oids)
ref(p, oids)
new_oids = []
for i in oids:
if i not in old_oids: new_oids.append(i)
# Sort new oids based on id of object
new_oidict = {}
for oid in new_oids:
try:
p, serial = load(oid, version)
o, p = reorderPickle(jar, p)
new_oidict[oid] = getattr(o, 'id', None)
except:
log.error('exportXML: could not load oid %r' % oid,
exc_info=True)
new_oidict[oid] = None # Ick, a broken reference
new_oids.sort(key=lambda x: new_oidict[x])
# Build new sorted oids
oids = list(old_oids) + new_oids
# Do real export # Do real export
for (oid, o, p) in reordered_pickle: if file is None:
write(XMLrecord(oid,len(p),p, id_mapping)) file = TemporaryFile()
elif isinstance(file, basestring):
file = open(file, 'w+b')
write = file.write
write('<?xml version="1.0"?>\n<ZopeData>\n')
for oid in reordered_oid_list:
p = getReorderedPickle(oid)
write(XMLrecord(oid, len(p), p, id_mapping))
write('</ZopeData>\n') write('</ZopeData>\n')
return file return file
......
...@@ -276,6 +276,7 @@ class Reference(Scalar): ...@@ -276,6 +276,7 @@ class Reference(Scalar):
def __init__(self, v, mapping): def __init__(self, v, mapping):
self._v=v self._v=v
self.mapping = mapping self.mapping = mapping
mapping.mark(v)
def __str__(self, indent=0): def __str__(self, indent=0):
v=self._v v=self._v
#LOG('Reference', 0, str(v)) #LOG('Reference', 0, str(v))
...@@ -284,7 +285,6 @@ class Reference(Scalar): ...@@ -284,7 +285,6 @@ class Reference(Scalar):
else: else:
name = self.__class__.__name__.lower() name = self.__class__.__name__.lower()
#LOG('noImmutable', 0, "%s mapped to %s" % (v, self.mapping[v])) #LOG('noImmutable', 0, "%s mapped to %s" % (v, self.mapping[v]))
self.mapping.mark(v)
value = '<%s id="%s"/>' % (name, self.mapping[v]) value = '<%s id="%s"/>' % (name, self.mapping[v])
return '%s%s\n' % (' '*indent, value) return '%s%s\n' % (' '*indent, value)
...@@ -315,6 +315,7 @@ ppml.NoBlanks = NoBlanks ...@@ -315,6 +315,7 @@ ppml.NoBlanks = NoBlanks
class IdentityMapping: class IdentityMapping:
def __init__(self): def __init__(self):
self.resetMapping()
self.immutable = {} self.immutable = {}
def resetMapping(self): def resetMapping(self):
...@@ -348,14 +349,6 @@ class IdentityMapping: ...@@ -348,14 +349,6 @@ class IdentityMapping:
ppml.IdentityMapping = IdentityMapping ppml.IdentityMapping = IdentityMapping
class MinimalMapping(IdentityMapping): class MinimalMapping(IdentityMapping):
def __init__(self):
self.mapped_id = {}
self.mapped_core_id = {}
self.last_sub_id = {}
self.last_id = 1
self.converted_aka = {}
self.marked_reference = {}
self.immutable = {}
def resetMapping(self): def resetMapping(self):
self.mapped_id = {} self.mapped_id = {}
...@@ -363,7 +356,7 @@ class MinimalMapping(IdentityMapping): ...@@ -363,7 +356,7 @@ class MinimalMapping(IdentityMapping):
self.last_sub_id = {} self.last_sub_id = {}
self.last_id = 1 self.last_id = 1
self.converted_aka = {} self.converted_aka = {}
self.marked_reference = {} self.marked_reference = set()
def __getitem__(self, id): def __getitem__(self, id):
id = str(id) id = str(id)
...@@ -402,10 +395,10 @@ class MinimalMapping(IdentityMapping): ...@@ -402,10 +395,10 @@ class MinimalMapping(IdentityMapping):
self.converted_aka[old] = new self.converted_aka[old] = new
def mark(self, v): def mark(self, v):
self.marked_reference[v] = 1 self.marked_reference.add(v)
def isMarked(self, v): def isMarked(self, v):
return self.marked_reference.has_key(v) return v in self.marked_reference
def __str__(self, a): def __str__(self, a):
return "Error here" return "Error here"
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment