Commit 0a973366 authored by Julien Muchembled's avatar Julien Muchembled

Speed up installation of Python Scripts and prepare clean up of XML

Before this patch, a Python Script was compiled several times during its
installation. Now, it is compiled only once.

'_code', 'func_code' and 'func_defaults' attributes are also not required
anymore in the XML representation of Python Scripts. This will allow to remove
this garbage for XML in the future.

For the moment, we have to provide forward compatibility with old revision of
ERP5 product, so this patch does not change exported XML.

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@38348 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 4d075606
...@@ -577,7 +577,7 @@ class BaseTemplateItem(Implicit, Persistent): ...@@ -577,7 +577,7 @@ class BaseTemplateItem(Implicit, Persistent):
def importFile(self, bta, **kw): def importFile(self, bta, **kw):
bta.importFiles(item=self) bta.importFiles(item=self)
def removeProperties(self, obj): def removeProperties(self, obj, export):
""" """
Remove unneeded properties for export Remove unneeded properties for export
""" """
...@@ -587,6 +587,7 @@ class BaseTemplateItem(Implicit, Persistent): ...@@ -587,6 +587,7 @@ class BaseTemplateItem(Implicit, Persistent):
attr_list = [ '_dav_writelocks', '_filepath', '_owner', 'uid', attr_list = [ '_dav_writelocks', '_filepath', '_owner', 'uid',
'workflow_history', '__ac_local_roles__' ] 'workflow_history', '__ac_local_roles__' ]
if export:
attr_list += { attr_list += {
'ERP5 Python Script': ('_lazy_compilation', 'Python_magic'), 'ERP5 Python Script': ('_lazy_compilation', 'Python_magic'),
}.get(meta_type, ()) }.get(meta_type, ())
...@@ -599,7 +600,13 @@ class BaseTemplateItem(Implicit, Persistent): ...@@ -599,7 +600,13 @@ class BaseTemplateItem(Implicit, Persistent):
if not obj.getProperty('business_template_include_content', 1): if not obj.getProperty('business_template_include_content', 1):
obj.deletePdfContent() obj.deletePdfContent()
elif meta_type == 'ERP5 Python Script': elif meta_type == 'ERP5 Python Script':
if export:
# XXX forward compatibility: set to None instead of deleting '_code'
# so that old BT code can import recent BT
obj._code = None obj._code = None
else:
# save result of automatic compilation
obj._p_changed = 1
elif interfaces.IIdGenerator.providedBy(obj): elif interfaces.IIdGenerator.providedBy(obj):
for dict_name in ('last_max_id_dict', 'last_id_dict'): for dict_name in ('last_max_id_dict', 'last_id_dict'):
if getattr(obj, dict_name, None) is not None: if getattr(obj, dict_name, None) is not None:
...@@ -717,7 +724,7 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -717,7 +724,7 @@ class ObjectTemplateItem(BaseTemplateItem):
relative_url = '/'.join([url,id]) relative_url = '/'.join([url,id])
obj = p.unrestrictedTraverse(relative_url) obj = p.unrestrictedTraverse(relative_url)
obj = obj._getCopy(context) obj = obj._getCopy(context)
obj = self.removeProperties(obj) obj = self.removeProperties(obj, 1)
id_list = obj.objectIds() # FIXME duplicated variable name id_list = obj.objectIds() # FIXME duplicated variable name
if hasattr(aq_base(obj), 'groups'): # XXX should check metatype instead if hasattr(aq_base(obj), 'groups'): # XXX should check metatype instead
# we must keep groups because they are deleted along with subobjects # we must keep groups because they are deleted along with subobjects
...@@ -744,7 +751,7 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -744,7 +751,7 @@ class ObjectTemplateItem(BaseTemplateItem):
except AttributeError: except AttributeError:
raise AttributeError, "Could not find object '%s' during business template processing." % relative_url raise AttributeError, "Could not find object '%s' during business template processing." % relative_url
_recursiveRemoveUid(obj) _recursiveRemoveUid(obj)
obj = self.removeProperties(obj) obj = self.removeProperties(obj, 1)
id_list = obj.objectIds() id_list = obj.objectIds()
if hasattr(aq_base(obj), 'groups'): # XXX should check metatype instead if hasattr(aq_base(obj), 'groups'): # XXX should check metatype instead
# we must keep groups because they are deleted along with subobjects # we must keep groups because they are deleted along with subobjects
...@@ -836,7 +843,7 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -836,7 +843,7 @@ class ObjectTemplateItem(BaseTemplateItem):
# FIXME: Why not use the importXML function directly? Are there any BT5s # FIXME: Why not use the importXML function directly? Are there any BT5s
# with actual .zexp files on the wild? # with actual .zexp files on the wild?
obj = connection.importFile(file_obj, customImporters=customImporters) obj = connection.importFile(file_obj, customImporters=customImporters)
self.removeProperties(obj) self.removeProperties(obj, 0)
self._objects[file_name[:-4]] = obj self._objects[file_name[:-4]] = obj
def preinstall(self, context, installed_item, **kw): def preinstall(self, context, installed_item, **kw):
...@@ -847,7 +854,7 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -847,7 +854,7 @@ class ObjectTemplateItem(BaseTemplateItem):
for path in self._objects: for path in self._objects:
if installed_item._objects.has_key(path): if installed_item._objects.has_key(path):
upgrade_list.append((path, upgrade_list.append((path,
self.removeProperties(installed_item._objects[path]))) self.removeProperties(installed_item._objects[path], 0)))
else: # new object else: # new object
modified_object_list[path] = 'New', type_name modified_object_list[path] = 'New', type_name
# update _p_jar property of objects cleaned by removeProperties # update _p_jar property of objects cleaned by removeProperties
...@@ -1038,10 +1045,6 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -1038,10 +1045,6 @@ class ObjectTemplateItem(BaseTemplateItem):
# install object # install object
obj = self._objects[path] obj = self._objects[path]
if getattr(obj, 'meta_type', None) in ('Script (Python)',
'ERP5 Python Script'):
if getattr(obj, '_code') is None:
obj._compile()
if getattr(aq_base(obj), 'groups', None) is not None: if getattr(aq_base(obj), 'groups', None) is not None:
# we must keep original order groups # we must keep original order groups
# because they change when we add subobjects # because they change when we add subobjects
...@@ -1379,7 +1382,7 @@ class PathTemplateItem(ObjectTemplateItem): ...@@ -1379,7 +1382,7 @@ class PathTemplateItem(ObjectTemplateItem):
obj = obj.__of__(context) obj = obj.__of__(context)
_recursiveRemoveUid(obj) _recursiveRemoveUid(obj)
id_list = obj.objectIds() id_list = obj.objectIds()
obj = self.removeProperties(obj) obj = self.removeProperties(obj, 1)
if hasattr(aq_base(obj), 'groups'): if hasattr(aq_base(obj), 'groups'):
# we must keep groups because it's ereased when we delete subobjects # we must keep groups because it's ereased when we delete subobjects
groups = deepcopy(obj.groups) groups = deepcopy(obj.groups)
...@@ -1496,7 +1499,7 @@ class CategoryTemplateItem(ObjectTemplateItem): ...@@ -1496,7 +1499,7 @@ class CategoryTemplateItem(ObjectTemplateItem):
relative_url = '/'.join([url,id]) relative_url = '/'.join([url,id])
obj = p.unrestrictedTraverse(relative_url) obj = p.unrestrictedTraverse(relative_url)
obj = obj._getCopy(context) obj = obj._getCopy(context)
obj = self.removeProperties(obj) obj = self.removeProperties(obj, 1)
id_list = obj.objectIds() id_list = obj.objectIds()
if id_list: if id_list:
self.build_sub_objects(context, id_list, relative_url) self.build_sub_objects(context, id_list, relative_url)
...@@ -1512,7 +1515,7 @@ class CategoryTemplateItem(ObjectTemplateItem): ...@@ -1512,7 +1515,7 @@ class CategoryTemplateItem(ObjectTemplateItem):
obj = p.unrestrictedTraverse(relative_url) obj = p.unrestrictedTraverse(relative_url)
obj = obj._getCopy(context) obj = obj._getCopy(context)
_recursiveRemoveUid(obj) _recursiveRemoveUid(obj)
obj = self.removeProperties(obj) obj = self.removeProperties(obj, 1)
include_sub_categories = obj.__of__(context).getProperty('business_template_include_sub_categories', 0) include_sub_categories = obj.__of__(context).getProperty('business_template_include_sub_categories', 0)
id_list = obj.objectIds() id_list = obj.objectIds()
if len(id_list) > 0 and include_sub_categories: if len(id_list) > 0 and include_sub_categories:
...@@ -1828,10 +1831,6 @@ class WorkflowTemplateItem(ObjectTemplateItem): ...@@ -1828,10 +1831,6 @@ class WorkflowTemplateItem(ObjectTemplateItem):
self._backupObject(action, trashbin, container_path, object_id, keep_subobjects=1) self._backupObject(action, trashbin, container_path, object_id, keep_subobjects=1)
container.manage_delObjects([object_id]) container.manage_delObjects([object_id])
obj = self._objects[path] obj = self._objects[path]
if getattr(obj, 'meta_type', None) in ('Script (Python)',
'ERP5 Python Script'):
if getattr(obj, '_code') is None:
obj._compile()
obj = obj._getCopy(container) obj = obj._getCopy(container)
container._setObject(object_id, obj) container._setObject(object_id, obj)
obj = container._getOb(object_id) obj = container._getOb(object_id)
...@@ -2634,7 +2633,7 @@ class CatalogMethodTemplateItem(ObjectTemplateItem): ...@@ -2634,7 +2633,7 @@ class CatalogMethodTemplateItem(ObjectTemplateItem):
obj=obj.aq_parent obj=obj.aq_parent
connection=obj._p_jar connection=obj._p_jar
obj = connection.importFile(file, customImporters=customImporters) obj = connection.importFile(file, customImporters=customImporters)
self.removeProperties(obj) self.removeProperties(obj, 0)
self._objects[file_name[:-4]] = obj self._objects[file_name[:-4]] = obj
else: else:
LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, )) LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
...@@ -2727,7 +2726,7 @@ class ActionTemplateItem(ObjectTemplateItem): ...@@ -2727,7 +2726,7 @@ class ActionTemplateItem(ObjectTemplateItem):
continue continue
raise NotFound('Action %r not found' % id) raise NotFound('Action %r not found' % id)
key = posixpath.join(url[-2], url[-1], value) key = posixpath.join(url[-2], url[-1], value)
self._objects[key] = self.removeProperties(action) self._objects[key] = self.removeProperties(action, 1)
self._objects[key].wl_clearLocks() self._objects[key].wl_clearLocks()
def install(self, context, trashbin, **kw): def install(self, context, trashbin, **kw):
...@@ -5060,8 +5059,8 @@ Business Template is a set of definitions, such as skins, portal types and categ ...@@ -5060,8 +5059,8 @@ Business Template is a set of definitions, such as skins, portal types and categ
f1 = StringIO() # for XML export of New Object f1 = StringIO() # for XML export of New Object
f2 = StringIO() # For XML export of Installed Object f2 = StringIO() # For XML export of Installed Object
# Remove unneeded properties # Remove unneeded properties
new_object = new_item.removeProperties(new_object) new_object = new_item.removeProperties(new_object, 1)
installed_object = installed_item.removeProperties(installed_object) installed_object = installed_item.removeProperties(installed_object, 1)
# XML Export in memory # XML Export in memory
OFS.XMLExportImport.exportXML(new_object._p_jar, new_object._p_oid, f1) OFS.XMLExportImport.exportXML(new_object._p_jar, new_object._p_oid, f1)
OFS.XMLExportImport.exportXML(installed_object._p_jar, OFS.XMLExportImport.exportXML(installed_object._p_jar,
......
...@@ -1230,6 +1230,24 @@ class ZEOServerTestCase(ERP5TypeTestCase): ...@@ -1230,6 +1230,24 @@ class ZEOServerTestCase(ERP5TypeTestCase):
self.zeo_server.close_server() self.zeo_server.close_server()
class lazy_func_prop(object):
"""Descriptor to delay the compilations of Python Scripts
until some of their attributes are accessed.
"""
default_dict = {}
def __init__(self, name, default):
self.name = name
self.default_dict[name] = default
def __get__(self, instance, owner):
if self.name not in instance.__dict__:
instance.__dict__.update(self.default_dict)
instance._orig_compile()
return instance.__dict__[self.name]
def __set__(self, instance, value):
instance.__dict__[self.name] = value
def __delete__(self, instance):
del instance.__dict__[self.name]
@onsetup @onsetup
def optimize(): def optimize():
'''Significantly reduces portal creation time.''' '''Significantly reduces portal creation time.'''
...@@ -1241,26 +1259,44 @@ def optimize(): ...@@ -1241,26 +1259,44 @@ def optimize():
# Delay the compilations of Python Scripts until they are really executed. # Delay the compilations of Python Scripts until they are really executed.
from Products.PythonScripts.PythonScript import PythonScript from Products.PythonScripts.PythonScript import PythonScript
PythonScript_compile = PythonScript._compile # In the future, Python Scripts will be exported without those 2 attributes:
PythonScript.func_code = lazy_func_prop('func_code', None)
PythonScript.func_defaults = lazy_func_prop('func_defaults', None)
PythonScript._orig_compile = PythonScript._compile
def _compile(self): def _compile(self):
self._lazy_compilation = 1 # mark the script as being not compiled
for name in lazy_func_prop.default_dict:
self.__dict__.pop(name, None)
PythonScript._compile = _compile PythonScript._compile = _compile
PythonScript_exec = PythonScript._exec PythonScript_exec = PythonScript._exec
def _exec(self, *args): def _exec(self, *args):
if getattr(self, '_lazy_compilation', 0): self.func_code # trigger compilation if needed
self._lazy_compilation = 0
PythonScript_compile(self)
return PythonScript_exec(self, *args) return PythonScript_exec(self, *args)
PythonScript._exec = _exec PythonScript._exec = _exec
from Acquisition import aq_parent from Acquisition import aq_parent
def _makeFunction(self, dummy=0): # CMFCore.FSPythonScript uses dummy arg. def _makeFunction(self, dummy=0): # CMFCore.FSPythonScript uses dummy arg.
self.ZCacheable_invalidate() self.ZCacheable_invalidate()
PythonScript_compile(self) self.__dict__.update(lazy_func_prop.default_dict)
self._orig_compile()
if not (aq_parent(self) is None or hasattr(self, '_filepath')): if not (aq_parent(self) is None or hasattr(self, '_filepath')):
# It needs a _filepath, and has an acquisition wrapper. # It needs a _filepath, and has an acquisition wrapper.
self._filepath = self.get_filepath() self._filepath = self.get_filepath()
PythonScript._makeFunction = _makeFunction PythonScript._makeFunction = _makeFunction
# XXX Previous implementation of this 'optimize' function requires that
# Python Scripts always contain up-to-date data for 'func_code' and
# 'func_defaults' properties, so make sure we always export them.
# This compatibility code is not required for normal ERP5 instances
# because scripts are compiled at BT installation.
from Products.ERP5Type.Document.BusinessTemplate import BaseTemplateItem
BaseTemplateItem_removeProperties = BaseTemplateItem.removeProperties
def removeProperties(self, obj, export):
if export and isinstance(obj, PythonScript):
obj.func_code # trigger compilation if needed
return BaseTemplateItem_removeProperties(self, obj, export)
BaseTemplateItem.removeProperties = removeProperties
# Do not reindex portal types sub objects by default # Do not reindex portal types sub objects by default
# We will probably disable reindexing for other types later # We will probably disable reindexing for other types later
full_indexing_set = set(os.environ.get('enable_full_indexing', '').split(',')) full_indexing_set = set(os.environ.get('enable_full_indexing', '').split(','))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment