Commit 0511e0de authored by Arnaud Fontaine's avatar Arnaud Fontaine

Optimize Component loading by using a registry, similar to ERP5Type.document_class_registry.

Upon startup, a registry (dict) is initialized for each Component
module. Then, everytime a Component is validated/invalidated, this registry is
updated.

This avoids looking whether a Component exists in ZODB/Catalog and is
validated in Component Tool on every imports or when loading a Component,
instead the registry is used. Moreover, the Catalog was used before as the
Component are matched by their reference and their validation state, thus
making bootstrap tricky.
parent fb5912f8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>state_change[\'object\'].addToRegistry()\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>state_change</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>addToRegistry</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -50,8 +50,7 @@ ...@@ -50,8 +50,7 @@
</item> </item>
<item> <item>
<key> <string>_body</string> </key> <key> <string>_body</string> </key>
<value> <string>object = state_change[\'object\']\n <value> <string>state_change[\'object\'].Base_checkConsistency()\n
object.Base_checkConsistency()\n
</string> </value> </string> </value>
</item> </item>
<item> <item>
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>state_change[\'object\'].deleteFromRegistry()\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>state_change</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>deleteFromRegistry</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -10,6 +10,10 @@ ...@@ -10,6 +10,10 @@
<key> <string>actbox_category</string> </key> <key> <string>actbox_category</string> </key>
<value> <string>workflow</string> </value> <value> <string>workflow</string> </value>
</item> </item>
<item>
<key> <string>actbox_icon</string> </key>
<value> <string></string> </value>
</item>
<item> <item>
<key> <string>actbox_name</string> </key> <key> <string>actbox_name</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
...@@ -42,7 +46,7 @@ ...@@ -42,7 +46,7 @@
</item> </item>
<item> <item>
<key> <string>script_name</string> </key> <key> <string>script_name</string> </key>
<value> <string></string> </value> <value> <string>deleteFromRegistry</string> </value>
</item> </item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
......
...@@ -10,6 +10,10 @@ ...@@ -10,6 +10,10 @@
<key> <string>actbox_category</string> </key> <key> <string>actbox_category</string> </key>
<value> <string>workflow</string> </value> <value> <string>workflow</string> </value>
</item> </item>
<item>
<key> <string>actbox_icon</string> </key>
<value> <string></string> </value>
</item>
<item> <item>
<key> <string>actbox_name</string> </key> <key> <string>actbox_name</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
...@@ -42,7 +46,7 @@ ...@@ -42,7 +46,7 @@
</item> </item>
<item> <item>
<key> <string>script_name</string> </key> <key> <string>script_name</string> </key>
<value> <string></string> </value> <value> <string>addToRegistry</string> </value>
</item> </item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
......
2012-02-11 arnaud.fontaine
* Update component module registry upon validate/invalidate.
2012-02-07 arnaud.fontaine 2012-02-07 arnaud.fontaine
* Use resetOnceAtTransactionBoundary when a Component has been modified. * Use resetOnceAtTransactionBoundary when a Component has been modified.
......
41000 41001
\ No newline at end of file \ No newline at end of file
...@@ -87,6 +87,29 @@ class Component(Base): ...@@ -87,6 +87,29 @@ class Component(Base):
return [] return []
def addToRegistry(self):
"""
Add the Component to its appropriate module registry
"""
namespace_fullname = self._getDynamicModuleNamespace()
namespace_module = __import__(namespace_fullname, {}, {},
fromlist=[namespace_fullname])
reference = self.getReference()
namespace_module._registry_dict[reference] = {
'component': self,
'module_name': "%s.%s" % (namespace_fullname, reference)}
def deleteFromRegistry(self):
"""
Delete the Component from its appropriate module registry
"""
namespace_fullname = self._getDynamicModuleNamespace()
namespace_module = __import__(namespace_fullname, {}, {},
fromlist=[namespace_fullname])
del namespace_module._registry_dict[self.getReference()]
def _setTextContent(self, text_content): def _setTextContent(self, text_content):
""" """
When the validation state is already 'validated', set the new value to When the validation state is already 'validated', set the new value to
...@@ -236,7 +259,6 @@ class Component(Base): ...@@ -236,7 +259,6 @@ class Component(Base):
exec source_code in namespace_dict exec source_code in namespace_dict
return context.newContent(id=id, return context.newContent(id=id,
# XXX-arnau: useless field?
reference=class_name, reference=class_name,
text_content=source_code, text_content=source_code,
portal_type=cls.portal_type) portal_type=cls.portal_type)
...@@ -206,11 +206,8 @@ class TypesTool(TypeProvider): ...@@ -206,11 +206,8 @@ class TypesTool(TypeProvider):
from Products.ERP5Type import document_class_registry from Products.ERP5Type import document_class_registry
document_type_set = set(document_class_registry) document_type_set = set(document_class_registry)
# XXX-arnau: should be cached and reference? import erp5.component.document
component_tool = self.getPortalObject().portal_components document_type_set.update(erp5.component.document._registry_dict)
for obj in component_tool.searchFolder(portal_type='Document Component',
validation_state='validated'):
document_type_set.add(obj.getReference())
return sorted(document_type_set) return sorted(document_type_set)
......
...@@ -54,6 +54,7 @@ class ComponentDynamicPackage(ModuleType): ...@@ -54,6 +54,7 @@ class ComponentDynamicPackage(ModuleType):
# Necessary otherwise imports will fail because an object is considered a # Necessary otherwise imports will fail because an object is considered a
# package only if __path__ is defined # package only if __path__ is defined
__path__ = [] __path__ = []
__registry_dict = None
def __init__(self, namespace, portal_type): def __init__(self, namespace, portal_type):
super(ComponentDynamicPackage, self).__init__(namespace) super(ComponentDynamicPackage, self).__init__(namespace)
...@@ -70,6 +71,43 @@ class ComponentDynamicPackage(ModuleType): ...@@ -70,6 +71,43 @@ class ComponentDynamicPackage(ModuleType):
# Add the import hook # Add the import hook
sys.meta_path.append(self) sys.meta_path.append(self)
@property
def _registry_dict(self):
"""
Create the component registry, this is very similar to
Products.ERP5Type.document_class_registry and avoids checking whether a
Component exists at each call at the expense to increase startup
time. Moreover, it allows to handle reference easily.
XXX-arnau: handle different versions of a Component, perhaps something
like erp5.component.extension.VERSION.REFERENCE perhaps but there should
be a a way to specify priorities such as portal_skins maybe?
"""
if self.__registry_dict is None:
try:
component_tool = getSite().portal_components
# XXX-arnau: When installing ERP5 site, erp5_core_components has not
# been installed yet, thus this will obviously failed...
except AttributeError:
return {}
self.__registry_dict = {}
# XXX-arnau: contentValues should not be used as there may be a large
# number of objects, but as this is done only once, that should perhaps
# not be a problem after all, and using the Catalog is too risky as it
# lags behind and depends upon objects being reindexed
for component in component_tool.contentValues(portal_type=self._portal_type):
# Only consider modified or validated states as state transition will
# be handled by component_validation_workflow which will take care of
# updating the registry
if component.getValidationState() in ('modified', 'validated'):
reference = component.getReference()
self.__registry_dict[reference] = {
'component': component,
'module_name': self._namespace_prefix + reference}
return self.__registry_dict
def find_module(self, fullname, path=None): def find_module(self, fullname, path=None):
# Ignore any absolute imports which does not start with this package # Ignore any absolute imports which does not start with this package
# prefix, None there means that "normal" sys.path will be used # prefix, None there means that "normal" sys.path will be used
...@@ -86,29 +124,9 @@ class ComponentDynamicPackage(ModuleType): ...@@ -86,29 +124,9 @@ class ComponentDynamicPackage(ModuleType):
# Skip components not available, otherwise Products for example could be # Skip components not available, otherwise Products for example could be
# wrongly considered as importable and thus the actual filesystem class # wrongly considered as importable and thus the actual filesystem class
# ignored # ignored
# if name not in self._registry_dict:
# XXX-arnau: This must use reference rather than ID
site = getSite()
component = getattr(site.portal_components.aq_explicit, fullname, None)
if not (component and
component.getValidationState() in ('modified', 'validated')):
return None return None
# XXX-arnau: Using the Catalog should be preferred however it is not
# really possible for two reasons: 1/ the Catalog lags behind the ZODB
# thus immediately after adding/removing a Component, it will fail to load
# a Component because of reindexing 2/ this is unsurprisingly really slow
# compared to a ZODB access.
#
# site = getSite()
# found = list(site.portal_catalog.unrestrictedSearchResults(
# reference=name,
# portal_type=self._portal_type,
# parent_uid=site.portal_components.getUid(),
# validation_state=('validated', 'modified')))
# if not found:
# return None
return self return self
def load_module(self, fullname): def load_module(self, fullname):
...@@ -124,22 +142,11 @@ class ComponentDynamicPackage(ModuleType): ...@@ -124,22 +142,11 @@ class ComponentDynamicPackage(ModuleType):
site = getSite() site = getSite()
# XXX-arnau: erp5.component.extension.VERSION.REFERENCE perhaps but there
# should be a a way to specify priorities such as portal_skins maybe?
component_name = fullname.replace(self._namespace_prefix, '') component_name = fullname.replace(self._namespace_prefix, '')
component_id = '%s.%s' % (self._namespace, component_name) component_id = '%s.%s' % (self._namespace, component_name)
try: try:
# XXX-arnau: Performances (~ 200x slower than direct access to ZODB) and component = self._registry_dict[component_name]['component']
# also lag behind the ZODB (e.g. reindexing), so this is certainly not a except KeyError:
# good solution
component = site.portal_catalog.unrestrictedSearchResults(
parent_uid=site.portal_components.getUid(),
reference=component_name,
validation_state=('validated', 'modified'),
portal_type=self._portal_type)[0].getObject()
# component = getattr(site.portal_components, component_id)
except IndexError:
LOG("ERP5Type.dynamic", INFO, LOG("ERP5Type.dynamic", INFO,
"Could not find %s or it has not been validated or it has not been " "Could not find %s or it has not been validated or it has not been "
"migrated yet?" % component_id) "migrated yet?" % component_id)
......
...@@ -42,12 +42,13 @@ from Products.ERP5Type.TransactionalVariable import TransactionalResource ...@@ -42,12 +42,13 @@ from Products.ERP5Type.TransactionalVariable import TransactionalResource
from zLOG import LOG, ERROR, INFO, WARNING, PANIC from zLOG import LOG, ERROR, INFO, WARNING, PANIC
def _importClass(classpath): def _importClass(classpath, is_zodb_document=False):
try: try:
module_path, class_name = classpath.rsplit('.', 1) module_path, class_name = classpath.rsplit('.', 1)
module = __import__(module_path, {}, {}, (module_path,)) module = __import__(module_path, {}, {}, (module_path,))
klass = getattr(module, class_name) klass = getattr(module, class_name)
if not is_zodb_document:
# XXX is this required? (here?) # XXX is this required? (here?)
setDefaultClassProperties(klass) setDefaultClassProperties(klass)
InitializeClass(klass) InitializeClass(klass)
...@@ -87,12 +88,6 @@ core_portal_type_class_dict = { ...@@ -87,12 +88,6 @@ core_portal_type_class_dict = {
'generating': False}, 'generating': False},
'Solver Tool': {'type_class': 'SolverTool', 'Solver Tool': {'type_class': 'SolverTool',
'generating': False}, 'generating': False},
# Needed to load Components
#
# XXX-arnau: only for now as the Catalog is being used (parent_uid
# especially), but it will later be replaced anyway...
'Category Tool': {'type_class': 'CategoryTool',
'generating': False}
} }
def generatePortalTypeClass(site, portal_type_name): def generatePortalTypeClass(site, portal_type_name):
...@@ -188,35 +183,45 @@ def generatePortalTypeClass(site, portal_type_name): ...@@ -188,35 +183,45 @@ def generatePortalTypeClass(site, portal_type_name):
raise AttributeError('Document class is not defined on Portal Type %s' \ raise AttributeError('Document class is not defined on Portal Type %s' \
% portal_type_name) % portal_type_name)
is_zodb_document = False
klass = None klass = None
if '.' in type_class: if '.' in type_class:
type_class_path = type_class type_class_path = type_class
else: else:
type_class_path = None
# Skip any document within ERP5Type Product as it is needed for # Skip any document within ERP5Type Product as it is needed for
# bootstrapping anyway # bootstrapping anyway
type_class_namespace = document_class_registry.get(type_class, '') type_class_namespace = document_class_registry.get(type_class, '')
if not (type_class_namespace.startswith('Products.ERP5Type') or if not (type_class_namespace.startswith('Products.ERP5Type') or
portal_type_name in core_portal_type_class_dict): portal_type_name in core_portal_type_class_dict):
try: import erp5.component.document
klass = getattr(__import__('erp5.component.document.' + type_class, module_info_dict = erp5.component.document._registry_dict.get(type_class,
fromlist=['erp5.component.document'], None)
level=0), if module_info_dict:
type_class) type_class_path = "%s.%s" % (module_info_dict['module_name'], type_class)
is_zodb_document = True
except (ImportError, AttributeError):
pass if type_class_path is None:
if klass is None:
type_class_path = document_class_registry.get(type_class, None) type_class_path = document_class_registry.get(type_class, None)
if type_class_path is None:
if klass is None and type_class_path is None:
raise AttributeError('Document class %s has not been registered:' raise AttributeError('Document class %s has not been registered:'
' cannot import it as base of Portal Type %s' ' cannot import it as base of Portal Type %s'
% (type_class, portal_type_name)) % (type_class, portal_type_name))
if klass is None: try:
klass = _importClass(type_class_path, is_zodb_document)
except ImportError:
# A Document Component should always have a class matching its reference,
# so this should never happen...
if is_zodb_document:
type_class_path = document_class_registry.get(type_class, None)
if type_class_path is not None:
klass = _importClass(type_class_path) klass = _importClass(type_class_path)
if klass is None:
raise
global property_sheet_generating_portal_type_set global property_sheet_generating_portal_type_set
accessor_holder_list = [] accessor_holder_list = []
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment