Commit 0511e0de authored by Arnaud Fontaine's avatar Arnaud Fontaine

Optimize Component loading by using a registry, similar to ERP5Type.document_class_registry.

Upon startup, a registry (dict) is initialized for each Component
module. Then, everytime a Component is validated/invalidated, this registry is
updated.

This avoids looking whether a Component exists in ZODB/Catalog and is
validated in Component Tool on every imports or when loading a Component,
instead the registry is used. Moreover, the Catalog was used before as the
Component are matched by their reference and their validation state, thus
making bootstrap tricky.
parent fb5912f8
No related merge requests found
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>state_change[\'object\'].addToRegistry()\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>state_change</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>addToRegistry</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -50,8 +50,7 @@ ...@@ -50,8 +50,7 @@
</item> </item>
<item> <item>
<key> <string>_body</string> </key> <key> <string>_body</string> </key>
<value> <string>object = state_change[\'object\']\n <value> <string>state_change[\'object\'].Base_checkConsistency()\n
object.Base_checkConsistency()\n
</string> </value> </string> </value>
</item> </item>
<item> <item>
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>state_change[\'object\'].deleteFromRegistry()\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>state_change</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>deleteFromRegistry</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -10,6 +10,10 @@ ...@@ -10,6 +10,10 @@
<key> <string>actbox_category</string> </key> <key> <string>actbox_category</string> </key>
<value> <string>workflow</string> </value> <value> <string>workflow</string> </value>
</item> </item>
<item>
<key> <string>actbox_icon</string> </key>
<value> <string></string> </value>
</item>
<item> <item>
<key> <string>actbox_name</string> </key> <key> <string>actbox_name</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
...@@ -42,7 +46,7 @@ ...@@ -42,7 +46,7 @@
</item> </item>
<item> <item>
<key> <string>script_name</string> </key> <key> <string>script_name</string> </key>
<value> <string></string> </value> <value> <string>deleteFromRegistry</string> </value>
</item> </item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
......
...@@ -10,6 +10,10 @@ ...@@ -10,6 +10,10 @@
<key> <string>actbox_category</string> </key> <key> <string>actbox_category</string> </key>
<value> <string>workflow</string> </value> <value> <string>workflow</string> </value>
</item> </item>
<item>
<key> <string>actbox_icon</string> </key>
<value> <string></string> </value>
</item>
<item> <item>
<key> <string>actbox_name</string> </key> <key> <string>actbox_name</string> </key>
<value> <string></string> </value> <value> <string></string> </value>
...@@ -42,7 +46,7 @@ ...@@ -42,7 +46,7 @@
</item> </item>
<item> <item>
<key> <string>script_name</string> </key> <key> <string>script_name</string> </key>
<value> <string></string> </value> <value> <string>addToRegistry</string> </value>
</item> </item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
......
2012-02-11 arnaud.fontaine
* Update component module registry upon validate/invalidate.
2012-02-07 arnaud.fontaine 2012-02-07 arnaud.fontaine
* Use resetOnceAtTransactionBoundary when a Component has been modified. * Use resetOnceAtTransactionBoundary when a Component has been modified.
......
41000 41001
\ No newline at end of file \ No newline at end of file
...@@ -87,6 +87,29 @@ class Component(Base): ...@@ -87,6 +87,29 @@ class Component(Base):
return [] return []
def addToRegistry(self):
"""
Add the Component to its appropriate module registry
"""
namespace_fullname = self._getDynamicModuleNamespace()
namespace_module = __import__(namespace_fullname, {}, {},
fromlist=[namespace_fullname])
reference = self.getReference()
namespace_module._registry_dict[reference] = {
'component': self,
'module_name': "%s.%s" % (namespace_fullname, reference)}
def deleteFromRegistry(self):
"""
Delete the Component from its appropriate module registry
"""
namespace_fullname = self._getDynamicModuleNamespace()
namespace_module = __import__(namespace_fullname, {}, {},
fromlist=[namespace_fullname])
del namespace_module._registry_dict[self.getReference()]
def _setTextContent(self, text_content): def _setTextContent(self, text_content):
""" """
When the validation state is already 'validated', set the new value to When the validation state is already 'validated', set the new value to
...@@ -236,7 +259,6 @@ class Component(Base): ...@@ -236,7 +259,6 @@ class Component(Base):
exec source_code in namespace_dict exec source_code in namespace_dict
return context.newContent(id=id, return context.newContent(id=id,
# XXX-arnau: useless field?
reference=class_name, reference=class_name,
text_content=source_code, text_content=source_code,
portal_type=cls.portal_type) portal_type=cls.portal_type)
...@@ -206,11 +206,8 @@ class TypesTool(TypeProvider): ...@@ -206,11 +206,8 @@ class TypesTool(TypeProvider):
from Products.ERP5Type import document_class_registry from Products.ERP5Type import document_class_registry
document_type_set = set(document_class_registry) document_type_set = set(document_class_registry)
# XXX-arnau: should be cached and reference? import erp5.component.document
component_tool = self.getPortalObject().portal_components document_type_set.update(erp5.component.document._registry_dict)
for obj in component_tool.searchFolder(portal_type='Document Component',
validation_state='validated'):
document_type_set.add(obj.getReference())
return sorted(document_type_set) return sorted(document_type_set)
......
...@@ -54,6 +54,7 @@ class ComponentDynamicPackage(ModuleType): ...@@ -54,6 +54,7 @@ class ComponentDynamicPackage(ModuleType):
# Necessary otherwise imports will fail because an object is considered a # Necessary otherwise imports will fail because an object is considered a
# package only if __path__ is defined # package only if __path__ is defined
__path__ = [] __path__ = []
__registry_dict = None
def __init__(self, namespace, portal_type): def __init__(self, namespace, portal_type):
super(ComponentDynamicPackage, self).__init__(namespace) super(ComponentDynamicPackage, self).__init__(namespace)
...@@ -70,6 +71,43 @@ class ComponentDynamicPackage(ModuleType): ...@@ -70,6 +71,43 @@ class ComponentDynamicPackage(ModuleType):
# Add the import hook # Add the import hook
sys.meta_path.append(self) sys.meta_path.append(self)
@property
def _registry_dict(self):
"""
Create the component registry, this is very similar to
Products.ERP5Type.document_class_registry and avoids checking whether a
Component exists at each call at the expense to increase startup
time. Moreover, it allows to handle reference easily.
XXX-arnau: handle different versions of a Component, perhaps something
like erp5.component.extension.VERSION.REFERENCE perhaps but there should
be a a way to specify priorities such as portal_skins maybe?
"""
if self.__registry_dict is None:
try:
component_tool = getSite().portal_components
# XXX-arnau: When installing ERP5 site, erp5_core_components has not
# been installed yet, thus this will obviously failed...
except AttributeError:
return {}
self.__registry_dict = {}
# XXX-arnau: contentValues should not be used as there may be a large
# number of objects, but as this is done only once, that should perhaps
# not be a problem after all, and using the Catalog is too risky as it
# lags behind and depends upon objects being reindexed
for component in component_tool.contentValues(portal_type=self._portal_type):
# Only consider modified or validated states as state transition will
# be handled by component_validation_workflow which will take care of
# updating the registry
if component.getValidationState() in ('modified', 'validated'):
reference = component.getReference()
self.__registry_dict[reference] = {
'component': component,
'module_name': self._namespace_prefix + reference}
return self.__registry_dict
def find_module(self, fullname, path=None): def find_module(self, fullname, path=None):
# Ignore any absolute imports which does not start with this package # Ignore any absolute imports which does not start with this package
# prefix, None there means that "normal" sys.path will be used # prefix, None there means that "normal" sys.path will be used
...@@ -86,29 +124,9 @@ class ComponentDynamicPackage(ModuleType): ...@@ -86,29 +124,9 @@ class ComponentDynamicPackage(ModuleType):
# Skip components not available, otherwise Products for example could be # Skip components not available, otherwise Products for example could be
# wrongly considered as importable and thus the actual filesystem class # wrongly considered as importable and thus the actual filesystem class
# ignored # ignored
# if name not in self._registry_dict:
# XXX-arnau: This must use reference rather than ID
site = getSite()
component = getattr(site.portal_components.aq_explicit, fullname, None)
if not (component and
component.getValidationState() in ('modified', 'validated')):
return None return None
# XXX-arnau: Using the Catalog should be preferred however it is not
# really possible for two reasons: 1/ the Catalog lags behind the ZODB
# thus immediately after adding/removing a Component, it will fail to load
# a Component because of reindexing 2/ this is unsurprisingly really slow
# compared to a ZODB access.
#
# site = getSite()
# found = list(site.portal_catalog.unrestrictedSearchResults(
# reference=name,
# portal_type=self._portal_type,
# parent_uid=site.portal_components.getUid(),
# validation_state=('validated', 'modified')))
# if not found:
# return None
return self return self
def load_module(self, fullname): def load_module(self, fullname):
...@@ -124,22 +142,11 @@ class ComponentDynamicPackage(ModuleType): ...@@ -124,22 +142,11 @@ class ComponentDynamicPackage(ModuleType):
site = getSite() site = getSite()
# XXX-arnau: erp5.component.extension.VERSION.REFERENCE perhaps but there
# should be a a way to specify priorities such as portal_skins maybe?
component_name = fullname.replace(self._namespace_prefix, '') component_name = fullname.replace(self._namespace_prefix, '')
component_id = '%s.%s' % (self._namespace, component_name) component_id = '%s.%s' % (self._namespace, component_name)
try: try:
# XXX-arnau: Performances (~ 200x slower than direct access to ZODB) and component = self._registry_dict[component_name]['component']
# also lag behind the ZODB (e.g. reindexing), so this is certainly not a except KeyError:
# good solution
component = site.portal_catalog.unrestrictedSearchResults(
parent_uid=site.portal_components.getUid(),
reference=component_name,
validation_state=('validated', 'modified'),
portal_type=self._portal_type)[0].getObject()
# component = getattr(site.portal_components, component_id)
except IndexError:
LOG("ERP5Type.dynamic", INFO, LOG("ERP5Type.dynamic", INFO,
"Could not find %s or it has not been validated or it has not been " "Could not find %s or it has not been validated or it has not been "
"migrated yet?" % component_id) "migrated yet?" % component_id)
......
...@@ -42,15 +42,16 @@ from Products.ERP5Type.TransactionalVariable import TransactionalResource ...@@ -42,15 +42,16 @@ from Products.ERP5Type.TransactionalVariable import TransactionalResource
from zLOG import LOG, ERROR, INFO, WARNING, PANIC from zLOG import LOG, ERROR, INFO, WARNING, PANIC
def _importClass(classpath): def _importClass(classpath, is_zodb_document=False):
try: try:
module_path, class_name = classpath.rsplit('.', 1) module_path, class_name = classpath.rsplit('.', 1)
module = __import__(module_path, {}, {}, (module_path,)) module = __import__(module_path, {}, {}, (module_path,))
klass = getattr(module, class_name) klass = getattr(module, class_name)
# XXX is this required? (here?) if not is_zodb_document:
setDefaultClassProperties(klass) # XXX is this required? (here?)
InitializeClass(klass) setDefaultClassProperties(klass)
InitializeClass(klass)
return klass return klass
except StandardError: except StandardError:
...@@ -87,12 +88,6 @@ core_portal_type_class_dict = { ...@@ -87,12 +88,6 @@ core_portal_type_class_dict = {
'generating': False}, 'generating': False},
'Solver Tool': {'type_class': 'SolverTool', 'Solver Tool': {'type_class': 'SolverTool',
'generating': False}, 'generating': False},
# Needed to load Components
#
# XXX-arnau: only for now as the Catalog is being used (parent_uid
# especially), but it will later be replaced anyway...
'Category Tool': {'type_class': 'CategoryTool',
'generating': False}
} }
def generatePortalTypeClass(site, portal_type_name): def generatePortalTypeClass(site, portal_type_name):
...@@ -188,34 +183,44 @@ def generatePortalTypeClass(site, portal_type_name): ...@@ -188,34 +183,44 @@ def generatePortalTypeClass(site, portal_type_name):
raise AttributeError('Document class is not defined on Portal Type %s' \ raise AttributeError('Document class is not defined on Portal Type %s' \
% portal_type_name) % portal_type_name)
is_zodb_document = False
klass = None klass = None
if '.' in type_class: if '.' in type_class:
type_class_path = type_class type_class_path = type_class
else: else:
type_class_path = None
# Skip any document within ERP5Type Product as it is needed for # Skip any document within ERP5Type Product as it is needed for
# bootstrapping anyway # bootstrapping anyway
type_class_namespace = document_class_registry.get(type_class, '') type_class_namespace = document_class_registry.get(type_class, '')
if not (type_class_namespace.startswith('Products.ERP5Type') or if not (type_class_namespace.startswith('Products.ERP5Type') or
portal_type_name in core_portal_type_class_dict): portal_type_name in core_portal_type_class_dict):
try: import erp5.component.document
klass = getattr(__import__('erp5.component.document.' + type_class, module_info_dict = erp5.component.document._registry_dict.get(type_class,
fromlist=['erp5.component.document'], None)
level=0), if module_info_dict:
type_class) type_class_path = "%s.%s" % (module_info_dict['module_name'], type_class)
is_zodb_document = True
except (ImportError, AttributeError):
pass if type_class_path is None:
if klass is None:
type_class_path = document_class_registry.get(type_class, None) type_class_path = document_class_registry.get(type_class, None)
if type_class_path is None:
raise AttributeError('Document class %s has not been registered:'
' cannot import it as base of Portal Type %s'
% (type_class, portal_type_name))
if klass is None and type_class_path is None: try:
raise AttributeError('Document class %s has not been registered:' klass = _importClass(type_class_path, is_zodb_document)
' cannot import it as base of Portal Type %s' except ImportError:
% (type_class, portal_type_name)) # A Document Component should always have a class matching its reference,
# so this should never happen...
if is_zodb_document:
type_class_path = document_class_registry.get(type_class, None)
if type_class_path is not None:
klass = _importClass(type_class_path)
if klass is None: if klass is None:
klass = _importClass(type_class_path) raise
global property_sheet_generating_portal_type_set global property_sheet_generating_portal_type_set
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment