Commit dcec7e2f authored by Yoshinori Okuji's avatar Yoshinori Okuji

Make the ZEXP cache hack not to interfere too much.

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@31093 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 4e8987de
...@@ -79,6 +79,19 @@ from difflib import unified_diff ...@@ -79,6 +79,19 @@ from difflib import unified_diff
import posixpath import posixpath
import transaction import transaction
import gdbm
import threading
CACHE_DATABASE_PATH = None
try:
if int(os.getenv('ERP5_BT5_CACHE', 0)):
from App.config import getConfiguration
instancehome = getConfiguration().instancehome
CACHE_DATABASE_PATH = os.path.join(instancehome, 'bt5cache.db')
except TypeError:
pass
cache_database = threading.local()
# those attributes from CatalogMethodTemplateItem are kept for # those attributes from CatalogMethodTemplateItem are kept for
# backward compatibility # backward compatibility
catalog_method_list = ('_is_catalog_list_method_archive', catalog_method_list = ('_is_catalog_list_method_archive',
...@@ -352,16 +365,26 @@ class BusinessTemplateFolder(BusinessTemplateArchive): ...@@ -352,16 +365,26 @@ class BusinessTemplateFolder(BusinessTemplateArchive):
class_name = item.__class__.__name__ class_name = item.__class__.__name__
root_path_len = self.root_path_len root_path_len = self.root_path_len
prefix_len = root_path_len + len(class_name) + len(os.sep) prefix_len = root_path_len + len(class_name) + len(os.sep)
for file_path in self.file_list_dict.get(class_name, ()): if CACHE_DATABASE_PATH:
if os.path.isfile(file_path): try:
file = open(file_path, 'rb') cache_database.db = gdbm.open(CACHE_DATABASE_PATH, 'cf')
try: except gdbm.error:
file_name = file_path[prefix_len:] cache_database.db = gdbm.open(CACHE_DATABASE_PATH, 'nf')
if '%' in file_name: try:
file_name = unquote(file_name) for file_path in self.file_list_dict.get(class_name, ()):
item._importFile(file_name, file) if os.path.isfile(file_path):
finally: file = open(file_path, 'rb')
file.close() try:
file_name = file_path[prefix_len:]
if '%' in file_name:
file_name = unquote(file_name)
item._importFile(file_name, file)
finally:
file.close()
finally:
if hasattr(cache_database, 'db'):
cache_database.db.close()
del cache_database.db
class BusinessTemplateTarball(BusinessTemplateArchive): class BusinessTemplateTarball(BusinessTemplateArchive):
""" """
...@@ -650,41 +673,70 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -650,41 +673,70 @@ class ObjectTemplateItem(BaseTemplateItem):
obj.wl_clearLocks() obj.wl_clearLocks()
def _compileXML(self, file): def _compileXML(self, file):
name, ext = os.path.splitext(file.name) # This method converts XML to ZEXP. Because the conversion
compiled_file = name + '.zexp' # is quite heavy, a persistent cache database is used to
if not os.path.exists(compiled_file) or os.path.getmtime(file.name) > os.path.getmtime(compiled_file): # store ZEXP, so the second run wouldn't have to re-generate
LOG('Business Template', 0, 'Compiling %s to %s...' % (file.name, compiled_file)) # identical data again.
try: #
from Shared.DC.xml import ppml # For now, a pair of the path to an XML file and its modification time
from OFS.XMLExportImport import start_zopedata, save_record, save_zopedata # are used as a unique key. In theory, a checksum of the content could
import pyexpat # be used instead, and it could be more reliable, as modification time
outfile=open(compiled_file, 'wb') # might not be updated in some insane filesystems correctly. However,
try: # in practice, checksums consume a lot of CPU time, so when the cache
data=file.read() # does not hit, the increased overhead is significant. In addition, it
F=ppml.xmlPickler() # does rarely happen that two XML files in Business Templates contain
F.end_handlers['record'] = save_record # the same data, so it may not be expected to have more cache hits
F.end_handlers['ZopeData'] = save_zopedata # with this approach.
F.start_handlers['ZopeData'] = start_zopedata #
F.binary=1 # The disadvantage is that this wouldn't work with the archive format,
F.file=outfile # because each entry in an archive does not have a mtime in itself.
p=pyexpat.ParserCreate() # However, the plan is to have an archive to retain ZEXP directly
p.CharacterDataHandler=F.handle_data # instead of XML, so the idea of caching would be completely useless
p.StartElementHandler=F.unknown_starttag # with the archive format.
p.EndElementHandler=F.unknown_endtag name = file.name
r=p.Parse(data) mtime = os.path.getmtime(file.name)
finally: key = '%s:%s' % (name, mtime)
outfile.close()
except: try:
if os.path.exists(compiled_file): return StringIO(cache_database.db[key])
os.remove(compiled_file) except:
raise pass
return open(compiled_file)
#LOG('Business Template', 0, 'Compiling %s...' % (name,))
from Shared.DC.xml import ppml
from OFS.XMLExportImport import start_zopedata, save_record, save_zopedata
import xml.parsers.expat
outfile=StringIO()
try:
data=file.read()
F=ppml.xmlPickler()
F.end_handlers['record'] = save_record
F.end_handlers['ZopeData'] = save_zopedata
F.start_handlers['ZopeData'] = start_zopedata
F.binary=1
F.file=outfile
p=xml.parsers.expat.ParserCreate('utf-8')
p.returns_unicode = False
p.CharacterDataHandler=F.handle_data
p.StartElementHandler=F.unknown_starttag
p.EndElementHandler=F.unknown_endtag
r=p.Parse(data)
try:
cache_database.db[key] = outfile.getvalue()
except:
pass
outfile.seek(0)
return outfile
except:
outfile.close()
raise
def _importFile(self, file_name, file_obj): def _importFile(self, file_name, file_obj):
# import xml file # import xml file
if not file_name.endswith('.xml'): if not file_name.endswith('.xml'):
if not file_name.endswith('.zexp'): LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
return return
obj = self obj = self
connection = None connection = None
...@@ -692,13 +744,8 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -692,13 +744,8 @@ class ObjectTemplateItem(BaseTemplateItem):
obj=obj.aq_parent obj=obj.aq_parent
connection=obj._p_jar connection=obj._p_jar
__traceback_info__ = 'Importing %s' % file_name __traceback_info__ = 'Importing %s' % file_name
# The pre-compilation hack is disabled, because the design is not if hasattr(cache_database, 'db') and isinstance(file_obj, file):
# nice. Do not enable it without yo's approval. obj = connection.importFile(self._compileXML(file_obj))
if 0:
if isinstance(file_obj, file):
obj = connection.importFile(self._compileXML(file_obj))
else:
obj = connection.importFile(file_obj, customImporters=customImporters)
else: else:
# FIXME: Why not use the importXML function directly? Are there any BT5s # FIXME: Why not use the importXML function directly? Are there any BT5s
# with actual .zexp files on the wild? # with actual .zexp files on the wild?
...@@ -1506,8 +1553,7 @@ class RegisteredSkinSelectionTemplateItem(BaseTemplateItem): ...@@ -1506,8 +1553,7 @@ class RegisteredSkinSelectionTemplateItem(BaseTemplateItem):
def _importFile(self, file_name, file): def _importFile(self, file_name, file):
if not file_name.endswith('.xml'): if not file_name.endswith('.xml'):
if not file_name.endswith('.zexp'): LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
return return
# import workflow chain for portal_type # import workflow chain for portal_type
skin_selection_dict = {} skin_selection_dict = {}
...@@ -1863,8 +1909,7 @@ class PortalTypeWorkflowChainTemplateItem(BaseTemplateItem): ...@@ -1863,8 +1909,7 @@ class PortalTypeWorkflowChainTemplateItem(BaseTemplateItem):
def _importFile(self, file_name, file): def _importFile(self, file_name, file):
if not file_name.endswith('.xml'): if not file_name.endswith('.xml'):
if not file_name.endswith('.zexp'): LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
return return
# import workflow chain for portal_type # import workflow chain for portal_type
dict = {} dict = {}
...@@ -1975,8 +2020,7 @@ class PortalTypeAllowedContentTypeTemplateItem(BaseTemplateItem): ...@@ -1975,8 +2020,7 @@ class PortalTypeAllowedContentTypeTemplateItem(BaseTemplateItem):
def _importFile(self, file_name, file): def _importFile(self, file_name, file):
if not file_name.endswith('.xml'): if not file_name.endswith('.xml'):
if not file_name.endswith('.zexp'): LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
return return
path, name = posixpath.split(file_name) path, name = posixpath.split(file_name)
xml = parse(file) xml = parse(file)
...@@ -2748,8 +2792,7 @@ class SitePropertyTemplateItem(BaseTemplateItem): ...@@ -2748,8 +2792,7 @@ class SitePropertyTemplateItem(BaseTemplateItem):
def _importFile(self, file_name, file): def _importFile(self, file_name, file):
# recreate list of site property from xml file # recreate list of site property from xml file
if not file_name.endswith('.xml'): if not file_name.endswith('.xml'):
if not file_name.endswith('.zexp'): LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
return return
xml = parse(file) xml = parse(file)
property_list = xml.getElementsByTagName('property') property_list = xml.getElementsByTagName('property')
...@@ -3219,8 +3262,7 @@ class RoleTemplateItem(BaseTemplateItem): ...@@ -3219,8 +3262,7 @@ class RoleTemplateItem(BaseTemplateItem):
def _importFile(self, file_name, file): def _importFile(self, file_name, file):
if not file_name.endswith('.xml'): if not file_name.endswith('.xml'):
if not file_name.endswith('.zexp'): LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
return return
xml = parse(file) xml = parse(file)
role_list = xml.getElementsByTagName('role') role_list = xml.getElementsByTagName('role')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment