Make the ZEXP cache hack not to interfere too much.

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@31093 20353a03-c40f-0410-a6d1-a30d3c3de9de

product/ERP5/Document/BusinessTemplate.py

@@ -79,6 +79,19 @@ from difflib import unified_diff
 import posixpath
 import transaction
 
+import gdbm
+import threading
+
+CACHE_DATABASE_PATH = None
+try:
+  if int(os.getenv('ERP5_BT5_CACHE', 0)):
+    from App.config import getConfiguration
+    instancehome = getConfiguration().instancehome
+    CACHE_DATABASE_PATH = os.path.join(instancehome, 'bt5cache.db')
+except TypeError:
+  pass
+cache_database = threading.local()
+
 # those attributes from CatalogMethodTemplateItem are kept for
 # backward compatibility
 catalog_method_list = ('_is_catalog_list_method_archive',
@@ -352,6 +365,12 @@ class BusinessTemplateFolder(BusinessTemplateArchive):
     class_name = item.__class__.__name__
     root_path_len = self.root_path_len
     prefix_len = root_path_len + len(class_name) + len(os.sep)
+    if CACHE_DATABASE_PATH:
+      try:
+        cache_database.db = gdbm.open(CACHE_DATABASE_PATH, 'cf')
+      except gdbm.error:
+        cache_database.db = gdbm.open(CACHE_DATABASE_PATH, 'nf')
+    try:
       for file_path in self.file_list_dict.get(class_name, ()):
         if os.path.isfile(file_path):
           file = open(file_path, 'rb')
@@ -362,6 +381,10 @@ class BusinessTemplateFolder(BusinessTemplateArchive):
             item._importFile(file_name, file)
           finally:
             file.close()
+    finally:
+      if hasattr(cache_database, 'db'):
+        cache_database.db.close()
+        del cache_database.db
 
 class BusinessTemplateTarball(BusinessTemplateArchive):
   """
@@ -650,15 +673,40 @@ class ObjectTemplateItem(BaseTemplateItem):
       obj.wl_clearLocks()
 
   def _compileXML(self, file):
-      name, ext = os.path.splitext(file.name)
-      compiled_file = name + '.zexp'
-      if not os.path.exists(compiled_file) or os.path.getmtime(file.name) > os.path.getmtime(compiled_file):
-          LOG('Business Template', 0, 'Compiling %s to %s...' % (file.name, compiled_file))
+    # This method converts XML to ZEXP. Because the conversion
+    # is quite heavy, a persistent cache database is used to
+    # store ZEXP, so the second run wouldn't have to re-generate
+    # identical data again.
+    #
+    # For now, a pair of the path to an XML file and its modification time
+    # are used as a unique key. In theory, a checksum of the content could
+    # be used instead, and it could be more reliable, as modification time
+    # might not be updated in some insane filesystems correctly. However,
+    # in practice, checksums consume a lot of CPU time, so when the cache
+    # does not hit, the increased overhead is significant. In addition, it
+    # does rarely happen that two XML files in Business Templates contain
+    # the same data, so it may not be expected to have more cache hits
+    # with this approach.
+    #
+    # The disadvantage is that this wouldn't work with the archive format,
+    # because each entry in an archive does not have a mtime in itself.
+    # However, the plan is to have an archive to retain ZEXP directly
+    # instead of XML, so the idea of caching would be completely useless
+    # with the archive format.
+    name = file.name
+    mtime = os.path.getmtime(file.name)
+    key = '%s:%s' % (name, mtime)
+
     try:
+      return StringIO(cache_database.db[key])
+    except:
+      pass
+
+    #LOG('Business Template', 0, 'Compiling %s...' % (name,))
     from Shared.DC.xml import ppml
     from OFS.XMLExportImport import start_zopedata, save_record, save_zopedata
-              import pyexpat
-              outfile=open(compiled_file, 'wb')
+    import xml.parsers.expat
+    outfile=StringIO()
     try:
       data=file.read()
       F=ppml.xmlPickler()
@@ -667,23 +715,27 @@ class ObjectTemplateItem(BaseTemplateItem):
       F.start_handlers['ZopeData'] = start_zopedata
       F.binary=1
       F.file=outfile
-                  p=pyexpat.ParserCreate()
+      p=xml.parsers.expat.ParserCreate('utf-8')
+      p.returns_unicode = False
       p.CharacterDataHandler=F.handle_data
       p.StartElementHandler=F.unknown_starttag
       p.EndElementHandler=F.unknown_endtag
       r=p.Parse(data)
-              finally:
-                  outfile.close()
+
+      try:
+        cache_database.db[key] = outfile.getvalue()
       except:
-              if os.path.exists(compiled_file):
-                  os.remove(compiled_file)
+        pass
+
+      outfile.seek(0)
+      return outfile
+    except:
+      outfile.close()
       raise
-      return open(compiled_file)
 
   def _importFile(self, file_name, file_obj):
     # import xml file
     if not file_name.endswith('.xml'):
-      if not file_name.endswith('.zexp'):
       LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
       return
     obj = self
@@ -692,13 +744,8 @@ class ObjectTemplateItem(BaseTemplateItem):
       obj=obj.aq_parent
       connection=obj._p_jar
     __traceback_info__ = 'Importing %s' % file_name
-    # The pre-compilation hack is disabled, because the design is not
-    # nice. Do not enable it without yo's approval.
-    if 0:
-      if isinstance(file_obj, file):
+    if hasattr(cache_database, 'db') and isinstance(file_obj, file):
       obj = connection.importFile(self._compileXML(file_obj))
-      else:
-        obj = connection.importFile(file_obj, customImporters=customImporters)
     else:
       # FIXME: Why not use the importXML function directly? Are there any BT5s
       # with actual .zexp files on the wild?
@@ -1506,7 +1553,6 @@ class RegisteredSkinSelectionTemplateItem(BaseTemplateItem):
 
   def _importFile(self, file_name, file):
     if not file_name.endswith('.xml'):
-      if not file_name.endswith('.zexp'):
       LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
       return
     # import workflow chain for portal_type
@@ -1863,7 +1909,6 @@ class PortalTypeWorkflowChainTemplateItem(BaseTemplateItem):
 
   def _importFile(self, file_name, file):
     if not file_name.endswith('.xml'):
-      if not file_name.endswith('.zexp'):
       LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
       return
     # import workflow chain for portal_type
@@ -1975,7 +2020,6 @@ class PortalTypeAllowedContentTypeTemplateItem(BaseTemplateItem):
 
   def _importFile(self, file_name, file):
     if not file_name.endswith('.xml'):
-      if not file_name.endswith('.zexp'):
       LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
       return
     path, name = posixpath.split(file_name)
@@ -2748,7 +2792,6 @@ class SitePropertyTemplateItem(BaseTemplateItem):
   def _importFile(self, file_name, file):
     # recreate list of site property from xml file
     if not file_name.endswith('.xml'):
-      if not file_name.endswith('.zexp'):
       LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
       return
     xml = parse(file)
@@ -3219,7 +3262,6 @@ class RoleTemplateItem(BaseTemplateItem):
 
   def _importFile(self, file_name, file):
     if not file_name.endswith('.xml'):
-      if not file_name.endswith('.zexp'):
       LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
       return
     xml = parse(file)