Fix import of text files into ZODB.

When importing back ace.js containing a last line with whitespaces only, the
last line was not imported, and thus the file could never been downloaded
completely as its really size was less than the one on the File ZODB object.

product/ERP5/tests/testBusinessTemplate.py

@@ -6830,6 +6830,84 @@ class TestBusinessTemplate(BusinessTemplateMixin):
     sequence_list.addSequenceString(sequence_string)
     sequence_list.play(self)
 
+  def stepCreateTextFile(self, sequence=None, **kw):
+    skin_tool = self.getSkinsTool()
+    skin_folder = skin_tool._getOb(sequence.get('skin_folder_id'))
+
+    file_id = 'fake_js_file'
+    file_content = """  
+   var 
+ 
+        debug =  [42  
+ ];
+ 
+    """
+
+    from Products.ERP5Type.tests.utils import createZODBFile
+    createZODBFile(skin_folder, file_id, 'text/javascript', file_content)
+
+    sequence.edit(fake_file_id=file_id,
+                  fake_file_content=file_content)
+
+  def stepCheckTextFileContent(self, sequence=None, **kw):
+    skin_folder = self.getSkinsTool()._getOb(sequence.get('skin_folder_id'))
+    file_content = getattr(skin_folder, sequence.get('fake_file_id')).data
+    expected_file_content = sequence.get('fake_file_content')
+
+    self.assertEquals(file_content, expected_file_content)
+    self.assertEquals(len(file_content), len(expected_file_content))
+
+  def test_text_file_import_export(self):
+    """
+    When importing back ace.js containing a last line with whitespaces only,
+    the last line was not imported, and thus the file could never been
+    downloaded completely as its really size was less than the one on the File
+    ZODB object
+    """
+    sequence_list = SequenceList()
+    sequence_string = """
+       CreateSkinFolder
+       CreateTextFile
+       CheckTextFileContent
+
+       CreateNewBusinessTemplate
+       UseExportBusinessTemplate
+       AddSkinFolderToBusinessTemplate
+       CheckModifiedBuildingState
+       CheckNotInstalledInstallationState
+       BuildBusinessTemplate
+       CheckBuiltBuildingState
+       CheckNotInstalledInstallationState
+       CheckObjectPropertiesInBusinessTemplate
+       SaveBusinessTemplate
+       CheckBuiltBuildingState
+       CheckNotInstalledInstallationState
+       RemoveSkinFolder
+       RemoveBusinessTemplate
+       RemoveAllTrashBins
+
+       ImportBusinessTemplate
+       UseImportBusinessTemplate
+       CheckBuiltBuildingState
+       CheckNotInstalledInstallationState
+       InstallWithoutForceBusinessTemplate
+       Tic
+       CheckInstalledInstallationState
+       CheckBuiltBuildingState
+       CheckNoTrashBin
+       CheckSkinsLayers
+       CheckSkinFolderExists
+       CheckTextFileContent
+
+       UninstallBusinessTemplate
+       CheckBuiltBuildingState
+       CheckNotInstalledInstallationState
+       CheckSkinFolderRemoved
+       """
+
+    sequence_list.addSequenceString(sequence_string)
+    sequence_list.play(self)
+
 from Products.ERP5Type.Core.DocumentComponent import DocumentComponent
 
 class TestDocumentTemplateItem(BusinessTemplateMixin):

product/ERP5Type/patches/ppml.py

@@ -309,32 +309,52 @@ ppml.Object = Object
 
 blanck_line_expression = re.compile('^ +$')
 
-# For optmization.
 class NoBlanks:
-
+    """
+    This allows to ignore at least whitespaces between elements and also
+    correctly handle string/unicode
+    """
     previous_stack_end = None
     previous_discarded_data = None
 
     def handle_data(self, data):
+        """
+        Called for each character lines of element data, twice in this
+        example:
+
+        <string>abc
+        bar</string>
+        """
+        # Ignore element data between elements (eg '<e> <f> </f> </e>')...
         if data.strip():
-            # Horrible conditions to try fixing some weird removal of spaces.
-            # It happened that javascript files with a line like
-            # "  ];\n" was replaced by "];\n", so the indent was lost.
-            # Indeed the parser was calling this handle_data function first
-            # for "  ", then for "];". So original code was dropping the "  ".
-            # Disabling the above if was the initial idea, but it give
-            # other troubles, so such conditions were introduced.
-            if data.startswith(']') \
-                  and self.previous_discarded_data \
-                  and blanck_line_expression.match(self.previous_discarded_data) \
-                  and self._stack[-1] == self.previous_stack_end:
-              data = self.previous_discarded_data + data
             if isinstance(data, unicode):
                 data = data.encode('raw_unicode_escape')
             self.append(data)
-        else:
-          self.previous_stack_end = self._stack[-1]
-          self.previous_discarded_data = data
+
+        # Except for strings and unicode data as whitespaces should be
+        # kept. It happened that javascript files with a line like " ];\n" was
+        # replaced by "];\n", so the indent was lost. Indeed the parser was
+        # calling this handle_data function first for " ", then for "];". So
+        # original code was dropping the " ".
+        elif (isinstance(self._stack[-1], list) and
+              self._stack[-1][0] in ('string', 'unicode')):
+            # If the first character data of this element is a whitespace, it
+            # will be concatenated with the next line (if any, but at
+            # this point it is not possible to know anyway)
+            if len(self._stack[-1]) == 2:
+                self.previous_stack_end = self._stack[-1]
+                self.previous_discarded_data = data
+            else:
+                if (self._stack[-1] == self.previous_stack_end and
+                    self.previous_discarded_data):
+                    data = self.previous_discarded_data + data
+                    self.previous_discarded_data = None
+                    self.previous_stack_end = None
+
+                if isinstance(data, unicode):
+                    data = data.encode('raw_unicode_escape')
+
+                self.append(data)
 
 ppml.NoBlanks = NoBlanks
 

product/ERP5Type/tests/utils.py

@@ -208,6 +208,19 @@ def removeZODBPythonScript(container, script_id):
   """
   container.manage_delObjects([script_id])
 
+def createZODBFile(container, file_id, file_content_type, file_content):
+  if file_id in container.objectIds():
+    removeZODBFileScript(container, file_id)
+
+  container.manage_addProduct['OFSP'].manage_addFile(id=file_id)
+  zodb_file = container._getOb(file_id)
+  zodb_file.manage_edit(title='',
+                        content_type=file_content_type,
+                        filedata=file_content)
+
+  container.portal_url.getPortalObject().changeSkin(None)
+  return zodb_file
+
 # class tool
 def installRealClassTool(portal):
   """Replaces portal_classes by a real class tool object.