renderjs_ui: "Touch" cache manifest in Post-Upgrade

With the current architecture of cache manifest, it was required that
developpers change the cache manifests referencing a web page every time
they change a web page. In practice, developers were never doing this,
so we sometimes had issues where client keep using old version of
website even though a new version has been deployed, when after
deployment we did not change the cache manifest.

To automate the scenario of modifying the cache manifest, introduce a
post-upgrade constraint that will check that the manifest is newer than
all of the referenced pages. If that's the case, the constraint can fix
by modifying the manifest content, which also changes the modification
date of the manifest, which might be used in "if-modified-since"
negociations.

According to spec [1], cache manifest is updated if the HTTP responses
is not "304 not modified" and if the manifest content is not
byte-for-byte identical to the previously cached version.

[1] https://www.w3.org/TR/2008/WD-html5-20080122/#updating1

/reviewed-on nexedi/erp5!1009

bt5/erp5_web_renderjs_ui/PortalTypePropertySheetTemplateItem/property_sheet_list.xml

@@ -7,6 +7,9 @@
   <item>Reference</item>
   <item>SortIndex</item>
  </portal_type>
+ <portal_type id="Web Site">
+  <item>WebSiteRenderJSUpgradeConstraint</item>
+ </portal_type>
  <portal_type id="Web Style">
   <item>Reference</item>
   <item>SortIndex</item>

bt5/erp5_web_renderjs_ui/PropertySheetTemplateItem/portal_property_sheets/WebSiteRenderJSUpgradeConstraint.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Property Sheet" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_count</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>_mt_index</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>_tree</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>WebSiteRenderJSUpgradeConstraint</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Property Sheet</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="Length" module="BTrees.Length"/>
+    </pickle>
+    <pickle> <int>0</int> </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="OOBTree" module="BTrees.OOBTree"/>
+    </pickle>
+    <pickle>
+      <none/>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="OOBTree" module="BTrees.OOBTree"/>
+    </pickle>
+    <pickle>
+      <none/>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_web_renderjs_ui/PropertySheetTemplateItem/portal_property_sheets/WebSiteRenderJSUpgradeConstraint/cache_modification_date_constraint.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Script Constraint" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_identity_criterion</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>_range_criterion</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>categories</string> </key>
+            <value>
+              <tuple>
+                <string>constraint_type/post_upgrade</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>cache_modification_date_constraint</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Script Constraint</string> </value>
+        </item>
+        <item>
+            <key> <string>script_id</string> </key>
+            <value> <string>WebSite_checkCacheModificationDateConsistency</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_web_renderjs_ui/SkinTemplateItem/portal_skins/erp5_web_renderjs/WebSite_checkCacheModificationDateConsistency.py

+from DateTime import DateTime
+appcache_reference = context.getLayoutProperty("configuration_manifest_url")
+
+getDocumentValue = context.getDocumentValue
+
+error_list = []
+if appcache_reference:
+  url_list = context.Base_getListFileFromAppcache()
+  # Check that the manifest is newer than all cached resources.
+  appcache_manifest = getDocumentValue(appcache_reference).getObject()
+  appcache_manifest_modification_date = appcache_manifest.getModificationDate()
+  for url in url_list:
+    if url:
+      referenced_document = getDocumentValue(url)
+      if referenced_document is not None and (
+          referenced_document.getModificationDate() >
+          appcache_manifest_modification_date):
+        error_list.append(
+            "Document {} is newer than cache manifest".format(url))
+
+  if error_list and fixit:
+    appcache_manifest.edit(
+        text_content='''# Last modified by {} on {}
+{}
+'''.format(script.getId(), DateTime(), appcache_manifest.getTextContent()))
+
+return error_list

bt5/erp5_web_renderjs_ui/SkinTemplateItem/portal_skins/erp5_web_renderjs/WebSite_checkCacheModificationDateConsistency.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string>fixit=False</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>WebSite_checkCacheModificationDateConsistency</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_web_renderjs_ui/bt/template_portal_type_property_sheet_list

@@ -2,5 +2,6 @@ Web Manifest | Reference
 Web Manifest | SortIndex
 Web Script | Reference
 Web Script | SortIndex
+Web Site | WebSiteRenderJSUpgradeConstraint
 Web Style | Reference
 Web Style | SortIndex
\ No newline at end of file

bt5/erp5_web_renderjs_ui/bt/template_property_sheet_id_list

+WebSiteRenderJSUpgradeConstraint
\ No newline at end of file

bt5/erp5_web_renderjs_ui_test/TestTemplateItem/portal_components/test.erp5.testRJSUpgrader.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import textwrap
+import time
+
+from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
+
+
+class TestRenderJSUpgrade(ERP5TypeTestCase):
+  """Test Upgrader scripts for renderjs UI.
+  """
+  def afterSetUp(self):
+    self.login()
+    self.web_site = self.portal.web_site_module.newContent(
+        portal_type='Web Site',
+        skin_selection_name='RJS',
+    )
+    self.web_site.publish()
+    self.manifest = self.portal.web_page_module.newContent(
+        portal_type='Web Manifest',
+        text_content="# empty",
+        reference='{}.appcache'.format(self.id()),
+    )
+    self.manifest.publish()
+    self.html_page = self.portal.web_page_module.newContent(
+        portal_type='Web Page',
+        text_content="<b>content</b>",
+        reference='{}.html'.format(self.id()))
+    self.html_page.publish()
+    self.javascript = self.portal.web_page_module.newContent(
+        portal_type='Web Script',
+        text_content="alert('hello !')",
+        reference='{}.js'.format(self.id()))
+    self.javascript.publish()
+
+  def test_upgrade_empty_site(self):
+    self.assertEqual([], self.web_site.checkConsistency())
+    self.assertEqual([], self.web_site.fixConsistency())
+
+  def test_upgrade_fix_pages_modification_date(self):
+    # ERP5JS Web Sites define the list of pages to be cached using an
+    # application cache manifest. We have a post-upgrade constraint which
+    # checks that the manifest is more recent that the referenced files.
+    manifest_content = textwrap.dedent(
+        '''\
+        CACHE MANIFEST
+        # v1 - 2011-08-13
+        # This is a comment.
+        http://www.example.com/index.html
+
+        {}
+        {}
+        NETWORK:
+        *
+        ''').format(
+            self.html_page.getReference(), self.javascript.getReference())
+    self.manifest.edit(text_content=manifest_content)
+    self.web_site.setProperty(
+        'configuration_manifest_url', self.manifest.getReference())
+    self.tic()
+    time.sleep(1)
+    self.javascript.edit(text_content="alert('hello again !')")
+    self.tic()
+
+    self.assertLess(
+        self.manifest.getModificationDate(),
+        self.javascript.getModificationDate())
+    self.assertEqual(
+        [
+            'Document {} is newer than cache manifest'.format(
+                self.javascript.getReference()),
+        ], [str(m.getMessage()) for m in self.web_site.checkConsistency()])
+    self.web_site.fixConsistency()
+    self.tic()
+
+    self.assertEqual(
+        [], [str(m.getMessage()) for m in self.web_site.checkConsistency()])
+    self.assertGreater(
+        self.manifest.getModificationDate(),
+        self.javascript.getModificationDate())
+    self.assertIn(
+        'Last modified by WebSite_checkCacheModificationDateConsistency on',
+        self.manifest.getTextContent())
+    self.assertIn(manifest_content, self.manifest.getTextContent())

bt5/erp5_web_renderjs_ui_test/TestTemplateItem/portal_components/test.erp5.testRJSUpgrader.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Test Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>testRJSUpgrader</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>test.erp5.testRJSUpgrader</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Test Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_log</string> </key>
+            <value>
+              <list>
+                <dictionary>
+                  <item>
+                      <key> <string>action</string> </key>
+                      <value> <string>validate</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>validation_state</string> </key>
+                      <value> <string>validated</string> </value>
+                  </item>
+                </dictionary>
+              </list>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_web_renderjs_ui_test/bt/template_test_id_list

@@ -23,4 +23,5 @@ test.erp5.testFunctionalRJSEditorGadget
 test.erp5.testFunctionalRJSRecoverPassword
 test.erp5.testFunctionalRJSInterfaceValidator
 test.erp5.testFunctionalRJSDms
-test.erp5.testRJSPortalType
+test.erp5.testRJSPortalType
\ No newline at end of file
+test.erp5.testRJSUpgrader
\ No newline at end of file