base: Configure substitution's "ignore missing" on notification messages

TextDocument substitution API already supported an `ignore_missing` argument,
so that we can programmatically control wether missing entries should be
rendered as ${variable} or should raise an error. This makes sense for some
"important" notifications, where not sending a message would be less problematic
than sending a message where some variables have not been substituted.

This extends this concept by allowing to configure as a property on the
notification message wether missing entries should be ignored. This default to
"ignore" to maintain compatbility.

bt5/erp5_base/SkinTemplateItem/portal_skins/erp5_base/NotificationMessage_viewEditor.xml

@@ -59,8 +59,8 @@
             <key> <string>group_list</string> </key>
             <value>
               <list>
-                <string>left (Page Properties)</string>
-                <string>right (Publication)</string>
+                <string>left</string>
+                <string>right</string>
                 <string>center</string>
                 <string>bottom</string>
               </list>
@@ -85,18 +85,20 @@
                     </value>
                 </item>
                 <item>
-                    <key> <string>left (Page Properties)</string> </key>
+                    <key> <string>left</string> </key>
                     <value>
                       <list>
                         <string>my_content_type</string>
-                        <string>my_text_content_substitution_mapping_method_id</string>
                       </list>
                     </value>
                 </item>
                 <item>
-                    <key> <string>right (Publication)</string> </key>
+                    <key> <string>right</string> </key>
                     <value>
-                      <list/>
+                      <list>
+                        <string>my_text_content_substitution_mapping_method_id</string>
+                        <string>my_text_content_substitution_mapping_ignore_missing</string>
+                      </list>
                     </value>
                 </item>
               </dictionary>

bt5/erp5_base/SkinTemplateItem/portal_skins/erp5_base/NotificationMessage_viewEditor/my_text_content_substitution_mapping_ignore_missing.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>delegated_list</string> </key>
+            <value>
+              <list>
+                <string>title</string>
+              </list>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>my_text_content_substitution_mapping_ignore_missing</string> </value>
+        </item>
+        <item>
+            <key> <string>message_values</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>external_validator_failed</string> </key>
+                    <value> <string>The input failed the external validator.</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>overrides</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>tales</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>values</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string>my_checkbox</string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string>Base_viewFieldLibrary</string> </value>
+                </item>
+                <item>
+                    <key> <string>title</string> </key>
+                    <value> <string>Ignore Missing Substitution Variables</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testNotificationMessageModule.py

@@ -152,8 +152,9 @@ class TestNotificationMessageModule(ERP5TypeTestCase):
     self.assertEqual('substitution text: b', text.rstrip())
 
   def test_safe_substitution_content(self):
-    """Tests that 'safe' substitution is performed, unless safe_substitute is
-    explicitly passed to False.
+    """Tests that 'safe' substitution is performed, unless the notification
+    message is configured text_content_substitution_mapping_ignore_missing set
+    to false, or safe_substitute is passed as False.
     """
     module = self.portal.notification_message_module
     createZODBPythonScript(self.portal,
@@ -175,6 +176,23 @@ class TestNotificationMessageModule(ERP5TypeTestCase):
     self.assertRaises(KeyError, doc.convert, 'html', safe_substitute=False)
     self.assertRaises(KeyError, doc.asSubjectText, safe_substitute=False)
 
+    doc.setTextContentSubstitutionMappingIgnoreMissing(False)
+    self.assertRaises(KeyError, doc.convert, 'txt')
+    self.assertRaises(KeyError, doc.convert, 'html')
+    self.assertRaises(KeyError, doc.asSubjectText)
+    mime, text = doc.convert('txt', safe_substitute=True)
+    self.assertEqual('substitution text: ${b}', text.rstrip())
+    self.assertEqual('${b}', doc.asSubjectText(safe_substitute=True))
+
+    self.tic()
+    # even when notification messages are configured to not ignore missing
+    # entries from mapping, they are indexed in full text.
+    self.assertEqual(
+        [brain.getObject() for brain in self.portal.portal_catalog(
+            SearchableText='substitution text',
+            uid=doc.getUid())],
+        [doc])
+
   def test_substitution_lazy_dict(self):
     """Substitution script just needs to return an object implementing
     __getitem__ protocol.

product/ERP5/bootstrap/erp5_core/DocumentTemplateItem/portal_components/document.erp5.TextDocument.py

@@ -75,7 +75,7 @@ class TextDocument(CachedConvertableMixin, BaseConvertableFileMixin, TextContent
                     , PropertySheet.Reference
                     )
 
-  def _substituteTextContent(self, text, safe_substitute=True, **kw):
+  def _substituteTextContent(self, text, safe_substitute=_MARKER, **kw):
     # If a method for string substitutions of the text content, perform it.
     # Decode everything into unicode before the substitutions, in order to
     # avoid encoding errors.
@@ -104,6 +104,8 @@ class TextDocument(CachedConvertableMixin, BaseConvertableFileMixin, TextContent
           return v
       unicode_mapping = UnicodeMapping()
 
+      if safe_substitute is _MARKER:
+        safe_substitute = self.isTextContentSubstitutionMappingIgnoreMissing()
       if safe_substitute:
         text = Template(text).safe_substitute(unicode_mapping)
       else:
@@ -116,7 +118,7 @@ class TextDocument(CachedConvertableMixin, BaseConvertableFileMixin, TextContent
     return text
 
   security.declareProtected(Permissions.AccessContentsInformation, 'asSubjectText')
-  def asSubjectText(self, substitution_method_parameter_dict=None, safe_substitute=True, **kw):
+  def asSubjectText(self, substitution_method_parameter_dict=None, safe_substitute=_MARKER, **kw):
     """
       Converts the subject of the document to a textual representation.
     """
@@ -127,7 +129,7 @@ class TextDocument(CachedConvertableMixin, BaseConvertableFileMixin, TextContent
                                        **substitution_method_parameter_dict)
 
   def _convert(self, format, substitution_method_parameter_dict=None, # pylint: disable=redefined-builtin
-              safe_substitute=True, charset=None, text_content=None, substitute=True, **kw):
+              safe_substitute=_MARKER, charset=None, text_content=None, substitute=True, **kw):
     """
       Convert text using portal_transforms or oood
     """

product/ERP5/bootstrap/erp5_core/InterfaceTemplateItem/portal_components/interface.erp5.ITextDocument.py

@@ -35,8 +35,16 @@ class ITextDocument(Interface):
 
   Document which implement ITextDocument can handle text content in multiple
   format (html, structured-text, text).
+
   Substitution mapping can occurs on result if
   text_content_substitution_mapping_method_id is defined.
+
+  Substitutions are made using python string templates described by PEP-0292
+  ( https://www.python.org/dev/peps/pep-0292 ). The substitution is done using
+  "safe_subsitute" method, ie. in the case of missing variables, the substitution
+  marker will be kept as-is. To make missing variables an error, one can either
+  define the text_content_substitution_mapping_ignore_missing property to False
+  on the text document, or pass safe_substitute=False to methods.
   """
 
   def getTextContent(default=None):

product/ERP5/bootstrap/erp5_property_sheets/PropertySheetTemplateItem/portal_property_sheets/TextDocument/text_content_substitution_mapping_ignore_missing_property.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Standard Property" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_local_properties</string> </key>
+            <value>
+              <tuple>
+                <dictionary>
+                  <item>
+                      <key> <string>id</string> </key>
+                      <value> <string>mode</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>type</string> </key>
+                      <value> <string>string</string> </value>
+                  </item>
+                </dictionary>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>categories</string> </key>
+            <value>
+              <tuple>
+                <string>elementary_type/boolean</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value> <string>Ignore missing entries from substitution mapping.\n
+\n
+When ignoring, missing entries will be kept as is, otherwise substitution will be an error.</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>text_content_substitution_mapping_ignore_missing_property</string> </value>
+        </item>
+        <item>
+            <key> <string>mode</string> </key>
+            <value> <string>w</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Standard Property</string> </value>
+        </item>
+        <item>
+            <key> <string>property_default</string> </key>
+            <value> <string>python: True</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>