Merge remote-tracking branch 'origin/master' into zope4py2

bt5/erp5_accounting/SkinTemplateItem/portal_skins/erp5_accounting/AccountModule_viewDialogFieldLibrary/your_mirror_section_category_list.xml

@@ -263,7 +263,7 @@
       <dictionary>
         <item>
             <key> <string>_text</string> </key>
-            <value> <string>python:here.portal_categories.role.getCategoryChildTranslatedLogicalPathItemList(base=True)</string> </value>
+            <value> <string>python:here.portal_categories.role.getCategoryChildTranslatedLogicalPathItemList(base=True, display_none_category=False)</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/erp5_accounting/SkinTemplateItem/portal_skins/erp5_accounting/Account_getGapItemList.py

@@ -7,32 +7,35 @@ categories from all available GAP.
 
 portal = context.getPortalObject()
 
-display_cache = {}
 def display(x):
-  if x not in display_cache:
-    gap_id = x.getReference()
-    if gap_id:
-      display_cache[x] = '%s - %s' % (
-        gap_id,
-        x.getTranslatedShortTitle() or x.getTranslatedTitle())
-    else:
-      display_cache[x] = x.getIndentedTitle()
+  gap_id = x.getReference()
+  if gap_id:
+    return '%s - %s' % (
+      gap_id,
+      x.getTranslatedShortTitle() or x.getTranslatedTitle())
+  return x.getIndentedTitle()
 
-  return display_cache[x]
 
 def getGapItemList(only_preferred_gap, gap_root=None):
-  if only_preferred_gap:
-    if gap_root:
-      return portal.portal_categories.resolveCategory(gap_root).getCategoryChildItemList(
-        base=False, is_self_excluded=True, display_method=display,
+  if only_preferred_gap and gap_root:
+    return portal.portal_categories.resolveCategory(gap_root).getCategoryChildItemList(
+        base=False,
+        is_self_excluded=True,
+        display_method=display,
+        display_none_category=False,
         local_sort_id=('int_index', 'reference', 'id'))
 
   result = []
-  for country in portal.portal_categories.gap.contentValues():
-    for gap_root in country.contentValues():
-      result.extend(gap_root.getCategoryChildItemList(
-        base=False, is_self_excluded=True, display_method=display,
-        local_sort_id=('int_index', 'reference', 'id')))
+  for gap_root_title, gap_root in context.AccountModule_getAvailableGapList():
+    if gap_root:
+      result.append((gap_root_title, None))
+      result.extend(
+          portal.portal_categories.resolveCategory(gap_root).getCategoryChildItemList(
+              base=False,
+              is_self_excluded=True,
+              display_method=display,
+              display_none_category=False,
+              local_sort_id=('int_index', 'reference', 'id')))
   return result
 
 from Products.ERP5Type.Cache import CachingMethod

bt5/erp5_accounting/SkinTemplateItem/portal_skins/erp5_accounting/Account_hashGapItemList.py

@@ -12,6 +12,8 @@ def getSubFieldDict():
     for item in item_list:
       # Get value of the item
       item_value = item[int(not is_right_display)]
+      if item_value is None:
+        continue
 
       # Hash key from item_value
       item_split = item_value.split('/')

bt5/erp5_accounting/SkinTemplateItem/portal_skins/erp5_accounting/AccountingTransactionModule_viewSearchDialog/your_node.xml

@@ -278,7 +278,7 @@
       <dictionary>
         <item>
             <key> <string>_text</string> </key>
-            <value> <string>python: context.ERP5Site_getAccountItemList(section_category=request.get(\'your_section_category\', preferences.getPreferredAccountingTransactionSectionCategory()), section_category_strict=request.get(\'your_section_category_strict\', preferences.getPreferredAccountingSectionCategoryStrict()), from_date=request.get(\'your_from_date\', preferences.getPreferredAccountingTransactionFromDate()))</string> </value>
+            <value> <string>python: [item for item in context.ERP5Site_getAccountItemList(section_category=request.get(\'your_section_category\', preferences.getPreferredAccountingTransactionSectionCategory()), section_category_strict=request.get(\'your_section_category_strict\', preferences.getPreferredAccountingSectionCategoryStrict()), from_date=request.get(\'your_from_date\', preferences.getPreferredAccountingTransactionFromDate())) if item[0]]</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/erp5_accounting/SkinTemplateItem/portal_skins/erp5_accounting/AccountingTransactionModule_viewSearchDialog/your_simulation_state.xml

@@ -278,7 +278,7 @@
       <dictionary>
         <item>
             <key> <string>_text</string> </key>
-            <value> <string>python: context.ERP5Site_getWorkflowStateItemList(portal_type=\'Accounting Transaction\', state_var=\'simulation_state\')</string> </value>
+            <value> <string>python: context.ERP5Site_getWorkflowStateItemList(portal_type=\'Accounting Transaction\', state_var=\'simulation_state\', display_none_category=False)</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/erp5_base/DocumentTemplateItem/portal_components/document.erp5.Login.py

@@ -34,6 +34,8 @@ from Products.ERP5Type.XMLObject import XMLObject
 
 
 class Login(EncryptedPasswordMixin, XMLObject, LoginAccountProviderMixin):
+  """Login
+  """
   meta_type = 'ERP5 Login'
   portal_type = 'Login'
   add_permission = Permissions.AddPortalContent

bt5/erp5_base/MixinTemplateItem/portal_components/mixin.erp5.EncryptedPasswordMixin.py

@@ -40,7 +40,8 @@ from Products.CMFCore.utils import _checkPermission
 from Products.CMFCore.exceptions import AccessControl_Unauthorized
 
 class EncryptedPasswordMixin(object):
-
+  """Encrypted Password Mixin
+  """
   # Declarative security
   security = ClassSecurityInfo()
   security.declareObjectProtected(Permissions.AccessContentsInformation)

bt5/erp5_budget/SkinTemplateItem/portal_skins/erp5_budget/BudgetModule_viewBudgetConsumptionReportDialog/your_specialise.xml

@@ -114,7 +114,7 @@
       <dictionary>
         <item>
             <key> <string>_text</string> </key>
-            <value> <string>python: [(\'\', \'\')] + [(x.getTitle(), x.getRelativeUrl()) for x in context.portal_catalog(portal_type=\'Budget Model\')]</string> </value>
+            <value> <string>python: [(x.getTranslatedTitle(), x.getRelativeUrl()) for x in context.portal_catalog(portal_type=\'Budget Model\')]</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/erp5_budget/SkinTemplateItem/portal_skins/erp5_budget/BudgetModule_viewBudgetConsumptionReportDialog/your_validation_state.xml

@@ -280,7 +280,7 @@
       <dictionary>
         <item>
             <key> <string>_text</string> </key>
-            <value> <string>python: context.ERP5Site_getWorkflowStateItemList(portal_type=\'Budget\', state_var=\'validation_state\')</string> </value>
+            <value> <string>python: context.ERP5Site_getWorkflowStateItemList(portal_type=\'Budget\', state_var=\'validation_state\', display_none_category=False)</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/erp5_core_proxy_field_legacy/SkinTemplateItem/portal_skins/erp5_core_proxy_field_legacy/Base_viewDialogFieldLibrary/your_category_list.xml

@@ -263,7 +263,7 @@
       <dictionary>
         <item>
             <key> <string>_text</string> </key>
-            <value> <string>python: here.portal_categories[field.getId().replace(\'your_\', \'\', 1).replace(\'_list\', \'\')].getCategoryChildCompactLogicalPathItemList(local_sort_id="int_index", checked_permission=\'View\')</string> </value>
+            <value> <string>python: here.portal_categories[field.getId().replace(\'your_\', \'\', 1).replace(\'_list\', \'\')].getCategoryChildCompactLogicalPathItemList(local_sort_id="int_index", checked_permission=\'View\', display_none_category=False)</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testFields.py

@@ -50,6 +50,7 @@ from Products.ERP5Form.Form import field_value_cache
 from Products.ERP5Form.Form import getFieldValue
 from Products.ERP5Form import ProxyField
 from DateTime import DateTime
+import lxml.html
 
 from Products.Formulator.Widget import NSMAP
 ODG_XML_WRAPPING_XPATH = 'draw:text-box/text:p/text:span'
@@ -534,6 +535,8 @@ class TestListField(ERP5TypeTestCase):
 
   def afterSetUp(self):
     self.field = ListField('test_field')
+    self.field.values['items'] = [('My first Line', '1'), ('My Second Line', '2')]
+    self.field.values['default'] = '2'
     self.widget = self.field.widget
     self.createCategories()
     self.tic()
@@ -553,9 +556,6 @@ class TestListField(ERP5TypeTestCase):
                                       int_index=2)
 
   def test_render_odt(self):
-    items = [('My first Line', '1'), ('My Second Line', '2')]
-    self.field.values['items'] = items
-    self.field.values['default'] = '2'
     element = self.field.render_odt(as_string=False)
     self.assertEqual('{%(text)s}p' % NSMAP, element.tag)
     self.assertEqual('My Second Line', element.text)
@@ -565,6 +565,44 @@ class TestListField(ERP5TypeTestCase):
     element = self.field.render_odt(as_string=False)
     self.assertEqual('??? (3)', element.text)
 
+  def test_render(self):
+    select, input_element, = lxml.html.fragments_fromstring(self.field.render())  # pylint:disable=unbalanced-tuple-unpacking
+    # listfields render an input to confirm that the field was posted
+    # in the form's action script
+    self.assertEqual(input_element.name, 'default_field_test_field:int')
+    self.assertEqual(input_element.type, 'hidden')
+
+    self.assertEqual(select.tag, 'select')
+    first, second = select
+    self.assertEqual(first.tag, 'option')
+    self.assertEqual(first.text_content(), 'My first Line')
+    self.assertEqual(first.attrib['value'], '1')
+
+    self.assertEqual(second.tag, 'option')
+    self.assertEqual(second.text_content(), 'My Second Line')
+    self.assertEqual(second.attrib['value'], '2',)
+    self.assertTrue(second.attrib['selected'])
+
+  def test_render_escape_html(self):
+    self.field.values['default'] = ''
+    self.field.values['items'] = [
+        ('<script>alert("text content")</script>', '<script>alert("value")</script>'),]
+    (script, ), _, = lxml.html.fragments_fromstring(self.field.render())  # pylint:disable=unbalanced-tuple-unpacking
+    self.assertEqual(script.attrib['value'], '<script>alert("value")</script>')
+    self.assertEqual(script.text_content(), '<script>alert("text content")</script>')
+    # selected
+    self.field.values['default'] = self.field.values['items'][0][1]
+    (script, ), _, = lxml.html.fragments_fromstring(self.field.render())  # pylint:disable=unbalanced-tuple-unpacking
+    self.assertEqual(script.attrib['value'], '<script>alert("value")</script>')
+    self.assertEqual(script.text_content(), '<script>alert("text content")</script>')
+
+  def test_render_disabled(self):
+    self.field.values['default'] = ''
+    # None items are rendered as disabled
+    self.field.values['items'] = [('Disabled', None)]
+    (disabled, ), _, = lxml.html.fragments_fromstring(self.field.render())  # pylint:disable=unbalanced-tuple-unpacking
+    self.assertTrue(disabled.attrib['disabled'])
+    self.assertFalse(disabled.attrib.get('value'))
 
   def test_listField_value_order(self):
     '''This test check the list field value order
@@ -606,6 +644,46 @@ class TestMultiListField(ERP5TypeTestCase):
     self.field.values['items'] = [('A', 'a',), ('B', 'b')]
     self.field.values['default'] = ['a', 'b']
 
+  def test_render(self):
+    select, input_element, = lxml.html.fragments_fromstring(self.field.render())  # pylint:disable=unbalanced-tuple-unpacking
+    # listfields render an input to confirm that the field was posted
+    # in the form's action script
+    self.assertEqual(input_element.name, 'default_field_test_field:int')
+    self.assertEqual(input_element.type, 'hidden')
+
+    self.assertEqual(select.tag, 'select')
+    first, second = select
+    self.assertEqual(first.tag, 'option')
+    self.assertEqual(first.text_content(), 'A')
+    self.assertEqual(first.attrib['value'], 'a')
+    self.assertTrue(second.attrib['selected'])
+
+    self.assertEqual(second.tag, 'option')
+    self.assertEqual(second.text_content(), 'B')
+    self.assertEqual(second.attrib['value'], 'b',)
+    self.assertTrue(second.attrib['selected'])
+
+  def test_render_escape_html(self):
+    self.field.values['default'] = []
+    self.field.values['items'] = [
+        ('<script>alert("text content")</script>', '<script>alert("value")</script>')]
+    (script, ), _, = lxml.html.fragments_fromstring(self.field.render())  # pylint:disable=unbalanced-tuple-unpacking
+    self.assertEqual(script.attrib['value'], '<script>alert("value")</script>')
+    self.assertEqual(script.text_content(), '<script>alert("text content")</script>')
+    # selected
+    self.field.values['default'] = [self.field.values['items'][0][1]]
+    (script, ), _, = lxml.html.fragments_fromstring(self.field.render())  # pylint:disable=unbalanced-tuple-unpacking
+    self.assertEqual(script.attrib['value'], '<script>alert("value")</script>')
+    self.assertEqual(script.text_content(), '<script>alert("text content")</script>')
+
+  def test_render_disabled(self):
+    # None items are rendered as disabled
+    self.field.values['default'] = []
+    self.field.values['items'] = [('Disabled', None)]
+    (disabled, ), _, = lxml.html.fragments_fromstring(self.field.render())  # pylint:disable=unbalanced-tuple-unpacking
+    self.assertTrue(disabled.attrib['disabled'])
+    self.assertFalse(disabled.attrib.get('value'))
+
   def test_render_view(self):
     self.assertEqual('A<br />\nB', self.field.render_view(value=['a', 'b']))
 

bt5/erp5_corporate_identity/SkinTemplateItem/portal_skins/erp5_corporate_identity/WebPage_createTableOfContent.py

@@ -56,7 +56,7 @@ for header in re.findall("<h[1-6].*?</h[1-6]>", doc_content or blank):
     html_quote(header_reference_prefix),
     '_anchor">',
     html_quote(header_reference),
-    '</div></a>']
+    '</a></div>']
   )
 
 closer = int(header_current) * '</ol>'

bt5/erp5_corporate_identity_test/PathTemplateItem/web_page_module/template_test_book_output_expected_002_en_html.html

@@ -316,7 +316,7 @@
     </tal:block-->
   
             
-            <section class="ci-book-table-of-content"><p class="ci-book-toc-faux-h1">Table of Contents</p><ol><li><div><a href="#introduction0_anchor">Introduction</div></a><ol><li><div><a href="#references1_anchor">References</div></a><ol><li><div><a href="#applicable-documents2_anchor">Applicable Documents</div></a></li><li><div><a href="#referenced-documents3_anchor">Referenced Documents</div></a></li></ol><li><div><a href="#abbreviations4_anchor">Abbreviations</div></a></li><li><div><a href="#figures5_anchor">Figures</div></a></li><li><div><a href="#tables6_anchor">Tables</div></a></li></ol><li><div><a href="#synthese7_anchor">Synthese</div></a><ol><li><div><a href="#risks-and-measures8_anchor">Risks and measures</div></a><ol><li><div><a href="#risk-of-casualties-if-device-does-not-stop-and-destroys-itself9_anchor">Risk of casualties if device does not stop and destroys itself</div></a></li><li><div><a href="#risk-of-killing-birds-and-polluting-the-environment10_anchor">Risk of killing birds and polluting the environment</div></a></li><li><div><a href="#noise-and-vibration-levels11_anchor">Noise and vibration levels</div></a></li></ol></li></ol><li><div><a href="#header-embedded-document12_anchor">Header Embedded Document</div></a><ol><li><div><a href="#header-embedded-document-content-examples13_anchor">Header Embedded Document content Examples</div></a></li></ol><li><div><a href="#reports14_anchor">Reports</div></a><ol><li><div><a href="#project-reports15_anchor">Project Reports</div></a></li><li><div><a href="#sale-order-reports16_anchor">Sale Order Reports</div></a></ol></ol></section>
+            <section class="ci-book-table-of-content"><p class="ci-book-toc-faux-h1">Table of Contents</p><ol><li><div><a href="#introduction0_anchor">Introduction</a></div><ol><li><div><a href="#references1_anchor">References</a></div><ol><li><div><a href="#applicable-documents2_anchor">Applicable Documents</a></div></li><li><div><a href="#referenced-documents3_anchor">Referenced Documents</a></div></li></ol><li><div><a href="#abbreviations4_anchor">Abbreviations</a></div></li><li><div><a href="#figures5_anchor">Figures</a></div></li><li><div><a href="#tables6_anchor">Tables</a></div></li></ol><li><div><a href="#synthese7_anchor">Synthese</a></div><ol><li><div><a href="#risks-and-measures8_anchor">Risks and measures</a></div><ol><li><div><a href="#risk-of-casualties-if-device-does-not-stop-and-destroys-itself9_anchor">Risk of casualties if device does not stop and destroys itself</a></div></li><li><div><a href="#risk-of-killing-birds-and-polluting-the-environment10_anchor">Risk of killing birds and polluting the environment</a></div></li><li><div><a href="#noise-and-vibration-levels11_anchor">Noise and vibration levels</a></div></li></ol></li></ol><li><div><a href="#header-embedded-document12_anchor">Header Embedded Document</a></div><ol><li><div><a href="#header-embedded-document-content-examples13_anchor">Header Embedded Document content Examples</a></div></li></ol><li><div><a href="#reports14_anchor">Reports</a></div><ol><li><div><a href="#project-reports15_anchor">Project Reports</a></div></li><li><div><a href="#sale-order-reports16_anchor">Sale Order Reports</a></div></ol></ol></section>
           
           
     

bt5/erp5_corporate_identity_test/PathTemplateItem/web_page_module/template_test_book_output_expected_003_de_html.html

@@ -89,7 +89,7 @@
   
       	    
   	        
-            <section class="ci-book-table-of-content"><p class="ci-book-toc-faux-h1">Inhaltsverzeichnis</p><ol><li><div><a href="#synthese0_anchor">Synthese</div></a><ol><li><div><a href="#risks-and-measures1_anchor">Risks and measures</div></a><ol><li><div><a href="#risk-of-casualties-if-device-does-not-stop-and-destroys-itself2_anchor">Risk of casualties if device does not stop and destroys itself</div></a></li><li><div><a href="#risk-of-killing-birds-and-polluting-the-environment3_anchor">Risk of killing birds and polluting the environment</div></a></li><li><div><a href="#noise-and-vibration-levels4_anchor">Noise and vibration levels</div></a></li></ol></li></ol><li><div><a href="#reports5_anchor">Reports</div></a><ol><li><div><a href="#project-reports6_anchor">Project Reports</div></a></li><li><div><a href="#sale-order-reports7_anchor">Sale Order Reports</div></a></ol></ol></section>
+            <section class="ci-book-table-of-content"><p class="ci-book-toc-faux-h1">Inhaltsverzeichnis</p><ol><li><div><a href="#synthese0_anchor">Synthese</a></div><ol><li><div><a href="#risks-and-measures1_anchor">Risks and measures</a></div><ol><li><div><a href="#risk-of-casualties-if-device-does-not-stop-and-destroys-itself2_anchor">Risk of casualties if device does not stop and destroys itself</a></div></li><li><div><a href="#risk-of-killing-birds-and-polluting-the-environment3_anchor">Risk of killing birds and polluting the environment</a></div></li><li><div><a href="#noise-and-vibration-levels4_anchor">Noise and vibration levels</a></div></li></ol></li></ol><li><div><a href="#reports5_anchor">Reports</a></div><ol><li><div><a href="#project-reports6_anchor">Project Reports</a></div></li><li><div><a href="#sale-order-reports7_anchor">Sale Order Reports</a></div></ol></ol></section>
           
           
     <h1><a name="synthese0_anchor"></a>Synthese<a class="custom-para" href=#synthese0_anchor><span style="font-size:.75em;line-height:1em;padding-left:.5em;">&para;</span></a></h1> 

bt5/erp5_dms/SkinTemplateItem/portal_skins/erp5_dms/Base_viewAdvancedSearchDialog/your_publication_section_relative_url.xml

@@ -118,7 +118,7 @@
       <dictionary>
         <item>
             <key> <string>_text</string> </key>
-            <value> <string>python: getattr(here.portal_categories[field.getId().replace(\'your_\', \'\', 1).replace(\'_relative_url\', \'\')], preferences.getPreference(\'preferred_category_child_item_list_method_id\', \'getCategoryChildCompactLogicalPathItemList\'))(local_sort_id=(\'int_index\', \'translated_title\'), checked_permission=\'View\', base=1)</string> </value>
+            <value> <string>python: getattr(here.portal_categories[field.getId().replace(\'your_\', \'\', 1).replace(\'_relative_url\', \'\')], preferences.getPreference(\'preferred_category_child_item_list_method_id\', \'getCategoryChildCompactLogicalPathItemList\'))(local_sort_id=(\'int_index\', \'translated_title\'), checked_permission=\'View\', base=1,  display_none_category=False)</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/erp5_officejs_ooffice/PathTemplateItem/web_site_module/ooffice_presentation.xml

@@ -397,7 +397,7 @@
         </item>
         <item>
             <key> <string>configuration_content_security_policy</string> </key>
-            <value> <string>default-src \'self\'; img-src \'self\' data:; media-src \'self\' blob:; connect-src \'self\' https://content.dropboxapi.com https://api.dropboxapi.com mail.tiolive.com data: *.host.vifib.net *.node.vifib.com *.erp5.net *.nexedi.net https://netdna.bootstrapcdn.com; script-src \'self\' \'unsafe-eval\' \'unsafe-inline\'; font-src \'self\' netdna.bootstrapcdn.com; style-src \'self\' netdna.bootstrapcdn.com \'unsafe-inline\' data:; frame-src \'self\' data:</string> </value>
+            <value> <string>default-src \'self\'; img-src \'self\' data:; media-src \'self\' blob:; connect-src \'self\' https://content.dropboxapi.com https://api.dropboxapi.com mail.tiolive.com data: *.host.vifib.net *.node.vifib.com *.erp5.net *.nexedi.net https://netdna.bootstrapcdn.com; script-src \'self\' \'unsafe-eval\' \'unsafe-inline\'; font-src \'self\' netdna.bootstrapcdn.com; style-src \'self\' netdna.bootstrapcdn.com \'unsafe-inline\' data:; frame-src \'self\' data:</string> </value>
         </item>
         <item>
             <key> <string>configuration_default_view_action_reference</string> </key>

bt5/erp5_officejs_ooffice/PathTemplateItem/web_site_module/ooffice_presentation/app.xml

@@ -367,7 +367,7 @@
         </item>
         <item>
             <key> <string>configuration_content_security_policy</string> </key>
-            <value> <string>default-src \'self\'; img-src \'self\' data:; media-src \'self\' blob:; connect-src \'self\' https://content.dropboxapi.com https://api.dropboxapi.com mail.tiolive.com data: *.host.vifib.net *.node.vifib.com *.erp5.net *.nexedi.net https://netdna.bootstrapcdn.com; script-src \'self\' \'unsafe-eval\' \'unsafe-inline\'; font-src \'self\' netdna.bootstrapcdn.com; style-src \'self\' netdna.bootstrapcdn.com \'unsafe-inline\' data:; frame-src \'self\' data:</string> </value>
+            <value> <string>default-src \'self\'; img-src \'self\' data:; media-src \'self\' blob:; connect-src \'self\' https://content.dropboxapi.com https://api.dropboxapi.com mail.tiolive.com data: *.host.vifib.net *.node.vifib.com *.erp5.net *.nexedi.net https://netdna.bootstrapcdn.com; script-src \'self\' \'unsafe-eval\' \'unsafe-inline\'; font-src \'self\' netdna.bootstrapcdn.com; style-src \'self\' netdna.bootstrapcdn.com \'unsafe-inline\' data:; frame-src \'self\' data:</string> </value>
         </item>
         <item>
             <key> <string>configuration_default_jio_document_page_gadget_url</string> </key>

bt5/erp5_officejs_ooffice/PathTemplateItem/web_site_module/ooffice_spreadsheet.xml

@@ -397,7 +397,7 @@
         </item>
         <item>
             <key> <string>configuration_content_security_policy</string> </key>
-            <value> <string>default-src \'self\'; img-src \'self\' data:; media-src \'self\' blob:; connect-src \'self\' https://content.dropboxapi.com https://api.dropboxapi.com mail.tiolive.com data: *.host.vifib.net *.node.vifib.com *.erp5.net *.nexedi.net https://netdna.bootstrapcdn.com; script-src \'self\' \'unsafe-eval\' \'unsafe-inline\'; font-src \'self\' netdna.bootstrapcdn.com; style-src \'self\' netdna.bootstrapcdn.com \'unsafe-inline\' data:; frame-src \'self\' data:</string> </value>
+            <value> <string>default-src \'self\'; img-src \'self\' data:; media-src \'self\' blob:; connect-src \'self\' https://content.dropboxapi.com https://api.dropboxapi.com mail.tiolive.com data: *.host.vifib.net *.node.vifib.com *.erp5.net *.nexedi.net https://netdna.bootstrapcdn.com; script-src \'self\' \'unsafe-eval\' \'unsafe-inline\'; font-src \'self\' netdna.bootstrapcdn.com; style-src \'self\' netdna.bootstrapcdn.com \'unsafe-inline\' data:; frame-src \'self\' data:</string> </value>
         </item>
         <item>
             <key> <string>configuration_default_view_action_reference</string> </key>

bt5/erp5_officejs_ooffice/PathTemplateItem/web_site_module/ooffice_spreadsheet/app.xml

@@ -395,7 +395,7 @@
         </item>
         <item>
             <key> <string>configuration_content_security_policy</string> </key>
-            <value> <string>default-src \'self\'; img-src \'self\' data:; media-src \'self\' blob:; connect-src \'self\' https://content.dropboxapi.com https://api.dropboxapi.com mail.tiolive.com data: *.host.vifib.net *.node.vifib.com *.erp5.net *.nexedi.net https://netdna.bootstrapcdn.com; script-src \'self\' \'unsafe-eval\' \'unsafe-inline\'; font-src \'self\' netdna.bootstrapcdn.com; style-src \'self\' netdna.bootstrapcdn.com \'unsafe-inline\' data:; frame-src \'self\' data:</string> </value>
+            <value> <string>default-src \'self\'; img-src \'self\' data:; media-src \'self\' blob:; connect-src \'self\' https://content.dropboxapi.com https://api.dropboxapi.com mail.tiolive.com data: *.host.vifib.net *.node.vifib.com *.erp5.net *.nexedi.net https://netdna.bootstrapcdn.com; script-src \'self\' \'unsafe-eval\' \'unsafe-inline\'; font-src \'self\' netdna.bootstrapcdn.com; style-src \'self\' netdna.bootstrapcdn.com \'unsafe-inline\' data:; frame-src \'self\' data:</string> </value>
         </item>
         <item>
             <key> <string>configuration_default_jio_document_page_gadget_url</string> </key>

bt5/erp5_officejs_ooffice/PathTemplateItem/web_site_module/ooffice_text.xml

@@ -397,7 +397,7 @@
         </item>
         <item>
             <key> <string>configuration_content_security_policy</string> </key>
-            <value> <string>default-src \'self\'; img-src \'self\' data:; media-src \'self\' blob:; connect-src \'self\' https://content.dropboxapi.com https://api.dropboxapi.com mail.tiolive.com data: *.host.vifib.net *.node.vifib.com *.erp5.net *.nexedi.net https://netdna.bootstrapcdn.com; script-src \'self\' \'unsafe-eval\' \'unsafe-inline\'; font-src \'self\' netdna.bootstrapcdn.com; style-src \'self\' netdna.bootstrapcdn.com \'unsafe-inline\' data:; frame-src \'self\' data:</string> </value>
+            <value> <string>default-src \'self\'; img-src \'self\' data:; media-src \'self\' blob:; connect-src \'self\' https://content.dropboxapi.com https://api.dropboxapi.com mail.tiolive.com data: *.host.vifib.net *.node.vifib.com *.erp5.net *.nexedi.net https://netdna.bootstrapcdn.com; script-src \'self\' \'unsafe-eval\' \'unsafe-inline\'; font-src \'self\' netdna.bootstrapcdn.com; style-src \'self\' netdna.bootstrapcdn.com \'unsafe-inline\' data:; frame-src \'self\' data:</string> </value>
         </item>
         <item>
             <key> <string>configuration_default_view_action_reference</string> </key>

bt5/erp5_officejs_ooffice/PathTemplateItem/web_site_module/ooffice_text/app.xml

@@ -367,7 +367,7 @@
         </item>
         <item>
             <key> <string>configuration_content_security_policy</string> </key>
-            <value> <string>default-src \'self\'; img-src \'self\' data:; media-src \'self\' blob:; connect-src \'self\' https://content.dropboxapi.com https://api.dropboxapi.com mail.tiolive.com data: *.host.vifib.net *.node.vifib.com *.erp5.net *.nexedi.net https://netdna.bootstrapcdn.com; script-src \'self\' \'unsafe-eval\' \'unsafe-inline\'; font-src \'self\' netdna.bootstrapcdn.com; style-src \'self\' netdna.bootstrapcdn.com \'unsafe-inline\' data:; frame-src \'self\' data:</string> </value>
+            <value> <string>default-src \'self\'; img-src \'self\' data:; media-src \'self\' blob:; connect-src \'self\' https://content.dropboxapi.com https://api.dropboxapi.com mail.tiolive.com data: *.host.vifib.net *.node.vifib.com *.erp5.net *.nexedi.net https://netdna.bootstrapcdn.com; script-src \'self\' \'unsafe-eval\' \'unsafe-inline\'; font-src \'self\' netdna.bootstrapcdn.com; style-src \'self\' netdna.bootstrapcdn.com \'unsafe-inline\' data:; frame-src \'self\' data:</string> </value>
         </item>
         <item>
             <key> <string>configuration_default_jio_document_page_gadget_url</string> </key>

bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/Base_viewTradeFieldLibrary/my_dialog_mode_use_list.xml

@@ -118,7 +118,7 @@
       <dictionary>
         <item>
             <key> <string>_text</string> </key>
-            <value> <string>python: getattr(here.portal_categories.use, preferences.getPreference(\'preferred_category_child_item_list_method_id\', \'getCategoryChildCompactLogicalPathItemList\'))(local_sort_id=(\'int_index\', \'translated_title\'), checked_permission=\'View\', base=True)</string> </value>
+            <value> <string>python: getattr(here.portal_categories.use, preferences.getPreference(\'preferred_category_child_item_list_method_id\', \'getCategoryChildCompactLogicalPathItemList\'))(local_sort_id=(\'int_index\', \'translated_title\'), checked_permission=\'View\', base=True,  display_none_category=False)</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/erp5_web_renderjs_ui/PathTemplateItem/web_page_module/rjs_gadget_erp5_pt_language_js.js

@@ -22,7 +22,11 @@
     .declareMethod("render", function () {
       var gadget = this;
 
-      return new RSVP.Queue()
+      return new RSVP.Queue(
+	  // Stabilise UI tests with a tiny delay, which should be unnoticeable
+	  // to users.
+          RSVP.delay(200)
+        )
         .push(function () {
           return RSVP.all([
             gadget.getUrlForList([{command: 'history_previous'}]),

bt5/erp5_web_renderjs_ui_test/PathTemplateItem/portal_tests/renderjs_ui_relation_field_zuite/testRelationFieldPercent.zpt

@@ -61,7 +61,7 @@
 </tr>
 <tr>
   <td>assertEval</td>
-  <td>(function() { selenium.browserbot.getCurrentWindow().document.getElementById("field_my_foo_category_title").value = "%"; return true;})()
+  <td>(function() { selenium.browserbot.getCurrentWindow().document.getElementById("field_my_foo_category_title").value = "Qua%"; return true;})()
   <td>true</td>
 </tr>
 <tr>

product/CMFActivity/Activity/SQLBase.py

@@ -68,6 +68,12 @@ UID_SAFE_BITSIZE = 63
 # enough while yielding one order of magnitude collision probability
 # improvement.
 UID_ALLOCATION_TRY_COUNT = 10
+# Limit the number of UNION-joined subqueries per query when looking for
+# blocking activities. Used to take a slice from a list.
+# XXX: 5000 is known to work on a case on "after_tag" dependency, which fails
+# at 5400 with:
+#   ProgrammingError: (1064, "memory exhausted near [...]")
+_MAX_DEPENDENCY_UNION_SUBQUERY_COUNT = -5000
 
 def sort_message_key(message):
   # same sort key as in SQLBase.getMessageList
@@ -181,37 +187,42 @@ def _validate_after_tag_and_method_id(value, render_string):
 #   same order as the dependency value items expected by the next item
 # - callable rendering given values into an SQL condition
 #   (value, render_string) -> str
+# - minimal applicable processing_node value (excluded)
 _DEPENDENCY_TESTER_DICT = {
   'after_method_id': (
     ('method_id', ),
     sqltest_dict['method_id'],
+    DEPENDENCY_IGNORED_ERROR_STATE,
   ),
   'after_path': (
     ('path', ),
     sqltest_dict['path'],
+    DEPENDENCY_IGNORED_ERROR_STATE,
   ),
   'after_message_uid': (
     ('uid', ),
     sqltest_dict['uid'],
+    DEPENDENCY_IGNORED_ERROR_STATE,
   ),
   'after_path_and_method_id': (
     ('path', 'method_id'),
     _validate_after_path_and_method_id,
+    DEPENDENCY_IGNORED_ERROR_STATE,
   ),
   'after_tag': (
     ('tag', ),
     sqltest_dict['tag'],
+    DEPENDENCY_IGNORED_ERROR_STATE,
   ),
   'after_tag_and_method_id': (
     ('tag', 'method_id'),
     _validate_after_tag_and_method_id,
+    DEPENDENCY_IGNORED_ERROR_STATE,
   ),
   'serialization_tag': (
     ('serialization_tag', ),
-    lambda value, render_string: (
-      'processing_node > -1 AND ' +
-      sqltest_dict['serialization_tag'](value, render_string)
-    ),
+    sqltest_dict['serialization_tag'],
+    -1,
   ),
 }
 
@@ -460,7 +471,7 @@ CREATE TABLE %s (
         dependency_value,
       ) in message.activity_kw.iteritems():
         try:
-          column_list, _ = dependency_tester_dict[dependency_name]
+          column_list, _, _ = dependency_tester_dict[dependency_name]
         except KeyError:
           continue
         # There are 2 types of dependencies:
@@ -569,14 +580,16 @@ CREATE TABLE %s (
       if not dependency_value_dict:
         # No more non-blocked message for this dependency, skip it.
         continue
-      column_list, to_sql = dependency_tester_dict[dependency_name]
+      column_list, to_sql, min_processing_node = dependency_tester_dict[
+        dependency_name
+      ]
       row2key = (
         _ITEMGETTER0
         if len(column_list) == 1 else
         _IDENTITY
       )
       base_sql_suffix = ' WHERE processing_node > %i AND (%%s) LIMIT 1)' % (
-        DEPENDENCY_IGNORED_ERROR_STATE,
+        min_processing_node,
       )
       sql_suffix_list = [
         base_sql_suffix % to_sql(dependency_value, quote)
@@ -585,22 +598,30 @@ CREATE TABLE %s (
       base_sql_prefix = '(SELECT %s FROM ' % (
         ','.join(column_list),
       )
-      for row in db.query(
-        ' UNION '.join(
-          base_sql_prefix + table_name + sql_suffix
-          for table_name in table_name_list
-          for sql_suffix in sql_suffix_list
-        ),
-        max_rows=0,
-      )[1]:
-        # Each row is a value which blocks some activities.
-        dependent_message_set = dependency_value_dict[row2key(row)]
-        # queue blocked messages for processing in the beginning of next
-        # outermost iteration.
-        new_blocked_message_set.update(dependent_message_set)
-        # ...but update result immediately, in case there is no next
-        # outermost iteration.
-        result.difference_update(dependent_message_set)
+      subquery_list = [
+        base_sql_prefix + table_name + sql_suffix
+        for table_name in table_name_list
+        for sql_suffix in sql_suffix_list
+      ]
+      while subquery_list:
+        # Join queries with a UNION, to reduce per-query latency.
+        # Also, limit the number of subqueries per query, as their number can
+        # largely exceed the number of activities being considered multiplied
+        # by the number of activty tables: it is also proportional to the
+        # number of distinct values being looked for in the current column.
+        for row in db.query(
+          ' UNION '.join(subquery_list[_MAX_DEPENDENCY_UNION_SUBQUERY_COUNT:]),
+          max_rows=0,
+        )[1]:
+          # Each row is a value which blocks some activities.
+          dependent_message_set = dependency_value_dict[row2key(row)]
+          # queue blocked messages for processing in the beginning of next
+          # outermost iteration.
+          new_blocked_message_set |= dependent_message_set
+          # ...but update result immediately, in case there is no next
+          # outermost iteration.
+          result -= dependent_message_set
+        del subquery_list[_MAX_DEPENDENCY_UNION_SUBQUERY_COUNT:]
       dependency_value_dict.clear()
     return result
 

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Base_viewDocumentList/listbox.xml

@@ -116,10 +116,6 @@
                     <key> <string>all_columns</string> </key>
                     <value>
                       <list>
-                        <tuple>
-                          <string>language</string>
-                          <string>Language</string>
-                        </tuple>
                         <tuple>
                           <string>revision</string>
                           <string>Revision</string>
@@ -185,6 +181,10 @@
                           <string>version</string>
                           <string>Version</string>
                         </tuple>
+                        <tuple>
+                          <string>language</string>
+                          <string>Language</string>
+                        </tuple>
                         <tuple>
                           <string>description</string>
                           <string>Description</string>

product/ERP5/bootstrap/erp5_core/bt/template_path_list

@@ -6,28 +6,92 @@ portal_caches/erp5_content_medium/default_ram_cache
 portal_caches/erp5_content_short
 portal_caches/erp5_content_short/default_ram_cache
 portal_caches/erp5_session_cache
-portal_caches/erp5_session_cache/**
+portal_caches/erp5_session_cache/2
 portal_caches/erp5_ui_long
 portal_caches/erp5_ui_long/default_ram_cache
 portal_caches/erp5_ui_medium
 portal_caches/erp5_ui_medium/default_ram_cache
 portal_caches/erp5_ui_short
 portal_caches/erp5_ui_short/default_ram_cache
-portal_categories/action_type/*
-portal_categories/callable_type/**
+portal_categories/action_type/global
+portal_categories/action_type/object_action
+portal_categories/action_type/object_button
+portal_categories/action_type/object_dialog
+portal_categories/action_type/object_exchange
+portal_categories/action_type/object_exchange_raw
+portal_categories/action_type/object_fast_input
+portal_categories/action_type/object_fast_input_raw
+portal_categories/action_type/object_icon
+portal_categories/action_type/object_jio_action
+portal_categories/action_type/object_jio_action_raw
+portal_categories/action_type/object_jio_button
+portal_categories/action_type/object_jio_button_raw
+portal_categories/action_type/object_jio_exchange
+portal_categories/action_type/object_jio_exchange_raw
+portal_categories/action_type/object_jio_fast_input
+portal_categories/action_type/object_jio_fast_input_raw
+portal_categories/action_type/object_jio_js_script
+portal_categories/action_type/object_jio_jump
+portal_categories/action_type/object_jio_jump_raw
+portal_categories/action_type/object_jio_print
+portal_categories/action_type/object_jio_print_raw
+portal_categories/action_type/object_jio_report
+portal_categories/action_type/object_jio_report_raw
+portal_categories/action_type/object_jio_search
+portal_categories/action_type/object_jio_view
+portal_categories/action_type/object_jump
+portal_categories/action_type/object_list
+portal_categories/action_type/object_onlyjio_action
+portal_categories/action_type/object_onlyjio_action_raw
+portal_categories/action_type/object_onlyjio_button
+portal_categories/action_type/object_onlyjio_button_raw
+portal_categories/action_type/object_onlyjio_jump
+portal_categories/action_type/object_onlyjio_jump_raw
+portal_categories/action_type/object_onlyjio_view
+portal_categories/action_type/object_onlyxhtml_view
+portal_categories/action_type/object_print
+portal_categories/action_type/object_print_raw
+portal_categories/action_type/object_report
+portal_categories/action_type/object_report_raw
+portal_categories/action_type/object_search
+portal_categories/action_type/object_view
+portal_categories/action_type/object_web_view
+portal_categories/action_type/workflow
+portal_categories/callable_type/script
+portal_categories/callable_type/script/diff_filter
 portal_categories/constraint_type/audit
 portal_categories/constraint_type/configuration
 portal_categories/constraint_type/default
 portal_categories/constraint_type/post_upgrade
 portal_categories/constraint_type/pre_upgrade
 portal_categories/constraint_type/upgrader
-portal_categories/elementary_type/**
+portal_categories/elementary_type/boolean
+portal_categories/elementary_type/content
+portal_categories/elementary_type/data
+portal_categories/elementary_type/date
+portal_categories/elementary_type/float
+portal_categories/elementary_type/int
+portal_categories/elementary_type/lines
+portal_categories/elementary_type/long
+portal_categories/elementary_type/multiple selection
+portal_categories/elementary_type/object
+portal_categories/elementary_type/selection
+portal_categories/elementary_type/string
+portal_categories/elementary_type/tales
+portal_categories/elementary_type/text
+portal_categories/elementary_type/tokens
 portal_contribution_registry/default_predicate
 portal_domains/base_day_domain
 portal_domains/base_month_domain
 portal_domains/base_week_domain
 portal_domains/base_year_domain
 portal_domains/parent_domain
-portal_ids/**
+portal_ids/1
+portal_ids/2
+portal_ids/3
+portal_ids/4
+portal_ids/5
+portal_ids/6
+portal_ids/mysql_non_continuous_increasing_non_zodb
 portal_memcached/default_memcached_plugin
 portal_preferences/default_site_preference
\ No newline at end of file

product/ERP5/tests/testInvalidationBug.py

@@ -88,7 +88,11 @@ class TestInvalidationBug(ERP5TypeTestCase):
     self.assertEqual(result_list[1], [0,0])  # activity buffer first
     self.assertEqual(result_list[-3], [1,0]) # catalog
     self.assertEqual(result_list[-2], None)  # ZODB
-    self.assertEqual(result_list[-1], [1,1]) # activity tables last
+    result_catalog_count, result_activity_count = result_list[-1]
+    # activity tables last (there may be multiple activities, but there must be
+    # at least one).
+    self.assertEqual(result_catalog_count, 1)
+    self.assertGreaterEqual(result_activity_count, 1)
 
   # TODO: - skip this test for ZEO>=5 because it's covered upstream
   #         (and later remove it)
@@ -168,8 +172,9 @@ class TestInvalidationBug(ERP5TypeTestCase):
       unpatch()
       activity_tool.manage_addToProcessingList(node_list)
       self.commit()
-    ## When the bug is not fixed, we get a -3 failed activity
-    self.assertNoPendingMessage()
+    ## When the bug is not fixed, we get a failed activity
+    ## which would cause tic to fail
+    self.tic()
 
   def _testReindex(self):
     print("To reproduce bugs easily, distribution step should be skipped for"

product/ERP5/tests/testXHTML.py

@@ -27,6 +27,7 @@
 #
 ##############################################################################
 
+import cgi
 import unittest
 import os
 import requests
@@ -34,6 +35,7 @@ from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 
 from subprocess import Popen, PIPE
+from AccessControl import getSecurityManager
 from Testing import ZopeTestCase
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from Products.ERP5Type.tests.utils import addUserToDeveloperRole, findContentChain
@@ -674,9 +676,20 @@ def makeTestMethod(validator, portal_type, view_name, bt_name):
         self.portal,
         portal_type)
     document = createSubContent(module, portal_type_list)
-    view = getattr(document, view_name)
+
+    response = self.publish(
+        '%s/%s' % (document.getPath(), view_name),
+        user=str(getSecurityManager().getUser()),
+        handle_errors=False,
+    )
+    charset = 'iso8859-15'
+    content_type = response.getHeader('content-type')
+    if content_type:
+      _, params = cgi.parse_header(content_type)
+      charset = params.get('charset', charset)
+
     self.assert_(*validate_xhtml( validator=validator,
-                                  source=view(),
+                                  source=response.getBody().decode(charset),
                                   view_name=view_name,
                                   bt_name=bt_name))
   return testMethod
@@ -742,6 +755,19 @@ if validator_to_use == 'tidy':
 elif validator_to_use == 'nu':
   validator = NuValidator(show_warnings)
 
+
+expected_failure_list = (
+    # this view needs VCS preference set (this test suite does not support
+    # setting preferences, but this might be a way to fix this).
+    'test_erp5_forge_Business_Template_BusinessTemplate_viewVcsStatus',
+    # this view only works when solver decision has a relation to a solver.
+    # One way to fix this would be to allow a custom "init script" to be called
+    # on a portal type.
+    'test_erp5_simulation_Solver_Decision_SolverDecision_viewConfiguration',
+    # this view redirects to an external URL
+    'test_erp5_web_Static_Web_Site_WebSite_view',
+)
+
 def test_suite():
   # add the tests
   if validator is not None:
@@ -749,15 +775,7 @@ def test_suite():
     # on getBusinessTemplateList call
     addTestMethodDynamically(TestXHTML, validator,
       ('erp5_core',) + TestXHTML.getBusinessTemplateList(),
-      expected_failure_list=(
-          # this view needs VCS preference set (this test suite does not support
-          # setting preferences, but this might be a way to fix this).
-          'test_erp5_forge_Business_Template_BusinessTemplate_viewVcsStatus',
-          # this view only works when solver decision has a relation to a solver.
-          # One way to fix this would be to allow a custom "init script" to be called
-          # on a portal type.
-          'test_erp5_simulation_Solver_Decision_SolverDecision_viewConfiguration',
-      ))
+      expected_failure_list=expected_failure_list)
   suite = unittest.TestSuite()
   suite.addTest(unittest.makeSuite(TestXHTML))
   return suite

product/ERP5Type/Log.py

@@ -38,7 +38,7 @@ marker_ = []
 def log(description, content=marker_, level=INFO):
     """Deprecated method
 
-    Use erp5.componement.Log instead.
+    Use erp5.component.module.Log instead.
     
     Kept for compatbility to allow instance upgrade.
     """

product/ERP5Type/tests/CodingStyleTest.py

@@ -114,15 +114,7 @@ def test_suite():
       testclass,
       testXHTML.validator,
       tested_business_template_list,
-      expected_failure_list=(
-          # this view needs VCS preference set (this test suite does not support
-          # setting preferences, but this might be a way to fix this).
-          'test_erp5_forge_Business_Template_BusinessTemplate_viewVcsStatus',
-          # this view only works when solver decision has a relation to a solver.
-          # One way to fix this would be to allow a custom "init script" to be called
-          # on a portal type.
-          'test_erp5_simulation_Solver_Decision_SolverDecision_viewConfiguration',
-      ),
+      expected_failure_list=testXHTML.expected_failure_list,
   )
 
   # required to create content in portal_components

product/Formulator/Widget.py

@@ -1027,7 +1027,7 @@ class MultiItemsWidget(ItemsWidget):
       if item_value in value:
         rendered_item = self.render_selected_item(
             escape(ustr(item_text)),
-            escape(ustr(item_value)),
+            item_value,
             key,
             css_class,
             extra_item)
@@ -1037,7 +1037,7 @@ class MultiItemsWidget(ItemsWidget):
       else:
         rendered_item = self.render_item(
             escape(ustr(item_text)),
-            escape(ustr(item_value)),
+            item_value,
             key,
             css_class,
             extra_item)