Changes so that document ids with non-ascii characters work

See merge request nexedi/erp5!1950

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testBase.py

@@ -1405,6 +1405,25 @@ class TestBase(ERP5TypeTestCase, ZopeTestCase.Functional):
     newSecurityManager(None, test_user)
     assertActorHistoryEqual([user_1_title, user_2_title, user_3_user_id, existing_non_erp5_user_id])
 
+  def test_idWithSpecialCharacter(self, quiet=quiet, run=run_all_test):
+    """
+      Test that an id with non-ascii characters
+      can well be used an indexed
+    """
+    portal = self.getPortal()
+    portal_type = "Organisation"
+    module = portal.getDefaultModule(portal_type=portal_type)
+    obj = module.newContent(
+      id='tést',
+      portal_type=portal_type
+    )
+    self.tic() # no error in indexation
+    # Check that the new object is retreivable by its id
+    self.assertEqual(obj.getId(), 'tést')
+    self.assertIsInstance(obj.getId(), str)
+    self.assertEqual(module._getOb(obj.getId()).getId(), obj.getId())
+    self.assertIsInstance(module._getOb(obj.getId()).getId(), str)
+
 
 class TestERP5PropertyManager(unittest.TestCase):
   """Tests for ERP5PropertyManager.

product/CMFActivity/ActivityTool.py

@@ -27,7 +27,7 @@ from __future__ import absolute_import
 #
 ##############################################################################
 from six import string_types as basestring
-from Products.ERP5Type.Utils import ensure_list
+from Products.ERP5Type.Utils import ensure_list, str2unicode
 
 import copy
 import socket
@@ -359,7 +359,10 @@ class Message(BaseMessage):
           #      that method !
           method = getattr(obj, self.method_id)
           transaction.get().note(
-            u'CMFActivity {}/{}'.format('/'.join(self.object_path), self.method_id)
+            u'CMFActivity {}/{}'.format(
+              '/'.join([str2unicode(x) for x in self.object_path]),
+              str2unicode(self.method_id),
+            )
           )
           # Store site info
           setSite(activity_tool.getParentValue())

product/ERP5Type/patches/sqltest.py

@@ -174,13 +174,13 @@ else: # For easy diff with original (ZSQLMethods 3.14)
             else:
                 if not isinstance(v, StringTypes):
                     v = str(v)
-                if isinstance(v, six.binary_type):
-                    v = v.decode('utf-8')
+                if six.PY3 and isinstance(v, six.binary_type):
+                    v = v.decode(self.encoding or 'UTF-8')
                 # The call to sql_quote__ can return something that is not
                 # a native string anymore!
                 v = md.getitem('sql_quote__', 0)(v)
-                if isinstance(v, six.binary_type):
-                    v = v.decode('utf-8')
+                if six.PY3 and isinstance(v, six.binary_type):
+                    v = v.decode(self.encoding or 'UTF-8')
                 # if v.find("\'") >= 0: v="''".(v.split("\'"))
                 # v="'%s'" % v
             vs.append(v)