Localizer: support python3 and polib 1.2.0

Fix po_import behavior in Python 2.

Newer versions of polib accept only unicode strings in the
`pofile` function (because they check if they start by the decoded
version of the BOM).
I changed the `data` that is passed to `pofile` to be a unicode
string in Python 2 too. This seems to work locally in my old
version of polib, so that at least the old behavior should be
kept the same.
Co-authored-by: Carlos Ramos Carreño <carlos.ramos@nexedi.com>
Co-authored-by: Jérome Perrin <jerome@nexedi.com>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_updateTranslationTable.py

-from Products.ERP5Type.Utils import getMessageIdWithContext
+from Products.ERP5Type.Utils import getMessageIdWithContext, str2unicode, unicode2str
 
 supported_languages = context.Localizer.get_supported_languages()
 translated_keys = {} # This dict prevents entering the same key twice
@@ -34,7 +34,7 @@ for portal_type in portal_type_list:
       for lang in supported_languages:
         key = (lang, portal_type.getId(), state_var, state_reference)
         if key not in translated_keys:
-          translated_message = context.Localizer.erp5_ui.gettext(state_reference, lang=lang).encode('utf-8')
+          translated_message = unicode2str(context.Localizer.erp5_ui.gettext(state_reference, lang=lang))
           translated_keys[key] = None # mark as translated
           object_list.append(dict(language=lang, message_context=state_var, portal_type=portal_type.getId(), original_message=state_reference,
                              translated_message=translated_message))
@@ -43,10 +43,10 @@ for portal_type in portal_type_list:
         if state.getTitle() is not None and state.getTitle() != '':
           state_var_title = '%s_title' % state_var
           msg_id = getMessageIdWithContext(state.getTitle(), 'state', wf_id)
-          translated_message = context.Localizer.erp5_ui.gettext(msg_id, default='', lang=lang).encode('utf-8')
+          translated_message = unicode2str(context.Localizer.erp5_ui.gettext(msg_id, default='', lang=lang))
           if translated_message == '':
             msg_id = state.getTitle()
-            translated_message = context.Localizer.erp5_ui.gettext(state.getTitle().decode('utf-8'), lang=lang).encode('utf-8')
+            translated_message = unicode2str(context.Localizer.erp5_ui.gettext(str2unicode(state.getTitle()), lang=lang))
           key = (lang, portal_type.getId(), state_var_title, state_reference, msg_id)
           if key not in translated_keys:
             translated_keys[key] = None # mark as translated
@@ -67,7 +67,7 @@ for ptype in context.portal_types.objectValues():
     if key not in translated_keys:
       translated_keys[key] = None # mark as translated
       object_list.append(dict(language=lang, message_context='portal_type', portal_type=portal_type, original_message=portal_type,
-                         translated_message=context.Localizer.erp5_ui.gettext(portal_type, lang=lang).encode('utf-8')))
+                         translated_message=unicode2str(context.Localizer.erp5_ui.gettext(portal_type, lang=lang))))
 if object_list:
   catalog_translation_list(object_list)
 

product/Localizer/MessageCatalog.py

@@ -106,15 +106,11 @@ def message_decode(message):
     To be used in the user interface, to avoid problems with the
     encodings, HTML entities, etc..
     """
-    message = decodebytes(message)
+    message = decodebytes(message.encode())
     encoding = HTTPRequest.default_encoding
     return six.text_type(message, encoding)
 
 
-def filter_sort(x, y):
-    return cmp(to_unicode(x), to_unicode(y))
-
-
 def get_url(url, batch_start, batch_size, regex, lang, empty, **kw):
     params = []
     for key, value in six.iteritems(kw):
@@ -396,7 +392,7 @@ class MessageCatalog(LanguageManager, ObjectManager, SimpleItem):
         for m, t in six.iteritems(self._messages):
             if query.search(m) and (not empty or not t.get(lang, '').strip()):
                 messages.append(m)
-        messages.sort(filter_sort)
+        messages.sort(key=lambda m: to_unicode(m))
         # How many messages
         n = len(messages)
         namespace['n_messages'] = n
@@ -664,6 +660,8 @@ class MessageCatalog(LanguageManager, ObjectManager, SimpleItem):
         messages = self._messages
 
         # Load the data
+        if isinstance(data, bytes):
+            data = data.decode("utf-8")
         po = polib.pofile(data)
         encoding = to_str(po.encoding)
         for entry in po:
@@ -688,8 +686,10 @@ class MessageCatalog(LanguageManager, ObjectManager, SimpleItem):
     def manage_import(self, lang, file, REQUEST=None, RESPONSE=None):
         """ """
         # XXX For backwards compatibility only, use "po_import" instead.
-        if isinstance(file, str):
+        if isinstance(file, str): # six.PY2
             content = file
+        elif isinstance(file, bytes): # six.PY3
+            content = file.decode()
         else:
             content = file.read()
 

product/Localizer/__init__.py

@@ -23,9 +23,7 @@ from App.ImageFile import ImageFile
 from DocumentTemplate.DT_String import String
 
 # Import from Localizer
-import six
-if six.PY2:
-  from . import patches as _
+from . import patches as _
 from . import Localizer, MessageCatalog
 from .LocalFiles import LocalDTMLFile