py3: DummyMailHostMixin message is bytes().

Co-authored-by: Arnaud Fontaine <arnaud.fontaine@nexedi.com>

bt5/erp5_accounting_l10n_fr/TestTemplateItem/portal_components/test.erp5.testAccounting_l10n_fr.py

@@ -30,9 +30,13 @@
 """Test suite for erp5_accounting_l10n_fr
 """
 
+import six
 import unittest
 import zipfile
-import email
+if six.PY2:
+  from email import message_from_string as message_from_bytes
+else:
+  from email import message_from_bytes
 import os.path
 import io
 from DateTime import DateTime
@@ -118,7 +122,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     self.assertNotEqual((), last_message)
     _, mto, message_text = last_message
     self.assertEqual('"%s" <%s>' % (self.first_name, self.recipient_email_address), mto[0])
-    mail_message = email.message_from_string(message_text)
+    mail_message = message_from_bytes(message_text)
     for part in mail_message.walk():
       content_type = part.get_content_type()
       file_name = part.get_filename()

bt5/erp5_authentication_policy/TestTemplateItem/portal_components/test.erp5.testAuthenticationPolicy.py

@@ -644,7 +644,7 @@ class TestAuthenticationPolicy(ERP5TypeTestCase):
     self.tic()
     _, (to,), message = self.portal.MailHost._last_message
     self.assertEqual(to, 'user@example.com')
-    self.assertIn('Password Recovery', message)
+    self.assertIn(b'Password Recovery', message)
 
   def test_HttpRequest(self):
     """

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testNotificationTool.py

@@ -26,6 +26,7 @@
 #
 ##############################################################################
 
+import six
 import unittest
 
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
@@ -33,7 +34,10 @@ from AccessControl.SecurityManagement import newSecurityManager
 from AccessControl.SecurityManagement import getSecurityManager
 from Products.ERP5Type.tests.Sequence import SequenceList
 from Products.ERP5Type.tests.utils import DummyMailHost
-import email
+if six.PY2:
+  from email import message_from_string as message_from_bytes
+else:
+  from email import message_from_bytes
 from email.header import decode_header, make_header
 from email.utils import parseaddr
 
@@ -47,7 +51,7 @@ def decode_email(file_):
     'headers': {}
   }
   # Get Message
-  msg = email.message_from_string(file_)
+  msg = message_from_bytes(file_)
   # Back up original file
   theMail['__original__'] = file_
   for key, value in msg.items():

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testPasswordTool.py

@@ -125,7 +125,7 @@ class TestPasswordTool(ERP5TypeTestCase):
     self.assertEqual(['userA@example.invalid'], mto)
     reset_key, = list(six.iterkeys(self.portal.portal_password._password_request_dict))
     self.assertIn(
-      ('PasswordTool_viewResetPassword?reset_key=' + reset_key).encode(),
+      ('PasswordTool_viewResetPassword?reset_key=' + reset_key),
       mbody)
 
     ret = self.portal.portal_password.changeUserPassword(

bt5/erp5_credential/TestTemplateItem/portal_components/test.erp5.testERP5Credential.py

@@ -26,13 +26,18 @@
 #
 ##############################################################################
 
+import six
 import unittest
 from Products.ERP5Type.tests.utils import reindex
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from Products.ERP5Type.tests.utils import DummyMailHost
 from Products.ERP5Type.tests.Sequence import SequenceList
 from DateTime import DateTime
-import email, re
+import re
+if six.PY2:
+  from email import message_from_string as message_from_bytes
+else:
+  from email import message_from_bytes
 from email.header import decode_header, make_header
 from email.utils import parseaddr
 import six.moves.urllib.parse
@@ -172,7 +177,7 @@ class TestERP5Credential(ERP5TypeTestCase):
       'headers': {}
     }
     # Get Message
-    msg = email.message_from_string(file_)
+    msg = message_from_bytes(file_)
     # Back up original file
     theMail['__original__'] = file_
     for key, value in msg.items():

bt5/erp5_crm/TestTemplateItem/portal_components/test.erp5.testCRM.py

@@ -26,6 +26,7 @@
 #
 ##############################################################################
 
+import six
 import unittest
 import six.moves.urllib.parse
 import os
@@ -40,7 +41,11 @@ from email.header import decode_header
 from email.mime.multipart import MIMEMultipart
 from email.mime.base import MIMEBase
 from email.mime.text import MIMEText
-from email import encoders, message_from_string
+from email import encoders
+if six.PY2:
+  from email import message_from_string as message_from_bytes
+else:
+  from email import message_from_bytes
 from DateTime import DateTime
 
 import Products.ERP5.tests
@@ -883,7 +888,7 @@ class TestCRMMailIngestion(BaseTestCRM):
     portal = self.portal
     message_string = self.makeFileUpload('simple').read().decode()
 
-    message = message_from_string(message_string)
+    message = message_from_bytes(message_string)
     message.replace_header('subject', 'Visit:Company A')
     data = message.as_string().encode()
     self._ingestMail(data=data)
@@ -891,7 +896,7 @@ class TestCRMMailIngestion(BaseTestCRM):
     document = getLastCreatedEvent(portal.event_module)
     self.assertEqual(document.getPortalType(), 'Visit')
 
-    message = message_from_string(message_string)
+    message = message_from_bytes(message_string)
     message.replace_header('subject', 'Fax:Company B')
     data = message.as_string().encode()
     self._ingestMail(data=data)
@@ -899,7 +904,7 @@ class TestCRMMailIngestion(BaseTestCRM):
     document = getLastCreatedEvent(portal.event_module)
     self.assertEqual(document.getPortalType(), 'Fax Message')
 
-    message = message_from_string(message_string)
+    message = message_from_bytes(message_string)
     message.replace_header('subject', 'TEST:Company B')
     data = message.as_string().encode()
     self._ingestMail(data=data)
@@ -907,7 +912,7 @@ class TestCRMMailIngestion(BaseTestCRM):
     document = getLastCreatedEvent(portal.event_module)
     self.assertEqual(document.getPortalType(), 'Mail Message')
 
-    message = message_from_string(message_string)
+    message = message_from_bytes(message_string)
     message.replace_header('subject', 'visit:Company A')
     data = message.as_string().encode()
     self._ingestMail(data=data)
@@ -915,7 +920,7 @@ class TestCRMMailIngestion(BaseTestCRM):
     document = getLastCreatedEvent(portal.event_module)
     self.assertEqual(document.getPortalType(), 'Visit')
 
-    message = message_from_string(message_string)
+    message = message_from_bytes(message_string)
     message.replace_header('subject', 'phone:Company B')
     data = message.as_string().encode()
     self._ingestMail(data=data)
@@ -923,7 +928,7 @@ class TestCRMMailIngestion(BaseTestCRM):
     document = portal.event_module[portal.event_module.objectIds()[-1]]
     self.assertEqual(document.getPortalType(), 'Phone Call')
 
-    message = message_from_string(message_string)
+    message = message_from_bytes(message_string)
     message.replace_header('subject', 'LETTER:Company C')
     data = message.as_string().encode()
     self._ingestMail(data=data)
@@ -931,7 +936,7 @@ class TestCRMMailIngestion(BaseTestCRM):
     document = getLastCreatedEvent(portal.event_module)
     self.assertEqual(document.getPortalType(), 'Letter')
 
-    message = message_from_string(message_string)
+    message = message_from_bytes(message_string)
     body = message.get_payload()
     message.set_payload('Visit:%s' % body)
     data = message.as_string().encode()
@@ -940,7 +945,7 @@ class TestCRMMailIngestion(BaseTestCRM):
     document = getLastCreatedEvent(portal.event_module)
     self.assertEqual(document.getPortalType(), 'Visit')
 
-    message = message_from_string(message_string)
+    message = message_from_bytes(message_string)
     body = message.get_payload()
     message.set_payload('PHONE CALL:%s' % body)
     data = message.as_string().encode()
@@ -1152,7 +1157,7 @@ class TestCRMMailSend(BaseTestCRM):
     self.assertEqual('"Me," <me@erp5.org>', mfrom)
     self.assertEqual(['"Recipient," <recipient@example.com>'], mto)
     self.assertEqual(event.getTextContent(), text_content)
-    message = message_from_string(messageText)
+    message = message_from_bytes(messageText)
 
     self.assertEqual('A Mail', decode_header(message['Subject'])[0][0])
     part = None
@@ -1236,7 +1241,7 @@ class TestCRMMailSend(BaseTestCRM):
     self.assertEqual('"Me," <me@erp5.org>', mfrom)
     self.assertEqual(['"Recipient," <recipient@example.com>'], mto)
 
-    message = message_from_string(messageText)
+    message = message_from_bytes(messageText)
     part = None
     for i in message.get_payload():
       if i.get_content_type()=='text/html':
@@ -1259,7 +1264,7 @@ class TestCRMMailSend(BaseTestCRM):
     self.assertEqual('=?utf-8?q?Me=2C_=F0=9F=90=88_fan?= <me@erp5.org>', mfrom)
     self.assertEqual(['=?utf-8?q?Recipient=2C_=F0=9F=90=88_fan?= <recipient@example.com>'], mto)
 
-    message = message_from_string(messageText)
+    message = message_from_bytes(messageText)
 
     self.assertEqual(u'Héhé', decode_header(message['Subject'])[0][0].decode('utf-8'))
     self.assertEqual(u'Me, 🐈 fan', decode_header(message['From'])[0][0].decode('utf-8'))
@@ -1301,7 +1306,7 @@ class TestCRMMailSend(BaseTestCRM):
     mail_text = event.send(download=True)
 
     # Check mail text.
-    message = message_from_string(mail_text)
+    message = message_from_bytes(mail_text)
     part = None
     for i in message.get_payload():
       if i.get_content_type()=='text/plain':
@@ -1348,7 +1353,7 @@ class TestCRMMailSend(BaseTestCRM):
     mail_text = event.send(download=True)
 
     # Check mail text.
-    message = message_from_string(mail_text)
+    message = message_from_bytes(mail_text)
     part = None
     for i in message.get_payload():
       if i.get_content_type()=='text/plain':
@@ -1393,7 +1398,7 @@ class TestCRMMailSend(BaseTestCRM):
 
     mail_text = event.send(download=True)
     # Check mail text.
-    message = message_from_string(mail_text)
+    message = message_from_bytes(mail_text)
     part = None
     for i in message.get_payload():
       if i.get_content_type()=='text/plain':
@@ -1440,7 +1445,7 @@ class TestCRMMailSend(BaseTestCRM):
     mail_text = event.send(download=True)
 
     # Check mail text.
-    message = message_from_string(mail_text)
+    message = message_from_bytes(mail_text)
     part = None
     for i in message.get_payload():
       if i.get_content_type()=='text/plain':
@@ -1487,7 +1492,7 @@ class TestCRMMailSend(BaseTestCRM):
     mail_text = event.send(download=True)
 
     # Check mail text.
-    message = message_from_string(mail_text)
+    message = message_from_bytes(mail_text)
     part = None
     for i in message.get_payload():
       if i.get_content_type()=='text/plain':
@@ -1538,7 +1543,7 @@ class TestCRMMailSend(BaseTestCRM):
     mail_text = event.send(download=True)
 
     # Check mail text.
-    message = message_from_string(mail_text)
+    message = message_from_bytes(mail_text)
     part = None
     for i in message.get_payload():
       if i.get_content_type()=='text/plain':
@@ -1586,7 +1591,7 @@ class TestCRMMailSend(BaseTestCRM):
 
     mail_text = event.send(download=True)
     # Check mail text.
-    message = message_from_string(mail_text)
+    message = message_from_bytes(mail_text)
     part = None
     for i in message.get_payload():
       if i.get_content_type()=='text/plain':
@@ -1717,7 +1722,7 @@ class TestCRMMailSend(BaseTestCRM):
     mail_text = event.send(download=True)
 
     # Check mail text.
-    message = message_from_string(mail_text)
+    message = message_from_bytes(mail_text)
     part = None
     for i in message.get_payload():
       if i.get_content_type()=='text/plain':
@@ -1775,7 +1780,7 @@ class TestCRMMailSend(BaseTestCRM):
     mail_text = event.send(download=True)
 
     # Check mail text.
-    message = message_from_string(mail_text)
+    message = message_from_bytes(mail_text)
     part = None
     for i in message.get_payload():
       if i.get_content_type()=='text/plain':
@@ -2035,8 +2040,8 @@ class TestCRMMailSend(BaseTestCRM):
 
     self.assertEqual(5, len(self.portal.MailHost._message_list))
     for message_info in self.portal.MailHost._message_list:
-      self.assertIn(mail_text_content, message_info[-1])
-      message = message_from_string(message_info[-1])
+      self.assertIn(mail_text_content, message_info[-1].decode())
+      message = message_from_bytes(message_info[-1])
       self.assertTrue(DateTime(message.get("Date")).isCurrentDay())
 
   def test_MailMessage_send_simple_case(self):
@@ -2053,7 +2058,7 @@ class TestCRMMailSend(BaseTestCRM):
                       attachment_list=[])
     self.tic()
     (from_url, to_url, last_message,), = self.portal.MailHost._message_list
-    self.assertIn("Body Simple Case", last_message)
+    self.assertIn(b"Body Simple Case", last_message)
     self.assertEqual('FG ER <eee@eee.com>', from_url)
     self.assertEqual(['Expert User <expert@in24.test>'], to_url)
 
@@ -2068,7 +2073,7 @@ class TestCRMMailSend(BaseTestCRM):
     mail_message.send(extra_header_dict={"X-test-header": "test"})
     self.tic()
     (_, _, last_message,), = self.portal.MailHost._message_list
-    message = message_from_string(last_message)
+    message = message_from_bytes(last_message)
     self.assertEqual("test", message.get("X-test-header"))
 
   def test_MailMessage_send_security(self):

bt5/erp5_forge/TestTemplateItem/portal_components/test.erp5.testBug.py

@@ -29,12 +29,16 @@
 ##############################################################################
 
 
+import six
 import unittest
 from DateTime import DateTime
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from Products.ERP5Type.tests.Sequence import SequenceList
 from Products.ERP5Type.tests.utils import DummyMailHost
-from email import message_from_string
+if six.PY2:
+  from email import message_from_string as message_from_bytes
+else:
+  from email import message_from_bytes
 from email.header import decode_header
 
 
@@ -218,7 +222,7 @@ class TestBug(ERP5TypeTestCase):
     mfrom, mto, messageText = last_message
     self.assertEqual('dummy <loggedperson@localhost>', mfrom)
     self.assertEqual(['person1@localhost'], mto)
-    message = message_from_string(messageText)
+    message = message_from_bytes(messageText)
     self.assertTrue(decode_header(message['Subject'])[0][0].endswith(bug.getTitle()))
 
   def stepCheckBugMessageNotification(self, sequence=None,
@@ -232,7 +236,7 @@ class TestBug(ERP5TypeTestCase):
     mfrom, mto, messageText = last_message
     self.assertEqual('person2@localhost', mfrom)
     self.assertEqual(['person1@localhost'], mto)
-    message = message_from_string(messageText)
+    message = message_from_bytes(messageText)
     self.assertTrue(decode_header(message['Subject'])[0][0].endswith(bug.getTitle()))
 
   def stepSetSourceProject(self, sequence=None, sequence_list=None, **kw):

product/CMFActivity/tests/testCMFActivity.py

@@ -1141,6 +1141,7 @@ class TestCMFActivity(ERP5TypeTestCase, LogInterceptor):
       self.flushAllActivities(silent=1, loop_size=100)
       # Check there is a traceback in the email notification
       sender, recipients, mail = message_list.pop()
+      mail = mail.decode()
       self.assertIn("Module %s, line %s, in failingMethod" % (
         __name__, inspect.getsourcelines(failingMethod)[1]), mail)
       self.assertIn("ValueError:", mail)

product/ERP5OOo/tests/testDeferredStyle.py

@@ -26,6 +26,7 @@
 #
 ##############################################################################
 
+import six
 import textwrap
 import unittest
 import textwrap
@@ -36,7 +37,11 @@ from AccessControl.SecurityManagement import newSecurityManager
 from Acquisition import aq_base
 from Products.ERP5OOo.tests.utils import Validator
 from lxml import html
-import email, six.moves.urllib.parse, six.moves.http_client
+if six.PY2:
+  from email import message_from_string as message_from_bytes
+else:
+  from email import message_from_bytes
+import six.moves.urllib.parse, six.moves.http_client
 from Products.Formulator.MethodField import Method
 
 
@@ -133,7 +138,7 @@ class TestDeferredStyleBase(DeferredStyleTestCase):
     self.assertNotEqual(last_message, ())
     mfrom, mto, message_text = last_message
     self.assertEqual('"%s" <%s>' % (self.first_name, self.recipient_email_address), mto[0])
-    mail_message = email.message_from_string(message_text)
+    mail_message = message_from_bytes(message_text)
     for part in mail_message.walk():
       content_type = part.get_content_type()
       file_name = part.get_filename()
@@ -198,7 +203,7 @@ class TestDeferredStyleBase(DeferredStyleTestCase):
     self.assertNotEqual(last_message, ())
     mfrom, mto, message_text = last_message
     self.assertEqual('"%s" <%s>' % (self.first_name, self.recipient_email_address), mto[0])
-    mail_message = email.message_from_string(message_text)
+    mail_message = message_from_bytes(message_text)
     for part in mail_message.walk():
       content_type = part.get_content_type()
       if content_type == "text/html":
@@ -236,7 +241,7 @@ class TestDeferredStyleBase(DeferredStyleTestCase):
     self.assertNotEqual(last_message, ())
     mfrom, mto, message_text = last_message
     self.assertEqual('"%s" <%s>' % (self.first_name, self.recipient_email_address), mto[0])
-    mail_message = email.message_from_string(message_text)
+    mail_message = message_from_bytes(message_text)
     for part in mail_message.walk():
       content_type = part.get_content_type()
       if content_type == "text/html":
@@ -260,7 +265,7 @@ class TestDeferredStyleBase(DeferredStyleTestCase):
     self.assertNotEqual(last_message, ())
     mfrom, mto, message_text = last_message
     self.assertEqual('"%s" <%s>' % (self.first_name, self.recipient_email_address), mto[0])
-    mail_message = email.message_from_string(message_text)
+    mail_message = message_from_bytes(message_text)
     for part in mail_message.walk():
       content_type = part.get_content_type()
       if content_type == "text/html":
@@ -285,7 +290,7 @@ class TestDeferredStyleBase(DeferredStyleTestCase):
     self.assertNotEqual(last_message, ())
     mfrom, mto, message_text = last_message
     self.assertEqual('"%s" <%s>' % (self.first_name, self.recipient_email_address), mto[0])
-    mail_message = email.message_from_string(message_text)
+    mail_message = message_from_bytes(message_text)
     for part in mail_message.walk():
       content_type = part.get_content_type()
       if content_type == self.content_type:
@@ -315,7 +320,7 @@ class TestDeferredStyleBase(DeferredStyleTestCase):
           'HTTP_ACCEPT_LANGUAGE': 'fr;q=0.9,en;q=0.8',
           })
     self.tic()
-    mail_message = email.message_from_string(self.portal.MailHost._last_message[2])
+    mail_message = message_from_bytes(self.portal.MailHost._last_message[2])
     # mail subject is translated
     self.assertEqual('Historique', mail_message['subject'])
     # content is translated
@@ -345,11 +350,11 @@ class TestDeferredStyleBase(DeferredStyleTestCase):
           'HTTP_COOKIE': 'LOCALIZER_LANGUAGE="fr"',
           })
     self.tic()
-    mail_message = email.message_from_string(self.portal.MailHost._last_message[2])
+    mail_message = message_from_bytes(self.portal.MailHost._last_message[2])
     # mail subject is translated
     self.assertEqual('Historique', mail_message['subject'])
     # content is translated
-    mail_message = email.message_from_string(self.portal.MailHost._last_message[2])
+    mail_message = message_from_bytes(self.portal.MailHost._last_message[2])
     part, = [x for x in mail_message.walk() if x.get_content_type() == self.content_type]
     self.assertIn(
         b'Historique',
@@ -450,7 +455,7 @@ class TestDeferredStyleBase(DeferredStyleTestCase):
 
     # inspect the report as text and check the selection was initialized from
     # request parameter.
-    mail_message = email.message_from_string(self.portal.MailHost._last_message[2])
+    mail_message = message_from_bytes(self.portal.MailHost._last_message[2])
     part, = [x for x in mail_message.walk() if x.get_content_type() == self.content_type]
 
     doc = self.portal.document_module.newContent(
@@ -487,7 +492,7 @@ class TestODSDeferredStyle(TestDeferredStyleBase):
     self.assertNotEqual(last_message, ())
     mfrom, mto, message_text = last_message
     self.assertEqual('"%s" <%s>' % (self.first_name, self.recipient_email_address), mto[0])
-    mail_message = email.message_from_string(message_text)
+    mail_message = message_from_bytes(message_text)
     for part in mail_message.walk():
       content_type = part.get_content_type()
       file_name = part.get_filename()

product/ERP5Type/tests/utils.py

@@ -49,7 +49,6 @@ from Zope2.Startup.datatypes import ZopeDatabase
 from Testing import ZopeTestCase
 import Products.ERP5Type
 from Products.MailHost.MailHost import MailHost
-from email import message_from_string
 from Products.ERP5Type.Globals import PersistentMapping
 from Products.ERP5Type.Utils import simple_decorator
 from Products.ZSQLCatalog.SQLCatalog import Catalog
@@ -59,9 +58,11 @@ import lxml.html
 
 if six.PY2:
   FileIO = file
+  from email import message_from_string as message_from_bytes
 else:
   from io import FileIO
   from importlib import reload
+  from email import message_from_bytes
 
 
 def canonical_html(html):
@@ -106,12 +107,15 @@ class DummyMailHostMixin(object):
 
   @staticmethod
   def _decodeMessage(messageText):
+    # type: (bytes) -> str
     """ Decode message"""
     message_text = messageText
-    for part in message_from_string(messageText).walk():
+    for part in message_from_bytes(messageText).walk():
       if part.get_content_type() in ['text/plain', 'text/html' ] \
                   and not part.is_multipart():
-        message_text = part.get_payload(decode=1)
+        message_text = part.get_payload(decode=True)
+        if six.PY3:
+          message_text = message_text.decode(part.get_content_charset('ascii'))
     return message_text
 
   security.declarePrivate('getMessageList')