Merge #11605: don't use set/get_payload in feedparser; they do conversions.

3cfbd3cc · R David Murray · 73c60d0d · a181d8d9 · 3cfbd3cc · 3cfbd3cc
Commit 3cfbd3cc authored Apr 06, 2011 by R David Murray
Hide whitespace changes
Inline Side-by-side

Showing with 52 additions and 2 deletions

Lib/email/feedparser.py Lib/email/feedparser.py +2 -2

Lib/test/test_email/test_email.py Lib/test/test_email/test_email.py +47 -0

Misc/NEWS Misc/NEWS +3 -0

No files found.
--- a/Lib/email/feedparser.py
+++ b/Lib/email/feedparser.py
@@ -368,12 +368,12 @@ class FeedParser:
                                end = len(mo.group(0))
                                self._last.epilogue = epilogue[:-end]
                    else:
-                        payload = self._last.get_payload()
+                        payload = self._last._payload
                        if isinstance(payload, str):
                            mo = NLCRE_eol.search(payload)
                            if mo:
                                payload = payload[:-len(mo.group(0))]
-                                self._last.set_payload(payload)
+                                self._last._payload = payload
                    self._input.pop_eof_matcher()
                    self._pop_message()
                    # Set the multipart up for newline cleansing, which will

--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -3143,6 +3143,53 @@ class Test8BitBytesHandling(unittest.TestCase):
        g = email.generator.BytesGenerator(s)
        g.flatten(msg, linesep='\r\n')
        self.assertEqual(s.getvalue(), text)
+
+    def test_8bit_multipart(self):
+        # Issue 11605
+        source = textwrap.dedent("""\
+            Date: Fri, 18 Mar 2011 17:15:43 +0100
+            To: foo@example.com
+            From: foodwatch-Newsletter <bar@example.com>
+            Subject: Aktuelles zu Japan, Klonfleisch und Smiley-System
+            Message-ID: <76a486bee62b0d200f33dc2ca08220ad@localhost.localdomain>
+            MIME-Version: 1.0
+            Content-Type: multipart/alternative;
+                    boundary="b1_76a486bee62b0d200f33dc2ca08220ad"
+
+            --b1_76a486bee62b0d200f33dc2ca08220ad
+            Content-Type: text/plain; charset="utf-8"
+            Content-Transfer-Encoding: 8bit
+
+            Guten Tag, ,
+
+            mit großer Betroffenheit verfolgen auch wir im foodwatch-Team die
+            Nachrichten aus Japan.
+
+
+            --b1_76a486bee62b0d200f33dc2ca08220ad
+            Content-Type: text/html; charset="utf-8"
+            Content-Transfer-Encoding: 8bit
+
+            <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+                "http://www.w3.org/TR/html4/loose.dtd">
+            <html lang="de">
+            <head>
+                    <title>foodwatch - Newsletter</title>
+            </head>
+            <body>
+              <p>mit gro&szlig;er Betroffenheit verfolgen auch wir im foodwatch-Team
+                 die Nachrichten aus Japan.</p>
+            </body>
+            </html>
+            --b1_76a486bee62b0d200f33dc2ca08220ad--
+
+            """).encode('utf-8')
+        msg = email.message_from_bytes(source)
+        s = BytesIO()
+        g = email.generator.BytesGenerator(s)
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), source)
+
    maxDiff = None



--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -94,6 +94,9 @@ Core and Builtins
 Library
 -------

+- Issue #11605: email.parser.BytesFeedParser was incorrectly converting multipart
+  subpararts with an 8bit CTE into unicode instead of preserving the bytes.
+
 - Issue #10963: Ensure that subprocess.communicate() never raises EPIPE.

 - Issue #10791: Implement missing method GzipFile.read1(), allowing GzipFile