Fixes for SF #1076485, which I'll apply to the CVS head too. The problem was

caused by a self._input.readline() call that wasn't checking for the
NeedsMoreData marker.

msg_43.txt contains a message that illustrates the problem, when
email.message_from_*() is called.  That interface uses the Parser API, which
splits reads into 8192 byte chunks.  It so happens that for the test message,
the 8192 chunk falls inside a message/delivery-status, which is where in the
FeedParser the readline() call was that didn't check for NeedsMoreData.

I also added an assert to unreadline() so it'll be more evident if an attempt
to push back NeedsMoreData ever happens again.

Bump the email package version number.

Lib/email/FeedParser.py

@@ -87,6 +87,7 @@ class BufferedSubFile(object):
 
     def unreadline(self, line):
         # Let the consumer push a line back into the buffer.
+        assert line is not NeedMoreData
         self._lines.append(line)
 
     def push(self, data):
@@ -242,8 +243,18 @@ class FeedParser:
                 # EOF.  We want to see if we're at the end of this subpart, so
                 # first consume the blank line, then test the next line to see
                 # if we're at this subpart's EOF.
+                while True:
                     line = self._input.readline()
+                    if line is NeedMoreData:
+                        yield NeedMoreData
+                        continue
+                    break
+                while True:
                     line = self._input.readline()
+                    if line is NeedMoreData:
+                        yield NeedMoreData
+                        continue
+                    break
                 if line == '':
                     break
                 # Not at EOF so this is a line we're going to need.

Lib/email/__init__.py

@@ -4,7 +4,7 @@
 
 """A package for parsing, handling, and generating email messages."""
 
-__version__ = '3.0'
+__version__ = '3.0+'
 
 __all__ = [
     'base64MIME',

Lib/email/test/data/msg_43.txt

+From SRS0=aO/p=ON=bag.python.org=None@bounce2.pobox.com  Fri Nov 26 21:40:36 2004
+X-VM-v5-Data: ([nil nil nil nil nil nil nil nil nil]
+	[nil nil nil nil nil nil nil "MAILER DAEMON <>" "MAILER DAEMON <>" nil nil "Banned file: auto__mail.python.bat in mail from you" "^From:" nil nil nil nil "Banned file: auto__mail.python.bat in mail from you" nil nil nil nil nil nil nil]
+	nil)
+MIME-Version: 1.0
+Message-Id: <edab.7804f5cb8070@python.org>
+Content-Type: multipart/report; report-type=delivery-status;
+    charset=utf-8;
+    boundary="----------=_1101526904-1956-5"
+X-Virus-Scanned: by XS4ALL Virus Scanner
+X-UIDL: 4\G!!!<c"!UV["!M7C!!
+From: MAILER DAEMON <>
+To: <webmaster@python.org>
+Subject: Banned file: auto__mail.python.bat in mail from you
+Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+This is a multi-part message in MIME format...
+
+------------=_1101526904-1956-5
+Content-Type: text/plain; charset="utf-8"
+Content-Disposition: inline
+Content-Transfer-Encoding: 7bit
+
+BANNED FILENAME ALERT
+
+Your message to: xxxxxxx@dot.ca.gov, xxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxx@dot.ca.gov, xxxxxx@dot.ca.gov, xxxxxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxx@dot.ca.gov, xxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxxxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxx@dot.ca.gov, xxxxxxxx@dot.ca.gov, xxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxx@dot.ca.gov, xxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxxxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxxx@dot.ca.gov, xxxx@dot.ca.gov, xxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxxxxxxxx@dot.ca.gov
+was blocked by our Spam Firewall. The email you sent with the following subject has NOT BEEN DELIVERED:
+
+Subject: Delivery_failure_notice
+
+An attachment in that mail was of a file type that the Spam Firewall is set to block.
+
+
+
+------------=_1101526904-1956-5
+Content-Type: message/delivery-status
+Content-Disposition: inline
+Content-Transfer-Encoding: 7bit
+Content-Description: Delivery error report
+
+Reporting-MTA: dns; sacspam01.dot.ca.gov
+Received-From-MTA: smtp; sacspam01.dot.ca.gov ([127.0.0.1])
+Arrival-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+Final-Recipient: rfc822; xxxxxxx@dot.ca.gov
+Action: failed
+Status: 5.7.1
+Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat
+Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST)
+
+------------=_1101526904-1956-5
+Content-Type: text/rfc822-headers
+Content-Disposition: inline
+Content-Transfer-Encoding: 7bit
+Content-Description: Undelivered-message headers
+
+Received: from kgsav.org (ppp-70-242-162-63.dsl.spfdmo.swbell.net [70.242.162.63])
+	by sacspam01.dot.ca.gov (Spam Firewall) with SMTP
+	id A232AD03DE3A; Fri, 26 Nov 2004 19:41:35 -0800 (PST)
+From: webmaster@python.org
+To: xxxxx@dot.ca.gov
+Date: Sat, 27 Nov 2004 03:35:30 UTC
+Subject: Delivery_failure_notice
+Importance: Normal
+X-Priority: 3 (Normal)
+X-MSMail-Priority: Normal
+Message-ID: <edab.7804f5cb8070@python.org>
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="====67bd2b7a5.f99f7"
+Content-Transfer-Encoding: 7bit
+
+------------=_1101526904-1956-5--
+

Lib/email/test/test_email.py

@@ -1973,7 +1973,7 @@ class TestIdempotent(TestEmailBase):
 
 
 # Test various other bits of the package's functionality
-class TestMiscellaneous(unittest.TestCase):
+class TestMiscellaneous(TestEmailBase):
     def test_message_from_string(self):
         fp = openfile('msg_01.txt')
         try:
@@ -2222,6 +2222,48 @@ class TestMiscellaneous(unittest.TestCase):
         uc = Charset('US-ASCII')
         self.assertEqual(lc.get_body_encoding(), uc.get_body_encoding())
 
+    def test_partial_falls_inside_message_delivery_status(self):
+        eq = self.ndiffAssertEqual
+        # The Parser interface provides chunks of data to FeedParser in 8192
+        # byte gulps.  SF bug #1076485 found one of those chunks inside
+        # message/delivery-status header block, which triggered an
+        # unreadline() of NeedMoreData.
+        msg = self._msgobj('msg_43.txt')
+        sfp = StringIO()
+        Iterators._structure(msg, sfp)
+        eq(sfp.getvalue(), """\
+multipart/report
+    text/plain
+    message/delivery-status
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+        text/plain
+    text/rfc822-headers
+""")
+
 
 
 # Test the iterator/generators