merge heads

Doc/library/smtplib.rst

@@ -323,21 +323,32 @@ An :class:`SMTP` instance has the following methods:
    .. versionchanged:: 3.2 *msg* may be a byte string.
 
 
-.. method:: SMTP.send_message(msg, from_addr=None, to_addrs=None, mail_options=[], rcpt_options=[])
+.. method:: SMTP.send_message(msg, from_addr=None, to_addrs=None, \
+                              mail_options=[], rcpt_options=[])
 
    This is a convenience method for calling :meth:`sendmail` with the message
    represented by an :class:`email.message.Message` object.  The arguments have
    the same meaning as for :meth:`sendmail`, except that *msg* is a ``Message``
    object.
 
-   If *from_addr* is ``None``, ``send_message`` sets its value to the value of
-   the :mailheader:`From` header from *msg*.  If *to_addrs* is ``None``,
-   ``send_message`` combines the values (if any) of the :mailheader:`To`,
-   :mailheader:`CC`, and :mailheader:`Bcc` fields from *msg*.  Regardless of
-   the values of *from_addr* and *to_addrs*, ``send_message`` deletes any  Bcc
-   field from *msg*.  It then serializes *msg* using
+   If *from_addr* is ``None`` or *to_addrs* is ``None``, ``send_message`` fills
+   those arguments with addresses extracted from the headers of *msg* as
+   specified in :rfc:`2822`\: *from_addr* is set to the :mailheader:`Sender`
+   field if it is present, and otherwise to the :mailheader:`From` field.
+   *to_adresses* combines the values (if any) of the :mailheader:`To`,
+   :mailheader:`Cc`, and :mailheader:`Bcc` fields from *msg*.  If exactly one
+   set of :mailheader:`Resent-*` headers appear in the message, the regular
+   headers are ignored and the :mailheader:`Resent-*` headers are used instead.
+   If the message contains more than one set of :mailheader:`Resent-*` headers,
+   a :exc:`ValueError` is raised, since there is no way to unambiguously detect
+   the most recent set of :mailheader:`Resent-` headers.
+
+   ``send_message`` serializes *msg* using
    :class:`~email.generator.BytesGenerator` with ``\r\n`` as the *linesep*, and
-   calls :meth:`sendmail` to transmit the resulting message.
+   calls :meth:`sendmail` to transmit the resulting message.  Regardless of the
+   values of *from_addr* and *to_addrs*, ``send_message`` does not transmit any
+   :mailheader:`Bcc` or :mailheader:`Resent-Bcc` headers that may appear
+   in *msg*.
 
    .. versionadded:: 3.2
 

Doc/library/sqlite3.rst

@@ -599,43 +599,43 @@ Row Objects
 
 Let's assume we initialize a table as in the example given above::
 
-    conn = sqlite3.connect(":memory:")
-    c = conn.cursor()
-    c.execute('''create table stocks
-    (date text, trans text, symbol text,
-     qty real, price real)''')
-    c.execute("""insert into stocks
-              values ('2006-01-05','BUY','RHAT',100,35.14)""")
-    conn.commit()
-    c.close()
+   conn = sqlite3.connect(":memory:")
+   c = conn.cursor()
+   c.execute('''create table stocks
+   (date text, trans text, symbol text,
+    qty real, price real)''')
+   c.execute("""insert into stocks
+             values ('2006-01-05','BUY','RHAT',100,35.14)""")
+   conn.commit()
+   c.close()
 
 Now we plug :class:`Row` in::
 
-    >>> conn.row_factory = sqlite3.Row
-    >>> c = conn.cursor()
-    >>> c.execute('select * from stocks')
-    <sqlite3.Cursor object at 0x7f4e7dd8fa80>
-    >>> r = c.fetchone()
-    >>> type(r)
-    <class 'sqlite3.Row'>
-    >>> tuple(r)
-    ('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14)
-    >>> len(r)
-    5
-    >>> r[2]
-    'RHAT'
-    >>> r.keys()
-    ['date', 'trans', 'symbol', 'qty', 'price']
-    >>> r['qty']
-    100.0
-    >>> for member in r:
-    ...     print(member)
-    ...
-    2006-01-05
-    BUY
-    RHAT
-    100.0
-    35.14
+   >>> conn.row_factory = sqlite3.Row
+   >>> c = conn.cursor()
+   >>> c.execute('select * from stocks')
+   <sqlite3.Cursor object at 0x7f4e7dd8fa80>
+   >>> r = c.fetchone()
+   >>> type(r)
+   <class 'sqlite3.Row'>
+   >>> tuple(r)
+   ('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14)
+   >>> len(r)
+   5
+   >>> r[2]
+   'RHAT'
+   >>> r.keys()
+   ['date', 'trans', 'symbol', 'qty', 'price']
+   >>> r['qty']
+   100.0
+   >>> for member in r:
+   ...     print(member)
+   ...
+   2006-01-05
+   BUY
+   RHAT
+   100.0
+   35.14
 
 
 .. _sqlite3-types:
@@ -886,6 +886,7 @@ only makes sense to call from a different thread.
 .. rubric:: Footnotes
 
 .. [#f1] The sqlite3 module is not built with loadable extension support by
-  default, because some platforms (notably Mac OS X) have SQLite libraries which
-  are compiled without this feature. To get loadable extension support, you must
-  pass --enable-loadable-sqlite-extensions to configure.
+   default, because some platforms (notably Mac OS X) have SQLite
+   libraries which are compiled without this feature. To get loadable
+   extension support, you must pass --enable-loadable-sqlite-extensions to
+   configure.

Lib/importlib/test/source/test_file_loader.py

@@ -214,7 +214,7 @@ class BadBytecodeTest(unittest.TestCase):
                                                 lambda bc: bc[:8] + b'<test>',
                                                 del_source=del_source)
             file_path = mapping['_temp'] if not del_source else bytecode_path
-            with self.assertRaises(ValueError):
+            with self.assertRaises(EOFError):
                 self.import_(file_path, '_temp')
 
     def _test_bad_magic(self, test, *, del_source=False):

Lib/smtplib.py

@@ -49,6 +49,7 @@ import email.message
 import email.generator
 import base64
 import hmac
+import copy
 from email.base64mime import body_encode as encode_base64
 from sys import stderr
 
@@ -674,7 +675,7 @@ class SMTP:
 
         msg may be a string containing characters in the ASCII range, or a byte
         string.  A string is encoded to bytes using the ascii codec, and lone
-        \r and \n characters are converted to \r\n characters.
+        \\r and \\n characters are converted to \\r\\n characters.
 
         If there has been no previous EHLO or HELO command this session, this
         method tries ESMTP EHLO first.  If the server does ESMTP, message size
@@ -757,24 +758,49 @@ class SMTP:
         """Converts message to a bytestring and passes it to sendmail.
 
         The arguments are as for sendmail, except that msg is an
-        email.message.Message object.  If from_addr is None, the from_addr is
-        taken from the 'From' header of the Message.  If to_addrs is None, its
-        value is composed from the addresses listed in the 'To', 'CC', and
-        'Bcc' fields.  Regardless of the values of from_addr and to_addr, any
-        Bcc field in the Message object is deleted.  The Message object is then
-        serialized using email.generator.BytesGenerator and sendmail is called
-        to transmit the message.
+        email.message.Message object.  If from_addr is None or to_addrs is
+        None, these arguments are taken from the headers of the Message as
+        described in RFC 2822 (a ValueError is raised if there is more than
+        one set of 'Resent-' headers).  Regardless of the values of from_addr and
+        to_addr, any Bcc field (or Resent-Bcc field, when the Message is a
+        resent) of the Message object won't be transmitted.  The Message
+        object is then serialized using email.generator.BytesGenerator and
+        sendmail is called to transmit the message.
+
         """
+        # 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822
+        # Section 3.6.6). In such a case, we use the 'Resent-*' fields.  However,
+        # if there is more than one 'Resent-' block there's no way to
+        # unambiguously determine which one is the most recent in all cases,
+        # so rather than guess we raise a ValueError in that case.
+        #
+        # TODO implement heuristics to guess the correct Resent-* block with an
+        # option allowing the user to enable the heuristics.  (It should be
+        # possible to guess correctly almost all of the time.)
+        resent =msg.get_all('Resent-Date')
+        if resent is None:
+            header_prefix = ''
+        elif len(resent) == 1:
+            header_prefix = 'Resent-'
+        else:
+            raise ValueError("message has more than one 'Resent-' header block")
         if from_addr is None:
-            from_addr = msg['From']
+            # Prefer the sender field per RFC 2822:3.6.2.
+            from_addr = (msg[header_prefix+'Sender']
+                           if (header_prefix+'Sender') in msg
+                           else msg[header_prefix+'From'])
         if to_addrs is None:
-            addr_fields = [f for f in (msg['To'], msg['Bcc'], msg['CC'])
-                            if f is not None]
+            addr_fields = [f for f in (msg[header_prefix+'To'],
+                                       msg[header_prefix+'Bcc'],
+                                       msg[header_prefix+'Cc']) if f is not None]
             to_addrs = [a[1] for a in email.utils.getaddresses(addr_fields)]
-        del msg['Bcc']
+        # Make a local copy so we can delete the bcc headers.
+        msg_copy = copy.copy(msg)
+        del msg_copy['Bcc']
+        del msg_copy['Resent-Bcc']
         with io.BytesIO() as bytesmsg:
             g = email.generator.BytesGenerator(bytesmsg)
-            g.flatten(msg, linesep='\r\n')
+            g.flatten(msg_copy, linesep='\r\n')
             flatmsg = bytesmsg.getvalue()
         return self.sendmail(from_addr, to_addrs, flatmsg, mail_options,
                              rcpt_options)

Lib/test/test_marshal.py

@@ -211,6 +211,30 @@ class BugsTestCase(unittest.TestCase):
         invalid_string = b'l\x02\x00\x00\x00\x00\x00\x00\x00'
         self.assertRaises(ValueError, marshal.loads, invalid_string)
 
+    def test_multiple_dumps_and_loads(self):
+        # Issue 12291: marshal.load() should be callable multiple times
+        # with interleaved data written by non-marshal code
+        # Adapted from a patch by Engelbert Gruber.
+        data = (1, 'abc', b'def', 1.0, (2, 'a', ['b', b'c']))
+        for interleaved in (b'', b'0123'):
+            ilen = len(interleaved)
+            positions = []
+            try:
+                with open(support.TESTFN, 'wb') as f:
+                    for d in data:
+                        marshal.dump(d, f)
+                        if ilen:
+                            f.write(interleaved)
+                        positions.append(f.tell())
+                with open(support.TESTFN, 'rb') as f:
+                    for i, d in enumerate(data):
+                        self.assertEqual(d, marshal.load(f))
+                        if ilen:
+                            f.read(ilen)
+                        self.assertEqual(positions[i], f.tell())
+            finally:
+                support.unlink(support.TESTFN)
+
 
 def test_main():
     support.run_unittest(IntTestCase,

Lib/test/test_smtplib.py

@@ -320,13 +320,16 @@ class DebuggingServerTests(unittest.TestCase):
         # XXX (see comment in testSend)
         time.sleep(0.01)
         smtp.quit()
+        # make sure the Bcc header is still in the message.
+        self.assertEqual(m['Bcc'], 'John Root <root@localhost>, "Dinsdale" '
+                                    '<warped@silly.walks.com>')
 
         self.client_evt.set()
         self.serv_evt.wait()
         self.output.flush()
         # Add the X-Peer header that DebuggingServer adds
         m['X-Peer'] = socket.gethostbyname('localhost')
-        # The Bcc header is deleted before serialization.
+        # The Bcc header should not be transmitted.
         del m['Bcc']
         mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END)
         self.assertEqual(self.output.getvalue(), mexpect)
@@ -365,6 +368,112 @@ class DebuggingServerTests(unittest.TestCase):
                                  re.MULTILINE)
             self.assertRegex(debugout, to_addr)
 
+    def testSendMessageWithSpecifiedAddresses(self):
+        # Make sure addresses specified in call override those in message.
+        m = email.mime.text.MIMEText('A test message')
+        m['From'] = 'foo@bar.com'
+        m['To'] = 'John, Dinsdale'
+        smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
+        smtp.send_message(m, from_addr='joe@example.com', to_addrs='foo@example.net')
+        # XXX (see comment in testSend)
+        time.sleep(0.01)
+        smtp.quit()
+
+        self.client_evt.set()
+        self.serv_evt.wait()
+        self.output.flush()
+        # Add the X-Peer header that DebuggingServer adds
+        m['X-Peer'] = socket.gethostbyname('localhost')
+        mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END)
+        self.assertEqual(self.output.getvalue(), mexpect)
+        debugout = smtpd.DEBUGSTREAM.getvalue()
+        sender = re.compile("^sender: joe@example.com$", re.MULTILINE)
+        self.assertRegex(debugout, sender)
+        for addr in ('John', 'Dinsdale'):
+            to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr),
+                                 re.MULTILINE)
+            self.assertNotRegex(debugout, to_addr)
+        recip = re.compile(r"^recips: .*'foo@example.net'.*$", re.MULTILINE)
+        self.assertRegex(debugout, recip)
+
+    def testSendMessageWithMultipleFrom(self):
+        # Sender overrides To
+        m = email.mime.text.MIMEText('A test message')
+        m['From'] = 'Bernard, Bianca'
+        m['Sender'] = 'the_rescuers@Rescue-Aid-Society.com'
+        m['To'] = 'John, Dinsdale'
+        smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
+        smtp.send_message(m)
+        # XXX (see comment in testSend)
+        time.sleep(0.01)
+        smtp.quit()
+
+        self.client_evt.set()
+        self.serv_evt.wait()
+        self.output.flush()
+        # Add the X-Peer header that DebuggingServer adds
+        m['X-Peer'] = socket.gethostbyname('localhost')
+        mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END)
+        self.assertEqual(self.output.getvalue(), mexpect)
+        debugout = smtpd.DEBUGSTREAM.getvalue()
+        sender = re.compile("^sender: the_rescuers@Rescue-Aid-Society.com$", re.MULTILINE)
+        self.assertRegex(debugout, sender)
+        for addr in ('John', 'Dinsdale'):
+            to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr),
+                                 re.MULTILINE)
+            self.assertRegex(debugout, to_addr)
+
+    def testSendMessageResent(self):
+        m = email.mime.text.MIMEText('A test message')
+        m['From'] = 'foo@bar.com'
+        m['To'] = 'John'
+        m['CC'] = 'Sally, Fred'
+        m['Bcc'] = 'John Root <root@localhost>, "Dinsdale" <warped@silly.walks.com>'
+        m['Resent-Date'] = 'Thu, 1 Jan 1970 17:42:00 +0000'
+        m['Resent-From'] = 'holy@grail.net'
+        m['Resent-To'] = 'Martha <my_mom@great.cooker.com>, Jeff'
+        m['Resent-Bcc'] = 'doe@losthope.net'
+        smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
+        smtp.send_message(m)
+        # XXX (see comment in testSend)
+        time.sleep(0.01)
+        smtp.quit()
+
+        self.client_evt.set()
+        self.serv_evt.wait()
+        self.output.flush()
+        # The Resent-Bcc headers are deleted before serialization.
+        del m['Bcc']
+        del m['Resent-Bcc']
+        # Add the X-Peer header that DebuggingServer adds
+        m['X-Peer'] = socket.gethostbyname('localhost')
+        mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END)
+        self.assertEqual(self.output.getvalue(), mexpect)
+        debugout = smtpd.DEBUGSTREAM.getvalue()
+        sender = re.compile("^sender: holy@grail.net$", re.MULTILINE)
+        self.assertRegex(debugout, sender)
+        for addr in ('my_mom@great.cooker.com', 'Jeff', 'doe@losthope.net'):
+            to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr),
+                                 re.MULTILINE)
+            self.assertRegex(debugout, to_addr)
+
+    def testSendMessageMultipleResentRaises(self):
+        m = email.mime.text.MIMEText('A test message')
+        m['From'] = 'foo@bar.com'
+        m['To'] = 'John'
+        m['CC'] = 'Sally, Fred'
+        m['Bcc'] = 'John Root <root@localhost>, "Dinsdale" <warped@silly.walks.com>'
+        m['Resent-Date'] = 'Thu, 1 Jan 1970 17:42:00 +0000'
+        m['Resent-From'] = 'holy@grail.net'
+        m['Resent-To'] = 'Martha <my_mom@great.cooker.com>, Jeff'
+        m['Resent-Bcc'] = 'doe@losthope.net'
+        m['Resent-Date'] = 'Thu, 2 Jan 1970 17:42:00 +0000'
+        m['Resent-To'] = 'holy@grail.net'
+        m['Resent-From'] = 'Martha <my_mom@great.cooker.com>, Jeff'
+        smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
+        with self.assertRaises(ValueError):
+            smtp.send_message(m)
+        smtp.close()
 
 class NonConnectingTests(unittest.TestCase):
 

Makefile.pre.in

@@ -274,7 +274,7 @@ AST_ASDL=	$(srcdir)/Parser/Python.asdl
 
 ASDLGEN_FILES=	$(srcdir)/Parser/asdl.py $(srcdir)/Parser/asdl_c.py
 # XXX Note that a build now requires Python exist before the build starts
-ASDLGEN=	$(srcdir)/Parser/asdl_c.py
+ASDLGEN=	@DISABLE_ASDLGEN@ $(srcdir)/Parser/asdl_c.py
 
 ##########################################################################
 # Python

Misc/ACKS

@@ -265,6 +265,7 @@ Michael Ernst
 Ben Escoto
 Andy Eskilsson
 Stefan Esser
+Nicolas Estibals
 Stephen D Evans
 Carey Evans
 Tim Everett

Misc/NEWS

@@ -27,6 +27,9 @@ What's New in Python 3.2.1 release candidate 2?
 Core and Builtins
 -----------------
 
+- Issue #12291: You can now load multiple marshalled objects from a stream,
+  with other data interleaved between marshalled objects.
+
 - Issue #12084: os.stat on Windows now works properly with relative symbolic
   links when called from any directory.
 
@@ -42,6 +45,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #12147: Adjust the new-in-3.2 smtplib.send_message method for better
+  conformance to the RFCs:  correctly handle Sender and Resent- headers.
+
 - Issue #12352: Fix a deadlock in multiprocessing.Heap when a block is freed by
   the garbage collector while the Heap lock is held.
 

configure

@@ -644,6 +644,8 @@ LN
 INSTALL_DATA
 INSTALL_SCRIPT
 INSTALL_PROGRAM
+HAS_PYTHON
+DISABLE_ASDLGEN
 HAS_HG
 HGBRANCH
 HGTAG
@@ -5204,6 +5206,9 @@ fi
 
 
 
+
+if test -e $srcdir/.hg/00changelog.i
+then
 # Extract the first word of "hg", so it can be a program name with args.
 set dummy hg; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
@@ -5242,6 +5247,9 @@ $as_echo "no" >&6; }
 fi
 
 
+else
+HAS_HG=no-repository
+fi
 if test $HAS_HG = found
 then
     HGVERSION="hg id -i \$(srcdir)"
@@ -5253,6 +5261,52 @@ else
     HGBRANCH=""
 fi
 
+
+DISABLE_ASDLGEN=""
+# Extract the first word of "python", so it can be a program name with args.
+set dummy python; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_HAS_PYTHON+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  if test -n "$HAS_PYTHON"; then
+  ac_cv_prog_HAS_PYTHON="$HAS_PYTHON" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+    ac_cv_prog_HAS_PYTHON="found"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  test -z "$ac_cv_prog_HAS_PYTHON" && ac_cv_prog_HAS_PYTHON="not-found"
+fi
+fi
+HAS_PYTHON=$ac_cv_prog_HAS_PYTHON
+if test -n "$HAS_PYTHON"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_PYTHON" >&5
+$as_echo "$HAS_PYTHON" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+if test $HAS_HG != found -o $HAS_PYTHON != found
+then
+    DISABLE_ASDLGEN="@echo hg: $HAS_HG, python: $HAS_PYTHON! cannot run \$(srcdir)/Parser/asdl_c.py #"
+fi
+
+
 case $MACHDEP in
 bsdos*|hp*|HP*)
 	# install -d does not work on BSDI or HP-UX

configure.in

@@ -832,7 +832,13 @@ fi
 AC_SUBST(HGVERSION)
 AC_SUBST(HGTAG)
 AC_SUBST(HGBRANCH)
+
+if test -e $srcdir/.hg/00changelog.i
+then
 AC_CHECK_PROG(HAS_HG, hg, found, not-found)
+else
+HAS_HG=no-repository
+fi
 if test $HAS_HG = found
 then
     HGVERSION="hg id -i \$(srcdir)"
@@ -844,6 +850,15 @@ else
     HGBRANCH=""
 fi
 
+AC_SUBST(DISABLE_ASDLGEN)
+DISABLE_ASDLGEN=""
+AC_CHECK_PROG(HAS_PYTHON, python, found, not-found)
+if test $HAS_HG != found -o $HAS_PYTHON != found
+then
+    DISABLE_ASDLGEN="@echo hg: $HAS_HG, python: $HAS_PYTHON! cannot run \$(srcdir)/Parser/asdl_c.py #"
+fi
+
+
 case $MACHDEP in
 bsdos*|hp*|HP*)
 	# install -d does not work on BSDI or HP-UX