#12586: add provisional email policy with new header parsing and folding.

When the new policies are used (and only when the new policies are explicitly
used) headers turn into objects that have attributes based on their parsed
values, and can be set using objects that encapsulate the values, as well as
set directly from unicode strings.  The folding algorithm then takes care of
encoding unicode where needed, and folding according to the highest level
syntactic objects.

With this patch only date and time headers are parsed as anything other than
unstructured, but that is all the helper methods in the existing API handle.
I do plan to add more parsers, and complete the set specified in the RFC
before the package becomes stable.

Lib/email/_encoded_words.py

+""" Routines for manipulating RFC2047 encoded words.
+
+This is currently a package-private API, but will be considered for promotion
+to a public API if there is demand.
+
+"""
+
+# An ecoded word looks like this:
+#
+#        =?charset[*lang]?cte?encoded_string?=
+#
+# for more information about charset see the charset module.  Here it is one
+# of the preferred MIME charset names (hopefully; you never know when parsing).
+# cte (Content Transfer Encoding) is either 'q' or 'b' (ignoring case).  In
+# theory other letters could be used for other encodings, but in practice this
+# (almost?) never happens.  There could be a public API for adding entries
+# to to the CTE tables, but YAGNI for now.  'q' is Quoted Printable, 'b' is
+# Base64.  The meaning of encoded_string should be obvious.  'lang' is optional
+# as indicated by the brackets (they are not part of the syntax) but is almost
+# never encountered in practice.
+#
+# The general interface for a CTE decoder is that it takes the encoded_string
+# as its argument, and returns a tuple (cte_decoded_string, defects).  The
+# cte_decoded_string is the original binary that was encoded using the
+# specified cte.  'defects' is a list of MessageDefect instances indicating any
+# problems encountered during conversion.  'charset' and 'lang' are the
+# corresponding strings extracted from the EW, case preserved.
+#
+# The general interface for a CTE encoder is that it takes a binary sequence
+# as input and returns the cte_encoded_string, which is an ascii-only string.
+#
+# Each decoder must also supply a length function that takes the binary
+# sequence as its argument and returns the length of the resulting encoded
+# string.
+#
+# The main API functions for the module are decode, which calls the decoder
+# referenced by the cte specifier, and encode, which adds the appropriate
+# RFC 2047 "chrome" to the encoded string, and can optionally automatically
+# select the shortest possible encoding.  See their docstrings below for
+# details.
+
+import re
+import base64
+import binascii
+import functools
+from string import ascii_letters, digits
+from email import errors
+
+#
+# Quoted Printable
+#
+
+# regex based decoder.
+_q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub,
+        lambda m: bytes([int(m.group(1), 16)]))
+
+def decode_q(encoded):
+    encoded = encoded.replace(b'_', b' ')
+    return _q_byte_subber(encoded), []
+
+
+# dict mapping bytes to their encoded form
+class QByteMap(dict):
+
+    safe = b'-!*+/' + ascii_letters.encode('ascii') + digits.encode('ascii')
+
+    def __missing__(self, key):
+        if key in self.safe:
+            self[key] = chr(key)
+        else:
+            self[key] = "={:02X}".format(key)
+        return self[key]
+
+_q_byte_map = QByteMap()
+
+# In headers spaces are mapped to '_'.
+_q_byte_map[ord(' ')] = '_'
+
+def encode_q(bstring):
+    return ''.join(_q_byte_map[x] for x in bstring)
+
+def len_q(bstring):
+    return sum(len(_q_byte_map[x]) for x in bstring)
+
+
+#
+# Base64
+#
+
+def decode_b(encoded):
+    defects = []
+    pad_err = len(encoded) % 4
+    if pad_err:
+        defects.append(errors.InvalidBase64PaddingDefect())
+        padded_encoded = encoded + b'==='[:4-pad_err]
+    else:
+        padded_encoded = encoded
+    try:
+        return base64.b64decode(padded_encoded, validate=True), defects
+    except binascii.Error:
+        # Since we had correct padding, this must an invalid char error.
+        defects = [errors.InvalidBase64CharactersDefect()]
+        # The non-alphabet characters are ignored as far as padding
+        # goes, but we don't know how many there are.  So we'll just
+        # try various padding lengths until something works.
+        for i in 0, 1, 2, 3:
+            try:
+                return base64.b64decode(encoded+b'='*i, validate=False), defects
+            except binascii.Error:
+                if i==0:
+                    defects.append(errors.InvalidBase64PaddingDefect())
+        else:
+            # This should never happen.
+            raise AssertionError("unexpected binascii.Error")
+
+def encode_b(bstring):
+    return base64.b64encode(bstring).decode('ascii')
+
+def len_b(bstring):
+    groups_of_3, leftover = divmod(len(bstring), 3)
+    # 4 bytes out for each 3 bytes (or nonzero fraction thereof) in.
+    return groups_of_3 * 4 + (4 if leftover else 0)
+
+
+_cte_decoders = {
+    'q': decode_q,
+    'b': decode_b,
+    }
+
+def decode(ew):
+    """Decode encoded word and return (string, charset, lang, defects) tuple.
+
+    An RFC 2047/2243 encoded word has the form:
+
+        =?charset*lang?cte?encoded_string?=
+
+    where '*lang' may be omitted but the other parts may not be.
+
+    This function expects exactly such a string (that is, it does not check the
+    syntax and may raise errors if the string is not well formed), and returns
+    the encoded_string decoded first from its Content Transfer Encoding and
+    then from the resulting bytes into unicode using the specified charset.  If
+    the cte-decoded string does not successfully decode using the specified
+    character set, a defect is added to the defects list and the unknown octets
+    are replaced by the unicode 'unknown' character \uFDFF.
+
+    The specified charset and language are returned.  The default for language,
+    which is rarely if ever encountered, is the empty string.
+
+    """
+    _, charset, cte, cte_string, _ = ew.split('?')
+    charset, _, lang = charset.partition('*')
+    cte = cte.lower()
+    # Recover the original bytes and do CTE decoding.
+    bstring = cte_string.encode('ascii', 'surrogateescape')
+    bstring, defects = _cte_decoders[cte](bstring)
+    # Turn the CTE decoded bytes into unicode.
+    try:
+        string = bstring.decode(charset)
+    except UnicodeError:
+        defects.append(errors.UndecodableBytesDefect("Encoded word "
+            "contains bytes not decodable using {} charset".format(charset)))
+        string = bstring.decode(charset, 'surrogateescape')
+    except LookupError:
+        string = bstring.decode('ascii', 'surrogateescape')
+        if charset.lower() != 'unknown-8bit':
+            defects.append(errors.CharsetError("Unknown charset {} "
+                "in encoded word; decoded as unknown bytes".format(charset)))
+    return string, charset, lang, defects
+
+
+_cte_encoders = {
+    'q': encode_q,
+    'b': encode_b,
+    }
+
+_cte_encode_length = {
+    'q': len_q,
+    'b': len_b,
+    }
+
+def encode(string, charset='utf-8', encoding=None, lang=''):
+    """Encode string using the CTE encoding that produces the shorter result.
+
+    Produces an RFC 2047/2243 encoded word of the form:
+
+        =?charset*lang?cte?encoded_string?=
+
+    where '*lang' is omitted unless the 'lang' parameter is given a value.
+    Optional argument charset (defaults to utf-8) specifies the charset to use
+    to encode the string to binary before CTE encoding it.  Optional argument
+    'encoding' is the cte specifier for the encoding that should be used ('q'
+    or 'b'); if it is None (the default) the encoding which produces the
+    shortest encoded sequence is used, except that 'q' is preferred if it is up
+    to five characters longer.  Optional argument 'lang' (default '') gives the
+    RFC 2243 language string to specify in the encoded word.
+
+    """
+    if charset == 'unknown-8bit':
+        bstring = string.encode('ascii', 'surrogateescape')
+    else:
+        bstring = string.encode(charset)
+    if encoding is None:
+        qlen = _cte_encode_length['q'](bstring)
+        blen = _cte_encode_length['b'](bstring)
+        # Bias toward q.  5 is arbitrary.
+        encoding = 'q' if qlen - blen < 5 else 'b'
+    encoded = _cte_encoders[encoding](bstring)
+    if lang:
+        lang = '*' + lang
+    return "=?{}{}?{}?{}?=".format(charset, lang, encoding, encoded)

Lib/email/_policybase.py

@@ -64,10 +64,16 @@ class _PolicyBase:
         except for the changes passed in as keyword arguments.
 
         """
+        newpolicy = self.__class__.__new__(self.__class__)
         for attr, value in self.__dict__.items():
-            if attr not in kw:
-                kw[attr] = value
-        return self.__class__(**kw)
+            object.__setattr__(newpolicy, attr, value)
+        for attr, value in kw.items():
+            if not hasattr(self, attr):
+                raise TypeError(
+                    "{!r} is an invalid keyword argument for {}".format(
+                        attr, self.__class__.__name__))
+            object.__setattr__(newpolicy, attr, value)
+        return newpolicy
 
     def __setattr__(self, name, value):
         if hasattr(self, name):

Lib/email/errors.py

@@ -5,7 +5,6 @@
 """email package exception classes."""
 
 
-
 class MessageError(Exception):
     """Base class for errors in the email package."""
 
@@ -30,9 +29,8 @@ class CharsetError(MessageError):
     """An illegal charset was given."""
 
 
-
 # These are parsing defects which the parser was able to work around.
-class MessageDefect(Exception):
+class MessageDefect(ValueError):
     """Base class for a message defect."""
 
     def __init__(self, line=None):
@@ -58,3 +56,42 @@ class MultipartInvariantViolationDefect(MessageDefect):
 
 class InvalidMultipartContentTransferEncodingDefect(MessageDefect):
     """An invalid content transfer encoding was set on the multipart itself."""
+
+class UndecodableBytesDefect(MessageDefect):
+    """Header contained bytes that could not be decoded"""
+
+class InvalidBase64PaddingDefect(MessageDefect):
+    """base64 encoded sequence had an incorrect length"""
+
+class InvalidBase64CharactersDefect(MessageDefect):
+    """base64 encoded sequence had characters not in base64 alphabet"""
+
+# These errors are specific to header parsing.
+
+class HeaderDefect(MessageDefect):
+    """Base class for a header defect."""
+
+class InvalidHeaderDefect(HeaderDefect):
+    """Header is not valid, message gives details."""
+
+class HeaderMissingRequiredValue(HeaderDefect):
+    """A header that must have a value had none"""
+
+class NonPrintableDefect(HeaderDefect):
+    """ASCII characters outside the ascii-printable range found"""
+
+    def __init__(self, non_printables):
+        super().__init__(non_printables)
+        self.non_printables = non_printables
+
+    def __str__(self):
+        return ("the following ASCII non-printables found in header: "
+            "{}".format(self.non_printables))
+
+class ObsoleteHeaderDefect(HeaderDefect):
+    """Header uses syntax declared obsolete by RFC 5322"""
+
+class NonASCIILocalPartDefect(HeaderDefect):
+    """local_part contains non-ASCII characters"""
+    # This defect only occurs during unicode parsing, not when
+    # parsing messages decoded from binary.

Lib/email/generator.py

@@ -95,9 +95,15 @@ class Generator:
         self._encoded_NL = self._encode(self._NL)
         self._EMPTY = ''
         self._encoded_EMTPY = self._encode('')
-        p = self.policy
+        # Because we use clone (below) when we recursively process message
+        # subparts, and because clone uses the computed policy (not None),
+        # submessages will automatically get set to the computed policy when
+        # they are processed by this code.
+        old_gen_policy = self.policy
+        old_msg_policy = msg.policy
         try:
             self.policy = policy
+            msg.policy = policy
             if unixfrom:
                 ufrom = msg.get_unixfrom()
                 if not ufrom:
@@ -105,7 +111,8 @@ class Generator:
                 self.write(ufrom + self._NL)
             self._write(msg)
         finally:
-            self.policy = p
+            self.policy = old_gen_policy
+            msg.policy = old_msg_policy
 
     def clone(self, fp):
         """Clone this generator with the exact same options."""

Lib/email/policy.py

@@ -2,11 +2,178 @@
 code that adds all the email6 features.
 """
 
-from email._policybase import Policy, compat32, Compat32
+from email._policybase import Policy, Compat32, compat32
+from email.utils import _has_surrogates
+from email._headerregistry import HeaderRegistry as _HeaderRegistry
 
-# XXX: temporarily derive everything from compat32.
+__all__ = [
+    'Compat32',
+    'compat32',
+    'Policy',
+    'EmailPolicy',
+    'default',
+    'strict',
+    'SMTP',
+    'HTTP',
+    ]
 
-default = compat32
+class EmailPolicy(Policy):
+
+    """+
+    PROVISIONAL
+
+    The API extensions enabled by this this policy are currently provisional.
+    Refer to the documentation for details.
+
+    This policy adds new header parsing and folding algorithms.  Instead of
+    simple strings, headers are custom objects with custom attributes
+    depending on the type of the field.  The folding algorithm fully
+    implements RFCs 2047 and 5322.
+
+    In addition to the settable attributes listed above that apply to
+    all Policies, this policy adds the following additional attributes:
+
+    refold_source       -- if the value for a header in the Message object
+                           came from the parsing of some source, this attribute
+                           indicates whether or not a generator should refold
+                           that value when transforming the message back into
+                           stream form.  The possible values are:
+
+                           none  -- all source values use original folding
+                           long  -- source values that have any line that is
+                                    longer than max_line_length will be
+                                    refolded
+                           all  -- all values are refolded.
+
+                           The default is 'long'.
+
+    header_factory      -- a callable that takes two arguments, 'name' and
+                           'value', where 'name' is a header field name and
+                           'value' is an unfolded header field value, and
+                           returns a string-like object that represents that
+                           header.  A default header_factory is provided that
+                           understands some of the RFC5322 header field types.
+                           (Currently address fields and date fields have
+                           special treatment, while all other fields are
+                           treated as unstructured.  This list will be
+                           completed before the extension is marked stable.)
+    """
+
+    refold_source = 'long'
+    header_factory = _HeaderRegistry()
+
+    def __init__(self, **kw):
+        # Ensure that each new instance gets a unique header factory
+        # (as opposed to clones, which share the factory).
+        if 'header_factory' not in kw:
+            object.__setattr__(self, 'header_factory', _HeaderRegistry())
+        super().__init__(**kw)
+
+    # The logic of the next three methods is chosen such that it is possible to
+    # switch a Message object between a Compat32 policy and a policy derived
+    # from this class and have the results stay consistent.  This allows a
+    # Message object constructed with this policy to be passed to a library
+    # that only handles Compat32 objects, or to receive such an object and
+    # convert it to use the newer style by just changing its policy.  It is
+    # also chosen because it postpones the relatively expensive full rfc5322
+    # parse until as late as possible when parsing from source, since in many
+    # applications only a few headers will actually be inspected.
+
+    def header_source_parse(self, sourcelines):
+        """+
+        The name is parsed as everything up to the ':' and returned unmodified.
+        The value is determined by stripping leading whitespace off the
+        remainder of the first line, joining all subsequent lines together, and
+        stripping any trailing carriage return or linefeed characters.  (This
+        is the same as Compat32).
+
+        """
+        name, value = sourcelines[0].split(':', 1)
+        value = value.lstrip(' \t') + ''.join(sourcelines[1:])
+        return (name, value.rstrip('\r\n'))
+
+    def header_store_parse(self, name, value):
+        """+
+        The name is returned unchanged.  If the input value has a 'name'
+        attribute and it matches the name ignoring case, the value is returned
+        unchanged.  Otherwise the name and value are passed to header_factory
+        method, and the resulting custom header object is returned as the
+        value.  In this case a ValueError is raised if the input value contains
+        CR or LF characters.
+
+        """
+        if hasattr(value, 'name') and value.name.lower() == name.lower():
+            return (name, value)
+        if len(value.splitlines())>1:
+            raise ValueError("Header values may not contain linefeed "
+                             "or carriage return characters")
+        return (name, self.header_factory(name, value))
+
+    def header_fetch_parse(self, name, value):
+        """+
+        If the value has a 'name' attribute, it is returned to unmodified.
+        Otherwise the name and the value with any linesep characters removed
+        are passed to the header_factory method, and the resulting custom
+        header object is returned.  Any surrogateescaped bytes get turned
+        into the unicode unknown-character glyph.
+
+        """
+        if hasattr(value, 'name'):
+            return value
+        return self.header_factory(name, ''.join(value.splitlines()))
+
+    def fold(self, name, value):
+        """+
+        Header folding is controlled by the refold_source policy setting.  A
+        value is considered to be a 'source value' if and only if it does not
+        have a 'name' attribute (having a 'name' attribute means it is a header
+        object of some sort).  If a source value needs to be refolded according
+        to the policy, it is converted into a custom header object by passing
+        the name and the value with any linesep characters removed to the
+        header_factory method.  Folding of a custom header object is done by
+        calling its fold method with the current policy.
+
+        Source values are split into lines using splitlines.  If the value is
+        not to be refolded, the lines are rejoined using the linesep from the
+        policy and returned.  The exception is lines containing non-ascii
+        binary data.  In that case the value is refolded regardless of the
+        refold_source setting, which causes the binary data to be CTE encoded
+        using the unknown-8bit charset.
+
+        """
+        return self._fold(name, value, refold_binary=True)
+
+    def fold_binary(self, name, value):
+        """+
+        The same as fold if cte_type is 7bit, except that the returned value is
+        bytes.
+
+        If cte_type is 8bit, non-ASCII binary data is converted back into
+        bytes.  Headers with binary data are not refolded, regardless of the
+        refold_header setting, since there is no way to know whether the binary
+        data consists of single byte characters or multibyte characters.
+
+        """
+        folded = self._fold(name, value, refold_binary=self.cte_type=='7bit')
+        return folded.encode('ascii', 'surrogateescape')
+
+    def _fold(self, name, value, refold_binary=False):
+        if hasattr(value, 'name'):
+            return value.fold(policy=self)
+        maxlen = self.max_line_length if self.max_line_length else float('inf')
+        lines = value.splitlines()
+        refold = (self.refold_source == 'all' or
+                  self.refold_source == 'long' and
+                    (len(lines[0])+len(name)+2 > maxlen or
+                     any(len(x) > maxlen for x in lines[1:])))
+        if refold or refold_binary and _has_surrogates(value):
+            return self.header_factory(name, ''.join(lines)).fold(policy=self)
+        return name + ': ' + self.linesep.join(lines) + self.linesep
+
+
+default = EmailPolicy()
+# Make the default policy use the class default header_factory
+del default.header_factory
 strict = default.clone(raise_on_defect=True)
 SMTP = default.clone(linesep='\r\n')
 HTTP = default.clone(linesep='\r\n', max_line_length=None)

Lib/email/utils.py

@@ -62,6 +62,13 @@ escapesre = re.compile(r'[\\"]')
 _has_surrogates = re.compile(
     '([^\ud800-\udbff]|\A)[\udc00-\udfff]([^\udc00-\udfff]|\Z)').search
 
+# How to deal with a string containing bytes before handing it to the
+# application through the 'normal' interface.
+def _sanitize(string):
+    # Turn any escaped bytes into unicode 'unknown' char.
+    original_bytes = string.encode('ascii', 'surrogateescape')
+    return original_bytes.decode('ascii', 'replace')
+
 
 # Helpers
 

Lib/test/test_email/__init__.py

@@ -65,3 +65,9 @@ class TestEmailBase(unittest.TestCase):
     def assertBytesEqual(self, first, second, msg):
         """Our byte strings are really encoded strings; improve diff output"""
         self.assertEqual(self._bytes_repr(first), self._bytes_repr(second))
+
+    def assertDefectsEqual(self, actual, expected):
+        self.assertEqual(len(actual), len(expected), actual)
+        for i in range(len(actual)):
+            self.assertIsInstance(actual[i], expected[i],
+                                    'item {}'.format(i))

Lib/test/test_email/test__encoded_words.py

+import unittest
+from email import _encoded_words as _ew
+from email import errors
+from test.test_email import TestEmailBase
+
+
+class TestDecodeQ(TestEmailBase):
+
+    def _test(self, source, ex_result, ex_defects=[]):
+        result, defects = _ew.decode_q(source)
+        self.assertEqual(result, ex_result)
+        self.assertDefectsEqual(defects, ex_defects)
+
+    def test_no_encoded(self):
+        self._test(b'foobar', b'foobar')
+
+    def test_spaces(self):
+        self._test(b'foo=20bar=20', b'foo bar ')
+        self._test(b'foo_bar_', b'foo bar ')
+
+    def test_run_of_encoded(self):
+        self._test(b'foo=20=20=21=2Cbar', b'foo  !,bar')
+
+
+class TestDecodeB(TestEmailBase):
+
+    def _test(self, source, ex_result, ex_defects=[]):
+        result, defects = _ew.decode_b(source)
+        self.assertEqual(result, ex_result)
+        self.assertDefectsEqual(defects, ex_defects)
+
+    def test_simple(self):
+        self._test(b'Zm9v', b'foo')
+
+    def test_missing_padding(self):
+        self._test(b'dmk', b'vi', [errors.InvalidBase64PaddingDefect])
+
+    def test_invalid_character(self):
+        self._test(b'dm\x01k===', b'vi', [errors.InvalidBase64CharactersDefect])
+
+    def test_invalid_character_and_bad_padding(self):
+        self._test(b'dm\x01k', b'vi', [errors.InvalidBase64CharactersDefect,
+                                       errors.InvalidBase64PaddingDefect])
+
+
+class TestDecode(TestEmailBase):
+
+    def test_wrong_format_input_raises(self):
+        with self.assertRaises(ValueError):
+            _ew.decode('=?badone?=')
+        with self.assertRaises(ValueError):
+            _ew.decode('=?')
+        with self.assertRaises(ValueError):
+            _ew.decode('')
+
+    def _test(self, source, result, charset='us-ascii', lang='', defects=[]):
+        res, char, l, d = _ew.decode(source)
+        self.assertEqual(res, result)
+        self.assertEqual(char, charset)
+        self.assertEqual(l, lang)
+        self.assertDefectsEqual(d, defects)
+
+    def test_simple_q(self):
+        self._test('=?us-ascii?q?foo?=', 'foo')
+
+    def test_simple_b(self):
+        self._test('=?us-ascii?b?dmk=?=', 'vi')
+
+    def test_q_case_ignored(self):
+        self._test('=?us-ascii?Q?foo?=', 'foo')
+
+    def test_b_case_ignored(self):
+        self._test('=?us-ascii?B?dmk=?=', 'vi')
+
+    def test_non_trivial_q(self):
+        self._test('=?latin-1?q?=20F=fcr=20Elise=20?=', ' Für Elise ', 'latin-1')
+
+    def test_q_escpaed_bytes_preserved(self):
+        self._test(b'=?us-ascii?q?=20\xACfoo?='.decode('us-ascii',
+                                                       'surrogateescape'),
+                   ' \uDCACfoo',
+                   defects = [errors.UndecodableBytesDefect])
+
+    def test_b_undecodable_bytes_ignored_with_defect(self):
+        self._test(b'=?us-ascii?b?dm\xACk?='.decode('us-ascii',
+                                                   'surrogateescape'),
+                   'vi',
+                   defects = [
+                    errors.InvalidBase64CharactersDefect,
+                    errors.InvalidBase64PaddingDefect])
+
+    def test_b_invalid_bytes_ignored_with_defect(self):
+        self._test('=?us-ascii?b?dm\x01k===?=',
+                   'vi',
+                   defects = [errors.InvalidBase64CharactersDefect])
+
+    def test_b_invalid_bytes_incorrect_padding(self):
+        self._test('=?us-ascii?b?dm\x01k?=',
+                   'vi',
+                   defects = [
+                    errors.InvalidBase64CharactersDefect,
+                    errors.InvalidBase64PaddingDefect])
+
+    def test_b_padding_defect(self):
+        self._test('=?us-ascii?b?dmk?=',
+                   'vi',
+                    defects = [errors.InvalidBase64PaddingDefect])
+
+    def test_nonnull_lang(self):
+        self._test('=?us-ascii*jive?q?test?=', 'test', lang='jive')
+
+    def test_unknown_8bit_charset(self):
+        self._test('=?unknown-8bit?q?foo=ACbar?=',
+                   b'foo\xacbar'.decode('ascii', 'surrogateescape'),
+                   charset = 'unknown-8bit',
+                   defects = [])
+
+    def test_unknown_charset(self):
+        self._test('=?foobar?q?foo=ACbar?=',
+                   b'foo\xacbar'.decode('ascii', 'surrogateescape'),
+                   charset = 'foobar',
+                   # XXX Should this be a new Defect instead?
+                   defects = [errors.CharsetError])
+
+
+class TestEncodeQ(TestEmailBase):
+
+    def _test(self, src, expected):
+        self.assertEqual(_ew.encode_q(src), expected)
+
+    def test_all_safe(self):
+        self._test(b'foobar', 'foobar')
+
+    def test_spaces(self):
+        self._test(b'foo bar ', 'foo_bar_')
+
+    def test_run_of_encodables(self):
+        self._test(b'foo  ,,bar', 'foo__=2C=2Cbar')
+
+
+class TestEncodeB(TestEmailBase):
+
+    def test_simple(self):
+        self.assertEqual(_ew.encode_b(b'foo'), 'Zm9v')
+
+    def test_padding(self):
+        self.assertEqual(_ew.encode_b(b'vi'), 'dmk=')
+
+
+class TestEncode(TestEmailBase):
+
+    def test_q(self):
+        self.assertEqual(_ew.encode('foo', 'utf-8', 'q'), '=?utf-8?q?foo?=')
+
+    def test_b(self):
+        self.assertEqual(_ew.encode('foo', 'utf-8', 'b'), '=?utf-8?b?Zm9v?=')
+
+    def test_auto_q(self):
+        self.assertEqual(_ew.encode('foo', 'utf-8'), '=?utf-8?q?foo?=')
+
+    def test_auto_q_if_short_mostly_safe(self):
+        self.assertEqual(_ew.encode('vi.', 'utf-8'), '=?utf-8?q?vi=2E?=')
+
+    def test_auto_b_if_enough_unsafe(self):
+        self.assertEqual(_ew.encode('.....', 'utf-8'), '=?utf-8?b?Li4uLi4=?=')
+
+    def test_auto_b_if_long_unsafe(self):
+        self.assertEqual(_ew.encode('vi.vi.vi.vi.vi.', 'utf-8'),
+                         '=?utf-8?b?dmkudmkudmkudmkudmku?=')
+
+    def test_auto_q_if_long_mostly_safe(self):
+        self.assertEqual(_ew.encode('vi vi vi.vi ', 'utf-8'),
+                         '=?utf-8?q?vi_vi_vi=2Evi_?=')
+
+    def test_utf8_default(self):
+        self.assertEqual(_ew.encode('foo'), '=?utf-8?q?foo?=')
+
+    def test_lang(self):
+        self.assertEqual(_ew.encode('foo', lang='jive'), '=?utf-8*jive?q?foo?=')
+
+    def test_unknown_8bit(self):
+        self.assertEqual(_ew.encode('foo\uDCACbar', charset='unknown-8bit'),
+                         '=?unknown-8bit?q?foo=ACbar?=')
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_email/test_generator.py

@@ -6,14 +6,16 @@ from email.generator import Generator, BytesGenerator
 from email import policy
 from test.test_email import TestEmailBase
 
-# XXX: move generator tests from test_email into here at some point.
 
+class TestGeneratorBase:
 
-class TestGeneratorBase():
+    policy = policy.default
 
-    policy = policy.compat32
+    def msgmaker(self, msg, policy=None):
+        policy = self.policy if policy is None else policy
+        return self.msgfunc(msg, policy=policy)
 
-    long_subject = {
+    refold_long_expected = {
         0: textwrap.dedent("""\
             To: whom_it_may_concern@example.com
             From: nobody_you_want_to_know@example.com
@@ -23,33 +25,32 @@ class TestGeneratorBase():
 
             None
             """),
+        # From is wrapped because wrapped it fits in 40.
         40: textwrap.dedent("""\
             To: whom_it_may_concern@example.com
-            From:\x20
+            From:
              nobody_you_want_to_know@example.com
             Subject: We the willing led by the
-             unknowing are doing the
-             impossible for the ungrateful. We have
-             done so much for so long with so little
-             we are now qualified to do anything
-             with nothing.
+             unknowing are doing the impossible for
+             the ungrateful. We have done so much
+             for so long with so little we are now
+             qualified to do anything with nothing.
 
             None
             """),
+        # Neither to nor from fit even if put on a new line,
+        # so we leave them sticking out on the first line.
         20: textwrap.dedent("""\
-            To:\x20
-             whom_it_may_concern@example.com
-            From:\x20
-             nobody_you_want_to_know@example.com
+            To: whom_it_may_concern@example.com
+            From: nobody_you_want_to_know@example.com
             Subject: We the
              willing led by the
              unknowing are doing
-             the
-             impossible for the
-             ungrateful. We have
-             done so much for so
-             long with so little
-             we are now
+             the impossible for
+             the ungrateful. We
+             have done so much
+             for so long with so
+             little we are now
              qualified to do
              anything with
              nothing.
@@ -57,65 +58,90 @@ class TestGeneratorBase():
             None
             """),
         }
-    long_subject[100] = long_subject[0]
-
-    def maxheaderlen_parameter_test(self, n):
-        msg = self.msgmaker(self.typ(self.long_subject[0]))
+    refold_long_expected[100] = refold_long_expected[0]
+
+    refold_all_expected = refold_long_expected.copy()
+    refold_all_expected[0] = (
+            "To: whom_it_may_concern@example.com\n"
+            "From: nobody_you_want_to_know@example.com\n"
+            "Subject: We the willing led by the unknowing are doing the "
+              "impossible for the ungrateful. We have done so much for "
+              "so long with so little we are now qualified to do anything "
+              "with nothing.\n"
+              "\n"
+              "None\n")
+    refold_all_expected[100] = (
+            "To: whom_it_may_concern@example.com\n"
+            "From: nobody_you_want_to_know@example.com\n"
+            "Subject: We the willing led by the unknowing are doing the "
+                "impossible for the ungrateful. We have\n"
+              " done so much for so long with so little we are now qualified "
+                "to do anything with nothing.\n"
+              "\n"
+              "None\n")
+
+    def _test_maxheaderlen_parameter(self, n):
+        msg = self.msgmaker(self.typ(self.refold_long_expected[0]))
         s = self.ioclass()
-        g = self.genclass(s, maxheaderlen=n)
+        g = self.genclass(s, maxheaderlen=n, policy=self.policy)
         g.flatten(msg)
-        self.assertEqual(s.getvalue(), self.typ(self.long_subject[n]))
+        self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[n]))
 
-    def test_maxheaderlen_parameter_0(self):
-        self.maxheaderlen_parameter_test(0)
+    for n in refold_long_expected:
+        locals()['test_maxheaderlen_parameter_' + str(n)] = (
+            lambda self, n=n:
+                self._test_maxheaderlen_parameter(n))
 
-    def test_maxheaderlen_parameter_100(self):
-        self.maxheaderlen_parameter_test(100)
-
-    def test_maxheaderlen_parameter_40(self):
-        self.maxheaderlen_parameter_test(40)
-
-    def test_maxheaderlen_parameter_20(self):
-        self.maxheaderlen_parameter_test(20)
-
-    def maxheaderlen_policy_test(self, n):
-        msg = self.msgmaker(self.typ(self.long_subject[0]))
+    def _test_max_line_length_policy(self, n):
+        msg = self.msgmaker(self.typ(self.refold_long_expected[0]))
         s = self.ioclass()
-        g = self.genclass(s, policy=policy.default.clone(max_line_length=n))
+        g = self.genclass(s, policy=self.policy.clone(max_line_length=n))
         g.flatten(msg)
-        self.assertEqual(s.getvalue(), self.typ(self.long_subject[n]))
-
-    def test_maxheaderlen_policy_0(self):
-        self.maxheaderlen_policy_test(0)
-
-    def test_maxheaderlen_policy_100(self):
-        self.maxheaderlen_policy_test(100)
-
-    def test_maxheaderlen_policy_40(self):
-        self.maxheaderlen_policy_test(40)
+        self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[n]))
 
-    def test_maxheaderlen_policy_20(self):
-        self.maxheaderlen_policy_test(20)
+    for n in refold_long_expected:
+        locals()['test_max_line_length_policy' + str(n)] = (
+            lambda self, n=n:
+                self._test_max_line_length_policy(n))
 
-    def maxheaderlen_parm_overrides_policy_test(self, n):
-        msg = self.msgmaker(self.typ(self.long_subject[0]))
+    def _test_maxheaderlen_parm_overrides_policy(self, n):
+        msg = self.msgmaker(self.typ(self.refold_long_expected[0]))
         s = self.ioclass()
         g = self.genclass(s, maxheaderlen=n,
-                          policy=policy.default.clone(max_line_length=10))
+                          policy=self.policy.clone(max_line_length=10))
         g.flatten(msg)
-        self.assertEqual(s.getvalue(), self.typ(self.long_subject[n]))
+        self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[n]))
 
-    def test_maxheaderlen_parm_overrides_policy_0(self):
-        self.maxheaderlen_parm_overrides_policy_test(0)
+    for n in refold_long_expected:
+        locals()['test_maxheaderlen_parm_overrides_policy' + str(n)] = (
+            lambda self, n=n:
+                self._test_maxheaderlen_parm_overrides_policy(n))
 
-    def test_maxheaderlen_parm_overrides_policy_100(self):
-        self.maxheaderlen_parm_overrides_policy_test(100)
+    def _test_refold_none_does_not_fold(self, n):
+        msg = self.msgmaker(self.typ(self.refold_long_expected[0]))
+        s = self.ioclass()
+        g = self.genclass(s, policy=self.policy.clone(refold_source='none',
+                                                      max_line_length=n))
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[0]))
+
+    for n in refold_long_expected:
+        locals()['test_refold_none_does_not_fold' + str(n)] = (
+            lambda self, n=n:
+                self._test_refold_none_does_not_fold(n))
 
-    def test_maxheaderlen_parm_overrides_policy_40(self):
-        self.maxheaderlen_parm_overrides_policy_test(40)
+    def _test_refold_all(self, n):
+        msg = self.msgmaker(self.typ(self.refold_long_expected[0]))
+        s = self.ioclass()
+        g = self.genclass(s, policy=self.policy.clone(refold_source='all',
+                                                      max_line_length=n))
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), self.typ(self.refold_all_expected[n]))
 
-    def test_maxheaderlen_parm_overrides_policy_20(self):
-        self.maxheaderlen_parm_overrides_policy_test(20)
+    for n in refold_long_expected:
+        locals()['test_refold_all' + str(n)] = (
+            lambda self, n=n:
+                self._test_refold_all(n))
 
     def test_crlf_control_via_policy(self):
         source = "Subject: test\r\n\r\ntest body\r\n"
@@ -138,30 +164,24 @@ class TestGeneratorBase():
 
 class TestGenerator(TestGeneratorBase, TestEmailBase):
 
+    msgfunc = staticmethod(message_from_string)
     genclass = Generator
     ioclass = io.StringIO
     typ = str
 
-    def msgmaker(self, msg, policy=None):
-        policy = self.policy if policy is None else policy
-        return message_from_string(msg, policy=policy)
-
 
 class TestBytesGenerator(TestGeneratorBase, TestEmailBase):
 
+    msgfunc = staticmethod(message_from_bytes)
     genclass = BytesGenerator
     ioclass = io.BytesIO
     typ = lambda self, x: x.encode('ascii')
 
-    def msgmaker(self, msg, policy=None):
-        policy = self.policy if policy is None else policy
-        return message_from_bytes(msg, policy=policy)
-
     def test_cte_type_7bit_handles_unknown_8bit(self):
         source = ("Subject: Maintenant je vous présente mon "
                  "collègue\n\n").encode('utf-8')
-        expected = ('Subject: =?unknown-8bit?q?Maintenant_je_vous_pr=C3=A9sente_mon_'
-                    'coll=C3=A8gue?=\n\n').encode('ascii')
+        expected = ('Subject: Maintenant je vous =?unknown-8bit?q?'
+                    'pr=C3=A9sente_mon_coll=C3=A8gue?=\n\n').encode('ascii')
         msg = message_from_bytes(source)
         s = io.BytesIO()
         g = BytesGenerator(s, policy=self.policy.clone(cte_type='7bit'))

Lib/test/test_email/test_pickleable.py

+import unittest
+import textwrap
+import copy
+import pickle
+from email import policy
+from email import message_from_string
+from email._headerregistry import HeaderRegistry
+from test.test_email import TestEmailBase
+
+class TestPickleCopyHeader(TestEmailBase):
+
+    unstructured = HeaderRegistry()('subject', 'this is a test')
+
+    def test_deepcopy_unstructured(self):
+        h = copy.deepcopy(self.unstructured)
+        self.assertEqual(str(h), str(self.unstructured))
+
+    def test_pickle_unstructured(self):
+        p = pickle.dumps(self.unstructured)
+        h = pickle.loads(p)
+        self.assertEqual(str(h), str(self.unstructured))
+
+    address = HeaderRegistry()('from', 'frodo@mordor.net')
+
+    def test_deepcopy_address(self):
+        h = copy.deepcopy(self.address)
+        self.assertEqual(str(h), str(self.address))
+
+    def test_pickle_address(self):
+        p = pickle.dumps(self.address)
+        h = pickle.loads(p)
+        self.assertEqual(str(h), str(self.address))
+
+
+class TestPickleCopyMessage(TestEmailBase):
+
+    testmsg = message_from_string(textwrap.dedent("""\
+            From: frodo@mordor.net
+            To: bilbo@underhill.org
+            Subject: help
+
+            I think I forgot the ring.
+            """), policy=policy.default)
+
+    def test_deepcopy(self):
+        msg2 = copy.deepcopy(self.testmsg)
+        self.assertEqual(msg2.as_string(), self.testmsg.as_string())
+
+    def test_pickle(self):
+        p = pickle.dumps(self.testmsg)
+        msg2 = pickle.loads(p)
+        self.assertEqual(msg2.as_string(), self.testmsg.as_string())
+
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_email/test_policy.py

@@ -5,49 +5,70 @@ import unittest
 import email.policy
 import email.parser
 import email.generator
+from email import _headerregistry
+
+def make_defaults(base_defaults, differences):
+    defaults = base_defaults.copy()
+    defaults.update(differences)
+    return defaults
 
 class PolicyAPITests(unittest.TestCase):
 
     longMessage = True
 
-    # These default values are the ones set on email.policy.default.
-    # If any of these defaults change, the docs must be updated.
-    policy_defaults = {
+    # Base default values.
+    compat32_defaults = {
         'max_line_length':          78,
         'linesep':                  '\n',
         'cte_type':                 '8bit',
         'raise_on_defect':          False,
         }
-
-    # For each policy under test, we give here the values of the attributes
-    # that are different from the defaults for that policy.
+    # These default values are the ones set on email.policy.default.
+    # If any of these defaults change, the docs must be updated.
+    policy_defaults = compat32_defaults.copy()
+    policy_defaults.update({
+        'raise_on_defect':          False,
+        'header_factory':           email.policy.EmailPolicy.header_factory,
+        'refold_source':            'long',
+        })
+
+    # For each policy under test, we give here what we expect the defaults to
+    # be for that policy.  The second argument to make defaults is the
+    # difference between the base defaults and that for the particular policy.
+    new_policy = email.policy.EmailPolicy()
     policies = {
-        email.policy.Compat32(): {},
-        email.policy.compat32: {},
-        email.policy.default: {},
-        email.policy.SMTP: {'linesep': '\r\n'},
-        email.policy.HTTP: {'linesep': '\r\n', 'max_line_length': None},
-        email.policy.strict: {'raise_on_defect': True},
+        email.policy.compat32: make_defaults(compat32_defaults, {}),
+        email.policy.default: make_defaults(policy_defaults, {}),
+        email.policy.SMTP: make_defaults(policy_defaults,
+                                         {'linesep': '\r\n'}),
+        email.policy.HTTP: make_defaults(policy_defaults,
+                                         {'linesep': '\r\n',
+                                          'max_line_length': None}),
+        email.policy.strict: make_defaults(policy_defaults,
+                                           {'raise_on_defect': True}),
+        new_policy: make_defaults(policy_defaults, {}),
         }
+    # Creating a new policy creates a new header factory.  There is a test
+    # later that proves this.
+    policies[new_policy]['header_factory'] = new_policy.header_factory
 
     def test_defaults(self):
-        for policy, changed_defaults in self.policies.items():
-            expected = self.policy_defaults.copy()
-            expected.update(changed_defaults)
+        for policy, expected in self.policies.items():
             for attr, value in expected.items():
                 self.assertEqual(getattr(policy, attr), value,
                                 ("change {} docs/docstrings if defaults have "
                                 "changed").format(policy))
 
     def test_all_attributes_covered(self):
-        for attr in dir(email.policy.default):
-            if (attr.startswith('_') or
-               isinstance(getattr(email.policy.Policy, attr),
-                          types.FunctionType)):
-                continue
-            else:
-                self.assertIn(attr, self.policy_defaults,
-                              "{} is not fully tested".format(attr))
+        for policy, expected in self.policies.items():
+            for attr in dir(policy):
+                if (attr.startswith('_') or
+                        isinstance(getattr(email.policy.EmailPolicy, attr),
+                              types.FunctionType)):
+                    continue
+                else:
+                    self.assertIn(attr, expected,
+                                  "{} is not fully tested".format(attr))
 
     def test_abc(self):
         with self.assertRaises(TypeError) as cm:
@@ -62,18 +83,20 @@ class PolicyAPITests(unittest.TestCase):
             self.assertIn(method, msg)
 
     def test_policy_is_immutable(self):
-        for policy in self.policies:
-            for attr in self.policy_defaults:
+        for policy, defaults in self.policies.items():
+            for attr in defaults:
                 with self.assertRaisesRegex(AttributeError, attr+".*read-only"):
                     setattr(policy, attr, None)
             with self.assertRaisesRegex(AttributeError, 'no attribute.*foo'):
                 policy.foo = None
 
-    def test_set_policy_attrs_when_calledl(self):
-        testattrdict = { attr: None for attr in self.policy_defaults }
-        for policyclass in self.policies:
+    def test_set_policy_attrs_when_cloned(self):
+        # None of the attributes has a default value of None, so we set them
+        # all to None in the clone call and check that it worked.
+        for policyclass, defaults in self.policies.items():
+            testattrdict = {attr: None for attr in defaults}
             policy = policyclass.clone(**testattrdict)
-            for attr in self.policy_defaults:
+            for attr in defaults:
                 self.assertIsNone(getattr(policy, attr))
 
     def test_reject_non_policy_keyword_when_called(self):
@@ -105,7 +128,7 @@ class PolicyAPITests(unittest.TestCase):
                 self.defects = []
         obj = Dummy()
         defect = object()
-        policy = email.policy.Compat32()
+        policy = email.policy.EmailPolicy()
         policy.register_defect(obj, defect)
         self.assertEqual(obj.defects, [defect])
         defect2 = object()
@@ -134,7 +157,7 @@ class PolicyAPITests(unittest.TestCase):
         email.policy.default.handle_defect(foo, defect2)
         self.assertEqual(foo.defects, [defect1, defect2])
 
-    class MyPolicy(email.policy.Compat32):
+    class MyPolicy(email.policy.EmailPolicy):
         defects = None
         def __init__(self, *args, **kw):
             super().__init__(*args, defects=[], **kw)
@@ -159,6 +182,49 @@ class PolicyAPITests(unittest.TestCase):
         self.assertEqual(my_policy.defects, [defect1, defect2])
         self.assertEqual(foo.defects, [])
 
+    def test_default_header_factory(self):
+        h = email.policy.default.header_factory('Test', 'test')
+        self.assertEqual(h.name, 'Test')
+        self.assertIsInstance(h, _headerregistry.UnstructuredHeader)
+        self.assertIsInstance(h, _headerregistry.BaseHeader)
+
+    class Foo:
+        parse = _headerregistry.UnstructuredHeader.parse
+
+    def test_each_Policy_gets_unique_factory(self):
+        policy1 = email.policy.EmailPolicy()
+        policy2 = email.policy.EmailPolicy()
+        policy1.header_factory.map_to_type('foo', self.Foo)
+        h = policy1.header_factory('foo', 'test')
+        self.assertIsInstance(h, self.Foo)
+        self.assertNotIsInstance(h, _headerregistry.UnstructuredHeader)
+        h = policy2.header_factory('foo', 'test')
+        self.assertNotIsInstance(h, self.Foo)
+        self.assertIsInstance(h, _headerregistry.UnstructuredHeader)
+
+    def test_clone_copies_factory(self):
+        policy1 = email.policy.EmailPolicy()
+        policy2 = policy1.clone()
+        policy1.header_factory.map_to_type('foo', self.Foo)
+        h = policy1.header_factory('foo', 'test')
+        self.assertIsInstance(h, self.Foo)
+        h = policy2.header_factory('foo', 'test')
+        self.assertIsInstance(h, self.Foo)
+
+    def test_new_factory_overrides_default(self):
+        mypolicy = email.policy.EmailPolicy()
+        myfactory = mypolicy.header_factory
+        newpolicy = mypolicy + email.policy.strict
+        self.assertEqual(newpolicy.header_factory, myfactory)
+        newpolicy = email.policy.strict + mypolicy
+        self.assertEqual(newpolicy.header_factory, myfactory)
+
+    def test_adding_default_policies_preserves_default_factory(self):
+        newpolicy = email.policy.default + email.policy.strict
+        self.assertEqual(newpolicy.header_factory,
+                         email.policy.EmailPolicy.header_factory)
+        self.assertEqual(newpolicy.__dict__, {'raise_on_defect': True})
+
     # XXX: Need subclassing tests.
     # For adding subclassed objects, make sure the usual rules apply (subclass
     # wins), but that the order still works (right overrides left).