SF patch #1157027, cookielib mis-handles RFC 2109 cookies in Netscape mode

Doc/lib/libcookielib.tex

@@ -18,17 +18,18 @@ the server in later HTTP requests.
 Both the regular Netscape cookie protocol and the protocol defined by
 \rfc{2965} are handled.  RFC 2965 handling is switched off by default.
 \rfc{2109} cookies are parsed as Netscape cookies and subsequently
-treated as RFC 2965 cookies.  Note that the great majority of cookies
-on the Internet are Netscape cookies.  \module{cookielib} attempts to
-follow the de-facto Netscape cookie protocol (which differs
-substantially from that set out in the original Netscape
-specification), including taking note of the \code{max-age} and
-\code{port} cookie-attributes introduced with RFC 2109.  \note{The
-various named parameters found in \mailheader{Set-Cookie} and
-\mailheader{Set-Cookie2} headers (eg. \code{domain} and
-\code{expires}) are conventionally referred to as \dfn{attributes}.
-To distinguish them from Python attributes, the documentation for this
-module uses the term \dfn{cookie-attribute} instead}.
+treated either as Netscape or RFC 2965 cookies according to the
+'policy' in effect.  Note that the great majority of cookies on the
+Internet are Netscape cookies.  \module{cookielib} attempts to follow
+the de-facto Netscape cookie protocol (which differs substantially
+from that set out in the original Netscape specification), including
+taking note of the \code{max-age} and \code{port} cookie-attributes
+introduced with RFC 2109.  \note{The various named parameters found in
+\mailheader{Set-Cookie} and \mailheader{Set-Cookie2} headers
+(eg. \code{domain} and \code{expires}) are conventionally referred to
+as \dfn{attributes}.  To distinguish them from Python attributes, the
+documentation for this module uses the term \dfn{cookie-attribute}
+instead}.
 
 
 The module defines the following exception:
@@ -74,6 +75,7 @@ accepted from / returned to the server.
     blocked_domains=\constant{None},
     allowed_domains=\constant{None},
     netscape=\constant{True}, rfc2965=\constant{False},
+    rfc2109_as_netscape=\constant{None},
     hide_cookie2=\constant{False},
     strict_domain=\constant{False},
     strict_rfc2965_unverifiable=\constant{True},
@@ -92,10 +94,14 @@ documentation for \class{CookiePolicy} and \class{DefaultCookiePolicy}
 objects.
 
 \class{DefaultCookiePolicy} implements the standard accept / reject
-rules for Netscape and RFC 2965 cookies.  RFC 2109 cookies
+rules for Netscape and RFC 2965 cookies.  By default, RFC 2109 cookies
 (ie. cookies received in a \mailheader{Set-Cookie} header with a
 version cookie-attribute of 1) are treated according to the RFC 2965
-rules.  \class{DefaultCookiePolicy} also provides some parameters to
+rules.  However, if RFC 2965 handling is turned off or
+\member{rfc2109_as_netscape} is True, RFC 2109 cookies are
+'downgraded' by the \class{CookieJar} instance to Netscape cookies, by
+setting the \member{version} attribute of the \class{Cookie} instance
+to 0.  \class{DefaultCookiePolicy} also provides some parameters to
 allow some fine-tuning of policy.
 \end{classdesc}
 
@@ -493,6 +499,17 @@ receiving cookies.
 which are all initialised from the constructor arguments of the same
 name, and which may all be assigned to.
 
+\begin{memberdesc}{rfc2109_as_netscape}
+If true, request that the \class{CookieJar} instance downgrade RFC
+2109 cookies (ie. cookies received in a \mailheader{Set-Cookie} header
+with a version cookie-attribute of 1) to Netscape cookies by setting
+the version attribute of the \class{Cookie} instance to 0.  The
+default value is \constant{None}, in which case RFC 2109 cookies are
+downgraded if and only if RFC 2965 handling is turned off.  Therefore,
+RFC 2109 cookies are downgraded by default.
+\versionadded{2.5}
+\end{memberdesc}
+
 General strictness switches:
 
 \begin{memberdesc}{strict_domain}
@@ -567,9 +584,10 @@ Equivalent to \code{DomainStrictNoDots|DomainStrictNonDomain}.
 \class{Cookie} instances have Python attributes roughly corresponding
 to the standard cookie-attributes specified in the various cookie
 standards.  The correspondence is not one-to-one, because there are
-complicated rules for assigning default values, and because the
+complicated rules for assigning default values, because the
 \code{max-age} and \code{expires} cookie-attributes contain equivalent
-information.
+information, and because RFC 2109 cookies may be 'downgraded' by
+\module{cookielib} from version 1 to version 0 (Netscape) cookies.
 
 Assignment to these attributes should not be necessary other than in
 rare circumstances in a \class{CookiePolicy} method.  The class does
@@ -577,8 +595,10 @@ not enforce internal consistency, so you should know what you're
 doing if you do that.
 
 \begin{memberdesc}[Cookie]{version}
-Integer or \constant{None}.  Netscape cookies have version 0.  RFC
-2965 and RFC 2109 cookies have version 1.
+Integer or \constant{None}.  Netscape cookies have \member{version} 0.
+RFC 2965 and RFC 2109 cookies have a \code{version} cookie-attribute
+of 1.  However, note that \module{cookielib} may 'downgrade' RFC 2109
+cookies to Netscape cookies, in which case \member{version} is 0.
 \end{memberdesc}
 \begin{memberdesc}[Cookie]{name}
 Cookie name (a string).
@@ -611,6 +631,14 @@ or \constant{None}.
 URL linking to a comment from the server explaining the function of
 this cookie, or \constant{None}.
 \end{memberdesc}
+\begin{memberdesc}[Cookie]{rfc2109}
+True if this cookie was received as an RFC 2109 cookie (ie. the cookie
+arrived in a \mailheader{Set-Cookie} header, and the value of the
+Version cookie-attribute in that header was 1).  This attribute is
+provided because \module{cookielib} may 'downgrade' RFC 2109 cookies
+to Netscape cookies, in which case \member{version} is 0.
+\versionadded{2.5}
+\end{memberdesc}
 
 \begin{memberdesc}[Cookie]{port_specified}
 True if a port or set of ports was explicitly specified by the server

Lib/cookielib.py

@@ -460,10 +460,7 @@ def parse_ns_headers(ns_headers):
                 if lc in known_attrs:
                     k = lc
                 if k == "version":
-                    # This is an RFC 2109 cookie.  Will be treated as RFC 2965
-                    # cookie in rest of code.
-                    # Probably it should be parsed with split_header_words, but
-                    # that's too much hassle.
+                    # This is an RFC 2109 cookie.
                     version_set = True
                 if k == "expires":
                     # convert expires date to seconds since epoch
@@ -723,7 +720,9 @@ class Cookie:
                  discard,
                  comment,
                  comment_url,
-                 rest):
+                 rest,
+                 rfc2109=False,
+                 ):
 
         if version is not None: version = int(version)
         if expires is not None: expires = int(expires)
@@ -750,6 +749,7 @@ class Cookie:
         self.discard = discard
         self.comment = comment
         self.comment_url = comment_url
+        self.rfc2109 = rfc2109
 
         self._rest = copy.copy(rest)
 
@@ -787,6 +787,7 @@ class Cookie:
             attr = getattr(self, name)
             args.append("%s=%s" % (name, repr(attr)))
         args.append("rest=%s" % repr(self._rest))
+        args.append("rfc2109=%s" % repr(self.rfc2109))
         return "Cookie(%s)" % ", ".join(args)
 
 
@@ -836,6 +837,7 @@ class DefaultCookiePolicy(CookiePolicy):
     def __init__(self,
                  blocked_domains=None, allowed_domains=None,
                  netscape=True, rfc2965=False,
+                 rfc2109_as_netscape=None,
                  hide_cookie2=False,
                  strict_domain=False,
                  strict_rfc2965_unverifiable=True,
@@ -847,6 +849,7 @@ class DefaultCookiePolicy(CookiePolicy):
         """Constructor arguments should be passed as keyword arguments only."""
         self.netscape = netscape
         self.rfc2965 = rfc2965
+        self.rfc2109_as_netscape = rfc2109_as_netscape
         self.hide_cookie2 = hide_cookie2
         self.strict_domain = strict_domain
         self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable
@@ -1518,6 +1521,18 @@ class CookieJar:
             if cookie: cookies.append(cookie)
         return cookies
 
+    def _process_rfc2109_cookies(self, cookies):
+        rfc2109_as_ns = getattr(self._policy, 'rfc2109_as_netscape', None)
+        if rfc2109_as_ns is None:
+            rfc2109_as_ns = not self._policy.rfc2965
+        for cookie in cookies:
+            if cookie.version == 1:
+                cookie.rfc2109 = True
+                if rfc2109_as_ns: 
+                    # treat 2109 cookies as Netscape cookies rather than
+                    # as RFC2965 cookies
+                    cookie.version = 0
+
     def make_cookies(self, response, request):
         """Return sequence of Cookie objects extracted from response object."""
         # get cookie-attributes for RFC 2965 and Netscape protocols
@@ -1543,11 +1558,13 @@ class CookieJar:
 
         if ns_hdrs and netscape:
             try:
+                # RFC 2109 and Netscape cookies
                 ns_cookies = self._cookies_from_attrs_set(
                     parse_ns_headers(ns_hdrs), request)
             except:
                 reraise_unmasked_exceptions()
                 ns_cookies = []
+            self._process_rfc2109_cookies(ns_cookies)
 
             # Look for Netscape cookies (from Set-Cookie headers) that match
             # corresponding RFC 2965 cookies (from Set-Cookie2 headers).

Lib/test/test_cookielib.py

@@ -386,6 +386,39 @@ class CookieTests(TestCase):
         self.assertEquals(interact_netscape(c, "http://www.acme.com/foo/"),
                           '"spam"; eggs')
 
+    def test_rfc2109_handling(self):
+        # RFC 2109 cookies are handled as RFC 2965 or Netscape cookies,
+        # dependent on policy settings
+        from cookielib import CookieJar, DefaultCookiePolicy
+
+        for rfc2109_as_netscape, rfc2965, version in [
+            # default according to rfc2965 if not explicitly specified
+            (None, False, 0),
+            (None, True, 1),
+            # explicit rfc2109_as_netscape
+            (False, False, None),  # version None here means no cookie stored
+            (False, True, 1),
+            (True, False, 0),
+            (True, True, 0),
+            ]:
+            policy = DefaultCookiePolicy(
+                rfc2109_as_netscape=rfc2109_as_netscape,
+                rfc2965=rfc2965)
+            c = CookieJar(policy)
+            interact_netscape(c, "http://www.example.com/", "ni=ni; Version=1")
+            try:
+                cookie = c._cookies["www.example.com"]["/"]["ni"]
+            except KeyError:
+                self.assert_(version is None)  # didn't expect a stored cookie
+            else:
+                self.assertEqual(cookie.version, version)
+                # 2965 cookies are unaffected
+                interact_2965(c, "http://www.example.com/",
+                              "foo=bar; Version=1")
+                if rfc2965:
+                    cookie2965 = c._cookies["www.example.com"]["/"]["foo"]
+                    self.assertEqual(cookie2965.version, 1)
+
     def test_ns_parser(self):
         from cookielib import CookieJar, DEFAULT_HTTP_PORT