merge heads

Doc/library/hmac.rst

@@ -83,6 +83,7 @@ This module also provides the following helper function:
       contents of the inputs via a timing attack, it does leak the length
       of the inputs. However, this generally is not a security risk.
 
+   .. versionadded:: 3.3
 
 .. seealso::
 

Lib/os.py

@@ -353,13 +353,23 @@ if _exists("openat"):
         names = flistdir(topfd)
         dirs, nondirs = [], []
         for name in names:
-            # Here, we don't use AT_SYMLINK_NOFOLLOW to be consistent with
-            # walk() which reports symlinks to directories as directories. We do
-            # however check for symlinks before recursing into a subdirectory.
-            if st.S_ISDIR(fstatat(topfd, name).st_mode):
-                dirs.append(name)
-            else:
-                nondirs.append(name)
+            try:
+                # Here, we don't use AT_SYMLINK_NOFOLLOW to be consistent with
+                # walk() which reports symlinks to directories as directories.
+                # We do however check for symlinks before recursing into
+                # a subdirectory.
+                if st.S_ISDIR(fstatat(topfd, name).st_mode):
+                    dirs.append(name)
+                else:
+                    nondirs.append(name)
+            except FileNotFoundError:
+                try:
+                    # Add dangling symlinks, ignore disappeared files
+                    if st.S_ISLNK(fstatat(topfd, name, AT_SYMLINK_NOFOLLOW)
+                                .st_mode):
+                        nondirs.append(name)
+                except FileNotFoundError:
+                    continue
 
         if topdown:
             yield toppath, dirs, nondirs, topfd

Lib/test/test_os.py

@@ -651,6 +651,7 @@ class WalkTests(unittest.TestCase):
         #         SUB2/             a file kid and a dirsymlink kid
         #           tmp3
         #           link/           a symlink to TESTFN.2
+        #           broken_link
         #       TEST2/
         #         tmp4              a lone file
         walk_path = join(support.TESTFN, "TEST1")
@@ -663,6 +664,8 @@ class WalkTests(unittest.TestCase):
         link_path = join(sub2_path, "link")
         t2_path = join(support.TESTFN, "TEST2")
         tmp4_path = join(support.TESTFN, "TEST2", "tmp4")
+        link_path = join(sub2_path, "link")
+        broken_link_path = join(sub2_path, "broken_link")
 
         # Create stuff.
         os.makedirs(sub11_path)
@@ -679,7 +682,8 @@ class WalkTests(unittest.TestCase):
             else:
                 symlink_to_dir = os.symlink
             symlink_to_dir(os.path.abspath(t2_path), link_path)
-            sub2_tree = (sub2_path, ["link"], ["tmp3"])
+            symlink_to_dir('broken', broken_link_path)
+            sub2_tree = (sub2_path, ["link"], ["broken_link", "tmp3"])
         else:
             sub2_tree = (sub2_path, [], ["tmp3"])
 
@@ -691,6 +695,7 @@ class WalkTests(unittest.TestCase):
         #     flipped:  TESTFN, SUB2, SUB1, SUB11
         flipped = all[0][1][0] != "SUB1"
         all[0][1].sort()
+        all[3 - 2 * flipped][-1].sort()
         self.assertEqual(all[0], (walk_path, ["SUB1", "SUB2"], ["tmp1"]))
         self.assertEqual(all[1 + flipped], (sub1_path, ["SUB11"], ["tmp2"]))
         self.assertEqual(all[2 + flipped], (sub11_path, [], []))
@@ -706,6 +711,7 @@ class WalkTests(unittest.TestCase):
                 dirs.remove('SUB1')
         self.assertEqual(len(all), 2)
         self.assertEqual(all[0], (walk_path, ["SUB2"], ["tmp1"]))
+        all[1][-1].sort()
         self.assertEqual(all[1], sub2_tree)
 
         # Walk bottom-up.
@@ -716,6 +722,7 @@ class WalkTests(unittest.TestCase):
         #     flipped:  SUB2, SUB11, SUB1, TESTFN
         flipped = all[3][1][0] != "SUB1"
         all[3][1].sort()
+        all[2 - 2 * flipped][-1].sort()
         self.assertEqual(all[3], (walk_path, ["SUB1", "SUB2"], ["tmp1"]))
         self.assertEqual(all[flipped], (sub11_path, [], []))
         self.assertEqual(all[flipped + 1], (sub1_path, ["SUB11"], ["tmp2"]))

Lib/test/test_urllib2.py

@@ -1252,6 +1252,22 @@ class HandlerTests(unittest.TestCase):
     def test_basic_auth_with_single_quoted_realm(self):
         self.test_basic_auth(quote_char="'")
 
+    def test_basic_auth_with_unquoted_realm(self):
+        opener = OpenerDirector()
+        password_manager = MockPasswordManager()
+        auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
+        realm = "ACME Widget Store"
+        http_handler = MockHTTPHandler(
+            401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm)
+        opener.add_handler(auth_handler)
+        opener.add_handler(http_handler)
+        with self.assertWarns(UserWarning):
+            self._test_basic_auth(opener, auth_handler, "Authorization",
+                                realm, http_handler, password_manager,
+                                "http://acme.example.com/protected",
+                                "http://acme.example.com/protected",
+                                )
+
     def test_proxy_basic_auth(self):
         opener = OpenerDirector()
         ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))

Lib/urllib/request.py

@@ -895,7 +895,7 @@ class AbstractBasicAuthHandler:
     # allow for double- and single-quoted realm values
     # (single quotes are a violation of the RFC, but appear in the wild)
     rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
-                    'realm=(["\'])(.*?)\\2', re.I)
+                    'realm=(["\']?)([^"\']*)\\2', re.I)
 
     # XXX could pre-emptively send auth info already accepted (RFC 2617,
     # end of section 2, and section 1.2 immediately after "credentials"
@@ -934,6 +934,9 @@ class AbstractBasicAuthHandler:
                 mo = AbstractBasicAuthHandler.rx.search(authreq)
                 if mo:
                     scheme, quote, realm = mo.groups()
+                    if quote not in ['"',"'"]:
+                        warnings.warn("Basic Auth Realm was unquoted",
+                                      UserWarning, 2)
                     if scheme.lower() == 'basic':
                         response = self.retry_http_basic_auth(host, req, realm)
                         if response and response.code != 401:

Misc/NEWS

@@ -31,7 +31,12 @@ Core and Builtins
 Library
 -------
 
-- Issue 14807: move undocumented tarfile.filemode() to stat.filemode() and add
+- Issue #14773: Fix os.fwalk() failing on dangling symlinks.
+
+- Issue #12541: Be lenient with quotes around Realm field of HTTP Basic
+  Authentation in urllib2.
+
+- Issue #14807: move undocumented tarfile.filemode() to stat.filemode() and add
   doc entry. Add tarfile.filemode alias with deprecation warning.
 
 - Issue #13815: TarFile.extractfile() now returns io.BufferedReader objects.

PCbuild/pythoncore.vcxproj

@@ -564,12 +564,6 @@
     <ClCompile Include="..\Modules\zlib\compress.c" />
     <ClCompile Include="..\Modules\zlib\crc32.c" />
     <ClCompile Include="..\Modules\zlib\deflate.c" />
-    <ClCompile Include="..\Modules\zlib\gzio.c">
-      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='PGInstrument|Win32'">_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='PGUpdate|Win32'">_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-    </ClCompile>
     <ClCompile Include="..\Modules\zlib\infback.c" />
     <ClCompile Include="..\Modules\zlib\inffast.c" />
     <ClCompile Include="..\Modules\zlib\inflate.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -578,9 +578,6 @@
     <ClCompile Include="..\Modules\zlib\deflate.c">
       <Filter>Modules\zlib</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\zlib\gzio.c">
-      <Filter>Modules\zlib</Filter>
-    </ClCompile>
     <ClCompile Include="..\Modules\zlib\infback.c">
       <Filter>Modules\zlib</Filter>
     </ClCompile>

Parser/asdl_c.py

@@ -784,7 +784,7 @@ static int add_attributes(PyTypeObject* type, char**attrs, int num_fields)
 
 static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
 {
-    int i, n = asdl_seq_LEN(seq);
+    Py_ssize_t i, n = asdl_seq_LEN(seq);
     PyObject *result = PyList_New(n);
     PyObject *value;
     if (!result)
@@ -1106,7 +1106,7 @@ class ObjVisitor(PickleVisitor):
                 # While the sequence elements are stored as void*,
                 # ast2obj_cmpop expects an enum
                 self.emit("{", depth)
-                self.emit("int i, n = asdl_seq_LEN(%s);" % value, depth+1)
+                self.emit("Py_ssize_t i, n = asdl_seq_LEN(%s);" % value, depth+1)
                 self.emit("value = PyList_New(n);", depth+1)
                 self.emit("if (!value) goto failed;", depth+1)
                 self.emit("for(i = 0; i < n; i++)", depth+1)

Python/Python-ast.c

@@ -636,7 +636,7 @@ static int add_attributes(PyTypeObject* type, char**attrs, int num_fields)
 
 static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
 {
-    int i, n = asdl_seq_LEN(seq);
+    Py_ssize_t i, n = asdl_seq_LEN(seq);
     PyObject *result = PyList_New(n);
     PyObject *value;
     if (!result)
@@ -2857,7 +2857,7 @@ ast2obj_expr(void* _o)
                         goto failed;
                 Py_DECREF(value);
                 {
-                        int i, n = asdl_seq_LEN(o->v.Compare.ops);
+                        Py_ssize_t i, n = asdl_seq_LEN(o->v.Compare.ops);
                         value = PyList_New(n);
                         if (!value) goto failed;
                         for(i = 0; i < n; i++)