#16618: Make glob.glob match consistently across strings and bytes

Fixes handling of leading dots.

Patch by Serhiy Storchaka.

Lib/glob.py

@@ -58,8 +58,8 @@ def glob1(dirname, pattern):
         names = os.listdir(dirname)
     except os.error:
         return []
-    if pattern[0] != '.':
-        names = [x for x in names if x[0] != '.']
+    if not _ishidden(pattern):
+        names = [x for x in names if not _ishidden(x)]
     return fnmatch.filter(names, pattern)
 
 def glob0(dirname, basename):
@@ -83,3 +83,6 @@ def has_magic(s):
     else:
         match = magic_check.search(s)
     return match is not None
+
+def _ishidden(path):
+    return path[0] in ('.', b'.'[0])

Lib/test/test_glob.py

-import unittest
-from test.support import run_unittest, TESTFN, skip_unless_symlink, can_symlink
 import glob
 import os
 import shutil
 import sys
+import unittest
+
+from test.support import run_unittest, TESTFN, skip_unless_symlink, can_symlink
 
 
 class GlobTests(unittest.TestCase):
@@ -31,7 +32,8 @@ class GlobTests(unittest.TestCase):
         self.mktemp('a', 'bcd', 'efg', 'ha')
         if can_symlink():
             os.symlink(self.norm('broken'), self.norm('sym1'))
-            os.symlink(self.norm('broken'), self.norm('sym2'))
+            os.symlink('broken', self.norm('sym2'))
+            os.symlink(os.path.join('a', 'bcd'), self.norm('sym3'))
 
     def tearDown(self):
         shutil.rmtree(self.tempdir)
@@ -44,10 +46,16 @@ class GlobTests(unittest.TestCase):
         p = os.path.join(self.tempdir, pattern)
         res = glob.glob(p)
         self.assertEqual(list(glob.iglob(p)), res)
+        bres = [os.fsencode(x) for x in res]
+        self.assertEqual(glob.glob(os.fsencode(p)), bres)
+        self.assertEqual(list(glob.iglob(os.fsencode(p))), bres)
         return res
 
     def assertSequencesEqual_noorder(self, l1, l2):
+        l1 = list(l1)
+        l2 = list(l2)
         self.assertEqual(set(l1), set(l2))
+        self.assertEqual(sorted(l1), sorted(l2))
 
     def test_glob_literal(self):
         eq = self.assertSequencesEqual_noorder
@@ -56,15 +64,15 @@ class GlobTests(unittest.TestCase):
         eq(self.glob('aab'), [self.norm('aab')])
         eq(self.glob('zymurgy'), [])
 
-        # test return types are unicode, but only if os.listdir
-        # returns unicode filenames
-        uniset = set([str])
-        tmp = os.listdir('.')
-        if set(type(x) for x in tmp) == uniset:
-            u1 = glob.glob('*')
-            u2 = glob.glob('./*')
-            self.assertEqual(set(type(r) for r in u1), uniset)
-            self.assertEqual(set(type(r) for r in u2), uniset)
+        res = glob.glob('*')
+        self.assertEqual({type(r) for r in res}, {str})
+        res = glob.glob(os.path.join(os.curdir, '*'))
+        self.assertEqual({type(r) for r in res}, {str})
+
+        res = glob.glob(b'*')
+        self.assertEqual({type(r) for r in res}, {bytes})
+        res = glob.glob(os.path.join(os.fsencode(os.curdir), b'*'))
+        self.assertEqual({type(r) for r in res}, {bytes})
 
     def test_glob_one_directory(self):
         eq = self.assertSequencesEqual_noorder
@@ -93,20 +101,20 @@ class GlobTests(unittest.TestCase):
         eq(self.glob('*', '*a'), [])
         eq(self.glob('a', '*', '*', '*a'),
            [self.norm('a', 'bcd', 'efg', 'ha')])
-        eq(self.glob('?a?', '*F'), map(self.norm, [os.path.join('aaa', 'zzzF'),
-                                                   os.path.join('aab', 'F')]))
+        eq(self.glob('?a?', '*F'), [self.norm('aaa', 'zzzF'),
+                                    self.norm('aab', 'F')])
 
     def test_glob_directory_with_trailing_slash(self):
         # Patterns ending with a slash shouldn't match non-dirs
-        res = glob.glob(os.path.join(self.tempdir, 'Z*Z') + os.sep)
+        res = glob.glob(self.norm('Z*Z') + os.sep)
         self.assertEqual(res, [])
-        res = glob.glob(os.path.join(self.tempdir, 'ZZZ') + os.sep)
+        res = glob.glob(self.norm('ZZZ') + os.sep)
         self.assertEqual(res, [])
-        # When there is wildcard pattern which ends with os.sep, glob()
+        # When there is a wildcard pattern which ends with os.sep, glob()
         # doesn't blow up.
-        res = glob.glob(os.path.join(self.tempdir, 'aa*') + os.sep)
+        res = glob.glob(self.norm('aa*') + os.sep)
         self.assertEqual(len(res), 2)
-        # either of these results are reasonable
+        # either of these results is reasonable
         self.assertIn(set(res), [
                       {self.norm('aaa'), self.norm('aab')},
                       {self.norm('aaa') + os.sep, self.norm('aab') + os.sep},
@@ -115,22 +123,37 @@ class GlobTests(unittest.TestCase):
     def test_glob_bytes_directory_with_trailing_slash(self):
         # Same as test_glob_directory_with_trailing_slash, but with a
         # bytes argument.
-        res = glob.glob(os.fsencode(os.path.join(self.tempdir, 'Z*Z') + os.sep))
+        res = glob.glob(os.fsencode(self.norm('Z*Z') + os.sep))
         self.assertEqual(res, [])
-        res = glob.glob(os.fsencode(os.path.join(self.tempdir, 'ZZZ') + os.sep))
+        res = glob.glob(os.fsencode(self.norm('ZZZ') + os.sep))
         self.assertEqual(res, [])
-        res = glob.glob(os.fsencode(os.path.join(self.tempdir, 'aa*') + os.sep))
+        res = glob.glob(os.fsencode(self.norm('aa*') + os.sep))
         self.assertEqual(len(res), 2)
-        # either of these results are reasonable
-        self.assertIn({os.fsdecode(x) for x in res}, [
-                      {self.norm('aaa'), self.norm('aab')},
-                      {self.norm('aaa') + os.sep, self.norm('aab') + os.sep},
+        # either of these results is reasonable
+        self.assertIn(set(res), [
+                      {os.fsencode(self.norm('aaa')),
+                       os.fsencode(self.norm('aab'))},
+                      {os.fsencode(self.norm('aaa') + os.sep),
+                       os.fsencode(self.norm('aab') + os.sep)},
                       ])
 
+    @skip_unless_symlink
+    def test_glob_symlinks(self):
+        eq = self.assertSequencesEqual_noorder
+        eq(self.glob('sym3'), [self.norm('sym3')])
+        eq(self.glob('sym3', '*'), [self.norm('sym3', 'EF'),
+                                    self.norm('sym3', 'efg')])
+        self.assertIn(self.glob('sym3' + os.sep),
+                      [[self.norm('sym3')], [self.norm('sym3') + os.sep]])
+        eq(self.glob('*', '*F'),
+           [self.norm('aaa', 'zzzF'),
+            self.norm('aab', 'F'), self.norm('sym3', 'EF')])
+
     @skip_unless_symlink
     def test_glob_broken_symlinks(self):
         eq = self.assertSequencesEqual_noorder
-        eq(self.glob('sym*'), [self.norm('sym1'), self.norm('sym2')])
+        eq(self.glob('sym*'), [self.norm('sym1'), self.norm('sym2'),
+                               self.norm('sym3')])
         eq(self.glob('sym1'), [self.norm('sym1')])
         eq(self.glob('sym2'), [self.norm('sym2')])
 

Misc/NEWS

@@ -179,6 +179,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #16618: Make glob.glob match consistently across strings and bytes
+  regarding leading dots.  Patch by Serhiy Storchaka.
+
 - Issue #16702: test_urllib2_localnet tests now correctly ignores proxies for
   localhost tests.