Issue #14626: Large refactoring of functions / parameters in the os module.

Many functions now support "dir_fd" and "follow_symlinks" parameters;
some also support accepting an open file descriptor in place of of a path
string.  Added os.support_* collections as LBYL helpers.  Removed many
functions only previously seen in 3.3 alpha releases (often starting with
"f" or "l", or ending with "at").  Originally suggested by Serhiy Storchaka;
implemented by Larry Hastings.

Lib/os.py

@@ -56,9 +56,10 @@ if 'posix' in _names:
         pass
     import posixpath as path
 
-    import posix
-    __all__.extend(_get_exports_list(posix))
-    del posix
+    try:
+        from posix import _have_functions
+    except ImportError:
+        pass
 
 elif 'nt' in _names:
     name = 'nt'
@@ -75,6 +76,11 @@ elif 'nt' in _names:
     __all__.extend(_get_exports_list(nt))
     del nt
 
+    try:
+        from nt import _have_functions
+    except ImportError:
+        pass
+
 elif 'os2' in _names:
     name = 'os2'
     linesep = '\r\n'
@@ -94,6 +100,11 @@ elif 'os2' in _names:
     __all__.extend(_get_exports_list(os2))
     del os2
 
+    try:
+        from os2 import _have_functions
+    except ImportError:
+        pass
+
 elif 'ce' in _names:
     name = 'ce'
     linesep = '\r\n'
@@ -110,6 +121,11 @@ elif 'ce' in _names:
     __all__.extend(_get_exports_list(ce))
     del ce
 
+    try:
+        from ce import _have_functions
+    except ImportError:
+        pass
+
 else:
     raise ImportError('no os specific module found')
 
@@ -119,6 +135,84 @@ from os.path import (curdir, pardir, sep, pathsep, defpath, extsep, altsep,
 
 del _names
 
+
+if _exists("_have_functions"):
+    _globals = globals()
+    def _add(str, fn):
+        if (fn in _globals) and (str in _have_functions):
+            _set.add(_globals[fn])
+
+    _set = set()
+    _add("HAVE_FACCESSAT",  "access")
+    _add("HAVE_FCHMODAT",   "chmod")
+    _add("HAVE_FCHOWNAT",   "chown")
+    _add("HAVE_FSTATAT",    "stat")
+    _add("HAVE_FUTIMESAT",  "utime")
+    _add("HAVE_LINKAT",     "link")
+    _add("HAVE_MKDIRAT",    "mkdir")
+    _add("HAVE_MKFIFOAT",   "mkfifo")
+    _add("HAVE_MKNODAT",    "mknod")
+    _add("HAVE_OPENAT",     "open")
+    _add("HAVE_READLINKAT", "readlink")
+    _add("HAVE_RENAMEAT",   "rename")
+    _add("HAVE_SYMLINKAT",  "symlink")
+    _add("HAVE_UNLINKAT",   "unlink")
+    _add("HAVE_UTIMENSAT",  "utime")
+    supports_dir_fd = _set
+
+    _set = set()
+    _add("HAVE_FACCESSAT",  "access")
+    supports_effective_ids = _set
+
+    _set = set()
+    _add("HAVE_FCHDIR",     "chdir")
+    _add("HAVE_FCHMOD",     "chmod")
+    _add("HAVE_FCHOWN",     "chown")
+    _add("HAVE_FDOPENDIR",  "listdir")
+    _add("HAVE_FEXECVE",    "execve")
+    _set.add(stat) # fstat always works
+    _add("HAVE_FUTIMENS",   "utime")
+    _add("HAVE_FUTIMES",    "utime")
+    if _exists("statvfs") and _exists("fstatvfs"): # mac os x10.3
+        _add("HAVE_FSTATVFS", "statvfs")
+    supports_fd = _set
+
+    _set = set()
+    _add("HAVE_FACCESSAT",  "access")
+    # Current linux (kernel 3.2, glibc 2.15) doesn't support lchmod.
+    # (The function exists, but it's a stub that always returns ENOSUP.)
+    # Now, linux *does* have fchmodat, which says it can ignore
+    # symbolic links.  But that doesn't work either (also returns ENOSUP).
+    # I'm guessing that if they fix fchmodat, they'll also add lchmod at
+    # the same time.  So, for now, assume that fchmodat doesn't support
+    # follow_symlinks unless lchmod works.
+    if ((sys.platform != "linux") or
+        ("HAVE_LCHMOD" in _have_functions)):
+        _add("HAVE_FCHMODAT",   "chmod")
+    _add("HAVE_FCHOWNAT",   "chown")
+    _add("HAVE_FSTATAT",    "stat")
+    _add("HAVE_LCHFLAGS",   "chflags")
+    _add("HAVE_LCHMOD",     "chmod")
+    if _exists("lchown"): # mac os x10.3
+        _add("HAVE_LCHOWN", "chown")
+    _add("HAVE_LINKAT",     "link")
+    _add("HAVE_LUTIMES",    "utime")
+    _add("HAVE_LSTAT",      "stat")
+    _add("HAVE_FSTATAT",    "stat")
+    _add("HAVE_UTIMENSAT",  "utime")
+    _add("MS_WINDOWS",      "stat")
+    supports_follow_symlinks = _set
+
+    _set = set()
+    _add("HAVE_UNLINKAT",   "unlink")
+    supports_remove_directory = _set
+
+    del _set
+    del _have_functions
+    del _globals
+    del _add
+
+
 # Python uses fixed values for the SEEK_ constants; they are mapped
 # to native constants if necessary in posixmodule.c
 # Other possible SEEK values are directly imported from posixmodule.c
@@ -318,7 +412,7 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
 
 __all__.append("walk")
 
-if _exists("openat"):
+if open in supports_dir_fd:
 
     def fwalk(top, topdown=True, onerror=None, followlinks=False):
         """Directory tree generator.
@@ -343,7 +437,7 @@ if _exists("openat"):
         import os
         for root, dirs, files, rootfd in os.fwalk('python/Lib/email'):
             print(root, "consumes", end="")
-            print(sum([os.fstatat(rootfd, name).st_size for name in files]),
+            print(sum([os.stat(name, dir_fd=rootfd).st_size for name in files]),
                   end="")
             print("bytes in", len(files), "non-directory files")
             if 'CVS' in dirs:
@@ -365,10 +459,7 @@ if _exists("openat"):
         # necessary, it can be adapted to only require O(1) FDs, see issue
         # #13734.
 
-        # whether to follow symlinks
-        flag = 0 if followlinks else AT_SYMLINK_NOFOLLOW
-
-        names = flistdir(topfd)
+        names = listdir(topfd)
         dirs, nondirs = [], []
         for name in names:
             try:
@@ -376,14 +467,14 @@ if _exists("openat"):
                 # walk() which reports symlinks to directories as directories.
                 # We do however check for symlinks before recursing into
                 # a subdirectory.
-                if st.S_ISDIR(fstatat(topfd, name).st_mode):
+                if st.S_ISDIR(stat(name, dir_fd=topfd).st_mode):
                     dirs.append(name)
                 else:
                     nondirs.append(name)
             except FileNotFoundError:
                 try:
                     # Add dangling symlinks, ignore disappeared files
-                    if st.S_ISLNK(fstatat(topfd, name, AT_SYMLINK_NOFOLLOW)
+                    if st.S_ISLNK(stat(name, dir_fd=topfd, follow_symlinks=False)
                                 .st_mode):
                         nondirs.append(name)
                 except FileNotFoundError:
@@ -394,8 +485,8 @@ if _exists("openat"):
 
         for name in dirs:
             try:
-                orig_st = fstatat(topfd, name, flag)
-                dirfd = openat(topfd, name, O_RDONLY)
+                orig_st = stat(name, dir_fd=topfd, follow_symlinks=followlinks)
+                dirfd = open(name, O_RDONLY, dir_fd=topfd)
             except error as err:
                 if onerror is not None:
                     onerror(err)

Lib/shutil.py

@@ -139,27 +139,45 @@ def copystat(src, dst, symlinks=False):
     only if both `src` and `dst` are symlinks.
 
     """
-    def _nop(*args, ns=None):
+    def _nop(*args, ns=None, follow_symlinks=None):
         pass
 
-    if symlinks and os.path.islink(src) and os.path.islink(dst):
-        stat_func = os.lstat
-        utime_func = os.lutimes if hasattr(os, 'lutimes') else _nop
-        chmod_func = os.lchmod if hasattr(os, 'lchmod') else _nop
-        chflags_func = os.lchflags if hasattr(os, 'lchflags') else _nop
+    # follow symlinks (aka don't not follow symlinks)
+    follow = not (symlinks and os.path.islink(src) and os.path.islink(dst))
+    if follow:
+        # use the real function if it exists
+        def lookup(name):
+            return getattr(os, name, _nop)
     else:
-        stat_func = os.stat
-        utime_func = os.utime if hasattr(os, 'utime') else _nop
-        chmod_func = os.chmod if hasattr(os, 'chmod') else _nop
-        chflags_func = os.chflags if hasattr(os, 'chflags') else _nop
-
-    st = stat_func(src)
+        # use the real function only if it exists
+        # *and* it supports follow_symlinks
+        def lookup(name):
+            fn = getattr(os, name, _nop)
+            if fn in os.supports_follow_symlinks:
+                return fn
+            return _nop
+
+    st = lookup("stat")(src, follow_symlinks=follow)
     mode = stat.S_IMODE(st.st_mode)
-    utime_func(dst, ns=(st.st_atime_ns, st.st_mtime_ns))
-    chmod_func(dst, mode)
+    lookup("utime")(dst, ns=(st.st_atime_ns, st.st_mtime_ns),
+        follow_symlinks=follow)
+    try:
+        lookup("chmod")(dst, mode, follow_symlinks=follow)
+    except NotImplementedError:
+        # if we got a NotImplementedError, it's because
+        #   * follow_symlinks=False,
+        #   * lchown() is unavailable, and
+        #   * either
+        #       * fchownat() is unvailable or
+        #       * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW.
+        #         (it returned ENOSUP.)
+        # therefore we're out of options--we simply cannot chown the
+        # symlink.  give up, suppress the error.
+        # (which is what shutil always did in this circumstance.)
+        pass
     if hasattr(st, 'st_flags'):
         try:
-            chflags_func(dst, st.st_flags)
+            lookup("chflags")(dst, st.st_flags, follow_symlinks=follow)
         except OSError as why:
             for err in 'EOPNOTSUPP', 'ENOTSUP':
                 if hasattr(errno, err) and why.errno == getattr(errno, err):
@@ -176,20 +194,11 @@ if hasattr(os, 'listxattr'):
         If the optional flag `symlinks` is set, symlinks won't be followed.
 
         """
-        if symlinks:
-            listxattr = os.llistxattr
-            removexattr = os.lremovexattr
-            setxattr = os.lsetxattr
-            getxattr = os.lgetxattr
-        else:
-            listxattr = os.listxattr
-            removexattr = os.removexattr
-            setxattr = os.setxattr
-            getxattr = os.getxattr
 
-        for attr in listxattr(src):
+        for name in os.listxattr(src, follow_symlinks=symlinks):
             try:
-                setxattr(dst, attr, getxattr(src, attr))
+                value = os.getxattr(src, name, follow_symlinks=symlinks)
+                os.setxattr(dst, name, value, follow_symlinks=symlinks)
             except OSError as e:
                 if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA):
                     raise

Lib/test/support.py

@@ -1703,8 +1703,8 @@ def can_xattr():
                 try:
                     # TESTFN & tempfile may use different file systems with
                     # different capabilities
-                    os.fsetxattr(tmp_fp, b"user.test", b"")
-                    os.fsetxattr(fp.fileno(), b"user.test", b"")
+                    os.setxattr(tmp_fp, b"user.test", b"")
+                    os.setxattr(fp.fileno(), b"user.test", b"")
                     # Kernels < 2.6.39 don't respect setxattr flags.
                     kernel_version = platform.release()
                     m = re.match("2.6.(\d{1,2})", kernel_version)

Lib/test/test_shutil.py

@@ -268,7 +268,7 @@ class TestShutil(unittest.TestCase):
         # don't follow
         shutil.copystat(src_link, dst_link, symlinks=True)
         dst_link_stat = os.lstat(dst_link)
-        if hasattr(os, 'lutimes'):
+        if os.utime in os.supports_follow_symlinks:
             for attr in 'st_atime', 'st_mtime':
                 # The modification times may be truncated in the new file.
                 self.assertLessEqual(getattr(src_link_stat, attr),
@@ -334,11 +334,11 @@ class TestShutil(unittest.TestCase):
         write_file(dst, 'bar')
         os_error = OSError(errno.EPERM, 'EPERM')
 
-        def _raise_on_user_foo(fname, attr, val):
+        def _raise_on_user_foo(fname, attr, val, **kwargs):
             if attr == 'user.foo':
                 raise os_error
             else:
-                orig_setxattr(fname, attr, val)
+                orig_setxattr(fname, attr, val, **kwargs)
         try:
             orig_setxattr = os.setxattr
             os.setxattr = _raise_on_user_foo
@@ -361,13 +361,13 @@ class TestShutil(unittest.TestCase):
         write_file(src, 'foo')
         os.symlink(src, src_link)
         os.setxattr(src, 'trusted.foo', b'42')
-        os.lsetxattr(src_link, 'trusted.foo', b'43')
+        os.setxattr(src_link, 'trusted.foo', b'43', follow_symlinks=False)
         dst = os.path.join(tmp_dir, 'bar')
         dst_link = os.path.join(tmp_dir, 'qux')
         write_file(dst, 'bar')
         os.symlink(dst, dst_link)
         shutil._copyxattr(src_link, dst_link, symlinks=True)
-        self.assertEqual(os.lgetxattr(dst_link, 'trusted.foo'), b'43')
+        self.assertEqual(os.getxattr(dst_link, 'trusted.foo', follow_symlinks=False), b'43')
         self.assertRaises(OSError, os.getxattr, dst, 'trusted.foo')
         shutil._copyxattr(src_link, dst, symlinks=True)
         self.assertEqual(os.getxattr(dst, 'trusted.foo'), b'43')
@@ -419,7 +419,7 @@ class TestShutil(unittest.TestCase):
         self.assertTrue(os.path.islink(dst))
         self.assertEqual(os.readlink(dst), os.readlink(src_link))
         dst_stat = os.lstat(dst)
-        if hasattr(os, 'lutimes'):
+        if os.utime in os.supports_follow_symlinks:
             for attr in 'st_atime', 'st_mtime':
                 # The modification times may be truncated in the new file.
                 self.assertLessEqual(getattr(src_link_stat, attr),

Misc/NEWS

@@ -40,6 +40,14 @@ Core and Builtins
 Library
 -------
 
+- Issue #14626: Large refactoring of functions / parameters in the os module.
+  Many functions now support "dir_fd" and "follow_symlinks" parameters;
+  some also support accepting an open file descriptor in place of of a path
+  string.  Added os.support_* collections as LBYL helpers.  Removed many
+  functions only previously seen in 3.3 alpha releases (often starting with
+  "f" or "l", or ending with "at").  Originally suggested by Serhiy Storchaka;
+  implemented by Larry Hastings.
+
 - Issue #15008: Implement PEP 362 "Signature Objects".
   Patch by Yury Selivanov.