shutil.py 49.1 KB
Newer Older
1
"""Utility functions for copying and archiving files and directory trees.
2

3
XXX The functions here don't copy the resource fork or other metadata on Mac.
4 5

"""
Guido van Rossum's avatar
Guido van Rossum committed
6

Guido van Rossum's avatar
Guido van Rossum committed
7
import os
8
import sys
9
import stat
Georg Brandl's avatar
Georg Brandl committed
10
import fnmatch
11
import collections
12
import errno
13 14 15 16 17 18 19

try:
    import zlib
    del zlib
    _ZLIB_SUPPORTED = True
except ImportError:
    _ZLIB_SUPPORTED = False
20

21 22
try:
    import bz2
Florent Xicluna's avatar
Florent Xicluna committed
23
    del bz2
24
    _BZ2_SUPPORTED = True
25
except ImportError:
26 27
    _BZ2_SUPPORTED = False

28 29 30 31 32 33 34
try:
    import lzma
    del lzma
    _LZMA_SUPPORTED = True
except ImportError:
    _LZMA_SUPPORTED = False

35 36
try:
    from pwd import getpwnam
37
except ImportError:
38 39 40 41
    getpwnam = None

try:
    from grp import getgrnam
42
except ImportError:
43
    getgrnam = None
Guido van Rossum's avatar
Guido van Rossum committed
44

45
_WINDOWS = os.name == 'nt'
46 47 48
posix = nt = None
if os.name == 'posix':
    import posix
49
elif _WINDOWS:
50 51
    import nt

52
COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024
53
_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux")
54
_HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile")  # macOS
55

56 57 58
__all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
           "copytree", "move", "rmtree", "Error", "SpecialFileError",
           "ExecError", "make_archive", "get_archive_formats",
59 60
           "register_archive_format", "unregister_archive_format",
           "get_unpack_formats", "register_unpack_format",
Éric Araujo's avatar
Éric Araujo committed
61
           "unregister_unpack_format", "unpack_archive",
62 63
           "ignore_patterns", "chown", "which", "get_terminal_size",
           "SameFileError"]
Éric Araujo's avatar
Éric Araujo committed
64
           # disk_usage is added later, if available on the platform
65

66
class Error(OSError):
67
    pass
Guido van Rossum's avatar
Guido van Rossum committed
68

69 70 71
class SameFileError(Error):
    """Raised when source and destination are the same file."""

72
class SpecialFileError(OSError):
73 74 75
    """Raised when trying to do a kind of operation (e.g. copying) which is
    not supported on a special file (e.g. a named pipe)"""

76
class ExecError(OSError):
77 78
    """Raised when a command could not be executed"""

79
class ReadError(OSError):
80 81 82
    """Raised when an archive cannot be read"""

class RegistryError(Exception):
83
    """Raised when a registry operation with the archiving
84
    and unpacking registries fails"""
85

86 87 88 89 90
class _GiveupOnFastCopy(Exception):
    """Raised as a signal to fallback on using raw read()/write()
    file copy when fast-copy functions fail to do so.
    """

91
def _fastcopy_fcopyfile(fsrc, fdst, flags):
92
    """Copy a regular file content or metadata by using high-performance
93
    fcopyfile(3) syscall (macOS).
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
    """
    try:
        infd = fsrc.fileno()
        outfd = fdst.fileno()
    except Exception as err:
        raise _GiveupOnFastCopy(err)  # not a regular file

    try:
        posix._fcopyfile(infd, outfd, flags)
    except OSError as err:
        err.filename = fsrc.name
        err.filename2 = fdst.name
        if err.errno in {errno.EINVAL, errno.ENOTSUP}:
            raise _GiveupOnFastCopy(err)
        else:
            raise err from None

def _fastcopy_sendfile(fsrc, fdst):
    """Copy data from one regular mmap-like fd to another by using
    high-performance sendfile(2) syscall.
114
    This should work on Linux >= 2.6.33 only.
115 116 117 118 119 120 121 122 123 124
    """
    # Note: copyfileobj() is left alone in order to not introduce any
    # unexpected breakage. Possible risks by using zero-copy calls
    # in copyfileobj() are:
    # - fdst cannot be open in "a"(ppend) mode
    # - fsrc and fdst may be open in "t"(ext) mode
    # - fsrc may be a BufferedReader (which hides unread data in a buffer),
    #   GzipFile (which decompresses data), HTTPResponse (which decodes
    #   chunks).
    # - possibly others (e.g. encrypted fs/partition?)
125
    global _USE_CP_SENDFILE
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
    try:
        infd = fsrc.fileno()
        outfd = fdst.fileno()
    except Exception as err:
        raise _GiveupOnFastCopy(err)  # not a regular file

    # Hopefully the whole file will be copied in a single call.
    # sendfile() is called in a loop 'till EOF is reached (0 return)
    # so a bufsize smaller or bigger than the actual file size
    # should not make any difference, also in case the file content
    # changes while being copied.
    try:
        blocksize = max(os.fstat(infd).st_size, 2 ** 23)  # min 8MB
    except Exception:
        blocksize = 2 ** 27  # 128MB

    offset = 0
    while True:
        try:
            sent = os.sendfile(outfd, infd, offset, blocksize)
        except OSError as err:
            # ...in oder to have a more informative exception.
            err.filename = fsrc.name
            err.filename2 = fdst.name

            if err.errno == errno.ENOTSOCK:
                # sendfile() on this platform (probably Linux < 2.6.33)
                # does not support copies between regular files (only
                # sockets).
155
                _USE_CP_SENDFILE = False
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
                raise _GiveupOnFastCopy(err)

            if err.errno == errno.ENOSPC:  # filesystem is full
                raise err from None

            # Give up on first call and if no data was copied.
            if offset == 0 and os.lseek(outfd, 0, os.SEEK_CUR) == 0:
                raise _GiveupOnFastCopy(err)

            raise err
        else:
            if sent == 0:
                break  # EOF
            offset += sent

171 172 173 174 175
def _copyfileobj_readinto(fsrc, fdst, length=COPY_BUFSIZE):
    """readinto()/memoryview() based variant of copyfileobj().
    *fsrc* must support readinto() method and both files must be
    open in binary mode.
    """
176 177 178 179 180 181 182 183 184
    # Localize variable access to minimize overhead.
    fsrc_readinto = fsrc.readinto
    fdst_write = fdst.write
    with memoryview(bytearray(length)) as mv:
        while True:
            n = fsrc_readinto(mv)
            if not n:
                break
            elif n < length:
185 186
                with mv[:n] as smv:
                    fdst.write(smv)
187 188 189
            else:
                fdst_write(mv)

190
def copyfileobj(fsrc, fdst, length=0):
191
    """copy data from file-like object fsrc to file-like object fdst"""
192
    # Localize variable access to minimize overhead.
193 194
    if not length:
        length = COPY_BUFSIZE
195 196 197 198 199 200 201
    fsrc_read = fsrc.read
    fdst_write = fdst.write
    while True:
        buf = fsrc_read(length)
        if not buf:
            break
        fdst_write(buf)
202

203 204
def _samefile(src, dst):
    # Macintosh, Unix.
205 206 207 208 209 210
    if isinstance(src, os.DirEntry) and hasattr(os.path, 'samestat'):
        try:
            return os.path.samestat(src.stat(), os.stat(dst))
        except OSError:
            return False

211
    if hasattr(os.path, 'samefile'):
212 213 214 215
        try:
            return os.path.samefile(src, dst)
        except OSError:
            return False
216 217 218 219

    # All other platforms: check for same pathname.
    return (os.path.normcase(os.path.abspath(src)) ==
            os.path.normcase(os.path.abspath(dst)))
Tim Peters's avatar
Tim Peters committed
220

221 222 223 224 225 226
def _stat(fn):
    return fn.stat() if isinstance(fn, os.DirEntry) else os.stat(fn)

def _islink(fn):
    return fn.is_symlink() if isinstance(fn, os.DirEntry) else os.path.islink(fn)

227
def copyfile(src, dst, *, follow_symlinks=True):
228
    """Copy data from src to dst in the most efficient way possible.
229

230
    If follow_symlinks is not set and src is a symbolic link, a new
231 232 233
    symlink will be created instead of copying the file it points to.

    """
234
    if _samefile(src, dst):
235
        raise SameFileError("{!r} and {!r} are the same file".format(src, dst))
236

237 238
    file_size = 0
    for i, fn in enumerate([src, dst]):
239
        try:
240
            st = _stat(fn)
241 242 243
        except OSError:
            # File most likely does not exist
            pass
244 245 246
        else:
            # XXX What about other special files? (sockets, devices...)
            if stat.S_ISFIFO(st.st_mode):
247
                fn = fn.path if isinstance(fn, os.DirEntry) else fn
248
                raise SpecialFileError("`%s` is a named pipe" % fn)
249 250
            if _WINDOWS and i == 0:
                file_size = st.st_size
251

252
    if not follow_symlinks and _islink(src):
253 254
        os.symlink(os.readlink(src), dst)
    else:
255
        with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
256 257
            # macOS
            if _HAS_FCOPYFILE:
258
                try:
259
                    _fastcopy_fcopyfile(fsrc, fdst, posix._COPYFILE_DATA)
260 261 262
                    return dst
                except _GiveupOnFastCopy:
                    pass
263 264
            # Linux
            elif _USE_CP_SENDFILE:
265
                try:
266
                    _fastcopy_sendfile(fsrc, fdst)
267 268 269
                    return dst
                except _GiveupOnFastCopy:
                    pass
270 271 272 273 274
            # Windows, see:
            # https://github.com/python/cpython/pull/7160#discussion_r195405230
            elif _WINDOWS and file_size > 0:
                _copyfileobj_readinto(fsrc, fdst, min(file_size, COPY_BUFSIZE))
                return dst
275

276
            copyfileobj(fsrc, fdst)
277

278
    return dst
279

280
def copymode(src, dst, *, follow_symlinks=True):
281
    """Copy mode bits from src to dst.
Guido van Rossum's avatar
Guido van Rossum committed
282

283 284 285
    If follow_symlinks is not set, symlinks aren't followed if and only
    if both `src` and `dst` are symlinks.  If `lchmod` isn't available
    (e.g. Linux) this method does nothing.
286 287

    """
288
    if not follow_symlinks and _islink(src) and os.path.islink(dst):
289 290 291 292 293
        if hasattr(os, 'lchmod'):
            stat_func, chmod_func = os.lstat, os.lchmod
        else:
            return
    else:
294
        stat_func, chmod_func = _stat, os.chmod
Guido van Rossum's avatar
Guido van Rossum committed
295

296 297 298
    st = stat_func(src)
    chmod_func(dst, stat.S_IMODE(st.st_mode))

299
if hasattr(os, 'listxattr'):
300
    def _copyxattr(src, dst, *, follow_symlinks=True):
301 302 303 304
        """Copy extended filesystem attributes from `src` to `dst`.

        Overwrite existing attributes.

305
        If `follow_symlinks` is false, symlinks won't be followed.
306 307 308

        """

309 310 311
        try:
            names = os.listxattr(src, follow_symlinks=follow_symlinks)
        except OSError as e:
312
            if e.errno not in (errno.ENOTSUP, errno.ENODATA, errno.EINVAL):
313 314 315
                raise
            return
        for name in names:
316
            try:
317 318
                value = os.getxattr(src, name, follow_symlinks=follow_symlinks)
                os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
319
            except OSError as e:
320 321
                if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA,
                                   errno.EINVAL):
322 323 324 325 326
                    raise
else:
    def _copyxattr(*args, **kwargs):
        pass

327
def copystat(src, dst, *, follow_symlinks=True):
328
    """Copy file metadata
329

330 331 332 333
    Copy the permission bits, last access time, last modification time, and
    flags from `src` to `dst`. On Linux, copystat() also copies the "extended
    attributes" where possible. The file contents, owner, and group are
    unaffected. `src` and `dst` are path names given as strings.
334

335 336
    If the optional flag `follow_symlinks` is not set, symlinks aren't
    followed if and only if both `src` and `dst` are symlinks.
337
    """
338
    def _nop(*args, ns=None, follow_symlinks=None):
339 340
        pass

341
    # follow symlinks (aka don't not follow symlinks)
342
    follow = follow_symlinks or not (_islink(src) and os.path.islink(dst))
343 344 345 346
    if follow:
        # use the real function if it exists
        def lookup(name):
            return getattr(os, name, _nop)
347
    else:
348 349 350 351 352 353 354 355
        # use the real function only if it exists
        # *and* it supports follow_symlinks
        def lookup(name):
            fn = getattr(os, name, _nop)
            if fn in os.supports_follow_symlinks:
                return fn
            return _nop

356 357 358 359
    if isinstance(src, os.DirEntry):
        st = src.stat(follow_symlinks=follow)
    else:
        st = lookup("stat")(src, follow_symlinks=follow)
360
    mode = stat.S_IMODE(st.st_mode)
361 362
    lookup("utime")(dst, ns=(st.st_atime_ns, st.st_mtime_ns),
        follow_symlinks=follow)
363 364 365
    # We must copy extended attributes before the file is (potentially)
    # chmod()'ed read-only, otherwise setxattr() will error with -EACCES.
    _copyxattr(src, dst, follow_symlinks=follow)
366 367 368 369 370 371 372
    try:
        lookup("chmod")(dst, mode, follow_symlinks=follow)
    except NotImplementedError:
        # if we got a NotImplementedError, it's because
        #   * follow_symlinks=False,
        #   * lchown() is unavailable, and
        #   * either
373
        #       * fchownat() is unavailable or
374 375 376 377 378 379
        #       * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW.
        #         (it returned ENOSUP.)
        # therefore we're out of options--we simply cannot chown the
        # symlink.  give up, suppress the error.
        # (which is what shutil always did in this circumstance.)
        pass
380
    if hasattr(st, 'st_flags'):
381
        try:
382
            lookup("chflags")(dst, st.st_flags, follow_symlinks=follow)
383
        except OSError as why:
384 385 386 387
            for err in 'EOPNOTSUPP', 'ENOTSUP':
                if hasattr(errno, err) and why.errno == getattr(errno, err):
                    break
            else:
388
                raise
389

390
def copy(src, dst, *, follow_symlinks=True):
391
    """Copy data and mode bits ("cp src dst"). Return the file's destination.
Tim Peters's avatar
Tim Peters committed
392

393 394
    The destination may be a directory.

395
    If follow_symlinks is false, symlinks won't be followed. This
396 397
    resembles GNU's "cp -P src dst".

398 399 400
    If source and destination are the same file, a SameFileError will be
    raised.

401
    """
Guido van Rossum's avatar
Guido van Rossum committed
402
    if os.path.isdir(dst):
403
        dst = os.path.join(dst, os.path.basename(src))
404 405
    copyfile(src, dst, follow_symlinks=follow_symlinks)
    copymode(src, dst, follow_symlinks=follow_symlinks)
406
    return dst
Guido van Rossum's avatar
Guido van Rossum committed
407

408
def copy2(src, dst, *, follow_symlinks=True):
409 410 411 412
    """Copy data and metadata. Return the file's destination.

    Metadata is copied with copystat(). Please see the copystat function
    for more information.
413 414 415

    The destination may be a directory.

416
    If follow_symlinks is false, symlinks won't be followed. This
417
    resembles GNU's "cp -P src dst".
418
    """
Guido van Rossum's avatar
Guido van Rossum committed
419
    if os.path.isdir(dst):
420
        dst = os.path.join(dst, os.path.basename(src))
421 422
    copyfile(src, dst, follow_symlinks=follow_symlinks)
    copystat(src, dst, follow_symlinks=follow_symlinks)
423
    return dst
Guido van Rossum's avatar
Guido van Rossum committed
424

Georg Brandl's avatar
Georg Brandl committed
425 426
def ignore_patterns(*patterns):
    """Function that can be used as copytree() ignore parameter.
427

Georg Brandl's avatar
Georg Brandl committed
428 429 430 431 432 433 434 435 436
    Patterns is a sequence of glob-style patterns
    that are used to exclude files"""
    def _ignore_patterns(path, names):
        ignored_names = []
        for pattern in patterns:
            ignored_names.extend(fnmatch.filter(names, pattern))
        return set(ignored_names)
    return _ignore_patterns

437
def _copytree(entries, src, dst, symlinks, ignore, copy_function,
438
              ignore_dangling_symlinks, dirs_exist_ok=False):
Georg Brandl's avatar
Georg Brandl committed
439
    if ignore is not None:
440
        ignored_names = ignore(src, set(os.listdir(src)))
Georg Brandl's avatar
Georg Brandl committed
441 442 443
    else:
        ignored_names = set()

444
    os.makedirs(dst, exist_ok=dirs_exist_ok)
445
    errors = []
446 447 448 449
    use_srcentry = copy_function is copy2 or copy_function is copy

    for srcentry in entries:
        if srcentry.name in ignored_names:
Georg Brandl's avatar
Georg Brandl committed
450
            continue
451 452 453
        srcname = os.path.join(src, srcentry.name)
        dstname = os.path.join(dst, srcentry.name)
        srcobj = srcentry if use_srcentry else srcname
454
        try:
455 456 457 458 459 460 461 462
            is_symlink = srcentry.is_symlink()
            if is_symlink and os.name == 'nt':
                # Special check for directory junctions, which appear as
                # symlinks but we want to recurse.
                lstat = srcentry.stat(follow_symlinks=False)
                if lstat.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT:
                    is_symlink = False
            if is_symlink:
463
                linkto = os.readlink(srcname)
464
                if symlinks:
465 466 467
                    # We can't just leave it to `copy_function` because legacy
                    # code with a custom `copy_function` may rely on copytree
                    # doing the right thing.
468
                    os.symlink(linkto, dstname)
469
                    copystat(srcobj, dstname, follow_symlinks=not symlinks)
470 471 472 473
                else:
                    # ignore dangling symlink if the flag is on
                    if not os.path.exists(linkto) and ignore_dangling_symlinks:
                        continue
474
                    # otherwise let the copy occur. copy2 will raise an error
475 476
                    if srcentry.is_dir():
                        copytree(srcobj, dstname, symlinks, ignore,
477
                                 copy_function, dirs_exist_ok=dirs_exist_ok)
478
                    else:
479 480
                        copy_function(srcobj, dstname)
            elif srcentry.is_dir():
481 482
                copytree(srcobj, dstname, symlinks, ignore, copy_function,
                         dirs_exist_ok=dirs_exist_ok)
483
            else:
484
                # Will raise a SpecialFileError for unsupported file types
485
                copy_function(srcobj, dstname)
486 487
        # catch the Error from the recursive copytree so that we can
        # continue with other files
488
        except Error as err:
489
            errors.extend(err.args[0])
490
        except OSError as why:
491
            errors.append((srcname, dstname, str(why)))
492 493
    try:
        copystat(src, dst)
494
    except OSError as why:
495
        # Copying file access times may fail on Windows
496
        if getattr(why, 'winerror', None) is None:
497
            errors.append((src, dst, str(why)))
498
    if errors:
499
        raise Error(errors)
500
    return dst
501

502
def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2,
503 504 505 506 507
             ignore_dangling_symlinks=False, dirs_exist_ok=False):
    """Recursively copy a directory tree and return the destination directory.

    dirs_exist_ok dictates whether to raise an exception in case dst or any
    missing parent directory already exists.
508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539

    If exception(s) occur, an Error is raised with a list of reasons.

    If the optional symlinks flag is true, symbolic links in the
    source tree result in symbolic links in the destination tree; if
    it is false, the contents of the files pointed to by symbolic
    links are copied. If the file pointed by the symlink doesn't
    exist, an exception will be added in the list of errors raised in
    an Error exception at the end of the copy process.

    You can set the optional ignore_dangling_symlinks flag to true if you
    want to silence this exception. Notice that this has no effect on
    platforms that don't support os.symlink.

    The optional ignore argument is a callable. If given, it
    is called with the `src` parameter, which is the directory
    being visited by copytree(), and `names` which is the list of
    `src` contents, as returned by os.listdir():

        callable(src, names) -> ignored_names

    Since copytree() is called recursively, the callable will be
    called once for each directory that is copied. It returns a
    list of names relative to the `src` directory that should
    not be copied.

    The optional copy_function argument is a callable that will be used
    to copy each file. It will be called with the source path and the
    destination path as arguments. By default, copy2() is used, but any
    function that supports the same signature (like copy()) can be used.

    """
540
    sys.audit("shutil.copytree", src, dst)
541 542 543
    with os.scandir(src) as entries:
        return _copytree(entries=entries, src=src, dst=dst, symlinks=symlinks,
                         ignore=ignore, copy_function=copy_function,
544 545
                         ignore_dangling_symlinks=ignore_dangling_symlinks,
                         dirs_exist_ok=dirs_exist_ok)
546

547
if hasattr(os.stat_result, 'st_file_attributes'):
548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577
    # Special handling for directory junctions to make them behave like
    # symlinks for shutil.rmtree, since in general they do not appear as
    # regular links.
    def _rmtree_isdir(entry):
        try:
            st = entry.stat(follow_symlinks=False)
            return (stat.S_ISDIR(st.st_mode) and not
                (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT
                 and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT))
        except OSError:
            return False

    def _rmtree_islink(path):
        try:
            st = os.lstat(path)
            return (stat.S_ISLNK(st.st_mode) or
                (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT
                 and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT))
        except OSError:
            return False
else:
    def _rmtree_isdir(entry):
        try:
            return entry.is_dir(follow_symlinks=False)
        except OSError:
            return False

    def _rmtree_islink(path):
        return os.path.islink(path)

578 579
# version vulnerable to race conditions
def _rmtree_unsafe(path, onerror):
580
    try:
581 582
        with os.scandir(path) as scandir_it:
            entries = list(scandir_it)
583
    except OSError:
584 585 586 587
        onerror(os.scandir, path, sys.exc_info())
        entries = []
    for entry in entries:
        fullname = entry.path
588
        if _rmtree_isdir(entry):
589 590 591 592 593 594 595 596 597
            try:
                if entry.is_symlink():
                    # This can only happen if someone replaces
                    # a directory with a symlink after the call to
                    # os.scandir or entry.is_dir above.
                    raise OSError("Cannot call rmtree on a symbolic link")
            except OSError:
                onerror(os.path.islink, fullname, sys.exc_info())
                continue
598
            _rmtree_unsafe(fullname, onerror)
599
        else:
600
            try:
601
                os.unlink(fullname)
602
            except OSError:
603
                onerror(os.unlink, fullname, sys.exc_info())
604 605
    try:
        os.rmdir(path)
606
    except OSError:
607
        onerror(os.rmdir, path, sys.exc_info())
608

609 610 611
# Version using fd-based APIs to protect against races
def _rmtree_safe_fd(topfd, path, onerror):
    try:
612 613
        with os.scandir(topfd) as scandir_it:
            entries = list(scandir_it)
614 615
    except OSError as err:
        err.filename = path
616 617 618 619
        onerror(os.scandir, path, sys.exc_info())
        return
    for entry in entries:
        fullname = os.path.join(path, entry.name)
620
        try:
621
            is_dir = entry.is_dir(follow_symlinks=False)
622
        except OSError:
623
            is_dir = False
624 625 626 627 628 629 630 631
        else:
            if is_dir:
                try:
                    orig_st = entry.stat(follow_symlinks=False)
                    is_dir = stat.S_ISDIR(orig_st.st_mode)
                except OSError:
                    onerror(os.lstat, fullname, sys.exc_info())
                    continue
632
        if is_dir:
633
            try:
634
                dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd)
635
            except OSError:
636
                onerror(os.open, fullname, sys.exc_info())
637 638 639 640
            else:
                try:
                    if os.path.samestat(orig_st, os.fstat(dirfd)):
                        _rmtree_safe_fd(dirfd, fullname, onerror)
641
                        try:
642
                            os.rmdir(entry.name, dir_fd=topfd)
643
                        except OSError:
644
                            onerror(os.rmdir, fullname, sys.exc_info())
645 646 647 648
                    else:
                        try:
                            # This can only happen if someone replaces
                            # a directory with a symlink after the call to
649
                            # os.scandir or stat.S_ISDIR above.
650 651 652 653
                            raise OSError("Cannot call rmtree on a symbolic "
                                          "link")
                        except OSError:
                            onerror(os.path.islink, fullname, sys.exc_info())
654 655 656 657
                finally:
                    os.close(dirfd)
        else:
            try:
658
                os.unlink(entry.name, dir_fd=topfd)
659
            except OSError:
660
                onerror(os.unlink, fullname, sys.exc_info())
661

662 663
_use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <=
                     os.supports_dir_fd and
664
                     os.scandir in os.supports_fd and
665
                     os.stat in os.supports_follow_symlinks)
666

667 668 669 670 671
def rmtree(path, ignore_errors=False, onerror=None):
    """Recursively delete a directory tree.

    If ignore_errors is set, errors are ignored; otherwise, if onerror
    is set, it is called to handle the error with arguments (func,
672
    path, exc_info) where func is platform and implementation dependent;
673 674 675 676 677
    path is the argument to that function that caused it to fail; and
    exc_info is a tuple returned by sys.exc_info().  If ignore_errors
    is false and onerror is None, an exception is raised.

    """
678
    sys.audit("shutil.rmtree", path)
679 680 681 682 683 684 685
    if ignore_errors:
        def onerror(*args):
            pass
    elif onerror is None:
        def onerror(*args):
            raise
    if _use_fd_functions:
686 687 688
        # While the unsafe rmtree works fine on bytes, the fd based does not.
        if isinstance(path, bytes):
            path = os.fsdecode(path)
689 690 691 692 693 694 695 696 697 698 699 700 701
        # Note: To guard against symlink races, we use the standard
        # lstat()/open()/fstat() trick.
        try:
            orig_st = os.lstat(path)
        except Exception:
            onerror(os.lstat, path, sys.exc_info())
            return
        try:
            fd = os.open(path, os.O_RDONLY)
        except Exception:
            onerror(os.lstat, path, sys.exc_info())
            return
        try:
702
            if os.path.samestat(orig_st, os.fstat(fd)):
703
                _rmtree_safe_fd(fd, path, onerror)
704 705
                try:
                    os.rmdir(path)
706
                except OSError:
707
                    onerror(os.rmdir, path, sys.exc_info())
708
            else:
709 710 711 712 713
                try:
                    # symlinks to directories are forbidden, see bug #1669
                    raise OSError("Cannot call rmtree on a symbolic link")
                except OSError:
                    onerror(os.path.islink, path, sys.exc_info())
714 715 716
        finally:
            os.close(fd)
    else:
717
        try:
718
            if _rmtree_islink(path):
719 720 721 722 723 724
                # symlinks to directories are forbidden, see bug #1669
                raise OSError("Cannot call rmtree on a symbolic link")
        except OSError:
            onerror(os.path.islink, path, sys.exc_info())
            # can't continue even if onerror hook returns
            return
725 726
        return _rmtree_unsafe(path, onerror)

727 728 729
# Allow introspection of whether or not the hardening against symlink
# attacks is supported on the current platform
rmtree.avoids_symlink_attacks = _use_fd_functions
730 731 732 733

def _basename(path):
    # A basename() variant which first strips the trailing slash, if present.
    # Thus we always get the last component of the path, even for directories.
734 735
    sep = os.path.sep + (os.path.altsep or '')
    return os.path.basename(path.rstrip(sep))
736

737
def move(src, dst, copy_function=copy2):
738
    """Recursively move a file or directory to another location. This is
739 740
    similar to the Unix "mv" command. Return the file or directory's
    destination.
741 742 743 744

    If the destination is a directory or a symlink to a directory, the source
    is moved inside the directory. The destination path must not already
    exist.
745

746 747 748 749
    If the destination already exists but is not a directory, it may be
    overwritten depending on os.rename() semantics.

    If the destination is on our current filesystem, then rename() is used.
750 751 752 753
    Otherwise, src is copied to the destination and then removed. Symlinks are
    recreated under the new name if os.rename() fails because of cross
    filesystem renames.

754 755 756 757 758
    The optional `copy_function` argument is a callable that will be used
    to copy the source or it will be delegated to `copytree`.
    By default, copy2() is used, but any function that supports the same
    signature (like copy()) can be used.

759 760 761 762
    A lot more could be done here...  A look at a mv.c shows a lot of
    the issues this implementation glosses over.

    """
763 764
    real_dst = dst
    if os.path.isdir(dst):
765 766 767 768 769 770
        if _samefile(src, dst):
            # We might be on a case insensitive filesystem,
            # perform the rename anyway.
            os.rename(src, dst)
            return

771 772 773
        real_dst = os.path.join(dst, _basename(src))
        if os.path.exists(real_dst):
            raise Error("Destination path '%s' already exists" % real_dst)
774
    try:
775
        os.rename(src, real_dst)
776
    except OSError:
777 778 779 780 781
        if os.path.islink(src):
            linkto = os.readlink(src)
            os.symlink(linkto, real_dst)
            os.unlink(src)
        elif os.path.isdir(src):
782
            if _destinsrc(src, dst):
783 784 785 786
                raise Error("Cannot move a directory '%s' into itself"
                            " '%s'." % (src, dst))
            copytree(src, real_dst, copy_function=copy_function,
                     symlinks=True)
787 788
            rmtree(src)
        else:
789
            copy_function(src, real_dst)
790
            os.unlink(src)
791
    return real_dst
792

793
def _destinsrc(src, dst):
794 795
    src = os.path.abspath(src)
    dst = os.path.abspath(dst)
796 797 798 799 800
    if not src.endswith(os.path.sep):
        src += os.path.sep
    if not dst.endswith(os.path.sep):
        dst += os.path.sep
    return dst.startswith(src)
801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830

def _get_gid(name):
    """Returns a gid, given a group name."""
    if getgrnam is None or name is None:
        return None
    try:
        result = getgrnam(name)
    except KeyError:
        result = None
    if result is not None:
        return result[2]
    return None

def _get_uid(name):
    """Returns an uid, given a user name."""
    if getpwnam is None or name is None:
        return None
    try:
        result = getpwnam(name)
    except KeyError:
        result = None
    if result is not None:
        return result[2]
    return None

def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0,
                  owner=None, group=None, logger=None):
    """Create a (possibly compressed) tar file from all the files under
    'base_dir'.

831
    'compress' must be "gzip" (the default), "bzip2", "xz", or None.
832 833 834 835 836

    'owner' and 'group' can be used to define an owner and a group for the
    archive that is being built. If not provided, the current owner and group
    will be used.

837
    The output tar file will be named 'base_name' +  ".tar", possibly plus
838
    the appropriate compression extension (".gz", ".bz2", or ".xz").
839 840 841

    Returns the output filename.
    """
842 843 844 845 846 847 848 849 850
    if compress is None:
        tar_compression = ''
    elif _ZLIB_SUPPORTED and compress == 'gzip':
        tar_compression = 'gz'
    elif _BZ2_SUPPORTED and compress == 'bzip2':
        tar_compression = 'bz2'
    elif _LZMA_SUPPORTED and compress == 'xz':
        tar_compression = 'xz'
    else:
851 852
        raise ValueError("bad value for 'compress', or compression format not "
                         "supported : {0}".format(compress))
853

854 855 856 857
    import tarfile  # late import for breaking circular dependency

    compress_ext = '.' + tar_compression if compress else ''
    archive_name = base_name + '.tar' + compress_ext
858
    archive_dir = os.path.dirname(archive_name)
859

860
    if archive_dir and not os.path.exists(archive_dir):
861
        if logger is not None:
Éric Araujo's avatar
Éric Araujo committed
862
            logger.info("creating %s", archive_dir)
863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882
        if not dry_run:
            os.makedirs(archive_dir)

    # creating the tarball
    if logger is not None:
        logger.info('Creating tar archive')

    uid = _get_uid(owner)
    gid = _get_gid(group)

    def _set_uid_gid(tarinfo):
        if gid is not None:
            tarinfo.gid = gid
            tarinfo.gname = group
        if uid is not None:
            tarinfo.uid = uid
            tarinfo.uname = owner
        return tarinfo

    if not dry_run:
883
        tar = tarfile.open(archive_name, 'w|%s' % tar_compression)
884 885 886 887 888 889 890 891 892 893
        try:
            tar.add(base_dir, filter=_set_uid_gid)
        finally:
            tar.close()

    return archive_name

def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None):
    """Create a zip file from all the files under 'base_dir'.

894 895
    The output zip file will be named 'base_name' + ".zip".  Returns the
    name of the output zip file.
896
    """
897
    import zipfile  # late import for breaking circular dependency
898

899 900 901
    zip_filename = base_name + ".zip"
    archive_dir = os.path.dirname(base_name)

902
    if archive_dir and not os.path.exists(archive_dir):
903 904 905 906 907
        if logger is not None:
            logger.info("creating %s", archive_dir)
        if not dry_run:
            os.makedirs(archive_dir)

908 909 910
    if logger is not None:
        logger.info("creating '%s' and adding '%s' to it",
                    zip_filename, base_dir)
911

912 913 914
    if not dry_run:
        with zipfile.ZipFile(zip_filename, "w",
                             compression=zipfile.ZIP_DEFLATED) as zf:
915
            path = os.path.normpath(base_dir)
916 917 918 919
            if path != os.curdir:
                zf.write(path, path)
                if logger is not None:
                    logger.info("adding '%s'", path)
920
            for dirpath, dirnames, filenames in os.walk(base_dir):
921 922 923 924 925
                for name in sorted(dirnames):
                    path = os.path.normpath(os.path.join(dirpath, name))
                    zf.write(path, path)
                    if logger is not None:
                        logger.info("adding '%s'", path)
926 927 928 929 930 931
                for name in filenames:
                    path = os.path.normpath(os.path.join(dirpath, name))
                    if os.path.isfile(path):
                        zf.write(path, path)
                        if logger is not None:
                            logger.info("adding '%s'", path)
932 933 934 935 936

    return zip_filename

_ARCHIVE_FORMATS = {
    'tar':   (_make_tarball, [('compress', None)], "uncompressed tar file"),
937 938 939 940 941 942
}

if _ZLIB_SUPPORTED:
    _ARCHIVE_FORMATS['gztar'] = (_make_tarball, [('compress', 'gzip')],
                                "gzip'ed tar-file")
    _ARCHIVE_FORMATS['zip'] = (_make_zipfile, [], "ZIP file")
943

944 945 946 947
if _BZ2_SUPPORTED:
    _ARCHIVE_FORMATS['bztar'] = (_make_tarball, [('compress', 'bzip2')],
                                "bzip2'ed tar-file")

948 949 950 951
if _LZMA_SUPPORTED:
    _ARCHIVE_FORMATS['xztar'] = (_make_tarball, [('compress', 'xz')],
                                "xz'ed tar-file")

952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972
def get_archive_formats():
    """Returns a list of supported formats for archiving and unarchiving.

    Each element of the returned sequence is a tuple (name, description)
    """
    formats = [(name, registry[2]) for name, registry in
               _ARCHIVE_FORMATS.items()]
    formats.sort()
    return formats

def register_archive_format(name, function, extra_args=None, description=''):
    """Registers an archive format.

    name is the name of the format. function is the callable that will be
    used to create archives. If provided, extra_args is a sequence of
    (name, value) tuples that will be passed as arguments to the callable.
    description can be provided to describe the format, and will be returned
    by the get_archive_formats() function.
    """
    if extra_args is None:
        extra_args = []
973
    if not callable(function):
974 975 976 977
        raise TypeError('The %s object is not callable' % function)
    if not isinstance(extra_args, (tuple, list)):
        raise TypeError('extra_args needs to be a sequence')
    for element in extra_args:
978
        if not isinstance(element, (tuple, list)) or len(element) !=2:
979 980 981 982 983 984 985 986 987 988 989 990
            raise TypeError('extra_args elements are : (arg_name, value)')

    _ARCHIVE_FORMATS[name] = (function, extra_args, description)

def unregister_archive_format(name):
    del _ARCHIVE_FORMATS[name]

def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
                 dry_run=0, owner=None, group=None, logger=None):
    """Create an archive file (eg. zip or tar).

    'base_name' is the name of the file to create, minus any format-specific
991 992
    extension; 'format' is the archive format: one of "zip", "tar", "gztar",
    "bztar", or "xztar".  Or any other registered format.
993 994 995 996 997 998 999 1000 1001 1002 1003

    'root_dir' is a directory that will be the root directory of the
    archive; ie. we typically chdir into 'root_dir' before creating the
    archive.  'base_dir' is the directory where we start archiving from;
    ie. 'base_dir' will be the common prefix of all files and
    directories in the archive.  'root_dir' and 'base_dir' both default
    to the current directory.  Returns the name of the archive file.

    'owner' and 'group' are used when creating a tar archive. By default,
    uses the current owner and group.
    """
1004
    sys.audit("shutil.make_archive", base_name, format, root_dir, base_dir)
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
    save_cwd = os.getcwd()
    if root_dir is not None:
        if logger is not None:
            logger.debug("changing into '%s'", root_dir)
        base_name = os.path.abspath(base_name)
        if not dry_run:
            os.chdir(root_dir)

    if base_dir is None:
        base_dir = os.curdir

    kwargs = {'dry_run': dry_run, 'logger': logger}

    try:
        format_info = _ARCHIVE_FORMATS[format]
    except KeyError:
1021
        raise ValueError("unknown archive format '%s'" % format) from None
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039

    func = format_info[0]
    for arg, val in format_info[1]:
        kwargs[arg] = val

    if format != 'zip':
        kwargs['owner'] = owner
        kwargs['group'] = group

    try:
        filename = func(base_name, base_dir, **kwargs)
    finally:
        if root_dir is not None:
            if logger is not None:
                logger.debug("changing back to '%s'", save_cwd)
            os.chdir(save_cwd)

    return filename
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066


def get_unpack_formats():
    """Returns a list of supported formats for unpacking.

    Each element of the returned sequence is a tuple
    (name, extensions, description)
    """
    formats = [(name, info[0], info[3]) for name, info in
               _UNPACK_FORMATS.items()]
    formats.sort()
    return formats

def _check_unpack_options(extensions, function, extra_args):
    """Checks what gets registered as an unpacker."""
    # first make sure no other unpacker is registered for this extension
    existing_extensions = {}
    for name, info in _UNPACK_FORMATS.items():
        for ext in info[0]:
            existing_extensions[ext] = name

    for extension in extensions:
        if extension in existing_extensions:
            msg = '%s is already registered for "%s"'
            raise RegistryError(msg % (extension,
                                       existing_extensions[extension]))

1067
    if not callable(function):
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093
        raise TypeError('The registered function must be a callable')


def register_unpack_format(name, extensions, function, extra_args=None,
                           description=''):
    """Registers an unpack format.

    `name` is the name of the format. `extensions` is a list of extensions
    corresponding to the format.

    `function` is the callable that will be
    used to unpack archives. The callable will receive archives to unpack.
    If it's unable to handle an archive, it needs to raise a ReadError
    exception.

    If provided, `extra_args` is a sequence of
    (name, value) tuples that will be passed as arguments to the callable.
    description can be provided to describe the format, and will be returned
    by the get_unpack_formats() function.
    """
    if extra_args is None:
        extra_args = []
    _check_unpack_options(extensions, function, extra_args)
    _UNPACK_FORMATS[name] = extensions, function, extra_args, description

def unregister_unpack_format(name):
1094
    """Removes the pack format from the registry."""
1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
    del _UNPACK_FORMATS[name]

def _ensure_directory(path):
    """Ensure that the parent directory of `path` exists"""
    dirname = os.path.dirname(path)
    if not os.path.isdir(dirname):
        os.makedirs(dirname)

def _unpack_zipfile(filename, extract_dir):
    """Unpack zip `filename` to `extract_dir`
    """
1106
    import zipfile  # late import for breaking circular dependency
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127

    if not zipfile.is_zipfile(filename):
        raise ReadError("%s is not a zip file" % filename)

    zip = zipfile.ZipFile(filename)
    try:
        for info in zip.infolist():
            name = info.filename

            # don't extract absolute paths or ones with .. in them
            if name.startswith('/') or '..' in name:
                continue

            target = os.path.join(extract_dir, *name.split('/'))
            if not target:
                continue

            _ensure_directory(target)
            if not name.endswith('/'):
                # file
                data = zip.read(info.filename)
1128
                f = open(target, 'wb')
1129 1130 1131 1132 1133 1134 1135 1136 1137
                try:
                    f.write(data)
                finally:
                    f.close()
                    del data
    finally:
        zip.close()

def _unpack_tarfile(filename, extract_dir):
1138
    """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir`
1139
    """
1140
    import tarfile  # late import for breaking circular dependency
1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152
    try:
        tarobj = tarfile.open(filename)
    except tarfile.TarError:
        raise ReadError(
            "%s is not a compressed or uncompressed tar file" % filename)
    try:
        tarobj.extractall(extract_dir)
    finally:
        tarobj.close()

_UNPACK_FORMATS = {
    'tar':   (['.tar'], _unpack_tarfile, [], "uncompressed tar file"),
1153 1154 1155 1156 1157 1158
    'zip':   (['.zip'], _unpack_zipfile, [], "ZIP file"),
}

if _ZLIB_SUPPORTED:
    _UNPACK_FORMATS['gztar'] = (['.tar.gz', '.tgz'], _unpack_tarfile, [],
                                "gzip'ed tar-file")
1159

1160
if _BZ2_SUPPORTED:
1161
    _UNPACK_FORMATS['bztar'] = (['.tar.bz2', '.tbz2'], _unpack_tarfile, [],
1162 1163
                                "bzip2'ed tar-file")

1164 1165 1166 1167
if _LZMA_SUPPORTED:
    _UNPACK_FORMATS['xztar'] = (['.tar.xz', '.txz'], _unpack_tarfile, [],
                                "xz'ed tar-file")

1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
def _find_unpack_format(filename):
    for name, info in _UNPACK_FORMATS.items():
        for extension in info[0]:
            if filename.endswith(extension):
                return name
    return None

def unpack_archive(filename, extract_dir=None, format=None):
    """Unpack an archive.

    `filename` is the name of the archive.

    `extract_dir` is the name of the target directory, where the archive
    is unpacked. If not provided, the current working directory is used.

1183 1184 1185 1186
    `format` is the archive format: one of "zip", "tar", "gztar", "bztar",
    or "xztar".  Or any other registered format.  If not provided,
    unpack_archive will use the filename extension and see if an unpacker
    was registered for that extension.
1187 1188 1189 1190 1191 1192

    In case none is found, a ValueError is raised.
    """
    if extract_dir is None:
        extract_dir = os.getcwd()

1193 1194 1195
    extract_dir = os.fspath(extract_dir)
    filename = os.fspath(filename)

1196 1197 1198 1199
    if format is not None:
        try:
            format_info = _UNPACK_FORMATS[format]
        except KeyError:
1200
            raise ValueError("Unknown unpack format '{0}'".format(format)) from None
1201

1202 1203
        func = format_info[1]
        func(filename, extract_dir, **dict(format_info[2]))
1204 1205 1206 1207 1208 1209 1210 1211 1212
    else:
        # we need to look at the registered unpackers supported extensions
        format = _find_unpack_format(filename)
        if format is None:
            raise ReadError("Unknown archive format '{0}'".format(filename))

        func = _UNPACK_FORMATS[format][1]
        kwargs = dict(_UNPACK_FORMATS[format][2])
        func(filename, extract_dir, **kwargs)
1213

Éric Araujo's avatar
Éric Araujo committed
1214 1215 1216 1217 1218

if hasattr(os, 'statvfs'):

    __all__.append('disk_usage')
    _ntuple_diskusage = collections.namedtuple('usage', 'total used free')
1219 1220 1221
    _ntuple_diskusage.total.__doc__ = 'Total space in bytes'
    _ntuple_diskusage.used.__doc__ = 'Used space in bytes'
    _ntuple_diskusage.free.__doc__ = 'Free space in bytes'
1222 1223

    def disk_usage(path):
Éric Araujo's avatar
Éric Araujo committed
1224 1225
        """Return disk usage statistics about the given path.

1226
        Returned value is a named tuple with attributes 'total', 'used' and
Éric Araujo's avatar
Éric Araujo committed
1227
        'free', which are the amount of total, used and free space, in bytes.
1228
        """
Éric Araujo's avatar
Éric Araujo committed
1229 1230 1231 1232 1233 1234
        st = os.statvfs(path)
        free = st.f_bavail * st.f_frsize
        total = st.f_blocks * st.f_frsize
        used = (st.f_blocks - st.f_bfree) * st.f_frsize
        return _ntuple_diskusage(total, used, free)

1235
elif _WINDOWS:
Éric Araujo's avatar
Éric Araujo committed
1236 1237 1238 1239 1240 1241 1242

    __all__.append('disk_usage')
    _ntuple_diskusage = collections.namedtuple('usage', 'total used free')

    def disk_usage(path):
        """Return disk usage statistics about the given path.

1243
        Returned values is a named tuple with attributes 'total', 'used' and
Éric Araujo's avatar
Éric Araujo committed
1244 1245 1246 1247
        'free', which are the amount of total, used and free space, in bytes.
        """
        total, free = nt._getdiskusage(path)
        used = total - free
1248
        return _ntuple_diskusage(total, used, free)
1249

1250

1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280
def chown(path, user=None, group=None):
    """Change owner user and group of the given path.

    user and group can be the uid/gid or the user/group names, and in that case,
    they are converted to their respective uid/gid.
    """

    if user is None and group is None:
        raise ValueError("user and/or group must be set")

    _user = user
    _group = group

    # -1 means don't change it
    if user is None:
        _user = -1
    # user can either be an int (the uid) or a string (the system username)
    elif isinstance(user, str):
        _user = _get_uid(user)
        if _user is None:
            raise LookupError("no such user: {!r}".format(user))

    if group is None:
        _group = -1
    elif not isinstance(group, int):
        _group = _get_gid(group)
        if _group is None:
            raise LookupError("no such group: {!r}".format(group))

    os.chown(path, _user, _group)
1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315

def get_terminal_size(fallback=(80, 24)):
    """Get the size of the terminal window.

    For each of the two dimensions, the environment variable, COLUMNS
    and LINES respectively, is checked. If the variable is defined and
    the value is a positive integer, it is used.

    When COLUMNS or LINES is not defined, which is the common case,
    the terminal connected to sys.__stdout__ is queried
    by invoking os.get_terminal_size.

    If the terminal size cannot be successfully queried, either because
    the system doesn't support querying, or because we are not
    connected to a terminal, the value given in fallback parameter
    is used. Fallback defaults to (80, 24) which is the default
    size used by many terminal emulators.

    The value returned is a named tuple of type os.terminal_size.
    """
    # columns, lines are the working values
    try:
        columns = int(os.environ['COLUMNS'])
    except (KeyError, ValueError):
        columns = 0

    try:
        lines = int(os.environ['LINES'])
    except (KeyError, ValueError):
        lines = 0

    # only query if necessary
    if columns <= 0 or lines <= 0:
        try:
            size = os.get_terminal_size(sys.__stdout__.fileno())
1316 1317 1318
        except (AttributeError, ValueError, OSError):
            # stdout is None, closed, detached, or not a terminal, or
            # os.get_terminal_size() is unsupported
1319 1320 1321 1322 1323 1324 1325
            size = os.terminal_size(fallback)
        if columns <= 0:
            columns = size.columns
        if lines <= 0:
            lines = size.lines

    return os.terminal_size((columns, lines))
1326

1327 1328 1329 1330 1331 1332 1333 1334 1335

# Check that a given file can be accessed with the correct mode.
# Additionally check that `file` is not a directory, as on Windows
# directories pass the os.access check.
def _access_check(fn, mode):
    return (os.path.exists(fn) and os.access(fn, mode)
            and not os.path.isdir(fn))


1336
def which(cmd, mode=os.F_OK | os.X_OK, path=None):
1337
    """Given a command, mode, and a PATH string, return the path which
1338 1339 1340 1341 1342 1343 1344 1345
    conforms to the given mode on the PATH, or None if there is no such
    file.

    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
    of os.environ.get("PATH"), or can be overridden with a custom search
    path.

    """
1346 1347 1348 1349 1350 1351 1352
    # If we're given a path with a directory part, look it up directly rather
    # than referring to PATH directories. This includes checking relative to the
    # current directory, e.g. ./script
    if os.path.dirname(cmd):
        if _access_check(cmd, mode):
            return cmd
        return None
1353

1354 1355
    use_bytes = isinstance(cmd, bytes)

1356
    if path is None:
1357 1358 1359 1360 1361 1362 1363 1364
        path = os.environ.get("PATH", None)
        if path is None:
            try:
                path = os.confstr("CS_PATH")
            except (AttributeError, ValueError):
                # os.confstr() or CS_PATH is not available
                path = os.defpath
        # bpo-35755: Don't use os.defpath if the PATH environment variable is
1365
        # set to an empty string
1366 1367

    # PATH='' doesn't match, whereas PATH=':' looks in the current directory
1368 1369
    if not path:
        return None
1370

1371 1372 1373 1374 1375 1376
    if use_bytes:
        path = os.fsencode(path)
        path = path.split(os.fsencode(os.pathsep))
    else:
        path = os.fsdecode(path)
        path = path.split(os.pathsep)
1377 1378 1379

    if sys.platform == "win32":
        # The current directory takes precedence on Windows.
1380 1381 1382 1383 1384
        curdir = os.curdir
        if use_bytes:
            curdir = os.fsencode(curdir)
        if curdir not in path:
            path.insert(0, curdir)
1385 1386 1387

        # PATHEXT is necessary to check on Windows.
        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
1388 1389
        if use_bytes:
            pathext = [os.fsencode(ext) for ext in pathext]
1390 1391
        # See if the given file matches any of the expected path extensions.
        # This will allow us to short circuit when given "python.exe".
1392 1393
        # If it does match, only test that one, otherwise we have to try
        # others.
1394 1395 1396 1397
        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
            files = [cmd]
        else:
            files = [cmd + ext for ext in pathext]
1398 1399 1400 1401 1402 1403 1404
    else:
        # On other platforms you don't have things like PATHEXT to tell you
        # what file suffixes are executable, so just pass on cmd as-is.
        files = [cmd]

    seen = set()
    for dir in path:
1405 1406 1407
        normdir = os.path.normcase(dir)
        if not normdir in seen:
            seen.add(normdir)
1408 1409 1410 1411 1412
            for thefile in files:
                name = os.path.join(dir, thefile)
                if _access_check(name, mode):
                    return name
    return None