tarfile.py 90.2 KB
Newer Older
1
#!/usr/bin/env python3
2 3 4
#-------------------------------------------------------------------
# tarfile.py
#-------------------------------------------------------------------
5
# Copyright (C) 2002 Lars Gustaebel <lars@gustaebel.de>
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
# All rights reserved.
#
# Permission  is  hereby granted,  free  of charge,  to  any person
# obtaining a  copy of  this software  and associated documentation
# files  (the  "Software"),  to   deal  in  the  Software   without
# restriction,  including  without limitation  the  rights to  use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies  of  the  Software,  and to  permit  persons  to  whom the
# Software  is  furnished  to  do  so,  subject  to  the  following
# conditions:
#
# The above copyright  notice and this  permission notice shall  be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS  IS", WITHOUT WARRANTY OF ANY  KIND,
# EXPRESS OR IMPLIED, INCLUDING  BUT NOT LIMITED TO  THE WARRANTIES
# OF  MERCHANTABILITY,  FITNESS   FOR  A  PARTICULAR   PURPOSE  AND
# NONINFRINGEMENT.  IN  NO  EVENT SHALL  THE  AUTHORS  OR COPYRIGHT
# HOLDERS  BE LIABLE  FOR ANY  CLAIM, DAMAGES  OR OTHER  LIABILITY,
# WHETHER  IN AN  ACTION OF  CONTRACT, TORT  OR OTHERWISE,  ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
"""Read from and write to tar format archives.
"""

__version__ = "$Revision$"

34
version     = "0.9.0"
35
__author__  = "Lars Gust\u00e4bel (lars@gustaebel.de)"
36 37
__date__    = "$Date$"
__cvsid__   = "$Id$"
38
__credits__ = "Gustavo Niemeyer, Niels Gust\u00e4bel, Richard Townsend."
39 40 41 42 43 44 45 46 47 48 49

#---------
# Imports
#---------
import sys
import os
import shutil
import stat
import errno
import time
import struct
50
import copy
51
import re
52 53 54 55 56 57

try:
    import grp, pwd
except ImportError:
    grp = pwd = None

58 59 60 61 62 63 64 65 66
# os.symlink on Windows prior to 6.0 raises NotImplementedError
symlink_exception = (AttributeError, NotImplementedError)
try:
    # WindowsError (1314) will be raised if the caller does not hold the
    # SeCreateSymbolicLinkPrivilege privilege
    symlink_exception += (WindowsError,)
except NameError:
    pass

67 68 69
# from tarfile import *
__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"]

70
from builtins import open as _open # Since 'open' is TarFile.open
71

72 73 74
#---------------------------------------------------------
# tar constants
#---------------------------------------------------------
75
NUL = b"\0"                     # the null character
76
BLOCKSIZE = 512                 # length of processing blocks
77
RECORDSIZE = BLOCKSIZE * 20     # length of records
78 79
GNU_MAGIC = b"ustar  \0"        # magic gnu tar string
POSIX_MAGIC = b"ustar\x0000"    # magic posix tar string
80

81 82 83
LENGTH_NAME = 100               # maximum length of a filename
LENGTH_LINK = 100               # maximum length of a linkname
LENGTH_PREFIX = 155             # maximum length of the prefix field
84

85 86 87 88 89 90 91 92 93
REGTYPE = b"0"                  # regular file
AREGTYPE = b"\0"                # regular file
LNKTYPE = b"1"                  # link (inside tarfile)
SYMTYPE = b"2"                  # symbolic link
CHRTYPE = b"3"                  # character special device
BLKTYPE = b"4"                  # block special device
DIRTYPE = b"5"                  # directory
FIFOTYPE = b"6"                 # fifo special device
CONTTYPE = b"7"                 # contiguous file
94

95 96 97
GNUTYPE_LONGNAME = b"L"         # GNU tar longname
GNUTYPE_LONGLINK = b"K"         # GNU tar longlink
GNUTYPE_SPARSE = b"S"           # GNU tar sparse file
98

99 100 101
XHDTYPE = b"x"                  # POSIX.1-2001 extended header
XGLTYPE = b"g"                  # POSIX.1-2001 global header
SOLARIS_XHDTYPE = b"X"          # Solaris extended header
102 103 104 105 106

USTAR_FORMAT = 0                # POSIX.1-1988 (ustar) format
GNU_FORMAT = 1                  # GNU tar format
PAX_FORMAT = 2                  # POSIX.1-2001 (pax) format
DEFAULT_FORMAT = GNU_FORMAT
107 108 109 110

#---------------------------------------------------------
# tarfile constants
#---------------------------------------------------------
111 112 113
# File types that tarfile supports:
SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE,
                   SYMTYPE, DIRTYPE, FIFOTYPE,
114 115 116 117
                   CONTTYPE, CHRTYPE, BLKTYPE,
                   GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
                   GNUTYPE_SPARSE)

118 119 120 121 122 123 124 125 126 127 128
# File types that will be treated as a regular file.
REGULAR_TYPES = (REGTYPE, AREGTYPE,
                 CONTTYPE, GNUTYPE_SPARSE)

# File types that are part of the GNU tar format.
GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
             GNUTYPE_SPARSE)

# Fields from a pax header that override a TarInfo attribute.
PAX_FIELDS = ("path", "linkpath", "size", "mtime",
              "uid", "gid", "uname", "gname")
129

130 131 132
# Fields from a pax header that are affected by hdrcharset.
PAX_NAME_FIELDS = {"path", "linkpath", "uname", "gname"}

133 134 135 136 137 138 139 140 141 142 143
# Fields in a pax header that are numbers, all other fields
# are treated as strings.
PAX_NUMBER_FIELDS = {
    "atime": float,
    "ctime": float,
    "mtime": float,
    "uid": int,
    "gid": int,
    "size": int
}

144 145 146
#---------------------------------------------------------
# Bits used in the mode field, values in octal.
#---------------------------------------------------------
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
S_IFLNK = 0o120000        # symbolic link
S_IFREG = 0o100000        # regular file
S_IFBLK = 0o060000        # block device
S_IFDIR = 0o040000        # directory
S_IFCHR = 0o020000        # character device
S_IFIFO = 0o010000        # fifo

TSUID   = 0o4000          # set UID on execution
TSGID   = 0o2000          # set GID on execution
TSVTX   = 0o1000          # reserved

TUREAD  = 0o400           # read by owner
TUWRITE = 0o200           # write by owner
TUEXEC  = 0o100           # execute/search by owner
TGREAD  = 0o040           # read by group
TGWRITE = 0o020           # write by group
TGEXEC  = 0o010           # execute/search by group
TOREAD  = 0o004           # read by other
TOWRITE = 0o002           # write by other
TOEXEC  = 0o001           # execute/search by other
167

168 169 170
#---------------------------------------------------------
# initialization
#---------------------------------------------------------
171 172 173 174
if os.name in ("nt", "ce"):
    ENCODING = "utf-8"
else:
    ENCODING = sys.getfilesystemencoding()
175

176 177 178
#---------------------------------------------------------
# Some useful functions
#---------------------------------------------------------
179

180 181
def stn(s, length, encoding, errors):
    """Convert a string to a null-terminated bytes object.
182
    """
183
    s = s.encode(encoding, errors)
184
    return s[:length] + (length - len(s)) * NUL
185

186 187
def nts(s, encoding, errors):
    """Convert a null-terminated bytes object to a string.
188
    """
189 190 191 192
    p = s.find(b"\0")
    if p != -1:
        s = s[:p]
    return s.decode(encoding, errors)
193

194 195 196 197 198
def nti(s):
    """Convert a number field to a python number.
    """
    # There are two possible encodings for a number field, see
    # itn() below.
199
    if s[0] != chr(0o200):
200
        try:
201
            n = int(nts(s, "ascii", "strict") or "0", 8)
202
        except ValueError:
203
            raise InvalidHeaderError("invalid header")
204
    else:
205
        n = 0
206
        for i in range(len(s) - 1):
207 208 209 210
            n <<= 8
            n += ord(s[i + 1])
    return n

211
def itn(n, digits=8, format=DEFAULT_FORMAT):
212 213 214 215 216
    """Convert a python number to a number field.
    """
    # POSIX 1003.1-1988 requires numbers to be encoded as a string of
    # octal digits followed by a null-byte, this allows values up to
    # (8**(digits-1))-1. GNU tar allows storing numbers greater than
217
    # that if necessary. A leading 0o200 byte indicates this particular
218 219 220
    # encoding, the following digits-1 bytes are a big-endian
    # representation. This allows values up to (256**(digits-1))-1.
    if 0 <= n < 8 ** (digits - 1):
221
        s = bytes("%0*o" % (digits - 1, n), "ascii") + NUL
222
    else:
223
        if format != GNU_FORMAT or n >= 256 ** (digits - 1):
224 225 226 227 228 229 230
            raise ValueError("overflow in number field")

        if n < 0:
            # XXX We mimic GNU tar's behaviour with negative numbers,
            # this could raise OverflowError.
            n = struct.unpack("L", struct.pack("l", n))[0]

231
        s = bytearray()
232
        for i in range(digits - 1):
233
            s.insert(0, n & 0o377)
234
            n >>= 8
235
        s.insert(0, 0o200)
236 237 238 239 240 241 242 243 244 245
    return s

def calc_chksums(buf):
    """Calculate the checksum for a member's header by summing up all
       characters except for the chksum field which is treated as if
       it was filled with spaces. According to the GNU tar sources,
       some tars (Sun and NeXT) calculate chksum with signed char,
       which will be different if there are chars in the buffer with
       the high bit set. So we calculate two checksums, unsigned and
       signed.
246
    """
247 248 249
    unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
    signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
    return unsigned_chksum, signed_chksum
250 251 252 253 254 255 256 257 258 259 260 261 262

def copyfileobj(src, dst, length=None):
    """Copy length bytes from fileobj src to fileobj dst.
       If length is None, copy the entire content.
    """
    if length == 0:
        return
    if length is None:
        shutil.copyfileobj(src, dst)
        return

    BUFSIZE = 16 * 1024
    blocks, remainder = divmod(length, BUFSIZE)
263
    for b in range(blocks):
264 265
        buf = src.read(BUFSIZE)
        if len(buf) < BUFSIZE:
266
            raise IOError("end of file reached")
267 268 269 270 271
        dst.write(buf)

    if remainder != 0:
        buf = src.read(remainder)
        if len(buf) < remainder:
272
            raise IOError("end of file reached")
273 274 275 276
        dst.write(buf)
    return

filemode_table = (
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
    ((S_IFLNK,      "l"),
     (S_IFREG,      "-"),
     (S_IFBLK,      "b"),
     (S_IFDIR,      "d"),
     (S_IFCHR,      "c"),
     (S_IFIFO,      "p")),

    ((TUREAD,       "r"),),
    ((TUWRITE,      "w"),),
    ((TUEXEC|TSUID, "s"),
     (TSUID,        "S"),
     (TUEXEC,       "x")),

    ((TGREAD,       "r"),),
    ((TGWRITE,      "w"),),
    ((TGEXEC|TSGID, "s"),
     (TSGID,        "S"),
     (TGEXEC,       "x")),

    ((TOREAD,       "r"),),
    ((TOWRITE,      "w"),),
    ((TOEXEC|TSVTX, "t"),
     (TSVTX,        "T"),
     (TOEXEC,       "x"))
)
302 303 304 305 306 307

def filemode(mode):
    """Convert a file's mode to a string of the form
       -rwxrwxrwx.
       Used by TarFile.list()
    """
308 309 310 311 312 313 314 315 316
    perm = []
    for table in filemode_table:
        for bit, char in table:
            if mode & bit == bit:
                perm.append(char)
                break
        else:
            perm.append("-")
    return "".join(perm)
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332

class TarError(Exception):
    """Base exception."""
    pass
class ExtractError(TarError):
    """General exception for extract errors."""
    pass
class ReadError(TarError):
    """Exception for unreadble tar archives."""
    pass
class CompressionError(TarError):
    """Exception for unavailable compression methods."""
    pass
class StreamError(TarError):
    """Exception for unsupported operations on stream-like TarFiles."""
    pass
333
class HeaderError(TarError):
334 335 336 337 338 339 340 341 342 343 344 345
    """Base exception for header errors."""
    pass
class EmptyHeaderError(HeaderError):
    """Exception for empty headers."""
    pass
class TruncatedHeaderError(HeaderError):
    """Exception for truncated headers."""
    pass
class EOFHeaderError(HeaderError):
    """Exception for end of file headers."""
    pass
class InvalidHeaderError(HeaderError):
346 347
    """Exception for invalid headers."""
    pass
348 349 350
class SubsequentHeaderError(HeaderError):
    """Exception for missing and invalid extended headers."""
    pass
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367

#---------------------------
# internal stream interface
#---------------------------
class _LowLevelFile:
    """Low-level file object. Supports reading and writing.
       It is used instead of a regular file object for streaming
       access.
    """

    def __init__(self, name, mode):
        mode = {
            "r": os.O_RDONLY,
            "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
        }[mode]
        if hasattr(os, "O_BINARY"):
            mode |= os.O_BINARY
368
        self.fd = os.open(name, mode, 0o666)
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389

    def close(self):
        os.close(self.fd)

    def read(self, size):
        return os.read(self.fd, size)

    def write(self, s):
        os.write(self.fd, s)

class _Stream:
    """Class that serves as an adapter between TarFile and
       a stream-like object.  The stream-like object only
       needs to have a read() or write() method and is accessed
       blockwise.  Use of gzip or bzip2 compression is possible.
       A stream-like object could be for example: sys.stdin,
       sys.stdout, a socket, a tape device etc.

       _Stream is intended to be used only internally.
    """

390
    def __init__(self, name, mode, comptype, fileobj, bufsize):
391 392 393 394 395 396 397
        """Construct a _Stream object.
        """
        self._extfileobj = True
        if fileobj is None:
            fileobj = _LowLevelFile(name, mode)
            self._extfileobj = False

398 399 400 401 402 403 404 405 406 407 408
        if comptype == '*':
            # Enable transparent compression detection for the
            # stream interface
            fileobj = _StreamProxy(fileobj)
            comptype = fileobj.getcomptype()

        self.name     = name or ""
        self.mode     = mode
        self.comptype = comptype
        self.fileobj  = fileobj
        self.bufsize  = bufsize
409
        self.buf      = b""
410
        self.pos      = 0
411 412
        self.closed   = False

413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
        try:
            if comptype == "gz":
                try:
                    import zlib
                except ImportError:
                    raise CompressionError("zlib module is not available")
                self.zlib = zlib
                self.crc = zlib.crc32(b"")
                if mode == "r":
                    self._init_read_gz()
                else:
                    self._init_write_gz()

            if comptype == "bz2":
                try:
                    import bz2
                except ImportError:
                    raise CompressionError("bz2 module is not available")
                if mode == "r":
                    self.dbuf = b""
                    self.cmp = bz2.BZ2Decompressor()
                else:
                    self.cmp = bz2.BZ2Compressor()
        except:
            if not self._extfileobj:
                self.fileobj.close()
            self.closed = True
            raise
441 442

    def __del__(self):
443
        if hasattr(self, "closed") and not self.closed:
444 445 446 447 448 449 450 451 452
            self.close()

    def _init_write_gz(self):
        """Initialize for writing with gzip compression.
        """
        self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED,
                                            -self.zlib.MAX_WBITS,
                                            self.zlib.DEF_MEM_LEVEL,
                                            0)
453
        timestamp = struct.pack("<L", int(time.time()))
454
        self.__write(b"\037\213\010\010" + timestamp + b"\002\377")
455 456
        if self.name.endswith(".gz"):
            self.name = self.name[:-3]
457 458
        # RFC1952 says we must use ISO-8859-1 for the FNAME field.
        self.__write(self.name.encode("iso-8859-1", "replace") + NUL)
459 460 461 462

    def write(self, s):
        """Write string s to the stream.
        """
463
        if self.comptype == "gz":
464 465
            self.crc = self.zlib.crc32(s, self.crc)
        self.pos += len(s)
466
        if self.comptype != "tar":
467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485
            s = self.cmp.compress(s)
        self.__write(s)

    def __write(self, s):
        """Write string s to the stream if a whole new block
           is ready to be written.
        """
        self.buf += s
        while len(self.buf) > self.bufsize:
            self.fileobj.write(self.buf[:self.bufsize])
            self.buf = self.buf[self.bufsize:]

    def close(self):
        """Close the _Stream object. No operation should be
           done on it afterwards.
        """
        if self.closed:
            return

486
        if self.mode == "w" and self.comptype != "tar":
487
            self.buf += self.cmp.flush()
488

489 490
        if self.mode == "w" and self.buf:
            self.fileobj.write(self.buf)
491
            self.buf = b""
492
            if self.comptype == "gz":
493 494 495 496 497 498
                # The native zlib crc is an unsigned 32-bit integer, but
                # the Python wrapper implicitly casts that to a signed C
                # long.  So, on a 32-bit box self.crc may "look negative",
                # while the same crc on a 64-bit box may "look positive".
                # To avoid irksome warnings from the `struct` module, force
                # it to look positive on all boxes.
499 500
                self.fileobj.write(struct.pack("<L", self.crc & 0xffffffff))
                self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFF))
501 502 503 504 505 506 507 508 509 510

        if not self._extfileobj:
            self.fileobj.close()

        self.closed = True

    def _init_read_gz(self):
        """Initialize for reading a gzip compressed fileobj.
        """
        self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS)
511
        self.dbuf = b""
512 513

        # taken from gzip.GzipFile with some alterations
514
        if self.__read(2) != b"\037\213":
515
            raise ReadError("not a gzip file")
516
        if self.__read(1) != b"\010":
517
            raise CompressionError("unsupported compression method")
518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548

        flag = ord(self.__read(1))
        self.__read(6)

        if flag & 4:
            xlen = ord(self.__read(1)) + 256 * ord(self.__read(1))
            self.read(xlen)
        if flag & 8:
            while True:
                s = self.__read(1)
                if not s or s == NUL:
                    break
        if flag & 16:
            while True:
                s = self.__read(1)
                if not s or s == NUL:
                    break
        if flag & 2:
            self.__read(2)

    def tell(self):
        """Return the stream's file pointer position.
        """
        return self.pos

    def seek(self, pos=0):
        """Set the stream's file pointer to pos. Negative seeking
           is forbidden.
        """
        if pos - self.pos >= 0:
            blocks, remainder = divmod(pos - self.pos, self.bufsize)
549
            for i in range(blocks):
550 551 552
                self.read(self.bufsize)
            self.read(remainder)
        else:
553
            raise StreamError("seeking backwards is not allowed")
554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
        return self.pos

    def read(self, size=None):
        """Return the next size number of bytes from the stream.
           If size is not defined, return all bytes of the stream
           up to EOF.
        """
        if size is None:
            t = []
            while True:
                buf = self._read(self.bufsize)
                if not buf:
                    break
                t.append(buf)
            buf = "".join(t)
        else:
            buf = self._read(size)
        self.pos += len(buf)
        return buf

    def _read(self, size):
        """Return size bytes from the stream.
        """
577
        if self.comptype == "tar":
578 579 580 581 582 583 584
            return self.__read(size)

        c = len(self.dbuf)
        while c < size:
            buf = self.__read(self.bufsize)
            if not buf:
                break
585 586 587 588
            try:
                buf = self.cmp.decompress(buf)
            except IOError:
                raise ReadError("invalid compressed data")
589
            self.dbuf += buf
590
            c += len(buf)
591 592 593
        buf = self.dbuf[:size]
        self.dbuf = self.dbuf[size:]
        return buf
594 595 596 597 598 599 600 601 602 603

    def __read(self, size):
        """Return size bytes from stream. If internal buffer is empty,
           read another block from the stream.
        """
        c = len(self.buf)
        while c < size:
            buf = self.fileobj.read(self.bufsize)
            if not buf:
                break
604
            self.buf += buf
605
            c += len(buf)
606 607 608
        buf = self.buf[:size]
        self.buf = self.buf[size:]
        return buf
609 610
# class _Stream

611 612 613 614 615 616 617 618 619 620 621 622 623 624
class _StreamProxy(object):
    """Small proxy class that enables transparent compression
       detection for the Stream interface (mode 'r|*').
    """

    def __init__(self, fileobj):
        self.fileobj = fileobj
        self.buf = self.fileobj.read(BLOCKSIZE)

    def read(self, size):
        self.read = self.fileobj.read
        return self.buf

    def getcomptype(self):
625
        if self.buf.startswith(b"\037\213\010"):
626
            return "gz"
627
        if self.buf.startswith(b"BZh91"):
628 629 630 631 632 633 634
            return "bz2"
        return "tar"

    def close(self):
        self.fileobj.close()
# class StreamProxy

635 636 637 638 639 640 641 642 643 644 645 646 647
class _BZ2Proxy(object):
    """Small proxy class that enables external file object
       support for "r:bz2" and "w:bz2" modes. This is actually
       a workaround for a limitation in bz2 module's BZ2File
       class which (unlike gzip.GzipFile) has no support for
       a file object argument.
    """

    blocksize = 16 * 1024

    def __init__(self, fileobj, mode):
        self.fileobj = fileobj
        self.mode = mode
648
        self.name = getattr(self.fileobj, "name", None)
649 650 651 652 653 654 655 656
        self.init()

    def init(self):
        import bz2
        self.pos = 0
        if self.mode == "r":
            self.bz2obj = bz2.BZ2Decompressor()
            self.fileobj.seek(0)
657
            self.buf = b""
658 659 660 661 662 663
        else:
            self.bz2obj = bz2.BZ2Compressor()

    def read(self, size):
        x = len(self.buf)
        while x < size:
664 665
            raw = self.fileobj.read(self.blocksize)
            if not raw:
666
                break
667 668
            data = self.bz2obj.decompress(raw)
            self.buf += data
669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
            x += len(data)

        buf = self.buf[:size]
        self.buf = self.buf[size:]
        self.pos += len(buf)
        return buf

    def seek(self, pos):
        if pos < self.pos:
            self.init()
        self.read(pos - self.pos)

    def tell(self):
        return self.pos

    def write(self, data):
        self.pos += len(data)
        raw = self.bz2obj.compress(data)
        self.fileobj.write(raw)

    def close(self):
        if self.mode == "w":
            raw = self.bz2obj.flush()
            self.fileobj.write(raw)
# class _BZ2Proxy

695 696 697
#------------------------
# Extraction file object
#------------------------
698 699 700 701
class _FileInFile(object):
    """A thin wrapper around an existing file object that
       provides a part of its data as an individual file
       object.
702 703
    """

704
    def __init__(self, fileobj, offset, size, blockinfo=None):
705 706 707 708
        self.fileobj = fileobj
        self.offset = offset
        self.size = size
        self.position = 0
709

710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726
        if blockinfo is None:
            blockinfo = [(0, size)]

        # Construct a map with data and zero blocks.
        self.map_index = 0
        self.map = []
        lastpos = 0
        realpos = self.offset
        for offset, size in blockinfo:
            if offset > lastpos:
                self.map.append((False, lastpos, offset, None))
            self.map.append((True, offset, offset + size, realpos))
            realpos += size
            lastpos = offset + size
        if lastpos < self.size:
            self.map.append((False, lastpos, self.size, None))

727 728 729 730 731 732
    def seekable(self):
        if not hasattr(self.fileobj, "seekable"):
            # XXX gzip.GzipFile and bz2.BZ2File
            return True
        return self.fileobj.seekable()

733 734
    def tell(self):
        """Return the current file position.
735
        """
736
        return self.position
737

738 739
    def seek(self, position):
        """Seek to a position in the file.
740
        """
741
        self.position = position
742

743 744 745 746 747
    def read(self, size=None):
        """Read data from the file.
        """
        if size is None:
            size = self.size - self.position
748
        else:
749
            size = min(size, self.size - self.position)
750

751
        buf = b""
752
        while size > 0:
753 754 755 756 757 758 759 760 761 762
            while True:
                data, start, stop, offset = self.map[self.map_index]
                if start <= self.position < stop:
                    break
                else:
                    self.map_index += 1
                    if self.map_index == len(self.map):
                        self.map_index = 0
            length = min(size, stop - self.position)
            if data:
763 764
                self.fileobj.seek(offset + (self.position - start))
                buf += self.fileobj.read(length)
765 766 767 768 769
            else:
                buf += NUL * length
            size -= length
            self.position += length
        return buf
770 771 772 773 774 775 776 777 778 779 780 781 782
#class _FileInFile


class ExFileObject(object):
    """File-like object for reading an archive member.
       Is returned by TarFile.extractfile().
    """
    blocksize = 1024

    def __init__(self, tarfile, tarinfo):
        self.fileobj = _FileInFile(tarfile.fileobj,
                                   tarinfo.offset_data,
                                   tarinfo.size,
783
                                   tarinfo.sparse)
784 785 786 787 788 789
        self.name = tarinfo.name
        self.mode = "r"
        self.closed = False
        self.size = tarinfo.size

        self.position = 0
790 791 792 793 794 795 796 797 798 799
        self.buffer = b""

    def readable(self):
        return True

    def writable(self):
        return False

    def seekable(self):
        return self.fileobj.seekable()
800 801 802 803 804 805 806 807

    def read(self, size=None):
        """Read at most size bytes from the file. If size is not
           present or None, read all data until EOF is reached.
        """
        if self.closed:
            raise ValueError("I/O operation on closed file")

808
        buf = b""
809 810 811
        if self.buffer:
            if size is None:
                buf = self.buffer
812
                self.buffer = b""
813 814 815 816 817 818 819 820 821 822 823 824
            else:
                buf = self.buffer[:size]
                self.buffer = self.buffer[size:]

        if size is None:
            buf += self.fileobj.read()
        else:
            buf += self.fileobj.read(size - len(buf))

        self.position += len(buf)
        return buf

825 826 827
    # XXX TextIOWrapper uses the read1() method.
    read1 = read

828 829 830 831 832 833 834 835
    def readline(self, size=-1):
        """Read one entire line from the file. If size is present
           and non-negative, return a string with at most that
           size, which may be an incomplete line.
        """
        if self.closed:
            raise ValueError("I/O operation on closed file")

836 837 838
        pos = self.buffer.find(b"\n") + 1
        if pos == 0:
            # no newline found.
839 840
            while True:
                buf = self.fileobj.read(self.blocksize)
841 842 843
                self.buffer += buf
                if not buf or b"\n" in buf:
                    pos = self.buffer.find(b"\n") + 1
844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865
                    if pos == 0:
                        # no newline found.
                        pos = len(self.buffer)
                    break

        if size != -1:
            pos = min(size, pos)

        buf = self.buffer[:pos]
        self.buffer = self.buffer[pos:]
        self.position += len(buf)
        return buf

    def readlines(self):
        """Return a list with all remaining lines.
        """
        result = []
        while True:
            line = self.readline()
            if not line: break
            result.append(line)
        return result
866 867 868 869

    def tell(self):
        """Return the current file position.
        """
870 871 872 873
        if self.closed:
            raise ValueError("I/O operation on closed file")

        return self.position
874

875
    def seek(self, pos, whence=os.SEEK_SET):
876 877
        """Seek to a position in the file.
        """
878 879 880 881 882 883
        if self.closed:
            raise ValueError("I/O operation on closed file")

        if whence == os.SEEK_SET:
            self.position = min(max(pos, 0), self.size)
        elif whence == os.SEEK_CUR:
884
            if pos < 0:
885
                self.position = max(self.position + pos, 0)
886
            else:
887 888 889 890 891 892
                self.position = min(self.position + pos, self.size)
        elif whence == os.SEEK_END:
            self.position = max(min(self.size + pos, self.size), 0)
        else:
            raise ValueError("Invalid argument")

893
        self.buffer = b""
894
        self.fileobj.seek(self.position)
895 896 897 898 899

    def close(self):
        """Close the file object.
        """
        self.closed = True
900 901

    def __iter__(self):
902
        """Get an iterator over the file's lines.
903
        """
904 905 906 907 908
        while True:
            line = self.readline()
            if not line:
                break
            yield line
909 910 911 912 913 914 915 916 917 918 919 920 921
#class ExFileObject

#------------------
# Exported Classes
#------------------
class TarInfo(object):
    """Informational class which holds the details about an
       archive member given by a tar header block.
       TarInfo objects are returned by TarFile.getmember(),
       TarFile.getmembers() and TarFile.gettarinfo() and are
       usually created internally.
    """

922 923 924 925 926 927
    __slots__ = ("name", "mode", "uid", "gid", "size", "mtime",
                 "chksum", "type", "linkname", "uname", "gname",
                 "devmajor", "devminor",
                 "offset", "offset_data", "pax_headers", "sparse",
                 "tarfile", "_sparse_structs", "_link_target")

928 929 930 931
    def __init__(self, name=""):
        """Construct a TarInfo object. name is the optional name
           of the member.
        """
932
        self.name = name        # member name
933
        self.mode = 0o644       # file permissions
934 935 936 937 938 939 940
        self.uid = 0            # user id
        self.gid = 0            # group id
        self.size = 0           # file size
        self.mtime = 0          # modification time
        self.chksum = 0         # header checksum
        self.type = REGTYPE     # member type
        self.linkname = ""      # link name
941 942
        self.uname = ""         # user name
        self.gname = ""         # group name
943 944 945 946 947
        self.devmajor = 0       # device major number
        self.devminor = 0       # device minor number

        self.offset = 0         # the tar header starts here
        self.offset_data = 0    # the file's data starts here
948

949
        self.sparse = None      # sparse member information
950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965
        self.pax_headers = {}   # pax header information

    # In pax headers the "name" and "linkname" field are called
    # "path" and "linkpath".
    def _getpath(self):
        return self.name
    def _setpath(self, name):
        self.name = name
    path = property(_getpath, _setpath)

    def _getlinkpath(self):
        return self.linkname
    def _setlinkpath(self, linkname):
        self.linkname = linkname
    linkpath = property(_getlinkpath, _setlinkpath)

966 967 968
    def __repr__(self):
        return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self))

969
    def get_info(self):
970 971 972
        """Return the TarInfo's attributes as a dictionary.
        """
        info = {
973
            "name":     self.name,
974
            "mode":     self.mode & 0o7777,
975 976 977 978 979 980
            "uid":      self.uid,
            "gid":      self.gid,
            "size":     self.size,
            "mtime":    self.mtime,
            "chksum":   self.chksum,
            "type":     self.type,
981
            "linkname": self.linkname,
982 983 984 985 986 987 988 989 990 991 992
            "uname":    self.uname,
            "gname":    self.gname,
            "devmajor": self.devmajor,
            "devminor": self.devminor
        }

        if info["type"] == DIRTYPE and not info["name"].endswith("/"):
            info["name"] += "/"

        return info

993
    def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="surrogateescape"):
994 995
        """Return a tar header as a string of 512 byte blocks.
        """
996
        info = self.get_info()
997

998
        if format == USTAR_FORMAT:
999
            return self.create_ustar_header(info, encoding, errors)
1000
        elif format == GNU_FORMAT:
1001
            return self.create_gnu_header(info, encoding, errors)
1002
        elif format == PAX_FORMAT:
1003
            return self.create_pax_header(info, encoding)
1004 1005 1006
        else:
            raise ValueError("invalid format")

1007
    def create_ustar_header(self, info, encoding, errors):
1008 1009 1010 1011 1012 1013 1014 1015 1016 1017
        """Return the object as a ustar header block.
        """
        info["magic"] = POSIX_MAGIC

        if len(info["linkname"]) > LENGTH_LINK:
            raise ValueError("linkname is too long")

        if len(info["name"]) > LENGTH_NAME:
            info["prefix"], info["name"] = self._posix_split_name(info["name"])

1018
        return self._create_header(info, USTAR_FORMAT, encoding, errors)
1019

1020
    def create_gnu_header(self, info, encoding, errors):
1021 1022 1023 1024
        """Return the object as a GNU header block sequence.
        """
        info["magic"] = GNU_MAGIC

1025
        buf = b""
1026
        if len(info["linkname"]) > LENGTH_LINK:
1027
            buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK, encoding, errors)
1028 1029

        if len(info["name"]) > LENGTH_NAME:
1030
            buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME, encoding, errors)
1031

1032
        return buf + self._create_header(info, GNU_FORMAT, encoding, errors)
1033

1034
    def create_pax_header(self, info, encoding):
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047
        """Return the object as a ustar header block. If it cannot be
           represented this way, prepend a pax extended header sequence
           with supplement information.
        """
        info["magic"] = POSIX_MAGIC
        pax_headers = self.pax_headers.copy()

        # Test string fields for values that exceed the field length or cannot
        # be represented in ASCII encoding.
        for name, hname, length in (
                ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK),
                ("uname", "uname", 32), ("gname", "gname", 32)):

1048 1049 1050 1051
            if hname in pax_headers:
                # The pax header has priority.
                continue

1052 1053
            # Try to encode the string as ASCII.
            try:
1054
                info[name].encode("ascii", "strict")
1055
            except UnicodeEncodeError:
1056
                pax_headers[hname] = info[name]
1057 1058
                continue

1059
            if len(info[name]) > length:
1060
                pax_headers[hname] = info[name]
1061 1062 1063 1064

        # Test number fields for values that exceed the field limit or values
        # that like to be stored as float.
        for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)):
1065 1066 1067 1068 1069
            if name in pax_headers:
                # The pax header has priority. Avoid overflow.
                info[name] = 0
                continue

1070 1071
            val = info[name]
            if not 0 <= val < 8 ** (digits - 1) or isinstance(val, float):
1072
                pax_headers[name] = str(val)
1073 1074
                info[name] = 0

1075
        # Create a pax extended header if necessary.
1076
        if pax_headers:
1077
            buf = self._create_pax_generic_header(pax_headers, XHDTYPE, encoding)
1078
        else:
1079
            buf = b""
1080

1081
        return buf + self._create_header(info, USTAR_FORMAT, "ascii", "replace")
1082 1083

    @classmethod
1084
    def create_pax_global_header(cls, pax_headers):
1085 1086
        """Return the object as a pax global header block sequence.
        """
1087
        return cls._create_pax_generic_header(pax_headers, XGLTYPE, "utf-8")
1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104

    def _posix_split_name(self, name):
        """Split a name longer than 100 chars into a prefix
           and a name part.
        """
        prefix = name[:LENGTH_PREFIX + 1]
        while prefix and prefix[-1] != "/":
            prefix = prefix[:-1]

        name = name[len(prefix):]
        prefix = prefix[:-1]

        if not prefix or len(name) > LENGTH_NAME:
            raise ValueError("name is too long")
        return prefix, name

    @staticmethod
1105
    def _create_header(info, format, encoding, errors):
1106 1107 1108 1109
        """Return a header block. info is a dictionary with file
           information, format must be one of the *_FORMAT constants.
        """
        parts = [
1110
            stn(info.get("name", ""), 100, encoding, errors),
1111
            itn(info.get("mode", 0) & 0o7777, 8, format),
1112 1113 1114 1115
            itn(info.get("uid", 0), 8, format),
            itn(info.get("gid", 0), 8, format),
            itn(info.get("size", 0), 12, format),
            itn(info.get("mtime", 0), 12, format),
1116
            b"        ", # checksum field
1117
            info.get("type", REGTYPE),
1118 1119
            stn(info.get("linkname", ""), 100, encoding, errors),
            info.get("magic", POSIX_MAGIC),
1120 1121
            stn(info.get("uname", ""), 32, encoding, errors),
            stn(info.get("gname", ""), 32, encoding, errors),
1122 1123
            itn(info.get("devmajor", 0), 8, format),
            itn(info.get("devminor", 0), 8, format),
1124
            stn(info.get("prefix", ""), 155, encoding, errors)
1125 1126
        ]

1127
        buf = struct.pack("%ds" % BLOCKSIZE, b"".join(parts))
1128
        chksum = calc_chksums(buf[-BLOCKSIZE:])[0]
1129
        buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:]
1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142
        return buf

    @staticmethod
    def _create_payload(payload):
        """Return the string payload filled with zero bytes
           up to the next 512 byte border.
        """
        blocks, remainder = divmod(len(payload), BLOCKSIZE)
        if remainder > 0:
            payload += (BLOCKSIZE - remainder) * NUL
        return payload

    @classmethod
1143
    def _create_gnu_long_header(cls, name, type, encoding, errors):
1144 1145 1146
        """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence
           for name.
        """
1147
        name = name.encode(encoding, errors) + NUL
1148 1149 1150 1151 1152 1153 1154 1155

        info = {}
        info["name"] = "././@LongLink"
        info["type"] = type
        info["size"] = len(name)
        info["magic"] = GNU_MAGIC

        # create extended header + name blocks.
1156
        return cls._create_header(info, USTAR_FORMAT, encoding, errors) + \
1157 1158 1159
                cls._create_payload(name)

    @classmethod
1160 1161
    def _create_pax_generic_header(cls, pax_headers, type, encoding):
        """Return a POSIX.1-2008 extended or global header sequence
1162
           that contains a list of keyword, value pairs. The values
1163
           must be strings.
1164
        """
1165 1166 1167 1168 1169
        # Check if one of the fields contains surrogate characters and thereby
        # forces hdrcharset=BINARY, see _proc_pax() for more information.
        binary = False
        for keyword, value in pax_headers.items():
            try:
1170
                value.encode("utf-8", "strict")
1171 1172 1173 1174
            except UnicodeEncodeError:
                binary = True
                break

1175
        records = b""
1176 1177 1178 1179
        if binary:
            # Put the hdrcharset field at the beginning of the header.
            records += b"21 hdrcharset=BINARY\n"

1180
        for keyword, value in pax_headers.items():
1181
            keyword = keyword.encode("utf-8")
1182 1183 1184 1185 1186
            if binary:
                # Try to restore the original byte representation of `value'.
                # Needless to say, that the encoding must match the string.
                value = value.encode(encoding, "surrogateescape")
            else:
1187
                value = value.encode("utf-8")
1188

1189 1190 1191 1192 1193 1194 1195
            l = len(keyword) + len(value) + 3   # ' ' + '=' + '\n'
            n = p = 0
            while True:
                n = l + len(str(p))
                if n == p:
                    break
                p = n
1196
            records += bytes(str(p), "ascii") + b" " + keyword + b"=" + value + b"\n"
1197 1198 1199 1200 1201 1202 1203 1204 1205 1206

        # We use a hardcoded "././@PaxHeader" name like star does
        # instead of the one that POSIX recommends.
        info = {}
        info["name"] = "././@PaxHeader"
        info["type"] = type
        info["size"] = len(records)
        info["magic"] = POSIX_MAGIC

        # Create pax header + record blocks.
1207
        return cls._create_header(info, USTAR_FORMAT, "ascii", "replace") + \
1208 1209
                cls._create_payload(records)

Guido van Rossum's avatar
Guido van Rossum committed
1210
    @classmethod
1211 1212
    def frombuf(cls, buf, encoding, errors):
        """Construct a TarInfo object from a 512 byte bytes object.
1213
        """
1214 1215
        if len(buf) == 0:
            raise EmptyHeaderError("empty header")
1216
        if len(buf) != BLOCKSIZE:
1217
            raise TruncatedHeaderError("truncated header")
1218
        if buf.count(NUL) == BLOCKSIZE:
1219
            raise EOFHeaderError("end of file header")
1220 1221 1222

        chksum = nti(buf[148:156])
        if chksum not in calc_chksums(buf):
1223
            raise InvalidHeaderError("bad checksum")
1224

1225
        obj = cls()
1226
        obj.name = nts(buf[0:100], encoding, errors)
1227 1228 1229 1230 1231 1232 1233
        obj.mode = nti(buf[100:108])
        obj.uid = nti(buf[108:116])
        obj.gid = nti(buf[116:124])
        obj.size = nti(buf[124:136])
        obj.mtime = nti(buf[136:148])
        obj.chksum = chksum
        obj.type = buf[156:157]
1234 1235 1236
        obj.linkname = nts(buf[157:257], encoding, errors)
        obj.uname = nts(buf[265:297], encoding, errors)
        obj.gname = nts(buf[297:329], encoding, errors)
1237 1238
        obj.devmajor = nti(buf[329:337])
        obj.devminor = nti(buf[337:345])
1239
        prefix = nts(buf[345:500], encoding, errors)
1240 1241 1242 1243 1244

        # Old V7 tar format represents a directory as a regular
        # file with a trailing slash.
        if obj.type == AREGTYPE and obj.name.endswith("/"):
            obj.type = DIRTYPE
1245

1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263
        # The old GNU sparse format occupies some of the unused
        # space in the buffer for up to 4 sparse structures.
        # Save the them for later processing in _proc_sparse().
        if obj.type == GNUTYPE_SPARSE:
            pos = 386
            structs = []
            for i in range(4):
                try:
                    offset = nti(buf[pos:pos + 12])
                    numbytes = nti(buf[pos + 12:pos + 24])
                except ValueError:
                    break
                structs.append((offset, numbytes))
                pos += 24
            isextended = bool(buf[482])
            origsize = nti(buf[483:495])
            obj._sparse_structs = (structs, isextended, origsize)

1264 1265 1266
        # Remove redundant slashes from directories.
        if obj.isdir():
            obj.name = obj.name.rstrip("/")
1267

1268 1269 1270 1271
        # Reconstruct a ustar longname.
        if prefix and obj.type not in GNU_TYPES:
            obj.name = prefix + "/" + obj.name
        return obj
1272

1273 1274 1275 1276 1277 1278
    @classmethod
    def fromtarfile(cls, tarfile):
        """Return the next TarInfo object from TarFile object
           tarfile.
        """
        buf = tarfile.fileobj.read(BLOCKSIZE)
1279
        obj = cls.frombuf(buf, tarfile.encoding, tarfile.errors)
1280 1281
        obj.offset = tarfile.fileobj.tell() - BLOCKSIZE
        return obj._proc_member(tarfile)
1282

1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
    #--------------------------------------------------------------------------
    # The following are methods that are called depending on the type of a
    # member. The entry point is _proc_member() which can be overridden in a
    # subclass to add custom _proc_*() methods. A _proc_*() method MUST
    # implement the following
    # operations:
    # 1. Set self.offset_data to the position where the data blocks begin,
    #    if there is data that follows.
    # 2. Set tarfile.offset to the position where the next member's header will
    #    begin.
    # 3. Return self or another valid TarInfo object.
    def _proc_member(self, tarfile):
        """Choose the right processing method depending on
           the type and call it.
        """
        if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK):
            return self._proc_gnulong(tarfile)
        elif self.type == GNUTYPE_SPARSE:
            return self._proc_sparse(tarfile)
        elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE):
            return self._proc_pax(tarfile)
1304
        else:
1305
            return self._proc_builtin(tarfile)
1306

1307 1308 1309 1310 1311 1312 1313 1314 1315 1316
    def _proc_builtin(self, tarfile):
        """Process a builtin type or an unknown type which
           will be treated as a regular file.
        """
        self.offset_data = tarfile.fileobj.tell()
        offset = self.offset_data
        if self.isreg() or self.type not in SUPPORTED_TYPES:
            # Skip the following data blocks.
            offset += self._block(self.size)
        tarfile.offset = offset
1317

1318
        # Patch the TarInfo object with saved global
1319
        # header information.
1320
        self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors)
1321

1322
        return self
1323

1324 1325 1326 1327 1328
    def _proc_gnulong(self, tarfile):
        """Process the blocks that hold a GNU longname
           or longlink member.
        """
        buf = tarfile.fileobj.read(self._block(self.size))
1329

1330
        # Fetch the next header and process it.
1331 1332 1333 1334
        try:
            next = self.fromtarfile(tarfile)
        except HeaderError:
            raise SubsequentHeaderError("missing or bad subsequent header")
1335

1336 1337 1338 1339
        # Patch the TarInfo object from the next header with
        # the longname information.
        next.offset = self.offset
        if self.type == GNUTYPE_LONGNAME:
1340
            next.name = nts(buf, tarfile.encoding, tarfile.errors)
1341
        elif self.type == GNUTYPE_LONGLINK:
1342
            next.linkname = nts(buf, tarfile.encoding, tarfile.errors)
1343

1344
        return next
1345

1346 1347 1348
    def _proc_sparse(self, tarfile):
        """Process a GNU sparse header plus extra headers.
        """
1349 1350 1351
        # We already collected some sparse structures in frombuf().
        structs, isextended, origsize = self._sparse_structs
        del self._sparse_structs
1352

1353
        # Collect sparse structures from extended header blocks.
1354
        while isextended:
1355 1356
            buf = tarfile.fileobj.read(BLOCKSIZE)
            pos = 0
1357
            for i in range(21):
1358 1359 1360 1361 1362
                try:
                    offset = nti(buf[pos:pos + 12])
                    numbytes = nti(buf[pos + 12:pos + 24])
                except ValueError:
                    break
1363 1364
                if offset and numbytes:
                    structs.append((offset, numbytes))
1365
                pos += 24
1366
            isextended = bool(buf[504])
1367
        self.sparse = structs
1368 1369 1370 1371 1372 1373 1374 1375

        self.offset_data = tarfile.fileobj.tell()
        tarfile.offset = self.offset_data + self._block(self.size)
        self.size = origsize
        return self

    def _proc_pax(self, tarfile):
        """Process an extended or global header as described in
1376
           POSIX.1-2008.
1377
        """
1378 1379
        # Read the header information.
        buf = tarfile.fileobj.read(self._block(self.size))
1380

1381 1382 1383 1384 1385 1386 1387 1388
        # A pax header stores supplemental information for either
        # the following file (extended) or all following files
        # (global).
        if self.type == XGLTYPE:
            pax_headers = tarfile.pax_headers
        else:
            pax_headers = tarfile.pax_headers.copy()

1389 1390 1391 1392 1393 1394 1395
        # Check if the pax header contains a hdrcharset field. This tells us
        # the encoding of the path, linkpath, uname and gname fields. Normally,
        # these fields are UTF-8 encoded but since POSIX.1-2008 tar
        # implementations are allowed to store them as raw binary strings if
        # the translation to UTF-8 fails.
        match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf)
        if match is not None:
1396
            pax_headers["hdrcharset"] = match.group(1).decode("utf-8")
1397 1398 1399 1400 1401 1402 1403 1404

        # For the time being, we don't care about anything other than "BINARY".
        # The only other value that is currently allowed by the standard is
        # "ISO-IR 10646 2000 UTF-8" in other words UTF-8.
        hdrcharset = pax_headers.get("hdrcharset")
        if hdrcharset == "BINARY":
            encoding = tarfile.encoding
        else:
1405
            encoding = "utf-8"
1406

1407 1408 1409
        # Parse pax header information. A record looks like that:
        # "%d %s=%s\n" % (length, keyword, value). length is the size
        # of the complete record including the length field itself and
1410
        # the newline. keyword and value are both UTF-8 encoded strings.
1411
        regex = re.compile(br"(\d+) ([^=]+)=")
1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
        pos = 0
        while True:
            match = regex.match(buf, pos)
            if not match:
                break

            length, keyword = match.groups()
            length = int(length)
            value = buf[match.end(2) + 1:match.start(1) + length - 1]

1422
            # Normally, we could just use "utf-8" as the encoding and "strict"
1423 1424 1425 1426 1427 1428
            # as the error handler, but we better not take the risk. For
            # example, GNU tar <= 1.23 is known to store filenames it cannot
            # translate to UTF-8 as raw strings (unfortunately without a
            # hdrcharset=BINARY header).
            # We first try the strict standard encoding, and if that fails we
            # fall back on the user's encoding and error handler.
1429
            keyword = self._decode_pax_field(keyword, "utf-8", "utf-8",
1430 1431 1432 1433 1434
                    tarfile.errors)
            if keyword in PAX_NAME_FIELDS:
                value = self._decode_pax_field(value, encoding, tarfile.encoding,
                        tarfile.errors)
            else:
1435
                value = self._decode_pax_field(value, "utf-8", "utf-8",
1436
                        tarfile.errors)
1437 1438 1439 1440

            pax_headers[keyword] = value
            pos += length

1441
        # Fetch the next header.
1442 1443 1444 1445
        try:
            next = self.fromtarfile(tarfile)
        except HeaderError:
            raise SubsequentHeaderError("missing or bad subsequent header")
1446

1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
        # Process GNU sparse information.
        if "GNU.sparse.map" in pax_headers:
            # GNU extended sparse format version 0.1.
            self._proc_gnusparse_01(next, pax_headers)

        elif "GNU.sparse.size" in pax_headers:
            # GNU extended sparse format version 0.0.
            self._proc_gnusparse_00(next, pax_headers, buf)

        elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0":
            # GNU extended sparse format version 1.0.
            self._proc_gnusparse_10(next, pax_headers, tarfile)

1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475
        if self.type in (XHDTYPE, SOLARIS_XHDTYPE):
            # Patch the TarInfo object with the extended header info.
            next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors)
            next.offset = self.offset

            if "size" in pax_headers:
                # If the extended header replaces the size field,
                # we need to recalculate the offset where the next
                # header starts.
                offset = next.offset_data
                if next.isreg() or next.type not in SUPPORTED_TYPES:
                    offset += next._block(next.size)
                tarfile.offset = offset

        return next

1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508
    def _proc_gnusparse_00(self, next, pax_headers, buf):
        """Process a GNU tar extended sparse header, version 0.0.
        """
        offsets = []
        for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf):
            offsets.append(int(match.group(1)))
        numbytes = []
        for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf):
            numbytes.append(int(match.group(1)))
        next.sparse = list(zip(offsets, numbytes))

    def _proc_gnusparse_01(self, next, pax_headers):
        """Process a GNU tar extended sparse header, version 0.1.
        """
        sparse = [int(x) for x in pax_headers["GNU.sparse.map"].split(",")]
        next.sparse = list(zip(sparse[::2], sparse[1::2]))

    def _proc_gnusparse_10(self, next, pax_headers, tarfile):
        """Process a GNU tar extended sparse header, version 1.0.
        """
        fields = None
        sparse = []
        buf = tarfile.fileobj.read(BLOCKSIZE)
        fields, buf = buf.split(b"\n", 1)
        fields = int(fields)
        while len(sparse) < fields * 2:
            if b"\n" not in buf:
                buf += tarfile.fileobj.read(BLOCKSIZE)
            number, buf = buf.split(b"\n", 1)
            sparse.append(int(number))
        next.offset_data = tarfile.fileobj.tell()
        next.sparse = list(zip(sparse[::2], sparse[1::2]))

1509 1510 1511 1512 1513
    def _apply_pax_info(self, pax_headers, encoding, errors):
        """Replace fields with supplemental information from a previous
           pax extended or global header.
        """
        for keyword, value in pax_headers.items():
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528
            if keyword == "GNU.sparse.name":
                setattr(self, "path", value)
            elif keyword == "GNU.sparse.size":
                setattr(self, "size", int(value))
            elif keyword == "GNU.sparse.realsize":
                setattr(self, "size", int(value))
            elif keyword in PAX_FIELDS:
                if keyword in PAX_NUMBER_FIELDS:
                    try:
                        value = PAX_NUMBER_FIELDS[keyword](value)
                    except ValueError:
                        value = 0
                if keyword == "path":
                    value = value.rstrip("/")
                setattr(self, keyword, value)
1529 1530

        self.pax_headers = pax_headers.copy()
1531

1532 1533 1534 1535 1536 1537 1538 1539
    def _decode_pax_field(self, value, encoding, fallback_encoding, fallback_errors):
        """Decode a single field from a pax record.
        """
        try:
            return value.decode(encoding, "strict")
        except UnicodeDecodeError:
            return value.decode(fallback_encoding, fallback_errors)

1540 1541 1542 1543 1544 1545 1546 1547
    def _block(self, count):
        """Round up a byte count by BLOCKSIZE and return it,
           e.g. _block(834) => 1024.
        """
        blocks, remainder = divmod(count, BLOCKSIZE)
        if remainder:
            blocks += 1
        return blocks * BLOCKSIZE
1548

1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565
    def isreg(self):
        return self.type in REGULAR_TYPES
    def isfile(self):
        return self.isreg()
    def isdir(self):
        return self.type == DIRTYPE
    def issym(self):
        return self.type == SYMTYPE
    def islnk(self):
        return self.type == LNKTYPE
    def ischr(self):
        return self.type == CHRTYPE
    def isblk(self):
        return self.type == BLKTYPE
    def isfifo(self):
        return self.type == FIFOTYPE
    def issparse(self):
1566
        return self.sparse is not None
1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
    def isdev(self):
        return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE)
# class TarInfo

class TarFile(object):
    """The TarFile Class provides an interface to tar archives.
    """

    debug = 0                   # May be set from 0 (no msgs) to 3 (all msgs)

    dereference = False         # If true, add content of linked file to the
                                # tar file, else the link.

    ignore_zeros = False        # If true, skips empty or invalid blocks and
                                # continues processing.

1583
    errorlevel = 1              # If 0, fatal errors only appear in debug
1584 1585 1586
                                # messages (if debug >= 0). If > 0, errors
                                # are passed to the caller as exceptions.

1587 1588
    format = DEFAULT_FORMAT     # The format to use when creating an archive.

1589 1590 1591
    encoding = ENCODING         # Encoding for 8-bit character strings.

    errors = None               # Error handler for unicode conversion.
1592

1593
    tarinfo = TarInfo           # The default TarInfo class to use.
1594

1595 1596 1597 1598
    fileobject = ExFileObject   # The default ExFileObject class to use.

    def __init__(self, name=None, mode="r", fileobj=None, format=None,
            tarinfo=None, dereference=None, ignore_zeros=None, encoding=None,
1599
            errors="surrogateescape", pax_headers=None, debug=None, errorlevel=None):
1600 1601 1602 1603 1604 1605 1606 1607 1608
        """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to
           read from an existing archive, 'a' to append data to an existing
           file or 'w' to create a new file overwriting an existing one. `mode'
           defaults to 'r'.
           If `fileobj' is given, it is used for reading or writing data. If it
           can be determined, `mode' is overridden by `fileobj's mode.
           `fileobj' is not closed, when TarFile is closed.
        """
        if len(mode) > 1 or mode not in "raw":
1609
            raise ValueError("mode must be 'r', 'a' or 'w'")
1610 1611
        self.mode = mode
        self._mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode]
1612 1613

        if not fileobj:
1614
            if self.mode == "a" and not os.path.exists(name):
1615
                # Create nonexistent files in append mode.
1616 1617
                self.mode = "w"
                self._mode = "wb"
1618
            fileobj = bltn_open(name, self._mode)
1619 1620
            self._extfileobj = False
        else:
1621 1622
            if name is None and hasattr(fileobj, "name"):
                name = fileobj.name
1623
            if hasattr(fileobj, "mode"):
1624
                self._mode = fileobj.mode
1625
            self._extfileobj = True
1626
        self.name = os.path.abspath(name) if name else None
1627 1628
        self.fileobj = fileobj

1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639
        # Init attributes.
        if format is not None:
            self.format = format
        if tarinfo is not None:
            self.tarinfo = tarinfo
        if dereference is not None:
            self.dereference = dereference
        if ignore_zeros is not None:
            self.ignore_zeros = ignore_zeros
        if encoding is not None:
            self.encoding = encoding
1640
        self.errors = errors
1641 1642 1643 1644 1645 1646

        if pax_headers is not None and self.format == PAX_FORMAT:
            self.pax_headers = pax_headers
        else:
            self.pax_headers = {}

1647 1648 1649 1650 1651 1652
        if debug is not None:
            self.debug = debug
        if errorlevel is not None:
            self.errorlevel = errorlevel

        # Init datastructures.
1653 1654 1655
        self.closed = False
        self.members = []       # list of members as TarInfo objects
        self._loaded = False    # flag if all members have been read
1656 1657
        self.offset = self.fileobj.tell()
                                # current position in the archive file
1658 1659
        self.inodes = {}        # dictionary caching the inodes of
                                # archive members already added
1660

1661 1662 1663 1664 1665 1666 1667 1668 1669
        try:
            if self.mode == "r":
                self.firstmember = None
                self.firstmember = self.next()

            if self.mode == "a":
                # Move to the end of the archive,
                # before the first empty block.
                while True:
1670 1671 1672 1673 1674 1675
                    self.fileobj.seek(self.offset)
                    try:
                        tarinfo = self.tarinfo.fromtarfile(self)
                        self.members.append(tarinfo)
                    except EOFHeaderError:
                        self.fileobj.seek(self.offset)
1676
                        break
1677 1678
                    except HeaderError as e:
                        raise ReadError(str(e))
1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691

            if self.mode in "aw":
                self._loaded = True

                if self.pax_headers:
                    buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy())
                    self.fileobj.write(buf)
                    self.offset += len(buf)
        except:
            if not self._extfileobj:
                self.fileobj.close()
            self.closed = True
            raise
1692

1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703
    #--------------------------------------------------------------------------
    # Below are the classmethods which act as alternate constructors to the
    # TarFile class. The open() method is the only one that is needed for
    # public use; it is the "super"-constructor and is able to select an
    # adequate "sub"-constructor for a particular compression using the mapping
    # from OPEN_METH.
    #
    # This concept allows one to subclass TarFile without losing the comfort of
    # the super-constructor. A sub-constructor is registered and made available
    # by adding it to the mapping in OPEN_METH.

Guido van Rossum's avatar
Guido van Rossum committed
1704
    @classmethod
1705
    def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs):
1706 1707 1708 1709
        """Open a tar archive for reading, writing or appending. Return
           an appropriate TarFile class.

           mode:
1710
           'r' or 'r:*' open for reading with transparent compression
1711 1712 1713
           'r:'         open for reading exclusively uncompressed
           'r:gz'       open for reading with gzip compression
           'r:bz2'      open for reading with bzip2 compression
1714
           'a' or 'a:'  open for appending, creating the file if necessary
1715 1716 1717
           'w' or 'w:'  open for writing without compression
           'w:gz'       open for writing with gzip compression
           'w:bz2'      open for writing with bzip2 compression
1718 1719

           'r|*'        open a stream of tar blocks with transparent compression
1720 1721 1722 1723 1724 1725 1726 1727 1728
           'r|'         open an uncompressed stream of tar blocks for reading
           'r|gz'       open a gzip compressed stream of tar blocks
           'r|bz2'      open a bzip2 compressed stream of tar blocks
           'w|'         open an uncompressed stream for writing
           'w|gz'       open a gzip compressed stream for writing
           'w|bz2'      open a bzip2 compressed stream for writing
        """

        if not name and not fileobj:
1729
            raise ValueError("nothing to open")
1730

1731 1732 1733 1734
        if mode in ("r", "r:*"):
            # Find out which *open() is appropriate for opening the file.
            for comptype in cls.OPEN_METH:
                func = getattr(cls, cls.OPEN_METH[comptype])
1735 1736
                if fileobj is not None:
                    saved_pos = fileobj.tell()
1737
                try:
1738 1739
                    return func(name, "r", fileobj, **kwargs)
                except (ReadError, CompressionError) as e:
1740 1741
                    if fileobj is not None:
                        fileobj.seek(saved_pos)
1742
                    continue
1743
            raise ReadError("file could not be opened successfully")
1744 1745

        elif ":" in mode:
1746 1747 1748 1749 1750 1751 1752 1753 1754
            filemode, comptype = mode.split(":", 1)
            filemode = filemode or "r"
            comptype = comptype or "tar"

            # Select the *open() function according to
            # given compression.
            if comptype in cls.OPEN_METH:
                func = getattr(cls, cls.OPEN_METH[comptype])
            else:
1755
                raise CompressionError("unknown compression type %r" % comptype)
1756
            return func(name, filemode, fileobj, **kwargs)
1757 1758 1759 1760 1761 1762 1763

        elif "|" in mode:
            filemode, comptype = mode.split("|", 1)
            filemode = filemode or "r"
            comptype = comptype or "tar"

            if filemode not in "rw":
1764
                raise ValueError("mode must be 'r' or 'w'")
1765

1766 1767 1768 1769 1770 1771
            stream = _Stream(name, filemode, comptype, fileobj, bufsize)
            try:
                t = cls(name, filemode, stream, **kwargs)
            except:
                stream.close()
                raise
1772 1773 1774 1775
            t._extfileobj = False
            return t

        elif mode in "aw":
1776
            return cls.taropen(name, mode, fileobj, **kwargs)
1777

1778
        raise ValueError("undiscernible mode")
1779

Guido van Rossum's avatar
Guido van Rossum committed
1780
    @classmethod
1781
    def taropen(cls, name, mode="r", fileobj=None, **kwargs):
1782 1783 1784
        """Open uncompressed tar archive name for reading or writing.
        """
        if len(mode) > 1 or mode not in "raw":
1785
            raise ValueError("mode must be 'r', 'a' or 'w'")
1786
        return cls(name, mode, fileobj, **kwargs)
1787

Guido van Rossum's avatar
Guido van Rossum committed
1788
    @classmethod
1789
    def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs):
1790 1791 1792 1793
        """Open gzip compressed tar archive name for reading or writing.
           Appending is not allowed.
        """
        if len(mode) > 1 or mode not in "rw":
1794
            raise ValueError("mode must be 'r' or 'w'")
1795 1796 1797

        try:
            import gzip
1798 1799
            gzip.GzipFile
        except (ImportError, AttributeError):
1800
            raise CompressionError("gzip module is not available")
1801

1802
        extfileobj = fileobj is not None
1803
        try:
1804 1805
            fileobj = gzip.GzipFile(name, mode + "b", compresslevel, fileobj)
            t = cls.taropen(name, mode, fileobj, **kwargs)
1806
        except IOError:
1807 1808
            if not extfileobj:
                fileobj.close()
1809
            raise ReadError("not a gzip file")
1810 1811 1812 1813
        except:
            if not extfileobj:
                fileobj.close()
            raise
1814
        t._extfileobj = extfileobj
1815 1816
        return t

Guido van Rossum's avatar
Guido van Rossum committed
1817
    @classmethod
1818
    def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs):
1819 1820 1821 1822
        """Open bzip2 compressed tar archive name for reading or writing.
           Appending is not allowed.
        """
        if len(mode) > 1 or mode not in "rw":
1823
            raise ValueError("mode must be 'r' or 'w'.")
1824 1825 1826 1827

        try:
            import bz2
        except ImportError:
1828
            raise CompressionError("bz2 module is not available")
1829 1830

        if fileobj is not None:
1831 1832 1833
            fileobj = _BZ2Proxy(fileobj, mode)
        else:
            fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
1834 1835

        try:
1836
            t = cls.taropen(name, mode, fileobj, **kwargs)
1837
        except (IOError, EOFError):
1838
            fileobj.close()
1839
            raise ReadError("not a bzip2 file")
1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859
        t._extfileobj = False
        return t

    # All *open() methods are registered here.
    OPEN_METH = {
        "tar": "taropen",   # uncompressed tar
        "gz":  "gzopen",    # gzip compressed tar
        "bz2": "bz2open"    # bzip2 compressed tar
    }

    #--------------------------------------------------------------------------
    # The public methods which TarFile provides:

    def close(self):
        """Close the TarFile. In write-mode, two finishing zero blocks are
           appended to the archive.
        """
        if self.closed:
            return

1860
        if self.mode in "aw":
1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875
            self.fileobj.write(NUL * (BLOCKSIZE * 2))
            self.offset += (BLOCKSIZE * 2)
            # fill up the end with zero-blocks
            # (like option -b20 for tar does)
            blocks, remainder = divmod(self.offset, RECORDSIZE)
            if remainder > 0:
                self.fileobj.write(NUL * (RECORDSIZE - remainder))

        if not self._extfileobj:
            self.fileobj.close()
        self.closed = True

    def getmember(self, name):
        """Return a TarInfo object for member `name'. If `name' can not be
           found in the archive, KeyError is raised. If a member occurs more
1876
           than once in the archive, its last occurrence is assumed to be the
1877 1878
           most up-to-date version.
        """
1879 1880
        tarinfo = self._getmember(name)
        if tarinfo is None:
1881
            raise KeyError("filename %r not found" % name)
1882
        return tarinfo
1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897

    def getmembers(self):
        """Return the members of the archive as a list of TarInfo objects. The
           list has the same order as the members in the archive.
        """
        self._check()
        if not self._loaded:    # if we want to obtain a list of
            self._load()        # all members, we first have to
                                # scan the whole archive.
        return self.members

    def getnames(self):
        """Return the members of the archive as a list of their names. It has
           the same order as the list returned by getmembers().
        """
1898
        return [tarinfo.name for tarinfo in self.getmembers()]
1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919

    def gettarinfo(self, name=None, arcname=None, fileobj=None):
        """Create a TarInfo object for either the file `name' or the file
           object `fileobj' (using os.fstat on its file descriptor). You can
           modify some of the TarInfo's attributes before you add it using
           addfile(). If given, `arcname' specifies an alternative name for the
           file in the archive.
        """
        self._check("aw")

        # When fileobj is given, replace name by
        # fileobj's real name.
        if fileobj is not None:
            name = fileobj.name

        # Building the name of the member in the archive.
        # Backward slashes are converted to forward slashes,
        # Absolute paths are turned to relative paths.
        if arcname is None:
            arcname = name
        drv, arcname = os.path.splitdrive(arcname)
1920 1921
        arcname = arcname.replace(os.sep, "/")
        arcname = arcname.lstrip("/")
1922 1923 1924

        # Now, fill the TarInfo object with
        # information specific for the file.
1925 1926
        tarinfo = self.tarinfo()
        tarinfo.tarfile = self
1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941

        # Use os.stat or os.lstat, depending on platform
        # and if symlinks shall be resolved.
        if fileobj is None:
            if hasattr(os, "lstat") and not self.dereference:
                statres = os.lstat(name)
            else:
                statres = os.stat(name)
        else:
            statres = os.fstat(fileobj.fileno())
        linkname = ""

        stmd = statres.st_mode
        if stat.S_ISREG(stmd):
            inode = (statres.st_ino, statres.st_dev)
1942 1943
            if not self.dereference and statres.st_nlink > 1 and \
                    inode in self.inodes and arcname != self.inodes[inode]:
1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969
                # Is it a hardlink to an already
                # archived file?
                type = LNKTYPE
                linkname = self.inodes[inode]
            else:
                # The inode is added only if its valid.
                # For win32 it is always 0.
                type = REGTYPE
                if inode[0]:
                    self.inodes[inode] = arcname
        elif stat.S_ISDIR(stmd):
            type = DIRTYPE
        elif stat.S_ISFIFO(stmd):
            type = FIFOTYPE
        elif stat.S_ISLNK(stmd):
            type = SYMTYPE
            linkname = os.readlink(name)
        elif stat.S_ISCHR(stmd):
            type = CHRTYPE
        elif stat.S_ISBLK(stmd):
            type = BLKTYPE
        else:
            return None

        # Fill the TarInfo object with all
        # information we can get.
1970 1971 1972 1973
        tarinfo.name = arcname
        tarinfo.mode = stmd
        tarinfo.uid = statres.st_uid
        tarinfo.gid = statres.st_gid
1974
        if type == REGTYPE:
1975
            tarinfo.size = statres.st_size
1976
        else:
1977
            tarinfo.size = 0
1978
        tarinfo.mtime = statres.st_mtime
1979
        tarinfo.type = type
1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006
        tarinfo.linkname = linkname
        if pwd:
            try:
                tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0]
            except KeyError:
                pass
        if grp:
            try:
                tarinfo.gname = grp.getgrgid(tarinfo.gid)[0]
            except KeyError:
                pass

        if type in (CHRTYPE, BLKTYPE):
            if hasattr(os, "major") and hasattr(os, "minor"):
                tarinfo.devmajor = os.major(statres.st_rdev)
                tarinfo.devminor = os.minor(statres.st_rdev)
        return tarinfo

    def list(self, verbose=True):
        """Print a table of contents to sys.stdout. If `verbose' is False, only
           the names of the members are printed. If it is True, an `ls -l'-like
           output is produced.
        """
        self._check()

        for tarinfo in self:
            if verbose:
2007 2008 2009
                print(filemode(tarinfo.mode), end=' ')
                print("%s/%s" % (tarinfo.uname or tarinfo.uid,
                                 tarinfo.gname or tarinfo.gid), end=' ')
2010
                if tarinfo.ischr() or tarinfo.isblk():
2011 2012
                    print("%10s" % ("%d,%d" \
                                    % (tarinfo.devmajor, tarinfo.devminor)), end=' ')
2013
                else:
2014 2015 2016
                    print("%10d" % tarinfo.size, end=' ')
                print("%d-%02d-%02d %02d:%02d:%02d" \
                      % time.localtime(tarinfo.mtime)[:6], end=' ')
2017

2018
            print(tarinfo.name + ("/" if tarinfo.isdir() else ""), end=' ')
2019 2020 2021

            if verbose:
                if tarinfo.issym():
2022
                    print("->", tarinfo.linkname, end=' ')
2023
                if tarinfo.islnk():
2024 2025
                    print("link to", tarinfo.linkname, end=' ')
            print()
2026

2027
    def add(self, name, arcname=None, recursive=True, exclude=None, *, filter=None):
2028 2029 2030 2031
        """Add the file `name' to the archive. `name' may be any type of file
           (directory, fifo, symbolic link, etc.). If given, `arcname'
           specifies an alternative name for the file in the archive.
           Directories are added recursively by default. This can be avoided by
2032
           setting `recursive' to False. `exclude' is a function that should
2033 2034 2035 2036
           return True for each filename to be excluded. `filter' is a function
           that expects a TarInfo object argument and returns the changed
           TarInfo object, if it returns None the TarInfo object will be
           excluded from the archive.
2037 2038 2039 2040 2041 2042
        """
        self._check("aw")

        if arcname is None:
            arcname = name

2043
        # Exclude pathnames.
2044 2045 2046 2047 2048 2049 2050
        if exclude is not None:
            import warnings
            warnings.warn("use the filter argument instead",
                    DeprecationWarning, 2)
            if exclude(name):
                self._dbg(2, "tarfile: Excluded %r" % name)
                return
2051

2052
        # Skip if somebody tries to archive the archive...
2053
        if self.name is not None and os.path.abspath(name) == self.name:
2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065
            self._dbg(2, "tarfile: Skipped %r" % name)
            return

        self._dbg(1, name)

        # Create a TarInfo object from the file.
        tarinfo = self.gettarinfo(name, arcname)

        if tarinfo is None:
            self._dbg(1, "tarfile: Unsupported type %r" % name)
            return

2066 2067 2068 2069 2070 2071 2072
        # Change or exclude the TarInfo object.
        if filter is not None:
            tarinfo = filter(tarinfo)
            if tarinfo is None:
                self._dbg(2, "tarfile: Excluded %r" % name)
                return

2073 2074
        # Append the tar header and data to the archive.
        if tarinfo.isreg():
2075
            f = bltn_open(name, "rb")
2076 2077 2078
            self.addfile(tarinfo, f)
            f.close()

2079
        elif tarinfo.isdir():
2080 2081 2082
            self.addfile(tarinfo)
            if recursive:
                for f in os.listdir(name):
2083
                    self.add(os.path.join(name, f), os.path.join(arcname, f),
2084
                            recursive, exclude, filter=filter)
2085

2086 2087 2088
        else:
            self.addfile(tarinfo)

2089 2090 2091 2092 2093 2094 2095 2096 2097
    def addfile(self, tarinfo, fileobj=None):
        """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is
           given, tarinfo.size bytes are read from it and added to the archive.
           You can create TarInfo objects using gettarinfo().
           On Windows platforms, `fileobj' should always be opened with mode
           'rb' to avoid irritation about the file size.
        """
        self._check("aw")

2098
        tarinfo = copy.copy(tarinfo)
2099

2100
        buf = tarinfo.tobuf(self.format, self.encoding, self.errors)
2101 2102
        self.fileobj.write(buf)
        self.offset += len(buf)
2103 2104 2105 2106 2107 2108 2109 2110 2111 2112

        # If there's data to follow, append it.
        if fileobj is not None:
            copyfileobj(fileobj, self.fileobj, tarinfo.size)
            blocks, remainder = divmod(tarinfo.size, BLOCKSIZE)
            if remainder > 0:
                self.fileobj.write(NUL * (BLOCKSIZE - remainder))
                blocks += 1
            self.offset += blocks * BLOCKSIZE

2113
        self.members.append(tarinfo)
2114

2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128
    def extractall(self, path=".", members=None):
        """Extract all members from the archive to the current working
           directory and set owner, modification time and permissions on
           directories afterwards. `path' specifies a different directory
           to extract to. `members' is optional and must be a subset of the
           list returned by getmembers().
        """
        directories = []

        if members is None:
            members = self

        for tarinfo in members:
            if tarinfo.isdir():
Christian Heimes's avatar
Christian Heimes committed
2129
                # Extract directories with a safe mode.
2130
                directories.append(tarinfo)
Christian Heimes's avatar
Christian Heimes committed
2131 2132
                tarinfo = copy.copy(tarinfo)
                tarinfo.mode = 0o700
2133 2134
            # Do not set_attrs directories, as we will do that further down
            self.extract(tarinfo, path, set_attrs=not tarinfo.isdir())
2135 2136

        # Reverse sort directories.
2137
        directories.sort(key=lambda a: a.name)
2138 2139 2140 2141
        directories.reverse()

        # Set correct owner, mtime and filemode on directories.
        for tarinfo in directories:
2142
            dirpath = os.path.join(path, tarinfo.name)
2143
            try:
2144 2145 2146
                self.chown(tarinfo, dirpath)
                self.utime(tarinfo, dirpath)
                self.chmod(tarinfo, dirpath)
2147
            except ExtractError as e:
2148 2149 2150 2151 2152
                if self.errorlevel > 1:
                    raise
                else:
                    self._dbg(1, "tarfile: %s" % e)

2153
    def extract(self, member, path="", set_attrs=True):
2154 2155 2156
        """Extract a member from the archive to the current working directory,
           using its full name. Its file information is extracted as accurately
           as possible. `member' may be a filename or a TarInfo object. You can
2157 2158
           specify a different directory using `path'. File attributes (owner,
           mtime, mode) are set unless `set_attrs' is False.
2159 2160 2161
        """
        self._check("r")

2162
        if isinstance(member, str):
2163
            tarinfo = self.getmember(member)
2164 2165
        else:
            tarinfo = member
2166

2167 2168 2169 2170
        # Prepare the link target for makelink().
        if tarinfo.islnk():
            tarinfo._link_target = os.path.join(path, tarinfo.linkname)

2171
        try:
2172 2173
            self._extract_member(tarinfo, os.path.join(path, tarinfo.name),
                                 set_attrs=set_attrs)
2174
        except EnvironmentError as e:
2175 2176 2177 2178 2179 2180 2181
            if self.errorlevel > 0:
                raise
            else:
                if e.filename is None:
                    self._dbg(1, "tarfile: %s" % e.strerror)
                else:
                    self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename))
2182
        except ExtractError as e:
2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198
            if self.errorlevel > 1:
                raise
            else:
                self._dbg(1, "tarfile: %s" % e)

    def extractfile(self, member):
        """Extract a member from the archive as a file object. `member' may be
           a filename or a TarInfo object. If `member' is a regular file, a
           file-like object is returned. If `member' is a link, a file-like
           object is constructed from the link's target. If `member' is none of
           the above, None is returned.
           The file-like object is read-only and provides the following
           methods: read(), readline(), readlines(), seek() and tell()
        """
        self._check("r")

2199
        if isinstance(member, str):
2200
            tarinfo = self.getmember(member)
2201 2202
        else:
            tarinfo = member
2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216

        if tarinfo.isreg():
            return self.fileobject(self, tarinfo)

        elif tarinfo.type not in SUPPORTED_TYPES:
            # If a member's type is unknown, it is treated as a
            # regular file.
            return self.fileobject(self, tarinfo)

        elif tarinfo.islnk() or tarinfo.issym():
            if isinstance(self.fileobj, _Stream):
                # A small but ugly workaround for the case that someone tries
                # to extract a (sym)link as a file-object from a non-seekable
                # stream of tar blocks.
2217
                raise StreamError("cannot extract (sym)link as file object")
2218
            else:
2219
                # A (sym)link's file object is its target's file object.
2220
                return self.extractfile(self._find_link_target(tarinfo))
2221 2222 2223 2224 2225
        else:
            # If there's no data associated with the member (directory, chrdev,
            # blkdev, etc.), return None instead of a file object.
            return None

2226
    def _extract_member(self, tarinfo, targetpath, set_attrs=True):
2227 2228 2229 2230 2231 2232
        """Extract the TarInfo object tarinfo to a physical
           file called targetpath.
        """
        # Fetch the TarInfo object for the given name
        # and build the destination pathname, replacing
        # forward slashes to platform specific separators.
2233 2234
        targetpath = targetpath.rstrip("/")
        targetpath = targetpath.replace("/", os.sep)
2235 2236 2237 2238

        # Create all upper directories.
        upperdirs = os.path.dirname(targetpath)
        if upperdirs and not os.path.exists(upperdirs):
Christian Heimes's avatar
Christian Heimes committed
2239 2240
            # Create directories that are not part of the archive with
            # default permissions.
2241
            os.makedirs(upperdirs)
2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262

        if tarinfo.islnk() or tarinfo.issym():
            self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname))
        else:
            self._dbg(1, tarinfo.name)

        if tarinfo.isreg():
            self.makefile(tarinfo, targetpath)
        elif tarinfo.isdir():
            self.makedir(tarinfo, targetpath)
        elif tarinfo.isfifo():
            self.makefifo(tarinfo, targetpath)
        elif tarinfo.ischr() or tarinfo.isblk():
            self.makedev(tarinfo, targetpath)
        elif tarinfo.islnk() or tarinfo.issym():
            self.makelink(tarinfo, targetpath)
        elif tarinfo.type not in SUPPORTED_TYPES:
            self.makeunknown(tarinfo, targetpath)
        else:
            self.makefile(tarinfo, targetpath)

2263 2264 2265 2266 2267
        if set_attrs:
            self.chown(tarinfo, targetpath)
            if not tarinfo.issym():
                self.chmod(tarinfo, targetpath)
                self.utime(tarinfo, targetpath)
2268 2269 2270 2271 2272 2273 2274 2275 2276 2277

    #--------------------------------------------------------------------------
    # Below are the different file methods. They are called via
    # _extract_member() when extract() is called. They can be replaced in a
    # subclass to implement other functionality.

    def makedir(self, tarinfo, targetpath):
        """Make a directory called targetpath.
        """
        try:
Christian Heimes's avatar
Christian Heimes committed
2278 2279 2280
            # Use a safe mode for the directory, the real mode is set
            # later in _extract_member().
            os.mkdir(targetpath, 0o700)
2281
        except EnvironmentError as e:
2282 2283 2284 2285 2286 2287
            if e.errno != errno.EEXIST:
                raise

    def makefile(self, tarinfo, targetpath):
        """Make a file called targetpath.
        """
2288 2289
        source = self.fileobj
        source.seek(tarinfo.offset_data)
2290
        target = bltn_open(targetpath, "wb")
2291 2292 2293 2294 2295 2296 2297 2298
        if tarinfo.sparse is not None:
            for offset, size in tarinfo.sparse:
                target.seek(offset)
                copyfileobj(source, target, size)
        else:
            copyfileobj(source, target, tarinfo.size)
        target.seek(tarinfo.size)
        target.truncate()
2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314
        target.close()

    def makeunknown(self, tarinfo, targetpath):
        """Make a file from a TarInfo object with an unknown type
           at targetpath.
        """
        self.makefile(tarinfo, targetpath)
        self._dbg(1, "tarfile: Unknown file type %r, " \
                     "extracted as regular file." % tarinfo.type)

    def makefifo(self, tarinfo, targetpath):
        """Make a fifo called targetpath.
        """
        if hasattr(os, "mkfifo"):
            os.mkfifo(targetpath)
        else:
2315
            raise ExtractError("fifo not supported by system")
2316 2317 2318 2319 2320

    def makedev(self, tarinfo, targetpath):
        """Make a character or block device called targetpath.
        """
        if not hasattr(os, "mknod") or not hasattr(os, "makedev"):
2321
            raise ExtractError("special devices not supported by system")
2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336

        mode = tarinfo.mode
        if tarinfo.isblk():
            mode |= stat.S_IFBLK
        else:
            mode |= stat.S_IFCHR

        os.mknod(targetpath, mode,
                 os.makedev(tarinfo.devmajor, tarinfo.devminor))

    def makelink(self, tarinfo, targetpath):
        """Make a (symbolic) link called targetpath. If it cannot be created
          (platform limitation), we try to make a copy of the referenced file
          instead of a link.
        """
2337
        try:
2338
            # For systems that support symbolic and hard links.
2339
            if tarinfo.issym():
2340
                os.symlink(tarinfo.linkname, targetpath)
2341
            else:
2342
                # See extract().
2343 2344 2345
                if os.path.exists(tarinfo._link_target):
                    os.link(tarinfo._link_target, targetpath)
                else:
2346 2347
                    self._extract_member(self._find_link_target(tarinfo),
                                         targetpath)
2348
        except symlink_exception:
2349
            if tarinfo.issym():
2350 2351
                linkpath = os.path.join(os.path.dirname(tarinfo.name),
                                        tarinfo.linkname)
2352 2353
            else:
                linkpath = tarinfo.linkname
2354
        else:
2355
            try:
2356 2357
                self._extract_member(self._find_link_target(tarinfo),
                                     targetpath)
2358 2359
            except KeyError:
                raise ExtractError("unable to resolve link inside archive")
2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383

    def chown(self, tarinfo, targetpath):
        """Set owner of targetpath according to tarinfo.
        """
        if pwd and hasattr(os, "geteuid") and os.geteuid() == 0:
            # We have to be root to do so.
            try:
                g = grp.getgrnam(tarinfo.gname)[2]
            except KeyError:
                try:
                    g = grp.getgrgid(tarinfo.gid)[2]
                except KeyError:
                    g = os.getgid()
            try:
                u = pwd.getpwnam(tarinfo.uname)[2]
            except KeyError:
                try:
                    u = pwd.getpwuid(tarinfo.uid)[2]
                except KeyError:
                    u = os.getuid()
            try:
                if tarinfo.issym() and hasattr(os, "lchown"):
                    os.lchown(targetpath, u, g)
                else:
2384 2385
                    if sys.platform != "os2emx":
                        os.chown(targetpath, u, g)
2386
            except EnvironmentError as e:
2387
                raise ExtractError("could not change owner")
2388 2389 2390 2391

    def chmod(self, tarinfo, targetpath):
        """Set file permissions of targetpath according to tarinfo.
        """
2392 2393 2394
        if hasattr(os, 'chmod'):
            try:
                os.chmod(targetpath, tarinfo.mode)
2395
            except EnvironmentError as e:
2396
                raise ExtractError("could not change mode")
2397 2398 2399 2400

    def utime(self, tarinfo, targetpath):
        """Set modification time of targetpath according to tarinfo.
        """
2401
        if not hasattr(os, 'utime'):
2402
            return
2403 2404
        try:
            os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime))
2405
        except EnvironmentError as e:
2406
            raise ExtractError("could not change modification time")
2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420

    #--------------------------------------------------------------------------
    def next(self):
        """Return the next member of the archive as a TarInfo object, when
           TarFile is opened for reading. Return None if there is no more
           available.
        """
        self._check("ra")
        if self.firstmember is not None:
            m = self.firstmember
            self.firstmember = None
            return m

        # Read the next block.
Andrew M. Kuchling's avatar
Andrew M. Kuchling committed
2421
        self.fileobj.seek(self.offset)
2422
        tarinfo = None
2423 2424
        while True:
            try:
2425
                tarinfo = self.tarinfo.fromtarfile(self)
2426
            except EOFHeaderError as e:
2427
                if self.ignore_zeros:
2428
                    self._dbg(2, "0x%X: %s" % (self.offset, e))
2429 2430
                    self.offset += BLOCKSIZE
                    continue
2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445
            except InvalidHeaderError as e:
                if self.ignore_zeros:
                    self._dbg(2, "0x%X: %s" % (self.offset, e))
                    self.offset += BLOCKSIZE
                    continue
                elif self.offset == 0:
                    raise ReadError(str(e))
            except EmptyHeaderError:
                if self.offset == 0:
                    raise ReadError("empty file")
            except TruncatedHeaderError as e:
                if self.offset == 0:
                    raise ReadError(str(e))
            except SubsequentHeaderError as e:
                raise ReadError(str(e))
2446 2447
            break

2448 2449 2450 2451 2452
        if tarinfo is not None:
            self.members.append(tarinfo)
        else:
            self._loaded = True

2453
        return tarinfo
2454 2455 2456 2457

    #--------------------------------------------------------------------------
    # Little helper methods:

2458
    def _getmember(self, name, tarinfo=None, normalize=False):
2459 2460 2461
        """Find an archive member by name from bottom to top.
           If tarinfo is given, it is used as the starting point.
        """
2462 2463 2464
        # Ensure that all members have been loaded.
        members = self.getmembers()

2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476
        # Limit the member search list up to tarinfo.
        if tarinfo is not None:
            members = members[:members.index(tarinfo)]

        if normalize:
            name = os.path.normpath(name)

        for member in reversed(members):
            if normalize:
                member_name = os.path.normpath(member.name)
            else:
                member_name = member.name
2477

2478 2479
            if name == member_name:
                return member
Andrew M. Kuchling's avatar
Andrew M. Kuchling committed
2480

2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495
    def _load(self):
        """Read through the entire archive file and look for readable
           members.
        """
        while True:
            tarinfo = self.next()
            if tarinfo is None:
                break
        self._loaded = True

    def _check(self, mode=None):
        """Check if TarFile is still open, and if the operation's mode
           corresponds to TarFile's mode.
        """
        if self.closed:
2496
            raise IOError("%s is closed" % self.__class__.__name__)
2497 2498
        if mode is not None and self.mode not in mode:
            raise IOError("bad operation for mode %r" % self.mode)
2499

2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518
    def _find_link_target(self, tarinfo):
        """Find the target member of a symlink or hardlink member in the
           archive.
        """
        if tarinfo.issym():
            # Always search the entire archive.
            linkname = os.path.dirname(tarinfo.name) + "/" + tarinfo.linkname
            limit = None
        else:
            # Search the archive before the link, because a hard link is
            # just a reference to an already archived file.
            linkname = tarinfo.linkname
            limit = tarinfo

        member = self._getmember(linkname, tarinfo=limit, normalize=True)
        if member is None:
            raise KeyError("linkname %r not found" % linkname)
        return member

2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
    def __iter__(self):
        """Provide an iterator object.
        """
        if self._loaded:
            return iter(self.members)
        else:
            return TarIter(self)

    def _dbg(self, level, msg):
        """Write debugging output to sys.stderr.
        """
        if level <= self.debug:
2531
            print(msg, file=sys.stderr)
2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545

    def __enter__(self):
        self._check()
        return self

    def __exit__(self, type, value, traceback):
        if type is None:
            self.close()
        else:
            # An exception occurred. We must not call close() because
            # it would try to write end-of-archive blocks and padding.
            if not self._extfileobj:
                self.fileobj.close()
            self.closed = True
2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558
# class TarFile

class TarIter:
    """Iterator Class.

       for tarinfo in TarFile(...):
           suite...
    """

    def __init__(self, tarfile):
        """Construct a TarIter object.
        """
        self.tarfile = tarfile
2559
        self.index = 0
2560 2561 2562 2563
    def __iter__(self):
        """Return iterator object.
        """
        return self
2564
    def __next__(self):
2565 2566 2567
        """Return the next item using TarFile's next() method.
           When all members have been read, set TarFile as _loaded.
        """
2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581
        # Fix for SF #1100429: Under rare circumstances it can
        # happen that getmembers() is called during iteration,
        # which will cause TarIter to stop prematurely.
        if not self.tarfile._loaded:
            tarinfo = self.tarfile.next()
            if not tarinfo:
                self.tarfile._loaded = True
                raise StopIteration
        else:
            try:
                tarinfo = self.tarfile.members[self.index]
            except IndexError:
                raise StopIteration
        self.index += 1
2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597
        return tarinfo

#--------------------
# exported functions
#--------------------
def is_tarfile(name):
    """Return True if name points to a tar archive that we
       are able to handle, else return False.
    """
    try:
        t = open(name)
        t.close()
        return True
    except TarError:
        return False

2598
bltn_open = open
2599
open = TarFile.open