1package archive // import "github.com/docker/docker/pkg/archive" 2 3import ( 4 "archive/tar" 5 "bufio" 6 "bytes" 7 "compress/bzip2" 8 "compress/gzip" 9 "context" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "os" 14 "os/exec" 15 "path/filepath" 16 "runtime" 17 "strconv" 18 "strings" 19 "syscall" 20 "time" 21 22 "github.com/docker/docker/pkg/fileutils" 23 "github.com/docker/docker/pkg/idtools" 24 "github.com/docker/docker/pkg/ioutils" 25 "github.com/docker/docker/pkg/pools" 26 "github.com/docker/docker/pkg/system" 27 "github.com/sirupsen/logrus" 28) 29 30var unpigzPath string 31 32func init() { 33 if path, err := exec.LookPath("unpigz"); err != nil { 34 logrus.Debug("unpigz binary not found in PATH, falling back to go gzip library") 35 } else { 36 logrus.Debugf("Using unpigz binary found at path %s", path) 37 unpigzPath = path 38 } 39} 40 41type ( 42 // Compression is the state represents if compressed or not. 43 Compression int 44 // WhiteoutFormat is the format of whiteouts unpacked 45 WhiteoutFormat int 46 47 // TarOptions wraps the tar options. 48 TarOptions struct { 49 IncludeFiles []string 50 ExcludePatterns []string 51 Compression Compression 52 NoLchown bool 53 UIDMaps []idtools.IDMap 54 GIDMaps []idtools.IDMap 55 ChownOpts *idtools.IDPair 56 IncludeSourceDir bool 57 // WhiteoutFormat is the expected on disk format for whiteout files. 58 // This format will be converted to the standard format on pack 59 // and from the standard format on unpack. 60 WhiteoutFormat WhiteoutFormat 61 // When unpacking, specifies whether overwriting a directory with a 62 // non-directory is allowed and vice versa. 63 NoOverwriteDirNonDir bool 64 // For each include when creating an archive, the included name will be 65 // replaced with the matching name from this map. 66 RebaseNames map[string]string 67 InUserNS bool 68 } 69) 70 71// Archiver implements the Archiver interface and allows the reuse of most utility functions of 72// this package with a pluggable Untar function. Also, to facilitate the passing of specific id 73// mappings for untar, an Archiver can be created with maps which will then be passed to Untar operations. 74type Archiver struct { 75 Untar func(io.Reader, string, *TarOptions) error 76 IDMappingsVar *idtools.IDMappings 77} 78 79// NewDefaultArchiver returns a new Archiver without any IDMappings 80func NewDefaultArchiver() *Archiver { 81 return &Archiver{Untar: Untar, IDMappingsVar: &idtools.IDMappings{}} 82} 83 84// breakoutError is used to differentiate errors related to breaking out 85// When testing archive breakout in the unit tests, this error is expected 86// in order for the test to pass. 87type breakoutError error 88 89const ( 90 // Uncompressed represents the uncompressed. 91 Uncompressed Compression = iota 92 // Bzip2 is bzip2 compression algorithm. 93 Bzip2 94 // Gzip is gzip compression algorithm. 95 Gzip 96 // Xz is xz compression algorithm. 97 Xz 98) 99 100const ( 101 // AUFSWhiteoutFormat is the default format for whiteouts 102 AUFSWhiteoutFormat WhiteoutFormat = iota 103 // OverlayWhiteoutFormat formats whiteout according to the overlay 104 // standard. 105 OverlayWhiteoutFormat 106) 107 108const ( 109 modeISDIR = 040000 // Directory 110 modeISFIFO = 010000 // FIFO 111 modeISREG = 0100000 // Regular file 112 modeISLNK = 0120000 // Symbolic link 113 modeISBLK = 060000 // Block special file 114 modeISCHR = 020000 // Character special file 115 modeISSOCK = 0140000 // Socket 116) 117 118// IsArchivePath checks if the (possibly compressed) file at the given path 119// starts with a tar file header. 120func IsArchivePath(path string) bool { 121 file, err := os.Open(path) 122 if err != nil { 123 return false 124 } 125 defer file.Close() 126 rdr, err := DecompressStream(file) 127 if err != nil { 128 return false 129 } 130 defer rdr.Close() 131 r := tar.NewReader(rdr) 132 _, err = r.Next() 133 return err == nil 134} 135 136// DetectCompression detects the compression algorithm of the source. 137func DetectCompression(source []byte) Compression { 138 for compression, m := range map[Compression][]byte{ 139 Bzip2: {0x42, 0x5A, 0x68}, 140 Gzip: {0x1F, 0x8B, 0x08}, 141 Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, 142 } { 143 if len(source) < len(m) { 144 logrus.Debug("Len too short") 145 continue 146 } 147 if bytes.Equal(m, source[:len(m)]) { 148 return compression 149 } 150 } 151 return Uncompressed 152} 153 154func xzDecompress(ctx context.Context, archive io.Reader) (io.ReadCloser, error) { 155 args := []string{"xz", "-d", "-c", "-q"} 156 157 return cmdStream(exec.CommandContext(ctx, args[0], args[1:]...), archive) 158} 159 160func gzDecompress(ctx context.Context, buf io.Reader) (io.ReadCloser, error) { 161 if unpigzPath == "" { 162 return gzip.NewReader(buf) 163 } 164 165 disablePigzEnv := os.Getenv("MOBY_DISABLE_PIGZ") 166 if disablePigzEnv != "" { 167 if disablePigz, err := strconv.ParseBool(disablePigzEnv); err != nil { 168 return nil, err 169 } else if disablePigz { 170 return gzip.NewReader(buf) 171 } 172 } 173 174 return cmdStream(exec.CommandContext(ctx, unpigzPath, "-d", "-c"), buf) 175} 176 177func wrapReadCloser(readBuf io.ReadCloser, cancel context.CancelFunc) io.ReadCloser { 178 return ioutils.NewReadCloserWrapper(readBuf, func() error { 179 cancel() 180 return readBuf.Close() 181 }) 182} 183 184// DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive. 185func DecompressStream(archive io.Reader) (io.ReadCloser, error) { 186 p := pools.BufioReader32KPool 187 buf := p.Get(archive) 188 bs, err := buf.Peek(10) 189 if err != nil && err != io.EOF { 190 // Note: we'll ignore any io.EOF error because there are some odd 191 // cases where the layer.tar file will be empty (zero bytes) and 192 // that results in an io.EOF from the Peek() call. So, in those 193 // cases we'll just treat it as a non-compressed stream and 194 // that means just create an empty layer. 195 // See Issue 18170 196 return nil, err 197 } 198 199 compression := DetectCompression(bs) 200 switch compression { 201 case Uncompressed: 202 readBufWrapper := p.NewReadCloserWrapper(buf, buf) 203 return readBufWrapper, nil 204 case Gzip: 205 ctx, cancel := context.WithCancel(context.Background()) 206 207 gzReader, err := gzDecompress(ctx, buf) 208 if err != nil { 209 cancel() 210 return nil, err 211 } 212 readBufWrapper := p.NewReadCloserWrapper(buf, gzReader) 213 return wrapReadCloser(readBufWrapper, cancel), nil 214 case Bzip2: 215 bz2Reader := bzip2.NewReader(buf) 216 readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader) 217 return readBufWrapper, nil 218 case Xz: 219 ctx, cancel := context.WithCancel(context.Background()) 220 221 xzReader, err := xzDecompress(ctx, buf) 222 if err != nil { 223 cancel() 224 return nil, err 225 } 226 readBufWrapper := p.NewReadCloserWrapper(buf, xzReader) 227 return wrapReadCloser(readBufWrapper, cancel), nil 228 default: 229 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 230 } 231} 232 233// CompressStream compresses the dest with specified compression algorithm. 234func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) { 235 p := pools.BufioWriter32KPool 236 buf := p.Get(dest) 237 switch compression { 238 case Uncompressed: 239 writeBufWrapper := p.NewWriteCloserWrapper(buf, buf) 240 return writeBufWrapper, nil 241 case Gzip: 242 gzWriter := gzip.NewWriter(dest) 243 writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter) 244 return writeBufWrapper, nil 245 case Bzip2, Xz: 246 // archive/bzip2 does not support writing, and there is no xz support at all 247 // However, this is not a problem as docker only currently generates gzipped tars 248 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 249 default: 250 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 251 } 252} 253 254// TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper to 255// modify the contents or header of an entry in the archive. If the file already 256// exists in the archive the TarModifierFunc will be called with the Header and 257// a reader which will return the files content. If the file does not exist both 258// header and content will be nil. 259type TarModifierFunc func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) 260 261// ReplaceFileTarWrapper converts inputTarStream to a new tar stream. Files in the 262// tar stream are modified if they match any of the keys in mods. 263func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModifierFunc) io.ReadCloser { 264 pipeReader, pipeWriter := io.Pipe() 265 266 go func() { 267 tarReader := tar.NewReader(inputTarStream) 268 tarWriter := tar.NewWriter(pipeWriter) 269 defer inputTarStream.Close() 270 defer tarWriter.Close() 271 272 modify := func(name string, original *tar.Header, modifier TarModifierFunc, tarReader io.Reader) error { 273 header, data, err := modifier(name, original, tarReader) 274 switch { 275 case err != nil: 276 return err 277 case header == nil: 278 return nil 279 } 280 281 header.Name = name 282 header.Size = int64(len(data)) 283 if err := tarWriter.WriteHeader(header); err != nil { 284 return err 285 } 286 if len(data) != 0 { 287 if _, err := tarWriter.Write(data); err != nil { 288 return err 289 } 290 } 291 return nil 292 } 293 294 var err error 295 var originalHeader *tar.Header 296 for { 297 originalHeader, err = tarReader.Next() 298 if err == io.EOF { 299 break 300 } 301 if err != nil { 302 pipeWriter.CloseWithError(err) 303 return 304 } 305 306 modifier, ok := mods[originalHeader.Name] 307 if !ok { 308 // No modifiers for this file, copy the header and data 309 if err := tarWriter.WriteHeader(originalHeader); err != nil { 310 pipeWriter.CloseWithError(err) 311 return 312 } 313 if _, err := pools.Copy(tarWriter, tarReader); err != nil { 314 pipeWriter.CloseWithError(err) 315 return 316 } 317 continue 318 } 319 delete(mods, originalHeader.Name) 320 321 if err := modify(originalHeader.Name, originalHeader, modifier, tarReader); err != nil { 322 pipeWriter.CloseWithError(err) 323 return 324 } 325 } 326 327 // Apply the modifiers that haven't matched any files in the archive 328 for name, modifier := range mods { 329 if err := modify(name, nil, modifier, nil); err != nil { 330 pipeWriter.CloseWithError(err) 331 return 332 } 333 } 334 335 pipeWriter.Close() 336 337 }() 338 return pipeReader 339} 340 341// Extension returns the extension of a file that uses the specified compression algorithm. 342func (compression *Compression) Extension() string { 343 switch *compression { 344 case Uncompressed: 345 return "tar" 346 case Bzip2: 347 return "tar.bz2" 348 case Gzip: 349 return "tar.gz" 350 case Xz: 351 return "tar.xz" 352 } 353 return "" 354} 355 356// FileInfoHeader creates a populated Header from fi. 357// Compared to archive pkg this function fills in more information. 358// Also, regardless of Go version, this function fills file type bits (e.g. hdr.Mode |= modeISDIR), 359// which have been deleted since Go 1.9 archive/tar. 360func FileInfoHeader(name string, fi os.FileInfo, link string) (*tar.Header, error) { 361 hdr, err := tar.FileInfoHeader(fi, link) 362 if err != nil { 363 return nil, err 364 } 365 hdr.Format = tar.FormatPAX 366 hdr.ModTime = hdr.ModTime.Truncate(time.Second) 367 hdr.AccessTime = time.Time{} 368 hdr.ChangeTime = time.Time{} 369 hdr.Mode = fillGo18FileTypeBits(int64(chmodTarEntry(os.FileMode(hdr.Mode))), fi) 370 name, err = canonicalTarName(name, fi.IsDir()) 371 if err != nil { 372 return nil, fmt.Errorf("tar: cannot canonicalize path: %v", err) 373 } 374 hdr.Name = name 375 if err := setHeaderForSpecialDevice(hdr, name, fi.Sys()); err != nil { 376 return nil, err 377 } 378 return hdr, nil 379} 380 381// fillGo18FileTypeBits fills type bits which have been removed on Go 1.9 archive/tar 382// https://github.com/golang/go/commit/66b5a2f 383func fillGo18FileTypeBits(mode int64, fi os.FileInfo) int64 { 384 fm := fi.Mode() 385 switch { 386 case fm.IsRegular(): 387 mode |= modeISREG 388 case fi.IsDir(): 389 mode |= modeISDIR 390 case fm&os.ModeSymlink != 0: 391 mode |= modeISLNK 392 case fm&os.ModeDevice != 0: 393 if fm&os.ModeCharDevice != 0 { 394 mode |= modeISCHR 395 } else { 396 mode |= modeISBLK 397 } 398 case fm&os.ModeNamedPipe != 0: 399 mode |= modeISFIFO 400 case fm&os.ModeSocket != 0: 401 mode |= modeISSOCK 402 } 403 return mode 404} 405 406// ReadSecurityXattrToTarHeader reads security.capability xattr from filesystem 407// to a tar header 408func ReadSecurityXattrToTarHeader(path string, hdr *tar.Header) error { 409 capability, _ := system.Lgetxattr(path, "security.capability") 410 if capability != nil { 411 hdr.Xattrs = make(map[string]string) 412 hdr.Xattrs["security.capability"] = string(capability) 413 } 414 return nil 415} 416 417type tarWhiteoutConverter interface { 418 ConvertWrite(*tar.Header, string, os.FileInfo) (*tar.Header, error) 419 ConvertRead(*tar.Header, string) (bool, error) 420} 421 422type tarAppender struct { 423 TarWriter *tar.Writer 424 Buffer *bufio.Writer 425 426 // for hardlink mapping 427 SeenFiles map[uint64]string 428 IDMappings *idtools.IDMappings 429 ChownOpts *idtools.IDPair 430 431 // For packing and unpacking whiteout files in the 432 // non standard format. The whiteout files defined 433 // by the AUFS standard are used as the tar whiteout 434 // standard. 435 WhiteoutConverter tarWhiteoutConverter 436} 437 438func newTarAppender(idMapping *idtools.IDMappings, writer io.Writer, chownOpts *idtools.IDPair) *tarAppender { 439 return &tarAppender{ 440 SeenFiles: make(map[uint64]string), 441 TarWriter: tar.NewWriter(writer), 442 Buffer: pools.BufioWriter32KPool.Get(nil), 443 IDMappings: idMapping, 444 ChownOpts: chownOpts, 445 } 446} 447 448// canonicalTarName provides a platform-independent and consistent posix-style 449//path for files and directories to be archived regardless of the platform. 450func canonicalTarName(name string, isDir bool) (string, error) { 451 name, err := CanonicalTarNameForPath(name) 452 if err != nil { 453 return "", err 454 } 455 456 // suffix with '/' for directories 457 if isDir && !strings.HasSuffix(name, "/") { 458 name += "/" 459 } 460 return name, nil 461} 462 463// addTarFile adds to the tar archive a file from `path` as `name` 464func (ta *tarAppender) addTarFile(path, name string) error { 465 fi, err := os.Lstat(path) 466 if err != nil { 467 return err 468 } 469 470 var link string 471 if fi.Mode()&os.ModeSymlink != 0 { 472 var err error 473 link, err = os.Readlink(path) 474 if err != nil { 475 return err 476 } 477 } 478 479 hdr, err := FileInfoHeader(name, fi, link) 480 if err != nil { 481 return err 482 } 483 if err := ReadSecurityXattrToTarHeader(path, hdr); err != nil { 484 return err 485 } 486 487 // if it's not a directory and has more than 1 link, 488 // it's hard linked, so set the type flag accordingly 489 if !fi.IsDir() && hasHardlinks(fi) { 490 inode, err := getInodeFromStat(fi.Sys()) 491 if err != nil { 492 return err 493 } 494 // a link should have a name that it links too 495 // and that linked name should be first in the tar archive 496 if oldpath, ok := ta.SeenFiles[inode]; ok { 497 hdr.Typeflag = tar.TypeLink 498 hdr.Linkname = oldpath 499 hdr.Size = 0 // This Must be here for the writer math to add up! 500 } else { 501 ta.SeenFiles[inode] = name 502 } 503 } 504 505 //check whether the file is overlayfs whiteout 506 //if yes, skip re-mapping container ID mappings. 507 isOverlayWhiteout := fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 508 509 //handle re-mapping container ID mappings back to host ID mappings before 510 //writing tar headers/files. We skip whiteout files because they were written 511 //by the kernel and already have proper ownership relative to the host 512 if !isOverlayWhiteout && 513 !strings.HasPrefix(filepath.Base(hdr.Name), WhiteoutPrefix) && 514 !ta.IDMappings.Empty() { 515 fileIDPair, err := getFileUIDGID(fi.Sys()) 516 if err != nil { 517 return err 518 } 519 hdr.Uid, hdr.Gid, err = ta.IDMappings.ToContainer(fileIDPair) 520 if err != nil { 521 return err 522 } 523 } 524 525 // explicitly override with ChownOpts 526 if ta.ChownOpts != nil { 527 hdr.Uid = ta.ChownOpts.UID 528 hdr.Gid = ta.ChownOpts.GID 529 } 530 531 if ta.WhiteoutConverter != nil { 532 wo, err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi) 533 if err != nil { 534 return err 535 } 536 537 // If a new whiteout file exists, write original hdr, then 538 // replace hdr with wo to be written after. Whiteouts should 539 // always be written after the original. Note the original 540 // hdr may have been updated to be a whiteout with returning 541 // a whiteout header 542 if wo != nil { 543 if err := ta.TarWriter.WriteHeader(hdr); err != nil { 544 return err 545 } 546 if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 { 547 return fmt.Errorf("tar: cannot use whiteout for non-empty file") 548 } 549 hdr = wo 550 } 551 } 552 553 if err := ta.TarWriter.WriteHeader(hdr); err != nil { 554 return err 555 } 556 557 if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 { 558 // We use system.OpenSequential to ensure we use sequential file 559 // access on Windows to avoid depleting the standby list. 560 // On Linux, this equates to a regular os.Open. 561 file, err := system.OpenSequential(path) 562 if err != nil { 563 return err 564 } 565 566 ta.Buffer.Reset(ta.TarWriter) 567 defer ta.Buffer.Reset(nil) 568 _, err = io.Copy(ta.Buffer, file) 569 file.Close() 570 if err != nil { 571 return err 572 } 573 err = ta.Buffer.Flush() 574 if err != nil { 575 return err 576 } 577 } 578 579 return nil 580} 581 582func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *idtools.IDPair, inUserns bool) error { 583 // hdr.Mode is in linux format, which we can use for sycalls, 584 // but for os.Foo() calls we need the mode converted to os.FileMode, 585 // so use hdrInfo.Mode() (they differ for e.g. setuid bits) 586 hdrInfo := hdr.FileInfo() 587 588 switch hdr.Typeflag { 589 case tar.TypeDir: 590 // Create directory unless it exists as a directory already. 591 // In that case we just want to merge the two 592 if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { 593 if err := os.Mkdir(path, hdrInfo.Mode()); err != nil { 594 return err 595 } 596 } 597 598 case tar.TypeReg, tar.TypeRegA: 599 // Source is regular file. We use system.OpenFileSequential to use sequential 600 // file access to avoid depleting the standby list on Windows. 601 // On Linux, this equates to a regular os.OpenFile 602 file, err := system.OpenFileSequential(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode()) 603 if err != nil { 604 return err 605 } 606 if _, err := io.Copy(file, reader); err != nil { 607 file.Close() 608 return err 609 } 610 file.Close() 611 612 case tar.TypeBlock, tar.TypeChar: 613 if inUserns { // cannot create devices in a userns 614 return nil 615 } 616 // Handle this is an OS-specific way 617 if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { 618 return err 619 } 620 621 case tar.TypeFifo: 622 // Handle this is an OS-specific way 623 if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { 624 return err 625 } 626 627 case tar.TypeLink: 628 targetPath := filepath.Join(extractDir, hdr.Linkname) 629 // check for hardlink breakout 630 if !strings.HasPrefix(targetPath, extractDir) { 631 return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname)) 632 } 633 if err := os.Link(targetPath, path); err != nil { 634 return err 635 } 636 637 case tar.TypeSymlink: 638 // path -> hdr.Linkname = targetPath 639 // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file 640 targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname) 641 642 // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because 643 // that symlink would first have to be created, which would be caught earlier, at this very check: 644 if !strings.HasPrefix(targetPath, extractDir) { 645 return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) 646 } 647 if err := os.Symlink(hdr.Linkname, path); err != nil { 648 return err 649 } 650 651 case tar.TypeXGlobalHeader: 652 logrus.Debug("PAX Global Extended Headers found and ignored") 653 return nil 654 655 default: 656 return fmt.Errorf("unhandled tar header type %d", hdr.Typeflag) 657 } 658 659 // Lchown is not supported on Windows. 660 if Lchown && runtime.GOOS != "windows" { 661 if chownOpts == nil { 662 chownOpts = &idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid} 663 } 664 if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil { 665 return err 666 } 667 } 668 669 var errors []string 670 for key, value := range hdr.Xattrs { 671 if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil { 672 if err == syscall.ENOTSUP { 673 // We ignore errors here because not all graphdrivers support 674 // xattrs *cough* old versions of AUFS *cough*. However only 675 // ENOTSUP should be emitted in that case, otherwise we still 676 // bail. 677 errors = append(errors, err.Error()) 678 continue 679 } 680 return err 681 } 682 683 } 684 685 if len(errors) > 0 { 686 logrus.WithFields(logrus.Fields{ 687 "errors": errors, 688 }).Warn("ignored xattrs in archive: underlying filesystem doesn't support them") 689 } 690 691 // There is no LChmod, so ignore mode for symlink. Also, this 692 // must happen after chown, as that can modify the file mode 693 if err := handleLChmod(hdr, path, hdrInfo); err != nil { 694 return err 695 } 696 697 aTime := hdr.AccessTime 698 if aTime.Before(hdr.ModTime) { 699 // Last access time should never be before last modified time. 700 aTime = hdr.ModTime 701 } 702 703 // system.Chtimes doesn't support a NOFOLLOW flag atm 704 if hdr.Typeflag == tar.TypeLink { 705 if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { 706 if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { 707 return err 708 } 709 } 710 } else if hdr.Typeflag != tar.TypeSymlink { 711 if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { 712 return err 713 } 714 } else { 715 ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)} 716 if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { 717 return err 718 } 719 } 720 return nil 721} 722 723// Tar creates an archive from the directory at `path`, and returns it as a 724// stream of bytes. 725func Tar(path string, compression Compression) (io.ReadCloser, error) { 726 return TarWithOptions(path, &TarOptions{Compression: compression}) 727} 728 729// TarWithOptions creates an archive from the directory at `path`, only including files whose relative 730// paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`. 731func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) { 732 733 // Fix the source path to work with long path names. This is a no-op 734 // on platforms other than Windows. 735 srcPath = fixVolumePathPrefix(srcPath) 736 737 pm, err := fileutils.NewPatternMatcher(options.ExcludePatterns) 738 if err != nil { 739 return nil, err 740 } 741 742 pipeReader, pipeWriter := io.Pipe() 743 744 compressWriter, err := CompressStream(pipeWriter, options.Compression) 745 if err != nil { 746 return nil, err 747 } 748 749 go func() { 750 ta := newTarAppender( 751 idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps), 752 compressWriter, 753 options.ChownOpts, 754 ) 755 ta.WhiteoutConverter = getWhiteoutConverter(options.WhiteoutFormat) 756 757 defer func() { 758 // Make sure to check the error on Close. 759 if err := ta.TarWriter.Close(); err != nil { 760 logrus.Errorf("Can't close tar writer: %s", err) 761 } 762 if err := compressWriter.Close(); err != nil { 763 logrus.Errorf("Can't close compress writer: %s", err) 764 } 765 if err := pipeWriter.Close(); err != nil { 766 logrus.Errorf("Can't close pipe writer: %s", err) 767 } 768 }() 769 770 // this buffer is needed for the duration of this piped stream 771 defer pools.BufioWriter32KPool.Put(ta.Buffer) 772 773 // In general we log errors here but ignore them because 774 // during e.g. a diff operation the container can continue 775 // mutating the filesystem and we can see transient errors 776 // from this 777 778 stat, err := os.Lstat(srcPath) 779 if err != nil { 780 return 781 } 782 783 if !stat.IsDir() { 784 // We can't later join a non-dir with any includes because the 785 // 'walk' will error if "file/." is stat-ed and "file" is not a 786 // directory. So, we must split the source path and use the 787 // basename as the include. 788 if len(options.IncludeFiles) > 0 { 789 logrus.Warn("Tar: Can't archive a file with includes") 790 } 791 792 dir, base := SplitPathDirEntry(srcPath) 793 srcPath = dir 794 options.IncludeFiles = []string{base} 795 } 796 797 if len(options.IncludeFiles) == 0 { 798 options.IncludeFiles = []string{"."} 799 } 800 801 seen := make(map[string]bool) 802 803 for _, include := range options.IncludeFiles { 804 rebaseName := options.RebaseNames[include] 805 806 walkRoot := getWalkRoot(srcPath, include) 807 filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error { 808 if err != nil { 809 logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err) 810 return nil 811 } 812 813 relFilePath, err := filepath.Rel(srcPath, filePath) 814 if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) { 815 // Error getting relative path OR we are looking 816 // at the source directory path. Skip in both situations. 817 return nil 818 } 819 820 if options.IncludeSourceDir && include == "." && relFilePath != "." { 821 relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator)) 822 } 823 824 skip := false 825 826 // If "include" is an exact match for the current file 827 // then even if there's an "excludePatterns" pattern that 828 // matches it, don't skip it. IOW, assume an explicit 'include' 829 // is asking for that file no matter what - which is true 830 // for some files, like .dockerignore and Dockerfile (sometimes) 831 if include != relFilePath { 832 skip, err = pm.Matches(relFilePath) 833 if err != nil { 834 logrus.Errorf("Error matching %s: %v", relFilePath, err) 835 return err 836 } 837 } 838 839 if skip { 840 // If we want to skip this file and its a directory 841 // then we should first check to see if there's an 842 // excludes pattern (e.g. !dir/file) that starts with this 843 // dir. If so then we can't skip this dir. 844 845 // Its not a dir then so we can just return/skip. 846 if !f.IsDir() { 847 return nil 848 } 849 850 // No exceptions (!...) in patterns so just skip dir 851 if !pm.Exclusions() { 852 return filepath.SkipDir 853 } 854 855 dirSlash := relFilePath + string(filepath.Separator) 856 857 for _, pat := range pm.Patterns() { 858 if !pat.Exclusion() { 859 continue 860 } 861 if strings.HasPrefix(pat.String()+string(filepath.Separator), dirSlash) { 862 // found a match - so can't skip this dir 863 return nil 864 } 865 } 866 867 // No matching exclusion dir so just skip dir 868 return filepath.SkipDir 869 } 870 871 if seen[relFilePath] { 872 return nil 873 } 874 seen[relFilePath] = true 875 876 // Rename the base resource. 877 if rebaseName != "" { 878 var replacement string 879 if rebaseName != string(filepath.Separator) { 880 // Special case the root directory to replace with an 881 // empty string instead so that we don't end up with 882 // double slashes in the paths. 883 replacement = rebaseName 884 } 885 886 relFilePath = strings.Replace(relFilePath, include, replacement, 1) 887 } 888 889 if err := ta.addTarFile(filePath, relFilePath); err != nil { 890 logrus.Errorf("Can't add file %s to tar: %s", filePath, err) 891 // if pipe is broken, stop writing tar stream to it 892 if err == io.ErrClosedPipe { 893 return err 894 } 895 } 896 return nil 897 }) 898 } 899 }() 900 901 return pipeReader, nil 902} 903 904// Unpack unpacks the decompressedArchive to dest with options. 905func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error { 906 tr := tar.NewReader(decompressedArchive) 907 trBuf := pools.BufioReader32KPool.Get(nil) 908 defer pools.BufioReader32KPool.Put(trBuf) 909 910 var dirs []*tar.Header 911 idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps) 912 rootIDs := idMappings.RootPair() 913 whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat) 914 915 // Iterate through the files in the archive. 916loop: 917 for { 918 hdr, err := tr.Next() 919 if err == io.EOF { 920 // end of tar archive 921 break 922 } 923 if err != nil { 924 return err 925 } 926 927 // Normalize name, for safety and for a simple is-root check 928 // This keeps "../" as-is, but normalizes "/../" to "/". Or Windows: 929 // This keeps "..\" as-is, but normalizes "\..\" to "\". 930 hdr.Name = filepath.Clean(hdr.Name) 931 932 for _, exclude := range options.ExcludePatterns { 933 if strings.HasPrefix(hdr.Name, exclude) { 934 continue loop 935 } 936 } 937 938 // After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in 939 // the filepath format for the OS on which the daemon is running. Hence 940 // the check for a slash-suffix MUST be done in an OS-agnostic way. 941 if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { 942 // Not the root directory, ensure that the parent directory exists 943 parent := filepath.Dir(hdr.Name) 944 parentPath := filepath.Join(dest, parent) 945 if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { 946 err = idtools.MkdirAllAndChownNew(parentPath, 0777, rootIDs) 947 if err != nil { 948 return err 949 } 950 } 951 } 952 953 path := filepath.Join(dest, hdr.Name) 954 rel, err := filepath.Rel(dest, path) 955 if err != nil { 956 return err 957 } 958 if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { 959 return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) 960 } 961 962 // If path exits we almost always just want to remove and replace it 963 // The only exception is when it is a directory *and* the file from 964 // the layer is also a directory. Then we want to merge them (i.e. 965 // just apply the metadata from the layer). 966 if fi, err := os.Lstat(path); err == nil { 967 if options.NoOverwriteDirNonDir && fi.IsDir() && hdr.Typeflag != tar.TypeDir { 968 // If NoOverwriteDirNonDir is true then we cannot replace 969 // an existing directory with a non-directory from the archive. 970 return fmt.Errorf("cannot overwrite directory %q with non-directory %q", path, dest) 971 } 972 973 if options.NoOverwriteDirNonDir && !fi.IsDir() && hdr.Typeflag == tar.TypeDir { 974 // If NoOverwriteDirNonDir is true then we cannot replace 975 // an existing non-directory with a directory from the archive. 976 return fmt.Errorf("cannot overwrite non-directory %q with directory %q", path, dest) 977 } 978 979 if fi.IsDir() && hdr.Name == "." { 980 continue 981 } 982 983 if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { 984 if err := os.RemoveAll(path); err != nil { 985 return err 986 } 987 } 988 } 989 trBuf.Reset(tr) 990 991 if err := remapIDs(idMappings, hdr); err != nil { 992 return err 993 } 994 995 if whiteoutConverter != nil { 996 writeFile, err := whiteoutConverter.ConvertRead(hdr, path) 997 if err != nil { 998 return err 999 } 1000 if !writeFile { 1001 continue 1002 } 1003 } 1004 1005 if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts, options.InUserNS); err != nil { 1006 return err 1007 } 1008 1009 // Directory mtimes must be handled at the end to avoid further 1010 // file creation in them to modify the directory mtime 1011 if hdr.Typeflag == tar.TypeDir { 1012 dirs = append(dirs, hdr) 1013 } 1014 } 1015 1016 for _, hdr := range dirs { 1017 path := filepath.Join(dest, hdr.Name) 1018 1019 if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil { 1020 return err 1021 } 1022 } 1023 return nil 1024} 1025 1026// Untar reads a stream of bytes from `archive`, parses it as a tar archive, 1027// and unpacks it into the directory at `dest`. 1028// The archive may be compressed with one of the following algorithms: 1029// identity (uncompressed), gzip, bzip2, xz. 1030// FIXME: specify behavior when target path exists vs. doesn't exist. 1031func Untar(tarArchive io.Reader, dest string, options *TarOptions) error { 1032 return untarHandler(tarArchive, dest, options, true) 1033} 1034 1035// UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive, 1036// and unpacks it into the directory at `dest`. 1037// The archive must be an uncompressed stream. 1038func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error { 1039 return untarHandler(tarArchive, dest, options, false) 1040} 1041 1042// Handler for teasing out the automatic decompression 1043func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error { 1044 if tarArchive == nil { 1045 return fmt.Errorf("Empty archive") 1046 } 1047 dest = filepath.Clean(dest) 1048 if options == nil { 1049 options = &TarOptions{} 1050 } 1051 if options.ExcludePatterns == nil { 1052 options.ExcludePatterns = []string{} 1053 } 1054 1055 r := tarArchive 1056 if decompress { 1057 decompressedArchive, err := DecompressStream(tarArchive) 1058 if err != nil { 1059 return err 1060 } 1061 defer decompressedArchive.Close() 1062 r = decompressedArchive 1063 } 1064 1065 return Unpack(r, dest, options) 1066} 1067 1068// TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. 1069// If either Tar or Untar fails, TarUntar aborts and returns the error. 1070func (archiver *Archiver) TarUntar(src, dst string) error { 1071 logrus.Debugf("TarUntar(%s %s)", src, dst) 1072 archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed}) 1073 if err != nil { 1074 return err 1075 } 1076 defer archive.Close() 1077 options := &TarOptions{ 1078 UIDMaps: archiver.IDMappingsVar.UIDs(), 1079 GIDMaps: archiver.IDMappingsVar.GIDs(), 1080 } 1081 return archiver.Untar(archive, dst, options) 1082} 1083 1084// UntarPath untar a file from path to a destination, src is the source tar file path. 1085func (archiver *Archiver) UntarPath(src, dst string) error { 1086 archive, err := os.Open(src) 1087 if err != nil { 1088 return err 1089 } 1090 defer archive.Close() 1091 options := &TarOptions{ 1092 UIDMaps: archiver.IDMappingsVar.UIDs(), 1093 GIDMaps: archiver.IDMappingsVar.GIDs(), 1094 } 1095 return archiver.Untar(archive, dst, options) 1096} 1097 1098// CopyWithTar creates a tar archive of filesystem path `src`, and 1099// unpacks it at filesystem path `dst`. 1100// The archive is streamed directly with fixed buffering and no 1101// intermediary disk IO. 1102func (archiver *Archiver) CopyWithTar(src, dst string) error { 1103 srcSt, err := os.Stat(src) 1104 if err != nil { 1105 return err 1106 } 1107 if !srcSt.IsDir() { 1108 return archiver.CopyFileWithTar(src, dst) 1109 } 1110 1111 // if this Archiver is set up with ID mapping we need to create 1112 // the new destination directory with the remapped root UID/GID pair 1113 // as owner 1114 rootIDs := archiver.IDMappingsVar.RootPair() 1115 // Create dst, copy src's content into it 1116 logrus.Debugf("Creating dest directory: %s", dst) 1117 if err := idtools.MkdirAllAndChownNew(dst, 0755, rootIDs); err != nil { 1118 return err 1119 } 1120 logrus.Debugf("Calling TarUntar(%s, %s)", src, dst) 1121 return archiver.TarUntar(src, dst) 1122} 1123 1124// CopyFileWithTar emulates the behavior of the 'cp' command-line 1125// for a single file. It copies a regular file from path `src` to 1126// path `dst`, and preserves all its metadata. 1127func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { 1128 logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst) 1129 srcSt, err := os.Stat(src) 1130 if err != nil { 1131 return err 1132 } 1133 1134 if srcSt.IsDir() { 1135 return fmt.Errorf("Can't copy a directory") 1136 } 1137 1138 // Clean up the trailing slash. This must be done in an operating 1139 // system specific manner. 1140 if dst[len(dst)-1] == os.PathSeparator { 1141 dst = filepath.Join(dst, filepath.Base(src)) 1142 } 1143 // Create the holding directory if necessary 1144 if err := system.MkdirAll(filepath.Dir(dst), 0700, ""); err != nil { 1145 return err 1146 } 1147 1148 r, w := io.Pipe() 1149 errC := make(chan error, 1) 1150 1151 go func() { 1152 defer close(errC) 1153 1154 errC <- func() error { 1155 defer w.Close() 1156 1157 srcF, err := os.Open(src) 1158 if err != nil { 1159 return err 1160 } 1161 defer srcF.Close() 1162 1163 hdr, err := tar.FileInfoHeader(srcSt, "") 1164 if err != nil { 1165 return err 1166 } 1167 hdr.Format = tar.FormatPAX 1168 hdr.ModTime = hdr.ModTime.Truncate(time.Second) 1169 hdr.AccessTime = time.Time{} 1170 hdr.ChangeTime = time.Time{} 1171 hdr.Name = filepath.Base(dst) 1172 hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) 1173 1174 if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil { 1175 return err 1176 } 1177 1178 tw := tar.NewWriter(w) 1179 defer tw.Close() 1180 if err := tw.WriteHeader(hdr); err != nil { 1181 return err 1182 } 1183 if _, err := io.Copy(tw, srcF); err != nil { 1184 return err 1185 } 1186 return nil 1187 }() 1188 }() 1189 defer func() { 1190 if er := <-errC; err == nil && er != nil { 1191 err = er 1192 } 1193 }() 1194 1195 err = archiver.Untar(r, filepath.Dir(dst), nil) 1196 if err != nil { 1197 r.CloseWithError(err) 1198 } 1199 return err 1200} 1201 1202// IDMappings returns the IDMappings of the archiver. 1203func (archiver *Archiver) IDMappings() *idtools.IDMappings { 1204 return archiver.IDMappingsVar 1205} 1206 1207func remapIDs(idMappings *idtools.IDMappings, hdr *tar.Header) error { 1208 ids, err := idMappings.ToHost(idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}) 1209 hdr.Uid, hdr.Gid = ids.UID, ids.GID 1210 return err 1211} 1212 1213// cmdStream executes a command, and returns its stdout as a stream. 1214// If the command fails to run or doesn't complete successfully, an error 1215// will be returned, including anything written on stderr. 1216func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, error) { 1217 cmd.Stdin = input 1218 pipeR, pipeW := io.Pipe() 1219 cmd.Stdout = pipeW 1220 var errBuf bytes.Buffer 1221 cmd.Stderr = &errBuf 1222 1223 // Run the command and return the pipe 1224 if err := cmd.Start(); err != nil { 1225 return nil, err 1226 } 1227 1228 // Copy stdout to the returned pipe 1229 go func() { 1230 if err := cmd.Wait(); err != nil { 1231 pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String())) 1232 } else { 1233 pipeW.Close() 1234 } 1235 }() 1236 1237 return pipeR, nil 1238} 1239 1240// NewTempArchive reads the content of src into a temporary file, and returns the contents 1241// of that file as an archive. The archive can only be read once - as soon as reading completes, 1242// the file will be deleted. 1243func NewTempArchive(src io.Reader, dir string) (*TempArchive, error) { 1244 f, err := ioutil.TempFile(dir, "") 1245 if err != nil { 1246 return nil, err 1247 } 1248 if _, err := io.Copy(f, src); err != nil { 1249 return nil, err 1250 } 1251 if _, err := f.Seek(0, 0); err != nil { 1252 return nil, err 1253 } 1254 st, err := f.Stat() 1255 if err != nil { 1256 return nil, err 1257 } 1258 size := st.Size() 1259 return &TempArchive{File: f, Size: size}, nil 1260} 1261 1262// TempArchive is a temporary archive. The archive can only be read once - as soon as reading completes, 1263// the file will be deleted. 1264type TempArchive struct { 1265 *os.File 1266 Size int64 // Pre-computed from Stat().Size() as a convenience 1267 read int64 1268 closed bool 1269} 1270 1271// Close closes the underlying file if it's still open, or does a no-op 1272// to allow callers to try to close the TempArchive multiple times safely. 1273func (archive *TempArchive) Close() error { 1274 if archive.closed { 1275 return nil 1276 } 1277 1278 archive.closed = true 1279 1280 return archive.File.Close() 1281} 1282 1283func (archive *TempArchive) Read(data []byte) (int, error) { 1284 n, err := archive.File.Read(data) 1285 archive.read += int64(n) 1286 if err != nil || archive.read == archive.Size { 1287 archive.Close() 1288 os.Remove(archive.File.Name()) 1289 } 1290 return n, err 1291} 1292