1package archive // import "github.com/ory/dockertest/v3/docker/pkg/archive" 2 3import ( 4 "archive/tar" 5 "bufio" 6 "bytes" 7 "compress/bzip2" 8 "compress/gzip" 9 "context" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "os" 14 "os/exec" 15 "path/filepath" 16 "runtime" 17 "strconv" 18 "strings" 19 "syscall" 20 21 "github.com/ory/dockertest/v3/docker/pkg/fileutils" 22 "github.com/ory/dockertest/v3/docker/pkg/idtools" 23 "github.com/ory/dockertest/v3/docker/pkg/ioutils" 24 "github.com/ory/dockertest/v3/docker/pkg/pools" 25 "github.com/ory/dockertest/v3/docker/pkg/system" 26 "github.com/sirupsen/logrus" 27) 28 29var unpigzPath string 30 31func init() { 32 if path, err := exec.LookPath("unpigz"); err != nil { 33 logrus.Debug("unpigz binary not found in PATH, falling back to go gzip library") 34 } else { 35 logrus.Debugf("Using unpigz binary found at path %s", path) 36 unpigzPath = path 37 } 38} 39 40type ( 41 // Compression is the state represents if compressed or not. 42 Compression int 43 // WhiteoutFormat is the format of whiteouts unpacked 44 WhiteoutFormat int 45 46 // TarOptions wraps the tar options. 47 TarOptions struct { 48 IncludeFiles []string 49 ExcludePatterns []string 50 Compression Compression 51 NoLchown bool 52 UIDMaps []idtools.IDMap 53 GIDMaps []idtools.IDMap 54 ChownOpts *idtools.IDPair 55 IncludeSourceDir bool 56 // WhiteoutFormat is the expected on disk format for whiteout files. 57 // This format will be converted to the standard format on pack 58 // and from the standard format on unpack. 59 WhiteoutFormat WhiteoutFormat 60 // When unpacking, specifies whether overwriting a directory with a 61 // non-directory is allowed and vice versa. 62 NoOverwriteDirNonDir bool 63 // For each include when creating an archive, the included name will be 64 // replaced with the matching name from this map. 65 RebaseNames map[string]string 66 InUserNS bool 67 } 68) 69 70// Archiver implements the Archiver interface and allows the reuse of most utility functions of 71// this package with a pluggable Untar function. Also, to facilitate the passing of specific id 72// mappings for untar, an Archiver can be created with maps which will then be passed to Untar operations. 73type Archiver struct { 74 Untar func(io.Reader, string, *TarOptions) error 75 IDMappingsVar *idtools.IDMappings 76} 77 78// NewDefaultArchiver returns a new Archiver without any IDMappings 79func NewDefaultArchiver() *Archiver { 80 return &Archiver{Untar: Untar, IDMappingsVar: &idtools.IDMappings{}} 81} 82 83// breakoutError is used to differentiate errors related to breaking out 84// When testing archive breakout in the unit tests, this error is expected 85// in order for the test to pass. 86type breakoutError error 87 88const ( 89 // Uncompressed represents the uncompressed. 90 Uncompressed Compression = iota 91 // Bzip2 is bzip2 compression algorithm. 92 Bzip2 93 // Gzip is gzip compression algorithm. 94 Gzip 95 // Xz is xz compression algorithm. 96 Xz 97) 98 99const ( 100 // AUFSWhiteoutFormat is the default format for whiteouts 101 AUFSWhiteoutFormat WhiteoutFormat = iota 102 // OverlayWhiteoutFormat formats whiteout according to the overlay 103 // standard. 104 OverlayWhiteoutFormat 105) 106 107const ( 108 modeISDIR = 040000 // Directory 109 modeISFIFO = 010000 // FIFO 110 modeISREG = 0100000 // Regular file 111 modeISLNK = 0120000 // Symbolic link 112 modeISBLK = 060000 // Block special file 113 modeISCHR = 020000 // Character special file 114 modeISSOCK = 0140000 // Socket 115) 116 117// IsArchivePath checks if the (possibly compressed) file at the given path 118// starts with a tar file header. 119func IsArchivePath(path string) bool { 120 file, err := os.Open(path) 121 if err != nil { 122 return false 123 } 124 defer file.Close() 125 rdr, err := DecompressStream(file) 126 if err != nil { 127 return false 128 } 129 r := tar.NewReader(rdr) 130 _, err = r.Next() 131 return err == nil 132} 133 134// DetectCompression detects the compression algorithm of the source. 135func DetectCompression(source []byte) Compression { 136 for compression, m := range map[Compression][]byte{ 137 Bzip2: {0x42, 0x5A, 0x68}, 138 Gzip: {0x1F, 0x8B, 0x08}, 139 Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, 140 } { 141 if len(source) < len(m) { 142 logrus.Debug("Len too short") 143 continue 144 } 145 if bytes.Equal(m, source[:len(m)]) { 146 return compression 147 } 148 } 149 return Uncompressed 150} 151 152func xzDecompress(ctx context.Context, archive io.Reader) (io.ReadCloser, error) { 153 args := []string{"xz", "-d", "-c", "-q"} 154 155 return cmdStream(exec.CommandContext(ctx, args[0], args[1:]...), archive) 156} 157 158func gzDecompress(ctx context.Context, buf io.Reader) (io.ReadCloser, error) { 159 if unpigzPath == "" { 160 return gzip.NewReader(buf) 161 } 162 163 disablePigzEnv := os.Getenv("MOBY_DISABLE_PIGZ") 164 if disablePigzEnv != "" { 165 if disablePigz, err := strconv.ParseBool(disablePigzEnv); err != nil { 166 return nil, err 167 } else if disablePigz { 168 return gzip.NewReader(buf) 169 } 170 } 171 172 return cmdStream(exec.CommandContext(ctx, unpigzPath, "-d", "-c"), buf) 173} 174 175func wrapReadCloser(readBuf io.ReadCloser, cancel context.CancelFunc) io.ReadCloser { 176 return ioutils.NewReadCloserWrapper(readBuf, func() error { 177 cancel() 178 return readBuf.Close() 179 }) 180} 181 182// DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive. 183func DecompressStream(archive io.Reader) (io.ReadCloser, error) { 184 p := pools.BufioReader32KPool 185 buf := p.Get(archive) 186 bs, err := buf.Peek(10) 187 if err != nil && err != io.EOF { 188 // Note: we'll ignore any io.EOF error because there are some odd 189 // cases where the layer.tar file will be empty (zero bytes) and 190 // that results in an io.EOF from the Peek() call. So, in those 191 // cases we'll just treat it as a non-compressed stream and 192 // that means just create an empty layer. 193 // See Issue 18170 194 return nil, err 195 } 196 197 compression := DetectCompression(bs) 198 switch compression { 199 case Uncompressed: 200 readBufWrapper := p.NewReadCloserWrapper(buf, buf) 201 return readBufWrapper, nil 202 case Gzip: 203 ctx, cancel := context.WithCancel(context.Background()) 204 205 gzReader, err := gzDecompress(ctx, buf) 206 if err != nil { 207 cancel() 208 return nil, err 209 } 210 readBufWrapper := p.NewReadCloserWrapper(buf, gzReader) 211 return wrapReadCloser(readBufWrapper, cancel), nil 212 case Bzip2: 213 bz2Reader := bzip2.NewReader(buf) 214 readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader) 215 return readBufWrapper, nil 216 case Xz: 217 ctx, cancel := context.WithCancel(context.Background()) 218 219 xzReader, err := xzDecompress(ctx, buf) 220 if err != nil { 221 cancel() 222 return nil, err 223 } 224 readBufWrapper := p.NewReadCloserWrapper(buf, xzReader) 225 return wrapReadCloser(readBufWrapper, cancel), nil 226 default: 227 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 228 } 229} 230 231// CompressStream compresses the dest with specified compression algorithm. 232func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) { 233 p := pools.BufioWriter32KPool 234 buf := p.Get(dest) 235 switch compression { 236 case Uncompressed: 237 writeBufWrapper := p.NewWriteCloserWrapper(buf, buf) 238 return writeBufWrapper, nil 239 case Gzip: 240 gzWriter := gzip.NewWriter(dest) 241 writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter) 242 return writeBufWrapper, nil 243 case Bzip2, Xz: 244 // archive/bzip2 does not support writing, and there is no xz support at all 245 // However, this is not a problem as docker only currently generates gzipped tars 246 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 247 default: 248 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 249 } 250} 251 252// TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper to 253// modify the contents or header of an entry in the archive. If the file already 254// exists in the archive the TarModifierFunc will be called with the Header and 255// a reader which will return the files content. If the file does not exist both 256// header and content will be nil. 257type TarModifierFunc func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) 258 259// ReplaceFileTarWrapper converts inputTarStream to a new tar stream. Files in the 260// tar stream are modified if they match any of the keys in mods. 261func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModifierFunc) io.ReadCloser { 262 pipeReader, pipeWriter := io.Pipe() 263 264 go func() { 265 tarReader := tar.NewReader(inputTarStream) 266 tarWriter := tar.NewWriter(pipeWriter) 267 defer inputTarStream.Close() 268 defer tarWriter.Close() 269 270 modify := func(name string, original *tar.Header, modifier TarModifierFunc, tarReader io.Reader) error { 271 header, data, err := modifier(name, original, tarReader) 272 switch { 273 case err != nil: 274 return err 275 case header == nil: 276 return nil 277 } 278 279 header.Name = name 280 header.Size = int64(len(data)) 281 if err := tarWriter.WriteHeader(header); err != nil { 282 return err 283 } 284 if len(data) != 0 { 285 if _, err := tarWriter.Write(data); err != nil { 286 return err 287 } 288 } 289 return nil 290 } 291 292 var err error 293 var originalHeader *tar.Header 294 for { 295 originalHeader, err = tarReader.Next() 296 if err == io.EOF { 297 break 298 } 299 if err != nil { 300 pipeWriter.CloseWithError(err) 301 return 302 } 303 304 modifier, ok := mods[originalHeader.Name] 305 if !ok { 306 // No modifiers for this file, copy the header and data 307 if err := tarWriter.WriteHeader(originalHeader); err != nil { 308 pipeWriter.CloseWithError(err) 309 return 310 } 311 if _, err := pools.Copy(tarWriter, tarReader); err != nil { 312 pipeWriter.CloseWithError(err) 313 return 314 } 315 continue 316 } 317 delete(mods, originalHeader.Name) 318 319 if err := modify(originalHeader.Name, originalHeader, modifier, tarReader); err != nil { 320 pipeWriter.CloseWithError(err) 321 return 322 } 323 } 324 325 // Apply the modifiers that haven't matched any files in the archive 326 for name, modifier := range mods { 327 if err := modify(name, nil, modifier, nil); err != nil { 328 pipeWriter.CloseWithError(err) 329 return 330 } 331 } 332 333 pipeWriter.Close() 334 335 }() 336 return pipeReader 337} 338 339// Extension returns the extension of a file that uses the specified compression algorithm. 340func (compression *Compression) Extension() string { 341 switch *compression { 342 case Uncompressed: 343 return "tar" 344 case Bzip2: 345 return "tar.bz2" 346 case Gzip: 347 return "tar.gz" 348 case Xz: 349 return "tar.xz" 350 } 351 return "" 352} 353 354// FileInfoHeader creates a populated Header from fi. 355// Compared to archive pkg this function fills in more information. 356// Also, regardless of Go version, this function fills file type bits (e.g. hdr.Mode |= modeISDIR), 357// which have been deleted since Go 1.9 archive/tar. 358func FileInfoHeader(name string, fi os.FileInfo, link string) (*tar.Header, error) { 359 hdr, err := tar.FileInfoHeader(fi, link) 360 if err != nil { 361 return nil, err 362 } 363 hdr.Mode = fillGo18FileTypeBits(int64(chmodTarEntry(os.FileMode(hdr.Mode))), fi) 364 name, err = canonicalTarName(name, fi.IsDir()) 365 if err != nil { 366 return nil, fmt.Errorf("tar: cannot canonicalize path: %v", err) 367 } 368 hdr.Name = name 369 if err := setHeaderForSpecialDevice(hdr, name, fi.Sys()); err != nil { 370 return nil, err 371 } 372 return hdr, nil 373} 374 375// fillGo18FileTypeBits fills type bits which have been removed on Go 1.9 archive/tar 376// https://github.com/golang/go/commit/66b5a2f 377func fillGo18FileTypeBits(mode int64, fi os.FileInfo) int64 { 378 fm := fi.Mode() 379 switch { 380 case fm.IsRegular(): 381 mode |= modeISREG 382 case fi.IsDir(): 383 mode |= modeISDIR 384 case fm&os.ModeSymlink != 0: 385 mode |= modeISLNK 386 case fm&os.ModeDevice != 0: 387 if fm&os.ModeCharDevice != 0 { 388 mode |= modeISCHR 389 } else { 390 mode |= modeISBLK 391 } 392 case fm&os.ModeNamedPipe != 0: 393 mode |= modeISFIFO 394 case fm&os.ModeSocket != 0: 395 mode |= modeISSOCK 396 } 397 return mode 398} 399 400// ReadSecurityXattrToTarHeader reads security.capability xattr from filesystem 401// to a tar header 402func ReadSecurityXattrToTarHeader(path string, hdr *tar.Header) error { 403 capability, _ := system.Lgetxattr(path, "security.capability") 404 if capability != nil { 405 hdr.Xattrs = make(map[string]string) 406 hdr.Xattrs["security.capability"] = string(capability) 407 } 408 return nil 409} 410 411type tarWhiteoutConverter interface { 412 ConvertWrite(*tar.Header, string, os.FileInfo) (*tar.Header, error) 413 ConvertRead(*tar.Header, string) (bool, error) 414} 415 416type tarAppender struct { 417 TarWriter *tar.Writer 418 Buffer *bufio.Writer 419 420 // for hardlink mapping 421 SeenFiles map[uint64]string 422 IDMappings *idtools.IDMappings 423 ChownOpts *idtools.IDPair 424 425 // For packing and unpacking whiteout files in the 426 // non standard format. The whiteout files defined 427 // by the AUFS standard are used as the tar whiteout 428 // standard. 429 WhiteoutConverter tarWhiteoutConverter 430} 431 432func newTarAppender(idMapping *idtools.IDMappings, writer io.Writer, chownOpts *idtools.IDPair) *tarAppender { 433 return &tarAppender{ 434 SeenFiles: make(map[uint64]string), 435 TarWriter: tar.NewWriter(writer), 436 Buffer: pools.BufioWriter32KPool.Get(nil), 437 IDMappings: idMapping, 438 ChownOpts: chownOpts, 439 } 440} 441 442// canonicalTarName provides a platform-independent and consistent posix-style 443//path for files and directories to be archived regardless of the platform. 444func canonicalTarName(name string, isDir bool) (string, error) { 445 name, err := CanonicalTarNameForPath(name) 446 if err != nil { 447 return "", err 448 } 449 450 // suffix with '/' for directories 451 if isDir && !strings.HasSuffix(name, "/") { 452 name += "/" 453 } 454 return name, nil 455} 456 457// addTarFile adds to the tar archive a file from `path` as `name` 458func (ta *tarAppender) addTarFile(path, name string) error { 459 fi, err := os.Lstat(path) 460 if err != nil { 461 return err 462 } 463 464 var link string 465 if fi.Mode()&os.ModeSymlink != 0 { 466 var err error 467 link, err = os.Readlink(path) 468 if err != nil { 469 return err 470 } 471 } 472 473 hdr, err := FileInfoHeader(name, fi, link) 474 if err != nil { 475 return err 476 } 477 if err := ReadSecurityXattrToTarHeader(path, hdr); err != nil { 478 return err 479 } 480 481 // if it's not a directory and has more than 1 link, 482 // it's hard linked, so set the type flag accordingly 483 if !fi.IsDir() && hasHardlinks(fi) { 484 inode, err := getInodeFromStat(fi.Sys()) 485 if err != nil { 486 return err 487 } 488 // a link should have a name that it links too 489 // and that linked name should be first in the tar archive 490 if oldpath, ok := ta.SeenFiles[inode]; ok { 491 hdr.Typeflag = tar.TypeLink 492 hdr.Linkname = oldpath 493 hdr.Size = 0 // This Must be here for the writer math to add up! 494 } else { 495 ta.SeenFiles[inode] = name 496 } 497 } 498 499 //check whether the file is overlayfs whiteout 500 //if yes, skip re-mapping container ID mappings. 501 isOverlayWhiteout := fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 502 503 //handle re-mapping container ID mappings back to host ID mappings before 504 //writing tar headers/files. We skip whiteout files because they were written 505 //by the kernel and already have proper ownership relative to the host 506 if !isOverlayWhiteout && 507 !strings.HasPrefix(filepath.Base(hdr.Name), WhiteoutPrefix) && 508 !ta.IDMappings.Empty() { 509 fileIDPair, err := getFileUIDGID(fi.Sys()) 510 if err != nil { 511 return err 512 } 513 hdr.Uid, hdr.Gid, err = ta.IDMappings.ToContainer(fileIDPair) 514 if err != nil { 515 return err 516 } 517 } 518 519 // explicitly override with ChownOpts 520 if ta.ChownOpts != nil { 521 hdr.Uid = ta.ChownOpts.UID 522 hdr.Gid = ta.ChownOpts.GID 523 } 524 525 if ta.WhiteoutConverter != nil { 526 wo, err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi) 527 if err != nil { 528 return err 529 } 530 531 // If a new whiteout file exists, write original hdr, then 532 // replace hdr with wo to be written after. Whiteouts should 533 // always be written after the original. Note the original 534 // hdr may have been updated to be a whiteout with returning 535 // a whiteout header 536 if wo != nil { 537 if err := ta.TarWriter.WriteHeader(hdr); err != nil { 538 return err 539 } 540 if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 { 541 return fmt.Errorf("tar: cannot use whiteout for non-empty file") 542 } 543 hdr = wo 544 } 545 } 546 547 if err := ta.TarWriter.WriteHeader(hdr); err != nil { 548 return err 549 } 550 551 if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 { 552 // We use system.OpenSequential to ensure we use sequential file 553 // access on Windows to avoid depleting the standby list. 554 // On Linux, this equates to a regular os.Open. 555 file, err := system.OpenSequential(path) 556 if err != nil { 557 return err 558 } 559 560 ta.Buffer.Reset(ta.TarWriter) 561 defer ta.Buffer.Reset(nil) 562 _, err = io.Copy(ta.Buffer, file) 563 file.Close() 564 if err != nil { 565 return err 566 } 567 err = ta.Buffer.Flush() 568 if err != nil { 569 return err 570 } 571 } 572 573 return nil 574} 575 576func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *idtools.IDPair, inUserns bool) error { 577 // hdr.Mode is in linux format, which we can use for sycalls, 578 // but for os.Foo() calls we need the mode converted to os.FileMode, 579 // so use hdrInfo.Mode() (they differ for e.g. setuid bits) 580 hdrInfo := hdr.FileInfo() 581 582 switch hdr.Typeflag { 583 case tar.TypeDir: 584 // Create directory unless it exists as a directory already. 585 // In that case we just want to merge the two 586 if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { 587 if err := os.Mkdir(path, hdrInfo.Mode()); err != nil { 588 return err 589 } 590 } 591 592 case tar.TypeReg, tar.TypeRegA: 593 // Source is regular file. We use system.OpenFileSequential to use sequential 594 // file access to avoid depleting the standby list on Windows. 595 // On Linux, this equates to a regular os.OpenFile 596 file, err := system.OpenFileSequential(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode()) 597 if err != nil { 598 return err 599 } 600 if _, err := io.Copy(file, reader); err != nil { 601 file.Close() 602 return err 603 } 604 file.Close() 605 606 case tar.TypeBlock, tar.TypeChar: 607 if inUserns { // cannot create devices in a userns 608 return nil 609 } 610 // Handle this is an OS-specific way 611 if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { 612 return err 613 } 614 615 case tar.TypeFifo: 616 // Handle this is an OS-specific way 617 if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { 618 return err 619 } 620 621 case tar.TypeLink: 622 targetPath := filepath.Join(extractDir, hdr.Linkname) 623 // check for hardlink breakout 624 if !strings.HasPrefix(targetPath, extractDir) { 625 return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname)) 626 } 627 if err := os.Link(targetPath, path); err != nil { 628 return err 629 } 630 631 case tar.TypeSymlink: 632 // path -> hdr.Linkname = targetPath 633 // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file 634 targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname) 635 636 // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because 637 // that symlink would first have to be created, which would be caught earlier, at this very check: 638 if !strings.HasPrefix(targetPath, extractDir) { 639 return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) 640 } 641 if err := os.Symlink(hdr.Linkname, path); err != nil { 642 return err 643 } 644 645 case tar.TypeXGlobalHeader: 646 logrus.Debug("PAX Global Extended Headers found and ignored") 647 return nil 648 649 default: 650 return fmt.Errorf("unhandled tar header type %d", hdr.Typeflag) 651 } 652 653 // Lchown is not supported on Windows. 654 if Lchown && runtime.GOOS != "windows" { 655 if chownOpts == nil { 656 chownOpts = &idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid} 657 } 658 if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil { 659 return err 660 } 661 } 662 663 var errors []string 664 for key, value := range hdr.Xattrs { 665 if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil { 666 if err == syscall.ENOTSUP { 667 // We ignore errors here because not all graphdrivers support 668 // xattrs *cough* old versions of AUFS *cough*. However only 669 // ENOTSUP should be emitted in that case, otherwise we still 670 // bail. 671 errors = append(errors, err.Error()) 672 continue 673 } 674 return err 675 } 676 677 } 678 679 if len(errors) > 0 { 680 logrus.WithFields(logrus.Fields{ 681 "errors": errors, 682 }).Warn("ignored xattrs in archive: underlying filesystem doesn't support them") 683 } 684 685 // There is no LChmod, so ignore mode for symlink. Also, this 686 // must happen after chown, as that can modify the file mode 687 if err := handleLChmod(hdr, path, hdrInfo); err != nil { 688 return err 689 } 690 691 aTime := hdr.AccessTime 692 if aTime.Before(hdr.ModTime) { 693 // Last access time should never be before last modified time. 694 aTime = hdr.ModTime 695 } 696 697 // system.Chtimes doesn't support a NOFOLLOW flag atm 698 if hdr.Typeflag == tar.TypeLink { 699 if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { 700 if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { 701 return err 702 } 703 } 704 } else if hdr.Typeflag != tar.TypeSymlink { 705 if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { 706 return err 707 } 708 } else { 709 ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)} 710 if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { 711 return err 712 } 713 } 714 return nil 715} 716 717// Tar creates an archive from the directory at `path`, and returns it as a 718// stream of bytes. 719func Tar(path string, compression Compression) (io.ReadCloser, error) { 720 return TarWithOptions(path, &TarOptions{Compression: compression}) 721} 722 723// TarWithOptions creates an archive from the directory at `path`, only including files whose relative 724// paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`. 725func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) { 726 727 // Fix the source path to work with long path names. This is a no-op 728 // on platforms other than Windows. 729 srcPath = fixVolumePathPrefix(srcPath) 730 731 pm, err := fileutils.NewPatternMatcher(options.ExcludePatterns) 732 if err != nil { 733 return nil, err 734 } 735 736 pipeReader, pipeWriter := io.Pipe() 737 738 compressWriter, err := CompressStream(pipeWriter, options.Compression) 739 if err != nil { 740 return nil, err 741 } 742 743 go func() { 744 ta := newTarAppender( 745 idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps), 746 compressWriter, 747 options.ChownOpts, 748 ) 749 ta.WhiteoutConverter = getWhiteoutConverter(options.WhiteoutFormat) 750 751 defer func() { 752 // Make sure to check the error on Close. 753 if err := ta.TarWriter.Close(); err != nil { 754 logrus.Errorf("Can't close tar writer: %s", err) 755 } 756 if err := compressWriter.Close(); err != nil { 757 logrus.Errorf("Can't close compress writer: %s", err) 758 } 759 if err := pipeWriter.Close(); err != nil { 760 logrus.Errorf("Can't close pipe writer: %s", err) 761 } 762 }() 763 764 // this buffer is needed for the duration of this piped stream 765 defer pools.BufioWriter32KPool.Put(ta.Buffer) 766 767 // In general we log errors here but ignore them because 768 // during e.g. a diff operation the container can continue 769 // mutating the filesystem and we can see transient errors 770 // from this 771 772 stat, err := os.Lstat(srcPath) 773 if err != nil { 774 return 775 } 776 777 if !stat.IsDir() { 778 // We can't later join a non-dir with any includes because the 779 // 'walk' will error if "file/." is stat-ed and "file" is not a 780 // directory. So, we must split the source path and use the 781 // basename as the include. 782 if len(options.IncludeFiles) > 0 { 783 logrus.Warn("Tar: Can't archive a file with includes") 784 } 785 786 dir, base := SplitPathDirEntry(srcPath) 787 srcPath = dir 788 options.IncludeFiles = []string{base} 789 } 790 791 if len(options.IncludeFiles) == 0 { 792 options.IncludeFiles = []string{"."} 793 } 794 795 seen := make(map[string]bool) 796 797 for _, include := range options.IncludeFiles { 798 rebaseName := options.RebaseNames[include] 799 800 walkRoot := getWalkRoot(srcPath, include) 801 filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error { 802 if err != nil { 803 logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err) 804 return nil 805 } 806 807 relFilePath, err := filepath.Rel(srcPath, filePath) 808 if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) { 809 // Error getting relative path OR we are looking 810 // at the source directory path. Skip in both situations. 811 return nil 812 } 813 814 if options.IncludeSourceDir && include == "." && relFilePath != "." { 815 relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator)) 816 } 817 818 skip := false 819 820 // If "include" is an exact match for the current file 821 // then even if there's an "excludePatterns" pattern that 822 // matches it, don't skip it. IOW, assume an explicit 'include' 823 // is asking for that file no matter what - which is true 824 // for some files, like .dockerignore and Dockerfile (sometimes) 825 if include != relFilePath { 826 skip, err = pm.Matches(relFilePath) 827 if err != nil { 828 logrus.Errorf("Error matching %s: %v", relFilePath, err) 829 return err 830 } 831 } 832 833 if skip { 834 // If we want to skip this file and its a directory 835 // then we should first check to see if there's an 836 // excludes pattern (e.g. !dir/file) that starts with this 837 // dir. If so then we can't skip this dir. 838 839 // Its not a dir then so we can just return/skip. 840 if !f.IsDir() { 841 return nil 842 } 843 844 // No exceptions (!...) in patterns so just skip dir 845 if !pm.Exclusions() { 846 return filepath.SkipDir 847 } 848 849 dirSlash := relFilePath + string(filepath.Separator) 850 851 for _, pat := range pm.Patterns() { 852 if !pat.Exclusion() { 853 continue 854 } 855 if strings.HasPrefix(pat.String()+string(filepath.Separator), dirSlash) { 856 // found a match - so can't skip this dir 857 return nil 858 } 859 } 860 861 // No matching exclusion dir so just skip dir 862 return filepath.SkipDir 863 } 864 865 if seen[relFilePath] { 866 return nil 867 } 868 seen[relFilePath] = true 869 870 // Rename the base resource. 871 if rebaseName != "" { 872 var replacement string 873 if rebaseName != string(filepath.Separator) { 874 // Special case the root directory to replace with an 875 // empty string instead so that we don't end up with 876 // double slashes in the paths. 877 replacement = rebaseName 878 } 879 880 relFilePath = strings.Replace(relFilePath, include, replacement, 1) 881 } 882 883 if err := ta.addTarFile(filePath, relFilePath); err != nil { 884 logrus.Errorf("Can't add file %s to tar: %s", filePath, err) 885 // if pipe is broken, stop writing tar stream to it 886 if err == io.ErrClosedPipe { 887 return err 888 } 889 } 890 return nil 891 }) 892 } 893 }() 894 895 return pipeReader, nil 896} 897 898// Unpack unpacks the decompressedArchive to dest with options. 899func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error { 900 tr := tar.NewReader(decompressedArchive) 901 trBuf := pools.BufioReader32KPool.Get(nil) 902 defer pools.BufioReader32KPool.Put(trBuf) 903 904 var dirs []*tar.Header 905 idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps) 906 rootIDs := idMappings.RootPair() 907 whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat) 908 909 // Iterate through the files in the archive. 910loop: 911 for { 912 hdr, err := tr.Next() 913 if err == io.EOF { 914 // end of tar archive 915 break 916 } 917 if err != nil { 918 return err 919 } 920 921 // Normalize name, for safety and for a simple is-root check 922 // This keeps "../" as-is, but normalizes "/../" to "/". Or Windows: 923 // This keeps "..\" as-is, but normalizes "\..\" to "\". 924 hdr.Name = filepath.Clean(hdr.Name) 925 926 for _, exclude := range options.ExcludePatterns { 927 if strings.HasPrefix(hdr.Name, exclude) { 928 continue loop 929 } 930 } 931 932 // After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in 933 // the filepath format for the OS on which the daemon is running. Hence 934 // the check for a slash-suffix MUST be done in an OS-agnostic way. 935 if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { 936 // Not the root directory, ensure that the parent directory exists 937 parent := filepath.Dir(hdr.Name) 938 parentPath := filepath.Join(dest, parent) 939 if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { 940 err = idtools.MkdirAllAndChownNew(parentPath, 0777, rootIDs) 941 if err != nil { 942 return err 943 } 944 } 945 } 946 947 path := filepath.Join(dest, hdr.Name) 948 rel, err := filepath.Rel(dest, path) 949 if err != nil { 950 return err 951 } 952 if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { 953 return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) 954 } 955 956 // If path exits we almost always just want to remove and replace it 957 // The only exception is when it is a directory *and* the file from 958 // the layer is also a directory. Then we want to merge them (i.e. 959 // just apply the metadata from the layer). 960 if fi, err := os.Lstat(path); err == nil { 961 if options.NoOverwriteDirNonDir && fi.IsDir() && hdr.Typeflag != tar.TypeDir { 962 // If NoOverwriteDirNonDir is true then we cannot replace 963 // an existing directory with a non-directory from the archive. 964 return fmt.Errorf("cannot overwrite directory %q with non-directory %q", path, dest) 965 } 966 967 if options.NoOverwriteDirNonDir && !fi.IsDir() && hdr.Typeflag == tar.TypeDir { 968 // If NoOverwriteDirNonDir is true then we cannot replace 969 // an existing non-directory with a directory from the archive. 970 return fmt.Errorf("cannot overwrite non-directory %q with directory %q", path, dest) 971 } 972 973 if fi.IsDir() && hdr.Name == "." { 974 continue 975 } 976 977 if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { 978 if err := os.RemoveAll(path); err != nil { 979 return err 980 } 981 } 982 } 983 trBuf.Reset(tr) 984 985 if err := remapIDs(idMappings, hdr); err != nil { 986 return err 987 } 988 989 if whiteoutConverter != nil { 990 writeFile, err := whiteoutConverter.ConvertRead(hdr, path) 991 if err != nil { 992 return err 993 } 994 if !writeFile { 995 continue 996 } 997 } 998 999 if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts, options.InUserNS); err != nil { 1000 return err 1001 } 1002 1003 // Directory mtimes must be handled at the end to avoid further 1004 // file creation in them to modify the directory mtime 1005 if hdr.Typeflag == tar.TypeDir { 1006 dirs = append(dirs, hdr) 1007 } 1008 } 1009 1010 for _, hdr := range dirs { 1011 path := filepath.Join(dest, hdr.Name) 1012 1013 if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil { 1014 return err 1015 } 1016 } 1017 return nil 1018} 1019 1020// Untar reads a stream of bytes from `archive`, parses it as a tar archive, 1021// and unpacks it into the directory at `dest`. 1022// The archive may be compressed with one of the following algorithms: 1023// identity (uncompressed), gzip, bzip2, xz. 1024// FIXME: specify behavior when target path exists vs. doesn't exist. 1025func Untar(tarArchive io.Reader, dest string, options *TarOptions) error { 1026 return untarHandler(tarArchive, dest, options, true) 1027} 1028 1029// UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive, 1030// and unpacks it into the directory at `dest`. 1031// The archive must be an uncompressed stream. 1032func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error { 1033 return untarHandler(tarArchive, dest, options, false) 1034} 1035 1036// Handler for teasing out the automatic decompression 1037func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error { 1038 if tarArchive == nil { 1039 return fmt.Errorf("Empty archive") 1040 } 1041 dest = filepath.Clean(dest) 1042 if options == nil { 1043 options = &TarOptions{} 1044 } 1045 if options.ExcludePatterns == nil { 1046 options.ExcludePatterns = []string{} 1047 } 1048 1049 r := tarArchive 1050 if decompress { 1051 decompressedArchive, err := DecompressStream(tarArchive) 1052 if err != nil { 1053 return err 1054 } 1055 defer decompressedArchive.Close() 1056 r = decompressedArchive 1057 } 1058 1059 return Unpack(r, dest, options) 1060} 1061 1062// TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. 1063// If either Tar or Untar fails, TarUntar aborts and returns the error. 1064func (archiver *Archiver) TarUntar(src, dst string) error { 1065 logrus.Debugf("TarUntar(%s %s)", src, dst) 1066 archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed}) 1067 if err != nil { 1068 return err 1069 } 1070 defer archive.Close() 1071 options := &TarOptions{ 1072 UIDMaps: archiver.IDMappingsVar.UIDs(), 1073 GIDMaps: archiver.IDMappingsVar.GIDs(), 1074 } 1075 return archiver.Untar(archive, dst, options) 1076} 1077 1078// UntarPath untar a file from path to a destination, src is the source tar file path. 1079func (archiver *Archiver) UntarPath(src, dst string) error { 1080 archive, err := os.Open(src) 1081 if err != nil { 1082 return err 1083 } 1084 defer archive.Close() 1085 options := &TarOptions{ 1086 UIDMaps: archiver.IDMappingsVar.UIDs(), 1087 GIDMaps: archiver.IDMappingsVar.GIDs(), 1088 } 1089 return archiver.Untar(archive, dst, options) 1090} 1091 1092// CopyWithTar creates a tar archive of filesystem path `src`, and 1093// unpacks it at filesystem path `dst`. 1094// The archive is streamed directly with fixed buffering and no 1095// intermediary disk IO. 1096func (archiver *Archiver) CopyWithTar(src, dst string) error { 1097 srcSt, err := os.Stat(src) 1098 if err != nil { 1099 return err 1100 } 1101 if !srcSt.IsDir() { 1102 return archiver.CopyFileWithTar(src, dst) 1103 } 1104 1105 // if this Archiver is set up with ID mapping we need to create 1106 // the new destination directory with the remapped root UID/GID pair 1107 // as owner 1108 rootIDs := archiver.IDMappingsVar.RootPair() 1109 // Create dst, copy src's content into it 1110 logrus.Debugf("Creating dest directory: %s", dst) 1111 if err := idtools.MkdirAllAndChownNew(dst, 0755, rootIDs); err != nil { 1112 return err 1113 } 1114 logrus.Debugf("Calling TarUntar(%s, %s)", src, dst) 1115 return archiver.TarUntar(src, dst) 1116} 1117 1118// CopyFileWithTar emulates the behavior of the 'cp' command-line 1119// for a single file. It copies a regular file from path `src` to 1120// path `dst`, and preserves all its metadata. 1121func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { 1122 logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst) 1123 srcSt, err := os.Stat(src) 1124 if err != nil { 1125 return err 1126 } 1127 1128 if srcSt.IsDir() { 1129 return fmt.Errorf("Can't copy a directory") 1130 } 1131 1132 // Clean up the trailing slash. This must be done in an operating 1133 // system specific manner. 1134 if dst[len(dst)-1] == os.PathSeparator { 1135 dst = filepath.Join(dst, filepath.Base(src)) 1136 } 1137 // Create the holding directory if necessary 1138 if err := system.MkdirAll(filepath.Dir(dst), 0700, ""); err != nil { 1139 return err 1140 } 1141 1142 r, w := io.Pipe() 1143 errC := make(chan error, 1) 1144 1145 go func() { 1146 defer close(errC) 1147 1148 errC <- func() error { 1149 defer w.Close() 1150 1151 srcF, err := os.Open(src) 1152 if err != nil { 1153 return err 1154 } 1155 defer srcF.Close() 1156 1157 hdr, err := tar.FileInfoHeader(srcSt, "") 1158 if err != nil { 1159 return err 1160 } 1161 hdr.Name = filepath.Base(dst) 1162 hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) 1163 1164 if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil { 1165 return err 1166 } 1167 1168 tw := tar.NewWriter(w) 1169 defer tw.Close() 1170 if err := tw.WriteHeader(hdr); err != nil { 1171 return err 1172 } 1173 if _, err := io.Copy(tw, srcF); err != nil { 1174 return err 1175 } 1176 return nil 1177 }() 1178 }() 1179 defer func() { 1180 if er := <-errC; err == nil && er != nil { 1181 err = er 1182 } 1183 }() 1184 1185 err = archiver.Untar(r, filepath.Dir(dst), nil) 1186 if err != nil { 1187 r.CloseWithError(err) 1188 } 1189 return err 1190} 1191 1192// IDMappings returns the IDMappings of the archiver. 1193func (archiver *Archiver) IDMappings() *idtools.IDMappings { 1194 return archiver.IDMappingsVar 1195} 1196 1197func remapIDs(idMappings *idtools.IDMappings, hdr *tar.Header) error { 1198 ids, err := idMappings.ToHost(idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}) 1199 hdr.Uid, hdr.Gid = ids.UID, ids.GID 1200 return err 1201} 1202 1203// cmdStream executes a command, and returns its stdout as a stream. 1204// If the command fails to run or doesn't complete successfully, an error 1205// will be returned, including anything written on stderr. 1206func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, error) { 1207 cmd.Stdin = input 1208 pipeR, pipeW := io.Pipe() 1209 cmd.Stdout = pipeW 1210 var errBuf bytes.Buffer 1211 cmd.Stderr = &errBuf 1212 1213 // Run the command and return the pipe 1214 if err := cmd.Start(); err != nil { 1215 return nil, err 1216 } 1217 1218 // Copy stdout to the returned pipe 1219 go func() { 1220 if err := cmd.Wait(); err != nil { 1221 pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String())) 1222 } else { 1223 pipeW.Close() 1224 } 1225 }() 1226 1227 return pipeR, nil 1228} 1229 1230// NewTempArchive reads the content of src into a temporary file, and returns the contents 1231// of that file as an archive. The archive can only be read once - as soon as reading completes, 1232// the file will be deleted. 1233func NewTempArchive(src io.Reader, dir string) (*TempArchive, error) { 1234 f, err := ioutil.TempFile(dir, "") 1235 if err != nil { 1236 return nil, err 1237 } 1238 if _, err := io.Copy(f, src); err != nil { 1239 return nil, err 1240 } 1241 if _, err := f.Seek(0, 0); err != nil { 1242 return nil, err 1243 } 1244 st, err := f.Stat() 1245 if err != nil { 1246 return nil, err 1247 } 1248 size := st.Size() 1249 return &TempArchive{File: f, Size: size}, nil 1250} 1251 1252// TempArchive is a temporary archive. The archive can only be read once - as soon as reading completes, 1253// the file will be deleted. 1254type TempArchive struct { 1255 *os.File 1256 Size int64 // Pre-computed from Stat().Size() as a convenience 1257 read int64 1258 closed bool 1259} 1260 1261// Close closes the underlying file if it's still open, or does a no-op 1262// to allow callers to try to close the TempArchive multiple times safely. 1263func (archive *TempArchive) Close() error { 1264 if archive.closed { 1265 return nil 1266 } 1267 1268 archive.closed = true 1269 1270 return archive.File.Close() 1271} 1272 1273func (archive *TempArchive) Read(data []byte) (int, error) { 1274 n, err := archive.File.Read(data) 1275 archive.read += int64(n) 1276 if err != nil || archive.read == archive.Size { 1277 archive.Close() 1278 os.Remove(archive.File.Name()) 1279 } 1280 return n, err 1281} 1282