1// Copyright 2009 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package tar 6 7import ( 8 "bytes" 9 "io" 10 "strconv" 11 "strings" 12 "time" 13) 14 15// Reader provides sequential access to the contents of a tar archive. 16// Reader.Next advances to the next file in the archive (including the first), 17// and then Reader can be treated as an io.Reader to access the file's data. 18type Reader struct { 19 r io.Reader 20 pad int64 // Amount of padding (ignored) after current file entry 21 curr fileReader // Reader for current file entry 22 blk block // Buffer to use as temporary local storage 23 24 // err is a persistent error. 25 // It is only the responsibility of every exported method of Reader to 26 // ensure that this error is sticky. 27 err error 28} 29 30type fileReader interface { 31 io.Reader 32 fileState 33 34 WriteTo(io.Writer) (int64, error) 35} 36 37// NewReader creates a new Reader reading from r. 38func NewReader(r io.Reader) *Reader { 39 return &Reader{r: r, curr: ®FileReader{r, 0}} 40} 41 42// Next advances to the next entry in the tar archive. 43// The Header.Size determines how many bytes can be read for the next file. 44// Any remaining data in the current file is automatically discarded. 45// 46// io.EOF is returned at the end of the input. 47func (tr *Reader) Next() (*Header, error) { 48 if tr.err != nil { 49 return nil, tr.err 50 } 51 hdr, err := tr.next() 52 tr.err = err 53 return hdr, err 54} 55 56func (tr *Reader) next() (*Header, error) { 57 var paxHdrs map[string]string 58 var gnuLongName, gnuLongLink string 59 60 // Externally, Next iterates through the tar archive as if it is a series of 61 // files. Internally, the tar format often uses fake "files" to add meta 62 // data that describes the next file. These meta data "files" should not 63 // normally be visible to the outside. As such, this loop iterates through 64 // one or more "header files" until it finds a "normal file". 65 format := FormatUSTAR | FormatPAX | FormatGNU 66 for { 67 // Discard the remainder of the file and any padding. 68 if err := discard(tr.r, tr.curr.physicalRemaining()); err != nil { 69 return nil, err 70 } 71 if _, err := tryReadFull(tr.r, tr.blk[:tr.pad]); err != nil { 72 return nil, err 73 } 74 tr.pad = 0 75 76 hdr, rawHdr, err := tr.readHeader() 77 if err != nil { 78 return nil, err 79 } 80 if err := tr.handleRegularFile(hdr); err != nil { 81 return nil, err 82 } 83 format.mayOnlyBe(hdr.Format) 84 85 // Check for PAX/GNU special headers and files. 86 switch hdr.Typeflag { 87 case TypeXHeader, TypeXGlobalHeader: 88 format.mayOnlyBe(FormatPAX) 89 paxHdrs, err = parsePAX(tr) 90 if err != nil { 91 return nil, err 92 } 93 if hdr.Typeflag == TypeXGlobalHeader { 94 mergePAX(hdr, paxHdrs) 95 return &Header{ 96 Name: hdr.Name, 97 Typeflag: hdr.Typeflag, 98 Xattrs: hdr.Xattrs, 99 PAXRecords: hdr.PAXRecords, 100 Format: format, 101 }, nil 102 } 103 continue // This is a meta header affecting the next header 104 case TypeGNULongName, TypeGNULongLink: 105 format.mayOnlyBe(FormatGNU) 106 realname, err := io.ReadAll(tr) 107 if err != nil { 108 return nil, err 109 } 110 111 var p parser 112 switch hdr.Typeflag { 113 case TypeGNULongName: 114 gnuLongName = p.parseString(realname) 115 case TypeGNULongLink: 116 gnuLongLink = p.parseString(realname) 117 } 118 continue // This is a meta header affecting the next header 119 default: 120 // The old GNU sparse format is handled here since it is technically 121 // just a regular file with additional attributes. 122 123 if err := mergePAX(hdr, paxHdrs); err != nil { 124 return nil, err 125 } 126 if gnuLongName != "" { 127 hdr.Name = gnuLongName 128 } 129 if gnuLongLink != "" { 130 hdr.Linkname = gnuLongLink 131 } 132 if hdr.Typeflag == TypeRegA { 133 if strings.HasSuffix(hdr.Name, "/") { 134 hdr.Typeflag = TypeDir // Legacy archives use trailing slash for directories 135 } else { 136 hdr.Typeflag = TypeReg 137 } 138 } 139 140 // The extended headers may have updated the size. 141 // Thus, setup the regFileReader again after merging PAX headers. 142 if err := tr.handleRegularFile(hdr); err != nil { 143 return nil, err 144 } 145 146 // Sparse formats rely on being able to read from the logical data 147 // section; there must be a preceding call to handleRegularFile. 148 if err := tr.handleSparseFile(hdr, rawHdr); err != nil { 149 return nil, err 150 } 151 152 // Set the final guess at the format. 153 if format.has(FormatUSTAR) && format.has(FormatPAX) { 154 format.mayOnlyBe(FormatUSTAR) 155 } 156 hdr.Format = format 157 return hdr, nil // This is a file, so stop 158 } 159 } 160} 161 162// handleRegularFile sets up the current file reader and padding such that it 163// can only read the following logical data section. It will properly handle 164// special headers that contain no data section. 165func (tr *Reader) handleRegularFile(hdr *Header) error { 166 nb := hdr.Size 167 if isHeaderOnlyType(hdr.Typeflag) { 168 nb = 0 169 } 170 if nb < 0 { 171 return ErrHeader 172 } 173 174 tr.pad = blockPadding(nb) 175 tr.curr = ®FileReader{r: tr.r, nb: nb} 176 return nil 177} 178 179// handleSparseFile checks if the current file is a sparse format of any type 180// and sets the curr reader appropriately. 181func (tr *Reader) handleSparseFile(hdr *Header, rawHdr *block) error { 182 var spd sparseDatas 183 var err error 184 if hdr.Typeflag == TypeGNUSparse { 185 spd, err = tr.readOldGNUSparseMap(hdr, rawHdr) 186 } else { 187 spd, err = tr.readGNUSparsePAXHeaders(hdr) 188 } 189 190 // If sp is non-nil, then this is a sparse file. 191 // Note that it is possible for len(sp) == 0. 192 if err == nil && spd != nil { 193 if isHeaderOnlyType(hdr.Typeflag) || !validateSparseEntries(spd, hdr.Size) { 194 return ErrHeader 195 } 196 sph := invertSparseEntries(spd, hdr.Size) 197 tr.curr = &sparseFileReader{tr.curr, sph, 0} 198 } 199 return err 200} 201 202// readGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. 203// If they are found, then this function reads the sparse map and returns it. 204// This assumes that 0.0 headers have already been converted to 0.1 headers 205// by the PAX header parsing logic. 206func (tr *Reader) readGNUSparsePAXHeaders(hdr *Header) (sparseDatas, error) { 207 // Identify the version of GNU headers. 208 var is1x0 bool 209 major, minor := hdr.PAXRecords[paxGNUSparseMajor], hdr.PAXRecords[paxGNUSparseMinor] 210 switch { 211 case major == "0" && (minor == "0" || minor == "1"): 212 is1x0 = false 213 case major == "1" && minor == "0": 214 is1x0 = true 215 case major != "" || minor != "": 216 return nil, nil // Unknown GNU sparse PAX version 217 case hdr.PAXRecords[paxGNUSparseMap] != "": 218 is1x0 = false // 0.0 and 0.1 did not have explicit version records, so guess 219 default: 220 return nil, nil // Not a PAX format GNU sparse file. 221 } 222 hdr.Format.mayOnlyBe(FormatPAX) 223 224 // Update hdr from GNU sparse PAX headers. 225 if name := hdr.PAXRecords[paxGNUSparseName]; name != "" { 226 hdr.Name = name 227 } 228 size := hdr.PAXRecords[paxGNUSparseSize] 229 if size == "" { 230 size = hdr.PAXRecords[paxGNUSparseRealSize] 231 } 232 if size != "" { 233 n, err := strconv.ParseInt(size, 10, 64) 234 if err != nil { 235 return nil, ErrHeader 236 } 237 hdr.Size = n 238 } 239 240 // Read the sparse map according to the appropriate format. 241 if is1x0 { 242 return readGNUSparseMap1x0(tr.curr) 243 } 244 return readGNUSparseMap0x1(hdr.PAXRecords) 245} 246 247// mergePAX merges paxHdrs into hdr for all relevant fields of Header. 248func mergePAX(hdr *Header, paxHdrs map[string]string) (err error) { 249 for k, v := range paxHdrs { 250 if v == "" { 251 continue // Keep the original USTAR value 252 } 253 var id64 int64 254 switch k { 255 case paxPath: 256 hdr.Name = v 257 case paxLinkpath: 258 hdr.Linkname = v 259 case paxUname: 260 hdr.Uname = v 261 case paxGname: 262 hdr.Gname = v 263 case paxUid: 264 id64, err = strconv.ParseInt(v, 10, 64) 265 hdr.Uid = int(id64) // Integer overflow possible 266 case paxGid: 267 id64, err = strconv.ParseInt(v, 10, 64) 268 hdr.Gid = int(id64) // Integer overflow possible 269 case paxAtime: 270 hdr.AccessTime, err = parsePAXTime(v) 271 case paxMtime: 272 hdr.ModTime, err = parsePAXTime(v) 273 case paxCtime: 274 hdr.ChangeTime, err = parsePAXTime(v) 275 case paxSize: 276 hdr.Size, err = strconv.ParseInt(v, 10, 64) 277 default: 278 if strings.HasPrefix(k, paxSchilyXattr) { 279 if hdr.Xattrs == nil { 280 hdr.Xattrs = make(map[string]string) 281 } 282 hdr.Xattrs[k[len(paxSchilyXattr):]] = v 283 } 284 } 285 if err != nil { 286 return ErrHeader 287 } 288 } 289 hdr.PAXRecords = paxHdrs 290 return nil 291} 292 293// parsePAX parses PAX headers. 294// If an extended header (type 'x') is invalid, ErrHeader is returned 295func parsePAX(r io.Reader) (map[string]string, error) { 296 buf, err := io.ReadAll(r) 297 if err != nil { 298 return nil, err 299 } 300 sbuf := string(buf) 301 302 // For GNU PAX sparse format 0.0 support. 303 // This function transforms the sparse format 0.0 headers into format 0.1 304 // headers since 0.0 headers were not PAX compliant. 305 var sparseMap []string 306 307 paxHdrs := make(map[string]string) 308 for len(sbuf) > 0 { 309 key, value, residual, err := parsePAXRecord(sbuf) 310 if err != nil { 311 return nil, ErrHeader 312 } 313 sbuf = residual 314 315 switch key { 316 case paxGNUSparseOffset, paxGNUSparseNumBytes: 317 // Validate sparse header order and value. 318 if (len(sparseMap)%2 == 0 && key != paxGNUSparseOffset) || 319 (len(sparseMap)%2 == 1 && key != paxGNUSparseNumBytes) || 320 strings.Contains(value, ",") { 321 return nil, ErrHeader 322 } 323 sparseMap = append(sparseMap, value) 324 default: 325 paxHdrs[key] = value 326 } 327 } 328 if len(sparseMap) > 0 { 329 paxHdrs[paxGNUSparseMap] = strings.Join(sparseMap, ",") 330 } 331 return paxHdrs, nil 332} 333 334// readHeader reads the next block header and assumes that the underlying reader 335// is already aligned to a block boundary. It returns the raw block of the 336// header in case further processing is required. 337// 338// The err will be set to io.EOF only when one of the following occurs: 339// * Exactly 0 bytes are read and EOF is hit. 340// * Exactly 1 block of zeros is read and EOF is hit. 341// * At least 2 blocks of zeros are read. 342func (tr *Reader) readHeader() (*Header, *block, error) { 343 // Two blocks of zero bytes marks the end of the archive. 344 if _, err := io.ReadFull(tr.r, tr.blk[:]); err != nil { 345 return nil, nil, err // EOF is okay here; exactly 0 bytes read 346 } 347 if bytes.Equal(tr.blk[:], zeroBlock[:]) { 348 if _, err := io.ReadFull(tr.r, tr.blk[:]); err != nil { 349 return nil, nil, err // EOF is okay here; exactly 1 block of zeros read 350 } 351 if bytes.Equal(tr.blk[:], zeroBlock[:]) { 352 return nil, nil, io.EOF // normal EOF; exactly 2 block of zeros read 353 } 354 return nil, nil, ErrHeader // Zero block and then non-zero block 355 } 356 357 // Verify the header matches a known format. 358 format := tr.blk.getFormat() 359 if format == FormatUnknown { 360 return nil, nil, ErrHeader 361 } 362 363 var p parser 364 hdr := new(Header) 365 366 // Unpack the V7 header. 367 v7 := tr.blk.toV7() 368 hdr.Typeflag = v7.typeFlag()[0] 369 hdr.Name = p.parseString(v7.name()) 370 hdr.Linkname = p.parseString(v7.linkName()) 371 hdr.Size = p.parseNumeric(v7.size()) 372 hdr.Mode = p.parseNumeric(v7.mode()) 373 hdr.Uid = int(p.parseNumeric(v7.uid())) 374 hdr.Gid = int(p.parseNumeric(v7.gid())) 375 hdr.ModTime = time.Unix(p.parseNumeric(v7.modTime()), 0) 376 377 // Unpack format specific fields. 378 if format > formatV7 { 379 ustar := tr.blk.toUSTAR() 380 hdr.Uname = p.parseString(ustar.userName()) 381 hdr.Gname = p.parseString(ustar.groupName()) 382 hdr.Devmajor = p.parseNumeric(ustar.devMajor()) 383 hdr.Devminor = p.parseNumeric(ustar.devMinor()) 384 385 var prefix string 386 switch { 387 case format.has(FormatUSTAR | FormatPAX): 388 hdr.Format = format 389 ustar := tr.blk.toUSTAR() 390 prefix = p.parseString(ustar.prefix()) 391 392 // For Format detection, check if block is properly formatted since 393 // the parser is more liberal than what USTAR actually permits. 394 notASCII := func(r rune) bool { return r >= 0x80 } 395 if bytes.IndexFunc(tr.blk[:], notASCII) >= 0 { 396 hdr.Format = FormatUnknown // Non-ASCII characters in block. 397 } 398 nul := func(b []byte) bool { return int(b[len(b)-1]) == 0 } 399 if !(nul(v7.size()) && nul(v7.mode()) && nul(v7.uid()) && nul(v7.gid()) && 400 nul(v7.modTime()) && nul(ustar.devMajor()) && nul(ustar.devMinor())) { 401 hdr.Format = FormatUnknown // Numeric fields must end in NUL 402 } 403 case format.has(formatSTAR): 404 star := tr.blk.toSTAR() 405 prefix = p.parseString(star.prefix()) 406 hdr.AccessTime = time.Unix(p.parseNumeric(star.accessTime()), 0) 407 hdr.ChangeTime = time.Unix(p.parseNumeric(star.changeTime()), 0) 408 case format.has(FormatGNU): 409 hdr.Format = format 410 var p2 parser 411 gnu := tr.blk.toGNU() 412 if b := gnu.accessTime(); b[0] != 0 { 413 hdr.AccessTime = time.Unix(p2.parseNumeric(b), 0) 414 } 415 if b := gnu.changeTime(); b[0] != 0 { 416 hdr.ChangeTime = time.Unix(p2.parseNumeric(b), 0) 417 } 418 419 // Prior to Go1.8, the Writer had a bug where it would output 420 // an invalid tar file in certain rare situations because the logic 421 // incorrectly believed that the old GNU format had a prefix field. 422 // This is wrong and leads to an output file that mangles the 423 // atime and ctime fields, which are often left unused. 424 // 425 // In order to continue reading tar files created by former, buggy 426 // versions of Go, we skeptically parse the atime and ctime fields. 427 // If we are unable to parse them and the prefix field looks like 428 // an ASCII string, then we fallback on the pre-Go1.8 behavior 429 // of treating these fields as the USTAR prefix field. 430 // 431 // Note that this will not use the fallback logic for all possible 432 // files generated by a pre-Go1.8 toolchain. If the generated file 433 // happened to have a prefix field that parses as valid 434 // atime and ctime fields (e.g., when they are valid octal strings), 435 // then it is impossible to distinguish between a valid GNU file 436 // and an invalid pre-Go1.8 file. 437 // 438 // See https://golang.org/issues/12594 439 // See https://golang.org/issues/21005 440 if p2.err != nil { 441 hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{} 442 ustar := tr.blk.toUSTAR() 443 if s := p.parseString(ustar.prefix()); isASCII(s) { 444 prefix = s 445 } 446 hdr.Format = FormatUnknown // Buggy file is not GNU 447 } 448 } 449 if len(prefix) > 0 { 450 hdr.Name = prefix + "/" + hdr.Name 451 } 452 } 453 return hdr, &tr.blk, p.err 454} 455 456// readOldGNUSparseMap reads the sparse map from the old GNU sparse format. 457// The sparse map is stored in the tar header if it's small enough. 458// If it's larger than four entries, then one or more extension headers are used 459// to store the rest of the sparse map. 460// 461// The Header.Size does not reflect the size of any extended headers used. 462// Thus, this function will read from the raw io.Reader to fetch extra headers. 463// This method mutates blk in the process. 464func (tr *Reader) readOldGNUSparseMap(hdr *Header, blk *block) (sparseDatas, error) { 465 // Make sure that the input format is GNU. 466 // Unfortunately, the STAR format also has a sparse header format that uses 467 // the same type flag but has a completely different layout. 468 if blk.getFormat() != FormatGNU { 469 return nil, ErrHeader 470 } 471 hdr.Format.mayOnlyBe(FormatGNU) 472 473 var p parser 474 hdr.Size = p.parseNumeric(blk.toGNU().realSize()) 475 if p.err != nil { 476 return nil, p.err 477 } 478 s := blk.toGNU().sparse() 479 spd := make(sparseDatas, 0, s.maxEntries()) 480 for { 481 for i := 0; i < s.maxEntries(); i++ { 482 // This termination condition is identical to GNU and BSD tar. 483 if s.entry(i).offset()[0] == 0x00 { 484 break // Don't return, need to process extended headers (even if empty) 485 } 486 offset := p.parseNumeric(s.entry(i).offset()) 487 length := p.parseNumeric(s.entry(i).length()) 488 if p.err != nil { 489 return nil, p.err 490 } 491 spd = append(spd, sparseEntry{Offset: offset, Length: length}) 492 } 493 494 if s.isExtended()[0] > 0 { 495 // There are more entries. Read an extension header and parse its entries. 496 if _, err := mustReadFull(tr.r, blk[:]); err != nil { 497 return nil, err 498 } 499 s = blk.toSparse() 500 continue 501 } 502 return spd, nil // Done 503 } 504} 505 506// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format 507// version 1.0. The format of the sparse map consists of a series of 508// newline-terminated numeric fields. The first field is the number of entries 509// and is always present. Following this are the entries, consisting of two 510// fields (offset, length). This function must stop reading at the end 511// boundary of the block containing the last newline. 512// 513// Note that the GNU manual says that numeric values should be encoded in octal 514// format. However, the GNU tar utility itself outputs these values in decimal. 515// As such, this library treats values as being encoded in decimal. 516func readGNUSparseMap1x0(r io.Reader) (sparseDatas, error) { 517 var ( 518 cntNewline int64 519 buf bytes.Buffer 520 blk block 521 ) 522 523 // feedTokens copies data in blocks from r into buf until there are 524 // at least cnt newlines in buf. It will not read more blocks than needed. 525 feedTokens := func(n int64) error { 526 for cntNewline < n { 527 if _, err := mustReadFull(r, blk[:]); err != nil { 528 return err 529 } 530 buf.Write(blk[:]) 531 for _, c := range blk { 532 if c == '\n' { 533 cntNewline++ 534 } 535 } 536 } 537 return nil 538 } 539 540 // nextToken gets the next token delimited by a newline. This assumes that 541 // at least one newline exists in the buffer. 542 nextToken := func() string { 543 cntNewline-- 544 tok, _ := buf.ReadString('\n') 545 return strings.TrimRight(tok, "\n") 546 } 547 548 // Parse for the number of entries. 549 // Use integer overflow resistant math to check this. 550 if err := feedTokens(1); err != nil { 551 return nil, err 552 } 553 numEntries, err := strconv.ParseInt(nextToken(), 10, 0) // Intentionally parse as native int 554 if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { 555 return nil, ErrHeader 556 } 557 558 // Parse for all member entries. 559 // numEntries is trusted after this since a potential attacker must have 560 // committed resources proportional to what this library used. 561 if err := feedTokens(2 * numEntries); err != nil { 562 return nil, err 563 } 564 spd := make(sparseDatas, 0, numEntries) 565 for i := int64(0); i < numEntries; i++ { 566 offset, err1 := strconv.ParseInt(nextToken(), 10, 64) 567 length, err2 := strconv.ParseInt(nextToken(), 10, 64) 568 if err1 != nil || err2 != nil { 569 return nil, ErrHeader 570 } 571 spd = append(spd, sparseEntry{Offset: offset, Length: length}) 572 } 573 return spd, nil 574} 575 576// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format 577// version 0.1. The sparse map is stored in the PAX headers. 578func readGNUSparseMap0x1(paxHdrs map[string]string) (sparseDatas, error) { 579 // Get number of entries. 580 // Use integer overflow resistant math to check this. 581 numEntriesStr := paxHdrs[paxGNUSparseNumBlocks] 582 numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) // Intentionally parse as native int 583 if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { 584 return nil, ErrHeader 585 } 586 587 // There should be two numbers in sparseMap for each entry. 588 sparseMap := strings.Split(paxHdrs[paxGNUSparseMap], ",") 589 if len(sparseMap) == 1 && sparseMap[0] == "" { 590 sparseMap = sparseMap[:0] 591 } 592 if int64(len(sparseMap)) != 2*numEntries { 593 return nil, ErrHeader 594 } 595 596 // Loop through the entries in the sparse map. 597 // numEntries is trusted now. 598 spd := make(sparseDatas, 0, numEntries) 599 for len(sparseMap) >= 2 { 600 offset, err1 := strconv.ParseInt(sparseMap[0], 10, 64) 601 length, err2 := strconv.ParseInt(sparseMap[1], 10, 64) 602 if err1 != nil || err2 != nil { 603 return nil, ErrHeader 604 } 605 spd = append(spd, sparseEntry{Offset: offset, Length: length}) 606 sparseMap = sparseMap[2:] 607 } 608 return spd, nil 609} 610 611// Read reads from the current file in the tar archive. 612// It returns (0, io.EOF) when it reaches the end of that file, 613// until Next is called to advance to the next file. 614// 615// If the current file is sparse, then the regions marked as a hole 616// are read back as NUL-bytes. 617// 618// Calling Read on special types like TypeLink, TypeSymlink, TypeChar, 619// TypeBlock, TypeDir, and TypeFifo returns (0, io.EOF) regardless of what 620// the Header.Size claims. 621func (tr *Reader) Read(b []byte) (int, error) { 622 if tr.err != nil { 623 return 0, tr.err 624 } 625 n, err := tr.curr.Read(b) 626 if err != nil && err != io.EOF { 627 tr.err = err 628 } 629 return n, err 630} 631 632// writeTo writes the content of the current file to w. 633// The bytes written matches the number of remaining bytes in the current file. 634// 635// If the current file is sparse and w is an io.WriteSeeker, 636// then writeTo uses Seek to skip past holes defined in Header.SparseHoles, 637// assuming that skipped regions are filled with NULs. 638// This always writes the last byte to ensure w is the right size. 639// 640// TODO(dsnet): Re-export this when adding sparse file support. 641// See https://golang.org/issue/22735 642func (tr *Reader) writeTo(w io.Writer) (int64, error) { 643 if tr.err != nil { 644 return 0, tr.err 645 } 646 n, err := tr.curr.WriteTo(w) 647 if err != nil { 648 tr.err = err 649 } 650 return n, err 651} 652 653// regFileReader is a fileReader for reading data from a regular file entry. 654type regFileReader struct { 655 r io.Reader // Underlying Reader 656 nb int64 // Number of remaining bytes to read 657} 658 659func (fr *regFileReader) Read(b []byte) (n int, err error) { 660 if int64(len(b)) > fr.nb { 661 b = b[:fr.nb] 662 } 663 if len(b) > 0 { 664 n, err = fr.r.Read(b) 665 fr.nb -= int64(n) 666 } 667 switch { 668 case err == io.EOF && fr.nb > 0: 669 return n, io.ErrUnexpectedEOF 670 case err == nil && fr.nb == 0: 671 return n, io.EOF 672 default: 673 return n, err 674 } 675} 676 677func (fr *regFileReader) WriteTo(w io.Writer) (int64, error) { 678 return io.Copy(w, struct{ io.Reader }{fr}) 679} 680 681// logicalRemaining implements fileState.logicalRemaining. 682func (fr regFileReader) logicalRemaining() int64 { 683 return fr.nb 684} 685 686// logicalRemaining implements fileState.physicalRemaining. 687func (fr regFileReader) physicalRemaining() int64 { 688 return fr.nb 689} 690 691// sparseFileReader is a fileReader for reading data from a sparse file entry. 692type sparseFileReader struct { 693 fr fileReader // Underlying fileReader 694 sp sparseHoles // Normalized list of sparse holes 695 pos int64 // Current position in sparse file 696} 697 698func (sr *sparseFileReader) Read(b []byte) (n int, err error) { 699 finished := int64(len(b)) >= sr.logicalRemaining() 700 if finished { 701 b = b[:sr.logicalRemaining()] 702 } 703 704 b0 := b 705 endPos := sr.pos + int64(len(b)) 706 for endPos > sr.pos && err == nil { 707 var nf int // Bytes read in fragment 708 holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset() 709 if sr.pos < holeStart { // In a data fragment 710 bf := b[:min(int64(len(b)), holeStart-sr.pos)] 711 nf, err = tryReadFull(sr.fr, bf) 712 } else { // In a hole fragment 713 bf := b[:min(int64(len(b)), holeEnd-sr.pos)] 714 nf, err = tryReadFull(zeroReader{}, bf) 715 } 716 b = b[nf:] 717 sr.pos += int64(nf) 718 if sr.pos >= holeEnd && len(sr.sp) > 1 { 719 sr.sp = sr.sp[1:] // Ensure last fragment always remains 720 } 721 } 722 723 n = len(b0) - len(b) 724 switch { 725 case err == io.EOF: 726 return n, errMissData // Less data in dense file than sparse file 727 case err != nil: 728 return n, err 729 case sr.logicalRemaining() == 0 && sr.physicalRemaining() > 0: 730 return n, errUnrefData // More data in dense file than sparse file 731 case finished: 732 return n, io.EOF 733 default: 734 return n, nil 735 } 736} 737 738func (sr *sparseFileReader) WriteTo(w io.Writer) (n int64, err error) { 739 ws, ok := w.(io.WriteSeeker) 740 if ok { 741 if _, err := ws.Seek(0, io.SeekCurrent); err != nil { 742 ok = false // Not all io.Seeker can really seek 743 } 744 } 745 if !ok { 746 return io.Copy(w, struct{ io.Reader }{sr}) 747 } 748 749 var writeLastByte bool 750 pos0 := sr.pos 751 for sr.logicalRemaining() > 0 && !writeLastByte && err == nil { 752 var nf int64 // Size of fragment 753 holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset() 754 if sr.pos < holeStart { // In a data fragment 755 nf = holeStart - sr.pos 756 nf, err = io.CopyN(ws, sr.fr, nf) 757 } else { // In a hole fragment 758 nf = holeEnd - sr.pos 759 if sr.physicalRemaining() == 0 { 760 writeLastByte = true 761 nf-- 762 } 763 _, err = ws.Seek(nf, io.SeekCurrent) 764 } 765 sr.pos += nf 766 if sr.pos >= holeEnd && len(sr.sp) > 1 { 767 sr.sp = sr.sp[1:] // Ensure last fragment always remains 768 } 769 } 770 771 // If the last fragment is a hole, then seek to 1-byte before EOF, and 772 // write a single byte to ensure the file is the right size. 773 if writeLastByte && err == nil { 774 _, err = ws.Write([]byte{0}) 775 sr.pos++ 776 } 777 778 n = sr.pos - pos0 779 switch { 780 case err == io.EOF: 781 return n, errMissData // Less data in dense file than sparse file 782 case err != nil: 783 return n, err 784 case sr.logicalRemaining() == 0 && sr.physicalRemaining() > 0: 785 return n, errUnrefData // More data in dense file than sparse file 786 default: 787 return n, nil 788 } 789} 790 791func (sr sparseFileReader) logicalRemaining() int64 { 792 return sr.sp[len(sr.sp)-1].endOffset() - sr.pos 793} 794func (sr sparseFileReader) physicalRemaining() int64 { 795 return sr.fr.physicalRemaining() 796} 797 798type zeroReader struct{} 799 800func (zeroReader) Read(b []byte) (int, error) { 801 for i := range b { 802 b[i] = 0 803 } 804 return len(b), nil 805} 806 807// mustReadFull is like io.ReadFull except it returns 808// io.ErrUnexpectedEOF when io.EOF is hit before len(b) bytes are read. 809func mustReadFull(r io.Reader, b []byte) (int, error) { 810 n, err := tryReadFull(r, b) 811 if err == io.EOF { 812 err = io.ErrUnexpectedEOF 813 } 814 return n, err 815} 816 817// tryReadFull is like io.ReadFull except it returns 818// io.EOF when it is hit before len(b) bytes are read. 819func tryReadFull(r io.Reader, b []byte) (n int, err error) { 820 for len(b) > n && err == nil { 821 var nn int 822 nn, err = r.Read(b[n:]) 823 n += nn 824 } 825 if len(b) == n && err == io.EOF { 826 err = nil 827 } 828 return n, err 829} 830 831// discard skips n bytes in r, reporting an error if unable to do so. 832func discard(r io.Reader, n int64) error { 833 // If possible, Seek to the last byte before the end of the data section. 834 // Do this because Seek is often lazy about reporting errors; this will mask 835 // the fact that the stream may be truncated. We can rely on the 836 // io.CopyN done shortly afterwards to trigger any IO errors. 837 var seekSkipped int64 // Number of bytes skipped via Seek 838 if sr, ok := r.(io.Seeker); ok && n > 1 { 839 // Not all io.Seeker can actually Seek. For example, os.Stdin implements 840 // io.Seeker, but calling Seek always returns an error and performs 841 // no action. Thus, we try an innocent seek to the current position 842 // to see if Seek is really supported. 843 pos1, err := sr.Seek(0, io.SeekCurrent) 844 if pos1 >= 0 && err == nil { 845 // Seek seems supported, so perform the real Seek. 846 pos2, err := sr.Seek(n-1, io.SeekCurrent) 847 if pos2 < 0 || err != nil { 848 return err 849 } 850 seekSkipped = pos2 - pos1 851 } 852 } 853 854 copySkipped, err := io.CopyN(io.Discard, r, n-seekSkipped) 855 if err == io.EOF && seekSkipped+copySkipped < n { 856 err = io.ErrUnexpectedEOF 857 } 858 return err 859} 860