1package filesystem 2 3import ( 4 "bytes" 5 "io" 6 "os" 7 "time" 8 9 "github.com/go-git/go-git/v5/plumbing" 10 "github.com/go-git/go-git/v5/plumbing/cache" 11 "github.com/go-git/go-git/v5/plumbing/format/idxfile" 12 "github.com/go-git/go-git/v5/plumbing/format/objfile" 13 "github.com/go-git/go-git/v5/plumbing/format/packfile" 14 "github.com/go-git/go-git/v5/plumbing/storer" 15 "github.com/go-git/go-git/v5/storage/filesystem/dotgit" 16 "github.com/go-git/go-git/v5/utils/ioutil" 17 18 "github.com/go-git/go-billy/v5" 19) 20 21type ObjectStorage struct { 22 options Options 23 24 // objectCache is an object cache uses to cache delta's bases and also recently 25 // loaded loose objects 26 objectCache cache.Object 27 28 dir *dotgit.DotGit 29 index map[plumbing.Hash]idxfile.Index 30 31 packList []plumbing.Hash 32 packListIdx int 33 packfiles map[plumbing.Hash]*packfile.Packfile 34} 35 36// NewObjectStorage creates a new ObjectStorage with the given .git directory and cache. 37func NewObjectStorage(dir *dotgit.DotGit, objectCache cache.Object) *ObjectStorage { 38 return NewObjectStorageWithOptions(dir, objectCache, Options{}) 39} 40 41// NewObjectStorageWithOptions creates a new ObjectStorage with the given .git directory, cache and extra options 42func NewObjectStorageWithOptions(dir *dotgit.DotGit, objectCache cache.Object, ops Options) *ObjectStorage { 43 return &ObjectStorage{ 44 options: ops, 45 objectCache: objectCache, 46 dir: dir, 47 } 48} 49 50func (s *ObjectStorage) requireIndex() error { 51 if s.index != nil { 52 return nil 53 } 54 55 s.index = make(map[plumbing.Hash]idxfile.Index) 56 packs, err := s.dir.ObjectPacks() 57 if err != nil { 58 return err 59 } 60 61 for _, h := range packs { 62 if err := s.loadIdxFile(h); err != nil { 63 return err 64 } 65 } 66 67 return nil 68} 69 70// Reindex indexes again all packfiles. Useful if git changed packfiles externally 71func (s *ObjectStorage) Reindex() { 72 s.index = nil 73} 74 75func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) { 76 f, err := s.dir.ObjectPackIdx(h) 77 if err != nil { 78 return err 79 } 80 81 defer ioutil.CheckClose(f, &err) 82 83 idxf := idxfile.NewMemoryIndex() 84 d := idxfile.NewDecoder(f) 85 if err = d.Decode(idxf); err != nil { 86 return err 87 } 88 89 s.index[h] = idxf 90 return err 91} 92 93func (s *ObjectStorage) NewEncodedObject() plumbing.EncodedObject { 94 return &plumbing.MemoryObject{} 95} 96 97func (s *ObjectStorage) PackfileWriter() (io.WriteCloser, error) { 98 if err := s.requireIndex(); err != nil { 99 return nil, err 100 } 101 102 w, err := s.dir.NewObjectPack() 103 if err != nil { 104 return nil, err 105 } 106 107 w.Notify = func(h plumbing.Hash, writer *idxfile.Writer) { 108 index, err := writer.Index() 109 if err == nil { 110 s.index[h] = index 111 } 112 } 113 114 return w, nil 115} 116 117// SetEncodedObject adds a new object to the storage. 118func (s *ObjectStorage) SetEncodedObject(o plumbing.EncodedObject) (h plumbing.Hash, err error) { 119 if o.Type() == plumbing.OFSDeltaObject || o.Type() == plumbing.REFDeltaObject { 120 return plumbing.ZeroHash, plumbing.ErrInvalidType 121 } 122 123 ow, err := s.dir.NewObject() 124 if err != nil { 125 return plumbing.ZeroHash, err 126 } 127 128 defer ioutil.CheckClose(ow, &err) 129 130 or, err := o.Reader() 131 if err != nil { 132 return plumbing.ZeroHash, err 133 } 134 135 defer ioutil.CheckClose(or, &err) 136 137 if err = ow.WriteHeader(o.Type(), o.Size()); err != nil { 138 return plumbing.ZeroHash, err 139 } 140 141 if _, err = io.Copy(ow, or); err != nil { 142 return plumbing.ZeroHash, err 143 } 144 145 return o.Hash(), err 146} 147 148// HasEncodedObject returns nil if the object exists, without actually 149// reading the object data from storage. 150func (s *ObjectStorage) HasEncodedObject(h plumbing.Hash) (err error) { 151 // Check unpacked objects 152 f, err := s.dir.Object(h) 153 if err != nil { 154 if !os.IsNotExist(err) { 155 return err 156 } 157 // Fall through to check packed objects. 158 } else { 159 defer ioutil.CheckClose(f, &err) 160 return nil 161 } 162 163 // Check packed objects. 164 if err := s.requireIndex(); err != nil { 165 return err 166 } 167 _, _, offset := s.findObjectInPackfile(h) 168 if offset == -1 { 169 return plumbing.ErrObjectNotFound 170 } 171 return nil 172} 173 174func (s *ObjectStorage) encodedObjectSizeFromUnpacked(h plumbing.Hash) ( 175 size int64, err error) { 176 f, err := s.dir.Object(h) 177 if err != nil { 178 if os.IsNotExist(err) { 179 return 0, plumbing.ErrObjectNotFound 180 } 181 182 return 0, err 183 } 184 185 r, err := objfile.NewReader(f) 186 if err != nil { 187 return 0, err 188 } 189 defer ioutil.CheckClose(r, &err) 190 191 _, size, err = r.Header() 192 return size, err 193} 194 195func (s *ObjectStorage) packfile(idx idxfile.Index, pack plumbing.Hash) (*packfile.Packfile, error) { 196 if p := s.packfileFromCache(pack); p != nil { 197 return p, nil 198 } 199 200 f, err := s.dir.ObjectPack(pack) 201 if err != nil { 202 return nil, err 203 } 204 205 var p *packfile.Packfile 206 if s.objectCache != nil { 207 p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache) 208 } else { 209 p = packfile.NewPackfile(idx, s.dir.Fs(), f) 210 } 211 212 return p, s.storePackfileInCache(pack, p) 213} 214 215func (s *ObjectStorage) packfileFromCache(hash plumbing.Hash) *packfile.Packfile { 216 if s.packfiles == nil { 217 if s.options.KeepDescriptors { 218 s.packfiles = make(map[plumbing.Hash]*packfile.Packfile) 219 } else if s.options.MaxOpenDescriptors > 0 { 220 s.packList = make([]plumbing.Hash, s.options.MaxOpenDescriptors) 221 s.packfiles = make(map[plumbing.Hash]*packfile.Packfile, s.options.MaxOpenDescriptors) 222 } 223 } 224 225 return s.packfiles[hash] 226} 227 228func (s *ObjectStorage) storePackfileInCache(hash plumbing.Hash, p *packfile.Packfile) error { 229 if s.options.KeepDescriptors { 230 s.packfiles[hash] = p 231 return nil 232 } 233 234 if s.options.MaxOpenDescriptors <= 0 { 235 return nil 236 } 237 238 // start over as the limit of packList is hit 239 if s.packListIdx >= len(s.packList) { 240 s.packListIdx = 0 241 } 242 243 // close the existing packfile if open 244 if next := s.packList[s.packListIdx]; !next.IsZero() { 245 open := s.packfiles[next] 246 delete(s.packfiles, next) 247 if open != nil { 248 if err := open.Close(); err != nil { 249 return err 250 } 251 } 252 } 253 254 // cache newly open packfile 255 s.packList[s.packListIdx] = hash 256 s.packfiles[hash] = p 257 s.packListIdx++ 258 259 return nil 260} 261 262func (s *ObjectStorage) encodedObjectSizeFromPackfile(h plumbing.Hash) ( 263 size int64, err error) { 264 if err := s.requireIndex(); err != nil { 265 return 0, err 266 } 267 268 pack, _, offset := s.findObjectInPackfile(h) 269 if offset == -1 { 270 return 0, plumbing.ErrObjectNotFound 271 } 272 273 idx := s.index[pack] 274 hash, err := idx.FindHash(offset) 275 if err == nil { 276 obj, ok := s.objectCache.Get(hash) 277 if ok { 278 return obj.Size(), nil 279 } 280 } else if err != nil && err != plumbing.ErrObjectNotFound { 281 return 0, err 282 } 283 284 p, err := s.packfile(idx, pack) 285 if err != nil { 286 return 0, err 287 } 288 289 if !s.options.KeepDescriptors && s.options.MaxOpenDescriptors == 0 { 290 defer ioutil.CheckClose(p, &err) 291 } 292 293 return p.GetSizeByOffset(offset) 294} 295 296// EncodedObjectSize returns the plaintext size of the given object, 297// without actually reading the full object data from storage. 298func (s *ObjectStorage) EncodedObjectSize(h plumbing.Hash) ( 299 size int64, err error) { 300 size, err = s.encodedObjectSizeFromUnpacked(h) 301 if err != nil && err != plumbing.ErrObjectNotFound { 302 return 0, err 303 } else if err == nil { 304 return size, nil 305 } 306 307 return s.encodedObjectSizeFromPackfile(h) 308} 309 310// EncodedObject returns the object with the given hash, by searching for it in 311// the packfile and the git object directories. 312func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) { 313 var obj plumbing.EncodedObject 314 var err error 315 316 if s.index != nil { 317 obj, err = s.getFromPackfile(h, false) 318 if err == plumbing.ErrObjectNotFound { 319 obj, err = s.getFromUnpacked(h) 320 } 321 } else { 322 obj, err = s.getFromUnpacked(h) 323 if err == plumbing.ErrObjectNotFound { 324 obj, err = s.getFromPackfile(h, false) 325 } 326 } 327 328 // If the error is still object not found, check if it's a shared object 329 // repository. 330 if err == plumbing.ErrObjectNotFound { 331 dotgits, e := s.dir.Alternates() 332 if e == nil { 333 // Create a new object storage with the DotGit(s) and check for the 334 // required hash object. Skip when not found. 335 for _, dg := range dotgits { 336 o := NewObjectStorage(dg, s.objectCache) 337 enobj, enerr := o.EncodedObject(t, h) 338 if enerr != nil { 339 continue 340 } 341 return enobj, nil 342 } 343 } 344 } 345 346 if err != nil { 347 return nil, err 348 } 349 350 if plumbing.AnyObject != t && obj.Type() != t { 351 return nil, plumbing.ErrObjectNotFound 352 } 353 354 return obj, nil 355} 356 357// DeltaObject returns the object with the given hash, by searching for 358// it in the packfile and the git object directories. 359func (s *ObjectStorage) DeltaObject(t plumbing.ObjectType, 360 h plumbing.Hash) (plumbing.EncodedObject, error) { 361 obj, err := s.getFromUnpacked(h) 362 if err == plumbing.ErrObjectNotFound { 363 obj, err = s.getFromPackfile(h, true) 364 } 365 366 if err != nil { 367 return nil, err 368 } 369 370 if plumbing.AnyObject != t && obj.Type() != t { 371 return nil, plumbing.ErrObjectNotFound 372 } 373 374 return obj, nil 375} 376 377func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedObject, err error) { 378 f, err := s.dir.Object(h) 379 if err != nil { 380 if os.IsNotExist(err) { 381 return nil, plumbing.ErrObjectNotFound 382 } 383 384 return nil, err 385 } 386 defer ioutil.CheckClose(f, &err) 387 388 if cacheObj, found := s.objectCache.Get(h); found { 389 return cacheObj, nil 390 } 391 392 obj = s.NewEncodedObject() 393 r, err := objfile.NewReader(f) 394 if err != nil { 395 return nil, err 396 } 397 398 defer ioutil.CheckClose(r, &err) 399 400 t, size, err := r.Header() 401 if err != nil { 402 return nil, err 403 } 404 405 obj.SetType(t) 406 obj.SetSize(size) 407 w, err := obj.Writer() 408 if err != nil { 409 return nil, err 410 } 411 412 defer ioutil.CheckClose(w, &err) 413 414 s.objectCache.Put(obj) 415 416 _, err = io.Copy(w, r) 417 return obj, err 418} 419 420// Get returns the object with the given hash, by searching for it in 421// the packfile. 422func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( 423 plumbing.EncodedObject, error) { 424 425 if err := s.requireIndex(); err != nil { 426 return nil, err 427 } 428 429 pack, hash, offset := s.findObjectInPackfile(h) 430 if offset == -1 { 431 return nil, plumbing.ErrObjectNotFound 432 } 433 434 idx := s.index[pack] 435 p, err := s.packfile(idx, pack) 436 if err != nil { 437 return nil, err 438 } 439 440 if !s.options.KeepDescriptors && s.options.MaxOpenDescriptors == 0 { 441 defer ioutil.CheckClose(p, &err) 442 } 443 444 if canBeDelta { 445 return s.decodeDeltaObjectAt(p, offset, hash) 446 } 447 448 return s.decodeObjectAt(p, offset) 449} 450 451func (s *ObjectStorage) decodeObjectAt( 452 p *packfile.Packfile, 453 offset int64, 454) (plumbing.EncodedObject, error) { 455 hash, err := p.FindHash(offset) 456 if err == nil { 457 obj, ok := s.objectCache.Get(hash) 458 if ok { 459 return obj, nil 460 } 461 } 462 463 if err != nil && err != plumbing.ErrObjectNotFound { 464 return nil, err 465 } 466 467 return p.GetByOffset(offset) 468} 469 470func (s *ObjectStorage) decodeDeltaObjectAt( 471 p *packfile.Packfile, 472 offset int64, 473 hash plumbing.Hash, 474) (plumbing.EncodedObject, error) { 475 scan := p.Scanner() 476 header, err := scan.SeekObjectHeader(offset) 477 if err != nil { 478 return nil, err 479 } 480 481 var ( 482 base plumbing.Hash 483 ) 484 485 switch header.Type { 486 case plumbing.REFDeltaObject: 487 base = header.Reference 488 case plumbing.OFSDeltaObject: 489 base, err = p.FindHash(header.OffsetReference) 490 if err != nil { 491 return nil, err 492 } 493 default: 494 return s.decodeObjectAt(p, offset) 495 } 496 497 obj := &plumbing.MemoryObject{} 498 obj.SetType(header.Type) 499 w, err := obj.Writer() 500 if err != nil { 501 return nil, err 502 } 503 504 if _, _, err := scan.NextObject(w); err != nil { 505 return nil, err 506 } 507 508 return newDeltaObject(obj, hash, base, header.Length), nil 509} 510 511func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, plumbing.Hash, int64) { 512 for packfile, index := range s.index { 513 offset, err := index.FindOffset(h) 514 if err == nil { 515 return packfile, h, offset 516 } 517 } 518 519 return plumbing.ZeroHash, plumbing.ZeroHash, -1 520} 521 522func (s *ObjectStorage) HashesWithPrefix(prefix []byte) ([]plumbing.Hash, error) { 523 hashes, err := s.dir.ObjectsWithPrefix(prefix) 524 if err != nil { 525 return nil, err 526 } 527 528 // TODO: This could be faster with some idxfile changes, 529 // or diving into the packfile. 530 for _, index := range s.index { 531 ei, err := index.Entries() 532 if err != nil { 533 return nil, err 534 } 535 for { 536 e, err := ei.Next() 537 if err == io.EOF { 538 break 539 } else if err != nil { 540 return nil, err 541 } 542 if bytes.HasPrefix(e.Hash[:], prefix) { 543 hashes = append(hashes, e.Hash) 544 } 545 } 546 ei.Close() 547 } 548 549 return hashes, nil 550} 551 552// IterEncodedObjects returns an iterator for all the objects in the packfile 553// with the given type. 554func (s *ObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (storer.EncodedObjectIter, error) { 555 objects, err := s.dir.Objects() 556 if err != nil { 557 return nil, err 558 } 559 560 seen := make(map[plumbing.Hash]struct{}) 561 var iters []storer.EncodedObjectIter 562 if len(objects) != 0 { 563 iters = append(iters, &objectsIter{s: s, t: t, h: objects}) 564 seen = hashListAsMap(objects) 565 } 566 567 packi, err := s.buildPackfileIters(t, seen) 568 if err != nil { 569 return nil, err 570 } 571 572 iters = append(iters, packi) 573 return storer.NewMultiEncodedObjectIter(iters), nil 574} 575 576func (s *ObjectStorage) buildPackfileIters( 577 t plumbing.ObjectType, 578 seen map[plumbing.Hash]struct{}, 579) (storer.EncodedObjectIter, error) { 580 if err := s.requireIndex(); err != nil { 581 return nil, err 582 } 583 584 packs, err := s.dir.ObjectPacks() 585 if err != nil { 586 return nil, err 587 } 588 return &lazyPackfilesIter{ 589 hashes: packs, 590 open: func(h plumbing.Hash) (storer.EncodedObjectIter, error) { 591 pack, err := s.dir.ObjectPack(h) 592 if err != nil { 593 return nil, err 594 } 595 return newPackfileIter( 596 s.dir.Fs(), pack, t, seen, s.index[h], 597 s.objectCache, s.options.KeepDescriptors, 598 ) 599 }, 600 }, nil 601} 602 603// Close closes all opened files. 604func (s *ObjectStorage) Close() error { 605 var firstError error 606 if s.options.KeepDescriptors || s.options.MaxOpenDescriptors > 0 { 607 for _, packfile := range s.packfiles { 608 err := packfile.Close() 609 if firstError == nil && err != nil { 610 firstError = err 611 } 612 } 613 } 614 615 s.packfiles = nil 616 s.dir.Close() 617 618 return firstError 619} 620 621type lazyPackfilesIter struct { 622 hashes []plumbing.Hash 623 open func(h plumbing.Hash) (storer.EncodedObjectIter, error) 624 cur storer.EncodedObjectIter 625} 626 627func (it *lazyPackfilesIter) Next() (plumbing.EncodedObject, error) { 628 for { 629 if it.cur == nil { 630 if len(it.hashes) == 0 { 631 return nil, io.EOF 632 } 633 h := it.hashes[0] 634 it.hashes = it.hashes[1:] 635 636 sub, err := it.open(h) 637 if err == io.EOF { 638 continue 639 } else if err != nil { 640 return nil, err 641 } 642 it.cur = sub 643 } 644 ob, err := it.cur.Next() 645 if err == io.EOF { 646 it.cur.Close() 647 it.cur = nil 648 continue 649 } else if err != nil { 650 return nil, err 651 } 652 return ob, nil 653 } 654} 655 656func (it *lazyPackfilesIter) ForEach(cb func(plumbing.EncodedObject) error) error { 657 return storer.ForEachIterator(it, cb) 658} 659 660func (it *lazyPackfilesIter) Close() { 661 if it.cur != nil { 662 it.cur.Close() 663 it.cur = nil 664 } 665 it.hashes = nil 666} 667 668type packfileIter struct { 669 pack billy.File 670 iter storer.EncodedObjectIter 671 seen map[plumbing.Hash]struct{} 672 673 // tells whether the pack file should be left open after iteration or not 674 keepPack bool 675} 676 677// NewPackfileIter returns a new EncodedObjectIter for the provided packfile 678// and object type. Packfile and index file will be closed after they're 679// used. If keepPack is true the packfile won't be closed after the iteration 680// finished. 681func NewPackfileIter( 682 fs billy.Filesystem, 683 f billy.File, 684 idxFile billy.File, 685 t plumbing.ObjectType, 686 keepPack bool, 687) (storer.EncodedObjectIter, error) { 688 idx := idxfile.NewMemoryIndex() 689 if err := idxfile.NewDecoder(idxFile).Decode(idx); err != nil { 690 return nil, err 691 } 692 693 if err := idxFile.Close(); err != nil { 694 return nil, err 695 } 696 697 seen := make(map[plumbing.Hash]struct{}) 698 return newPackfileIter(fs, f, t, seen, idx, nil, keepPack) 699} 700 701func newPackfileIter( 702 fs billy.Filesystem, 703 f billy.File, 704 t plumbing.ObjectType, 705 seen map[plumbing.Hash]struct{}, 706 index idxfile.Index, 707 cache cache.Object, 708 keepPack bool, 709) (storer.EncodedObjectIter, error) { 710 var p *packfile.Packfile 711 if cache != nil { 712 p = packfile.NewPackfileWithCache(index, fs, f, cache) 713 } else { 714 p = packfile.NewPackfile(index, fs, f) 715 } 716 717 iter, err := p.GetByType(t) 718 if err != nil { 719 return nil, err 720 } 721 722 return &packfileIter{ 723 pack: f, 724 iter: iter, 725 seen: seen, 726 keepPack: keepPack, 727 }, nil 728} 729 730func (iter *packfileIter) Next() (plumbing.EncodedObject, error) { 731 for { 732 obj, err := iter.iter.Next() 733 if err != nil { 734 return nil, err 735 } 736 737 if _, ok := iter.seen[obj.Hash()]; ok { 738 continue 739 } 740 741 return obj, nil 742 } 743} 744 745func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { 746 for { 747 o, err := iter.Next() 748 if err != nil { 749 if err == io.EOF { 750 iter.Close() 751 return nil 752 } 753 return err 754 } 755 756 if err := cb(o); err != nil { 757 return err 758 } 759 } 760} 761 762func (iter *packfileIter) Close() { 763 iter.iter.Close() 764 if !iter.keepPack { 765 _ = iter.pack.Close() 766 } 767} 768 769type objectsIter struct { 770 s *ObjectStorage 771 t plumbing.ObjectType 772 h []plumbing.Hash 773} 774 775func (iter *objectsIter) Next() (plumbing.EncodedObject, error) { 776 if len(iter.h) == 0 { 777 return nil, io.EOF 778 } 779 780 obj, err := iter.s.getFromUnpacked(iter.h[0]) 781 iter.h = iter.h[1:] 782 783 if err != nil { 784 return nil, err 785 } 786 787 if iter.t != plumbing.AnyObject && iter.t != obj.Type() { 788 return iter.Next() 789 } 790 791 return obj, err 792} 793 794func (iter *objectsIter) ForEach(cb func(plumbing.EncodedObject) error) error { 795 for { 796 o, err := iter.Next() 797 if err != nil { 798 if err == io.EOF { 799 return nil 800 } 801 return err 802 } 803 804 if err := cb(o); err != nil { 805 return err 806 } 807 } 808} 809 810func (iter *objectsIter) Close() { 811 iter.h = []plumbing.Hash{} 812} 813 814func hashListAsMap(l []plumbing.Hash) map[plumbing.Hash]struct{} { 815 m := make(map[plumbing.Hash]struct{}, len(l)) 816 for _, h := range l { 817 m[h] = struct{}{} 818 } 819 return m 820} 821 822func (s *ObjectStorage) ForEachObjectHash(fun func(plumbing.Hash) error) error { 823 err := s.dir.ForEachObjectHash(fun) 824 if err == storer.ErrStop { 825 return nil 826 } 827 return err 828} 829 830func (s *ObjectStorage) LooseObjectTime(hash plumbing.Hash) (time.Time, error) { 831 fi, err := s.dir.ObjectStat(hash) 832 if err != nil { 833 return time.Time{}, err 834 } 835 return fi.ModTime(), nil 836} 837 838func (s *ObjectStorage) DeleteLooseObject(hash plumbing.Hash) error { 839 return s.dir.ObjectDelete(hash) 840} 841 842func (s *ObjectStorage) ObjectPacks() ([]plumbing.Hash, error) { 843 return s.dir.ObjectPacks() 844} 845 846func (s *ObjectStorage) DeleteOldObjectPackAndIndex(h plumbing.Hash, t time.Time) error { 847 return s.dir.DeleteOldObjectPackAndIndex(h, t) 848} 849