1package storage 2 3import ( 4 "context" 5 "fmt" 6 7 "github.com/docker/distribution" 8 "github.com/docker/distribution/reference" 9 "github.com/docker/distribution/registry/storage/driver" 10 "github.com/opencontainers/go-digest" 11) 12 13func emit(format string, a ...interface{}) { 14 fmt.Printf(format+"\n", a...) 15} 16 17// GCOpts contains options for garbage collector 18type GCOpts struct { 19 DryRun bool 20 RemoveUntagged bool 21} 22 23// ManifestDel contains manifest structure which will be deleted 24type ManifestDel struct { 25 Name string 26 Digest digest.Digest 27 Tags []string 28} 29 30// MarkAndSweep performs a mark and sweep of registry data 31func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, opts GCOpts) error { 32 repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator) 33 if !ok { 34 return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator") 35 } 36 37 // mark 38 markSet := make(map[digest.Digest]struct{}) 39 manifestArr := make([]ManifestDel, 0) 40 err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error { 41 emit(repoName) 42 43 var err error 44 named, err := reference.WithName(repoName) 45 if err != nil { 46 return fmt.Errorf("failed to parse repo name %s: %v", repoName, err) 47 } 48 repository, err := registry.Repository(ctx, named) 49 if err != nil { 50 return fmt.Errorf("failed to construct repository: %v", err) 51 } 52 53 manifestService, err := repository.Manifests(ctx) 54 if err != nil { 55 return fmt.Errorf("failed to construct manifest service: %v", err) 56 } 57 58 manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator) 59 if !ok { 60 return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator") 61 } 62 63 err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error { 64 if opts.RemoveUntagged { 65 // fetch all tags where this manifest is the latest one 66 tags, err := repository.Tags(ctx).Lookup(ctx, distribution.Descriptor{Digest: dgst}) 67 if err != nil { 68 return fmt.Errorf("failed to retrieve tags for digest %v: %v", dgst, err) 69 } 70 if len(tags) == 0 { 71 emit("manifest eligible for deletion: %s", dgst) 72 // fetch all tags from repository 73 // all of these tags could contain manifest in history 74 // which means that we need check (and delete) those references when deleting manifest 75 allTags, err := repository.Tags(ctx).All(ctx) 76 if err != nil { 77 return fmt.Errorf("failed to retrieve tags %v", err) 78 } 79 manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) 80 return nil 81 } 82 } 83 // Mark the manifest's blob 84 emit("%s: marking manifest %s ", repoName, dgst) 85 markSet[dgst] = struct{}{} 86 87 manifest, err := manifestService.Get(ctx, dgst) 88 if err != nil { 89 return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) 90 } 91 92 descriptors := manifest.References() 93 for _, descriptor := range descriptors { 94 markSet[descriptor.Digest] = struct{}{} 95 emit("%s: marking blob %s", repoName, descriptor.Digest) 96 } 97 98 return nil 99 }) 100 101 // In certain situations such as unfinished uploads, deleting all 102 // tags in S3 or removing the _manifests folder manually, this 103 // error may be of type PathNotFound. 104 // 105 // In these cases we can continue marking other manifests safely. 106 if _, ok := err.(driver.PathNotFoundError); ok { 107 return nil 108 } 109 110 return err 111 }) 112 113 if err != nil { 114 return fmt.Errorf("failed to mark: %v", err) 115 } 116 117 // sweep 118 vacuum := NewVacuum(ctx, storageDriver) 119 if !opts.DryRun { 120 for _, obj := range manifestArr { 121 err = vacuum.RemoveManifest(obj.Name, obj.Digest, obj.Tags) 122 if err != nil { 123 return fmt.Errorf("failed to delete manifest %s: %v", obj.Digest, err) 124 } 125 } 126 } 127 blobService := registry.Blobs() 128 deleteSet := make(map[digest.Digest]struct{}) 129 err = blobService.Enumerate(ctx, func(dgst digest.Digest) error { 130 // check if digest is in markSet. If not, delete it! 131 if _, ok := markSet[dgst]; !ok { 132 deleteSet[dgst] = struct{}{} 133 } 134 return nil 135 }) 136 if err != nil { 137 return fmt.Errorf("error enumerating blobs: %v", err) 138 } 139 emit("\n%d blobs marked, %d blobs and %d manifests eligible for deletion", len(markSet), len(deleteSet), len(manifestArr)) 140 for dgst := range deleteSet { 141 emit("blob eligible for deletion: %s", dgst) 142 if opts.DryRun { 143 continue 144 } 145 err = vacuum.RemoveBlob(string(dgst)) 146 if err != nil { 147 return fmt.Errorf("failed to delete blob %s: %v", dgst, err) 148 } 149 } 150 151 return err 152} 153