1package storage 2 3import ( 4 "context" 5 "fmt" 6 7 "github.com/docker/distribution" 8 "github.com/docker/distribution/reference" 9 "github.com/docker/distribution/registry/storage/driver" 10 "github.com/opencontainers/go-digest" 11) 12 13func emit(format string, a ...interface{}) { 14 fmt.Printf(format+"\n", a...) 15} 16 17// GCOpts contains options for garbage collector 18type GCOpts struct { 19 DryRun bool 20 RemoveUntagged bool 21} 22 23// ManifestDel contains manifest structure which will be deleted 24type ManifestDel struct { 25 Name string 26 Digest digest.Digest 27 Tags []string 28} 29 30// MarkAndSweep performs a mark and sweep of registry data 31func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, opts GCOpts) error { 32 repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator) 33 if !ok { 34 return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator") 35 } 36 37 // mark 38 markSet := make(map[digest.Digest]struct{}) 39 manifestArr := make([]ManifestDel, 0) 40 err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error { 41 emit(repoName) 42 43 var err error 44 named, err := reference.WithName(repoName) 45 if err != nil { 46 return fmt.Errorf("failed to parse repo name %s: %v", repoName, err) 47 } 48 repository, err := registry.Repository(ctx, named) 49 if err != nil { 50 return fmt.Errorf("failed to construct repository: %v", err) 51 } 52 53 manifestService, err := repository.Manifests(ctx) 54 if err != nil { 55 return fmt.Errorf("failed to construct manifest service: %v", err) 56 } 57 58 manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator) 59 if !ok { 60 return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator") 61 } 62 63 err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error { 64 if opts.RemoveUntagged { 65 // fetch all tags where this manifest is the latest one 66 tags, err := repository.Tags(ctx).Lookup(ctx, distribution.Descriptor{Digest: dgst}) 67 if err != nil { 68 return fmt.Errorf("failed to retrieve tags for digest %v: %v", dgst, err) 69 } 70 if len(tags) == 0 { 71 emit("manifest eligible for deletion: %s", dgst) 72 // fetch all tags from repository 73 // all of these tags could contain manifest in history 74 // which means that we need check (and delete) those references when deleting manifest 75 allTags, err := repository.Tags(ctx).All(ctx) 76 if err != nil { 77 return fmt.Errorf("failed to retrieve tags %v", err) 78 } 79 manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) 80 return nil 81 } 82 } 83 // Mark the manifest's blob 84 emit("%s: marking manifest %s ", repoName, dgst) 85 markSet[dgst] = struct{}{} 86 87 manifest, err := manifestService.Get(ctx, dgst) 88 if err != nil { 89 return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) 90 } 91 92 descriptors := manifest.References() 93 for _, descriptor := range descriptors { 94 markSet[descriptor.Digest] = struct{}{} 95 emit("%s: marking blob %s", repoName, descriptor.Digest) 96 } 97 98 return nil 99 }) 100 101 if err != nil { 102 // In certain situations such as unfinished uploads, deleting all 103 // tags in S3 or removing the _manifests folder manually, this 104 // error may be of type PathNotFound. 105 // 106 // In these cases we can continue marking other manifests safely. 107 if _, ok := err.(driver.PathNotFoundError); ok { 108 return nil 109 } 110 } 111 112 return err 113 }) 114 115 if err != nil { 116 return fmt.Errorf("failed to mark: %v", err) 117 } 118 119 // sweep 120 vacuum := NewVacuum(ctx, storageDriver) 121 if !opts.DryRun { 122 for _, obj := range manifestArr { 123 err = vacuum.RemoveManifest(obj.Name, obj.Digest, obj.Tags) 124 if err != nil { 125 return fmt.Errorf("failed to delete manifest %s: %v", obj.Digest, err) 126 } 127 } 128 } 129 blobService := registry.Blobs() 130 deleteSet := make(map[digest.Digest]struct{}) 131 err = blobService.Enumerate(ctx, func(dgst digest.Digest) error { 132 // check if digest is in markSet. If not, delete it! 133 if _, ok := markSet[dgst]; !ok { 134 deleteSet[dgst] = struct{}{} 135 } 136 return nil 137 }) 138 if err != nil { 139 return fmt.Errorf("error enumerating blobs: %v", err) 140 } 141 emit("\n%d blobs marked, %d blobs and %d manifests eligible for deletion", len(markSet), len(deleteSet), len(manifestArr)) 142 for dgst := range deleteSet { 143 emit("blob eligible for deletion: %s", dgst) 144 if opts.DryRun { 145 continue 146 } 147 err = vacuum.RemoveBlob(string(dgst)) 148 if err != nil { 149 return fmt.Errorf("failed to delete blob %s: %v", dgst, err) 150 } 151 } 152 153 return err 154} 155