1package storage
2
3import (
4	"context"
5	"fmt"
6
7	"github.com/docker/distribution"
8	"github.com/docker/distribution/reference"
9	"github.com/docker/distribution/registry/storage/driver"
10	"github.com/opencontainers/go-digest"
11)
12
13func emit(format string, a ...interface{}) {
14	fmt.Printf(format+"\n", a...)
15}
16
17// GCOpts contains options for garbage collector
18type GCOpts struct {
19	DryRun         bool
20	RemoveUntagged bool
21}
22
23// ManifestDel contains manifest structure which will be deleted
24type ManifestDel struct {
25	Name   string
26	Digest digest.Digest
27	Tags   []string
28}
29
30// MarkAndSweep performs a mark and sweep of registry data
31func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, opts GCOpts) error {
32	repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
33	if !ok {
34		return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator")
35	}
36
37	// mark
38	markSet := make(map[digest.Digest]struct{})
39	manifestArr := make([]ManifestDel, 0)
40	err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
41		emit(repoName)
42
43		var err error
44		named, err := reference.WithName(repoName)
45		if err != nil {
46			return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
47		}
48		repository, err := registry.Repository(ctx, named)
49		if err != nil {
50			return fmt.Errorf("failed to construct repository: %v", err)
51		}
52
53		manifestService, err := repository.Manifests(ctx)
54		if err != nil {
55			return fmt.Errorf("failed to construct manifest service: %v", err)
56		}
57
58		manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
59		if !ok {
60			return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator")
61		}
62
63		err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
64			if opts.RemoveUntagged {
65				// fetch all tags where this manifest is the latest one
66				tags, err := repository.Tags(ctx).Lookup(ctx, distribution.Descriptor{Digest: dgst})
67				if err != nil {
68					return fmt.Errorf("failed to retrieve tags for digest %v: %v", dgst, err)
69				}
70				if len(tags) == 0 {
71					emit("manifest eligible for deletion: %s", dgst)
72					// fetch all tags from repository
73					// all of these tags could contain manifest in history
74					// which means that we need check (and delete) those references when deleting manifest
75					allTags, err := repository.Tags(ctx).All(ctx)
76					if err != nil {
77						return fmt.Errorf("failed to retrieve tags %v", err)
78					}
79					manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags})
80					return nil
81				}
82			}
83			// Mark the manifest's blob
84			emit("%s: marking manifest %s ", repoName, dgst)
85			markSet[dgst] = struct{}{}
86
87			manifest, err := manifestService.Get(ctx, dgst)
88			if err != nil {
89				return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err)
90			}
91
92			descriptors := manifest.References()
93			for _, descriptor := range descriptors {
94				markSet[descriptor.Digest] = struct{}{}
95				emit("%s: marking blob %s", repoName, descriptor.Digest)
96			}
97
98			return nil
99		})
100
101		// In certain situations such as unfinished uploads, deleting all
102		// tags in S3 or removing the _manifests folder manually, this
103		// error may be of type PathNotFound.
104		//
105		// In these cases we can continue marking other manifests safely.
106		if _, ok := err.(driver.PathNotFoundError); ok {
107			return nil
108		}
109
110		return err
111	})
112
113	if err != nil {
114		return fmt.Errorf("failed to mark: %v", err)
115	}
116
117	// sweep
118	vacuum := NewVacuum(ctx, storageDriver)
119	if !opts.DryRun {
120		for _, obj := range manifestArr {
121			err = vacuum.RemoveManifest(obj.Name, obj.Digest, obj.Tags)
122			if err != nil {
123				return fmt.Errorf("failed to delete manifest %s: %v", obj.Digest, err)
124			}
125		}
126	}
127	blobService := registry.Blobs()
128	deleteSet := make(map[digest.Digest]struct{})
129	err = blobService.Enumerate(ctx, func(dgst digest.Digest) error {
130		// check if digest is in markSet. If not, delete it!
131		if _, ok := markSet[dgst]; !ok {
132			deleteSet[dgst] = struct{}{}
133		}
134		return nil
135	})
136	if err != nil {
137		return fmt.Errorf("error enumerating blobs: %v", err)
138	}
139	emit("\n%d blobs marked, %d blobs and %d manifests eligible for deletion", len(markSet), len(deleteSet), len(manifestArr))
140	for dgst := range deleteSet {
141		emit("blob eligible for deletion: %s", dgst)
142		if opts.DryRun {
143			continue
144		}
145		err = vacuum.RemoveBlob(string(dgst))
146		if err != nil {
147			return fmt.Errorf("failed to delete blob %s: %v", dgst, err)
148		}
149	}
150
151	return err
152}
153