1/*
2   Copyright The containerd Authors.
3
4   Licensed under the Apache License, Version 2.0 (the "License");
5   you may not use this file except in compliance with the License.
6   You may obtain a copy of the License at
7
8       http://www.apache.org/licenses/LICENSE-2.0
9
10   Unless required by applicable law or agreed to in writing, software
11   distributed under the License is distributed on an "AS IS" BASIS,
12   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   See the License for the specific language governing permissions and
14   limitations under the License.
15*/
16
17// Package archive provides a Docker and OCI compatible importer
18package archive
19
20import (
21	"archive/tar"
22	"bytes"
23	"context"
24	"encoding/json"
25	"fmt"
26	"io"
27	"io/ioutil"
28	"path"
29
30	"github.com/containerd/containerd/archive/compression"
31	"github.com/containerd/containerd/content"
32	"github.com/containerd/containerd/errdefs"
33	"github.com/containerd/containerd/images"
34	"github.com/containerd/containerd/log"
35	digest "github.com/opencontainers/go-digest"
36	specs "github.com/opencontainers/image-spec/specs-go"
37	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
38	"github.com/pkg/errors"
39)
40
41type importOpts struct {
42	compress bool
43}
44
45// ImportOpt is an option for importing an OCI index
46type ImportOpt func(*importOpts) error
47
48// WithImportCompression compresses uncompressed layers on import.
49// This is used for import formats which do not include the manifest.
50func WithImportCompression() ImportOpt {
51	return func(io *importOpts) error {
52		io.compress = true
53		return nil
54	}
55}
56
57// ImportIndex imports an index from a tar archive image bundle
58// - implements Docker v1.1, v1.2 and OCI v1.
59// - prefers OCI v1 when provided
60// - creates OCI index for Docker formats
61// - normalizes Docker references and adds as OCI ref name
62//      e.g. alpine:latest -> docker.io/library/alpine:latest
63// - existing OCI reference names are untouched
64func ImportIndex(ctx context.Context, store content.Store, reader io.Reader, opts ...ImportOpt) (ocispec.Descriptor, error) {
65	var (
66		tr = tar.NewReader(reader)
67
68		ociLayout ocispec.ImageLayout
69		mfsts     []struct {
70			Config   string
71			RepoTags []string
72			Layers   []string
73		}
74		symlinks = make(map[string]string)
75		blobs    = make(map[string]ocispec.Descriptor)
76		iopts    importOpts
77	)
78
79	for _, o := range opts {
80		if err := o(&iopts); err != nil {
81			return ocispec.Descriptor{}, err
82		}
83	}
84
85	for {
86		hdr, err := tr.Next()
87		if err == io.EOF {
88			break
89		}
90		if err != nil {
91			return ocispec.Descriptor{}, err
92		}
93		if hdr.Typeflag == tar.TypeSymlink {
94			symlinks[hdr.Name] = path.Join(path.Dir(hdr.Name), hdr.Linkname)
95		}
96
97		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
98			if hdr.Typeflag != tar.TypeDir {
99				log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored")
100			}
101			continue
102		}
103
104		hdrName := path.Clean(hdr.Name)
105		if hdrName == ocispec.ImageLayoutFile {
106			if err = onUntarJSON(tr, &ociLayout); err != nil {
107				return ocispec.Descriptor{}, errors.Wrapf(err, "untar oci layout %q", hdr.Name)
108			}
109		} else if hdrName == "manifest.json" {
110			if err = onUntarJSON(tr, &mfsts); err != nil {
111				return ocispec.Descriptor{}, errors.Wrapf(err, "untar manifest %q", hdr.Name)
112			}
113		} else {
114			dgst, err := onUntarBlob(ctx, tr, store, hdr.Size, "tar-"+hdrName)
115			if err != nil {
116				return ocispec.Descriptor{}, errors.Wrapf(err, "failed to ingest %q", hdr.Name)
117			}
118
119			blobs[hdrName] = ocispec.Descriptor{
120				Digest: dgst,
121				Size:   hdr.Size,
122			}
123		}
124	}
125
126	// If OCI layout was given, interpret the tar as an OCI layout.
127	// When not provided, the layout of the tar will be interpreted
128	// as Docker v1.1 or v1.2.
129	if ociLayout.Version != "" {
130		if ociLayout.Version != ocispec.ImageLayoutVersion {
131			return ocispec.Descriptor{}, errors.Errorf("unsupported OCI version %s", ociLayout.Version)
132		}
133
134		idx, ok := blobs["index.json"]
135		if !ok {
136			return ocispec.Descriptor{}, errors.Errorf("missing index.json in OCI layout %s", ocispec.ImageLayoutVersion)
137		}
138
139		idx.MediaType = ocispec.MediaTypeImageIndex
140		return idx, nil
141	}
142
143	if mfsts == nil {
144		return ocispec.Descriptor{}, errors.Errorf("unrecognized image format")
145	}
146
147	for name, linkname := range symlinks {
148		desc, ok := blobs[linkname]
149		if !ok {
150			return ocispec.Descriptor{}, errors.Errorf("no target for symlink layer from %q to %q", name, linkname)
151		}
152		blobs[name] = desc
153	}
154
155	idx := ocispec.Index{
156		Versioned: specs.Versioned{
157			SchemaVersion: 2,
158		},
159	}
160	for _, mfst := range mfsts {
161		config, ok := blobs[mfst.Config]
162		if !ok {
163			return ocispec.Descriptor{}, errors.Errorf("image config %q not found", mfst.Config)
164		}
165		config.MediaType = images.MediaTypeDockerSchema2Config
166
167		layers, err := resolveLayers(ctx, store, mfst.Layers, blobs, iopts.compress)
168		if err != nil {
169			return ocispec.Descriptor{}, errors.Wrap(err, "failed to resolve layers")
170		}
171
172		manifest := struct {
173			SchemaVersion int                  `json:"schemaVersion"`
174			MediaType     string               `json:"mediaType"`
175			Config        ocispec.Descriptor   `json:"config"`
176			Layers        []ocispec.Descriptor `json:"layers"`
177		}{
178			SchemaVersion: 2,
179			MediaType:     images.MediaTypeDockerSchema2Manifest,
180			Config:        config,
181			Layers:        layers,
182		}
183
184		desc, err := writeManifest(ctx, store, manifest, manifest.MediaType)
185		if err != nil {
186			return ocispec.Descriptor{}, errors.Wrap(err, "write docker manifest")
187		}
188
189		platforms, err := images.Platforms(ctx, store, desc)
190		if err != nil {
191			return ocispec.Descriptor{}, errors.Wrap(err, "unable to resolve platform")
192		}
193		if len(platforms) > 0 {
194			// Only one platform can be resolved from non-index manifest,
195			// The platform can only come from the config included above,
196			// if the config has no platform it can be safely omitted.
197			desc.Platform = &platforms[0]
198		}
199
200		if len(mfst.RepoTags) == 0 {
201			idx.Manifests = append(idx.Manifests, desc)
202		} else {
203			// Add descriptor per tag
204			for _, ref := range mfst.RepoTags {
205				mfstdesc := desc
206
207				normalized, err := normalizeReference(ref)
208				if err != nil {
209					return ocispec.Descriptor{}, err
210				}
211
212				mfstdesc.Annotations = map[string]string{
213					images.AnnotationImageName: normalized,
214					ocispec.AnnotationRefName:  ociReferenceName(normalized),
215				}
216
217				idx.Manifests = append(idx.Manifests, mfstdesc)
218			}
219		}
220	}
221
222	return writeManifest(ctx, store, idx, ocispec.MediaTypeImageIndex)
223}
224
225func onUntarJSON(r io.Reader, j interface{}) error {
226	b, err := ioutil.ReadAll(r)
227	if err != nil {
228		return err
229	}
230	return json.Unmarshal(b, j)
231}
232
233func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size int64, ref string) (digest.Digest, error) {
234	dgstr := digest.Canonical.Digester()
235
236	if err := content.WriteBlob(ctx, store, ref, io.TeeReader(r, dgstr.Hash()), ocispec.Descriptor{Size: size}); err != nil {
237		return "", err
238	}
239
240	return dgstr.Digest(), nil
241}
242
243func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor, compress bool) ([]ocispec.Descriptor, error) {
244	layers := make([]ocispec.Descriptor, len(layerFiles))
245	descs := map[digest.Digest]*ocispec.Descriptor{}
246	filters := []string{}
247	for i, f := range layerFiles {
248		desc, ok := blobs[f]
249		if !ok {
250			return nil, errors.Errorf("layer %q not found", f)
251		}
252		layers[i] = desc
253		descs[desc.Digest] = &layers[i]
254		filters = append(filters, "labels.\"containerd.io/uncompressed\"=="+desc.Digest.String())
255	}
256
257	err := store.Walk(ctx, func(info content.Info) error {
258		dgst, ok := info.Labels["containerd.io/uncompressed"]
259		if ok {
260			desc := descs[digest.Digest(dgst)]
261			if desc != nil {
262				desc.MediaType = images.MediaTypeDockerSchema2LayerGzip
263				desc.Digest = info.Digest
264				desc.Size = info.Size
265			}
266		}
267		return nil
268	}, filters...)
269	if err != nil {
270		return nil, errors.Wrap(err, "failure checking for compressed blobs")
271	}
272
273	for i, desc := range layers {
274		if desc.MediaType != "" {
275			continue
276		}
277		// Open blob, resolve media type
278		ra, err := store.ReaderAt(ctx, desc)
279		if err != nil {
280			return nil, errors.Wrapf(err, "failed to open %q (%s)", layerFiles[i], desc.Digest)
281		}
282		s, err := compression.DecompressStream(content.NewReader(ra))
283		if err != nil {
284			return nil, errors.Wrapf(err, "failed to detect compression for %q", layerFiles[i])
285		}
286		if s.GetCompression() == compression.Uncompressed {
287			if compress {
288				ref := fmt.Sprintf("compress-blob-%s-%s", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
289				labels := map[string]string{
290					"containerd.io/uncompressed": desc.Digest.String(),
291				}
292				layers[i], err = compressBlob(ctx, store, s, ref, content.WithLabels(labels))
293				if err != nil {
294					s.Close()
295					return nil, err
296				}
297				layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
298			} else {
299				layers[i].MediaType = images.MediaTypeDockerSchema2Layer
300			}
301		} else {
302			layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
303		}
304		s.Close()
305
306	}
307	return layers, nil
308}
309
310func compressBlob(ctx context.Context, cs content.Store, r io.Reader, ref string, opts ...content.Opt) (desc ocispec.Descriptor, err error) {
311	w, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
312	if err != nil {
313		return ocispec.Descriptor{}, errors.Wrap(err, "failed to open writer")
314	}
315
316	defer func() {
317		w.Close()
318		if err != nil {
319			cs.Abort(ctx, ref)
320		}
321	}()
322	if err := w.Truncate(0); err != nil {
323		return ocispec.Descriptor{}, errors.Wrap(err, "failed to truncate writer")
324	}
325
326	cw, err := compression.CompressStream(w, compression.Gzip)
327	if err != nil {
328		return ocispec.Descriptor{}, err
329	}
330
331	if _, err := io.Copy(cw, r); err != nil {
332		return ocispec.Descriptor{}, err
333	}
334	if err := cw.Close(); err != nil {
335		return ocispec.Descriptor{}, err
336	}
337
338	cst, err := w.Status()
339	if err != nil {
340		return ocispec.Descriptor{}, errors.Wrap(err, "failed to get writer status")
341	}
342
343	desc.Digest = w.Digest()
344	desc.Size = cst.Offset
345
346	if err := w.Commit(ctx, desc.Size, desc.Digest, opts...); err != nil {
347		if !errdefs.IsAlreadyExists(err) {
348			return ocispec.Descriptor{}, errors.Wrap(err, "failed to commit")
349		}
350	}
351
352	return desc, nil
353}
354
355func writeManifest(ctx context.Context, cs content.Ingester, manifest interface{}, mediaType string) (ocispec.Descriptor, error) {
356	manifestBytes, err := json.Marshal(manifest)
357	if err != nil {
358		return ocispec.Descriptor{}, err
359	}
360
361	desc := ocispec.Descriptor{
362		MediaType: mediaType,
363		Digest:    digest.FromBytes(manifestBytes),
364		Size:      int64(len(manifestBytes)),
365	}
366	if err := content.WriteBlob(ctx, cs, "manifest-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil {
367		return ocispec.Descriptor{}, err
368	}
369
370	return desc, nil
371}
372