1/* 2 Copyright The containerd Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15*/ 16 17package schema1 18 19import ( 20 "bytes" 21 "context" 22 "encoding/base64" 23 "encoding/json" 24 "fmt" 25 "io" 26 "io/ioutil" 27 "strconv" 28 "strings" 29 "sync" 30 "time" 31 32 "golang.org/x/sync/errgroup" 33 34 "github.com/containerd/containerd/archive/compression" 35 "github.com/containerd/containerd/content" 36 "github.com/containerd/containerd/errdefs" 37 "github.com/containerd/containerd/images" 38 "github.com/containerd/containerd/log" 39 "github.com/containerd/containerd/remotes" 40 digest "github.com/opencontainers/go-digest" 41 specs "github.com/opencontainers/image-spec/specs-go" 42 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 43 "github.com/pkg/errors" 44) 45 46const ( 47 manifestSizeLimit = 8e6 // 8MB 48 labelDockerSchema1EmptyLayer = "containerd.io/docker.schema1.empty-layer" 49) 50 51type blobState struct { 52 diffID digest.Digest 53 empty bool 54} 55 56// Converter converts schema1 manifests to schema2 on fetch 57type Converter struct { 58 contentStore content.Store 59 fetcher remotes.Fetcher 60 61 pulledManifest *manifest 62 63 mu sync.Mutex 64 blobMap map[digest.Digest]blobState 65 layerBlobs map[digest.Digest]ocispec.Descriptor 66} 67 68// NewConverter returns a new converter 69func NewConverter(contentStore content.Store, fetcher remotes.Fetcher) *Converter { 70 return &Converter{ 71 contentStore: contentStore, 72 fetcher: fetcher, 73 blobMap: map[digest.Digest]blobState{}, 74 layerBlobs: map[digest.Digest]ocispec.Descriptor{}, 75 } 76} 77 78// Handle fetching descriptors for a docker media type 79func (c *Converter) Handle(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 80 switch desc.MediaType { 81 case images.MediaTypeDockerSchema1Manifest: 82 if err := c.fetchManifest(ctx, desc); err != nil { 83 return nil, err 84 } 85 86 m := c.pulledManifest 87 if len(m.FSLayers) != len(m.History) { 88 return nil, errors.New("invalid schema 1 manifest, history and layer mismatch") 89 } 90 descs := make([]ocispec.Descriptor, 0, len(c.pulledManifest.FSLayers)) 91 92 for i := range m.FSLayers { 93 if _, ok := c.blobMap[c.pulledManifest.FSLayers[i].BlobSum]; !ok { 94 empty, err := isEmptyLayer([]byte(m.History[i].V1Compatibility)) 95 if err != nil { 96 return nil, err 97 } 98 99 // Do no attempt to download a known empty blob 100 if !empty { 101 descs = append([]ocispec.Descriptor{ 102 { 103 MediaType: images.MediaTypeDockerSchema2LayerGzip, 104 Digest: c.pulledManifest.FSLayers[i].BlobSum, 105 Size: -1, 106 }, 107 }, descs...) 108 } 109 c.blobMap[c.pulledManifest.FSLayers[i].BlobSum] = blobState{ 110 empty: empty, 111 } 112 } 113 } 114 return descs, nil 115 case images.MediaTypeDockerSchema2LayerGzip: 116 if c.pulledManifest == nil { 117 return nil, errors.New("manifest required for schema 1 blob pull") 118 } 119 return nil, c.fetchBlob(ctx, desc) 120 default: 121 return nil, fmt.Errorf("%v not support for schema 1 manifests", desc.MediaType) 122 } 123} 124 125// ConvertOptions provides options on converting a docker schema1 manifest. 126type ConvertOptions struct { 127 // ManifestMediaType specifies the media type of the manifest OCI descriptor. 128 ManifestMediaType string 129 130 // ConfigMediaType specifies the media type of the manifest config OCI 131 // descriptor. 132 ConfigMediaType string 133} 134 135// ConvertOpt allows configuring a convert operation. 136type ConvertOpt func(context.Context, *ConvertOptions) error 137 138// UseDockerSchema2 is used to indicate that a schema1 manifest should be 139// converted into the media types for a docker schema2 manifest. 140func UseDockerSchema2() ConvertOpt { 141 return func(ctx context.Context, o *ConvertOptions) error { 142 o.ManifestMediaType = images.MediaTypeDockerSchema2Manifest 143 o.ConfigMediaType = images.MediaTypeDockerSchema2Config 144 return nil 145 } 146} 147 148// Convert a docker manifest to an OCI descriptor 149func (c *Converter) Convert(ctx context.Context, opts ...ConvertOpt) (ocispec.Descriptor, error) { 150 co := ConvertOptions{ 151 ManifestMediaType: ocispec.MediaTypeImageManifest, 152 ConfigMediaType: ocispec.MediaTypeImageConfig, 153 } 154 for _, opt := range opts { 155 if err := opt(ctx, &co); err != nil { 156 return ocispec.Descriptor{}, err 157 } 158 } 159 160 history, diffIDs, err := c.schema1ManifestHistory() 161 if err != nil { 162 return ocispec.Descriptor{}, errors.Wrap(err, "schema 1 conversion failed") 163 } 164 165 var img ocispec.Image 166 if err := json.Unmarshal([]byte(c.pulledManifest.History[0].V1Compatibility), &img); err != nil { 167 return ocispec.Descriptor{}, errors.Wrap(err, "failed to unmarshal image from schema 1 history") 168 } 169 170 img.History = history 171 img.RootFS = ocispec.RootFS{ 172 Type: "layers", 173 DiffIDs: diffIDs, 174 } 175 176 b, err := json.MarshalIndent(img, "", " ") 177 if err != nil { 178 return ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal image") 179 } 180 181 config := ocispec.Descriptor{ 182 MediaType: co.ConfigMediaType, 183 Digest: digest.Canonical.FromBytes(b), 184 Size: int64(len(b)), 185 } 186 187 layers := make([]ocispec.Descriptor, len(diffIDs)) 188 for i, diffID := range diffIDs { 189 layers[i] = c.layerBlobs[diffID] 190 } 191 192 manifest := ocispec.Manifest{ 193 Versioned: specs.Versioned{ 194 SchemaVersion: 2, 195 }, 196 Config: config, 197 Layers: layers, 198 } 199 200 mb, err := json.MarshalIndent(manifest, "", " ") 201 if err != nil { 202 return ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal image") 203 } 204 205 desc := ocispec.Descriptor{ 206 MediaType: co.ManifestMediaType, 207 Digest: digest.Canonical.FromBytes(mb), 208 Size: int64(len(mb)), 209 } 210 211 labels := map[string]string{} 212 labels["containerd.io/gc.ref.content.0"] = manifest.Config.Digest.String() 213 for i, ch := range manifest.Layers { 214 labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = ch.Digest.String() 215 } 216 217 ref := remotes.MakeRefKey(ctx, desc) 218 if err := content.WriteBlob(ctx, c.contentStore, ref, bytes.NewReader(mb), desc, content.WithLabels(labels)); err != nil { 219 return ocispec.Descriptor{}, errors.Wrap(err, "failed to write config") 220 } 221 222 ref = remotes.MakeRefKey(ctx, config) 223 if err := content.WriteBlob(ctx, c.contentStore, ref, bytes.NewReader(b), config); err != nil { 224 return ocispec.Descriptor{}, errors.Wrap(err, "failed to write config") 225 } 226 227 return desc, nil 228} 229 230func (c *Converter) fetchManifest(ctx context.Context, desc ocispec.Descriptor) error { 231 log.G(ctx).Debug("fetch schema 1") 232 233 rc, err := c.fetcher.Fetch(ctx, desc) 234 if err != nil { 235 return err 236 } 237 238 b, err := ioutil.ReadAll(io.LimitReader(rc, manifestSizeLimit)) // limit to 8MB 239 rc.Close() 240 if err != nil { 241 return err 242 } 243 244 b, err = stripSignature(b) 245 if err != nil { 246 return err 247 } 248 249 var m manifest 250 if err := json.Unmarshal(b, &m); err != nil { 251 return err 252 } 253 c.pulledManifest = &m 254 255 return nil 256} 257 258func (c *Converter) fetchBlob(ctx context.Context, desc ocispec.Descriptor) error { 259 log.G(ctx).Debug("fetch blob") 260 261 var ( 262 ref = remotes.MakeRefKey(ctx, desc) 263 calc = newBlobStateCalculator() 264 compressMethod = compression.Gzip 265 ) 266 267 // size may be unknown, set to zero for content ingest 268 ingestDesc := desc 269 if ingestDesc.Size == -1 { 270 ingestDesc.Size = 0 271 } 272 273 cw, err := content.OpenWriter(ctx, c.contentStore, content.WithRef(ref), content.WithDescriptor(ingestDesc)) 274 if err != nil { 275 if !errdefs.IsAlreadyExists(err) { 276 return err 277 } 278 279 reuse, err := c.reuseLabelBlobState(ctx, desc) 280 if err != nil { 281 return err 282 } 283 284 if reuse { 285 return nil 286 } 287 288 ra, err := c.contentStore.ReaderAt(ctx, desc) 289 if err != nil { 290 return err 291 } 292 defer ra.Close() 293 294 r, err := compression.DecompressStream(content.NewReader(ra)) 295 if err != nil { 296 return err 297 } 298 299 compressMethod = r.GetCompression() 300 _, err = io.Copy(calc, r) 301 r.Close() 302 if err != nil { 303 return err 304 } 305 } else { 306 defer cw.Close() 307 308 rc, err := c.fetcher.Fetch(ctx, desc) 309 if err != nil { 310 return err 311 } 312 defer rc.Close() 313 314 eg, _ := errgroup.WithContext(ctx) 315 pr, pw := io.Pipe() 316 317 eg.Go(func() error { 318 r, err := compression.DecompressStream(pr) 319 if err != nil { 320 return err 321 } 322 323 compressMethod = r.GetCompression() 324 _, err = io.Copy(calc, r) 325 r.Close() 326 pr.CloseWithError(err) 327 return err 328 }) 329 330 eg.Go(func() error { 331 defer pw.Close() 332 333 return content.Copy(ctx, cw, io.TeeReader(rc, pw), ingestDesc.Size, ingestDesc.Digest) 334 }) 335 336 if err := eg.Wait(); err != nil { 337 return err 338 } 339 } 340 341 if desc.Size == -1 { 342 info, err := c.contentStore.Info(ctx, desc.Digest) 343 if err != nil { 344 return errors.Wrap(err, "failed to get blob info") 345 } 346 desc.Size = info.Size 347 } 348 349 if compressMethod == compression.Uncompressed { 350 log.G(ctx).WithField("id", desc.Digest).Debugf("changed media type for uncompressed schema1 layer blob") 351 desc.MediaType = images.MediaTypeDockerSchema2Layer 352 } 353 354 state := calc.State() 355 356 cinfo := content.Info{ 357 Digest: desc.Digest, 358 Labels: map[string]string{ 359 "containerd.io/uncompressed": state.diffID.String(), 360 labelDockerSchema1EmptyLayer: strconv.FormatBool(state.empty), 361 }, 362 } 363 364 if _, err := c.contentStore.Update(ctx, cinfo, "labels.containerd.io/uncompressed", fmt.Sprintf("labels.%s", labelDockerSchema1EmptyLayer)); err != nil { 365 return errors.Wrap(err, "failed to update uncompressed label") 366 } 367 368 c.mu.Lock() 369 c.blobMap[desc.Digest] = state 370 c.layerBlobs[state.diffID] = desc 371 c.mu.Unlock() 372 373 return nil 374} 375 376func (c *Converter) reuseLabelBlobState(ctx context.Context, desc ocispec.Descriptor) (bool, error) { 377 cinfo, err := c.contentStore.Info(ctx, desc.Digest) 378 if err != nil { 379 return false, errors.Wrap(err, "failed to get blob info") 380 } 381 desc.Size = cinfo.Size 382 383 diffID, ok := cinfo.Labels["containerd.io/uncompressed"] 384 if !ok { 385 return false, nil 386 } 387 388 emptyVal, ok := cinfo.Labels[labelDockerSchema1EmptyLayer] 389 if !ok { 390 return false, nil 391 } 392 393 isEmpty, err := strconv.ParseBool(emptyVal) 394 if err != nil { 395 log.G(ctx).WithField("id", desc.Digest).Warnf("failed to parse bool from label %s: %v", labelDockerSchema1EmptyLayer, isEmpty) 396 return false, nil 397 } 398 399 bState := blobState{empty: isEmpty} 400 401 if bState.diffID, err = digest.Parse(diffID); err != nil { 402 log.G(ctx).WithField("id", desc.Digest).Warnf("failed to parse digest from label containerd.io/uncompressed: %v", diffID) 403 return false, nil 404 } 405 406 // NOTE: there is no need to read header to get compression method 407 // because there are only two kinds of methods. 408 if bState.diffID == desc.Digest { 409 desc.MediaType = images.MediaTypeDockerSchema2Layer 410 } else { 411 desc.MediaType = images.MediaTypeDockerSchema2LayerGzip 412 } 413 414 c.mu.Lock() 415 c.blobMap[desc.Digest] = bState 416 c.layerBlobs[bState.diffID] = desc 417 c.mu.Unlock() 418 return true, nil 419} 420 421func (c *Converter) schema1ManifestHistory() ([]ocispec.History, []digest.Digest, error) { 422 if c.pulledManifest == nil { 423 return nil, nil, errors.New("missing schema 1 manifest for conversion") 424 } 425 m := *c.pulledManifest 426 427 if len(m.History) == 0 { 428 return nil, nil, errors.New("no history") 429 } 430 431 history := make([]ocispec.History, len(m.History)) 432 diffIDs := []digest.Digest{} 433 for i := range m.History { 434 var h v1History 435 if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), &h); err != nil { 436 return nil, nil, errors.Wrap(err, "failed to unmarshal history") 437 } 438 439 blobSum := m.FSLayers[i].BlobSum 440 441 state := c.blobMap[blobSum] 442 443 history[len(history)-i-1] = ocispec.History{ 444 Author: h.Author, 445 Comment: h.Comment, 446 Created: &h.Created, 447 CreatedBy: strings.Join(h.ContainerConfig.Cmd, " "), 448 EmptyLayer: state.empty, 449 } 450 451 if !state.empty { 452 diffIDs = append([]digest.Digest{state.diffID}, diffIDs...) 453 454 } 455 } 456 457 return history, diffIDs, nil 458} 459 460type fsLayer struct { 461 BlobSum digest.Digest `json:"blobSum"` 462} 463 464type history struct { 465 V1Compatibility string `json:"v1Compatibility"` 466} 467 468type manifest struct { 469 FSLayers []fsLayer `json:"fsLayers"` 470 History []history `json:"history"` 471} 472 473type v1History struct { 474 Author string `json:"author,omitempty"` 475 Created time.Time `json:"created"` 476 Comment string `json:"comment,omitempty"` 477 ThrowAway *bool `json:"throwaway,omitempty"` 478 Size *int `json:"Size,omitempty"` // used before ThrowAway field 479 ContainerConfig struct { 480 Cmd []string `json:"Cmd,omitempty"` 481 } `json:"container_config,omitempty"` 482} 483 484// isEmptyLayer returns whether the v1 compatibility history describes an 485// empty layer. A return value of true indicates the layer is empty, 486// however false does not indicate non-empty. 487func isEmptyLayer(compatHistory []byte) (bool, error) { 488 var h v1History 489 if err := json.Unmarshal(compatHistory, &h); err != nil { 490 return false, err 491 } 492 493 if h.ThrowAway != nil { 494 return *h.ThrowAway, nil 495 } 496 if h.Size != nil { 497 return *h.Size == 0, nil 498 } 499 500 // If no `Size` or `throwaway` field is given, then 501 // it cannot be determined whether the layer is empty 502 // from the history, return false 503 return false, nil 504} 505 506type signature struct { 507 Signatures []jsParsedSignature `json:"signatures"` 508} 509 510type jsParsedSignature struct { 511 Protected string `json:"protected"` 512} 513 514type protectedBlock struct { 515 Length int `json:"formatLength"` 516 Tail string `json:"formatTail"` 517} 518 519// joseBase64UrlDecode decodes the given string using the standard base64 url 520// decoder but first adds the appropriate number of trailing '=' characters in 521// accordance with the jose specification. 522// http://tools.ietf.org/html/draft-ietf-jose-json-web-signature-31#section-2 523func joseBase64UrlDecode(s string) ([]byte, error) { 524 switch len(s) % 4 { 525 case 0: 526 case 2: 527 s += "==" 528 case 3: 529 s += "=" 530 default: 531 return nil, errors.New("illegal base64url string") 532 } 533 return base64.URLEncoding.DecodeString(s) 534} 535 536func stripSignature(b []byte) ([]byte, error) { 537 var sig signature 538 if err := json.Unmarshal(b, &sig); err != nil { 539 return nil, err 540 } 541 if len(sig.Signatures) == 0 { 542 return nil, errors.New("no signatures") 543 } 544 pb, err := joseBase64UrlDecode(sig.Signatures[0].Protected) 545 if err != nil { 546 return nil, errors.Wrapf(err, "could not decode %s", sig.Signatures[0].Protected) 547 } 548 549 var protected protectedBlock 550 if err := json.Unmarshal(pb, &protected); err != nil { 551 return nil, err 552 } 553 554 if protected.Length > len(b) { 555 return nil, errors.New("invalid protected length block") 556 } 557 558 tail, err := joseBase64UrlDecode(protected.Tail) 559 if err != nil { 560 return nil, errors.Wrap(err, "invalid tail base 64 value") 561 } 562 563 return append(b[:protected.Length], tail...), nil 564} 565 566type blobStateCalculator struct { 567 empty bool 568 digester digest.Digester 569} 570 571func newBlobStateCalculator() *blobStateCalculator { 572 return &blobStateCalculator{ 573 empty: true, 574 digester: digest.Canonical.Digester(), 575 } 576} 577 578func (c *blobStateCalculator) Write(p []byte) (int, error) { 579 if c.empty { 580 for _, b := range p { 581 if b != 0x00 { 582 c.empty = false 583 break 584 } 585 } 586 } 587 return c.digester.Hash().Write(p) 588} 589 590func (c *blobStateCalculator) State() blobState { 591 return blobState{ 592 empty: c.empty, 593 diffID: c.digester.Digest(), 594 } 595} 596