1/* 2 Copyright The containerd Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15*/ 16 17package schema1 18 19import ( 20 "bytes" 21 "context" 22 "encoding/base64" 23 "encoding/json" 24 "fmt" 25 "io" 26 "io/ioutil" 27 "strconv" 28 "strings" 29 "sync" 30 "time" 31 32 "golang.org/x/sync/errgroup" 33 34 "github.com/containerd/containerd/archive/compression" 35 "github.com/containerd/containerd/content" 36 "github.com/containerd/containerd/errdefs" 37 "github.com/containerd/containerd/images" 38 "github.com/containerd/containerd/log" 39 "github.com/containerd/containerd/remotes" 40 digest "github.com/opencontainers/go-digest" 41 specs "github.com/opencontainers/image-spec/specs-go" 42 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 43 "github.com/pkg/errors" 44) 45 46const ( 47 manifestSizeLimit = 8e6 // 8MB 48 labelDockerSchema1EmptyLayer = "containerd.io/docker.schema1.empty-layer" 49) 50 51type blobState struct { 52 diffID digest.Digest 53 empty bool 54} 55 56// Converter converts schema1 manifests to schema2 on fetch 57type Converter struct { 58 contentStore content.Store 59 fetcher remotes.Fetcher 60 61 pulledManifest *manifest 62 63 mu sync.Mutex 64 blobMap map[digest.Digest]blobState 65 layerBlobs map[digest.Digest]ocispec.Descriptor 66} 67 68// NewConverter returns a new converter 69func NewConverter(contentStore content.Store, fetcher remotes.Fetcher) *Converter { 70 return &Converter{ 71 contentStore: contentStore, 72 fetcher: fetcher, 73 blobMap: map[digest.Digest]blobState{}, 74 layerBlobs: map[digest.Digest]ocispec.Descriptor{}, 75 } 76} 77 78// Handle fetching descriptors for a docker media type 79func (c *Converter) Handle(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 80 switch desc.MediaType { 81 case images.MediaTypeDockerSchema1Manifest: 82 if err := c.fetchManifest(ctx, desc); err != nil { 83 return nil, err 84 } 85 86 m := c.pulledManifest 87 if len(m.FSLayers) != len(m.History) { 88 return nil, errors.New("invalid schema 1 manifest, history and layer mismatch") 89 } 90 descs := make([]ocispec.Descriptor, 0, len(c.pulledManifest.FSLayers)) 91 92 for i := range m.FSLayers { 93 if _, ok := c.blobMap[c.pulledManifest.FSLayers[i].BlobSum]; !ok { 94 empty, err := isEmptyLayer([]byte(m.History[i].V1Compatibility)) 95 if err != nil { 96 return nil, err 97 } 98 99 // Do no attempt to download a known empty blob 100 if !empty { 101 descs = append([]ocispec.Descriptor{ 102 { 103 MediaType: images.MediaTypeDockerSchema2LayerGzip, 104 Digest: c.pulledManifest.FSLayers[i].BlobSum, 105 Size: -1, 106 }, 107 }, descs...) 108 } 109 c.blobMap[c.pulledManifest.FSLayers[i].BlobSum] = blobState{ 110 empty: empty, 111 } 112 } 113 } 114 return descs, nil 115 case images.MediaTypeDockerSchema2LayerGzip: 116 if c.pulledManifest == nil { 117 return nil, errors.New("manifest required for schema 1 blob pull") 118 } 119 return nil, c.fetchBlob(ctx, desc) 120 default: 121 return nil, fmt.Errorf("%v not support for schema 1 manifests", desc.MediaType) 122 } 123} 124 125// ConvertOptions provides options on converting a docker schema1 manifest. 126type ConvertOptions struct { 127 // ManifestMediaType specifies the media type of the manifest OCI descriptor. 128 ManifestMediaType string 129 130 // ConfigMediaType specifies the media type of the manifest config OCI 131 // descriptor. 132 ConfigMediaType string 133} 134 135// ConvertOpt allows configuring a convert operation. 136type ConvertOpt func(context.Context, *ConvertOptions) error 137 138// UseDockerSchema2 is used to indicate that a schema1 manifest should be 139// converted into the media types for a docker schema2 manifest. 140func UseDockerSchema2() ConvertOpt { 141 return func(ctx context.Context, o *ConvertOptions) error { 142 o.ManifestMediaType = images.MediaTypeDockerSchema2Manifest 143 o.ConfigMediaType = images.MediaTypeDockerSchema2Config 144 return nil 145 } 146} 147 148// Convert a docker manifest to an OCI descriptor 149func (c *Converter) Convert(ctx context.Context, opts ...ConvertOpt) (ocispec.Descriptor, error) { 150 co := ConvertOptions{ 151 ManifestMediaType: ocispec.MediaTypeImageManifest, 152 ConfigMediaType: ocispec.MediaTypeImageConfig, 153 } 154 for _, opt := range opts { 155 if err := opt(ctx, &co); err != nil { 156 return ocispec.Descriptor{}, err 157 } 158 } 159 160 history, diffIDs, err := c.schema1ManifestHistory() 161 if err != nil { 162 return ocispec.Descriptor{}, errors.Wrap(err, "schema 1 conversion failed") 163 } 164 165 var img ocispec.Image 166 if err := json.Unmarshal([]byte(c.pulledManifest.History[0].V1Compatibility), &img); err != nil { 167 return ocispec.Descriptor{}, errors.Wrap(err, "failed to unmarshal image from schema 1 history") 168 } 169 170 img.History = history 171 img.RootFS = ocispec.RootFS{ 172 Type: "layers", 173 DiffIDs: diffIDs, 174 } 175 176 b, err := json.MarshalIndent(img, "", " ") 177 if err != nil { 178 return ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal image") 179 } 180 181 config := ocispec.Descriptor{ 182 MediaType: co.ConfigMediaType, 183 Digest: digest.Canonical.FromBytes(b), 184 Size: int64(len(b)), 185 } 186 187 layers := make([]ocispec.Descriptor, len(diffIDs)) 188 for i, diffID := range diffIDs { 189 layers[i] = c.layerBlobs[diffID] 190 } 191 192 manifest := ocispec.Manifest{ 193 Versioned: specs.Versioned{ 194 SchemaVersion: 2, 195 }, 196 Config: config, 197 Layers: layers, 198 } 199 200 mb, err := json.MarshalIndent(manifest, "", " ") 201 if err != nil { 202 return ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal image") 203 } 204 205 desc := ocispec.Descriptor{ 206 MediaType: co.ManifestMediaType, 207 Digest: digest.Canonical.FromBytes(mb), 208 Size: int64(len(mb)), 209 } 210 211 labels := map[string]string{} 212 labels["containerd.io/gc.ref.content.0"] = manifest.Config.Digest.String() 213 for i, ch := range manifest.Layers { 214 labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = ch.Digest.String() 215 } 216 217 ref := remotes.MakeRefKey(ctx, desc) 218 if err := content.WriteBlob(ctx, c.contentStore, ref, bytes.NewReader(mb), desc, content.WithLabels(labels)); err != nil { 219 return ocispec.Descriptor{}, errors.Wrap(err, "failed to write image manifest") 220 } 221 222 ref = remotes.MakeRefKey(ctx, config) 223 if err := content.WriteBlob(ctx, c.contentStore, ref, bytes.NewReader(b), config); err != nil { 224 return ocispec.Descriptor{}, errors.Wrap(err, "failed to write image config") 225 } 226 227 return desc, nil 228} 229 230// ReadStripSignature reads in a schema1 manifest and returns a byte array 231// with the "signatures" field stripped 232func ReadStripSignature(schema1Blob io.Reader) ([]byte, error) { 233 b, err := ioutil.ReadAll(io.LimitReader(schema1Blob, manifestSizeLimit)) // limit to 8MB 234 if err != nil { 235 return nil, err 236 } 237 238 return stripSignature(b) 239} 240 241func (c *Converter) fetchManifest(ctx context.Context, desc ocispec.Descriptor) error { 242 log.G(ctx).Debug("fetch schema 1") 243 244 rc, err := c.fetcher.Fetch(ctx, desc) 245 if err != nil { 246 return err 247 } 248 249 b, err := ReadStripSignature(rc) 250 rc.Close() 251 if err != nil { 252 return err 253 } 254 255 var m manifest 256 if err := json.Unmarshal(b, &m); err != nil { 257 return err 258 } 259 c.pulledManifest = &m 260 261 return nil 262} 263 264func (c *Converter) fetchBlob(ctx context.Context, desc ocispec.Descriptor) error { 265 log.G(ctx).Debug("fetch blob") 266 267 var ( 268 ref = remotes.MakeRefKey(ctx, desc) 269 calc = newBlobStateCalculator() 270 compressMethod = compression.Gzip 271 ) 272 273 // size may be unknown, set to zero for content ingest 274 ingestDesc := desc 275 if ingestDesc.Size == -1 { 276 ingestDesc.Size = 0 277 } 278 279 cw, err := content.OpenWriter(ctx, c.contentStore, content.WithRef(ref), content.WithDescriptor(ingestDesc)) 280 if err != nil { 281 if !errdefs.IsAlreadyExists(err) { 282 return err 283 } 284 285 reuse, err := c.reuseLabelBlobState(ctx, desc) 286 if err != nil { 287 return err 288 } 289 290 if reuse { 291 return nil 292 } 293 294 ra, err := c.contentStore.ReaderAt(ctx, desc) 295 if err != nil { 296 return err 297 } 298 defer ra.Close() 299 300 r, err := compression.DecompressStream(content.NewReader(ra)) 301 if err != nil { 302 return err 303 } 304 305 compressMethod = r.GetCompression() 306 _, err = io.Copy(calc, r) 307 r.Close() 308 if err != nil { 309 return err 310 } 311 } else { 312 defer cw.Close() 313 314 rc, err := c.fetcher.Fetch(ctx, desc) 315 if err != nil { 316 return err 317 } 318 defer rc.Close() 319 320 eg, _ := errgroup.WithContext(ctx) 321 pr, pw := io.Pipe() 322 323 eg.Go(func() error { 324 r, err := compression.DecompressStream(pr) 325 if err != nil { 326 return err 327 } 328 329 compressMethod = r.GetCompression() 330 _, err = io.Copy(calc, r) 331 r.Close() 332 pr.CloseWithError(err) 333 return err 334 }) 335 336 eg.Go(func() error { 337 defer pw.Close() 338 339 return content.Copy(ctx, cw, io.TeeReader(rc, pw), ingestDesc.Size, ingestDesc.Digest) 340 }) 341 342 if err := eg.Wait(); err != nil { 343 return err 344 } 345 } 346 347 if desc.Size == -1 { 348 info, err := c.contentStore.Info(ctx, desc.Digest) 349 if err != nil { 350 return errors.Wrap(err, "failed to get blob info") 351 } 352 desc.Size = info.Size 353 } 354 355 if compressMethod == compression.Uncompressed { 356 log.G(ctx).WithField("id", desc.Digest).Debugf("changed media type for uncompressed schema1 layer blob") 357 desc.MediaType = images.MediaTypeDockerSchema2Layer 358 } 359 360 state := calc.State() 361 362 cinfo := content.Info{ 363 Digest: desc.Digest, 364 Labels: map[string]string{ 365 "containerd.io/uncompressed": state.diffID.String(), 366 labelDockerSchema1EmptyLayer: strconv.FormatBool(state.empty), 367 }, 368 } 369 370 if _, err := c.contentStore.Update(ctx, cinfo, "labels.containerd.io/uncompressed", fmt.Sprintf("labels.%s", labelDockerSchema1EmptyLayer)); err != nil { 371 return errors.Wrap(err, "failed to update uncompressed label") 372 } 373 374 c.mu.Lock() 375 c.blobMap[desc.Digest] = state 376 c.layerBlobs[state.diffID] = desc 377 c.mu.Unlock() 378 379 return nil 380} 381 382func (c *Converter) reuseLabelBlobState(ctx context.Context, desc ocispec.Descriptor) (bool, error) { 383 cinfo, err := c.contentStore.Info(ctx, desc.Digest) 384 if err != nil { 385 return false, errors.Wrap(err, "failed to get blob info") 386 } 387 desc.Size = cinfo.Size 388 389 diffID, ok := cinfo.Labels["containerd.io/uncompressed"] 390 if !ok { 391 return false, nil 392 } 393 394 emptyVal, ok := cinfo.Labels[labelDockerSchema1EmptyLayer] 395 if !ok { 396 return false, nil 397 } 398 399 isEmpty, err := strconv.ParseBool(emptyVal) 400 if err != nil { 401 log.G(ctx).WithField("id", desc.Digest).Warnf("failed to parse bool from label %s: %v", labelDockerSchema1EmptyLayer, isEmpty) 402 return false, nil 403 } 404 405 bState := blobState{empty: isEmpty} 406 407 if bState.diffID, err = digest.Parse(diffID); err != nil { 408 log.G(ctx).WithField("id", desc.Digest).Warnf("failed to parse digest from label containerd.io/uncompressed: %v", diffID) 409 return false, nil 410 } 411 412 // NOTE: there is no need to read header to get compression method 413 // because there are only two kinds of methods. 414 if bState.diffID == desc.Digest { 415 desc.MediaType = images.MediaTypeDockerSchema2Layer 416 } else { 417 desc.MediaType = images.MediaTypeDockerSchema2LayerGzip 418 } 419 420 c.mu.Lock() 421 c.blobMap[desc.Digest] = bState 422 c.layerBlobs[bState.diffID] = desc 423 c.mu.Unlock() 424 return true, nil 425} 426 427func (c *Converter) schema1ManifestHistory() ([]ocispec.History, []digest.Digest, error) { 428 if c.pulledManifest == nil { 429 return nil, nil, errors.New("missing schema 1 manifest for conversion") 430 } 431 m := *c.pulledManifest 432 433 if len(m.History) == 0 { 434 return nil, nil, errors.New("no history") 435 } 436 437 history := make([]ocispec.History, len(m.History)) 438 diffIDs := []digest.Digest{} 439 for i := range m.History { 440 var h v1History 441 if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), &h); err != nil { 442 return nil, nil, errors.Wrap(err, "failed to unmarshal history") 443 } 444 445 blobSum := m.FSLayers[i].BlobSum 446 447 state := c.blobMap[blobSum] 448 449 history[len(history)-i-1] = ocispec.History{ 450 Author: h.Author, 451 Comment: h.Comment, 452 Created: &h.Created, 453 CreatedBy: strings.Join(h.ContainerConfig.Cmd, " "), 454 EmptyLayer: state.empty, 455 } 456 457 if !state.empty { 458 diffIDs = append([]digest.Digest{state.diffID}, diffIDs...) 459 460 } 461 } 462 463 return history, diffIDs, nil 464} 465 466type fsLayer struct { 467 BlobSum digest.Digest `json:"blobSum"` 468} 469 470type history struct { 471 V1Compatibility string `json:"v1Compatibility"` 472} 473 474type manifest struct { 475 FSLayers []fsLayer `json:"fsLayers"` 476 History []history `json:"history"` 477} 478 479type v1History struct { 480 Author string `json:"author,omitempty"` 481 Created time.Time `json:"created"` 482 Comment string `json:"comment,omitempty"` 483 ThrowAway *bool `json:"throwaway,omitempty"` 484 Size *int `json:"Size,omitempty"` // used before ThrowAway field 485 ContainerConfig struct { 486 Cmd []string `json:"Cmd,omitempty"` 487 } `json:"container_config,omitempty"` 488} 489 490// isEmptyLayer returns whether the v1 compatibility history describes an 491// empty layer. A return value of true indicates the layer is empty, 492// however false does not indicate non-empty. 493func isEmptyLayer(compatHistory []byte) (bool, error) { 494 var h v1History 495 if err := json.Unmarshal(compatHistory, &h); err != nil { 496 return false, err 497 } 498 499 if h.ThrowAway != nil { 500 return *h.ThrowAway, nil 501 } 502 if h.Size != nil { 503 return *h.Size == 0, nil 504 } 505 506 // If no `Size` or `throwaway` field is given, then 507 // it cannot be determined whether the layer is empty 508 // from the history, return false 509 return false, nil 510} 511 512type signature struct { 513 Signatures []jsParsedSignature `json:"signatures"` 514} 515 516type jsParsedSignature struct { 517 Protected string `json:"protected"` 518} 519 520type protectedBlock struct { 521 Length int `json:"formatLength"` 522 Tail string `json:"formatTail"` 523} 524 525// joseBase64UrlDecode decodes the given string using the standard base64 url 526// decoder but first adds the appropriate number of trailing '=' characters in 527// accordance with the jose specification. 528// http://tools.ietf.org/html/draft-ietf-jose-json-web-signature-31#section-2 529func joseBase64UrlDecode(s string) ([]byte, error) { 530 switch len(s) % 4 { 531 case 0: 532 case 2: 533 s += "==" 534 case 3: 535 s += "=" 536 default: 537 return nil, errors.New("illegal base64url string") 538 } 539 return base64.URLEncoding.DecodeString(s) 540} 541 542func stripSignature(b []byte) ([]byte, error) { 543 var sig signature 544 if err := json.Unmarshal(b, &sig); err != nil { 545 return nil, err 546 } 547 if len(sig.Signatures) == 0 { 548 return nil, errors.New("no signatures") 549 } 550 pb, err := joseBase64UrlDecode(sig.Signatures[0].Protected) 551 if err != nil { 552 return nil, errors.Wrapf(err, "could not decode %s", sig.Signatures[0].Protected) 553 } 554 555 var protected protectedBlock 556 if err := json.Unmarshal(pb, &protected); err != nil { 557 return nil, err 558 } 559 560 if protected.Length > len(b) { 561 return nil, errors.New("invalid protected length block") 562 } 563 564 tail, err := joseBase64UrlDecode(protected.Tail) 565 if err != nil { 566 return nil, errors.Wrap(err, "invalid tail base 64 value") 567 } 568 569 return append(b[:protected.Length], tail...), nil 570} 571 572type blobStateCalculator struct { 573 empty bool 574 digester digest.Digester 575} 576 577func newBlobStateCalculator() *blobStateCalculator { 578 return &blobStateCalculator{ 579 empty: true, 580 digester: digest.Canonical.Digester(), 581 } 582} 583 584func (c *blobStateCalculator) Write(p []byte) (int, error) { 585 if c.empty { 586 for _, b := range p { 587 if b != 0x00 { 588 c.empty = false 589 break 590 } 591 } 592 } 593 return c.digester.Hash().Write(p) 594} 595 596func (c *blobStateCalculator) State() blobState { 597 return blobState{ 598 empty: c.empty, 599 diffID: c.digester.Digest(), 600 } 601} 602