1package v1 // import "github.com/docker/docker/migrate/v1" 2 3import ( 4 "encoding/json" 5 "errors" 6 "fmt" 7 "io/ioutil" 8 "os" 9 "path/filepath" 10 "runtime" 11 "strconv" 12 "sync" 13 "time" 14 15 "github.com/docker/distribution/reference" 16 "github.com/docker/docker/distribution/metadata" 17 "github.com/docker/docker/image" 18 imagev1 "github.com/docker/docker/image/v1" 19 "github.com/docker/docker/layer" 20 "github.com/docker/docker/pkg/ioutils" 21 refstore "github.com/docker/docker/reference" 22 "github.com/opencontainers/go-digest" 23 "github.com/sirupsen/logrus" 24) 25 26type graphIDRegistrar interface { 27 RegisterByGraphID(string, layer.ChainID, layer.DiffID, string, int64) (layer.Layer, error) 28 Release(layer.Layer) ([]layer.Metadata, error) 29} 30 31type graphIDMounter interface { 32 CreateRWLayerByGraphID(string, string, layer.ChainID) error 33} 34 35type checksumCalculator interface { 36 ChecksumForGraphID(id, parent, oldTarDataPath, newTarDataPath string) (diffID layer.DiffID, size int64, err error) 37} 38 39const ( 40 graphDirName = "graph" 41 tarDataFileName = "tar-data.json.gz" 42 migrationFileName = ".migration-v1-images.json" 43 migrationTagsFileName = ".migration-v1-tags" 44 migrationDiffIDFileName = ".migration-diffid" 45 migrationSizeFileName = ".migration-size" 46 migrationTarDataFileName = ".migration-tardata" 47 containersDirName = "containers" 48 configFileNameLegacy = "config.json" 49 configFileName = "config.v2.json" 50 repositoriesFilePrefixLegacy = "repositories-" 51) 52 53var ( 54 errUnsupported = errors.New("migration is not supported") 55) 56 57// Migrate takes an old graph directory and transforms the metadata into the 58// new format. 59func Migrate(root, driverName string, ls layer.Store, is image.Store, rs refstore.Store, ms metadata.Store) error { 60 graphDir := filepath.Join(root, graphDirName) 61 if _, err := os.Lstat(graphDir); os.IsNotExist(err) { 62 return nil 63 } 64 65 mappings, err := restoreMappings(root) 66 if err != nil { 67 return err 68 } 69 70 if cc, ok := ls.(checksumCalculator); ok { 71 CalculateLayerChecksums(root, cc, mappings) 72 } 73 74 if registrar, ok := ls.(graphIDRegistrar); !ok { 75 return errUnsupported 76 } else if err := migrateImages(root, registrar, is, ms, mappings); err != nil { 77 return err 78 } 79 80 err = saveMappings(root, mappings) 81 if err != nil { 82 return err 83 } 84 85 if mounter, ok := ls.(graphIDMounter); !ok { 86 return errUnsupported 87 } else if err := migrateContainers(root, mounter, is, mappings); err != nil { 88 return err 89 } 90 91 return migrateRefs(root, driverName, rs, mappings) 92} 93 94// CalculateLayerChecksums walks an old graph directory and calculates checksums 95// for each layer. These checksums are later used for migration. 96func CalculateLayerChecksums(root string, ls checksumCalculator, mappings map[string]image.ID) { 97 graphDir := filepath.Join(root, graphDirName) 98 // spawn some extra workers also for maximum performance because the process is bounded by both cpu and io 99 workers := runtime.NumCPU() * 3 100 workQueue := make(chan string, workers) 101 102 wg := sync.WaitGroup{} 103 104 for i := 0; i < workers; i++ { 105 wg.Add(1) 106 go func() { 107 for id := range workQueue { 108 start := time.Now() 109 if err := calculateLayerChecksum(graphDir, id, ls); err != nil { 110 logrus.Errorf("could not calculate checksum for %q, %q", id, err) 111 } 112 elapsed := time.Since(start) 113 logrus.Debugf("layer %s took %.2f seconds", id, elapsed.Seconds()) 114 } 115 wg.Done() 116 }() 117 } 118 119 dir, err := ioutil.ReadDir(graphDir) 120 if err != nil { 121 logrus.Errorf("could not read directory %q", graphDir) 122 return 123 } 124 for _, v := range dir { 125 v1ID := v.Name() 126 if err := imagev1.ValidateID(v1ID); err != nil { 127 continue 128 } 129 if _, ok := mappings[v1ID]; ok { // support old migrations without helper files 130 continue 131 } 132 workQueue <- v1ID 133 } 134 close(workQueue) 135 wg.Wait() 136} 137 138func calculateLayerChecksum(graphDir, id string, ls checksumCalculator) error { 139 diffIDFile := filepath.Join(graphDir, id, migrationDiffIDFileName) 140 if _, err := os.Lstat(diffIDFile); err == nil { 141 return nil 142 } else if !os.IsNotExist(err) { 143 return err 144 } 145 146 parent, err := getParent(filepath.Join(graphDir, id)) 147 if err != nil { 148 return err 149 } 150 151 diffID, size, err := ls.ChecksumForGraphID(id, parent, filepath.Join(graphDir, id, tarDataFileName), filepath.Join(graphDir, id, migrationTarDataFileName)) 152 if err != nil { 153 return err 154 } 155 156 if err := ioutil.WriteFile(filepath.Join(graphDir, id, migrationSizeFileName), []byte(strconv.Itoa(int(size))), 0600); err != nil { 157 return err 158 } 159 160 if err := ioutils.AtomicWriteFile(filepath.Join(graphDir, id, migrationDiffIDFileName), []byte(diffID), 0600); err != nil { 161 return err 162 } 163 164 logrus.Infof("calculated checksum for layer %s: %s", id, diffID) 165 return nil 166} 167 168func restoreMappings(root string) (map[string]image.ID, error) { 169 mappings := make(map[string]image.ID) 170 171 mfile := filepath.Join(root, migrationFileName) 172 f, err := os.Open(mfile) 173 if err != nil && !os.IsNotExist(err) { 174 return nil, err 175 } else if err == nil { 176 err := json.NewDecoder(f).Decode(&mappings) 177 if err != nil { 178 f.Close() 179 return nil, err 180 } 181 f.Close() 182 } 183 184 return mappings, nil 185} 186 187func saveMappings(root string, mappings map[string]image.ID) error { 188 mfile := filepath.Join(root, migrationFileName) 189 f, err := os.OpenFile(mfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) 190 if err != nil { 191 return err 192 } 193 defer f.Close() 194 return json.NewEncoder(f).Encode(mappings) 195} 196 197func migrateImages(root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) error { 198 graphDir := filepath.Join(root, graphDirName) 199 200 dir, err := ioutil.ReadDir(graphDir) 201 if err != nil { 202 return err 203 } 204 for _, v := range dir { 205 v1ID := v.Name() 206 if err := imagev1.ValidateID(v1ID); err != nil { 207 continue 208 } 209 if _, exists := mappings[v1ID]; exists { 210 continue 211 } 212 if err := migrateImage(v1ID, root, ls, is, ms, mappings); err != nil { 213 continue 214 } 215 } 216 217 return nil 218} 219 220func migrateContainers(root string, ls graphIDMounter, is image.Store, imageMappings map[string]image.ID) error { 221 containersDir := filepath.Join(root, containersDirName) 222 dir, err := ioutil.ReadDir(containersDir) 223 if err != nil { 224 return err 225 } 226 for _, v := range dir { 227 id := v.Name() 228 229 if _, err := os.Stat(filepath.Join(containersDir, id, configFileName)); err == nil { 230 continue 231 } 232 233 containerJSON, err := ioutil.ReadFile(filepath.Join(containersDir, id, configFileNameLegacy)) 234 if err != nil { 235 logrus.Errorf("migrate container error: %v", err) 236 continue 237 } 238 239 var c map[string]*json.RawMessage 240 if err := json.Unmarshal(containerJSON, &c); err != nil { 241 logrus.Errorf("migrate container error: %v", err) 242 continue 243 } 244 245 imageStrJSON, ok := c["Image"] 246 if !ok { 247 return fmt.Errorf("invalid container configuration for %v", id) 248 } 249 250 var image string 251 if err := json.Unmarshal([]byte(*imageStrJSON), &image); err != nil { 252 logrus.Errorf("migrate container error: %v", err) 253 continue 254 } 255 256 imageID, ok := imageMappings[image] 257 if !ok { 258 logrus.Errorf("image not migrated %v", imageID) // non-fatal error 259 continue 260 } 261 262 c["Image"] = rawJSON(imageID) 263 264 containerJSON, err = json.Marshal(c) 265 if err != nil { 266 return err 267 } 268 269 if err := ioutil.WriteFile(filepath.Join(containersDir, id, configFileName), containerJSON, 0600); err != nil { 270 return err 271 } 272 273 img, err := is.Get(imageID) 274 if err != nil { 275 return err 276 } 277 278 if err := ls.CreateRWLayerByGraphID(id, id, img.RootFS.ChainID()); err != nil { 279 logrus.Errorf("migrate container error: %v", err) 280 continue 281 } 282 283 logrus.Infof("migrated container %s to point to %s", id, imageID) 284 285 } 286 return nil 287} 288 289type refAdder interface { 290 AddTag(ref reference.Named, id digest.Digest, force bool) error 291 AddDigest(ref reference.Canonical, id digest.Digest, force bool) error 292} 293 294func migrateRefs(root, driverName string, rs refAdder, mappings map[string]image.ID) error { 295 migrationFile := filepath.Join(root, migrationTagsFileName) 296 if _, err := os.Lstat(migrationFile); !os.IsNotExist(err) { 297 return err 298 } 299 300 type repositories struct { 301 Repositories map[string]map[string]string 302 } 303 304 var repos repositories 305 306 f, err := os.Open(filepath.Join(root, repositoriesFilePrefixLegacy+driverName)) 307 if err != nil { 308 if os.IsNotExist(err) { 309 return nil 310 } 311 return err 312 } 313 defer f.Close() 314 if err := json.NewDecoder(f).Decode(&repos); err != nil { 315 return err 316 } 317 318 for name, repo := range repos.Repositories { 319 for tag, id := range repo { 320 if strongID, exists := mappings[id]; exists { 321 ref, err := reference.ParseNormalizedNamed(name) 322 if err != nil { 323 logrus.Errorf("migrate tags: invalid name %q, %q", name, err) 324 continue 325 } 326 if !reference.IsNameOnly(ref) { 327 logrus.Errorf("migrate tags: invalid name %q, unexpected tag or digest", name) 328 continue 329 } 330 if dgst, err := digest.Parse(tag); err == nil { 331 canonical, err := reference.WithDigest(reference.TrimNamed(ref), dgst) 332 if err != nil { 333 logrus.Errorf("migrate tags: invalid digest %q, %q", dgst, err) 334 continue 335 } 336 if err := rs.AddDigest(canonical, strongID.Digest(), false); err != nil { 337 logrus.Errorf("can't migrate digest %q for %q, err: %q", reference.FamiliarString(ref), strongID, err) 338 } 339 } else { 340 tagRef, err := reference.WithTag(ref, tag) 341 if err != nil { 342 logrus.Errorf("migrate tags: invalid tag %q, %q", tag, err) 343 continue 344 } 345 if err := rs.AddTag(tagRef, strongID.Digest(), false); err != nil { 346 logrus.Errorf("can't migrate tag %q for %q, err: %q", reference.FamiliarString(ref), strongID, err) 347 } 348 } 349 logrus.Infof("migrated tag %s:%s to point to %s", name, tag, strongID) 350 } 351 } 352 } 353 354 mf, err := os.Create(migrationFile) 355 if err != nil { 356 return err 357 } 358 mf.Close() 359 360 return nil 361} 362 363func getParent(confDir string) (string, error) { 364 jsonFile := filepath.Join(confDir, "json") 365 imageJSON, err := ioutil.ReadFile(jsonFile) 366 if err != nil { 367 return "", err 368 } 369 var parent struct { 370 Parent string 371 ParentID digest.Digest `json:"parent_id"` 372 } 373 if err := json.Unmarshal(imageJSON, &parent); err != nil { 374 return "", err 375 } 376 if parent.Parent == "" && parent.ParentID != "" { // v1.9 377 parent.Parent = parent.ParentID.Hex() 378 } 379 // compatibilityID for parent 380 parentCompatibilityID, err := ioutil.ReadFile(filepath.Join(confDir, "parent")) 381 if err == nil && len(parentCompatibilityID) > 0 { 382 parent.Parent = string(parentCompatibilityID) 383 } 384 return parent.Parent, nil 385} 386 387func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) (err error) { 388 defer func() { 389 if err != nil { 390 logrus.Errorf("migration failed for %v, err: %v", id, err) 391 } 392 }() 393 394 parent, err := getParent(filepath.Join(root, graphDirName, id)) 395 if err != nil { 396 return err 397 } 398 399 var parentID image.ID 400 if parent != "" { 401 var exists bool 402 if parentID, exists = mappings[parent]; !exists { 403 if err := migrateImage(parent, root, ls, is, ms, mappings); err != nil { 404 // todo: fail or allow broken chains? 405 return err 406 } 407 parentID = mappings[parent] 408 } 409 } 410 411 rootFS := image.NewRootFS() 412 var history []image.History 413 414 if parentID != "" { 415 parentImg, err := is.Get(parentID) 416 if err != nil { 417 return err 418 } 419 420 rootFS = parentImg.RootFS 421 history = parentImg.History 422 } 423 424 diffIDData, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationDiffIDFileName)) 425 if err != nil { 426 return err 427 } 428 diffID, err := digest.Parse(string(diffIDData)) 429 if err != nil { 430 return err 431 } 432 433 sizeStr, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationSizeFileName)) 434 if err != nil { 435 return err 436 } 437 size, err := strconv.ParseInt(string(sizeStr), 10, 64) 438 if err != nil { 439 return err 440 } 441 442 layer, err := ls.RegisterByGraphID(id, rootFS.ChainID(), layer.DiffID(diffID), filepath.Join(root, graphDirName, id, migrationTarDataFileName), size) 443 if err != nil { 444 return err 445 } 446 logrus.Infof("migrated layer %s to %s", id, layer.DiffID()) 447 448 jsonFile := filepath.Join(root, graphDirName, id, "json") 449 imageJSON, err := ioutil.ReadFile(jsonFile) 450 if err != nil { 451 return err 452 } 453 454 h, err := imagev1.HistoryFromConfig(imageJSON, false) 455 if err != nil { 456 return err 457 } 458 history = append(history, h) 459 460 rootFS.Append(layer.DiffID()) 461 462 config, err := imagev1.MakeConfigFromV1Config(imageJSON, rootFS, history) 463 if err != nil { 464 return err 465 } 466 strongID, err := is.Create(config) 467 if err != nil { 468 return err 469 } 470 logrus.Infof("migrated image %s to %s", id, strongID) 471 472 if parentID != "" { 473 if err := is.SetParent(strongID, parentID); err != nil { 474 return err 475 } 476 } 477 478 checksum, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, "checksum")) 479 if err == nil { // best effort 480 dgst, err := digest.Parse(string(checksum)) 481 if err == nil { 482 V2MetadataService := metadata.NewV2MetadataService(ms) 483 V2MetadataService.Add(layer.DiffID(), metadata.V2Metadata{Digest: dgst}) 484 } 485 } 486 _, err = ls.Release(layer) 487 if err != nil { 488 return err 489 } 490 491 mappings[id] = strongID 492 return 493} 494 495func rawJSON(value interface{}) *json.RawMessage { 496 jsonval, err := json.Marshal(value) 497 if err != nil { 498 return nil 499 } 500 return (*json.RawMessage)(&jsonval) 501} 502