1package v1 // import "github.com/docker/docker/migrate/v1"
2
3import (
4	"encoding/json"
5	"errors"
6	"fmt"
7	"io/ioutil"
8	"os"
9	"path/filepath"
10	"runtime"
11	"strconv"
12	"sync"
13	"time"
14
15	"github.com/docker/distribution/reference"
16	"github.com/docker/docker/distribution/metadata"
17	"github.com/docker/docker/image"
18	imagev1 "github.com/docker/docker/image/v1"
19	"github.com/docker/docker/layer"
20	"github.com/docker/docker/pkg/ioutils"
21	refstore "github.com/docker/docker/reference"
22	"github.com/opencontainers/go-digest"
23	"github.com/sirupsen/logrus"
24)
25
26type graphIDRegistrar interface {
27	RegisterByGraphID(string, layer.ChainID, layer.DiffID, string, int64) (layer.Layer, error)
28	Release(layer.Layer) ([]layer.Metadata, error)
29}
30
31type graphIDMounter interface {
32	CreateRWLayerByGraphID(string, string, layer.ChainID) error
33}
34
35type checksumCalculator interface {
36	ChecksumForGraphID(id, parent, oldTarDataPath, newTarDataPath string) (diffID layer.DiffID, size int64, err error)
37}
38
39const (
40	graphDirName                 = "graph"
41	tarDataFileName              = "tar-data.json.gz"
42	migrationFileName            = ".migration-v1-images.json"
43	migrationTagsFileName        = ".migration-v1-tags"
44	migrationDiffIDFileName      = ".migration-diffid"
45	migrationSizeFileName        = ".migration-size"
46	migrationTarDataFileName     = ".migration-tardata"
47	containersDirName            = "containers"
48	configFileNameLegacy         = "config.json"
49	configFileName               = "config.v2.json"
50	repositoriesFilePrefixLegacy = "repositories-"
51)
52
53var (
54	errUnsupported = errors.New("migration is not supported")
55)
56
57// Migrate takes an old graph directory and transforms the metadata into the
58// new format.
59func Migrate(root, driverName string, ls layer.Store, is image.Store, rs refstore.Store, ms metadata.Store) error {
60	graphDir := filepath.Join(root, graphDirName)
61	if _, err := os.Lstat(graphDir); os.IsNotExist(err) {
62		return nil
63	}
64
65	mappings, err := restoreMappings(root)
66	if err != nil {
67		return err
68	}
69
70	if cc, ok := ls.(checksumCalculator); ok {
71		CalculateLayerChecksums(root, cc, mappings)
72	}
73
74	if registrar, ok := ls.(graphIDRegistrar); !ok {
75		return errUnsupported
76	} else if err := migrateImages(root, registrar, is, ms, mappings); err != nil {
77		return err
78	}
79
80	err = saveMappings(root, mappings)
81	if err != nil {
82		return err
83	}
84
85	if mounter, ok := ls.(graphIDMounter); !ok {
86		return errUnsupported
87	} else if err := migrateContainers(root, mounter, is, mappings); err != nil {
88		return err
89	}
90
91	return migrateRefs(root, driverName, rs, mappings)
92}
93
94// CalculateLayerChecksums walks an old graph directory and calculates checksums
95// for each layer. These checksums are later used for migration.
96func CalculateLayerChecksums(root string, ls checksumCalculator, mappings map[string]image.ID) {
97	graphDir := filepath.Join(root, graphDirName)
98	// spawn some extra workers also for maximum performance because the process is bounded by both cpu and io
99	workers := runtime.NumCPU() * 3
100	workQueue := make(chan string, workers)
101
102	wg := sync.WaitGroup{}
103
104	for i := 0; i < workers; i++ {
105		wg.Add(1)
106		go func() {
107			for id := range workQueue {
108				start := time.Now()
109				if err := calculateLayerChecksum(graphDir, id, ls); err != nil {
110					logrus.Errorf("could not calculate checksum for %q, %q", id, err)
111				}
112				elapsed := time.Since(start)
113				logrus.Debugf("layer %s took %.2f seconds", id, elapsed.Seconds())
114			}
115			wg.Done()
116		}()
117	}
118
119	dir, err := ioutil.ReadDir(graphDir)
120	if err != nil {
121		logrus.Errorf("could not read directory %q", graphDir)
122		return
123	}
124	for _, v := range dir {
125		v1ID := v.Name()
126		if err := imagev1.ValidateID(v1ID); err != nil {
127			continue
128		}
129		if _, ok := mappings[v1ID]; ok { // support old migrations without helper files
130			continue
131		}
132		workQueue <- v1ID
133	}
134	close(workQueue)
135	wg.Wait()
136}
137
138func calculateLayerChecksum(graphDir, id string, ls checksumCalculator) error {
139	diffIDFile := filepath.Join(graphDir, id, migrationDiffIDFileName)
140	if _, err := os.Lstat(diffIDFile); err == nil {
141		return nil
142	} else if !os.IsNotExist(err) {
143		return err
144	}
145
146	parent, err := getParent(filepath.Join(graphDir, id))
147	if err != nil {
148		return err
149	}
150
151	diffID, size, err := ls.ChecksumForGraphID(id, parent, filepath.Join(graphDir, id, tarDataFileName), filepath.Join(graphDir, id, migrationTarDataFileName))
152	if err != nil {
153		return err
154	}
155
156	if err := ioutil.WriteFile(filepath.Join(graphDir, id, migrationSizeFileName), []byte(strconv.Itoa(int(size))), 0600); err != nil {
157		return err
158	}
159
160	if err := ioutils.AtomicWriteFile(filepath.Join(graphDir, id, migrationDiffIDFileName), []byte(diffID), 0600); err != nil {
161		return err
162	}
163
164	logrus.Infof("calculated checksum for layer %s: %s", id, diffID)
165	return nil
166}
167
168func restoreMappings(root string) (map[string]image.ID, error) {
169	mappings := make(map[string]image.ID)
170
171	mfile := filepath.Join(root, migrationFileName)
172	f, err := os.Open(mfile)
173	if err != nil && !os.IsNotExist(err) {
174		return nil, err
175	} else if err == nil {
176		err := json.NewDecoder(f).Decode(&mappings)
177		if err != nil {
178			f.Close()
179			return nil, err
180		}
181		f.Close()
182	}
183
184	return mappings, nil
185}
186
187func saveMappings(root string, mappings map[string]image.ID) error {
188	mfile := filepath.Join(root, migrationFileName)
189	f, err := os.OpenFile(mfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
190	if err != nil {
191		return err
192	}
193	defer f.Close()
194	return json.NewEncoder(f).Encode(mappings)
195}
196
197func migrateImages(root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) error {
198	graphDir := filepath.Join(root, graphDirName)
199
200	dir, err := ioutil.ReadDir(graphDir)
201	if err != nil {
202		return err
203	}
204	for _, v := range dir {
205		v1ID := v.Name()
206		if err := imagev1.ValidateID(v1ID); err != nil {
207			continue
208		}
209		if _, exists := mappings[v1ID]; exists {
210			continue
211		}
212		if err := migrateImage(v1ID, root, ls, is, ms, mappings); err != nil {
213			continue
214		}
215	}
216
217	return nil
218}
219
220func migrateContainers(root string, ls graphIDMounter, is image.Store, imageMappings map[string]image.ID) error {
221	containersDir := filepath.Join(root, containersDirName)
222	dir, err := ioutil.ReadDir(containersDir)
223	if err != nil {
224		return err
225	}
226	for _, v := range dir {
227		id := v.Name()
228
229		if _, err := os.Stat(filepath.Join(containersDir, id, configFileName)); err == nil {
230			continue
231		}
232
233		containerJSON, err := ioutil.ReadFile(filepath.Join(containersDir, id, configFileNameLegacy))
234		if err != nil {
235			logrus.Errorf("migrate container error: %v", err)
236			continue
237		}
238
239		var c map[string]*json.RawMessage
240		if err := json.Unmarshal(containerJSON, &c); err != nil {
241			logrus.Errorf("migrate container error: %v", err)
242			continue
243		}
244
245		imageStrJSON, ok := c["Image"]
246		if !ok {
247			return fmt.Errorf("invalid container configuration for %v", id)
248		}
249
250		var image string
251		if err := json.Unmarshal([]byte(*imageStrJSON), &image); err != nil {
252			logrus.Errorf("migrate container error: %v", err)
253			continue
254		}
255
256		imageID, ok := imageMappings[image]
257		if !ok {
258			logrus.Errorf("image not migrated %v", imageID) // non-fatal error
259			continue
260		}
261
262		c["Image"] = rawJSON(imageID)
263
264		containerJSON, err = json.Marshal(c)
265		if err != nil {
266			return err
267		}
268
269		if err := ioutil.WriteFile(filepath.Join(containersDir, id, configFileName), containerJSON, 0600); err != nil {
270			return err
271		}
272
273		img, err := is.Get(imageID)
274		if err != nil {
275			return err
276		}
277
278		if err := ls.CreateRWLayerByGraphID(id, id, img.RootFS.ChainID()); err != nil {
279			logrus.Errorf("migrate container error: %v", err)
280			continue
281		}
282
283		logrus.Infof("migrated container %s to point to %s", id, imageID)
284
285	}
286	return nil
287}
288
289type refAdder interface {
290	AddTag(ref reference.Named, id digest.Digest, force bool) error
291	AddDigest(ref reference.Canonical, id digest.Digest, force bool) error
292}
293
294func migrateRefs(root, driverName string, rs refAdder, mappings map[string]image.ID) error {
295	migrationFile := filepath.Join(root, migrationTagsFileName)
296	if _, err := os.Lstat(migrationFile); !os.IsNotExist(err) {
297		return err
298	}
299
300	type repositories struct {
301		Repositories map[string]map[string]string
302	}
303
304	var repos repositories
305
306	f, err := os.Open(filepath.Join(root, repositoriesFilePrefixLegacy+driverName))
307	if err != nil {
308		if os.IsNotExist(err) {
309			return nil
310		}
311		return err
312	}
313	defer f.Close()
314	if err := json.NewDecoder(f).Decode(&repos); err != nil {
315		return err
316	}
317
318	for name, repo := range repos.Repositories {
319		for tag, id := range repo {
320			if strongID, exists := mappings[id]; exists {
321				ref, err := reference.ParseNormalizedNamed(name)
322				if err != nil {
323					logrus.Errorf("migrate tags: invalid name %q, %q", name, err)
324					continue
325				}
326				if !reference.IsNameOnly(ref) {
327					logrus.Errorf("migrate tags: invalid name %q, unexpected tag or digest", name)
328					continue
329				}
330				if dgst, err := digest.Parse(tag); err == nil {
331					canonical, err := reference.WithDigest(reference.TrimNamed(ref), dgst)
332					if err != nil {
333						logrus.Errorf("migrate tags: invalid digest %q, %q", dgst, err)
334						continue
335					}
336					if err := rs.AddDigest(canonical, strongID.Digest(), false); err != nil {
337						logrus.Errorf("can't migrate digest %q for %q, err: %q", reference.FamiliarString(ref), strongID, err)
338					}
339				} else {
340					tagRef, err := reference.WithTag(ref, tag)
341					if err != nil {
342						logrus.Errorf("migrate tags: invalid tag %q, %q", tag, err)
343						continue
344					}
345					if err := rs.AddTag(tagRef, strongID.Digest(), false); err != nil {
346						logrus.Errorf("can't migrate tag %q for %q, err: %q", reference.FamiliarString(ref), strongID, err)
347					}
348				}
349				logrus.Infof("migrated tag %s:%s to point to %s", name, tag, strongID)
350			}
351		}
352	}
353
354	mf, err := os.Create(migrationFile)
355	if err != nil {
356		return err
357	}
358	mf.Close()
359
360	return nil
361}
362
363func getParent(confDir string) (string, error) {
364	jsonFile := filepath.Join(confDir, "json")
365	imageJSON, err := ioutil.ReadFile(jsonFile)
366	if err != nil {
367		return "", err
368	}
369	var parent struct {
370		Parent   string
371		ParentID digest.Digest `json:"parent_id"`
372	}
373	if err := json.Unmarshal(imageJSON, &parent); err != nil {
374		return "", err
375	}
376	if parent.Parent == "" && parent.ParentID != "" { // v1.9
377		parent.Parent = parent.ParentID.Hex()
378	}
379	// compatibilityID for parent
380	parentCompatibilityID, err := ioutil.ReadFile(filepath.Join(confDir, "parent"))
381	if err == nil && len(parentCompatibilityID) > 0 {
382		parent.Parent = string(parentCompatibilityID)
383	}
384	return parent.Parent, nil
385}
386
387func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) (err error) {
388	defer func() {
389		if err != nil {
390			logrus.Errorf("migration failed for %v, err: %v", id, err)
391		}
392	}()
393
394	parent, err := getParent(filepath.Join(root, graphDirName, id))
395	if err != nil {
396		return err
397	}
398
399	var parentID image.ID
400	if parent != "" {
401		var exists bool
402		if parentID, exists = mappings[parent]; !exists {
403			if err := migrateImage(parent, root, ls, is, ms, mappings); err != nil {
404				// todo: fail or allow broken chains?
405				return err
406			}
407			parentID = mappings[parent]
408		}
409	}
410
411	rootFS := image.NewRootFS()
412	var history []image.History
413
414	if parentID != "" {
415		parentImg, err := is.Get(parentID)
416		if err != nil {
417			return err
418		}
419
420		rootFS = parentImg.RootFS
421		history = parentImg.History
422	}
423
424	diffIDData, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationDiffIDFileName))
425	if err != nil {
426		return err
427	}
428	diffID, err := digest.Parse(string(diffIDData))
429	if err != nil {
430		return err
431	}
432
433	sizeStr, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationSizeFileName))
434	if err != nil {
435		return err
436	}
437	size, err := strconv.ParseInt(string(sizeStr), 10, 64)
438	if err != nil {
439		return err
440	}
441
442	layer, err := ls.RegisterByGraphID(id, rootFS.ChainID(), layer.DiffID(diffID), filepath.Join(root, graphDirName, id, migrationTarDataFileName), size)
443	if err != nil {
444		return err
445	}
446	logrus.Infof("migrated layer %s to %s", id, layer.DiffID())
447
448	jsonFile := filepath.Join(root, graphDirName, id, "json")
449	imageJSON, err := ioutil.ReadFile(jsonFile)
450	if err != nil {
451		return err
452	}
453
454	h, err := imagev1.HistoryFromConfig(imageJSON, false)
455	if err != nil {
456		return err
457	}
458	history = append(history, h)
459
460	rootFS.Append(layer.DiffID())
461
462	config, err := imagev1.MakeConfigFromV1Config(imageJSON, rootFS, history)
463	if err != nil {
464		return err
465	}
466	strongID, err := is.Create(config)
467	if err != nil {
468		return err
469	}
470	logrus.Infof("migrated image %s to %s", id, strongID)
471
472	if parentID != "" {
473		if err := is.SetParent(strongID, parentID); err != nil {
474			return err
475		}
476	}
477
478	checksum, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, "checksum"))
479	if err == nil { // best effort
480		dgst, err := digest.Parse(string(checksum))
481		if err == nil {
482			V2MetadataService := metadata.NewV2MetadataService(ms)
483			V2MetadataService.Add(layer.DiffID(), metadata.V2Metadata{Digest: dgst})
484		}
485	}
486	_, err = ls.Release(layer)
487	if err != nil {
488		return err
489	}
490
491	mappings[id] = strongID
492	return
493}
494
495func rawJSON(value interface{}) *json.RawMessage {
496	jsonval, err := json.Marshal(value)
497	if err != nil {
498		return nil
499	}
500	return (*json.RawMessage)(&jsonval)
501}
502