1package v1
2
3import (
4	"errors"
5	"fmt"
6	"io/ioutil"
7	"os"
8	"path/filepath"
9	"runtime"
10	"strconv"
11	"sync"
12	"time"
13
14	"encoding/json"
15
16	"github.com/docker/distribution/reference"
17	"github.com/docker/docker/distribution/metadata"
18	"github.com/docker/docker/image"
19	imagev1 "github.com/docker/docker/image/v1"
20	"github.com/docker/docker/layer"
21	"github.com/docker/docker/pkg/ioutils"
22	refstore "github.com/docker/docker/reference"
23	"github.com/opencontainers/go-digest"
24	"github.com/sirupsen/logrus"
25)
26
27type graphIDRegistrar interface {
28	RegisterByGraphID(string, layer.ChainID, layer.DiffID, string, int64) (layer.Layer, error)
29	Release(layer.Layer) ([]layer.Metadata, error)
30}
31
32type graphIDMounter interface {
33	CreateRWLayerByGraphID(string, string, layer.ChainID) error
34}
35
36type checksumCalculator interface {
37	ChecksumForGraphID(id, parent, oldTarDataPath, newTarDataPath string) (diffID layer.DiffID, size int64, err error)
38}
39
40const (
41	graphDirName                 = "graph"
42	tarDataFileName              = "tar-data.json.gz"
43	migrationFileName            = ".migration-v1-images.json"
44	migrationTagsFileName        = ".migration-v1-tags"
45	migrationDiffIDFileName      = ".migration-diffid"
46	migrationSizeFileName        = ".migration-size"
47	migrationTarDataFileName     = ".migration-tardata"
48	containersDirName            = "containers"
49	configFileNameLegacy         = "config.json"
50	configFileName               = "config.v2.json"
51	repositoriesFilePrefixLegacy = "repositories-"
52)
53
54var (
55	errUnsupported = errors.New("migration is not supported")
56)
57
58// Migrate takes an old graph directory and transforms the metadata into the
59// new format.
60func Migrate(root, driverName string, ls layer.Store, is image.Store, rs refstore.Store, ms metadata.Store) error {
61	graphDir := filepath.Join(root, graphDirName)
62	if _, err := os.Lstat(graphDir); os.IsNotExist(err) {
63		return nil
64	}
65
66	mappings, err := restoreMappings(root)
67	if err != nil {
68		return err
69	}
70
71	if cc, ok := ls.(checksumCalculator); ok {
72		CalculateLayerChecksums(root, cc, mappings)
73	}
74
75	if registrar, ok := ls.(graphIDRegistrar); !ok {
76		return errUnsupported
77	} else if err := migrateImages(root, registrar, is, ms, mappings); err != nil {
78		return err
79	}
80
81	err = saveMappings(root, mappings)
82	if err != nil {
83		return err
84	}
85
86	if mounter, ok := ls.(graphIDMounter); !ok {
87		return errUnsupported
88	} else if err := migrateContainers(root, mounter, is, mappings); err != nil {
89		return err
90	}
91
92	if err := migrateRefs(root, driverName, rs, mappings); err != nil {
93		return err
94	}
95
96	return nil
97}
98
99// CalculateLayerChecksums walks an old graph directory and calculates checksums
100// for each layer. These checksums are later used for migration.
101func CalculateLayerChecksums(root string, ls checksumCalculator, mappings map[string]image.ID) {
102	graphDir := filepath.Join(root, graphDirName)
103	// spawn some extra workers also for maximum performance because the process is bounded by both cpu and io
104	workers := runtime.NumCPU() * 3
105	workQueue := make(chan string, workers)
106
107	wg := sync.WaitGroup{}
108
109	for i := 0; i < workers; i++ {
110		wg.Add(1)
111		go func() {
112			for id := range workQueue {
113				start := time.Now()
114				if err := calculateLayerChecksum(graphDir, id, ls); err != nil {
115					logrus.Errorf("could not calculate checksum for %q, %q", id, err)
116				}
117				elapsed := time.Since(start)
118				logrus.Debugf("layer %s took %.2f seconds", id, elapsed.Seconds())
119			}
120			wg.Done()
121		}()
122	}
123
124	dir, err := ioutil.ReadDir(graphDir)
125	if err != nil {
126		logrus.Errorf("could not read directory %q", graphDir)
127		return
128	}
129	for _, v := range dir {
130		v1ID := v.Name()
131		if err := imagev1.ValidateID(v1ID); err != nil {
132			continue
133		}
134		if _, ok := mappings[v1ID]; ok { // support old migrations without helper files
135			continue
136		}
137		workQueue <- v1ID
138	}
139	close(workQueue)
140	wg.Wait()
141}
142
143func calculateLayerChecksum(graphDir, id string, ls checksumCalculator) error {
144	diffIDFile := filepath.Join(graphDir, id, migrationDiffIDFileName)
145	if _, err := os.Lstat(diffIDFile); err == nil {
146		return nil
147	} else if !os.IsNotExist(err) {
148		return err
149	}
150
151	parent, err := getParent(filepath.Join(graphDir, id))
152	if err != nil {
153		return err
154	}
155
156	diffID, size, err := ls.ChecksumForGraphID(id, parent, filepath.Join(graphDir, id, tarDataFileName), filepath.Join(graphDir, id, migrationTarDataFileName))
157	if err != nil {
158		return err
159	}
160
161	if err := ioutil.WriteFile(filepath.Join(graphDir, id, migrationSizeFileName), []byte(strconv.Itoa(int(size))), 0600); err != nil {
162		return err
163	}
164
165	if err := ioutils.AtomicWriteFile(filepath.Join(graphDir, id, migrationDiffIDFileName), []byte(diffID), 0600); err != nil {
166		return err
167	}
168
169	logrus.Infof("calculated checksum for layer %s: %s", id, diffID)
170	return nil
171}
172
173func restoreMappings(root string) (map[string]image.ID, error) {
174	mappings := make(map[string]image.ID)
175
176	mfile := filepath.Join(root, migrationFileName)
177	f, err := os.Open(mfile)
178	if err != nil && !os.IsNotExist(err) {
179		return nil, err
180	} else if err == nil {
181		err := json.NewDecoder(f).Decode(&mappings)
182		if err != nil {
183			f.Close()
184			return nil, err
185		}
186		f.Close()
187	}
188
189	return mappings, nil
190}
191
192func saveMappings(root string, mappings map[string]image.ID) error {
193	mfile := filepath.Join(root, migrationFileName)
194	f, err := os.OpenFile(mfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
195	if err != nil {
196		return err
197	}
198	defer f.Close()
199	return json.NewEncoder(f).Encode(mappings)
200}
201
202func migrateImages(root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) error {
203	graphDir := filepath.Join(root, graphDirName)
204
205	dir, err := ioutil.ReadDir(graphDir)
206	if err != nil {
207		return err
208	}
209	for _, v := range dir {
210		v1ID := v.Name()
211		if err := imagev1.ValidateID(v1ID); err != nil {
212			continue
213		}
214		if _, exists := mappings[v1ID]; exists {
215			continue
216		}
217		if err := migrateImage(v1ID, root, ls, is, ms, mappings); err != nil {
218			continue
219		}
220	}
221
222	return nil
223}
224
225func migrateContainers(root string, ls graphIDMounter, is image.Store, imageMappings map[string]image.ID) error {
226	containersDir := filepath.Join(root, containersDirName)
227	dir, err := ioutil.ReadDir(containersDir)
228	if err != nil {
229		return err
230	}
231	for _, v := range dir {
232		id := v.Name()
233
234		if _, err := os.Stat(filepath.Join(containersDir, id, configFileName)); err == nil {
235			continue
236		}
237
238		containerJSON, err := ioutil.ReadFile(filepath.Join(containersDir, id, configFileNameLegacy))
239		if err != nil {
240			logrus.Errorf("migrate container error: %v", err)
241			continue
242		}
243
244		var c map[string]*json.RawMessage
245		if err := json.Unmarshal(containerJSON, &c); err != nil {
246			logrus.Errorf("migrate container error: %v", err)
247			continue
248		}
249
250		imageStrJSON, ok := c["Image"]
251		if !ok {
252			return fmt.Errorf("invalid container configuration for %v", id)
253		}
254
255		var image string
256		if err := json.Unmarshal([]byte(*imageStrJSON), &image); err != nil {
257			logrus.Errorf("migrate container error: %v", err)
258			continue
259		}
260
261		imageID, ok := imageMappings[image]
262		if !ok {
263			logrus.Errorf("image not migrated %v", imageID) // non-fatal error
264			continue
265		}
266
267		c["Image"] = rawJSON(imageID)
268
269		containerJSON, err = json.Marshal(c)
270		if err != nil {
271			return err
272		}
273
274		if err := ioutil.WriteFile(filepath.Join(containersDir, id, configFileName), containerJSON, 0600); err != nil {
275			return err
276		}
277
278		img, err := is.Get(imageID)
279		if err != nil {
280			return err
281		}
282
283		if err := ls.CreateRWLayerByGraphID(id, id, img.RootFS.ChainID()); err != nil {
284			logrus.Errorf("migrate container error: %v", err)
285			continue
286		}
287
288		logrus.Infof("migrated container %s to point to %s", id, imageID)
289
290	}
291	return nil
292}
293
294type refAdder interface {
295	AddTag(ref reference.Named, id digest.Digest, force bool) error
296	AddDigest(ref reference.Canonical, id digest.Digest, force bool) error
297}
298
299func migrateRefs(root, driverName string, rs refAdder, mappings map[string]image.ID) error {
300	migrationFile := filepath.Join(root, migrationTagsFileName)
301	if _, err := os.Lstat(migrationFile); !os.IsNotExist(err) {
302		return err
303	}
304
305	type repositories struct {
306		Repositories map[string]map[string]string
307	}
308
309	var repos repositories
310
311	f, err := os.Open(filepath.Join(root, repositoriesFilePrefixLegacy+driverName))
312	if err != nil {
313		if os.IsNotExist(err) {
314			return nil
315		}
316		return err
317	}
318	defer f.Close()
319	if err := json.NewDecoder(f).Decode(&repos); err != nil {
320		return err
321	}
322
323	for name, repo := range repos.Repositories {
324		for tag, id := range repo {
325			if strongID, exists := mappings[id]; exists {
326				ref, err := reference.ParseNormalizedNamed(name)
327				if err != nil {
328					logrus.Errorf("migrate tags: invalid name %q, %q", name, err)
329					continue
330				}
331				if !reference.IsNameOnly(ref) {
332					logrus.Errorf("migrate tags: invalid name %q, unexpected tag or digest", name)
333					continue
334				}
335				if dgst, err := digest.Parse(tag); err == nil {
336					canonical, err := reference.WithDigest(reference.TrimNamed(ref), dgst)
337					if err != nil {
338						logrus.Errorf("migrate tags: invalid digest %q, %q", dgst, err)
339						continue
340					}
341					if err := rs.AddDigest(canonical, strongID.Digest(), false); err != nil {
342						logrus.Errorf("can't migrate digest %q for %q, err: %q", reference.FamiliarString(ref), strongID, err)
343					}
344				} else {
345					tagRef, err := reference.WithTag(ref, tag)
346					if err != nil {
347						logrus.Errorf("migrate tags: invalid tag %q, %q", tag, err)
348						continue
349					}
350					if err := rs.AddTag(tagRef, strongID.Digest(), false); err != nil {
351						logrus.Errorf("can't migrate tag %q for %q, err: %q", reference.FamiliarString(ref), strongID, err)
352					}
353				}
354				logrus.Infof("migrated tag %s:%s to point to %s", name, tag, strongID)
355			}
356		}
357	}
358
359	mf, err := os.Create(migrationFile)
360	if err != nil {
361		return err
362	}
363	mf.Close()
364
365	return nil
366}
367
368func getParent(confDir string) (string, error) {
369	jsonFile := filepath.Join(confDir, "json")
370	imageJSON, err := ioutil.ReadFile(jsonFile)
371	if err != nil {
372		return "", err
373	}
374	var parent struct {
375		Parent   string
376		ParentID digest.Digest `json:"parent_id"`
377	}
378	if err := json.Unmarshal(imageJSON, &parent); err != nil {
379		return "", err
380	}
381	if parent.Parent == "" && parent.ParentID != "" { // v1.9
382		parent.Parent = parent.ParentID.Hex()
383	}
384	// compatibilityID for parent
385	parentCompatibilityID, err := ioutil.ReadFile(filepath.Join(confDir, "parent"))
386	if err == nil && len(parentCompatibilityID) > 0 {
387		parent.Parent = string(parentCompatibilityID)
388	}
389	return parent.Parent, nil
390}
391
392func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) (err error) {
393	defer func() {
394		if err != nil {
395			logrus.Errorf("migration failed for %v, err: %v", id, err)
396		}
397	}()
398
399	parent, err := getParent(filepath.Join(root, graphDirName, id))
400	if err != nil {
401		return err
402	}
403
404	var parentID image.ID
405	if parent != "" {
406		var exists bool
407		if parentID, exists = mappings[parent]; !exists {
408			if err := migrateImage(parent, root, ls, is, ms, mappings); err != nil {
409				// todo: fail or allow broken chains?
410				return err
411			}
412			parentID = mappings[parent]
413		}
414	}
415
416	rootFS := image.NewRootFS()
417	var history []image.History
418
419	if parentID != "" {
420		parentImg, err := is.Get(parentID)
421		if err != nil {
422			return err
423		}
424
425		rootFS = parentImg.RootFS
426		history = parentImg.History
427	}
428
429	diffIDData, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationDiffIDFileName))
430	if err != nil {
431		return err
432	}
433	diffID, err := digest.Parse(string(diffIDData))
434	if err != nil {
435		return err
436	}
437
438	sizeStr, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationSizeFileName))
439	if err != nil {
440		return err
441	}
442	size, err := strconv.ParseInt(string(sizeStr), 10, 64)
443	if err != nil {
444		return err
445	}
446
447	layer, err := ls.RegisterByGraphID(id, rootFS.ChainID(), layer.DiffID(diffID), filepath.Join(root, graphDirName, id, migrationTarDataFileName), size)
448	if err != nil {
449		return err
450	}
451	logrus.Infof("migrated layer %s to %s", id, layer.DiffID())
452
453	jsonFile := filepath.Join(root, graphDirName, id, "json")
454	imageJSON, err := ioutil.ReadFile(jsonFile)
455	if err != nil {
456		return err
457	}
458
459	h, err := imagev1.HistoryFromConfig(imageJSON, false)
460	if err != nil {
461		return err
462	}
463	history = append(history, h)
464
465	rootFS.Append(layer.DiffID())
466
467	config, err := imagev1.MakeConfigFromV1Config(imageJSON, rootFS, history)
468	if err != nil {
469		return err
470	}
471	strongID, err := is.Create(config)
472	if err != nil {
473		return err
474	}
475	logrus.Infof("migrated image %s to %s", id, strongID)
476
477	if parentID != "" {
478		if err := is.SetParent(strongID, parentID); err != nil {
479			return err
480		}
481	}
482
483	checksum, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, "checksum"))
484	if err == nil { // best effort
485		dgst, err := digest.Parse(string(checksum))
486		if err == nil {
487			V2MetadataService := metadata.NewV2MetadataService(ms)
488			V2MetadataService.Add(layer.DiffID(), metadata.V2Metadata{Digest: dgst})
489		}
490	}
491	_, err = ls.Release(layer)
492	if err != nil {
493		return err
494	}
495
496	mappings[id] = strongID
497	return
498}
499
500func rawJSON(value interface{}) *json.RawMessage {
501	jsonval, err := json.Marshal(value)
502	if err != nil {
503		return nil
504	}
505	return (*json.RawMessage)(&jsonval)
506}
507