1// +build !windows
2
3/*
4   Copyright The containerd Authors.
5
6   Licensed under the Apache License, Version 2.0 (the "License");
7   you may not use this file except in compliance with the License.
8   You may obtain a copy of the License at
9
10       http://www.apache.org/licenses/LICENSE-2.0
11
12   Unless required by applicable law or agreed to in writing, software
13   distributed under the License is distributed on an "AS IS" BASIS,
14   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   See the License for the specific language governing permissions and
16   limitations under the License.
17*/
18
19package archive
20
21import (
22	"archive/tar"
23	"bytes"
24	"context"
25	"fmt"
26	"io"
27	"io/ioutil"
28	"os"
29	"os/exec"
30	"path/filepath"
31	"testing"
32	"time"
33
34	_ "crypto/sha256"
35
36	"github.com/containerd/containerd/archive/tartest"
37	"github.com/containerd/containerd/pkg/testutil"
38	"github.com/containerd/continuity/fs"
39	"github.com/containerd/continuity/fs/fstest"
40	"github.com/pkg/errors"
41)
42
43const tarCmd = "tar"
44
45// baseApplier creates a basic filesystem layout
46// with multiple types of files for basic tests.
47var baseApplier = fstest.Apply(
48	fstest.CreateDir("/etc/", 0755),
49	fstest.CreateFile("/etc/hosts", []byte("127.0.0.1 localhost"), 0644),
50	fstest.Link("/etc/hosts", "/etc/hosts.allow"),
51	fstest.CreateDir("/usr/local/lib", 0755),
52	fstest.CreateFile("/usr/local/lib/libnothing.so", []byte{0x00, 0x00}, 0755),
53	fstest.Symlink("libnothing.so", "/usr/local/lib/libnothing.so.2"),
54	fstest.CreateDir("/home", 0755),
55	fstest.CreateDir("/home/derek", 0700),
56)
57
58func TestUnpack(t *testing.T) {
59	requireTar(t)
60
61	if err := testApply(baseApplier); err != nil {
62		t.Fatalf("Test apply failed: %+v", err)
63	}
64}
65
66func TestBaseDiff(t *testing.T) {
67	requireTar(t)
68
69	if err := testBaseDiff(baseApplier); err != nil {
70		t.Fatalf("Test base diff failed: %+v", err)
71	}
72}
73
74func TestRelativeSymlinks(t *testing.T) {
75	breakoutLinks := []fstest.Applier{
76		fstest.Apply(
77			baseApplier,
78			fstest.Symlink("../other", "/home/derek/other"),
79			fstest.Symlink("../../etc", "/home/derek/etc"),
80			fstest.Symlink("up/../../other", "/home/derek/updown"),
81		),
82		fstest.Apply(
83			baseApplier,
84			fstest.Symlink("../../../breakout", "/home/derek/breakout"),
85		),
86		fstest.Apply(
87			baseApplier,
88			fstest.Symlink("../../breakout", "/breakout"),
89		),
90		fstest.Apply(
91			baseApplier,
92			fstest.Symlink("etc/../../upandout", "/breakout"),
93		),
94		fstest.Apply(
95			baseApplier,
96			fstest.Symlink("derek/../../../downandout", "/home/breakout"),
97		),
98		fstest.Apply(
99			baseApplier,
100			fstest.Symlink("/etc", "localetc"),
101		),
102	}
103
104	for _, bo := range breakoutLinks {
105		if err := testDiffApply(bo); err != nil {
106			t.Fatalf("Test apply failed: %+v", err)
107		}
108	}
109}
110
111func TestSymlinks(t *testing.T) {
112	links := [][2]fstest.Applier{
113		{
114			fstest.Apply(
115				fstest.CreateDir("/bin/", 0755),
116				fstest.CreateFile("/bin/superbinary", []byte{0x00, 0x00}, 0755),
117				fstest.Symlink("../bin/superbinary", "/bin/other1"),
118			),
119			fstest.Apply(
120				fstest.Remove("/bin/other1"),
121				fstest.Symlink("/bin/superbinary", "/bin/other1"),
122				fstest.Symlink("../bin/superbinary", "/bin/other2"),
123				fstest.Symlink("superbinary", "/bin/other3"),
124			),
125		},
126		{
127			fstest.Apply(
128				fstest.CreateDir("/bin/", 0755),
129				fstest.CreateDir("/sbin/", 0755),
130				fstest.CreateFile("/sbin/superbinary", []byte{0x00, 0x00}, 0755),
131				fstest.Symlink("/sbin/superbinary", "/bin/superbinary"),
132				fstest.Symlink("../bin/superbinary", "/bin/other1"),
133			),
134			fstest.Apply(
135				fstest.Remove("/bin/other1"),
136				fstest.Symlink("/bin/superbinary", "/bin/other1"),
137				fstest.Symlink("superbinary", "/bin/other2"),
138			),
139		},
140		{
141			fstest.Apply(
142				fstest.CreateDir("/bin/", 0755),
143				fstest.CreateDir("/sbin/", 0755),
144				fstest.CreateFile("/sbin/superbinary", []byte{0x00, 0x00}, 0755),
145				fstest.Symlink("../sbin/superbinary", "/bin/superbinary"),
146				fstest.Symlink("../bin/superbinary", "/bin/other1"),
147			),
148			fstest.Apply(
149				fstest.Remove("/bin/other1"),
150				fstest.Symlink("/bin/superbinary", "/bin/other1"),
151			),
152		},
153		{
154			fstest.Apply(
155				fstest.CreateDir("/bin/", 0755),
156				fstest.CreateFile("/bin/actualbinary", []byte{0x00, 0x00}, 0755),
157				fstest.Symlink("actualbinary", "/bin/superbinary"),
158				fstest.Symlink("../bin/superbinary", "/bin/other1"),
159				fstest.Symlink("superbinary", "/bin/other2"),
160			),
161			fstest.Apply(
162				fstest.Remove("/bin/other1"),
163				fstest.Remove("/bin/other2"),
164				fstest.Symlink("/bin/superbinary", "/bin/other1"),
165				fstest.Symlink("superbinary", "/bin/other2"),
166			),
167		},
168		{
169			fstest.Apply(
170				fstest.CreateDir("/bin/", 0755),
171				fstest.CreateFile("/bin/actualbinary", []byte{0x00, 0x00}, 0755),
172				fstest.Symlink("actualbinary", "/bin/myapp"),
173			),
174			fstest.Apply(
175				fstest.Remove("/bin/myapp"),
176				fstest.CreateDir("/bin/myapp", 0755),
177			),
178		},
179	}
180
181	for i, l := range links {
182		if err := testDiffApply(l[0], l[1]); err != nil {
183			t.Fatalf("Test[%d] apply failed: %+v", i+1, err)
184		}
185	}
186}
187
188func TestTarWithXattr(t *testing.T) {
189	testutil.RequiresRoot(t)
190
191	fileXattrExist := func(f1, xattrKey, xattrValue string) func(string) error {
192		return func(root string) error {
193			values, err := getxattr(filepath.Join(root, f1), xattrKey)
194			if err != nil {
195				return err
196			}
197			if xattrValue != string(values) {
198				return fmt.Errorf("file xattrs expect to be %s, actually get %s", xattrValue, values)
199			}
200			return nil
201		}
202	}
203
204	tests := []struct {
205		name  string
206		key   string
207		value string
208		err   error
209	}{
210		{
211			name:  "WithXattrsUser",
212			key:   "user.key",
213			value: "value",
214		},
215		{
216			// security related xattrs need root permission to test
217			name:  "WithXattrSelinux",
218			key:   "security.selinux",
219			value: "unconfined_u:object_r:default_t:s0\x00",
220		},
221	}
222	for _, at := range tests {
223		tc := tartest.TarContext{}.WithUIDGID(os.Getuid(), os.Getgid()).WithModTime(time.Now().UTC()).WithXattrs(map[string]string{
224			at.key: at.value,
225		})
226		w := tartest.TarAll(tc.File("/file", []byte{}, 0755))
227		validator := fileXattrExist("file", at.key, at.value)
228		t.Run(at.name, makeWriterToTarTest(w, nil, validator, at.err))
229	}
230}
231
232func TestBreakouts(t *testing.T) {
233	tc := tartest.TarContext{}.WithUIDGID(os.Getuid(), os.Getgid()).WithModTime(time.Now().UTC())
234	expected := "unbroken"
235	unbrokenCheck := func(root string) error {
236		b, err := ioutil.ReadFile(filepath.Join(root, "etc", "unbroken"))
237		if err != nil {
238			return errors.Wrap(err, "failed to read unbroken")
239		}
240		if string(b) != expected {
241			return errors.Errorf("/etc/unbroken: unexpected value %s, expected %s", b, expected)
242		}
243		return nil
244	}
245	errFileDiff := errors.New("files differ")
246
247	isSymlinkFile := func(f string) func(string) error {
248		return func(root string) error {
249			fi, err := os.Lstat(filepath.Join(root, f))
250			if err != nil {
251				return err
252			}
253
254			if got := fi.Mode() & os.ModeSymlink; got != os.ModeSymlink {
255				return errors.Errorf("%s should be symlink", fi.Name())
256			}
257			return nil
258		}
259	}
260
261	sameSymlinkFile := func(f1, f2 string) func(string) error {
262		checkF1, checkF2 := isSymlinkFile(f1), isSymlinkFile(f2)
263		return func(root string) error {
264			if err := checkF1(root); err != nil {
265				return err
266			}
267
268			if err := checkF2(root); err != nil {
269				return err
270			}
271
272			t1, err := os.Readlink(filepath.Join(root, f1))
273			if err != nil {
274				return err
275			}
276
277			t2, err := os.Readlink(filepath.Join(root, f2))
278			if err != nil {
279				return err
280			}
281
282			if t1 != t2 {
283				return errors.Wrapf(errFileDiff, "%#v and %#v", t1, t2)
284			}
285			return nil
286		}
287	}
288
289	sameFile := func(f1, f2 string) func(string) error {
290		return func(root string) error {
291			p1, err := fs.RootPath(root, f1)
292			if err != nil {
293				return err
294			}
295			p2, err := fs.RootPath(root, f2)
296			if err != nil {
297				return err
298			}
299			s1, err := os.Stat(p1)
300			if err != nil {
301				return err
302			}
303			s2, err := os.Stat(p2)
304			if err != nil {
305				return err
306			}
307			if !os.SameFile(s1, s2) {
308				return errors.Wrapf(errFileDiff, "%#v and %#v", s1, s2)
309			}
310			return nil
311		}
312	}
313	notSameFile := func(f1, f2 string) func(string) error {
314		same := sameFile(f1, f2)
315		return func(root string) error {
316			err := same(root)
317			if err == nil {
318				return errors.New("files are the same, expected diff")
319			}
320			if !errors.Is(err, errFileDiff) {
321				return err
322			}
323			return nil
324		}
325	}
326	fileValue := func(f1 string, content []byte) func(string) error {
327		return func(root string) error {
328			b, err := ioutil.ReadFile(filepath.Join(root, f1))
329			if err != nil {
330				return err
331			}
332			if !bytes.Equal(b, content) {
333				return errors.Errorf("content differs: expected %v, got %v", content, b)
334			}
335			return nil
336		}
337	}
338	fileNotExists := func(f1 string) func(string) error {
339		return func(root string) error {
340			_, err := os.Lstat(filepath.Join(root, f1))
341			if err == nil {
342				return errors.New("file exists")
343			} else if !os.IsNotExist(err) {
344				return err
345			}
346			return nil
347		}
348
349	}
350	all := func(funcs ...func(string) error) func(string) error {
351		return func(root string) error {
352			for _, f := range funcs {
353				if err := f(root); err != nil {
354					return err
355				}
356			}
357			return nil
358		}
359	}
360
361	breakouts := []struct {
362		name      string
363		w         tartest.WriterToTar
364		apply     fstest.Applier
365		validator func(string) error
366		err       error
367	}{
368		{
369			name: "SymlinkAbsolute",
370			w: tartest.TarAll(
371				tc.Dir("etc", 0755),
372				tc.Symlink("/etc", "localetc"),
373				tc.File("/localetc/unbroken", []byte(expected), 0644),
374			),
375			validator: unbrokenCheck,
376		},
377		{
378			name: "SymlinkUpAndOut",
379			w: tartest.TarAll(
380				tc.Dir("etc", 0755),
381				tc.Dir("dummy", 0755),
382				tc.Symlink("/dummy/../etc", "localetc"),
383				tc.File("/localetc/unbroken", []byte(expected), 0644),
384			),
385			validator: unbrokenCheck,
386		},
387		{
388			name: "SymlinkMultipleAbsolute",
389			w: tartest.TarAll(
390				tc.Dir("etc", 0755),
391				tc.Dir("dummy", 0755),
392				tc.Symlink("/etc", "/dummy/etc"),
393				tc.Symlink("/dummy/etc", "localetc"),
394				tc.File("/dummy/etc/unbroken", []byte(expected), 0644),
395			),
396			validator: unbrokenCheck,
397		},
398		{
399			name: "SymlinkMultipleRelative",
400			w: tartest.TarAll(
401				tc.Dir("etc", 0755),
402				tc.Dir("dummy", 0755),
403				tc.Symlink("/etc", "/dummy/etc"),
404				tc.Symlink("./dummy/etc", "localetc"),
405				tc.File("/dummy/etc/unbroken", []byte(expected), 0644),
406			),
407			validator: unbrokenCheck,
408		},
409		{
410			name: "SymlinkEmptyFile",
411			w: tartest.TarAll(
412				tc.Dir("etc", 0755),
413				tc.File("etc/emptied", []byte("notempty"), 0644),
414				tc.Symlink("/etc", "localetc"),
415				tc.File("/localetc/emptied", []byte{}, 0644),
416			),
417			validator: func(root string) error {
418				b, err := ioutil.ReadFile(filepath.Join(root, "etc", "emptied"))
419				if err != nil {
420					return errors.Wrap(err, "failed to read unbroken")
421				}
422				if len(b) > 0 {
423					return errors.Errorf("/etc/emptied: non-empty")
424				}
425				return nil
426			},
427		},
428		{
429			name: "HardlinkRelative",
430			w: tartest.TarAll(
431				tc.Dir("etc", 0770),
432				tc.File("/etc/passwd", []byte("inside"), 0644),
433				tc.Dir("breakouts", 0755),
434				tc.Symlink("../../etc", "breakouts/d1"),
435				tc.Link("/breakouts/d1/passwd", "breakouts/mypasswd"),
436			),
437			validator: sameFile("/breakouts/mypasswd", "/etc/passwd"),
438		},
439		{
440			name: "HardlinkDownAndOut",
441			w: tartest.TarAll(
442				tc.Dir("etc", 0770),
443				tc.File("/etc/passwd", []byte("inside"), 0644),
444				tc.Dir("breakouts", 0755),
445				tc.Dir("downandout", 0755),
446				tc.Symlink("../downandout/../../etc", "breakouts/d1"),
447				tc.Link("/breakouts/d1/passwd", "breakouts/mypasswd"),
448			),
449			validator: sameFile("/breakouts/mypasswd", "/etc/passwd"),
450		},
451		{
452			name: "HardlinkAbsolute",
453			w: tartest.TarAll(
454				tc.Dir("etc", 0770),
455				tc.File("/etc/passwd", []byte("inside"), 0644),
456				tc.Symlink("/etc", "localetc"),
457				tc.Link("/localetc/passwd", "localpasswd"),
458			),
459			validator: sameFile("localpasswd", "/etc/passwd"),
460		},
461		{
462			name: "HardlinkRelativeLong",
463			w: tartest.TarAll(
464				tc.Dir("etc", 0770),
465				tc.File("/etc/passwd", []byte("inside"), 0644),
466				tc.Symlink("../../../../../../../etc", "localetc"),
467				tc.Link("/localetc/passwd", "localpasswd"),
468			),
469			validator: sameFile("localpasswd", "/etc/passwd"),
470		},
471		{
472			name: "HardlinkRelativeUpAndOut",
473			w: tartest.TarAll(
474				tc.Dir("etc", 0770),
475				tc.File("/etc/passwd", []byte("inside"), 0644),
476				tc.Symlink("upandout/../../../etc", "localetc"),
477				tc.Link("/localetc/passwd", "localpasswd"),
478			),
479			validator: sameFile("localpasswd", "/etc/passwd"),
480		},
481		{
482			name: "HardlinkDirectRelative",
483			w: tartest.TarAll(
484				tc.Dir("etc", 0770),
485				tc.File("/etc/passwd", []byte("inside"), 0644),
486				tc.Link("../../../../../etc/passwd", "localpasswd"),
487			),
488			validator: sameFile("localpasswd", "/etc/passwd"),
489		},
490		{
491			name: "HardlinkDirectAbsolute",
492			w: tartest.TarAll(
493				tc.Dir("etc", 0770),
494				tc.File("/etc/passwd", []byte("inside"), 0644),
495				tc.Link("/etc/passwd", "localpasswd"),
496			),
497			validator: sameFile("localpasswd", "/etc/passwd"),
498		},
499		{
500			name: "HardlinkSymlinkBeforeCreateTarget",
501			w: tartest.TarAll(
502				tc.Dir("etc", 0770),
503				tc.Symlink("/etc/passwd", "localpasswd"),
504				tc.Link("localpasswd", "localpasswd-dup"),
505				tc.File("/etc/passwd", []byte("after"), 0644),
506			),
507			validator: sameFile("localpasswd-dup", "/etc/passwd"),
508		},
509		{
510			name: "HardlinkSymlinkRelative",
511			w: tartest.TarAll(
512				tc.Dir("etc", 0770),
513				tc.File("/etc/passwd", []byte("inside"), 0644),
514				tc.Symlink("../../../../../etc/passwd", "passwdlink"),
515				tc.Link("/passwdlink", "localpasswd"),
516			),
517			validator: all(
518				sameSymlinkFile("/localpasswd", "/passwdlink"),
519				sameFile("/localpasswd", "/etc/passwd"),
520			),
521		},
522		{
523			name: "HardlinkSymlinkAbsolute",
524			w: tartest.TarAll(
525				tc.Dir("etc", 0770),
526				tc.File("/etc/passwd", []byte("inside"), 0644),
527				tc.Symlink("/etc/passwd", "passwdlink"),
528				tc.Link("/passwdlink", "localpasswd"),
529			),
530			validator: all(
531				sameSymlinkFile("/localpasswd", "/passwdlink"),
532				sameFile("/localpasswd", "/etc/passwd"),
533			),
534		},
535		{
536			name: "SymlinkParentDirectory",
537			w: tartest.TarAll(
538				tc.Dir("etc", 0770),
539				tc.File("/etc/passwd", []byte("inside"), 0644),
540				tc.Symlink("/etc/", ".."),
541				tc.Link("/etc/passwd", "localpasswd"),
542			),
543			validator: sameFile("/localpasswd", "/etc/passwd"),
544		},
545		{
546			name: "SymlinkEmptyFilename",
547			w: tartest.TarAll(
548				tc.Dir("etc", 0770),
549				tc.File("/etc/passwd", []byte("inside"), 0644),
550				tc.Symlink("/etc/", ""),
551				tc.Link("/etc/passwd", "localpasswd"),
552			),
553			validator: sameFile("/localpasswd", "/etc/passwd"),
554		},
555		{
556			name: "SymlinkParentRelative",
557			w: tartest.TarAll(
558				tc.Dir("etc", 0770),
559				tc.File("/etc/passwd", []byte("inside"), 0644),
560				tc.Symlink("/etc/", "localetc/sub/.."),
561				tc.Link("/etc/passwd", "/localetc/localpasswd"),
562			),
563			validator: sameFile("/localetc/localpasswd", "/etc/passwd"),
564		},
565		{
566			name: "SymlinkSlashEnded",
567			w: tartest.TarAll(
568				tc.Dir("etc", 0770),
569				tc.File("/etc/passwd", []byte("inside"), 0644),
570				tc.Dir("localetc/", 0770),
571				tc.Link("/etc/passwd", "/localetc/localpasswd"),
572			),
573			validator: sameFile("/localetc/localpasswd", "/etc/passwd"),
574		},
575		{
576			name: "SymlinkOverrideDirectory",
577			apply: fstest.Apply(
578				fstest.CreateDir("/etc/", 0755),
579				fstest.CreateFile("/etc/passwd", []byte("inside"), 0644),
580				fstest.CreateDir("/localetc/", 0755),
581			),
582			w: tartest.TarAll(
583				tc.Symlink("/etc", "localetc"),
584				tc.Link("/etc/passwd", "/localetc/localpasswd"),
585			),
586			validator: sameFile("/localetc/localpasswd", "/etc/passwd"),
587		},
588		{
589			name: "SymlinkOverrideDirectoryRelative",
590			apply: fstest.Apply(
591				fstest.CreateDir("/etc/", 0755),
592				fstest.CreateFile("/etc/passwd", []byte("inside"), 0644),
593				fstest.CreateDir("/localetc/", 0755),
594			),
595			w: tartest.TarAll(
596				tc.Symlink("../../etc", "localetc"),
597				tc.Link("/etc/passwd", "/localetc/localpasswd"),
598			),
599			validator: sameFile("/localetc/localpasswd", "/etc/passwd"),
600		},
601		{
602			name: "DirectoryOverrideSymlink",
603			apply: fstest.Apply(
604				fstest.CreateDir("/etc/", 0755),
605				fstest.CreateFile("/etc/passwd", []byte("inside"), 0644),
606				fstest.Symlink("/etc", "localetc"),
607			),
608			w: tartest.TarAll(
609				tc.Dir("/localetc/", 0755),
610				tc.Link("/etc/passwd", "/localetc/localpasswd"),
611			),
612			validator: sameFile("/localetc/localpasswd", "/etc/passwd"),
613		},
614		{
615			name: "DirectoryOverrideSymlinkAndHardlink",
616			apply: fstest.Apply(
617				fstest.CreateDir("/etc/", 0755),
618				fstest.CreateFile("/etc/passwd", []byte("inside"), 0644),
619				fstest.Symlink("etc", "localetc"),
620				fstest.Link("/etc/passwd", "/localetc/localpasswd"),
621			),
622			w: tartest.TarAll(
623				tc.Dir("/localetc/", 0755),
624				tc.File("/localetc/localpasswd", []byte("different"), 0644),
625			),
626			validator: notSameFile("/localetc/localpasswd", "/etc/passwd"),
627		},
628		{
629			name: "WhiteoutRootParent",
630			apply: fstest.Apply(
631				fstest.CreateDir("/etc/", 0755),
632				fstest.CreateFile("/etc/passwd", []byte("inside"), 0644),
633			),
634			w: tartest.TarAll(
635				tc.File(".wh...", []byte{}, 0644), // Should wipe out whole directory
636			),
637			err: errInvalidArchive,
638		},
639		{
640			name: "WhiteoutParent",
641			apply: fstest.Apply(
642				fstest.CreateDir("/etc/", 0755),
643				fstest.CreateFile("/etc/passwd", []byte("inside"), 0644),
644			),
645			w: tartest.TarAll(
646				tc.File("etc/.wh...", []byte{}, 0644),
647			),
648			err: errInvalidArchive,
649		},
650		{
651			name: "WhiteoutRoot",
652			apply: fstest.Apply(
653				fstest.CreateDir("/etc/", 0755),
654				fstest.CreateFile("/etc/passwd", []byte("inside"), 0644),
655			),
656			w: tartest.TarAll(
657				tc.File(".wh..", []byte{}, 0644),
658			),
659			err: errInvalidArchive,
660		},
661		{
662			name: "WhiteoutCurrentDirectory",
663			apply: fstest.Apply(
664				fstest.CreateDir("/etc/", 0755),
665				fstest.CreateFile("/etc/passwd", []byte("inside"), 0644),
666			),
667			w: tartest.TarAll(
668				tc.File("etc/.wh..", []byte{}, 0644), // Should wipe out whole directory
669			),
670			err: errInvalidArchive,
671		},
672		{
673			name: "WhiteoutSymlink",
674			apply: fstest.Apply(
675				fstest.CreateDir("/etc/", 0755),
676				fstest.CreateFile("/etc/passwd", []byte("all users"), 0644),
677				fstest.Symlink("/etc", "localetc"),
678			),
679			w: tartest.TarAll(
680				tc.File(".wh.localetc", []byte{}, 0644), // Should wipe out whole directory
681			),
682			validator: all(
683				fileValue("etc/passwd", []byte("all users")),
684				fileNotExists("localetc"),
685			),
686		},
687		{
688			// TODO: This test should change once archive apply is disallowing
689			// symlinks as parents in the name
690			name: "WhiteoutSymlinkPath",
691			apply: fstest.Apply(
692				fstest.CreateDir("/etc/", 0755),
693				fstest.CreateFile("/etc/passwd", []byte("all users"), 0644),
694				fstest.CreateFile("/etc/whitedout", []byte("ahhhh whiteout"), 0644),
695				fstest.Symlink("/etc", "localetc"),
696			),
697			w: tartest.TarAll(
698				tc.File("localetc/.wh.whitedout", []byte{}, 0644),
699			),
700			validator: all(
701				fileValue("etc/passwd", []byte("all users")),
702				fileNotExists("etc/whitedout"),
703			),
704		},
705		{
706			name: "WhiteoutDirectoryName",
707			apply: fstest.Apply(
708				fstest.CreateDir("/etc/", 0755),
709				fstest.CreateFile("/etc/passwd", []byte("all users"), 0644),
710				fstest.CreateFile("/etc/whitedout", []byte("ahhhh whiteout"), 0644),
711				fstest.Symlink("/etc", "localetc"),
712			),
713			w: tartest.TarAll(
714				tc.File(".wh.etc/somefile", []byte("non-empty"), 0644),
715			),
716			validator: all(
717				fileValue("etc/passwd", []byte("all users")),
718				fileValue(".wh.etc/somefile", []byte("non-empty")),
719			),
720		},
721		{
722			name: "WhiteoutDeadSymlinkParent",
723			apply: fstest.Apply(
724				fstest.CreateDir("/etc/", 0755),
725				fstest.CreateFile("/etc/passwd", []byte("all users"), 0644),
726				fstest.Symlink("/dne", "localetc"),
727			),
728			w: tartest.TarAll(
729				tc.File("localetc/.wh.etc", []byte{}, 0644),
730			),
731			// no-op, remove does not
732			validator: fileValue("etc/passwd", []byte("all users")),
733		},
734		{
735			name: "WhiteoutRelativePath",
736			apply: fstest.Apply(
737				fstest.CreateDir("/etc/", 0755),
738				fstest.CreateFile("/etc/passwd", []byte("all users"), 0644),
739				fstest.Symlink("/dne", "localetc"),
740			),
741			w: tartest.TarAll(
742				tc.File("dne/../.wh.etc", []byte{}, 0644),
743			),
744			// resolution ends up just removing etc
745			validator: fileNotExists("etc/passwd"),
746		},
747	}
748
749	for _, bo := range breakouts {
750		t.Run(bo.name, makeWriterToTarTest(bo.w, bo.apply, bo.validator, bo.err))
751	}
752}
753
754func TestDiffApply(t *testing.T) {
755	fstest.FSSuite(t, diffApplier{})
756}
757
758func TestApplyTar(t *testing.T) {
759	tc := tartest.TarContext{}.WithUIDGID(os.Getuid(), os.Getgid()).WithModTime(time.Now().UTC())
760	directoriesExist := func(dirs ...string) func(string) error {
761		return func(root string) error {
762			for _, d := range dirs {
763				p, err := fs.RootPath(root, d)
764				if err != nil {
765					return err
766				}
767				if _, err := os.Stat(p); err != nil {
768					return errors.Wrapf(err, "failure checking existence for %v", d)
769				}
770			}
771			return nil
772		}
773	}
774
775	tests := []struct {
776		name      string
777		w         tartest.WriterToTar
778		apply     fstest.Applier
779		validator func(string) error
780		err       error
781	}{
782		{
783			name: "DirectoryCreation",
784			apply: fstest.Apply(
785				fstest.CreateDir("/etc/", 0755),
786			),
787			w: tartest.TarAll(
788				tc.Dir("/etc/subdir", 0755),
789				tc.Dir("/etc/subdir2/", 0755),
790				tc.Dir("/etc/subdir2/more", 0755),
791				tc.Dir("/other/noparent-1/1", 0755),
792				tc.Dir("/other/noparent-2/2/", 0755),
793			),
794			validator: directoriesExist(
795				"etc/subdir",
796				"etc/subdir2",
797				"etc/subdir2/more",
798				"other/noparent-1/1",
799				"other/noparent-2/2",
800			),
801		},
802	}
803
804	for _, at := range tests {
805		t.Run(at.name, makeWriterToTarTest(at.w, at.apply, at.validator, at.err))
806	}
807}
808
809func testApply(a fstest.Applier) error {
810	td, err := ioutil.TempDir("", "test-apply-")
811	if err != nil {
812		return errors.Wrap(err, "failed to create temp dir")
813	}
814	defer os.RemoveAll(td)
815	dest, err := ioutil.TempDir("", "test-apply-dest-")
816	if err != nil {
817		return errors.Wrap(err, "failed to create temp dir")
818	}
819	defer os.RemoveAll(dest)
820
821	if err := a.Apply(td); err != nil {
822		return errors.Wrap(err, "failed to apply filesystem changes")
823	}
824
825	tarArgs := []string{"c", "-C", td}
826	names, err := readDirNames(td)
827	if err != nil {
828		return errors.Wrap(err, "failed to read directory names")
829	}
830	tarArgs = append(tarArgs, names...)
831
832	cmd := exec.Command(tarCmd, tarArgs...)
833
834	arch, err := cmd.StdoutPipe()
835	if err != nil {
836		return errors.Wrap(err, "failed to create stdout pipe")
837	}
838
839	if err := cmd.Start(); err != nil {
840		return errors.Wrap(err, "failed to start command")
841	}
842
843	if _, err := Apply(context.Background(), dest, arch); err != nil {
844		return errors.Wrap(err, "failed to apply tar stream")
845	}
846
847	return fstest.CheckDirectoryEqual(td, dest)
848}
849
850func testBaseDiff(a fstest.Applier) error {
851	td, err := ioutil.TempDir("", "test-base-diff-")
852	if err != nil {
853		return errors.Wrap(err, "failed to create temp dir")
854	}
855	defer os.RemoveAll(td)
856	dest, err := ioutil.TempDir("", "test-base-diff-dest-")
857	if err != nil {
858		return errors.Wrap(err, "failed to create temp dir")
859	}
860	defer os.RemoveAll(dest)
861
862	if err := a.Apply(td); err != nil {
863		return errors.Wrap(err, "failed to apply filesystem changes")
864	}
865
866	arch := Diff(context.Background(), "", td)
867
868	cmd := exec.Command(tarCmd, "x", "-C", dest)
869	cmd.Stdin = arch
870	if err := cmd.Run(); err != nil {
871		return errors.Wrap(err, "tar command failed")
872	}
873
874	return fstest.CheckDirectoryEqual(td, dest)
875}
876
877func testDiffApply(appliers ...fstest.Applier) error {
878	td, err := ioutil.TempDir("", "test-diff-apply-")
879	if err != nil {
880		return errors.Wrap(err, "failed to create temp dir")
881	}
882	defer os.RemoveAll(td)
883	dest, err := ioutil.TempDir("", "test-diff-apply-dest-")
884	if err != nil {
885		return errors.Wrap(err, "failed to create temp dir")
886	}
887	defer os.RemoveAll(dest)
888
889	for _, a := range appliers {
890		if err := a.Apply(td); err != nil {
891			return errors.Wrap(err, "failed to apply filesystem changes")
892		}
893	}
894
895	// Apply base changes before diff
896	if len(appliers) > 1 {
897		for _, a := range appliers[:len(appliers)-1] {
898			if err := a.Apply(dest); err != nil {
899				return errors.Wrap(err, "failed to apply base filesystem changes")
900			}
901		}
902	}
903
904	diffBytes, err := ioutil.ReadAll(Diff(context.Background(), dest, td))
905	if err != nil {
906		return errors.Wrap(err, "failed to create diff")
907	}
908
909	if _, err := Apply(context.Background(), dest, bytes.NewReader(diffBytes)); err != nil {
910		return errors.Wrap(err, "failed to apply tar stream")
911	}
912
913	return fstest.CheckDirectoryEqual(td, dest)
914}
915
916func makeWriterToTarTest(wt tartest.WriterToTar, a fstest.Applier, validate func(string) error, applyErr error) func(*testing.T) {
917	return func(t *testing.T) {
918		td, err := ioutil.TempDir("", "test-writer-to-tar-")
919		if err != nil {
920			t.Fatalf("Failed to create temp dir: %v", err)
921		}
922		defer os.RemoveAll(td)
923
924		if a != nil {
925			if err := a.Apply(td); err != nil {
926				t.Fatalf("Failed to apply filesystem to directory: %v", err)
927			}
928		}
929
930		tr := tartest.TarFromWriterTo(wt)
931
932		if _, err := Apply(context.Background(), td, tr); err != nil {
933			if applyErr == nil {
934				t.Fatalf("Failed to apply tar: %v", err)
935			} else if !errors.Is(err, applyErr) {
936				t.Fatalf("Unexpected apply error: %v, expected %v", err, applyErr)
937			}
938			return
939		} else if applyErr != nil {
940			t.Fatalf("Expected apply error, got none: %v", applyErr)
941		}
942
943		if validate != nil {
944			if err := validate(td); err != nil {
945				t.Errorf("Validation failed: %v", err)
946			}
947
948		}
949
950	}
951}
952
953func TestDiffTar(t *testing.T) {
954	tests := []struct {
955		name       string
956		validators []tarEntryValidator
957		a          fstest.Applier
958		b          fstest.Applier
959	}{
960		{
961			name:       "EmptyDiff",
962			validators: []tarEntryValidator{},
963			a: fstest.Apply(
964				fstest.CreateDir("/etc/", 0755),
965			),
966			b: fstest.Apply(),
967		},
968		{
969			name: "ParentInclusion",
970			validators: []tarEntryValidator{
971				dirEntry("d1/", 0755),
972				dirEntry("d1/d/", 0700),
973				dirEntry("d2/", 0770),
974				fileEntry("d2/f", []byte("ok"), 0644),
975			},
976			a: fstest.Apply(
977				fstest.CreateDir("/d1/", 0755),
978				fstest.CreateDir("/d2/", 0770),
979			),
980			b: fstest.Apply(
981				fstest.CreateDir("/d1/d", 0700),
982				fstest.CreateFile("/d2/f", []byte("ok"), 0644),
983			),
984		},
985		{
986			name: "HardlinkParentInclusion",
987			validators: []tarEntryValidator{
988				dirEntry("d2/", 0755),
989				fileEntry("d2/l1", []byte("link me"), 0644),
990				// d1/f1 and its parent is included after the new link,
991				// before the new link was included, these files would
992				// not have been needed
993				dirEntry("d1/", 0755),
994				linkEntry("d1/f1", "d2/l1"),
995				dirEntry("d3/", 0755),
996				fileEntry("d3/l1", []byte("link me"), 0644),
997				dirEntry("d4/", 0755),
998				linkEntry("d4/f1", "d3/l1"),
999				dirEntry("d6/", 0755),
1000				whiteoutEntry("d6/l1"),
1001				whiteoutEntry("d6/l2"),
1002			},
1003			a: fstest.Apply(
1004				fstest.CreateDir("/d1/", 0755),
1005				fstest.CreateFile("/d1/f1", []byte("link me"), 0644),
1006				fstest.CreateDir("/d2/", 0755),
1007				fstest.CreateFile("/d2/f1", []byte("link me"), 0644),
1008				fstest.CreateDir("/d3/", 0755),
1009				fstest.CreateDir("/d4/", 0755),
1010				fstest.CreateFile("/d4/f1", []byte("link me"), 0644),
1011				fstest.CreateDir("/d5/", 0755),
1012				fstest.CreateFile("/d5/f1", []byte("link me"), 0644),
1013				fstest.CreateDir("/d6/", 0755),
1014				fstest.Link("/d1/f1", "/d6/l1"),
1015				fstest.Link("/d5/f1", "/d6/l2"),
1016			),
1017			b: fstest.Apply(
1018				fstest.Link("/d1/f1", "/d2/l1"),
1019				fstest.Link("/d4/f1", "/d3/l1"),
1020				fstest.Remove("/d6/l1"),
1021				fstest.Remove("/d6/l2"),
1022			),
1023		},
1024		{
1025			name: "UpdateDirectoryPermission",
1026			validators: []tarEntryValidator{
1027				dirEntry("d1/", 0777),
1028				dirEntry("d1/d/", 0700),
1029				dirEntry("d2/", 0770),
1030				fileEntry("d2/f", []byte("ok"), 0644),
1031			},
1032			a: fstest.Apply(
1033				fstest.CreateDir("/d1/", 0755),
1034				fstest.CreateDir("/d2/", 0770),
1035			),
1036			b: fstest.Apply(
1037				fstest.Chmod("/d1", 0777),
1038				fstest.CreateDir("/d1/d", 0700),
1039				fstest.CreateFile("/d2/f", []byte("ok"), 0644),
1040			),
1041		},
1042		{
1043			name: "HardlinkUpdatedParent",
1044			validators: []tarEntryValidator{
1045				dirEntry("d1/", 0777),
1046				dirEntry("d2/", 0755),
1047				fileEntry("d2/l1", []byte("link me"), 0644),
1048				// d1/f1 is included after the new link, its
1049				// parent has already changed and therefore
1050				// only the linked file is included
1051				linkEntry("d1/f1", "d2/l1"),
1052				dirEntry("d4/", 0777),
1053				fileEntry("d4/l1", []byte("link me"), 0644),
1054				dirEntry("d3/", 0755),
1055				linkEntry("d3/f1", "d4/l1"),
1056			},
1057			a: fstest.Apply(
1058				fstest.CreateDir("/d1/", 0755),
1059				fstest.CreateFile("/d1/f1", []byte("link me"), 0644),
1060				fstest.CreateDir("/d2/", 0755),
1061				fstest.CreateFile("/d2/f1", []byte("link me"), 0644),
1062				fstest.CreateDir("/d3/", 0755),
1063				fstest.CreateFile("/d3/f1", []byte("link me"), 0644),
1064				fstest.CreateDir("/d4/", 0755),
1065			),
1066			b: fstest.Apply(
1067				fstest.Chmod("/d1", 0777),
1068				fstest.Link("/d1/f1", "/d2/l1"),
1069				fstest.Chmod("/d4", 0777),
1070				fstest.Link("/d3/f1", "/d4/l1"),
1071			),
1072		},
1073		{
1074			name: "WhiteoutIncludesParents",
1075			validators: []tarEntryValidator{
1076				dirEntry("d1/", 0755),
1077				whiteoutEntry("d1/f1"),
1078				dirEntry("d2/", 0755),
1079				whiteoutEntry("d2/f1"),
1080				fileEntry("d2/f2", []byte("content"), 0777),
1081				dirEntry("d3/", 0755),
1082				whiteoutEntry("d3/f1"),
1083				fileEntry("d3/f2", []byte("content"), 0644),
1084				dirEntry("d4/", 0755),
1085				fileEntry("d4/f0", []byte("content"), 0644),
1086				whiteoutEntry("d4/f1"),
1087				whiteoutEntry("d5"),
1088			},
1089			a: fstest.Apply(
1090				fstest.CreateDir("/d1/", 0755),
1091				fstest.CreateFile("/d1/f1", []byte("content"), 0644),
1092				fstest.CreateDir("/d2/", 0755),
1093				fstest.CreateFile("/d2/f1", []byte("content"), 0644),
1094				fstest.CreateFile("/d2/f2", []byte("content"), 0644),
1095				fstest.CreateDir("/d3/", 0755),
1096				fstest.CreateFile("/d3/f1", []byte("content"), 0644),
1097				fstest.CreateDir("/d4/", 0755),
1098				fstest.CreateFile("/d4/f1", []byte("content"), 0644),
1099				fstest.CreateDir("/d5/", 0755),
1100				fstest.CreateFile("/d5/f1", []byte("content"), 0644),
1101			),
1102			b: fstest.Apply(
1103				fstest.Remove("/d1/f1"),
1104				fstest.Remove("/d2/f1"),
1105				fstest.Chmod("/d2/f2", 0777),
1106				fstest.Remove("/d3/f1"),
1107				fstest.CreateFile("/d3/f2", []byte("content"), 0644),
1108				fstest.Remove("/d4/f1"),
1109				fstest.CreateFile("/d4/f0", []byte("content"), 0644),
1110				fstest.RemoveAll("/d5"),
1111			),
1112		},
1113		{
1114			name: "WhiteoutParentRemoval",
1115			validators: []tarEntryValidator{
1116				whiteoutEntry("d1"),
1117				whiteoutEntry("d2"),
1118				dirEntry("d3/", 0755),
1119			},
1120			a: fstest.Apply(
1121				fstest.CreateDir("/d1/", 0755),
1122				fstest.CreateDir("/d2/", 0755),
1123				fstest.CreateFile("/d2/f1", []byte("content"), 0644),
1124			),
1125			b: fstest.Apply(
1126				fstest.RemoveAll("/d1"),
1127				fstest.RemoveAll("/d2"),
1128				fstest.CreateDir("/d3/", 0755),
1129			),
1130		},
1131		{
1132			name: "IgnoreSockets",
1133			validators: []tarEntryValidator{
1134				fileEntry("f2", []byte("content"), 0644),
1135				// There should be _no_ socket here, despite the fstest.CreateSocket below
1136				fileEntry("f3", []byte("content"), 0644),
1137			},
1138			a: fstest.Apply(
1139				fstest.CreateFile("/f1", []byte("content"), 0644),
1140			),
1141			b: fstest.Apply(
1142				fstest.CreateFile("/f2", []byte("content"), 0644),
1143				fstest.CreateSocket("/s0", 0644),
1144				fstest.CreateFile("/f3", []byte("content"), 0644),
1145			),
1146		},
1147	}
1148
1149	for _, at := range tests {
1150		t.Run(at.name, makeDiffTarTest(at.validators, at.a, at.b))
1151	}
1152}
1153
1154type tarEntryValidator func(*tar.Header, []byte) error
1155
1156func dirEntry(name string, mode int) tarEntryValidator {
1157	return func(hdr *tar.Header, b []byte) error {
1158		if hdr.Typeflag != tar.TypeDir {
1159			return errors.New("not directory type")
1160		}
1161		if hdr.Name != name {
1162			return errors.Errorf("wrong name %q, expected %q", hdr.Name, name)
1163		}
1164		if hdr.Mode != int64(mode) {
1165			return errors.Errorf("wrong mode %o, expected %o", hdr.Mode, mode)
1166		}
1167		return nil
1168	}
1169}
1170
1171func fileEntry(name string, expected []byte, mode int) tarEntryValidator {
1172	return func(hdr *tar.Header, b []byte) error {
1173		if hdr.Typeflag != tar.TypeReg {
1174			return errors.New("not file type")
1175		}
1176		if hdr.Name != name {
1177			return errors.Errorf("wrong name %q, expected %q", hdr.Name, name)
1178		}
1179		if hdr.Mode != int64(mode) {
1180			return errors.Errorf("wrong mode %o, expected %o", hdr.Mode, mode)
1181		}
1182		if !bytes.Equal(b, expected) {
1183			return errors.Errorf("different file content")
1184		}
1185		return nil
1186	}
1187}
1188
1189func linkEntry(name, link string) tarEntryValidator {
1190	return func(hdr *tar.Header, b []byte) error {
1191		if hdr.Typeflag != tar.TypeLink {
1192			return errors.New("not link type")
1193		}
1194		if hdr.Name != name {
1195			return errors.Errorf("wrong name %q, expected %q", hdr.Name, name)
1196		}
1197		if hdr.Linkname != link {
1198			return errors.Errorf("wrong link %q, expected %q", hdr.Linkname, link)
1199		}
1200		return nil
1201	}
1202}
1203
1204func whiteoutEntry(name string) tarEntryValidator {
1205	whiteOutDir := filepath.Dir(name)
1206	whiteOutBase := filepath.Base(name)
1207	whiteOut := filepath.Join(whiteOutDir, whiteoutPrefix+whiteOutBase)
1208
1209	return func(hdr *tar.Header, b []byte) error {
1210		if hdr.Typeflag != tar.TypeReg {
1211			return errors.Errorf("not file type: %q", hdr.Typeflag)
1212		}
1213		if hdr.Name != whiteOut {
1214			return errors.Errorf("wrong name %q, expected whiteout %q", hdr.Name, name)
1215		}
1216		return nil
1217	}
1218}
1219
1220func makeDiffTarTest(validators []tarEntryValidator, a, b fstest.Applier) func(*testing.T) {
1221	return func(t *testing.T) {
1222		ad, err := ioutil.TempDir("", "test-make-diff-tar-")
1223		if err != nil {
1224			t.Fatalf("failed to create temp dir: %v", err)
1225		}
1226		defer os.RemoveAll(ad)
1227		if err := a.Apply(ad); err != nil {
1228			t.Fatalf("failed to apply a: %v", err)
1229		}
1230
1231		bd, err := ioutil.TempDir("", "test-make-diff-tar-")
1232		if err != nil {
1233			t.Fatalf("failed to create temp dir: %v", err)
1234		}
1235		defer os.RemoveAll(bd)
1236		if err := fs.CopyDir(bd, ad); err != nil {
1237			t.Fatalf("failed to copy dir: %v", err)
1238		}
1239		if err := b.Apply(bd); err != nil {
1240			t.Fatalf("failed to apply b: %v", err)
1241		}
1242
1243		rc := Diff(context.Background(), ad, bd)
1244		defer rc.Close()
1245
1246		tr := tar.NewReader(rc)
1247		for i := 0; ; i++ {
1248			hdr, err := tr.Next()
1249			if err != nil {
1250				if err == io.EOF {
1251					break
1252				}
1253				t.Fatalf("tar read error: %v", err)
1254			}
1255			var b []byte
1256			if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 {
1257				b, err = ioutil.ReadAll(tr)
1258				if err != nil {
1259					t.Fatalf("tar read file error: %v", err)
1260				}
1261			}
1262			if i >= len(validators) {
1263				t.Fatal("no validator for entry")
1264			}
1265			if err := validators[i](hdr, b); err != nil {
1266				t.Fatalf("tar entry[%d] validation fail: %#v", i, err)
1267			}
1268		}
1269	}
1270}
1271
1272type diffApplier struct{}
1273
1274func (d diffApplier) TestContext(ctx context.Context) (context.Context, func(), error) {
1275	base, err := ioutil.TempDir("", "test-diff-apply-")
1276	if err != nil {
1277		return ctx, nil, errors.Wrap(err, "failed to create temp dir")
1278	}
1279	return context.WithValue(ctx, d, base), func() {
1280		os.RemoveAll(base)
1281	}, nil
1282}
1283
1284func (d diffApplier) Apply(ctx context.Context, a fstest.Applier) (string, func(), error) {
1285	base := ctx.Value(d).(string)
1286
1287	applyCopy, err := ioutil.TempDir("", "test-diffapply-apply-copy-")
1288	if err != nil {
1289		return "", nil, errors.Wrap(err, "failed to create temp dir")
1290	}
1291	defer os.RemoveAll(applyCopy)
1292	if err = fs.CopyDir(applyCopy, base); err != nil {
1293		return "", nil, errors.Wrap(err, "failed to copy base")
1294	}
1295	if err := a.Apply(applyCopy); err != nil {
1296		return "", nil, errors.Wrap(err, "failed to apply changes to copy of base")
1297	}
1298
1299	diffBytes, err := ioutil.ReadAll(Diff(ctx, base, applyCopy))
1300	if err != nil {
1301		return "", nil, errors.Wrap(err, "failed to create diff")
1302	}
1303
1304	if _, err = Apply(ctx, base, bytes.NewReader(diffBytes)); err != nil {
1305		return "", nil, errors.Wrap(err, "failed to apply tar stream")
1306	}
1307
1308	return base, nil, nil
1309}
1310
1311func readDirNames(p string) ([]string, error) {
1312	fis, err := ioutil.ReadDir(p)
1313	if err != nil {
1314		return nil, err
1315	}
1316	names := make([]string, len(fis))
1317	for i, fi := range fis {
1318		names[i] = fi.Name()
1319	}
1320	return names, nil
1321}
1322
1323func requireTar(t *testing.T) {
1324	if _, err := exec.LookPath(tarCmd); err != nil {
1325		t.Skipf("%s not found, skipping", tarCmd)
1326	}
1327}
1328