1package executor
2
3import (
4	"context"
5	"fmt"
6	"io/ioutil"
7	"os"
8	"path/filepath"
9	"regexp"
10	"strconv"
11	"strings"
12	"testing"
13	"time"
14
15	"github.com/hashicorp/nomad/client/allocdir"
16	"github.com/hashicorp/nomad/client/taskenv"
17	"github.com/hashicorp/nomad/client/testutil"
18	"github.com/hashicorp/nomad/drivers/shared/capabilities"
19	"github.com/hashicorp/nomad/helper/testlog"
20	"github.com/hashicorp/nomad/nomad/mock"
21	"github.com/hashicorp/nomad/plugins/drivers"
22	tu "github.com/hashicorp/nomad/testutil"
23	"github.com/opencontainers/runc/libcontainer/cgroups"
24	lconfigs "github.com/opencontainers/runc/libcontainer/configs"
25	"github.com/opencontainers/runc/libcontainer/devices"
26	"github.com/stretchr/testify/require"
27	"golang.org/x/sys/unix"
28)
29
30func init() {
31	executorFactories["LibcontainerExecutor"] = libcontainerFactory
32}
33
34var libcontainerFactory = executorFactory{
35	new: NewExecutorWithIsolation,
36	configureExecCmd: func(t *testing.T, cmd *ExecCommand) {
37		cmd.ResourceLimits = true
38		setupRootfs(t, cmd.TaskDir)
39	},
40}
41
42// testExecutorContextWithChroot returns an ExecutorContext and AllocDir with
43// chroot. Use testExecutorContext if you don't need a chroot.
44//
45// The caller is responsible for calling AllocDir.Destroy() to cleanup.
46func testExecutorCommandWithChroot(t *testing.T) *testExecCmd {
47	chrootEnv := map[string]string{
48		"/etc/ld.so.cache":  "/etc/ld.so.cache",
49		"/etc/ld.so.conf":   "/etc/ld.so.conf",
50		"/etc/ld.so.conf.d": "/etc/ld.so.conf.d",
51		"/etc/passwd":       "/etc/passwd",
52		"/lib":              "/lib",
53		"/lib64":            "/lib64",
54		"/usr/lib":          "/usr/lib",
55		"/bin/ls":           "/bin/ls",
56		"/bin/cat":          "/bin/cat",
57		"/bin/echo":         "/bin/echo",
58		"/bin/bash":         "/bin/bash",
59		"/bin/sleep":        "/bin/sleep",
60		"/foobar":           "/does/not/exist",
61	}
62
63	alloc := mock.Alloc()
64	task := alloc.Job.TaskGroups[0].Tasks[0]
65	taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, "global").Build()
66
67	allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), filepath.Join(os.TempDir(), alloc.ID))
68	if err := allocDir.Build(); err != nil {
69		t.Fatalf("AllocDir.Build() failed: %v", err)
70	}
71	if err := allocDir.NewTaskDir(task.Name).Build(true, chrootEnv); err != nil {
72		allocDir.Destroy()
73		t.Fatalf("allocDir.NewTaskDir(%q) failed: %v", task.Name, err)
74	}
75	td := allocDir.TaskDirs[task.Name]
76	cmd := &ExecCommand{
77		Env:     taskEnv.List(),
78		TaskDir: td.Dir,
79		Resources: &drivers.Resources{
80			NomadResources: alloc.AllocatedResources.Tasks[task.Name],
81		},
82	}
83
84	testCmd := &testExecCmd{
85		command:  cmd,
86		allocDir: allocDir,
87	}
88	configureTLogging(t, testCmd)
89	return testCmd
90}
91
92func TestExecutor_configureNamespaces(t *testing.T) {
93	t.Run("host host", func(t *testing.T) {
94		require.Equal(t, lconfigs.Namespaces{
95			{Type: lconfigs.NEWNS},
96		}, configureNamespaces("host", "host"))
97	})
98
99	t.Run("host private", func(t *testing.T) {
100		require.Equal(t, lconfigs.Namespaces{
101			{Type: lconfigs.NEWNS},
102			{Type: lconfigs.NEWIPC},
103		}, configureNamespaces("host", "private"))
104	})
105
106	t.Run("private host", func(t *testing.T) {
107		require.Equal(t, lconfigs.Namespaces{
108			{Type: lconfigs.NEWNS},
109			{Type: lconfigs.NEWPID},
110		}, configureNamespaces("private", "host"))
111	})
112
113	t.Run("private private", func(t *testing.T) {
114		require.Equal(t, lconfigs.Namespaces{
115			{Type: lconfigs.NEWNS},
116			{Type: lconfigs.NEWPID},
117			{Type: lconfigs.NEWIPC},
118		}, configureNamespaces("private", "private"))
119	})
120}
121
122func TestExecutor_Isolation_PID_and_IPC_hostMode(t *testing.T) {
123	t.Parallel()
124	r := require.New(t)
125	testutil.ExecCompatible(t)
126
127	testExecCmd := testExecutorCommandWithChroot(t)
128	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
129	execCmd.Cmd = "/bin/ls"
130	execCmd.Args = []string{"-F", "/", "/etc/"}
131	defer allocDir.Destroy()
132
133	execCmd.ResourceLimits = true
134	execCmd.ModePID = "host" // disable PID namespace
135	execCmd.ModeIPC = "host" // disable IPC namespace
136
137	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
138	defer executor.Shutdown("SIGKILL", 0)
139
140	ps, err := executor.Launch(execCmd)
141	r.NoError(err)
142	r.NotZero(ps.Pid)
143
144	estate, err := executor.Wait(context.Background())
145	r.NoError(err)
146	r.Zero(estate.ExitCode)
147
148	lexec, ok := executor.(*LibcontainerExecutor)
149	r.True(ok)
150
151	// Check that namespaces were applied to the container config
152	config := lexec.container.Config()
153
154	r.Contains(config.Namespaces, lconfigs.Namespace{Type: lconfigs.NEWNS})
155	r.NotContains(config.Namespaces, lconfigs.Namespace{Type: lconfigs.NEWPID})
156	r.NotContains(config.Namespaces, lconfigs.Namespace{Type: lconfigs.NEWIPC})
157
158	// Shut down executor
159	r.NoError(executor.Shutdown("", 0))
160	executor.Wait(context.Background())
161}
162
163func TestExecutor_IsolationAndConstraints(t *testing.T) {
164	t.Parallel()
165	r := require.New(t)
166	testutil.ExecCompatible(t)
167
168	testExecCmd := testExecutorCommandWithChroot(t)
169	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
170	execCmd.Cmd = "/bin/ls"
171	execCmd.Args = []string{"-F", "/", "/etc/"}
172	defer allocDir.Destroy()
173
174	execCmd.ResourceLimits = true
175	execCmd.ModePID = "private"
176	execCmd.ModeIPC = "private"
177
178	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
179	defer executor.Shutdown("SIGKILL", 0)
180
181	ps, err := executor.Launch(execCmd)
182	r.NoError(err)
183	r.NotZero(ps.Pid)
184
185	estate, err := executor.Wait(context.Background())
186	r.NoError(err)
187	r.Zero(estate.ExitCode)
188
189	lexec, ok := executor.(*LibcontainerExecutor)
190	r.True(ok)
191
192	// Check if the resource constraints were applied
193	state, err := lexec.container.State()
194	r.NoError(err)
195
196	memLimits := filepath.Join(state.CgroupPaths["memory"], "memory.limit_in_bytes")
197	data, err := ioutil.ReadFile(memLimits)
198	r.NoError(err)
199
200	expectedMemLim := strconv.Itoa(int(execCmd.Resources.NomadResources.Memory.MemoryMB * 1024 * 1024))
201	actualMemLim := strings.TrimSpace(string(data))
202	r.Equal(actualMemLim, expectedMemLim)
203
204	// Check that namespaces were applied to the container config
205	config := lexec.container.Config()
206
207	r.Contains(config.Namespaces, lconfigs.Namespace{Type: lconfigs.NEWNS})
208	r.Contains(config.Namespaces, lconfigs.Namespace{Type: lconfigs.NEWPID})
209	r.Contains(config.Namespaces, lconfigs.Namespace{Type: lconfigs.NEWIPC})
210
211	// Shut down executor
212	r.NoError(executor.Shutdown("", 0))
213	executor.Wait(context.Background())
214
215	// Check if Nomad has actually removed the cgroups
216	tu.WaitForResult(func() (bool, error) {
217		_, err = os.Stat(memLimits)
218		if err == nil {
219			return false, fmt.Errorf("expected an error from os.Stat %s", memLimits)
220		}
221		return true, nil
222	}, func(err error) { t.Error(err) })
223
224	expected := `/:
225alloc/
226bin/
227dev/
228etc/
229lib/
230lib64/
231local/
232proc/
233secrets/
234sys/
235tmp/
236usr/
237
238/etc/:
239ld.so.cache
240ld.so.conf
241ld.so.conf.d/
242passwd`
243	tu.WaitForResult(func() (bool, error) {
244		output := testExecCmd.stdout.String()
245		act := strings.TrimSpace(string(output))
246		if act != expected {
247			return false, fmt.Errorf("Command output incorrectly: want %v; got %v", expected, act)
248		}
249		return true, nil
250	}, func(err error) { t.Error(err) })
251}
252
253// TestExecutor_CgroupPaths asserts that process starts with independent cgroups
254// hierarchy created for this process
255func TestExecutor_CgroupPaths(t *testing.T) {
256	t.Parallel()
257	require := require.New(t)
258	testutil.ExecCompatible(t)
259
260	testExecCmd := testExecutorCommandWithChroot(t)
261	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
262	execCmd.Cmd = "/bin/bash"
263	execCmd.Args = []string{"-c", "sleep 0.2; cat /proc/self/cgroup"}
264	defer allocDir.Destroy()
265
266	execCmd.ResourceLimits = true
267
268	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
269	defer executor.Shutdown("SIGKILL", 0)
270
271	ps, err := executor.Launch(execCmd)
272	require.NoError(err)
273	require.NotZero(ps.Pid)
274
275	state, err := executor.Wait(context.Background())
276	require.NoError(err)
277	require.Zero(state.ExitCode)
278
279	tu.WaitForResult(func() (bool, error) {
280		output := strings.TrimSpace(testExecCmd.stdout.String())
281		// Verify that we got some cgroups
282		if !strings.Contains(output, ":devices:") {
283			return false, fmt.Errorf("was expected cgroup files but found:\n%v", output)
284		}
285		lines := strings.Split(output, "\n")
286		for _, line := range lines {
287			// Every cgroup entry should be /nomad/$ALLOC_ID
288			if line == "" {
289				continue
290			}
291
292			// Skip rdma subsystem; rdma was added in most recent kernels and libcontainer/docker
293			// don't isolate it by default.
294			// :: filters out odd empty cgroup found in latest Ubuntu lines, e.g. 0::/user.slice/user-1000.slice/session-17.scope
295			// that is also not used for isolation
296			if strings.Contains(line, ":rdma:") || strings.Contains(line, "::") {
297				continue
298			}
299
300			if !strings.Contains(line, ":/nomad/") {
301				return false, fmt.Errorf("Not a member of the alloc's cgroup: expected=...:/nomad/... -- found=%q", line)
302			}
303		}
304		return true, nil
305	}, func(err error) { t.Error(err) })
306}
307
308// TestExecutor_CgroupPaths asserts that all cgroups created for a task
309// are destroyed on shutdown
310func TestExecutor_CgroupPathsAreDestroyed(t *testing.T) {
311	t.Parallel()
312	require := require.New(t)
313	testutil.ExecCompatible(t)
314
315	testExecCmd := testExecutorCommandWithChroot(t)
316	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
317	execCmd.Cmd = "/bin/bash"
318	execCmd.Args = []string{"-c", "sleep 0.2; cat /proc/self/cgroup"}
319	defer allocDir.Destroy()
320
321	execCmd.ResourceLimits = true
322
323	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
324	defer executor.Shutdown("SIGKILL", 0)
325
326	ps, err := executor.Launch(execCmd)
327	require.NoError(err)
328	require.NotZero(ps.Pid)
329
330	state, err := executor.Wait(context.Background())
331	require.NoError(err)
332	require.Zero(state.ExitCode)
333
334	var cgroupsPaths string
335	tu.WaitForResult(func() (bool, error) {
336		output := strings.TrimSpace(testExecCmd.stdout.String())
337		// Verify that we got some cgroups
338		if !strings.Contains(output, ":devices:") {
339			return false, fmt.Errorf("was expected cgroup files but found:\n%v", output)
340		}
341		lines := strings.Split(output, "\n")
342		for _, line := range lines {
343			// Every cgroup entry should be /nomad/$ALLOC_ID
344			if line == "" {
345				continue
346			}
347
348			// Skip rdma subsystem; rdma was added in most recent kernels and libcontainer/docker
349			// don't isolate it by default.
350			if strings.Contains(line, ":rdma:") || strings.Contains(line, "::") {
351				continue
352			}
353
354			if !strings.Contains(line, ":/nomad/") {
355				return false, fmt.Errorf("Not a member of the alloc's cgroup: expected=...:/nomad/... -- found=%q", line)
356			}
357		}
358
359		cgroupsPaths = output
360		return true, nil
361	}, func(err error) { t.Error(err) })
362
363	// shutdown executor and test that cgroups are destroyed
364	executor.Shutdown("SIGKILL", 0)
365
366	// test that the cgroup paths are not visible
367	tmpFile, err := ioutil.TempFile("", "")
368	require.NoError(err)
369	defer os.Remove(tmpFile.Name())
370
371	_, err = tmpFile.WriteString(cgroupsPaths)
372	require.NoError(err)
373	tmpFile.Close()
374
375	subsystems, err := cgroups.ParseCgroupFile(tmpFile.Name())
376	require.NoError(err)
377
378	for subsystem, cgroup := range subsystems {
379		if !strings.Contains(cgroup, "nomad/") {
380			// this should only be rdma at this point
381			continue
382		}
383
384		p, err := getCgroupPathHelper(subsystem, cgroup)
385		require.NoError(err)
386		require.Falsef(cgroups.PathExists(p), "cgroup for %s %s still exists", subsystem, cgroup)
387	}
388}
389
390func TestUniversalExecutor_LookupTaskBin(t *testing.T) {
391	t.Parallel()
392	require := require.New(t)
393
394	// Create a temp dir
395	tmpDir, err := ioutil.TempDir("", "")
396	require.Nil(err)
397	defer os.Remove(tmpDir)
398
399	// Create the command
400	cmd := &ExecCommand{Env: []string{"PATH=/bin"}, TaskDir: tmpDir}
401
402	// Make a foo subdir
403	os.MkdirAll(filepath.Join(tmpDir, "foo"), 0700)
404
405	// Write a file under foo
406	filePath := filepath.Join(tmpDir, "foo", "tmp.txt")
407	err = ioutil.WriteFile(filePath, []byte{1, 2}, os.ModeAppend)
408	require.NoError(err)
409
410	// Lookout with an absolute path to the binary
411	cmd.Cmd = "/foo/tmp.txt"
412	_, err = lookupTaskBin(cmd)
413	require.NoError(err)
414
415	// Write a file under local subdir
416	os.MkdirAll(filepath.Join(tmpDir, "local"), 0700)
417	filePath2 := filepath.Join(tmpDir, "local", "tmp.txt")
418	ioutil.WriteFile(filePath2, []byte{1, 2}, os.ModeAppend)
419
420	// Lookup with file name, should find the one we wrote above
421	cmd.Cmd = "tmp.txt"
422	_, err = lookupTaskBin(cmd)
423	require.NoError(err)
424
425	// Lookup a host absolute path
426	cmd.Cmd = "/bin/sh"
427	_, err = lookupTaskBin(cmd)
428	require.Error(err)
429}
430
431// Exec Launch looks for the binary only inside the chroot
432func TestExecutor_EscapeContainer(t *testing.T) {
433	t.Parallel()
434	require := require.New(t)
435	testutil.ExecCompatible(t)
436
437	testExecCmd := testExecutorCommandWithChroot(t)
438	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
439	execCmd.Cmd = "/bin/kill" // missing from the chroot container
440	defer allocDir.Destroy()
441
442	execCmd.ResourceLimits = true
443
444	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
445	defer executor.Shutdown("SIGKILL", 0)
446
447	_, err := executor.Launch(execCmd)
448	require.Error(err)
449	require.Regexp("^file /bin/kill not found under path", err)
450
451	// Bare files are looked up using the system path, inside the container
452	allocDir.Destroy()
453	testExecCmd = testExecutorCommandWithChroot(t)
454	execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir
455	execCmd.Cmd = "kill"
456	_, err = executor.Launch(execCmd)
457	require.Error(err)
458	require.Regexp("^file kill not found under path", err)
459
460	allocDir.Destroy()
461	testExecCmd = testExecutorCommandWithChroot(t)
462	execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir
463	execCmd.Cmd = "echo"
464	_, err = executor.Launch(execCmd)
465	require.NoError(err)
466}
467
468func TestExecutor_Capabilities(t *testing.T) {
469	t.Parallel()
470	testutil.ExecCompatible(t)
471
472	cases := []struct {
473		user string
474		caps string
475	}{
476		{
477			user: "nobody",
478			caps: `
479CapInh: 0000000000000000
480CapPrm: 0000000000000000
481CapEff: 0000000000000000
482CapBnd: 00000000a80405fb
483CapAmb: 0000000000000000`,
484		},
485		{
486			user: "root",
487			caps: `
488CapInh: 0000000000000000
489CapPrm: 0000003fffffffff
490CapEff: 0000003fffffffff
491CapBnd: 0000003fffffffff
492CapAmb: 0000000000000000`,
493		},
494	}
495
496	for _, c := range cases {
497		t.Run(c.user, func(t *testing.T) {
498
499			testExecCmd := testExecutorCommandWithChroot(t)
500			execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
501			defer allocDir.Destroy()
502
503			execCmd.User = c.user
504			execCmd.ResourceLimits = true
505			execCmd.Cmd = "/bin/bash"
506			execCmd.Args = []string{"-c", "cat /proc/$$/status"}
507			execCmd.Capabilities = capabilities.NomadDefaults().Slice(true)
508
509			executor := NewExecutorWithIsolation(testlog.HCLogger(t))
510			defer executor.Shutdown("SIGKILL", 0)
511
512			_, err := executor.Launch(execCmd)
513			require.NoError(t, err)
514
515			ch := make(chan interface{})
516			go func() {
517				executor.Wait(context.Background())
518				close(ch)
519			}()
520
521			select {
522			case <-ch:
523				// all good
524			case <-time.After(5 * time.Second):
525				require.Fail(t, "timeout waiting for exec to shutdown")
526			}
527
528			canonical := func(s string) string {
529				s = strings.TrimSpace(s)
530				s = regexp.MustCompile("[ \t]+").ReplaceAllString(s, " ")
531				s = regexp.MustCompile("[\n\r]+").ReplaceAllString(s, "\n")
532				return s
533			}
534
535			expected := canonical(c.caps)
536			tu.WaitForResult(func() (bool, error) {
537				output := canonical(testExecCmd.stdout.String())
538				if !strings.Contains(output, expected) {
539					return false, fmt.Errorf("capabilities didn't match: want\n%v\n; got:\n%v\n", expected, output)
540				}
541				return true, nil
542			}, func(err error) { require.NoError(t, err) })
543		})
544	}
545
546}
547
548func TestExecutor_ClientCleanup(t *testing.T) {
549	t.Parallel()
550	testutil.ExecCompatible(t)
551	require := require.New(t)
552
553	testExecCmd := testExecutorCommandWithChroot(t)
554	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
555	defer allocDir.Destroy()
556
557	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
558	defer executor.Shutdown("", 0)
559
560	// Need to run a command which will produce continuous output but not
561	// too quickly to ensure executor.Exit() stops the process.
562	execCmd.Cmd = "/bin/bash"
563	execCmd.Args = []string{"-c", "while true; do /bin/echo X; /bin/sleep 1; done"}
564	execCmd.ResourceLimits = true
565
566	ps, err := executor.Launch(execCmd)
567
568	require.NoError(err)
569	require.NotZero(ps.Pid)
570	time.Sleep(500 * time.Millisecond)
571	require.NoError(executor.Shutdown("SIGINT", 100*time.Millisecond))
572
573	ch := make(chan interface{})
574	go func() {
575		executor.Wait(context.Background())
576		close(ch)
577	}()
578
579	select {
580	case <-ch:
581		// all good
582	case <-time.After(5 * time.Second):
583		require.Fail("timeout waiting for exec to shutdown")
584	}
585
586	output := testExecCmd.stdout.String()
587	require.NotZero(len(output))
588	time.Sleep(2 * time.Second)
589	output1 := testExecCmd.stdout.String()
590	require.Equal(len(output), len(output1))
591}
592
593func TestExecutor_cmdDevices(t *testing.T) {
594	input := []*drivers.DeviceConfig{
595		{
596			HostPath:    "/dev/null",
597			TaskPath:    "/task/dev/null",
598			Permissions: "rwm",
599		},
600	}
601
602	expected := &devices.Device{
603		Rule: devices.Rule{
604			Type:        99,
605			Major:       1,
606			Minor:       3,
607			Permissions: "rwm",
608		},
609		Path: "/task/dev/null",
610	}
611
612	found, err := cmdDevices(input)
613	require.NoError(t, err)
614	require.Len(t, found, 1)
615
616	// ignore file permission and ownership
617	// as they are host specific potentially
618	d := found[0]
619	d.FileMode = 0
620	d.Uid = 0
621	d.Gid = 0
622
623	require.EqualValues(t, expected, d)
624}
625
626func TestExecutor_cmdMounts(t *testing.T) {
627	input := []*drivers.MountConfig{
628		{
629			HostPath: "/host/path-ro",
630			TaskPath: "/task/path-ro",
631			Readonly: true,
632		},
633		{
634			HostPath: "/host/path-rw",
635			TaskPath: "/task/path-rw",
636			Readonly: false,
637		},
638	}
639
640	expected := []*lconfigs.Mount{
641		{
642			Source:           "/host/path-ro",
643			Destination:      "/task/path-ro",
644			Flags:            unix.MS_BIND | unix.MS_RDONLY,
645			Device:           "bind",
646			PropagationFlags: []int{unix.MS_PRIVATE | unix.MS_REC},
647		},
648		{
649			Source:           "/host/path-rw",
650			Destination:      "/task/path-rw",
651			Flags:            unix.MS_BIND,
652			Device:           "bind",
653			PropagationFlags: []int{unix.MS_PRIVATE | unix.MS_REC},
654		},
655	}
656
657	require.EqualValues(t, expected, cmdMounts(input))
658}
659
660// TestUniversalExecutor_NoCgroup asserts that commands are executed in the
661// same cgroup as parent process
662func TestUniversalExecutor_NoCgroup(t *testing.T) {
663	t.Parallel()
664	testutil.ExecCompatible(t)
665
666	expectedBytes, err := ioutil.ReadFile("/proc/self/cgroup")
667	require.NoError(t, err)
668
669	expected := strings.TrimSpace(string(expectedBytes))
670
671	testExecCmd := testExecutorCommand(t)
672	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
673	execCmd.Cmd = "/bin/cat"
674	execCmd.Args = []string{"/proc/self/cgroup"}
675	defer allocDir.Destroy()
676
677	execCmd.BasicProcessCgroup = false
678	execCmd.ResourceLimits = false
679
680	executor := NewExecutor(testlog.HCLogger(t))
681	defer executor.Shutdown("SIGKILL", 0)
682
683	_, err = executor.Launch(execCmd)
684	require.NoError(t, err)
685
686	_, err = executor.Wait(context.Background())
687	require.NoError(t, err)
688
689	tu.WaitForResult(func() (bool, error) {
690		act := strings.TrimSpace(string(testExecCmd.stdout.String()))
691		if expected != act {
692			return false, fmt.Errorf("expected:\n%s actual:\n%s", expected, act)
693		}
694		return true, nil
695	}, func(err error) {
696		stderr := strings.TrimSpace(string(testExecCmd.stderr.String()))
697		t.Logf("stderr: %v", stderr)
698		require.NoError(t, err)
699	})
700
701}
702