1package executor 2 3import ( 4 "context" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "path/filepath" 9 "regexp" 10 "strconv" 11 "strings" 12 "testing" 13 "time" 14 15 "github.com/hashicorp/nomad/client/allocdir" 16 "github.com/hashicorp/nomad/client/taskenv" 17 "github.com/hashicorp/nomad/client/testutil" 18 "github.com/hashicorp/nomad/drivers/shared/capabilities" 19 "github.com/hashicorp/nomad/helper/testlog" 20 "github.com/hashicorp/nomad/nomad/mock" 21 "github.com/hashicorp/nomad/plugins/drivers" 22 tu "github.com/hashicorp/nomad/testutil" 23 "github.com/opencontainers/runc/libcontainer/cgroups" 24 lconfigs "github.com/opencontainers/runc/libcontainer/configs" 25 "github.com/opencontainers/runc/libcontainer/devices" 26 "github.com/stretchr/testify/require" 27 "golang.org/x/sys/unix" 28) 29 30func init() { 31 executorFactories["LibcontainerExecutor"] = libcontainerFactory 32} 33 34var libcontainerFactory = executorFactory{ 35 new: NewExecutorWithIsolation, 36 configureExecCmd: func(t *testing.T, cmd *ExecCommand) { 37 cmd.ResourceLimits = true 38 setupRootfs(t, cmd.TaskDir) 39 }, 40} 41 42// testExecutorContextWithChroot returns an ExecutorContext and AllocDir with 43// chroot. Use testExecutorContext if you don't need a chroot. 44// 45// The caller is responsible for calling AllocDir.Destroy() to cleanup. 46func testExecutorCommandWithChroot(t *testing.T) *testExecCmd { 47 chrootEnv := map[string]string{ 48 "/etc/ld.so.cache": "/etc/ld.so.cache", 49 "/etc/ld.so.conf": "/etc/ld.so.conf", 50 "/etc/ld.so.conf.d": "/etc/ld.so.conf.d", 51 "/etc/passwd": "/etc/passwd", 52 "/lib": "/lib", 53 "/lib64": "/lib64", 54 "/usr/lib": "/usr/lib", 55 "/bin/ls": "/bin/ls", 56 "/bin/cat": "/bin/cat", 57 "/bin/echo": "/bin/echo", 58 "/bin/bash": "/bin/bash", 59 "/bin/sleep": "/bin/sleep", 60 "/foobar": "/does/not/exist", 61 } 62 63 alloc := mock.Alloc() 64 task := alloc.Job.TaskGroups[0].Tasks[0] 65 taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, "global").Build() 66 67 allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), filepath.Join(os.TempDir(), alloc.ID)) 68 if err := allocDir.Build(); err != nil { 69 t.Fatalf("AllocDir.Build() failed: %v", err) 70 } 71 if err := allocDir.NewTaskDir(task.Name).Build(true, chrootEnv); err != nil { 72 allocDir.Destroy() 73 t.Fatalf("allocDir.NewTaskDir(%q) failed: %v", task.Name, err) 74 } 75 td := allocDir.TaskDirs[task.Name] 76 cmd := &ExecCommand{ 77 Env: taskEnv.List(), 78 TaskDir: td.Dir, 79 Resources: &drivers.Resources{ 80 NomadResources: alloc.AllocatedResources.Tasks[task.Name], 81 }, 82 } 83 84 testCmd := &testExecCmd{ 85 command: cmd, 86 allocDir: allocDir, 87 } 88 configureTLogging(t, testCmd) 89 return testCmd 90} 91 92func TestExecutor_configureNamespaces(t *testing.T) { 93 t.Run("host host", func(t *testing.T) { 94 require.Equal(t, lconfigs.Namespaces{ 95 {Type: lconfigs.NEWNS}, 96 }, configureNamespaces("host", "host")) 97 }) 98 99 t.Run("host private", func(t *testing.T) { 100 require.Equal(t, lconfigs.Namespaces{ 101 {Type: lconfigs.NEWNS}, 102 {Type: lconfigs.NEWIPC}, 103 }, configureNamespaces("host", "private")) 104 }) 105 106 t.Run("private host", func(t *testing.T) { 107 require.Equal(t, lconfigs.Namespaces{ 108 {Type: lconfigs.NEWNS}, 109 {Type: lconfigs.NEWPID}, 110 }, configureNamespaces("private", "host")) 111 }) 112 113 t.Run("private private", func(t *testing.T) { 114 require.Equal(t, lconfigs.Namespaces{ 115 {Type: lconfigs.NEWNS}, 116 {Type: lconfigs.NEWPID}, 117 {Type: lconfigs.NEWIPC}, 118 }, configureNamespaces("private", "private")) 119 }) 120} 121 122func TestExecutor_Isolation_PID_and_IPC_hostMode(t *testing.T) { 123 t.Parallel() 124 r := require.New(t) 125 testutil.ExecCompatible(t) 126 127 testExecCmd := testExecutorCommandWithChroot(t) 128 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 129 execCmd.Cmd = "/bin/ls" 130 execCmd.Args = []string{"-F", "/", "/etc/"} 131 defer allocDir.Destroy() 132 133 execCmd.ResourceLimits = true 134 execCmd.ModePID = "host" // disable PID namespace 135 execCmd.ModeIPC = "host" // disable IPC namespace 136 137 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 138 defer executor.Shutdown("SIGKILL", 0) 139 140 ps, err := executor.Launch(execCmd) 141 r.NoError(err) 142 r.NotZero(ps.Pid) 143 144 estate, err := executor.Wait(context.Background()) 145 r.NoError(err) 146 r.Zero(estate.ExitCode) 147 148 lexec, ok := executor.(*LibcontainerExecutor) 149 r.True(ok) 150 151 // Check that namespaces were applied to the container config 152 config := lexec.container.Config() 153 154 r.Contains(config.Namespaces, lconfigs.Namespace{Type: lconfigs.NEWNS}) 155 r.NotContains(config.Namespaces, lconfigs.Namespace{Type: lconfigs.NEWPID}) 156 r.NotContains(config.Namespaces, lconfigs.Namespace{Type: lconfigs.NEWIPC}) 157 158 // Shut down executor 159 r.NoError(executor.Shutdown("", 0)) 160 executor.Wait(context.Background()) 161} 162 163func TestExecutor_IsolationAndConstraints(t *testing.T) { 164 t.Parallel() 165 r := require.New(t) 166 testutil.ExecCompatible(t) 167 168 testExecCmd := testExecutorCommandWithChroot(t) 169 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 170 execCmd.Cmd = "/bin/ls" 171 execCmd.Args = []string{"-F", "/", "/etc/"} 172 defer allocDir.Destroy() 173 174 execCmd.ResourceLimits = true 175 execCmd.ModePID = "private" 176 execCmd.ModeIPC = "private" 177 178 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 179 defer executor.Shutdown("SIGKILL", 0) 180 181 ps, err := executor.Launch(execCmd) 182 r.NoError(err) 183 r.NotZero(ps.Pid) 184 185 estate, err := executor.Wait(context.Background()) 186 r.NoError(err) 187 r.Zero(estate.ExitCode) 188 189 lexec, ok := executor.(*LibcontainerExecutor) 190 r.True(ok) 191 192 // Check if the resource constraints were applied 193 state, err := lexec.container.State() 194 r.NoError(err) 195 196 memLimits := filepath.Join(state.CgroupPaths["memory"], "memory.limit_in_bytes") 197 data, err := ioutil.ReadFile(memLimits) 198 r.NoError(err) 199 200 expectedMemLim := strconv.Itoa(int(execCmd.Resources.NomadResources.Memory.MemoryMB * 1024 * 1024)) 201 actualMemLim := strings.TrimSpace(string(data)) 202 r.Equal(actualMemLim, expectedMemLim) 203 204 // Check that namespaces were applied to the container config 205 config := lexec.container.Config() 206 207 r.Contains(config.Namespaces, lconfigs.Namespace{Type: lconfigs.NEWNS}) 208 r.Contains(config.Namespaces, lconfigs.Namespace{Type: lconfigs.NEWPID}) 209 r.Contains(config.Namespaces, lconfigs.Namespace{Type: lconfigs.NEWIPC}) 210 211 // Shut down executor 212 r.NoError(executor.Shutdown("", 0)) 213 executor.Wait(context.Background()) 214 215 // Check if Nomad has actually removed the cgroups 216 tu.WaitForResult(func() (bool, error) { 217 _, err = os.Stat(memLimits) 218 if err == nil { 219 return false, fmt.Errorf("expected an error from os.Stat %s", memLimits) 220 } 221 return true, nil 222 }, func(err error) { t.Error(err) }) 223 224 expected := `/: 225alloc/ 226bin/ 227dev/ 228etc/ 229lib/ 230lib64/ 231local/ 232proc/ 233secrets/ 234sys/ 235tmp/ 236usr/ 237 238/etc/: 239ld.so.cache 240ld.so.conf 241ld.so.conf.d/ 242passwd` 243 tu.WaitForResult(func() (bool, error) { 244 output := testExecCmd.stdout.String() 245 act := strings.TrimSpace(string(output)) 246 if act != expected { 247 return false, fmt.Errorf("Command output incorrectly: want %v; got %v", expected, act) 248 } 249 return true, nil 250 }, func(err error) { t.Error(err) }) 251} 252 253// TestExecutor_CgroupPaths asserts that process starts with independent cgroups 254// hierarchy created for this process 255func TestExecutor_CgroupPaths(t *testing.T) { 256 t.Parallel() 257 require := require.New(t) 258 testutil.ExecCompatible(t) 259 260 testExecCmd := testExecutorCommandWithChroot(t) 261 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 262 execCmd.Cmd = "/bin/bash" 263 execCmd.Args = []string{"-c", "sleep 0.2; cat /proc/self/cgroup"} 264 defer allocDir.Destroy() 265 266 execCmd.ResourceLimits = true 267 268 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 269 defer executor.Shutdown("SIGKILL", 0) 270 271 ps, err := executor.Launch(execCmd) 272 require.NoError(err) 273 require.NotZero(ps.Pid) 274 275 state, err := executor.Wait(context.Background()) 276 require.NoError(err) 277 require.Zero(state.ExitCode) 278 279 tu.WaitForResult(func() (bool, error) { 280 output := strings.TrimSpace(testExecCmd.stdout.String()) 281 // Verify that we got some cgroups 282 if !strings.Contains(output, ":devices:") { 283 return false, fmt.Errorf("was expected cgroup files but found:\n%v", output) 284 } 285 lines := strings.Split(output, "\n") 286 for _, line := range lines { 287 // Every cgroup entry should be /nomad/$ALLOC_ID 288 if line == "" { 289 continue 290 } 291 292 // Skip rdma subsystem; rdma was added in most recent kernels and libcontainer/docker 293 // don't isolate it by default. 294 // :: filters out odd empty cgroup found in latest Ubuntu lines, e.g. 0::/user.slice/user-1000.slice/session-17.scope 295 // that is also not used for isolation 296 if strings.Contains(line, ":rdma:") || strings.Contains(line, "::") { 297 continue 298 } 299 300 if !strings.Contains(line, ":/nomad/") { 301 return false, fmt.Errorf("Not a member of the alloc's cgroup: expected=...:/nomad/... -- found=%q", line) 302 } 303 } 304 return true, nil 305 }, func(err error) { t.Error(err) }) 306} 307 308// TestExecutor_CgroupPaths asserts that all cgroups created for a task 309// are destroyed on shutdown 310func TestExecutor_CgroupPathsAreDestroyed(t *testing.T) { 311 t.Parallel() 312 require := require.New(t) 313 testutil.ExecCompatible(t) 314 315 testExecCmd := testExecutorCommandWithChroot(t) 316 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 317 execCmd.Cmd = "/bin/bash" 318 execCmd.Args = []string{"-c", "sleep 0.2; cat /proc/self/cgroup"} 319 defer allocDir.Destroy() 320 321 execCmd.ResourceLimits = true 322 323 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 324 defer executor.Shutdown("SIGKILL", 0) 325 326 ps, err := executor.Launch(execCmd) 327 require.NoError(err) 328 require.NotZero(ps.Pid) 329 330 state, err := executor.Wait(context.Background()) 331 require.NoError(err) 332 require.Zero(state.ExitCode) 333 334 var cgroupsPaths string 335 tu.WaitForResult(func() (bool, error) { 336 output := strings.TrimSpace(testExecCmd.stdout.String()) 337 // Verify that we got some cgroups 338 if !strings.Contains(output, ":devices:") { 339 return false, fmt.Errorf("was expected cgroup files but found:\n%v", output) 340 } 341 lines := strings.Split(output, "\n") 342 for _, line := range lines { 343 // Every cgroup entry should be /nomad/$ALLOC_ID 344 if line == "" { 345 continue 346 } 347 348 // Skip rdma subsystem; rdma was added in most recent kernels and libcontainer/docker 349 // don't isolate it by default. 350 if strings.Contains(line, ":rdma:") || strings.Contains(line, "::") { 351 continue 352 } 353 354 if !strings.Contains(line, ":/nomad/") { 355 return false, fmt.Errorf("Not a member of the alloc's cgroup: expected=...:/nomad/... -- found=%q", line) 356 } 357 } 358 359 cgroupsPaths = output 360 return true, nil 361 }, func(err error) { t.Error(err) }) 362 363 // shutdown executor and test that cgroups are destroyed 364 executor.Shutdown("SIGKILL", 0) 365 366 // test that the cgroup paths are not visible 367 tmpFile, err := ioutil.TempFile("", "") 368 require.NoError(err) 369 defer os.Remove(tmpFile.Name()) 370 371 _, err = tmpFile.WriteString(cgroupsPaths) 372 require.NoError(err) 373 tmpFile.Close() 374 375 subsystems, err := cgroups.ParseCgroupFile(tmpFile.Name()) 376 require.NoError(err) 377 378 for subsystem, cgroup := range subsystems { 379 if !strings.Contains(cgroup, "nomad/") { 380 // this should only be rdma at this point 381 continue 382 } 383 384 p, err := getCgroupPathHelper(subsystem, cgroup) 385 require.NoError(err) 386 require.Falsef(cgroups.PathExists(p), "cgroup for %s %s still exists", subsystem, cgroup) 387 } 388} 389 390func TestUniversalExecutor_LookupTaskBin(t *testing.T) { 391 t.Parallel() 392 require := require.New(t) 393 394 // Create a temp dir 395 tmpDir, err := ioutil.TempDir("", "") 396 require.Nil(err) 397 defer os.Remove(tmpDir) 398 399 // Create the command 400 cmd := &ExecCommand{Env: []string{"PATH=/bin"}, TaskDir: tmpDir} 401 402 // Make a foo subdir 403 os.MkdirAll(filepath.Join(tmpDir, "foo"), 0700) 404 405 // Write a file under foo 406 filePath := filepath.Join(tmpDir, "foo", "tmp.txt") 407 err = ioutil.WriteFile(filePath, []byte{1, 2}, os.ModeAppend) 408 require.NoError(err) 409 410 // Lookout with an absolute path to the binary 411 cmd.Cmd = "/foo/tmp.txt" 412 _, err = lookupTaskBin(cmd) 413 require.NoError(err) 414 415 // Write a file under local subdir 416 os.MkdirAll(filepath.Join(tmpDir, "local"), 0700) 417 filePath2 := filepath.Join(tmpDir, "local", "tmp.txt") 418 ioutil.WriteFile(filePath2, []byte{1, 2}, os.ModeAppend) 419 420 // Lookup with file name, should find the one we wrote above 421 cmd.Cmd = "tmp.txt" 422 _, err = lookupTaskBin(cmd) 423 require.NoError(err) 424 425 // Lookup a host absolute path 426 cmd.Cmd = "/bin/sh" 427 _, err = lookupTaskBin(cmd) 428 require.Error(err) 429} 430 431// Exec Launch looks for the binary only inside the chroot 432func TestExecutor_EscapeContainer(t *testing.T) { 433 t.Parallel() 434 require := require.New(t) 435 testutil.ExecCompatible(t) 436 437 testExecCmd := testExecutorCommandWithChroot(t) 438 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 439 execCmd.Cmd = "/bin/kill" // missing from the chroot container 440 defer allocDir.Destroy() 441 442 execCmd.ResourceLimits = true 443 444 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 445 defer executor.Shutdown("SIGKILL", 0) 446 447 _, err := executor.Launch(execCmd) 448 require.Error(err) 449 require.Regexp("^file /bin/kill not found under path", err) 450 451 // Bare files are looked up using the system path, inside the container 452 allocDir.Destroy() 453 testExecCmd = testExecutorCommandWithChroot(t) 454 execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir 455 execCmd.Cmd = "kill" 456 _, err = executor.Launch(execCmd) 457 require.Error(err) 458 require.Regexp("^file kill not found under path", err) 459 460 allocDir.Destroy() 461 testExecCmd = testExecutorCommandWithChroot(t) 462 execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir 463 execCmd.Cmd = "echo" 464 _, err = executor.Launch(execCmd) 465 require.NoError(err) 466} 467 468func TestExecutor_Capabilities(t *testing.T) { 469 t.Parallel() 470 testutil.ExecCompatible(t) 471 472 cases := []struct { 473 user string 474 caps string 475 }{ 476 { 477 user: "nobody", 478 caps: ` 479CapInh: 0000000000000000 480CapPrm: 0000000000000000 481CapEff: 0000000000000000 482CapBnd: 00000000a80405fb 483CapAmb: 0000000000000000`, 484 }, 485 { 486 user: "root", 487 caps: ` 488CapInh: 0000000000000000 489CapPrm: 0000003fffffffff 490CapEff: 0000003fffffffff 491CapBnd: 0000003fffffffff 492CapAmb: 0000000000000000`, 493 }, 494 } 495 496 for _, c := range cases { 497 t.Run(c.user, func(t *testing.T) { 498 499 testExecCmd := testExecutorCommandWithChroot(t) 500 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 501 defer allocDir.Destroy() 502 503 execCmd.User = c.user 504 execCmd.ResourceLimits = true 505 execCmd.Cmd = "/bin/bash" 506 execCmd.Args = []string{"-c", "cat /proc/$$/status"} 507 execCmd.Capabilities = capabilities.NomadDefaults().Slice(true) 508 509 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 510 defer executor.Shutdown("SIGKILL", 0) 511 512 _, err := executor.Launch(execCmd) 513 require.NoError(t, err) 514 515 ch := make(chan interface{}) 516 go func() { 517 executor.Wait(context.Background()) 518 close(ch) 519 }() 520 521 select { 522 case <-ch: 523 // all good 524 case <-time.After(5 * time.Second): 525 require.Fail(t, "timeout waiting for exec to shutdown") 526 } 527 528 canonical := func(s string) string { 529 s = strings.TrimSpace(s) 530 s = regexp.MustCompile("[ \t]+").ReplaceAllString(s, " ") 531 s = regexp.MustCompile("[\n\r]+").ReplaceAllString(s, "\n") 532 return s 533 } 534 535 expected := canonical(c.caps) 536 tu.WaitForResult(func() (bool, error) { 537 output := canonical(testExecCmd.stdout.String()) 538 if !strings.Contains(output, expected) { 539 return false, fmt.Errorf("capabilities didn't match: want\n%v\n; got:\n%v\n", expected, output) 540 } 541 return true, nil 542 }, func(err error) { require.NoError(t, err) }) 543 }) 544 } 545 546} 547 548func TestExecutor_ClientCleanup(t *testing.T) { 549 t.Parallel() 550 testutil.ExecCompatible(t) 551 require := require.New(t) 552 553 testExecCmd := testExecutorCommandWithChroot(t) 554 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 555 defer allocDir.Destroy() 556 557 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 558 defer executor.Shutdown("", 0) 559 560 // Need to run a command which will produce continuous output but not 561 // too quickly to ensure executor.Exit() stops the process. 562 execCmd.Cmd = "/bin/bash" 563 execCmd.Args = []string{"-c", "while true; do /bin/echo X; /bin/sleep 1; done"} 564 execCmd.ResourceLimits = true 565 566 ps, err := executor.Launch(execCmd) 567 568 require.NoError(err) 569 require.NotZero(ps.Pid) 570 time.Sleep(500 * time.Millisecond) 571 require.NoError(executor.Shutdown("SIGINT", 100*time.Millisecond)) 572 573 ch := make(chan interface{}) 574 go func() { 575 executor.Wait(context.Background()) 576 close(ch) 577 }() 578 579 select { 580 case <-ch: 581 // all good 582 case <-time.After(5 * time.Second): 583 require.Fail("timeout waiting for exec to shutdown") 584 } 585 586 output := testExecCmd.stdout.String() 587 require.NotZero(len(output)) 588 time.Sleep(2 * time.Second) 589 output1 := testExecCmd.stdout.String() 590 require.Equal(len(output), len(output1)) 591} 592 593func TestExecutor_cmdDevices(t *testing.T) { 594 input := []*drivers.DeviceConfig{ 595 { 596 HostPath: "/dev/null", 597 TaskPath: "/task/dev/null", 598 Permissions: "rwm", 599 }, 600 } 601 602 expected := &devices.Device{ 603 Rule: devices.Rule{ 604 Type: 99, 605 Major: 1, 606 Minor: 3, 607 Permissions: "rwm", 608 }, 609 Path: "/task/dev/null", 610 } 611 612 found, err := cmdDevices(input) 613 require.NoError(t, err) 614 require.Len(t, found, 1) 615 616 // ignore file permission and ownership 617 // as they are host specific potentially 618 d := found[0] 619 d.FileMode = 0 620 d.Uid = 0 621 d.Gid = 0 622 623 require.EqualValues(t, expected, d) 624} 625 626func TestExecutor_cmdMounts(t *testing.T) { 627 input := []*drivers.MountConfig{ 628 { 629 HostPath: "/host/path-ro", 630 TaskPath: "/task/path-ro", 631 Readonly: true, 632 }, 633 { 634 HostPath: "/host/path-rw", 635 TaskPath: "/task/path-rw", 636 Readonly: false, 637 }, 638 } 639 640 expected := []*lconfigs.Mount{ 641 { 642 Source: "/host/path-ro", 643 Destination: "/task/path-ro", 644 Flags: unix.MS_BIND | unix.MS_RDONLY, 645 Device: "bind", 646 PropagationFlags: []int{unix.MS_PRIVATE | unix.MS_REC}, 647 }, 648 { 649 Source: "/host/path-rw", 650 Destination: "/task/path-rw", 651 Flags: unix.MS_BIND, 652 Device: "bind", 653 PropagationFlags: []int{unix.MS_PRIVATE | unix.MS_REC}, 654 }, 655 } 656 657 require.EqualValues(t, expected, cmdMounts(input)) 658} 659 660// TestUniversalExecutor_NoCgroup asserts that commands are executed in the 661// same cgroup as parent process 662func TestUniversalExecutor_NoCgroup(t *testing.T) { 663 t.Parallel() 664 testutil.ExecCompatible(t) 665 666 expectedBytes, err := ioutil.ReadFile("/proc/self/cgroup") 667 require.NoError(t, err) 668 669 expected := strings.TrimSpace(string(expectedBytes)) 670 671 testExecCmd := testExecutorCommand(t) 672 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 673 execCmd.Cmd = "/bin/cat" 674 execCmd.Args = []string{"/proc/self/cgroup"} 675 defer allocDir.Destroy() 676 677 execCmd.BasicProcessCgroup = false 678 execCmd.ResourceLimits = false 679 680 executor := NewExecutor(testlog.HCLogger(t)) 681 defer executor.Shutdown("SIGKILL", 0) 682 683 _, err = executor.Launch(execCmd) 684 require.NoError(t, err) 685 686 _, err = executor.Wait(context.Background()) 687 require.NoError(t, err) 688 689 tu.WaitForResult(func() (bool, error) { 690 act := strings.TrimSpace(string(testExecCmd.stdout.String())) 691 if expected != act { 692 return false, fmt.Errorf("expected:\n%s actual:\n%s", expected, act) 693 } 694 return true, nil 695 }, func(err error) { 696 stderr := strings.TrimSpace(string(testExecCmd.stderr.String())) 697 t.Logf("stderr: %v", stderr) 698 require.NoError(t, err) 699 }) 700 701} 702