1// +build integration,kubernetes 2 3package kubernetes_test 4 5import ( 6 "bytes" 7 "context" 8 "fmt" 9 "io/ioutil" 10 "net/http" 11 "net/http/httptest" 12 "net/url" 13 "os" 14 "regexp" 15 "strings" 16 "testing" 17 "time" 18 19 "github.com/gorilla/websocket" 20 "github.com/stretchr/testify/assert" 21 "github.com/stretchr/testify/require" 22 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 "k8s.io/apimachinery/pkg/labels" 24 k8s "k8s.io/client-go/kubernetes" 25 26 "gitlab.com/gitlab-org/gitlab-runner/common" 27 "gitlab.com/gitlab-org/gitlab-runner/common/buildtest" 28 "gitlab.com/gitlab-org/gitlab-runner/executors/kubernetes" 29 "gitlab.com/gitlab-org/gitlab-runner/executors/kubernetes/internal/pull" 30 "gitlab.com/gitlab-org/gitlab-runner/helpers" 31 "gitlab.com/gitlab-org/gitlab-runner/helpers/container/helperimage" 32 "gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags" 33 "gitlab.com/gitlab-org/gitlab-runner/session" 34 "gitlab.com/gitlab-org/gitlab-runner/shells" 35) 36 37type featureFlagTest func(t *testing.T, flagName string, flagValue bool) 38 39func TestRunIntegrationTestsWithFeatureFlag(t *testing.T) { 40 tests := map[string]featureFlagTest{ 41 "testKubernetesSuccessRun": testKubernetesSuccessRunFeatureFlag, 42 "testKubernetesMultistepRun": testKubernetesMultistepRunFeatureFlag, 43 "testKubernetesTimeoutRun": testKubernetesTimeoutRunFeatureFlag, 44 "testKubernetesBuildFail": testKubernetesBuildFailFeatureFlag, 45 "testKubernetesBuildCancel": testKubernetesBuildCancelFeatureFlag, 46 "testKubernetesBuildLogLimitExceeded": testKubernetesBuildLogLimitExceededFeatureFlag, 47 "testKubernetesBuildMasking": testKubernetesBuildMaskingFeatureFlag, 48 "testKubernetesCustomClonePath": testKubernetesCustomClonePathFeatureFlag, 49 "testKubernetesNoRootImage": testKubernetesNoRootImageFeatureFlag, 50 "testKubernetesMissingImage": testKubernetesMissingImageFeatureFlag, 51 "testKubernetesMissingTag": testKubernetesMissingTagFeatureFlag, 52 "testKubernetesFailingToPullImageTwiceFeatureFlag": testKubernetesFailingToPullImageTwiceFeatureFlag, 53 "testKubernetesFailingToPullServiceImageTwiceFeatureFlag": testKubernetesFailingToPullSvcImageTwiceFeatureFlag, 54 "testKubernetesFailingToPullHelperTwiceFeatureFlag": testKubernetesFailingToPullHelperTwiceFeatureFlag, 55 "testOverwriteNamespaceNotMatch": testOverwriteNamespaceNotMatchFeatureFlag, 56 "testOverwriteServiceAccountNotMatch": testOverwriteServiceAccountNotMatchFeatureFlag, 57 "testInteractiveTerminal": testInteractiveTerminalFeatureFlag, 58 "testKubernetesReplaceEnvFeatureFlag": testKubernetesReplaceEnvFeatureFlag, 59 "testKubernetesReplaceMissingEnvVarFeatureFlag": testKubernetesReplaceMissingEnvVarFeatureFlag, 60 "testKubernetesWithNonRootSecurityContext": testKubernetesWithNonRootSecurityContext, 61 "testConfiguredBuildDirVolumeMountFeatureFlag": testBuildDirVolumeMountFeatureFlag, 62 "testUserConfiguredBuildDirVolumeMountFeatureFlag": testUserConfiguredBuildDirVolumeMountFeatureFlag, 63 "testKubernetesPwshFeatureFlag": testKubernetesPwshFeatureFlag, 64 } 65 66 featureFlags := []string{ 67 featureflags.UseLegacyKubernetesExecutionStrategy, 68 } 69 70 for tn, tt := range tests { 71 for _, ff := range featureFlags { 72 t.Run(fmt.Sprintf("%s %s true", tn, ff), func(t *testing.T) { 73 tt(t, ff, true) 74 }) 75 76 t.Run(fmt.Sprintf("%s %s false", tn, ff), func(t *testing.T) { 77 tt(t, ff, false) 78 }) 79 } 80 } 81} 82 83func testKubernetesSuccessRunFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 84 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 85 86 build := getTestBuild(t, common.GetRemoteSuccessfulBuild) 87 build.Image.Name = common.TestDockerGitImage 88 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 89 90 err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout}) 91 assert.NoError(t, err) 92} 93 94func testKubernetesMultistepRunFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 95 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 96 97 successfulBuild, err := common.GetRemoteSuccessfulMultistepBuild() 98 require.NoError(t, err) 99 100 failingScriptBuild, err := common.GetRemoteFailingMultistepBuild(common.StepNameScript) 101 require.NoError(t, err) 102 103 failingReleaseBuild, err := common.GetRemoteFailingMultistepBuild("release") 104 require.NoError(t, err) 105 106 successfulBuild.Image.Name = common.TestDockerGitImage 107 failingScriptBuild.Image.Name = common.TestDockerGitImage 108 failingReleaseBuild.Image.Name = common.TestDockerGitImage 109 110 tests := map[string]struct { 111 jobResponse common.JobResponse 112 expectedOutput []string 113 unwantedOutput []string 114 errExpected bool 115 }{ 116 "Successful build with release and after_script step": { 117 jobResponse: successfulBuild, 118 expectedOutput: []string{ 119 "echo Hello World", 120 "echo Release", 121 "echo After Script", 122 }, 123 errExpected: false, 124 }, 125 "Failure on script step. Release is skipped. After script runs.": { 126 jobResponse: failingScriptBuild, 127 expectedOutput: []string{ 128 "echo Hello World", 129 "echo After Script", 130 }, 131 unwantedOutput: []string{ 132 "echo Release", 133 }, 134 errExpected: true, 135 }, 136 "Failure on release step. After script runs.": { 137 jobResponse: failingReleaseBuild, 138 expectedOutput: []string{ 139 "echo Hello World", 140 "echo Release", 141 "echo After Script", 142 }, 143 errExpected: true, 144 }, 145 } 146 147 for tn, tt := range tests { 148 t.Run(tn, func(t *testing.T) { 149 build := getTestBuild(t, func() (common.JobResponse, error) { 150 return tt.jobResponse, nil 151 }) 152 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 153 154 var buf bytes.Buffer 155 err := build.Run(&common.Config{}, &common.Trace{Writer: &buf}) 156 157 out := buf.String() 158 for _, output := range tt.expectedOutput { 159 assert.Contains(t, out, output) 160 } 161 162 for _, output := range tt.unwantedOutput { 163 assert.NotContains(t, out, output) 164 } 165 166 if tt.errExpected { 167 var buildErr *common.BuildError 168 assert.ErrorAs(t, err, &buildErr) 169 return 170 } 171 assert.NoError(t, err) 172 }) 173 } 174} 175 176func testKubernetesTimeoutRunFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 177 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 178 179 build := getTestBuild(t, common.GetRemoteLongRunningBuild) 180 build.Image.Name = common.TestDockerGitImage 181 build.RunnerInfo.Timeout = 10 // seconds 182 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 183 184 err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout}) 185 require.Error(t, err) 186 var buildError *common.BuildError 187 assert.ErrorAs(t, err, &buildError) 188 assert.Equal(t, common.JobExecutionTimeout, buildError.FailureReason) 189} 190 191func testKubernetesBuildFailFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 192 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 193 194 build := getTestBuild(t, common.GetRemoteFailedBuild) 195 build.Image.Name = common.TestDockerGitImage 196 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 197 198 err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout}) 199 require.Error(t, err, "error") 200 var buildError *common.BuildError 201 assert.ErrorAs(t, err, &buildError) 202 assert.Contains(t, err.Error(), "command terminated with exit code 1") 203 assert.Equal(t, 1, buildError.ExitCode) 204} 205 206func testKubernetesBuildCancelFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 207 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 208 209 build := getTestBuild(t, func() (common.JobResponse, error) { 210 return common.JobResponse{}, nil 211 }) 212 buildtest.RunBuildWithCancel( 213 t, 214 build.Runner, 215 func(build *common.Build) { 216 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 217 }, 218 ) 219} 220 221func testKubernetesBuildLogLimitExceededFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 222 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 223 224 build := getTestBuild(t, func() (common.JobResponse, error) { 225 return common.JobResponse{}, nil 226 }) 227 buildtest.RunRemoteBuildWithJobOutputLimitExceeded( 228 t, 229 build.Runner, 230 func(build *common.Build) { 231 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 232 }, 233 ) 234} 235 236func testKubernetesBuildMaskingFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 237 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 238 239 build := getTestBuild(t, func() (common.JobResponse, error) { 240 return common.JobResponse{}, nil 241 }) 242 buildtest.RunBuildWithMasking( 243 t, 244 build.Runner, 245 func(build *common.Build) { 246 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 247 }, 248 ) 249} 250 251func testKubernetesCustomClonePathFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 252 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 253 254 jobResponse, err := common.GetRemoteBuildResponse( 255 "ls -al $CI_BUILDS_DIR/go/src/gitlab.com/gitlab-org/repo", 256 ) 257 require.NoError(t, err) 258 259 tests := map[string]struct { 260 clonePath string 261 expectedErr bool 262 }{ 263 "uses custom clone path": { 264 clonePath: "$CI_BUILDS_DIR/go/src/gitlab.com/gitlab-org/repo", 265 expectedErr: false, 266 }, 267 "path has to be within CI_BUILDS_DIR": { 268 clonePath: "/unknown/go/src/gitlab.com/gitlab-org/repo", 269 expectedErr: true, 270 }, 271 } 272 273 for name, test := range tests { 274 t.Run(name, func(t *testing.T) { 275 build := getTestBuild(t, func() (common.JobResponse, error) { 276 return jobResponse, nil 277 }) 278 build.Runner.Environment = []string{ 279 "GIT_CLONE_PATH=" + test.clonePath, 280 } 281 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 282 283 err = build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout}) 284 if test.expectedErr { 285 var buildErr *common.BuildError 286 assert.ErrorAs(t, err, &buildErr) 287 return 288 } 289 290 assert.NoError(t, err) 291 }) 292 } 293} 294 295func testKubernetesNoRootImageFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 296 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 297 298 build := getTestBuild(t, common.GetRemoteSuccessfulBuildWithDumpedVariables) 299 build.Image.Name = common.TestAlpineNoRootImage 300 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 301 302 err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout}) 303 assert.NoError(t, err) 304} 305 306func testKubernetesMissingImageFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 307 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 308 309 build := getTestBuild(t, common.GetRemoteFailedBuild) 310 build.Image.Name = "some/non-existing/image" 311 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 312 313 err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout}) 314 require.Error(t, err) 315 var buildErr *common.BuildError 316 assert.ErrorAs(t, err, &buildErr) 317 assert.Contains(t, err.Error(), "image pull failed") 318} 319 320func testKubernetesMissingTagFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 321 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 322 323 build := getTestBuild(t, common.GetRemoteFailedBuild) 324 build.Image.Name = "docker:missing-tag" 325 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 326 327 err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout}) 328 require.Error(t, err) 329 var buildErr *common.BuildError 330 assert.ErrorAs(t, err, &buildErr) 331 assert.Contains(t, err.Error(), "image pull failed") 332} 333 334func testKubernetesFailingToPullImageTwiceFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 335 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 336 337 build := getTestBuild(t, common.GetRemoteFailedBuild) 338 build.Image.Name = "some/non-existing/image" 339 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 340 341 err := runMultiPullPolicyBuild(t, build) 342 343 var imagePullErr *pull.ImagePullError 344 require.ErrorAs(t, err, &imagePullErr) 345 assert.Equal(t, build.Image.Name, imagePullErr.Image) 346} 347 348func testKubernetesFailingToPullSvcImageTwiceFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 349 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 350 351 build := getTestBuild(t, common.GetRemoteFailedBuild) 352 build.Services = common.Services{ 353 { 354 Name: "some/non-existing/image", 355 }, 356 } 357 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 358 359 err := runMultiPullPolicyBuild(t, build) 360 361 var imagePullErr *pull.ImagePullError 362 require.ErrorAs(t, err, &imagePullErr) 363 assert.Equal(t, build.Services[0].Name, imagePullErr.Image) 364} 365 366func testKubernetesFailingToPullHelperTwiceFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 367 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 368 369 build := getTestBuild(t, common.GetRemoteFailedBuild) 370 build.Runner.Kubernetes.HelperImage = "some/non-existing/image" 371 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 372 373 err := runMultiPullPolicyBuild(t, build) 374 375 var imagePullErr *pull.ImagePullError 376 require.ErrorAs(t, err, &imagePullErr) 377 assert.Equal(t, build.Runner.Kubernetes.HelperImage, imagePullErr.Image) 378} 379 380func testOverwriteNamespaceNotMatchFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 381 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 382 383 build := getTestBuild(t, func() (common.JobResponse, error) { 384 return common.JobResponse{ 385 GitInfo: common.GitInfo{ 386 Sha: "1234567890", 387 }, 388 Image: common.Image{ 389 Name: "test-image", 390 }, 391 Variables: []common.JobVariable{ 392 {Key: kubernetes.NamespaceOverwriteVariableName, Value: "namespace"}, 393 }, 394 }, nil 395 }) 396 build.Runner.Kubernetes.NamespaceOverwriteAllowed = "^not_a_match$" 397 build.SystemInterrupt = make(chan os.Signal, 1) 398 build.Image.Name = common.TestDockerGitImage 399 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 400 401 err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout}) 402 require.Error(t, err) 403 assert.Contains(t, err.Error(), "does not match") 404} 405 406func testOverwriteServiceAccountNotMatchFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 407 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 408 409 build := getTestBuild(t, func() (common.JobResponse, error) { 410 return common.JobResponse{ 411 GitInfo: common.GitInfo{ 412 Sha: "1234567890", 413 }, 414 Image: common.Image{ 415 Name: "test-image", 416 }, 417 Variables: []common.JobVariable{ 418 {Key: kubernetes.ServiceAccountOverwriteVariableName, Value: "service-account"}, 419 }, 420 }, nil 421 }) 422 build.Runner.Kubernetes.ServiceAccountOverwriteAllowed = "^not_a_match$" 423 build.SystemInterrupt = make(chan os.Signal, 1) 424 build.Image.Name = common.TestDockerGitImage 425 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 426 427 err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout}) 428 require.Error(t, err) 429 assert.Contains(t, err.Error(), "does not match") 430} 431 432func testInteractiveTerminalFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 433 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 434 435 if os.Getenv("GITLAB_CI") == "true" { 436 t.Skip("Skipping inside of GitLab CI check https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26421") 437 } 438 439 client := getTestKubeClusterClient(t) 440 secrets, err := client.CoreV1().Secrets("default").List(context.Background(), metav1.ListOptions{}) 441 require.NoError(t, err) 442 443 build := getTestBuild(t, func() (common.JobResponse, error) { 444 return common.GetRemoteBuildResponse("sleep 5") 445 }) 446 build.Image.Name = "docker:git" 447 build.Runner.Kubernetes.BearerToken = string(secrets.Items[0].Data["token"]) 448 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 449 450 sess, err := session.NewSession(nil) 451 build.Session = sess 452 453 outBuffer := bytes.NewBuffer(nil) 454 outCh := make(chan string) 455 456 go func() { 457 err = build.Run( 458 &common.Config{ 459 SessionServer: common.SessionServer{ 460 SessionTimeout: 2, 461 }, 462 }, 463 &common.Trace{Writer: outBuffer}, 464 ) 465 require.NoError(t, err) 466 467 outCh <- outBuffer.String() 468 }() 469 470 for build.Session.Mux() == nil { 471 time.Sleep(10 * time.Millisecond) 472 } 473 474 time.Sleep(5 * time.Second) 475 476 srv := httptest.NewServer(build.Session.Mux()) 477 defer srv.Close() 478 479 u := url.URL{ 480 Scheme: "ws", 481 Host: srv.Listener.Addr().String(), 482 Path: build.Session.Endpoint + "/exec", 483 } 484 headers := http.Header{ 485 "Authorization": []string{build.Session.Token}, 486 } 487 conn, resp, err := websocket.DefaultDialer.Dial(u.String(), headers) 488 defer func() { 489 resp.Body.Close() 490 if conn != nil { 491 _ = conn.Close() 492 } 493 }() 494 require.NoError(t, err) 495 assert.Equal(t, resp.StatusCode, http.StatusSwitchingProtocols) 496 497 out := <-outCh 498 t.Log(out) 499 500 assert.Contains(t, out, "Terminal is connected, will time out in 2s...") 501} 502 503func testKubernetesReplaceEnvFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 504 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 505 build := getTestBuild(t, common.GetRemoteSuccessfulBuild) 506 build.Image.Name = "$IMAGE:$VERSION" 507 build.JobResponse.Variables = append( 508 build.JobResponse.Variables, 509 common.JobVariable{Key: "IMAGE", Value: "alpine"}, 510 common.JobVariable{Key: "VERSION", Value: "latest"}, 511 ) 512 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 513 out, err := buildtest.RunBuildReturningOutput(t, build) 514 require.NoError(t, err) 515 assert.Contains(t, out, "alpine:latest") 516} 517 518func testKubernetesReplaceMissingEnvVarFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 519 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 520 build := getTestBuild(t, common.GetRemoteSuccessfulBuild) 521 build.Image.Name = "alpine:$NOT_EXISTING_VARIABLE" 522 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 523 524 err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout}) 525 require.Error(t, err) 526 assert.Contains(t, err.Error(), "image pull failed: Failed to apply default image tag \"alpine:\"") 527} 528 529func testBuildDirVolumeMountFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 530 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 531 532 build := getTestBuild(t, common.GetRemoteSuccessfulBuild) 533 build.Image.Name = common.TestDockerGitImage 534 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 535 build.Runner.Kubernetes.Volumes = common.KubernetesVolumes{ 536 EmptyDirs: []common.KubernetesEmptyDir{ 537 { 538 Name: "build", 539 MountPath: "/builds", 540 Medium: "Memory", 541 }, 542 }, 543 } 544 545 err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout}) 546 assert.NoError(t, err) 547} 548 549func testUserConfiguredBuildDirVolumeMountFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 550 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 551 552 build := getTestBuild(t, common.GetRemoteSuccessfulBuild) 553 build.Image.Name = common.TestDockerGitImage 554 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 555 build.Runner.BuildsDir = "/some/path" 556 build.Runner.Kubernetes.Volumes = common.KubernetesVolumes{ 557 EmptyDirs: []common.KubernetesEmptyDir{ 558 { 559 Name: "build", 560 MountPath: "/some/path", 561 Medium: "Memory", 562 }, 563 }, 564 } 565 566 err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout}) 567 assert.NoError(t, err) 568} 569 570// TestLogDeletionAttach tests the outcome when the log files are all deleted 571func TestLogDeletionAttach(t *testing.T) { 572 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 573 574 t.Skip("Log deletion test temporary skipped: issue https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27755") 575 576 tests := []struct { 577 stage string 578 outputAssertions func(t *testing.T, out string, pod string) 579 }{ 580 { 581 stage: "step_", // Any script the user defined 582 outputAssertions: func(t *testing.T, out string, pod string) { 583 assert.Contains( 584 t, 585 out, 586 "ERROR: Job failed: command terminated with exit code 100", 587 ) 588 }, 589 }, 590 { 591 stage: string(common.BuildStagePrepare), 592 outputAssertions: func(t *testing.T, out string, pod string) { 593 assert.Contains( 594 t, 595 out, 596 "ERROR: Job failed: command terminated with exit code 100", 597 ) 598 }, 599 }, 600 } 601 602 for _, tt := range tests { 603 t.Run(tt.stage, func(t *testing.T) { 604 build := getTestBuild(t, func() (common.JobResponse, error) { 605 return common.GetRemoteBuildResponse( 606 "sleep 5000", 607 ) 608 }) 609 buildtest.SetBuildFeatureFlag(build, featureflags.UseLegacyKubernetesExecutionStrategy, false) 610 611 deletedPodNameCh := make(chan string) 612 defer buildtest.OnUserStage(build, func() { 613 client := getTestKubeClusterClient(t) 614 pods, err := client. 615 CoreV1(). 616 Pods("default"). 617 List(context.Background(), metav1.ListOptions{ 618 LabelSelector: labels.Set(build.Runner.Kubernetes.PodLabels).String(), 619 }) 620 require.NoError(t, err) 621 require.NotEmpty(t, pods.Items) 622 pod := pods.Items[0] 623 config, err := kubernetes.GetKubeClientConfig(new(common.KubernetesConfig)) 624 require.NoError(t, err) 625 logsPath := fmt.Sprintf("/logs-%d-%d", build.JobInfo.ProjectID, build.JobResponse.ID) 626 opts := kubernetes.ExecOptions{ 627 Namespace: pod.Namespace, 628 PodName: pod.Name, 629 Client: client, 630 Stdin: true, 631 In: strings.NewReader(fmt.Sprintf("rm -rf %s/*", logsPath)), 632 Out: ioutil.Discard, 633 Command: []string{"/bin/sh"}, 634 Config: config, 635 Executor: &kubernetes.DefaultRemoteExecutor{}, 636 } 637 err = opts.Run() 638 require.NoError(t, err) 639 640 deletedPodNameCh <- pod.Name 641 })() 642 643 out, err := buildtest.RunBuildReturningOutput(t, build) 644 require.Error(t, err) 645 assert.True(t, err != nil, "No error returned") 646 647 tt.outputAssertions(t, out, <-deletedPodNameCh) 648 }) 649 } 650} 651 652// This test reproduces the bug reported in https://gitlab.com/gitlab-org/gitlab-runner/issues/2583 653func TestPrepareIssue2583(t *testing.T) { 654 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 655 656 namespace := "my_namespace" 657 serviceAccount := "my_account" 658 659 runnerConfig := &common.RunnerConfig{ 660 RunnerSettings: common.RunnerSettings{ 661 Executor: "kubernetes", 662 Kubernetes: &common.KubernetesConfig{ 663 Image: "an/image:latest", 664 Namespace: namespace, 665 NamespaceOverwriteAllowed: ".*", 666 ServiceAccount: serviceAccount, 667 ServiceAccountOverwriteAllowed: ".*", 668 }, 669 }, 670 } 671 672 build := getTestBuild(t, func() (common.JobResponse, error) { 673 return common.JobResponse{ 674 Variables: []common.JobVariable{ 675 {Key: kubernetes.NamespaceOverwriteVariableName, Value: "namespace"}, 676 {Key: kubernetes.ServiceAccountOverwriteVariableName, Value: "sa"}, 677 }, 678 }, nil 679 }) 680 681 e := kubernetes.NewDefaultExecutorForTest() 682 683 // TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932 684 prepareOptions := common.ExecutorPrepareOptions{ 685 Config: runnerConfig, 686 Build: build, 687 Context: context.TODO(), 688 } 689 690 err := e.Prepare(prepareOptions) 691 assert.NoError(t, err) 692 assert.Equal(t, namespace, runnerConfig.Kubernetes.Namespace) 693 assert.Equal(t, serviceAccount, runnerConfig.Kubernetes.ServiceAccount) 694} 695 696func TestHelperImageRegistryLogs(t *testing.T) { 697 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 698 699 for _, tt := range []bool{true, false} { 700 t.Run(fmt.Sprintf("%s is %t", featureflags.GitLabRegistryHelperImage, tt), func(t *testing.T) { 701 build := getTestBuild(t, common.GetRemoteSuccessfulBuild) 702 build.Runner.Kubernetes.PullPolicy = common.StringOrArray{common.PullPolicyAlways} 703 buildtest.SetBuildFeatureFlag(build, featureflags.GitLabRegistryHelperImage, tt) 704 705 trace := bytes.Buffer{} 706 err := build.Run(&common.Config{}, &common.Trace{Writer: &trace}) 707 require.NoError(t, err) 708 709 if !tt { 710 assert.Contains( 711 t, 712 trace.String(), 713 helperimage.DockerHubWarningMessage, 714 ) 715 return 716 } 717 assert.NotContains( 718 t, 719 trace.String(), 720 helperimage.DockerHubWarningMessage, 721 ) 722 }) 723 } 724} 725 726func TestDeletedPodSystemFailureDuringExecution(t *testing.T) { 727 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 728 729 tests := []struct { 730 stage string 731 outputAssertions func(t *testing.T, out string, pod string) 732 }{ 733 { 734 stage: "step_", // Any script the user defined 735 outputAssertions: func(t *testing.T, out string, pod string) { 736 assert.Contains( 737 t, 738 out, 739 fmt.Sprintf("ERROR: Job failed (system failure): pods %q not found", pod), 740 ) 741 }, 742 }, 743 { 744 stage: string(common.BuildStagePrepare), 745 outputAssertions: func(t *testing.T, out string, pod string) { 746 assert.Contains( 747 t, 748 out, 749 "ERROR: Job failed (system failure):", 750 ) 751 752 assert.Contains( 753 t, 754 out, 755 fmt.Sprintf("pods %q not found", pod), 756 ) 757 }, 758 }, 759 } 760 761 for _, tt := range tests { 762 t.Run(tt.stage, func(t *testing.T) { 763 build := getTestBuild(t, common.GetRemoteLongRunningBuild) 764 765 // It's not possible to get this kind of information on the legacy execution path. 766 buildtest.SetBuildFeatureFlag(build, featureflags.UseLegacyKubernetesExecutionStrategy, false) 767 768 deletedPodNameCh := make(chan string) 769 defer buildtest.OnStage(build, tt.stage, func() { 770 client := getTestKubeClusterClient(t) 771 pods, err := client.CoreV1().Pods("default").List( 772 context.Background(), 773 metav1.ListOptions{ 774 LabelSelector: labels.Set(build.Runner.Kubernetes.PodLabels).String(), 775 }, 776 ) 777 require.NoError(t, err) 778 require.NotEmpty(t, pods.Items) 779 pod := pods.Items[0] 780 err = client. 781 CoreV1(). 782 Pods("default"). 783 Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) 784 require.NoError(t, err) 785 786 deletedPodNameCh <- pod.Name 787 })() 788 789 out, err := buildtest.RunBuildReturningOutput(t, build) 790 assert.True(t, kubernetes.IsKubernetesPodNotFoundError(err), "expected err NotFound, but got %T", err) 791 792 tt.outputAssertions(t, out, <-deletedPodNameCh) 793 }) 794 } 795} 796 797func testKubernetesWithNonRootSecurityContext(t *testing.T, featureFlagName string, featureFlagValue bool) { 798 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 799 800 build := getTestBuild(t, func() (common.JobResponse, error) { 801 return common.GetRemoteBuildResponse("id") 802 }) 803 build.Image.Name = common.TestAlpineNoRootImage 804 805 runAsNonRoot := true 806 runAsUser := int64(1895034) 807 build.Runner.Kubernetes.PodSecurityContext = common.KubernetesPodSecurityContext{ 808 RunAsNonRoot: &runAsNonRoot, 809 RunAsUser: &runAsUser, 810 } 811 812 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 813 814 out, err := buildtest.RunBuildReturningOutput(t, build) 815 assert.NoError(t, err) 816 assert.Contains(t, out, fmt.Sprintf("uid=%d gid=0(root)", runAsUser)) 817} 818 819func testKubernetesPwshFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) { 820 helpers.SkipIntegrationTests(t, "kubectl", "cluster-info") 821 822 build := getTestBuild(t, common.GetRemoteSuccessfulBuild) 823 buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue) 824 825 build.Image.Name = common.TestPwshImage 826 build.Runner.Shell = shells.SNPwsh 827 build.JobResponse.Steps = common.Steps{ 828 common.Step{ 829 Name: common.StepNameScript, 830 Script: []string{ 831 "Write-Output $PSVersionTable", 832 }, 833 }, 834 } 835 836 out, err := buildtest.RunBuildReturningOutput(t, build) 837 assert.NoError(t, err) 838 assert.Regexp(t, regexp.MustCompile("PSEdition +Core"), out) 839} 840 841func getTestBuild(t *testing.T, getJobResponse func() (common.JobResponse, error)) *common.Build { 842 jobResponse, err := getJobResponse() 843 assert.NoError(t, err) 844 845 podUUID, err := helpers.GenerateRandomUUID(8) 846 require.NoError(t, err) 847 848 return &common.Build{ 849 JobResponse: jobResponse, 850 Runner: &common.RunnerConfig{ 851 RunnerSettings: common.RunnerSettings{ 852 Executor: "kubernetes", 853 Kubernetes: &common.KubernetesConfig{ 854 Image: common.TestAlpineImage, 855 PullPolicy: common.StringOrArray{common.PullPolicyIfNotPresent}, 856 PodLabels: map[string]string{ 857 "test.k8s.gitlab.com/name": podUUID, 858 }, 859 }, 860 }, 861 }, 862 } 863} 864 865func getTestKubeClusterClient(t *testing.T) *k8s.Clientset { 866 config, err := kubernetes.GetKubeClientConfig(new(common.KubernetesConfig)) 867 require.NoError(t, err) 868 client, err := k8s.NewForConfig(config) 869 require.NoError(t, err) 870 871 return client 872} 873 874func runMultiPullPolicyBuild(t *testing.T, build *common.Build) error { 875 build.Runner.Kubernetes.PullPolicy = common.StringOrArray{ 876 common.PullPolicyAlways, 877 common.PullPolicyIfNotPresent, 878 } 879 880 outBuffer := bytes.NewBuffer(nil) 881 882 err := build.Run(&common.Config{}, &common.Trace{Writer: outBuffer}) 883 require.Error(t, err) 884 var buildErr *common.BuildError 885 assert.ErrorAs(t, err, &buildErr) 886 887 assert.Regexp( 888 t, 889 `(?s).*WARNING: Failed to pull image with policy "Always": image pull failed:.*`+ 890 `Attempt #2: Trying "IfNotPresent" pull policy for "some\/non-existing\/image" image.*`+ 891 `WARNING: Failed to pull image with policy "IfNotPresent":.*`+ 892 `image pull failed:.*`, 893 outBuffer.String(), 894 ) 895 896 return err 897} 898