1// +build integration,kubernetes
2
3package kubernetes_test
4
5import (
6	"bytes"
7	"context"
8	"fmt"
9	"io/ioutil"
10	"net/http"
11	"net/http/httptest"
12	"net/url"
13	"os"
14	"regexp"
15	"strings"
16	"testing"
17	"time"
18
19	"github.com/gorilla/websocket"
20	"github.com/stretchr/testify/assert"
21	"github.com/stretchr/testify/require"
22	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
23	"k8s.io/apimachinery/pkg/labels"
24	k8s "k8s.io/client-go/kubernetes"
25
26	"gitlab.com/gitlab-org/gitlab-runner/common"
27	"gitlab.com/gitlab-org/gitlab-runner/common/buildtest"
28	"gitlab.com/gitlab-org/gitlab-runner/executors/kubernetes"
29	"gitlab.com/gitlab-org/gitlab-runner/executors/kubernetes/internal/pull"
30	"gitlab.com/gitlab-org/gitlab-runner/helpers"
31	"gitlab.com/gitlab-org/gitlab-runner/helpers/container/helperimage"
32	"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
33	"gitlab.com/gitlab-org/gitlab-runner/session"
34	"gitlab.com/gitlab-org/gitlab-runner/shells"
35)
36
37type featureFlagTest func(t *testing.T, flagName string, flagValue bool)
38
39func TestRunIntegrationTestsWithFeatureFlag(t *testing.T) {
40	tests := map[string]featureFlagTest{
41		"testKubernetesSuccessRun":                                testKubernetesSuccessRunFeatureFlag,
42		"testKubernetesMultistepRun":                              testKubernetesMultistepRunFeatureFlag,
43		"testKubernetesTimeoutRun":                                testKubernetesTimeoutRunFeatureFlag,
44		"testKubernetesBuildFail":                                 testKubernetesBuildFailFeatureFlag,
45		"testKubernetesBuildCancel":                               testKubernetesBuildCancelFeatureFlag,
46		"testKubernetesBuildLogLimitExceeded":                     testKubernetesBuildLogLimitExceededFeatureFlag,
47		"testKubernetesBuildMasking":                              testKubernetesBuildMaskingFeatureFlag,
48		"testKubernetesCustomClonePath":                           testKubernetesCustomClonePathFeatureFlag,
49		"testKubernetesNoRootImage":                               testKubernetesNoRootImageFeatureFlag,
50		"testKubernetesMissingImage":                              testKubernetesMissingImageFeatureFlag,
51		"testKubernetesMissingTag":                                testKubernetesMissingTagFeatureFlag,
52		"testKubernetesFailingToPullImageTwiceFeatureFlag":        testKubernetesFailingToPullImageTwiceFeatureFlag,
53		"testKubernetesFailingToPullServiceImageTwiceFeatureFlag": testKubernetesFailingToPullSvcImageTwiceFeatureFlag,
54		"testKubernetesFailingToPullHelperTwiceFeatureFlag":       testKubernetesFailingToPullHelperTwiceFeatureFlag,
55		"testOverwriteNamespaceNotMatch":                          testOverwriteNamespaceNotMatchFeatureFlag,
56		"testOverwriteServiceAccountNotMatch":                     testOverwriteServiceAccountNotMatchFeatureFlag,
57		"testInteractiveTerminal":                                 testInteractiveTerminalFeatureFlag,
58		"testKubernetesReplaceEnvFeatureFlag":                     testKubernetesReplaceEnvFeatureFlag,
59		"testKubernetesReplaceMissingEnvVarFeatureFlag":           testKubernetesReplaceMissingEnvVarFeatureFlag,
60		"testKubernetesWithNonRootSecurityContext":                testKubernetesWithNonRootSecurityContext,
61		"testConfiguredBuildDirVolumeMountFeatureFlag":            testBuildDirVolumeMountFeatureFlag,
62		"testUserConfiguredBuildDirVolumeMountFeatureFlag":        testUserConfiguredBuildDirVolumeMountFeatureFlag,
63		"testKubernetesPwshFeatureFlag":                           testKubernetesPwshFeatureFlag,
64	}
65
66	featureFlags := []string{
67		featureflags.UseLegacyKubernetesExecutionStrategy,
68	}
69
70	for tn, tt := range tests {
71		for _, ff := range featureFlags {
72			t.Run(fmt.Sprintf("%s %s true", tn, ff), func(t *testing.T) {
73				tt(t, ff, true)
74			})
75
76			t.Run(fmt.Sprintf("%s %s false", tn, ff), func(t *testing.T) {
77				tt(t, ff, false)
78			})
79		}
80	}
81}
82
83func testKubernetesSuccessRunFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
84	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
85
86	build := getTestBuild(t, common.GetRemoteSuccessfulBuild)
87	build.Image.Name = common.TestDockerGitImage
88	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
89
90	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
91	assert.NoError(t, err)
92}
93
94func testKubernetesMultistepRunFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
95	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
96
97	successfulBuild, err := common.GetRemoteSuccessfulMultistepBuild()
98	require.NoError(t, err)
99
100	failingScriptBuild, err := common.GetRemoteFailingMultistepBuild(common.StepNameScript)
101	require.NoError(t, err)
102
103	failingReleaseBuild, err := common.GetRemoteFailingMultistepBuild("release")
104	require.NoError(t, err)
105
106	successfulBuild.Image.Name = common.TestDockerGitImage
107	failingScriptBuild.Image.Name = common.TestDockerGitImage
108	failingReleaseBuild.Image.Name = common.TestDockerGitImage
109
110	tests := map[string]struct {
111		jobResponse    common.JobResponse
112		expectedOutput []string
113		unwantedOutput []string
114		errExpected    bool
115	}{
116		"Successful build with release and after_script step": {
117			jobResponse: successfulBuild,
118			expectedOutput: []string{
119				"echo Hello World",
120				"echo Release",
121				"echo After Script",
122			},
123			errExpected: false,
124		},
125		"Failure on script step. Release is skipped. After script runs.": {
126			jobResponse: failingScriptBuild,
127			expectedOutput: []string{
128				"echo Hello World",
129				"echo After Script",
130			},
131			unwantedOutput: []string{
132				"echo Release",
133			},
134			errExpected: true,
135		},
136		"Failure on release step. After script runs.": {
137			jobResponse: failingReleaseBuild,
138			expectedOutput: []string{
139				"echo Hello World",
140				"echo Release",
141				"echo After Script",
142			},
143			errExpected: true,
144		},
145	}
146
147	for tn, tt := range tests {
148		t.Run(tn, func(t *testing.T) {
149			build := getTestBuild(t, func() (common.JobResponse, error) {
150				return tt.jobResponse, nil
151			})
152			buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
153
154			var buf bytes.Buffer
155			err := build.Run(&common.Config{}, &common.Trace{Writer: &buf})
156
157			out := buf.String()
158			for _, output := range tt.expectedOutput {
159				assert.Contains(t, out, output)
160			}
161
162			for _, output := range tt.unwantedOutput {
163				assert.NotContains(t, out, output)
164			}
165
166			if tt.errExpected {
167				var buildErr *common.BuildError
168				assert.ErrorAs(t, err, &buildErr)
169				return
170			}
171			assert.NoError(t, err)
172		})
173	}
174}
175
176func testKubernetesTimeoutRunFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
177	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
178
179	build := getTestBuild(t, common.GetRemoteLongRunningBuild)
180	build.Image.Name = common.TestDockerGitImage
181	build.RunnerInfo.Timeout = 10 // seconds
182	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
183
184	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
185	require.Error(t, err)
186	var buildError *common.BuildError
187	assert.ErrorAs(t, err, &buildError)
188	assert.Equal(t, common.JobExecutionTimeout, buildError.FailureReason)
189}
190
191func testKubernetesBuildFailFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
192	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
193
194	build := getTestBuild(t, common.GetRemoteFailedBuild)
195	build.Image.Name = common.TestDockerGitImage
196	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
197
198	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
199	require.Error(t, err, "error")
200	var buildError *common.BuildError
201	assert.ErrorAs(t, err, &buildError)
202	assert.Contains(t, err.Error(), "command terminated with exit code 1")
203	assert.Equal(t, 1, buildError.ExitCode)
204}
205
206func testKubernetesBuildCancelFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
207	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
208
209	build := getTestBuild(t, func() (common.JobResponse, error) {
210		return common.JobResponse{}, nil
211	})
212	buildtest.RunBuildWithCancel(
213		t,
214		build.Runner,
215		func(build *common.Build) {
216			buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
217		},
218	)
219}
220
221func testKubernetesBuildLogLimitExceededFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
222	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
223
224	build := getTestBuild(t, func() (common.JobResponse, error) {
225		return common.JobResponse{}, nil
226	})
227	buildtest.RunRemoteBuildWithJobOutputLimitExceeded(
228		t,
229		build.Runner,
230		func(build *common.Build) {
231			buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
232		},
233	)
234}
235
236func testKubernetesBuildMaskingFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
237	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
238
239	build := getTestBuild(t, func() (common.JobResponse, error) {
240		return common.JobResponse{}, nil
241	})
242	buildtest.RunBuildWithMasking(
243		t,
244		build.Runner,
245		func(build *common.Build) {
246			buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
247		},
248	)
249}
250
251func testKubernetesCustomClonePathFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
252	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
253
254	jobResponse, err := common.GetRemoteBuildResponse(
255		"ls -al $CI_BUILDS_DIR/go/src/gitlab.com/gitlab-org/repo",
256	)
257	require.NoError(t, err)
258
259	tests := map[string]struct {
260		clonePath   string
261		expectedErr bool
262	}{
263		"uses custom clone path": {
264			clonePath:   "$CI_BUILDS_DIR/go/src/gitlab.com/gitlab-org/repo",
265			expectedErr: false,
266		},
267		"path has to be within CI_BUILDS_DIR": {
268			clonePath:   "/unknown/go/src/gitlab.com/gitlab-org/repo",
269			expectedErr: true,
270		},
271	}
272
273	for name, test := range tests {
274		t.Run(name, func(t *testing.T) {
275			build := getTestBuild(t, func() (common.JobResponse, error) {
276				return jobResponse, nil
277			})
278			build.Runner.Environment = []string{
279				"GIT_CLONE_PATH=" + test.clonePath,
280			}
281			buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
282
283			err = build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
284			if test.expectedErr {
285				var buildErr *common.BuildError
286				assert.ErrorAs(t, err, &buildErr)
287				return
288			}
289
290			assert.NoError(t, err)
291		})
292	}
293}
294
295func testKubernetesNoRootImageFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
296	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
297
298	build := getTestBuild(t, common.GetRemoteSuccessfulBuildWithDumpedVariables)
299	build.Image.Name = common.TestAlpineNoRootImage
300	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
301
302	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
303	assert.NoError(t, err)
304}
305
306func testKubernetesMissingImageFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
307	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
308
309	build := getTestBuild(t, common.GetRemoteFailedBuild)
310	build.Image.Name = "some/non-existing/image"
311	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
312
313	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
314	require.Error(t, err)
315	var buildErr *common.BuildError
316	assert.ErrorAs(t, err, &buildErr)
317	assert.Contains(t, err.Error(), "image pull failed")
318}
319
320func testKubernetesMissingTagFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
321	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
322
323	build := getTestBuild(t, common.GetRemoteFailedBuild)
324	build.Image.Name = "docker:missing-tag"
325	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
326
327	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
328	require.Error(t, err)
329	var buildErr *common.BuildError
330	assert.ErrorAs(t, err, &buildErr)
331	assert.Contains(t, err.Error(), "image pull failed")
332}
333
334func testKubernetesFailingToPullImageTwiceFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
335	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
336
337	build := getTestBuild(t, common.GetRemoteFailedBuild)
338	build.Image.Name = "some/non-existing/image"
339	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
340
341	err := runMultiPullPolicyBuild(t, build)
342
343	var imagePullErr *pull.ImagePullError
344	require.ErrorAs(t, err, &imagePullErr)
345	assert.Equal(t, build.Image.Name, imagePullErr.Image)
346}
347
348func testKubernetesFailingToPullSvcImageTwiceFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
349	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
350
351	build := getTestBuild(t, common.GetRemoteFailedBuild)
352	build.Services = common.Services{
353		{
354			Name: "some/non-existing/image",
355		},
356	}
357	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
358
359	err := runMultiPullPolicyBuild(t, build)
360
361	var imagePullErr *pull.ImagePullError
362	require.ErrorAs(t, err, &imagePullErr)
363	assert.Equal(t, build.Services[0].Name, imagePullErr.Image)
364}
365
366func testKubernetesFailingToPullHelperTwiceFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
367	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
368
369	build := getTestBuild(t, common.GetRemoteFailedBuild)
370	build.Runner.Kubernetes.HelperImage = "some/non-existing/image"
371	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
372
373	err := runMultiPullPolicyBuild(t, build)
374
375	var imagePullErr *pull.ImagePullError
376	require.ErrorAs(t, err, &imagePullErr)
377	assert.Equal(t, build.Runner.Kubernetes.HelperImage, imagePullErr.Image)
378}
379
380func testOverwriteNamespaceNotMatchFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
381	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
382
383	build := getTestBuild(t, func() (common.JobResponse, error) {
384		return common.JobResponse{
385			GitInfo: common.GitInfo{
386				Sha: "1234567890",
387			},
388			Image: common.Image{
389				Name: "test-image",
390			},
391			Variables: []common.JobVariable{
392				{Key: kubernetes.NamespaceOverwriteVariableName, Value: "namespace"},
393			},
394		}, nil
395	})
396	build.Runner.Kubernetes.NamespaceOverwriteAllowed = "^not_a_match$"
397	build.SystemInterrupt = make(chan os.Signal, 1)
398	build.Image.Name = common.TestDockerGitImage
399	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
400
401	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
402	require.Error(t, err)
403	assert.Contains(t, err.Error(), "does not match")
404}
405
406func testOverwriteServiceAccountNotMatchFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
407	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
408
409	build := getTestBuild(t, func() (common.JobResponse, error) {
410		return common.JobResponse{
411			GitInfo: common.GitInfo{
412				Sha: "1234567890",
413			},
414			Image: common.Image{
415				Name: "test-image",
416			},
417			Variables: []common.JobVariable{
418				{Key: kubernetes.ServiceAccountOverwriteVariableName, Value: "service-account"},
419			},
420		}, nil
421	})
422	build.Runner.Kubernetes.ServiceAccountOverwriteAllowed = "^not_a_match$"
423	build.SystemInterrupt = make(chan os.Signal, 1)
424	build.Image.Name = common.TestDockerGitImage
425	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
426
427	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
428	require.Error(t, err)
429	assert.Contains(t, err.Error(), "does not match")
430}
431
432func testInteractiveTerminalFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
433	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
434
435	if os.Getenv("GITLAB_CI") == "true" {
436		t.Skip("Skipping inside of GitLab CI check https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26421")
437	}
438
439	client := getTestKubeClusterClient(t)
440	secrets, err := client.CoreV1().Secrets("default").List(context.Background(), metav1.ListOptions{})
441	require.NoError(t, err)
442
443	build := getTestBuild(t, func() (common.JobResponse, error) {
444		return common.GetRemoteBuildResponse("sleep 5")
445	})
446	build.Image.Name = "docker:git"
447	build.Runner.Kubernetes.BearerToken = string(secrets.Items[0].Data["token"])
448	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
449
450	sess, err := session.NewSession(nil)
451	build.Session = sess
452
453	outBuffer := bytes.NewBuffer(nil)
454	outCh := make(chan string)
455
456	go func() {
457		err = build.Run(
458			&common.Config{
459				SessionServer: common.SessionServer{
460					SessionTimeout: 2,
461				},
462			},
463			&common.Trace{Writer: outBuffer},
464		)
465		require.NoError(t, err)
466
467		outCh <- outBuffer.String()
468	}()
469
470	for build.Session.Mux() == nil {
471		time.Sleep(10 * time.Millisecond)
472	}
473
474	time.Sleep(5 * time.Second)
475
476	srv := httptest.NewServer(build.Session.Mux())
477	defer srv.Close()
478
479	u := url.URL{
480		Scheme: "ws",
481		Host:   srv.Listener.Addr().String(),
482		Path:   build.Session.Endpoint + "/exec",
483	}
484	headers := http.Header{
485		"Authorization": []string{build.Session.Token},
486	}
487	conn, resp, err := websocket.DefaultDialer.Dial(u.String(), headers)
488	defer func() {
489		resp.Body.Close()
490		if conn != nil {
491			_ = conn.Close()
492		}
493	}()
494	require.NoError(t, err)
495	assert.Equal(t, resp.StatusCode, http.StatusSwitchingProtocols)
496
497	out := <-outCh
498	t.Log(out)
499
500	assert.Contains(t, out, "Terminal is connected, will time out in 2s...")
501}
502
503func testKubernetesReplaceEnvFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
504	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
505	build := getTestBuild(t, common.GetRemoteSuccessfulBuild)
506	build.Image.Name = "$IMAGE:$VERSION"
507	build.JobResponse.Variables = append(
508		build.JobResponse.Variables,
509		common.JobVariable{Key: "IMAGE", Value: "alpine"},
510		common.JobVariable{Key: "VERSION", Value: "latest"},
511	)
512	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
513	out, err := buildtest.RunBuildReturningOutput(t, build)
514	require.NoError(t, err)
515	assert.Contains(t, out, "alpine:latest")
516}
517
518func testKubernetesReplaceMissingEnvVarFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
519	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
520	build := getTestBuild(t, common.GetRemoteSuccessfulBuild)
521	build.Image.Name = "alpine:$NOT_EXISTING_VARIABLE"
522	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
523
524	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
525	require.Error(t, err)
526	assert.Contains(t, err.Error(), "image pull failed: Failed to apply default image tag \"alpine:\"")
527}
528
529func testBuildDirVolumeMountFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
530	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
531
532	build := getTestBuild(t, common.GetRemoteSuccessfulBuild)
533	build.Image.Name = common.TestDockerGitImage
534	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
535	build.Runner.Kubernetes.Volumes = common.KubernetesVolumes{
536		EmptyDirs: []common.KubernetesEmptyDir{
537			{
538				Name:      "build",
539				MountPath: "/builds",
540				Medium:    "Memory",
541			},
542		},
543	}
544
545	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
546	assert.NoError(t, err)
547}
548
549func testUserConfiguredBuildDirVolumeMountFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
550	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
551
552	build := getTestBuild(t, common.GetRemoteSuccessfulBuild)
553	build.Image.Name = common.TestDockerGitImage
554	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
555	build.Runner.BuildsDir = "/some/path"
556	build.Runner.Kubernetes.Volumes = common.KubernetesVolumes{
557		EmptyDirs: []common.KubernetesEmptyDir{
558			{
559				Name:      "build",
560				MountPath: "/some/path",
561				Medium:    "Memory",
562			},
563		},
564	}
565
566	err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
567	assert.NoError(t, err)
568}
569
570// TestLogDeletionAttach tests the outcome when the log files are all deleted
571func TestLogDeletionAttach(t *testing.T) {
572	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
573
574	t.Skip("Log deletion test temporary skipped: issue https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27755")
575
576	tests := []struct {
577		stage            string
578		outputAssertions func(t *testing.T, out string, pod string)
579	}{
580		{
581			stage: "step_", // Any script the user defined
582			outputAssertions: func(t *testing.T, out string, pod string) {
583				assert.Contains(
584					t,
585					out,
586					"ERROR: Job failed: command terminated with exit code 100",
587				)
588			},
589		},
590		{
591			stage: string(common.BuildStagePrepare),
592			outputAssertions: func(t *testing.T, out string, pod string) {
593				assert.Contains(
594					t,
595					out,
596					"ERROR: Job failed: command terminated with exit code 100",
597				)
598			},
599		},
600	}
601
602	for _, tt := range tests {
603		t.Run(tt.stage, func(t *testing.T) {
604			build := getTestBuild(t, func() (common.JobResponse, error) {
605				return common.GetRemoteBuildResponse(
606					"sleep 5000",
607				)
608			})
609			buildtest.SetBuildFeatureFlag(build, featureflags.UseLegacyKubernetesExecutionStrategy, false)
610
611			deletedPodNameCh := make(chan string)
612			defer buildtest.OnUserStage(build, func() {
613				client := getTestKubeClusterClient(t)
614				pods, err := client.
615					CoreV1().
616					Pods("default").
617					List(context.Background(), metav1.ListOptions{
618						LabelSelector: labels.Set(build.Runner.Kubernetes.PodLabels).String(),
619					})
620				require.NoError(t, err)
621				require.NotEmpty(t, pods.Items)
622				pod := pods.Items[0]
623				config, err := kubernetes.GetKubeClientConfig(new(common.KubernetesConfig))
624				require.NoError(t, err)
625				logsPath := fmt.Sprintf("/logs-%d-%d", build.JobInfo.ProjectID, build.JobResponse.ID)
626				opts := kubernetes.ExecOptions{
627					Namespace: pod.Namespace,
628					PodName:   pod.Name,
629					Client:    client,
630					Stdin:     true,
631					In:        strings.NewReader(fmt.Sprintf("rm -rf %s/*", logsPath)),
632					Out:       ioutil.Discard,
633					Command:   []string{"/bin/sh"},
634					Config:    config,
635					Executor:  &kubernetes.DefaultRemoteExecutor{},
636				}
637				err = opts.Run()
638				require.NoError(t, err)
639
640				deletedPodNameCh <- pod.Name
641			})()
642
643			out, err := buildtest.RunBuildReturningOutput(t, build)
644			require.Error(t, err)
645			assert.True(t, err != nil, "No error returned")
646
647			tt.outputAssertions(t, out, <-deletedPodNameCh)
648		})
649	}
650}
651
652// This test reproduces the bug reported in https://gitlab.com/gitlab-org/gitlab-runner/issues/2583
653func TestPrepareIssue2583(t *testing.T) {
654	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
655
656	namespace := "my_namespace"
657	serviceAccount := "my_account"
658
659	runnerConfig := &common.RunnerConfig{
660		RunnerSettings: common.RunnerSettings{
661			Executor: "kubernetes",
662			Kubernetes: &common.KubernetesConfig{
663				Image:                          "an/image:latest",
664				Namespace:                      namespace,
665				NamespaceOverwriteAllowed:      ".*",
666				ServiceAccount:                 serviceAccount,
667				ServiceAccountOverwriteAllowed: ".*",
668			},
669		},
670	}
671
672	build := getTestBuild(t, func() (common.JobResponse, error) {
673		return common.JobResponse{
674			Variables: []common.JobVariable{
675				{Key: kubernetes.NamespaceOverwriteVariableName, Value: "namespace"},
676				{Key: kubernetes.ServiceAccountOverwriteVariableName, Value: "sa"},
677			},
678		}, nil
679	})
680
681	e := kubernetes.NewDefaultExecutorForTest()
682
683	// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
684	prepareOptions := common.ExecutorPrepareOptions{
685		Config:  runnerConfig,
686		Build:   build,
687		Context: context.TODO(),
688	}
689
690	err := e.Prepare(prepareOptions)
691	assert.NoError(t, err)
692	assert.Equal(t, namespace, runnerConfig.Kubernetes.Namespace)
693	assert.Equal(t, serviceAccount, runnerConfig.Kubernetes.ServiceAccount)
694}
695
696func TestHelperImageRegistryLogs(t *testing.T) {
697	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
698
699	for _, tt := range []bool{true, false} {
700		t.Run(fmt.Sprintf("%s is %t", featureflags.GitLabRegistryHelperImage, tt), func(t *testing.T) {
701			build := getTestBuild(t, common.GetRemoteSuccessfulBuild)
702			build.Runner.Kubernetes.PullPolicy = common.StringOrArray{common.PullPolicyAlways}
703			buildtest.SetBuildFeatureFlag(build, featureflags.GitLabRegistryHelperImage, tt)
704
705			trace := bytes.Buffer{}
706			err := build.Run(&common.Config{}, &common.Trace{Writer: &trace})
707			require.NoError(t, err)
708
709			if !tt {
710				assert.Contains(
711					t,
712					trace.String(),
713					helperimage.DockerHubWarningMessage,
714				)
715				return
716			}
717			assert.NotContains(
718				t,
719				trace.String(),
720				helperimage.DockerHubWarningMessage,
721			)
722		})
723	}
724}
725
726func TestDeletedPodSystemFailureDuringExecution(t *testing.T) {
727	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
728
729	tests := []struct {
730		stage            string
731		outputAssertions func(t *testing.T, out string, pod string)
732	}{
733		{
734			stage: "step_", // Any script the user defined
735			outputAssertions: func(t *testing.T, out string, pod string) {
736				assert.Contains(
737					t,
738					out,
739					fmt.Sprintf("ERROR: Job failed (system failure): pods %q not found", pod),
740				)
741			},
742		},
743		{
744			stage: string(common.BuildStagePrepare),
745			outputAssertions: func(t *testing.T, out string, pod string) {
746				assert.Contains(
747					t,
748					out,
749					"ERROR: Job failed (system failure):",
750				)
751
752				assert.Contains(
753					t,
754					out,
755					fmt.Sprintf("pods %q not found", pod),
756				)
757			},
758		},
759	}
760
761	for _, tt := range tests {
762		t.Run(tt.stage, func(t *testing.T) {
763			build := getTestBuild(t, common.GetRemoteLongRunningBuild)
764
765			// It's not possible to get this kind of information on the legacy execution path.
766			buildtest.SetBuildFeatureFlag(build, featureflags.UseLegacyKubernetesExecutionStrategy, false)
767
768			deletedPodNameCh := make(chan string)
769			defer buildtest.OnStage(build, tt.stage, func() {
770				client := getTestKubeClusterClient(t)
771				pods, err := client.CoreV1().Pods("default").List(
772					context.Background(),
773					metav1.ListOptions{
774						LabelSelector: labels.Set(build.Runner.Kubernetes.PodLabels).String(),
775					},
776				)
777				require.NoError(t, err)
778				require.NotEmpty(t, pods.Items)
779				pod := pods.Items[0]
780				err = client.
781					CoreV1().
782					Pods("default").
783					Delete(context.Background(), pod.Name, metav1.DeleteOptions{})
784				require.NoError(t, err)
785
786				deletedPodNameCh <- pod.Name
787			})()
788
789			out, err := buildtest.RunBuildReturningOutput(t, build)
790			assert.True(t, kubernetes.IsKubernetesPodNotFoundError(err), "expected err NotFound, but got %T", err)
791
792			tt.outputAssertions(t, out, <-deletedPodNameCh)
793		})
794	}
795}
796
797func testKubernetesWithNonRootSecurityContext(t *testing.T, featureFlagName string, featureFlagValue bool) {
798	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
799
800	build := getTestBuild(t, func() (common.JobResponse, error) {
801		return common.GetRemoteBuildResponse("id")
802	})
803	build.Image.Name = common.TestAlpineNoRootImage
804
805	runAsNonRoot := true
806	runAsUser := int64(1895034)
807	build.Runner.Kubernetes.PodSecurityContext = common.KubernetesPodSecurityContext{
808		RunAsNonRoot: &runAsNonRoot,
809		RunAsUser:    &runAsUser,
810	}
811
812	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
813
814	out, err := buildtest.RunBuildReturningOutput(t, build)
815	assert.NoError(t, err)
816	assert.Contains(t, out, fmt.Sprintf("uid=%d gid=0(root)", runAsUser))
817}
818
819func testKubernetesPwshFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
820	helpers.SkipIntegrationTests(t, "kubectl", "cluster-info")
821
822	build := getTestBuild(t, common.GetRemoteSuccessfulBuild)
823	buildtest.SetBuildFeatureFlag(build, featureFlagName, featureFlagValue)
824
825	build.Image.Name = common.TestPwshImage
826	build.Runner.Shell = shells.SNPwsh
827	build.JobResponse.Steps = common.Steps{
828		common.Step{
829			Name: common.StepNameScript,
830			Script: []string{
831				"Write-Output $PSVersionTable",
832			},
833		},
834	}
835
836	out, err := buildtest.RunBuildReturningOutput(t, build)
837	assert.NoError(t, err)
838	assert.Regexp(t, regexp.MustCompile("PSEdition +Core"), out)
839}
840
841func getTestBuild(t *testing.T, getJobResponse func() (common.JobResponse, error)) *common.Build {
842	jobResponse, err := getJobResponse()
843	assert.NoError(t, err)
844
845	podUUID, err := helpers.GenerateRandomUUID(8)
846	require.NoError(t, err)
847
848	return &common.Build{
849		JobResponse: jobResponse,
850		Runner: &common.RunnerConfig{
851			RunnerSettings: common.RunnerSettings{
852				Executor: "kubernetes",
853				Kubernetes: &common.KubernetesConfig{
854					Image:      common.TestAlpineImage,
855					PullPolicy: common.StringOrArray{common.PullPolicyIfNotPresent},
856					PodLabels: map[string]string{
857						"test.k8s.gitlab.com/name": podUUID,
858					},
859				},
860			},
861		},
862	}
863}
864
865func getTestKubeClusterClient(t *testing.T) *k8s.Clientset {
866	config, err := kubernetes.GetKubeClientConfig(new(common.KubernetesConfig))
867	require.NoError(t, err)
868	client, err := k8s.NewForConfig(config)
869	require.NoError(t, err)
870
871	return client
872}
873
874func runMultiPullPolicyBuild(t *testing.T, build *common.Build) error {
875	build.Runner.Kubernetes.PullPolicy = common.StringOrArray{
876		common.PullPolicyAlways,
877		common.PullPolicyIfNotPresent,
878	}
879
880	outBuffer := bytes.NewBuffer(nil)
881
882	err := build.Run(&common.Config{}, &common.Trace{Writer: outBuffer})
883	require.Error(t, err)
884	var buildErr *common.BuildError
885	assert.ErrorAs(t, err, &buildErr)
886
887	assert.Regexp(
888		t,
889		`(?s).*WARNING: Failed to pull image with policy "Always": image pull failed:.*`+
890			`Attempt #2: Trying "IfNotPresent" pull policy for "some\/non-existing\/image" image.*`+
891			`WARNING: Failed to pull image with policy "IfNotPresent":.*`+
892			`image pull failed:.*`,
893		outBuffer.String(),
894	)
895
896	return err
897}
898