1// +build !integration
2
3package kubernetes
4
5import (
6	"bytes"
7	"context"
8	"encoding/json"
9	"errors"
10	"fmt"
11	"io"
12	"io/ioutil"
13	"net/http"
14	"net/url"
15	"os"
16	"runtime"
17	"strconv"
18	"strings"
19	"testing"
20	"time"
21
22	"github.com/sirupsen/logrus"
23	"github.com/stretchr/testify/assert"
24	"github.com/stretchr/testify/mock"
25	"github.com/stretchr/testify/require"
26	api "k8s.io/api/core/v1"
27	kubeerrors "k8s.io/apimachinery/pkg/api/errors"
28	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29	"k8s.io/apimachinery/pkg/util/intstr"
30	"k8s.io/client-go/rest/fake"
31	"k8s.io/client-go/util/exec"
32
33	"gitlab.com/gitlab-org/gitlab-runner/common"
34	"gitlab.com/gitlab-org/gitlab-runner/common/buildtest"
35	"gitlab.com/gitlab-org/gitlab-runner/executors"
36	"gitlab.com/gitlab-org/gitlab-runner/executors/kubernetes/internal/pull"
37	"gitlab.com/gitlab-org/gitlab-runner/helpers/container/helperimage"
38	dns_test "gitlab.com/gitlab-org/gitlab-runner/helpers/dns/test"
39	"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
40	"gitlab.com/gitlab-org/gitlab-runner/session/proxy"
41	"gitlab.com/gitlab-org/gitlab-runner/shells"
42)
43
44type featureFlagTest func(t *testing.T, flagName string, flagValue bool)
45
46func mustCreateResourceList(t *testing.T, cpu, memory, ephemeralStorage string) api.ResourceList {
47	resources, err := createResourceList(cpu, memory, ephemeralStorage)
48	require.NoError(t, err)
49
50	return resources
51}
52
53func TestRunTestsWithFeatureFlag(t *testing.T) {
54	tests := map[string]featureFlagTest{
55		"testVolumeMounts":                      testVolumeMountsFeatureFlag,
56		"testVolumes":                           testVolumesFeatureFlag,
57		"testSetupBuildPodServiceCreationError": testSetupBuildPodServiceCreationErrorFeatureFlag,
58		"testSetupBuildPodFailureGetPullPolicy": testSetupBuildPodFailureGetPullPolicyFeatureFlag,
59	}
60
61	featureFlags := []string{
62		featureflags.UseLegacyKubernetesExecutionStrategy,
63	}
64
65	for tn, tt := range tests {
66		for _, ff := range featureFlags {
67			t.Run(fmt.Sprintf("%s %s true", tn, ff), func(t *testing.T) {
68				tt(t, ff, true)
69			})
70
71			t.Run(fmt.Sprintf("%s %s false", tn, ff), func(t *testing.T) {
72				tt(t, ff, false)
73			})
74		}
75	}
76}
77
78func testVolumeMountsFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
79	tests := map[string]struct {
80		GlobalConfig *common.Config
81		RunnerConfig common.RunnerConfig
82		Build        *common.Build
83
84		Expected []api.VolumeMount
85	}{
86		"no custom volumes": {
87			GlobalConfig: &common.Config{},
88			RunnerConfig: common.RunnerConfig{
89				RunnerSettings: common.RunnerSettings{
90					Kubernetes: &common.KubernetesConfig{},
91				},
92			},
93			Build: &common.Build{
94				Runner: &common.RunnerConfig{},
95			},
96			Expected: []api.VolumeMount{
97				{Name: "repo"},
98			},
99		},
100		"custom volumes": {
101			GlobalConfig: &common.Config{},
102			RunnerConfig: common.RunnerConfig{
103				RunnerSettings: common.RunnerSettings{
104					Kubernetes: &common.KubernetesConfig{
105						Volumes: common.KubernetesVolumes{
106							HostPaths: []common.KubernetesHostPath{
107								{Name: "docker", MountPath: "/var/run/docker.sock", HostPath: "/var/run/docker.sock"},
108								{Name: "host-path", MountPath: "/path/two", HostPath: "/path/one"},
109								{
110									Name:      "host-subpath",
111									MountPath: "/subpath",
112									HostPath:  "/path/one",
113									SubPath:   "subpath",
114								},
115							},
116							Secrets: []common.KubernetesSecret{
117								{Name: "Secret", MountPath: "/path/to/whatever"},
118								{
119									Name:      "Secret-subpath",
120									MountPath: "/path/to/whatever",
121									SubPath:   "secret-subpath",
122								},
123							},
124							PVCs: []common.KubernetesPVC{
125								{Name: "PVC", MountPath: "/path/to/whatever"},
126								{
127									Name:      "PVC-subpath",
128									MountPath: "/path/to/whatever",
129									SubPath:   "PVC-subpath",
130								},
131							},
132							ConfigMaps: []common.KubernetesConfigMap{
133								{Name: "ConfigMap", MountPath: "/path/to/whatever"},
134								{
135									Name:      "ConfigMap-subpath",
136									MountPath: "/path/to/whatever",
137									SubPath:   "ConfigMap-subpath",
138								},
139							},
140							EmptyDirs: []common.KubernetesEmptyDir{
141								{Name: "emptyDir", MountPath: "/path/to/empty/dir"},
142								{
143									Name:      "emptyDir-subpath",
144									MountPath: "/subpath",
145									SubPath:   "empty-subpath",
146								},
147							},
148							CSIs: []common.KubernetesCSI{
149								{Name: "csi", MountPath: "/path/to/csi/volume", Driver: "some-driver"},
150								{
151									Name:      "csi-subpath",
152									MountPath: "/path/to/csi/volume",
153									Driver:    "some-driver",
154									SubPath:   "subpath",
155								},
156							},
157						},
158					},
159				},
160			},
161			Build: &common.Build{
162				Runner: &common.RunnerConfig{},
163			},
164			Expected: []api.VolumeMount{
165				{Name: "docker", MountPath: "/var/run/docker.sock"},
166				{Name: "host-path", MountPath: "/path/two"},
167				{Name: "host-subpath", MountPath: "/subpath", SubPath: "subpath"},
168				{Name: "Secret", MountPath: "/path/to/whatever"},
169				{Name: "Secret-subpath", MountPath: "/path/to/whatever", SubPath: "secret-subpath"},
170				{Name: "PVC", MountPath: "/path/to/whatever"},
171				{Name: "PVC-subpath", MountPath: "/path/to/whatever", SubPath: "PVC-subpath"},
172				{Name: "ConfigMap", MountPath: "/path/to/whatever"},
173				{Name: "ConfigMap-subpath", MountPath: "/path/to/whatever", SubPath: "ConfigMap-subpath"},
174				{Name: "emptyDir", MountPath: "/path/to/empty/dir"},
175				{Name: "emptyDir-subpath", MountPath: "/subpath", SubPath: "empty-subpath"},
176				{Name: "csi", MountPath: "/path/to/csi/volume"},
177				{Name: "csi-subpath", MountPath: "/path/to/csi/volume", SubPath: "subpath"},
178				{Name: "repo"},
179			},
180		},
181		"custom volumes with read-only settings": {
182			GlobalConfig: &common.Config{},
183			RunnerConfig: common.RunnerConfig{
184				RunnerSettings: common.RunnerSettings{
185					Kubernetes: &common.KubernetesConfig{
186						Volumes: common.KubernetesVolumes{
187							HostPaths: []common.KubernetesHostPath{
188								{
189									Name:      "test",
190									MountPath: "/opt/test/readonly",
191									ReadOnly:  true,
192									HostPath:  "/opt/test/rw",
193								},
194								{Name: "docker", MountPath: "/var/run/docker.sock"},
195							},
196							ConfigMaps: []common.KubernetesConfigMap{
197								{Name: "configMap", MountPath: "/path/to/configmap", ReadOnly: true},
198							},
199							Secrets: []common.KubernetesSecret{
200								{Name: "secret", MountPath: "/path/to/secret", ReadOnly: true},
201							},
202							CSIs: []common.KubernetesCSI{
203								{Name: "csi", MountPath: "/path/to/csi/volume", Driver: "some-driver", ReadOnly: true},
204							},
205						},
206					},
207				},
208			},
209			Build: &common.Build{
210				Runner: &common.RunnerConfig{},
211			},
212			Expected: []api.VolumeMount{
213				{Name: "test", MountPath: "/opt/test/readonly", ReadOnly: true},
214				{Name: "docker", MountPath: "/var/run/docker.sock"},
215				{Name: "secret", MountPath: "/path/to/secret", ReadOnly: true},
216				{Name: "configMap", MountPath: "/path/to/configmap", ReadOnly: true},
217				{Name: "csi", MountPath: "/path/to/csi/volume", ReadOnly: true},
218				{Name: "repo"},
219			},
220		},
221		"default volume with build dir": {
222			GlobalConfig: &common.Config{},
223			RunnerConfig: common.RunnerConfig{
224				RunnerSettings: common.RunnerSettings{
225					Kubernetes: &common.KubernetesConfig{
226						Volumes: common.KubernetesVolumes{},
227					},
228				},
229			},
230			Build: &common.Build{
231				RootDir: "/path/to/builds/dir",
232				Runner:  &common.RunnerConfig{},
233			},
234			Expected: []api.VolumeMount{
235				{
236					Name:      "repo",
237					MountPath: "/path/to/builds/dir",
238				},
239			},
240		},
241		"user-provided volume with build dir": {
242			GlobalConfig: &common.Config{},
243			RunnerConfig: common.RunnerConfig{
244				RunnerSettings: common.RunnerSettings{
245					Kubernetes: &common.KubernetesConfig{
246						Volumes: common.KubernetesVolumes{
247							HostPaths: []common.KubernetesHostPath{
248								{Name: "user-provided", MountPath: "/path/to/builds/dir"},
249							},
250						},
251					},
252				},
253			},
254			Build: &common.Build{
255				RootDir: "/path/to/builds/dir",
256				Runner:  &common.RunnerConfig{},
257			},
258			Expected: []api.VolumeMount{
259				{Name: "user-provided", MountPath: "/path/to/builds/dir"},
260			},
261		},
262	}
263
264	for tn, tt := range tests {
265		t.Run(tn, func(t *testing.T) {
266			e := &executor{
267				AbstractExecutor: executors.AbstractExecutor{
268					ExecutorOptions: executorOptions,
269					Build:           tt.Build,
270					Config:          tt.RunnerConfig,
271				},
272			}
273
274			buildtest.SetBuildFeatureFlag(e.Build, featureFlagName, featureFlagValue)
275			assert.Equal(t, tt.Expected, e.getVolumeMounts())
276		})
277	}
278}
279
280func testVolumesFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
281	csiVolFSType := "ext4"
282	csiVolReadOnly := false
283	mode := int32(0777)
284	optional := false
285	//nolint:lll
286	tests := map[string]struct {
287		GlobalConfig *common.Config
288		RunnerConfig common.RunnerConfig
289		Build        *common.Build
290
291		Expected []api.Volume
292	}{
293		"no custom volumes": {
294			GlobalConfig: &common.Config{},
295			RunnerConfig: common.RunnerConfig{
296				RunnerSettings: common.RunnerSettings{
297					Kubernetes: &common.KubernetesConfig{},
298				},
299			},
300			Build: &common.Build{
301				Runner: &common.RunnerConfig{},
302			},
303			Expected: []api.Volume{
304				{Name: "repo", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
305				{
306					Name: "scripts", VolumeSource: api.VolumeSource{
307						ConfigMap: &api.ConfigMapVolumeSource{
308							LocalObjectReference: api.LocalObjectReference{
309								Name: fakeConfigMap().Name,
310							},
311							DefaultMode: &mode,
312							Optional:    &optional,
313						},
314					},
315				},
316				{Name: "logs", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
317			},
318		},
319		"custom volumes": {
320			GlobalConfig: &common.Config{},
321			RunnerConfig: common.RunnerConfig{
322				RunnerSettings: common.RunnerSettings{
323					Kubernetes: &common.KubernetesConfig{
324						Volumes: common.KubernetesVolumes{
325							HostPaths: []common.KubernetesHostPath{
326								{Name: "docker", MountPath: "/var/run/docker.sock"},
327								{Name: "host-path", MountPath: "/path/two", HostPath: "/path/one"},
328								{
329									Name:      "host-subpath",
330									MountPath: "/subpath",
331									HostPath:  "/path/one",
332									SubPath:   "subpath",
333								},
334							},
335							PVCs: []common.KubernetesPVC{
336								{Name: "PVC", MountPath: "/path/to/whatever"},
337								{
338									Name:      "PVC-subpath",
339									MountPath: "/subpath",
340									SubPath:   "subpath",
341								},
342							},
343							ConfigMaps: []common.KubernetesConfigMap{
344								{Name: "ConfigMap", MountPath: "/path/to/config", Items: map[string]string{"key_1": "/path/to/key_1"}},
345								{
346									Name:      "ConfigMap-subpath",
347									MountPath: "/subpath",
348									Items:     map[string]string{"key_1": "/path/to/key_1"},
349									SubPath:   "subpath",
350								},
351							},
352							Secrets: []common.KubernetesSecret{
353								{Name: "secret", MountPath: "/path/to/secret", ReadOnly: true, Items: map[string]string{"secret_1": "/path/to/secret_1"}},
354								{
355									Name:      "secret-subpath",
356									MountPath: "/subpath",
357									ReadOnly:  true,
358									Items:     map[string]string{"secret_1": "/path/to/secret_1"},
359									SubPath:   "subpath",
360								},
361							},
362							EmptyDirs: []common.KubernetesEmptyDir{
363								{Name: "emptyDir", MountPath: "/path/to/empty/dir", Medium: "Memory"},
364								{
365									Name:      "emptyDir-subpath",
366									MountPath: "/subpath",
367									Medium:    "Memory",
368									SubPath:   "subpath",
369								},
370							},
371							CSIs: []common.KubernetesCSI{
372								{
373									Name:             "csi",
374									MountPath:        "/path/to/csi/volume",
375									Driver:           "some-driver",
376									FSType:           csiVolFSType,
377									ReadOnly:         csiVolReadOnly,
378									VolumeAttributes: map[string]string{"key": "value"},
379								},
380							},
381						},
382					},
383				},
384			},
385			Build: &common.Build{
386				Runner: &common.RunnerConfig{},
387			},
388			Expected: []api.Volume{
389				{Name: "docker", VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/var/run/docker.sock"}}},
390				{Name: "host-path", VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/path/one"}}},
391				{Name: "host-subpath", VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/path/one"}}},
392				{
393					Name: "secret",
394					VolumeSource: api.VolumeSource{
395						Secret: &api.SecretVolumeSource{
396							SecretName: "secret",
397							Items:      []api.KeyToPath{{Key: "secret_1", Path: "/path/to/secret_1"}},
398						},
399					},
400				},
401				{
402					Name: "secret-subpath",
403					VolumeSource: api.VolumeSource{
404						Secret: &api.SecretVolumeSource{
405							SecretName: "secret-subpath",
406							Items:      []api.KeyToPath{{Key: "secret_1", Path: "/path/to/secret_1"}},
407						},
408					},
409				},
410				{Name: "PVC", VolumeSource: api.VolumeSource{PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{ClaimName: "PVC"}}},
411				{Name: "PVC-subpath", VolumeSource: api.VolumeSource{PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{ClaimName: "PVC-subpath"}}},
412				{
413					Name: "ConfigMap",
414					VolumeSource: api.VolumeSource{
415						ConfigMap: &api.ConfigMapVolumeSource{
416							LocalObjectReference: api.LocalObjectReference{Name: "ConfigMap"},
417							Items:                []api.KeyToPath{{Key: "key_1", Path: "/path/to/key_1"}},
418						},
419					},
420				},
421				{
422					Name: "ConfigMap-subpath",
423					VolumeSource: api.VolumeSource{
424						ConfigMap: &api.ConfigMapVolumeSource{
425							LocalObjectReference: api.LocalObjectReference{Name: "ConfigMap-subpath"},
426							Items:                []api.KeyToPath{{Key: "key_1", Path: "/path/to/key_1"}},
427						},
428					},
429				},
430				{Name: "emptyDir", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{Medium: "Memory"}}},
431				{Name: "emptyDir-subpath", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{Medium: "Memory"}}},
432				{
433					Name: "csi",
434					VolumeSource: api.VolumeSource{
435						CSI: &api.CSIVolumeSource{
436							Driver:           "some-driver",
437							FSType:           &csiVolFSType,
438							ReadOnly:         &csiVolReadOnly,
439							VolumeAttributes: map[string]string{"key": "value"},
440						},
441					},
442				},
443				{Name: "repo", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
444				{
445					Name: "scripts", VolumeSource: api.VolumeSource{
446						ConfigMap: &api.ConfigMapVolumeSource{
447							LocalObjectReference: api.LocalObjectReference{
448								Name: fakeConfigMap().Name,
449							},
450							DefaultMode: &mode,
451							Optional:    &optional,
452						},
453					},
454				},
455				{Name: "logs", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
456			},
457		},
458		"default volume with build dir": {
459			GlobalConfig: &common.Config{},
460			RunnerConfig: common.RunnerConfig{
461				RunnerSettings: common.RunnerSettings{
462					Kubernetes: &common.KubernetesConfig{
463						Volumes: common.KubernetesVolumes{},
464					},
465				},
466			},
467			Build: &common.Build{
468				RootDir: "/path/to/builds/dir",
469				Runner:  &common.RunnerConfig{},
470			},
471			Expected: []api.Volume{
472				{Name: "repo", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
473				{
474					Name: "scripts", VolumeSource: api.VolumeSource{
475						ConfigMap: &api.ConfigMapVolumeSource{
476							LocalObjectReference: api.LocalObjectReference{
477								Name: fakeConfigMap().Name,
478							},
479							DefaultMode: &mode,
480							Optional:    &optional,
481						},
482					},
483				},
484				{Name: "logs", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
485			},
486		},
487		"user-provided volume with build dir": {
488			GlobalConfig: &common.Config{},
489			RunnerConfig: common.RunnerConfig{
490				RunnerSettings: common.RunnerSettings{
491					Kubernetes: &common.KubernetesConfig{
492						Volumes: common.KubernetesVolumes{
493							HostPaths: []common.KubernetesHostPath{
494								{Name: "user-provided", MountPath: "/path/to/builds/dir"},
495							},
496						},
497					},
498				},
499			},
500			Build: &common.Build{
501				RootDir: "/path/to/builds/dir",
502				Runner:  &common.RunnerConfig{},
503			},
504			Expected: []api.Volume{
505				{Name: "user-provided", VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/path/to/builds/dir"}}},
506				{
507					Name: "scripts", VolumeSource: api.VolumeSource{
508						ConfigMap: &api.ConfigMapVolumeSource{
509							LocalObjectReference: api.LocalObjectReference{
510								Name: fakeConfigMap().Name,
511							},
512							DefaultMode: &mode,
513							Optional:    &optional,
514						},
515					},
516				},
517				{Name: "logs", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
518			},
519		},
520	}
521
522	for tn, tt := range tests {
523		t.Run(tn, func(t *testing.T) {
524			e := &executor{
525				AbstractExecutor: executors.AbstractExecutor{
526					ExecutorOptions: executorOptions,
527					Build:           tt.Build,
528					Config:          tt.RunnerConfig,
529				},
530				configMap: fakeConfigMap(),
531			}
532
533			buildtest.SetBuildFeatureFlag(e.Build, featureFlagName, featureFlagValue)
534			assert.Equal(t, tt.Expected, e.getVolumes())
535		})
536	}
537}
538
539func testSetupBuildPodServiceCreationErrorFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
540	version, _ := testVersionAndCodec()
541	helperImageInfo, err := helperimage.Get(common.REVISION, helperimage.Config{
542		OSType:       helperimage.OSTypeLinux,
543		Architecture: "amd64",
544	})
545	require.NoError(t, err)
546
547	runnerConfig := common.RunnerConfig{
548		RunnerSettings: common.RunnerSettings{
549			Kubernetes: &common.KubernetesConfig{
550				Namespace:   "default",
551				HelperImage: "custom/helper-image",
552			},
553		},
554	}
555
556	fakeRoundTripper := func(req *http.Request) (*http.Response, error) {
557		body, errRT := ioutil.ReadAll(req.Body)
558		if !assert.NoError(t, errRT, "failed to read request body") {
559			return nil, errRT
560		}
561
562		p := new(api.Pod)
563		errRT = json.Unmarshal(body, p)
564		if !assert.NoError(t, errRT, "failed to read request body") {
565			return nil, errRT
566		}
567
568		if req.URL.Path == "/api/v1/namespaces/default/services" {
569			return nil, fmt.Errorf("foobar")
570		}
571
572		resp := &http.Response{
573			StatusCode: http.StatusOK,
574			Body: FakeReadCloser{
575				Reader: bytes.NewBuffer(body),
576			},
577		}
578		resp.Header = make(http.Header)
579		resp.Header.Add("Content-Type", "application/json")
580
581		return resp, nil
582	}
583
584	mockFc := &mockFeatureChecker{}
585	mockFc.On("IsHostAliasSupported").Return(true, nil)
586	mockPullManager := &pull.MockManager{}
587	defer mockPullManager.AssertExpectations(t)
588	ex := executor{
589		kubeClient: testKubernetesClient(version, fake.CreateHTTPClient(fakeRoundTripper)),
590		options: &kubernetesOptions{
591			Image: common.Image{
592				Name:  "test-image",
593				Ports: []common.Port{{Number: 80}},
594			},
595			Services: common.Services{
596				{
597					Name:  "test-service",
598					Alias: "custom_name",
599					Ports: []common.Port{
600						{
601							Number:   81,
602							Name:     "custom_port_name",
603							Protocol: "http",
604						},
605					},
606				},
607			},
608		},
609		AbstractExecutor: executors.AbstractExecutor{
610			Config:     runnerConfig,
611			BuildShell: &common.ShellConfiguration{},
612			Build: &common.Build{
613				JobResponse: common.JobResponse{
614					Variables: []common.JobVariable{},
615				},
616				Runner: &runnerConfig,
617			},
618			ProxyPool: proxy.NewPool(),
619		},
620		helperImageInfo: helperImageInfo,
621		featureChecker:  mockFc,
622		configMap:       fakeConfigMap(),
623		pullManager:     mockPullManager,
624	}
625	buildtest.SetBuildFeatureFlag(ex.Build, featureFlagName, featureFlagValue)
626
627	mockPullManager.On("GetPullPolicyFor", ex.options.Services[0].Name).
628		Return(api.PullAlways, nil).
629		Once()
630	mockPullManager.On("GetPullPolicyFor", ex.options.Image.Name).
631		Return(api.PullAlways, nil).
632		Once()
633	mockPullManager.On("GetPullPolicyFor", runnerConfig.RunnerSettings.Kubernetes.HelperImage).
634		Return(api.PullAlways, nil).
635		Once()
636
637	err = ex.prepareOverwrites(make(common.JobVariables, 0))
638	assert.NoError(t, err)
639
640	err = ex.setupBuildPod(nil)
641	assert.Error(t, err)
642	assert.Contains(t, err.Error(), "error creating the proxy service")
643}
644
645func testSetupBuildPodFailureGetPullPolicyFeatureFlag(t *testing.T, featureFlagName string, featureFlagValue bool) {
646	for _, failOnImage := range []string{
647		"test-service",
648		"test-helper",
649		"test-build",
650	} {
651		t.Run(failOnImage, func(t *testing.T) {
652			runnerConfig := common.RunnerConfig{
653				RunnerSettings: common.RunnerSettings{
654					Kubernetes: &common.KubernetesConfig{
655						HelperImage: "test-helper",
656					},
657				},
658			}
659
660			mockFc := &mockFeatureChecker{}
661			defer mockFc.AssertExpectations(t)
662			mockFc.On("IsHostAliasSupported").Return(true, nil).Maybe()
663
664			mockPullManager := &pull.MockManager{}
665			defer mockPullManager.AssertExpectations(t)
666
667			e := executor{
668				options: &kubernetesOptions{
669					Image: common.Image{
670						Name: "test-build",
671					},
672					Services: common.Services{
673						{
674							Name: "test-service",
675						},
676					},
677				},
678				AbstractExecutor: executors.AbstractExecutor{
679					Config:     runnerConfig,
680					BuildShell: &common.ShellConfiguration{},
681					Build: &common.Build{
682						JobResponse: common.JobResponse{},
683						Runner:      &runnerConfig,
684					},
685				},
686				featureChecker: mockFc,
687				pullManager:    mockPullManager,
688			}
689			buildtest.SetBuildFeatureFlag(e.Build, featureFlagName, featureFlagValue)
690
691			mockPullManager.On("GetPullPolicyFor", failOnImage).
692				Return(api.PullAlways, assert.AnError).
693				Once()
694
695			mockPullManager.On("GetPullPolicyFor", mock.Anything).
696				Return(api.PullAlways, nil).
697				Maybe()
698
699			err := e.prepareOverwrites(make(common.JobVariables, 0))
700			assert.NoError(t, err)
701
702			err = e.setupBuildPod(nil)
703			assert.ErrorIs(t, err, assert.AnError)
704			assert.Error(t, err)
705		})
706	}
707}
708
709func TestCleanup(t *testing.T) {
710	version, _ := testVersionAndCodec()
711	objectMeta := metav1.ObjectMeta{Name: "test-resource", Namespace: "test-ns"}
712	podsEndpointURI := "/api/" + version + "/namespaces/" + objectMeta.Namespace + "/pods/" + objectMeta.Name
713	servicesEndpointURI := "/api/" + version + "/namespaces/" + objectMeta.Namespace + "/services/" + objectMeta.Name
714	secretsEndpointURI := "/api/" + version + "/namespaces/" + objectMeta.Namespace + "/secrets/" + objectMeta.Name
715	configMapsEndpointURI :=
716		"/api/" + version + "/namespaces/" + objectMeta.Namespace + "/configmaps/" + objectMeta.Name
717
718	tests := []struct {
719		Name        string
720		Pod         *api.Pod
721		ConfigMap   *api.ConfigMap
722		Credentials *api.Secret
723		ClientFunc  func(*http.Request) (*http.Response, error)
724		Services    []api.Service
725		Error       bool
726	}{
727		{
728			Name: "Proper Cleanup",
729			Pod:  &api.Pod{ObjectMeta: objectMeta},
730			ClientFunc: func(req *http.Request) (*http.Response, error) {
731				switch p, m := req.URL.Path, req.Method; {
732				case m == http.MethodDelete && p == podsEndpointURI:
733					return fakeKubeDeleteResponse(http.StatusOK), nil
734				default:
735					return nil, fmt.Errorf("unexpected request. method: %s, path: %s", m, p)
736				}
737			},
738		},
739		{
740			Name: "Delete failure",
741			Pod:  &api.Pod{ObjectMeta: objectMeta},
742			ClientFunc: func(req *http.Request) (*http.Response, error) {
743				return nil, fmt.Errorf("delete failed")
744			},
745			Error: true,
746		},
747		{
748			Name: "POD already deleted",
749			Pod:  &api.Pod{ObjectMeta: objectMeta},
750			ClientFunc: func(req *http.Request) (*http.Response, error) {
751				switch p, m := req.URL.Path, req.Method; {
752				case m == http.MethodDelete && p == podsEndpointURI:
753					return fakeKubeDeleteResponse(http.StatusNotFound), nil
754				default:
755					return nil, fmt.Errorf("unexpected request. method: %s, path: %s", m, p)
756				}
757			},
758			Error: true,
759		},
760		{
761			Name:        "POD creation failed, Secrets provided",
762			Pod:         nil, // a failed POD create request will cause a nil Pod
763			Credentials: &api.Secret{ObjectMeta: objectMeta},
764			ClientFunc: func(req *http.Request) (*http.Response, error) {
765				switch p, m := req.URL.Path, req.Method; {
766				case m == http.MethodDelete && p == secretsEndpointURI:
767					return fakeKubeDeleteResponse(http.StatusNotFound), nil
768				default:
769					return nil, fmt.Errorf("unexpected request. method: %s, path: %s", m, p)
770				}
771			},
772			Error: true,
773		},
774		{
775			Name:     "POD created, Services created",
776			Pod:      &api.Pod{ObjectMeta: objectMeta},
777			Services: []api.Service{{ObjectMeta: objectMeta}},
778			ClientFunc: func(req *http.Request) (*http.Response, error) {
779				switch p, m := req.URL.Path, req.Method; {
780				case m == http.MethodDelete && ((p == servicesEndpointURI) || (p == podsEndpointURI)):
781					return fakeKubeDeleteResponse(http.StatusOK), nil
782				default:
783					return nil, fmt.Errorf("unexpected request. method: %s, path: %s", m, p)
784				}
785			},
786		},
787		{
788			Name:     "POD created, Services creation failed",
789			Pod:      &api.Pod{ObjectMeta: objectMeta},
790			Services: []api.Service{{ObjectMeta: objectMeta}},
791			ClientFunc: func(req *http.Request) (*http.Response, error) {
792				switch p, m := req.URL.Path, req.Method; {
793				case m == http.MethodDelete && p == servicesEndpointURI:
794					return fakeKubeDeleteResponse(http.StatusNotFound), nil
795				case m == http.MethodDelete && p == podsEndpointURI:
796					return fakeKubeDeleteResponse(http.StatusOK), nil
797				default:
798					return nil, fmt.Errorf("unexpected request. method: %s, path: %s", m, p)
799				}
800			},
801			Error: true,
802		},
803		{
804			Name:     "POD creation failed, Services created",
805			Pod:      nil, // a failed POD create request will cause a nil Pod
806			Services: []api.Service{{ObjectMeta: objectMeta}},
807			ClientFunc: func(req *http.Request) (*http.Response, error) {
808				switch p, m := req.URL.Path, req.Method; {
809				case m == http.MethodDelete && p == servicesEndpointURI:
810					return fakeKubeDeleteResponse(http.StatusOK), nil
811				default:
812					return nil, fmt.Errorf("unexpected request. method: %s, path: %s", m, p)
813				}
814			},
815		},
816		{
817			Name:     "POD creation failed, Services cleanup failed",
818			Pod:      nil, // a failed POD create request will cause a nil Pod
819			Services: []api.Service{{ObjectMeta: objectMeta}},
820			ClientFunc: func(req *http.Request) (*http.Response, error) {
821				switch p, m := req.URL.Path, req.Method; {
822				case m == http.MethodDelete && p == servicesEndpointURI:
823					return fakeKubeDeleteResponse(http.StatusNotFound), nil
824				default:
825					return nil, fmt.Errorf("unexpected request. method: %s, path: %s", m, p)
826				}
827			},
828			Error: true,
829		},
830		{
831			Name:      "ConfigMap cleanup",
832			ConfigMap: &api.ConfigMap{ObjectMeta: objectMeta},
833			ClientFunc: func(req *http.Request) (*http.Response, error) {
834				switch p, m := req.URL.Path, req.Method; {
835				case m == http.MethodDelete && p == configMapsEndpointURI:
836					return fakeKubeDeleteResponse(http.StatusOK), nil
837				default:
838					return nil, fmt.Errorf("unexpected request. method: %s, path: %s", m, p)
839				}
840			},
841		},
842		{
843			Name:      "ConfigMap cleanup failed",
844			ConfigMap: &api.ConfigMap{ObjectMeta: objectMeta},
845			ClientFunc: func(req *http.Request) (*http.Response, error) {
846				switch p, m := req.URL.Path, req.Method; {
847				case m == http.MethodDelete && p == configMapsEndpointURI:
848					return fakeKubeDeleteResponse(http.StatusNotFound), nil
849				default:
850					return nil, fmt.Errorf("unexpected request. method: %s, path: %s", m, p)
851				}
852			},
853			Error: true,
854		},
855	}
856
857	for _, test := range tests {
858		t.Run(test.Name, func(t *testing.T) {
859			ex := executor{
860				kubeClient:  testKubernetesClient(version, fake.CreateHTTPClient(test.ClientFunc)),
861				pod:         test.Pod,
862				credentials: test.Credentials,
863				services:    test.Services,
864				configMap:   test.ConfigMap,
865			}
866			ex.configurationOverwrites = &overwrites{namespace: "test-ns"}
867
868			errored := false
869			buildTrace := FakeBuildTrace{
870				testWriter: testWriter{
871					call: func(b []byte) (int, error) {
872						if !errored {
873							if s := string(b); strings.Contains(s, "Error cleaning up") {
874								errored = true
875							} else if test.Error {
876								t.Errorf("expected failure. got: '%s'", string(b))
877							}
878						}
879						return len(b), nil
880					},
881				},
882			}
883			ex.AbstractExecutor.Trace = buildTrace
884			ex.AbstractExecutor.BuildLogger = common.NewBuildLogger(buildTrace, logrus.WithFields(logrus.Fields{}))
885
886			ex.Cleanup()
887
888			if test.Error && !errored {
889				t.Errorf("expected cleanup to fail but it didn't")
890			} else if !test.Error && errored {
891				t.Errorf("expected cleanup not to fail but it did")
892			}
893		})
894	}
895}
896
897func TestPrepare(t *testing.T) {
898	helperImageTag := "latest"
899	// common.REVISION is overridden at build time.
900	if common.REVISION != "HEAD" {
901		helperImageTag = common.REVISION
902	}
903
904	defaultHelperImage := helperimage.Info{
905		Architecture:            "x86_64",
906		Name:                    helperimage.GitLabRegistryName,
907		Tag:                     fmt.Sprintf("x86_64-%s", helperImageTag),
908		IsSupportingLocalImport: true,
909		Cmd:                     []string{"gitlab-runner-build"},
910	}
911	os := helperimage.OSTypeLinux
912	if runtime.GOOS == helperimage.OSTypeWindows {
913		os = helperimage.OSTypeWindows
914	}
915	pwshHelperImage, err := helperimage.Get(helperImageTag, helperimage.Config{
916		Architecture:   "x86_64",
917		OSType:         os,
918		Shell:          shells.SNPwsh,
919		GitLabRegistry: true,
920	})
921	require.NoError(t, err)
922
923	tests := []struct {
924		Name  string
925		Error string
926
927		GlobalConfig *common.Config
928		RunnerConfig *common.RunnerConfig
929		Build        *common.Build
930
931		Expected           *executor
932		ExpectedPullPolicy api.PullPolicy
933	}{
934		{
935			Name:         "all with limits",
936			GlobalConfig: &common.Config{},
937			RunnerConfig: &common.RunnerConfig{
938				RunnerSettings: common.RunnerSettings{
939					Kubernetes: &common.KubernetesConfig{
940						Host:                         "test-server",
941						ServiceCPULimit:              "100m",
942						ServiceMemoryLimit:           "200Mi",
943						ServiceEphemeralStorageLimit: "1Gi",
944						CPULimit:                     "1.5",
945						MemoryLimit:                  "4Gi",
946						EphemeralStorageLimit:        "6Gi",
947						HelperCPULimit:               "50m",
948						HelperMemoryLimit:            "100Mi",
949						HelperEphemeralStorageLimit:  "200Mi",
950						Privileged:                   true,
951						PullPolicy:                   common.StringOrArray{"if-not-present"},
952					},
953				},
954			},
955			Build: &common.Build{
956				JobResponse: common.JobResponse{
957					GitInfo: common.GitInfo{
958						Sha: "1234567890",
959					},
960					Image: common.Image{
961						Name: "test-image",
962					},
963					Variables: []common.JobVariable{
964						{Key: "privileged", Value: "true"},
965					},
966				},
967				Runner: &common.RunnerConfig{},
968			},
969			Expected: &executor{
970				options: &kubernetesOptions{
971					Image: common.Image{
972						Name: "test-image",
973					},
974				},
975				configurationOverwrites: &overwrites{
976					namespace:       "default",
977					buildLimits:     mustCreateResourceList(t, "1.5", "4Gi", "6Gi"),
978					serviceLimits:   mustCreateResourceList(t, "100m", "200Mi", "1Gi"),
979					helperLimits:    mustCreateResourceList(t, "50m", "100Mi", "200Mi"),
980					buildRequests:   api.ResourceList{},
981					serviceRequests: api.ResourceList{},
982					helperRequests:  api.ResourceList{},
983				},
984				helperImageInfo: defaultHelperImage,
985			},
986			ExpectedPullPolicy: api.PullIfNotPresent,
987		},
988		{
989			Name:         "all with limits and requests",
990			GlobalConfig: &common.Config{},
991			RunnerConfig: &common.RunnerConfig{
992				RunnerSettings: common.RunnerSettings{
993					Kubernetes: &common.KubernetesConfig{
994						Host:                           "test-server",
995						ServiceAccount:                 "default",
996						ServiceAccountOverwriteAllowed: ".*",
997						BearerTokenOverwriteAllowed:    true,
998						ServiceCPULimit:                "100m",
999						ServiceMemoryLimit:             "200Mi",
1000						ServiceEphemeralStorageLimit:   "2Gi",
1001						CPULimit:                       "1.5",
1002						MemoryLimit:                    "4Gi",
1003						EphemeralStorageLimit:          "3Gi",
1004						HelperCPULimit:                 "50m",
1005						HelperMemoryLimit:              "100Mi",
1006						HelperEphemeralStorageLimit:    "300Mi",
1007						ServiceCPURequest:              "99m",
1008						ServiceMemoryRequest:           "5Mi",
1009						ServiceEphemeralStorageRequest: "200Mi",
1010						CPURequest:                     "1",
1011						MemoryRequest:                  "1.5Gi",
1012						EphemeralStorageRequest:        "1.3Gi",
1013						HelperCPURequest:               "0.5m",
1014						HelperMemoryRequest:            "42Mi",
1015						HelperEphemeralStorageRequest:  "99Mi",
1016						Privileged:                     false,
1017					},
1018				},
1019			},
1020			Build: &common.Build{
1021				JobResponse: common.JobResponse{
1022					GitInfo: common.GitInfo{
1023						Sha: "1234567890",
1024					},
1025					Image: common.Image{
1026						Name: "test-image",
1027					},
1028					Variables: []common.JobVariable{
1029						{Key: ServiceAccountOverwriteVariableName, Value: "not-default"},
1030					},
1031				},
1032				Runner: &common.RunnerConfig{},
1033			},
1034			Expected: &executor{
1035				options: &kubernetesOptions{
1036					Image: common.Image{
1037						Name: "test-image",
1038					},
1039				},
1040				configurationOverwrites: &overwrites{
1041					namespace:       "default",
1042					serviceAccount:  "not-default",
1043					buildLimits:     mustCreateResourceList(t, "1.5", "4Gi", "3Gi"),
1044					buildRequests:   mustCreateResourceList(t, "1", "1.5Gi", "1.3Gi"),
1045					serviceLimits:   mustCreateResourceList(t, "100m", "200Mi", "2Gi"),
1046					serviceRequests: mustCreateResourceList(t, "99m", "5Mi", "200Mi"),
1047					helperLimits:    mustCreateResourceList(t, "50m", "100Mi", "300Mi"),
1048					helperRequests:  mustCreateResourceList(t, "0.5m", "42Mi", "99Mi"),
1049				},
1050				helperImageInfo: defaultHelperImage,
1051			},
1052		},
1053		{
1054			Name:         "unmatched service account",
1055			Error:        "couldn't prepare overwrites: provided value \"not-default\" does not match \"allowed-.*\"",
1056			GlobalConfig: &common.Config{},
1057			RunnerConfig: &common.RunnerConfig{
1058				RunnerSettings: common.RunnerSettings{
1059					Kubernetes: &common.KubernetesConfig{
1060						Host:                           "test-server",
1061						ServiceAccount:                 "default",
1062						ServiceAccountOverwriteAllowed: "allowed-.*",
1063						ServiceCPULimit:                "100m",
1064						ServiceMemoryLimit:             "200Mi",
1065						ServiceEphemeralStorageLimit:   "300Mi",
1066						CPULimit:                       "1.5",
1067						MemoryLimit:                    "4Gi",
1068						EphemeralStorageLimit:          "5Gi",
1069						HelperCPULimit:                 "50m",
1070						HelperMemoryLimit:              "100Mi",
1071						HelperEphemeralStorageLimit:    "200Mi",
1072						ServiceCPURequest:              "99m",
1073						ServiceMemoryRequest:           "5Mi",
1074						ServiceEphemeralStorageRequest: "50Mi",
1075						CPURequest:                     "1",
1076						MemoryRequest:                  "1.5Gi",
1077						EphemeralStorageRequest:        "40Mi",
1078						HelperCPURequest:               "0.5m",
1079						HelperMemoryRequest:            "42Mi",
1080						HelperEphemeralStorageRequest:  "52Mi",
1081						Privileged:                     false,
1082					},
1083				},
1084			},
1085			Build: &common.Build{
1086				JobResponse: common.JobResponse{
1087					GitInfo: common.GitInfo{
1088						Sha: "1234567890",
1089					},
1090					Image: common.Image{
1091						Name: "test-image",
1092					},
1093					Variables: []common.JobVariable{
1094						{Key: ServiceAccountOverwriteVariableName, Value: "not-default"},
1095					},
1096				},
1097				Runner: &common.RunnerConfig{},
1098			},
1099		},
1100		{
1101			Name:         "regexp match on service account and namespace",
1102			GlobalConfig: &common.Config{},
1103			RunnerConfig: &common.RunnerConfig{
1104				RunnerSettings: common.RunnerSettings{
1105					Kubernetes: &common.KubernetesConfig{
1106						Host:                           "test-server",
1107						Namespace:                      "namespace",
1108						ServiceAccount:                 "a_service_account",
1109						ServiceAccountOverwriteAllowed: ".*",
1110						NamespaceOverwriteAllowed:      "^n.*?e$",
1111						ServiceCPULimit:                "100m",
1112						ServiceMemoryLimit:             "200Mi",
1113						ServiceEphemeralStorageLimit:   "300Mi",
1114						CPULimit:                       "1.5",
1115						MemoryLimit:                    "4Gi",
1116						EphemeralStorageLimit:          "5Gi",
1117						HelperCPULimit:                 "50m",
1118						HelperMemoryLimit:              "100Mi",
1119						HelperEphemeralStorageLimit:    "300Mi",
1120						ServiceCPURequest:              "99m",
1121						ServiceMemoryRequest:           "5Mi",
1122						ServiceEphemeralStorageRequest: "15Mi",
1123						CPURequest:                     "1",
1124						MemoryRequest:                  "1.5Gi",
1125						EphemeralStorageRequest:        "1.7Gi",
1126						HelperCPURequest:               "0.5m",
1127						HelperMemoryRequest:            "42Mi",
1128						HelperEphemeralStorageRequest:  "32Mi",
1129						Privileged:                     false,
1130					},
1131				},
1132			},
1133			Build: &common.Build{
1134				JobResponse: common.JobResponse{
1135					GitInfo: common.GitInfo{
1136						Sha: "1234567890",
1137					},
1138					Image: common.Image{
1139						Name: "test-image",
1140					},
1141					Variables: []common.JobVariable{
1142						{Key: NamespaceOverwriteVariableName, Value: "new-namespace-name"},
1143					},
1144				},
1145				Runner: &common.RunnerConfig{},
1146			},
1147			Expected: &executor{
1148				options: &kubernetesOptions{
1149					Image: common.Image{
1150						Name: "test-image",
1151					},
1152				},
1153				configurationOverwrites: &overwrites{
1154					namespace:       "new-namespace-name",
1155					serviceAccount:  "a_service_account",
1156					buildLimits:     mustCreateResourceList(t, "1.5", "4Gi", "5Gi"),
1157					buildRequests:   mustCreateResourceList(t, "1", "1.5Gi", "1.7Gi"),
1158					serviceLimits:   mustCreateResourceList(t, "100m", "200Mi", "300Mi"),
1159					serviceRequests: mustCreateResourceList(t, "99m", "5Mi", "15Mi"),
1160					helperLimits:    mustCreateResourceList(t, "50m", "100Mi", "300Mi"),
1161					helperRequests:  mustCreateResourceList(t, "0.5m", "42Mi", "32Mi"),
1162				},
1163				helperImageInfo: defaultHelperImage,
1164			},
1165		},
1166		{
1167			Name:         "regexp match on namespace",
1168			GlobalConfig: &common.Config{},
1169			RunnerConfig: &common.RunnerConfig{
1170				RunnerSettings: common.RunnerSettings{
1171					Kubernetes: &common.KubernetesConfig{
1172						Namespace:                 "namespace",
1173						Host:                      "test-server",
1174						NamespaceOverwriteAllowed: "^namespace-[0-9]$",
1175					},
1176				},
1177			},
1178			Build: &common.Build{
1179				JobResponse: common.JobResponse{
1180					GitInfo: common.GitInfo{
1181						Sha: "1234567890",
1182					},
1183					Image: common.Image{
1184						Name: "test-image",
1185					},
1186					Variables: []common.JobVariable{
1187						{Key: NamespaceOverwriteVariableName, Value: "namespace-$CI_CONCURRENT_ID"},
1188					},
1189				},
1190				Runner: &common.RunnerConfig{},
1191			},
1192			Expected: &executor{
1193				options: &kubernetesOptions{
1194					Image: common.Image{
1195						Name: "test-image",
1196					},
1197				},
1198				configurationOverwrites: &overwrites{
1199					namespace:       "namespace-0",
1200					serviceLimits:   api.ResourceList{},
1201					buildLimits:     api.ResourceList{},
1202					helperLimits:    api.ResourceList{},
1203					serviceRequests: api.ResourceList{},
1204					buildRequests:   api.ResourceList{},
1205					helperRequests:  api.ResourceList{},
1206				},
1207				helperImageInfo: defaultHelperImage,
1208			},
1209		},
1210		{
1211			Name:         "minimal configuration",
1212			GlobalConfig: &common.Config{},
1213			RunnerConfig: &common.RunnerConfig{
1214				RunnerSettings: common.RunnerSettings{
1215					Kubernetes: &common.KubernetesConfig{
1216						Image: "test-image",
1217						Host:  "test-server",
1218					},
1219				},
1220			},
1221			Build: &common.Build{
1222				JobResponse: common.JobResponse{
1223					GitInfo: common.GitInfo{
1224						Sha: "1234567890",
1225					},
1226				},
1227				Runner: &common.RunnerConfig{},
1228			},
1229			Expected: &executor{
1230				options: &kubernetesOptions{
1231					Image: common.Image{
1232						Name: "test-image",
1233					},
1234				},
1235				configurationOverwrites: &overwrites{
1236					namespace:       "default",
1237					serviceLimits:   api.ResourceList{},
1238					buildLimits:     api.ResourceList{},
1239					helperLimits:    api.ResourceList{},
1240					serviceRequests: api.ResourceList{},
1241					buildRequests:   api.ResourceList{},
1242					helperRequests:  api.ResourceList{},
1243				},
1244				helperImageInfo: defaultHelperImage,
1245			},
1246		},
1247		{
1248			Name:         "minimal configuration with pwsh shell",
1249			GlobalConfig: &common.Config{},
1250			RunnerConfig: &common.RunnerConfig{
1251				RunnerSettings: common.RunnerSettings{
1252					Shell: shells.SNPwsh,
1253					Kubernetes: &common.KubernetesConfig{
1254						Image: "test-image",
1255						Host:  "test-server",
1256					},
1257				},
1258			},
1259			Build: &common.Build{
1260				JobResponse: common.JobResponse{
1261					GitInfo: common.GitInfo{
1262						Sha: "1234567890",
1263					},
1264				},
1265				Runner: &common.RunnerConfig{},
1266			},
1267			Expected: &executor{
1268				options: &kubernetesOptions{
1269					Image: common.Image{
1270						Name: "test-image",
1271					},
1272				},
1273				configurationOverwrites: &overwrites{
1274					namespace:       "default",
1275					serviceLimits:   api.ResourceList{},
1276					buildLimits:     api.ResourceList{},
1277					helperLimits:    api.ResourceList{},
1278					serviceRequests: api.ResourceList{},
1279					buildRequests:   api.ResourceList{},
1280					helperRequests:  api.ResourceList{},
1281				},
1282				helperImageInfo: pwshHelperImage,
1283			},
1284		},
1285		{
1286			Name:         "image and one service",
1287			GlobalConfig: &common.Config{},
1288			RunnerConfig: &common.RunnerConfig{
1289				RunnerSettings: common.RunnerSettings{
1290					Kubernetes: &common.KubernetesConfig{
1291						Host: "test-server",
1292					},
1293				},
1294			},
1295			Build: &common.Build{
1296				JobResponse: common.JobResponse{
1297					GitInfo: common.GitInfo{
1298						Sha: "1234567890",
1299					},
1300					Image: common.Image{
1301						Name:       "test-image",
1302						Entrypoint: []string{"/init", "run"},
1303					},
1304					Services: common.Services{
1305						{
1306							Name:       "test-service",
1307							Entrypoint: []string{"/init", "run"},
1308							Command:    []string{"application", "--debug"},
1309						},
1310					},
1311				},
1312				Runner: &common.RunnerConfig{},
1313			},
1314			Expected: &executor{
1315				options: &kubernetesOptions{
1316					Image: common.Image{
1317						Name:       "test-image",
1318						Entrypoint: []string{"/init", "run"},
1319					},
1320					Services: common.Services{
1321						{
1322							Name:       "test-service",
1323							Entrypoint: []string{"/init", "run"},
1324							Command:    []string{"application", "--debug"},
1325						},
1326					},
1327				},
1328				configurationOverwrites: &overwrites{
1329					namespace:       "default",
1330					serviceLimits:   api.ResourceList{},
1331					buildLimits:     api.ResourceList{},
1332					helperLimits:    api.ResourceList{},
1333					serviceRequests: api.ResourceList{},
1334					buildRequests:   api.ResourceList{},
1335					helperRequests:  api.ResourceList{},
1336				},
1337				helperImageInfo: defaultHelperImage,
1338			},
1339		},
1340		{
1341			Name:         "merge services",
1342			GlobalConfig: &common.Config{},
1343			RunnerConfig: &common.RunnerConfig{
1344				RunnerSettings: common.RunnerSettings{
1345					Kubernetes: &common.KubernetesConfig{
1346						Host: "test-server",
1347						Services: []common.Service{
1348							{Name: "test-service-k8s", Alias: "alias"},
1349							{Name: "test-service-k8s2"},
1350							{Name: ""},
1351							{
1352								Name:    "test-service-k8s3",
1353								Command: []string{"executable", "param1", "param2"},
1354							},
1355							{
1356								Name:       "test-service-k8s4",
1357								Entrypoint: []string{"executable", "param3", "param4"},
1358							},
1359							{
1360								Name:       "test-service-k8s5",
1361								Alias:      "alias5",
1362								Command:    []string{"executable", "param1", "param2"},
1363								Entrypoint: []string{"executable", "param3", "param4"},
1364							},
1365						},
1366					},
1367				},
1368			},
1369			Build: &common.Build{
1370				JobResponse: common.JobResponse{
1371					GitInfo: common.GitInfo{
1372						Sha: "1234567890",
1373					},
1374					Image: common.Image{
1375						Name:       "test-image",
1376						Entrypoint: []string{"/init", "run"},
1377					},
1378					Services: common.Services{
1379						{
1380							Name:       "test-service",
1381							Alias:      "test-alias",
1382							Entrypoint: []string{"/init", "run"},
1383							Command:    []string{"application", "--debug"},
1384						},
1385						{
1386							Name: "",
1387						},
1388					},
1389				},
1390				Runner: &common.RunnerConfig{},
1391			},
1392			Expected: &executor{
1393				options: &kubernetesOptions{
1394					Image: common.Image{
1395						Name:       "test-image",
1396						Entrypoint: []string{"/init", "run"},
1397					},
1398					Services: common.Services{
1399						{
1400							Name:  "test-service-k8s",
1401							Alias: "alias",
1402						},
1403						{
1404							Name: "test-service-k8s2",
1405						},
1406						{
1407							Name:    "test-service-k8s3",
1408							Command: []string{"executable", "param1", "param2"},
1409						},
1410						{
1411							Name:       "test-service-k8s4",
1412							Entrypoint: []string{"executable", "param3", "param4"},
1413						},
1414						{
1415							Name:       "test-service-k8s5",
1416							Alias:      "alias5",
1417							Command:    []string{"executable", "param1", "param2"},
1418							Entrypoint: []string{"executable", "param3", "param4"},
1419						},
1420						{
1421							Name:       "test-service",
1422							Alias:      "test-alias",
1423							Entrypoint: []string{"/init", "run"},
1424							Command:    []string{"application", "--debug"},
1425						},
1426					},
1427				},
1428				configurationOverwrites: &overwrites{
1429					namespace:       "default",
1430					serviceLimits:   api.ResourceList{},
1431					buildLimits:     api.ResourceList{},
1432					helperLimits:    api.ResourceList{},
1433					serviceRequests: api.ResourceList{},
1434					buildRequests:   api.ResourceList{},
1435					helperRequests:  api.ResourceList{},
1436				},
1437				helperImageInfo: defaultHelperImage,
1438			},
1439		},
1440		{
1441			Name:         "Default helper image",
1442			GlobalConfig: &common.Config{},
1443			RunnerConfig: &common.RunnerConfig{
1444				RunnerSettings: common.RunnerSettings{
1445					Kubernetes: &common.KubernetesConfig{
1446						Host: "test-server",
1447					},
1448				},
1449			},
1450			Build: &common.Build{
1451				JobResponse: common.JobResponse{
1452					Image: common.Image{
1453						Name: "test-image",
1454					},
1455				},
1456				Runner: &common.RunnerConfig{},
1457			},
1458			Expected: &executor{
1459				options: &kubernetesOptions{
1460					Image: common.Image{
1461						Name: "test-image",
1462					},
1463				},
1464				helperImageInfo: defaultHelperImage,
1465				configurationOverwrites: &overwrites{
1466					namespace:       "default",
1467					serviceLimits:   api.ResourceList{},
1468					buildLimits:     api.ResourceList{},
1469					helperLimits:    api.ResourceList{},
1470					serviceRequests: api.ResourceList{},
1471					buildRequests:   api.ResourceList{},
1472					helperRequests:  api.ResourceList{},
1473				},
1474			},
1475		},
1476		{
1477			Name:         "DockerHub helper image",
1478			GlobalConfig: &common.Config{},
1479			RunnerConfig: &common.RunnerConfig{
1480				RunnerSettings: common.RunnerSettings{
1481					Kubernetes: &common.KubernetesConfig{
1482						Host: "test-server",
1483					},
1484				},
1485			},
1486			Build: &common.Build{
1487				JobResponse: common.JobResponse{
1488					Image: common.Image{
1489						Name: "test-image",
1490					},
1491					Variables: common.JobVariables{
1492						common.JobVariable{
1493							Key:      featureflags.GitLabRegistryHelperImage,
1494							Value:    "false",
1495							Public:   false,
1496							Internal: false,
1497							File:     false,
1498							Masked:   false,
1499							Raw:      false,
1500						},
1501					},
1502				},
1503				Runner: &common.RunnerConfig{},
1504			},
1505			Expected: &executor{
1506				options: &kubernetesOptions{
1507					Image: common.Image{
1508						Name: "test-image",
1509					},
1510				},
1511				helperImageInfo: helperimage.Info{
1512					Architecture:            "x86_64",
1513					Name:                    helperimage.DockerHubName,
1514					Tag:                     fmt.Sprintf("x86_64-%s", helperImageTag),
1515					IsSupportingLocalImport: true,
1516					Cmd:                     []string{"gitlab-runner-build"},
1517				},
1518				configurationOverwrites: &overwrites{
1519					namespace:       "default",
1520					serviceLimits:   api.ResourceList{},
1521					buildLimits:     api.ResourceList{},
1522					helperLimits:    api.ResourceList{},
1523					serviceRequests: api.ResourceList{},
1524					buildRequests:   api.ResourceList{},
1525					helperRequests:  api.ResourceList{},
1526				},
1527			},
1528		},
1529		{
1530			Name:         "helper image with ubuntu flavour default registry",
1531			GlobalConfig: &common.Config{},
1532			RunnerConfig: &common.RunnerConfig{
1533				RunnerSettings: common.RunnerSettings{
1534					Kubernetes: &common.KubernetesConfig{
1535						Host:              "test-server",
1536						HelperImageFlavor: "ubuntu",
1537					},
1538				},
1539			},
1540			Build: &common.Build{
1541				JobResponse: common.JobResponse{
1542					Image: common.Image{
1543						Name: "test-image",
1544					},
1545				},
1546				Runner: &common.RunnerConfig{},
1547			},
1548			Expected: &executor{
1549				options: &kubernetesOptions{
1550					Image: common.Image{
1551						Name: "test-image",
1552					},
1553				},
1554				configurationOverwrites: &overwrites{
1555					namespace:       "default",
1556					serviceLimits:   api.ResourceList{},
1557					buildLimits:     api.ResourceList{},
1558					helperLimits:    api.ResourceList{},
1559					serviceRequests: api.ResourceList{},
1560					buildRequests:   api.ResourceList{},
1561					helperRequests:  api.ResourceList{},
1562				},
1563				helperImageInfo: helperimage.Info{
1564					Architecture:            "x86_64",
1565					Name:                    helperimage.GitLabRegistryName,
1566					Tag:                     fmt.Sprintf("ubuntu-x86_64-%s", helperImageTag),
1567					IsSupportingLocalImport: true,
1568					Cmd:                     []string{"gitlab-runner-build"},
1569				},
1570			},
1571		},
1572		{
1573			Name:         "helper image with ubuntu flavour DockerHub registry",
1574			GlobalConfig: &common.Config{},
1575			RunnerConfig: &common.RunnerConfig{
1576				RunnerSettings: common.RunnerSettings{
1577					Kubernetes: &common.KubernetesConfig{
1578						Host:              "test-server",
1579						HelperImageFlavor: "ubuntu",
1580					},
1581				},
1582			},
1583			Build: &common.Build{
1584				JobResponse: common.JobResponse{
1585					Image: common.Image{
1586						Name: "test-image",
1587					},
1588					Variables: common.JobVariables{
1589						common.JobVariable{
1590							Key:      featureflags.GitLabRegistryHelperImage,
1591							Value:    "false",
1592							Public:   false,
1593							Internal: false,
1594							File:     false,
1595							Masked:   false,
1596							Raw:      false,
1597						},
1598					},
1599				},
1600				Runner: &common.RunnerConfig{},
1601			},
1602			Expected: &executor{
1603				options: &kubernetesOptions{
1604					Image: common.Image{
1605						Name: "test-image",
1606					},
1607				},
1608				configurationOverwrites: &overwrites{
1609					namespace:       "default",
1610					serviceLimits:   api.ResourceList{},
1611					buildLimits:     api.ResourceList{},
1612					helperLimits:    api.ResourceList{},
1613					serviceRequests: api.ResourceList{},
1614					buildRequests:   api.ResourceList{},
1615					helperRequests:  api.ResourceList{},
1616				},
1617				helperImageInfo: helperimage.Info{
1618					Architecture:            "x86_64",
1619					Name:                    helperimage.DockerHubName,
1620					Tag:                     fmt.Sprintf("ubuntu-x86_64-%s", helperImageTag),
1621					IsSupportingLocalImport: true,
1622					Cmd:                     []string{"gitlab-runner-build"},
1623				},
1624			},
1625		},
1626	}
1627
1628	for _, test := range tests {
1629		t.Run(test.Name, func(t *testing.T) {
1630			e := &executor{
1631				AbstractExecutor: executors.AbstractExecutor{
1632					ExecutorOptions: executorOptions,
1633				},
1634			}
1635
1636			// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
1637			prepareOptions := common.ExecutorPrepareOptions{
1638				Config:  test.RunnerConfig,
1639				Build:   test.Build,
1640				Context: context.TODO(),
1641			}
1642			prepareOptions.Build.Runner.Executor = "kubernetes"
1643
1644			err := e.Prepare(prepareOptions)
1645			if err != nil {
1646				assert.False(t, test.Build.IsSharedEnv())
1647			}
1648			if test.Error != "" {
1649				assert.Error(t, err)
1650				assert.Contains(t, err.Error(), test.Error)
1651				return
1652			}
1653
1654			// Set this to nil so we aren't testing the functionality of the
1655			// base AbstractExecutor's Prepare method
1656			e.AbstractExecutor = executors.AbstractExecutor{}
1657
1658			pullPolicy, err := e.pullManager.GetPullPolicyFor(prepareOptions.Build.Image.Name)
1659			assert.NoError(t, err)
1660			assert.Equal(t, test.ExpectedPullPolicy, pullPolicy)
1661
1662			e.kubeClient = nil
1663			e.kubeConfig = nil
1664			e.featureChecker = nil
1665			e.pullManager = nil
1666
1667			assert.NoError(t, err)
1668			assert.Equal(t, test.Expected, e)
1669		})
1670	}
1671}
1672
1673func TestSetupCredentials(t *testing.T) {
1674	version, _ := testVersionAndCodec()
1675
1676	type testDef struct {
1677		RunnerCredentials *common.RunnerCredentials
1678		Credentials       []common.Credentials
1679		VerifyFn          func(*testing.T, testDef, *api.Secret)
1680	}
1681	tests := map[string]testDef{
1682		"no credentials": {
1683			// don't execute VerifyFn
1684			VerifyFn: nil,
1685		},
1686		"registry credentials": {
1687			Credentials: []common.Credentials{
1688				{
1689					Type:     "registry",
1690					URL:      "http://example.com",
1691					Username: "user",
1692					Password: "password",
1693				},
1694			},
1695			VerifyFn: func(t *testing.T, test testDef, secret *api.Secret) {
1696				assert.Equal(t, api.SecretTypeDockercfg, secret.Type)
1697				assert.NotEmpty(t, secret.Data[api.DockerConfigKey])
1698			},
1699		},
1700		"other credentials": {
1701			Credentials: []common.Credentials{
1702				{
1703					Type:     "other",
1704					URL:      "http://example.com",
1705					Username: "user",
1706					Password: "password",
1707				},
1708			},
1709			// don't execute VerifyFn
1710			VerifyFn: nil,
1711		},
1712		"non-DNS-1123-compatible-token": {
1713			RunnerCredentials: &common.RunnerCredentials{
1714				Token: "ToK3_?OF",
1715			},
1716			Credentials: []common.Credentials{
1717				{
1718					Type:     "registry",
1719					URL:      "http://example.com",
1720					Username: "user",
1721					Password: "password",
1722				},
1723			},
1724			VerifyFn: func(t *testing.T, test testDef, secret *api.Secret) {
1725				dns_test.AssertRFC1123Compatibility(t, secret.GetGenerateName())
1726			},
1727		},
1728	}
1729
1730	executed := false
1731	fakeClientRoundTripper := func(test testDef) func(req *http.Request) (*http.Response, error) {
1732		return func(req *http.Request) (resp *http.Response, err error) {
1733			podBytes, err := ioutil.ReadAll(req.Body)
1734			executed = true
1735
1736			if err != nil {
1737				t.Errorf("failed to read request body: %s", err.Error())
1738				return
1739			}
1740
1741			p := new(api.Secret)
1742
1743			err = json.Unmarshal(podBytes, p)
1744
1745			if err != nil {
1746				t.Errorf("error decoding pod: %s", err.Error())
1747				return
1748			}
1749
1750			if test.VerifyFn != nil {
1751				test.VerifyFn(t, test, p)
1752			}
1753
1754			resp = &http.Response{StatusCode: http.StatusOK, Body: FakeReadCloser{
1755				Reader: bytes.NewBuffer(podBytes),
1756			}}
1757			resp.Header = make(http.Header)
1758			resp.Header.Add("Content-Type", "application/json")
1759
1760			return
1761		}
1762	}
1763
1764	for testName, test := range tests {
1765		t.Run(testName, func(t *testing.T) {
1766			ex := executor{
1767				kubeClient: testKubernetesClient(version, fake.CreateHTTPClient(fakeClientRoundTripper(test))),
1768				options:    &kubernetesOptions{},
1769				AbstractExecutor: executors.AbstractExecutor{
1770					Config: common.RunnerConfig{
1771						RunnerSettings: common.RunnerSettings{
1772							Kubernetes: &common.KubernetesConfig{
1773								Namespace: "default",
1774							},
1775						},
1776					},
1777					BuildShell: &common.ShellConfiguration{},
1778					Build: &common.Build{
1779						JobResponse: common.JobResponse{
1780							Variables:   []common.JobVariable{},
1781							Credentials: test.Credentials,
1782						},
1783						Runner: &common.RunnerConfig{},
1784					},
1785				},
1786			}
1787
1788			if test.RunnerCredentials != nil {
1789				ex.Build.Runner = &common.RunnerConfig{
1790					RunnerCredentials: *test.RunnerCredentials,
1791				}
1792			}
1793
1794			executed = false
1795
1796			err := ex.prepareOverwrites(make(common.JobVariables, 0))
1797			assert.NoError(t, err)
1798
1799			err = ex.setupCredentials()
1800			assert.NoError(t, err)
1801
1802			if test.VerifyFn != nil {
1803				assert.True(t, executed)
1804			} else {
1805				assert.False(t, executed)
1806			}
1807		})
1808	}
1809}
1810
1811type setupBuildPodTestDef struct {
1812	RunnerConfig             common.RunnerConfig
1813	Variables                []common.JobVariable
1814	Options                  *kubernetesOptions
1815	InitContainers           []api.Container
1816	PrepareFn                func(*testing.T, setupBuildPodTestDef, *executor)
1817	VerifyFn                 func(*testing.T, setupBuildPodTestDef, *api.Pod)
1818	VerifyExecutorFn         func(*testing.T, setupBuildPodTestDef, *executor)
1819	VerifySetupBuildPodErrFn func(*testing.T, error)
1820}
1821
1822type setupBuildPodFakeRoundTripper struct {
1823	t        *testing.T
1824	test     setupBuildPodTestDef
1825	executed bool
1826}
1827
1828func (rt *setupBuildPodFakeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
1829	rt.executed = true
1830	podBytes, err := ioutil.ReadAll(req.Body)
1831	if !assert.NoError(rt.t, err, "failed to read request body") {
1832		return nil, err
1833	}
1834
1835	p := new(api.Pod)
1836	err = json.Unmarshal(podBytes, p)
1837	if !assert.NoError(rt.t, err, "failed to read request body") {
1838		return nil, err
1839	}
1840
1841	if rt.test.VerifyFn != nil {
1842		rt.test.VerifyFn(rt.t, rt.test, p)
1843	}
1844
1845	resp := &http.Response{
1846		StatusCode: http.StatusOK,
1847		Body: FakeReadCloser{
1848			Reader: bytes.NewBuffer(podBytes),
1849		},
1850	}
1851	resp.Header = make(http.Header)
1852	resp.Header.Add("Content-Type", "application/json")
1853
1854	return resp, nil
1855}
1856
1857func TestSetupBuildPod(t *testing.T) {
1858	version, _ := testVersionAndCodec()
1859	testErr := errors.New("fail")
1860	ndotsValue := "2"
1861
1862	tests := map[string]setupBuildPodTestDef{
1863		"passes node selector setting": {
1864			RunnerConfig: common.RunnerConfig{
1865				RunnerSettings: common.RunnerSettings{
1866					Kubernetes: &common.KubernetesConfig{
1867						Namespace: "default",
1868						NodeSelector: map[string]string{
1869							"a-selector":       "first",
1870							"another-selector": "second",
1871						},
1872					},
1873				},
1874			},
1875			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
1876				assert.Equal(t, test.RunnerConfig.RunnerSettings.Kubernetes.NodeSelector, pod.Spec.NodeSelector)
1877			},
1878		},
1879		"uses configured credentials": {
1880			RunnerConfig: common.RunnerConfig{
1881				RunnerSettings: common.RunnerSettings{
1882					Kubernetes: &common.KubernetesConfig{
1883						Namespace: "default",
1884					},
1885				},
1886			},
1887			PrepareFn: func(t *testing.T, test setupBuildPodTestDef, e *executor) {
1888				e.credentials = &api.Secret{
1889					ObjectMeta: metav1.ObjectMeta{
1890						Name: "job-credentials",
1891					},
1892				}
1893			},
1894			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
1895				secrets := []api.LocalObjectReference{{Name: "job-credentials"}}
1896				assert.Equal(t, secrets, pod.Spec.ImagePullSecrets)
1897			},
1898		},
1899		"uses configured image pull secrets": {
1900			RunnerConfig: common.RunnerConfig{
1901				RunnerSettings: common.RunnerSettings{
1902					Kubernetes: &common.KubernetesConfig{
1903						Namespace: "default",
1904						ImagePullSecrets: []string{
1905							"docker-registry-credentials",
1906						},
1907					},
1908				},
1909			},
1910			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
1911				secrets := []api.LocalObjectReference{{Name: "docker-registry-credentials"}}
1912				assert.Equal(t, secrets, pod.Spec.ImagePullSecrets)
1913			},
1914		},
1915		"uses default security context flags for containers": {
1916			RunnerConfig: common.RunnerConfig{
1917				RunnerSettings: common.RunnerSettings{
1918					Kubernetes: &common.KubernetesConfig{
1919						Namespace: "default",
1920					},
1921				},
1922			},
1923			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
1924				for _, c := range pod.Spec.Containers {
1925					assert.Empty(
1926						t,
1927						c.SecurityContext.Privileged,
1928						"Container security context Privileged should be empty",
1929					)
1930					assert.Nil(
1931						t,
1932						c.SecurityContext.AllowPrivilegeEscalation,
1933						"Container security context AllowPrivilegeEscalation should be empty",
1934					)
1935				}
1936			},
1937		},
1938		"configures security context flags for un-privileged containers": {
1939			RunnerConfig: common.RunnerConfig{
1940				RunnerSettings: common.RunnerSettings{
1941					Kubernetes: &common.KubernetesConfig{
1942						Namespace:                "default",
1943						Privileged:               false,
1944						AllowPrivilegeEscalation: func(b bool) *bool { return &b }(false),
1945					},
1946				},
1947			},
1948			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
1949				for _, c := range pod.Spec.Containers {
1950					require.NotNil(t, c.SecurityContext.Privileged)
1951					assert.False(t, *c.SecurityContext.Privileged)
1952					require.NotNil(t, c.SecurityContext.AllowPrivilegeEscalation)
1953					assert.False(t, *c.SecurityContext.AllowPrivilegeEscalation)
1954				}
1955			},
1956		},
1957		"configures security context flags for privileged containers": {
1958			RunnerConfig: common.RunnerConfig{
1959				RunnerSettings: common.RunnerSettings{
1960					Kubernetes: &common.KubernetesConfig{
1961						Namespace:                "default",
1962						Privileged:               true,
1963						AllowPrivilegeEscalation: func(b bool) *bool { return &b }(true),
1964					},
1965				},
1966			},
1967			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
1968				for _, c := range pod.Spec.Containers {
1969					require.NotNil(t, c.SecurityContext.Privileged)
1970					assert.True(t, *c.SecurityContext.Privileged)
1971					require.NotNil(t, c.SecurityContext.AllowPrivilegeEscalation)
1972					assert.True(t, *c.SecurityContext.AllowPrivilegeEscalation)
1973				}
1974			},
1975		},
1976		"configures helper container": {
1977			RunnerConfig: common.RunnerConfig{
1978				RunnerSettings: common.RunnerSettings{
1979					Kubernetes: &common.KubernetesConfig{
1980						Namespace: "default",
1981					},
1982				},
1983			},
1984			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
1985				hasHelper := false
1986				for _, c := range pod.Spec.Containers {
1987					if c.Name == helperContainerName {
1988						hasHelper = true
1989					}
1990				}
1991				assert.True(t, hasHelper)
1992			},
1993		},
1994		"uses configured helper image": {
1995			RunnerConfig: common.RunnerConfig{
1996				RunnerSettings: common.RunnerSettings{
1997					Kubernetes: &common.KubernetesConfig{
1998						Namespace:   "default",
1999						HelperImage: "custom/helper-image",
2000					},
2001				},
2002			},
2003			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2004				for _, c := range pod.Spec.Containers {
2005					if c.Name == "helper" {
2006						assert.Equal(t, test.RunnerConfig.RunnerSettings.Kubernetes.HelperImage, c.Image)
2007					}
2008				}
2009			},
2010		},
2011		"expands variables for pod labels": {
2012			RunnerConfig: common.RunnerConfig{
2013				RunnerSettings: common.RunnerSettings{
2014					Kubernetes: &common.KubernetesConfig{
2015						Namespace: "default",
2016						PodLabels: map[string]string{
2017							"test":    "label",
2018							"another": "label",
2019							"var":     "$test",
2020						},
2021					},
2022				},
2023			},
2024			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2025				assert.Equal(t, map[string]string{
2026					"test":    "label",
2027					"another": "label",
2028					"var":     "sometestvar",
2029					"pod":     pod.GenerateName,
2030				}, pod.ObjectMeta.Labels)
2031			},
2032			Variables: []common.JobVariable{
2033				{Key: "test", Value: "sometestvar"},
2034			},
2035		},
2036		"expands variables for pod annotations": {
2037			RunnerConfig: common.RunnerConfig{
2038				RunnerSettings: common.RunnerSettings{
2039					Kubernetes: &common.KubernetesConfig{
2040						Namespace: "default",
2041						PodAnnotations: map[string]string{
2042							"test":    "annotation",
2043							"another": "annotation",
2044							"var":     "$test",
2045						},
2046					},
2047				},
2048			},
2049			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2050				assert.Equal(t, map[string]string{
2051					"test":    "annotation",
2052					"another": "annotation",
2053					"var":     "sometestvar",
2054				}, pod.ObjectMeta.Annotations)
2055			},
2056			Variables: []common.JobVariable{
2057				{Key: "test", Value: "sometestvar"},
2058			},
2059		},
2060		"expands variables for helper image": {
2061			RunnerConfig: common.RunnerConfig{
2062				RunnerSettings: common.RunnerSettings{
2063					Kubernetes: &common.KubernetesConfig{
2064						Namespace:   "default",
2065						HelperImage: "custom/helper-image:${CI_RUNNER_REVISION}",
2066					},
2067				},
2068			},
2069			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2070				for _, c := range pod.Spec.Containers {
2071					if c.Name == "helper" {
2072						assert.Equal(t, "custom/helper-image:"+common.REVISION, c.Image)
2073					}
2074				}
2075			},
2076		},
2077		"support setting kubernetes pod taint tolerations": {
2078			RunnerConfig: common.RunnerConfig{
2079				RunnerSettings: common.RunnerSettings{
2080					Kubernetes: &common.KubernetesConfig{
2081						Namespace: "default",
2082						NodeTolerations: map[string]string{
2083							"node-role.kubernetes.io/master": "NoSchedule",
2084							"custom.toleration=value":        "NoSchedule",
2085							"empty.value=":                   "PreferNoSchedule",
2086							"onlyKey":                        "",
2087						},
2088					},
2089				},
2090			},
2091			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2092				expectedTolerations := []api.Toleration{
2093					{
2094						Key:      "node-role.kubernetes.io/master",
2095						Operator: api.TolerationOpExists,
2096						Effect:   api.TaintEffectNoSchedule,
2097					},
2098					{
2099						Key:      "custom.toleration",
2100						Operator: api.TolerationOpEqual,
2101						Value:    "value",
2102						Effect:   api.TaintEffectNoSchedule,
2103					},
2104					{
2105
2106						Key:      "empty.value",
2107						Operator: api.TolerationOpEqual,
2108						Value:    "",
2109						Effect:   api.TaintEffectPreferNoSchedule,
2110					},
2111					{
2112						Key:      "onlyKey",
2113						Operator: api.TolerationOpExists,
2114						Effect:   "",
2115					},
2116				}
2117				assert.ElementsMatch(t, expectedTolerations, pod.Spec.Tolerations)
2118			},
2119		},
2120		"supports extended docker configuration for image and services": {
2121			RunnerConfig: common.RunnerConfig{
2122				RunnerSettings: common.RunnerSettings{
2123					Kubernetes: &common.KubernetesConfig{
2124						Namespace:   "default",
2125						HelperImage: "custom/helper-image",
2126					},
2127				},
2128			},
2129			Options: &kubernetesOptions{
2130				Image: common.Image{
2131					Name:       "test-image",
2132					Entrypoint: []string{"/init", "run"},
2133				},
2134				Services: common.Services{
2135					{
2136						Name:       "test-service",
2137						Entrypoint: []string{"/init", "run"},
2138						Command:    []string{"application", "--debug"},
2139					},
2140					{
2141						Name:    "test-service-2",
2142						Command: []string{"application", "--debug"},
2143					},
2144				},
2145			},
2146			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2147				require.Len(t, pod.Spec.Containers, 4)
2148
2149				assert.Equal(t, "build", pod.Spec.Containers[0].Name)
2150				assert.Equal(t, "test-image", pod.Spec.Containers[0].Image)
2151				assert.Equal(t, []string{"/init", "run"}, pod.Spec.Containers[0].Command)
2152				assert.Empty(t, pod.Spec.Containers[0].Args, "Build container args should be empty")
2153
2154				assert.Equal(t, "helper", pod.Spec.Containers[1].Name)
2155				assert.Equal(t, "custom/helper-image", pod.Spec.Containers[1].Image)
2156				assert.Empty(t, pod.Spec.Containers[1].Command, "Helper container command should be empty")
2157				assert.Empty(t, pod.Spec.Containers[1].Args, "Helper container args should be empty")
2158
2159				assert.Equal(t, "svc-0", pod.Spec.Containers[2].Name)
2160				assert.Equal(t, "test-service", pod.Spec.Containers[2].Image)
2161				assert.Equal(t, []string{"/init", "run"}, pod.Spec.Containers[2].Command)
2162				assert.Equal(t, []string{"application", "--debug"}, pod.Spec.Containers[2].Args)
2163
2164				assert.Equal(t, "svc-1", pod.Spec.Containers[3].Name)
2165				assert.Equal(t, "test-service-2", pod.Spec.Containers[3].Image)
2166				assert.Empty(t, pod.Spec.Containers[3].Command, "Service container command should be empty")
2167				assert.Equal(t, []string{"application", "--debug"}, pod.Spec.Containers[3].Args)
2168			},
2169		},
2170		"creates services in kubernetes if ports are set": {
2171			RunnerConfig: common.RunnerConfig{
2172				RunnerSettings: common.RunnerSettings{
2173					Kubernetes: &common.KubernetesConfig{
2174						Namespace:   "default",
2175						HelperImage: "custom/helper-image",
2176					},
2177				},
2178			},
2179			Options: &kubernetesOptions{
2180				Image: common.Image{
2181					Name: "test-image",
2182					Ports: []common.Port{
2183						{
2184							Number: 80,
2185						},
2186					},
2187				},
2188				Services: common.Services{
2189					{
2190						Name: "test-service",
2191						Ports: []common.Port{
2192							{
2193								Number: 82,
2194							},
2195							{
2196								Number: 84,
2197							},
2198						},
2199					},
2200					{
2201						Name: "test-service2",
2202						Ports: []common.Port{
2203							{
2204								Number: 85,
2205							},
2206						},
2207					},
2208					{
2209						Name: "test-service3",
2210					},
2211				},
2212			},
2213			VerifyExecutorFn: func(t *testing.T, test setupBuildPodTestDef, e *executor) {
2214				expectedServices := []api.Service{
2215					{
2216						ObjectMeta: metav1.ObjectMeta{
2217							GenerateName: "build",
2218							Namespace:    "default",
2219						},
2220						Spec: api.ServiceSpec{
2221							Ports: []api.ServicePort{
2222								{
2223									Port:       80,
2224									TargetPort: intstr.FromInt(80),
2225									Name:       "build-80",
2226								},
2227							},
2228							Selector: map[string]string{"pod": e.pod.GenerateName},
2229							Type:     api.ServiceTypeClusterIP,
2230						},
2231					},
2232					{
2233						ObjectMeta: metav1.ObjectMeta{
2234							GenerateName: "proxy-svc-0",
2235							Namespace:    "default",
2236						},
2237						Spec: api.ServiceSpec{
2238							Ports: []api.ServicePort{
2239								{
2240									Port:       82,
2241									TargetPort: intstr.FromInt(82),
2242									Name:       "proxy-svc-0-82",
2243								},
2244								{
2245									Port:       84,
2246									TargetPort: intstr.FromInt(84),
2247									Name:       "proxy-svc-0-84",
2248								},
2249							},
2250							Selector: map[string]string{"pod": e.pod.GenerateName},
2251							Type:     api.ServiceTypeClusterIP,
2252						},
2253					},
2254					{
2255						ObjectMeta: metav1.ObjectMeta{
2256							GenerateName: "proxy-svc-1",
2257							Namespace:    "default",
2258						},
2259						Spec: api.ServiceSpec{
2260							Ports: []api.ServicePort{
2261								{
2262									Port:       85,
2263									TargetPort: intstr.FromInt(85),
2264									Name:       "proxy-svc-1-85",
2265								},
2266							},
2267							Selector: map[string]string{"pod": e.pod.GenerateName},
2268							Type:     api.ServiceTypeClusterIP,
2269						},
2270					},
2271				}
2272
2273				assert.ElementsMatch(t, expectedServices, e.services)
2274			},
2275		},
2276		"the default service name for the build container is build": {
2277			RunnerConfig: common.RunnerConfig{
2278				RunnerSettings: common.RunnerSettings{
2279					Kubernetes: &common.KubernetesConfig{
2280						Namespace:   "default",
2281						HelperImage: "custom/helper-image",
2282					},
2283				},
2284			},
2285			Options: &kubernetesOptions{
2286				Image: common.Image{
2287					Name: "test-image",
2288					Ports: []common.Port{
2289						{
2290							Number: 80,
2291						},
2292					},
2293				},
2294			},
2295			VerifyExecutorFn: func(t *testing.T, test setupBuildPodTestDef, e *executor) {
2296				assert.Equal(t, "build", e.services[0].GenerateName)
2297			},
2298		},
2299		"the services have a selector pointing to the 'pod' label in the pod": {
2300			RunnerConfig: common.RunnerConfig{
2301				RunnerSettings: common.RunnerSettings{
2302					Kubernetes: &common.KubernetesConfig{
2303						Namespace:   "default",
2304						HelperImage: "custom/helper-image",
2305					},
2306				},
2307			},
2308			Options: &kubernetesOptions{
2309				Image: common.Image{
2310					Name: "test-image",
2311					Ports: []common.Port{
2312						{
2313							Number: 80,
2314						},
2315					},
2316				},
2317				Services: common.Services{
2318					{
2319						Name: "test-service",
2320						Ports: []common.Port{
2321							{
2322								Number: 82,
2323							},
2324						},
2325					},
2326				},
2327			},
2328			VerifyExecutorFn: func(t *testing.T, test setupBuildPodTestDef, e *executor) {
2329				for _, service := range e.services {
2330					assert.Equal(t, map[string]string{"pod": e.pod.GenerateName}, service.Spec.Selector)
2331				}
2332			},
2333		},
2334		"the service is named as the alias if set": {
2335			RunnerConfig: common.RunnerConfig{
2336				RunnerSettings: common.RunnerSettings{
2337					Kubernetes: &common.KubernetesConfig{
2338						Namespace:   "default",
2339						HelperImage: "custom/helper-image",
2340					},
2341				},
2342			},
2343			Options: &kubernetesOptions{
2344				Image: common.Image{
2345					Name: "test-image",
2346				},
2347				Services: common.Services{
2348					{
2349						Name:  "test-service",
2350						Alias: "custom-name",
2351						Ports: []common.Port{
2352							{
2353								Number: 82,
2354							},
2355						},
2356					},
2357				},
2358			},
2359			VerifyExecutorFn: func(t *testing.T, test setupBuildPodTestDef, e *executor) {
2360				assert.Equal(t, "custom-name", e.services[0].GenerateName)
2361			},
2362		},
2363		"proxies are configured if services have been created": {
2364			RunnerConfig: common.RunnerConfig{
2365				RunnerSettings: common.RunnerSettings{
2366					Kubernetes: &common.KubernetesConfig{
2367						Namespace:   "default",
2368						HelperImage: "custom/helper-image",
2369					},
2370				},
2371			},
2372			Options: &kubernetesOptions{
2373				Image: common.Image{
2374					Name: "test-image",
2375					Ports: []common.Port{
2376						{
2377							Number: 80,
2378						},
2379					},
2380				},
2381				Services: common.Services{
2382					{
2383						Name:  "test-service",
2384						Alias: "custom_name",
2385						Ports: []common.Port{
2386							{
2387								Number:   81,
2388								Name:     "custom_port_name",
2389								Protocol: "http",
2390							},
2391						},
2392					},
2393					{
2394						Name: "test-service2",
2395						Ports: []common.Port{
2396							{
2397								Number: 82,
2398							},
2399						},
2400					},
2401				},
2402			},
2403			VerifyExecutorFn: func(t *testing.T, test setupBuildPodTestDef, e *executor) {
2404				require.Len(t, e.ProxyPool, 3)
2405
2406				assert.NotEmpty(t, "proxy-svc-1", e.ProxyPool)
2407				assert.NotEmpty(t, "custom_name", e.ProxyPool)
2408				assert.NotEmpty(t, "build", e.ProxyPool)
2409
2410				port := e.ProxyPool["proxy-svc-1"].Settings.Ports[0]
2411				assert.Equal(t, 82, port.Number)
2412
2413				port = e.ProxyPool["custom_name"].Settings.Ports[0]
2414				assert.Equal(t, 81, port.Number)
2415				assert.Equal(t, "custom_port_name", port.Name)
2416				assert.Equal(t, "http", port.Protocol)
2417
2418				port = e.ProxyPool["build"].Settings.Ports[0]
2419				assert.Equal(t, 80, port.Number)
2420			},
2421		},
2422		"makes service name compatible with RFC1123": {
2423			RunnerConfig: common.RunnerConfig{
2424				RunnerSettings: common.RunnerSettings{
2425					Kubernetes: &common.KubernetesConfig{
2426						Namespace:   "default",
2427						HelperImage: "custom/helper-image",
2428					},
2429				},
2430			},
2431			Options: &kubernetesOptions{
2432				Image: common.Image{
2433					Name: "test-image",
2434				},
2435				Services: common.Services{
2436					{
2437						Name:  "test-service",
2438						Alias: "service,name-.non-compat!ble",
2439						Ports: []common.Port{
2440							{
2441								Number:   81,
2442								Name:     "port,name-.non-compat!ble",
2443								Protocol: "http",
2444							},
2445						},
2446					},
2447				},
2448			},
2449			VerifyExecutorFn: func(t *testing.T, test setupBuildPodTestDef, e *executor) {
2450				assert.Equal(t, "servicename-non-compatble", e.services[0].GenerateName)
2451				assert.NotEmpty(t, e.ProxyPool["service,name-.non-compat!ble"])
2452				assert.Equal(
2453					t,
2454					"port,name-.non-compat!ble",
2455					e.ProxyPool["service,name-.non-compat!ble"].Settings.Ports[0].Name,
2456				)
2457			},
2458		},
2459		"sets command (entrypoint) and args": {
2460			RunnerConfig: common.RunnerConfig{
2461				RunnerSettings: common.RunnerSettings{
2462					Kubernetes: &common.KubernetesConfig{
2463						Namespace:   "default",
2464						HelperImage: "custom/helper-image",
2465					},
2466				},
2467			},
2468			Options: &kubernetesOptions{
2469				Image: common.Image{
2470					Name: "test-image",
2471				},
2472				Services: common.Services{
2473					{
2474						Name:    "test-service-0",
2475						Command: []string{"application", "--debug"},
2476					},
2477					{
2478						Name:       "test-service-1",
2479						Entrypoint: []string{"application", "--debug"},
2480					},
2481					{
2482						Name:       "test-service-2",
2483						Entrypoint: []string{"application", "--debug"},
2484						Command:    []string{"argument1", "argument2"},
2485					},
2486				},
2487			},
2488			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2489				require.Len(t, pod.Spec.Containers, 5)
2490
2491				assert.Equal(t, "build", pod.Spec.Containers[0].Name)
2492				assert.Equal(t, "test-image", pod.Spec.Containers[0].Image)
2493				assert.Empty(t, pod.Spec.Containers[0].Command, "Build container command should be empty")
2494				assert.Empty(t, pod.Spec.Containers[0].Args, "Build container args should be empty")
2495
2496				assert.Equal(t, "helper", pod.Spec.Containers[1].Name)
2497				assert.Equal(t, "custom/helper-image", pod.Spec.Containers[1].Image)
2498				assert.Empty(t, pod.Spec.Containers[1].Command, "Helper container command should be empty")
2499				assert.Empty(t, pod.Spec.Containers[1].Args, "Helper container args should be empty")
2500
2501				assert.Equal(t, "svc-0", pod.Spec.Containers[2].Name)
2502				assert.Equal(t, "test-service-0", pod.Spec.Containers[2].Image)
2503				assert.Empty(t, pod.Spec.Containers[2].Command, "Service container command should be empty")
2504				assert.Equal(t, []string{"application", "--debug"}, pod.Spec.Containers[2].Args)
2505
2506				assert.Equal(t, "svc-1", pod.Spec.Containers[3].Name)
2507				assert.Equal(t, "test-service-1", pod.Spec.Containers[3].Image)
2508				assert.Equal(t, []string{"application", "--debug"}, pod.Spec.Containers[3].Command)
2509				assert.Empty(t, pod.Spec.Containers[3].Args, "Service container args should be empty")
2510
2511				assert.Equal(t, "svc-2", pod.Spec.Containers[4].Name)
2512				assert.Equal(t, "test-service-2", pod.Spec.Containers[4].Image)
2513				assert.Equal(t, []string{"application", "--debug"}, pod.Spec.Containers[4].Command)
2514				assert.Equal(t, []string{"argument1", "argument2"}, pod.Spec.Containers[4].Args)
2515			},
2516		},
2517		"non-DNS-1123-compatible-token": {
2518			RunnerConfig: common.RunnerConfig{
2519				RunnerCredentials: common.RunnerCredentials{
2520					Token: "ToK3_?OF",
2521				},
2522				RunnerSettings: common.RunnerSettings{
2523					Kubernetes: &common.KubernetesConfig{
2524						Namespace: "default",
2525					},
2526				},
2527			},
2528			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2529				dns_test.AssertRFC1123Compatibility(t, pod.GetGenerateName())
2530			},
2531		},
2532		"supports pod security context": {
2533			RunnerConfig: common.RunnerConfig{
2534				RunnerSettings: common.RunnerSettings{
2535					Kubernetes: &common.KubernetesConfig{
2536						Namespace: "default",
2537						PodSecurityContext: common.KubernetesPodSecurityContext{
2538							FSGroup:            func() *int64 { i := int64(200); return &i }(),
2539							RunAsGroup:         func() *int64 { i := int64(200); return &i }(),
2540							RunAsNonRoot:       func() *bool { i := bool(true); return &i }(),
2541							RunAsUser:          func() *int64 { i := int64(200); return &i }(),
2542							SupplementalGroups: []int64{200},
2543						},
2544					},
2545				},
2546			},
2547			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2548				assert.Equal(t, int64(200), *pod.Spec.SecurityContext.FSGroup)
2549				assert.Equal(t, int64(200), *pod.Spec.SecurityContext.RunAsGroup)
2550				assert.Equal(t, int64(200), *pod.Spec.SecurityContext.RunAsUser)
2551				assert.Equal(t, true, *pod.Spec.SecurityContext.RunAsNonRoot)
2552				assert.Equal(t, []int64{200}, pod.Spec.SecurityContext.SupplementalGroups)
2553			},
2554		},
2555		"uses default security context when unspecified": {
2556			RunnerConfig: common.RunnerConfig{
2557				RunnerSettings: common.RunnerSettings{
2558					Kubernetes: &common.KubernetesConfig{
2559						Namespace: "default",
2560					},
2561				},
2562			},
2563			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2564				assert.Empty(t, pod.Spec.SecurityContext, "Security context should be empty")
2565			},
2566		},
2567		"supports pod node affinities": {
2568			RunnerConfig: common.RunnerConfig{
2569				RunnerSettings: common.RunnerSettings{
2570					Kubernetes: &common.KubernetesConfig{
2571						Namespace: "default",
2572						Affinity: common.KubernetesAffinity{
2573							NodeAffinity: &common.KubernetesNodeAffinity{
2574								PreferredDuringSchedulingIgnoredDuringExecution: []common.PreferredSchedulingTerm{
2575									{
2576										Weight: 100,
2577										Preference: common.NodeSelectorTerm{
2578											MatchExpressions: []common.NodeSelectorRequirement{
2579												{
2580													Key:      "cpu_speed",
2581													Operator: "In",
2582													Values:   []string{"fast"},
2583												},
2584											},
2585											MatchFields: []common.NodeSelectorRequirement{
2586												{
2587													Key:      "cpu_count",
2588													Operator: "Gt",
2589													Values:   []string{"12"},
2590												},
2591											},
2592										},
2593									},
2594									{
2595										Weight: 50,
2596										Preference: common.NodeSelectorTerm{
2597											MatchExpressions: []common.NodeSelectorRequirement{
2598												{
2599													Key:      "kubernetes.io/e2e-az-name",
2600													Operator: "In",
2601													Values:   []string{"e2e-az1", "e2e-az2"},
2602												},
2603												{
2604													Key:      "kubernetes.io/arch",
2605													Operator: "NotIn",
2606													Values:   []string{"arm"},
2607												},
2608											},
2609										},
2610									},
2611								},
2612								RequiredDuringSchedulingIgnoredDuringExecution: &common.NodeSelector{
2613									NodeSelectorTerms: []common.NodeSelectorTerm{
2614										{
2615											MatchExpressions: []common.NodeSelectorRequirement{
2616												{
2617													Key:      "kubernetes.io/e2e-az-name",
2618													Operator: "In",
2619													Values:   []string{"e2e-az1", "e2e-az2"},
2620												},
2621											},
2622										},
2623									},
2624								},
2625							},
2626						},
2627					},
2628				},
2629			},
2630			//nolint:lll
2631			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2632				require.NotNil(t, pod.Spec.Affinity)
2633				require.NotNil(t, pod.Spec.Affinity.NodeAffinity)
2634
2635				nodeAffinity := pod.Spec.Affinity.NodeAffinity
2636				preferredNodeAffinity := nodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution
2637
2638				require.Len(t, preferredNodeAffinity, 2)
2639				assert.Equal(t, int32(100), preferredNodeAffinity[0].Weight)
2640				require.Len(t, preferredNodeAffinity[0].Preference.MatchExpressions, 1)
2641				require.Len(t, preferredNodeAffinity[0].Preference.MatchFields, 1)
2642				assert.Equal(t, "cpu_speed", preferredNodeAffinity[0].Preference.MatchExpressions[0].Key)
2643				assert.Equal(t, api.NodeSelectorOperator("In"), preferredNodeAffinity[0].Preference.MatchExpressions[0].Operator)
2644				assert.Equal(t, []string{"fast"}, preferredNodeAffinity[0].Preference.MatchExpressions[0].Values)
2645				assert.Equal(t, "cpu_count", preferredNodeAffinity[0].Preference.MatchFields[0].Key)
2646				assert.Equal(t, api.NodeSelectorOperator("Gt"), preferredNodeAffinity[0].Preference.MatchFields[0].Operator)
2647				assert.Equal(t, []string{"12"}, preferredNodeAffinity[0].Preference.MatchFields[0].Values)
2648
2649				assert.Equal(t, int32(50), preferredNodeAffinity[1].Weight)
2650				require.Len(t, preferredNodeAffinity[1].Preference.MatchExpressions, 2)
2651				assert.Equal(t, "kubernetes.io/e2e-az-name", preferredNodeAffinity[1].Preference.MatchExpressions[0].Key)
2652				assert.Equal(t, api.NodeSelectorOperator("In"), preferredNodeAffinity[1].Preference.MatchExpressions[0].Operator)
2653				assert.Equal(t, []string{"e2e-az1", "e2e-az2"}, preferredNodeAffinity[1].Preference.MatchExpressions[0].Values)
2654				assert.Equal(t, "kubernetes.io/arch", preferredNodeAffinity[1].Preference.MatchExpressions[1].Key)
2655				assert.Equal(t, api.NodeSelectorOperator("NotIn"), preferredNodeAffinity[1].Preference.MatchExpressions[1].Operator)
2656				assert.Equal(t, []string{"arm"}, preferredNodeAffinity[1].Preference.MatchExpressions[1].Values)
2657
2658				require.NotNil(t, nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
2659				requiredNodeAffinity := nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution
2660
2661				require.Len(t, requiredNodeAffinity.NodeSelectorTerms, 1)
2662				require.Len(t, requiredNodeAffinity.NodeSelectorTerms[0].MatchExpressions, 1)
2663				require.Len(t, requiredNodeAffinity.NodeSelectorTerms[0].MatchFields, 0)
2664				assert.Equal(t, "kubernetes.io/e2e-az-name", requiredNodeAffinity.NodeSelectorTerms[0].MatchExpressions[0].Key)
2665				assert.Equal(t, api.NodeSelectorOperator("In"), requiredNodeAffinity.NodeSelectorTerms[0].MatchExpressions[0].Operator)
2666				assert.Equal(t, []string{"e2e-az1", "e2e-az2"}, requiredNodeAffinity.NodeSelectorTerms[0].MatchExpressions[0].Values)
2667			},
2668		},
2669		"supports services and setting extra hosts using HostAliases": {
2670			RunnerConfig: common.RunnerConfig{
2671				RunnerSettings: common.RunnerSettings{
2672					Kubernetes: &common.KubernetesConfig{
2673						Namespace: "default",
2674						HostAliases: []common.KubernetesHostAliases{
2675							{
2676								IP:        "127.0.0.1",
2677								Hostnames: []string{"redis"},
2678							},
2679							{
2680								IP:        "8.8.8.8",
2681								Hostnames: []string{"dns1", "dns2"},
2682							},
2683						},
2684					},
2685				},
2686			},
2687			Options: &kubernetesOptions{
2688				Services: common.Services{
2689					{
2690						Name:  "test-service",
2691						Alias: "svc-alias",
2692					},
2693					{
2694						Name: "docker:dind",
2695					},
2696					{
2697						Name: "service-with-port:dind",
2698						Ports: []common.Port{{
2699							Number:   0,
2700							Protocol: "",
2701							Name:     "",
2702						}},
2703					},
2704				},
2705			},
2706			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2707				// the second time this fn is called is to create the proxy service
2708				if pod.Kind == "Service" {
2709					return
2710				}
2711
2712				require.Len(t, pod.Spec.HostAliases, 3)
2713				assert.Equal(t, []api.HostAlias{
2714					{
2715						IP:        "127.0.0.1",
2716						Hostnames: []string{"test-service", "svc-alias", "docker"},
2717					},
2718					{
2719						IP:        "127.0.0.1",
2720						Hostnames: []string{"redis"},
2721					},
2722					{
2723						IP:        "8.8.8.8",
2724						Hostnames: []string{"dns1", "dns2"},
2725					},
2726				}, pod.Spec.HostAliases)
2727			},
2728		},
2729		"ignores non RFC1123 aliases": {
2730			RunnerConfig: common.RunnerConfig{
2731				RunnerSettings: common.RunnerSettings{
2732					Kubernetes: &common.KubernetesConfig{
2733						Namespace: "default",
2734					},
2735				},
2736			},
2737			Options: &kubernetesOptions{
2738				Services: common.Services{
2739					{
2740						Name:  "test-service",
2741						Alias: "INVALID_ALIAS",
2742					},
2743					{
2744						Name: "docker:dind",
2745					},
2746				},
2747			},
2748			VerifySetupBuildPodErrFn: func(t *testing.T, err error) {
2749				var expected *invalidHostAliasDNSError
2750				assert.ErrorAs(t, err, &expected)
2751				assert.True(t, expected.Is(err))
2752				errMsg := err.Error()
2753				assert.Contains(t, errMsg, "is invalid DNS")
2754				assert.Contains(t, errMsg, "INVALID_ALIAS")
2755				assert.Contains(t, errMsg, "test-service")
2756			},
2757		},
2758		"no host aliases when feature is not supported in kubernetes": {
2759			RunnerConfig: common.RunnerConfig{
2760				RunnerSettings: common.RunnerSettings{
2761					Kubernetes: &common.KubernetesConfig{
2762						Namespace: "default",
2763						HostAliases: []common.KubernetesHostAliases{
2764							{
2765								IP:        "127.0.0.1",
2766								Hostnames: []string{"redis"},
2767							},
2768							{
2769								IP:        "8.8.8.8",
2770								Hostnames: []string{"google"},
2771							},
2772						},
2773					},
2774				},
2775			},
2776			Options: &kubernetesOptions{
2777				Services: common.Services{
2778					{
2779						Name:  "test-service",
2780						Alias: "alias",
2781					},
2782				},
2783			},
2784			PrepareFn: func(t *testing.T, def setupBuildPodTestDef, e *executor) {
2785				mockFc := &mockFeatureChecker{}
2786				mockFc.On("IsHostAliasSupported").Return(false, nil)
2787				e.featureChecker = mockFc
2788			},
2789			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2790				assert.Len(t, pod.Spec.Containers, 3)
2791				assert.Nil(t, pod.Spec.HostAliases)
2792			},
2793		},
2794		"check host aliases with non kubernetes version error": {
2795			RunnerConfig: common.RunnerConfig{
2796				RunnerSettings: common.RunnerSettings{
2797					Kubernetes: &common.KubernetesConfig{
2798						Namespace: "default",
2799					},
2800				},
2801			},
2802			Options: &kubernetesOptions{
2803				Services: common.Services{
2804					{
2805						Name:  "test-service",
2806						Alias: "alias",
2807					},
2808				},
2809			},
2810			PrepareFn: func(t *testing.T, def setupBuildPodTestDef, e *executor) {
2811				mockFc := &mockFeatureChecker{}
2812				mockFc.On("IsHostAliasSupported").Return(false, testErr)
2813				e.featureChecker = mockFc
2814			},
2815			VerifySetupBuildPodErrFn: func(t *testing.T, err error) {
2816				assert.ErrorIs(t, err, testErr)
2817			},
2818		},
2819		"check host aliases with kubernetes version error": {
2820			RunnerConfig: common.RunnerConfig{
2821				RunnerSettings: common.RunnerSettings{
2822					Kubernetes: &common.KubernetesConfig{
2823						Namespace: "default",
2824					},
2825				},
2826			},
2827			Options: &kubernetesOptions{
2828				Services: common.Services{
2829					{
2830						Name:  "test-service",
2831						Alias: "alias",
2832					},
2833				},
2834			},
2835			PrepareFn: func(t *testing.T, def setupBuildPodTestDef, e *executor) {
2836				mockFc := &mockFeatureChecker{}
2837				mockFc.On("IsHostAliasSupported").Return(false, &badVersionError{})
2838				e.featureChecker = mockFc
2839			},
2840			VerifySetupBuildPodErrFn: func(t *testing.T, err error) {
2841				assert.NoError(t, err)
2842			},
2843		},
2844		"no init container defined": {
2845			RunnerConfig: common.RunnerConfig{
2846				RunnerSettings: common.RunnerSettings{
2847					Kubernetes: &common.KubernetesConfig{
2848						Namespace: "default",
2849					},
2850				},
2851			},
2852			InitContainers: []api.Container{},
2853			VerifyFn: func(t *testing.T, def setupBuildPodTestDef, pod *api.Pod) {
2854				assert.Nil(t, pod.Spec.InitContainers)
2855			},
2856		},
2857		"init container defined": {
2858			RunnerConfig: common.RunnerConfig{
2859				RunnerSettings: common.RunnerSettings{
2860					Kubernetes: &common.KubernetesConfig{
2861						Namespace: "default",
2862					},
2863				},
2864			},
2865			InitContainers: []api.Container{
2866				{
2867					Name:  "a-init-container",
2868					Image: "alpine",
2869				},
2870			},
2871			VerifyFn: func(t *testing.T, def setupBuildPodTestDef, pod *api.Pod) {
2872				require.Equal(t, def.InitContainers, pod.Spec.InitContainers)
2873			},
2874		},
2875		"support setting linux capabilities": {
2876			RunnerConfig: common.RunnerConfig{
2877				RunnerSettings: common.RunnerSettings{
2878					Kubernetes: &common.KubernetesConfig{
2879						Namespace: "default",
2880						CapAdd:    []string{"CAP_1", "CAP_2"},
2881						CapDrop:   []string{"CAP_3", "CAP_4"},
2882					},
2883				},
2884			},
2885			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2886				require.NotEmpty(t, pod.Spec.Containers)
2887				capabilities := pod.Spec.Containers[0].SecurityContext.Capabilities
2888				require.NotNil(t, capabilities)
2889				assert.Len(t, capabilities.Add, 2)
2890				assert.Contains(t, capabilities.Add, api.Capability("CAP_1"))
2891				assert.Contains(t, capabilities.Add, api.Capability("CAP_2"))
2892				assert.Len(t, capabilities.Drop, 3)
2893				assert.Contains(t, capabilities.Drop, api.Capability("CAP_3"))
2894				assert.Contains(t, capabilities.Drop, api.Capability("CAP_4"))
2895				assert.Contains(t, capabilities.Drop, api.Capability("NET_RAW"))
2896			},
2897		},
2898		"setting linux capabilities overriding defaults": {
2899			RunnerConfig: common.RunnerConfig{
2900				RunnerSettings: common.RunnerSettings{
2901					Kubernetes: &common.KubernetesConfig{
2902						Namespace: "default",
2903						CapAdd:    []string{"NET_RAW", "CAP_2"},
2904					},
2905				},
2906			},
2907			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2908				require.NotEmpty(t, pod.Spec.Containers)
2909				capabilities := pod.Spec.Containers[0].SecurityContext.Capabilities
2910				require.NotNil(t, capabilities)
2911				assert.Len(t, capabilities.Add, 2)
2912				assert.Contains(t, capabilities.Add, api.Capability("NET_RAW"))
2913				assert.Contains(t, capabilities.Add, api.Capability("CAP_2"))
2914				assert.Empty(t, capabilities.Drop)
2915			},
2916		},
2917		"setting same linux capabilities, drop wins": {
2918			RunnerConfig: common.RunnerConfig{
2919				RunnerSettings: common.RunnerSettings{
2920					Kubernetes: &common.KubernetesConfig{
2921						Namespace: "default",
2922						CapAdd:    []string{"CAP_1"},
2923						CapDrop:   []string{"CAP_1"},
2924					},
2925				},
2926			},
2927			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2928				require.NotEmpty(t, pod.Spec.Containers)
2929				capabilities := pod.Spec.Containers[0].SecurityContext.Capabilities
2930				require.NotNil(t, capabilities)
2931				assert.Empty(t, capabilities.Add)
2932				assert.Len(t, capabilities.Drop, 2)
2933				assert.Contains(t, capabilities.Drop, api.Capability("NET_RAW"))
2934				assert.Contains(t, capabilities.Drop, api.Capability("CAP_1"))
2935			},
2936		},
2937		"support setting linux capabilities on all containers": {
2938			RunnerConfig: common.RunnerConfig{
2939				RunnerSettings: common.RunnerSettings{
2940					Kubernetes: &common.KubernetesConfig{
2941						Namespace: "default",
2942						CapAdd:    []string{"CAP_1"},
2943						CapDrop:   []string{"CAP_2"},
2944					},
2945				},
2946			},
2947			Options: &kubernetesOptions{
2948				Services: common.Services{
2949					{
2950						Name:    "test-service-0",
2951						Command: []string{"application", "--debug"},
2952					},
2953					{
2954						Name:       "test-service-1",
2955						Entrypoint: []string{"application", "--debug"},
2956					},
2957				},
2958			},
2959			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2960				require.Len(t, pod.Spec.Containers, 4)
2961
2962				assertContainerCap := func(container api.Container) {
2963					t.Run("container-"+container.Name, func(t *testing.T) {
2964						capabilities := container.SecurityContext.Capabilities
2965						require.NotNil(t, capabilities)
2966						assert.Len(t, capabilities.Add, 1)
2967						assert.Contains(t, capabilities.Add, api.Capability("CAP_1"))
2968						assert.Len(t, capabilities.Drop, 2)
2969						assert.Contains(t, capabilities.Drop, api.Capability("CAP_2"))
2970						assert.Contains(t, capabilities.Drop, api.Capability("NET_RAW"))
2971					})
2972				}
2973
2974				assertContainerCap(pod.Spec.Containers[0])
2975				assertContainerCap(pod.Spec.Containers[1])
2976				assertContainerCap(pod.Spec.Containers[2])
2977				assertContainerCap(pod.Spec.Containers[3])
2978			},
2979		},
2980		"support setting DNS policy to empty string": {
2981			RunnerConfig: common.RunnerConfig{
2982				RunnerSettings: common.RunnerSettings{
2983					Kubernetes: &common.KubernetesConfig{
2984						Namespace: "default",
2985						DNSPolicy: "",
2986					},
2987				},
2988			},
2989			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
2990				assert.Equal(t, api.DNSClusterFirst, pod.Spec.DNSPolicy)
2991				assert.Nil(t, pod.Spec.DNSConfig)
2992			},
2993		},
2994		"support setting DNS policy to none": {
2995			RunnerConfig: common.RunnerConfig{
2996				RunnerSettings: common.RunnerSettings{
2997					Kubernetes: &common.KubernetesConfig{
2998						Namespace: "default",
2999						DNSPolicy: common.DNSPolicyNone,
3000					},
3001				},
3002			},
3003			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
3004				assert.Equal(t, api.DNSNone, pod.Spec.DNSPolicy)
3005			},
3006		},
3007		"support setting DNS policy to default": {
3008			RunnerConfig: common.RunnerConfig{
3009				RunnerSettings: common.RunnerSettings{
3010					Kubernetes: &common.KubernetesConfig{
3011						Namespace: "default",
3012						DNSPolicy: common.DNSPolicyDefault,
3013					},
3014				},
3015			},
3016			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
3017				assert.Equal(t, api.DNSDefault, pod.Spec.DNSPolicy)
3018				assert.Nil(t, pod.Spec.DNSConfig)
3019			},
3020		},
3021		"support setting DNS policy to cluster-first": {
3022			RunnerConfig: common.RunnerConfig{
3023				RunnerSettings: common.RunnerSettings{
3024					Kubernetes: &common.KubernetesConfig{
3025						Namespace: "default",
3026						DNSPolicy: common.DNSPolicyClusterFirst,
3027					},
3028				},
3029			},
3030			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
3031				assert.Equal(t, api.DNSClusterFirst, pod.Spec.DNSPolicy)
3032				assert.Nil(t, pod.Spec.DNSConfig)
3033			},
3034		},
3035		"support setting DNS policy to cluster-first-with-host-net": {
3036			RunnerConfig: common.RunnerConfig{
3037				RunnerSettings: common.RunnerSettings{
3038					Kubernetes: &common.KubernetesConfig{
3039						Namespace: "default",
3040						DNSPolicy: common.DNSPolicyClusterFirstWithHostNet,
3041					},
3042				},
3043			},
3044			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
3045				assert.Equal(t, api.DNSClusterFirstWithHostNet, pod.Spec.DNSPolicy)
3046				assert.Nil(t, pod.Spec.DNSConfig)
3047			},
3048		},
3049		"fail setting DNS policy to invalid value": {
3050			RunnerConfig: common.RunnerConfig{
3051				RunnerSettings: common.RunnerSettings{
3052					Kubernetes: &common.KubernetesConfig{
3053						Namespace: "default",
3054						DNSPolicy: "some-invalid-policy",
3055					},
3056				},
3057			},
3058			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
3059				assert.Empty(t, pod.Spec.DNSPolicy)
3060				assert.Nil(t, pod.Spec.DNSConfig)
3061			},
3062		},
3063		"support setting pod DNS config": {
3064			RunnerConfig: common.RunnerConfig{
3065				RunnerSettings: common.RunnerSettings{
3066					Kubernetes: &common.KubernetesConfig{
3067						Namespace: "default",
3068						DNSConfig: common.KubernetesDNSConfig{
3069							Nameservers: []string{"1.2.3.4"},
3070							Searches:    []string{"ns1.svc.cluster-domain.example", "my.dns.search.suffix"},
3071							Options: []common.KubernetesDNSConfigOption{
3072								{Name: "ndots", Value: &ndotsValue},
3073								{Name: "edns0"},
3074							},
3075						},
3076					},
3077				},
3078			},
3079			VerifyFn: func(t *testing.T, test setupBuildPodTestDef, pod *api.Pod) {
3080				require.NotNil(t, pod.Spec.DNSConfig)
3081
3082				assert.Equal(t, []string{"1.2.3.4"}, pod.Spec.DNSConfig.Nameservers)
3083				assert.Equal(
3084					t,
3085					[]string{
3086						"ns1.svc.cluster-domain.example",
3087						"my.dns.search.suffix",
3088					},
3089					pod.Spec.DNSConfig.Searches,
3090				)
3091
3092				options := pod.Spec.DNSConfig.Options
3093				require.Len(t, options, 2)
3094				assert.Equal(t, "ndots", options[0].Name)
3095				assert.Equal(t, "edns0", options[1].Name)
3096				require.NotNil(t, options[0].Value)
3097				assert.Equal(t, ndotsValue, *options[0].Value)
3098				assert.Nil(t, options[1].Value)
3099			},
3100		},
3101	}
3102
3103	for testName, test := range tests {
3104		t.Run(testName, func(t *testing.T) {
3105			helperImageInfo, err := helperimage.Get(common.REVISION, helperimage.Config{
3106				OSType:       helperimage.OSTypeLinux,
3107				Architecture: "amd64",
3108			})
3109			require.NoError(t, err)
3110
3111			vars := test.Variables
3112			if vars == nil {
3113				vars = []common.JobVariable{}
3114			}
3115
3116			options := test.Options
3117			if options == nil {
3118				options = &kubernetesOptions{}
3119			}
3120
3121			rt := setupBuildPodFakeRoundTripper{
3122				t:    t,
3123				test: test,
3124			}
3125
3126			mockFc := &mockFeatureChecker{}
3127			mockFc.On("IsHostAliasSupported").Return(true, nil)
3128
3129			mockPullManager := &pull.MockManager{}
3130			defer mockPullManager.AssertExpectations(t)
3131
3132			ex := executor{
3133				kubeClient: testKubernetesClient(version, fake.CreateHTTPClient(rt.RoundTrip)),
3134				options:    options,
3135				configMap:  fakeConfigMap(),
3136				AbstractExecutor: executors.AbstractExecutor{
3137					Config:     test.RunnerConfig,
3138					BuildShell: &common.ShellConfiguration{},
3139					Build: &common.Build{
3140						JobResponse: common.JobResponse{
3141							Variables: vars,
3142						},
3143						Runner: &test.RunnerConfig,
3144					},
3145					ProxyPool: proxy.NewPool(),
3146				},
3147				helperImageInfo: helperImageInfo,
3148				featureChecker:  mockFc,
3149				pullManager:     mockPullManager,
3150			}
3151
3152			if ex.options.Image.Name == "" {
3153				// Ensure we have a valid Docker image name in the configuration,
3154				// if nothing is specified in the test case
3155				ex.options.Image.Name = "build-image"
3156			}
3157
3158			if test.PrepareFn != nil {
3159				test.PrepareFn(t, test, &ex)
3160			}
3161
3162			if test.Options != nil && test.Options.Services != nil {
3163				for _, service := range test.Options.Services {
3164					mockPullManager.On("GetPullPolicyFor", service.Name).
3165						Return(api.PullAlways, nil).
3166						Once()
3167				}
3168			}
3169
3170			mockPullManager.On("GetPullPolicyFor", ex.getHelperImage()).
3171				Return(api.PullAlways, nil).
3172				Maybe()
3173			mockPullManager.On("GetPullPolicyFor", ex.options.Image.Name).
3174				Return(api.PullAlways, nil).
3175				Maybe()
3176
3177			err = ex.prepareOverwrites(make(common.JobVariables, 0))
3178			assert.NoError(t, err, "error preparing overwrites")
3179
3180			err = ex.setupBuildPod(test.InitContainers)
3181			if test.VerifySetupBuildPodErrFn == nil {
3182				assert.NoError(t, err, "error setting up build pod")
3183				assert.True(t, rt.executed, "RoundTrip for kubernetes client should be executed")
3184			} else {
3185				test.VerifySetupBuildPodErrFn(t, err)
3186			}
3187
3188			if test.VerifyExecutorFn != nil {
3189				test.VerifyExecutorFn(t, test, &ex)
3190			}
3191		})
3192	}
3193}
3194
3195func TestProcessLogs(t *testing.T) {
3196	tests := map[string]struct {
3197		lineCh           chan string
3198		errCh            chan error
3199		expectedExitCode int
3200		expectedScript   string
3201		run              func(ch chan string, errCh chan error)
3202	}{
3203		"Successful Processing": {
3204			lineCh:           make(chan string, 2),
3205			errCh:            make(chan error, 1),
3206			expectedExitCode: 1,
3207			expectedScript:   "script",
3208			run: func(ch chan string, errCh chan error) {
3209				b, err := json.Marshal(getCommandExitStatus(1, "script"))
3210				require.NoError(t, err)
3211				ch <- string(b)
3212			},
3213		},
3214		"Reattach failure with CodeExitError": {
3215			lineCh:           make(chan string, 1),
3216			errCh:            make(chan error, 1),
3217			expectedExitCode: 2,
3218			expectedScript:   "",
3219			run: func(ch chan string, errCh chan error) {
3220				errCh <- exec.CodeExitError{
3221					Err:  fmt.Errorf("giving up reattaching to log"),
3222					Code: 2,
3223				}
3224			},
3225		},
3226		"Reattach failure with EOF error": {
3227			lineCh:           make(chan string, 1),
3228			errCh:            make(chan error, 1),
3229			expectedExitCode: unknownLogProcessorExitCode,
3230			expectedScript:   "",
3231			run: func(ch chan string, errCh chan error) {
3232				errCh <- fmt.Errorf("Custom error for test with EOF %s", io.EOF)
3233			},
3234		},
3235		"Reattach failure with custom error": {
3236			lineCh:           make(chan string, 1),
3237			errCh:            make(chan error, 1),
3238			expectedExitCode: unknownLogProcessorExitCode,
3239			expectedScript:   "",
3240			run: func(ch chan string, errCh chan error) {
3241				errCh <- errors.New("Custom error")
3242			},
3243		},
3244		"Error channel closed before line channel": {
3245			lineCh:           make(chan string, 2),
3246			errCh:            make(chan error, 1),
3247			expectedExitCode: 3,
3248			expectedScript:   "script",
3249			run: func(ch chan string, errCh chan error) {
3250				close(errCh)
3251				b, err := json.Marshal(getCommandExitStatus(3, "script"))
3252				require.NoError(t, err)
3253				ch <- string(b)
3254				close(ch)
3255			},
3256		},
3257	}
3258
3259	for tn, tc := range tests {
3260		t.Run(tn, func(t *testing.T) {
3261			mockTrace := &common.MockJobTrace{}
3262			defer mockTrace.AssertExpectations(t)
3263			mockTrace.On("Write", []byte("line\n")).Return(0, nil).Once()
3264
3265			mockLogProcessor := new(mockLogProcessor)
3266			defer mockLogProcessor.AssertExpectations(t)
3267
3268			tc.lineCh <- "line"
3269			mockLogProcessor.On("Process", mock.Anything).
3270				Return((<-chan string)(tc.lineCh), (<-chan error)(tc.errCh)).
3271				Once()
3272
3273			tc.run(tc.lineCh, tc.errCh)
3274
3275			e := newExecutor()
3276			e.Trace = mockTrace
3277			e.pod = &api.Pod{}
3278			e.pod.Name = "pod_name"
3279			e.pod.Namespace = "namespace"
3280			e.newLogProcessor = func() logProcessor {
3281				return mockLogProcessor
3282			}
3283
3284			go e.processLogs(context.Background())
3285
3286			exitStatus := <-e.remoteProcessTerminated
3287			assert.Equal(t, tc.expectedExitCode, *exitStatus.CommandExitCode)
3288			if tc.expectedScript != "" {
3289				assert.Equal(t, tc.expectedScript, *exitStatus.Script)
3290			}
3291		})
3292	}
3293}
3294
3295func getCommandExitStatus(exitCode int, script string) shells.TrapCommandExitStatus {
3296	return shells.TrapCommandExitStatus{
3297		CommandExitCode: &exitCode,
3298		Script:          &script,
3299	}
3300}
3301
3302func TestRunAttachCheckPodStatus(t *testing.T) {
3303	version, codec := testVersionAndCodec()
3304
3305	respErr := errors.New("err")
3306
3307	type podResponse struct {
3308		response *http.Response
3309		err      error
3310	}
3311
3312	tests := map[string]struct {
3313		responses []podResponse
3314		verifyErr func(t *testing.T, errCh <-chan error)
3315	}{
3316		"no error": {
3317			responses: []podResponse{
3318				{
3319					response: &http.Response{StatusCode: http.StatusOK},
3320					err:      nil,
3321				},
3322			},
3323			verifyErr: func(t *testing.T, errCh <-chan error) {
3324				assert.NoError(t, <-errCh)
3325			},
3326		},
3327		"pod phase failed": {
3328			responses: []podResponse{
3329				{
3330					response: &http.Response{
3331						StatusCode: http.StatusOK,
3332						Body:       objBody(codec, execPodWithPhase(api.PodFailed)),
3333					},
3334					err: nil,
3335				},
3336			},
3337			verifyErr: func(t *testing.T, errCh <-chan error) {
3338				err := <-errCh
3339				require.Error(t, err)
3340				var phaseErr *podPhaseError
3341				assert.ErrorAs(t, err, &phaseErr)
3342				assert.Equal(t, api.PodFailed, phaseErr.phase)
3343			},
3344		},
3345		"pod not found": {
3346			responses: []podResponse{
3347				{
3348					response: nil,
3349					err: &kubeerrors.StatusError{
3350						ErrStatus: metav1.Status{
3351							Code: http.StatusNotFound,
3352							Details: &metav1.StatusDetails{
3353								Kind: "pods",
3354							},
3355						},
3356					},
3357				},
3358			},
3359			verifyErr: func(t *testing.T, errCh <-chan error) {
3360				err := <-errCh
3361				require.Error(t, err)
3362				var statusErr *kubeerrors.StatusError
3363				assert.ErrorAs(t, err, &statusErr)
3364				assert.Equal(t, int32(http.StatusNotFound), statusErr.ErrStatus.Code)
3365			},
3366		},
3367		"general error continues": {
3368			responses: []podResponse{
3369				{
3370					response: nil,
3371					err:      respErr,
3372				},
3373				{
3374					response: nil,
3375					err:      respErr,
3376				},
3377				{
3378					response: nil,
3379					err:      respErr,
3380				},
3381			},
3382			verifyErr: func(t *testing.T, errCh <-chan error) {
3383				select {
3384				case err, more := <-errCh:
3385					assert.False(t, more)
3386					assert.NoError(t, err)
3387				case <-time.After(10 * time.Second):
3388					require.Fail(t, "Should not get any error")
3389				}
3390			},
3391		},
3392	}
3393
3394	for tn, tt := range tests {
3395		t.Run(tn, func(t *testing.T) {
3396			ctx, cancel := context.WithCancel(context.Background())
3397			defer cancel()
3398
3399			i := 0
3400			fakeClient := fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) {
3401				switch p, m := req.URL.Path, req.Method; {
3402				case p == "/api/v1/namespaces/namespace/pods/pod" && m == http.MethodGet:
3403					res := tt.responses[i]
3404					i++
3405					if i == len(tt.responses) {
3406						cancel()
3407					}
3408
3409					if res.response == nil {
3410						return nil, res.err
3411					}
3412
3413					res.response.Header = map[string][]string{
3414						"Content-Type": {"application/json"},
3415					}
3416					if res.response.Body == nil {
3417						res.response.Body = objBody(codec, execPod())
3418					}
3419
3420					return res.response, nil
3421				default:
3422					return nil, fmt.Errorf("unexpected request")
3423				}
3424			})
3425
3426			client := testKubernetesClient(version, fakeClient)
3427
3428			e := executor{}
3429			e.Config = common.RunnerConfig{}
3430			e.Config.Kubernetes = &common.KubernetesConfig{
3431				PollInterval: 1,
3432				PollTimeout:  2,
3433			}
3434			e.kubeClient = client
3435			e.remoteProcessTerminated = make(chan shells.TrapCommandExitStatus)
3436			e.Trace = &common.Trace{Writer: os.Stdout}
3437			e.pod = &api.Pod{}
3438			e.pod.Name = "pod"
3439			e.pod.Namespace = "namespace"
3440
3441			tt.verifyErr(t, e.watchPodStatus(ctx))
3442		})
3443	}
3444}
3445
3446func fakeConfigMap() *api.ConfigMap {
3447	configMap := &api.ConfigMap{}
3448	configMap.Name = "fake"
3449	return configMap
3450}
3451
3452func fakeKubeDeleteResponse(status int) *http.Response {
3453	_, codec := testVersionAndCodec()
3454
3455	body := objBody(codec, &metav1.Status{Code: int32(status)})
3456	return &http.Response{StatusCode: status, Body: body, Header: map[string][]string{
3457		"Content-Type": {"application/json"},
3458	}}
3459}
3460
3461func TestNewLogStreamerStream(t *testing.T) {
3462	abortErr := errors.New("abort")
3463
3464	pod := new(api.Pod)
3465	pod.Namespace = "k8s_namespace"
3466	pod.Name = "k8s_pod_name"
3467
3468	client := mockKubernetesClientWithHost("", "", nil)
3469	output := new(bytes.Buffer)
3470	offset := 15
3471
3472	e := newExecutor()
3473	e.pod = pod
3474	e.Build = &common.Build{
3475		Runner: new(common.RunnerConfig),
3476	}
3477
3478	remoteExecutor := new(MockRemoteExecutor)
3479	urlMatcher := mock.MatchedBy(func(url *url.URL) bool {
3480		query := url.Query()
3481		assert.Equal(t, helperContainerName, query.Get("container"))
3482		assert.Equal(t, "true", query.Get("stdout"))
3483		assert.Equal(t, "true", query.Get("stderr"))
3484		command := query["command"]
3485		assert.Equal(t, []string{
3486			"gitlab-runner-helper",
3487			"read-logs",
3488			"--path",
3489			e.logFile(),
3490			"--offset",
3491			strconv.Itoa(offset),
3492			"--wait-file-timeout",
3493			waitLogFileTimeout.String(),
3494		}, command)
3495
3496		return true
3497	})
3498	remoteExecutor.
3499		On("Execute", http.MethodPost, urlMatcher, mock.Anything, nil, output, output, false).
3500		Return(abortErr)
3501
3502	p, ok := e.newLogProcessor().(*kubernetesLogProcessor)
3503	require.True(t, ok)
3504	p.logsOffset = int64(offset)
3505
3506	s, ok := p.logStreamer.(*kubernetesLogStreamer)
3507	require.True(t, ok)
3508	s.client = client
3509	s.executor = remoteExecutor
3510
3511	assert.Equal(t, pod.Name, s.pod)
3512	assert.Equal(t, pod.Namespace, s.namespace)
3513
3514	err := s.Stream(int64(offset), output)
3515	assert.ErrorIs(t, err, abortErr)
3516}
3517
3518type FakeReadCloser struct {
3519	io.Reader
3520}
3521
3522func (f FakeReadCloser) Close() error {
3523	return nil
3524}
3525
3526type FakeBuildTrace struct {
3527	testWriter
3528}
3529
3530func (f FakeBuildTrace) Success()                                          {}
3531func (f FakeBuildTrace) Fail(err error, failureData common.JobFailureData) {}
3532func (f FakeBuildTrace) Notify(func())                                     {}
3533func (f FakeBuildTrace) SetCancelFunc(cancelFunc context.CancelFunc)       {}
3534func (f FakeBuildTrace) Cancel() bool                                      { return false }
3535func (f FakeBuildTrace) SetAbortFunc(cancelFunc context.CancelFunc)        {}
3536func (f FakeBuildTrace) Abort() bool                                       { return false }
3537func (f FakeBuildTrace) SetFailuresCollector(fc common.FailuresCollector)  {}
3538func (f FakeBuildTrace) SetMasked(masked []string)                         {}
3539func (f FakeBuildTrace) IsStdout() bool {
3540	return false
3541}
3542
3543func TestCommandTerminatedError_Is(t *testing.T) {
3544	tests := map[string]struct {
3545		err error
3546
3547		expectedIsResult bool
3548	}{
3549		"nil": {
3550			err:              nil,
3551			expectedIsResult: false,
3552		},
3553		"EOF": {
3554			err:              io.EOF,
3555			expectedIsResult: false,
3556		},
3557		"commandTerminatedError": {
3558			err:              &commandTerminatedError{},
3559			expectedIsResult: true,
3560		},
3561	}
3562
3563	for tn, tt := range tests {
3564		t.Run(tn, func(t *testing.T) {
3565			if tt.expectedIsResult {
3566				assert.ErrorIs(t, tt.err, new(commandTerminatedError))
3567				return
3568			}
3569
3570			assert.NotErrorIs(t, tt.err, new(commandTerminatedError))
3571		})
3572	}
3573}
3574
3575func TestGenerateScripts(t *testing.T) {
3576	testErr := errors.New("testErr")
3577
3578	successfulResponse, err := common.GetRemoteSuccessfulMultistepBuild()
3579	require.NoError(t, err)
3580
3581	setupMockShellGenerateScript := func(m *common.MockShell, e *executor, stages []common.BuildStage) {
3582		for _, s := range stages {
3583			m.On("GenerateScript", s, e.ExecutorOptions.Shell).
3584				Return("OK", nil).
3585				Once()
3586		}
3587	}
3588
3589	setupScripts := func(e *executor, stages []common.BuildStage) map[string]string {
3590		scripts := map[string]string{}
3591		switch e.Shell().Shell {
3592		case shells.SNPwsh:
3593			scripts[parsePwshScriptName] = shells.PwshValidationScript
3594		default:
3595			scripts[detectShellScriptName] = shells.BashDetectShellScript
3596		}
3597
3598		for _, s := range stages {
3599			scripts[string(s)] = "OK"
3600		}
3601
3602		return scripts
3603	}
3604
3605	tests := map[string]struct {
3606		getExecutor        func() *executor
3607		setupMockShell     func(e *executor) *common.MockShell
3608		getExpectedScripts func(e *executor) map[string]string
3609		expectedErr        error
3610	}{
3611		"all stages OK": {
3612			getExecutor: func() *executor {
3613				return setupExecutor("bash", successfulResponse)
3614			},
3615			setupMockShell: func(e *executor) *common.MockShell {
3616				buildStages := e.Build.BuildStages()
3617				m := new(common.MockShell)
3618				setupMockShellGenerateScript(m, e, buildStages)
3619
3620				return m
3621			},
3622			getExpectedScripts: func(e *executor) map[string]string {
3623				buildStages := e.Build.BuildStages()
3624				return setupScripts(e, buildStages)
3625			},
3626			expectedErr: nil,
3627		},
3628		"all stages OK with pwsh": {
3629			getExecutor: func() *executor {
3630				return setupExecutor(shells.SNPwsh, successfulResponse)
3631			},
3632			setupMockShell: func(e *executor) *common.MockShell {
3633				buildStages := e.Build.BuildStages()
3634				m := new(common.MockShell)
3635				setupMockShellGenerateScript(m, e, buildStages)
3636
3637				return m
3638			},
3639			getExpectedScripts: func(e *executor) map[string]string {
3640				buildStages := e.Build.BuildStages()
3641				return setupScripts(e, buildStages)
3642			},
3643			expectedErr: nil,
3644		},
3645		"stage returns skip build stage error": {
3646			getExecutor: func() *executor {
3647				return setupExecutor("bash", successfulResponse)
3648			},
3649			setupMockShell: func(e *executor) *common.MockShell {
3650				buildStages := e.Build.BuildStages()
3651				m := new(common.MockShell)
3652				m.On("GenerateScript", buildStages[0], e.ExecutorOptions.Shell).
3653					Return("", common.ErrSkipBuildStage).
3654					Once()
3655
3656				setupMockShellGenerateScript(m, e, buildStages[1:])
3657
3658				return m
3659			},
3660			getExpectedScripts: func(e *executor) map[string]string {
3661				buildStages := e.Build.BuildStages()
3662				return setupScripts(e, buildStages[1:])
3663			},
3664			expectedErr: nil,
3665		},
3666		"stage returns error": {
3667			getExecutor: func() *executor {
3668				return setupExecutor("bash", successfulResponse)
3669			},
3670			setupMockShell: func(e *executor) *common.MockShell {
3671				buildStages := e.Build.BuildStages()
3672				m := new(common.MockShell)
3673				m.On("GenerateScript", buildStages[0], e.ExecutorOptions.Shell).
3674					Return("", testErr).
3675					Once()
3676
3677				return m
3678			},
3679			getExpectedScripts: func(e *executor) map[string]string {
3680				return nil
3681			},
3682			expectedErr: testErr,
3683		},
3684	}
3685
3686	for tn, tt := range tests {
3687		t.Run(tn, func(t *testing.T) {
3688			e := tt.getExecutor()
3689			expectedScripts := tt.getExpectedScripts(e)
3690			m := tt.setupMockShell(e)
3691			defer m.AssertExpectations(t)
3692
3693			scripts, err := e.generateScripts(m)
3694			assert.ErrorIs(t, err, tt.expectedErr)
3695			assert.Equal(t, expectedScripts, scripts)
3696		})
3697	}
3698}
3699
3700func TestExecutor_buildLogPermissionsInitContainer(t *testing.T) {
3701	dockerHub, err := helperimage.Get(common.REVISION, helperimage.Config{
3702		OSType:       helperimage.OSTypeLinux,
3703		Architecture: "amd64",
3704	})
3705	require.NoError(t, err)
3706
3707	gitlabRegistry, err := helperimage.Get(common.REVISION, helperimage.Config{
3708		OSType:         helperimage.OSTypeLinux,
3709		Architecture:   "amd64",
3710		GitLabRegistry: true,
3711	})
3712	require.NoError(t, err)
3713
3714	tests := map[string]struct {
3715		expectedImage string
3716		jobVariables  common.JobVariables
3717		config        common.RunnerConfig
3718	}{
3719		"default helper image": {
3720			expectedImage: gitlabRegistry.String(),
3721			config: common.RunnerConfig{
3722				RunnerSettings: common.RunnerSettings{
3723					Kubernetes: &common.KubernetesConfig{
3724						Image:      "alpine:3.12",
3725						PullPolicy: common.StringOrArray{common.PullPolicyIfNotPresent},
3726						Host:       "127.0.0.1",
3727					},
3728				},
3729			},
3730		},
3731		"helper image from DockerHub": {
3732			expectedImage: dockerHub.String(),
3733			jobVariables: []common.JobVariable{
3734				{
3735					Key:    featureflags.GitLabRegistryHelperImage,
3736					Value:  "false",
3737					Public: true,
3738				},
3739			},
3740			config: common.RunnerConfig{
3741				RunnerSettings: common.RunnerSettings{
3742					Kubernetes: &common.KubernetesConfig{
3743						Image:      "alpine:3.12",
3744						PullPolicy: common.StringOrArray{common.PullPolicyIfNotPresent},
3745						Host:       "127.0.0.1",
3746					},
3747				},
3748			},
3749		},
3750		"configured helper image": {
3751			expectedImage: "config-image",
3752			config: common.RunnerConfig{
3753				RunnerSettings: common.RunnerSettings{
3754					Kubernetes: &common.KubernetesConfig{
3755						HelperImage: "config-image",
3756						Image:       "alpine:3.12",
3757						PullPolicy:  common.StringOrArray{common.PullPolicyIfNotPresent},
3758						Host:        "127.0.0.1",
3759					},
3760				},
3761			},
3762		},
3763	}
3764
3765	for testName, tt := range tests {
3766		t.Run(testName, func(t *testing.T) {
3767			e := &executor{
3768				AbstractExecutor: executors.AbstractExecutor{
3769					ExecutorOptions: executorOptions,
3770					Build: &common.Build{
3771						JobResponse: common.JobResponse{
3772							Variables: tt.jobVariables,
3773						},
3774						Runner: &tt.config,
3775					},
3776					Config: tt.config,
3777				},
3778			}
3779
3780			prepareOptions := common.ExecutorPrepareOptions{
3781				Config:  &tt.config,
3782				Build:   e.Build,
3783				Context: context.Background(),
3784			}
3785
3786			err := e.Prepare(prepareOptions)
3787			require.NoError(t, err)
3788
3789			c, err := e.buildLogPermissionsInitContainer()
3790			assert.NoError(t, err)
3791			assert.Equal(t, tt.expectedImage, c.Image)
3792			assert.Equal(t, api.PullIfNotPresent, c.ImagePullPolicy)
3793			assert.Len(t, c.VolumeMounts, 1)
3794			assert.Len(t, c.Command, 3)
3795		})
3796	}
3797}
3798
3799func TestExecutor_buildLogPermissionsInitContainer_FailPullPolicy(t *testing.T) {
3800	mockPullManager := &pull.MockManager{}
3801	defer mockPullManager.AssertExpectations(t)
3802
3803	e := &executor{
3804		AbstractExecutor: executors.AbstractExecutor{
3805			ExecutorOptions: executorOptions,
3806			Build: &common.Build{
3807				Runner: &common.RunnerConfig{},
3808			},
3809			Config: common.RunnerConfig{
3810				RunnerSettings: common.RunnerSettings{
3811					Kubernetes: &common.KubernetesConfig{},
3812				},
3813			},
3814		},
3815		pullManager: mockPullManager,
3816	}
3817
3818	mockPullManager.On("GetPullPolicyFor", mock.Anything).
3819		Return(api.PullAlways, assert.AnError).
3820		Once()
3821
3822	_, err := e.buildLogPermissionsInitContainer()
3823	assert.ErrorIs(t, err, assert.AnError)
3824}
3825
3826func TestShellRetrieval(t *testing.T) {
3827	successfulResponse, err := common.GetRemoteSuccessfulMultistepBuild()
3828	require.NoError(t, err)
3829
3830	tests := map[string]struct {
3831		executor     *executor
3832		expectedName string
3833		expectedErr  error
3834	}{
3835		"retrieve bash": {
3836			executor:     setupExecutor("bash", successfulResponse),
3837			expectedName: "bash",
3838		},
3839		"retrieve pwsh": {
3840			executor:     setupExecutor(shells.SNPwsh, successfulResponse),
3841			expectedName: shells.SNPwsh,
3842		},
3843		"failure for no shell": {
3844			executor:    setupExecutor("no shell", successfulResponse),
3845			expectedErr: errIncorrectShellType,
3846		},
3847	}
3848
3849	for tn, tt := range tests {
3850		t.Run(tn, func(t *testing.T) {
3851			shell, err := tt.executor.retrieveShell()
3852			assert.Equal(t, err, tt.expectedErr, "The retrievalShell error and the expected one should be the same")
3853			if tt.expectedErr == nil {
3854				assert.Equal(t, tt.expectedName, shell.GetName())
3855			}
3856		})
3857	}
3858}
3859
3860func TestGetContainerInfo(t *testing.T) {
3861	successfulResponse, err := common.GetRemoteSuccessfulMultistepBuild()
3862	require.NoError(t, err)
3863
3864	tests := map[string]struct {
3865		executor              *executor
3866		command               common.ExecutorCommand
3867		expectedContainerName string
3868		getExpectedCommand    func(e *executor, cmd common.ExecutorCommand) []string
3869	}{
3870		"bash container info": {
3871			executor: setupExecutor("bash", successfulResponse),
3872			command: common.ExecutorCommand{
3873				Stage: common.BuildStagePrepare,
3874			},
3875			expectedContainerName: buildContainerName,
3876			getExpectedCommand: func(e *executor, cmd common.ExecutorCommand) []string {
3877				return []string{
3878					"sh",
3879					e.scriptPath(detectShellScriptName),
3880					e.scriptPath(cmd.Stage),
3881					e.buildRedirectionCmd(),
3882				}
3883			},
3884		},
3885		"predefined bash container info": {
3886			executor: setupExecutor("bash", successfulResponse),
3887			command: common.ExecutorCommand{
3888				Stage:      common.BuildStagePrepare,
3889				Predefined: true,
3890			},
3891			expectedContainerName: helperContainerName,
3892			getExpectedCommand: func(e *executor, cmd common.ExecutorCommand) []string {
3893				return append(
3894					e.helperImageInfo.Cmd,
3895					"<<<",
3896					e.scriptPath(cmd.Stage),
3897					e.buildRedirectionCmd(),
3898				)
3899			},
3900		},
3901		"pwsh container info": {
3902			executor: setupExecutor(shells.SNPwsh, successfulResponse),
3903			command: common.ExecutorCommand{
3904				Stage: common.BuildStagePrepare,
3905			},
3906			expectedContainerName: buildContainerName,
3907			getExpectedCommand: func(e *executor, cmd common.ExecutorCommand) []string {
3908				return []string{
3909					e.scriptPath(parsePwshScriptName),
3910					e.scriptPath(cmd.Stage),
3911					e.logFile(),
3912					e.buildRedirectionCmd(),
3913				}
3914			},
3915		},
3916		"predefined pwsh container info": {
3917			executor: setupExecutor(shells.SNPwsh, successfulResponse),
3918			command: common.ExecutorCommand{
3919				Stage:      common.BuildStagePrepare,
3920				Predefined: true,
3921			},
3922			expectedContainerName: helperContainerName,
3923			getExpectedCommand: func(e *executor, cmd common.ExecutorCommand) []string {
3924				commands := append([]string(nil), fmt.Sprintf("Get-Content -Path %s | ", e.scriptPath(cmd.Stage)))
3925				commands = append(commands, e.helperImageInfo.Cmd...)
3926				commands = append(commands, e.buildRedirectionCmd())
3927				return commands
3928			},
3929		},
3930	}
3931
3932	for tn, tt := range tests {
3933		t.Run(tn, func(t *testing.T) {
3934			containerName, containerCommand := tt.executor.getContainerInfo(tt.command)
3935			assert.Equal(t, tt.expectedContainerName, containerName)
3936			assert.Equal(t, tt.getExpectedCommand(tt.executor, tt.command), containerCommand)
3937		})
3938	}
3939}
3940
3941func setupExecutor(shell string, successfulResponse common.JobResponse) *executor {
3942	return &executor{
3943		helperImageInfo: helperimage.Info{
3944			Cmd: []string{"custom", "command"},
3945		},
3946		AbstractExecutor: executors.AbstractExecutor{
3947			ExecutorOptions: executors.ExecutorOptions{
3948				DefaultBuildsDir: "/builds",
3949				DefaultCacheDir:  "/cache",
3950				Shell: common.ShellScriptInfo{
3951					Shell: shell,
3952				},
3953			},
3954			Build: &common.Build{
3955				JobResponse: successfulResponse,
3956			},
3957		},
3958	}
3959}
3960