1/* 2 Copyright The containerd Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15*/ 16 17package server 18 19import ( 20 "os" 21 "path/filepath" 22 "testing" 23 24 imagespec "github.com/opencontainers/image-spec/specs-go/v1" 25 runtimespec "github.com/opencontainers/runtime-spec/specs-go" 26 "github.com/opencontainers/selinux/go-selinux" 27 "github.com/stretchr/testify/assert" 28 "github.com/stretchr/testify/require" 29 runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" 30 31 "github.com/containerd/containerd/pkg/cri/annotations" 32 "github.com/containerd/containerd/pkg/cri/opts" 33 ostesting "github.com/containerd/containerd/pkg/os/testing" 34) 35 36func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) { 37 config := &runtime.PodSandboxConfig{ 38 Metadata: &runtime.PodSandboxMetadata{ 39 Name: "test-name", 40 Uid: "test-uid", 41 Namespace: "test-ns", 42 Attempt: 1, 43 }, 44 Hostname: "test-hostname", 45 LogDirectory: "test-log-directory", 46 Labels: map[string]string{"a": "b"}, 47 Annotations: map[string]string{"c": "d"}, 48 Linux: &runtime.LinuxPodSandboxConfig{ 49 CgroupParent: "/test/cgroup/parent", 50 }, 51 } 52 imageConfig := &imagespec.ImageConfig{ 53 Env: []string{"a=b", "c=d"}, 54 Entrypoint: []string{"/pause"}, 55 Cmd: []string{"forever"}, 56 WorkingDir: "/workspace", 57 } 58 specCheck := func(t *testing.T, id string, spec *runtimespec.Spec) { 59 assert.Equal(t, "test-hostname", spec.Hostname) 60 assert.Equal(t, getCgroupsPath("/test/cgroup/parent", id), spec.Linux.CgroupsPath) 61 assert.Equal(t, relativeRootfsPath, spec.Root.Path) 62 assert.Equal(t, true, spec.Root.Readonly) 63 assert.Contains(t, spec.Process.Env, "a=b", "c=d") 64 assert.Equal(t, []string{"/pause", "forever"}, spec.Process.Args) 65 assert.Equal(t, "/workspace", spec.Process.Cwd) 66 assert.EqualValues(t, *spec.Linux.Resources.CPU.Shares, opts.DefaultSandboxCPUshares) 67 assert.EqualValues(t, *spec.Process.OOMScoreAdj, defaultSandboxOOMAdj) 68 69 t.Logf("Check PodSandbox annotations") 70 assert.Contains(t, spec.Annotations, annotations.SandboxID) 71 assert.EqualValues(t, spec.Annotations[annotations.SandboxID], id) 72 73 assert.Contains(t, spec.Annotations, annotations.ContainerType) 74 assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeSandbox) 75 76 assert.Contains(t, spec.Annotations, annotations.SandboxNamespace) 77 assert.EqualValues(t, spec.Annotations[annotations.SandboxNamespace], "test-ns") 78 79 assert.Contains(t, spec.Annotations, annotations.SandboxName) 80 assert.EqualValues(t, spec.Annotations[annotations.SandboxName], "test-name") 81 82 assert.Contains(t, spec.Annotations, annotations.SandboxLogDir) 83 assert.EqualValues(t, spec.Annotations[annotations.SandboxLogDir], "test-log-directory") 84 85 if selinux.GetEnabled() { 86 assert.NotEqual(t, "", spec.Process.SelinuxLabel) 87 assert.NotEqual(t, "", spec.Linux.MountLabel) 88 } 89 } 90 return config, imageConfig, specCheck 91} 92 93func TestLinuxSandboxContainerSpec(t *testing.T) { 94 testID := "test-id" 95 nsPath := "test-cni" 96 for desc, test := range map[string]struct { 97 configChange func(*runtime.PodSandboxConfig) 98 specCheck func(*testing.T, *runtimespec.Spec) 99 expectErr bool 100 }{ 101 "spec should reflect original config": { 102 specCheck: func(t *testing.T, spec *runtimespec.Spec) { 103 // runtime spec should have expected namespaces enabled by default. 104 require.NotNil(t, spec.Linux) 105 assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ 106 Type: runtimespec.NetworkNamespace, 107 Path: nsPath, 108 }) 109 assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ 110 Type: runtimespec.UTSNamespace, 111 }) 112 assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ 113 Type: runtimespec.PIDNamespace, 114 }) 115 assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ 116 Type: runtimespec.IPCNamespace, 117 }) 118 }, 119 }, 120 "host namespace": { 121 configChange: func(c *runtime.PodSandboxConfig) { 122 c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ 123 NamespaceOptions: &runtime.NamespaceOption{ 124 Network: runtime.NamespaceMode_NODE, 125 Pid: runtime.NamespaceMode_NODE, 126 Ipc: runtime.NamespaceMode_NODE, 127 }, 128 } 129 }, 130 specCheck: func(t *testing.T, spec *runtimespec.Spec) { 131 // runtime spec should disable expected namespaces in host mode. 132 require.NotNil(t, spec.Linux) 133 assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ 134 Type: runtimespec.NetworkNamespace, 135 }) 136 assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ 137 Type: runtimespec.UTSNamespace, 138 }) 139 assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ 140 Type: runtimespec.PIDNamespace, 141 }) 142 assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ 143 Type: runtimespec.IPCNamespace, 144 }) 145 }, 146 }, 147 "should set supplemental groups correctly": { 148 configChange: func(c *runtime.PodSandboxConfig) { 149 c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ 150 SupplementalGroups: []int64{1111, 2222}, 151 } 152 }, 153 specCheck: func(t *testing.T, spec *runtimespec.Spec) { 154 require.NotNil(t, spec.Process) 155 assert.Contains(t, spec.Process.User.AdditionalGids, uint32(1111)) 156 assert.Contains(t, spec.Process.User.AdditionalGids, uint32(2222)) 157 }, 158 }, 159 } { 160 t.Logf("TestCase %q", desc) 161 c := newTestCRIService() 162 config, imageConfig, specCheck := getRunPodSandboxTestData() 163 if test.configChange != nil { 164 test.configChange(config) 165 } 166 spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, nil) 167 if test.expectErr { 168 assert.Error(t, err) 169 assert.Nil(t, spec) 170 continue 171 } 172 assert.NoError(t, err) 173 assert.NotNil(t, spec) 174 specCheck(t, testID, spec) 175 if test.specCheck != nil { 176 test.specCheck(t, spec) 177 } 178 } 179} 180 181func TestSetupSandboxFiles(t *testing.T) { 182 const ( 183 testID = "test-id" 184 realhostname = "test-real-hostname" 185 ) 186 for desc, test := range map[string]struct { 187 dnsConfig *runtime.DNSConfig 188 hostname string 189 ipcMode runtime.NamespaceMode 190 expectedCalls []ostesting.CalledDetail 191 }{ 192 "should check host /dev/shm existence when ipc mode is NODE": { 193 ipcMode: runtime.NamespaceMode_NODE, 194 expectedCalls: []ostesting.CalledDetail{ 195 { 196 Name: "Hostname", 197 }, 198 { 199 Name: "WriteFile", 200 Arguments: []interface{}{ 201 filepath.Join(testRootDir, sandboxesDir, testID, "hostname"), 202 []byte(realhostname + "\n"), 203 os.FileMode(0644), 204 }, 205 }, 206 { 207 Name: "CopyFile", 208 Arguments: []interface{}{ 209 "/etc/hosts", 210 filepath.Join(testRootDir, sandboxesDir, testID, "hosts"), 211 os.FileMode(0644), 212 }, 213 }, 214 { 215 Name: "CopyFile", 216 Arguments: []interface{}{ 217 "/etc/resolv.conf", 218 filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"), 219 os.FileMode(0644), 220 }, 221 }, 222 { 223 Name: "Stat", 224 Arguments: []interface{}{"/dev/shm"}, 225 }, 226 }, 227 }, 228 "should create new /etc/resolv.conf if DNSOptions is set": { 229 dnsConfig: &runtime.DNSConfig{ 230 Servers: []string{"8.8.8.8"}, 231 Searches: []string{"114.114.114.114"}, 232 Options: []string{"timeout:1"}, 233 }, 234 ipcMode: runtime.NamespaceMode_NODE, 235 expectedCalls: []ostesting.CalledDetail{ 236 { 237 Name: "Hostname", 238 }, 239 { 240 Name: "WriteFile", 241 Arguments: []interface{}{ 242 filepath.Join(testRootDir, sandboxesDir, testID, "hostname"), 243 []byte(realhostname + "\n"), 244 os.FileMode(0644), 245 }, 246 }, 247 { 248 Name: "CopyFile", 249 Arguments: []interface{}{ 250 "/etc/hosts", 251 filepath.Join(testRootDir, sandboxesDir, testID, "hosts"), 252 os.FileMode(0644), 253 }, 254 }, 255 { 256 Name: "WriteFile", 257 Arguments: []interface{}{ 258 filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"), 259 []byte(`search 114.114.114.114 260nameserver 8.8.8.8 261options timeout:1 262`), os.FileMode(0644), 263 }, 264 }, 265 { 266 Name: "Stat", 267 Arguments: []interface{}{"/dev/shm"}, 268 }, 269 }, 270 }, 271 "should create sandbox shm when ipc namespace mode is not NODE": { 272 ipcMode: runtime.NamespaceMode_POD, 273 expectedCalls: []ostesting.CalledDetail{ 274 { 275 Name: "Hostname", 276 }, 277 { 278 Name: "WriteFile", 279 Arguments: []interface{}{ 280 filepath.Join(testRootDir, sandboxesDir, testID, "hostname"), 281 []byte(realhostname + "\n"), 282 os.FileMode(0644), 283 }, 284 }, 285 { 286 Name: "CopyFile", 287 Arguments: []interface{}{ 288 "/etc/hosts", 289 filepath.Join(testRootDir, sandboxesDir, testID, "hosts"), 290 os.FileMode(0644), 291 }, 292 }, 293 { 294 Name: "CopyFile", 295 Arguments: []interface{}{ 296 "/etc/resolv.conf", 297 filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"), 298 os.FileMode(0644), 299 }, 300 }, 301 { 302 Name: "MkdirAll", 303 Arguments: []interface{}{ 304 filepath.Join(testStateDir, sandboxesDir, testID, "shm"), 305 os.FileMode(0700), 306 }, 307 }, 308 { 309 Name: "Mount", 310 // Ignore arguments which are too complex to check. 311 }, 312 }, 313 }, 314 "should create /etc/hostname when hostname is set": { 315 hostname: "test-hostname", 316 ipcMode: runtime.NamespaceMode_NODE, 317 expectedCalls: []ostesting.CalledDetail{ 318 { 319 Name: "WriteFile", 320 Arguments: []interface{}{ 321 filepath.Join(testRootDir, sandboxesDir, testID, "hostname"), 322 []byte("test-hostname\n"), 323 os.FileMode(0644), 324 }, 325 }, 326 { 327 Name: "CopyFile", 328 Arguments: []interface{}{ 329 "/etc/hosts", 330 filepath.Join(testRootDir, sandboxesDir, testID, "hosts"), 331 os.FileMode(0644), 332 }, 333 }, 334 { 335 Name: "CopyFile", 336 Arguments: []interface{}{ 337 "/etc/resolv.conf", 338 filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"), 339 os.FileMode(0644), 340 }, 341 }, 342 { 343 Name: "Stat", 344 Arguments: []interface{}{"/dev/shm"}, 345 }, 346 }, 347 }, 348 } { 349 t.Logf("TestCase %q", desc) 350 c := newTestCRIService() 351 c.os.(*ostesting.FakeOS).HostnameFn = func() (string, error) { 352 return realhostname, nil 353 } 354 cfg := &runtime.PodSandboxConfig{ 355 Hostname: test.hostname, 356 DnsConfig: test.dnsConfig, 357 Linux: &runtime.LinuxPodSandboxConfig{ 358 SecurityContext: &runtime.LinuxSandboxSecurityContext{ 359 NamespaceOptions: &runtime.NamespaceOption{ 360 Ipc: test.ipcMode, 361 }, 362 }, 363 }, 364 } 365 c.setupSandboxFiles(testID, cfg) 366 calls := c.os.(*ostesting.FakeOS).GetCalls() 367 assert.Len(t, calls, len(test.expectedCalls)) 368 for i, expected := range test.expectedCalls { 369 if expected.Arguments == nil { 370 // Ignore arguments. 371 expected.Arguments = calls[i].Arguments 372 } 373 assert.Equal(t, expected, calls[i]) 374 } 375 } 376} 377 378func TestParseDNSOption(t *testing.T) { 379 for desc, test := range map[string]struct { 380 servers []string 381 searches []string 382 options []string 383 expectedContent string 384 expectErr bool 385 }{ 386 "empty dns options should return empty content": {}, 387 "non-empty dns options should return correct content": { 388 servers: []string{"8.8.8.8", "server.google.com"}, 389 searches: []string{"114.114.114.114"}, 390 options: []string{"timeout:1"}, 391 expectedContent: `search 114.114.114.114 392nameserver 8.8.8.8 393nameserver server.google.com 394options timeout:1 395`, 396 }, 397 "expanded dns config should return correct content on modern libc (e.g. glibc 2.26 and above)": { 398 servers: []string{"8.8.8.8", "server.google.com"}, 399 searches: []string{ 400 "server0.google.com", 401 "server1.google.com", 402 "server2.google.com", 403 "server3.google.com", 404 "server4.google.com", 405 "server5.google.com", 406 "server6.google.com", 407 }, 408 options: []string{"timeout:1"}, 409 expectedContent: `search server0.google.com server1.google.com server2.google.com server3.google.com server4.google.com server5.google.com server6.google.com 410nameserver 8.8.8.8 411nameserver server.google.com 412options timeout:1 413`, 414 }, 415 } { 416 t.Logf("TestCase %q", desc) 417 resolvContent, err := parseDNSOptions(test.servers, test.searches, test.options) 418 if test.expectErr { 419 assert.Error(t, err) 420 continue 421 } 422 assert.NoError(t, err) 423 assert.Equal(t, resolvContent, test.expectedContent) 424 } 425} 426 427func TestSandboxDisableCgroup(t *testing.T) { 428 config, imageConfig, _ := getRunPodSandboxTestData() 429 c := newTestCRIService() 430 c.config.DisableCgroup = true 431 spec, err := c.sandboxContainerSpec("test-id", config, imageConfig, "test-cni", []string{}) 432 require.NoError(t, err) 433 434 t.Log("resource limit should not be set") 435 assert.Nil(t, spec.Linux.Resources.Memory) 436 assert.Nil(t, spec.Linux.Resources.CPU) 437 438 t.Log("cgroup path should be empty") 439 assert.Empty(t, spec.Linux.CgroupsPath) 440} 441 442// TODO(random-liu): [P1] Add unit test for different error cases to make sure 443// the function cleans up on error properly. 444