1/* 2Copyright 2016 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package kubelet 18 19import ( 20 "fmt" 21 "testing" 22 23 "github.com/stretchr/testify/assert" 24 v1 "k8s.io/api/core/v1" 25 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 "k8s.io/apimachinery/pkg/runtime" 27 "k8s.io/apimachinery/pkg/types" 28 core "k8s.io/client-go/testing" 29 "k8s.io/kubernetes/pkg/volume" 30 volumetest "k8s.io/kubernetes/pkg/volume/testing" 31 "k8s.io/kubernetes/pkg/volume/util" 32) 33 34func TestListVolumesForPod(t *testing.T) { 35 testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) 36 defer testKubelet.Cleanup() 37 kubelet := testKubelet.kubelet 38 39 pod := podWithUIDNameNsSpec("12345678", "foo", "test", v1.PodSpec{ 40 Containers: []v1.Container{ 41 { 42 Name: "container1", 43 VolumeMounts: []v1.VolumeMount{ 44 { 45 Name: "vol1", 46 MountPath: "/mnt/vol1", 47 }, 48 { 49 Name: "vol2", 50 MountPath: "/mnt/vol2", 51 }, 52 }, 53 }, 54 }, 55 Volumes: []v1.Volume{ 56 { 57 Name: "vol1", 58 VolumeSource: v1.VolumeSource{ 59 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ 60 PDName: "fake-device1", 61 }, 62 }, 63 }, 64 { 65 Name: "vol2", 66 VolumeSource: v1.VolumeSource{ 67 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ 68 PDName: "fake-device2", 69 }, 70 }, 71 }, 72 }, 73 }) 74 75 stopCh := runVolumeManager(kubelet) 76 defer close(stopCh) 77 78 kubelet.podManager.SetPods([]*v1.Pod{pod}) 79 err := kubelet.volumeManager.WaitForAttachAndMount(pod) 80 assert.NoError(t, err) 81 82 podName := util.GetUniquePodName(pod) 83 84 volumesToReturn, volumeExsit := kubelet.ListVolumesForPod(types.UID(podName)) 85 assert.True(t, volumeExsit, "expected to find volumes for pod %q", podName) 86 87 outerVolumeSpecName1 := "vol1" 88 assert.NotNil(t, volumesToReturn[outerVolumeSpecName1], "key %s", outerVolumeSpecName1) 89 90 outerVolumeSpecName2 := "vol2" 91 assert.NotNil(t, volumesToReturn[outerVolumeSpecName2], "key %s", outerVolumeSpecName2) 92} 93 94func TestPodVolumesExist(t *testing.T) { 95 testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) 96 defer testKubelet.Cleanup() 97 kubelet := testKubelet.kubelet 98 99 pods := []*v1.Pod{ 100 { 101 ObjectMeta: metav1.ObjectMeta{ 102 Name: "pod1", 103 UID: "pod1uid", 104 }, 105 Spec: v1.PodSpec{ 106 Containers: []v1.Container{ 107 { 108 Name: "container1", 109 VolumeMounts: []v1.VolumeMount{ 110 { 111 Name: "vol1", 112 MountPath: "/mnt/vol1", 113 }, 114 }, 115 }, 116 }, 117 Volumes: []v1.Volume{ 118 { 119 Name: "vol1", 120 VolumeSource: v1.VolumeSource{ 121 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ 122 PDName: "fake-device1", 123 }, 124 }, 125 }, 126 }, 127 }, 128 }, 129 { 130 ObjectMeta: metav1.ObjectMeta{ 131 Name: "pod2", 132 UID: "pod2uid", 133 }, 134 Spec: v1.PodSpec{ 135 Containers: []v1.Container{ 136 { 137 Name: "container2", 138 VolumeMounts: []v1.VolumeMount{ 139 { 140 Name: "vol2", 141 MountPath: "/mnt/vol2", 142 }, 143 }, 144 }, 145 }, 146 Volumes: []v1.Volume{ 147 { 148 Name: "vol2", 149 VolumeSource: v1.VolumeSource{ 150 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ 151 PDName: "fake-device2", 152 }, 153 }, 154 }, 155 }, 156 }, 157 }, 158 { 159 ObjectMeta: metav1.ObjectMeta{ 160 Name: "pod3", 161 UID: "pod3uid", 162 }, 163 Spec: v1.PodSpec{ 164 Containers: []v1.Container{ 165 { 166 Name: "container3", 167 VolumeMounts: []v1.VolumeMount{ 168 { 169 Name: "vol3", 170 MountPath: "/mnt/vol3", 171 }, 172 }, 173 }, 174 }, 175 Volumes: []v1.Volume{ 176 { 177 Name: "vol3", 178 VolumeSource: v1.VolumeSource{ 179 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ 180 PDName: "fake-device3", 181 }, 182 }, 183 }, 184 }, 185 }, 186 }, 187 } 188 189 stopCh := runVolumeManager(kubelet) 190 defer close(stopCh) 191 192 kubelet.podManager.SetPods(pods) 193 for _, pod := range pods { 194 err := kubelet.volumeManager.WaitForAttachAndMount(pod) 195 assert.NoError(t, err) 196 } 197 198 for _, pod := range pods { 199 podVolumesExist := kubelet.podVolumesExist(pod.UID) 200 assert.True(t, podVolumesExist, "pod %q", pod.UID) 201 } 202} 203 204func TestVolumeAttachAndMountControllerDisabled(t *testing.T) { 205 testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) 206 defer testKubelet.Cleanup() 207 kubelet := testKubelet.kubelet 208 209 pod := podWithUIDNameNsSpec("12345678", "foo", "test", v1.PodSpec{ 210 Containers: []v1.Container{ 211 { 212 Name: "container1", 213 VolumeMounts: []v1.VolumeMount{ 214 { 215 Name: "vol1", 216 MountPath: "/mnt/vol1", 217 }, 218 }, 219 }, 220 }, 221 Volumes: []v1.Volume{ 222 { 223 Name: "vol1", 224 VolumeSource: v1.VolumeSource{ 225 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ 226 PDName: "fake-device", 227 }, 228 }, 229 }, 230 }, 231 }) 232 233 stopCh := runVolumeManager(kubelet) 234 defer close(stopCh) 235 236 kubelet.podManager.SetPods([]*v1.Pod{pod}) 237 err := kubelet.volumeManager.WaitForAttachAndMount(pod) 238 assert.NoError(t, err) 239 240 podVolumes := kubelet.volumeManager.GetMountedVolumesForPod( 241 util.GetUniquePodName(pod)) 242 243 expectedPodVolumes := []string{"vol1"} 244 assert.Len(t, podVolumes, len(expectedPodVolumes), "Volumes for pod %+v", pod) 245 for _, name := range expectedPodVolumes { 246 assert.Contains(t, podVolumes, name, "Volumes for pod %+v", pod) 247 } 248 assert.True(t, testKubelet.volumePlugin.GetNewAttacherCallCount() >= 1, "Expected plugin NewAttacher to be called at least once") 249 assert.NoError(t, volumetest.VerifyWaitForAttachCallCount( 250 1 /* expectedWaitForAttachCallCount */, testKubelet.volumePlugin)) 251 assert.NoError(t, volumetest.VerifyAttachCallCount( 252 1 /* expectedAttachCallCount */, testKubelet.volumePlugin)) 253 assert.NoError(t, volumetest.VerifyMountDeviceCallCount( 254 1 /* expectedMountDeviceCallCount */, testKubelet.volumePlugin)) 255 assert.NoError(t, volumetest.VerifySetUpCallCount( 256 1 /* expectedSetUpCallCount */, testKubelet.volumePlugin)) 257} 258 259func TestVolumeUnmountAndDetachControllerDisabled(t *testing.T) { 260 testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) 261 defer testKubelet.Cleanup() 262 kubelet := testKubelet.kubelet 263 264 pod := podWithUIDNameNsSpec("12345678", "foo", "test", v1.PodSpec{ 265 Containers: []v1.Container{ 266 { 267 Name: "container1", 268 VolumeMounts: []v1.VolumeMount{ 269 { 270 Name: "vol1", 271 MountPath: "/mnt/vol1", 272 }, 273 }, 274 }, 275 }, 276 Volumes: []v1.Volume{ 277 { 278 Name: "vol1", 279 VolumeSource: v1.VolumeSource{ 280 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ 281 PDName: "fake-device", 282 }, 283 }, 284 }, 285 }, 286 }) 287 288 stopCh := runVolumeManager(kubelet) 289 defer close(stopCh) 290 291 // Add pod 292 kubelet.podManager.SetPods([]*v1.Pod{pod}) 293 294 // Verify volumes attached 295 err := kubelet.volumeManager.WaitForAttachAndMount(pod) 296 assert.NoError(t, err) 297 298 podVolumes := kubelet.volumeManager.GetMountedVolumesForPod( 299 util.GetUniquePodName(pod)) 300 301 expectedPodVolumes := []string{"vol1"} 302 assert.Len(t, podVolumes, len(expectedPodVolumes), "Volumes for pod %+v", pod) 303 for _, name := range expectedPodVolumes { 304 assert.Contains(t, podVolumes, name, "Volumes for pod %+v", pod) 305 } 306 307 assert.True(t, testKubelet.volumePlugin.GetNewAttacherCallCount() >= 1, "Expected plugin NewAttacher to be called at least once") 308 assert.NoError(t, volumetest.VerifyWaitForAttachCallCount( 309 1 /* expectedWaitForAttachCallCount */, testKubelet.volumePlugin)) 310 assert.NoError(t, volumetest.VerifyAttachCallCount( 311 1 /* expectedAttachCallCount */, testKubelet.volumePlugin)) 312 assert.NoError(t, volumetest.VerifyMountDeviceCallCount( 313 1 /* expectedMountDeviceCallCount */, testKubelet.volumePlugin)) 314 assert.NoError(t, volumetest.VerifySetUpCallCount( 315 1 /* expectedSetUpCallCount */, testKubelet.volumePlugin)) 316 317 // Remove pod 318 // TODO: this may not be threadsafe (technically waitForVolumeUnmount) 319 kubelet.podWorkers.(*fakePodWorkers).removeRuntime = map[types.UID]bool{pod.UID: true} 320 kubelet.podManager.SetPods([]*v1.Pod{}) 321 322 assert.NoError(t, kubelet.volumeManager.WaitForUnmount(pod)) 323 if actual := kubelet.volumeManager.GetMountedVolumesForPod(util.GetUniquePodName(pod)); len(actual) > 0 { 324 t.Fatalf("expected volume unmount to wait for no volumes: %v", actual) 325 } 326 327 // Verify volumes unmounted 328 podVolumes = kubelet.volumeManager.GetMountedVolumesForPod( 329 util.GetUniquePodName(pod)) 330 331 assert.Len(t, podVolumes, 0, 332 "Expected volumes to be unmounted and detached. But some volumes are still mounted: %#v", podVolumes) 333 334 assert.NoError(t, volumetest.VerifyTearDownCallCount( 335 1 /* expectedTearDownCallCount */, testKubelet.volumePlugin)) 336 337 // Verify volumes detached and no longer reported as in use 338 assert.NoError(t, waitForVolumeDetach(v1.UniqueVolumeName("fake/fake-device"), kubelet.volumeManager)) 339 assert.True(t, testKubelet.volumePlugin.GetNewAttacherCallCount() >= 1, "Expected plugin NewAttacher to be called at least once") 340 assert.NoError(t, volumetest.VerifyDetachCallCount( 341 1 /* expectedDetachCallCount */, testKubelet.volumePlugin)) 342} 343 344func TestVolumeAttachAndMountControllerEnabled(t *testing.T) { 345 testKubelet := newTestKubelet(t, true /* controllerAttachDetachEnabled */) 346 defer testKubelet.Cleanup() 347 kubelet := testKubelet.kubelet 348 kubeClient := testKubelet.fakeKubeClient 349 kubeClient.AddReactor("get", "nodes", 350 func(action core.Action) (bool, runtime.Object, error) { 351 return true, &v1.Node{ 352 ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, 353 Status: v1.NodeStatus{ 354 VolumesAttached: []v1.AttachedVolume{ 355 { 356 Name: "fake/fake-device", 357 DevicePath: "fake/path", 358 }, 359 }}, 360 }, nil 361 }) 362 kubeClient.AddReactor("*", "*", func(action core.Action) (bool, runtime.Object, error) { 363 return true, nil, fmt.Errorf("no reaction implemented for %s", action) 364 }) 365 366 pod := podWithUIDNameNsSpec("12345678", "foo", "test", v1.PodSpec{ 367 Containers: []v1.Container{ 368 { 369 Name: "container1", 370 VolumeMounts: []v1.VolumeMount{ 371 { 372 Name: "vol1", 373 MountPath: "/mnt/vol1", 374 }, 375 }, 376 }, 377 }, 378 Volumes: []v1.Volume{ 379 { 380 Name: "vol1", 381 VolumeSource: v1.VolumeSource{ 382 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ 383 PDName: "fake-device", 384 }, 385 }, 386 }, 387 }, 388 }) 389 390 stopCh := runVolumeManager(kubelet) 391 defer close(stopCh) 392 393 kubelet.podManager.SetPods([]*v1.Pod{pod}) 394 395 // Fake node status update 396 go simulateVolumeInUseUpdate( 397 v1.UniqueVolumeName("fake/fake-device"), 398 stopCh, 399 kubelet.volumeManager) 400 401 assert.NoError(t, kubelet.volumeManager.WaitForAttachAndMount(pod)) 402 403 podVolumes := kubelet.volumeManager.GetMountedVolumesForPod( 404 util.GetUniquePodName(pod)) 405 allPodVolumes := kubelet.volumeManager.GetPossiblyMountedVolumesForPod( 406 util.GetUniquePodName(pod)) 407 assert.Equal(t, podVolumes, allPodVolumes, "GetMountedVolumesForPod and GetPossiblyMountedVolumesForPod should return the same volumes") 408 409 expectedPodVolumes := []string{"vol1"} 410 assert.Len(t, podVolumes, len(expectedPodVolumes), "Volumes for pod %+v", pod) 411 for _, name := range expectedPodVolumes { 412 assert.Contains(t, podVolumes, name, "Volumes for pod %+v", pod) 413 } 414 assert.True(t, testKubelet.volumePlugin.GetNewAttacherCallCount() >= 1, "Expected plugin NewAttacher to be called at least once") 415 assert.NoError(t, volumetest.VerifyWaitForAttachCallCount( 416 1 /* expectedWaitForAttachCallCount */, testKubelet.volumePlugin)) 417 assert.NoError(t, volumetest.VerifyZeroAttachCalls(testKubelet.volumePlugin)) 418 assert.NoError(t, volumetest.VerifyMountDeviceCallCount( 419 1 /* expectedMountDeviceCallCount */, testKubelet.volumePlugin)) 420 assert.NoError(t, volumetest.VerifySetUpCallCount( 421 1 /* expectedSetUpCallCount */, testKubelet.volumePlugin)) 422} 423 424func TestVolumeUnmountAndDetachControllerEnabled(t *testing.T) { 425 testKubelet := newTestKubelet(t, true /* controllerAttachDetachEnabled */) 426 defer testKubelet.Cleanup() 427 kubelet := testKubelet.kubelet 428 kubeClient := testKubelet.fakeKubeClient 429 kubeClient.AddReactor("get", "nodes", 430 func(action core.Action) (bool, runtime.Object, error) { 431 return true, &v1.Node{ 432 ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, 433 Status: v1.NodeStatus{ 434 VolumesAttached: []v1.AttachedVolume{ 435 { 436 Name: "fake/fake-device", 437 DevicePath: "fake/path", 438 }, 439 }}, 440 }, nil 441 }) 442 kubeClient.AddReactor("*", "*", func(action core.Action) (bool, runtime.Object, error) { 443 return true, nil, fmt.Errorf("no reaction implemented for %s", action) 444 }) 445 446 pod := podWithUIDNameNsSpec("12345678", "foo", "test", v1.PodSpec{ 447 Containers: []v1.Container{ 448 { 449 Name: "container1", 450 VolumeMounts: []v1.VolumeMount{ 451 { 452 Name: "vol1", 453 MountPath: "/mnt/vol1", 454 }, 455 }, 456 }, 457 }, 458 Volumes: []v1.Volume{ 459 { 460 Name: "vol1", 461 VolumeSource: v1.VolumeSource{ 462 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ 463 PDName: "fake-device", 464 }, 465 }, 466 }, 467 }, 468 }) 469 470 stopCh := runVolumeManager(kubelet) 471 defer close(stopCh) 472 473 // Add pod 474 kubelet.podManager.SetPods([]*v1.Pod{pod}) 475 476 // Fake node status update 477 go simulateVolumeInUseUpdate( 478 v1.UniqueVolumeName("fake/fake-device"), 479 stopCh, 480 kubelet.volumeManager) 481 482 // Verify volumes attached 483 assert.NoError(t, kubelet.volumeManager.WaitForAttachAndMount(pod)) 484 485 podVolumes := kubelet.volumeManager.GetMountedVolumesForPod( 486 util.GetUniquePodName(pod)) 487 allPodVolumes := kubelet.volumeManager.GetPossiblyMountedVolumesForPod( 488 util.GetUniquePodName(pod)) 489 assert.Equal(t, podVolumes, allPodVolumes, "GetMountedVolumesForPod and GetPossiblyMountedVolumesForPod should return the same volumes") 490 491 expectedPodVolumes := []string{"vol1"} 492 assert.Len(t, podVolumes, len(expectedPodVolumes), "Volumes for pod %+v", pod) 493 for _, name := range expectedPodVolumes { 494 assert.Contains(t, podVolumes, name, "Volumes for pod %+v", pod) 495 } 496 497 assert.True(t, testKubelet.volumePlugin.GetNewAttacherCallCount() >= 1, "Expected plugin NewAttacher to be called at least once") 498 assert.NoError(t, volumetest.VerifyWaitForAttachCallCount( 499 1 /* expectedWaitForAttachCallCount */, testKubelet.volumePlugin)) 500 assert.NoError(t, volumetest.VerifyZeroAttachCalls(testKubelet.volumePlugin)) 501 assert.NoError(t, volumetest.VerifyMountDeviceCallCount( 502 1 /* expectedMountDeviceCallCount */, testKubelet.volumePlugin)) 503 assert.NoError(t, volumetest.VerifySetUpCallCount( 504 1 /* expectedSetUpCallCount */, testKubelet.volumePlugin)) 505 506 // Remove pod 507 kubelet.podWorkers.(*fakePodWorkers).removeRuntime = map[types.UID]bool{pod.UID: true} 508 kubelet.podManager.SetPods([]*v1.Pod{}) 509 510 assert.NoError(t, waitForVolumeUnmount(kubelet.volumeManager, pod)) 511 512 // Verify volumes unmounted 513 podVolumes = kubelet.volumeManager.GetMountedVolumesForPod( 514 util.GetUniquePodName(pod)) 515 allPodVolumes = kubelet.volumeManager.GetPossiblyMountedVolumesForPod( 516 util.GetUniquePodName(pod)) 517 assert.Equal(t, podVolumes, allPodVolumes, "GetMountedVolumesForPod and GetPossiblyMountedVolumesForPod should return the same volumes") 518 519 assert.Len(t, podVolumes, 0, 520 "Expected volumes to be unmounted and detached. But some volumes are still mounted: %#v", podVolumes) 521 522 assert.NoError(t, volumetest.VerifyTearDownCallCount( 523 1 /* expectedTearDownCallCount */, testKubelet.volumePlugin)) 524 525 // Verify volumes detached and no longer reported as in use 526 assert.NoError(t, waitForVolumeDetach(v1.UniqueVolumeName("fake/fake-device"), kubelet.volumeManager)) 527 assert.True(t, testKubelet.volumePlugin.GetNewAttacherCallCount() >= 1, "Expected plugin NewAttacher to be called at least once") 528 assert.NoError(t, volumetest.VerifyZeroDetachCallCount(testKubelet.volumePlugin)) 529} 530 531type stubVolume struct { 532 path string 533 volume.MetricsNil 534} 535 536func (f *stubVolume) GetPath() string { 537 return f.path 538} 539 540func (f *stubVolume) GetAttributes() volume.Attributes { 541 return volume.Attributes{} 542} 543 544func (f *stubVolume) CanMount() error { 545 return nil 546} 547 548func (f *stubVolume) SetUp(mounterArgs volume.MounterArgs) error { 549 return nil 550} 551 552func (f *stubVolume) SetUpAt(dir string, mounterArgs volume.MounterArgs) error { 553 return nil 554} 555 556type stubBlockVolume struct { 557 dirPath string 558 volName string 559} 560 561func (f *stubBlockVolume) GetGlobalMapPath(spec *volume.Spec) (string, error) { 562 return "", nil 563} 564 565func (f *stubBlockVolume) GetPodDeviceMapPath() (string, string) { 566 return f.dirPath, f.volName 567} 568 569func (f *stubBlockVolume) SetUpDevice() (string, error) { 570 return "", nil 571} 572 573func (f stubBlockVolume) MapPodDevice() error { 574 return nil 575} 576 577func (f *stubBlockVolume) TearDownDevice(mapPath string, devicePath string) error { 578 return nil 579} 580 581func (f *stubBlockVolume) UnmapPodDevice() error { 582 return nil 583} 584 585func (f *stubBlockVolume) SupportsMetrics() bool { 586 return false 587} 588 589func (f *stubBlockVolume) GetMetrics() (*volume.Metrics, error) { 590 return nil, nil 591} 592