1/* 2Copyright 2017 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package pvcprotection 18 19import ( 20 "errors" 21 "reflect" 22 "testing" 23 "time" 24 25 "github.com/davecgh/go-spew/spew" 26 27 v1 "k8s.io/api/core/v1" 28 apierrors "k8s.io/apimachinery/pkg/api/errors" 29 "k8s.io/apimachinery/pkg/api/meta" 30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 "k8s.io/apimachinery/pkg/runtime" 32 "k8s.io/apimachinery/pkg/runtime/schema" 33 "k8s.io/apimachinery/pkg/types" 34 "k8s.io/client-go/informers" 35 "k8s.io/client-go/kubernetes/fake" 36 clienttesting "k8s.io/client-go/testing" 37 "k8s.io/klog/v2" 38 "k8s.io/kubernetes/pkg/controller" 39 volumeutil "k8s.io/kubernetes/pkg/volume/util" 40) 41 42type reaction struct { 43 verb string 44 resource string 45 reactorfn clienttesting.ReactionFunc 46} 47 48const ( 49 defaultNS = "default" 50 defaultPVCName = "pvc1" 51 defaultPodName = "pod1" 52 defaultNodeName = "node1" 53 defaultUID = "uid1" 54) 55 56func pod() *v1.Pod { 57 return &v1.Pod{ 58 ObjectMeta: metav1.ObjectMeta{ 59 Name: defaultPodName, 60 Namespace: defaultNS, 61 UID: defaultUID, 62 }, 63 Spec: v1.PodSpec{ 64 NodeName: defaultNodeName, 65 }, 66 Status: v1.PodStatus{ 67 Phase: v1.PodPending, 68 }, 69 } 70} 71 72func unscheduled(pod *v1.Pod) *v1.Pod { 73 pod.Spec.NodeName = "" 74 return pod 75} 76 77func withPVC(pvcName string, pod *v1.Pod) *v1.Pod { 78 volume := v1.Volume{ 79 Name: pvcName, 80 VolumeSource: v1.VolumeSource{ 81 PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ 82 ClaimName: pvcName, 83 }, 84 }, 85 } 86 pod.Spec.Volumes = append(pod.Spec.Volumes, volume) 87 return pod 88} 89 90func withEmptyDir(pod *v1.Pod) *v1.Pod { 91 volume := v1.Volume{ 92 Name: "emptyDir", 93 VolumeSource: v1.VolumeSource{ 94 EmptyDir: &v1.EmptyDirVolumeSource{}, 95 }, 96 } 97 pod.Spec.Volumes = append(pod.Spec.Volumes, volume) 98 return pod 99} 100 101func withStatus(phase v1.PodPhase, pod *v1.Pod) *v1.Pod { 102 pod.Status.Phase = phase 103 return pod 104} 105 106func withUID(uid types.UID, pod *v1.Pod) *v1.Pod { 107 pod.ObjectMeta.UID = uid 108 return pod 109} 110 111func pvc() *v1.PersistentVolumeClaim { 112 return &v1.PersistentVolumeClaim{ 113 ObjectMeta: metav1.ObjectMeta{ 114 Name: defaultPVCName, 115 Namespace: defaultNS, 116 }, 117 } 118} 119 120func withProtectionFinalizer(pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { 121 pvc.Finalizers = append(pvc.Finalizers, volumeutil.PVCProtectionFinalizer) 122 return pvc 123} 124 125func deleted(pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { 126 pvc.DeletionTimestamp = &metav1.Time{} 127 return pvc 128} 129 130func generateUpdateErrorFunc(t *testing.T, failures int) clienttesting.ReactionFunc { 131 i := 0 132 return func(action clienttesting.Action) (bool, runtime.Object, error) { 133 i++ 134 if i <= failures { 135 // Update fails 136 update, ok := action.(clienttesting.UpdateAction) 137 138 if !ok { 139 t.Fatalf("Reactor got non-update action: %+v", action) 140 } 141 acc, _ := meta.Accessor(update.GetObject()) 142 return true, nil, apierrors.NewForbidden(update.GetResource().GroupResource(), acc.GetName(), errors.New("Mock error")) 143 } 144 // Update succeeds 145 return false, nil, nil 146 } 147} 148 149func testPVCProtectionController(t *testing.T, genericEphemeralVolumeFeatureEnabled bool) { 150 pvcGVR := schema.GroupVersionResource{ 151 Group: v1.GroupName, 152 Version: "v1", 153 Resource: "persistentvolumeclaims", 154 } 155 podGVR := schema.GroupVersionResource{ 156 Group: v1.GroupName, 157 Version: "v1", 158 Resource: "pods", 159 } 160 podGVK := schema.GroupVersionKind{ 161 Group: v1.GroupName, 162 Version: "v1", 163 Kind: "Pod", 164 } 165 166 tests := []struct { 167 name string 168 // Object to insert into fake kubeclient before the test starts. 169 initialObjects []runtime.Object 170 // Whether not to insert the content of initialObjects into the 171 // informers before the test starts. Set it to true to simulate the case 172 // where informers have not been notified yet of certain API objects. 173 informersAreLate bool 174 // Optional client reactors. 175 reactors []reaction 176 // PVC event to simulate. This PVC will be automatically added to 177 // initialObjects. 178 updatedPVC *v1.PersistentVolumeClaim 179 // Pod event to simulate. This Pod will be automatically added to 180 // initialObjects. 181 updatedPod *v1.Pod 182 // Pod event to simulate. This Pod is *not* added to 183 // initialObjects. 184 deletedPod *v1.Pod 185 // List of expected kubeclient actions that should happen during the 186 // test. 187 expectedActions []clienttesting.Action 188 storageObjectInUseProtectionEnabled bool 189 }{ 190 // 191 // PVC events 192 // 193 { 194 name: "StorageObjectInUseProtection Enabled, PVC without finalizer -> finalizer is added", 195 updatedPVC: pvc(), 196 expectedActions: []clienttesting.Action{ 197 clienttesting.NewUpdateAction(pvcGVR, defaultNS, withProtectionFinalizer(pvc())), 198 }, 199 storageObjectInUseProtectionEnabled: true, 200 }, 201 { 202 name: "StorageObjectInUseProtection Disabled, PVC without finalizer -> finalizer is not added", 203 updatedPVC: pvc(), 204 expectedActions: []clienttesting.Action{}, 205 storageObjectInUseProtectionEnabled: false, 206 }, 207 { 208 name: "PVC with finalizer -> no action", 209 updatedPVC: withProtectionFinalizer(pvc()), 210 expectedActions: []clienttesting.Action{}, 211 storageObjectInUseProtectionEnabled: true, 212 }, 213 { 214 name: "saving PVC finalizer fails -> controller retries", 215 updatedPVC: pvc(), 216 reactors: []reaction{ 217 { 218 verb: "update", 219 resource: "persistentvolumeclaims", 220 reactorfn: generateUpdateErrorFunc(t, 2 /* update fails twice*/), 221 }, 222 }, 223 expectedActions: []clienttesting.Action{ 224 // This fails 225 clienttesting.NewUpdateAction(pvcGVR, defaultNS, withProtectionFinalizer(pvc())), 226 // This fails too 227 clienttesting.NewUpdateAction(pvcGVR, defaultNS, withProtectionFinalizer(pvc())), 228 // This succeeds 229 clienttesting.NewUpdateAction(pvcGVR, defaultNS, withProtectionFinalizer(pvc())), 230 }, 231 storageObjectInUseProtectionEnabled: true, 232 }, 233 { 234 name: "StorageObjectInUseProtection Enabled, deleted PVC with finalizer -> finalizer is removed", 235 updatedPVC: deleted(withProtectionFinalizer(pvc())), 236 expectedActions: []clienttesting.Action{ 237 clienttesting.NewListAction(podGVR, podGVK, defaultNS, metav1.ListOptions{}), 238 clienttesting.NewUpdateAction(pvcGVR, defaultNS, deleted(pvc())), 239 }, 240 storageObjectInUseProtectionEnabled: true, 241 }, 242 { 243 name: "StorageObjectInUseProtection Disabled, deleted PVC with finalizer -> finalizer is removed", 244 updatedPVC: deleted(withProtectionFinalizer(pvc())), 245 expectedActions: []clienttesting.Action{ 246 clienttesting.NewListAction(podGVR, podGVK, defaultNS, metav1.ListOptions{}), 247 clienttesting.NewUpdateAction(pvcGVR, defaultNS, deleted(pvc())), 248 }, 249 storageObjectInUseProtectionEnabled: false, 250 }, 251 { 252 name: "finalizer removal fails -> controller retries", 253 updatedPVC: deleted(withProtectionFinalizer(pvc())), 254 reactors: []reaction{ 255 { 256 verb: "update", 257 resource: "persistentvolumeclaims", 258 reactorfn: generateUpdateErrorFunc(t, 2 /* update fails twice*/), 259 }, 260 }, 261 expectedActions: []clienttesting.Action{ 262 clienttesting.NewListAction(podGVR, podGVK, defaultNS, metav1.ListOptions{}), 263 // Fails 264 clienttesting.NewUpdateAction(pvcGVR, defaultNS, deleted(pvc())), 265 clienttesting.NewListAction(podGVR, podGVK, defaultNS, metav1.ListOptions{}), 266 // Fails too 267 clienttesting.NewUpdateAction(pvcGVR, defaultNS, deleted(pvc())), 268 clienttesting.NewListAction(podGVR, podGVK, defaultNS, metav1.ListOptions{}), 269 // Succeeds 270 clienttesting.NewUpdateAction(pvcGVR, defaultNS, deleted(pvc())), 271 }, 272 storageObjectInUseProtectionEnabled: true, 273 }, 274 { 275 name: "deleted PVC with finalizer + pod with the PVC exists -> finalizer is not removed", 276 initialObjects: []runtime.Object{ 277 withPVC(defaultPVCName, pod()), 278 }, 279 updatedPVC: deleted(withProtectionFinalizer(pvc())), 280 expectedActions: []clienttesting.Action{}, 281 }, 282 { 283 name: "deleted PVC with finalizer + pod with unrelated PVC and EmptyDir exists -> finalizer is removed", 284 initialObjects: []runtime.Object{ 285 withEmptyDir(withPVC("unrelatedPVC", pod())), 286 }, 287 updatedPVC: deleted(withProtectionFinalizer(pvc())), 288 expectedActions: []clienttesting.Action{ 289 clienttesting.NewListAction(podGVR, podGVK, defaultNS, metav1.ListOptions{}), 290 clienttesting.NewUpdateAction(pvcGVR, defaultNS, deleted(pvc())), 291 }, 292 storageObjectInUseProtectionEnabled: true, 293 }, 294 { 295 name: "deleted PVC with finalizer + pod with the PVC finished but is not deleted -> finalizer is not removed", 296 initialObjects: []runtime.Object{ 297 withStatus(v1.PodFailed, withPVC(defaultPVCName, pod())), 298 }, 299 updatedPVC: deleted(withProtectionFinalizer(pvc())), 300 expectedActions: []clienttesting.Action{}, 301 storageObjectInUseProtectionEnabled: true, 302 }, 303 { 304 name: "deleted PVC with finalizer + pod with the PVC exists but is not in the Informer's cache yet -> finalizer is not removed", 305 initialObjects: []runtime.Object{ 306 withPVC(defaultPVCName, pod()), 307 }, 308 informersAreLate: true, 309 updatedPVC: deleted(withProtectionFinalizer(pvc())), 310 expectedActions: []clienttesting.Action{ 311 clienttesting.NewListAction(podGVR, podGVK, defaultNS, metav1.ListOptions{}), 312 }, 313 storageObjectInUseProtectionEnabled: true, 314 }, 315 // 316 // Pod events 317 // 318 { 319 name: "updated running Pod -> no action", 320 initialObjects: []runtime.Object{ 321 deleted(withProtectionFinalizer(pvc())), 322 }, 323 updatedPod: withStatus(v1.PodRunning, withPVC(defaultPVCName, pod())), 324 expectedActions: []clienttesting.Action{}, 325 storageObjectInUseProtectionEnabled: true, 326 }, 327 { 328 name: "updated finished Pod -> finalizer is not removed", 329 initialObjects: []runtime.Object{ 330 deleted(withProtectionFinalizer(pvc())), 331 }, 332 updatedPod: withStatus(v1.PodSucceeded, withPVC(defaultPVCName, pod())), 333 expectedActions: []clienttesting.Action{}, 334 storageObjectInUseProtectionEnabled: true, 335 }, 336 { 337 name: "updated unscheduled Pod -> finalizer is removed", 338 initialObjects: []runtime.Object{ 339 deleted(withProtectionFinalizer(pvc())), 340 }, 341 updatedPod: unscheduled(withPVC(defaultPVCName, pod())), 342 expectedActions: []clienttesting.Action{ 343 clienttesting.NewListAction(podGVR, podGVK, defaultNS, metav1.ListOptions{}), 344 clienttesting.NewUpdateAction(pvcGVR, defaultNS, deleted(pvc())), 345 }, 346 storageObjectInUseProtectionEnabled: true, 347 }, 348 { 349 name: "deleted running Pod -> finalizer is removed", 350 initialObjects: []runtime.Object{ 351 deleted(withProtectionFinalizer(pvc())), 352 }, 353 deletedPod: withStatus(v1.PodRunning, withPVC(defaultPVCName, pod())), 354 expectedActions: []clienttesting.Action{ 355 clienttesting.NewListAction(podGVR, podGVK, defaultNS, metav1.ListOptions{}), 356 clienttesting.NewUpdateAction(pvcGVR, defaultNS, deleted(pvc())), 357 }, 358 storageObjectInUseProtectionEnabled: true, 359 }, 360 { 361 name: "pod delete and create with same namespaced name seen as an update, old pod used deleted PVC -> finalizer is removed", 362 initialObjects: []runtime.Object{ 363 deleted(withProtectionFinalizer(pvc())), 364 }, 365 deletedPod: withPVC(defaultPVCName, pod()), 366 updatedPod: withUID("uid2", pod()), 367 expectedActions: []clienttesting.Action{ 368 clienttesting.NewListAction(podGVR, podGVK, defaultNS, metav1.ListOptions{}), 369 clienttesting.NewUpdateAction(pvcGVR, defaultNS, deleted(pvc())), 370 }, 371 storageObjectInUseProtectionEnabled: true, 372 }, 373 { 374 name: "pod delete and create with same namespaced name seen as an update, old pod used non-deleted PVC -> finalizer is not removed", 375 initialObjects: []runtime.Object{ 376 withProtectionFinalizer(pvc()), 377 }, 378 deletedPod: withPVC(defaultPVCName, pod()), 379 updatedPod: withUID("uid2", pod()), 380 expectedActions: []clienttesting.Action{}, 381 storageObjectInUseProtectionEnabled: true, 382 }, 383 { 384 name: "pod delete and create with same namespaced name seen as an update, both pods reference deleted PVC -> finalizer is not removed", 385 initialObjects: []runtime.Object{ 386 deleted(withProtectionFinalizer(pvc())), 387 }, 388 deletedPod: withPVC(defaultPVCName, pod()), 389 updatedPod: withUID("uid2", withPVC(defaultPVCName, pod())), 390 expectedActions: []clienttesting.Action{}, 391 storageObjectInUseProtectionEnabled: true, 392 }, 393 { 394 name: "pod update from unscheduled to scheduled, deleted PVC is referenced -> finalizer is not removed", 395 initialObjects: []runtime.Object{ 396 deleted(withProtectionFinalizer(pvc())), 397 }, 398 deletedPod: unscheduled(withPVC(defaultPVCName, pod())), 399 updatedPod: withPVC(defaultPVCName, pod()), 400 expectedActions: []clienttesting.Action{}, 401 storageObjectInUseProtectionEnabled: true, 402 }, 403 } 404 405 for _, test := range tests { 406 // Create initial data for client and informers. 407 var ( 408 clientObjs []runtime.Object 409 informersObjs []runtime.Object 410 ) 411 if test.updatedPVC != nil { 412 clientObjs = append(clientObjs, test.updatedPVC) 413 informersObjs = append(informersObjs, test.updatedPVC) 414 } 415 if test.updatedPod != nil { 416 clientObjs = append(clientObjs, test.updatedPod) 417 informersObjs = append(informersObjs, test.updatedPod) 418 } 419 clientObjs = append(clientObjs, test.initialObjects...) 420 if !test.informersAreLate { 421 informersObjs = append(informersObjs, test.initialObjects...) 422 } 423 424 // Create client with initial data 425 client := fake.NewSimpleClientset(clientObjs...) 426 427 // Create informers 428 informers := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) 429 pvcInformer := informers.Core().V1().PersistentVolumeClaims() 430 podInformer := informers.Core().V1().Pods() 431 432 // Create the controller 433 ctrl, err := NewPVCProtectionController(pvcInformer, podInformer, client, test.storageObjectInUseProtectionEnabled, genericEphemeralVolumeFeatureEnabled) 434 if err != nil { 435 t.Fatalf("unexpected error: %v", err) 436 } 437 438 // Populate the informers with initial objects so the controller can 439 // Get() and List() it. 440 for _, obj := range informersObjs { 441 switch obj.(type) { 442 case *v1.PersistentVolumeClaim: 443 pvcInformer.Informer().GetStore().Add(obj) 444 case *v1.Pod: 445 podInformer.Informer().GetStore().Add(obj) 446 default: 447 t.Fatalf("Unknown initalObject type: %+v", obj) 448 } 449 } 450 451 // Add reactor to inject test errors. 452 for _, reactor := range test.reactors { 453 client.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorfn) 454 } 455 456 // Start the test by simulating an event 457 if test.updatedPVC != nil { 458 ctrl.pvcAddedUpdated(test.updatedPVC) 459 } 460 switch { 461 case test.deletedPod != nil && test.updatedPod != nil && test.deletedPod.Namespace == test.updatedPod.Namespace && test.deletedPod.Name == test.updatedPod.Name: 462 ctrl.podAddedDeletedUpdated(test.deletedPod, test.updatedPod, false) 463 case test.updatedPod != nil: 464 ctrl.podAddedDeletedUpdated(nil, test.updatedPod, false) 465 case test.deletedPod != nil: 466 ctrl.podAddedDeletedUpdated(nil, test.deletedPod, true) 467 } 468 469 // Process the controller queue until we get expected results 470 timeout := time.Now().Add(10 * time.Second) 471 lastReportedActionCount := 0 472 for { 473 if time.Now().After(timeout) { 474 t.Errorf("Test %q: timed out", test.name) 475 break 476 } 477 if ctrl.queue.Len() > 0 { 478 klog.V(5).Infof("Test %q: %d events queue, processing one", test.name, ctrl.queue.Len()) 479 ctrl.processNextWorkItem() 480 } 481 if ctrl.queue.Len() > 0 { 482 // There is still some work in the queue, process it now 483 continue 484 } 485 currentActionCount := len(client.Actions()) 486 if currentActionCount < len(test.expectedActions) { 487 // Do not log every wait, only when the action count changes. 488 if lastReportedActionCount < currentActionCount { 489 klog.V(5).Infof("Test %q: got %d actions out of %d, waiting for the rest", test.name, currentActionCount, len(test.expectedActions)) 490 lastReportedActionCount = currentActionCount 491 } 492 // The test expected more to happen, wait for the actions. 493 // Most probably it's exponential backoff 494 time.Sleep(10 * time.Millisecond) 495 continue 496 } 497 break 498 } 499 actions := client.Actions() 500 for i, action := range actions { 501 if len(test.expectedActions) < i+1 { 502 t.Errorf("Test %q: %d unexpected actions: %+v", test.name, len(actions)-len(test.expectedActions), spew.Sdump(actions[i:])) 503 break 504 } 505 506 expectedAction := test.expectedActions[i] 507 if !reflect.DeepEqual(expectedAction, action) { 508 t.Errorf("Test %q: action %d\nExpected:\n%s\ngot:\n%s", test.name, i, spew.Sdump(expectedAction), spew.Sdump(action)) 509 } 510 } 511 512 if len(test.expectedActions) > len(actions) { 513 t.Errorf("Test %q: %d additional expected actions", test.name, len(test.expectedActions)-len(actions)) 514 for _, a := range test.expectedActions[len(actions):] { 515 t.Logf(" %+v", a) 516 } 517 } 518 519 } 520} 521 522func TestPVCProtectionController(t *testing.T) { 523 t.Run("with-GenericEphemeralVolume", func(t *testing.T) { testPVCProtectionController(t, true) }) 524 t.Run("without-GenericEphemeralVolume", func(t *testing.T) { testPVCProtectionController(t, false) }) 525} 526