1/* 2Copyright 2016 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package controller 18 19import ( 20 "encoding/json" 21 "fmt" 22 "sync" 23 24 apps "k8s.io/api/apps/v1" 25 v1 "k8s.io/api/core/v1" 26 "k8s.io/apimachinery/pkg/api/errors" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 "k8s.io/apimachinery/pkg/labels" 29 "k8s.io/apimachinery/pkg/runtime/schema" 30 "k8s.io/apimachinery/pkg/types" 31 utilerrors "k8s.io/apimachinery/pkg/util/errors" 32 "k8s.io/klog/v2" 33) 34 35type BaseControllerRefManager struct { 36 Controller metav1.Object 37 Selector labels.Selector 38 39 canAdoptErr error 40 canAdoptOnce sync.Once 41 CanAdoptFunc func() error 42} 43 44func (m *BaseControllerRefManager) CanAdopt() error { 45 m.canAdoptOnce.Do(func() { 46 if m.CanAdoptFunc != nil { 47 m.canAdoptErr = m.CanAdoptFunc() 48 } 49 }) 50 return m.canAdoptErr 51} 52 53// ClaimObject tries to take ownership of an object for this controller. 54// 55// It will reconcile the following: 56// * Adopt orphans if the match function returns true. 57// * Release owned objects if the match function returns false. 58// 59// A non-nil error is returned if some form of reconciliation was attempted and 60// failed. Usually, controllers should try again later in case reconciliation 61// is still needed. 62// 63// If the error is nil, either the reconciliation succeeded, or no 64// reconciliation was necessary. The returned boolean indicates whether you now 65// own the object. 66// 67// No reconciliation will be attempted if the controller is being deleted. 68func (m *BaseControllerRefManager) ClaimObject(obj metav1.Object, match func(metav1.Object) bool, adopt, release func(metav1.Object) error) (bool, error) { 69 controllerRef := metav1.GetControllerOfNoCopy(obj) 70 if controllerRef != nil { 71 if controllerRef.UID != m.Controller.GetUID() { 72 // Owned by someone else. Ignore. 73 return false, nil 74 } 75 if match(obj) { 76 // We already own it and the selector matches. 77 // Return true (successfully claimed) before checking deletion timestamp. 78 // We're still allowed to claim things we already own while being deleted 79 // because doing so requires taking no actions. 80 return true, nil 81 } 82 // Owned by us but selector doesn't match. 83 // Try to release, unless we're being deleted. 84 if m.Controller.GetDeletionTimestamp() != nil { 85 return false, nil 86 } 87 if err := release(obj); err != nil { 88 // If the pod no longer exists, ignore the error. 89 if errors.IsNotFound(err) { 90 return false, nil 91 } 92 // Either someone else released it, or there was a transient error. 93 // The controller should requeue and try again if it's still stale. 94 return false, err 95 } 96 // Successfully released. 97 return false, nil 98 } 99 100 // It's an orphan. 101 if m.Controller.GetDeletionTimestamp() != nil || !match(obj) { 102 // Ignore if we're being deleted or selector doesn't match. 103 return false, nil 104 } 105 if obj.GetDeletionTimestamp() != nil { 106 // Ignore if the object is being deleted 107 return false, nil 108 } 109 // Selector matches. Try to adopt. 110 if err := adopt(obj); err != nil { 111 // If the pod no longer exists, ignore the error. 112 if errors.IsNotFound(err) { 113 return false, nil 114 } 115 // Either someone else claimed it first, or there was a transient error. 116 // The controller should requeue and try again if it's still orphaned. 117 return false, err 118 } 119 // Successfully adopted. 120 return true, nil 121} 122 123type PodControllerRefManager struct { 124 BaseControllerRefManager 125 controllerKind schema.GroupVersionKind 126 podControl PodControlInterface 127 finalizers []string 128} 129 130// NewPodControllerRefManager returns a PodControllerRefManager that exposes 131// methods to manage the controllerRef of pods. 132// 133// The CanAdopt() function can be used to perform a potentially expensive check 134// (such as a live GET from the API server) prior to the first adoption. 135// It will only be called (at most once) if an adoption is actually attempted. 136// If CanAdopt() returns a non-nil error, all adoptions will fail. 137// 138// NOTE: Once CanAdopt() is called, it will not be called again by the same 139// PodControllerRefManager instance. Create a new instance if it makes 140// sense to check CanAdopt() again (e.g. in a different sync pass). 141func NewPodControllerRefManager( 142 podControl PodControlInterface, 143 controller metav1.Object, 144 selector labels.Selector, 145 controllerKind schema.GroupVersionKind, 146 canAdopt func() error, 147 finalizers ...string, 148) *PodControllerRefManager { 149 return &PodControllerRefManager{ 150 BaseControllerRefManager: BaseControllerRefManager{ 151 Controller: controller, 152 Selector: selector, 153 CanAdoptFunc: canAdopt, 154 }, 155 controllerKind: controllerKind, 156 podControl: podControl, 157 finalizers: finalizers, 158 } 159} 160 161// ClaimPods tries to take ownership of a list of Pods. 162// 163// It will reconcile the following: 164// * Adopt orphans if the selector matches. 165// * Release owned objects if the selector no longer matches. 166// 167// Optional: If one or more filters are specified, a Pod will only be claimed if 168// all filters return true. 169// 170// A non-nil error is returned if some form of reconciliation was attempted and 171// failed. Usually, controllers should try again later in case reconciliation 172// is still needed. 173// 174// If the error is nil, either the reconciliation succeeded, or no 175// reconciliation was necessary. The list of Pods that you now own is returned. 176func (m *PodControllerRefManager) ClaimPods(pods []*v1.Pod, filters ...func(*v1.Pod) bool) ([]*v1.Pod, error) { 177 var claimed []*v1.Pod 178 var errlist []error 179 180 match := func(obj metav1.Object) bool { 181 pod := obj.(*v1.Pod) 182 // Check selector first so filters only run on potentially matching Pods. 183 if !m.Selector.Matches(labels.Set(pod.Labels)) { 184 return false 185 } 186 for _, filter := range filters { 187 if !filter(pod) { 188 return false 189 } 190 } 191 return true 192 } 193 adopt := func(obj metav1.Object) error { 194 return m.AdoptPod(obj.(*v1.Pod)) 195 } 196 release := func(obj metav1.Object) error { 197 return m.ReleasePod(obj.(*v1.Pod)) 198 } 199 200 for _, pod := range pods { 201 ok, err := m.ClaimObject(pod, match, adopt, release) 202 if err != nil { 203 errlist = append(errlist, err) 204 continue 205 } 206 if ok { 207 claimed = append(claimed, pod) 208 } 209 } 210 return claimed, utilerrors.NewAggregate(errlist) 211} 212 213// AdoptPod sends a patch to take control of the pod. It returns the error if 214// the patching fails. 215func (m *PodControllerRefManager) AdoptPod(pod *v1.Pod) error { 216 if err := m.CanAdopt(); err != nil { 217 return fmt.Errorf("can't adopt Pod %v/%v (%v): %v", pod.Namespace, pod.Name, pod.UID, err) 218 } 219 // Note that ValidateOwnerReferences() will reject this patch if another 220 // OwnerReference exists with controller=true. 221 222 patchBytes, err := ownerRefControllerPatch(m.Controller, m.controllerKind, pod.UID, m.finalizers...) 223 if err != nil { 224 return err 225 } 226 return m.podControl.PatchPod(pod.Namespace, pod.Name, patchBytes) 227} 228 229// ReleasePod sends a patch to free the pod from the control of the controller. 230// It returns the error if the patching fails. 404 and 422 errors are ignored. 231func (m *PodControllerRefManager) ReleasePod(pod *v1.Pod) error { 232 klog.V(2).Infof("patching pod %s_%s to remove its controllerRef to %s/%s:%s", 233 pod.Namespace, pod.Name, m.controllerKind.GroupVersion(), m.controllerKind.Kind, m.Controller.GetName()) 234 patchBytes, err := deleteOwnerRefStrategicMergePatch(pod.UID, m.Controller.GetUID(), m.finalizers...) 235 if err != nil { 236 return err 237 } 238 err = m.podControl.PatchPod(pod.Namespace, pod.Name, patchBytes) 239 if err != nil { 240 if errors.IsNotFound(err) { 241 // If the pod no longer exists, ignore it. 242 return nil 243 } 244 if errors.IsInvalid(err) { 245 // Invalid error will be returned in two cases: 1. the pod 246 // has no owner reference, 2. the uid of the pod doesn't 247 // match, which means the pod is deleted and then recreated. 248 // In both cases, the error can be ignored. 249 250 // TODO: If the pod has owner references, but none of them 251 // has the owner.UID, server will silently ignore the patch. 252 // Investigate why. 253 return nil 254 } 255 } 256 return err 257} 258 259// ReplicaSetControllerRefManager is used to manage controllerRef of ReplicaSets. 260// Three methods are defined on this object 1: Classify 2: AdoptReplicaSet and 261// 3: ReleaseReplicaSet which are used to classify the ReplicaSets into appropriate 262// categories and accordingly adopt or release them. See comments on these functions 263// for more details. 264type ReplicaSetControllerRefManager struct { 265 BaseControllerRefManager 266 controllerKind schema.GroupVersionKind 267 rsControl RSControlInterface 268} 269 270// NewReplicaSetControllerRefManager returns a ReplicaSetControllerRefManager that exposes 271// methods to manage the controllerRef of ReplicaSets. 272// 273// The CanAdopt() function can be used to perform a potentially expensive check 274// (such as a live GET from the API server) prior to the first adoption. 275// It will only be called (at most once) if an adoption is actually attempted. 276// If CanAdopt() returns a non-nil error, all adoptions will fail. 277// 278// NOTE: Once CanAdopt() is called, it will not be called again by the same 279// ReplicaSetControllerRefManager instance. Create a new instance if it 280// makes sense to check CanAdopt() again (e.g. in a different sync pass). 281func NewReplicaSetControllerRefManager( 282 rsControl RSControlInterface, 283 controller metav1.Object, 284 selector labels.Selector, 285 controllerKind schema.GroupVersionKind, 286 canAdopt func() error, 287) *ReplicaSetControllerRefManager { 288 return &ReplicaSetControllerRefManager{ 289 BaseControllerRefManager: BaseControllerRefManager{ 290 Controller: controller, 291 Selector: selector, 292 CanAdoptFunc: canAdopt, 293 }, 294 controllerKind: controllerKind, 295 rsControl: rsControl, 296 } 297} 298 299// ClaimReplicaSets tries to take ownership of a list of ReplicaSets. 300// 301// It will reconcile the following: 302// * Adopt orphans if the selector matches. 303// * Release owned objects if the selector no longer matches. 304// 305// A non-nil error is returned if some form of reconciliation was attempted and 306// failed. Usually, controllers should try again later in case reconciliation 307// is still needed. 308// 309// If the error is nil, either the reconciliation succeeded, or no 310// reconciliation was necessary. The list of ReplicaSets that you now own is 311// returned. 312func (m *ReplicaSetControllerRefManager) ClaimReplicaSets(sets []*apps.ReplicaSet) ([]*apps.ReplicaSet, error) { 313 var claimed []*apps.ReplicaSet 314 var errlist []error 315 316 match := func(obj metav1.Object) bool { 317 return m.Selector.Matches(labels.Set(obj.GetLabels())) 318 } 319 adopt := func(obj metav1.Object) error { 320 return m.AdoptReplicaSet(obj.(*apps.ReplicaSet)) 321 } 322 release := func(obj metav1.Object) error { 323 return m.ReleaseReplicaSet(obj.(*apps.ReplicaSet)) 324 } 325 326 for _, rs := range sets { 327 ok, err := m.ClaimObject(rs, match, adopt, release) 328 if err != nil { 329 errlist = append(errlist, err) 330 continue 331 } 332 if ok { 333 claimed = append(claimed, rs) 334 } 335 } 336 return claimed, utilerrors.NewAggregate(errlist) 337} 338 339// AdoptReplicaSet sends a patch to take control of the ReplicaSet. It returns 340// the error if the patching fails. 341func (m *ReplicaSetControllerRefManager) AdoptReplicaSet(rs *apps.ReplicaSet) error { 342 if err := m.CanAdopt(); err != nil { 343 return fmt.Errorf("can't adopt ReplicaSet %v/%v (%v): %v", rs.Namespace, rs.Name, rs.UID, err) 344 } 345 // Note that ValidateOwnerReferences() will reject this patch if another 346 // OwnerReference exists with controller=true. 347 patchBytes, err := ownerRefControllerPatch(m.Controller, m.controllerKind, rs.UID) 348 if err != nil { 349 return err 350 } 351 return m.rsControl.PatchReplicaSet(rs.Namespace, rs.Name, patchBytes) 352} 353 354// ReleaseReplicaSet sends a patch to free the ReplicaSet from the control of the Deployment controller. 355// It returns the error if the patching fails. 404 and 422 errors are ignored. 356func (m *ReplicaSetControllerRefManager) ReleaseReplicaSet(replicaSet *apps.ReplicaSet) error { 357 klog.V(2).Infof("patching ReplicaSet %s_%s to remove its controllerRef to %s/%s:%s", 358 replicaSet.Namespace, replicaSet.Name, m.controllerKind.GroupVersion(), m.controllerKind.Kind, m.Controller.GetName()) 359 patchBytes, err := deleteOwnerRefStrategicMergePatch(replicaSet.UID, m.Controller.GetUID()) 360 if err != nil { 361 return err 362 } 363 err = m.rsControl.PatchReplicaSet(replicaSet.Namespace, replicaSet.Name, patchBytes) 364 if err != nil { 365 if errors.IsNotFound(err) { 366 // If the ReplicaSet no longer exists, ignore it. 367 return nil 368 } 369 if errors.IsInvalid(err) { 370 // Invalid error will be returned in two cases: 1. the ReplicaSet 371 // has no owner reference, 2. the uid of the ReplicaSet doesn't 372 // match, which means the ReplicaSet is deleted and then recreated. 373 // In both cases, the error can be ignored. 374 return nil 375 } 376 } 377 return err 378} 379 380// RecheckDeletionTimestamp returns a CanAdopt() function to recheck deletion. 381// 382// The CanAdopt() function calls getObject() to fetch the latest value, 383// and denies adoption attempts if that object has a non-nil DeletionTimestamp. 384func RecheckDeletionTimestamp(getObject func() (metav1.Object, error)) func() error { 385 return func() error { 386 obj, err := getObject() 387 if err != nil { 388 return fmt.Errorf("can't recheck DeletionTimestamp: %v", err) 389 } 390 if obj.GetDeletionTimestamp() != nil { 391 return fmt.Errorf("%v/%v has just been deleted at %v", obj.GetNamespace(), obj.GetName(), obj.GetDeletionTimestamp()) 392 } 393 return nil 394 } 395} 396 397// ControllerRevisionControllerRefManager is used to manage controllerRef of ControllerRevisions. 398// Three methods are defined on this object 1: Classify 2: AdoptControllerRevision and 399// 3: ReleaseControllerRevision which are used to classify the ControllerRevisions into appropriate 400// categories and accordingly adopt or release them. See comments on these functions 401// for more details. 402type ControllerRevisionControllerRefManager struct { 403 BaseControllerRefManager 404 controllerKind schema.GroupVersionKind 405 crControl ControllerRevisionControlInterface 406} 407 408// NewControllerRevisionControllerRefManager returns a ControllerRevisionControllerRefManager that exposes 409// methods to manage the controllerRef of ControllerRevisions. 410// 411// The canAdopt() function can be used to perform a potentially expensive check 412// (such as a live GET from the API server) prior to the first adoption. 413// It will only be called (at most once) if an adoption is actually attempted. 414// If canAdopt() returns a non-nil error, all adoptions will fail. 415// 416// NOTE: Once canAdopt() is called, it will not be called again by the same 417// ControllerRevisionControllerRefManager instance. Create a new instance if it 418// makes sense to check canAdopt() again (e.g. in a different sync pass). 419func NewControllerRevisionControllerRefManager( 420 crControl ControllerRevisionControlInterface, 421 controller metav1.Object, 422 selector labels.Selector, 423 controllerKind schema.GroupVersionKind, 424 canAdopt func() error, 425) *ControllerRevisionControllerRefManager { 426 return &ControllerRevisionControllerRefManager{ 427 BaseControllerRefManager: BaseControllerRefManager{ 428 Controller: controller, 429 Selector: selector, 430 CanAdoptFunc: canAdopt, 431 }, 432 controllerKind: controllerKind, 433 crControl: crControl, 434 } 435} 436 437// ClaimControllerRevisions tries to take ownership of a list of ControllerRevisions. 438// 439// It will reconcile the following: 440// * Adopt orphans if the selector matches. 441// * Release owned objects if the selector no longer matches. 442// 443// A non-nil error is returned if some form of reconciliation was attempted and 444// failed. Usually, controllers should try again later in case reconciliation 445// is still needed. 446// 447// If the error is nil, either the reconciliation succeeded, or no 448// reconciliation was necessary. The list of ControllerRevisions that you now own is 449// returned. 450func (m *ControllerRevisionControllerRefManager) ClaimControllerRevisions(histories []*apps.ControllerRevision) ([]*apps.ControllerRevision, error) { 451 var claimed []*apps.ControllerRevision 452 var errlist []error 453 454 match := func(obj metav1.Object) bool { 455 return m.Selector.Matches(labels.Set(obj.GetLabels())) 456 } 457 adopt := func(obj metav1.Object) error { 458 return m.AdoptControllerRevision(obj.(*apps.ControllerRevision)) 459 } 460 release := func(obj metav1.Object) error { 461 return m.ReleaseControllerRevision(obj.(*apps.ControllerRevision)) 462 } 463 464 for _, h := range histories { 465 ok, err := m.ClaimObject(h, match, adopt, release) 466 if err != nil { 467 errlist = append(errlist, err) 468 continue 469 } 470 if ok { 471 claimed = append(claimed, h) 472 } 473 } 474 return claimed, utilerrors.NewAggregate(errlist) 475} 476 477// AdoptControllerRevision sends a patch to take control of the ControllerRevision. It returns the error if 478// the patching fails. 479func (m *ControllerRevisionControllerRefManager) AdoptControllerRevision(history *apps.ControllerRevision) error { 480 if err := m.CanAdopt(); err != nil { 481 return fmt.Errorf("can't adopt ControllerRevision %v/%v (%v): %v", history.Namespace, history.Name, history.UID, err) 482 } 483 // Note that ValidateOwnerReferences() will reject this patch if another 484 // OwnerReference exists with controller=true. 485 patchBytes, err := ownerRefControllerPatch(m.Controller, m.controllerKind, history.UID) 486 if err != nil { 487 return err 488 } 489 return m.crControl.PatchControllerRevision(history.Namespace, history.Name, patchBytes) 490} 491 492// ReleaseControllerRevision sends a patch to free the ControllerRevision from the control of its controller. 493// It returns the error if the patching fails. 404 and 422 errors are ignored. 494func (m *ControllerRevisionControllerRefManager) ReleaseControllerRevision(history *apps.ControllerRevision) error { 495 klog.V(2).Infof("patching ControllerRevision %s_%s to remove its controllerRef to %s/%s:%s", 496 history.Namespace, history.Name, m.controllerKind.GroupVersion(), m.controllerKind.Kind, m.Controller.GetName()) 497 patchBytes, err := deleteOwnerRefStrategicMergePatch(history.UID, m.Controller.GetUID()) 498 if err != nil { 499 return err 500 } 501 502 err = m.crControl.PatchControllerRevision(history.Namespace, history.Name, patchBytes) 503 if err != nil { 504 if errors.IsNotFound(err) { 505 // If the ControllerRevision no longer exists, ignore it. 506 return nil 507 } 508 if errors.IsInvalid(err) { 509 // Invalid error will be returned in two cases: 1. the ControllerRevision 510 // has no owner reference, 2. the uid of the ControllerRevision doesn't 511 // match, which means the ControllerRevision is deleted and then recreated. 512 // In both cases, the error can be ignored. 513 return nil 514 } 515 } 516 return err 517} 518 519type objectForDeleteOwnerRefStrategicMergePatch struct { 520 Metadata objectMetaForMergePatch `json:"metadata"` 521} 522 523type objectMetaForMergePatch struct { 524 UID types.UID `json:"uid"` 525 OwnerReferences []map[string]string `json:"ownerReferences"` 526 DeleteFinalizers []string `json:"$deleteFromPrimitiveList/finalizers,omitempty"` 527} 528 529func deleteOwnerRefStrategicMergePatch(dependentUID types.UID, ownerUID types.UID, finalizers ...string) ([]byte, error) { 530 patch := objectForDeleteOwnerRefStrategicMergePatch{ 531 Metadata: objectMetaForMergePatch{ 532 UID: dependentUID, 533 OwnerReferences: []map[string]string{ 534 { 535 "$patch": "delete", 536 "uid": string(ownerUID), 537 }, 538 }, 539 DeleteFinalizers: finalizers, 540 }, 541 } 542 patchBytes, err := json.Marshal(&patch) 543 if err != nil { 544 return nil, err 545 } 546 return patchBytes, nil 547} 548 549type objectForAddOwnerRefPatch struct { 550 Metadata objectMetaForPatch `json:"metadata"` 551} 552 553type objectMetaForPatch struct { 554 OwnerReferences []metav1.OwnerReference `json:"ownerReferences"` 555 UID types.UID `json:"uid"` 556 Finalizers []string `json:"finalizers,omitempty"` 557} 558 559func ownerRefControllerPatch(controller metav1.Object, controllerKind schema.GroupVersionKind, uid types.UID, finalizers ...string) ([]byte, error) { 560 blockOwnerDeletion := true 561 isController := true 562 addControllerPatch := objectForAddOwnerRefPatch{ 563 Metadata: objectMetaForPatch{ 564 UID: uid, 565 OwnerReferences: []metav1.OwnerReference{ 566 { 567 APIVersion: controllerKind.GroupVersion().String(), 568 Kind: controllerKind.Kind, 569 Name: controller.GetName(), 570 UID: controller.GetUID(), 571 Controller: &isController, 572 BlockOwnerDeletion: &blockOwnerDeletion, 573 }, 574 }, 575 Finalizers: finalizers, 576 }, 577 } 578 patchBytes, err := json.Marshal(&addControllerPatch) 579 if err != nil { 580 return nil, err 581 } 582 return patchBytes, nil 583} 584