1/* 2Copyright 2016 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package polymorphichelpers 18 19import ( 20 "bytes" 21 "context" 22 "fmt" 23 "sort" 24 25 appsv1 "k8s.io/api/apps/v1" 26 corev1 "k8s.io/api/core/v1" 27 apiequality "k8s.io/apimachinery/pkg/api/equality" 28 "k8s.io/apimachinery/pkg/api/meta" 29 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 "k8s.io/apimachinery/pkg/runtime" 31 "k8s.io/apimachinery/pkg/runtime/schema" 32 "k8s.io/apimachinery/pkg/types" 33 "k8s.io/apimachinery/pkg/util/json" 34 "k8s.io/apimachinery/pkg/util/strategicpatch" 35 "k8s.io/client-go/kubernetes" 36 "k8s.io/kubectl/pkg/apps" 37 cmdutil "k8s.io/kubectl/pkg/cmd/util" 38 "k8s.io/kubectl/pkg/scheme" 39 deploymentutil "k8s.io/kubectl/pkg/util/deployment" 40) 41 42const ( 43 rollbackSuccess = "rolled back" 44 rollbackSkipped = "skipped rollback" 45) 46 47// Rollbacker provides an interface for resources that can be rolled back. 48type Rollbacker interface { 49 Rollback(obj runtime.Object, updatedAnnotations map[string]string, toRevision int64, dryRunStrategy cmdutil.DryRunStrategy) (string, error) 50} 51 52type RollbackVisitor struct { 53 clientset kubernetes.Interface 54 result Rollbacker 55} 56 57func (v *RollbackVisitor) VisitDeployment(elem apps.GroupKindElement) { 58 v.result = &DeploymentRollbacker{v.clientset} 59} 60 61func (v *RollbackVisitor) VisitStatefulSet(kind apps.GroupKindElement) { 62 v.result = &StatefulSetRollbacker{v.clientset} 63} 64 65func (v *RollbackVisitor) VisitDaemonSet(kind apps.GroupKindElement) { 66 v.result = &DaemonSetRollbacker{v.clientset} 67} 68 69func (v *RollbackVisitor) VisitJob(kind apps.GroupKindElement) {} 70func (v *RollbackVisitor) VisitPod(kind apps.GroupKindElement) {} 71func (v *RollbackVisitor) VisitReplicaSet(kind apps.GroupKindElement) {} 72func (v *RollbackVisitor) VisitReplicationController(kind apps.GroupKindElement) {} 73func (v *RollbackVisitor) VisitCronJob(kind apps.GroupKindElement) {} 74 75// RollbackerFor returns an implementation of Rollbacker interface for the given schema kind 76func RollbackerFor(kind schema.GroupKind, c kubernetes.Interface) (Rollbacker, error) { 77 elem := apps.GroupKindElement(kind) 78 visitor := &RollbackVisitor{ 79 clientset: c, 80 } 81 82 err := elem.Accept(visitor) 83 84 if err != nil { 85 return nil, fmt.Errorf("error retrieving rollbacker for %q, %v", kind.String(), err) 86 } 87 88 if visitor.result == nil { 89 return nil, fmt.Errorf("no rollbacker has been implemented for %q", kind) 90 } 91 92 return visitor.result, nil 93} 94 95type DeploymentRollbacker struct { 96 c kubernetes.Interface 97} 98 99func (r *DeploymentRollbacker) Rollback(obj runtime.Object, updatedAnnotations map[string]string, toRevision int64, dryRunStrategy cmdutil.DryRunStrategy) (string, error) { 100 if toRevision < 0 { 101 return "", revisionNotFoundErr(toRevision) 102 } 103 accessor, err := meta.Accessor(obj) 104 if err != nil { 105 return "", fmt.Errorf("failed to create accessor for kind %v: %s", obj.GetObjectKind(), err.Error()) 106 } 107 name := accessor.GetName() 108 namespace := accessor.GetNamespace() 109 110 // TODO: Fix this after kubectl has been removed from core. It is not possible to convert the runtime.Object 111 // to the external appsv1 Deployment without round-tripping through an internal version of Deployment. We're 112 // currently getting rid of all internal versions of resources. So we specifically request the appsv1 version 113 // here. This follows the same pattern as for DaemonSet and StatefulSet. 114 deployment, err := r.c.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) 115 if err != nil { 116 return "", fmt.Errorf("failed to retrieve Deployment %s: %v", name, err) 117 } 118 119 rsForRevision, err := deploymentRevision(deployment, r.c, toRevision) 120 if err != nil { 121 return "", err 122 } 123 if dryRunStrategy == cmdutil.DryRunClient { 124 return printTemplate(&rsForRevision.Spec.Template) 125 } 126 if deployment.Spec.Paused { 127 return "", fmt.Errorf("you cannot rollback a paused deployment; resume it first with 'kubectl rollout resume deployment/%s' and try again", name) 128 } 129 130 // Skip if the revision already matches current Deployment 131 if equalIgnoreHash(&rsForRevision.Spec.Template, &deployment.Spec.Template) { 132 return fmt.Sprintf("%s (current template already matches revision %d)", rollbackSkipped, toRevision), nil 133 } 134 135 // remove hash label before patching back into the deployment 136 delete(rsForRevision.Spec.Template.Labels, appsv1.DefaultDeploymentUniqueLabelKey) 137 138 // compute deployment annotations 139 annotations := map[string]string{} 140 for k := range annotationsToSkip { 141 if v, ok := deployment.Annotations[k]; ok { 142 annotations[k] = v 143 } 144 } 145 for k, v := range rsForRevision.Annotations { 146 if !annotationsToSkip[k] { 147 annotations[k] = v 148 } 149 } 150 151 // make patch to restore 152 patchType, patch, err := getDeploymentPatch(&rsForRevision.Spec.Template, annotations) 153 if err != nil { 154 return "", fmt.Errorf("failed restoring revision %d: %v", toRevision, err) 155 } 156 157 patchOptions := metav1.PatchOptions{} 158 if dryRunStrategy == cmdutil.DryRunServer { 159 patchOptions.DryRun = []string{metav1.DryRunAll} 160 } 161 // Restore revision 162 if _, err = r.c.AppsV1().Deployments(namespace).Patch(context.TODO(), name, patchType, patch, patchOptions); err != nil { 163 return "", fmt.Errorf("failed restoring revision %d: %v", toRevision, err) 164 } 165 return rollbackSuccess, nil 166} 167 168// equalIgnoreHash returns true if two given podTemplateSpec are equal, ignoring the diff in value of Labels[pod-template-hash] 169// We ignore pod-template-hash because: 170// 1. The hash result would be different upon podTemplateSpec API changes 171// (e.g. the addition of a new field will cause the hash code to change) 172// 2. The deployment template won't have hash labels 173func equalIgnoreHash(template1, template2 *corev1.PodTemplateSpec) bool { 174 t1Copy := template1.DeepCopy() 175 t2Copy := template2.DeepCopy() 176 // Remove hash labels from template.Labels before comparing 177 delete(t1Copy.Labels, appsv1.DefaultDeploymentUniqueLabelKey) 178 delete(t2Copy.Labels, appsv1.DefaultDeploymentUniqueLabelKey) 179 return apiequality.Semantic.DeepEqual(t1Copy, t2Copy) 180} 181 182// annotationsToSkip lists the annotations that should be preserved from the deployment and not 183// copied from the replicaset when rolling a deployment back 184var annotationsToSkip = map[string]bool{ 185 corev1.LastAppliedConfigAnnotation: true, 186 deploymentutil.RevisionAnnotation: true, 187 deploymentutil.RevisionHistoryAnnotation: true, 188 deploymentutil.DesiredReplicasAnnotation: true, 189 deploymentutil.MaxReplicasAnnotation: true, 190 appsv1.DeprecatedRollbackTo: true, 191} 192 193// getPatch returns a patch that can be applied to restore a Deployment to a 194// previous version. If the returned error is nil the patch is valid. 195func getDeploymentPatch(podTemplate *corev1.PodTemplateSpec, annotations map[string]string) (types.PatchType, []byte, error) { 196 // Create a patch of the Deployment that replaces spec.template 197 patch, err := json.Marshal([]interface{}{ 198 map[string]interface{}{ 199 "op": "replace", 200 "path": "/spec/template", 201 "value": podTemplate, 202 }, 203 map[string]interface{}{ 204 "op": "replace", 205 "path": "/metadata/annotations", 206 "value": annotations, 207 }, 208 }) 209 return types.JSONPatchType, patch, err 210} 211 212func deploymentRevision(deployment *appsv1.Deployment, c kubernetes.Interface, toRevision int64) (revision *appsv1.ReplicaSet, err error) { 213 214 _, allOldRSs, newRS, err := deploymentutil.GetAllReplicaSets(deployment, c.AppsV1()) 215 if err != nil { 216 return nil, fmt.Errorf("failed to retrieve replica sets from deployment %s: %v", deployment.Name, err) 217 } 218 allRSs := allOldRSs 219 if newRS != nil { 220 allRSs = append(allRSs, newRS) 221 } 222 223 var ( 224 latestReplicaSet *appsv1.ReplicaSet 225 latestRevision = int64(-1) 226 previousReplicaSet *appsv1.ReplicaSet 227 previousRevision = int64(-1) 228 ) 229 for _, rs := range allRSs { 230 if v, err := deploymentutil.Revision(rs); err == nil { 231 if toRevision == 0 { 232 if latestRevision < v { 233 // newest one we've seen so far 234 previousRevision = latestRevision 235 previousReplicaSet = latestReplicaSet 236 latestRevision = v 237 latestReplicaSet = rs 238 } else if previousRevision < v { 239 // second newest one we've seen so far 240 previousRevision = v 241 previousReplicaSet = rs 242 } 243 } else if toRevision == v { 244 return rs, nil 245 } 246 } 247 } 248 249 if toRevision > 0 { 250 return nil, revisionNotFoundErr(toRevision) 251 } 252 253 if previousReplicaSet == nil { 254 return nil, fmt.Errorf("no rollout history found for deployment %q", deployment.Name) 255 } 256 return previousReplicaSet, nil 257} 258 259type DaemonSetRollbacker struct { 260 c kubernetes.Interface 261} 262 263func (r *DaemonSetRollbacker) Rollback(obj runtime.Object, updatedAnnotations map[string]string, toRevision int64, dryRunStrategy cmdutil.DryRunStrategy) (string, error) { 264 if toRevision < 0 { 265 return "", revisionNotFoundErr(toRevision) 266 } 267 accessor, err := meta.Accessor(obj) 268 if err != nil { 269 return "", fmt.Errorf("failed to create accessor for kind %v: %s", obj.GetObjectKind(), err.Error()) 270 } 271 ds, history, err := daemonSetHistory(r.c.AppsV1(), accessor.GetNamespace(), accessor.GetName()) 272 if err != nil { 273 return "", err 274 } 275 if toRevision == 0 && len(history) <= 1 { 276 return "", fmt.Errorf("no last revision to roll back to") 277 } 278 279 toHistory := findHistory(toRevision, history) 280 if toHistory == nil { 281 return "", revisionNotFoundErr(toRevision) 282 } 283 284 if dryRunStrategy == cmdutil.DryRunClient { 285 appliedDS, err := applyDaemonSetHistory(ds, toHistory) 286 if err != nil { 287 return "", err 288 } 289 return printPodTemplate(&appliedDS.Spec.Template) 290 } 291 292 // Skip if the revision already matches current DaemonSet 293 done, err := daemonSetMatch(ds, toHistory) 294 if err != nil { 295 return "", err 296 } 297 if done { 298 return fmt.Sprintf("%s (current template already matches revision %d)", rollbackSkipped, toRevision), nil 299 } 300 301 patchOptions := metav1.PatchOptions{} 302 if dryRunStrategy == cmdutil.DryRunServer { 303 patchOptions.DryRun = []string{metav1.DryRunAll} 304 } 305 // Restore revision 306 if _, err = r.c.AppsV1().DaemonSets(accessor.GetNamespace()).Patch(context.TODO(), accessor.GetName(), types.StrategicMergePatchType, toHistory.Data.Raw, patchOptions); err != nil { 307 return "", fmt.Errorf("failed restoring revision %d: %v", toRevision, err) 308 } 309 310 return rollbackSuccess, nil 311} 312 313// daemonMatch check if the given DaemonSet's template matches the template stored in the given history. 314func daemonSetMatch(ds *appsv1.DaemonSet, history *appsv1.ControllerRevision) (bool, error) { 315 patch, err := getDaemonSetPatch(ds) 316 if err != nil { 317 return false, err 318 } 319 return bytes.Equal(patch, history.Data.Raw), nil 320} 321 322// getPatch returns a strategic merge patch that can be applied to restore a Daemonset to a 323// previous version. If the returned error is nil the patch is valid. The current state that we save is just the 324// PodSpecTemplate. We can modify this later to encompass more state (or less) and remain compatible with previously 325// recorded patches. 326func getDaemonSetPatch(ds *appsv1.DaemonSet) ([]byte, error) { 327 dsBytes, err := json.Marshal(ds) 328 if err != nil { 329 return nil, err 330 } 331 var raw map[string]interface{} 332 err = json.Unmarshal(dsBytes, &raw) 333 if err != nil { 334 return nil, err 335 } 336 objCopy := make(map[string]interface{}) 337 specCopy := make(map[string]interface{}) 338 339 // Create a patch of the DaemonSet that replaces spec.template 340 spec := raw["spec"].(map[string]interface{}) 341 template := spec["template"].(map[string]interface{}) 342 specCopy["template"] = template 343 template["$patch"] = "replace" 344 objCopy["spec"] = specCopy 345 patch, err := json.Marshal(objCopy) 346 return patch, err 347} 348 349type StatefulSetRollbacker struct { 350 c kubernetes.Interface 351} 352 353// toRevision is a non-negative integer, with 0 being reserved to indicate rolling back to previous configuration 354func (r *StatefulSetRollbacker) Rollback(obj runtime.Object, updatedAnnotations map[string]string, toRevision int64, dryRunStrategy cmdutil.DryRunStrategy) (string, error) { 355 if toRevision < 0 { 356 return "", revisionNotFoundErr(toRevision) 357 } 358 accessor, err := meta.Accessor(obj) 359 if err != nil { 360 return "", fmt.Errorf("failed to create accessor for kind %v: %s", obj.GetObjectKind(), err.Error()) 361 } 362 sts, history, err := statefulSetHistory(r.c.AppsV1(), accessor.GetNamespace(), accessor.GetName()) 363 if err != nil { 364 return "", err 365 } 366 if toRevision == 0 && len(history) <= 1 { 367 return "", fmt.Errorf("no last revision to roll back to") 368 } 369 370 toHistory := findHistory(toRevision, history) 371 if toHistory == nil { 372 return "", revisionNotFoundErr(toRevision) 373 } 374 375 if dryRunStrategy == cmdutil.DryRunClient { 376 appliedSS, err := applyRevision(sts, toHistory) 377 if err != nil { 378 return "", err 379 } 380 return printPodTemplate(&appliedSS.Spec.Template) 381 } 382 383 // Skip if the revision already matches current StatefulSet 384 done, err := statefulsetMatch(sts, toHistory) 385 if err != nil { 386 return "", err 387 } 388 if done { 389 return fmt.Sprintf("%s (current template already matches revision %d)", rollbackSkipped, toRevision), nil 390 } 391 392 patchOptions := metav1.PatchOptions{} 393 if dryRunStrategy == cmdutil.DryRunServer { 394 patchOptions.DryRun = []string{metav1.DryRunAll} 395 } 396 // Restore revision 397 if _, err = r.c.AppsV1().StatefulSets(sts.Namespace).Patch(context.TODO(), sts.Name, types.StrategicMergePatchType, toHistory.Data.Raw, patchOptions); err != nil { 398 return "", fmt.Errorf("failed restoring revision %d: %v", toRevision, err) 399 } 400 401 return rollbackSuccess, nil 402} 403 404var appsCodec = scheme.Codecs.LegacyCodec(appsv1.SchemeGroupVersion) 405 406// applyRevision returns a new StatefulSet constructed by restoring the state in revision to set. If the returned error 407// is nil, the returned StatefulSet is valid. 408func applyRevision(set *appsv1.StatefulSet, revision *appsv1.ControllerRevision) (*appsv1.StatefulSet, error) { 409 patched, err := strategicpatch.StrategicMergePatch([]byte(runtime.EncodeOrDie(appsCodec, set)), revision.Data.Raw, set) 410 if err != nil { 411 return nil, err 412 } 413 result := &appsv1.StatefulSet{} 414 err = json.Unmarshal(patched, result) 415 if err != nil { 416 return nil, err 417 } 418 return result, nil 419} 420 421// statefulsetMatch check if the given StatefulSet's template matches the template stored in the given history. 422func statefulsetMatch(ss *appsv1.StatefulSet, history *appsv1.ControllerRevision) (bool, error) { 423 patch, err := getStatefulSetPatch(ss) 424 if err != nil { 425 return false, err 426 } 427 return bytes.Equal(patch, history.Data.Raw), nil 428} 429 430// getStatefulSetPatch returns a strategic merge patch that can be applied to restore a StatefulSet to a 431// previous version. If the returned error is nil the patch is valid. The current state that we save is just the 432// PodSpecTemplate. We can modify this later to encompass more state (or less) and remain compatible with previously 433// recorded patches. 434func getStatefulSetPatch(set *appsv1.StatefulSet) ([]byte, error) { 435 str, err := runtime.Encode(appsCodec, set) 436 if err != nil { 437 return nil, err 438 } 439 var raw map[string]interface{} 440 if err := json.Unmarshal([]byte(str), &raw); err != nil { 441 return nil, err 442 } 443 objCopy := make(map[string]interface{}) 444 specCopy := make(map[string]interface{}) 445 spec := raw["spec"].(map[string]interface{}) 446 template := spec["template"].(map[string]interface{}) 447 specCopy["template"] = template 448 template["$patch"] = "replace" 449 objCopy["spec"] = specCopy 450 patch, err := json.Marshal(objCopy) 451 return patch, err 452} 453 454// findHistory returns a controllerrevision of a specific revision from the given controllerrevisions. 455// It returns nil if no such controllerrevision exists. 456// If toRevision is 0, the last previously used history is returned. 457func findHistory(toRevision int64, allHistory []*appsv1.ControllerRevision) *appsv1.ControllerRevision { 458 if toRevision == 0 && len(allHistory) <= 1 { 459 return nil 460 } 461 462 // Find the history to rollback to 463 var toHistory *appsv1.ControllerRevision 464 if toRevision == 0 { 465 // If toRevision == 0, find the latest revision (2nd max) 466 sort.Sort(historiesByRevision(allHistory)) 467 toHistory = allHistory[len(allHistory)-2] 468 } else { 469 for _, h := range allHistory { 470 if h.Revision == toRevision { 471 // If toRevision != 0, find the history with matching revision 472 return h 473 } 474 } 475 } 476 477 return toHistory 478} 479 480// printPodTemplate converts a given pod template into a human-readable string. 481func printPodTemplate(specTemplate *corev1.PodTemplateSpec) (string, error) { 482 podSpec, err := printTemplate(specTemplate) 483 if err != nil { 484 return "", err 485 } 486 return fmt.Sprintf("will roll back to %s", podSpec), nil 487} 488 489func revisionNotFoundErr(r int64) error { 490 return fmt.Errorf("unable to find specified revision %v in history", r) 491} 492 493// TODO: copied from daemon controller, should extract to a library 494type historiesByRevision []*appsv1.ControllerRevision 495 496func (h historiesByRevision) Len() int { return len(h) } 497func (h historiesByRevision) Swap(i, j int) { h[i], h[j] = h[j], h[i] } 498func (h historiesByRevision) Less(i, j int) bool { 499 return h[i].Revision < h[j].Revision 500} 501