1/* 2Copyright The Helm Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package action 18 19import ( 20 "bytes" 21 "context" 22 "fmt" 23 "strings" 24 "time" 25 26 "github.com/pkg/errors" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 "k8s.io/cli-runtime/pkg/resource" 29 30 "helm.sh/helm/v3/pkg/chart" 31 "helm.sh/helm/v3/pkg/chartutil" 32 "helm.sh/helm/v3/pkg/kube" 33 "helm.sh/helm/v3/pkg/postrender" 34 "helm.sh/helm/v3/pkg/release" 35 "helm.sh/helm/v3/pkg/releaseutil" 36) 37 38// Upgrade is the action for upgrading releases. 39// 40// It provides the implementation of 'helm upgrade'. 41type Upgrade struct { 42 cfg *Configuration 43 44 ChartPathOptions 45 46 // Install is a purely informative flag that indicates whether this upgrade was done in "install" mode. 47 // 48 // Applications may use this to determine whether this Upgrade operation was done as part of a 49 // pure upgrade (Upgrade.Install == false) or as part of an install-or-upgrade operation 50 // (Upgrade.Install == true). 51 // 52 // Setting this to `true` will NOT cause `Upgrade` to perform an install if the release does not exist. 53 // That process must be handled by creating an Install action directly. See cmd/upgrade.go for an 54 // example of how this flag is used. 55 Install bool 56 // Devel indicates that the operation is done in devel mode. 57 Devel bool 58 // Namespace is the namespace in which this operation should be performed. 59 Namespace string 60 // SkipCRDs skips installing CRDs when install flag is enabled during upgrade 61 SkipCRDs bool 62 // Timeout is the timeout for this operation 63 Timeout time.Duration 64 // Wait determines whether the wait operation should be performed after the upgrade is requested. 65 Wait bool 66 // DisableHooks disables hook processing if set to true. 67 DisableHooks bool 68 // DryRun controls whether the operation is prepared, but not executed. 69 // If `true`, the upgrade is prepared but not performed. 70 DryRun bool 71 // Force will, if set to `true`, ignore certain warnings and perform the upgrade anyway. 72 // 73 // This should be used with caution. 74 Force bool 75 // ResetValues will reset the values to the chart's built-ins rather than merging with existing. 76 ResetValues bool 77 // ReuseValues will re-use the user's last supplied values. 78 ReuseValues bool 79 // Recreate will (if true) recreate pods after a rollback. 80 Recreate bool 81 // MaxHistory limits the maximum number of revisions saved per release 82 MaxHistory int 83 // Atomic, if true, will roll back on failure. 84 Atomic bool 85 // CleanupOnFail will, if true, cause the upgrade to delete newly-created resources on a failed update. 86 CleanupOnFail bool 87 // SubNotes determines whether sub-notes are rendered in the chart. 88 SubNotes bool 89 // Description is the description of this operation 90 Description string 91 // PostRender is an optional post-renderer 92 // 93 // If this is non-nil, then after templates are rendered, they will be sent to the 94 // post renderer before sending to the Kuberntes API server. 95 PostRenderer postrender.PostRenderer 96 // DisableOpenAPIValidation controls whether OpenAPI validation is enforced. 97 DisableOpenAPIValidation bool 98} 99 100// NewUpgrade creates a new Upgrade object with the given configuration. 101func NewUpgrade(cfg *Configuration) *Upgrade { 102 return &Upgrade{ 103 cfg: cfg, 104 } 105} 106 107// Run executes the upgrade on the given release. 108func (u *Upgrade) Run(name string, chart *chart.Chart, vals map[string]interface{}) (*release.Release, error) { 109 if err := u.cfg.KubeClient.IsReachable(); err != nil { 110 return nil, err 111 } 112 113 // Make sure if Atomic is set, that wait is set as well. This makes it so 114 // the user doesn't have to specify both 115 u.Wait = u.Wait || u.Atomic 116 117 if err := validateReleaseName(name); err != nil { 118 return nil, errors.Errorf("release name is invalid: %s", name) 119 } 120 u.cfg.Log("preparing upgrade for %s", name) 121 currentRelease, upgradedRelease, err := u.prepareUpgrade(name, chart, vals) 122 if err != nil { 123 return nil, err 124 } 125 126 u.cfg.Releases.MaxHistory = u.MaxHistory 127 128 u.cfg.Log("performing update for %s", name) 129 res, err := u.performUpgrade(currentRelease, upgradedRelease) 130 if err != nil { 131 return res, err 132 } 133 134 if !u.DryRun { 135 u.cfg.Log("updating status for upgraded release for %s", name) 136 if err := u.cfg.Releases.Update(upgradedRelease); err != nil { 137 return res, err 138 } 139 } 140 141 return res, nil 142} 143 144func validateReleaseName(releaseName string) error { 145 if releaseName == "" { 146 return errMissingRelease 147 } 148 149 if !ValidName.MatchString(releaseName) || (len(releaseName) > releaseNameMaxLen) { 150 return errInvalidName 151 } 152 153 return nil 154} 155 156// prepareUpgrade builds an upgraded release for an upgrade operation. 157func (u *Upgrade) prepareUpgrade(name string, chart *chart.Chart, vals map[string]interface{}) (*release.Release, *release.Release, error) { 158 if chart == nil { 159 return nil, nil, errMissingChart 160 } 161 162 // finds the deployed release with the given name 163 currentRelease, err := u.cfg.Releases.Deployed(name) 164 if err != nil { 165 return nil, nil, err 166 } 167 168 // determine if values will be reused 169 vals, err = u.reuseValues(chart, currentRelease, vals) 170 if err != nil { 171 return nil, nil, err 172 } 173 174 if err := chartutil.ProcessDependencies(chart, vals); err != nil { 175 return nil, nil, err 176 } 177 178 // finds the non-deleted release with the given name 179 lastRelease, err := u.cfg.Releases.Last(name) 180 if err != nil { 181 return nil, nil, err 182 } 183 184 // Increment revision count. This is passed to templates, and also stored on 185 // the release object. 186 revision := lastRelease.Version + 1 187 188 options := chartutil.ReleaseOptions{ 189 Name: name, 190 Namespace: currentRelease.Namespace, 191 Revision: revision, 192 IsUpgrade: true, 193 } 194 195 caps, err := u.cfg.getCapabilities() 196 if err != nil { 197 return nil, nil, err 198 } 199 valuesToRender, err := chartutil.ToRenderValues(chart, vals, options, caps) 200 if err != nil { 201 return nil, nil, err 202 } 203 204 hooks, manifestDoc, notesTxt, err := u.cfg.renderResources(chart, valuesToRender, "", "", u.SubNotes, false, false, u.PostRenderer, u.DryRun) 205 if err != nil { 206 return nil, nil, err 207 } 208 209 // Store an upgraded release. 210 upgradedRelease := &release.Release{ 211 Name: name, 212 Namespace: currentRelease.Namespace, 213 Chart: chart, 214 Config: vals, 215 Info: &release.Info{ 216 FirstDeployed: currentRelease.Info.FirstDeployed, 217 LastDeployed: Timestamper(), 218 Status: release.StatusPendingUpgrade, 219 Description: "Preparing upgrade", // This should be overwritten later. 220 }, 221 Version: revision, 222 Manifest: manifestDoc.String(), 223 Hooks: hooks, 224 } 225 226 if len(notesTxt) > 0 { 227 upgradedRelease.Info.Notes = notesTxt 228 } 229 err = validateManifest(u.cfg.KubeClient, manifestDoc.Bytes(), !u.DisableOpenAPIValidation) 230 return currentRelease, upgradedRelease, err 231} 232 233func (u *Upgrade) performUpgrade(originalRelease, upgradedRelease *release.Release) (*release.Release, error) { 234 current, err := u.cfg.KubeClient.Build(bytes.NewBufferString(originalRelease.Manifest), false) 235 if err != nil { 236 // Checking for removed Kubernetes API error so can provide a more informative error message to the user 237 // Ref: https://github.com/helm/helm/issues/7219 238 if strings.Contains(err.Error(), "unable to recognize \"\": no matches for kind") { 239 return upgradedRelease, errors.Wrap(err, "current release manifest contains removed kubernetes api(s) for this "+ 240 "kubernetes version and it is therefore unable to build the kubernetes "+ 241 "objects for performing the diff. error from kubernetes") 242 } 243 return upgradedRelease, errors.Wrap(err, "unable to build kubernetes objects from current release manifest") 244 } 245 target, err := u.cfg.KubeClient.Build(bytes.NewBufferString(upgradedRelease.Manifest), !u.DisableOpenAPIValidation) 246 if err != nil { 247 return upgradedRelease, errors.Wrap(err, "unable to build kubernetes objects from new release manifest") 248 } 249 250 // It is safe to use force only on target because these are resources currently rendered by the chart. 251 err = target.Visit(setMetadataVisitor(upgradedRelease.Name, upgradedRelease.Namespace, true)) 252 if err != nil { 253 return upgradedRelease, err 254 } 255 256 // Do a basic diff using gvk + name to figure out what new resources are being created so we can validate they don't already exist 257 existingResources := make(map[string]bool) 258 for _, r := range current { 259 existingResources[objectKey(r)] = true 260 } 261 262 var toBeCreated kube.ResourceList 263 for _, r := range target { 264 if !existingResources[objectKey(r)] { 265 toBeCreated = append(toBeCreated, r) 266 } 267 } 268 269 toBeUpdated, err := existingResourceConflict(toBeCreated, upgradedRelease.Name, upgradedRelease.Namespace) 270 if err != nil { 271 return nil, errors.Wrap(err, "rendered manifests contain a resource that already exists. Unable to continue with update") 272 } 273 274 toBeUpdated.Visit(func(r *resource.Info, err error) error { 275 if err != nil { 276 return err 277 } 278 current.Append(r) 279 return nil 280 }) 281 282 if u.DryRun { 283 u.cfg.Log("dry run for %s", upgradedRelease.Name) 284 if len(u.Description) > 0 { 285 upgradedRelease.Info.Description = u.Description 286 } else { 287 upgradedRelease.Info.Description = "Dry run complete" 288 } 289 return upgradedRelease, nil 290 } 291 292 u.cfg.Log("creating upgraded release for %s", upgradedRelease.Name) 293 if err := u.cfg.Releases.Create(upgradedRelease); err != nil { 294 return nil, err 295 } 296 297 // pre-upgrade hooks 298 if !u.DisableHooks { 299 if err := u.cfg.execHook(upgradedRelease, release.HookPreUpgrade, u.Timeout); err != nil { 300 return u.failRelease(upgradedRelease, kube.ResourceList{}, fmt.Errorf("pre-upgrade hooks failed: %s", err)) 301 } 302 } else { 303 u.cfg.Log("upgrade hooks disabled for %s", upgradedRelease.Name) 304 } 305 306 results, err := u.cfg.KubeClient.Update(current, target, u.Force) 307 if err != nil { 308 u.cfg.recordRelease(originalRelease) 309 return u.failRelease(upgradedRelease, results.Created, err) 310 } 311 312 if u.Recreate { 313 // NOTE: Because this is not critical for a release to succeed, we just 314 // log if an error occurs and continue onward. If we ever introduce log 315 // levels, we should make these error level logs so users are notified 316 // that they'll need to go do the cleanup on their own 317 if err := recreate(u.cfg, results.Updated); err != nil { 318 u.cfg.Log(err.Error()) 319 } 320 } 321 322 if u.Wait { 323 if err := u.cfg.KubeClient.Wait(target, u.Timeout); err != nil { 324 u.cfg.recordRelease(originalRelease) 325 return u.failRelease(upgradedRelease, results.Created, err) 326 } 327 } 328 329 // post-upgrade hooks 330 if !u.DisableHooks { 331 if err := u.cfg.execHook(upgradedRelease, release.HookPostUpgrade, u.Timeout); err != nil { 332 return u.failRelease(upgradedRelease, results.Created, fmt.Errorf("post-upgrade hooks failed: %s", err)) 333 } 334 } 335 336 originalRelease.Info.Status = release.StatusSuperseded 337 u.cfg.recordRelease(originalRelease) 338 339 upgradedRelease.Info.Status = release.StatusDeployed 340 if len(u.Description) > 0 { 341 upgradedRelease.Info.Description = u.Description 342 } else { 343 upgradedRelease.Info.Description = "Upgrade complete" 344 } 345 346 return upgradedRelease, nil 347} 348 349func (u *Upgrade) failRelease(rel *release.Release, created kube.ResourceList, err error) (*release.Release, error) { 350 msg := fmt.Sprintf("Upgrade %q failed: %s", rel.Name, err) 351 u.cfg.Log("warning: %s", msg) 352 353 rel.Info.Status = release.StatusFailed 354 rel.Info.Description = msg 355 u.cfg.recordRelease(rel) 356 if u.CleanupOnFail && len(created) > 0 { 357 u.cfg.Log("Cleanup on fail set, cleaning up %d resources", len(created)) 358 _, errs := u.cfg.KubeClient.Delete(created) 359 if errs != nil { 360 var errorList []string 361 for _, e := range errs { 362 errorList = append(errorList, e.Error()) 363 } 364 return rel, errors.Wrapf(fmt.Errorf("unable to cleanup resources: %s", strings.Join(errorList, ", ")), "an error occurred while cleaning up resources. original upgrade error: %s", err) 365 } 366 u.cfg.Log("Resource cleanup complete") 367 } 368 if u.Atomic { 369 u.cfg.Log("Upgrade failed and atomic is set, rolling back to last successful release") 370 371 // As a protection, get the last successful release before rollback. 372 // If there are no successful releases, bail out 373 hist := NewHistory(u.cfg) 374 fullHistory, herr := hist.Run(rel.Name) 375 if herr != nil { 376 return rel, errors.Wrapf(herr, "an error occurred while finding last successful release. original upgrade error: %s", err) 377 } 378 379 // There isn't a way to tell if a previous release was successful, but 380 // generally failed releases do not get superseded unless the next 381 // release is successful, so this should be relatively safe 382 filteredHistory := releaseutil.FilterFunc(func(r *release.Release) bool { 383 return r.Info.Status == release.StatusSuperseded || r.Info.Status == release.StatusDeployed 384 }).Filter(fullHistory) 385 if len(filteredHistory) == 0 { 386 return rel, errors.Wrap(err, "unable to find a previously successful release when attempting to rollback. original upgrade error") 387 } 388 389 releaseutil.Reverse(filteredHistory, releaseutil.SortByRevision) 390 391 rollin := NewRollback(u.cfg) 392 rollin.Version = filteredHistory[0].Version 393 rollin.Wait = true 394 rollin.DisableHooks = u.DisableHooks 395 rollin.Recreate = u.Recreate 396 rollin.Force = u.Force 397 rollin.Timeout = u.Timeout 398 if rollErr := rollin.Run(rel.Name); rollErr != nil { 399 return rel, errors.Wrapf(rollErr, "an error occurred while rolling back the release. original upgrade error: %s", err) 400 } 401 return rel, errors.Wrapf(err, "release %s failed, and has been rolled back due to atomic being set", rel.Name) 402 } 403 404 return rel, err 405} 406 407// reuseValues copies values from the current release to a new release if the 408// new release does not have any values. 409// 410// If the request already has values, or if there are no values in the current 411// release, this does nothing. 412// 413// This is skipped if the u.ResetValues flag is set, in which case the 414// request values are not altered. 415func (u *Upgrade) reuseValues(chart *chart.Chart, current *release.Release, newVals map[string]interface{}) (map[string]interface{}, error) { 416 if u.ResetValues { 417 // If ResetValues is set, we completely ignore current.Config. 418 u.cfg.Log("resetting values to the chart's original version") 419 return newVals, nil 420 } 421 422 // If the ReuseValues flag is set, we always copy the old values over the new config's values. 423 if u.ReuseValues { 424 u.cfg.Log("reusing the old release's values") 425 426 // We have to regenerate the old coalesced values: 427 oldVals, err := chartutil.CoalesceValues(current.Chart, current.Config) 428 if err != nil { 429 return nil, errors.Wrap(err, "failed to rebuild old values") 430 } 431 432 newVals = chartutil.CoalesceTables(newVals, current.Config) 433 434 chart.Values = oldVals 435 436 return newVals, nil 437 } 438 439 if len(newVals) == 0 && len(current.Config) > 0 { 440 u.cfg.Log("copying values from %s (v%d) to new release.", current.Name, current.Version) 441 newVals = current.Config 442 } 443 return newVals, nil 444} 445 446func validateManifest(c kube.Interface, manifest []byte, openAPIValidation bool) error { 447 _, err := c.Build(bytes.NewReader(manifest), openAPIValidation) 448 return err 449} 450 451// recreate captures all the logic for recreating pods for both upgrade and 452// rollback. If we end up refactoring rollback to use upgrade, this can just be 453// made an unexported method on the upgrade action. 454func recreate(cfg *Configuration, resources kube.ResourceList) error { 455 for _, res := range resources { 456 versioned := kube.AsVersioned(res) 457 selector, err := kube.SelectorsForObject(versioned) 458 if err != nil { 459 // If no selector is returned, it means this object is 460 // definitely not a pod, so continue onward 461 continue 462 } 463 464 client, err := cfg.KubernetesClientSet() 465 if err != nil { 466 return errors.Wrapf(err, "unable to recreate pods for object %s/%s because an error occurred", res.Namespace, res.Name) 467 } 468 469 pods, err := client.CoreV1().Pods(res.Namespace).List(context.Background(), metav1.ListOptions{ 470 LabelSelector: selector.String(), 471 }) 472 if err != nil { 473 return errors.Wrapf(err, "unable to recreate pods for object %s/%s because an error occurred", res.Namespace, res.Name) 474 } 475 476 // Restart pods 477 for _, pod := range pods.Items { 478 // Delete each pod for get them restarted with changed spec. 479 if err := client.CoreV1().Pods(pod.Namespace).Delete(context.Background(), pod.Name, *metav1.NewPreconditionDeleteOptions(string(pod.UID))); err != nil { 480 return errors.Wrapf(err, "unable to recreate pods for object %s/%s because an error occurred", res.Namespace, res.Name) 481 } 482 } 483 } 484 return nil 485} 486 487func objectKey(r *resource.Info) string { 488 gvk := r.Object.GetObjectKind().GroupVersionKind() 489 return fmt.Sprintf("%s/%s/%s/%s", gvk.GroupVersion().String(), gvk.Kind, r.Namespace, r.Name) 490} 491