1/* 2Copyright 2017 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package upgrade 18 19import ( 20 "fmt" 21 "io" 22 "io/ioutil" 23 "os" 24 "sort" 25 "strings" 26 "text/tabwriter" 27 28 kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" 29 outputapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/output" 30 "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" 31 "k8s.io/kubernetes/cmd/kubeadm/app/constants" 32 "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" 33 kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" 34 35 "k8s.io/apimachinery/pkg/util/version" 36 clientset "k8s.io/client-go/kubernetes" 37 "k8s.io/klog/v2" 38 39 "github.com/lithammer/dedent" 40 "github.com/pkg/errors" 41 "github.com/spf13/cobra" 42) 43 44type planFlags struct { 45 *applyPlanFlags 46} 47 48// newCmdPlan returns the cobra command for `kubeadm upgrade plan` 49func newCmdPlan(apf *applyPlanFlags) *cobra.Command { 50 flags := &planFlags{ 51 applyPlanFlags: apf, 52 } 53 54 cmd := &cobra.Command{ 55 Use: "plan [version] [flags]", 56 Short: "Check which versions are available to upgrade to and validate whether your current cluster is upgradeable. To skip the internet check, pass in the optional [version] parameter", 57 RunE: func(_ *cobra.Command, args []string) error { 58 return runPlan(flags, args) 59 }, 60 } 61 62 // Register the common flags for apply and plan 63 addApplyPlanFlags(cmd.Flags(), flags.applyPlanFlags) 64 return cmd 65} 66 67// runPlan takes care of outputting available versions to upgrade to for the user 68func runPlan(flags *planFlags, args []string) error { 69 // Start with the basics, verify that the cluster is healthy, build a client and a versionGetter. Never dry-run when planning. 70 klog.V(1).Infoln("[upgrade/plan] verifying health of cluster") 71 klog.V(1).Infoln("[upgrade/plan] retrieving configuration from cluster") 72 client, versionGetter, cfg, err := enforceRequirements(flags.applyPlanFlags, args, false, false) 73 if err != nil { 74 return err 75 } 76 77 // Currently this is the only method we have for distinguishing 78 // external etcd vs static pod etcd 79 isExternalEtcd := cfg.Etcd.External != nil 80 81 // Compute which upgrade possibilities there are 82 klog.V(1).Infoln("[upgrade/plan] computing upgrade possibilities") 83 availUpgrades, err := upgrade.GetAvailableUpgrades(versionGetter, flags.allowExperimentalUpgrades, flags.allowRCUpgrades, isExternalEtcd, client, constants.GetStaticPodDirectory()) 84 if err != nil { 85 return errors.Wrap(err, "[upgrade/versions] FATAL") 86 } 87 88 // Fetch the current state of the component configs 89 klog.V(1).Infoln("[upgrade/plan] analysing component config version states") 90 configVersionStates, err := getComponentConfigVersionStates(&cfg.ClusterConfiguration, client, flags.cfgPath) 91 if err != nil { 92 return errors.WithMessage(err, "[upgrade/versions] FATAL") 93 } 94 95 // No upgrades available 96 if len(availUpgrades) == 0 { 97 klog.V(1).Infoln("[upgrade/plan] Awesome, you're up-to-date! Enjoy!") 98 return nil 99 } 100 101 // Generate and print upgrade plans 102 for _, up := range availUpgrades { 103 plan, unstableVersionFlag, err := genUpgradePlan(&up, isExternalEtcd) 104 if err != nil { 105 return err 106 } 107 108 // Actually, this is needed for machine readable output only. 109 // printUpgradePlan won't output the configVersionStates as it will simply print the same table several times 110 // in the human readable output if it did so 111 plan.ConfigVersions = configVersionStates 112 113 printUpgradePlan(&up, plan, unstableVersionFlag, isExternalEtcd, os.Stdout) 114 } 115 116 // Finally, print the component config state table 117 printComponentConfigVersionStates(configVersionStates, os.Stdout) 118 119 return nil 120} 121 122// newComponentUpgradePlan helper creates outputapi.ComponentUpgradePlan object 123func newComponentUpgradePlan(name, currentVersion, newVersion string) outputapi.ComponentUpgradePlan { 124 return outputapi.ComponentUpgradePlan{ 125 Name: name, 126 CurrentVersion: currentVersion, 127 NewVersion: newVersion, 128 } 129} 130 131// TODO There is currently no way to cleanly output upgrades that involve adding, removing, or changing components 132// https://github.com/kubernetes/kubeadm/issues/810 was created to track addressing this. 133func appendDNSComponent(components []outputapi.ComponentUpgradePlan, up *upgrade.Upgrade, name string) []outputapi.ComponentUpgradePlan { 134 beforeVersion := up.Before.DNSVersion 135 afterVersion := up.After.DNSVersion 136 137 if beforeVersion != "" || afterVersion != "" { 138 components = append(components, newComponentUpgradePlan(name, beforeVersion, afterVersion)) 139 } 140 return components 141} 142 143// genUpgradePlan generates output-friendly upgrade plan out of upgrade.Upgrade structure 144func genUpgradePlan(up *upgrade.Upgrade, isExternalEtcd bool) (*outputapi.UpgradePlan, string, error) { 145 newK8sVersion, err := version.ParseSemantic(up.After.KubeVersion) 146 if err != nil { 147 return nil, "", errors.Wrapf(err, "Unable to parse normalized version %q as a semantic version", up.After.KubeVersion) 148 } 149 150 unstableVersionFlag := "" 151 if len(newK8sVersion.PreRelease()) != 0 { 152 if strings.HasPrefix(newK8sVersion.PreRelease(), "rc") { 153 unstableVersionFlag = " --allow-release-candidate-upgrades" 154 } else { 155 unstableVersionFlag = " --allow-experimental-upgrades" 156 } 157 } 158 159 components := []outputapi.ComponentUpgradePlan{} 160 161 if up.CanUpgradeKubelets() { 162 // The map is of the form <old-version>:<node-count>. Here all the keys are put into a slice and sorted 163 // in order to always get the right order. Then the map value is extracted separately 164 for _, oldVersion := range sortedSliceFromStringIntMap(up.Before.KubeletVersions) { 165 nodeCount := up.Before.KubeletVersions[oldVersion] 166 components = append(components, newComponentUpgradePlan(constants.Kubelet, fmt.Sprintf("%d x %s", nodeCount, oldVersion), up.After.KubeVersion)) 167 } 168 } 169 170 components = append(components, newComponentUpgradePlan(constants.KubeAPIServer, up.Before.KubeVersion, up.After.KubeVersion)) 171 components = append(components, newComponentUpgradePlan(constants.KubeControllerManager, up.Before.KubeVersion, up.After.KubeVersion)) 172 components = append(components, newComponentUpgradePlan(constants.KubeScheduler, up.Before.KubeVersion, up.After.KubeVersion)) 173 components = append(components, newComponentUpgradePlan(constants.KubeProxy, up.Before.KubeVersion, up.After.KubeVersion)) 174 175 components = appendDNSComponent(components, up, constants.CoreDNS) 176 177 if !isExternalEtcd { 178 components = append(components, newComponentUpgradePlan(constants.Etcd, up.Before.EtcdVersion, up.After.EtcdVersion)) 179 } 180 181 return &outputapi.UpgradePlan{Components: components}, unstableVersionFlag, nil 182} 183 184func getComponentConfigVersionStates(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, cfgPath string) ([]outputapi.ComponentConfigVersionState, error) { 185 docmap := kubeadmapi.DocumentMap{} 186 187 if cfgPath != "" { 188 bytes, err := ioutil.ReadFile(cfgPath) 189 if err != nil { 190 return nil, errors.Wrapf(err, "unable to read config file %q", cfgPath) 191 } 192 193 docmap, err = kubeadmutil.SplitYAMLDocuments(bytes) 194 if err != nil { 195 return nil, err 196 } 197 } 198 199 return componentconfigs.GetVersionStates(cfg, client, docmap) 200} 201 202// printUpgradePlan prints a UX-friendly overview of what versions are available to upgrade to 203func printUpgradePlan(up *upgrade.Upgrade, plan *outputapi.UpgradePlan, unstableVersionFlag string, isExternalEtcd bool, w io.Writer) { 204 // The tab writer writes to the "real" writer w 205 tabw := tabwriter.NewWriter(w, 10, 4, 3, ' ', 0) 206 207 // endOfTable helper function flashes table writer 208 endOfTable := func() { 209 tabw.Flush() 210 fmt.Fprintln(w, "") 211 } 212 213 printHeader := true 214 printManualUpgradeHeader := true 215 for _, component := range plan.Components { 216 if isExternalEtcd && component.Name == constants.Etcd { 217 // Don't print etcd if it's external 218 continue 219 } else if component.Name == constants.Kubelet { 220 if printManualUpgradeHeader { 221 fmt.Fprintln(w, "Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':") 222 fmt.Fprintln(tabw, "COMPONENT\tCURRENT\tTARGET") 223 fmt.Fprintf(tabw, "%s\t%s\t%s\n", component.Name, component.CurrentVersion, component.NewVersion) 224 printManualUpgradeHeader = false 225 } else { 226 fmt.Fprintf(tabw, "%s\t%s\t%s\n", "", component.CurrentVersion, component.NewVersion) 227 } 228 } else { 229 if printHeader { 230 // End of manual upgrades table 231 endOfTable() 232 233 fmt.Fprintf(w, "Upgrade to the latest %s:\n", up.Description) 234 fmt.Fprintln(w, "") 235 fmt.Fprintln(tabw, "COMPONENT\tCURRENT\tTARGET") 236 printHeader = false 237 } 238 fmt.Fprintf(tabw, "%s\t%s\t%s\n", component.Name, component.CurrentVersion, component.NewVersion) 239 } 240 } 241 // End of control plane table 242 endOfTable() 243 244 //fmt.Fprintln(w, "") 245 fmt.Fprintln(w, "You can now apply the upgrade by executing the following command:") 246 fmt.Fprintln(w, "") 247 fmt.Fprintf(w, "\tkubeadm upgrade apply %s%s\n", up.After.KubeVersion, unstableVersionFlag) 248 fmt.Fprintln(w, "") 249 250 if up.Before.KubeadmVersion != up.After.KubeadmVersion { 251 fmt.Fprintf(w, "Note: Before you can perform this upgrade, you have to update kubeadm to %s.\n", up.After.KubeadmVersion) 252 fmt.Fprintln(w, "") 253 } 254 255 printLineSeparator(w) 256} 257 258// sortedSliceFromStringIntMap returns a slice of the keys in the map sorted alphabetically 259func sortedSliceFromStringIntMap(strMap map[string]uint16) []string { 260 strSlice := []string{} 261 for k := range strMap { 262 strSlice = append(strSlice, k) 263 } 264 sort.Strings(strSlice) 265 return strSlice 266} 267 268func strOrDash(s string) string { 269 if s != "" { 270 return s 271 } 272 return "-" 273} 274 275func yesOrNo(b bool) string { 276 if b { 277 return "yes" 278 } 279 return "no" 280} 281 282func printLineSeparator(w io.Writer) { 283 fmt.Fprintln(w, "_____________________________________________________________________") 284 fmt.Fprintln(w, "") 285} 286 287func printComponentConfigVersionStates(versionStates []outputapi.ComponentConfigVersionState, w io.Writer) { 288 if len(versionStates) == 0 { 289 fmt.Fprintln(w, "No information available on component configs.") 290 return 291 } 292 293 fmt.Fprintln(w, dedent.Dedent(` 294 The table below shows the current state of component configs as understood by this version of kubeadm. 295 Configs that have a "yes" mark in the "MANUAL UPGRADE REQUIRED" column require manual config upgrade or 296 resetting to kubeadm defaults before a successful upgrade can be performed. The version to manually 297 upgrade to is denoted in the "PREFERRED VERSION" column. 298 `)) 299 300 tabw := tabwriter.NewWriter(w, 10, 4, 3, ' ', 0) 301 fmt.Fprintln(tabw, "API GROUP\tCURRENT VERSION\tPREFERRED VERSION\tMANUAL UPGRADE REQUIRED") 302 303 for _, state := range versionStates { 304 fmt.Fprintf(tabw, 305 "%s\t%s\t%s\t%s\n", 306 state.Group, 307 strOrDash(state.CurrentVersion), 308 strOrDash(state.PreferredVersion), 309 yesOrNo(state.ManualUpgradeRequired), 310 ) 311 } 312 313 tabw.Flush() 314 printLineSeparator(w) 315} 316