1/* 2Copyright 2016 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package controlplane 18 19import ( 20 "fmt" 21 "net" 22 "os" 23 "path/filepath" 24 "strconv" 25 "strings" 26 27 kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" 28 kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" 29 "k8s.io/kubernetes/cmd/kubeadm/app/features" 30 "k8s.io/kubernetes/cmd/kubeadm/app/images" 31 certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" 32 kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" 33 staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" 34 "k8s.io/kubernetes/cmd/kubeadm/app/util/users" 35 36 v1 "k8s.io/api/core/v1" 37 "k8s.io/klog/v2" 38 utilsnet "k8s.io/utils/net" 39 40 "github.com/pkg/errors" 41) 42 43// CreateInitStaticPodManifestFiles will write all static pod manifest files needed to bring up the control plane. 44func CreateInitStaticPodManifestFiles(manifestDir, patchesDir string, cfg *kubeadmapi.InitConfiguration, isDryRun bool) error { 45 klog.V(1).Infoln("[control-plane] creating static Pod files") 46 return CreateStaticPodFiles(manifestDir, patchesDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, isDryRun, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler) 47} 48 49// GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current configuration 50// NB. this methods holds the information about how kubeadm creates static pod manifests. 51func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) map[string]v1.Pod { 52 // Get the required hostpath mounts 53 mounts := getHostPathVolumesForTheControlPlane(cfg) 54 55 // Prepare static pod specs 56 staticPodSpecs := map[string]v1.Pod{ 57 kubeadmconstants.KubeAPIServer: staticpodutil.ComponentPod(v1.Container{ 58 Name: kubeadmconstants.KubeAPIServer, 59 Image: images.GetKubernetesImage(kubeadmconstants.KubeAPIServer, cfg), 60 ImagePullPolicy: v1.PullIfNotPresent, 61 Command: getAPIServerCommand(cfg, endpoint), 62 VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)), 63 LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS), 64 ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS), 65 StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), 66 Resources: staticpodutil.ComponentResources("250m"), 67 Env: kubeadmutil.GetProxyEnvVars(), 68 }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer), 69 map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}), 70 kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{ 71 Name: kubeadmconstants.KubeControllerManager, 72 Image: images.GetKubernetesImage(kubeadmconstants.KubeControllerManager, cfg), 73 ImagePullPolicy: v1.PullIfNotPresent, 74 Command: getControllerManagerCommand(cfg), 75 VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), 76 LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS), 77 StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), 78 Resources: staticpodutil.ComponentResources("200m"), 79 Env: kubeadmutil.GetProxyEnvVars(), 80 }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil), 81 kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ 82 Name: kubeadmconstants.KubeScheduler, 83 Image: images.GetKubernetesImage(kubeadmconstants.KubeScheduler, cfg), 84 ImagePullPolicy: v1.PullIfNotPresent, 85 Command: getSchedulerCommand(cfg), 86 VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), 87 LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), 88 StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), 89 Resources: staticpodutil.ComponentResources("100m"), 90 Env: kubeadmutil.GetProxyEnvVars(), 91 }, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil), 92 } 93 return staticPodSpecs 94} 95 96// CreateStaticPodFiles creates all the requested static pod files. 97func CreateStaticPodFiles(manifestDir, patchesDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, isDryRun bool, componentNames ...string) error { 98 // gets the StaticPodSpecs, actualized for the current ClusterConfiguration 99 klog.V(1).Infoln("[control-plane] getting StaticPodSpecs") 100 specs := GetStaticPodSpecs(cfg, endpoint) 101 102 var usersAndGroups *users.UsersAndGroups 103 var err error 104 if features.Enabled(cfg.FeatureGates, features.RootlessControlPlane) { 105 if isDryRun { 106 fmt.Printf("[dryrun] Would create users and groups for %+v to run as non-root\n", componentNames) 107 } else { 108 usersAndGroups, err = staticpodutil.GetUsersAndGroups() 109 if err != nil { 110 return errors.Wrap(err, "failed to create users and groups") 111 } 112 } 113 } 114 115 // creates required static pod specs 116 for _, componentName := range componentNames { 117 // retrieves the StaticPodSpec for given component 118 spec, exists := specs[componentName] 119 if !exists { 120 return errors.Errorf("couldn't retrieve StaticPodSpec for %q", componentName) 121 } 122 123 // print all volumes that are mounted 124 for _, v := range spec.Spec.Volumes { 125 klog.V(2).Infof("[control-plane] adding volume %q for component %q", v.Name, componentName) 126 } 127 128 if features.Enabled(cfg.FeatureGates, features.RootlessControlPlane) { 129 if isDryRun { 130 fmt.Printf("[dryrun] Would update static pod manifest for %q to run run as non-root\n", componentName) 131 } else { 132 if usersAndGroups != nil { 133 if err := staticpodutil.RunComponentAsNonRoot(componentName, &spec, usersAndGroups, cfg); err != nil { 134 return errors.Wrapf(err, "failed to run component %q as non-root", componentName) 135 } 136 } 137 } 138 } 139 140 // if patchesDir is defined, patch the static Pod manifest 141 if patchesDir != "" { 142 patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout) 143 if err != nil { 144 return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", componentName) 145 } 146 spec = *patchedSpec 147 } 148 149 // writes the StaticPodSpec to disk 150 if err := staticpodutil.WriteStaticPodToDisk(componentName, manifestDir, spec); err != nil { 151 return errors.Wrapf(err, "failed to create static pod manifest file for %q", componentName) 152 } 153 154 klog.V(1).Infof("[control-plane] wrote static Pod manifest for component %q to %q\n", componentName, kubeadmconstants.GetStaticPodFilepath(componentName, manifestDir)) 155 } 156 157 return nil 158} 159 160// getAPIServerCommand builds the right API server command from the given config object and version 161func getAPIServerCommand(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint *kubeadmapi.APIEndpoint) []string { 162 defaultArguments := map[string]string{ 163 "advertise-address": localAPIEndpoint.AdvertiseAddress, 164 "enable-admission-plugins": "NodeRestriction", 165 "service-cluster-ip-range": cfg.Networking.ServiceSubnet, 166 "service-account-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName), 167 "service-account-signing-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName), 168 "service-account-issuer": fmt.Sprintf("https://kubernetes.default.svc.%s", cfg.Networking.DNSDomain), 169 "client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), 170 "tls-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerCertName), 171 "tls-private-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKeyName), 172 "kubelet-client-certificate": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertName), 173 "kubelet-client-key": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName), 174 "enable-bootstrap-token-auth": "true", 175 "secure-port": fmt.Sprintf("%d", localAPIEndpoint.BindPort), 176 "allow-privileged": "true", 177 "kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", 178 // add options to configure the front proxy. Without the generated client cert, this will never be useable 179 // so add it unconditionally with recommended values 180 "requestheader-username-headers": "X-Remote-User", 181 "requestheader-group-headers": "X-Remote-Group", 182 "requestheader-extra-headers-prefix": "X-Remote-Extra-", 183 "requestheader-client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName), 184 "requestheader-allowed-names": "front-proxy-client", 185 "proxy-client-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName), 186 "proxy-client-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName), 187 } 188 189 command := []string{"kube-apiserver"} 190 191 // If the user set endpoints for an external etcd cluster 192 if cfg.Etcd.External != nil { 193 defaultArguments["etcd-servers"] = strings.Join(cfg.Etcd.External.Endpoints, ",") 194 195 // Use any user supplied etcd certificates 196 if cfg.Etcd.External.CAFile != "" { 197 defaultArguments["etcd-cafile"] = cfg.Etcd.External.CAFile 198 } 199 if cfg.Etcd.External.CertFile != "" && cfg.Etcd.External.KeyFile != "" { 200 defaultArguments["etcd-certfile"] = cfg.Etcd.External.CertFile 201 defaultArguments["etcd-keyfile"] = cfg.Etcd.External.KeyFile 202 } 203 } else { 204 // Default to etcd static pod on localhost 205 // localhost IP family should be the same that the AdvertiseAddress 206 etcdLocalhostAddress := "127.0.0.1" 207 if utilsnet.IsIPv6String(localAPIEndpoint.AdvertiseAddress) { 208 etcdLocalhostAddress = "::1" 209 } 210 defaultArguments["etcd-servers"] = fmt.Sprintf("https://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdListenClientPort))) 211 defaultArguments["etcd-cafile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName) 212 defaultArguments["etcd-certfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName) 213 defaultArguments["etcd-keyfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName) 214 215 // Apply user configurations for local etcd 216 if cfg.Etcd.Local != nil { 217 if value, ok := cfg.Etcd.Local.ExtraArgs["advertise-client-urls"]; ok { 218 defaultArguments["etcd-servers"] = value 219 } 220 } 221 } 222 223 // TODO: The following code should be removed after dual-stack is GA. 224 // Note: The user still retains the ability to explicitly set feature-gates and that value will overwrite this base value. 225 if enabled, present := cfg.FeatureGates[features.IPv6DualStack]; present { 226 defaultArguments["feature-gates"] = fmt.Sprintf("%s=%t", features.IPv6DualStack, enabled) 227 } 228 229 if cfg.APIServer.ExtraArgs == nil { 230 cfg.APIServer.ExtraArgs = map[string]string{} 231 } 232 cfg.APIServer.ExtraArgs["authorization-mode"] = getAuthzModes(cfg.APIServer.ExtraArgs["authorization-mode"]) 233 command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.APIServer.ExtraArgs)...) 234 235 return command 236} 237 238// getAuthzModes gets the authorization-related parameters to the api server 239// Node,RBAC is the default mode if nothing is passed to kubeadm. User provided modes override 240// the default. 241func getAuthzModes(authzModeExtraArgs string) string { 242 defaultMode := []string{ 243 kubeadmconstants.ModeNode, 244 kubeadmconstants.ModeRBAC, 245 } 246 247 if len(authzModeExtraArgs) > 0 { 248 mode := []string{} 249 for _, requested := range strings.Split(authzModeExtraArgs, ",") { 250 if isValidAuthzMode(requested) { 251 mode = append(mode, requested) 252 } else { 253 klog.Warningf("ignoring unknown kube-apiserver authorization-mode %q", requested) 254 } 255 } 256 257 // only return the user provided mode if at least one was valid 258 if len(mode) > 0 { 259 if !compareAuthzModes(defaultMode, mode) { 260 klog.Warningf("the default kube-apiserver authorization-mode is %q; using %q", 261 strings.Join(defaultMode, ","), 262 strings.Join(mode, ","), 263 ) 264 } 265 return strings.Join(mode, ",") 266 } 267 } 268 return strings.Join(defaultMode, ",") 269} 270 271// compareAuthzModes compares two given authz modes and returns false if they do not match 272func compareAuthzModes(a, b []string) bool { 273 if len(a) != len(b) { 274 return false 275 } 276 for i, m := range a { 277 if m != b[i] { 278 return false 279 } 280 } 281 return true 282} 283 284func isValidAuthzMode(authzMode string) bool { 285 allModes := []string{ 286 kubeadmconstants.ModeNode, 287 kubeadmconstants.ModeRBAC, 288 kubeadmconstants.ModeWebhook, 289 kubeadmconstants.ModeABAC, 290 kubeadmconstants.ModeAlwaysAllow, 291 kubeadmconstants.ModeAlwaysDeny, 292 } 293 294 for _, mode := range allModes { 295 if authzMode == mode { 296 return true 297 } 298 } 299 return false 300} 301 302// getControllerManagerCommand builds the right controller manager command from the given config object and version 303func getControllerManagerCommand(cfg *kubeadmapi.ClusterConfiguration) []string { 304 305 kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName) 306 caFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName) 307 308 defaultArguments := map[string]string{ 309 "port": "0", 310 "bind-address": "127.0.0.1", 311 "leader-elect": "true", 312 "kubeconfig": kubeconfigFile, 313 "authentication-kubeconfig": kubeconfigFile, 314 "authorization-kubeconfig": kubeconfigFile, 315 "client-ca-file": caFile, 316 "requestheader-client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName), 317 "root-ca-file": caFile, 318 "service-account-private-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName), 319 "cluster-signing-cert-file": caFile, 320 "cluster-signing-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName), 321 "use-service-account-credentials": "true", 322 "controllers": "*,bootstrapsigner,tokencleaner", 323 } 324 325 // If using external CA, pass empty string to controller manager instead of ca.key/ca.crt path, 326 // so that the csrsigning controller fails to start 327 if res, _ := certphase.UsingExternalCA(cfg); res { 328 defaultArguments["cluster-signing-key-file"] = "" 329 defaultArguments["cluster-signing-cert-file"] = "" 330 } 331 332 // Let the controller-manager allocate Node CIDRs for the Pod network. 333 // Each node will get a subspace of the address CIDR provided with --pod-network-cidr. 334 if cfg.Networking.PodSubnet != "" { 335 defaultArguments["allocate-node-cidrs"] = "true" 336 defaultArguments["cluster-cidr"] = cfg.Networking.PodSubnet 337 if cfg.Networking.ServiceSubnet != "" { 338 defaultArguments["service-cluster-ip-range"] = cfg.Networking.ServiceSubnet 339 } 340 } 341 342 // Set cluster name 343 if cfg.ClusterName != "" { 344 defaultArguments["cluster-name"] = cfg.ClusterName 345 } 346 347 // TODO: The following code should be remvoved after dual-stack is GA. 348 // Note: The user still retains the ability to explicitly set feature-gates and that value will overwrite this base value. 349 enabled, present := cfg.FeatureGates[features.IPv6DualStack] 350 if present { 351 defaultArguments["feature-gates"] = fmt.Sprintf("%s=%t", features.IPv6DualStack, enabled) 352 } 353 354 command := []string{"kube-controller-manager"} 355 command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.ControllerManager.ExtraArgs)...) 356 357 return command 358} 359 360// getSchedulerCommand builds the right scheduler command from the given config object and version 361func getSchedulerCommand(cfg *kubeadmapi.ClusterConfiguration) []string { 362 kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName) 363 defaultArguments := map[string]string{ 364 "port": "0", 365 "bind-address": "127.0.0.1", 366 "leader-elect": "true", 367 "kubeconfig": kubeconfigFile, 368 "authentication-kubeconfig": kubeconfigFile, 369 "authorization-kubeconfig": kubeconfigFile, 370 } 371 372 // TODO: The following code should be remvoved after dual-stack is GA. 373 // Note: The user still retains the ability to explicitly set feature-gates and that value will overwrite this base value. 374 if enabled, present := cfg.FeatureGates[features.IPv6DualStack]; present { 375 defaultArguments["feature-gates"] = fmt.Sprintf("%s=%t", features.IPv6DualStack, enabled) 376 } 377 378 command := []string{"kube-scheduler"} 379 command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Scheduler.ExtraArgs)...) 380 return command 381} 382