1/* 2Copyright 2014 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17// Package app does all of the work necessary to create a Kubernetes 18// APIServer by binding together the API, master and APIServer infrastructure. 19// It can be configured and called directly or via the hyperkube framework. 20package app 21 22import ( 23 "crypto/tls" 24 "fmt" 25 "net" 26 "net/http" 27 "net/url" 28 "os" 29 "strings" 30 "time" 31 32 "github.com/spf13/cobra" 33 "github.com/spf13/pflag" 34 35 extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" 36 utilerrors "k8s.io/apimachinery/pkg/util/errors" 37 utilnet "k8s.io/apimachinery/pkg/util/net" 38 "k8s.io/apimachinery/pkg/util/sets" 39 "k8s.io/apiserver/pkg/admission" 40 "k8s.io/apiserver/pkg/authorization/authorizer" 41 openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" 42 genericfeatures "k8s.io/apiserver/pkg/features" 43 genericapiserver "k8s.io/apiserver/pkg/server" 44 "k8s.io/apiserver/pkg/server/egressselector" 45 "k8s.io/apiserver/pkg/server/filters" 46 serveroptions "k8s.io/apiserver/pkg/server/options" 47 serverstorage "k8s.io/apiserver/pkg/server/storage" 48 utilfeature "k8s.io/apiserver/pkg/util/feature" 49 utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" 50 "k8s.io/apiserver/pkg/util/webhook" 51 clientgoinformers "k8s.io/client-go/informers" 52 clientgoclientset "k8s.io/client-go/kubernetes" 53 "k8s.io/client-go/rest" 54 "k8s.io/client-go/util/keyutil" 55 cliflag "k8s.io/component-base/cli/flag" 56 "k8s.io/component-base/cli/globalflag" 57 _ "k8s.io/component-base/metrics/prometheus/workqueue" // for workqueue metric registration 58 "k8s.io/component-base/term" 59 "k8s.io/component-base/version" 60 "k8s.io/component-base/version/verflag" 61 "k8s.io/klog/v2" 62 aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" 63 aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" 64 65 "k8s.io/kubernetes/cmd/kube-apiserver/app/options" 66 "k8s.io/kubernetes/pkg/api/legacyscheme" 67 "k8s.io/kubernetes/pkg/capabilities" 68 "k8s.io/kubernetes/pkg/controlplane" 69 "k8s.io/kubernetes/pkg/controlplane/reconcilers" 70 generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" 71 "k8s.io/kubernetes/pkg/kubeapiserver" 72 kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" 73 kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator" 74 "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" 75 rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" 76 "k8s.io/kubernetes/pkg/serviceaccount" 77) 78 79// TODO: delete this check after insecure flags removed in v1.24 80func checkNonZeroInsecurePort(fs *pflag.FlagSet) error { 81 for _, name := range options.InsecurePortFlags { 82 val, err := fs.GetInt(name) 83 if err != nil { 84 return err 85 } 86 if val != 0 { 87 return fmt.Errorf("invalid port value %d: only zero is allowed", val) 88 } 89 } 90 return nil 91} 92 93// NewAPIServerCommand creates a *cobra.Command object with default parameters 94func NewAPIServerCommand() *cobra.Command { 95 s := options.NewServerRunOptions() 96 cmd := &cobra.Command{ 97 Use: "kube-apiserver", 98 Long: `The Kubernetes API server validates and configures data 99for the api objects which include pods, services, replicationcontrollers, and 100others. The API Server services REST operations and provides the frontend to the 101cluster's shared state through which all other components interact.`, 102 103 // stop printing usage when the command errors 104 SilenceUsage: true, 105 PersistentPreRunE: func(*cobra.Command, []string) error { 106 // silence client-go warnings. 107 // kube-apiserver loopback clients should not log self-issued warnings. 108 rest.SetDefaultWarningHandler(rest.NoWarnings{}) 109 return nil 110 }, 111 RunE: func(cmd *cobra.Command, args []string) error { 112 verflag.PrintAndExitIfRequested() 113 fs := cmd.Flags() 114 cliflag.PrintFlags(fs) 115 116 err := checkNonZeroInsecurePort(fs) 117 if err != nil { 118 return err 119 } 120 // set default options 121 completedOptions, err := Complete(s) 122 if err != nil { 123 return err 124 } 125 126 // validate options 127 if errs := completedOptions.Validate(); len(errs) != 0 { 128 return utilerrors.NewAggregate(errs) 129 } 130 131 return Run(completedOptions, genericapiserver.SetupSignalHandler()) 132 }, 133 Args: func(cmd *cobra.Command, args []string) error { 134 for _, arg := range args { 135 if len(arg) > 0 { 136 return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args) 137 } 138 } 139 return nil 140 }, 141 } 142 143 fs := cmd.Flags() 144 namedFlagSets := s.Flags() 145 verflag.AddFlags(namedFlagSets.FlagSet("global")) 146 globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name()) 147 options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic")) 148 for _, f := range namedFlagSets.FlagSets { 149 fs.AddFlagSet(f) 150 } 151 152 cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) 153 cliflag.SetUsageAndHelpFunc(cmd, namedFlagSets, cols) 154 155 return cmd 156} 157 158// Run runs the specified APIServer. This should never exit. 159func Run(completeOptions completedServerRunOptions, stopCh <-chan struct{}) error { 160 // To help debugging, immediately log version 161 klog.Infof("Version: %+v", version.Get()) 162 163 server, err := CreateServerChain(completeOptions, stopCh) 164 if err != nil { 165 return err 166 } 167 168 prepared, err := server.PrepareRun() 169 if err != nil { 170 return err 171 } 172 173 return prepared.Run(stopCh) 174} 175 176// CreateServerChain creates the apiservers connected via delegation. 177func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan struct{}) (*aggregatorapiserver.APIAggregator, error) { 178 kubeAPIServerConfig, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions) 179 if err != nil { 180 return nil, err 181 } 182 183 // If additional API servers are added, they should be gated. 184 apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount, 185 serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIServerConfig.ExtraConfig.ProxyTransport, kubeAPIServerConfig.GenericConfig.EgressSelector, kubeAPIServerConfig.GenericConfig.LoopbackClientConfig, kubeAPIServerConfig.GenericConfig.TracerProvider)) 186 if err != nil { 187 return nil, err 188 } 189 apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.NewEmptyDelegate()) 190 if err != nil { 191 return nil, err 192 } 193 194 kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer) 195 if err != nil { 196 return nil, err 197 } 198 199 // aggregator comes last in the chain 200 aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, completedOptions.ServerRunOptions, kubeAPIServerConfig.ExtraConfig.VersionedInformers, serviceResolver, kubeAPIServerConfig.ExtraConfig.ProxyTransport, pluginInitializer) 201 if err != nil { 202 return nil, err 203 } 204 aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers) 205 if err != nil { 206 // we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines 207 return nil, err 208 } 209 210 return aggregatorServer, nil 211} 212 213// CreateKubeAPIServer creates and wires a workable kube-apiserver 214func CreateKubeAPIServer(kubeAPIServerConfig *controlplane.Config, delegateAPIServer genericapiserver.DelegationTarget) (*controlplane.Instance, error) { 215 kubeAPIServer, err := kubeAPIServerConfig.Complete().New(delegateAPIServer) 216 if err != nil { 217 return nil, err 218 } 219 220 return kubeAPIServer, nil 221} 222 223// CreateProxyTransport creates the dialer infrastructure to connect to the nodes. 224func CreateProxyTransport() *http.Transport { 225 var proxyDialerFn utilnet.DialFunc 226 // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname 227 proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} 228 proxyTransport := utilnet.SetTransportDefaults(&http.Transport{ 229 DialContext: proxyDialerFn, 230 TLSClientConfig: proxyTLSClientConfig, 231 }) 232 return proxyTransport 233} 234 235// CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them 236func CreateKubeAPIServerConfig(s completedServerRunOptions) ( 237 *controlplane.Config, 238 aggregatorapiserver.ServiceResolver, 239 []admission.PluginInitializer, 240 error, 241) { 242 proxyTransport := CreateProxyTransport() 243 244 genericConfig, versionedInformers, serviceResolver, pluginInitializers, admissionPostStartHook, storageFactory, err := buildGenericConfig(s.ServerRunOptions, proxyTransport) 245 if err != nil { 246 return nil, nil, nil, err 247 } 248 249 capabilities.Initialize(capabilities.Capabilities{ 250 AllowPrivileged: s.AllowPrivileged, 251 // TODO(vmarmol): Implement support for HostNetworkSources. 252 PrivilegedSources: capabilities.PrivilegedSources{ 253 HostNetworkSources: []string{}, 254 HostPIDSources: []string{}, 255 HostIPCSources: []string{}, 256 }, 257 PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, 258 }) 259 260 s.Metrics.Apply() 261 serviceaccount.RegisterMetrics() 262 263 s.Logs.Apply() 264 265 config := &controlplane.Config{ 266 GenericConfig: genericConfig, 267 ExtraConfig: controlplane.ExtraConfig{ 268 APIResourceConfigSource: storageFactory.APIResourceConfigSource, 269 StorageFactory: storageFactory, 270 EventTTL: s.EventTTL, 271 KubeletClientConfig: s.KubeletConfig, 272 EnableLogsSupport: s.EnableLogsHandler, 273 ProxyTransport: proxyTransport, 274 275 ServiceIPRange: s.PrimaryServiceClusterIPRange, 276 APIServerServiceIP: s.APIServerServiceIP, 277 SecondaryServiceIPRange: s.SecondaryServiceClusterIPRange, 278 279 APIServerServicePort: 443, 280 281 ServiceNodePortRange: s.ServiceNodePortRange, 282 KubernetesServiceNodePort: s.KubernetesServiceNodePort, 283 284 EndpointReconcilerType: reconcilers.Type(s.EndpointReconcilerType), 285 MasterCount: s.MasterCount, 286 287 ServiceAccountIssuer: s.ServiceAccountIssuer, 288 ServiceAccountMaxExpiration: s.ServiceAccountTokenMaxExpiration, 289 ExtendExpiration: s.Authentication.ServiceAccounts.ExtendExpiration, 290 291 VersionedInformers: versionedInformers, 292 293 IdentityLeaseDurationSeconds: s.IdentityLeaseDurationSeconds, 294 IdentityLeaseRenewIntervalSeconds: s.IdentityLeaseRenewIntervalSeconds, 295 }, 296 } 297 298 clientCAProvider, err := s.Authentication.ClientCert.GetClientCAContentProvider() 299 if err != nil { 300 return nil, nil, nil, err 301 } 302 config.ExtraConfig.ClusterAuthenticationInfo.ClientCA = clientCAProvider 303 304 requestHeaderConfig, err := s.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig() 305 if err != nil { 306 return nil, nil, nil, err 307 } 308 if requestHeaderConfig != nil { 309 config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider 310 config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames 311 config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes 312 config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders 313 config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders 314 } 315 316 if err := config.GenericConfig.AddPostStartHook("start-kube-apiserver-admission-initializer", admissionPostStartHook); err != nil { 317 return nil, nil, nil, err 318 } 319 320 if config.GenericConfig.EgressSelector != nil { 321 // Use the config.GenericConfig.EgressSelector lookup to find the dialer to connect to the kubelet 322 config.ExtraConfig.KubeletClientConfig.Lookup = config.GenericConfig.EgressSelector.Lookup 323 324 // Use the config.GenericConfig.EgressSelector lookup as the transport used by the "proxy" subresources. 325 networkContext := egressselector.Cluster.AsNetworkContext() 326 dialer, err := config.GenericConfig.EgressSelector.Lookup(networkContext) 327 if err != nil { 328 return nil, nil, nil, err 329 } 330 c := proxyTransport.Clone() 331 c.DialContext = dialer 332 config.ExtraConfig.ProxyTransport = c 333 } 334 335 // Load the public keys. 336 var pubKeys []interface{} 337 for _, f := range s.Authentication.ServiceAccounts.KeyFiles { 338 keys, err := keyutil.PublicKeysFromFile(f) 339 if err != nil { 340 return nil, nil, nil, fmt.Errorf("failed to parse key file %q: %v", f, err) 341 } 342 pubKeys = append(pubKeys, keys...) 343 } 344 // Plumb the required metadata through ExtraConfig. 345 config.ExtraConfig.ServiceAccountIssuerURL = s.Authentication.ServiceAccounts.Issuers[0] 346 config.ExtraConfig.ServiceAccountJWKSURI = s.Authentication.ServiceAccounts.JWKSURI 347 config.ExtraConfig.ServiceAccountPublicKeys = pubKeys 348 349 return config, serviceResolver, pluginInitializers, nil 350} 351 352// BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it 353func buildGenericConfig( 354 s *options.ServerRunOptions, 355 proxyTransport *http.Transport, 356) ( 357 genericConfig *genericapiserver.Config, 358 versionedInformers clientgoinformers.SharedInformerFactory, 359 serviceResolver aggregatorapiserver.ServiceResolver, 360 pluginInitializers []admission.PluginInitializer, 361 admissionPostStartHook genericapiserver.PostStartHookFunc, 362 storageFactory *serverstorage.DefaultStorageFactory, 363 lastErr error, 364) { 365 genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs) 366 genericConfig.MergedResourceConfig = controlplane.DefaultAPIResourceConfigSource() 367 368 if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil { 369 return 370 } 371 372 if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil { 373 return 374 } 375 if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil { 376 return 377 } 378 if lastErr = s.APIEnablement.ApplyTo(genericConfig, controlplane.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); lastErr != nil { 379 return 380 } 381 if lastErr = s.EgressSelector.ApplyTo(genericConfig); lastErr != nil { 382 return 383 } 384 if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) { 385 if lastErr = s.Traces.ApplyTo(genericConfig.EgressSelector, genericConfig); lastErr != nil { 386 return 387 } 388 } 389 390 genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme)) 391 genericConfig.OpenAPIConfig.Info.Title = "Kubernetes" 392 genericConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck( 393 sets.NewString("watch", "proxy"), 394 sets.NewString("attach", "exec", "proxy", "log", "portforward"), 395 ) 396 397 kubeVersion := version.Get() 398 genericConfig.Version = &kubeVersion 399 400 storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig() 401 storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig 402 completedStorageFactoryConfig, err := storageFactoryConfig.Complete(s.Etcd) 403 if err != nil { 404 lastErr = err 405 return 406 } 407 storageFactory, lastErr = completedStorageFactoryConfig.New() 408 if lastErr != nil { 409 return 410 } 411 if genericConfig.EgressSelector != nil { 412 storageFactory.StorageConfig.Transport.EgressLookup = genericConfig.EgressSelector.Lookup 413 } 414 if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) && genericConfig.TracerProvider != nil { 415 storageFactory.StorageConfig.Transport.TracerProvider = genericConfig.TracerProvider 416 } 417 if lastErr = s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); lastErr != nil { 418 return 419 } 420 421 // Use protobufs for self-communication. 422 // Since not every generic apiserver has to support protobufs, we 423 // cannot default to it in generic apiserver and need to explicitly 424 // set it in kube-apiserver. 425 genericConfig.LoopbackClientConfig.ContentConfig.ContentType = "application/vnd.kubernetes.protobuf" 426 // Disable compression for self-communication, since we are going to be 427 // on a fast local network 428 genericConfig.LoopbackClientConfig.DisableCompression = true 429 430 kubeClientConfig := genericConfig.LoopbackClientConfig 431 clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeClientConfig) 432 if err != nil { 433 lastErr = fmt.Errorf("failed to create real external clientset: %v", err) 434 return 435 } 436 versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) 437 438 // Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present 439 if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, clientgoExternalClient, versionedInformers); lastErr != nil { 440 return 441 } 442 443 genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, genericConfig.EgressSelector, versionedInformers) 444 if err != nil { 445 lastErr = fmt.Errorf("invalid authorization config: %v", err) 446 return 447 } 448 if !sets.NewString(s.Authorization.Modes...).Has(modes.ModeRBAC) { 449 genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName) 450 } 451 452 lastErr = s.Audit.ApplyTo(genericConfig) 453 if lastErr != nil { 454 return 455 } 456 457 admissionConfig := &kubeapiserveradmission.Config{ 458 ExternalInformers: versionedInformers, 459 LoopbackClientConfig: genericConfig.LoopbackClientConfig, 460 CloudConfigFile: s.CloudProvider.CloudConfigFile, 461 } 462 serviceResolver = buildServiceResolver(s.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers) 463 pluginInitializers, admissionPostStartHook, err = admissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider) 464 if err != nil { 465 lastErr = fmt.Errorf("failed to create admission plugin initializer: %v", err) 466 return 467 } 468 469 err = s.Admission.ApplyTo( 470 genericConfig, 471 versionedInformers, 472 kubeClientConfig, 473 utilfeature.DefaultFeatureGate, 474 pluginInitializers...) 475 if err != nil { 476 lastErr = fmt.Errorf("failed to initialize admission: %v", err) 477 return 478 } 479 480 if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && s.GenericServerRunOptions.EnablePriorityAndFairness { 481 genericConfig.FlowControl, lastErr = BuildPriorityAndFairness(s, clientgoExternalClient, versionedInformers) 482 } 483 484 return 485} 486 487// BuildAuthorizer constructs the authorizer 488func BuildAuthorizer(s *options.ServerRunOptions, EgressSelector *egressselector.EgressSelector, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) { 489 authorizationConfig := s.Authorization.ToAuthorizationConfig(versionedInformers) 490 491 if EgressSelector != nil { 492 egressDialer, err := EgressSelector.Lookup(egressselector.ControlPlane.AsNetworkContext()) 493 if err != nil { 494 return nil, nil, err 495 } 496 authorizationConfig.CustomDial = egressDialer 497 } 498 499 return authorizationConfig.New() 500} 501 502// BuildPriorityAndFairness constructs the guts of the API Priority and Fairness filter 503func BuildPriorityAndFairness(s *options.ServerRunOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (utilflowcontrol.Interface, error) { 504 if s.GenericServerRunOptions.MaxRequestsInFlight+s.GenericServerRunOptions.MaxMutatingRequestsInFlight <= 0 { 505 return nil, fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", s.GenericServerRunOptions.MaxRequestsInFlight, s.GenericServerRunOptions.MaxMutatingRequestsInFlight) 506 } 507 return utilflowcontrol.New( 508 versionedInformer, 509 extclient.FlowcontrolV1beta1(), 510 s.GenericServerRunOptions.MaxRequestsInFlight+s.GenericServerRunOptions.MaxMutatingRequestsInFlight, 511 s.GenericServerRunOptions.RequestTimeout/4, 512 ), nil 513} 514 515// completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked. 516type completedServerRunOptions struct { 517 *options.ServerRunOptions 518} 519 520// Complete set default ServerRunOptions. 521// Should be called after kube-apiserver flags parsed. 522func Complete(s *options.ServerRunOptions) (completedServerRunOptions, error) { 523 var options completedServerRunOptions 524 // set defaults 525 if err := s.GenericServerRunOptions.DefaultAdvertiseAddress(s.SecureServing.SecureServingOptions); err != nil { 526 return options, err 527 } 528 529 // process s.ServiceClusterIPRange from list to Primary and Secondary 530 // we process secondary only if provided by user 531 apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(s.ServiceClusterIPRanges) 532 if err != nil { 533 return options, err 534 } 535 s.PrimaryServiceClusterIPRange = primaryServiceIPRange 536 s.SecondaryServiceClusterIPRange = secondaryServiceIPRange 537 s.APIServerServiceIP = apiServerServiceIP 538 539 if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil { 540 return options, fmt.Errorf("error creating self-signed certificates: %v", err) 541 } 542 543 if len(s.GenericServerRunOptions.ExternalHost) == 0 { 544 if len(s.GenericServerRunOptions.AdvertiseAddress) > 0 { 545 s.GenericServerRunOptions.ExternalHost = s.GenericServerRunOptions.AdvertiseAddress.String() 546 } else { 547 if hostname, err := os.Hostname(); err == nil { 548 s.GenericServerRunOptions.ExternalHost = hostname 549 } else { 550 return options, fmt.Errorf("error finding host name: %v", err) 551 } 552 } 553 klog.Infof("external host was not specified, using %v", s.GenericServerRunOptions.ExternalHost) 554 } 555 556 s.Authentication.ApplyAuthorization(s.Authorization) 557 558 // Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling 559 // TokenRequest functionality. This defaulting was convenient, but messed up 560 // a lot of people when they rotated their serving cert with no idea it was 561 // connected to their service account keys. We are taking this opportunity to 562 // remove this problematic defaulting. 563 if s.ServiceAccountSigningKeyFile == "" { 564 // Default to the private server key for service account token signing 565 if len(s.Authentication.ServiceAccounts.KeyFiles) == 0 && s.SecureServing.ServerCert.CertKey.KeyFile != "" { 566 if kubeauthenticator.IsValidServiceAccountKeyFile(s.SecureServing.ServerCert.CertKey.KeyFile) { 567 s.Authentication.ServiceAccounts.KeyFiles = []string{s.SecureServing.ServerCert.CertKey.KeyFile} 568 } else { 569 klog.Warning("No TLS key provided, service account token authentication disabled") 570 } 571 } 572 } 573 574 if s.ServiceAccountSigningKeyFile != "" && len(s.Authentication.ServiceAccounts.Issuers) != 0 && s.Authentication.ServiceAccounts.Issuers[0] != "" { 575 sk, err := keyutil.PrivateKeyFromFile(s.ServiceAccountSigningKeyFile) 576 if err != nil { 577 return options, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err) 578 } 579 if s.Authentication.ServiceAccounts.MaxExpiration != 0 { 580 lowBound := time.Hour 581 upBound := time.Duration(1<<32) * time.Second 582 if s.Authentication.ServiceAccounts.MaxExpiration < lowBound || 583 s.Authentication.ServiceAccounts.MaxExpiration > upBound { 584 return options, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds") 585 } 586 if s.Authentication.ServiceAccounts.ExtendExpiration { 587 if s.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second { 588 klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, s.Authentication.ServiceAccounts.MaxExpiration) 589 } 590 if s.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second { 591 klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, s.Authentication.ServiceAccounts.MaxExpiration) 592 } 593 } 594 } 595 596 s.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(s.Authentication.ServiceAccounts.Issuers[0], sk) 597 if err != nil { 598 return options, fmt.Errorf("failed to build token generator: %v", err) 599 } 600 s.ServiceAccountTokenMaxExpiration = s.Authentication.ServiceAccounts.MaxExpiration 601 } 602 603 if s.Etcd.EnableWatchCache { 604 sizes := kubeapiserver.DefaultWatchCacheSizes() 605 // Ensure that overrides parse correctly. 606 userSpecified, err := serveroptions.ParseWatchCacheSizes(s.Etcd.WatchCacheSizes) 607 if err != nil { 608 return options, err 609 } 610 for resource, size := range userSpecified { 611 sizes[resource] = size 612 } 613 s.Etcd.WatchCacheSizes, err = serveroptions.WriteWatchCacheSizes(sizes) 614 if err != nil { 615 return options, err 616 } 617 } 618 619 for key, value := range s.APIEnablement.RuntimeConfig { 620 if key == "v1" || strings.HasPrefix(key, "v1/") || 621 key == "api/v1" || strings.HasPrefix(key, "api/v1/") { 622 delete(s.APIEnablement.RuntimeConfig, key) 623 s.APIEnablement.RuntimeConfig["/v1"] = value 624 } 625 if key == "api/legacy" { 626 delete(s.APIEnablement.RuntimeConfig, key) 627 } 628 } 629 630 options.ServerRunOptions = s 631 return options, nil 632} 633 634func buildServiceResolver(enabledAggregatorRouting bool, hostname string, informer clientgoinformers.SharedInformerFactory) webhook.ServiceResolver { 635 var serviceResolver webhook.ServiceResolver 636 if enabledAggregatorRouting { 637 serviceResolver = aggregatorapiserver.NewEndpointServiceResolver( 638 informer.Core().V1().Services().Lister(), 639 informer.Core().V1().Endpoints().Lister(), 640 ) 641 } else { 642 serviceResolver = aggregatorapiserver.NewClusterIPServiceResolver( 643 informer.Core().V1().Services().Lister(), 644 ) 645 } 646 // resolve kubernetes.default.svc locally 647 if localHost, err := url.Parse(hostname); err == nil { 648 serviceResolver = aggregatorapiserver.NewLoopbackServiceResolver(serviceResolver, localHost) 649 } 650 return serviceResolver 651} 652 653func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, net.IPNet, error) { 654 serviceClusterIPRangeList := []string{} 655 if serviceClusterIPRanges != "" { 656 serviceClusterIPRangeList = strings.Split(serviceClusterIPRanges, ",") 657 } 658 659 var apiServerServiceIP net.IP 660 var primaryServiceIPRange net.IPNet 661 var secondaryServiceIPRange net.IPNet 662 var err error 663 // nothing provided by user, use default range (only applies to the Primary) 664 if len(serviceClusterIPRangeList) == 0 { 665 var primaryServiceClusterCIDR net.IPNet 666 primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(primaryServiceClusterCIDR) 667 if err != nil { 668 return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges: %v", err) 669 } 670 return apiServerServiceIP, primaryServiceIPRange, net.IPNet{}, nil 671 } 672 673 _, primaryServiceClusterCIDR, err := net.ParseCIDR(serviceClusterIPRangeList[0]) 674 if err != nil { 675 return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[0] is not a valid cidr") 676 } 677 678 primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(*(primaryServiceClusterCIDR)) 679 if err != nil { 680 return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges for primary service cidr: %v", err) 681 } 682 683 // user provided at least two entries 684 // note: validation asserts that the list is max of two dual stack entries 685 if len(serviceClusterIPRangeList) > 1 { 686 _, secondaryServiceClusterCIDR, err := net.ParseCIDR(serviceClusterIPRangeList[1]) 687 if err != nil { 688 return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[1] is not an ip net") 689 } 690 secondaryServiceIPRange = *secondaryServiceClusterCIDR 691 } 692 return apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, nil 693} 694