1/* 2Copyright 2014 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17// Package options contains flags and options for initializing an apiserver 18package options 19 20import ( 21 "net" 22 "strings" 23 "time" 24 25 "github.com/spf13/pflag" 26 utilnet "k8s.io/apimachinery/pkg/util/net" 27 genericoptions "k8s.io/apiserver/pkg/server/options" 28 "k8s.io/apiserver/pkg/storage/storagebackend" 29 cliflag "k8s.io/component-base/cli/flag" 30 "k8s.io/component-base/logs" 31 "k8s.io/component-base/metrics" 32 33 api "k8s.io/kubernetes/pkg/apis/core" 34 "k8s.io/kubernetes/pkg/cluster/ports" 35 "k8s.io/kubernetes/pkg/controlplane/reconcilers" 36 _ "k8s.io/kubernetes/pkg/features" // add the kubernetes feature gates 37 kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" 38 kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" 39 "k8s.io/kubernetes/pkg/serviceaccount" 40) 41 42// InsecurePortFlags are dummy flags, they are kept only for compatibility and will be removed in v1.24. 43// TODO: remove these flags in v1.24. 44var InsecurePortFlags = []string{"insecure-port", "port"} 45 46// ServerRunOptions runs a kubernetes api server. 47type ServerRunOptions struct { 48 GenericServerRunOptions *genericoptions.ServerRunOptions 49 Etcd *genericoptions.EtcdOptions 50 SecureServing *genericoptions.SecureServingOptionsWithLoopback 51 Audit *genericoptions.AuditOptions 52 Features *genericoptions.FeatureOptions 53 Admission *kubeoptions.AdmissionOptions 54 Authentication *kubeoptions.BuiltInAuthenticationOptions 55 Authorization *kubeoptions.BuiltInAuthorizationOptions 56 CloudProvider *kubeoptions.CloudProviderOptions 57 APIEnablement *genericoptions.APIEnablementOptions 58 EgressSelector *genericoptions.EgressSelectorOptions 59 Metrics *metrics.Options 60 Logs *logs.Options 61 Traces *genericoptions.TracingOptions 62 63 AllowPrivileged bool 64 EnableLogsHandler bool 65 EventTTL time.Duration 66 KubeletConfig kubeletclient.KubeletClientConfig 67 KubernetesServiceNodePort int 68 MaxConnectionBytesPerSec int64 69 // ServiceClusterIPRange is mapped to input provided by user 70 ServiceClusterIPRanges string 71 // PrimaryServiceClusterIPRange and SecondaryServiceClusterIPRange are the results 72 // of parsing ServiceClusterIPRange into actual values 73 PrimaryServiceClusterIPRange net.IPNet 74 SecondaryServiceClusterIPRange net.IPNet 75 // APIServerServiceIP is the first valid IP from PrimaryServiceClusterIPRange 76 APIServerServiceIP net.IP 77 78 ServiceNodePortRange utilnet.PortRange 79 80 ProxyClientCertFile string 81 ProxyClientKeyFile string 82 83 EnableAggregatorRouting bool 84 85 MasterCount int 86 EndpointReconcilerType string 87 88 IdentityLeaseDurationSeconds int 89 IdentityLeaseRenewIntervalSeconds int 90 91 ServiceAccountSigningKeyFile string 92 ServiceAccountIssuer serviceaccount.TokenGenerator 93 ServiceAccountTokenMaxExpiration time.Duration 94 95 ShowHiddenMetricsForVersion string 96} 97 98// NewServerRunOptions creates a new ServerRunOptions object with default parameters 99func NewServerRunOptions() *ServerRunOptions { 100 s := ServerRunOptions{ 101 GenericServerRunOptions: genericoptions.NewServerRunOptions(), 102 Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)), 103 SecureServing: kubeoptions.NewSecureServingOptions(), 104 Audit: genericoptions.NewAuditOptions(), 105 Features: genericoptions.NewFeatureOptions(), 106 Admission: kubeoptions.NewAdmissionOptions(), 107 Authentication: kubeoptions.NewBuiltInAuthenticationOptions().WithAll(), 108 Authorization: kubeoptions.NewBuiltInAuthorizationOptions(), 109 CloudProvider: kubeoptions.NewCloudProviderOptions(), 110 APIEnablement: genericoptions.NewAPIEnablementOptions(), 111 EgressSelector: genericoptions.NewEgressSelectorOptions(), 112 Metrics: metrics.NewOptions(), 113 Logs: logs.NewOptions(), 114 Traces: genericoptions.NewTracingOptions(), 115 116 EnableLogsHandler: true, 117 EventTTL: 1 * time.Hour, 118 MasterCount: 1, 119 EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType), 120 IdentityLeaseDurationSeconds: 3600, 121 IdentityLeaseRenewIntervalSeconds: 10, 122 KubeletConfig: kubeletclient.KubeletClientConfig{ 123 Port: ports.KubeletPort, 124 ReadOnlyPort: ports.KubeletReadOnlyPort, 125 PreferredAddressTypes: []string{ 126 // --override-hostname 127 string(api.NodeHostName), 128 129 // internal, preferring DNS if reported 130 string(api.NodeInternalDNS), 131 string(api.NodeInternalIP), 132 133 // external, preferring DNS if reported 134 string(api.NodeExternalDNS), 135 string(api.NodeExternalIP), 136 }, 137 HTTPTimeout: time.Duration(5) * time.Second, 138 }, 139 ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange, 140 } 141 142 // Overwrite the default for storage data format. 143 s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf" 144 145 return &s 146} 147 148// TODO: remove these insecure flags in v1.24 149func addDummyInsecureFlags(fs *pflag.FlagSet) { 150 var ( 151 bindAddr = net.IPv4(127, 0, 0, 1) 152 bindPort int 153 ) 154 155 for _, name := range []string{"insecure-bind-address", "address"} { 156 fs.IPVar(&bindAddr, name, bindAddr, ""+ 157 "The IP address on which to serve the insecure port (set to 0.0.0.0 or :: for listening in all interfaces and IP families).") 158 fs.MarkDeprecated(name, "This flag has no effect now and will be removed in v1.24.") 159 } 160 161 for _, name := range InsecurePortFlags { 162 fs.IntVar(&bindPort, name, bindPort, ""+ 163 "The port on which to serve unsecured, unauthenticated access.") 164 fs.MarkDeprecated(name, "This flag has no effect now and will be removed in v1.24.") 165 } 166} 167 168// Flags returns flags for a specific APIServer by section name 169func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) { 170 // Add the generic flags. 171 s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic")) 172 s.Etcd.AddFlags(fss.FlagSet("etcd")) 173 s.SecureServing.AddFlags(fss.FlagSet("secure serving")) 174 addDummyInsecureFlags(fss.FlagSet("insecure serving")) 175 s.Audit.AddFlags(fss.FlagSet("auditing")) 176 s.Features.AddFlags(fss.FlagSet("features")) 177 s.Authentication.AddFlags(fss.FlagSet("authentication")) 178 s.Authorization.AddFlags(fss.FlagSet("authorization")) 179 s.CloudProvider.AddFlags(fss.FlagSet("cloud provider")) 180 s.APIEnablement.AddFlags(fss.FlagSet("API enablement")) 181 s.EgressSelector.AddFlags(fss.FlagSet("egress selector")) 182 s.Admission.AddFlags(fss.FlagSet("admission")) 183 s.Metrics.AddFlags(fss.FlagSet("metrics")) 184 s.Logs.AddFlags(fss.FlagSet("logs")) 185 s.Traces.AddFlags(fss.FlagSet("traces")) 186 187 // Note: the weird ""+ in below lines seems to be the only way to get gofmt to 188 // arrange these text blocks sensibly. Grrr. 189 fs := fss.FlagSet("misc") 190 fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL, 191 "Amount of time to retain events.") 192 193 fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged, 194 "If true, allow privileged containers. [default=false]") 195 196 fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler, 197 "If true, install a /logs handler for the apiserver logs.") 198 fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19") 199 200 fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+ 201 "If non-zero, throttle each user connection to this number of bytes/sec. "+ 202 "Currently only applies to long-running requests.") 203 204 fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount, 205 "The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)") 206 207 fs.StringVar(&s.EndpointReconcilerType, "endpoint-reconciler-type", string(s.EndpointReconcilerType), 208 "Use an endpoint reconciler ("+strings.Join(reconcilers.AllTypes.Names(), ", ")+")") 209 210 fs.IntVar(&s.IdentityLeaseDurationSeconds, "identity-lease-duration-seconds", s.IdentityLeaseDurationSeconds, 211 "The duration of kube-apiserver lease in seconds, must be a positive number. (In use when the APIServerIdentity feature gate is enabled.)") 212 213 fs.IntVar(&s.IdentityLeaseRenewIntervalSeconds, "identity-lease-renew-interval-seconds", s.IdentityLeaseRenewIntervalSeconds, 214 "The interval of kube-apiserver renewing its lease in seconds, must be a positive number. (In use when the APIServerIdentity feature gate is enabled.)") 215 216 // See #14282 for details on how to test/try this option out. 217 // TODO: remove this comment once this option is tested in CI. 218 fs.IntVar(&s.KubernetesServiceNodePort, "kubernetes-service-node-port", s.KubernetesServiceNodePort, ""+ 219 "If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be "+ 220 "of type NodePort, using this as the value of the port. If zero, the Kubernetes master "+ 221 "service will be of type ClusterIP.") 222 223 fs.StringVar(&s.ServiceClusterIPRanges, "service-cluster-ip-range", s.ServiceClusterIPRanges, ""+ 224 "A CIDR notation IP range from which to assign service cluster IPs. This must not "+ 225 "overlap with any IP ranges assigned to nodes or pods. Max of two dual-stack CIDRs is allowed.") 226 227 fs.Var(&s.ServiceNodePortRange, "service-node-port-range", ""+ 228 "A port range to reserve for services with NodePort visibility. "+ 229 "Example: '30000-32767'. Inclusive at both ends of the range.") 230 231 // Kubelet related flags: 232 fs.StringSliceVar(&s.KubeletConfig.PreferredAddressTypes, "kubelet-preferred-address-types", s.KubeletConfig.PreferredAddressTypes, 233 "List of the preferred NodeAddressTypes to use for kubelet connections.") 234 235 fs.UintVar(&s.KubeletConfig.Port, "kubelet-port", s.KubeletConfig.Port, 236 "DEPRECATED: kubelet port.") 237 fs.MarkDeprecated("kubelet-port", "kubelet-port is deprecated and will be removed.") 238 239 fs.UintVar(&s.KubeletConfig.ReadOnlyPort, "kubelet-read-only-port", s.KubeletConfig.ReadOnlyPort, 240 "DEPRECATED: kubelet read only port.") 241 fs.MarkDeprecated("kubelet-read-only-port", "kubelet-read-only-port is deprecated and will be removed.") 242 243 fs.DurationVar(&s.KubeletConfig.HTTPTimeout, "kubelet-timeout", s.KubeletConfig.HTTPTimeout, 244 "Timeout for kubelet operations.") 245 246 fs.StringVar(&s.KubeletConfig.CertFile, "kubelet-client-certificate", s.KubeletConfig.CertFile, 247 "Path to a client cert file for TLS.") 248 249 fs.StringVar(&s.KubeletConfig.KeyFile, "kubelet-client-key", s.KubeletConfig.KeyFile, 250 "Path to a client key file for TLS.") 251 252 fs.StringVar(&s.KubeletConfig.CAFile, "kubelet-certificate-authority", s.KubeletConfig.CAFile, 253 "Path to a cert file for the certificate authority.") 254 255 fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+ 256 "Client certificate used to prove the identity of the aggregator or kube-apiserver "+ 257 "when it must call out during a request. This includes proxying requests to a user "+ 258 "api-server and calling out to webhook admission plugins. It is expected that this "+ 259 "cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+ 260 "That CA is published in the 'extension-apiserver-authentication' configmap in "+ 261 "the kube-system namespace. Components receiving calls from kube-aggregator should "+ 262 "use that CA to perform their half of the mutual TLS verification.") 263 fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+ 264 "Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+ 265 "when it must call out during a request. This includes proxying requests to a user "+ 266 "api-server and calling out to webhook admission plugins.") 267 268 fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting, 269 "Turns on aggregator routing requests to endpoints IP rather than cluster IP.") 270 271 fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+ 272 "Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.") 273 274 return fss 275} 276