1/*
2Copyright 2014 The Kubernetes Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8    http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17// Package options contains flags and options for initializing an apiserver
18package options
19
20import (
21	"net"
22	"strings"
23	"time"
24
25	"github.com/spf13/pflag"
26	utilnet "k8s.io/apimachinery/pkg/util/net"
27	genericoptions "k8s.io/apiserver/pkg/server/options"
28	"k8s.io/apiserver/pkg/storage/storagebackend"
29	cliflag "k8s.io/component-base/cli/flag"
30	"k8s.io/component-base/logs"
31	"k8s.io/component-base/metrics"
32
33	api "k8s.io/kubernetes/pkg/apis/core"
34	"k8s.io/kubernetes/pkg/cluster/ports"
35	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
36	_ "k8s.io/kubernetes/pkg/features" // add the kubernetes feature gates
37	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
38	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
39	"k8s.io/kubernetes/pkg/serviceaccount"
40)
41
42// InsecurePortFlags are dummy flags, they are kept only for compatibility and will be removed in v1.24.
43// TODO: remove these flags in v1.24.
44var InsecurePortFlags = []string{"insecure-port", "port"}
45
46// ServerRunOptions runs a kubernetes api server.
47type ServerRunOptions struct {
48	GenericServerRunOptions *genericoptions.ServerRunOptions
49	Etcd                    *genericoptions.EtcdOptions
50	SecureServing           *genericoptions.SecureServingOptionsWithLoopback
51	Audit                   *genericoptions.AuditOptions
52	Features                *genericoptions.FeatureOptions
53	Admission               *kubeoptions.AdmissionOptions
54	Authentication          *kubeoptions.BuiltInAuthenticationOptions
55	Authorization           *kubeoptions.BuiltInAuthorizationOptions
56	CloudProvider           *kubeoptions.CloudProviderOptions
57	APIEnablement           *genericoptions.APIEnablementOptions
58	EgressSelector          *genericoptions.EgressSelectorOptions
59	Metrics                 *metrics.Options
60	Logs                    *logs.Options
61	Traces                  *genericoptions.TracingOptions
62
63	AllowPrivileged           bool
64	EnableLogsHandler         bool
65	EventTTL                  time.Duration
66	KubeletConfig             kubeletclient.KubeletClientConfig
67	KubernetesServiceNodePort int
68	MaxConnectionBytesPerSec  int64
69	// ServiceClusterIPRange is mapped to input provided by user
70	ServiceClusterIPRanges string
71	// PrimaryServiceClusterIPRange and SecondaryServiceClusterIPRange are the results
72	// of parsing ServiceClusterIPRange into actual values
73	PrimaryServiceClusterIPRange   net.IPNet
74	SecondaryServiceClusterIPRange net.IPNet
75	// APIServerServiceIP is the first valid IP from PrimaryServiceClusterIPRange
76	APIServerServiceIP net.IP
77
78	ServiceNodePortRange utilnet.PortRange
79
80	ProxyClientCertFile string
81	ProxyClientKeyFile  string
82
83	EnableAggregatorRouting bool
84
85	MasterCount            int
86	EndpointReconcilerType string
87
88	IdentityLeaseDurationSeconds      int
89	IdentityLeaseRenewIntervalSeconds int
90
91	ServiceAccountSigningKeyFile     string
92	ServiceAccountIssuer             serviceaccount.TokenGenerator
93	ServiceAccountTokenMaxExpiration time.Duration
94
95	ShowHiddenMetricsForVersion string
96}
97
98// NewServerRunOptions creates a new ServerRunOptions object with default parameters
99func NewServerRunOptions() *ServerRunOptions {
100	s := ServerRunOptions{
101		GenericServerRunOptions: genericoptions.NewServerRunOptions(),
102		Etcd:                    genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
103		SecureServing:           kubeoptions.NewSecureServingOptions(),
104		Audit:                   genericoptions.NewAuditOptions(),
105		Features:                genericoptions.NewFeatureOptions(),
106		Admission:               kubeoptions.NewAdmissionOptions(),
107		Authentication:          kubeoptions.NewBuiltInAuthenticationOptions().WithAll(),
108		Authorization:           kubeoptions.NewBuiltInAuthorizationOptions(),
109		CloudProvider:           kubeoptions.NewCloudProviderOptions(),
110		APIEnablement:           genericoptions.NewAPIEnablementOptions(),
111		EgressSelector:          genericoptions.NewEgressSelectorOptions(),
112		Metrics:                 metrics.NewOptions(),
113		Logs:                    logs.NewOptions(),
114		Traces:                  genericoptions.NewTracingOptions(),
115
116		EnableLogsHandler:                 true,
117		EventTTL:                          1 * time.Hour,
118		MasterCount:                       1,
119		EndpointReconcilerType:            string(reconcilers.LeaseEndpointReconcilerType),
120		IdentityLeaseDurationSeconds:      3600,
121		IdentityLeaseRenewIntervalSeconds: 10,
122		KubeletConfig: kubeletclient.KubeletClientConfig{
123			Port:         ports.KubeletPort,
124			ReadOnlyPort: ports.KubeletReadOnlyPort,
125			PreferredAddressTypes: []string{
126				// --override-hostname
127				string(api.NodeHostName),
128
129				// internal, preferring DNS if reported
130				string(api.NodeInternalDNS),
131				string(api.NodeInternalIP),
132
133				// external, preferring DNS if reported
134				string(api.NodeExternalDNS),
135				string(api.NodeExternalIP),
136			},
137			HTTPTimeout: time.Duration(5) * time.Second,
138		},
139		ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
140	}
141
142	// Overwrite the default for storage data format.
143	s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
144
145	return &s
146}
147
148// TODO: remove these insecure flags in v1.24
149func addDummyInsecureFlags(fs *pflag.FlagSet) {
150	var (
151		bindAddr = net.IPv4(127, 0, 0, 1)
152		bindPort int
153	)
154
155	for _, name := range []string{"insecure-bind-address", "address"} {
156		fs.IPVar(&bindAddr, name, bindAddr, ""+
157			"The IP address on which to serve the insecure port (set to 0.0.0.0 or :: for listening in all interfaces and IP families).")
158		fs.MarkDeprecated(name, "This flag has no effect now and will be removed in v1.24.")
159	}
160
161	for _, name := range InsecurePortFlags {
162		fs.IntVar(&bindPort, name, bindPort, ""+
163			"The port on which to serve unsecured, unauthenticated access.")
164		fs.MarkDeprecated(name, "This flag has no effect now and will be removed in v1.24.")
165	}
166}
167
168// Flags returns flags for a specific APIServer by section name
169func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
170	// Add the generic flags.
171	s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic"))
172	s.Etcd.AddFlags(fss.FlagSet("etcd"))
173	s.SecureServing.AddFlags(fss.FlagSet("secure serving"))
174	addDummyInsecureFlags(fss.FlagSet("insecure serving"))
175	s.Audit.AddFlags(fss.FlagSet("auditing"))
176	s.Features.AddFlags(fss.FlagSet("features"))
177	s.Authentication.AddFlags(fss.FlagSet("authentication"))
178	s.Authorization.AddFlags(fss.FlagSet("authorization"))
179	s.CloudProvider.AddFlags(fss.FlagSet("cloud provider"))
180	s.APIEnablement.AddFlags(fss.FlagSet("API enablement"))
181	s.EgressSelector.AddFlags(fss.FlagSet("egress selector"))
182	s.Admission.AddFlags(fss.FlagSet("admission"))
183	s.Metrics.AddFlags(fss.FlagSet("metrics"))
184	s.Logs.AddFlags(fss.FlagSet("logs"))
185	s.Traces.AddFlags(fss.FlagSet("traces"))
186
187	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
188	// arrange these text blocks sensibly. Grrr.
189	fs := fss.FlagSet("misc")
190	fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL,
191		"Amount of time to retain events.")
192
193	fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged,
194		"If true, allow privileged containers. [default=false]")
195
196	fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler,
197		"If true, install a /logs handler for the apiserver logs.")
198	fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19")
199
200	fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+
201		"If non-zero, throttle each user connection to this number of bytes/sec. "+
202		"Currently only applies to long-running requests.")
203
204	fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
205		"The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)")
206
207	fs.StringVar(&s.EndpointReconcilerType, "endpoint-reconciler-type", string(s.EndpointReconcilerType),
208		"Use an endpoint reconciler ("+strings.Join(reconcilers.AllTypes.Names(), ", ")+")")
209
210	fs.IntVar(&s.IdentityLeaseDurationSeconds, "identity-lease-duration-seconds", s.IdentityLeaseDurationSeconds,
211		"The duration of kube-apiserver lease in seconds, must be a positive number. (In use when the APIServerIdentity feature gate is enabled.)")
212
213	fs.IntVar(&s.IdentityLeaseRenewIntervalSeconds, "identity-lease-renew-interval-seconds", s.IdentityLeaseRenewIntervalSeconds,
214		"The interval of kube-apiserver renewing its lease in seconds, must be a positive number. (In use when the APIServerIdentity feature gate is enabled.)")
215
216	// See #14282 for details on how to test/try this option out.
217	// TODO: remove this comment once this option is tested in CI.
218	fs.IntVar(&s.KubernetesServiceNodePort, "kubernetes-service-node-port", s.KubernetesServiceNodePort, ""+
219		"If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be "+
220		"of type NodePort, using this as the value of the port. If zero, the Kubernetes master "+
221		"service will be of type ClusterIP.")
222
223	fs.StringVar(&s.ServiceClusterIPRanges, "service-cluster-ip-range", s.ServiceClusterIPRanges, ""+
224		"A CIDR notation IP range from which to assign service cluster IPs. This must not "+
225		"overlap with any IP ranges assigned to nodes or pods. Max of two dual-stack CIDRs is allowed.")
226
227	fs.Var(&s.ServiceNodePortRange, "service-node-port-range", ""+
228		"A port range to reserve for services with NodePort visibility. "+
229		"Example: '30000-32767'. Inclusive at both ends of the range.")
230
231	// Kubelet related flags:
232	fs.StringSliceVar(&s.KubeletConfig.PreferredAddressTypes, "kubelet-preferred-address-types", s.KubeletConfig.PreferredAddressTypes,
233		"List of the preferred NodeAddressTypes to use for kubelet connections.")
234
235	fs.UintVar(&s.KubeletConfig.Port, "kubelet-port", s.KubeletConfig.Port,
236		"DEPRECATED: kubelet port.")
237	fs.MarkDeprecated("kubelet-port", "kubelet-port is deprecated and will be removed.")
238
239	fs.UintVar(&s.KubeletConfig.ReadOnlyPort, "kubelet-read-only-port", s.KubeletConfig.ReadOnlyPort,
240		"DEPRECATED: kubelet read only port.")
241	fs.MarkDeprecated("kubelet-read-only-port", "kubelet-read-only-port is deprecated and will be removed.")
242
243	fs.DurationVar(&s.KubeletConfig.HTTPTimeout, "kubelet-timeout", s.KubeletConfig.HTTPTimeout,
244		"Timeout for kubelet operations.")
245
246	fs.StringVar(&s.KubeletConfig.CertFile, "kubelet-client-certificate", s.KubeletConfig.CertFile,
247		"Path to a client cert file for TLS.")
248
249	fs.StringVar(&s.KubeletConfig.KeyFile, "kubelet-client-key", s.KubeletConfig.KeyFile,
250		"Path to a client key file for TLS.")
251
252	fs.StringVar(&s.KubeletConfig.CAFile, "kubelet-certificate-authority", s.KubeletConfig.CAFile,
253		"Path to a cert file for the certificate authority.")
254
255	fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+
256		"Client certificate used to prove the identity of the aggregator or kube-apiserver "+
257		"when it must call out during a request. This includes proxying requests to a user "+
258		"api-server and calling out to webhook admission plugins. It is expected that this "+
259		"cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+
260		"That CA is published in the 'extension-apiserver-authentication' configmap in "+
261		"the kube-system namespace. Components receiving calls from kube-aggregator should "+
262		"use that CA to perform their half of the mutual TLS verification.")
263	fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+
264		"Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+
265		"when it must call out during a request. This includes proxying requests to a user "+
266		"api-server and calling out to webhook admission plugins.")
267
268	fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting,
269		"Turns on aggregator routing requests to endpoints IP rather than cluster IP.")
270
271	fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+
272		"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.")
273
274	return fss
275}
276