1/*
2Copyright 2014 The Kubernetes Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8    http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17package options
18
19import (
20	"net"
21	"reflect"
22	"testing"
23	"time"
24
25	"github.com/google/go-cmp/cmp"
26	"github.com/google/go-cmp/cmp/cmpopts"
27	"github.com/spf13/pflag"
28	"k8s.io/apiserver/pkg/admission"
29	apiserveroptions "k8s.io/apiserver/pkg/server/options"
30	"k8s.io/apiserver/pkg/storage/etcd3"
31	"k8s.io/apiserver/pkg/storage/storagebackend"
32	auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
33	audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
34	restclient "k8s.io/client-go/rest"
35	cliflag "k8s.io/component-base/cli/flag"
36	"k8s.io/component-base/logs"
37	"k8s.io/component-base/metrics"
38	kapi "k8s.io/kubernetes/pkg/apis/core"
39	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
40	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
41	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
42)
43
44func TestAddFlags(t *testing.T) {
45	fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError)
46	s := NewServerRunOptions()
47	for _, f := range s.Flags().FlagSets {
48		fs.AddFlagSet(f)
49	}
50
51	args := []string{
52		"--enable-admission-plugins=AlwaysDeny",
53		"--admission-control-config-file=/admission-control-config",
54		"--advertise-address=192.168.10.10",
55		"--allow-privileged=false",
56		"--anonymous-auth=false",
57		"--apiserver-count=5",
58		"--audit-log-maxage=11",
59		"--audit-log-maxbackup=12",
60		"--audit-log-maxsize=13",
61		"--audit-log-path=/var/log",
62		"--audit-log-mode=blocking",
63		"--audit-log-batch-buffer-size=46",
64		"--audit-log-batch-max-size=47",
65		"--audit-log-batch-max-wait=48s",
66		"--audit-log-batch-throttle-enable=true",
67		"--audit-log-batch-throttle-qps=49.5",
68		"--audit-log-batch-throttle-burst=50",
69		"--audit-log-truncate-enabled=true",
70		"--audit-log-truncate-max-batch-size=45",
71		"--audit-log-truncate-max-event-size=44",
72		"--audit-log-version=audit.k8s.io/v1alpha1",
73		"--audit-policy-file=/policy",
74		"--audit-webhook-config-file=/webhook-config",
75		"--audit-webhook-mode=blocking",
76		"--audit-webhook-batch-buffer-size=42",
77		"--audit-webhook-batch-max-size=43",
78		"--audit-webhook-batch-max-wait=1s",
79		"--audit-webhook-batch-throttle-enable=false",
80		"--audit-webhook-batch-throttle-qps=43.5",
81		"--audit-webhook-batch-throttle-burst=44",
82		"--audit-webhook-truncate-enabled=true",
83		"--audit-webhook-truncate-max-batch-size=43",
84		"--audit-webhook-truncate-max-event-size=42",
85		"--audit-webhook-initial-backoff=2s",
86		"--audit-webhook-version=audit.k8s.io/v1alpha1",
87		"--authentication-token-webhook-cache-ttl=3m",
88		"--authentication-token-webhook-config-file=/token-webhook-config",
89		"--authorization-mode=AlwaysDeny,RBAC",
90		"--authorization-policy-file=/policy",
91		"--authorization-webhook-cache-authorized-ttl=3m",
92		"--authorization-webhook-cache-unauthorized-ttl=1m",
93		"--authorization-webhook-config-file=/webhook-config",
94		"--bind-address=192.168.10.20",
95		"--client-ca-file=/client-ca",
96		"--cloud-config=/cloud-config",
97		"--cloud-provider=azure",
98		"--cors-allowed-origins=10.10.10.100,10.10.10.200",
99		"--contention-profiling=true",
100		"--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml",
101		"--enable-aggregator-routing=true",
102		"--enable-priority-and-fairness=false",
103		"--enable-logs-handler=false",
104		"--endpoint-reconciler-type=" + string(reconcilers.LeaseEndpointReconcilerType),
105		"--etcd-keyfile=/var/run/kubernetes/etcd.key",
106		"--etcd-certfile=/var/run/kubernetes/etcdce.crt",
107		"--etcd-cafile=/var/run/kubernetes/etcdca.crt",
108		"--http2-max-streams-per-connection=42",
109		"--kubelet-read-only-port=10255",
110		"--kubelet-timeout=5s",
111		"--kubelet-client-certificate=/var/run/kubernetes/ceserver.crt",
112		"--kubelet-client-key=/var/run/kubernetes/server.key",
113		"--kubelet-certificate-authority=/var/run/kubernetes/caserver.crt",
114		"--tracing-config-file=/var/run/kubernetes/tracing_config.yaml",
115		"--proxy-client-cert-file=/var/run/kubernetes/proxy.crt",
116		"--proxy-client-key-file=/var/run/kubernetes/proxy.key",
117		"--request-timeout=2m",
118		"--storage-backend=etcd3",
119		"--service-cluster-ip-range=192.168.128.0/17",
120		"--lease-reuse-duration-seconds=100",
121	}
122	fs.Parse(args)
123
124	// This is a snapshot of expected options parsed by args.
125	expected := &ServerRunOptions{
126		ServiceNodePortRange:   kubeoptions.DefaultServiceNodePortRange,
127		ServiceClusterIPRanges: (&net.IPNet{IP: net.ParseIP("192.168.128.0"), Mask: net.CIDRMask(17, 32)}).String(),
128		MasterCount:            5,
129		EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
130		AllowPrivileged:        false,
131		GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
132			AdvertiseAddress:            net.ParseIP("192.168.10.10"),
133			CorsAllowedOriginList:       []string{"10.10.10.100", "10.10.10.200"},
134			MaxRequestsInFlight:         400,
135			MaxMutatingRequestsInFlight: 200,
136			RequestTimeout:              time.Duration(2) * time.Minute,
137			MinRequestTimeout:           1800,
138			JSONPatchMaxCopyBytes:       int64(3 * 1024 * 1024),
139			MaxRequestBodyBytes:         int64(3 * 1024 * 1024),
140		},
141		Admission: &kubeoptions.AdmissionOptions{
142			GenericAdmission: &apiserveroptions.AdmissionOptions{
143				RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder,
144				DefaultOffPlugins:      s.Admission.GenericAdmission.DefaultOffPlugins,
145				EnablePlugins:          []string{"AlwaysDeny"},
146				ConfigFile:             "/admission-control-config",
147				Plugins:                s.Admission.GenericAdmission.Plugins,
148				Decorators:             s.Admission.GenericAdmission.Decorators,
149			},
150		},
151		Etcd: &apiserveroptions.EtcdOptions{
152			StorageConfig: storagebackend.Config{
153				Type: "etcd3",
154				Transport: storagebackend.TransportConfig{
155					ServerList:    nil,
156					KeyFile:       "/var/run/kubernetes/etcd.key",
157					TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
158					CertFile:      "/var/run/kubernetes/etcdce.crt",
159				},
160				Paging:                true,
161				Prefix:                "/registry",
162				CompactionInterval:    storagebackend.DefaultCompactInterval,
163				CountMetricPollPeriod: time.Minute,
164				DBMetricPollInterval:  storagebackend.DefaultDBMetricPollInterval,
165				HealthcheckTimeout:    storagebackend.DefaultHealthcheckTimeout,
166				LeaseManagerConfig: etcd3.LeaseManagerConfig{
167					ReuseDurationSeconds: 100,
168					MaxObjectCount:       1000,
169				},
170			},
171			DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
172			DeleteCollectionWorkers: 1,
173			EnableGarbageCollection: true,
174			EnableWatchCache:        true,
175			DefaultWatchCacheSize:   100,
176		},
177		SecureServing: (&apiserveroptions.SecureServingOptions{
178			BindAddress: net.ParseIP("192.168.10.20"),
179			BindPort:    6443,
180			ServerCert: apiserveroptions.GeneratableKeyCert{
181				CertDirectory: "/var/run/kubernetes",
182				PairName:      "apiserver",
183			},
184			HTTP2MaxStreamsPerConnection: 42,
185			Required:                     true,
186		}).WithLoopback(),
187		EventTTL: 1 * time.Hour,
188		KubeletConfig: kubeletclient.KubeletClientConfig{
189			Port:         10250,
190			ReadOnlyPort: 10255,
191			PreferredAddressTypes: []string{
192				string(kapi.NodeHostName),
193				string(kapi.NodeInternalDNS),
194				string(kapi.NodeInternalIP),
195				string(kapi.NodeExternalDNS),
196				string(kapi.NodeExternalIP),
197			},
198			HTTPTimeout: time.Duration(5) * time.Second,
199			TLSClientConfig: restclient.TLSClientConfig{
200				CertFile: "/var/run/kubernetes/ceserver.crt",
201				KeyFile:  "/var/run/kubernetes/server.key",
202				CAFile:   "/var/run/kubernetes/caserver.crt",
203			},
204		},
205		Audit: &apiserveroptions.AuditOptions{
206			LogOptions: apiserveroptions.AuditLogOptions{
207				Path:       "/var/log",
208				MaxAge:     11,
209				MaxBackups: 12,
210				MaxSize:    13,
211				Format:     "json",
212				BatchOptions: apiserveroptions.AuditBatchOptions{
213					Mode: "blocking",
214					BatchConfig: auditbuffered.BatchConfig{
215						BufferSize:     46,
216						MaxBatchSize:   47,
217						MaxBatchWait:   48 * time.Second,
218						ThrottleEnable: true,
219						ThrottleQPS:    49.5,
220						ThrottleBurst:  50,
221					},
222				},
223				TruncateOptions: apiserveroptions.AuditTruncateOptions{
224					Enabled: true,
225					TruncateConfig: audittruncate.Config{
226						MaxBatchSize: 45,
227						MaxEventSize: 44,
228					},
229				},
230				GroupVersionString: "audit.k8s.io/v1alpha1",
231			},
232			WebhookOptions: apiserveroptions.AuditWebhookOptions{
233				ConfigFile: "/webhook-config",
234				BatchOptions: apiserveroptions.AuditBatchOptions{
235					Mode: "blocking",
236					BatchConfig: auditbuffered.BatchConfig{
237						BufferSize:     42,
238						MaxBatchSize:   43,
239						MaxBatchWait:   1 * time.Second,
240						ThrottleEnable: false,
241						ThrottleQPS:    43.5,
242						ThrottleBurst:  44,
243						AsyncDelegate:  true,
244					},
245				},
246				TruncateOptions: apiserveroptions.AuditTruncateOptions{
247					Enabled: true,
248					TruncateConfig: audittruncate.Config{
249						MaxBatchSize: 43,
250						MaxEventSize: 42,
251					},
252				},
253				InitialBackoff:     2 * time.Second,
254				GroupVersionString: "audit.k8s.io/v1alpha1",
255			},
256			PolicyFile: "/policy",
257		},
258		Features: &apiserveroptions.FeatureOptions{
259			EnableProfiling:           true,
260			EnableContentionProfiling: true,
261		},
262		Authentication: &kubeoptions.BuiltInAuthenticationOptions{
263			Anonymous: &kubeoptions.AnonymousAuthenticationOptions{
264				Allow: false,
265			},
266			ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
267				ClientCA: "/client-ca",
268			},
269			WebHook: &kubeoptions.WebHookAuthenticationOptions{
270				CacheTTL:     180000000000,
271				ConfigFile:   "/token-webhook-config",
272				Version:      "v1beta1",
273				RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
274			},
275			BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{},
276			OIDC: &kubeoptions.OIDCAuthenticationOptions{
277				UsernameClaim: "sub",
278				SigningAlgs:   []string{"RS256"},
279			},
280			RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{},
281			ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
282				Lookup:           true,
283				ExtendExpiration: true,
284			},
285			TokenFile:            &kubeoptions.TokenFileAuthenticationOptions{},
286			TokenSuccessCacheTTL: 10 * time.Second,
287			TokenFailureCacheTTL: 0,
288		},
289		Authorization: &kubeoptions.BuiltInAuthorizationOptions{
290			Modes:                       []string{"AlwaysDeny", "RBAC"},
291			PolicyFile:                  "/policy",
292			WebhookConfigFile:           "/webhook-config",
293			WebhookCacheAuthorizedTTL:   180000000000,
294			WebhookCacheUnauthorizedTTL: 60000000000,
295			WebhookVersion:              "v1beta1",
296			WebhookRetryBackoff:         apiserveroptions.DefaultAuthWebhookRetryBackoff(),
297		},
298		CloudProvider: &kubeoptions.CloudProviderOptions{
299			CloudConfigFile: "/cloud-config",
300			CloudProvider:   "azure",
301		},
302		APIEnablement: &apiserveroptions.APIEnablementOptions{
303			RuntimeConfig: cliflag.ConfigurationMap{},
304		},
305		EgressSelector: &apiserveroptions.EgressSelectorOptions{
306			ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml",
307		},
308		EnableLogsHandler:       false,
309		EnableAggregatorRouting: true,
310		ProxyClientKeyFile:      "/var/run/kubernetes/proxy.key",
311		ProxyClientCertFile:     "/var/run/kubernetes/proxy.crt",
312		Metrics:                 &metrics.Options{},
313		Logs:                    logs.NewOptions(),
314		Traces: &apiserveroptions.TracingOptions{
315			ConfigFile: "/var/run/kubernetes/tracing_config.yaml",
316		},
317		IdentityLeaseDurationSeconds:      3600,
318		IdentityLeaseRenewIntervalSeconds: 10,
319	}
320
321	if !reflect.DeepEqual(expected, s) {
322		t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{})))
323	}
324}
325