1/* 2Copyright 2014 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package options 18 19import ( 20 "net" 21 "reflect" 22 "testing" 23 "time" 24 25 "github.com/google/go-cmp/cmp" 26 "github.com/google/go-cmp/cmp/cmpopts" 27 "github.com/spf13/pflag" 28 "k8s.io/apiserver/pkg/admission" 29 apiserveroptions "k8s.io/apiserver/pkg/server/options" 30 "k8s.io/apiserver/pkg/storage/etcd3" 31 "k8s.io/apiserver/pkg/storage/storagebackend" 32 auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" 33 audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" 34 restclient "k8s.io/client-go/rest" 35 cliflag "k8s.io/component-base/cli/flag" 36 "k8s.io/component-base/logs" 37 "k8s.io/component-base/metrics" 38 kapi "k8s.io/kubernetes/pkg/apis/core" 39 "k8s.io/kubernetes/pkg/controlplane/reconcilers" 40 kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" 41 kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" 42) 43 44func TestAddFlags(t *testing.T) { 45 fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError) 46 s := NewServerRunOptions() 47 for _, f := range s.Flags().FlagSets { 48 fs.AddFlagSet(f) 49 } 50 51 args := []string{ 52 "--enable-admission-plugins=AlwaysDeny", 53 "--admission-control-config-file=/admission-control-config", 54 "--advertise-address=192.168.10.10", 55 "--allow-privileged=false", 56 "--anonymous-auth=false", 57 "--apiserver-count=5", 58 "--audit-log-maxage=11", 59 "--audit-log-maxbackup=12", 60 "--audit-log-maxsize=13", 61 "--audit-log-path=/var/log", 62 "--audit-log-mode=blocking", 63 "--audit-log-batch-buffer-size=46", 64 "--audit-log-batch-max-size=47", 65 "--audit-log-batch-max-wait=48s", 66 "--audit-log-batch-throttle-enable=true", 67 "--audit-log-batch-throttle-qps=49.5", 68 "--audit-log-batch-throttle-burst=50", 69 "--audit-log-truncate-enabled=true", 70 "--audit-log-truncate-max-batch-size=45", 71 "--audit-log-truncate-max-event-size=44", 72 "--audit-log-version=audit.k8s.io/v1alpha1", 73 "--audit-policy-file=/policy", 74 "--audit-webhook-config-file=/webhook-config", 75 "--audit-webhook-mode=blocking", 76 "--audit-webhook-batch-buffer-size=42", 77 "--audit-webhook-batch-max-size=43", 78 "--audit-webhook-batch-max-wait=1s", 79 "--audit-webhook-batch-throttle-enable=false", 80 "--audit-webhook-batch-throttle-qps=43.5", 81 "--audit-webhook-batch-throttle-burst=44", 82 "--audit-webhook-truncate-enabled=true", 83 "--audit-webhook-truncate-max-batch-size=43", 84 "--audit-webhook-truncate-max-event-size=42", 85 "--audit-webhook-initial-backoff=2s", 86 "--audit-webhook-version=audit.k8s.io/v1alpha1", 87 "--authentication-token-webhook-cache-ttl=3m", 88 "--authentication-token-webhook-config-file=/token-webhook-config", 89 "--authorization-mode=AlwaysDeny,RBAC", 90 "--authorization-policy-file=/policy", 91 "--authorization-webhook-cache-authorized-ttl=3m", 92 "--authorization-webhook-cache-unauthorized-ttl=1m", 93 "--authorization-webhook-config-file=/webhook-config", 94 "--bind-address=192.168.10.20", 95 "--client-ca-file=/client-ca", 96 "--cloud-config=/cloud-config", 97 "--cloud-provider=azure", 98 "--cors-allowed-origins=10.10.10.100,10.10.10.200", 99 "--contention-profiling=true", 100 "--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml", 101 "--enable-aggregator-routing=true", 102 "--enable-priority-and-fairness=false", 103 "--enable-logs-handler=false", 104 "--endpoint-reconciler-type=" + string(reconcilers.LeaseEndpointReconcilerType), 105 "--etcd-keyfile=/var/run/kubernetes/etcd.key", 106 "--etcd-certfile=/var/run/kubernetes/etcdce.crt", 107 "--etcd-cafile=/var/run/kubernetes/etcdca.crt", 108 "--http2-max-streams-per-connection=42", 109 "--kubelet-read-only-port=10255", 110 "--kubelet-timeout=5s", 111 "--kubelet-client-certificate=/var/run/kubernetes/ceserver.crt", 112 "--kubelet-client-key=/var/run/kubernetes/server.key", 113 "--kubelet-certificate-authority=/var/run/kubernetes/caserver.crt", 114 "--tracing-config-file=/var/run/kubernetes/tracing_config.yaml", 115 "--proxy-client-cert-file=/var/run/kubernetes/proxy.crt", 116 "--proxy-client-key-file=/var/run/kubernetes/proxy.key", 117 "--request-timeout=2m", 118 "--storage-backend=etcd3", 119 "--service-cluster-ip-range=192.168.128.0/17", 120 "--lease-reuse-duration-seconds=100", 121 } 122 fs.Parse(args) 123 124 // This is a snapshot of expected options parsed by args. 125 expected := &ServerRunOptions{ 126 ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange, 127 ServiceClusterIPRanges: (&net.IPNet{IP: net.ParseIP("192.168.128.0"), Mask: net.CIDRMask(17, 32)}).String(), 128 MasterCount: 5, 129 EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType), 130 AllowPrivileged: false, 131 GenericServerRunOptions: &apiserveroptions.ServerRunOptions{ 132 AdvertiseAddress: net.ParseIP("192.168.10.10"), 133 CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"}, 134 MaxRequestsInFlight: 400, 135 MaxMutatingRequestsInFlight: 200, 136 RequestTimeout: time.Duration(2) * time.Minute, 137 MinRequestTimeout: 1800, 138 JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), 139 MaxRequestBodyBytes: int64(3 * 1024 * 1024), 140 }, 141 Admission: &kubeoptions.AdmissionOptions{ 142 GenericAdmission: &apiserveroptions.AdmissionOptions{ 143 RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder, 144 DefaultOffPlugins: s.Admission.GenericAdmission.DefaultOffPlugins, 145 EnablePlugins: []string{"AlwaysDeny"}, 146 ConfigFile: "/admission-control-config", 147 Plugins: s.Admission.GenericAdmission.Plugins, 148 Decorators: s.Admission.GenericAdmission.Decorators, 149 }, 150 }, 151 Etcd: &apiserveroptions.EtcdOptions{ 152 StorageConfig: storagebackend.Config{ 153 Type: "etcd3", 154 Transport: storagebackend.TransportConfig{ 155 ServerList: nil, 156 KeyFile: "/var/run/kubernetes/etcd.key", 157 TrustedCAFile: "/var/run/kubernetes/etcdca.crt", 158 CertFile: "/var/run/kubernetes/etcdce.crt", 159 }, 160 Paging: true, 161 Prefix: "/registry", 162 CompactionInterval: storagebackend.DefaultCompactInterval, 163 CountMetricPollPeriod: time.Minute, 164 DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval, 165 HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout, 166 LeaseManagerConfig: etcd3.LeaseManagerConfig{ 167 ReuseDurationSeconds: 100, 168 MaxObjectCount: 1000, 169 }, 170 }, 171 DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", 172 DeleteCollectionWorkers: 1, 173 EnableGarbageCollection: true, 174 EnableWatchCache: true, 175 DefaultWatchCacheSize: 100, 176 }, 177 SecureServing: (&apiserveroptions.SecureServingOptions{ 178 BindAddress: net.ParseIP("192.168.10.20"), 179 BindPort: 6443, 180 ServerCert: apiserveroptions.GeneratableKeyCert{ 181 CertDirectory: "/var/run/kubernetes", 182 PairName: "apiserver", 183 }, 184 HTTP2MaxStreamsPerConnection: 42, 185 Required: true, 186 }).WithLoopback(), 187 EventTTL: 1 * time.Hour, 188 KubeletConfig: kubeletclient.KubeletClientConfig{ 189 Port: 10250, 190 ReadOnlyPort: 10255, 191 PreferredAddressTypes: []string{ 192 string(kapi.NodeHostName), 193 string(kapi.NodeInternalDNS), 194 string(kapi.NodeInternalIP), 195 string(kapi.NodeExternalDNS), 196 string(kapi.NodeExternalIP), 197 }, 198 HTTPTimeout: time.Duration(5) * time.Second, 199 TLSClientConfig: restclient.TLSClientConfig{ 200 CertFile: "/var/run/kubernetes/ceserver.crt", 201 KeyFile: "/var/run/kubernetes/server.key", 202 CAFile: "/var/run/kubernetes/caserver.crt", 203 }, 204 }, 205 Audit: &apiserveroptions.AuditOptions{ 206 LogOptions: apiserveroptions.AuditLogOptions{ 207 Path: "/var/log", 208 MaxAge: 11, 209 MaxBackups: 12, 210 MaxSize: 13, 211 Format: "json", 212 BatchOptions: apiserveroptions.AuditBatchOptions{ 213 Mode: "blocking", 214 BatchConfig: auditbuffered.BatchConfig{ 215 BufferSize: 46, 216 MaxBatchSize: 47, 217 MaxBatchWait: 48 * time.Second, 218 ThrottleEnable: true, 219 ThrottleQPS: 49.5, 220 ThrottleBurst: 50, 221 }, 222 }, 223 TruncateOptions: apiserveroptions.AuditTruncateOptions{ 224 Enabled: true, 225 TruncateConfig: audittruncate.Config{ 226 MaxBatchSize: 45, 227 MaxEventSize: 44, 228 }, 229 }, 230 GroupVersionString: "audit.k8s.io/v1alpha1", 231 }, 232 WebhookOptions: apiserveroptions.AuditWebhookOptions{ 233 ConfigFile: "/webhook-config", 234 BatchOptions: apiserveroptions.AuditBatchOptions{ 235 Mode: "blocking", 236 BatchConfig: auditbuffered.BatchConfig{ 237 BufferSize: 42, 238 MaxBatchSize: 43, 239 MaxBatchWait: 1 * time.Second, 240 ThrottleEnable: false, 241 ThrottleQPS: 43.5, 242 ThrottleBurst: 44, 243 AsyncDelegate: true, 244 }, 245 }, 246 TruncateOptions: apiserveroptions.AuditTruncateOptions{ 247 Enabled: true, 248 TruncateConfig: audittruncate.Config{ 249 MaxBatchSize: 43, 250 MaxEventSize: 42, 251 }, 252 }, 253 InitialBackoff: 2 * time.Second, 254 GroupVersionString: "audit.k8s.io/v1alpha1", 255 }, 256 PolicyFile: "/policy", 257 }, 258 Features: &apiserveroptions.FeatureOptions{ 259 EnableProfiling: true, 260 EnableContentionProfiling: true, 261 }, 262 Authentication: &kubeoptions.BuiltInAuthenticationOptions{ 263 Anonymous: &kubeoptions.AnonymousAuthenticationOptions{ 264 Allow: false, 265 }, 266 ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{ 267 ClientCA: "/client-ca", 268 }, 269 WebHook: &kubeoptions.WebHookAuthenticationOptions{ 270 CacheTTL: 180000000000, 271 ConfigFile: "/token-webhook-config", 272 Version: "v1beta1", 273 RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), 274 }, 275 BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{}, 276 OIDC: &kubeoptions.OIDCAuthenticationOptions{ 277 UsernameClaim: "sub", 278 SigningAlgs: []string{"RS256"}, 279 }, 280 RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{}, 281 ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{ 282 Lookup: true, 283 ExtendExpiration: true, 284 }, 285 TokenFile: &kubeoptions.TokenFileAuthenticationOptions{}, 286 TokenSuccessCacheTTL: 10 * time.Second, 287 TokenFailureCacheTTL: 0, 288 }, 289 Authorization: &kubeoptions.BuiltInAuthorizationOptions{ 290 Modes: []string{"AlwaysDeny", "RBAC"}, 291 PolicyFile: "/policy", 292 WebhookConfigFile: "/webhook-config", 293 WebhookCacheAuthorizedTTL: 180000000000, 294 WebhookCacheUnauthorizedTTL: 60000000000, 295 WebhookVersion: "v1beta1", 296 WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), 297 }, 298 CloudProvider: &kubeoptions.CloudProviderOptions{ 299 CloudConfigFile: "/cloud-config", 300 CloudProvider: "azure", 301 }, 302 APIEnablement: &apiserveroptions.APIEnablementOptions{ 303 RuntimeConfig: cliflag.ConfigurationMap{}, 304 }, 305 EgressSelector: &apiserveroptions.EgressSelectorOptions{ 306 ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml", 307 }, 308 EnableLogsHandler: false, 309 EnableAggregatorRouting: true, 310 ProxyClientKeyFile: "/var/run/kubernetes/proxy.key", 311 ProxyClientCertFile: "/var/run/kubernetes/proxy.crt", 312 Metrics: &metrics.Options{}, 313 Logs: logs.NewOptions(), 314 Traces: &apiserveroptions.TracingOptions{ 315 ConfigFile: "/var/run/kubernetes/tracing_config.yaml", 316 }, 317 IdentityLeaseDurationSeconds: 3600, 318 IdentityLeaseRenewIntervalSeconds: 10, 319 } 320 321 if !reflect.DeepEqual(expected, s) { 322 t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}))) 323 } 324} 325