1/* 2Copyright 2016 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package bootstrap 18 19import ( 20 "context" 21 "crypto" 22 "crypto/sha512" 23 "crypto/x509" 24 "crypto/x509/pkix" 25 "encoding/base64" 26 "errors" 27 "fmt" 28 "os" 29 "path/filepath" 30 "time" 31 32 "k8s.io/klog/v2" 33 34 certificatesv1 "k8s.io/api/certificates/v1" 35 "k8s.io/apimachinery/pkg/types" 36 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 37 "k8s.io/apimachinery/pkg/util/wait" 38 clientset "k8s.io/client-go/kubernetes" 39 "k8s.io/client-go/kubernetes/scheme" 40 restclient "k8s.io/client-go/rest" 41 "k8s.io/client-go/tools/clientcmd" 42 clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 43 "k8s.io/client-go/transport" 44 certutil "k8s.io/client-go/util/cert" 45 "k8s.io/client-go/util/certificate" 46 "k8s.io/client-go/util/certificate/csr" 47 "k8s.io/client-go/util/keyutil" 48) 49 50const tmpPrivateKeyFile = "kubelet-client.key.tmp" 51 52// LoadClientConfig tries to load the appropriate client config for retrieving certs and for use by users. 53// If bootstrapPath is empty, only kubeconfigPath is checked. If bootstrap path is set and the contents 54// of kubeconfigPath are valid, both certConfig and userConfig will point to that file. Otherwise the 55// kubeconfigPath on disk is populated based on bootstrapPath but pointing to the location of the client cert 56// in certDir. This preserves the historical behavior of bootstrapping where on subsequent restarts the 57// most recent client cert is used to request new client certs instead of the initial token. 58func LoadClientConfig(kubeconfigPath, bootstrapPath, certDir string) (certConfig, userConfig *restclient.Config, err error) { 59 if len(bootstrapPath) == 0 { 60 clientConfig, err := loadRESTClientConfig(kubeconfigPath) 61 if err != nil { 62 return nil, nil, fmt.Errorf("unable to load kubeconfig: %v", err) 63 } 64 klog.V(2).InfoS("No bootstrapping requested, will use kubeconfig") 65 return clientConfig, restclient.CopyConfig(clientConfig), nil 66 } 67 68 store, err := certificate.NewFileStore("kubelet-client", certDir, certDir, "", "") 69 if err != nil { 70 return nil, nil, fmt.Errorf("unable to build bootstrap cert store") 71 } 72 73 ok, err := isClientConfigStillValid(kubeconfigPath) 74 if err != nil { 75 return nil, nil, err 76 } 77 78 // use the current client config 79 if ok { 80 clientConfig, err := loadRESTClientConfig(kubeconfigPath) 81 if err != nil { 82 return nil, nil, fmt.Errorf("unable to load kubeconfig: %v", err) 83 } 84 klog.V(2).InfoS("Current kubeconfig file contents are still valid, no bootstrap necessary") 85 return clientConfig, restclient.CopyConfig(clientConfig), nil 86 } 87 88 bootstrapClientConfig, err := loadRESTClientConfig(bootstrapPath) 89 if err != nil { 90 return nil, nil, fmt.Errorf("unable to load bootstrap kubeconfig: %v", err) 91 } 92 93 clientConfig := restclient.AnonymousClientConfig(bootstrapClientConfig) 94 pemPath := store.CurrentPath() 95 clientConfig.KeyFile = pemPath 96 clientConfig.CertFile = pemPath 97 if err := writeKubeconfigFromBootstrapping(clientConfig, kubeconfigPath, pemPath); err != nil { 98 return nil, nil, err 99 } 100 klog.V(2).InfoS("Use the bootstrap credentials to request a cert, and set kubeconfig to point to the certificate dir") 101 return bootstrapClientConfig, clientConfig, nil 102} 103 104// LoadClientCert requests a client cert for kubelet if the kubeconfigPath file does not exist. 105// The kubeconfig at bootstrapPath is used to request a client certificate from the API server. 106// On success, a kubeconfig file referencing the generated key and obtained certificate is written to kubeconfigPath. 107// The certificate and key file are stored in certDir. 108func LoadClientCert(ctx context.Context, kubeconfigPath, bootstrapPath, certDir string, nodeName types.NodeName) error { 109 // Short-circuit if the kubeconfig file exists and is valid. 110 ok, err := isClientConfigStillValid(kubeconfigPath) 111 if err != nil { 112 return err 113 } 114 if ok { 115 klog.V(2).InfoS("Kubeconfig exists and is valid, skipping bootstrap", "path", kubeconfigPath) 116 return nil 117 } 118 119 klog.V(2).InfoS("Using bootstrap kubeconfig to generate TLS client cert, key and kubeconfig file") 120 121 bootstrapClientConfig, err := loadRESTClientConfig(bootstrapPath) 122 if err != nil { 123 return fmt.Errorf("unable to load bootstrap kubeconfig: %v", err) 124 } 125 126 bootstrapClient, err := clientset.NewForConfig(bootstrapClientConfig) 127 if err != nil { 128 return fmt.Errorf("unable to create certificates signing request client: %v", err) 129 } 130 131 store, err := certificate.NewFileStore("kubelet-client", certDir, certDir, "", "") 132 if err != nil { 133 return fmt.Errorf("unable to build bootstrap cert store") 134 } 135 136 var keyData []byte 137 if cert, err := store.Current(); err == nil { 138 if cert.PrivateKey != nil { 139 keyData, err = keyutil.MarshalPrivateKeyToPEM(cert.PrivateKey) 140 if err != nil { 141 keyData = nil 142 } 143 } 144 } 145 // Cache the private key in a separate file until CSR succeeds. This has to 146 // be a separate file because store.CurrentPath() points to a symlink 147 // managed by the store. 148 privKeyPath := filepath.Join(certDir, tmpPrivateKeyFile) 149 if !verifyKeyData(keyData) { 150 klog.V(2).InfoS("No valid private key and/or certificate found, reusing existing private key or creating a new one") 151 // Note: always call LoadOrGenerateKeyFile so that private key is 152 // reused on next startup if CSR request fails. 153 keyData, _, err = keyutil.LoadOrGenerateKeyFile(privKeyPath) 154 if err != nil { 155 return err 156 } 157 } 158 159 if err := waitForServer(ctx, *bootstrapClientConfig, 1*time.Minute); err != nil { 160 klog.InfoS("Error waiting for apiserver to come up", "err", err) 161 } 162 163 certData, err := requestNodeCertificate(ctx, bootstrapClient, keyData, nodeName) 164 if err != nil { 165 return err 166 } 167 if _, err := store.Update(certData, keyData); err != nil { 168 return err 169 } 170 if err := os.Remove(privKeyPath); err != nil && !os.IsNotExist(err) { 171 klog.V(2).InfoS("Failed cleaning up private key file", "path", privKeyPath, "err", err) 172 } 173 174 return writeKubeconfigFromBootstrapping(bootstrapClientConfig, kubeconfigPath, store.CurrentPath()) 175} 176 177func writeKubeconfigFromBootstrapping(bootstrapClientConfig *restclient.Config, kubeconfigPath, pemPath string) error { 178 // Get the CA data from the bootstrap client config. 179 caFile, caData := bootstrapClientConfig.CAFile, []byte{} 180 if len(caFile) == 0 { 181 caData = bootstrapClientConfig.CAData 182 } 183 184 // Build resulting kubeconfig. 185 kubeconfigData := clientcmdapi.Config{ 186 // Define a cluster stanza based on the bootstrap kubeconfig. 187 Clusters: map[string]*clientcmdapi.Cluster{"default-cluster": { 188 Server: bootstrapClientConfig.Host, 189 InsecureSkipTLSVerify: bootstrapClientConfig.Insecure, 190 CertificateAuthority: caFile, 191 CertificateAuthorityData: caData, 192 }}, 193 // Define auth based on the obtained client cert. 194 AuthInfos: map[string]*clientcmdapi.AuthInfo{"default-auth": { 195 ClientCertificate: pemPath, 196 ClientKey: pemPath, 197 }}, 198 // Define a context that connects the auth info and cluster, and set it as the default 199 Contexts: map[string]*clientcmdapi.Context{"default-context": { 200 Cluster: "default-cluster", 201 AuthInfo: "default-auth", 202 Namespace: "default", 203 }}, 204 CurrentContext: "default-context", 205 } 206 207 // Marshal to disk 208 return clientcmd.WriteToFile(kubeconfigData, kubeconfigPath) 209} 210 211func loadRESTClientConfig(kubeconfig string) (*restclient.Config, error) { 212 // Load structured kubeconfig data from the given path. 213 loader := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig} 214 loadedConfig, err := loader.Load() 215 if err != nil { 216 return nil, err 217 } 218 // Flatten the loaded data to a particular restclient.Config based on the current context. 219 return clientcmd.NewNonInteractiveClientConfig( 220 *loadedConfig, 221 loadedConfig.CurrentContext, 222 &clientcmd.ConfigOverrides{}, 223 loader, 224 ).ClientConfig() 225} 226 227// isClientConfigStillValid checks the provided kubeconfig to see if it has a valid 228// client certificate. It returns true if the kubeconfig is valid, or an error if bootstrapping 229// should stop immediately. 230func isClientConfigStillValid(kubeconfigPath string) (bool, error) { 231 _, err := os.Stat(kubeconfigPath) 232 if os.IsNotExist(err) { 233 return false, nil 234 } 235 if err != nil { 236 return false, fmt.Errorf("error reading existing bootstrap kubeconfig %s: %v", kubeconfigPath, err) 237 } 238 bootstrapClientConfig, err := loadRESTClientConfig(kubeconfigPath) 239 if err != nil { 240 utilruntime.HandleError(fmt.Errorf("unable to read existing bootstrap client config from %s: %v", kubeconfigPath, err)) 241 return false, nil 242 } 243 transportConfig, err := bootstrapClientConfig.TransportConfig() 244 if err != nil { 245 utilruntime.HandleError(fmt.Errorf("unable to load transport configuration from existing bootstrap client config read from %s: %v", kubeconfigPath, err)) 246 return false, nil 247 } 248 // has side effect of populating transport config data fields 249 if _, err := transport.TLSConfigFor(transportConfig); err != nil { 250 utilruntime.HandleError(fmt.Errorf("unable to load TLS configuration from existing bootstrap client config read from %s: %v", kubeconfigPath, err)) 251 return false, nil 252 } 253 certs, err := certutil.ParseCertsPEM(transportConfig.TLS.CertData) 254 if err != nil { 255 utilruntime.HandleError(fmt.Errorf("unable to load TLS certificates from existing bootstrap client config read from %s: %v", kubeconfigPath, err)) 256 return false, nil 257 } 258 if len(certs) == 0 { 259 utilruntime.HandleError(fmt.Errorf("unable to read TLS certificates from existing bootstrap client config read from %s: %v", kubeconfigPath, err)) 260 return false, nil 261 } 262 now := time.Now() 263 for _, cert := range certs { 264 if now.After(cert.NotAfter) { 265 utilruntime.HandleError(fmt.Errorf("part of the existing bootstrap client certificate in %s is expired: %v", kubeconfigPath, cert.NotAfter)) 266 return false, nil 267 } 268 } 269 return true, nil 270} 271 272// verifyKeyData returns true if the provided data appears to be a valid private key. 273func verifyKeyData(data []byte) bool { 274 if len(data) == 0 { 275 return false 276 } 277 _, err := keyutil.ParsePrivateKeyPEM(data) 278 return err == nil 279} 280 281func waitForServer(ctx context.Context, cfg restclient.Config, deadline time.Duration) error { 282 cfg.NegotiatedSerializer = scheme.Codecs.WithoutConversion() 283 cfg.Timeout = 1 * time.Second 284 cli, err := restclient.UnversionedRESTClientFor(&cfg) 285 if err != nil { 286 return fmt.Errorf("couldn't create client: %v", err) 287 } 288 289 ctx, cancel := context.WithTimeout(ctx, deadline) 290 defer cancel() 291 292 var connected bool 293 wait.JitterUntil(func() { 294 if _, err := cli.Get().AbsPath("/healthz").Do(ctx).Raw(); err != nil { 295 klog.InfoS("Failed to connect to apiserver", "err", err) 296 return 297 } 298 cancel() 299 connected = true 300 }, 2*time.Second, 0.2, true, ctx.Done()) 301 302 if !connected { 303 return errors.New("timed out waiting to connect to apiserver") 304 } 305 return nil 306} 307 308// requestNodeCertificate will create a certificate signing request for a node 309// (Organization and CommonName for the CSR will be set as expected for node 310// certificates) and send it to API server, then it will watch the object's 311// status, once approved by API server, it will return the API server's issued 312// certificate (pem-encoded). If there is any errors, or the watch timeouts, it 313// will return an error. This is intended for use on nodes (kubelet and 314// kubeadm). 315func requestNodeCertificate(ctx context.Context, client clientset.Interface, privateKeyData []byte, nodeName types.NodeName) (certData []byte, err error) { 316 subject := &pkix.Name{ 317 Organization: []string{"system:nodes"}, 318 CommonName: "system:node:" + string(nodeName), 319 } 320 321 privateKey, err := keyutil.ParsePrivateKeyPEM(privateKeyData) 322 if err != nil { 323 return nil, fmt.Errorf("invalid private key for certificate request: %v", err) 324 } 325 csrData, err := certutil.MakeCSR(privateKey, subject, nil, nil) 326 if err != nil { 327 return nil, fmt.Errorf("unable to generate certificate request: %v", err) 328 } 329 330 usages := []certificatesv1.KeyUsage{ 331 certificatesv1.UsageDigitalSignature, 332 certificatesv1.UsageKeyEncipherment, 333 certificatesv1.UsageClientAuth, 334 } 335 336 // The Signer interface contains the Public() method to get the public key. 337 signer, ok := privateKey.(crypto.Signer) 338 if !ok { 339 return nil, fmt.Errorf("private key does not implement crypto.Signer") 340 } 341 342 name, err := digestedName(signer.Public(), subject, usages) 343 if err != nil { 344 return nil, err 345 } 346 347 reqName, reqUID, err := csr.RequestCertificate(client, csrData, name, certificatesv1.KubeAPIServerClientKubeletSignerName, nil, usages, privateKey) 348 if err != nil { 349 return nil, err 350 } 351 352 ctx, cancel := context.WithTimeout(ctx, 3600*time.Second) 353 defer cancel() 354 355 klog.V(2).InfoS("Waiting for client certificate to be issued") 356 return csr.WaitForCertificate(ctx, client, reqName, reqUID) 357} 358 359// This digest should include all the relevant pieces of the CSR we care about. 360// We can't directly hash the serialized CSR because of random padding that we 361// regenerate every loop and we include usages which are not contained in the 362// CSR. This needs to be kept up to date as we add new fields to the node 363// certificates and with ensureCompatible. 364func digestedName(publicKey interface{}, subject *pkix.Name, usages []certificatesv1.KeyUsage) (string, error) { 365 hash := sha512.New512_256() 366 367 // Here we make sure two different inputs can't write the same stream 368 // to the hash. This delimiter is not in the base64.URLEncoding 369 // alphabet so there is no way to have spill over collisions. Without 370 // it 'CN:foo,ORG:bar' hashes to the same value as 'CN:foob,ORG:ar' 371 const delimiter = '|' 372 encode := base64.RawURLEncoding.EncodeToString 373 374 write := func(data []byte) { 375 hash.Write([]byte(encode(data))) 376 hash.Write([]byte{delimiter}) 377 } 378 379 publicKeyData, err := x509.MarshalPKIXPublicKey(publicKey) 380 if err != nil { 381 return "", err 382 } 383 write(publicKeyData) 384 385 write([]byte(subject.CommonName)) 386 for _, v := range subject.Organization { 387 write([]byte(v)) 388 } 389 for _, v := range usages { 390 write([]byte(v)) 391 } 392 393 return fmt.Sprintf("node-csr-%s", encode(hash.Sum(nil))), nil 394} 395