1/* 2Copyright 2017 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package certificate 18 19import ( 20 "crypto/tls" 21 "crypto/x509" 22 "crypto/x509/pkix" 23 "fmt" 24 "math" 25 "net" 26 "sort" 27 "time" 28 29 certificates "k8s.io/api/certificates/v1" 30 v1 "k8s.io/api/core/v1" 31 "k8s.io/apimachinery/pkg/types" 32 clientset "k8s.io/client-go/kubernetes" 33 "k8s.io/client-go/util/certificate" 34 compbasemetrics "k8s.io/component-base/metrics" 35 "k8s.io/component-base/metrics/legacyregistry" 36 kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" 37 "k8s.io/kubernetes/pkg/kubelet/metrics" 38) 39 40// NewKubeletServerCertificateManager creates a certificate manager for the kubelet when retrieving a server certificate 41// or returns an error. 42func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *kubeletconfig.KubeletConfiguration, nodeName types.NodeName, getAddresses func() []v1.NodeAddress, certDirectory string) (certificate.Manager, error) { 43 var clientsetFn certificate.ClientsetFunc 44 if kubeClient != nil { 45 clientsetFn = func(current *tls.Certificate) (clientset.Interface, error) { 46 return kubeClient, nil 47 } 48 } 49 certificateStore, err := certificate.NewFileStore( 50 "kubelet-server", 51 certDirectory, 52 certDirectory, 53 kubeCfg.TLSCertFile, 54 kubeCfg.TLSPrivateKeyFile) 55 if err != nil { 56 return nil, fmt.Errorf("failed to initialize server certificate store: %v", err) 57 } 58 var certificateRenewFailure = compbasemetrics.NewCounter( 59 &compbasemetrics.CounterOpts{ 60 Subsystem: metrics.KubeletSubsystem, 61 Name: "server_expiration_renew_errors", 62 Help: "Counter of certificate renewal errors.", 63 StabilityLevel: compbasemetrics.ALPHA, 64 }, 65 ) 66 legacyregistry.MustRegister(certificateRenewFailure) 67 68 certificateRotationAge := compbasemetrics.NewHistogram( 69 &compbasemetrics.HistogramOpts{ 70 Subsystem: metrics.KubeletSubsystem, 71 Name: "certificate_manager_server_rotation_seconds", 72 Help: "Histogram of the number of seconds the previous certificate lived before being rotated.", 73 Buckets: []float64{ 74 60, // 1 minute 75 3600, // 1 hour 76 14400, // 4 hours 77 86400, // 1 day 78 604800, // 1 week 79 2592000, // 1 month 80 7776000, // 3 months 81 15552000, // 6 months 82 31104000, // 1 year 83 124416000, // 4 years 84 }, 85 StabilityLevel: compbasemetrics.ALPHA, 86 }, 87 ) 88 legacyregistry.MustRegister(certificateRotationAge) 89 90 getTemplate := func() *x509.CertificateRequest { 91 hostnames, ips := addressesToHostnamesAndIPs(getAddresses()) 92 // don't return a template if we have no addresses to request for 93 if len(hostnames) == 0 && len(ips) == 0 { 94 return nil 95 } 96 return &x509.CertificateRequest{ 97 Subject: pkix.Name{ 98 CommonName: fmt.Sprintf("system:node:%s", nodeName), 99 Organization: []string{"system:nodes"}, 100 }, 101 DNSNames: hostnames, 102 IPAddresses: ips, 103 } 104 } 105 106 m, err := certificate.NewManager(&certificate.Config{ 107 ClientsetFn: clientsetFn, 108 GetTemplate: getTemplate, 109 SignerName: certificates.KubeletServingSignerName, 110 Usages: []certificates.KeyUsage{ 111 // https://tools.ietf.org/html/rfc5280#section-4.2.1.3 112 // 113 // Digital signature allows the certificate to be used to verify 114 // digital signatures used during TLS negotiation. 115 certificates.UsageDigitalSignature, 116 // KeyEncipherment allows the cert/key pair to be used to encrypt 117 // keys, including the symmetric keys negotiated during TLS setup 118 // and used for data transfer. 119 certificates.UsageKeyEncipherment, 120 // ServerAuth allows the cert to be used by a TLS server to 121 // authenticate itself to a TLS client. 122 certificates.UsageServerAuth, 123 }, 124 CertificateStore: certificateStore, 125 CertificateRotation: certificateRotationAge, 126 CertificateRenewFailure: certificateRenewFailure, 127 }) 128 if err != nil { 129 return nil, fmt.Errorf("failed to initialize server certificate manager: %v", err) 130 } 131 legacyregistry.RawMustRegister(compbasemetrics.NewGaugeFunc( 132 compbasemetrics.GaugeOpts{ 133 Subsystem: metrics.KubeletSubsystem, 134 Name: "certificate_manager_server_ttl_seconds", 135 Help: "Gauge of the shortest TTL (time-to-live) of " + 136 "the Kubelet's serving certificate. The value is in seconds " + 137 "until certificate expiry (negative if already expired). If " + 138 "serving certificate is invalid or unused, the value will " + 139 "be +INF.", 140 StabilityLevel: compbasemetrics.ALPHA, 141 }, 142 func() float64 { 143 if c := m.Current(); c != nil && c.Leaf != nil { 144 return math.Trunc(c.Leaf.NotAfter.Sub(time.Now()).Seconds()) 145 } 146 return math.Inf(1) 147 }, 148 )) 149 return m, nil 150} 151 152func addressesToHostnamesAndIPs(addresses []v1.NodeAddress) (dnsNames []string, ips []net.IP) { 153 seenDNSNames := map[string]bool{} 154 seenIPs := map[string]bool{} 155 for _, address := range addresses { 156 if len(address.Address) == 0 { 157 continue 158 } 159 160 switch address.Type { 161 case v1.NodeHostName: 162 if ip := net.ParseIP(address.Address); ip != nil { 163 seenIPs[address.Address] = true 164 } else { 165 seenDNSNames[address.Address] = true 166 } 167 case v1.NodeExternalIP, v1.NodeInternalIP: 168 if ip := net.ParseIP(address.Address); ip != nil { 169 seenIPs[address.Address] = true 170 } 171 case v1.NodeExternalDNS, v1.NodeInternalDNS: 172 seenDNSNames[address.Address] = true 173 } 174 } 175 176 for dnsName := range seenDNSNames { 177 dnsNames = append(dnsNames, dnsName) 178 } 179 for ip := range seenIPs { 180 ips = append(ips, net.ParseIP(ip)) 181 } 182 183 // return in stable order 184 sort.Strings(dnsNames) 185 sort.Slice(ips, func(i, j int) bool { return ips[i].String() < ips[j].String() }) 186 187 return dnsNames, ips 188} 189 190// NewKubeletClientCertificateManager sets up a certificate manager without a 191// client that can be used to sign new certificates (or rotate). If a CSR 192// client is set later, it may begin rotating/renewing the client cert. 193func NewKubeletClientCertificateManager( 194 certDirectory string, 195 nodeName types.NodeName, 196 bootstrapCertData []byte, 197 bootstrapKeyData []byte, 198 certFile string, 199 keyFile string, 200 clientsetFn certificate.ClientsetFunc, 201) (certificate.Manager, error) { 202 203 certificateStore, err := certificate.NewFileStore( 204 "kubelet-client", 205 certDirectory, 206 certDirectory, 207 certFile, 208 keyFile) 209 if err != nil { 210 return nil, fmt.Errorf("failed to initialize client certificate store: %v", err) 211 } 212 var certificateRenewFailure = compbasemetrics.NewCounter( 213 &compbasemetrics.CounterOpts{ 214 Namespace: metrics.KubeletSubsystem, 215 Subsystem: "certificate_manager", 216 Name: "client_expiration_renew_errors", 217 Help: "Counter of certificate renewal errors.", 218 StabilityLevel: compbasemetrics.ALPHA, 219 }, 220 ) 221 legacyregistry.Register(certificateRenewFailure) 222 223 m, err := certificate.NewManager(&certificate.Config{ 224 ClientsetFn: clientsetFn, 225 Template: &x509.CertificateRequest{ 226 Subject: pkix.Name{ 227 CommonName: fmt.Sprintf("system:node:%s", nodeName), 228 Organization: []string{"system:nodes"}, 229 }, 230 }, 231 SignerName: certificates.KubeAPIServerClientKubeletSignerName, 232 Usages: []certificates.KeyUsage{ 233 // https://tools.ietf.org/html/rfc5280#section-4.2.1.3 234 // 235 // DigitalSignature allows the certificate to be used to verify 236 // digital signatures including signatures used during TLS 237 // negotiation. 238 certificates.UsageDigitalSignature, 239 // KeyEncipherment allows the cert/key pair to be used to encrypt 240 // keys, including the symmetric keys negotiated during TLS setup 241 // and used for data transfer.. 242 certificates.UsageKeyEncipherment, 243 // ClientAuth allows the cert to be used by a TLS client to 244 // authenticate itself to the TLS server. 245 certificates.UsageClientAuth, 246 }, 247 248 // For backwards compatibility, the kubelet supports the ability to 249 // provide a higher privileged certificate as initial data that will 250 // then be rotated immediately. This code path is used by kubeadm on 251 // the masters. 252 BootstrapCertificatePEM: bootstrapCertData, 253 BootstrapKeyPEM: bootstrapKeyData, 254 255 CertificateStore: certificateStore, 256 CertificateRenewFailure: certificateRenewFailure, 257 }) 258 if err != nil { 259 return nil, fmt.Errorf("failed to initialize client certificate manager: %v", err) 260 } 261 262 return m, nil 263} 264