1/* 2Copyright 2017 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package certificate 18 19import ( 20 "bytes" 21 "crypto/tls" 22 "crypto/x509" 23 "crypto/x509/pkix" 24 "fmt" 25 "net" 26 "strings" 27 "testing" 28 "time" 29 30 certificates "k8s.io/api/certificates/v1beta1" 31 "k8s.io/apimachinery/pkg/api/errors" 32 v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 33 "k8s.io/apimachinery/pkg/runtime/schema" 34 watch "k8s.io/apimachinery/pkg/watch" 35 certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" 36) 37 38var storeCertData = newCertificateData(`-----BEGIN CERTIFICATE----- 39MIICRzCCAfGgAwIBAgIJALMb7ecMIk3MMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV 40BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE 41CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD 42VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwIBcNMTcwNDI2MjMyNjUyWhgPMjExNzA0 43MDIyMzI2NTJaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV 44BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J 45VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwXDANBgkq 46hkiG9w0BAQEFAANLADBIAkEAtBMa7NWpv3BVlKTCPGO/LEsguKqWHBtKzweMY2CV 47tAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5MzP2H5QIDAQABo1AwTjAdBgNV 48HQ4EFgQU22iy8aWkNSxv0nBxFxerfsvnZVMwHwYDVR0jBBgwFoAU22iy8aWkNSxv 490nBxFxerfsvnZVMwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAEOefGbV 50NcHxklaW06w6OBYJPwpIhCVozC1qdxGX1dg8VkEKzjOzjgqVD30m59OFmSlBmHsl 51nkVA6wyOSDYBf3o= 52-----END CERTIFICATE-----`, `-----BEGIN RSA PRIVATE KEY----- 53MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEAtBMa7NWpv3BVlKTC 54PGO/LEsguKqWHBtKzweMY2CVtAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5M 55zP2H5QIDAQABAkAS9BfXab3OKpK3bIgNNyp+DQJKrZnTJ4Q+OjsqkpXvNltPJosf 56G8GsiKu/vAt4HGqI3eU77NvRI+mL4MnHRmXBAiEA3qM4FAtKSRBbcJzPxxLEUSwg 57XSCcosCktbkXvpYrS30CIQDPDxgqlwDEJQ0uKuHkZI38/SPWWqfUmkecwlbpXABK 58iQIgZX08DA8VfvcA5/Xj1Zjdey9FVY6POLXen6RPiabE97UCICp6eUW7ht+2jjar 59e35EltCRCjoejRHTuN9TC0uCoVipAiAXaJIx/Q47vGwiw6Y8KXsNU6y54gTbOSxX 6054LzHNk/+Q== 61-----END RSA PRIVATE KEY-----`) 62var expiredStoreCertData = newCertificateData(`-----BEGIN CERTIFICATE----- 63MIIBFzCBwgIJALhygXnxXmN1MA0GCSqGSIb3DQEBCwUAMBMxETAPBgNVBAMMCGhv 64c3QtMTIzMB4XDTE4MTEwNDIzNTc1NFoXDTE4MTEwNTIzNTc1NFowEzERMA8GA1UE 65AwwIaG9zdC0xMjMwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAtBMa7NWpv3BVlKTC 66PGO/LEsguKqWHBtKzweMY2CVtAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5M 67zP2H5QIDAQABMA0GCSqGSIb3DQEBCwUAA0EAN2DPFUtCzqnidL+5nh+46Sk6dkMI 68T5DD11UuuIjZusKvThsHKVCIsyJ2bDo7cTbI+/nklLRP+FcC2wESFUgXbA== 69-----END CERTIFICATE-----`, `-----BEGIN RSA PRIVATE KEY----- 70MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEAtBMa7NWpv3BVlKTC 71PGO/LEsguKqWHBtKzweMY2CVtAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5M 72zP2H5QIDAQABAkAS9BfXab3OKpK3bIgNNyp+DQJKrZnTJ4Q+OjsqkpXvNltPJosf 73G8GsiKu/vAt4HGqI3eU77NvRI+mL4MnHRmXBAiEA3qM4FAtKSRBbcJzPxxLEUSwg 74XSCcosCktbkXvpYrS30CIQDPDxgqlwDEJQ0uKuHkZI38/SPWWqfUmkecwlbpXABK 75iQIgZX08DA8VfvcA5/Xj1Zjdey9FVY6POLXen6RPiabE97UCICp6eUW7ht+2jjar 76e35EltCRCjoejRHTuN9TC0uCoVipAiAXaJIx/Q47vGwiw6Y8KXsNU6y54gTbOSxX 7754LzHNk/+Q== 78-----END RSA PRIVATE KEY-----`) 79var bootstrapCertData = newCertificateData( 80 `-----BEGIN CERTIFICATE----- 81MIICRzCCAfGgAwIBAgIJANXr+UzRFq4TMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV 82BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE 83CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD 84VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTEwIBcNMTcwNDI2MjMyNzMyWhgPMjExNzA0 85MDIyMzI3MzJaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV 86BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J 87VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTEwXDANBgkq 88hkiG9w0BAQEFAANLADBIAkEAqvbkN4RShH1rL37JFp4fZPnn0JUhVWWsrP8NOomJ 89pXdBDUMGWuEQIsZ1Gf9JrCQLu6ooRyHSKRFpAVbMQ3ABJwIDAQABo1AwTjAdBgNV 90HQ4EFgQUEGBc6YYheEZ/5MhwqSUYYPYRj2MwHwYDVR0jBBgwFoAUEGBc6YYheEZ/ 915MhwqSUYYPYRj2MwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAIyNmznk 925dgJY52FppEEcfQRdS5k4XFPc22SHPcz77AHf5oWZ1WG9VezOZZPp8NCiFDDlDL8 93yma33a5eMyTjLD8= 94-----END CERTIFICATE-----`, `-----BEGIN RSA PRIVATE KEY----- 95MIIBVAIBADANBgkqhkiG9w0BAQEFAASCAT4wggE6AgEAAkEAqvbkN4RShH1rL37J 96Fp4fZPnn0JUhVWWsrP8NOomJpXdBDUMGWuEQIsZ1Gf9JrCQLu6ooRyHSKRFpAVbM 97Q3ABJwIDAQABAkBC2OBpGLMPHN8BJijIUDFkURakBvuOoX+/8MYiYk7QxEmfLCk6 98L6r+GLNFMfXwXcBmXtMKfZKAIKutKf098JaBAiEA10azfqt3G/5owrNA00plSyT6 99ZmHPzY9Uq1p/QTR/uOcCIQDLTkfBkLHm0UKeobbO/fSm6ZflhyBRDINy4FvwmZMt 100wQIgYV/tmQJeIh91q3wBepFQOClFykG8CTMoDUol/YyNqUkCIHfp6Rr7fGL3JIMq 101QQgf9DCK8SPZqq8DYXjdan0kKBJBAiEAyDb+07o2gpggo8BYUKSaiRCiyXfaq87f 102eVqgpBq/QN4= 103-----END RSA PRIVATE KEY-----`) 104var apiServerCertData = newCertificateData( 105 `-----BEGIN CERTIFICATE----- 106MIICRzCCAfGgAwIBAgIJAIydTIADd+yqMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV 107BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE 108CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD 109VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTIwIBcNMTcwNDI2MjMyNDU4WhgPMjExNzA0 110MDIyMzI0NThaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV 111BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J 112VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTIwXDANBgkq 113hkiG9w0BAQEFAANLADBIAkEAuiRet28DV68Dk4A8eqCaqgXmymamUEjW/DxvIQqH 1143lbhtm8BwSnS9wUAajSLSWiq3fci2RbRgaSPjUrnbOHCLQIDAQABo1AwTjAdBgNV 115HQ4EFgQU0vhI4OPGEOqT+VAWwxdhVvcmgdIwHwYDVR0jBBgwFoAU0vhI4OPGEOqT 116+VAWwxdhVvcmgdIwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBALNeJGDe 117nV5cXbp9W1bC12Tc8nnNXn4ypLE2JTQAvyp51zoZ8hQoSnRVx/VCY55Yu+br8gQZ 118+tW+O/PoE7B3tuY= 119-----END CERTIFICATE-----`, `-----BEGIN RSA PRIVATE KEY----- 120MIIBVgIBADANBgkqhkiG9w0BAQEFAASCAUAwggE8AgEAAkEAuiRet28DV68Dk4A8 121eqCaqgXmymamUEjW/DxvIQqH3lbhtm8BwSnS9wUAajSLSWiq3fci2RbRgaSPjUrn 122bOHCLQIDAQABAkEArDR1g9IqD3aUImNikDgAngbzqpAokOGyMoxeavzpEaFOgCzi 123gi7HF7yHRmZkUt8CzdEvnHSqRjFuaaB0gGA+AQIhAOc8Z1h8ElLRSqaZGgI3jCTp 124Izx9HNY//U5NGrXD2+ttAiEAzhOqkqI4+nDab7FpiD7MXI6fO549mEXeVBPvPtsS 125OcECIQCIfkpOm+ZBBpO3JXaJynoqK4gGI6ALA/ik6LSUiIlfPQIhAISjd9hlfZME 126bDQT1r8Q3Gx+h9LRqQeHgPBQ3F5ylqqBAiBaJ0hkYvrIdWxNlcLqD3065bJpHQ4S 127WQkuZUQN1M/Xvg== 128-----END RSA PRIVATE KEY-----`) 129 130type certificateData struct { 131 keyPEM []byte 132 certificatePEM []byte 133 certificate *tls.Certificate 134} 135 136func newCertificateData(certificatePEM string, keyPEM string) *certificateData { 137 certificate, err := tls.X509KeyPair([]byte(certificatePEM), []byte(keyPEM)) 138 if err != nil { 139 panic(fmt.Sprintf("Unable to initialize certificate: %v", err)) 140 } 141 certs, err := x509.ParseCertificates(certificate.Certificate[0]) 142 if err != nil { 143 panic(fmt.Sprintf("Unable to initialize certificate leaf: %v", err)) 144 } 145 certificate.Leaf = certs[0] 146 return &certificateData{ 147 keyPEM: []byte(keyPEM), 148 certificatePEM: []byte(certificatePEM), 149 certificate: &certificate, 150 } 151} 152 153func TestNewManagerNoRotation(t *testing.T) { 154 store := &fakeStore{ 155 cert: storeCertData.certificate, 156 } 157 if _, err := NewManager(&Config{ 158 Template: &x509.CertificateRequest{}, 159 Usages: []certificates.KeyUsage{}, 160 CertificateStore: store, 161 }); err != nil { 162 t.Fatalf("Failed to initialize the certificate manager: %v", err) 163 } 164} 165 166type gaugeMock struct { 167 calls int 168 lastValue float64 169} 170 171func (g *gaugeMock) Set(v float64) { 172 g.calls++ 173 g.lastValue = v 174} 175 176func TestSetRotationDeadline(t *testing.T) { 177 defer func(original func(float64) time.Duration) { jitteryDuration = original }(jitteryDuration) 178 179 now := time.Now() 180 testCases := []struct { 181 name string 182 notBefore time.Time 183 notAfter time.Time 184 shouldRotate bool 185 }{ 186 {"just issued, still good", now.Add(-1 * time.Hour), now.Add(99 * time.Hour), false}, 187 {"half way expired, still good", now.Add(-24 * time.Hour), now.Add(24 * time.Hour), false}, 188 {"mostly expired, still good", now.Add(-69 * time.Hour), now.Add(31 * time.Hour), false}, 189 {"just about expired, should rotate", now.Add(-91 * time.Hour), now.Add(9 * time.Hour), true}, 190 {"nearly expired, should rotate", now.Add(-99 * time.Hour), now.Add(1 * time.Hour), true}, 191 {"already expired, should rotate", now.Add(-10 * time.Hour), now.Add(-1 * time.Hour), true}, 192 {"long duration", now.Add(-6 * 30 * 24 * time.Hour), now.Add(6 * 30 * 24 * time.Hour), true}, 193 {"short duration", now.Add(-30 * time.Second), now.Add(30 * time.Second), true}, 194 } 195 196 for _, tc := range testCases { 197 t.Run(tc.name, func(t *testing.T) { 198 g := gaugeMock{} 199 m := manager{ 200 cert: &tls.Certificate{ 201 Leaf: &x509.Certificate{ 202 NotBefore: tc.notBefore, 203 NotAfter: tc.notAfter, 204 }, 205 }, 206 getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} }, 207 usages: []certificates.KeyUsage{}, 208 certificateExpiration: &g, 209 } 210 jitteryDuration = func(float64) time.Duration { return time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7) } 211 lowerBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7)) 212 213 deadline := m.nextRotationDeadline() 214 215 if !deadline.Equal(lowerBound) { 216 t.Errorf("For notBefore %v, notAfter %v, the rotationDeadline %v should be %v.", 217 tc.notBefore, 218 tc.notAfter, 219 deadline, 220 lowerBound) 221 } 222 if g.calls != 1 { 223 t.Errorf("%d metrics were recorded, wanted %d", g.calls, 1) 224 } 225 if g.lastValue != float64(tc.notAfter.Unix()) { 226 t.Errorf("%f value for metric was recorded, wanted %d", g.lastValue, tc.notAfter.Unix()) 227 } 228 }) 229 } 230} 231 232func TestCertSatisfiesTemplate(t *testing.T) { 233 testCases := []struct { 234 name string 235 cert *x509.Certificate 236 template *x509.CertificateRequest 237 shouldSatisfy bool 238 }{ 239 { 240 name: "No certificate, no template", 241 cert: nil, 242 template: nil, 243 shouldSatisfy: false, 244 }, 245 { 246 name: "No certificate", 247 cert: nil, 248 template: &x509.CertificateRequest{}, 249 shouldSatisfy: false, 250 }, 251 { 252 name: "No template", 253 cert: &x509.Certificate{ 254 Subject: pkix.Name{ 255 CommonName: "system:node:fake-node-name", 256 }, 257 }, 258 template: nil, 259 shouldSatisfy: true, 260 }, 261 { 262 name: "Mismatched common name", 263 cert: &x509.Certificate{ 264 Subject: pkix.Name{ 265 CommonName: "system:node:fake-node-name-2", 266 }, 267 }, 268 template: &x509.CertificateRequest{ 269 Subject: pkix.Name{ 270 CommonName: "system:node:fake-node-name", 271 }, 272 }, 273 shouldSatisfy: false, 274 }, 275 { 276 name: "Missing orgs in certificate", 277 cert: &x509.Certificate{ 278 Subject: pkix.Name{ 279 Organization: []string{"system:nodes"}, 280 }, 281 }, 282 template: &x509.CertificateRequest{ 283 Subject: pkix.Name{ 284 Organization: []string{"system:nodes", "foobar"}, 285 }, 286 }, 287 shouldSatisfy: false, 288 }, 289 { 290 name: "Extra orgs in certificate", 291 cert: &x509.Certificate{ 292 Subject: pkix.Name{ 293 Organization: []string{"system:nodes", "foobar"}, 294 }, 295 }, 296 template: &x509.CertificateRequest{ 297 Subject: pkix.Name{ 298 Organization: []string{"system:nodes"}, 299 }, 300 }, 301 shouldSatisfy: true, 302 }, 303 { 304 name: "Missing DNS names in certificate", 305 cert: &x509.Certificate{ 306 Subject: pkix.Name{}, 307 DNSNames: []string{"foo.example.com"}, 308 }, 309 template: &x509.CertificateRequest{ 310 Subject: pkix.Name{}, 311 DNSNames: []string{"foo.example.com", "bar.example.com"}, 312 }, 313 shouldSatisfy: false, 314 }, 315 { 316 name: "Extra DNS names in certificate", 317 cert: &x509.Certificate{ 318 Subject: pkix.Name{}, 319 DNSNames: []string{"foo.example.com", "bar.example.com"}, 320 }, 321 template: &x509.CertificateRequest{ 322 Subject: pkix.Name{}, 323 DNSNames: []string{"foo.example.com"}, 324 }, 325 shouldSatisfy: true, 326 }, 327 { 328 name: "Missing IP addresses in certificate", 329 cert: &x509.Certificate{ 330 Subject: pkix.Name{}, 331 IPAddresses: []net.IP{net.ParseIP("192.168.1.1")}, 332 }, 333 template: &x509.CertificateRequest{ 334 Subject: pkix.Name{}, 335 IPAddresses: []net.IP{net.ParseIP("192.168.1.1"), net.ParseIP("192.168.1.2")}, 336 }, 337 shouldSatisfy: false, 338 }, 339 { 340 name: "Extra IP addresses in certificate", 341 cert: &x509.Certificate{ 342 Subject: pkix.Name{}, 343 IPAddresses: []net.IP{net.ParseIP("192.168.1.1"), net.ParseIP("192.168.1.2")}, 344 }, 345 template: &x509.CertificateRequest{ 346 Subject: pkix.Name{}, 347 IPAddresses: []net.IP{net.ParseIP("192.168.1.1")}, 348 }, 349 shouldSatisfy: true, 350 }, 351 { 352 name: "Matching certificate", 353 cert: &x509.Certificate{ 354 Subject: pkix.Name{ 355 CommonName: "system:node:fake-node-name", 356 Organization: []string{"system:nodes"}, 357 }, 358 DNSNames: []string{"foo.example.com"}, 359 IPAddresses: []net.IP{net.ParseIP("192.168.1.1")}, 360 }, 361 template: &x509.CertificateRequest{ 362 Subject: pkix.Name{ 363 CommonName: "system:node:fake-node-name", 364 Organization: []string{"system:nodes"}, 365 }, 366 DNSNames: []string{"foo.example.com"}, 367 IPAddresses: []net.IP{net.ParseIP("192.168.1.1")}, 368 }, 369 shouldSatisfy: true, 370 }, 371 } 372 373 for _, tc := range testCases { 374 t.Run(tc.name, func(t *testing.T) { 375 var tlsCert *tls.Certificate 376 377 if tc.cert != nil { 378 tlsCert = &tls.Certificate{ 379 Leaf: tc.cert, 380 } 381 } 382 383 m := manager{ 384 cert: tlsCert, 385 getTemplate: func() *x509.CertificateRequest { return tc.template }, 386 } 387 388 result := m.certSatisfiesTemplate() 389 if result != tc.shouldSatisfy { 390 t.Errorf("cert: %+v, template: %+v, certSatisfiesTemplate returned %v, want %v", m.cert, tc.template, result, tc.shouldSatisfy) 391 } 392 }) 393 } 394} 395 396func TestRotateCertCreateCSRError(t *testing.T) { 397 now := time.Now() 398 m := manager{ 399 cert: &tls.Certificate{ 400 Leaf: &x509.Certificate{ 401 NotBefore: now.Add(-2 * time.Hour), 402 NotAfter: now.Add(-1 * time.Hour), 403 }, 404 }, 405 getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} }, 406 usages: []certificates.KeyUsage{}, 407 clientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { 408 return fakeClient{failureType: createError}, nil 409 }, 410 } 411 412 if success, err := m.rotateCerts(); success { 413 t.Errorf("Got success from 'rotateCerts', wanted failure") 414 } else if err != nil { 415 t.Errorf("Got error %v from 'rotateCerts', wanted no error.", err) 416 } 417} 418 419func TestRotateCertWaitingForResultError(t *testing.T) { 420 now := time.Now() 421 m := manager{ 422 cert: &tls.Certificate{ 423 Leaf: &x509.Certificate{ 424 NotBefore: now.Add(-2 * time.Hour), 425 NotAfter: now.Add(-1 * time.Hour), 426 }, 427 }, 428 getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} }, 429 usages: []certificates.KeyUsage{}, 430 clientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { 431 return fakeClient{failureType: watchError}, nil 432 }, 433 } 434 435 defer func(t time.Duration) { certificateWaitTimeout = t }(certificateWaitTimeout) 436 certificateWaitTimeout = 1 * time.Millisecond 437 if success, err := m.rotateCerts(); success { 438 t.Errorf("Got success from 'rotateCerts', wanted failure.") 439 } else if err != nil { 440 t.Errorf("Got error %v from 'rotateCerts', wanted no error.", err) 441 } 442} 443 444func TestNewManagerBootstrap(t *testing.T) { 445 store := &fakeStore{} 446 447 var cm Manager 448 cm, err := NewManager(&Config{ 449 Template: &x509.CertificateRequest{}, 450 Usages: []certificates.KeyUsage{}, 451 CertificateStore: store, 452 BootstrapCertificatePEM: bootstrapCertData.certificatePEM, 453 BootstrapKeyPEM: bootstrapCertData.keyPEM, 454 }) 455 if err != nil { 456 t.Fatalf("Failed to initialize the certificate manager: %v", err) 457 } 458 459 cert := cm.Current() 460 461 if cert == nil { 462 t.Errorf("Certificate was nil, expected something.") 463 } 464 if m, ok := cm.(*manager); !ok { 465 t.Errorf("Expected a '*manager' from 'NewManager'") 466 } else if !m.forceRotation { 467 t.Errorf("Expected rotation should happen during bootstrap, but it won't.") 468 } 469} 470 471func TestNewManagerNoBootstrap(t *testing.T) { 472 now := time.Now() 473 cert, err := tls.X509KeyPair(storeCertData.certificatePEM, storeCertData.keyPEM) 474 if err != nil { 475 t.Fatalf("Unable to initialize a certificate: %v", err) 476 } 477 cert.Leaf = &x509.Certificate{ 478 NotBefore: now.Add(-24 * time.Hour), 479 NotAfter: now.Add(24 * time.Hour), 480 } 481 store := &fakeStore{ 482 cert: &cert, 483 } 484 485 cm, err := NewManager(&Config{ 486 Template: &x509.CertificateRequest{}, 487 Usages: []certificates.KeyUsage{}, 488 CertificateStore: store, 489 BootstrapCertificatePEM: bootstrapCertData.certificatePEM, 490 BootstrapKeyPEM: bootstrapCertData.keyPEM, 491 }) 492 493 if err != nil { 494 t.Fatalf("Failed to initialize the certificate manager: %v", err) 495 } 496 497 currentCert := cm.Current() 498 499 if currentCert == nil { 500 t.Errorf("Certificate was nil, expected something.") 501 } 502 if m, ok := cm.(*manager); !ok { 503 t.Errorf("Expected a '*manager' from 'NewManager'") 504 } else { 505 if m.forceRotation { 506 t.Errorf("Expected rotation should not happen during bootstrap, but it won't.") 507 } 508 } 509} 510 511func TestGetCurrentCertificateOrBootstrap(t *testing.T) { 512 testCases := []struct { 513 description string 514 storeCert *tls.Certificate 515 bootstrapCertData []byte 516 bootstrapKeyData []byte 517 expectedCert *tls.Certificate 518 expectedShouldRotate bool 519 expectedErrMsg string 520 }{ 521 { 522 "return cert from store", 523 storeCertData.certificate, 524 nil, 525 nil, 526 storeCertData.certificate, 527 false, 528 "", 529 }, 530 { 531 "no cert in store and no bootstrap cert", 532 nil, 533 nil, 534 nil, 535 nil, 536 true, 537 "", 538 }, 539 } 540 541 for _, tc := range testCases { 542 t.Run(tc.description, func(t *testing.T) { 543 store := &fakeStore{ 544 cert: tc.storeCert, 545 } 546 547 certResult, shouldRotate, err := getCurrentCertificateOrBootstrap( 548 store, 549 tc.bootstrapCertData, 550 tc.bootstrapKeyData) 551 if certResult == nil || certResult.Certificate == nil || tc.expectedCert == nil { 552 if certResult != nil && tc.expectedCert != nil { 553 t.Errorf("Got certificate %v, wanted %v", certResult, tc.expectedCert) 554 } 555 } else { 556 if !certificatesEqual(certResult, tc.expectedCert) { 557 t.Errorf("Got certificate %v, wanted %v", certResult, tc.expectedCert) 558 } 559 } 560 if shouldRotate != tc.expectedShouldRotate { 561 t.Errorf("Got shouldRotate %t, wanted %t", shouldRotate, tc.expectedShouldRotate) 562 } 563 if err == nil { 564 if tc.expectedErrMsg != "" { 565 t.Errorf("Got err %v, wanted %q", err, tc.expectedErrMsg) 566 } 567 } else { 568 if tc.expectedErrMsg == "" || !strings.Contains(err.Error(), tc.expectedErrMsg) { 569 t.Errorf("Got err %v, wanted %q", err, tc.expectedErrMsg) 570 } 571 } 572 }) 573 } 574} 575 576func TestInitializeCertificateSigningRequestClient(t *testing.T) { 577 var nilCertificate = &certificateData{} 578 testCases := []struct { 579 description string 580 storeCert *certificateData 581 bootstrapCert *certificateData 582 apiCert *certificateData 583 expectedCertBeforeStart *certificateData 584 expectedCertAfterStart *certificateData 585 }{ 586 { 587 description: "No current certificate, no bootstrap certificate", 588 storeCert: nilCertificate, 589 bootstrapCert: nilCertificate, 590 apiCert: apiServerCertData, 591 expectedCertBeforeStart: nilCertificate, 592 expectedCertAfterStart: apiServerCertData, 593 }, 594 { 595 description: "No current certificate, bootstrap certificate", 596 storeCert: nilCertificate, 597 bootstrapCert: bootstrapCertData, 598 apiCert: apiServerCertData, 599 expectedCertBeforeStart: bootstrapCertData, 600 expectedCertAfterStart: apiServerCertData, 601 }, 602 { 603 description: "Current certificate, no bootstrap certificate", 604 storeCert: storeCertData, 605 bootstrapCert: nilCertificate, 606 apiCert: apiServerCertData, 607 expectedCertBeforeStart: storeCertData, 608 expectedCertAfterStart: storeCertData, 609 }, 610 { 611 description: "Current certificate, bootstrap certificate", 612 storeCert: storeCertData, 613 bootstrapCert: bootstrapCertData, 614 apiCert: apiServerCertData, 615 expectedCertBeforeStart: storeCertData, 616 expectedCertAfterStart: storeCertData, 617 }, 618 { 619 description: "Current certificate expired, no bootstrap certificate", 620 storeCert: expiredStoreCertData, 621 bootstrapCert: nilCertificate, 622 apiCert: apiServerCertData, 623 expectedCertBeforeStart: nil, 624 expectedCertAfterStart: apiServerCertData, 625 }, 626 } 627 628 for _, tc := range testCases { 629 t.Run(tc.description, func(t *testing.T) { 630 certificateStore := &fakeStore{ 631 cert: tc.storeCert.certificate, 632 } 633 634 certificateManager, err := NewManager(&Config{ 635 Template: &x509.CertificateRequest{ 636 Subject: pkix.Name{ 637 Organization: []string{"system:nodes"}, 638 CommonName: "system:node:fake-node-name", 639 }, 640 }, 641 Usages: []certificates.KeyUsage{ 642 certificates.UsageDigitalSignature, 643 certificates.UsageKeyEncipherment, 644 certificates.UsageClientAuth, 645 }, 646 CertificateStore: certificateStore, 647 BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM, 648 BootstrapKeyPEM: tc.bootstrapCert.keyPEM, 649 ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { 650 return &fakeClient{ 651 certificatePEM: tc.apiCert.certificatePEM, 652 }, nil 653 }, 654 }) 655 if err != nil { 656 t.Errorf("Got %v, wanted no error.", err) 657 } 658 659 certificate := certificateManager.Current() 660 if tc.expectedCertBeforeStart == nil { 661 if certificate != nil { 662 t.Errorf("Expected certificate to be nil, was %s", certificate.Leaf.NotAfter) 663 } 664 } else { 665 if !certificatesEqual(certificate, tc.expectedCertBeforeStart.certificate) { 666 t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertBeforeStart.certificate)) 667 } 668 } 669 670 if m, ok := certificateManager.(*manager); !ok { 671 t.Errorf("Expected a '*manager' from 'NewManager'") 672 } else { 673 if m.forceRotation { 674 if success, err := m.rotateCerts(); !success { 675 t.Errorf("Got failure from 'rotateCerts', wanted success.") 676 } else if err != nil { 677 t.Errorf("Got error %v, expected none.", err) 678 } 679 } 680 } 681 682 certificate = certificateManager.Current() 683 if tc.expectedCertAfterStart == nil { 684 if certificate != nil { 685 t.Errorf("Expected certificate to be nil, was %s", certificate.Leaf.NotAfter) 686 } 687 return 688 } 689 if !certificatesEqual(certificate, tc.expectedCertAfterStart.certificate) { 690 t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertAfterStart.certificate)) 691 } 692 }) 693 } 694} 695 696func TestInitializeOtherRESTClients(t *testing.T) { 697 var nilCertificate = &certificateData{} 698 testCases := []struct { 699 description string 700 storeCert *certificateData 701 bootstrapCert *certificateData 702 apiCert *certificateData 703 expectedCertBeforeStart *certificateData 704 expectedCertAfterStart *certificateData 705 }{ 706 { 707 description: "No current certificate, no bootstrap certificate", 708 storeCert: nilCertificate, 709 bootstrapCert: nilCertificate, 710 apiCert: apiServerCertData, 711 expectedCertBeforeStart: nilCertificate, 712 expectedCertAfterStart: apiServerCertData, 713 }, 714 { 715 description: "No current certificate, bootstrap certificate", 716 storeCert: nilCertificate, 717 bootstrapCert: bootstrapCertData, 718 apiCert: apiServerCertData, 719 expectedCertBeforeStart: bootstrapCertData, 720 expectedCertAfterStart: apiServerCertData, 721 }, 722 { 723 description: "Current certificate, no bootstrap certificate", 724 storeCert: storeCertData, 725 bootstrapCert: nilCertificate, 726 apiCert: apiServerCertData, 727 expectedCertBeforeStart: storeCertData, 728 expectedCertAfterStart: storeCertData, 729 }, 730 { 731 description: "Current certificate, bootstrap certificate", 732 storeCert: storeCertData, 733 bootstrapCert: bootstrapCertData, 734 apiCert: apiServerCertData, 735 expectedCertBeforeStart: storeCertData, 736 expectedCertAfterStart: storeCertData, 737 }, 738 } 739 740 for _, tc := range testCases { 741 t.Run(tc.description, func(t *testing.T) { 742 certificateStore := &fakeStore{ 743 cert: tc.storeCert.certificate, 744 } 745 746 certificateManager, err := NewManager(&Config{ 747 Template: &x509.CertificateRequest{ 748 Subject: pkix.Name{ 749 Organization: []string{"system:nodes"}, 750 CommonName: "system:node:fake-node-name", 751 }, 752 }, 753 Usages: []certificates.KeyUsage{ 754 certificates.UsageDigitalSignature, 755 certificates.UsageKeyEncipherment, 756 certificates.UsageClientAuth, 757 }, 758 CertificateStore: certificateStore, 759 BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM, 760 BootstrapKeyPEM: tc.bootstrapCert.keyPEM, 761 ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { 762 return &fakeClient{ 763 certificatePEM: tc.apiCert.certificatePEM, 764 }, nil 765 }, 766 }) 767 if err != nil { 768 t.Errorf("Got %v, wanted no error.", err) 769 } 770 771 certificate := certificateManager.Current() 772 if !certificatesEqual(certificate, tc.expectedCertBeforeStart.certificate) { 773 t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertBeforeStart.certificate)) 774 } 775 776 if m, ok := certificateManager.(*manager); !ok { 777 t.Errorf("Expected a '*manager' from 'NewManager'") 778 } else { 779 if m.forceRotation { 780 success, err := certificateManager.(*manager).rotateCerts() 781 if err != nil { 782 t.Errorf("Got error %v, expected none.", err) 783 return 784 } 785 if !success { 786 t.Errorf("Unexpected response 'rotateCerts': %t", success) 787 return 788 } 789 } 790 } 791 792 certificate = certificateManager.Current() 793 if !certificatesEqual(certificate, tc.expectedCertAfterStart.certificate) { 794 t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertAfterStart.certificate)) 795 } 796 }) 797 } 798} 799 800func TestServerHealth(t *testing.T) { 801 type certs struct { 802 storeCert *certificateData 803 bootstrapCert *certificateData 804 apiCert *certificateData 805 expectedCertBeforeStart *certificateData 806 expectedCertAfterStart *certificateData 807 } 808 809 updatedCerts := certs{ 810 storeCert: storeCertData, 811 bootstrapCert: bootstrapCertData, 812 apiCert: apiServerCertData, 813 expectedCertBeforeStart: storeCertData, 814 expectedCertAfterStart: apiServerCertData, 815 } 816 817 currentCerts := certs{ 818 storeCert: storeCertData, 819 bootstrapCert: bootstrapCertData, 820 apiCert: apiServerCertData, 821 expectedCertBeforeStart: storeCertData, 822 expectedCertAfterStart: storeCertData, 823 } 824 825 testCases := []struct { 826 description string 827 certs 828 829 failureType fakeClientFailureType 830 clientErr error 831 832 expectRotateFail bool 833 expectHealthy bool 834 }{ 835 { 836 description: "Current certificate, bootstrap certificate", 837 certs: updatedCerts, 838 expectHealthy: true, 839 }, 840 { 841 description: "Generic error on create", 842 certs: currentCerts, 843 844 failureType: createError, 845 expectRotateFail: true, 846 }, 847 { 848 description: "Unauthorized error on create", 849 certs: currentCerts, 850 851 failureType: createError, 852 clientErr: errors.NewUnauthorized("unauthorized"), 853 expectRotateFail: true, 854 expectHealthy: true, 855 }, 856 { 857 description: "Generic unauthorized error on create", 858 certs: currentCerts, 859 860 failureType: createError, 861 clientErr: errors.NewGenericServerResponse(401, "POST", schema.GroupResource{}, "", "", 0, true), 862 expectRotateFail: true, 863 expectHealthy: true, 864 }, 865 { 866 description: "Generic not found error on create", 867 certs: currentCerts, 868 869 failureType: createError, 870 clientErr: errors.NewGenericServerResponse(404, "POST", schema.GroupResource{}, "", "", 0, true), 871 expectRotateFail: true, 872 expectHealthy: false, 873 }, 874 { 875 description: "Not found error on create", 876 certs: currentCerts, 877 878 failureType: createError, 879 clientErr: errors.NewGenericServerResponse(404, "POST", schema.GroupResource{}, "", "", 0, false), 880 expectRotateFail: true, 881 expectHealthy: true, 882 }, 883 } 884 885 for _, tc := range testCases { 886 t.Run(tc.description, func(t *testing.T) { 887 certificateStore := &fakeStore{ 888 cert: tc.storeCert.certificate, 889 } 890 891 certificateManager, err := NewManager(&Config{ 892 Template: &x509.CertificateRequest{ 893 Subject: pkix.Name{ 894 Organization: []string{"system:nodes"}, 895 CommonName: "system:node:fake-node-name", 896 }, 897 }, 898 Usages: []certificates.KeyUsage{ 899 certificates.UsageDigitalSignature, 900 certificates.UsageKeyEncipherment, 901 certificates.UsageClientAuth, 902 }, 903 CertificateStore: certificateStore, 904 BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM, 905 BootstrapKeyPEM: tc.bootstrapCert.keyPEM, 906 ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { 907 return &fakeClient{ 908 certificatePEM: tc.apiCert.certificatePEM, 909 failureType: tc.failureType, 910 err: tc.clientErr, 911 }, nil 912 }, 913 }) 914 if err != nil { 915 t.Errorf("Got %v, wanted no error.", err) 916 } 917 918 certificate := certificateManager.Current() 919 if !certificatesEqual(certificate, tc.expectedCertBeforeStart.certificate) { 920 t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertBeforeStart.certificate)) 921 } 922 923 if _, ok := certificateManager.(*manager); !ok { 924 t.Errorf("Expected a '*manager' from 'NewManager'") 925 } else { 926 success, err := certificateManager.(*manager).rotateCerts() 927 if err != nil { 928 t.Errorf("Got error %v, expected none.", err) 929 return 930 } 931 if !success != tc.expectRotateFail { 932 t.Errorf("Unexpected response 'rotateCerts': %t", success) 933 return 934 } 935 if actual := certificateManager.(*manager).ServerHealthy(); actual != tc.expectHealthy { 936 t.Errorf("Unexpected manager server health: %t", actual) 937 } 938 } 939 940 certificate = certificateManager.Current() 941 if !certificatesEqual(certificate, tc.expectedCertAfterStart.certificate) { 942 t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertAfterStart.certificate)) 943 } 944 }) 945 } 946} 947 948type fakeClientFailureType int 949 950const ( 951 none fakeClientFailureType = iota 952 createError 953 watchError 954 certificateSigningRequestDenied 955) 956 957type fakeClient struct { 958 certificatesclient.CertificateSigningRequestInterface 959 failureType fakeClientFailureType 960 certificatePEM []byte 961 err error 962} 963 964func (c fakeClient) List(opts v1.ListOptions) (*certificates.CertificateSigningRequestList, error) { 965 if c.failureType == watchError { 966 if c.err != nil { 967 return nil, c.err 968 } 969 return nil, fmt.Errorf("Watch error") 970 } 971 csrReply := certificates.CertificateSigningRequestList{ 972 Items: []certificates.CertificateSigningRequest{ 973 {ObjectMeta: v1.ObjectMeta{UID: "fake-uid"}}, 974 }, 975 } 976 return &csrReply, nil 977} 978 979func (c fakeClient) Create(*certificates.CertificateSigningRequest) (*certificates.CertificateSigningRequest, error) { 980 if c.failureType == createError { 981 if c.err != nil { 982 return nil, c.err 983 } 984 return nil, fmt.Errorf("Create error") 985 } 986 csrReply := certificates.CertificateSigningRequest{} 987 csrReply.UID = "fake-uid" 988 return &csrReply, nil 989} 990 991func (c fakeClient) Watch(opts v1.ListOptions) (watch.Interface, error) { 992 if c.failureType == watchError { 993 if c.err != nil { 994 return nil, c.err 995 } 996 return nil, fmt.Errorf("Watch error") 997 } 998 return &fakeWatch{ 999 failureType: c.failureType, 1000 certificatePEM: c.certificatePEM, 1001 }, nil 1002} 1003 1004type fakeWatch struct { 1005 failureType fakeClientFailureType 1006 certificatePEM []byte 1007} 1008 1009func (w *fakeWatch) Stop() { 1010} 1011 1012func (w *fakeWatch) ResultChan() <-chan watch.Event { 1013 var condition certificates.CertificateSigningRequestCondition 1014 if w.failureType == certificateSigningRequestDenied { 1015 condition = certificates.CertificateSigningRequestCondition{ 1016 Type: certificates.CertificateDenied, 1017 } 1018 } else { 1019 condition = certificates.CertificateSigningRequestCondition{ 1020 Type: certificates.CertificateApproved, 1021 } 1022 } 1023 1024 csr := certificates.CertificateSigningRequest{ 1025 Status: certificates.CertificateSigningRequestStatus{ 1026 Conditions: []certificates.CertificateSigningRequestCondition{ 1027 condition, 1028 }, 1029 Certificate: []byte(w.certificatePEM), 1030 }, 1031 } 1032 csr.UID = "fake-uid" 1033 1034 c := make(chan watch.Event, 1) 1035 c <- watch.Event{ 1036 Type: watch.Added, 1037 Object: &csr, 1038 } 1039 return c 1040} 1041 1042type fakeStore struct { 1043 cert *tls.Certificate 1044} 1045 1046func (s *fakeStore) Current() (*tls.Certificate, error) { 1047 if s.cert == nil { 1048 noKeyErr := NoCertKeyError("") 1049 return nil, &noKeyErr 1050 } 1051 return s.cert, nil 1052} 1053 1054// Accepts the PEM data for the cert/key pair and makes the new cert/key 1055// pair the 'current' pair, that will be returned by future calls to 1056// Current(). 1057func (s *fakeStore) Update(certPEM, keyPEM []byte) (*tls.Certificate, error) { 1058 // In order to make the mocking work, whenever a cert/key pair is passed in 1059 // to be updated in the mock store, assume that the certificate manager 1060 // generated the key, and then asked the mock CertificateSigningRequest API 1061 // to sign it, then the faked API returned a canned response. The canned 1062 // signing response will not match the generated key. In order to make 1063 // things work out, search here for the correct matching key and use that 1064 // instead of the passed in key. That way this file of test code doesn't 1065 // have to implement an actual certificate signing process. 1066 for _, tc := range []*certificateData{storeCertData, bootstrapCertData, apiServerCertData} { 1067 if bytes.Equal(tc.certificatePEM, certPEM) { 1068 keyPEM = tc.keyPEM 1069 } 1070 } 1071 cert, err := tls.X509KeyPair(certPEM, keyPEM) 1072 if err != nil { 1073 return nil, err 1074 } 1075 now := time.Now() 1076 s.cert = &cert 1077 s.cert.Leaf = &x509.Certificate{ 1078 NotBefore: now.Add(-24 * time.Hour), 1079 NotAfter: now.Add(24 * time.Hour), 1080 } 1081 return s.cert, nil 1082} 1083 1084func certificatesEqual(c1 *tls.Certificate, c2 *tls.Certificate) bool { 1085 if c1 == nil || c2 == nil { 1086 return c1 == c2 1087 } 1088 if len(c1.Certificate) != len(c2.Certificate) { 1089 return false 1090 } 1091 for i := 0; i < len(c1.Certificate); i++ { 1092 if !bytes.Equal(c1.Certificate[i], c2.Certificate[i]) { 1093 return false 1094 } 1095 } 1096 return true 1097} 1098 1099func certificateString(c *tls.Certificate) string { 1100 if c == nil { 1101 return "certificate == nil" 1102 } 1103 if c.Leaf == nil { 1104 return "certificate.Leaf == nil" 1105 } 1106 return c.Leaf.Subject.CommonName 1107} 1108