1/* 2Copyright 2016 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package bootstrap 18 19import ( 20 "context" 21 "fmt" 22 "io" 23 "io/ioutil" 24 "os" 25 "reflect" 26 "testing" 27 28 certificatesv1 "k8s.io/api/certificates/v1" 29 apierrors "k8s.io/apimachinery/pkg/api/errors" 30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 "k8s.io/apimachinery/pkg/runtime" 32 "k8s.io/apimachinery/pkg/util/diff" 33 "k8s.io/apimachinery/pkg/watch" 34 "k8s.io/client-go/kubernetes/fake" 35 certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" 36 restclient "k8s.io/client-go/rest" 37 clienttesting "k8s.io/client-go/testing" 38 "k8s.io/client-go/util/certificate" 39 "k8s.io/client-go/util/keyutil" 40) 41 42func copyFile(src, dst string) (err error) { 43 in, err := os.Open(src) 44 if err != nil { 45 return err 46 } 47 defer in.Close() 48 out, err := os.Create(dst) 49 if err != nil { 50 return err 51 } 52 defer func() { 53 cerr := out.Close() 54 if err == nil { 55 err = cerr 56 } 57 }() 58 _, err = io.Copy(out, in) 59 return err 60} 61 62func TestLoadClientConfig(t *testing.T) { 63 //Create a temporary folder under tmp to store the required certificate files and configuration files. 64 fileDir := t.TempDir() 65 //Copy the required certificate file to the temporary directory. 66 copyFile("./testdata/mycertinvalid.crt", fileDir+"/mycertinvalid.crt") 67 copyFile("./testdata/mycertvalid.crt", fileDir+"/mycertvalid.crt") 68 copyFile("./testdata/mycertinvalid.key", fileDir+"/mycertinvalid.key") 69 copyFile("./testdata/mycertvalid.key", fileDir+"/mycertvalid.key") 70 testDataValid := []byte(` 71apiVersion: v1 72kind: Config 73clusters: 74- cluster: 75 certificate-authority: ca-a.crt 76 server: https://cluster-a.com 77 name: cluster-a 78- cluster: 79 server: https://cluster-b.com 80 name: cluster-b 81contexts: 82- context: 83 cluster: cluster-a 84 namespace: ns-a 85 user: user-a 86 name: context-a 87- context: 88 cluster: cluster-b 89 namespace: ns-b 90 user: user-b 91 name: context-b 92current-context: context-b 93users: 94- name: user-a 95 user: 96 client-certificate: mycertvalid.crt 97 client-key: mycertvalid.key 98- name: user-b 99 user: 100 client-certificate: mycertvalid.crt 101 client-key: mycertvalid.key 102 103`) 104 filevalid, err := ioutil.TempFile(fileDir, "kubeconfigvalid") 105 if err != nil { 106 t.Fatal(err) 107 } 108 ioutil.WriteFile(filevalid.Name(), testDataValid, os.FileMode(0755)) 109 110 testDataInvalid := []byte(` 111apiVersion: v1 112kind: Config 113clusters: 114- cluster: 115 certificate-authority: ca-a.crt 116 server: https://cluster-a.com 117 name: cluster-a 118- cluster: 119 server: https://cluster-b.com 120 name: cluster-b 121contexts: 122- context: 123 cluster: cluster-a 124 namespace: ns-a 125 user: user-a 126 name: context-a 127- context: 128 cluster: cluster-b 129 namespace: ns-b 130 user: user-b 131 name: context-b 132current-context: context-b 133users: 134- name: user-a 135 user: 136 client-certificate: mycertinvalid.crt 137 client-key: mycertinvalid.key 138- name: user-b 139 user: 140 client-certificate: mycertinvalid.crt 141 client-key: mycertinvalid.key 142 143`) 144 fileinvalid, err := ioutil.TempFile(fileDir, "kubeconfiginvalid") 145 if err != nil { 146 t.Fatal(err) 147 } 148 ioutil.WriteFile(fileinvalid.Name(), testDataInvalid, os.FileMode(0755)) 149 150 testDatabootstrap := []byte(` 151apiVersion: v1 152kind: Config 153clusters: 154- cluster: 155 certificate-authority: ca-a.crt 156 server: https://cluster-a.com 157 name: cluster-a 158- cluster: 159 server: https://cluster-b.com 160 name: cluster-b 161contexts: 162- context: 163 cluster: cluster-a 164 namespace: ns-a 165 user: user-a 166 name: context-a 167- context: 168 cluster: cluster-b 169 namespace: ns-b 170 user: user-b 171 name: context-b 172current-context: context-b 173users: 174- name: user-a 175 user: 176 token: mytoken-b 177- name: user-b 178 user: 179 token: mytoken-b 180`) 181 fileboot, err := ioutil.TempFile(fileDir, "kubeconfig") 182 if err != nil { 183 t.Fatal(err) 184 } 185 ioutil.WriteFile(fileboot.Name(), testDatabootstrap, os.FileMode(0755)) 186 187 dir, err := ioutil.TempDir(fileDir, "k8s-test-certstore-current") 188 if err != nil { 189 t.Fatalf("Unable to create the test directory %q: %v", dir, err) 190 } 191 192 store, err := certificate.NewFileStore("kubelet-client", dir, dir, "", "") 193 if err != nil { 194 t.Errorf("unable to build bootstrap cert store") 195 } 196 197 tests := []struct { 198 name string 199 kubeconfigPath string 200 bootstrapPath string 201 certDir string 202 expectedCertConfig *restclient.Config 203 expectedClientConfig *restclient.Config 204 }{ 205 { 206 name: "bootstrapPath is empty", 207 kubeconfigPath: filevalid.Name(), 208 bootstrapPath: "", 209 certDir: dir, 210 expectedCertConfig: &restclient.Config{ 211 Host: "https://cluster-b.com", 212 TLSClientConfig: restclient.TLSClientConfig{ 213 CertFile: fileDir + "/mycertvalid.crt", 214 KeyFile: fileDir + "/mycertvalid.key", 215 }, 216 BearerToken: "", 217 }, 218 expectedClientConfig: &restclient.Config{ 219 Host: "https://cluster-b.com", 220 TLSClientConfig: restclient.TLSClientConfig{ 221 CertFile: fileDir + "/mycertvalid.crt", 222 KeyFile: fileDir + "/mycertvalid.key", 223 }, 224 BearerToken: "", 225 }, 226 }, 227 { 228 name: "bootstrap path is set and the contents of kubeconfigPath are valid", 229 kubeconfigPath: filevalid.Name(), 230 bootstrapPath: fileboot.Name(), 231 certDir: dir, 232 expectedCertConfig: &restclient.Config{ 233 Host: "https://cluster-b.com", 234 TLSClientConfig: restclient.TLSClientConfig{ 235 CertFile: fileDir + "/mycertvalid.crt", 236 KeyFile: fileDir + "/mycertvalid.key", 237 }, 238 BearerToken: "", 239 }, 240 expectedClientConfig: &restclient.Config{ 241 Host: "https://cluster-b.com", 242 TLSClientConfig: restclient.TLSClientConfig{ 243 CertFile: fileDir + "/mycertvalid.crt", 244 KeyFile: fileDir + "/mycertvalid.key", 245 }, 246 BearerToken: "", 247 }, 248 }, 249 { 250 name: "bootstrap path is set and the contents of kubeconfigPath are not valid", 251 kubeconfigPath: fileinvalid.Name(), 252 bootstrapPath: fileboot.Name(), 253 certDir: dir, 254 expectedCertConfig: &restclient.Config{ 255 Host: "https://cluster-b.com", 256 TLSClientConfig: restclient.TLSClientConfig{}, 257 BearerToken: "mytoken-b", 258 }, 259 expectedClientConfig: &restclient.Config{ 260 Host: "https://cluster-b.com", 261 TLSClientConfig: restclient.TLSClientConfig{ 262 CertFile: store.CurrentPath(), 263 KeyFile: store.CurrentPath(), 264 }, 265 BearerToken: "", 266 }, 267 }, 268 } 269 270 for _, test := range tests { 271 t.Run(test.name, func(t *testing.T) { 272 certConfig, clientConfig, err := LoadClientConfig(test.kubeconfigPath, test.bootstrapPath, test.certDir) 273 if err != nil { 274 t.Fatal(err) 275 } 276 if !reflect.DeepEqual(certConfig, test.expectedCertConfig) { 277 t.Errorf("Unexpected certConfig: %s", diff.ObjectDiff(certConfig, test.expectedCertConfig)) 278 } 279 if !reflect.DeepEqual(clientConfig, test.expectedClientConfig) { 280 t.Errorf("Unexpected clientConfig: %s", diff.ObjectDiff(clientConfig, test.expectedClientConfig)) 281 } 282 }) 283 } 284} 285 286func TestLoadRESTClientConfig(t *testing.T) { 287 testData := []byte(` 288apiVersion: v1 289kind: Config 290clusters: 291- cluster: 292 certificate-authority: ca-a.crt 293 server: https://cluster-a.com 294 name: cluster-a 295- cluster: 296 certificate-authority-data: VGVzdA== 297 server: https://cluster-b.com 298 name: cluster-b 299contexts: 300- context: 301 cluster: cluster-a 302 namespace: ns-a 303 user: user-a 304 name: context-a 305- context: 306 cluster: cluster-b 307 namespace: ns-b 308 user: user-b 309 name: context-b 310current-context: context-b 311users: 312- name: user-a 313 user: 314 token: mytoken-a 315- name: user-b 316 user: 317 token: mytoken-b 318`) 319 f, err := ioutil.TempFile("", "kubeconfig") 320 if err != nil { 321 t.Fatal(err) 322 } 323 defer os.Remove(f.Name()) 324 ioutil.WriteFile(f.Name(), testData, os.FileMode(0755)) 325 326 config, err := loadRESTClientConfig(f.Name()) 327 if err != nil { 328 t.Fatal(err) 329 } 330 331 expectedConfig := &restclient.Config{ 332 Host: "https://cluster-b.com", 333 TLSClientConfig: restclient.TLSClientConfig{ 334 CAData: []byte(`Test`), 335 }, 336 BearerToken: "mytoken-b", 337 } 338 339 if !reflect.DeepEqual(config, expectedConfig) { 340 t.Errorf("Unexpected config: %s", diff.ObjectDiff(config, expectedConfig)) 341 } 342} 343 344func TestRequestNodeCertificateNoKeyData(t *testing.T) { 345 certData, err := requestNodeCertificate(context.TODO(), newClientset(fakeClient{}), []byte{}, "fake-node-name") 346 if err == nil { 347 t.Errorf("Got no error, wanted error an error because there was an empty private key passed in.") 348 } 349 if certData != nil { 350 t.Errorf("Got cert data, wanted nothing as there should have been an error.") 351 } 352} 353 354func TestRequestNodeCertificateErrorCreatingCSR(t *testing.T) { 355 client := newClientset(fakeClient{ 356 failureType: createError, 357 }) 358 privateKeyData, err := keyutil.MakeEllipticPrivateKeyPEM() 359 if err != nil { 360 t.Fatalf("Unable to generate a new private key: %v", err) 361 } 362 363 certData, err := requestNodeCertificate(context.TODO(), client, privateKeyData, "fake-node-name") 364 if err == nil { 365 t.Errorf("Got no error, wanted error an error because client.Create failed.") 366 } 367 if certData != nil { 368 t.Errorf("Got cert data, wanted nothing as there should have been an error.") 369 } 370} 371 372func TestRequestNodeCertificate(t *testing.T) { 373 privateKeyData, err := keyutil.MakeEllipticPrivateKeyPEM() 374 if err != nil { 375 t.Fatalf("Unable to generate a new private key: %v", err) 376 } 377 378 certData, err := requestNodeCertificate(context.TODO(), newClientset(fakeClient{}), privateKeyData, "fake-node-name") 379 if err != nil { 380 t.Errorf("Got %v, wanted no error.", err) 381 } 382 if certData == nil { 383 t.Errorf("Got nothing, expected a CSR.") 384 } 385} 386 387type failureType int 388 389const ( 390 noError failureType = iota //nolint:deadcode,varcheck 391 createError 392 certificateSigningRequestDenied 393) 394 395type fakeClient struct { 396 certificatesclient.CertificateSigningRequestInterface 397 failureType failureType 398} 399 400func newClientset(opts fakeClient) *fake.Clientset { 401 f := fake.NewSimpleClientset() 402 switch opts.failureType { 403 case createError: 404 f.PrependReactor("create", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { 405 switch action.GetResource().Version { 406 case "v1": 407 return true, nil, fmt.Errorf("create error") 408 default: 409 return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") 410 } 411 }) 412 default: 413 f.PrependReactor("create", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { 414 switch action.GetResource().Version { 415 case "v1": 416 return true, &certificatesv1.CertificateSigningRequest{ObjectMeta: metav1.ObjectMeta{Name: "fake-certificate-signing-request-name", UID: "fake-uid"}}, nil 417 default: 418 return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") 419 } 420 }) 421 f.PrependReactor("list", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { 422 switch action.GetResource().Version { 423 case "v1": 424 return true, &certificatesv1.CertificateSigningRequestList{Items: []certificatesv1.CertificateSigningRequest{{ObjectMeta: metav1.ObjectMeta{Name: "fake-certificate-signing-request-name", UID: "fake-uid"}}}}, nil 425 default: 426 return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") 427 } 428 }) 429 f.PrependWatchReactor("certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) { 430 switch action.GetResource().Version { 431 case "v1": 432 w := watch.NewFakeWithChanSize(1, false) 433 w.Add(opts.generateCSR()) 434 w.Stop() 435 return true, w, nil 436 437 default: 438 return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") 439 } 440 }) 441 } 442 return f 443} 444 445func (c fakeClient) generateCSR() runtime.Object { 446 var condition certificatesv1.CertificateSigningRequestCondition 447 var certificateData []byte 448 if c.failureType == certificateSigningRequestDenied { 449 condition = certificatesv1.CertificateSigningRequestCondition{ 450 Type: certificatesv1.CertificateDenied, 451 } 452 } else { 453 condition = certificatesv1.CertificateSigningRequestCondition{ 454 Type: certificatesv1.CertificateApproved, 455 } 456 certificateData = []byte(`issued certificate`) 457 } 458 459 csr := certificatesv1.CertificateSigningRequest{ 460 ObjectMeta: metav1.ObjectMeta{ 461 UID: "fake-uid", 462 }, 463 Status: certificatesv1.CertificateSigningRequestStatus{ 464 Conditions: []certificatesv1.CertificateSigningRequestCondition{ 465 condition, 466 }, 467 Certificate: certificateData, 468 }, 469 } 470 return &csr 471} 472