1/* 2 * Copyright 2020 gRPC authors. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package cdsbalancer 18 19import ( 20 "context" 21 "errors" 22 "fmt" 23 "regexp" 24 "testing" 25 26 "github.com/google/go-cmp/cmp" 27 "google.golang.org/grpc/attributes" 28 "google.golang.org/grpc/balancer" 29 "google.golang.org/grpc/credentials/local" 30 "google.golang.org/grpc/credentials/tls/certprovider" 31 "google.golang.org/grpc/credentials/xds" 32 "google.golang.org/grpc/internal" 33 xdscredsinternal "google.golang.org/grpc/internal/credentials/xds" 34 "google.golang.org/grpc/internal/testutils" 35 "google.golang.org/grpc/internal/xds/matcher" 36 "google.golang.org/grpc/resolver" 37 xdstestutils "google.golang.org/grpc/xds/internal/testutils" 38 "google.golang.org/grpc/xds/internal/testutils/fakeclient" 39 "google.golang.org/grpc/xds/internal/xdsclient" 40 "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" 41) 42 43const ( 44 fakeProvider1Name = "fake-certificate-provider-1" 45 fakeProvider2Name = "fake-certificate-provider-2" 46 fakeConfig = "my fake config" 47 testSAN = "test-san" 48) 49 50var ( 51 testSANMatchers = []matcher.StringMatcher{ 52 matcher.StringMatcherForTesting(newStringP(testSAN), nil, nil, nil, nil, true), 53 matcher.StringMatcherForTesting(nil, newStringP(testSAN), nil, nil, nil, false), 54 matcher.StringMatcherForTesting(nil, nil, newStringP(testSAN), nil, nil, false), 55 matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(testSAN), false), 56 matcher.StringMatcherForTesting(nil, nil, nil, newStringP(testSAN), nil, false), 57 } 58 fpb1, fpb2 *fakeProviderBuilder 59 bootstrapConfig *bootstrap.Config 60 cdsUpdateWithGoodSecurityCfg = xdsclient.ClusterUpdate{ 61 ClusterName: serviceName, 62 SecurityCfg: &xdsclient.SecurityConfig{ 63 RootInstanceName: "default1", 64 IdentityInstanceName: "default2", 65 SubjectAltNameMatchers: testSANMatchers, 66 }, 67 } 68 cdsUpdateWithMissingSecurityCfg = xdsclient.ClusterUpdate{ 69 ClusterName: serviceName, 70 SecurityCfg: &xdsclient.SecurityConfig{ 71 RootInstanceName: "not-default", 72 }, 73 } 74) 75 76func newStringP(s string) *string { 77 return &s 78} 79 80func init() { 81 fpb1 = &fakeProviderBuilder{name: fakeProvider1Name} 82 fpb2 = &fakeProviderBuilder{name: fakeProvider2Name} 83 cfg1, _ := fpb1.ParseConfig(fakeConfig + "1111") 84 cfg2, _ := fpb2.ParseConfig(fakeConfig + "2222") 85 bootstrapConfig = &bootstrap.Config{ 86 CertProviderConfigs: map[string]*certprovider.BuildableConfig{ 87 "default1": cfg1, 88 "default2": cfg2, 89 }, 90 } 91 certprovider.Register(fpb1) 92 certprovider.Register(fpb2) 93} 94 95// fakeProviderBuilder builds new instances of fakeProvider and interprets the 96// config provided to it as a string. 97type fakeProviderBuilder struct { 98 name string 99} 100 101func (b *fakeProviderBuilder) ParseConfig(config interface{}) (*certprovider.BuildableConfig, error) { 102 s, ok := config.(string) 103 if !ok { 104 return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config) 105 } 106 return certprovider.NewBuildableConfig(b.name, []byte(s), func(certprovider.BuildOptions) certprovider.Provider { 107 return &fakeProvider{ 108 Distributor: certprovider.NewDistributor(), 109 config: s, 110 } 111 }), nil 112} 113 114func (b *fakeProviderBuilder) Name() string { 115 return b.name 116} 117 118// fakeProvider is an implementation of the Provider interface which provides a 119// method for tests to invoke to push new key materials. 120type fakeProvider struct { 121 *certprovider.Distributor 122 config string 123} 124 125// Close helps implement the Provider interface. 126func (p *fakeProvider) Close() { 127 p.Distributor.Stop() 128} 129 130// setupWithXDSCreds performs all the setup steps required for tests which use 131// xDSCredentials. 132func setupWithXDSCreds(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) { 133 t.Helper() 134 xdsC := fakeclient.NewClient() 135 builder := balancer.Get(cdsName) 136 if builder == nil { 137 t.Fatalf("balancer.Get(%q) returned nil", cdsName) 138 } 139 // Create and pass xdsCredentials while building the CDS balancer. 140 creds, err := xds.NewClientCredentials(xds.ClientOptions{ 141 FallbackCreds: local.NewCredentials(), // Placeholder fallback credentials. 142 }) 143 if err != nil { 144 t.Fatalf("Failed to create xDS client creds: %v", err) 145 } 146 // Create a new CDS balancer and pass it a fake balancer.ClientConn which we 147 // can use to inspect the different calls made by the balancer. 148 tcc := xdstestutils.NewTestClientConn(t) 149 cdsB := builder.Build(tcc, balancer.BuildOptions{DialCreds: creds}) 150 151 // Override the creation of the EDS balancer to return a fake EDS balancer 152 // implementation. 153 edsB := newTestEDSBalancer() 154 oldEDSBalancerBuilder := newChildBalancer 155 newChildBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) { 156 edsB.parentCC = cc 157 return edsB, nil 158 } 159 160 // Push a ClientConnState update to the CDS balancer with a cluster name. 161 if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil { 162 t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err) 163 } 164 165 // Make sure the CDS balancer registers a Cluster watch with the xDS client 166 // passed via attributes in the above update. 167 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 168 defer ctxCancel() 169 gotCluster, err := xdsC.WaitForWatchCluster(ctx) 170 if err != nil { 171 t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) 172 } 173 if gotCluster != clusterName { 174 t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, clusterName) 175 } 176 177 return xdsC, cdsB.(*cdsBalancer), edsB, tcc, func() { 178 newChildBalancer = oldEDSBalancerBuilder 179 xdsC.Close() 180 } 181} 182 183// makeNewSubConn invokes the NewSubConn() call on the balancer.ClientConn 184// passed to the EDS balancer, and verifies that the CDS balancer forwards the 185// call appropriately to its parent balancer.ClientConn with or without 186// attributes bases on the value of wantFallback. 187func makeNewSubConn(ctx context.Context, edsCC balancer.ClientConn, parentCC *xdstestutils.TestClientConn, wantFallback bool) (balancer.SubConn, error) { 188 dummyAddr := "foo-address" 189 addrs := []resolver.Address{{Addr: dummyAddr}} 190 sc, err := edsCC.NewSubConn(addrs, balancer.NewSubConnOptions{}) 191 if err != nil { 192 return nil, fmt.Errorf("NewSubConn(%+v) on parent ClientConn failed: %v", addrs, err) 193 } 194 195 select { 196 case <-ctx.Done(): 197 return nil, errors.New("timeout when waiting for new SubConn") 198 case gotAddrs := <-parentCC.NewSubConnAddrsCh: 199 if len(gotAddrs) != 1 { 200 return nil, fmt.Errorf("NewSubConn expected 1 address, got %d", len(gotAddrs)) 201 } 202 if got, want := gotAddrs[0].Addr, addrs[0].Addr; got != want { 203 return nil, fmt.Errorf("resolver.Address passed to parent ClientConn has address %q, want %q", got, want) 204 } 205 getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdscredsinternal.HandshakeInfo) 206 hi := getHI(gotAddrs[0].Attributes) 207 if hi == nil { 208 return nil, errors.New("resolver.Address passed to parent ClientConn doesn't contain attributes") 209 } 210 if gotFallback := hi.UseFallbackCreds(); gotFallback != wantFallback { 211 return nil, fmt.Errorf("resolver.Address HandshakeInfo uses fallback creds? %v, want %v", gotFallback, wantFallback) 212 } 213 if !wantFallback { 214 if diff := cmp.Diff(testSANMatchers, hi.GetSANMatchersForTesting(), cmp.AllowUnexported(regexp.Regexp{})); diff != "" { 215 return nil, fmt.Errorf("unexpected diff in the list of SAN matchers (-got, +want):\n%s", diff) 216 } 217 } 218 } 219 return sc, nil 220} 221 222// TestSecurityConfigWithoutXDSCreds tests the case where xdsCredentials are not 223// in use, but the CDS balancer receives a Cluster update with security 224// configuration. Verifies that no certificate providers are created, and that 225// the address attributes added as part of the intercepted NewSubConn() method 226// indicate the use of fallback credentials. 227func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) { 228 // This creates a CDS balancer, pushes a ClientConnState update with a fake 229 // xdsClient, and makes sure that the CDS balancer registers a watch on the 230 // provided xdsClient. 231 xdsC, cdsB, edsB, tcc, cancel := setupWithWatch(t) 232 defer func() { 233 cancel() 234 cdsB.Close() 235 }() 236 237 // Override the provider builder function to push on a channel. We do not 238 // expect this function to be called as part of this test. 239 providerCh := testutils.NewChannel() 240 origBuildProvider := buildProvider 241 buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) { 242 p, err := origBuildProvider(c, id, cert, wi, wr) 243 providerCh.Send(nil) 244 return p, err 245 } 246 defer func() { buildProvider = origBuildProvider }() 247 248 // Here we invoke the watch callback registered on the fake xdsClient. This 249 // will trigger the watch handler on the CDS balancer, which will attempt to 250 // create a new EDS balancer. The fake EDS balancer created above will be 251 // returned to the CDS balancer, because we have overridden the 252 // newChildBalancer function as part of test setup. 253 cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName} 254 wantCCS := edsCCS(serviceName, nil, false, nil) 255 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 256 defer ctxCancel() 257 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { 258 t.Fatal(err) 259 } 260 261 // Make a NewSubConn and verify that the HandshakeInfo does not contain any 262 // certificate providers, forcing the credentials implementation to use 263 // fallback creds. 264 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil { 265 t.Fatal(err) 266 } 267 268 // Again, since xdsCredentials are not in use, no certificate providers 269 // should have been initialized by the CDS balancer. 270 sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) 271 defer sCancel() 272 if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded { 273 t.Fatal("cds balancer created certificate providers when not using xds credentials") 274 } 275} 276 277// TestNoSecurityConfigWithXDSCreds tests the case where xdsCredentials are in 278// use, but the CDS balancer receives a Cluster update without security 279// configuration. Verifies that no certificate providers are created, and that 280// the address attributes added as part of the intercepted NewSubConn() method 281// indicate the use of fallback credentials. 282func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) { 283 // This creates a CDS balancer which uses xdsCredentials, pushes a 284 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 285 // balancer registers a watch on the provided xdsClient. 286 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 287 defer func() { 288 cancel() 289 cdsB.Close() 290 }() 291 292 // Override the provider builder function to push on a channel. We do not 293 // expect this function to be called as part of this test. 294 providerCh := testutils.NewChannel() 295 origBuildProvider := buildProvider 296 buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) { 297 p, err := origBuildProvider(c, id, cert, wi, wr) 298 providerCh.Send(nil) 299 return p, err 300 } 301 defer func() { buildProvider = origBuildProvider }() 302 303 // Here we invoke the watch callback registered on the fake xdsClient. This 304 // will trigger the watch handler on the CDS balancer, which will attempt to 305 // create a new EDS balancer. The fake EDS balancer created above will be 306 // returned to the CDS balancer, because we have overridden the 307 // newChildBalancer function as part of test setup. No security config is 308 // passed to the CDS balancer as part of this update. 309 cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName} 310 wantCCS := edsCCS(serviceName, nil, false, nil) 311 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 312 defer ctxCancel() 313 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { 314 t.Fatal(err) 315 } 316 317 // Make a NewSubConn and verify that the HandshakeInfo does not contain any 318 // certificate providers, forcing the credentials implementation to use 319 // fallback creds. 320 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil { 321 t.Fatal(err) 322 } 323 324 // Again, since no security configuration was received, no certificate 325 // providers should have been initialized by the CDS balancer. 326 sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) 327 defer sCancel() 328 if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded { 329 t.Fatal("cds balancer created certificate providers when not using xds credentials") 330 } 331} 332 333// TestSecurityConfigNotFoundInBootstrap tests the case where the security 334// config returned by the xDS server cannot be resolved based on the contents of 335// the bootstrap config. Verifies that the balancer puts the channel in a failed 336// state, and returns an error picker. 337func (s) TestSecurityConfigNotFoundInBootstrap(t *testing.T) { 338 // We test two cases here: 339 // 0: Bootstrap contains security config. But received plugin instance name 340 // is not found in the bootstrap config. 341 // 1: Bootstrap contains no security config. 342 for i := 0; i < 2; i++ { 343 // This creates a CDS balancer which uses xdsCredentials, pushes a 344 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 345 // balancer registers a watch on the provided xdsClient. 346 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 347 defer func() { 348 cancel() 349 cdsB.Close() 350 }() 351 352 if i == 0 { 353 // Set the bootstrap config used by the fake client. 354 xdsC.SetBootstrapConfig(bootstrapConfig) 355 } 356 357 // Here we invoke the watch callback registered on the fake xdsClient. A bad 358 // security config is passed here. So, we expect the CDS balancer to not 359 // create an EDS balancer and instead reject this update and put the channel 360 // in a bad state. 361 xdsC.InvokeWatchClusterCallback(cdsUpdateWithMissingSecurityCfg, nil) 362 363 // The CDS balancer has not yet created an EDS balancer. So, this bad 364 // watcher update should not be forwarded forwarded to our fake EDS balancer 365 // as an error. 366 sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) 367 defer sCancel() 368 if err := edsB.waitForResolverError(sCtx, nil); err != context.DeadlineExceeded { 369 t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)") 370 } 371 372 // Make sure the CDS balancer reports an error picker. 373 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 374 defer ctxCancel() 375 if err := tcc.WaitForErrPicker(ctx); err != nil { 376 t.Fatal(err) 377 } 378 } 379} 380 381// TestCertproviderStoreError tests the case where the certprovider.Store 382// returns an error when the CDS balancer attempts to create a provider. 383func (s) TestCertproviderStoreError(t *testing.T) { 384 // This creates a CDS balancer which uses xdsCredentials, pushes a 385 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 386 // balancer registers a watch on the provided xdsClient. 387 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 388 defer func() { 389 cancel() 390 cdsB.Close() 391 }() 392 393 // Override the provider builder function to return an error. 394 origBuildProvider := buildProvider 395 buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) { 396 return nil, errors.New("certprovider store error") 397 } 398 defer func() { buildProvider = origBuildProvider }() 399 400 // Set the bootstrap config used by the fake client. 401 xdsC.SetBootstrapConfig(bootstrapConfig) 402 403 // Here we invoke the watch callback registered on the fake xdsClient. Even 404 // though the received update is good, the certprovider.Store is configured 405 // to return an error. So, CDS balancer should reject this config and report 406 // an error. 407 xdsC.InvokeWatchClusterCallback(cdsUpdateWithGoodSecurityCfg, nil) 408 409 // The CDS balancer has not yet created an EDS balancer. So, this bad 410 // watcher update should not be forwarded forwarded to our fake EDS balancer 411 // as an error. 412 sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) 413 defer sCancel() 414 if err := edsB.waitForResolverError(sCtx, nil); err != context.DeadlineExceeded { 415 t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)") 416 } 417 418 // Make sure the CDS balancer reports an error picker. 419 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 420 defer ctxCancel() 421 if err := tcc.WaitForErrPicker(ctx); err != nil { 422 t.Fatal(err) 423 } 424} 425 426func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) { 427 // This creates a CDS balancer which uses xdsCredentials, pushes a 428 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 429 // balancer registers a watch on the provided xdsClient. 430 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 431 defer func() { 432 cancel() 433 cdsB.Close() 434 }() 435 436 // Set the bootstrap config used by the fake client. 437 xdsC.SetBootstrapConfig(bootstrapConfig) 438 439 // Here we invoke the watch callback registered on the fake xdsClient. A bad 440 // security config is passed here. So, we expect the CDS balancer to not 441 // create an EDS balancer and instead reject this update and put the channel 442 // in a bad state. 443 xdsC.InvokeWatchClusterCallback(cdsUpdateWithMissingSecurityCfg, nil) 444 445 // The CDS balancer has not yet created an EDS balancer. So, this bad 446 // watcher update should not be forwarded forwarded to our fake EDS balancer 447 // as an error. 448 sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) 449 defer sCancel() 450 if err := edsB.waitForResolverError(sCtx, nil); err != context.DeadlineExceeded { 451 t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)") 452 } 453 454 // Make sure the CDS balancer reports an error picker. 455 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 456 defer ctxCancel() 457 if err := tcc.WaitForErrPicker(ctx); err != nil { 458 t.Fatal(err) 459 } 460 461 // Here we invoke the watch callback registered on the fake xdsClient. This 462 // will trigger the watch handler on the CDS balancer, which will attempt to 463 // create a new EDS balancer. The fake EDS balancer created above will be 464 // returned to the CDS balancer, because we have overridden the 465 // newChildBalancer function as part of test setup. 466 wantCCS := edsCCS(serviceName, nil, false, nil) 467 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil { 468 t.Fatal(err) 469 } 470 471 // Make a NewSubConn and verify that attributes are added. 472 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil { 473 t.Fatal(err) 474 } 475} 476 477// TestGoodSecurityConfig tests the case where the CDS balancer receives 478// security configuration as part of the Cluster resource which can be 479// successfully resolved using the bootstrap file contents. Verifies that 480// certificate providers are created, and that the NewSubConn() call adds 481// appropriate address attributes. 482func (s) TestGoodSecurityConfig(t *testing.T) { 483 // This creates a CDS balancer which uses xdsCredentials, pushes a 484 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 485 // balancer registers a watch on the provided xdsClient. 486 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 487 defer func() { 488 cancel() 489 cdsB.Close() 490 }() 491 492 // Set the bootstrap config used by the fake client. 493 xdsC.SetBootstrapConfig(bootstrapConfig) 494 495 // Here we invoke the watch callback registered on the fake xdsClient. This 496 // will trigger the watch handler on the CDS balancer, which will attempt to 497 // create a new EDS balancer. The fake EDS balancer created above will be 498 // returned to the CDS balancer, because we have overridden the 499 // newChildBalancer function as part of test setup. 500 wantCCS := edsCCS(serviceName, nil, false, nil) 501 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 502 defer ctxCancel() 503 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil { 504 t.Fatal(err) 505 } 506 507 // Make a NewSubConn and verify that attributes are added. 508 sc, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false) 509 if err != nil { 510 t.Fatal(err) 511 } 512 513 // Invoke UpdateAddresses and verify that attributes are added. 514 dummyAddr := "bar-address" 515 addrs := []resolver.Address{{Addr: dummyAddr}} 516 edsB.parentCC.UpdateAddresses(sc, addrs) 517 select { 518 case <-ctx.Done(): 519 t.Fatal("timeout when waiting for addresses to be updated on the subConn") 520 case gotAddrs := <-tcc.UpdateAddressesAddrsCh: 521 if len(gotAddrs) != 1 { 522 t.Fatalf("UpdateAddresses expected 1 address, got %d", len(gotAddrs)) 523 } 524 if got, want := gotAddrs[0].Addr, addrs[0].Addr; got != want { 525 t.Fatalf("resolver.Address passed to parent ClientConn through UpdateAddresses() has address %q, want %q", got, want) 526 } 527 getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdscredsinternal.HandshakeInfo) 528 hi := getHI(gotAddrs[0].Attributes) 529 if hi == nil { 530 t.Fatal("resolver.Address passed to parent ClientConn through UpdateAddresses() doesn't contain attributes") 531 } 532 } 533} 534 535func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) { 536 // This creates a CDS balancer which uses xdsCredentials, pushes a 537 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 538 // balancer registers a watch on the provided xdsClient. 539 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 540 defer func() { 541 cancel() 542 cdsB.Close() 543 }() 544 545 // Set the bootstrap config used by the fake client. 546 xdsC.SetBootstrapConfig(bootstrapConfig) 547 548 // Here we invoke the watch callback registered on the fake xdsClient. This 549 // will trigger the watch handler on the CDS balancer, which will attempt to 550 // create a new EDS balancer. The fake EDS balancer created above will be 551 // returned to the CDS balancer, because we have overridden the 552 // newChildBalancer function as part of test setup. 553 wantCCS := edsCCS(serviceName, nil, false, nil) 554 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 555 defer ctxCancel() 556 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil { 557 t.Fatal(err) 558 } 559 560 // Make a NewSubConn and verify that attributes are added. 561 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil { 562 t.Fatal(err) 563 } 564 565 // Here we invoke the watch callback registered on the fake xdsClient with 566 // an update which contains bad security config. So, we expect the CDS 567 // balancer to forward this error to the EDS balancer and eventually the 568 // channel needs to be put in a bad state. 569 cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName} 570 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { 571 t.Fatal(err) 572 } 573 574 // Make a NewSubConn and verify that fallback creds are used. 575 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil { 576 t.Fatal(err) 577 } 578} 579 580// TestSecurityConfigUpdate_GoodToBad tests the case where the first security 581// config returned by the xDS server is successful, but the second update cannot 582// be resolved based on the contents of the bootstrap config. Verifies that the 583// error is forwarded to the EDS balancer (which was created as part of the 584// first successful update). 585func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) { 586 // This creates a CDS balancer which uses xdsCredentials, pushes a 587 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 588 // balancer registers a watch on the provided xdsClient. 589 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 590 defer func() { 591 cancel() 592 cdsB.Close() 593 }() 594 595 // Set the bootstrap config used by the fake client. 596 xdsC.SetBootstrapConfig(bootstrapConfig) 597 598 // Here we invoke the watch callback registered on the fake xdsClient. This 599 // will trigger the watch handler on the CDS balancer, which will attempt to 600 // create a new EDS balancer. The fake EDS balancer created above will be 601 // returned to the CDS balancer, because we have overridden the 602 // newChildBalancer function as part of test setup. 603 wantCCS := edsCCS(serviceName, nil, false, nil) 604 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 605 defer ctxCancel() 606 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil { 607 t.Fatal(err) 608 } 609 610 // Make a NewSubConn and verify that attributes are added. 611 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil { 612 t.Fatal(err) 613 } 614 615 // Here we invoke the watch callback registered on the fake xdsClient with 616 // an update which contains bad security config. So, we expect the CDS 617 // balancer to forward this error to the EDS balancer and eventually the 618 // channel needs to be put in a bad state. 619 xdsC.InvokeWatchClusterCallback(cdsUpdateWithMissingSecurityCfg, nil) 620 621 // We manually check that an error is forwarded to the EDS balancer instead 622 // of using one of the helper methods on the testEDSBalancer, because all we 623 // care here is whether an error is sent to it or not. We don't care about 624 // the exact error. 625 gotErr, err := edsB.resolverErrCh.Receive(ctx) 626 if err != nil { 627 t.Fatal("timeout waiting for CDS balancer to forward error to EDS balancer upon receipt of bad security config") 628 } 629 if gotErr == nil { 630 t.Fatal("CDS balancer did not forward error to EDS balancer upon receipt of bad security config") 631 } 632 633 // Since the error being pushed here is not a resource-not-found-error, the 634 // registered watch should not be cancelled. 635 sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) 636 defer sCancel() 637 if _, err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded { 638 t.Fatal("cluster watch cancelled for a non-resource-not-found-error") 639 } 640} 641 642// TestSecurityConfigUpdate_GoodToGood tests the case where the CDS balancer 643// receives two different but successful updates with security configuration. 644// Verifies that appropriate providers are created, and that address attributes 645// are added. 646func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) { 647 // This creates a CDS balancer which uses xdsCredentials, pushes a 648 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 649 // balancer registers a watch on the provided xdsClient. 650 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 651 defer func() { 652 cancel() 653 cdsB.Close() 654 }() 655 656 // Override the provider builder function to push on a channel. 657 providerCh := testutils.NewChannel() 658 origBuildProvider := buildProvider 659 buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) { 660 p, err := origBuildProvider(c, id, cert, wi, wr) 661 providerCh.Send(nil) 662 return p, err 663 } 664 defer func() { buildProvider = origBuildProvider }() 665 666 // Set the bootstrap config used by the fake client. 667 xdsC.SetBootstrapConfig(bootstrapConfig) 668 669 // Here we invoke the watch callback registered on the fake xdsClient. This 670 // will trigger the watch handler on the CDS balancer, which will attempt to 671 // create a new EDS balancer. The fake EDS balancer created above will be 672 // returned to the CDS balancer, because we have overridden the 673 // newChildBalancer function as part of test setup. 674 cdsUpdate := xdsclient.ClusterUpdate{ 675 ClusterName: serviceName, 676 SecurityCfg: &xdsclient.SecurityConfig{ 677 RootInstanceName: "default1", 678 SubjectAltNameMatchers: testSANMatchers, 679 }, 680 } 681 wantCCS := edsCCS(serviceName, nil, false, nil) 682 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 683 defer ctxCancel() 684 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { 685 t.Fatal(err) 686 } 687 688 // We specified only the root provider. So, expect only one provider here. 689 if _, err := providerCh.Receive(ctx); err != nil { 690 t.Fatalf("Failed to create certificate provider upon receipt of security config") 691 } 692 693 // Make a NewSubConn and verify that attributes are added. 694 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil { 695 t.Fatal(err) 696 } 697 698 // Push another update with a new security configuration. 699 cdsUpdate = xdsclient.ClusterUpdate{ 700 ClusterName: serviceName, 701 SecurityCfg: &xdsclient.SecurityConfig{ 702 RootInstanceName: "default2", 703 SubjectAltNameMatchers: testSANMatchers, 704 }, 705 } 706 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { 707 t.Fatal(err) 708 } 709 710 // We specified only the root provider. So, expect only one provider here. 711 if _, err := providerCh.Receive(ctx); err != nil { 712 t.Fatalf("Failed to create certificate provider upon receipt of security config") 713 } 714 715 // Make a NewSubConn and verify that attributes are added. 716 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil { 717 t.Fatal(err) 718 } 719 720 // The HandshakeInfo type does not expose its internals. So, we cannot 721 // verify that the HandshakeInfo carried by the attributes have actually 722 // been changed. This will be covered in e2e/interop tests. 723 // TODO(easwars): Remove this TODO once appropriate e2e/intertop tests have 724 // been added. 725} 726