1// +build go1.12 2 3/* 4 * Copyright 2020 gRPC authors. 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 19package cdsbalancer 20 21import ( 22 "context" 23 "errors" 24 "fmt" 25 "regexp" 26 "testing" 27 28 "github.com/google/go-cmp/cmp" 29 "google.golang.org/grpc/attributes" 30 "google.golang.org/grpc/balancer" 31 "google.golang.org/grpc/credentials/local" 32 "google.golang.org/grpc/credentials/tls/certprovider" 33 "google.golang.org/grpc/credentials/xds" 34 "google.golang.org/grpc/internal" 35 xdscredsinternal "google.golang.org/grpc/internal/credentials/xds" 36 "google.golang.org/grpc/internal/testutils" 37 "google.golang.org/grpc/internal/xds/matcher" 38 "google.golang.org/grpc/resolver" 39 xdstestutils "google.golang.org/grpc/xds/internal/testutils" 40 "google.golang.org/grpc/xds/internal/testutils/fakeclient" 41 "google.golang.org/grpc/xds/internal/xdsclient" 42 "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" 43) 44 45const ( 46 fakeProvider1Name = "fake-certificate-provider-1" 47 fakeProvider2Name = "fake-certificate-provider-2" 48 fakeConfig = "my fake config" 49 testSAN = "test-san" 50) 51 52var ( 53 testSANMatchers = []matcher.StringMatcher{ 54 matcher.StringMatcherForTesting(newStringP(testSAN), nil, nil, nil, nil, true), 55 matcher.StringMatcherForTesting(nil, newStringP(testSAN), nil, nil, nil, false), 56 matcher.StringMatcherForTesting(nil, nil, newStringP(testSAN), nil, nil, false), 57 matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(testSAN), false), 58 matcher.StringMatcherForTesting(nil, nil, nil, newStringP(testSAN), nil, false), 59 } 60 fpb1, fpb2 *fakeProviderBuilder 61 bootstrapConfig *bootstrap.Config 62 cdsUpdateWithGoodSecurityCfg = xdsclient.ClusterUpdate{ 63 ClusterName: serviceName, 64 SecurityCfg: &xdsclient.SecurityConfig{ 65 RootInstanceName: "default1", 66 IdentityInstanceName: "default2", 67 SubjectAltNameMatchers: testSANMatchers, 68 }, 69 } 70 cdsUpdateWithMissingSecurityCfg = xdsclient.ClusterUpdate{ 71 ClusterName: serviceName, 72 SecurityCfg: &xdsclient.SecurityConfig{ 73 RootInstanceName: "not-default", 74 }, 75 } 76) 77 78func newStringP(s string) *string { 79 return &s 80} 81 82func init() { 83 fpb1 = &fakeProviderBuilder{name: fakeProvider1Name} 84 fpb2 = &fakeProviderBuilder{name: fakeProvider2Name} 85 cfg1, _ := fpb1.ParseConfig(fakeConfig + "1111") 86 cfg2, _ := fpb2.ParseConfig(fakeConfig + "2222") 87 bootstrapConfig = &bootstrap.Config{ 88 CertProviderConfigs: map[string]*certprovider.BuildableConfig{ 89 "default1": cfg1, 90 "default2": cfg2, 91 }, 92 } 93 certprovider.Register(fpb1) 94 certprovider.Register(fpb2) 95} 96 97// fakeProviderBuilder builds new instances of fakeProvider and interprets the 98// config provided to it as a string. 99type fakeProviderBuilder struct { 100 name string 101} 102 103func (b *fakeProviderBuilder) ParseConfig(config interface{}) (*certprovider.BuildableConfig, error) { 104 s, ok := config.(string) 105 if !ok { 106 return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config) 107 } 108 return certprovider.NewBuildableConfig(b.name, []byte(s), func(certprovider.BuildOptions) certprovider.Provider { 109 return &fakeProvider{ 110 Distributor: certprovider.NewDistributor(), 111 config: s, 112 } 113 }), nil 114} 115 116func (b *fakeProviderBuilder) Name() string { 117 return b.name 118} 119 120// fakeProvider is an implementation of the Provider interface which provides a 121// method for tests to invoke to push new key materials. 122type fakeProvider struct { 123 *certprovider.Distributor 124 config string 125} 126 127// Close helps implement the Provider interface. 128func (p *fakeProvider) Close() { 129 p.Distributor.Stop() 130} 131 132// setupWithXDSCreds performs all the setup steps required for tests which use 133// xDSCredentials. 134func setupWithXDSCreds(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) { 135 t.Helper() 136 xdsC := fakeclient.NewClient() 137 builder := balancer.Get(cdsName) 138 if builder == nil { 139 t.Fatalf("balancer.Get(%q) returned nil", cdsName) 140 } 141 // Create and pass xdsCredentials while building the CDS balancer. 142 creds, err := xds.NewClientCredentials(xds.ClientOptions{ 143 FallbackCreds: local.NewCredentials(), // Placeholder fallback credentials. 144 }) 145 if err != nil { 146 t.Fatalf("Failed to create xDS client creds: %v", err) 147 } 148 // Create a new CDS balancer and pass it a fake balancer.ClientConn which we 149 // can use to inspect the different calls made by the balancer. 150 tcc := xdstestutils.NewTestClientConn(t) 151 cdsB := builder.Build(tcc, balancer.BuildOptions{DialCreds: creds}) 152 153 // Override the creation of the EDS balancer to return a fake EDS balancer 154 // implementation. 155 edsB := newTestEDSBalancer() 156 oldEDSBalancerBuilder := newChildBalancer 157 newChildBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) { 158 edsB.parentCC = cc 159 return edsB, nil 160 } 161 162 // Push a ClientConnState update to the CDS balancer with a cluster name. 163 if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil { 164 t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err) 165 } 166 167 // Make sure the CDS balancer registers a Cluster watch with the xDS client 168 // passed via attributes in the above update. 169 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 170 defer ctxCancel() 171 gotCluster, err := xdsC.WaitForWatchCluster(ctx) 172 if err != nil { 173 t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) 174 } 175 if gotCluster != clusterName { 176 t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, clusterName) 177 } 178 179 return xdsC, cdsB.(*cdsBalancer), edsB, tcc, func() { 180 newChildBalancer = oldEDSBalancerBuilder 181 xdsC.Close() 182 } 183} 184 185// makeNewSubConn invokes the NewSubConn() call on the balancer.ClientConn 186// passed to the EDS balancer, and verifies that the CDS balancer forwards the 187// call appropriately to its parent balancer.ClientConn with or without 188// attributes bases on the value of wantFallback. 189func makeNewSubConn(ctx context.Context, edsCC balancer.ClientConn, parentCC *xdstestutils.TestClientConn, wantFallback bool) (balancer.SubConn, error) { 190 dummyAddr := "foo-address" 191 addrs := []resolver.Address{{Addr: dummyAddr}} 192 sc, err := edsCC.NewSubConn(addrs, balancer.NewSubConnOptions{}) 193 if err != nil { 194 return nil, fmt.Errorf("NewSubConn(%+v) on parent ClientConn failed: %v", addrs, err) 195 } 196 197 select { 198 case <-ctx.Done(): 199 return nil, errors.New("timeout when waiting for new SubConn") 200 case gotAddrs := <-parentCC.NewSubConnAddrsCh: 201 if len(gotAddrs) != 1 { 202 return nil, fmt.Errorf("NewSubConn expected 1 address, got %d", len(gotAddrs)) 203 } 204 if got, want := gotAddrs[0].Addr, addrs[0].Addr; got != want { 205 return nil, fmt.Errorf("resolver.Address passed to parent ClientConn has address %q, want %q", got, want) 206 } 207 getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdscredsinternal.HandshakeInfo) 208 hi := getHI(gotAddrs[0].Attributes) 209 if hi == nil { 210 return nil, errors.New("resolver.Address passed to parent ClientConn doesn't contain attributes") 211 } 212 if gotFallback := hi.UseFallbackCreds(); gotFallback != wantFallback { 213 return nil, fmt.Errorf("resolver.Address HandshakeInfo uses fallback creds? %v, want %v", gotFallback, wantFallback) 214 } 215 if !wantFallback { 216 if diff := cmp.Diff(testSANMatchers, hi.GetSANMatchersForTesting(), cmp.AllowUnexported(regexp.Regexp{})); diff != "" { 217 return nil, fmt.Errorf("unexpected diff in the list of SAN matchers (-got, +want):\n%s", diff) 218 } 219 } 220 } 221 return sc, nil 222} 223 224// TestSecurityConfigWithoutXDSCreds tests the case where xdsCredentials are not 225// in use, but the CDS balancer receives a Cluster update with security 226// configuration. Verifies that no certificate providers are created, and that 227// the address attributes added as part of the intercepted NewSubConn() method 228// indicate the use of fallback credentials. 229func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) { 230 // This creates a CDS balancer, pushes a ClientConnState update with a fake 231 // xdsClient, and makes sure that the CDS balancer registers a watch on the 232 // provided xdsClient. 233 xdsC, cdsB, edsB, tcc, cancel := setupWithWatch(t) 234 defer func() { 235 cancel() 236 cdsB.Close() 237 }() 238 239 // Override the provider builder function to push on a channel. We do not 240 // expect this function to be called as part of this test. 241 providerCh := testutils.NewChannel() 242 origBuildProvider := buildProvider 243 buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) { 244 p, err := origBuildProvider(c, id, cert, wi, wr) 245 providerCh.Send(nil) 246 return p, err 247 } 248 defer func() { buildProvider = origBuildProvider }() 249 250 // Here we invoke the watch callback registered on the fake xdsClient. This 251 // will trigger the watch handler on the CDS balancer, which will attempt to 252 // create a new EDS balancer. The fake EDS balancer created above will be 253 // returned to the CDS balancer, because we have overridden the 254 // newChildBalancer function as part of test setup. 255 cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName} 256 wantCCS := edsCCS(serviceName, nil, false) 257 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 258 defer ctxCancel() 259 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { 260 t.Fatal(err) 261 } 262 263 // Make a NewSubConn and verify that the HandshakeInfo does not contain any 264 // certificate providers, forcing the credentials implementation to use 265 // fallback creds. 266 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil { 267 t.Fatal(err) 268 } 269 270 // Again, since xdsCredentials are not in use, no certificate providers 271 // should have been initialized by the CDS balancer. 272 sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) 273 defer sCancel() 274 if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded { 275 t.Fatal("cds balancer created certificate providers when not using xds credentials") 276 } 277} 278 279// TestNoSecurityConfigWithXDSCreds tests the case where xdsCredentials are in 280// use, but the CDS balancer receives a Cluster update without security 281// configuration. Verifies that no certificate providers are created, and that 282// the address attributes added as part of the intercepted NewSubConn() method 283// indicate the use of fallback credentials. 284func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) { 285 // This creates a CDS balancer which uses xdsCredentials, pushes a 286 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 287 // balancer registers a watch on the provided xdsClient. 288 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 289 defer func() { 290 cancel() 291 cdsB.Close() 292 }() 293 294 // Override the provider builder function to push on a channel. We do not 295 // expect this function to be called as part of this test. 296 providerCh := testutils.NewChannel() 297 origBuildProvider := buildProvider 298 buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) { 299 p, err := origBuildProvider(c, id, cert, wi, wr) 300 providerCh.Send(nil) 301 return p, err 302 } 303 defer func() { buildProvider = origBuildProvider }() 304 305 // Here we invoke the watch callback registered on the fake xdsClient. This 306 // will trigger the watch handler on the CDS balancer, which will attempt to 307 // create a new EDS balancer. The fake EDS balancer created above will be 308 // returned to the CDS balancer, because we have overridden the 309 // newChildBalancer function as part of test setup. No security config is 310 // passed to the CDS balancer as part of this update. 311 cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName} 312 wantCCS := edsCCS(serviceName, nil, false) 313 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 314 defer ctxCancel() 315 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { 316 t.Fatal(err) 317 } 318 319 // Make a NewSubConn and verify that the HandshakeInfo does not contain any 320 // certificate providers, forcing the credentials implementation to use 321 // fallback creds. 322 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil { 323 t.Fatal(err) 324 } 325 326 // Again, since no security configuration was received, no certificate 327 // providers should have been initialized by the CDS balancer. 328 sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) 329 defer sCancel() 330 if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded { 331 t.Fatal("cds balancer created certificate providers when not using xds credentials") 332 } 333} 334 335// TestSecurityConfigNotFoundInBootstrap tests the case where the security 336// config returned by the xDS server cannot be resolved based on the contents of 337// the bootstrap config. Verifies that the balancer puts the channel in a failed 338// state, and returns an error picker. 339func (s) TestSecurityConfigNotFoundInBootstrap(t *testing.T) { 340 // We test two cases here: 341 // 0: Bootstrap contains security config. But received plugin instance name 342 // is not found in the bootstrap config. 343 // 1: Bootstrap contains no security config. 344 for i := 0; i < 2; i++ { 345 // This creates a CDS balancer which uses xdsCredentials, pushes a 346 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 347 // balancer registers a watch on the provided xdsClient. 348 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 349 defer func() { 350 cancel() 351 cdsB.Close() 352 }() 353 354 if i == 0 { 355 // Set the bootstrap config used by the fake client. 356 xdsC.SetBootstrapConfig(bootstrapConfig) 357 } 358 359 // Here we invoke the watch callback registered on the fake xdsClient. A bad 360 // security config is passed here. So, we expect the CDS balancer to not 361 // create an EDS balancer and instead reject this update and put the channel 362 // in a bad state. 363 xdsC.InvokeWatchClusterCallback(cdsUpdateWithMissingSecurityCfg, nil) 364 365 // The CDS balancer has not yet created an EDS balancer. So, this bad 366 // watcher update should not be forwarded forwarded to our fake EDS balancer 367 // as an error. 368 sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) 369 defer sCancel() 370 if err := edsB.waitForResolverError(sCtx, nil); err != context.DeadlineExceeded { 371 t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)") 372 } 373 374 // Make sure the CDS balancer reports an error picker. 375 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 376 defer ctxCancel() 377 if err := tcc.WaitForErrPicker(ctx); err != nil { 378 t.Fatal(err) 379 } 380 } 381} 382 383// TestCertproviderStoreError tests the case where the certprovider.Store 384// returns an error when the CDS balancer attempts to create a provider. 385func (s) TestCertproviderStoreError(t *testing.T) { 386 // This creates a CDS balancer which uses xdsCredentials, pushes a 387 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 388 // balancer registers a watch on the provided xdsClient. 389 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 390 defer func() { 391 cancel() 392 cdsB.Close() 393 }() 394 395 // Override the provider builder function to return an error. 396 origBuildProvider := buildProvider 397 buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) { 398 return nil, errors.New("certprovider store error") 399 } 400 defer func() { buildProvider = origBuildProvider }() 401 402 // Set the bootstrap config used by the fake client. 403 xdsC.SetBootstrapConfig(bootstrapConfig) 404 405 // Here we invoke the watch callback registered on the fake xdsClient. Even 406 // though the received update is good, the certprovider.Store is configured 407 // to return an error. So, CDS balancer should reject this config and report 408 // an error. 409 xdsC.InvokeWatchClusterCallback(cdsUpdateWithGoodSecurityCfg, nil) 410 411 // The CDS balancer has not yet created an EDS balancer. So, this bad 412 // watcher update should not be forwarded forwarded to our fake EDS balancer 413 // as an error. 414 sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) 415 defer sCancel() 416 if err := edsB.waitForResolverError(sCtx, nil); err != context.DeadlineExceeded { 417 t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)") 418 } 419 420 // Make sure the CDS balancer reports an error picker. 421 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 422 defer ctxCancel() 423 if err := tcc.WaitForErrPicker(ctx); err != nil { 424 t.Fatal(err) 425 } 426} 427 428func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) { 429 // This creates a CDS balancer which uses xdsCredentials, pushes a 430 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 431 // balancer registers a watch on the provided xdsClient. 432 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 433 defer func() { 434 cancel() 435 cdsB.Close() 436 }() 437 438 // Set the bootstrap config used by the fake client. 439 xdsC.SetBootstrapConfig(bootstrapConfig) 440 441 // Here we invoke the watch callback registered on the fake xdsClient. A bad 442 // security config is passed here. So, we expect the CDS balancer to not 443 // create an EDS balancer and instead reject this update and put the channel 444 // in a bad state. 445 xdsC.InvokeWatchClusterCallback(cdsUpdateWithMissingSecurityCfg, nil) 446 447 // The CDS balancer has not yet created an EDS balancer. So, this bad 448 // watcher update should not be forwarded forwarded to our fake EDS balancer 449 // as an error. 450 sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) 451 defer sCancel() 452 if err := edsB.waitForResolverError(sCtx, nil); err != context.DeadlineExceeded { 453 t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)") 454 } 455 456 // Make sure the CDS balancer reports an error picker. 457 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 458 defer ctxCancel() 459 if err := tcc.WaitForErrPicker(ctx); err != nil { 460 t.Fatal(err) 461 } 462 463 // Here we invoke the watch callback registered on the fake xdsClient. This 464 // will trigger the watch handler on the CDS balancer, which will attempt to 465 // create a new EDS balancer. The fake EDS balancer created above will be 466 // returned to the CDS balancer, because we have overridden the 467 // newChildBalancer function as part of test setup. 468 wantCCS := edsCCS(serviceName, nil, false) 469 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil { 470 t.Fatal(err) 471 } 472 473 // Make a NewSubConn and verify that attributes are added. 474 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil { 475 t.Fatal(err) 476 } 477} 478 479// TestGoodSecurityConfig tests the case where the CDS balancer receives 480// security configuration as part of the Cluster resource which can be 481// successfully resolved using the bootstrap file contents. Verifies that 482// certificate providers are created, and that the NewSubConn() call adds 483// appropriate address attributes. 484func (s) TestGoodSecurityConfig(t *testing.T) { 485 // This creates a CDS balancer which uses xdsCredentials, pushes a 486 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 487 // balancer registers a watch on the provided xdsClient. 488 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 489 defer func() { 490 cancel() 491 cdsB.Close() 492 }() 493 494 // Set the bootstrap config used by the fake client. 495 xdsC.SetBootstrapConfig(bootstrapConfig) 496 497 // Here we invoke the watch callback registered on the fake xdsClient. This 498 // will trigger the watch handler on the CDS balancer, which will attempt to 499 // create a new EDS balancer. The fake EDS balancer created above will be 500 // returned to the CDS balancer, because we have overridden the 501 // newChildBalancer function as part of test setup. 502 wantCCS := edsCCS(serviceName, nil, false) 503 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 504 defer ctxCancel() 505 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil { 506 t.Fatal(err) 507 } 508 509 // Make a NewSubConn and verify that attributes are added. 510 sc, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false) 511 if err != nil { 512 t.Fatal(err) 513 } 514 515 // Invoke UpdateAddresses and verify that attributes are added. 516 dummyAddr := "bar-address" 517 addrs := []resolver.Address{{Addr: dummyAddr}} 518 edsB.parentCC.UpdateAddresses(sc, addrs) 519 select { 520 case <-ctx.Done(): 521 t.Fatal("timeout when waiting for addresses to be updated on the subConn") 522 case gotAddrs := <-tcc.UpdateAddressesAddrsCh: 523 if len(gotAddrs) != 1 { 524 t.Fatalf("UpdateAddresses expected 1 address, got %d", len(gotAddrs)) 525 } 526 if got, want := gotAddrs[0].Addr, addrs[0].Addr; got != want { 527 t.Fatalf("resolver.Address passed to parent ClientConn through UpdateAddresses() has address %q, want %q", got, want) 528 } 529 getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdscredsinternal.HandshakeInfo) 530 hi := getHI(gotAddrs[0].Attributes) 531 if hi == nil { 532 t.Fatal("resolver.Address passed to parent ClientConn through UpdateAddresses() doesn't contain attributes") 533 } 534 } 535} 536 537func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) { 538 // This creates a CDS balancer which uses xdsCredentials, pushes a 539 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 540 // balancer registers a watch on the provided xdsClient. 541 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 542 defer func() { 543 cancel() 544 cdsB.Close() 545 }() 546 547 // Set the bootstrap config used by the fake client. 548 xdsC.SetBootstrapConfig(bootstrapConfig) 549 550 // Here we invoke the watch callback registered on the fake xdsClient. This 551 // will trigger the watch handler on the CDS balancer, which will attempt to 552 // create a new EDS balancer. The fake EDS balancer created above will be 553 // returned to the CDS balancer, because we have overridden the 554 // newChildBalancer function as part of test setup. 555 wantCCS := edsCCS(serviceName, nil, false) 556 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 557 defer ctxCancel() 558 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil { 559 t.Fatal(err) 560 } 561 562 // Make a NewSubConn and verify that attributes are added. 563 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil { 564 t.Fatal(err) 565 } 566 567 // Here we invoke the watch callback registered on the fake xdsClient with 568 // an update which contains bad security config. So, we expect the CDS 569 // balancer to forward this error to the EDS balancer and eventually the 570 // channel needs to be put in a bad state. 571 cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName} 572 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { 573 t.Fatal(err) 574 } 575 576 // Make a NewSubConn and verify that fallback creds are used. 577 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil { 578 t.Fatal(err) 579 } 580} 581 582// TestSecurityConfigUpdate_GoodToBad tests the case where the first security 583// config returned by the xDS server is successful, but the second update cannot 584// be resolved based on the contents of the bootstrap config. Verifies that the 585// error is forwarded to the EDS balancer (which was created as part of the 586// first successful update). 587func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) { 588 // This creates a CDS balancer which uses xdsCredentials, pushes a 589 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 590 // balancer registers a watch on the provided xdsClient. 591 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 592 defer func() { 593 cancel() 594 cdsB.Close() 595 }() 596 597 // Set the bootstrap config used by the fake client. 598 xdsC.SetBootstrapConfig(bootstrapConfig) 599 600 // Here we invoke the watch callback registered on the fake xdsClient. This 601 // will trigger the watch handler on the CDS balancer, which will attempt to 602 // create a new EDS balancer. The fake EDS balancer created above will be 603 // returned to the CDS balancer, because we have overridden the 604 // newChildBalancer function as part of test setup. 605 wantCCS := edsCCS(serviceName, nil, false) 606 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 607 defer ctxCancel() 608 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil { 609 t.Fatal(err) 610 } 611 612 // Make a NewSubConn and verify that attributes are added. 613 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil { 614 t.Fatal(err) 615 } 616 617 // Here we invoke the watch callback registered on the fake xdsClient with 618 // an update which contains bad security config. So, we expect the CDS 619 // balancer to forward this error to the EDS balancer and eventually the 620 // channel needs to be put in a bad state. 621 xdsC.InvokeWatchClusterCallback(cdsUpdateWithMissingSecurityCfg, nil) 622 623 // We manually check that an error is forwarded to the EDS balancer instead 624 // of using one of the helper methods on the testEDSBalancer, because all we 625 // care here is whether an error is sent to it or not. We don't care about 626 // the exact error. 627 gotErr, err := edsB.resolverErrCh.Receive(ctx) 628 if err != nil { 629 t.Fatal("timeout waiting for CDS balancer to forward error to EDS balancer upon receipt of bad security config") 630 } 631 if gotErr == nil { 632 t.Fatal("CDS balancer did not forward error to EDS balancer upon receipt of bad security config") 633 } 634 635 // Since the error being pushed here is not a resource-not-found-error, the 636 // registered watch should not be cancelled. 637 sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) 638 defer sCancel() 639 if _, err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded { 640 t.Fatal("cluster watch cancelled for a non-resource-not-found-error") 641 } 642} 643 644// TestSecurityConfigUpdate_GoodToGood tests the case where the CDS balancer 645// receives two different but successful updates with security configuration. 646// Verifies that appropriate providers are created, and that address attributes 647// are added. 648func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) { 649 // This creates a CDS balancer which uses xdsCredentials, pushes a 650 // ClientConnState update with a fake xdsClient, and makes sure that the CDS 651 // balancer registers a watch on the provided xdsClient. 652 xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t) 653 defer func() { 654 cancel() 655 cdsB.Close() 656 }() 657 658 // Override the provider builder function to push on a channel. 659 providerCh := testutils.NewChannel() 660 origBuildProvider := buildProvider 661 buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) { 662 p, err := origBuildProvider(c, id, cert, wi, wr) 663 providerCh.Send(nil) 664 return p, err 665 } 666 defer func() { buildProvider = origBuildProvider }() 667 668 // Set the bootstrap config used by the fake client. 669 xdsC.SetBootstrapConfig(bootstrapConfig) 670 671 // Here we invoke the watch callback registered on the fake xdsClient. This 672 // will trigger the watch handler on the CDS balancer, which will attempt to 673 // create a new EDS balancer. The fake EDS balancer created above will be 674 // returned to the CDS balancer, because we have overridden the 675 // newChildBalancer function as part of test setup. 676 cdsUpdate := xdsclient.ClusterUpdate{ 677 ClusterName: serviceName, 678 SecurityCfg: &xdsclient.SecurityConfig{ 679 RootInstanceName: "default1", 680 SubjectAltNameMatchers: testSANMatchers, 681 }, 682 } 683 wantCCS := edsCCS(serviceName, nil, false) 684 ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) 685 defer ctxCancel() 686 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { 687 t.Fatal(err) 688 } 689 690 // We specified only the root provider. So, expect only one provider here. 691 if _, err := providerCh.Receive(ctx); err != nil { 692 t.Fatalf("Failed to create certificate provider upon receipt of security config") 693 } 694 695 // Make a NewSubConn and verify that attributes are added. 696 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil { 697 t.Fatal(err) 698 } 699 700 // Push another update with a new security configuration. 701 cdsUpdate = xdsclient.ClusterUpdate{ 702 ClusterName: serviceName, 703 SecurityCfg: &xdsclient.SecurityConfig{ 704 RootInstanceName: "default2", 705 SubjectAltNameMatchers: testSANMatchers, 706 }, 707 } 708 if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { 709 t.Fatal(err) 710 } 711 712 // We specified only the root provider. So, expect only one provider here. 713 if _, err := providerCh.Receive(ctx); err != nil { 714 t.Fatalf("Failed to create certificate provider upon receipt of security config") 715 } 716 717 // Make a NewSubConn and verify that attributes are added. 718 if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil { 719 t.Fatal(err) 720 } 721 722 // The HandshakeInfo type does not expose its internals. So, we cannot 723 // verify that the HandshakeInfo carried by the attributes have actually 724 // been changed. This will be covered in e2e/interop tests. 725 // TODO(easwars): Remove this TODO once appropriate e2e/intertop tests have 726 // been added. 727} 728