1/* 2 * Copyright 2019 gRPC authors. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package edsbalancer 18 19import ( 20 "context" 21 "fmt" 22 "net" 23 "reflect" 24 "strconv" 25 "testing" 26 27 typespb "github.com/golang/protobuf/ptypes/wrappers" 28 "google.golang.org/grpc/balancer" 29 "google.golang.org/grpc/balancer/roundrobin" 30 "google.golang.org/grpc/connectivity" 31 "google.golang.org/grpc/resolver" 32 "google.golang.org/grpc/xds/internal" 33 addresspb "google.golang.org/grpc/xds/internal/proto/envoy/api/v2/core/address" 34 basepb "google.golang.org/grpc/xds/internal/proto/envoy/api/v2/core/base" 35 edspb "google.golang.org/grpc/xds/internal/proto/envoy/api/v2/eds" 36 endpointpb "google.golang.org/grpc/xds/internal/proto/envoy/api/v2/endpoint/endpoint" 37 percentpb "google.golang.org/grpc/xds/internal/proto/envoy/type/percent" 38) 39 40var ( 41 testClusterNames = []string{"test-cluster-1", "test-cluster-2"} 42 testSubZones = []string{"I", "II", "III", "IV"} 43 testEndpointAddrs = []string{"1.1.1.1:1", "2.2.2.2:2", "3.3.3.3:3", "4.4.4.4:4"} 44) 45 46type clusterLoadAssignmentBuilder struct { 47 v *edspb.ClusterLoadAssignment 48} 49 50func newClusterLoadAssignmentBuilder(clusterName string, dropPercents []uint32) *clusterLoadAssignmentBuilder { 51 var drops []*edspb.ClusterLoadAssignment_Policy_DropOverload 52 for i, d := range dropPercents { 53 drops = append(drops, &edspb.ClusterLoadAssignment_Policy_DropOverload{ 54 Category: fmt.Sprintf("test-drop-%d", i), 55 DropPercentage: &percentpb.FractionalPercent{ 56 Numerator: d, 57 Denominator: percentpb.FractionalPercent_HUNDRED, 58 }, 59 }) 60 } 61 62 return &clusterLoadAssignmentBuilder{ 63 v: &edspb.ClusterLoadAssignment{ 64 ClusterName: clusterName, 65 Policy: &edspb.ClusterLoadAssignment_Policy{ 66 DropOverloads: drops, 67 }, 68 }, 69 } 70} 71 72func (clab *clusterLoadAssignmentBuilder) addLocality(subzone string, weight uint32, addrsWithPort []string) { 73 var lbEndPoints []*endpointpb.LbEndpoint 74 for _, a := range addrsWithPort { 75 host, portStr, err := net.SplitHostPort(a) 76 if err != nil { 77 panic("failed to split " + a) 78 } 79 port, err := strconv.Atoi(portStr) 80 if err != nil { 81 panic("failed to atoi " + portStr) 82 } 83 84 lbEndPoints = append(lbEndPoints, &endpointpb.LbEndpoint{ 85 HostIdentifier: &endpointpb.LbEndpoint_Endpoint{ 86 Endpoint: &endpointpb.Endpoint{ 87 Address: &addresspb.Address{ 88 Address: &addresspb.Address_SocketAddress{ 89 SocketAddress: &addresspb.SocketAddress{ 90 Protocol: addresspb.SocketAddress_TCP, 91 Address: host, 92 PortSpecifier: &addresspb.SocketAddress_PortValue{ 93 PortValue: uint32(port)}}}}}}}, 94 ) 95 } 96 97 clab.v.Endpoints = append(clab.v.Endpoints, &endpointpb.LocalityLbEndpoints{ 98 Locality: &basepb.Locality{ 99 Region: "", 100 Zone: "", 101 SubZone: subzone, 102 }, 103 LbEndpoints: lbEndPoints, 104 LoadBalancingWeight: &typespb.UInt32Value{Value: weight}, 105 }) 106} 107 108func (clab *clusterLoadAssignmentBuilder) build() *edspb.ClusterLoadAssignment { 109 return clab.v 110} 111 112// One locality 113// - add backend 114// - remove backend 115// - replace backend 116// - change drop rate 117func TestEDS_OneLocality(t *testing.T) { 118 cc := newTestClientConn(t) 119 edsb := NewXDSBalancer(cc, nil) 120 121 // One locality with one backend. 122 clab1 := newClusterLoadAssignmentBuilder(testClusterNames[0], nil) 123 clab1.addLocality(testSubZones[0], 1, testEndpointAddrs[:1]) 124 edsb.HandleEDSResponse(clab1.build()) 125 126 sc1 := <-cc.newSubConnCh 127 edsb.HandleSubConnStateChange(sc1, connectivity.Connecting) 128 edsb.HandleSubConnStateChange(sc1, connectivity.Ready) 129 130 // Pick with only the first backend. 131 p1 := <-cc.newPickerCh 132 for i := 0; i < 5; i++ { 133 gotSC, _, _ := p1.Pick(context.Background(), balancer.PickOptions{}) 134 if !reflect.DeepEqual(gotSC, sc1) { 135 t.Fatalf("picker.Pick, got %v, want %v", gotSC, sc1) 136 } 137 } 138 139 // The same locality, add one more backend. 140 clab2 := newClusterLoadAssignmentBuilder(testClusterNames[0], nil) 141 clab2.addLocality(testSubZones[0], 1, testEndpointAddrs[:2]) 142 edsb.HandleEDSResponse(clab2.build()) 143 144 sc2 := <-cc.newSubConnCh 145 edsb.HandleSubConnStateChange(sc2, connectivity.Connecting) 146 edsb.HandleSubConnStateChange(sc2, connectivity.Ready) 147 148 // Test roundrobin with two subconns. 149 p2 := <-cc.newPickerCh 150 want := []balancer.SubConn{sc1, sc2} 151 if err := isRoundRobin(want, func() balancer.SubConn { 152 sc, _, _ := p2.Pick(context.Background(), balancer.PickOptions{}) 153 return sc 154 }); err != nil { 155 t.Fatalf("want %v, got %v", want, err) 156 } 157 158 // The same locality, delete first backend. 159 clab3 := newClusterLoadAssignmentBuilder(testClusterNames[0], nil) 160 clab3.addLocality(testSubZones[0], 1, testEndpointAddrs[1:2]) 161 edsb.HandleEDSResponse(clab3.build()) 162 163 scToRemove := <-cc.removeSubConnCh 164 if !reflect.DeepEqual(scToRemove, sc1) { 165 t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove) 166 } 167 edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown) 168 169 // Test pick with only the second subconn. 170 p3 := <-cc.newPickerCh 171 for i := 0; i < 5; i++ { 172 gotSC, _, _ := p3.Pick(context.Background(), balancer.PickOptions{}) 173 if !reflect.DeepEqual(gotSC, sc2) { 174 t.Fatalf("picker.Pick, got %v, want %v", gotSC, sc2) 175 } 176 } 177 178 // The same locality, replace backend. 179 clab4 := newClusterLoadAssignmentBuilder(testClusterNames[0], nil) 180 clab4.addLocality(testSubZones[0], 1, testEndpointAddrs[2:3]) 181 edsb.HandleEDSResponse(clab4.build()) 182 183 sc3 := <-cc.newSubConnCh 184 edsb.HandleSubConnStateChange(sc3, connectivity.Connecting) 185 edsb.HandleSubConnStateChange(sc3, connectivity.Ready) 186 scToRemove = <-cc.removeSubConnCh 187 if !reflect.DeepEqual(scToRemove, sc2) { 188 t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove) 189 } 190 edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown) 191 192 // Test pick with only the third subconn. 193 p4 := <-cc.newPickerCh 194 for i := 0; i < 5; i++ { 195 gotSC, _, _ := p4.Pick(context.Background(), balancer.PickOptions{}) 196 if !reflect.DeepEqual(gotSC, sc3) { 197 t.Fatalf("picker.Pick, got %v, want %v", gotSC, sc3) 198 } 199 } 200 201 // The same locality, different drop rate, dropping 50%. 202 clab5 := newClusterLoadAssignmentBuilder(testClusterNames[0], []uint32{50}) 203 clab5.addLocality(testSubZones[0], 1, testEndpointAddrs[2:3]) 204 edsb.HandleEDSResponse(clab5.build()) 205 206 // Picks with drops. 207 p5 := <-cc.newPickerCh 208 for i := 0; i < 100; i++ { 209 _, _, err := p5.Pick(context.Background(), balancer.PickOptions{}) 210 // TODO: the dropping algorithm needs a design. When the dropping algorithm 211 // is fixed, this test also needs fix. 212 if i < 50 && err == nil { 213 t.Errorf("The first 50%% picks should be drops, got error <nil>") 214 } else if i > 50 && err != nil { 215 t.Errorf("The second 50%% picks should be non-drops, got error %v", err) 216 } 217 } 218} 219 220// 2 locality 221// - start with 2 locality 222// - add locality 223// - remove locality 224// - address change for the <not-the-first> locality 225// - update locality weight 226func TestEDS_TwoLocalities(t *testing.T) { 227 cc := newTestClientConn(t) 228 edsb := NewXDSBalancer(cc, nil) 229 230 // Two localities, each with one backend. 231 clab1 := newClusterLoadAssignmentBuilder(testClusterNames[0], nil) 232 clab1.addLocality(testSubZones[0], 1, testEndpointAddrs[:1]) 233 clab1.addLocality(testSubZones[1], 1, testEndpointAddrs[1:2]) 234 edsb.HandleEDSResponse(clab1.build()) 235 236 sc1 := <-cc.newSubConnCh 237 edsb.HandleSubConnStateChange(sc1, connectivity.Connecting) 238 edsb.HandleSubConnStateChange(sc1, connectivity.Ready) 239 sc2 := <-cc.newSubConnCh 240 edsb.HandleSubConnStateChange(sc2, connectivity.Connecting) 241 edsb.HandleSubConnStateChange(sc2, connectivity.Ready) 242 243 // Test roundrobin with two subconns. 244 p1 := <-cc.newPickerCh 245 want := []balancer.SubConn{sc1, sc2} 246 if err := isRoundRobin(want, func() balancer.SubConn { 247 sc, _, _ := p1.Pick(context.Background(), balancer.PickOptions{}) 248 return sc 249 }); err != nil { 250 t.Fatalf("want %v, got %v", want, err) 251 } 252 253 // Add another locality, with one backend. 254 clab2 := newClusterLoadAssignmentBuilder(testClusterNames[0], nil) 255 clab2.addLocality(testSubZones[0], 1, testEndpointAddrs[:1]) 256 clab2.addLocality(testSubZones[1], 1, testEndpointAddrs[1:2]) 257 clab2.addLocality(testSubZones[2], 1, testEndpointAddrs[2:3]) 258 edsb.HandleEDSResponse(clab2.build()) 259 260 sc3 := <-cc.newSubConnCh 261 edsb.HandleSubConnStateChange(sc3, connectivity.Connecting) 262 edsb.HandleSubConnStateChange(sc3, connectivity.Ready) 263 264 // Test roundrobin with three subconns. 265 p2 := <-cc.newPickerCh 266 want = []balancer.SubConn{sc1, sc2, sc3} 267 if err := isRoundRobin(want, func() balancer.SubConn { 268 sc, _, _ := p2.Pick(context.Background(), balancer.PickOptions{}) 269 return sc 270 }); err != nil { 271 t.Fatalf("want %v, got %v", want, err) 272 } 273 274 // Remove first locality. 275 clab3 := newClusterLoadAssignmentBuilder(testClusterNames[0], nil) 276 clab3.addLocality(testSubZones[1], 1, testEndpointAddrs[1:2]) 277 clab3.addLocality(testSubZones[2], 1, testEndpointAddrs[2:3]) 278 edsb.HandleEDSResponse(clab3.build()) 279 280 scToRemove := <-cc.removeSubConnCh 281 if !reflect.DeepEqual(scToRemove, sc1) { 282 t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove) 283 } 284 edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown) 285 286 // Test pick with two subconns (without the first one). 287 p3 := <-cc.newPickerCh 288 want = []balancer.SubConn{sc2, sc3} 289 if err := isRoundRobin(want, func() balancer.SubConn { 290 sc, _, _ := p3.Pick(context.Background(), balancer.PickOptions{}) 291 return sc 292 }); err != nil { 293 t.Fatalf("want %v, got %v", want, err) 294 } 295 296 // Add a backend to the last locality. 297 clab4 := newClusterLoadAssignmentBuilder(testClusterNames[0], nil) 298 clab4.addLocality(testSubZones[1], 1, testEndpointAddrs[1:2]) 299 clab4.addLocality(testSubZones[2], 1, testEndpointAddrs[2:4]) 300 edsb.HandleEDSResponse(clab4.build()) 301 302 sc4 := <-cc.newSubConnCh 303 edsb.HandleSubConnStateChange(sc4, connectivity.Connecting) 304 edsb.HandleSubConnStateChange(sc4, connectivity.Ready) 305 306 // Test pick with two subconns (without the first one). 307 p4 := <-cc.newPickerCh 308 // Locality-1 will be picked twice, and locality-2 will be picked twice. 309 // Locality-1 contains only sc2, locality-2 contains sc3 and sc4. So expect 310 // two sc2's and sc3, sc4. 311 want = []balancer.SubConn{sc2, sc2, sc3, sc4} 312 if err := isRoundRobin(want, func() balancer.SubConn { 313 sc, _, _ := p4.Pick(context.Background(), balancer.PickOptions{}) 314 return sc 315 }); err != nil { 316 t.Fatalf("want %v, got %v", want, err) 317 } 318 319 // Change weight of the locality[1]. 320 clab5 := newClusterLoadAssignmentBuilder(testClusterNames[0], nil) 321 clab5.addLocality(testSubZones[1], 2, testEndpointAddrs[1:2]) 322 clab5.addLocality(testSubZones[2], 1, testEndpointAddrs[2:4]) 323 edsb.HandleEDSResponse(clab5.build()) 324 325 // Test pick with two subconns different locality weight. 326 p5 := <-cc.newPickerCh 327 // Locality-1 will be picked four times, and locality-2 will be picked twice 328 // (weight 2 and 1). Locality-1 contains only sc2, locality-2 contains sc3 and 329 // sc4. So expect four sc2's and sc3, sc4. 330 want = []balancer.SubConn{sc2, sc2, sc2, sc2, sc3, sc4} 331 if err := isRoundRobin(want, func() balancer.SubConn { 332 sc, _, _ := p5.Pick(context.Background(), balancer.PickOptions{}) 333 return sc 334 }); err != nil { 335 t.Fatalf("want %v, got %v", want, err) 336 } 337 338 // Change weight of the locality[1] to 0, it should never be picked. 339 clab6 := newClusterLoadAssignmentBuilder(testClusterNames[0], nil) 340 clab6.addLocality(testSubZones[1], 0, testEndpointAddrs[1:2]) 341 clab6.addLocality(testSubZones[2], 1, testEndpointAddrs[2:4]) 342 edsb.HandleEDSResponse(clab6.build()) 343 344 // Test pick with two subconns different locality weight. 345 p6 := <-cc.newPickerCh 346 // Locality-1 will be not be picked, and locality-2 will be picked. 347 // Locality-2 contains sc3 and sc4. So expect sc3, sc4. 348 want = []balancer.SubConn{sc3, sc4} 349 if err := isRoundRobin(want, func() balancer.SubConn { 350 sc, _, _ := p6.Pick(context.Background(), balancer.PickOptions{}) 351 return sc 352 }); err != nil { 353 t.Fatalf("want %v, got %v", want, err) 354 } 355} 356 357func TestClose(t *testing.T) { 358 edsb := NewXDSBalancer(nil, nil) 359 // This is what could happen when switching between fallback and eds. This 360 // make sure it doesn't panic. 361 edsb.Close() 362} 363 364func init() { 365 balancer.Register(&testConstBalancerBuilder{}) 366} 367 368var errTestConstPicker = fmt.Errorf("const picker error") 369 370type testConstBalancerBuilder struct{} 371 372func (*testConstBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { 373 return &testConstBalancer{cc: cc} 374} 375 376func (*testConstBalancerBuilder) Name() string { 377 return "test-const-balancer" 378} 379 380type testConstBalancer struct { 381 cc balancer.ClientConn 382} 383 384func (tb *testConstBalancer) HandleSubConnStateChange(sc balancer.SubConn, state connectivity.State) { 385 tb.cc.UpdateBalancerState(connectivity.Ready, &testConstPicker{err: errTestConstPicker}) 386} 387 388func (tb *testConstBalancer) HandleResolvedAddrs([]resolver.Address, error) { 389 tb.cc.UpdateBalancerState(connectivity.Ready, &testConstPicker{err: errTestConstPicker}) 390} 391 392func (*testConstBalancer) Close() { 393} 394 395type testConstPicker struct { 396 err error 397 sc balancer.SubConn 398} 399 400func (tcp *testConstPicker) Pick(ctx context.Context, opts balancer.PickOptions) (conn balancer.SubConn, done func(balancer.DoneInfo), err error) { 401 if tcp.err != nil { 402 return nil, nil, tcp.err 403 } 404 return tcp.sc, nil, nil 405} 406 407// Create XDS balancer, and update sub-balancer before handling eds responses. 408// Then switch between round-robin and test-const-balancer after handling first 409// eds response. 410func TestEDS_UpdateSubBalancerName(t *testing.T) { 411 cc := newTestClientConn(t) 412 edsb := NewXDSBalancer(cc, nil) 413 414 t.Logf("update sub-balancer to test-const-balancer") 415 edsb.HandleChildPolicy("test-const-balancer", nil) 416 417 // Two localities, each with one backend. 418 clab1 := newClusterLoadAssignmentBuilder(testClusterNames[0], nil) 419 clab1.addLocality(testSubZones[0], 1, testEndpointAddrs[:1]) 420 clab1.addLocality(testSubZones[1], 1, testEndpointAddrs[1:2]) 421 edsb.HandleEDSResponse(clab1.build()) 422 423 p0 := <-cc.newPickerCh 424 for i := 0; i < 5; i++ { 425 _, _, err := p0.Pick(context.Background(), balancer.PickOptions{}) 426 if !reflect.DeepEqual(err, errTestConstPicker) { 427 t.Fatalf("picker.Pick, got err %q, want err %q", err, errTestConstPicker) 428 } 429 } 430 431 t.Logf("update sub-balancer to round-robin") 432 edsb.HandleChildPolicy(roundrobin.Name, nil) 433 434 sc1 := <-cc.newSubConnCh 435 edsb.HandleSubConnStateChange(sc1, connectivity.Connecting) 436 edsb.HandleSubConnStateChange(sc1, connectivity.Ready) 437 sc2 := <-cc.newSubConnCh 438 edsb.HandleSubConnStateChange(sc2, connectivity.Connecting) 439 edsb.HandleSubConnStateChange(sc2, connectivity.Ready) 440 441 // Test roundrobin with two subconns. 442 p1 := <-cc.newPickerCh 443 want := []balancer.SubConn{sc1, sc2} 444 if err := isRoundRobin(want, func() balancer.SubConn { 445 sc, _, _ := p1.Pick(context.Background(), balancer.PickOptions{}) 446 return sc 447 }); err != nil { 448 t.Fatalf("want %v, got %v", want, err) 449 } 450 451 t.Logf("update sub-balancer to test-const-balancer") 452 edsb.HandleChildPolicy("test-const-balancer", nil) 453 454 for i := 0; i < 2; i++ { 455 scToRemove := <-cc.removeSubConnCh 456 if !reflect.DeepEqual(scToRemove, sc1) && !reflect.DeepEqual(scToRemove, sc2) { 457 t.Fatalf("RemoveSubConn, want (%v or %v), got %v", sc1, sc2, scToRemove) 458 } 459 edsb.HandleSubConnStateChange(scToRemove, connectivity.Shutdown) 460 } 461 462 p2 := <-cc.newPickerCh 463 for i := 0; i < 5; i++ { 464 _, _, err := p2.Pick(context.Background(), balancer.PickOptions{}) 465 if !reflect.DeepEqual(err, errTestConstPicker) { 466 t.Fatalf("picker.Pick, got err %q, want err %q", err, errTestConstPicker) 467 } 468 } 469 470 t.Logf("update sub-balancer to round-robin") 471 edsb.HandleChildPolicy(roundrobin.Name, nil) 472 473 sc3 := <-cc.newSubConnCh 474 edsb.HandleSubConnStateChange(sc3, connectivity.Connecting) 475 edsb.HandleSubConnStateChange(sc3, connectivity.Ready) 476 sc4 := <-cc.newSubConnCh 477 edsb.HandleSubConnStateChange(sc4, connectivity.Connecting) 478 edsb.HandleSubConnStateChange(sc4, connectivity.Ready) 479 480 p3 := <-cc.newPickerCh 481 want = []balancer.SubConn{sc3, sc4} 482 if err := isRoundRobin(want, func() balancer.SubConn { 483 sc, _, _ := p3.Pick(context.Background(), balancer.PickOptions{}) 484 return sc 485 }); err != nil { 486 t.Fatalf("want %v, got %v", want, err) 487 } 488} 489 490func TestDropPicker(t *testing.T) { 491 const pickCount = 12 492 var constPicker = &testConstPicker{ 493 sc: testSubConns[0], 494 } 495 496 tests := []struct { 497 name string 498 drops []*dropper 499 }{ 500 { 501 name: "no drop", 502 drops: nil, 503 }, 504 { 505 name: "one drop", 506 drops: []*dropper{ 507 newDropper(1, 2, ""), 508 }, 509 }, 510 { 511 name: "two drops", 512 drops: []*dropper{ 513 newDropper(1, 3, ""), 514 newDropper(1, 2, ""), 515 }, 516 }, 517 { 518 name: "three drops", 519 drops: []*dropper{ 520 newDropper(1, 3, ""), 521 newDropper(1, 4, ""), 522 newDropper(1, 2, ""), 523 }, 524 }, 525 } 526 for _, tt := range tests { 527 t.Run(tt.name, func(t *testing.T) { 528 529 p := newDropPicker(constPicker, tt.drops, nil) 530 531 // scCount is the number of sc's returned by pick. The opposite of 532 // drop-count. 533 var ( 534 scCount int 535 wantCount = pickCount 536 ) 537 for _, dp := range tt.drops { 538 wantCount = wantCount * int(dp.denominator-dp.numerator) / int(dp.denominator) 539 } 540 541 for i := 0; i < pickCount; i++ { 542 _, _, err := p.Pick(context.Background(), balancer.PickOptions{}) 543 if err == nil { 544 scCount++ 545 } 546 } 547 548 if scCount != (wantCount) { 549 t.Errorf("drops: %+v, scCount %v, wantCount %v", tt.drops, scCount, wantCount) 550 } 551 }) 552 } 553} 554 555func TestEDS_LoadReport(t *testing.T) { 556 testLoadStore := newTestLoadStore() 557 558 cc := newTestClientConn(t) 559 edsb := NewXDSBalancer(cc, testLoadStore) 560 561 backendToBalancerID := make(map[balancer.SubConn]internal.Locality) 562 563 // Two localities, each with one backend. 564 clab1 := newClusterLoadAssignmentBuilder(testClusterNames[0], nil) 565 clab1.addLocality(testSubZones[0], 1, testEndpointAddrs[:1]) 566 clab1.addLocality(testSubZones[1], 1, testEndpointAddrs[1:2]) 567 edsb.HandleEDSResponse(clab1.build()) 568 569 sc1 := <-cc.newSubConnCh 570 edsb.HandleSubConnStateChange(sc1, connectivity.Connecting) 571 edsb.HandleSubConnStateChange(sc1, connectivity.Ready) 572 backendToBalancerID[sc1] = internal.Locality{ 573 SubZone: testSubZones[0], 574 } 575 sc2 := <-cc.newSubConnCh 576 edsb.HandleSubConnStateChange(sc2, connectivity.Connecting) 577 edsb.HandleSubConnStateChange(sc2, connectivity.Ready) 578 backendToBalancerID[sc2] = internal.Locality{ 579 SubZone: testSubZones[1], 580 } 581 582 // Test roundrobin with two subconns. 583 p1 := <-cc.newPickerCh 584 var ( 585 wantStart []internal.Locality 586 wantEnd []internal.Locality 587 ) 588 589 for i := 0; i < 10; i++ { 590 sc, done, _ := p1.Pick(context.Background(), balancer.PickOptions{}) 591 locality := backendToBalancerID[sc] 592 wantStart = append(wantStart, locality) 593 if done != nil && sc != sc1 { 594 done(balancer.DoneInfo{}) 595 wantEnd = append(wantEnd, backendToBalancerID[sc]) 596 } 597 } 598 599 if !reflect.DeepEqual(testLoadStore.callsStarted, wantStart) { 600 t.Fatalf("want started: %v, got: %v", testLoadStore.callsStarted, wantStart) 601 } 602 if !reflect.DeepEqual(testLoadStore.callsEnded, wantEnd) { 603 t.Fatalf("want ended: %v, got: %v", testLoadStore.callsEnded, wantEnd) 604 } 605} 606