1 /*
2 *
3 * Copyright 2017 gRPC authors.
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 *
17 */
18
19 #include <deque>
20 #include <memory>
21 #include <mutex>
22 #include <numeric>
23 #include <set>
24 #include <sstream>
25 #include <string>
26 #include <thread>
27 #include <vector>
28
29 #include <gmock/gmock.h>
30 #include <gtest/gtest.h>
31
32 #include "absl/memory/memory.h"
33 #include "absl/strings/str_cat.h"
34 #include "absl/types/optional.h"
35
36 #include <grpc/grpc.h>
37 #include <grpc/support/alloc.h>
38 #include <grpc/support/log.h>
39 #include <grpc/support/time.h>
40 #include <grpcpp/channel.h>
41 #include <grpcpp/client_context.h>
42 #include <grpcpp/create_channel.h>
43 #include <grpcpp/server.h>
44 #include <grpcpp/server_builder.h>
45
46 #include "src/core/ext/filters/client_channel/backup_poller.h"
47 #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
48 #include "src/core/ext/filters/client_channel/server_address.h"
49 #include "src/core/ext/xds/xds_api.h"
50 #include "src/core/ext/xds/xds_channel_args.h"
51 #include "src/core/ext/xds/xds_client.h"
52 #include "src/core/lib/channel/channel_args.h"
53 #include "src/core/lib/gpr/env.h"
54 #include "src/core/lib/gpr/tmpfile.h"
55 #include "src/core/lib/gprpp/map.h"
56 #include "src/core/lib/gprpp/ref_counted_ptr.h"
57 #include "src/core/lib/gprpp/sync.h"
58 #include "src/core/lib/iomgr/parse_address.h"
59 #include "src/core/lib/iomgr/sockaddr.h"
60 #include "src/core/lib/security/credentials/fake/fake_credentials.h"
61 #include "src/cpp/client/secure_credentials.h"
62 #include "src/cpp/server/secure_server_credentials.h"
63
64 #include "test/core/util/port.h"
65 #include "test/core/util/resolve_localhost_ip46.h"
66 #include "test/core/util/test_config.h"
67 #include "test/cpp/end2end/test_service_impl.h"
68
69 #include "src/proto/grpc/testing/echo.grpc.pb.h"
70 #include "src/proto/grpc/testing/xds/ads_for_test.grpc.pb.h"
71 #include "src/proto/grpc/testing/xds/cds_for_test.grpc.pb.h"
72 #include "src/proto/grpc/testing/xds/eds_for_test.grpc.pb.h"
73 #include "src/proto/grpc/testing/xds/lds_rds_for_test.grpc.pb.h"
74 #include "src/proto/grpc/testing/xds/lrs_for_test.grpc.pb.h"
75
76 #include "src/proto/grpc/testing/xds/v3/ads.grpc.pb.h"
77 #include "src/proto/grpc/testing/xds/v3/cluster.grpc.pb.h"
78 #include "src/proto/grpc/testing/xds/v3/discovery.grpc.pb.h"
79 #include "src/proto/grpc/testing/xds/v3/endpoint.grpc.pb.h"
80 #include "src/proto/grpc/testing/xds/v3/http_connection_manager.grpc.pb.h"
81 #include "src/proto/grpc/testing/xds/v3/listener.grpc.pb.h"
82 #include "src/proto/grpc/testing/xds/v3/lrs.grpc.pb.h"
83 #include "src/proto/grpc/testing/xds/v3/route.grpc.pb.h"
84
85 namespace grpc {
86 namespace testing {
87 namespace {
88
89 using std::chrono::system_clock;
90
91 using ::envoy::config::cluster::v3::CircuitBreakers;
92 using ::envoy::config::cluster::v3::Cluster;
93 using ::envoy::config::cluster::v3::RoutingPriority;
94 using ::envoy::config::endpoint::v3::ClusterLoadAssignment;
95 using ::envoy::config::endpoint::v3::HealthStatus;
96 using ::envoy::config::listener::v3::Listener;
97 using ::envoy::config::route::v3::RouteConfiguration;
98 using ::envoy::extensions::filters::network::http_connection_manager::v3::
99 HttpConnectionManager;
100 using ::envoy::type::v3::FractionalPercent;
101
102 constexpr char kLdsTypeUrl[] =
103 "type.googleapis.com/envoy.config.listener.v3.Listener";
104 constexpr char kRdsTypeUrl[] =
105 "type.googleapis.com/envoy.config.route.v3.RouteConfiguration";
106 constexpr char kCdsTypeUrl[] =
107 "type.googleapis.com/envoy.config.cluster.v3.Cluster";
108 constexpr char kEdsTypeUrl[] =
109 "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment";
110
111 constexpr char kLdsV2TypeUrl[] = "type.googleapis.com/envoy.api.v2.Listener";
112 constexpr char kRdsV2TypeUrl[] =
113 "type.googleapis.com/envoy.api.v2.RouteConfiguration";
114 constexpr char kCdsV2TypeUrl[] = "type.googleapis.com/envoy.api.v2.Cluster";
115 constexpr char kEdsV2TypeUrl[] =
116 "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment";
117
118 constexpr char kDefaultLocalityRegion[] = "xds_default_locality_region";
119 constexpr char kDefaultLocalityZone[] = "xds_default_locality_zone";
120 constexpr char kLbDropType[] = "lb";
121 constexpr char kThrottleDropType[] = "throttle";
122 constexpr char kServerName[] = "server.example.com";
123 constexpr char kDefaultRouteConfigurationName[] = "route_config_name";
124 constexpr char kDefaultClusterName[] = "cluster_name";
125 constexpr char kDefaultEdsServiceName[] = "eds_service_name";
126 constexpr int kDefaultLocalityWeight = 3;
127 constexpr int kDefaultLocalityPriority = 0;
128
129 constexpr char kRequestMessage[] = "Live long and prosper.";
130 constexpr char kDefaultServiceConfig[] =
131 "{\n"
132 " \"loadBalancingConfig\":[\n"
133 " { \"does_not_exist\":{} },\n"
134 " { \"eds_experimental\":{\n"
135 " \"clusterName\": \"server.example.com\",\n"
136 " \"lrsLoadReportingServerName\": \"\"\n"
137 " } }\n"
138 " ]\n"
139 "}";
140 constexpr char kDefaultServiceConfigWithoutLoadReporting[] =
141 "{\n"
142 " \"loadBalancingConfig\":[\n"
143 " { \"does_not_exist\":{} },\n"
144 " { \"eds_experimental\":{\n"
145 " \"clusterName\": \"server.example.com\"\n"
146 " } }\n"
147 " ]\n"
148 "}";
149
150 constexpr char kBootstrapFileV3[] =
151 "{\n"
152 " \"xds_servers\": [\n"
153 " {\n"
154 " \"server_uri\": \"fake:///xds_server\",\n"
155 " \"channel_creds\": [\n"
156 " {\n"
157 " \"type\": \"fake\"\n"
158 " }\n"
159 " ],\n"
160 " \"server_features\": [\"xds_v3\"]\n"
161 " }\n"
162 " ],\n"
163 " \"node\": {\n"
164 " \"id\": \"xds_end2end_test\",\n"
165 " \"cluster\": \"test\",\n"
166 " \"metadata\": {\n"
167 " \"foo\": \"bar\"\n"
168 " },\n"
169 " \"locality\": {\n"
170 " \"region\": \"corp\",\n"
171 " \"zone\": \"svl\",\n"
172 " \"subzone\": \"mp3\"\n"
173 " }\n"
174 " }\n"
175 "}\n";
176
177 constexpr char kBootstrapFileV2[] =
178 "{\n"
179 " \"xds_servers\": [\n"
180 " {\n"
181 " \"server_uri\": \"fake:///xds_server\",\n"
182 " \"channel_creds\": [\n"
183 " {\n"
184 " \"type\": \"fake\"\n"
185 " }\n"
186 " ]\n"
187 " }\n"
188 " ],\n"
189 " \"node\": {\n"
190 " \"id\": \"xds_end2end_test\",\n"
191 " \"cluster\": \"test\",\n"
192 " \"metadata\": {\n"
193 " \"foo\": \"bar\"\n"
194 " },\n"
195 " \"locality\": {\n"
196 " \"region\": \"corp\",\n"
197 " \"zone\": \"svl\",\n"
198 " \"subzone\": \"mp3\"\n"
199 " }\n"
200 " }\n"
201 "}\n";
202
203 char* g_bootstrap_file_v3;
204 char* g_bootstrap_file_v2;
205
WriteBootstrapFiles()206 void WriteBootstrapFiles() {
207 char* bootstrap_file;
208 FILE* out = gpr_tmpfile("xds_bootstrap_v3", &bootstrap_file);
209 fputs(kBootstrapFileV3, out);
210 fclose(out);
211 g_bootstrap_file_v3 = bootstrap_file;
212 out = gpr_tmpfile("xds_bootstrap_v2", &bootstrap_file);
213 fputs(kBootstrapFileV2, out);
214 fclose(out);
215 g_bootstrap_file_v2 = bootstrap_file;
216 }
217
218 // Helper class to minimize the number of unique ports we use for this test.
219 class PortSaver {
220 public:
GetPort()221 int GetPort() {
222 if (idx_ >= ports_.size()) {
223 ports_.push_back(grpc_pick_unused_port_or_die());
224 }
225 return ports_[idx_++];
226 }
227
Reset()228 void Reset() { idx_ = 0; }
229
230 private:
231 std::vector<int> ports_;
232 size_t idx_ = 0;
233 };
234
235 PortSaver* g_port_saver = nullptr;
236
237 template <typename ServiceType>
238 class CountedService : public ServiceType {
239 public:
request_count()240 size_t request_count() {
241 grpc_core::MutexLock lock(&mu_);
242 return request_count_;
243 }
244
response_count()245 size_t response_count() {
246 grpc_core::MutexLock lock(&mu_);
247 return response_count_;
248 }
249
IncreaseResponseCount()250 void IncreaseResponseCount() {
251 grpc_core::MutexLock lock(&mu_);
252 ++response_count_;
253 }
IncreaseRequestCount()254 void IncreaseRequestCount() {
255 grpc_core::MutexLock lock(&mu_);
256 ++request_count_;
257 }
258
ResetCounters()259 void ResetCounters() {
260 grpc_core::MutexLock lock(&mu_);
261 request_count_ = 0;
262 response_count_ = 0;
263 }
264
265 private:
266 grpc_core::Mutex mu_;
267 size_t request_count_ = 0;
268 size_t response_count_ = 0;
269 };
270
271 const char g_kCallCredsMdKey[] = "Balancer should not ...";
272 const char g_kCallCredsMdValue[] = "... receive me";
273
274 template <typename RpcService>
275 class BackendServiceImpl
276 : public CountedService<TestMultipleServiceImpl<RpcService>> {
277 public:
BackendServiceImpl()278 BackendServiceImpl() {}
279
Echo(ServerContext * context,const EchoRequest * request,EchoResponse * response)280 Status Echo(ServerContext* context, const EchoRequest* request,
281 EchoResponse* response) override {
282 // Backend should receive the call credentials metadata.
283 auto call_credentials_entry =
284 context->client_metadata().find(g_kCallCredsMdKey);
285 EXPECT_NE(call_credentials_entry, context->client_metadata().end());
286 if (call_credentials_entry != context->client_metadata().end()) {
287 EXPECT_EQ(call_credentials_entry->second, g_kCallCredsMdValue);
288 }
289 CountedService<TestMultipleServiceImpl<RpcService>>::IncreaseRequestCount();
290 const auto status =
291 TestMultipleServiceImpl<RpcService>::Echo(context, request, response);
292 CountedService<
293 TestMultipleServiceImpl<RpcService>>::IncreaseResponseCount();
294 AddClient(context->peer());
295 return status;
296 }
297
Echo1(ServerContext * context,const EchoRequest * request,EchoResponse * response)298 Status Echo1(ServerContext* context, const EchoRequest* request,
299 EchoResponse* response) override {
300 return Echo(context, request, response);
301 }
302
Echo2(ServerContext * context,const EchoRequest * request,EchoResponse * response)303 Status Echo2(ServerContext* context, const EchoRequest* request,
304 EchoResponse* response) override {
305 return Echo(context, request, response);
306 }
307
Start()308 void Start() {}
Shutdown()309 void Shutdown() {}
310
clients()311 std::set<std::string> clients() {
312 grpc_core::MutexLock lock(&clients_mu_);
313 return clients_;
314 }
315
316 private:
AddClient(const std::string & client)317 void AddClient(const std::string& client) {
318 grpc_core::MutexLock lock(&clients_mu_);
319 clients_.insert(client);
320 }
321
322 grpc_core::Mutex clients_mu_;
323 std::set<std::string> clients_;
324 };
325
326 class ClientStats {
327 public:
328 struct LocalityStats {
LocalityStatsgrpc::testing::__anon6f70f29a0111::ClientStats::LocalityStats329 LocalityStats() {}
330
331 // Converts from proto message class.
332 template <class UpstreamLocalityStats>
LocalityStatsgrpc::testing::__anon6f70f29a0111::ClientStats::LocalityStats333 LocalityStats(const UpstreamLocalityStats& upstream_locality_stats)
334 : total_successful_requests(
335 upstream_locality_stats.total_successful_requests()),
336 total_requests_in_progress(
337 upstream_locality_stats.total_requests_in_progress()),
338 total_error_requests(upstream_locality_stats.total_error_requests()),
339 total_issued_requests(
340 upstream_locality_stats.total_issued_requests()) {}
341
operator +=grpc::testing::__anon6f70f29a0111::ClientStats::LocalityStats342 LocalityStats& operator+=(const LocalityStats& other) {
343 total_successful_requests += other.total_successful_requests;
344 total_requests_in_progress += other.total_requests_in_progress;
345 total_error_requests += other.total_error_requests;
346 total_issued_requests += other.total_issued_requests;
347 return *this;
348 }
349
350 uint64_t total_successful_requests = 0;
351 uint64_t total_requests_in_progress = 0;
352 uint64_t total_error_requests = 0;
353 uint64_t total_issued_requests = 0;
354 };
355
ClientStats()356 ClientStats() {}
357
358 // Converts from proto message class.
359 template <class ClusterStats>
ClientStats(const ClusterStats & cluster_stats)360 explicit ClientStats(const ClusterStats& cluster_stats)
361 : cluster_name_(cluster_stats.cluster_name()),
362 total_dropped_requests_(cluster_stats.total_dropped_requests()) {
363 for (const auto& input_locality_stats :
364 cluster_stats.upstream_locality_stats()) {
365 locality_stats_.emplace(input_locality_stats.locality().sub_zone(),
366 LocalityStats(input_locality_stats));
367 }
368 for (const auto& input_dropped_requests :
369 cluster_stats.dropped_requests()) {
370 dropped_requests_.emplace(input_dropped_requests.category(),
371 input_dropped_requests.dropped_count());
372 }
373 }
374
cluster_name() const375 const std::string& cluster_name() const { return cluster_name_; }
376
locality_stats() const377 const std::map<std::string, LocalityStats>& locality_stats() const {
378 return locality_stats_;
379 }
total_successful_requests() const380 uint64_t total_successful_requests() const {
381 uint64_t sum = 0;
382 for (auto& p : locality_stats_) {
383 sum += p.second.total_successful_requests;
384 }
385 return sum;
386 }
total_requests_in_progress() const387 uint64_t total_requests_in_progress() const {
388 uint64_t sum = 0;
389 for (auto& p : locality_stats_) {
390 sum += p.second.total_requests_in_progress;
391 }
392 return sum;
393 }
total_error_requests() const394 uint64_t total_error_requests() const {
395 uint64_t sum = 0;
396 for (auto& p : locality_stats_) {
397 sum += p.second.total_error_requests;
398 }
399 return sum;
400 }
total_issued_requests() const401 uint64_t total_issued_requests() const {
402 uint64_t sum = 0;
403 for (auto& p : locality_stats_) {
404 sum += p.second.total_issued_requests;
405 }
406 return sum;
407 }
408
total_dropped_requests() const409 uint64_t total_dropped_requests() const { return total_dropped_requests_; }
410
dropped_requests(const std::string & category) const411 uint64_t dropped_requests(const std::string& category) const {
412 auto iter = dropped_requests_.find(category);
413 GPR_ASSERT(iter != dropped_requests_.end());
414 return iter->second;
415 }
416
operator +=(const ClientStats & other)417 ClientStats& operator+=(const ClientStats& other) {
418 for (const auto& p : other.locality_stats_) {
419 locality_stats_[p.first] += p.second;
420 }
421 total_dropped_requests_ += other.total_dropped_requests_;
422 for (const auto& p : other.dropped_requests_) {
423 dropped_requests_[p.first] += p.second;
424 }
425 return *this;
426 }
427
428 private:
429 std::string cluster_name_;
430 std::map<std::string, LocalityStats> locality_stats_;
431 uint64_t total_dropped_requests_ = 0;
432 std::map<std::string, uint64_t> dropped_requests_;
433 };
434
435 // TODO(roth) move all of the code that deals with default resource contents out
436 // of AdsServiceImpl and into XdsEnd2EndTest.
437 class AdsServiceImpl : public std::enable_shared_from_this<AdsServiceImpl> {
438 public:
439 struct ResponseState {
440 enum State { NOT_SENT, SENT, ACKED, NACKED };
441 State state = NOT_SENT;
442 std::string error_message;
443 };
444
445 struct EdsResourceArgs {
446 struct Locality {
Localitygrpc::testing::__anon6f70f29a0111::AdsServiceImpl::EdsResourceArgs::Locality447 Locality(std::string sub_zone, std::vector<int> ports,
448 int lb_weight = kDefaultLocalityWeight,
449 int priority = kDefaultLocalityPriority,
450 std::vector<HealthStatus> health_statuses = {})
451 : sub_zone(std::move(sub_zone)),
452 ports(std::move(ports)),
453 lb_weight(lb_weight),
454 priority(priority),
455 health_statuses(std::move(health_statuses)) {}
456
457 const std::string sub_zone;
458 std::vector<int> ports;
459 int lb_weight;
460 int priority;
461 std::vector<HealthStatus> health_statuses;
462 };
463
464 EdsResourceArgs() = default;
EdsResourceArgsgrpc::testing::__anon6f70f29a0111::AdsServiceImpl::EdsResourceArgs465 explicit EdsResourceArgs(std::vector<Locality> locality_list)
466 : locality_list(std::move(locality_list)) {}
467
468 std::vector<Locality> locality_list;
469 std::map<std::string, uint32_t> drop_categories;
470 FractionalPercent::DenominatorType drop_denominator =
471 FractionalPercent::MILLION;
472 };
473
AdsServiceImpl()474 AdsServiceImpl()
475 : v2_rpc_service_(this, /*is_v2=*/true),
476 v3_rpc_service_(this, /*is_v2=*/false) {}
477
seen_v2_client() const478 bool seen_v2_client() const { return seen_v2_client_; }
seen_v3_client() const479 bool seen_v3_client() const { return seen_v3_client_; }
480
481 ::envoy::service::discovery::v2::AggregatedDiscoveryService::Service*
v2_rpc_service()482 v2_rpc_service() {
483 return &v2_rpc_service_;
484 }
485
486 ::envoy::service::discovery::v3::AggregatedDiscoveryService::Service*
v3_rpc_service()487 v3_rpc_service() {
488 return &v3_rpc_service_;
489 }
490
lds_response_state()491 ResponseState lds_response_state() {
492 grpc_core::MutexLock lock(&ads_mu_);
493 return resource_type_response_state_[kLdsTypeUrl];
494 }
495
rds_response_state()496 ResponseState rds_response_state() {
497 grpc_core::MutexLock lock(&ads_mu_);
498 return resource_type_response_state_[kRdsTypeUrl];
499 }
500
cds_response_state()501 ResponseState cds_response_state() {
502 grpc_core::MutexLock lock(&ads_mu_);
503 return resource_type_response_state_[kCdsTypeUrl];
504 }
505
eds_response_state()506 ResponseState eds_response_state() {
507 grpc_core::MutexLock lock(&ads_mu_);
508 return resource_type_response_state_[kEdsTypeUrl];
509 }
510
SetResourceIgnore(const std::string & type_url)511 void SetResourceIgnore(const std::string& type_url) {
512 grpc_core::MutexLock lock(&ads_mu_);
513 resource_types_to_ignore_.emplace(type_url);
514 }
515
SetResourceMinVersion(const std::string & type_url,int version)516 void SetResourceMinVersion(const std::string& type_url, int version) {
517 grpc_core::MutexLock lock(&ads_mu_);
518 resource_type_min_versions_[type_url] = version;
519 }
520
UnsetResource(const std::string & type_url,const std::string & name)521 void UnsetResource(const std::string& type_url, const std::string& name) {
522 grpc_core::MutexLock lock(&ads_mu_);
523 ResourceTypeState& resource_type_state = resource_map_[type_url];
524 ++resource_type_state.resource_type_version;
525 ResourceState& resource_state = resource_type_state.resource_name_map[name];
526 resource_state.resource_type_version =
527 resource_type_state.resource_type_version;
528 resource_state.resource.reset();
529 gpr_log(GPR_INFO,
530 "ADS[%p]: Unsetting %s resource %s; resource_type_version now %u",
531 this, type_url.c_str(), name.c_str(),
532 resource_type_state.resource_type_version);
533 for (SubscriptionState* subscription : resource_state.subscriptions) {
534 subscription->update_queue->emplace_back(type_url, name);
535 }
536 }
537
SetResource(google::protobuf::Any resource,const std::string & type_url,const std::string & name)538 void SetResource(google::protobuf::Any resource, const std::string& type_url,
539 const std::string& name) {
540 grpc_core::MutexLock lock(&ads_mu_);
541 ResourceTypeState& resource_type_state = resource_map_[type_url];
542 ++resource_type_state.resource_type_version;
543 ResourceState& resource_state = resource_type_state.resource_name_map[name];
544 resource_state.resource_type_version =
545 resource_type_state.resource_type_version;
546 resource_state.resource = std::move(resource);
547 gpr_log(GPR_INFO,
548 "ADS[%p]: Updating %s resource %s; resource_type_version now %u",
549 this, type_url.c_str(), name.c_str(),
550 resource_type_state.resource_type_version);
551 for (SubscriptionState* subscription : resource_state.subscriptions) {
552 subscription->update_queue->emplace_back(type_url, name);
553 }
554 }
555
SetLdsResource(const Listener & listener)556 void SetLdsResource(const Listener& listener) {
557 google::protobuf::Any resource;
558 resource.PackFrom(listener);
559 SetResource(std::move(resource), kLdsTypeUrl, listener.name());
560 }
561
SetRdsResource(const RouteConfiguration & route)562 void SetRdsResource(const RouteConfiguration& route) {
563 google::protobuf::Any resource;
564 resource.PackFrom(route);
565 SetResource(std::move(resource), kRdsTypeUrl, route.name());
566 }
567
SetCdsResource(const Cluster & cluster)568 void SetCdsResource(const Cluster& cluster) {
569 google::protobuf::Any resource;
570 resource.PackFrom(cluster);
571 SetResource(std::move(resource), kCdsTypeUrl, cluster.name());
572 }
573
SetEdsResource(const ClusterLoadAssignment & assignment)574 void SetEdsResource(const ClusterLoadAssignment& assignment) {
575 google::protobuf::Any resource;
576 resource.PackFrom(assignment);
577 SetResource(std::move(resource), kEdsTypeUrl, assignment.cluster_name());
578 }
579
Start()580 void Start() {
581 grpc_core::MutexLock lock(&ads_mu_);
582 ads_done_ = false;
583 }
584
Shutdown()585 void Shutdown() {
586 {
587 grpc_core::MutexLock lock(&ads_mu_);
588 NotifyDoneWithAdsCallLocked();
589 resource_type_response_state_.clear();
590 }
591 gpr_log(GPR_INFO, "ADS[%p]: shut down", this);
592 }
593
NotifyDoneWithAdsCall()594 void NotifyDoneWithAdsCall() {
595 grpc_core::MutexLock lock(&ads_mu_);
596 NotifyDoneWithAdsCallLocked();
597 }
598
NotifyDoneWithAdsCallLocked()599 void NotifyDoneWithAdsCallLocked() {
600 if (!ads_done_) {
601 ads_done_ = true;
602 ads_cond_.Broadcast();
603 }
604 }
605
clients()606 std::set<std::string> clients() {
607 grpc_core::MutexLock lock(&clients_mu_);
608 return clients_;
609 }
610
611 private:
612 // A queue of resource type/name pairs that have changed since the client
613 // subscribed to them.
614 using UpdateQueue = std::deque<
615 std::pair<std::string /* type url */, std::string /* resource name */>>;
616
617 // A struct representing a client's subscription to a particular resource.
618 struct SubscriptionState {
619 // The queue upon which to place updates when the resource is updated.
620 UpdateQueue* update_queue;
621 };
622
623 // A struct representing the a client's subscription to all the resources.
624 using SubscriptionNameMap =
625 std::map<std::string /* resource_name */, SubscriptionState>;
626 using SubscriptionMap =
627 std::map<std::string /* type_url */, SubscriptionNameMap>;
628
629 // Sent state for a given resource type.
630 struct SentState {
631 int nonce = 0;
632 int resource_type_version = 0;
633 };
634
635 // A struct representing the current state for an individual resource.
636 struct ResourceState {
637 // The resource itself, if present.
638 absl::optional<google::protobuf::Any> resource;
639 // The resource type version that this resource was last updated in.
640 int resource_type_version = 0;
641 // A list of subscriptions to this resource.
642 std::set<SubscriptionState*> subscriptions;
643 };
644
645 // The current state for all individual resources of a given type.
646 using ResourceNameMap =
647 std::map<std::string /* resource_name */, ResourceState>;
648
649 struct ResourceTypeState {
650 int resource_type_version = 0;
651 ResourceNameMap resource_name_map;
652 };
653
654 using ResourceMap = std::map<std::string /* type_url */, ResourceTypeState>;
655
656 template <class RpcApi, class DiscoveryRequest, class DiscoveryResponse>
657 class RpcService : public RpcApi::Service {
658 public:
659 using Stream = ServerReaderWriter<DiscoveryResponse, DiscoveryRequest>;
660
RpcService(AdsServiceImpl * parent,bool is_v2)661 RpcService(AdsServiceImpl* parent, bool is_v2)
662 : parent_(parent), is_v2_(is_v2) {}
663
StreamAggregatedResources(ServerContext * context,Stream * stream)664 Status StreamAggregatedResources(ServerContext* context,
665 Stream* stream) override {
666 gpr_log(GPR_INFO, "ADS[%p]: StreamAggregatedResources starts", this);
667 parent_->AddClient(context->peer());
668 if (is_v2_) {
669 parent_->seen_v2_client_ = true;
670 } else {
671 parent_->seen_v3_client_ = true;
672 }
673 // Balancer shouldn't receive the call credentials metadata.
674 EXPECT_EQ(context->client_metadata().find(g_kCallCredsMdKey),
675 context->client_metadata().end());
676 // Take a reference of the AdsServiceImpl object, which will go
677 // out of scope when this request handler returns. This ensures
678 // that the parent won't be destroyed until this stream is complete.
679 std::shared_ptr<AdsServiceImpl> ads_service_impl =
680 parent_->shared_from_this();
681 // Resources (type/name pairs) that have changed since the client
682 // subscribed to them.
683 UpdateQueue update_queue;
684 // Resources that the client will be subscribed to keyed by resource type
685 // url.
686 SubscriptionMap subscription_map;
687 // Sent state for each resource type.
688 std::map<std::string /*type_url*/, SentState> sent_state_map;
689 // Spawn a thread to read requests from the stream.
690 // Requests will be delivered to this thread in a queue.
691 std::deque<DiscoveryRequest> requests;
692 bool stream_closed = false;
693 std::thread reader(std::bind(&RpcService::BlockingRead, this, stream,
694 &requests, &stream_closed));
695 // Main loop to process requests and updates.
696 while (true) {
697 // Boolean to keep track if the loop received any work to do: a
698 // request or an update; regardless whether a response was actually
699 // sent out.
700 bool did_work = false;
701 // Look for new requests and and decide what to handle.
702 absl::optional<DiscoveryResponse> response;
703 {
704 grpc_core::MutexLock lock(&parent_->ads_mu_);
705 // If the stream has been closed or our parent is being shut
706 // down, stop immediately.
707 if (stream_closed || parent_->ads_done_) break;
708 // Otherwise, see if there's a request to read from the queue.
709 if (!requests.empty()) {
710 DiscoveryRequest request = std::move(requests.front());
711 requests.pop_front();
712 did_work = true;
713 gpr_log(GPR_INFO,
714 "ADS[%p]: Received request for type %s with content %s",
715 this, request.type_url().c_str(),
716 request.DebugString().c_str());
717 const std::string v3_resource_type =
718 TypeUrlToV3(request.type_url());
719 SentState& sent_state = sent_state_map[v3_resource_type];
720 // Process request.
721 ProcessRequest(request, v3_resource_type, &update_queue,
722 &subscription_map, &sent_state, &response);
723 }
724 }
725 if (response.has_value()) {
726 gpr_log(GPR_INFO, "ADS[%p]: Sending response: %s", this,
727 response->DebugString().c_str());
728 stream->Write(response.value());
729 }
730 response.reset();
731 // Look for updates and decide what to handle.
732 {
733 grpc_core::MutexLock lock(&parent_->ads_mu_);
734 if (!update_queue.empty()) {
735 const std::string resource_type =
736 std::move(update_queue.front().first);
737 const std::string resource_name =
738 std::move(update_queue.front().second);
739 update_queue.pop_front();
740 did_work = true;
741 SentState& sent_state = sent_state_map[resource_type];
742 ProcessUpdate(resource_type, resource_name, &subscription_map,
743 &sent_state, &response);
744 }
745 }
746 if (response.has_value()) {
747 gpr_log(GPR_INFO, "ADS[%p]: Sending update response: %s", this,
748 response->DebugString().c_str());
749 stream->Write(response.value());
750 }
751 // If we didn't find anything to do, delay before the next loop
752 // iteration; otherwise, check whether we should exit and then
753 // immediately continue.
754 gpr_timespec deadline =
755 grpc_timeout_milliseconds_to_deadline(did_work ? 0 : 10);
756 {
757 grpc_core::MutexLock lock(&parent_->ads_mu_);
758 if (!parent_->ads_cond_.WaitUntil(
759 &parent_->ads_mu_, [this] { return parent_->ads_done_; },
760 deadline)) {
761 break;
762 }
763 }
764 }
765 // Done with main loop. Clean up before returning.
766 // Join reader thread.
767 reader.join();
768 // Clean up any subscriptions that were still active when the call
769 // finished.
770 {
771 grpc_core::MutexLock lock(&parent_->ads_mu_);
772 for (auto& p : subscription_map) {
773 const std::string& type_url = p.first;
774 SubscriptionNameMap& subscription_name_map = p.second;
775 for (auto& q : subscription_name_map) {
776 const std::string& resource_name = q.first;
777 SubscriptionState& subscription_state = q.second;
778 ResourceNameMap& resource_name_map =
779 parent_->resource_map_[type_url].resource_name_map;
780 ResourceState& resource_state = resource_name_map[resource_name];
781 resource_state.subscriptions.erase(&subscription_state);
782 }
783 }
784 }
785 gpr_log(GPR_INFO, "ADS[%p]: StreamAggregatedResources done", this);
786 parent_->RemoveClient(context->peer());
787 return Status::OK;
788 }
789
790 private:
791 // Processes a response read from the client.
792 // Populates response if needed.
ProcessRequest(const DiscoveryRequest & request,const std::string & v3_resource_type,UpdateQueue * update_queue,SubscriptionMap * subscription_map,SentState * sent_state,absl::optional<DiscoveryResponse> * response)793 void ProcessRequest(const DiscoveryRequest& request,
794 const std::string& v3_resource_type,
795 UpdateQueue* update_queue,
796 SubscriptionMap* subscription_map,
797 SentState* sent_state,
798 absl::optional<DiscoveryResponse>* response) {
799 // Determine client resource type version.
800 int client_resource_type_version = 0;
801 if (!request.version_info().empty()) {
802 GPR_ASSERT(absl::SimpleAtoi(request.version_info(),
803 &client_resource_type_version));
804 }
805 // Check the nonce sent by the client, if any.
806 // (This will be absent on the first request on a stream.)
807 if (request.response_nonce().empty()) {
808 EXPECT_GE(client_resource_type_version,
809 parent_->resource_type_min_versions_[v3_resource_type])
810 << "resource_type: " << v3_resource_type;
811 } else {
812 int client_nonce;
813 GPR_ASSERT(absl::SimpleAtoi(request.response_nonce(), &client_nonce));
814 // Ignore requests with stale nonces.
815 if (client_nonce < sent_state->nonce) return;
816 // Check for ACK or NACK.
817 auto it = parent_->resource_type_response_state_.find(v3_resource_type);
818 if (it != parent_->resource_type_response_state_.end()) {
819 if (client_resource_type_version ==
820 sent_state->resource_type_version) {
821 it->second.state = ResponseState::ACKED;
822 it->second.error_message.clear();
823 gpr_log(GPR_INFO,
824 "ADS[%p]: client ACKed resource_type=%s version=%s", this,
825 request.type_url().c_str(), request.version_info().c_str());
826 } else {
827 it->second.state = ResponseState::NACKED;
828 EXPECT_EQ(request.error_detail().code(),
829 GRPC_STATUS_INVALID_ARGUMENT);
830 it->second.error_message = request.error_detail().message();
831 gpr_log(GPR_INFO,
832 "ADS[%p]: client NACKed resource_type=%s version=%s: %s",
833 this, request.type_url().c_str(),
834 request.version_info().c_str(),
835 it->second.error_message.c_str());
836 }
837 }
838 }
839 // Ignore resource types as requested by tests.
840 if (parent_->resource_types_to_ignore_.find(v3_resource_type) !=
841 parent_->resource_types_to_ignore_.end()) {
842 return;
843 }
844 // Look at all the resource names in the request.
845 auto& subscription_name_map = (*subscription_map)[v3_resource_type];
846 auto& resource_type_state = parent_->resource_map_[v3_resource_type];
847 auto& resource_name_map = resource_type_state.resource_name_map;
848 std::set<std::string> resources_in_current_request;
849 std::set<std::string> resources_added_to_response;
850 for (const std::string& resource_name : request.resource_names()) {
851 resources_in_current_request.emplace(resource_name);
852 auto& subscription_state = subscription_name_map[resource_name];
853 auto& resource_state = resource_name_map[resource_name];
854 // Subscribe if needed.
855 // Send the resource in the response if either (a) this is
856 // a new subscription or (b) there is an updated version of
857 // this resource to send.
858 if (parent_->MaybeSubscribe(v3_resource_type, resource_name,
859 &subscription_state, &resource_state,
860 update_queue) ||
861 ClientNeedsResourceUpdate(resource_type_state, resource_state,
862 client_resource_type_version,
863 &subscription_state)) {
864 gpr_log(GPR_INFO, "ADS[%p]: Sending update for type=%s name=%s", this,
865 request.type_url().c_str(), resource_name.c_str());
866 resources_added_to_response.emplace(resource_name);
867 if (!response->has_value()) response->emplace();
868 if (resource_state.resource.has_value()) {
869 auto* resource = (*response)->add_resources();
870 resource->CopyFrom(resource_state.resource.value());
871 if (is_v2_) {
872 resource->set_type_url(request.type_url());
873 }
874 }
875 } else {
876 gpr_log(GPR_INFO,
877 "ADS[%p]: client does not need update for type=%s name=%s",
878 this, request.type_url().c_str(), resource_name.c_str());
879 }
880 }
881 // Process unsubscriptions for any resource no longer
882 // present in the request's resource list.
883 parent_->ProcessUnsubscriptions(
884 v3_resource_type, resources_in_current_request,
885 &subscription_name_map, &resource_name_map);
886 // Construct response if needed.
887 if (!resources_added_to_response.empty()) {
888 CompleteBuildingDiscoveryResponse(
889 v3_resource_type, request.type_url(),
890 resource_type_state.resource_type_version, subscription_name_map,
891 resources_added_to_response, sent_state, &response->value());
892 }
893 }
894
895 // Processes a resource update from the test.
896 // Populates response if needed.
ProcessUpdate(const std::string & resource_type,const std::string & resource_name,SubscriptionMap * subscription_map,SentState * sent_state,absl::optional<DiscoveryResponse> * response)897 void ProcessUpdate(const std::string& resource_type,
898 const std::string& resource_name,
899 SubscriptionMap* subscription_map, SentState* sent_state,
900 absl::optional<DiscoveryResponse>* response) {
901 const std::string v2_resource_type = TypeUrlToV2(resource_type);
902 gpr_log(GPR_INFO, "ADS[%p]: Received update for type=%s name=%s", this,
903 resource_type.c_str(), resource_name.c_str());
904 auto& subscription_name_map = (*subscription_map)[resource_type];
905 auto& resource_type_state = parent_->resource_map_[resource_type];
906 auto& resource_name_map = resource_type_state.resource_name_map;
907 auto it = subscription_name_map.find(resource_name);
908 if (it != subscription_name_map.end()) {
909 SubscriptionState& subscription_state = it->second;
910 ResourceState& resource_state = resource_name_map[resource_name];
911 if (ClientNeedsResourceUpdate(resource_type_state, resource_state,
912 sent_state->resource_type_version,
913 &subscription_state)) {
914 gpr_log(GPR_INFO, "ADS[%p]: Sending update for type=%s name=%s", this,
915 resource_type.c_str(), resource_name.c_str());
916 response->emplace();
917 if (resource_state.resource.has_value()) {
918 auto* resource = (*response)->add_resources();
919 resource->CopyFrom(resource_state.resource.value());
920 if (is_v2_) {
921 resource->set_type_url(v2_resource_type);
922 }
923 }
924 CompleteBuildingDiscoveryResponse(
925 resource_type, v2_resource_type,
926 resource_type_state.resource_type_version, subscription_name_map,
927 {resource_name}, sent_state, &response->value());
928 }
929 }
930 }
931
932 // Starting a thread to do blocking read on the stream until cancel.
BlockingRead(Stream * stream,std::deque<DiscoveryRequest> * requests,bool * stream_closed)933 void BlockingRead(Stream* stream, std::deque<DiscoveryRequest>* requests,
934 bool* stream_closed) {
935 DiscoveryRequest request;
936 bool seen_first_request = false;
937 while (stream->Read(&request)) {
938 if (!seen_first_request) {
939 EXPECT_TRUE(request.has_node());
940 ASSERT_FALSE(request.node().client_features().empty());
941 EXPECT_EQ(request.node().client_features(0),
942 "envoy.lb.does_not_support_overprovisioning");
943 CheckBuildVersion(request);
944 seen_first_request = true;
945 }
946 {
947 grpc_core::MutexLock lock(&parent_->ads_mu_);
948 requests->emplace_back(std::move(request));
949 }
950 }
951 gpr_log(GPR_INFO, "ADS[%p]: Null read, stream closed", this);
952 grpc_core::MutexLock lock(&parent_->ads_mu_);
953 *stream_closed = true;
954 }
955
956 // Completing the building a DiscoveryResponse by adding common information
957 // for all resources and by adding all subscribed resources for LDS and CDS.
CompleteBuildingDiscoveryResponse(const std::string & resource_type,const std::string & v2_resource_type,const int version,const SubscriptionNameMap & subscription_name_map,const std::set<std::string> & resources_added_to_response,SentState * sent_state,DiscoveryResponse * response)958 void CompleteBuildingDiscoveryResponse(
959 const std::string& resource_type, const std::string& v2_resource_type,
960 const int version, const SubscriptionNameMap& subscription_name_map,
961 const std::set<std::string>& resources_added_to_response,
962 SentState* sent_state, DiscoveryResponse* response) {
963 auto& response_state =
964 parent_->resource_type_response_state_[resource_type];
965 if (response_state.state == ResponseState::NOT_SENT) {
966 response_state.state = ResponseState::SENT;
967 }
968 response->set_type_url(is_v2_ ? v2_resource_type : resource_type);
969 response->set_version_info(std::to_string(version));
970 response->set_nonce(std::to_string(++sent_state->nonce));
971 if (resource_type == kLdsTypeUrl || resource_type == kCdsTypeUrl) {
972 // For LDS and CDS we must send back all subscribed resources
973 // (even the unchanged ones)
974 for (const auto& p : subscription_name_map) {
975 const std::string& resource_name = p.first;
976 if (resources_added_to_response.find(resource_name) ==
977 resources_added_to_response.end()) {
978 ResourceNameMap& resource_name_map =
979 parent_->resource_map_[resource_type].resource_name_map;
980 const ResourceState& resource_state =
981 resource_name_map[resource_name];
982 if (resource_state.resource.has_value()) {
983 auto* resource = response->add_resources();
984 resource->CopyFrom(resource_state.resource.value());
985 if (is_v2_) {
986 resource->set_type_url(v2_resource_type);
987 }
988 }
989 }
990 }
991 }
992 sent_state->resource_type_version = version;
993 }
994
TypeUrlToV2(const std::string & resource_type)995 static std::string TypeUrlToV2(const std::string& resource_type) {
996 if (resource_type == kLdsTypeUrl) return kLdsV2TypeUrl;
997 if (resource_type == kRdsTypeUrl) return kRdsV2TypeUrl;
998 if (resource_type == kCdsTypeUrl) return kCdsV2TypeUrl;
999 if (resource_type == kEdsTypeUrl) return kEdsV2TypeUrl;
1000 return resource_type;
1001 }
1002
TypeUrlToV3(const std::string & resource_type)1003 static std::string TypeUrlToV3(const std::string& resource_type) {
1004 if (resource_type == kLdsV2TypeUrl) return kLdsTypeUrl;
1005 if (resource_type == kRdsV2TypeUrl) return kRdsTypeUrl;
1006 if (resource_type == kCdsV2TypeUrl) return kCdsTypeUrl;
1007 if (resource_type == kEdsV2TypeUrl) return kEdsTypeUrl;
1008 return resource_type;
1009 }
1010
CheckBuildVersion(const::envoy::api::v2::DiscoveryRequest & request)1011 static void CheckBuildVersion(
1012 const ::envoy::api::v2::DiscoveryRequest& request) {
1013 EXPECT_FALSE(request.node().build_version().empty());
1014 }
1015
CheckBuildVersion(const::envoy::service::discovery::v3::DiscoveryRequest & request)1016 static void CheckBuildVersion(
1017 const ::envoy::service::discovery::v3::DiscoveryRequest& request) {}
1018
1019 AdsServiceImpl* parent_;
1020 const bool is_v2_;
1021 };
1022
1023 // Checks whether the client needs to receive a newer version of
1024 // the resource.
ClientNeedsResourceUpdate(const ResourceTypeState & resource_type_state,const ResourceState & resource_state,int client_resource_type_version,SubscriptionState * subscription_state)1025 static bool ClientNeedsResourceUpdate(
1026 const ResourceTypeState& resource_type_state,
1027 const ResourceState& resource_state, int client_resource_type_version,
1028 SubscriptionState* subscription_state) {
1029 return client_resource_type_version <
1030 resource_type_state.resource_type_version &&
1031 resource_state.resource_type_version <=
1032 resource_type_state.resource_type_version;
1033 }
1034
1035 // Subscribes to a resource if not already subscribed:
1036 // 1. Sets the update_queue field in subscription_state.
1037 // 2. Adds subscription_state to resource_state->subscriptions.
MaybeSubscribe(const std::string & resource_type,const std::string & resource_name,SubscriptionState * subscription_state,ResourceState * resource_state,UpdateQueue * update_queue)1038 bool MaybeSubscribe(const std::string& resource_type,
1039 const std::string& resource_name,
1040 SubscriptionState* subscription_state,
1041 ResourceState* resource_state,
1042 UpdateQueue* update_queue) {
1043 // The update_queue will be null if we were not previously subscribed.
1044 if (subscription_state->update_queue != nullptr) return false;
1045 subscription_state->update_queue = update_queue;
1046 resource_state->subscriptions.emplace(subscription_state);
1047 gpr_log(GPR_INFO, "ADS[%p]: subscribe to resource type %s name %s state %p",
1048 this, resource_type.c_str(), resource_name.c_str(),
1049 &subscription_state);
1050 return true;
1051 }
1052
1053 // Removes subscriptions for resources no longer present in the
1054 // current request.
ProcessUnsubscriptions(const std::string & resource_type,const std::set<std::string> & resources_in_current_request,SubscriptionNameMap * subscription_name_map,ResourceNameMap * resource_name_map)1055 void ProcessUnsubscriptions(
1056 const std::string& resource_type,
1057 const std::set<std::string>& resources_in_current_request,
1058 SubscriptionNameMap* subscription_name_map,
1059 ResourceNameMap* resource_name_map) {
1060 for (auto it = subscription_name_map->begin();
1061 it != subscription_name_map->end();) {
1062 const std::string& resource_name = it->first;
1063 SubscriptionState& subscription_state = it->second;
1064 if (resources_in_current_request.find(resource_name) !=
1065 resources_in_current_request.end()) {
1066 ++it;
1067 continue;
1068 }
1069 gpr_log(GPR_INFO, "ADS[%p]: Unsubscribe to type=%s name=%s state=%p",
1070 this, resource_type.c_str(), resource_name.c_str(),
1071 &subscription_state);
1072 auto resource_it = resource_name_map->find(resource_name);
1073 GPR_ASSERT(resource_it != resource_name_map->end());
1074 auto& resource_state = resource_it->second;
1075 resource_state.subscriptions.erase(&subscription_state);
1076 if (resource_state.subscriptions.empty() &&
1077 !resource_state.resource.has_value()) {
1078 resource_name_map->erase(resource_it);
1079 }
1080 it = subscription_name_map->erase(it);
1081 }
1082 }
1083
AddClient(const std::string & client)1084 void AddClient(const std::string& client) {
1085 grpc_core::MutexLock lock(&clients_mu_);
1086 clients_.insert(client);
1087 }
1088
RemoveClient(const std::string & client)1089 void RemoveClient(const std::string& client) {
1090 grpc_core::MutexLock lock(&clients_mu_);
1091 clients_.erase(client);
1092 }
1093
1094 RpcService<::envoy::service::discovery::v2::AggregatedDiscoveryService,
1095 ::envoy::api::v2::DiscoveryRequest,
1096 ::envoy::api::v2::DiscoveryResponse>
1097 v2_rpc_service_;
1098 RpcService<::envoy::service::discovery::v3::AggregatedDiscoveryService,
1099 ::envoy::service::discovery::v3::DiscoveryRequest,
1100 ::envoy::service::discovery::v3::DiscoveryResponse>
1101 v3_rpc_service_;
1102
1103 std::atomic_bool seen_v2_client_{false};
1104 std::atomic_bool seen_v3_client_{false};
1105
1106 grpc_core::CondVar ads_cond_;
1107 // Protect the members below.
1108 grpc_core::Mutex ads_mu_;
1109 bool ads_done_ = false;
1110 std::map<std::string /* type_url */, ResponseState>
1111 resource_type_response_state_;
1112 std::set<std::string /*resource_type*/> resource_types_to_ignore_;
1113 std::map<std::string /*resource_type*/, int> resource_type_min_versions_;
1114 // An instance data member containing the current state of all resources.
1115 // Note that an entry will exist whenever either of the following is true:
1116 // - The resource exists (i.e., has been created by SetResource() and has not
1117 // yet been destroyed by UnsetResource()).
1118 // - There is at least one subscription for the resource.
1119 ResourceMap resource_map_;
1120
1121 grpc_core::Mutex clients_mu_;
1122 std::set<std::string> clients_;
1123 };
1124
1125 class LrsServiceImpl : public std::enable_shared_from_this<LrsServiceImpl> {
1126 public:
LrsServiceImpl(int client_load_reporting_interval_seconds)1127 explicit LrsServiceImpl(int client_load_reporting_interval_seconds)
1128 : v2_rpc_service_(this),
1129 v3_rpc_service_(this),
1130 client_load_reporting_interval_seconds_(
1131 client_load_reporting_interval_seconds),
1132 cluster_names_({kDefaultClusterName}) {}
1133
1134 ::envoy::service::load_stats::v2::LoadReportingService::Service*
v2_rpc_service()1135 v2_rpc_service() {
1136 return &v2_rpc_service_;
1137 }
1138
1139 ::envoy::service::load_stats::v3::LoadReportingService::Service*
v3_rpc_service()1140 v3_rpc_service() {
1141 return &v3_rpc_service_;
1142 }
1143
request_count()1144 size_t request_count() {
1145 return v2_rpc_service_.request_count() + v3_rpc_service_.request_count();
1146 }
1147
response_count()1148 size_t response_count() {
1149 return v2_rpc_service_.response_count() + v3_rpc_service_.response_count();
1150 }
1151
1152 // Must be called before the LRS call is started.
set_send_all_clusters(bool send_all_clusters)1153 void set_send_all_clusters(bool send_all_clusters) {
1154 send_all_clusters_ = send_all_clusters;
1155 }
set_cluster_names(const std::set<std::string> & cluster_names)1156 void set_cluster_names(const std::set<std::string>& cluster_names) {
1157 cluster_names_ = cluster_names;
1158 }
1159
Start()1160 void Start() {
1161 lrs_done_ = false;
1162 result_queue_.clear();
1163 }
1164
Shutdown()1165 void Shutdown() {
1166 {
1167 grpc_core::MutexLock lock(&lrs_mu_);
1168 NotifyDoneWithLrsCallLocked();
1169 }
1170 gpr_log(GPR_INFO, "LRS[%p]: shut down", this);
1171 }
1172
WaitForLoadReport()1173 std::vector<ClientStats> WaitForLoadReport() {
1174 grpc_core::MutexLock lock(&load_report_mu_);
1175 grpc_core::CondVar cv;
1176 if (result_queue_.empty()) {
1177 load_report_cond_ = &cv;
1178 load_report_cond_->WaitUntil(&load_report_mu_,
1179 [this] { return !result_queue_.empty(); });
1180 load_report_cond_ = nullptr;
1181 }
1182 std::vector<ClientStats> result = std::move(result_queue_.front());
1183 result_queue_.pop_front();
1184 return result;
1185 }
1186
NotifyDoneWithLrsCall()1187 void NotifyDoneWithLrsCall() {
1188 grpc_core::MutexLock lock(&lrs_mu_);
1189 NotifyDoneWithLrsCallLocked();
1190 }
1191
1192 private:
1193 template <class RpcApi, class LoadStatsRequest, class LoadStatsResponse>
1194 class RpcService : public CountedService<typename RpcApi::Service> {
1195 public:
1196 using Stream = ServerReaderWriter<LoadStatsResponse, LoadStatsRequest>;
1197
RpcService(LrsServiceImpl * parent)1198 explicit RpcService(LrsServiceImpl* parent) : parent_(parent) {}
1199
StreamLoadStats(ServerContext *,Stream * stream)1200 Status StreamLoadStats(ServerContext* /*context*/,
1201 Stream* stream) override {
1202 gpr_log(GPR_INFO, "LRS[%p]: StreamLoadStats starts", this);
1203 EXPECT_GT(parent_->client_load_reporting_interval_seconds_, 0);
1204 // Take a reference of the LrsServiceImpl object, reference will go
1205 // out of scope after this method exits.
1206 std::shared_ptr<LrsServiceImpl> lrs_service_impl =
1207 parent_->shared_from_this();
1208 // Read initial request.
1209 LoadStatsRequest request;
1210 if (stream->Read(&request)) {
1211 CountedService<typename RpcApi::Service>::IncreaseRequestCount();
1212 // Verify client features.
1213 EXPECT_THAT(
1214 request.node().client_features(),
1215 ::testing::Contains("envoy.lrs.supports_send_all_clusters"));
1216 // Send initial response.
1217 LoadStatsResponse response;
1218 if (parent_->send_all_clusters_) {
1219 response.set_send_all_clusters(true);
1220 } else {
1221 for (const std::string& cluster_name : parent_->cluster_names_) {
1222 response.add_clusters(cluster_name);
1223 }
1224 }
1225 response.mutable_load_reporting_interval()->set_seconds(
1226 parent_->client_load_reporting_interval_seconds_);
1227 stream->Write(response);
1228 CountedService<typename RpcApi::Service>::IncreaseResponseCount();
1229 // Wait for report.
1230 request.Clear();
1231 while (stream->Read(&request)) {
1232 gpr_log(GPR_INFO, "LRS[%p]: received client load report message: %s",
1233 this, request.DebugString().c_str());
1234 std::vector<ClientStats> stats;
1235 for (const auto& cluster_stats : request.cluster_stats()) {
1236 stats.emplace_back(cluster_stats);
1237 }
1238 grpc_core::MutexLock lock(&parent_->load_report_mu_);
1239 parent_->result_queue_.emplace_back(std::move(stats));
1240 if (parent_->load_report_cond_ != nullptr) {
1241 parent_->load_report_cond_->Signal();
1242 }
1243 }
1244 // Wait until notified done.
1245 grpc_core::MutexLock lock(&parent_->lrs_mu_);
1246 parent_->lrs_cv_.WaitUntil(&parent_->lrs_mu_,
1247 [this] { return parent_->lrs_done_; });
1248 }
1249 gpr_log(GPR_INFO, "LRS[%p]: StreamLoadStats done", this);
1250 return Status::OK;
1251 }
1252
1253 private:
1254 LrsServiceImpl* parent_;
1255 };
1256
NotifyDoneWithLrsCallLocked()1257 void NotifyDoneWithLrsCallLocked() {
1258 if (!lrs_done_) {
1259 lrs_done_ = true;
1260 lrs_cv_.Broadcast();
1261 }
1262 }
1263
1264 RpcService<::envoy::service::load_stats::v2::LoadReportingService,
1265 ::envoy::service::load_stats::v2::LoadStatsRequest,
1266 ::envoy::service::load_stats::v2::LoadStatsResponse>
1267 v2_rpc_service_;
1268 RpcService<::envoy::service::load_stats::v3::LoadReportingService,
1269 ::envoy::service::load_stats::v3::LoadStatsRequest,
1270 ::envoy::service::load_stats::v3::LoadStatsResponse>
1271 v3_rpc_service_;
1272
1273 const int client_load_reporting_interval_seconds_;
1274 bool send_all_clusters_ = false;
1275 std::set<std::string> cluster_names_;
1276
1277 grpc_core::CondVar lrs_cv_;
1278 grpc_core::Mutex lrs_mu_; // Protects lrs_done_.
1279 bool lrs_done_ = false;
1280
1281 grpc_core::Mutex load_report_mu_; // Protects the members below.
1282 grpc_core::CondVar* load_report_cond_ = nullptr;
1283 std::deque<std::vector<ClientStats>> result_queue_;
1284 };
1285
1286 class TestType {
1287 public:
TestType(bool use_xds_resolver,bool enable_load_reporting,bool enable_rds_testing=false,bool use_v2=false)1288 TestType(bool use_xds_resolver, bool enable_load_reporting,
1289 bool enable_rds_testing = false, bool use_v2 = false)
1290 : use_xds_resolver_(use_xds_resolver),
1291 enable_load_reporting_(enable_load_reporting),
1292 enable_rds_testing_(enable_rds_testing),
1293 use_v2_(use_v2) {}
1294
use_xds_resolver() const1295 bool use_xds_resolver() const { return use_xds_resolver_; }
enable_load_reporting() const1296 bool enable_load_reporting() const { return enable_load_reporting_; }
enable_rds_testing() const1297 bool enable_rds_testing() const { return enable_rds_testing_; }
use_v2() const1298 bool use_v2() const { return use_v2_; }
1299
AsString() const1300 std::string AsString() const {
1301 std::string retval = (use_xds_resolver_ ? "XdsResolver" : "FakeResolver");
1302 retval += (use_v2_ ? "V2" : "V3");
1303 if (enable_load_reporting_) retval += "WithLoadReporting";
1304 if (enable_rds_testing_) retval += "Rds";
1305 return retval;
1306 }
1307
1308 private:
1309 const bool use_xds_resolver_;
1310 const bool enable_load_reporting_;
1311 const bool enable_rds_testing_;
1312 const bool use_v2_;
1313 };
1314
1315 class XdsEnd2endTest : public ::testing::TestWithParam<TestType> {
1316 protected:
XdsEnd2endTest(size_t num_backends,size_t num_balancers,int client_load_reporting_interval_seconds=100)1317 XdsEnd2endTest(size_t num_backends, size_t num_balancers,
1318 int client_load_reporting_interval_seconds = 100)
1319 : num_backends_(num_backends),
1320 num_balancers_(num_balancers),
1321 client_load_reporting_interval_seconds_(
1322 client_load_reporting_interval_seconds) {}
1323
SetUpTestCase()1324 static void SetUpTestCase() {
1325 // Make the backup poller poll very frequently in order to pick up
1326 // updates from all the subchannels's FDs.
1327 GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, 1);
1328 #if TARGET_OS_IPHONE
1329 // Workaround Apple CFStream bug
1330 gpr_setenv("grpc_cfstream", "0");
1331 #endif
1332 grpc_init();
1333 }
1334
TearDownTestCase()1335 static void TearDownTestCase() { grpc_shutdown(); }
1336
SetUp()1337 void SetUp() override {
1338 gpr_setenv("GRPC_XDS_EXPERIMENTAL_V3_SUPPORT", "true");
1339 gpr_setenv("GRPC_XDS_BOOTSTRAP",
1340 GetParam().use_v2() ? g_bootstrap_file_v2 : g_bootstrap_file_v3);
1341 g_port_saver->Reset();
1342 bool localhost_resolves_to_ipv4 = false;
1343 bool localhost_resolves_to_ipv6 = false;
1344 grpc_core::LocalhostResolves(&localhost_resolves_to_ipv4,
1345 &localhost_resolves_to_ipv6);
1346 ipv6_only_ = !localhost_resolves_to_ipv4 && localhost_resolves_to_ipv6;
1347 response_generator_ =
1348 grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
1349 // Inject xDS channel response generator.
1350 lb_channel_response_generator_ =
1351 grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
1352 xds_channel_args_to_add_.emplace_back(
1353 grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
1354 lb_channel_response_generator_.get()));
1355 if (xds_resource_does_not_exist_timeout_ms_ > 0) {
1356 xds_channel_args_to_add_.emplace_back(grpc_channel_arg_integer_create(
1357 const_cast<char*>(GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS),
1358 xds_resource_does_not_exist_timeout_ms_));
1359 }
1360 xds_channel_args_.num_args = xds_channel_args_to_add_.size();
1361 xds_channel_args_.args = xds_channel_args_to_add_.data();
1362 grpc_core::internal::SetXdsChannelArgsForTest(&xds_channel_args_);
1363 // Make sure each test creates a new XdsClient instance rather than
1364 // reusing the one from the previous test. This avoids spurious failures
1365 // caused when a load reporting test runs after a non-load reporting test
1366 // and the XdsClient is still talking to the old LRS server, which fails
1367 // because it's not expecting the client to connect. It also
1368 // ensures that each test can independently set the global channel
1369 // args for the xDS channel.
1370 grpc_core::internal::UnsetGlobalXdsClientForTest();
1371 // Initialize default xDS resources.
1372 // Construct LDS resource.
1373 default_listener_.set_name(kServerName);
1374 // Construct RDS resource.
1375 default_route_config_.set_name(kDefaultRouteConfigurationName);
1376 auto* virtual_host = default_route_config_.add_virtual_hosts();
1377 virtual_host->add_domains("*");
1378 auto* route = virtual_host->add_routes();
1379 route->mutable_match()->set_prefix("");
1380 route->mutable_route()->set_cluster(kDefaultClusterName);
1381 // Construct CDS resource.
1382 default_cluster_.set_name(kDefaultClusterName);
1383 default_cluster_.set_type(Cluster::EDS);
1384 auto* eds_config = default_cluster_.mutable_eds_cluster_config();
1385 eds_config->mutable_eds_config()->mutable_ads();
1386 eds_config->set_service_name(kDefaultEdsServiceName);
1387 default_cluster_.set_lb_policy(Cluster::ROUND_ROBIN);
1388 if (GetParam().enable_load_reporting()) {
1389 default_cluster_.mutable_lrs_server()->mutable_self();
1390 }
1391 // Start the backends.
1392 for (size_t i = 0; i < num_backends_; ++i) {
1393 backends_.emplace_back(new BackendServerThread);
1394 backends_.back()->Start();
1395 }
1396 // Start the load balancers.
1397 for (size_t i = 0; i < num_balancers_; ++i) {
1398 balancers_.emplace_back(
1399 new BalancerServerThread(GetParam().enable_load_reporting()
1400 ? client_load_reporting_interval_seconds_
1401 : 0));
1402 balancers_.back()->Start();
1403 // Initialize resources.
1404 SetListenerAndRouteConfiguration(i, default_listener_,
1405 default_route_config_);
1406 balancers_.back()->ads_service()->SetCdsResource(default_cluster_);
1407 }
1408 ResetStub();
1409 }
1410
DefaultEdsServiceName() const1411 const char* DefaultEdsServiceName() const {
1412 return GetParam().use_xds_resolver() ? kDefaultEdsServiceName : kServerName;
1413 }
1414
TearDown()1415 void TearDown() override {
1416 ShutdownAllBackends();
1417 for (auto& balancer : balancers_) balancer->Shutdown();
1418 // Clear global xDS channel args, since they will go out of scope
1419 // when this test object is destroyed.
1420 grpc_core::internal::SetXdsChannelArgsForTest(nullptr);
1421 }
1422
StartAllBackends()1423 void StartAllBackends() {
1424 for (auto& backend : backends_) backend->Start();
1425 }
1426
StartBackend(size_t index)1427 void StartBackend(size_t index) { backends_[index]->Start(); }
1428
ShutdownAllBackends()1429 void ShutdownAllBackends() {
1430 for (auto& backend : backends_) backend->Shutdown();
1431 }
1432
ShutdownBackend(size_t index)1433 void ShutdownBackend(size_t index) { backends_[index]->Shutdown(); }
1434
ResetStub(int failover_timeout=0)1435 void ResetStub(int failover_timeout = 0) {
1436 channel_ = CreateChannel(failover_timeout);
1437 stub_ = grpc::testing::EchoTestService::NewStub(channel_);
1438 stub1_ = grpc::testing::EchoTest1Service::NewStub(channel_);
1439 stub2_ = grpc::testing::EchoTest2Service::NewStub(channel_);
1440 }
1441
CreateChannel(int failover_timeout=0,const char * server_name=kServerName,grpc_core::FakeResolverResponseGenerator * response_generator=nullptr)1442 std::shared_ptr<Channel> CreateChannel(
1443 int failover_timeout = 0, const char* server_name = kServerName,
1444 grpc_core::FakeResolverResponseGenerator* response_generator = nullptr) {
1445 ChannelArguments args;
1446 if (failover_timeout > 0) {
1447 args.SetInt(GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS, failover_timeout);
1448 }
1449 // If the parent channel is using the fake resolver, we inject the
1450 // response generator here.
1451 if (!GetParam().use_xds_resolver()) {
1452 if (response_generator == nullptr) {
1453 response_generator = response_generator_.get();
1454 }
1455 args.SetPointer(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR,
1456 response_generator);
1457 }
1458 std::string uri = absl::StrCat(
1459 GetParam().use_xds_resolver() ? "xds" : "fake", ":///", server_name);
1460 // TODO(dgq): templatize tests to run everything using both secure and
1461 // insecure channel credentials.
1462 grpc_channel_credentials* channel_creds =
1463 grpc_fake_transport_security_credentials_create();
1464 grpc_call_credentials* call_creds = grpc_md_only_test_credentials_create(
1465 g_kCallCredsMdKey, g_kCallCredsMdValue, false);
1466 std::shared_ptr<ChannelCredentials> creds(
1467 new SecureChannelCredentials(grpc_composite_channel_credentials_create(
1468 channel_creds, call_creds, nullptr)));
1469 call_creds->Unref();
1470 channel_creds->Unref();
1471 return ::grpc::CreateCustomChannel(uri, creds, args);
1472 }
1473
1474 enum RpcService {
1475 SERVICE_ECHO,
1476 SERVICE_ECHO1,
1477 SERVICE_ECHO2,
1478 };
1479
1480 enum RpcMethod {
1481 METHOD_ECHO,
1482 METHOD_ECHO1,
1483 METHOD_ECHO2,
1484 };
1485
1486 struct RpcOptions {
1487 RpcService service = SERVICE_ECHO;
1488 RpcMethod method = METHOD_ECHO;
1489 int timeout_ms = 1000;
1490 bool wait_for_ready = false;
1491 bool server_fail = false;
1492 std::vector<std::pair<std::string, std::string>> metadata;
1493
RpcOptionsgrpc::testing::__anon6f70f29a0111::XdsEnd2endTest::RpcOptions1494 RpcOptions() {}
1495
set_rpc_servicegrpc::testing::__anon6f70f29a0111::XdsEnd2endTest::RpcOptions1496 RpcOptions& set_rpc_service(RpcService rpc_service) {
1497 service = rpc_service;
1498 return *this;
1499 }
1500
set_rpc_methodgrpc::testing::__anon6f70f29a0111::XdsEnd2endTest::RpcOptions1501 RpcOptions& set_rpc_method(RpcMethod rpc_method) {
1502 method = rpc_method;
1503 return *this;
1504 }
1505
set_timeout_msgrpc::testing::__anon6f70f29a0111::XdsEnd2endTest::RpcOptions1506 RpcOptions& set_timeout_ms(int rpc_timeout_ms) {
1507 timeout_ms = rpc_timeout_ms;
1508 return *this;
1509 }
1510
set_wait_for_readygrpc::testing::__anon6f70f29a0111::XdsEnd2endTest::RpcOptions1511 RpcOptions& set_wait_for_ready(bool rpc_wait_for_ready) {
1512 wait_for_ready = rpc_wait_for_ready;
1513 return *this;
1514 }
1515
set_server_failgrpc::testing::__anon6f70f29a0111::XdsEnd2endTest::RpcOptions1516 RpcOptions& set_server_fail(bool rpc_server_fail) {
1517 server_fail = rpc_server_fail;
1518 return *this;
1519 }
1520
set_metadatagrpc::testing::__anon6f70f29a0111::XdsEnd2endTest::RpcOptions1521 RpcOptions& set_metadata(
1522 std::vector<std::pair<std::string, std::string>> rpc_metadata) {
1523 metadata = std::move(rpc_metadata);
1524 return *this;
1525 }
1526 };
1527
1528 template <typename Stub>
SendRpcMethod(Stub * stub,const RpcOptions & rpc_options,ClientContext * context,EchoRequest & request,EchoResponse * response)1529 Status SendRpcMethod(Stub* stub, const RpcOptions& rpc_options,
1530 ClientContext* context, EchoRequest& request,
1531 EchoResponse* response) {
1532 switch (rpc_options.method) {
1533 case METHOD_ECHO:
1534 return (*stub)->Echo(context, request, response);
1535 case METHOD_ECHO1:
1536 return (*stub)->Echo1(context, request, response);
1537 case METHOD_ECHO2:
1538 return (*stub)->Echo2(context, request, response);
1539 }
1540 }
1541
ResetBackendCounters(size_t start_index=0,size_t stop_index=0)1542 void ResetBackendCounters(size_t start_index = 0, size_t stop_index = 0) {
1543 if (stop_index == 0) stop_index = backends_.size();
1544 for (size_t i = start_index; i < stop_index; ++i) {
1545 backends_[i]->backend_service()->ResetCounters();
1546 backends_[i]->backend_service1()->ResetCounters();
1547 backends_[i]->backend_service2()->ResetCounters();
1548 }
1549 }
1550
SeenAllBackends(size_t start_index=0,size_t stop_index=0,const RpcOptions & rpc_options=RpcOptions ())1551 bool SeenAllBackends(size_t start_index = 0, size_t stop_index = 0,
1552 const RpcOptions& rpc_options = RpcOptions()) {
1553 if (stop_index == 0) stop_index = backends_.size();
1554 for (size_t i = start_index; i < stop_index; ++i) {
1555 switch (rpc_options.service) {
1556 case SERVICE_ECHO:
1557 if (backends_[i]->backend_service()->request_count() == 0) {
1558 return false;
1559 }
1560 break;
1561 case SERVICE_ECHO1:
1562 if (backends_[i]->backend_service1()->request_count() == 0) {
1563 return false;
1564 }
1565 break;
1566 case SERVICE_ECHO2:
1567 if (backends_[i]->backend_service2()->request_count() == 0) {
1568 return false;
1569 }
1570 break;
1571 }
1572 }
1573 return true;
1574 }
1575
SendRpcAndCount(int * num_total,int * num_ok,int * num_failure,int * num_drops,const RpcOptions & rpc_options=RpcOptions ())1576 void SendRpcAndCount(int* num_total, int* num_ok, int* num_failure,
1577 int* num_drops,
1578 const RpcOptions& rpc_options = RpcOptions()) {
1579 const Status status = SendRpc(rpc_options);
1580 if (status.ok()) {
1581 ++*num_ok;
1582 } else {
1583 if (status.error_message() == "Call dropped by load balancing policy") {
1584 ++*num_drops;
1585 } else {
1586 ++*num_failure;
1587 }
1588 }
1589 ++*num_total;
1590 }
1591
WaitForAllBackends(size_t start_index=0,size_t stop_index=0,bool reset_counters=true,const RpcOptions & rpc_options=RpcOptions (),bool allow_failures=false)1592 std::tuple<int, int, int> WaitForAllBackends(
1593 size_t start_index = 0, size_t stop_index = 0, bool reset_counters = true,
1594 const RpcOptions& rpc_options = RpcOptions(),
1595 bool allow_failures = false) {
1596 int num_ok = 0;
1597 int num_failure = 0;
1598 int num_drops = 0;
1599 int num_total = 0;
1600 while (!SeenAllBackends(start_index, stop_index, rpc_options)) {
1601 SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_drops,
1602 rpc_options);
1603 }
1604 if (reset_counters) ResetBackendCounters();
1605 gpr_log(GPR_INFO,
1606 "Performed %d warm up requests against the backends. "
1607 "%d succeeded, %d failed, %d dropped.",
1608 num_total, num_ok, num_failure, num_drops);
1609 if (!allow_failures) EXPECT_EQ(num_failure, 0);
1610 return std::make_tuple(num_ok, num_failure, num_drops);
1611 }
1612
WaitForBackend(size_t backend_idx,bool reset_counters=true,bool require_success=false)1613 void WaitForBackend(size_t backend_idx, bool reset_counters = true,
1614 bool require_success = false) {
1615 gpr_log(GPR_INFO, "========= WAITING FOR BACKEND %lu ==========",
1616 static_cast<unsigned long>(backend_idx));
1617 do {
1618 Status status = SendRpc();
1619 if (require_success) {
1620 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
1621 << " message=" << status.error_message();
1622 }
1623 } while (backends_[backend_idx]->backend_service()->request_count() == 0);
1624 if (reset_counters) ResetBackendCounters();
1625 gpr_log(GPR_INFO, "========= BACKEND %lu READY ==========",
1626 static_cast<unsigned long>(backend_idx));
1627 }
1628
CreateAddressListFromPortList(const std::vector<int> & ports)1629 grpc_core::ServerAddressList CreateAddressListFromPortList(
1630 const std::vector<int>& ports) {
1631 grpc_core::ServerAddressList addresses;
1632 for (int port : ports) {
1633 std::string lb_uri_str =
1634 absl::StrCat(ipv6_only_ ? "ipv6:[::1]:" : "ipv4:127.0.0.1:", port);
1635 grpc_uri* lb_uri = grpc_uri_parse(lb_uri_str.c_str(), true);
1636 GPR_ASSERT(lb_uri != nullptr);
1637 grpc_resolved_address address;
1638 GPR_ASSERT(grpc_parse_uri(lb_uri, &address));
1639 addresses.emplace_back(address.addr, address.len, nullptr);
1640 grpc_uri_destroy(lb_uri);
1641 }
1642 return addresses;
1643 }
1644
SetNextResolution(const std::vector<int> & ports,grpc_core::FakeResolverResponseGenerator * response_generator=nullptr)1645 void SetNextResolution(
1646 const std::vector<int>& ports,
1647 grpc_core::FakeResolverResponseGenerator* response_generator = nullptr) {
1648 if (GetParam().use_xds_resolver()) return; // Not used with xds resolver.
1649 grpc_core::ExecCtx exec_ctx;
1650 grpc_core::Resolver::Result result;
1651 result.addresses = CreateAddressListFromPortList(ports);
1652 grpc_error* error = GRPC_ERROR_NONE;
1653 const char* service_config_json =
1654 GetParam().enable_load_reporting()
1655 ? kDefaultServiceConfig
1656 : kDefaultServiceConfigWithoutLoadReporting;
1657 result.service_config =
1658 grpc_core::ServiceConfig::Create(nullptr, service_config_json, &error);
1659 ASSERT_EQ(error, GRPC_ERROR_NONE) << grpc_error_string(error);
1660 ASSERT_NE(result.service_config.get(), nullptr);
1661 if (response_generator == nullptr) {
1662 response_generator = response_generator_.get();
1663 }
1664 response_generator->SetResponse(std::move(result));
1665 }
1666
SetNextResolutionForLbChannelAllBalancers(const char * service_config_json=nullptr,const char * expected_targets=nullptr)1667 void SetNextResolutionForLbChannelAllBalancers(
1668 const char* service_config_json = nullptr,
1669 const char* expected_targets = nullptr) {
1670 std::vector<int> ports;
1671 for (size_t i = 0; i < balancers_.size(); ++i) {
1672 ports.emplace_back(balancers_[i]->port());
1673 }
1674 SetNextResolutionForLbChannel(ports, service_config_json, expected_targets);
1675 }
1676
SetNextResolutionForLbChannel(const std::vector<int> & ports,const char * service_config_json=nullptr,const char * expected_targets=nullptr)1677 void SetNextResolutionForLbChannel(const std::vector<int>& ports,
1678 const char* service_config_json = nullptr,
1679 const char* expected_targets = nullptr) {
1680 grpc_core::ExecCtx exec_ctx;
1681 grpc_core::Resolver::Result result;
1682 result.addresses = CreateAddressListFromPortList(ports);
1683 if (service_config_json != nullptr) {
1684 grpc_error* error = GRPC_ERROR_NONE;
1685 result.service_config = grpc_core::ServiceConfig::Create(
1686 nullptr, service_config_json, &error);
1687 ASSERT_NE(result.service_config.get(), nullptr);
1688 ASSERT_EQ(error, GRPC_ERROR_NONE) << grpc_error_string(error);
1689 }
1690 if (expected_targets != nullptr) {
1691 grpc_arg expected_targets_arg = grpc_channel_arg_string_create(
1692 const_cast<char*>(GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS),
1693 const_cast<char*>(expected_targets));
1694 result.args =
1695 grpc_channel_args_copy_and_add(nullptr, &expected_targets_arg, 1);
1696 }
1697 lb_channel_response_generator_->SetResponse(std::move(result));
1698 }
1699
SetNextReresolutionResponse(const std::vector<int> & ports)1700 void SetNextReresolutionResponse(const std::vector<int>& ports) {
1701 grpc_core::ExecCtx exec_ctx;
1702 grpc_core::Resolver::Result result;
1703 result.addresses = CreateAddressListFromPortList(ports);
1704 response_generator_->SetReresolutionResponse(std::move(result));
1705 }
1706
GetBackendPorts(size_t start_index=0,size_t stop_index=0) const1707 const std::vector<int> GetBackendPorts(size_t start_index = 0,
1708 size_t stop_index = 0) const {
1709 if (stop_index == 0) stop_index = backends_.size();
1710 std::vector<int> backend_ports;
1711 for (size_t i = start_index; i < stop_index; ++i) {
1712 backend_ports.push_back(backends_[i]->port());
1713 }
1714 return backend_ports;
1715 }
1716
SendRpc(const RpcOptions & rpc_options=RpcOptions (),EchoResponse * response=nullptr)1717 Status SendRpc(const RpcOptions& rpc_options = RpcOptions(),
1718 EchoResponse* response = nullptr) {
1719 const bool local_response = (response == nullptr);
1720 if (local_response) response = new EchoResponse;
1721 EchoRequest request;
1722 ClientContext context;
1723 for (const auto& metadata : rpc_options.metadata) {
1724 context.AddMetadata(metadata.first, metadata.second);
1725 }
1726 if (rpc_options.timeout_ms != 0) {
1727 context.set_deadline(
1728 grpc_timeout_milliseconds_to_deadline(rpc_options.timeout_ms));
1729 }
1730 if (rpc_options.wait_for_ready) context.set_wait_for_ready(true);
1731 request.set_message(kRequestMessage);
1732 if (rpc_options.server_fail) {
1733 request.mutable_param()->mutable_expected_error()->set_code(
1734 GRPC_STATUS_FAILED_PRECONDITION);
1735 }
1736 Status status;
1737 switch (rpc_options.service) {
1738 case SERVICE_ECHO:
1739 status =
1740 SendRpcMethod(&stub_, rpc_options, &context, request, response);
1741 break;
1742 case SERVICE_ECHO1:
1743 status =
1744 SendRpcMethod(&stub1_, rpc_options, &context, request, response);
1745 break;
1746 case SERVICE_ECHO2:
1747 status =
1748 SendRpcMethod(&stub2_, rpc_options, &context, request, response);
1749 break;
1750 }
1751 if (local_response) delete response;
1752 return status;
1753 }
1754
CheckRpcSendOk(const size_t times=1,const RpcOptions & rpc_options=RpcOptions ())1755 void CheckRpcSendOk(const size_t times = 1,
1756 const RpcOptions& rpc_options = RpcOptions()) {
1757 for (size_t i = 0; i < times; ++i) {
1758 EchoResponse response;
1759 const Status status = SendRpc(rpc_options, &response);
1760 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
1761 << " message=" << status.error_message();
1762 EXPECT_EQ(response.message(), kRequestMessage);
1763 }
1764 }
1765
CheckRpcSendFailure(const size_t times=1,const RpcOptions & rpc_options=RpcOptions (),const StatusCode expected_error_code=StatusCode::OK)1766 void CheckRpcSendFailure(
1767 const size_t times = 1, const RpcOptions& rpc_options = RpcOptions(),
1768 const StatusCode expected_error_code = StatusCode::OK) {
1769 for (size_t i = 0; i < times; ++i) {
1770 const Status status = SendRpc(rpc_options);
1771 EXPECT_FALSE(status.ok());
1772 if (expected_error_code != StatusCode::OK) {
1773 EXPECT_EQ(expected_error_code, status.error_code());
1774 }
1775 }
1776 }
1777
BuildListener(const RouteConfiguration & route_config)1778 static Listener BuildListener(const RouteConfiguration& route_config) {
1779 HttpConnectionManager http_connection_manager;
1780 *(http_connection_manager.mutable_route_config()) = route_config;
1781 Listener listener;
1782 listener.set_name(kServerName);
1783 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
1784 http_connection_manager);
1785 return listener;
1786 }
1787
BuildEdsResource(const AdsServiceImpl::EdsResourceArgs & args,const char * eds_service_name=kDefaultEdsServiceName)1788 ClusterLoadAssignment BuildEdsResource(
1789 const AdsServiceImpl::EdsResourceArgs& args,
1790 const char* eds_service_name = kDefaultEdsServiceName) {
1791 ClusterLoadAssignment assignment;
1792 assignment.set_cluster_name(eds_service_name);
1793 for (const auto& locality : args.locality_list) {
1794 auto* endpoints = assignment.add_endpoints();
1795 endpoints->mutable_load_balancing_weight()->set_value(locality.lb_weight);
1796 endpoints->set_priority(locality.priority);
1797 endpoints->mutable_locality()->set_region(kDefaultLocalityRegion);
1798 endpoints->mutable_locality()->set_zone(kDefaultLocalityZone);
1799 endpoints->mutable_locality()->set_sub_zone(locality.sub_zone);
1800 for (size_t i = 0; i < locality.ports.size(); ++i) {
1801 const int& port = locality.ports[i];
1802 auto* lb_endpoints = endpoints->add_lb_endpoints();
1803 if (locality.health_statuses.size() > i &&
1804 locality.health_statuses[i] != HealthStatus::UNKNOWN) {
1805 lb_endpoints->set_health_status(locality.health_statuses[i]);
1806 }
1807 auto* endpoint = lb_endpoints->mutable_endpoint();
1808 auto* address = endpoint->mutable_address();
1809 auto* socket_address = address->mutable_socket_address();
1810 socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
1811 socket_address->set_port_value(port);
1812 }
1813 }
1814 if (!args.drop_categories.empty()) {
1815 auto* policy = assignment.mutable_policy();
1816 for (const auto& p : args.drop_categories) {
1817 const std::string& name = p.first;
1818 const uint32_t parts_per_million = p.second;
1819 auto* drop_overload = policy->add_drop_overloads();
1820 drop_overload->set_category(name);
1821 auto* drop_percentage = drop_overload->mutable_drop_percentage();
1822 drop_percentage->set_numerator(parts_per_million);
1823 drop_percentage->set_denominator(args.drop_denominator);
1824 }
1825 }
1826 return assignment;
1827 }
1828
SetListenerAndRouteConfiguration(int idx,Listener listener,const RouteConfiguration & route_config)1829 void SetListenerAndRouteConfiguration(
1830 int idx, Listener listener, const RouteConfiguration& route_config) {
1831 auto* api_listener =
1832 listener.mutable_api_listener()->mutable_api_listener();
1833 HttpConnectionManager http_connection_manager;
1834 api_listener->UnpackTo(&http_connection_manager);
1835 if (GetParam().enable_rds_testing()) {
1836 auto* rds = http_connection_manager.mutable_rds();
1837 rds->set_route_config_name(kDefaultRouteConfigurationName);
1838 rds->mutable_config_source()->mutable_ads();
1839 balancers_[idx]->ads_service()->SetRdsResource(route_config);
1840 } else {
1841 *http_connection_manager.mutable_route_config() = route_config;
1842 }
1843 api_listener->PackFrom(http_connection_manager);
1844 balancers_[idx]->ads_service()->SetLdsResource(listener);
1845 }
1846
SetRouteConfiguration(int idx,const RouteConfiguration & route_config)1847 void SetRouteConfiguration(int idx, const RouteConfiguration& route_config) {
1848 if (GetParam().enable_rds_testing()) {
1849 balancers_[idx]->ads_service()->SetRdsResource(route_config);
1850 } else {
1851 balancers_[idx]->ads_service()->SetLdsResource(
1852 BuildListener(route_config));
1853 }
1854 }
1855
RouteConfigurationResponseState(int idx) const1856 AdsServiceImpl::ResponseState RouteConfigurationResponseState(int idx) const {
1857 AdsServiceImpl* ads_service = balancers_[idx]->ads_service();
1858 if (GetParam().enable_rds_testing()) {
1859 return ads_service->rds_response_state();
1860 }
1861 return ads_service->lds_response_state();
1862 }
1863
1864 public:
1865 // This method could benefit test subclasses; to make it accessible
1866 // via bind with a qualified name, it needs to be public.
SetEdsResourceWithDelay(size_t i,const ClusterLoadAssignment & assignment,int delay_ms)1867 void SetEdsResourceWithDelay(size_t i,
1868 const ClusterLoadAssignment& assignment,
1869 int delay_ms) {
1870 GPR_ASSERT(delay_ms > 0);
1871 gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(delay_ms));
1872 balancers_[i]->ads_service()->SetEdsResource(assignment);
1873 }
1874
1875 protected:
1876 class ServerThread {
1877 public:
ServerThread()1878 ServerThread() : port_(g_port_saver->GetPort()) {}
~ServerThread()1879 virtual ~ServerThread(){};
1880
Start()1881 void Start() {
1882 gpr_log(GPR_INFO, "starting %s server on port %d", Type(), port_);
1883 GPR_ASSERT(!running_);
1884 running_ = true;
1885 StartAllServices();
1886 grpc_core::Mutex mu;
1887 // We need to acquire the lock here in order to prevent the notify_one
1888 // by ServerThread::Serve from firing before the wait below is hit.
1889 grpc_core::MutexLock lock(&mu);
1890 grpc_core::CondVar cond;
1891 thread_ = absl::make_unique<std::thread>(
1892 std::bind(&ServerThread::Serve, this, &mu, &cond));
1893 cond.Wait(&mu);
1894 gpr_log(GPR_INFO, "%s server startup complete", Type());
1895 }
1896
Serve(grpc_core::Mutex * mu,grpc_core::CondVar * cond)1897 void Serve(grpc_core::Mutex* mu, grpc_core::CondVar* cond) {
1898 // We need to acquire the lock here in order to prevent the notify_one
1899 // below from firing before its corresponding wait is executed.
1900 grpc_core::MutexLock lock(mu);
1901 std::ostringstream server_address;
1902 server_address << "localhost:" << port_;
1903 ServerBuilder builder;
1904 std::shared_ptr<ServerCredentials> creds(new SecureServerCredentials(
1905 grpc_fake_transport_security_server_credentials_create()));
1906 builder.AddListeningPort(server_address.str(), creds);
1907 RegisterAllServices(&builder);
1908 server_ = builder.BuildAndStart();
1909 cond->Signal();
1910 }
1911
Shutdown()1912 void Shutdown() {
1913 if (!running_) return;
1914 gpr_log(GPR_INFO, "%s about to shutdown", Type());
1915 ShutdownAllServices();
1916 server_->Shutdown(grpc_timeout_milliseconds_to_deadline(0));
1917 thread_->join();
1918 gpr_log(GPR_INFO, "%s shutdown completed", Type());
1919 running_ = false;
1920 }
1921
port() const1922 int port() const { return port_; }
1923
1924 private:
1925 virtual void RegisterAllServices(ServerBuilder* builder) = 0;
1926 virtual void StartAllServices() = 0;
1927 virtual void ShutdownAllServices() = 0;
1928
1929 virtual const char* Type() = 0;
1930
1931 const int port_;
1932 std::unique_ptr<Server> server_;
1933 std::unique_ptr<std::thread> thread_;
1934 bool running_ = false;
1935 };
1936
1937 class BackendServerThread : public ServerThread {
1938 public:
1939 BackendServiceImpl<::grpc::testing::EchoTestService::Service>*
backend_service()1940 backend_service() {
1941 return &backend_service_;
1942 }
1943 BackendServiceImpl<::grpc::testing::EchoTest1Service::Service>*
backend_service1()1944 backend_service1() {
1945 return &backend_service1_;
1946 }
1947 BackendServiceImpl<::grpc::testing::EchoTest2Service::Service>*
backend_service2()1948 backend_service2() {
1949 return &backend_service2_;
1950 }
1951
1952 private:
RegisterAllServices(ServerBuilder * builder)1953 void RegisterAllServices(ServerBuilder* builder) override {
1954 builder->RegisterService(&backend_service_);
1955 builder->RegisterService(&backend_service1_);
1956 builder->RegisterService(&backend_service2_);
1957 }
1958
StartAllServices()1959 void StartAllServices() override {
1960 backend_service_.Start();
1961 backend_service1_.Start();
1962 backend_service2_.Start();
1963 }
1964
ShutdownAllServices()1965 void ShutdownAllServices() override {
1966 backend_service_.Shutdown();
1967 backend_service1_.Shutdown();
1968 backend_service2_.Shutdown();
1969 }
1970
Type()1971 const char* Type() override { return "Backend"; }
1972
1973 BackendServiceImpl<::grpc::testing::EchoTestService::Service>
1974 backend_service_;
1975 BackendServiceImpl<::grpc::testing::EchoTest1Service::Service>
1976 backend_service1_;
1977 BackendServiceImpl<::grpc::testing::EchoTest2Service::Service>
1978 backend_service2_;
1979 };
1980
1981 class BalancerServerThread : public ServerThread {
1982 public:
BalancerServerThread(int client_load_reporting_interval=0)1983 explicit BalancerServerThread(int client_load_reporting_interval = 0)
1984 : ads_service_(new AdsServiceImpl()),
1985 lrs_service_(new LrsServiceImpl(client_load_reporting_interval)) {}
1986
ads_service()1987 AdsServiceImpl* ads_service() { return ads_service_.get(); }
lrs_service()1988 LrsServiceImpl* lrs_service() { return lrs_service_.get(); }
1989
1990 private:
RegisterAllServices(ServerBuilder * builder)1991 void RegisterAllServices(ServerBuilder* builder) override {
1992 builder->RegisterService(ads_service_->v2_rpc_service());
1993 builder->RegisterService(ads_service_->v3_rpc_service());
1994 builder->RegisterService(lrs_service_->v2_rpc_service());
1995 builder->RegisterService(lrs_service_->v3_rpc_service());
1996 }
1997
StartAllServices()1998 void StartAllServices() override {
1999 ads_service_->Start();
2000 lrs_service_->Start();
2001 }
2002
ShutdownAllServices()2003 void ShutdownAllServices() override {
2004 ads_service_->Shutdown();
2005 lrs_service_->Shutdown();
2006 }
2007
Type()2008 const char* Type() override { return "Balancer"; }
2009
2010 std::shared_ptr<AdsServiceImpl> ads_service_;
2011 std::shared_ptr<LrsServiceImpl> lrs_service_;
2012 };
2013
2014 class LongRunningRpc {
2015 public:
StartRpc(grpc::testing::EchoTestService::Stub * stub)2016 void StartRpc(grpc::testing::EchoTestService::Stub* stub) {
2017 sender_thread_ = std::thread([this, stub]() {
2018 EchoResponse response;
2019 EchoRequest request;
2020 request.mutable_param()->set_client_cancel_after_us(1 * 1000 * 1000);
2021 request.set_message(kRequestMessage);
2022 (void)stub->Echo(&context_, request, &response);
2023 });
2024 }
2025
CancelRpc()2026 void CancelRpc() {
2027 context_.TryCancel();
2028 sender_thread_.join();
2029 }
2030
2031 private:
2032 std::thread sender_thread_;
2033 ClientContext context_;
2034 };
2035
2036 const size_t num_backends_;
2037 const size_t num_balancers_;
2038 const int client_load_reporting_interval_seconds_;
2039 bool ipv6_only_ = false;
2040 std::shared_ptr<Channel> channel_;
2041 std::unique_ptr<grpc::testing::EchoTestService::Stub> stub_;
2042 std::unique_ptr<grpc::testing::EchoTest1Service::Stub> stub1_;
2043 std::unique_ptr<grpc::testing::EchoTest2Service::Stub> stub2_;
2044 std::vector<std::unique_ptr<BackendServerThread>> backends_;
2045 std::vector<std::unique_ptr<BalancerServerThread>> balancers_;
2046 grpc_core::RefCountedPtr<grpc_core::FakeResolverResponseGenerator>
2047 response_generator_;
2048 grpc_core::RefCountedPtr<grpc_core::FakeResolverResponseGenerator>
2049 lb_channel_response_generator_;
2050 int xds_resource_does_not_exist_timeout_ms_ = 0;
2051 absl::InlinedVector<grpc_arg, 2> xds_channel_args_to_add_;
2052 grpc_channel_args xds_channel_args_;
2053
2054 Listener default_listener_;
2055 RouteConfiguration default_route_config_;
2056 Cluster default_cluster_;
2057 };
2058
2059 class BasicTest : public XdsEnd2endTest {
2060 public:
BasicTest()2061 BasicTest() : XdsEnd2endTest(4, 1) {}
2062 };
2063
2064 // Tests that the balancer sends the correct response to the client, and the
2065 // client sends RPCs to the backends using the default child policy.
TEST_P(BasicTest,Vanilla)2066 TEST_P(BasicTest, Vanilla) {
2067 SetNextResolution({});
2068 SetNextResolutionForLbChannelAllBalancers();
2069 const size_t kNumRpcsPerAddress = 100;
2070 AdsServiceImpl::EdsResourceArgs args({
2071 {"locality0", GetBackendPorts()},
2072 });
2073 balancers_[0]->ads_service()->SetEdsResource(
2074 BuildEdsResource(args, DefaultEdsServiceName()));
2075 // Make sure that trying to connect works without a call.
2076 channel_->GetState(true /* try_to_connect */);
2077 // We need to wait for all backends to come online.
2078 WaitForAllBackends();
2079 // Send kNumRpcsPerAddress RPCs per server.
2080 CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
2081 // Each backend should have gotten 100 requests.
2082 for (size_t i = 0; i < backends_.size(); ++i) {
2083 EXPECT_EQ(kNumRpcsPerAddress,
2084 backends_[i]->backend_service()->request_count());
2085 }
2086 // Check LB policy name for the channel.
2087 EXPECT_EQ((GetParam().use_xds_resolver() ? "xds_cluster_manager_experimental"
2088 : "eds_experimental"),
2089 channel_->GetLoadBalancingPolicyName());
2090 }
2091
TEST_P(BasicTest,IgnoresUnhealthyEndpoints)2092 TEST_P(BasicTest, IgnoresUnhealthyEndpoints) {
2093 SetNextResolution({});
2094 SetNextResolutionForLbChannelAllBalancers();
2095 const size_t kNumRpcsPerAddress = 100;
2096 AdsServiceImpl::EdsResourceArgs args({
2097 {"locality0",
2098 GetBackendPorts(),
2099 kDefaultLocalityWeight,
2100 kDefaultLocalityPriority,
2101 {HealthStatus::DRAINING}},
2102 });
2103 balancers_[0]->ads_service()->SetEdsResource(
2104 BuildEdsResource(args, DefaultEdsServiceName()));
2105 // Make sure that trying to connect works without a call.
2106 channel_->GetState(true /* try_to_connect */);
2107 // We need to wait for all backends to come online.
2108 WaitForAllBackends(/*start_index=*/1);
2109 // Send kNumRpcsPerAddress RPCs per server.
2110 CheckRpcSendOk(kNumRpcsPerAddress * (num_backends_ - 1));
2111 // Each backend should have gotten 100 requests.
2112 for (size_t i = 1; i < backends_.size(); ++i) {
2113 EXPECT_EQ(kNumRpcsPerAddress,
2114 backends_[i]->backend_service()->request_count());
2115 }
2116 }
2117
2118 // Tests that subchannel sharing works when the same backend is listed multiple
2119 // times.
TEST_P(BasicTest,SameBackendListedMultipleTimes)2120 TEST_P(BasicTest, SameBackendListedMultipleTimes) {
2121 SetNextResolution({});
2122 SetNextResolutionForLbChannelAllBalancers();
2123 // Same backend listed twice.
2124 std::vector<int> ports(2, backends_[0]->port());
2125 AdsServiceImpl::EdsResourceArgs args({
2126 {"locality0", ports},
2127 });
2128 const size_t kNumRpcsPerAddress = 10;
2129 balancers_[0]->ads_service()->SetEdsResource(
2130 BuildEdsResource(args, DefaultEdsServiceName()));
2131 // We need to wait for the backend to come online.
2132 WaitForBackend(0);
2133 // Send kNumRpcsPerAddress RPCs per server.
2134 CheckRpcSendOk(kNumRpcsPerAddress * ports.size());
2135 // Backend should have gotten 20 requests.
2136 EXPECT_EQ(kNumRpcsPerAddress * ports.size(),
2137 backends_[0]->backend_service()->request_count());
2138 // And they should have come from a single client port, because of
2139 // subchannel sharing.
2140 EXPECT_EQ(1UL, backends_[0]->backend_service()->clients().size());
2141 }
2142
2143 // Tests that RPCs will be blocked until a non-empty serverlist is received.
TEST_P(BasicTest,InitiallyEmptyServerlist)2144 TEST_P(BasicTest, InitiallyEmptyServerlist) {
2145 SetNextResolution({});
2146 SetNextResolutionForLbChannelAllBalancers();
2147 const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor();
2148 const int kCallDeadlineMs = kServerlistDelayMs * 2;
2149 // First response is an empty serverlist, sent right away.
2150 AdsServiceImpl::EdsResourceArgs::Locality empty_locality("locality0", {});
2151 AdsServiceImpl::EdsResourceArgs args({
2152 empty_locality,
2153 });
2154 balancers_[0]->ads_service()->SetEdsResource(
2155 BuildEdsResource(args, DefaultEdsServiceName()));
2156 // Send non-empty serverlist only after kServerlistDelayMs.
2157 args = AdsServiceImpl::EdsResourceArgs({
2158 {"locality0", GetBackendPorts()},
2159 });
2160 std::thread delayed_resource_setter(std::bind(
2161 &BasicTest::SetEdsResourceWithDelay, this, 0,
2162 BuildEdsResource(args, DefaultEdsServiceName()), kServerlistDelayMs));
2163 const auto t0 = system_clock::now();
2164 // Client will block: LB will initially send empty serverlist.
2165 CheckRpcSendOk(
2166 1, RpcOptions().set_timeout_ms(kCallDeadlineMs).set_wait_for_ready(true));
2167 const auto ellapsed_ms =
2168 std::chrono::duration_cast<std::chrono::milliseconds>(
2169 system_clock::now() - t0);
2170 // but eventually, the LB sends a serverlist update that allows the call to
2171 // proceed. The call delay must be larger than the delay in sending the
2172 // populated serverlist but under the call's deadline (which is enforced by
2173 // the call's deadline).
2174 EXPECT_GT(ellapsed_ms.count(), kServerlistDelayMs);
2175 delayed_resource_setter.join();
2176 }
2177
2178 // Tests that RPCs will fail with UNAVAILABLE instead of DEADLINE_EXCEEDED if
2179 // all the servers are unreachable.
TEST_P(BasicTest,AllServersUnreachableFailFast)2180 TEST_P(BasicTest, AllServersUnreachableFailFast) {
2181 SetNextResolution({});
2182 SetNextResolutionForLbChannelAllBalancers();
2183 const size_t kNumUnreachableServers = 5;
2184 std::vector<int> ports;
2185 for (size_t i = 0; i < kNumUnreachableServers; ++i) {
2186 ports.push_back(g_port_saver->GetPort());
2187 }
2188 AdsServiceImpl::EdsResourceArgs args({
2189 {"locality0", ports},
2190 });
2191 balancers_[0]->ads_service()->SetEdsResource(
2192 BuildEdsResource(args, DefaultEdsServiceName()));
2193 const Status status = SendRpc();
2194 // The error shouldn't be DEADLINE_EXCEEDED.
2195 EXPECT_EQ(StatusCode::UNAVAILABLE, status.error_code());
2196 }
2197
2198 // Tests that RPCs fail when the backends are down, and will succeed again after
2199 // the backends are restarted.
TEST_P(BasicTest,BackendsRestart)2200 TEST_P(BasicTest, BackendsRestart) {
2201 SetNextResolution({});
2202 SetNextResolutionForLbChannelAllBalancers();
2203 AdsServiceImpl::EdsResourceArgs args({
2204 {"locality0", GetBackendPorts()},
2205 });
2206 balancers_[0]->ads_service()->SetEdsResource(
2207 BuildEdsResource(args, DefaultEdsServiceName()));
2208 WaitForAllBackends();
2209 // Stop backends. RPCs should fail.
2210 ShutdownAllBackends();
2211 // Sending multiple failed requests instead of just one to ensure that the
2212 // client notices that all backends are down before we restart them. If we
2213 // didn't do this, then a single RPC could fail here due to the race condition
2214 // between the LB pick and the GOAWAY from the chosen backend being shut down,
2215 // which would not actually prove that the client noticed that all of the
2216 // backends are down. Then, when we send another request below (which we
2217 // expect to succeed), if the callbacks happen in the wrong order, the same
2218 // race condition could happen again due to the client not yet having noticed
2219 // that the backends were all down.
2220 CheckRpcSendFailure(num_backends_);
2221 // Restart all backends. RPCs should start succeeding again.
2222 StartAllBackends();
2223 CheckRpcSendOk(1, RpcOptions().set_timeout_ms(2000).set_wait_for_ready(true));
2224 }
2225
TEST_P(BasicTest,IgnoresDuplicateUpdates)2226 TEST_P(BasicTest, IgnoresDuplicateUpdates) {
2227 const size_t kNumRpcsPerAddress = 100;
2228 SetNextResolution({});
2229 SetNextResolutionForLbChannelAllBalancers();
2230 AdsServiceImpl::EdsResourceArgs args({
2231 {"locality0", GetBackendPorts()},
2232 });
2233 balancers_[0]->ads_service()->SetEdsResource(
2234 BuildEdsResource(args, DefaultEdsServiceName()));
2235 // Wait for all backends to come online.
2236 WaitForAllBackends();
2237 // Send kNumRpcsPerAddress RPCs per server, but send an EDS update in
2238 // between. If the update is not ignored, this will cause the
2239 // round_robin policy to see an update, which will randomly reset its
2240 // position in the address list.
2241 for (size_t i = 0; i < kNumRpcsPerAddress; ++i) {
2242 CheckRpcSendOk(2);
2243 balancers_[0]->ads_service()->SetEdsResource(
2244 BuildEdsResource(args, DefaultEdsServiceName()));
2245 CheckRpcSendOk(2);
2246 }
2247 // Each backend should have gotten the right number of requests.
2248 for (size_t i = 1; i < backends_.size(); ++i) {
2249 EXPECT_EQ(kNumRpcsPerAddress,
2250 backends_[i]->backend_service()->request_count());
2251 }
2252 }
2253
2254 using XdsResolverOnlyTest = BasicTest;
2255
TEST_P(XdsResolverOnlyTest,ResourceTypeVersionPersistsAcrossStreamRestarts)2256 TEST_P(XdsResolverOnlyTest, ResourceTypeVersionPersistsAcrossStreamRestarts) {
2257 SetNextResolution({});
2258 SetNextResolutionForLbChannelAllBalancers();
2259 AdsServiceImpl::EdsResourceArgs args({
2260 {"locality0", GetBackendPorts(0, 1)},
2261 });
2262 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2263 // Wait for backends to come online.
2264 WaitForAllBackends(0, 1);
2265 // Stop balancer.
2266 balancers_[0]->Shutdown();
2267 // Tell balancer to require minimum version 1 for all resource types.
2268 balancers_[0]->ads_service()->SetResourceMinVersion(kLdsTypeUrl, 1);
2269 balancers_[0]->ads_service()->SetResourceMinVersion(kRdsTypeUrl, 1);
2270 balancers_[0]->ads_service()->SetResourceMinVersion(kCdsTypeUrl, 1);
2271 balancers_[0]->ads_service()->SetResourceMinVersion(kEdsTypeUrl, 1);
2272 // Update backend, just so we can be sure that the client has
2273 // reconnected to the balancer.
2274 AdsServiceImpl::EdsResourceArgs args2({
2275 {"locality0", GetBackendPorts(1, 2)},
2276 });
2277 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args2));
2278 // Restart balancer.
2279 balancers_[0]->Start();
2280 // Make sure client has reconnected.
2281 WaitForAllBackends(1, 2);
2282 }
2283
2284 // Tests switching over from one cluster to another.
TEST_P(XdsResolverOnlyTest,ChangeClusters)2285 TEST_P(XdsResolverOnlyTest, ChangeClusters) {
2286 const char* kNewClusterName = "new_cluster_name";
2287 const char* kNewEdsServiceName = "new_eds_service_name";
2288 SetNextResolution({});
2289 SetNextResolutionForLbChannelAllBalancers();
2290 AdsServiceImpl::EdsResourceArgs args({
2291 {"locality0", GetBackendPorts(0, 2)},
2292 });
2293 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2294 // We need to wait for all backends to come online.
2295 WaitForAllBackends(0, 2);
2296 // Populate new EDS resource.
2297 AdsServiceImpl::EdsResourceArgs args2({
2298 {"locality0", GetBackendPorts(2, 4)},
2299 });
2300 balancers_[0]->ads_service()->SetEdsResource(
2301 BuildEdsResource(args2, kNewEdsServiceName));
2302 // Populate new CDS resource.
2303 Cluster new_cluster = default_cluster_;
2304 new_cluster.set_name(kNewClusterName);
2305 new_cluster.mutable_eds_cluster_config()->set_service_name(
2306 kNewEdsServiceName);
2307 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
2308 // Change RDS resource to point to new cluster.
2309 RouteConfiguration new_route_config = default_route_config_;
2310 new_route_config.mutable_virtual_hosts(0)
2311 ->mutable_routes(0)
2312 ->mutable_route()
2313 ->set_cluster(kNewClusterName);
2314 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
2315 // Wait for all new backends to be used.
2316 std::tuple<int, int, int> counts = WaitForAllBackends(2, 4);
2317 // Make sure no RPCs failed in the transition.
2318 EXPECT_EQ(0, std::get<1>(counts));
2319 }
2320
2321 // Tests that we go into TRANSIENT_FAILURE if the Cluster disappears.
TEST_P(XdsResolverOnlyTest,ClusterRemoved)2322 TEST_P(XdsResolverOnlyTest, ClusterRemoved) {
2323 SetNextResolution({});
2324 SetNextResolutionForLbChannelAllBalancers();
2325 AdsServiceImpl::EdsResourceArgs args({
2326 {"locality0", GetBackendPorts()},
2327 });
2328 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2329 // We need to wait for all backends to come online.
2330 WaitForAllBackends();
2331 // Unset CDS resource.
2332 balancers_[0]->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName);
2333 // Wait for RPCs to start failing.
2334 do {
2335 } while (SendRpc(RpcOptions(), nullptr).ok());
2336 // Make sure RPCs are still failing.
2337 CheckRpcSendFailure(1000);
2338 // Make sure we ACK'ed the update.
2339 EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state,
2340 AdsServiceImpl::ResponseState::ACKED);
2341 }
2342
2343 // Tests that we restart all xDS requests when we reestablish the ADS call.
TEST_P(XdsResolverOnlyTest,RestartsRequestsUponReconnection)2344 TEST_P(XdsResolverOnlyTest, RestartsRequestsUponReconnection) {
2345 // Manually configure use of RDS.
2346 auto listener = default_listener_;
2347 HttpConnectionManager http_connection_manager;
2348 auto* rds = http_connection_manager.mutable_rds();
2349 rds->set_route_config_name(kDefaultRouteConfigurationName);
2350 rds->mutable_config_source()->mutable_ads();
2351 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2352 http_connection_manager);
2353 balancers_[0]->ads_service()->SetLdsResource(listener);
2354 balancers_[0]->ads_service()->SetRdsResource(default_route_config_);
2355 const char* kNewClusterName = "new_cluster_name";
2356 const char* kNewEdsServiceName = "new_eds_service_name";
2357 SetNextResolution({});
2358 SetNextResolutionForLbChannelAllBalancers();
2359 AdsServiceImpl::EdsResourceArgs args({
2360 {"locality0", GetBackendPorts(0, 2)},
2361 });
2362 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2363 // We need to wait for all backends to come online.
2364 WaitForAllBackends(0, 2);
2365 // Now shut down and restart the balancer. When the client
2366 // reconnects, it should automatically restart the requests for all
2367 // resource types.
2368 balancers_[0]->Shutdown();
2369 balancers_[0]->Start();
2370 // Make sure things are still working.
2371 CheckRpcSendOk(100);
2372 // Populate new EDS resource.
2373 AdsServiceImpl::EdsResourceArgs args2({
2374 {"locality0", GetBackendPorts(2, 4)},
2375 });
2376 balancers_[0]->ads_service()->SetEdsResource(
2377 BuildEdsResource(args2, kNewEdsServiceName));
2378 // Populate new CDS resource.
2379 Cluster new_cluster = default_cluster_;
2380 new_cluster.set_name(kNewClusterName);
2381 new_cluster.mutable_eds_cluster_config()->set_service_name(
2382 kNewEdsServiceName);
2383 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
2384 // Change RDS resource to point to new cluster.
2385 RouteConfiguration new_route_config = default_route_config_;
2386 new_route_config.mutable_virtual_hosts(0)
2387 ->mutable_routes(0)
2388 ->mutable_route()
2389 ->set_cluster(kNewClusterName);
2390 balancers_[0]->ads_service()->SetRdsResource(new_route_config);
2391 // Wait for all new backends to be used.
2392 std::tuple<int, int, int> counts = WaitForAllBackends(2, 4);
2393 // Make sure no RPCs failed in the transition.
2394 EXPECT_EQ(0, std::get<1>(counts));
2395 }
2396
TEST_P(XdsResolverOnlyTest,DefaultRouteSpecifiesSlashPrefix)2397 TEST_P(XdsResolverOnlyTest, DefaultRouteSpecifiesSlashPrefix) {
2398 RouteConfiguration route_config = default_route_config_;
2399 route_config.mutable_virtual_hosts(0)
2400 ->mutable_routes(0)
2401 ->mutable_match()
2402 ->set_prefix("/");
2403 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
2404 SetNextResolution({});
2405 SetNextResolutionForLbChannelAllBalancers();
2406 AdsServiceImpl::EdsResourceArgs args({
2407 {"locality0", GetBackendPorts()},
2408 });
2409 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2410 // We need to wait for all backends to come online.
2411 WaitForAllBackends();
2412 }
2413
TEST_P(XdsResolverOnlyTest,CircuitBreaking)2414 TEST_P(XdsResolverOnlyTest, CircuitBreaking) {
2415 gpr_setenv("GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING", "true");
2416 constexpr size_t kMaxConcurrentRequests = 10;
2417 SetNextResolution({});
2418 SetNextResolutionForLbChannelAllBalancers();
2419 // Populate new EDS resources.
2420 AdsServiceImpl::EdsResourceArgs args({
2421 {"locality0", GetBackendPorts(0, 1)},
2422 });
2423 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2424 // Update CDS resource to set max concurrent request.
2425 CircuitBreakers circuit_breaks;
2426 Cluster cluster = default_cluster_;
2427 auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
2428 threshold->set_priority(RoutingPriority::DEFAULT);
2429 threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
2430 balancers_[0]->ads_service()->SetCdsResource(cluster);
2431 // Send exactly max_concurrent_requests long RPCs.
2432 LongRunningRpc rpcs[kMaxConcurrentRequests];
2433 for (size_t i = 0; i < kMaxConcurrentRequests; ++i) {
2434 rpcs[i].StartRpc(stub_.get());
2435 }
2436 // Wait for all RPCs to be in flight.
2437 while (backends_[0]->backend_service()->RpcsWaitingForClientCancel() <
2438 kMaxConcurrentRequests) {
2439 gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
2440 gpr_time_from_micros(1 * 1000, GPR_TIMESPAN)));
2441 }
2442 // Sending a RPC now should fail, the error message should tell us
2443 // we hit the max concurrent requests limit and got dropped.
2444 Status status = SendRpc();
2445 EXPECT_FALSE(status.ok());
2446 EXPECT_EQ(status.error_message(), "Call dropped by load balancing policy");
2447 // Cancel one RPC to allow another one through
2448 rpcs[0].CancelRpc();
2449 status = SendRpc();
2450 EXPECT_TRUE(status.ok());
2451 for (size_t i = 1; i < kMaxConcurrentRequests; ++i) {
2452 rpcs[i].CancelRpc();
2453 }
2454 // Make sure RPCs go to the correct backend:
2455 EXPECT_EQ(kMaxConcurrentRequests + 1,
2456 backends_[0]->backend_service()->request_count());
2457 gpr_unsetenv("GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING");
2458 }
2459
TEST_P(XdsResolverOnlyTest,CircuitBreakingMultipleChannelsShareCallCounter)2460 TEST_P(XdsResolverOnlyTest, CircuitBreakingMultipleChannelsShareCallCounter) {
2461 gpr_setenv("GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING", "true");
2462 constexpr size_t kMaxConcurrentRequests = 10;
2463 // Populate new EDS resources.
2464 AdsServiceImpl::EdsResourceArgs args({
2465 {"locality0", GetBackendPorts(0, 1)},
2466 });
2467 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2468 // Update CDS resource to set max concurrent request.
2469 CircuitBreakers circuit_breaks;
2470 Cluster cluster = default_cluster_;
2471 auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
2472 threshold->set_priority(RoutingPriority::DEFAULT);
2473 threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
2474 balancers_[0]->ads_service()->SetCdsResource(cluster);
2475 // Create second channel.
2476 auto response_generator2 =
2477 grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
2478 auto channel2 = CreateChannel(
2479 /*failover_timeout=*/0, /*server_name=*/kServerName,
2480 response_generator2.get());
2481 auto stub2 = grpc::testing::EchoTestService::NewStub(channel2);
2482 // Set resolution results for both channels and for the xDS channel.
2483 SetNextResolution({});
2484 SetNextResolution({}, response_generator2.get());
2485 SetNextResolutionForLbChannelAllBalancers();
2486 // Send exactly max_concurrent_requests long RPCs, alternating between
2487 // the two channels.
2488 LongRunningRpc rpcs[kMaxConcurrentRequests];
2489 for (size_t i = 0; i < kMaxConcurrentRequests; ++i) {
2490 rpcs[i].StartRpc(i % 2 == 0 ? stub_.get() : stub2.get());
2491 }
2492 // Wait for all RPCs to be in flight.
2493 while (backends_[0]->backend_service()->RpcsWaitingForClientCancel() <
2494 kMaxConcurrentRequests) {
2495 gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
2496 gpr_time_from_micros(1 * 1000, GPR_TIMESPAN)));
2497 }
2498 // Sending a RPC now should fail, the error message should tell us
2499 // we hit the max concurrent requests limit and got dropped.
2500 Status status = SendRpc();
2501 EXPECT_FALSE(status.ok());
2502 EXPECT_EQ(status.error_message(), "Call dropped by load balancing policy");
2503 // Cancel one RPC to allow another one through
2504 rpcs[0].CancelRpc();
2505 status = SendRpc();
2506 EXPECT_TRUE(status.ok());
2507 for (size_t i = 1; i < kMaxConcurrentRequests; ++i) {
2508 rpcs[i].CancelRpc();
2509 }
2510 // Make sure RPCs go to the correct backend:
2511 EXPECT_EQ(kMaxConcurrentRequests + 1,
2512 backends_[0]->backend_service()->request_count());
2513 gpr_unsetenv("GRPC_XDS_EXPERIMENTAL_CIRCUIT_BREAKING");
2514 }
2515
TEST_P(XdsResolverOnlyTest,CircuitBreakingDisabled)2516 TEST_P(XdsResolverOnlyTest, CircuitBreakingDisabled) {
2517 constexpr size_t kMaxConcurrentRequests = 10;
2518 SetNextResolution({});
2519 SetNextResolutionForLbChannelAllBalancers();
2520 // Populate new EDS resources.
2521 AdsServiceImpl::EdsResourceArgs args({
2522 {"locality0", GetBackendPorts(0, 1)},
2523 });
2524 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2525 // Update CDS resource to set max concurrent request.
2526 CircuitBreakers circuit_breaks;
2527 Cluster cluster = default_cluster_;
2528 auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
2529 threshold->set_priority(RoutingPriority::DEFAULT);
2530 threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
2531 balancers_[0]->ads_service()->SetCdsResource(cluster);
2532 // Send exactly max_concurrent_requests long RPCs.
2533 LongRunningRpc rpcs[kMaxConcurrentRequests];
2534 for (size_t i = 0; i < kMaxConcurrentRequests; ++i) {
2535 rpcs[i].StartRpc(stub_.get());
2536 }
2537 // Wait for all RPCs to be in flight.
2538 while (backends_[0]->backend_service()->RpcsWaitingForClientCancel() <
2539 kMaxConcurrentRequests) {
2540 gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
2541 gpr_time_from_micros(1 * 1000, GPR_TIMESPAN)));
2542 }
2543 // Sending a RPC now should not fail as circuit breaking is disabled.
2544 Status status = SendRpc();
2545 EXPECT_TRUE(status.ok());
2546 for (size_t i = 0; i < kMaxConcurrentRequests; ++i) {
2547 rpcs[i].CancelRpc();
2548 }
2549 // Make sure RPCs go to the correct backend:
2550 EXPECT_EQ(kMaxConcurrentRequests + 1,
2551 backends_[0]->backend_service()->request_count());
2552 }
2553
TEST_P(XdsResolverOnlyTest,MultipleChannelsShareXdsClient)2554 TEST_P(XdsResolverOnlyTest, MultipleChannelsShareXdsClient) {
2555 const char* kNewServerName = "new-server.example.com";
2556 Listener listener = default_listener_;
2557 listener.set_name(kNewServerName);
2558 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
2559 SetNextResolution({});
2560 SetNextResolutionForLbChannelAllBalancers();
2561 AdsServiceImpl::EdsResourceArgs args({
2562 {"locality0", GetBackendPorts()},
2563 });
2564 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2565 WaitForAllBackends();
2566 // Create second channel and tell it to connect to kNewServerName.
2567 auto channel2 = CreateChannel(/*failover_timeout=*/0, kNewServerName);
2568 channel2->GetState(/*try_to_connect=*/true);
2569 ASSERT_TRUE(
2570 channel2->WaitForConnected(grpc_timeout_milliseconds_to_deadline(100)));
2571 // Make sure there's only one client connected.
2572 EXPECT_EQ(1UL, balancers_[0]->ads_service()->clients().size());
2573 }
2574
2575 class XdsResolverLoadReportingOnlyTest : public XdsEnd2endTest {
2576 public:
XdsResolverLoadReportingOnlyTest()2577 XdsResolverLoadReportingOnlyTest() : XdsEnd2endTest(4, 1, 3) {}
2578 };
2579
2580 // Tests load reporting when switching over from one cluster to another.
TEST_P(XdsResolverLoadReportingOnlyTest,ChangeClusters)2581 TEST_P(XdsResolverLoadReportingOnlyTest, ChangeClusters) {
2582 const char* kNewClusterName = "new_cluster_name";
2583 const char* kNewEdsServiceName = "new_eds_service_name";
2584 balancers_[0]->lrs_service()->set_cluster_names(
2585 {kDefaultClusterName, kNewClusterName});
2586 SetNextResolution({});
2587 SetNextResolutionForLbChannelAllBalancers();
2588 // cluster kDefaultClusterName -> locality0 -> backends 0 and 1
2589 AdsServiceImpl::EdsResourceArgs args({
2590 {"locality0", GetBackendPorts(0, 2)},
2591 });
2592 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2593 // cluster kNewClusterName -> locality1 -> backends 2 and 3
2594 AdsServiceImpl::EdsResourceArgs args2({
2595 {"locality1", GetBackendPorts(2, 4)},
2596 });
2597 balancers_[0]->ads_service()->SetEdsResource(
2598 BuildEdsResource(args2, kNewEdsServiceName));
2599 // CDS resource for kNewClusterName.
2600 Cluster new_cluster = default_cluster_;
2601 new_cluster.set_name(kNewClusterName);
2602 new_cluster.mutable_eds_cluster_config()->set_service_name(
2603 kNewEdsServiceName);
2604 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
2605 // Wait for all backends to come online.
2606 int num_ok = 0;
2607 int num_failure = 0;
2608 int num_drops = 0;
2609 std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(0, 2);
2610 // The load report received at the balancer should be correct.
2611 std::vector<ClientStats> load_report =
2612 balancers_[0]->lrs_service()->WaitForLoadReport();
2613 EXPECT_THAT(
2614 load_report,
2615 ::testing::ElementsAre(::testing::AllOf(
2616 ::testing::Property(&ClientStats::cluster_name, kDefaultClusterName),
2617 ::testing::Property(
2618 &ClientStats::locality_stats,
2619 ::testing::ElementsAre(::testing::Pair(
2620 "locality0",
2621 ::testing::AllOf(
2622 ::testing::Field(&ClientStats::LocalityStats::
2623 total_successful_requests,
2624 num_ok),
2625 ::testing::Field(&ClientStats::LocalityStats::
2626 total_requests_in_progress,
2627 0UL),
2628 ::testing::Field(
2629 &ClientStats::LocalityStats::total_error_requests,
2630 num_failure),
2631 ::testing::Field(
2632 &ClientStats::LocalityStats::total_issued_requests,
2633 num_failure + num_ok))))),
2634 ::testing::Property(&ClientStats::total_dropped_requests,
2635 num_drops))));
2636 // Change RDS resource to point to new cluster.
2637 RouteConfiguration new_route_config = default_route_config_;
2638 new_route_config.mutable_virtual_hosts(0)
2639 ->mutable_routes(0)
2640 ->mutable_route()
2641 ->set_cluster(kNewClusterName);
2642 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
2643 // Wait for all new backends to be used.
2644 std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(2, 4);
2645 // The load report received at the balancer should be correct.
2646 load_report = balancers_[0]->lrs_service()->WaitForLoadReport();
2647 EXPECT_THAT(
2648 load_report,
2649 ::testing::ElementsAre(
2650 ::testing::AllOf(
2651 ::testing::Property(&ClientStats::cluster_name,
2652 kDefaultClusterName),
2653 ::testing::Property(
2654 &ClientStats::locality_stats,
2655 ::testing::ElementsAre(::testing::Pair(
2656 "locality0",
2657 ::testing::AllOf(
2658 ::testing::Field(&ClientStats::LocalityStats::
2659 total_successful_requests,
2660 ::testing::Lt(num_ok)),
2661 ::testing::Field(&ClientStats::LocalityStats::
2662 total_requests_in_progress,
2663 0UL),
2664 ::testing::Field(
2665 &ClientStats::LocalityStats::total_error_requests,
2666 ::testing::Le(num_failure)),
2667 ::testing::Field(
2668 &ClientStats::LocalityStats::
2669 total_issued_requests,
2670 ::testing::Le(num_failure + num_ok)))))),
2671 ::testing::Property(&ClientStats::total_dropped_requests,
2672 num_drops)),
2673 ::testing::AllOf(
2674 ::testing::Property(&ClientStats::cluster_name, kNewClusterName),
2675 ::testing::Property(
2676 &ClientStats::locality_stats,
2677 ::testing::ElementsAre(::testing::Pair(
2678 "locality1",
2679 ::testing::AllOf(
2680 ::testing::Field(&ClientStats::LocalityStats::
2681 total_successful_requests,
2682 ::testing::Le(num_ok)),
2683 ::testing::Field(&ClientStats::LocalityStats::
2684 total_requests_in_progress,
2685 0UL),
2686 ::testing::Field(
2687 &ClientStats::LocalityStats::total_error_requests,
2688 ::testing::Le(num_failure)),
2689 ::testing::Field(
2690 &ClientStats::LocalityStats::
2691 total_issued_requests,
2692 ::testing::Le(num_failure + num_ok)))))),
2693 ::testing::Property(&ClientStats::total_dropped_requests,
2694 num_drops))));
2695 int total_ok = 0;
2696 int total_failure = 0;
2697 for (const ClientStats& client_stats : load_report) {
2698 total_ok += client_stats.total_successful_requests();
2699 total_failure += client_stats.total_error_requests();
2700 }
2701 EXPECT_EQ(total_ok, num_ok);
2702 EXPECT_EQ(total_failure, num_failure);
2703 // The LRS service got a single request, and sent a single response.
2704 EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count());
2705 EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count());
2706 }
2707
2708 using SecureNamingTest = BasicTest;
2709
2710 // Tests that secure naming check passes if target name is expected.
TEST_P(SecureNamingTest,TargetNameIsExpected)2711 TEST_P(SecureNamingTest, TargetNameIsExpected) {
2712 SetNextResolution({});
2713 SetNextResolutionForLbChannel({balancers_[0]->port()}, nullptr, "xds_server");
2714 AdsServiceImpl::EdsResourceArgs args({
2715 {"locality0", GetBackendPorts()},
2716 });
2717 balancers_[0]->ads_service()->SetEdsResource(
2718 BuildEdsResource(args, DefaultEdsServiceName()));
2719 CheckRpcSendOk();
2720 }
2721
2722 // Tests that secure naming check fails if target name is unexpected.
TEST_P(SecureNamingTest,TargetNameIsUnexpected)2723 TEST_P(SecureNamingTest, TargetNameIsUnexpected) {
2724 ::testing::FLAGS_gtest_death_test_style = "threadsafe";
2725 SetNextResolution({});
2726 SetNextResolutionForLbChannel({balancers_[0]->port()}, nullptr,
2727 "incorrect_server_name");
2728 AdsServiceImpl::EdsResourceArgs args({
2729 {"locality0", GetBackendPorts()},
2730 });
2731 balancers_[0]->ads_service()->SetEdsResource(
2732 BuildEdsResource(args, DefaultEdsServiceName()));
2733 // Make sure that we blow up (via abort() from the security connector) when
2734 // the name from the balancer doesn't match expectations.
2735 ASSERT_DEATH_IF_SUPPORTED({ CheckRpcSendOk(); }, "");
2736 }
2737
2738 using LdsTest = BasicTest;
2739
2740 // Tests that LDS client should send a NACK if there is no API listener in the
2741 // Listener in the LDS response.
TEST_P(LdsTest,NoApiListener)2742 TEST_P(LdsTest, NoApiListener) {
2743 auto listener = default_listener_;
2744 listener.clear_api_listener();
2745 balancers_[0]->ads_service()->SetLdsResource(listener);
2746 SetNextResolution({});
2747 SetNextResolutionForLbChannelAllBalancers();
2748 CheckRpcSendFailure();
2749 const auto& response_state =
2750 balancers_[0]->ads_service()->lds_response_state();
2751 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2752 EXPECT_EQ(response_state.error_message, "Listener has no ApiListener.");
2753 }
2754
2755 // Tests that LDS client should send a NACK if the route_specifier in the
2756 // http_connection_manager is neither inlined route_config nor RDS.
TEST_P(LdsTest,WrongRouteSpecifier)2757 TEST_P(LdsTest, WrongRouteSpecifier) {
2758 auto listener = default_listener_;
2759 HttpConnectionManager http_connection_manager;
2760 http_connection_manager.mutable_scoped_routes();
2761 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2762 http_connection_manager);
2763 balancers_[0]->ads_service()->SetLdsResource(listener);
2764 SetNextResolution({});
2765 SetNextResolutionForLbChannelAllBalancers();
2766 CheckRpcSendFailure();
2767 const auto& response_state =
2768 balancers_[0]->ads_service()->lds_response_state();
2769 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2770 EXPECT_EQ(response_state.error_message,
2771 "HttpConnectionManager neither has inlined route_config nor RDS.");
2772 }
2773
2774 // Tests that LDS client should send a NACK if the rds message in the
2775 // http_connection_manager is missing the config_source field.
TEST_P(LdsTest,RdsMissingConfigSource)2776 TEST_P(LdsTest, RdsMissingConfigSource) {
2777 auto listener = default_listener_;
2778 HttpConnectionManager http_connection_manager;
2779 http_connection_manager.mutable_rds()->set_route_config_name(
2780 kDefaultRouteConfigurationName);
2781 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2782 http_connection_manager);
2783 balancers_[0]->ads_service()->SetLdsResource(listener);
2784 SetNextResolution({});
2785 SetNextResolutionForLbChannelAllBalancers();
2786 CheckRpcSendFailure();
2787 const auto& response_state =
2788 balancers_[0]->ads_service()->lds_response_state();
2789 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2790 EXPECT_EQ(response_state.error_message,
2791 "HttpConnectionManager missing config_source for RDS.");
2792 }
2793
2794 // Tests that LDS client should send a NACK if the rds message in the
2795 // http_connection_manager has a config_source field that does not specify ADS.
TEST_P(LdsTest,RdsConfigSourceDoesNotSpecifyAds)2796 TEST_P(LdsTest, RdsConfigSourceDoesNotSpecifyAds) {
2797 auto listener = default_listener_;
2798 HttpConnectionManager http_connection_manager;
2799 auto* rds = http_connection_manager.mutable_rds();
2800 rds->set_route_config_name(kDefaultRouteConfigurationName);
2801 rds->mutable_config_source()->mutable_self();
2802 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2803 http_connection_manager);
2804 balancers_[0]->ads_service()->SetLdsResource(listener);
2805 SetNextResolution({});
2806 SetNextResolutionForLbChannelAllBalancers();
2807 CheckRpcSendFailure();
2808 const auto& response_state =
2809 balancers_[0]->ads_service()->lds_response_state();
2810 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2811 EXPECT_EQ(response_state.error_message,
2812 "HttpConnectionManager ConfigSource for RDS does not specify ADS.");
2813 }
2814
2815 using LdsRdsTest = BasicTest;
2816
2817 // Tests that LDS client should send an ACK upon correct LDS response (with
2818 // inlined RDS result).
TEST_P(LdsRdsTest,Vanilla)2819 TEST_P(LdsRdsTest, Vanilla) {
2820 SetNextResolution({});
2821 SetNextResolutionForLbChannelAllBalancers();
2822 (void)SendRpc();
2823 EXPECT_EQ(RouteConfigurationResponseState(0).state,
2824 AdsServiceImpl::ResponseState::ACKED);
2825 // Make sure we actually used the RPC service for the right version of xDS.
2826 EXPECT_EQ(balancers_[0]->ads_service()->seen_v2_client(),
2827 GetParam().use_v2());
2828 EXPECT_NE(balancers_[0]->ads_service()->seen_v3_client(),
2829 GetParam().use_v2());
2830 }
2831
2832 // Tests that we go into TRANSIENT_FAILURE if the Listener is removed.
TEST_P(LdsRdsTest,ListenerRemoved)2833 TEST_P(LdsRdsTest, ListenerRemoved) {
2834 SetNextResolution({});
2835 SetNextResolutionForLbChannelAllBalancers();
2836 AdsServiceImpl::EdsResourceArgs args({
2837 {"locality0", GetBackendPorts()},
2838 });
2839 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2840 // We need to wait for all backends to come online.
2841 WaitForAllBackends();
2842 // Unset LDS resource.
2843 balancers_[0]->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
2844 // Wait for RPCs to start failing.
2845 do {
2846 } while (SendRpc(RpcOptions(), nullptr).ok());
2847 // Make sure RPCs are still failing.
2848 CheckRpcSendFailure(1000);
2849 // Make sure we ACK'ed the update.
2850 EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state().state,
2851 AdsServiceImpl::ResponseState::ACKED);
2852 }
2853
2854 // Tests that LDS client ACKs but fails if matching domain can't be found in
2855 // the LDS response.
TEST_P(LdsRdsTest,NoMatchedDomain)2856 TEST_P(LdsRdsTest, NoMatchedDomain) {
2857 RouteConfiguration route_config = default_route_config_;
2858 route_config.mutable_virtual_hosts(0)->clear_domains();
2859 route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain");
2860 SetRouteConfiguration(0, route_config);
2861 SetNextResolution({});
2862 SetNextResolutionForLbChannelAllBalancers();
2863 CheckRpcSendFailure();
2864 // Do a bit of polling, to allow the ACK to get to the ADS server.
2865 channel_->WaitForConnected(grpc_timeout_milliseconds_to_deadline(100));
2866 const auto& response_state = RouteConfigurationResponseState(0);
2867 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
2868 }
2869
2870 // Tests that LDS client should choose the virtual host with matching domain if
2871 // multiple virtual hosts exist in the LDS response.
TEST_P(LdsRdsTest,ChooseMatchedDomain)2872 TEST_P(LdsRdsTest, ChooseMatchedDomain) {
2873 RouteConfiguration route_config = default_route_config_;
2874 *(route_config.add_virtual_hosts()) = route_config.virtual_hosts(0);
2875 route_config.mutable_virtual_hosts(0)->clear_domains();
2876 route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain");
2877 SetRouteConfiguration(0, route_config);
2878 SetNextResolution({});
2879 SetNextResolutionForLbChannelAllBalancers();
2880 (void)SendRpc();
2881 EXPECT_EQ(RouteConfigurationResponseState(0).state,
2882 AdsServiceImpl::ResponseState::ACKED);
2883 }
2884
2885 // Tests that LDS client should choose the last route in the virtual host if
2886 // multiple routes exist in the LDS response.
TEST_P(LdsRdsTest,ChooseLastRoute)2887 TEST_P(LdsRdsTest, ChooseLastRoute) {
2888 RouteConfiguration route_config = default_route_config_;
2889 *(route_config.mutable_virtual_hosts(0)->add_routes()) =
2890 route_config.virtual_hosts(0).routes(0);
2891 route_config.mutable_virtual_hosts(0)
2892 ->mutable_routes(0)
2893 ->mutable_route()
2894 ->mutable_cluster_header();
2895 SetRouteConfiguration(0, route_config);
2896 SetNextResolution({});
2897 SetNextResolutionForLbChannelAllBalancers();
2898 (void)SendRpc();
2899 EXPECT_EQ(RouteConfigurationResponseState(0).state,
2900 AdsServiceImpl::ResponseState::ACKED);
2901 }
2902
2903 // Tests that LDS client should ignore route which has query_parameters.
TEST_P(LdsRdsTest,RouteMatchHasQueryParameters)2904 TEST_P(LdsRdsTest, RouteMatchHasQueryParameters) {
2905 RouteConfiguration route_config = default_route_config_;
2906 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
2907 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
2908 route1->mutable_match()->add_query_parameters();
2909 SetRouteConfiguration(0, route_config);
2910 SetNextResolution({});
2911 SetNextResolutionForLbChannelAllBalancers();
2912 CheckRpcSendFailure();
2913 const auto& response_state = RouteConfigurationResponseState(0);
2914 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2915 EXPECT_EQ(response_state.error_message, "No valid routes specified.");
2916 }
2917
2918 // Tests that LDS client should send a ACK if route match has a prefix
2919 // that is either empty or a single slash
TEST_P(LdsRdsTest,RouteMatchHasValidPrefixEmptyOrSingleSlash)2920 TEST_P(LdsRdsTest, RouteMatchHasValidPrefixEmptyOrSingleSlash) {
2921 RouteConfiguration route_config = default_route_config_;
2922 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
2923 route1->mutable_match()->set_prefix("");
2924 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
2925 default_route->mutable_match()->set_prefix("/");
2926 default_route->mutable_route()->set_cluster(kDefaultClusterName);
2927 SetRouteConfiguration(0, route_config);
2928 SetNextResolution({});
2929 SetNextResolutionForLbChannelAllBalancers();
2930 (void)SendRpc();
2931 const auto& response_state = RouteConfigurationResponseState(0);
2932 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
2933 }
2934
2935 // Tests that LDS client should ignore route which has a path
2936 // prefix string does not start with "/".
TEST_P(LdsRdsTest,RouteMatchHasInvalidPrefixNoLeadingSlash)2937 TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNoLeadingSlash) {
2938 RouteConfiguration route_config = default_route_config_;
2939 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
2940 route1->mutable_match()->set_prefix("grpc.testing.EchoTest1Service/");
2941 SetRouteConfiguration(0, route_config);
2942 SetNextResolution({});
2943 SetNextResolutionForLbChannelAllBalancers();
2944 CheckRpcSendFailure();
2945 const auto& response_state = RouteConfigurationResponseState(0);
2946 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2947 EXPECT_EQ(response_state.error_message, "No valid routes specified.");
2948 }
2949
2950 // Tests that LDS client should ignore route which has a prefix
2951 // string with more than 2 slashes.
TEST_P(LdsRdsTest,RouteMatchHasInvalidPrefixExtraContent)2952 TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixExtraContent) {
2953 RouteConfiguration route_config = default_route_config_;
2954 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
2955 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/Echo1/");
2956 SetRouteConfiguration(0, route_config);
2957 SetNextResolution({});
2958 SetNextResolutionForLbChannelAllBalancers();
2959 CheckRpcSendFailure();
2960 const auto& response_state = RouteConfigurationResponseState(0);
2961 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2962 EXPECT_EQ(response_state.error_message, "No valid routes specified.");
2963 }
2964
2965 // Tests that LDS client should ignore route which has a prefix
2966 // string "//".
TEST_P(LdsRdsTest,RouteMatchHasInvalidPrefixDoubleSlash)2967 TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixDoubleSlash) {
2968 RouteConfiguration route_config = default_route_config_;
2969 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
2970 route1->mutable_match()->set_prefix("//");
2971 SetRouteConfiguration(0, route_config);
2972 SetNextResolution({});
2973 SetNextResolutionForLbChannelAllBalancers();
2974 CheckRpcSendFailure();
2975 const auto& response_state = RouteConfigurationResponseState(0);
2976 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2977 EXPECT_EQ(response_state.error_message, "No valid routes specified.");
2978 }
2979
2980 // Tests that LDS client should ignore route which has path
2981 // but it's empty.
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathEmptyPath)2982 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathEmptyPath) {
2983 RouteConfiguration route_config = default_route_config_;
2984 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
2985 route1->mutable_match()->set_path("");
2986 SetRouteConfiguration(0, route_config);
2987 SetNextResolution({});
2988 SetNextResolutionForLbChannelAllBalancers();
2989 CheckRpcSendFailure();
2990 const auto& response_state = RouteConfigurationResponseState(0);
2991 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2992 EXPECT_EQ(response_state.error_message, "No valid routes specified.");
2993 }
2994
2995 // Tests that LDS client should ignore route which has path
2996 // string does not start with "/".
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathNoLeadingSlash)2997 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathNoLeadingSlash) {
2998 RouteConfiguration route_config = default_route_config_;
2999 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3000 route1->mutable_match()->set_path("grpc.testing.EchoTest1Service/Echo1");
3001 SetRouteConfiguration(0, route_config);
3002 SetNextResolution({});
3003 SetNextResolutionForLbChannelAllBalancers();
3004 CheckRpcSendFailure();
3005 const auto& response_state = RouteConfigurationResponseState(0);
3006 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3007 EXPECT_EQ(response_state.error_message, "No valid routes specified.");
3008 }
3009
3010 // Tests that LDS client should ignore route which has path
3011 // string that has too many slashes; for example, ends with "/".
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathTooManySlashes)3012 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathTooManySlashes) {
3013 RouteConfiguration route_config = default_route_config_;
3014 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3015 route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1/");
3016 SetRouteConfiguration(0, route_config);
3017 SetNextResolution({});
3018 SetNextResolutionForLbChannelAllBalancers();
3019 CheckRpcSendFailure();
3020 const auto& response_state = RouteConfigurationResponseState(0);
3021 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3022 EXPECT_EQ(response_state.error_message, "No valid routes specified.");
3023 }
3024
3025 // Tests that LDS client should ignore route which has path
3026 // string that has only 1 slash: missing "/" between service and method.
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathOnlyOneSlash)3027 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathOnlyOneSlash) {
3028 RouteConfiguration route_config = default_route_config_;
3029 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3030 route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service.Echo1");
3031 SetRouteConfiguration(0, route_config);
3032 SetNextResolution({});
3033 SetNextResolutionForLbChannelAllBalancers();
3034 CheckRpcSendFailure();
3035 const auto& response_state = RouteConfigurationResponseState(0);
3036 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3037 EXPECT_EQ(response_state.error_message, "No valid routes specified.");
3038 }
3039
3040 // Tests that LDS client should ignore route which has path
3041 // string that is missing service.
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathMissingService)3042 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingService) {
3043 RouteConfiguration route_config = default_route_config_;
3044 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3045 route1->mutable_match()->set_path("//Echo1");
3046 SetRouteConfiguration(0, route_config);
3047 SetNextResolution({});
3048 SetNextResolutionForLbChannelAllBalancers();
3049 CheckRpcSendFailure();
3050 const auto& response_state = RouteConfigurationResponseState(0);
3051 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3052 EXPECT_EQ(response_state.error_message, "No valid routes specified.");
3053 }
3054
3055 // Tests that LDS client should ignore route which has path
3056 // string that is missing method.
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathMissingMethod)3057 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingMethod) {
3058 RouteConfiguration route_config = default_route_config_;
3059 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3060 route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/");
3061 SetRouteConfiguration(0, route_config);
3062 SetNextResolution({});
3063 SetNextResolutionForLbChannelAllBalancers();
3064 CheckRpcSendFailure();
3065 const auto& response_state = RouteConfigurationResponseState(0);
3066 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3067 EXPECT_EQ(response_state.error_message, "No valid routes specified.");
3068 }
3069
3070 // Test that LDS client should reject route which has invalid path regex.
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathRegex)3071 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathRegex) {
3072 const char* kNewCluster1Name = "new_cluster_1";
3073 RouteConfiguration route_config = default_route_config_;
3074 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3075 route1->mutable_match()->mutable_safe_regex()->set_regex("a[z-a]");
3076 route1->mutable_route()->set_cluster(kNewCluster1Name);
3077 SetRouteConfiguration(0, route_config);
3078 SetNextResolution({});
3079 SetNextResolutionForLbChannelAllBalancers();
3080 CheckRpcSendFailure();
3081 const auto& response_state = RouteConfigurationResponseState(0);
3082 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3083 EXPECT_EQ(response_state.error_message,
3084 "Invalid regex string specified in path matcher.");
3085 }
3086
3087 // Tests that LDS client should send a NACK if route has an action other than
3088 // RouteAction in the LDS response.
TEST_P(LdsRdsTest,RouteHasNoRouteAction)3089 TEST_P(LdsRdsTest, RouteHasNoRouteAction) {
3090 RouteConfiguration route_config = default_route_config_;
3091 route_config.mutable_virtual_hosts(0)->mutable_routes(0)->mutable_redirect();
3092 SetRouteConfiguration(0, route_config);
3093 SetNextResolution({});
3094 SetNextResolutionForLbChannelAllBalancers();
3095 CheckRpcSendFailure();
3096 const auto& response_state = RouteConfigurationResponseState(0);
3097 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3098 EXPECT_EQ(response_state.error_message, "No RouteAction found in route.");
3099 }
3100
TEST_P(LdsRdsTest,RouteActionClusterHasEmptyClusterName)3101 TEST_P(LdsRdsTest, RouteActionClusterHasEmptyClusterName) {
3102 RouteConfiguration route_config = default_route_config_;
3103 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3104 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3105 route1->mutable_route()->set_cluster("");
3106 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
3107 default_route->mutable_match()->set_prefix("");
3108 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3109 SetRouteConfiguration(0, route_config);
3110 SetNextResolution({});
3111 SetNextResolutionForLbChannelAllBalancers();
3112 CheckRpcSendFailure();
3113 const auto& response_state = RouteConfigurationResponseState(0);
3114 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3115 EXPECT_EQ(response_state.error_message,
3116 "RouteAction cluster contains empty cluster name.");
3117 }
3118
TEST_P(LdsRdsTest,RouteActionWeightedTargetHasIncorrectTotalWeightSet)3119 TEST_P(LdsRdsTest, RouteActionWeightedTargetHasIncorrectTotalWeightSet) {
3120 const size_t kWeight75 = 75;
3121 const char* kNewCluster1Name = "new_cluster_1";
3122 RouteConfiguration route_config = default_route_config_;
3123 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3124 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3125 auto* weighted_cluster1 =
3126 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3127 weighted_cluster1->set_name(kNewCluster1Name);
3128 weighted_cluster1->mutable_weight()->set_value(kWeight75);
3129 route1->mutable_route()
3130 ->mutable_weighted_clusters()
3131 ->mutable_total_weight()
3132 ->set_value(kWeight75 + 1);
3133 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
3134 default_route->mutable_match()->set_prefix("");
3135 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3136 SetRouteConfiguration(0, route_config);
3137 SetNextResolution({});
3138 SetNextResolutionForLbChannelAllBalancers();
3139 CheckRpcSendFailure();
3140 const auto& response_state = RouteConfigurationResponseState(0);
3141 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3142 EXPECT_EQ(response_state.error_message,
3143 "RouteAction weighted_cluster has incorrect total weight");
3144 }
3145
TEST_P(LdsRdsTest,RouteActionWeightedTargetClusterHasEmptyClusterName)3146 TEST_P(LdsRdsTest, RouteActionWeightedTargetClusterHasEmptyClusterName) {
3147 const size_t kWeight75 = 75;
3148 RouteConfiguration route_config = default_route_config_;
3149 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3150 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3151 auto* weighted_cluster1 =
3152 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3153 weighted_cluster1->set_name("");
3154 weighted_cluster1->mutable_weight()->set_value(kWeight75);
3155 route1->mutable_route()
3156 ->mutable_weighted_clusters()
3157 ->mutable_total_weight()
3158 ->set_value(kWeight75);
3159 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
3160 default_route->mutable_match()->set_prefix("");
3161 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3162 SetRouteConfiguration(0, route_config);
3163 SetNextResolution({});
3164 SetNextResolutionForLbChannelAllBalancers();
3165 CheckRpcSendFailure();
3166 const auto& response_state = RouteConfigurationResponseState(0);
3167 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3168 EXPECT_EQ(
3169 response_state.error_message,
3170 "RouteAction weighted_cluster cluster contains empty cluster name.");
3171 }
3172
TEST_P(LdsRdsTest,RouteActionWeightedTargetClusterHasNoWeight)3173 TEST_P(LdsRdsTest, RouteActionWeightedTargetClusterHasNoWeight) {
3174 const size_t kWeight75 = 75;
3175 const char* kNewCluster1Name = "new_cluster_1";
3176 RouteConfiguration route_config = default_route_config_;
3177 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3178 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3179 auto* weighted_cluster1 =
3180 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3181 weighted_cluster1->set_name(kNewCluster1Name);
3182 route1->mutable_route()
3183 ->mutable_weighted_clusters()
3184 ->mutable_total_weight()
3185 ->set_value(kWeight75);
3186 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
3187 default_route->mutable_match()->set_prefix("");
3188 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3189 SetRouteConfiguration(0, route_config);
3190 SetNextResolution({});
3191 SetNextResolutionForLbChannelAllBalancers();
3192 CheckRpcSendFailure();
3193 const auto& response_state = RouteConfigurationResponseState(0);
3194 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3195 EXPECT_EQ(response_state.error_message,
3196 "RouteAction weighted_cluster cluster missing weight");
3197 }
3198
TEST_P(LdsRdsTest,RouteHeaderMatchInvalidRegex)3199 TEST_P(LdsRdsTest, RouteHeaderMatchInvalidRegex) {
3200 const char* kNewCluster1Name = "new_cluster_1";
3201 RouteConfiguration route_config = default_route_config_;
3202 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3203 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3204 auto* header_matcher1 = route1->mutable_match()->add_headers();
3205 header_matcher1->set_name("header1");
3206 header_matcher1->mutable_safe_regex_match()->set_regex("a[z-a]");
3207 route1->mutable_route()->set_cluster(kNewCluster1Name);
3208 SetRouteConfiguration(0, route_config);
3209 SetNextResolution({});
3210 SetNextResolutionForLbChannelAllBalancers();
3211 CheckRpcSendFailure();
3212 const auto& response_state = RouteConfigurationResponseState(0);
3213 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3214 EXPECT_EQ(response_state.error_message,
3215 "Invalid regex string specified in header matcher.");
3216 }
3217
TEST_P(LdsRdsTest,RouteHeaderMatchInvalidRange)3218 TEST_P(LdsRdsTest, RouteHeaderMatchInvalidRange) {
3219 const char* kNewCluster1Name = "new_cluster_1";
3220 RouteConfiguration route_config = default_route_config_;
3221 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3222 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3223 auto* header_matcher1 = route1->mutable_match()->add_headers();
3224 header_matcher1->set_name("header1");
3225 header_matcher1->mutable_range_match()->set_start(1001);
3226 header_matcher1->mutable_range_match()->set_end(1000);
3227 route1->mutable_route()->set_cluster(kNewCluster1Name);
3228 SetRouteConfiguration(0, route_config);
3229 SetNextResolution({});
3230 SetNextResolutionForLbChannelAllBalancers();
3231 CheckRpcSendFailure();
3232 const auto& response_state = RouteConfigurationResponseState(0);
3233 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3234 EXPECT_EQ(response_state.error_message,
3235 "Invalid range header matcher specifier specified: end "
3236 "cannot be smaller than start.");
3237 }
3238
3239 // Tests that LDS client should choose the default route (with no matching
3240 // specified) after unable to find a match with previous routes.
TEST_P(LdsRdsTest,XdsRoutingPathMatching)3241 TEST_P(LdsRdsTest, XdsRoutingPathMatching) {
3242 const char* kNewCluster1Name = "new_cluster_1";
3243 const char* kNewEdsService1Name = "new_eds_service_name_1";
3244 const char* kNewCluster2Name = "new_cluster_2";
3245 const char* kNewEdsService2Name = "new_eds_service_name_2";
3246 const size_t kNumEcho1Rpcs = 10;
3247 const size_t kNumEcho2Rpcs = 20;
3248 const size_t kNumEchoRpcs = 30;
3249 SetNextResolution({});
3250 SetNextResolutionForLbChannelAllBalancers();
3251 // Populate new EDS resources.
3252 AdsServiceImpl::EdsResourceArgs args({
3253 {"locality0", GetBackendPorts(0, 2)},
3254 });
3255 AdsServiceImpl::EdsResourceArgs args1({
3256 {"locality0", GetBackendPorts(2, 3)},
3257 });
3258 AdsServiceImpl::EdsResourceArgs args2({
3259 {"locality0", GetBackendPorts(3, 4)},
3260 });
3261 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3262 balancers_[0]->ads_service()->SetEdsResource(
3263 BuildEdsResource(args1, kNewEdsService1Name));
3264 balancers_[0]->ads_service()->SetEdsResource(
3265 BuildEdsResource(args2, kNewEdsService2Name));
3266 // Populate new CDS resources.
3267 Cluster new_cluster1 = default_cluster_;
3268 new_cluster1.set_name(kNewCluster1Name);
3269 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3270 kNewEdsService1Name);
3271 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3272 Cluster new_cluster2 = default_cluster_;
3273 new_cluster2.set_name(kNewCluster2Name);
3274 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3275 kNewEdsService2Name);
3276 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3277 // Populating Route Configurations for LDS.
3278 RouteConfiguration new_route_config = default_route_config_;
3279 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3280 route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1");
3281 route1->mutable_route()->set_cluster(kNewCluster1Name);
3282 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3283 route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
3284 route2->mutable_route()->set_cluster(kNewCluster2Name);
3285 auto* route3 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3286 route3->mutable_match()->set_path("/grpc.testing.EchoTest3Service/Echo3");
3287 route3->mutable_route()->set_cluster(kDefaultClusterName);
3288 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3289 default_route->mutable_match()->set_prefix("");
3290 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3291 SetRouteConfiguration(0, new_route_config);
3292 WaitForAllBackends(0, 2);
3293 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
3294 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
3295 .set_rpc_service(SERVICE_ECHO1)
3296 .set_rpc_method(METHOD_ECHO1)
3297 .set_wait_for_ready(true));
3298 CheckRpcSendOk(kNumEcho2Rpcs, RpcOptions()
3299 .set_rpc_service(SERVICE_ECHO2)
3300 .set_rpc_method(METHOD_ECHO2)
3301 .set_wait_for_ready(true));
3302 // Make sure RPCs all go to the correct backend.
3303 for (size_t i = 0; i < 2; ++i) {
3304 EXPECT_EQ(kNumEchoRpcs / 2,
3305 backends_[i]->backend_service()->request_count());
3306 EXPECT_EQ(0, backends_[i]->backend_service1()->request_count());
3307 EXPECT_EQ(0, backends_[i]->backend_service2()->request_count());
3308 }
3309 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3310 EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
3311 EXPECT_EQ(0, backends_[2]->backend_service2()->request_count());
3312 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
3313 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
3314 EXPECT_EQ(kNumEcho2Rpcs, backends_[3]->backend_service2()->request_count());
3315 }
3316
TEST_P(LdsRdsTest,XdsRoutingPathMatchingCaseInsensitive)3317 TEST_P(LdsRdsTest, XdsRoutingPathMatchingCaseInsensitive) {
3318 const char* kNewCluster1Name = "new_cluster_1";
3319 const char* kNewEdsService1Name = "new_eds_service_name_1";
3320 const char* kNewCluster2Name = "new_cluster_2";
3321 const char* kNewEdsService2Name = "new_eds_service_name_2";
3322 const size_t kNumEcho1Rpcs = 10;
3323 const size_t kNumEchoRpcs = 30;
3324 SetNextResolution({});
3325 SetNextResolutionForLbChannelAllBalancers();
3326 // Populate new EDS resources.
3327 AdsServiceImpl::EdsResourceArgs args({
3328 {"locality0", GetBackendPorts(0, 1)},
3329 });
3330 AdsServiceImpl::EdsResourceArgs args1({
3331 {"locality0", GetBackendPorts(1, 2)},
3332 });
3333 AdsServiceImpl::EdsResourceArgs args2({
3334 {"locality0", GetBackendPorts(2, 3)},
3335 });
3336 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3337 balancers_[0]->ads_service()->SetEdsResource(
3338 BuildEdsResource(args1, kNewEdsService1Name));
3339 balancers_[0]->ads_service()->SetEdsResource(
3340 BuildEdsResource(args2, kNewEdsService2Name));
3341 // Populate new CDS resources.
3342 Cluster new_cluster1 = default_cluster_;
3343 new_cluster1.set_name(kNewCluster1Name);
3344 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3345 kNewEdsService1Name);
3346 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3347 Cluster new_cluster2 = default_cluster_;
3348 new_cluster2.set_name(kNewCluster2Name);
3349 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3350 kNewEdsService2Name);
3351 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3352 // Populating Route Configurations for LDS.
3353 RouteConfiguration new_route_config = default_route_config_;
3354 // First route will not match, since it's case-sensitive.
3355 // Second route will match with same path.
3356 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3357 route1->mutable_match()->set_path("/GrPc.TeStInG.EcHoTeSt1SErViCe/EcHo1");
3358 route1->mutable_route()->set_cluster(kNewCluster1Name);
3359 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3360 route2->mutable_match()->set_path("/GrPc.TeStInG.EcHoTeSt1SErViCe/EcHo1");
3361 route2->mutable_match()->mutable_case_sensitive()->set_value(false);
3362 route2->mutable_route()->set_cluster(kNewCluster2Name);
3363 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3364 default_route->mutable_match()->set_prefix("");
3365 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3366 SetRouteConfiguration(0, new_route_config);
3367 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
3368 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
3369 .set_rpc_service(SERVICE_ECHO1)
3370 .set_rpc_method(METHOD_ECHO1)
3371 .set_wait_for_ready(true));
3372 // Make sure RPCs all go to the correct backend.
3373 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
3374 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
3375 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
3376 EXPECT_EQ(0, backends_[1]->backend_service1()->request_count());
3377 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3378 EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
3379 }
3380
TEST_P(LdsRdsTest,XdsRoutingPrefixMatching)3381 TEST_P(LdsRdsTest, XdsRoutingPrefixMatching) {
3382 const char* kNewCluster1Name = "new_cluster_1";
3383 const char* kNewEdsService1Name = "new_eds_service_name_1";
3384 const char* kNewCluster2Name = "new_cluster_2";
3385 const char* kNewEdsService2Name = "new_eds_service_name_2";
3386 const size_t kNumEcho1Rpcs = 10;
3387 const size_t kNumEcho2Rpcs = 20;
3388 const size_t kNumEchoRpcs = 30;
3389 SetNextResolution({});
3390 SetNextResolutionForLbChannelAllBalancers();
3391 // Populate new EDS resources.
3392 AdsServiceImpl::EdsResourceArgs args({
3393 {"locality0", GetBackendPorts(0, 2)},
3394 });
3395 AdsServiceImpl::EdsResourceArgs args1({
3396 {"locality0", GetBackendPorts(2, 3)},
3397 });
3398 AdsServiceImpl::EdsResourceArgs args2({
3399 {"locality0", GetBackendPorts(3, 4)},
3400 });
3401 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3402 balancers_[0]->ads_service()->SetEdsResource(
3403 BuildEdsResource(args1, kNewEdsService1Name));
3404 balancers_[0]->ads_service()->SetEdsResource(
3405 BuildEdsResource(args2, kNewEdsService2Name));
3406 // Populate new CDS resources.
3407 Cluster new_cluster1 = default_cluster_;
3408 new_cluster1.set_name(kNewCluster1Name);
3409 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3410 kNewEdsService1Name);
3411 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3412 Cluster new_cluster2 = default_cluster_;
3413 new_cluster2.set_name(kNewCluster2Name);
3414 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3415 kNewEdsService2Name);
3416 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3417 // Populating Route Configurations for LDS.
3418 RouteConfiguration new_route_config = default_route_config_;
3419 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3420 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3421 route1->mutable_route()->set_cluster(kNewCluster1Name);
3422 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3423 route2->mutable_match()->set_prefix("/grpc.testing.EchoTest2Service/");
3424 route2->mutable_route()->set_cluster(kNewCluster2Name);
3425 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3426 default_route->mutable_match()->set_prefix("");
3427 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3428 SetRouteConfiguration(0, new_route_config);
3429 WaitForAllBackends(0, 2);
3430 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
3431 CheckRpcSendOk(
3432 kNumEcho1Rpcs,
3433 RpcOptions().set_rpc_service(SERVICE_ECHO1).set_wait_for_ready(true));
3434 CheckRpcSendOk(
3435 kNumEcho2Rpcs,
3436 RpcOptions().set_rpc_service(SERVICE_ECHO2).set_wait_for_ready(true));
3437 // Make sure RPCs all go to the correct backend.
3438 for (size_t i = 0; i < 2; ++i) {
3439 EXPECT_EQ(kNumEchoRpcs / 2,
3440 backends_[i]->backend_service()->request_count());
3441 EXPECT_EQ(0, backends_[i]->backend_service1()->request_count());
3442 EXPECT_EQ(0, backends_[i]->backend_service2()->request_count());
3443 }
3444 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3445 EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
3446 EXPECT_EQ(0, backends_[2]->backend_service2()->request_count());
3447 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
3448 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
3449 EXPECT_EQ(kNumEcho2Rpcs, backends_[3]->backend_service2()->request_count());
3450 }
3451
TEST_P(LdsRdsTest,XdsRoutingPrefixMatchingCaseInsensitive)3452 TEST_P(LdsRdsTest, XdsRoutingPrefixMatchingCaseInsensitive) {
3453 const char* kNewCluster1Name = "new_cluster_1";
3454 const char* kNewEdsService1Name = "new_eds_service_name_1";
3455 const char* kNewCluster2Name = "new_cluster_2";
3456 const char* kNewEdsService2Name = "new_eds_service_name_2";
3457 const size_t kNumEcho1Rpcs = 10;
3458 const size_t kNumEchoRpcs = 30;
3459 SetNextResolution({});
3460 SetNextResolutionForLbChannelAllBalancers();
3461 // Populate new EDS resources.
3462 AdsServiceImpl::EdsResourceArgs args({
3463 {"locality0", GetBackendPorts(0, 1)},
3464 });
3465 AdsServiceImpl::EdsResourceArgs args1({
3466 {"locality0", GetBackendPorts(1, 2)},
3467 });
3468 AdsServiceImpl::EdsResourceArgs args2({
3469 {"locality0", GetBackendPorts(2, 3)},
3470 });
3471 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3472 balancers_[0]->ads_service()->SetEdsResource(
3473 BuildEdsResource(args1, kNewEdsService1Name));
3474 balancers_[0]->ads_service()->SetEdsResource(
3475 BuildEdsResource(args2, kNewEdsService2Name));
3476 // Populate new CDS resources.
3477 Cluster new_cluster1 = default_cluster_;
3478 new_cluster1.set_name(kNewCluster1Name);
3479 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3480 kNewEdsService1Name);
3481 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3482 Cluster new_cluster2 = default_cluster_;
3483 new_cluster2.set_name(kNewCluster2Name);
3484 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3485 kNewEdsService2Name);
3486 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3487 // Populating Route Configurations for LDS.
3488 RouteConfiguration new_route_config = default_route_config_;
3489 // First route will not match, since it's case-sensitive.
3490 // Second route will match with same path.
3491 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3492 route1->mutable_match()->set_prefix("/GrPc.TeStInG.EcHoTeSt1SErViCe");
3493 route1->mutable_route()->set_cluster(kNewCluster1Name);
3494 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3495 route2->mutable_match()->set_prefix("/GrPc.TeStInG.EcHoTeSt1SErViCe");
3496 route2->mutable_match()->mutable_case_sensitive()->set_value(false);
3497 route2->mutable_route()->set_cluster(kNewCluster2Name);
3498 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3499 default_route->mutable_match()->set_prefix("");
3500 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3501 SetRouteConfiguration(0, new_route_config);
3502 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
3503 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
3504 .set_rpc_service(SERVICE_ECHO1)
3505 .set_rpc_method(METHOD_ECHO1)
3506 .set_wait_for_ready(true));
3507 // Make sure RPCs all go to the correct backend.
3508 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
3509 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
3510 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
3511 EXPECT_EQ(0, backends_[1]->backend_service1()->request_count());
3512 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3513 EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
3514 }
3515
TEST_P(LdsRdsTest,XdsRoutingPathRegexMatching)3516 TEST_P(LdsRdsTest, XdsRoutingPathRegexMatching) {
3517 const char* kNewCluster1Name = "new_cluster_1";
3518 const char* kNewEdsService1Name = "new_eds_service_name_1";
3519 const char* kNewCluster2Name = "new_cluster_2";
3520 const char* kNewEdsService2Name = "new_eds_service_name_2";
3521 const size_t kNumEcho1Rpcs = 10;
3522 const size_t kNumEcho2Rpcs = 20;
3523 const size_t kNumEchoRpcs = 30;
3524 SetNextResolution({});
3525 SetNextResolutionForLbChannelAllBalancers();
3526 // Populate new EDS resources.
3527 AdsServiceImpl::EdsResourceArgs args({
3528 {"locality0", GetBackendPorts(0, 2)},
3529 });
3530 AdsServiceImpl::EdsResourceArgs args1({
3531 {"locality0", GetBackendPorts(2, 3)},
3532 });
3533 AdsServiceImpl::EdsResourceArgs args2({
3534 {"locality0", GetBackendPorts(3, 4)},
3535 });
3536 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3537 balancers_[0]->ads_service()->SetEdsResource(
3538 BuildEdsResource(args1, kNewEdsService1Name));
3539 balancers_[0]->ads_service()->SetEdsResource(
3540 BuildEdsResource(args2, kNewEdsService2Name));
3541 // Populate new CDS resources.
3542 Cluster new_cluster1 = default_cluster_;
3543 new_cluster1.set_name(kNewCluster1Name);
3544 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3545 kNewEdsService1Name);
3546 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3547 Cluster new_cluster2 = default_cluster_;
3548 new_cluster2.set_name(kNewCluster2Name);
3549 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3550 kNewEdsService2Name);
3551 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3552 // Populating Route Configurations for LDS.
3553 RouteConfiguration new_route_config = default_route_config_;
3554 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3555 // Will match "/grpc.testing.EchoTest1Service/"
3556 route1->mutable_match()->mutable_safe_regex()->set_regex(".*1.*");
3557 route1->mutable_route()->set_cluster(kNewCluster1Name);
3558 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3559 // Will match "/grpc.testing.EchoTest2Service/"
3560 route2->mutable_match()->mutable_safe_regex()->set_regex(".*2.*");
3561 route2->mutable_route()->set_cluster(kNewCluster2Name);
3562 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3563 default_route->mutable_match()->set_prefix("");
3564 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3565 SetRouteConfiguration(0, new_route_config);
3566 WaitForAllBackends(0, 2);
3567 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
3568 CheckRpcSendOk(
3569 kNumEcho1Rpcs,
3570 RpcOptions().set_rpc_service(SERVICE_ECHO1).set_wait_for_ready(true));
3571 CheckRpcSendOk(
3572 kNumEcho2Rpcs,
3573 RpcOptions().set_rpc_service(SERVICE_ECHO2).set_wait_for_ready(true));
3574 // Make sure RPCs all go to the correct backend.
3575 for (size_t i = 0; i < 2; ++i) {
3576 EXPECT_EQ(kNumEchoRpcs / 2,
3577 backends_[i]->backend_service()->request_count());
3578 EXPECT_EQ(0, backends_[i]->backend_service1()->request_count());
3579 EXPECT_EQ(0, backends_[i]->backend_service2()->request_count());
3580 }
3581 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3582 EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
3583 EXPECT_EQ(0, backends_[2]->backend_service2()->request_count());
3584 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
3585 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
3586 EXPECT_EQ(kNumEcho2Rpcs, backends_[3]->backend_service2()->request_count());
3587 }
3588
TEST_P(LdsRdsTest,XdsRoutingPathRegexMatchingCaseInsensitive)3589 TEST_P(LdsRdsTest, XdsRoutingPathRegexMatchingCaseInsensitive) {
3590 const char* kNewCluster1Name = "new_cluster_1";
3591 const char* kNewEdsService1Name = "new_eds_service_name_1";
3592 const char* kNewCluster2Name = "new_cluster_2";
3593 const char* kNewEdsService2Name = "new_eds_service_name_2";
3594 const size_t kNumEcho1Rpcs = 10;
3595 const size_t kNumEchoRpcs = 30;
3596 SetNextResolution({});
3597 SetNextResolutionForLbChannelAllBalancers();
3598 // Populate new EDS resources.
3599 AdsServiceImpl::EdsResourceArgs args({
3600 {"locality0", GetBackendPorts(0, 1)},
3601 });
3602 AdsServiceImpl::EdsResourceArgs args1({
3603 {"locality0", GetBackendPorts(1, 2)},
3604 });
3605 AdsServiceImpl::EdsResourceArgs args2({
3606 {"locality0", GetBackendPorts(2, 3)},
3607 });
3608 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3609 balancers_[0]->ads_service()->SetEdsResource(
3610 BuildEdsResource(args1, kNewEdsService1Name));
3611 balancers_[0]->ads_service()->SetEdsResource(
3612 BuildEdsResource(args2, kNewEdsService2Name));
3613 // Populate new CDS resources.
3614 Cluster new_cluster1 = default_cluster_;
3615 new_cluster1.set_name(kNewCluster1Name);
3616 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3617 kNewEdsService1Name);
3618 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3619 Cluster new_cluster2 = default_cluster_;
3620 new_cluster2.set_name(kNewCluster2Name);
3621 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3622 kNewEdsService2Name);
3623 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3624 // Populating Route Configurations for LDS.
3625 RouteConfiguration new_route_config = default_route_config_;
3626 // First route will not match, since it's case-sensitive.
3627 // Second route will match with same path.
3628 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3629 route1->mutable_match()->mutable_safe_regex()->set_regex(
3630 ".*EcHoTeSt1SErViCe.*");
3631 route1->mutable_route()->set_cluster(kNewCluster1Name);
3632 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3633 route2->mutable_match()->mutable_safe_regex()->set_regex(
3634 ".*EcHoTeSt1SErViCe.*");
3635 route2->mutable_match()->mutable_case_sensitive()->set_value(false);
3636 route2->mutable_route()->set_cluster(kNewCluster2Name);
3637 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3638 default_route->mutable_match()->set_prefix("");
3639 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3640 SetRouteConfiguration(0, new_route_config);
3641 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
3642 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
3643 .set_rpc_service(SERVICE_ECHO1)
3644 .set_rpc_method(METHOD_ECHO1)
3645 .set_wait_for_ready(true));
3646 // Make sure RPCs all go to the correct backend.
3647 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
3648 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
3649 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
3650 EXPECT_EQ(0, backends_[1]->backend_service1()->request_count());
3651 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3652 EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
3653 }
3654
TEST_P(LdsRdsTest,XdsRoutingWeightedCluster)3655 TEST_P(LdsRdsTest, XdsRoutingWeightedCluster) {
3656 const char* kNewCluster1Name = "new_cluster_1";
3657 const char* kNewEdsService1Name = "new_eds_service_name_1";
3658 const char* kNewCluster2Name = "new_cluster_2";
3659 const char* kNewEdsService2Name = "new_eds_service_name_2";
3660 const size_t kNumEcho1Rpcs = 1000;
3661 const size_t kNumEchoRpcs = 10;
3662 const size_t kWeight75 = 75;
3663 const size_t kWeight25 = 25;
3664 SetNextResolution({});
3665 SetNextResolutionForLbChannelAllBalancers();
3666 // Populate new EDS resources.
3667 AdsServiceImpl::EdsResourceArgs args({
3668 {"locality0", GetBackendPorts(0, 1)},
3669 });
3670 AdsServiceImpl::EdsResourceArgs args1({
3671 {"locality0", GetBackendPorts(1, 2)},
3672 });
3673 AdsServiceImpl::EdsResourceArgs args2({
3674 {"locality0", GetBackendPorts(2, 3)},
3675 });
3676 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3677 balancers_[0]->ads_service()->SetEdsResource(
3678 BuildEdsResource(args1, kNewEdsService1Name));
3679 balancers_[0]->ads_service()->SetEdsResource(
3680 BuildEdsResource(args2, kNewEdsService2Name));
3681 // Populate new CDS resources.
3682 Cluster new_cluster1 = default_cluster_;
3683 new_cluster1.set_name(kNewCluster1Name);
3684 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3685 kNewEdsService1Name);
3686 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3687 Cluster new_cluster2 = default_cluster_;
3688 new_cluster2.set_name(kNewCluster2Name);
3689 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3690 kNewEdsService2Name);
3691 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3692 // Populating Route Configurations for LDS.
3693 RouteConfiguration new_route_config = default_route_config_;
3694 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3695 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3696 auto* weighted_cluster1 =
3697 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3698 weighted_cluster1->set_name(kNewCluster1Name);
3699 weighted_cluster1->mutable_weight()->set_value(kWeight75);
3700 auto* weighted_cluster2 =
3701 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3702 weighted_cluster2->set_name(kNewCluster2Name);
3703 weighted_cluster2->mutable_weight()->set_value(kWeight25);
3704 route1->mutable_route()
3705 ->mutable_weighted_clusters()
3706 ->mutable_total_weight()
3707 ->set_value(kWeight75 + kWeight25);
3708 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3709 default_route->mutable_match()->set_prefix("");
3710 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3711 SetRouteConfiguration(0, new_route_config);
3712 WaitForAllBackends(0, 1);
3713 WaitForAllBackends(1, 3, true, RpcOptions().set_rpc_service(SERVICE_ECHO1));
3714 CheckRpcSendOk(kNumEchoRpcs);
3715 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions().set_rpc_service(SERVICE_ECHO1));
3716 // Make sure RPCs all go to the correct backend.
3717 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
3718 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
3719 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
3720 const int weight_75_request_count =
3721 backends_[1]->backend_service1()->request_count();
3722 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3723 const int weight_25_request_count =
3724 backends_[2]->backend_service1()->request_count();
3725 const double kErrorTolerance = 0.2;
3726 EXPECT_THAT(weight_75_request_count,
3727 ::testing::AllOf(::testing::Ge(kNumEcho1Rpcs * kWeight75 / 100 *
3728 (1 - kErrorTolerance)),
3729 ::testing::Le(kNumEcho1Rpcs * kWeight75 / 100 *
3730 (1 + kErrorTolerance))));
3731 // TODO: (@donnadionne) Reduce tolerance: increased the tolerance to keep the
3732 // test from flaking while debugging potential root cause.
3733 const double kErrorToleranceSmallLoad = 0.3;
3734 gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
3735 weight_75_request_count, weight_25_request_count);
3736 EXPECT_THAT(weight_25_request_count,
3737 ::testing::AllOf(::testing::Ge(kNumEcho1Rpcs * kWeight25 / 100 *
3738 (1 - kErrorToleranceSmallLoad)),
3739 ::testing::Le(kNumEcho1Rpcs * kWeight25 / 100 *
3740 (1 + kErrorToleranceSmallLoad))));
3741 }
3742
TEST_P(LdsRdsTest,RouteActionWeightedTargetDefaultRoute)3743 TEST_P(LdsRdsTest, RouteActionWeightedTargetDefaultRoute) {
3744 const char* kNewCluster1Name = "new_cluster_1";
3745 const char* kNewEdsService1Name = "new_eds_service_name_1";
3746 const char* kNewCluster2Name = "new_cluster_2";
3747 const char* kNewEdsService2Name = "new_eds_service_name_2";
3748 const size_t kNumEchoRpcs = 1000;
3749 const size_t kWeight75 = 75;
3750 const size_t kWeight25 = 25;
3751 SetNextResolution({});
3752 SetNextResolutionForLbChannelAllBalancers();
3753 // Populate new EDS resources.
3754 AdsServiceImpl::EdsResourceArgs args({
3755 {"locality0", GetBackendPorts(0, 1)},
3756 });
3757 AdsServiceImpl::EdsResourceArgs args1({
3758 {"locality0", GetBackendPorts(1, 2)},
3759 });
3760 AdsServiceImpl::EdsResourceArgs args2({
3761 {"locality0", GetBackendPorts(2, 3)},
3762 });
3763 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3764 balancers_[0]->ads_service()->SetEdsResource(
3765 BuildEdsResource(args1, kNewEdsService1Name));
3766 balancers_[0]->ads_service()->SetEdsResource(
3767 BuildEdsResource(args2, kNewEdsService2Name));
3768 // Populate new CDS resources.
3769 Cluster new_cluster1 = default_cluster_;
3770 new_cluster1.set_name(kNewCluster1Name);
3771 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3772 kNewEdsService1Name);
3773 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3774 Cluster new_cluster2 = default_cluster_;
3775 new_cluster2.set_name(kNewCluster2Name);
3776 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3777 kNewEdsService2Name);
3778 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3779 // Populating Route Configurations for LDS.
3780 RouteConfiguration new_route_config = default_route_config_;
3781 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3782 route1->mutable_match()->set_prefix("");
3783 auto* weighted_cluster1 =
3784 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3785 weighted_cluster1->set_name(kNewCluster1Name);
3786 weighted_cluster1->mutable_weight()->set_value(kWeight75);
3787 auto* weighted_cluster2 =
3788 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3789 weighted_cluster2->set_name(kNewCluster2Name);
3790 weighted_cluster2->mutable_weight()->set_value(kWeight25);
3791 route1->mutable_route()
3792 ->mutable_weighted_clusters()
3793 ->mutable_total_weight()
3794 ->set_value(kWeight75 + kWeight25);
3795 SetRouteConfiguration(0, new_route_config);
3796 WaitForAllBackends(1, 3);
3797 CheckRpcSendOk(kNumEchoRpcs);
3798 // Make sure RPCs all go to the correct backend.
3799 EXPECT_EQ(0, backends_[0]->backend_service()->request_count());
3800 const int weight_75_request_count =
3801 backends_[1]->backend_service()->request_count();
3802 const int weight_25_request_count =
3803 backends_[2]->backend_service()->request_count();
3804 const double kErrorTolerance = 0.2;
3805 EXPECT_THAT(weight_75_request_count,
3806 ::testing::AllOf(::testing::Ge(kNumEchoRpcs * kWeight75 / 100 *
3807 (1 - kErrorTolerance)),
3808 ::testing::Le(kNumEchoRpcs * kWeight75 / 100 *
3809 (1 + kErrorTolerance))));
3810 // TODO: (@donnadionne) Reduce tolerance: increased the tolerance to keep the
3811 // test from flaking while debugging potential root cause.
3812 const double kErrorToleranceSmallLoad = 0.3;
3813 gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
3814 weight_75_request_count, weight_25_request_count);
3815 EXPECT_THAT(weight_25_request_count,
3816 ::testing::AllOf(::testing::Ge(kNumEchoRpcs * kWeight25 / 100 *
3817 (1 - kErrorToleranceSmallLoad)),
3818 ::testing::Le(kNumEchoRpcs * kWeight25 / 100 *
3819 (1 + kErrorToleranceSmallLoad))));
3820 }
3821
TEST_P(LdsRdsTest,XdsRoutingWeightedClusterUpdateWeights)3822 TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateWeights) {
3823 const char* kNewCluster1Name = "new_cluster_1";
3824 const char* kNewEdsService1Name = "new_eds_service_name_1";
3825 const char* kNewCluster2Name = "new_cluster_2";
3826 const char* kNewEdsService2Name = "new_eds_service_name_2";
3827 const char* kNewCluster3Name = "new_cluster_3";
3828 const char* kNewEdsService3Name = "new_eds_service_name_3";
3829 const size_t kNumEcho1Rpcs = 1000;
3830 const size_t kNumEchoRpcs = 10;
3831 const size_t kWeight75 = 75;
3832 const size_t kWeight25 = 25;
3833 const size_t kWeight50 = 50;
3834 SetNextResolution({});
3835 SetNextResolutionForLbChannelAllBalancers();
3836 // Populate new EDS resources.
3837 AdsServiceImpl::EdsResourceArgs args({
3838 {"locality0", GetBackendPorts(0, 1)},
3839 });
3840 AdsServiceImpl::EdsResourceArgs args1({
3841 {"locality0", GetBackendPorts(1, 2)},
3842 });
3843 AdsServiceImpl::EdsResourceArgs args2({
3844 {"locality0", GetBackendPorts(2, 3)},
3845 });
3846 AdsServiceImpl::EdsResourceArgs args3({
3847 {"locality0", GetBackendPorts(3, 4)},
3848 });
3849 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3850 balancers_[0]->ads_service()->SetEdsResource(
3851 BuildEdsResource(args1, kNewEdsService1Name));
3852 balancers_[0]->ads_service()->SetEdsResource(
3853 BuildEdsResource(args2, kNewEdsService2Name));
3854 balancers_[0]->ads_service()->SetEdsResource(
3855 BuildEdsResource(args3, kNewEdsService3Name));
3856 // Populate new CDS resources.
3857 Cluster new_cluster1 = default_cluster_;
3858 new_cluster1.set_name(kNewCluster1Name);
3859 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3860 kNewEdsService1Name);
3861 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3862 Cluster new_cluster2 = default_cluster_;
3863 new_cluster2.set_name(kNewCluster2Name);
3864 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3865 kNewEdsService2Name);
3866 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3867 Cluster new_cluster3 = default_cluster_;
3868 new_cluster3.set_name(kNewCluster3Name);
3869 new_cluster3.mutable_eds_cluster_config()->set_service_name(
3870 kNewEdsService3Name);
3871 balancers_[0]->ads_service()->SetCdsResource(new_cluster3);
3872 // Populating Route Configurations.
3873 RouteConfiguration new_route_config = default_route_config_;
3874 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3875 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3876 auto* weighted_cluster1 =
3877 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3878 weighted_cluster1->set_name(kNewCluster1Name);
3879 weighted_cluster1->mutable_weight()->set_value(kWeight75);
3880 auto* weighted_cluster2 =
3881 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3882 weighted_cluster2->set_name(kNewCluster2Name);
3883 weighted_cluster2->mutable_weight()->set_value(kWeight25);
3884 route1->mutable_route()
3885 ->mutable_weighted_clusters()
3886 ->mutable_total_weight()
3887 ->set_value(kWeight75 + kWeight25);
3888 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3889 default_route->mutable_match()->set_prefix("");
3890 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3891 SetRouteConfiguration(0, new_route_config);
3892 WaitForAllBackends(0, 1);
3893 WaitForAllBackends(1, 3, true, RpcOptions().set_rpc_service(SERVICE_ECHO1));
3894 CheckRpcSendOk(kNumEchoRpcs);
3895 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions().set_rpc_service(SERVICE_ECHO1));
3896 // Make sure RPCs all go to the correct backend.
3897 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
3898 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
3899 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
3900 const int weight_75_request_count =
3901 backends_[1]->backend_service1()->request_count();
3902 EXPECT_EQ(0, backends_[1]->backend_service2()->request_count());
3903 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3904 const int weight_25_request_count =
3905 backends_[2]->backend_service1()->request_count();
3906 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
3907 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
3908 const double kErrorTolerance = 0.2;
3909 EXPECT_THAT(weight_75_request_count,
3910 ::testing::AllOf(::testing::Ge(kNumEcho1Rpcs * kWeight75 / 100 *
3911 (1 - kErrorTolerance)),
3912 ::testing::Le(kNumEcho1Rpcs * kWeight75 / 100 *
3913 (1 + kErrorTolerance))));
3914 // TODO: (@donnadionne) Reduce tolerance: increased the tolerance to keep the
3915 // test from flaking while debugging potential root cause.
3916 const double kErrorToleranceSmallLoad = 0.3;
3917 gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
3918 weight_75_request_count, weight_25_request_count);
3919 EXPECT_THAT(weight_25_request_count,
3920 ::testing::AllOf(::testing::Ge(kNumEcho1Rpcs * kWeight25 / 100 *
3921 (1 - kErrorToleranceSmallLoad)),
3922 ::testing::Le(kNumEcho1Rpcs * kWeight25 / 100 *
3923 (1 + kErrorToleranceSmallLoad))));
3924 // Change Route Configurations: same clusters different weights.
3925 weighted_cluster1->mutable_weight()->set_value(kWeight50);
3926 weighted_cluster2->mutable_weight()->set_value(kWeight50);
3927 // Change default route to a new cluster to help to identify when new polices
3928 // are seen by the client.
3929 default_route->mutable_route()->set_cluster(kNewCluster3Name);
3930 SetRouteConfiguration(0, new_route_config);
3931 ResetBackendCounters();
3932 WaitForAllBackends(3, 4);
3933 CheckRpcSendOk(kNumEchoRpcs);
3934 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions().set_rpc_service(SERVICE_ECHO1));
3935 // Make sure RPCs all go to the correct backend.
3936 EXPECT_EQ(0, backends_[0]->backend_service()->request_count());
3937 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
3938 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
3939 const int weight_50_request_count_1 =
3940 backends_[1]->backend_service1()->request_count();
3941 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3942 const int weight_50_request_count_2 =
3943 backends_[2]->backend_service1()->request_count();
3944 EXPECT_EQ(kNumEchoRpcs, backends_[3]->backend_service()->request_count());
3945 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
3946 EXPECT_THAT(weight_50_request_count_1,
3947 ::testing::AllOf(::testing::Ge(kNumEcho1Rpcs * kWeight50 / 100 *
3948 (1 - kErrorTolerance)),
3949 ::testing::Le(kNumEcho1Rpcs * kWeight50 / 100 *
3950 (1 + kErrorTolerance))));
3951 EXPECT_THAT(weight_50_request_count_2,
3952 ::testing::AllOf(::testing::Ge(kNumEcho1Rpcs * kWeight50 / 100 *
3953 (1 - kErrorTolerance)),
3954 ::testing::Le(kNumEcho1Rpcs * kWeight50 / 100 *
3955 (1 + kErrorTolerance))));
3956 }
3957
TEST_P(LdsRdsTest,XdsRoutingWeightedClusterUpdateClusters)3958 TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateClusters) {
3959 const char* kNewCluster1Name = "new_cluster_1";
3960 const char* kNewEdsService1Name = "new_eds_service_name_1";
3961 const char* kNewCluster2Name = "new_cluster_2";
3962 const char* kNewEdsService2Name = "new_eds_service_name_2";
3963 const char* kNewCluster3Name = "new_cluster_3";
3964 const char* kNewEdsService3Name = "new_eds_service_name_3";
3965 const size_t kNumEcho1Rpcs = 1000;
3966 const size_t kNumEchoRpcs = 10;
3967 const size_t kWeight75 = 75;
3968 const size_t kWeight25 = 25;
3969 const size_t kWeight50 = 50;
3970 SetNextResolution({});
3971 SetNextResolutionForLbChannelAllBalancers();
3972 // Populate new EDS resources.
3973 AdsServiceImpl::EdsResourceArgs args({
3974 {"locality0", GetBackendPorts(0, 1)},
3975 });
3976 AdsServiceImpl::EdsResourceArgs args1({
3977 {"locality0", GetBackendPorts(1, 2)},
3978 });
3979 AdsServiceImpl::EdsResourceArgs args2({
3980 {"locality0", GetBackendPorts(2, 3)},
3981 });
3982 AdsServiceImpl::EdsResourceArgs args3({
3983 {"locality0", GetBackendPorts(3, 4)},
3984 });
3985 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3986 balancers_[0]->ads_service()->SetEdsResource(
3987 BuildEdsResource(args1, kNewEdsService1Name));
3988 balancers_[0]->ads_service()->SetEdsResource(
3989 BuildEdsResource(args2, kNewEdsService2Name));
3990 balancers_[0]->ads_service()->SetEdsResource(
3991 BuildEdsResource(args3, kNewEdsService3Name));
3992 // Populate new CDS resources.
3993 Cluster new_cluster1 = default_cluster_;
3994 new_cluster1.set_name(kNewCluster1Name);
3995 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3996 kNewEdsService1Name);
3997 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3998 Cluster new_cluster2 = default_cluster_;
3999 new_cluster2.set_name(kNewCluster2Name);
4000 new_cluster2.mutable_eds_cluster_config()->set_service_name(
4001 kNewEdsService2Name);
4002 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4003 Cluster new_cluster3 = default_cluster_;
4004 new_cluster3.set_name(kNewCluster3Name);
4005 new_cluster3.mutable_eds_cluster_config()->set_service_name(
4006 kNewEdsService3Name);
4007 balancers_[0]->ads_service()->SetCdsResource(new_cluster3);
4008 // Populating Route Configurations.
4009 RouteConfiguration new_route_config = default_route_config_;
4010 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4011 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4012 auto* weighted_cluster1 =
4013 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4014 weighted_cluster1->set_name(kNewCluster1Name);
4015 weighted_cluster1->mutable_weight()->set_value(kWeight75);
4016 auto* weighted_cluster2 =
4017 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4018 weighted_cluster2->set_name(kDefaultClusterName);
4019 weighted_cluster2->mutable_weight()->set_value(kWeight25);
4020 route1->mutable_route()
4021 ->mutable_weighted_clusters()
4022 ->mutable_total_weight()
4023 ->set_value(kWeight75 + kWeight25);
4024 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
4025 default_route->mutable_match()->set_prefix("");
4026 default_route->mutable_route()->set_cluster(kDefaultClusterName);
4027 SetRouteConfiguration(0, new_route_config);
4028 WaitForAllBackends(0, 1);
4029 WaitForAllBackends(1, 2, true, RpcOptions().set_rpc_service(SERVICE_ECHO1));
4030 CheckRpcSendOk(kNumEchoRpcs);
4031 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions().set_rpc_service(SERVICE_ECHO1));
4032 // Make sure RPCs all go to the correct backend.
4033 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4034 int weight_25_request_count =
4035 backends_[0]->backend_service1()->request_count();
4036 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4037 int weight_75_request_count =
4038 backends_[1]->backend_service1()->request_count();
4039 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4040 EXPECT_EQ(0, backends_[2]->backend_service1()->request_count());
4041 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
4042 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
4043 const double kErrorTolerance = 0.2;
4044 EXPECT_THAT(weight_75_request_count,
4045 ::testing::AllOf(::testing::Ge(kNumEcho1Rpcs * kWeight75 / 100 *
4046 (1 - kErrorTolerance)),
4047 ::testing::Le(kNumEcho1Rpcs * kWeight75 / 100 *
4048 (1 + kErrorTolerance))));
4049 // TODO: (@donnadionne) Reduce tolerance: increased the tolerance to keep the
4050 // test from flaking while debugging potential root cause.
4051 const double kErrorToleranceSmallLoad = 0.3;
4052 gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
4053 weight_75_request_count, weight_25_request_count);
4054 EXPECT_THAT(weight_25_request_count,
4055 ::testing::AllOf(::testing::Ge(kNumEcho1Rpcs * kWeight25 / 100 *
4056 (1 - kErrorToleranceSmallLoad)),
4057 ::testing::Le(kNumEcho1Rpcs * kWeight25 / 100 *
4058 (1 + kErrorToleranceSmallLoad))));
4059 // Change Route Configurations: new set of clusters with different weights.
4060 weighted_cluster1->mutable_weight()->set_value(kWeight50);
4061 weighted_cluster2->set_name(kNewCluster2Name);
4062 weighted_cluster2->mutable_weight()->set_value(kWeight50);
4063 SetRouteConfiguration(0, new_route_config);
4064 ResetBackendCounters();
4065 WaitForAllBackends(2, 3, true, RpcOptions().set_rpc_service(SERVICE_ECHO1));
4066 CheckRpcSendOk(kNumEchoRpcs);
4067 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions().set_rpc_service(SERVICE_ECHO1));
4068 // Make sure RPCs all go to the correct backend.
4069 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4070 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
4071 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4072 const int weight_50_request_count_1 =
4073 backends_[1]->backend_service1()->request_count();
4074 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4075 const int weight_50_request_count_2 =
4076 backends_[2]->backend_service1()->request_count();
4077 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
4078 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
4079 EXPECT_THAT(weight_50_request_count_1,
4080 ::testing::AllOf(::testing::Ge(kNumEcho1Rpcs * kWeight50 / 100 *
4081 (1 - kErrorTolerance)),
4082 ::testing::Le(kNumEcho1Rpcs * kWeight50 / 100 *
4083 (1 + kErrorTolerance))));
4084 EXPECT_THAT(weight_50_request_count_2,
4085 ::testing::AllOf(::testing::Ge(kNumEcho1Rpcs * kWeight50 / 100 *
4086 (1 - kErrorTolerance)),
4087 ::testing::Le(kNumEcho1Rpcs * kWeight50 / 100 *
4088 (1 + kErrorTolerance))));
4089 // Change Route Configurations.
4090 weighted_cluster1->mutable_weight()->set_value(kWeight75);
4091 weighted_cluster2->set_name(kNewCluster3Name);
4092 weighted_cluster2->mutable_weight()->set_value(kWeight25);
4093 SetRouteConfiguration(0, new_route_config);
4094 ResetBackendCounters();
4095 WaitForAllBackends(3, 4, true, RpcOptions().set_rpc_service(SERVICE_ECHO1));
4096 CheckRpcSendOk(kNumEchoRpcs);
4097 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions().set_rpc_service(SERVICE_ECHO1));
4098 // Make sure RPCs all go to the correct backend.
4099 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4100 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
4101 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4102 weight_75_request_count = backends_[1]->backend_service1()->request_count();
4103 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4104 EXPECT_EQ(0, backends_[2]->backend_service1()->request_count());
4105 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
4106 weight_25_request_count = backends_[3]->backend_service1()->request_count();
4107 EXPECT_THAT(weight_75_request_count,
4108 ::testing::AllOf(::testing::Ge(kNumEcho1Rpcs * kWeight75 / 100 *
4109 (1 - kErrorTolerance)),
4110 ::testing::Le(kNumEcho1Rpcs * kWeight75 / 100 *
4111 (1 + kErrorTolerance))));
4112 // TODO: (@donnadionne) Reduce tolerance: increased the tolerance to keep the
4113 // test from flaking while debugging potential root cause.
4114 gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
4115 weight_75_request_count, weight_25_request_count);
4116 EXPECT_THAT(weight_25_request_count,
4117 ::testing::AllOf(::testing::Ge(kNumEcho1Rpcs * kWeight25 / 100 *
4118 (1 - kErrorToleranceSmallLoad)),
4119 ::testing::Le(kNumEcho1Rpcs * kWeight25 / 100 *
4120 (1 + kErrorToleranceSmallLoad))));
4121 }
4122
TEST_P(LdsRdsTest,XdsRoutingClusterUpdateClusters)4123 TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClusters) {
4124 const char* kNewClusterName = "new_cluster";
4125 const char* kNewEdsServiceName = "new_eds_service_name";
4126 const size_t kNumEchoRpcs = 5;
4127 SetNextResolution({});
4128 SetNextResolutionForLbChannelAllBalancers();
4129 // Populate new EDS resources.
4130 AdsServiceImpl::EdsResourceArgs args({
4131 {"locality0", GetBackendPorts(0, 1)},
4132 });
4133 AdsServiceImpl::EdsResourceArgs args1({
4134 {"locality0", GetBackendPorts(1, 2)},
4135 });
4136 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4137 balancers_[0]->ads_service()->SetEdsResource(
4138 BuildEdsResource(args1, kNewEdsServiceName));
4139 // Populate new CDS resources.
4140 Cluster new_cluster = default_cluster_;
4141 new_cluster.set_name(kNewClusterName);
4142 new_cluster.mutable_eds_cluster_config()->set_service_name(
4143 kNewEdsServiceName);
4144 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
4145 // Send Route Configuration.
4146 RouteConfiguration new_route_config = default_route_config_;
4147 SetRouteConfiguration(0, new_route_config);
4148 WaitForAllBackends(0, 1);
4149 CheckRpcSendOk(kNumEchoRpcs);
4150 // Make sure RPCs all go to the correct backend.
4151 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4152 // Change Route Configurations: new default cluster.
4153 auto* default_route =
4154 new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4155 default_route->mutable_route()->set_cluster(kNewClusterName);
4156 SetRouteConfiguration(0, new_route_config);
4157 WaitForAllBackends(1, 2);
4158 CheckRpcSendOk(kNumEchoRpcs);
4159 // Make sure RPCs all go to the correct backend.
4160 EXPECT_EQ(kNumEchoRpcs, backends_[1]->backend_service()->request_count());
4161 }
4162
TEST_P(LdsRdsTest,XdsRoutingClusterUpdateClustersWithPickingDelays)4163 TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) {
4164 const char* kNewClusterName = "new_cluster";
4165 const char* kNewEdsServiceName = "new_eds_service_name";
4166 SetNextResolution({});
4167 SetNextResolutionForLbChannelAllBalancers();
4168 // Populate new EDS resources.
4169 AdsServiceImpl::EdsResourceArgs args({
4170 {"locality0", GetBackendPorts(0, 1)},
4171 });
4172 AdsServiceImpl::EdsResourceArgs args1({
4173 {"locality0", GetBackendPorts(1, 2)},
4174 });
4175 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4176 balancers_[0]->ads_service()->SetEdsResource(
4177 BuildEdsResource(args1, kNewEdsServiceName));
4178 // Populate new CDS resources.
4179 Cluster new_cluster = default_cluster_;
4180 new_cluster.set_name(kNewClusterName);
4181 new_cluster.mutable_eds_cluster_config()->set_service_name(
4182 kNewEdsServiceName);
4183 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
4184 // Bring down the current backend: 0, this will delay route picking time,
4185 // resulting in un-committed RPCs.
4186 ShutdownBackend(0);
4187 // Send a RouteConfiguration with a default route that points to
4188 // backend 0.
4189 RouteConfiguration new_route_config = default_route_config_;
4190 SetRouteConfiguration(0, new_route_config);
4191 // Send exactly one RPC with no deadline and with wait_for_ready=true.
4192 // This RPC will not complete until after backend 0 is started.
4193 std::thread sending_rpc([this]() {
4194 CheckRpcSendOk(1, RpcOptions().set_wait_for_ready(true).set_timeout_ms(0));
4195 });
4196 // Send a non-wait_for_ready RPC which should fail, this will tell us
4197 // that the client has received the update and attempted to connect.
4198 const Status status = SendRpc(RpcOptions().set_timeout_ms(0));
4199 EXPECT_FALSE(status.ok());
4200 // Send a update RouteConfiguration to use backend 1.
4201 auto* default_route =
4202 new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4203 default_route->mutable_route()->set_cluster(kNewClusterName);
4204 SetRouteConfiguration(0, new_route_config);
4205 // Wait for RPCs to go to the new backend: 1, this ensures that the client has
4206 // processed the update.
4207 WaitForAllBackends(1, 2, false, RpcOptions(), true);
4208 // Bring up the previous backend: 0, this will allow the delayed RPC to
4209 // finally call on_call_committed upon completion.
4210 StartBackend(0);
4211 sending_rpc.join();
4212 // Make sure RPCs go to the correct backend:
4213 EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
4214 EXPECT_EQ(1, backends_[1]->backend_service()->request_count());
4215 }
4216
TEST_P(LdsRdsTest,XdsRoutingApplyXdsTimeout)4217 TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
4218 // TODO(https://github.com/grpc/grpc/issues/24549): TSAN won't work here.
4219 if (BuiltUnderAsan() || BuiltUnderTsan()) return;
4220
4221 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT", "true");
4222 const int64_t kTimeoutNano = 500000000;
4223 const int64_t kTimeoutGrpcTimeoutHeaderMaxSecond = 1;
4224 const int64_t kTimeoutMaxStreamDurationSecond = 2;
4225 const int64_t kTimeoutHttpMaxStreamDurationSecond = 3;
4226 const int64_t kTimeoutApplicationSecond = 4;
4227 const char* kNewCluster1Name = "new_cluster_1";
4228 const char* kNewEdsService1Name = "new_eds_service_name_1";
4229 const char* kNewCluster2Name = "new_cluster_2";
4230 const char* kNewEdsService2Name = "new_eds_service_name_2";
4231 const char* kNewCluster3Name = "new_cluster_3";
4232 const char* kNewEdsService3Name = "new_eds_service_name_3";
4233 SetNextResolution({});
4234 SetNextResolutionForLbChannelAllBalancers();
4235 // Populate new EDS resources.
4236 AdsServiceImpl::EdsResourceArgs args({
4237 {"locality0", {g_port_saver->GetPort()}},
4238 });
4239 AdsServiceImpl::EdsResourceArgs args1({
4240 {"locality0", {g_port_saver->GetPort()}},
4241 });
4242 AdsServiceImpl::EdsResourceArgs args2({
4243 {"locality0", {g_port_saver->GetPort()}},
4244 });
4245 AdsServiceImpl::EdsResourceArgs args3({
4246 {"locality0", {g_port_saver->GetPort()}},
4247 });
4248 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4249 balancers_[0]->ads_service()->SetEdsResource(
4250 BuildEdsResource(args1, kNewEdsService1Name));
4251 balancers_[0]->ads_service()->SetEdsResource(
4252 BuildEdsResource(args2, kNewEdsService2Name));
4253 balancers_[0]->ads_service()->SetEdsResource(
4254 BuildEdsResource(args3, kNewEdsService3Name));
4255 // Populate new CDS resources.
4256 Cluster new_cluster1 = default_cluster_;
4257 new_cluster1.set_name(kNewCluster1Name);
4258 new_cluster1.mutable_eds_cluster_config()->set_service_name(
4259 kNewEdsService1Name);
4260 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4261 Cluster new_cluster2 = default_cluster_;
4262 new_cluster2.set_name(kNewCluster2Name);
4263 new_cluster2.mutable_eds_cluster_config()->set_service_name(
4264 kNewEdsService2Name);
4265 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4266 Cluster new_cluster3 = default_cluster_;
4267 new_cluster3.set_name(kNewCluster3Name);
4268 new_cluster3.mutable_eds_cluster_config()->set_service_name(
4269 kNewEdsService3Name);
4270 balancers_[0]->ads_service()->SetCdsResource(new_cluster3);
4271 // Construct listener.
4272 auto listener = default_listener_;
4273 HttpConnectionManager http_connection_manager;
4274 // Set up HTTP max_stream_duration of 3.5 seconds
4275 auto* duration =
4276 http_connection_manager.mutable_common_http_protocol_options()
4277 ->mutable_max_stream_duration();
4278 duration->set_seconds(kTimeoutHttpMaxStreamDurationSecond);
4279 duration->set_nanos(kTimeoutNano);
4280 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
4281 http_connection_manager);
4282 // Construct route config.
4283 RouteConfiguration new_route_config = default_route_config_;
4284 // route 1: Set max_stream_duration of 2.5 seconds, Set
4285 // grpc_timeout_header_max of 1.5
4286 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4287 route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1");
4288 route1->mutable_route()->set_cluster(kNewCluster1Name);
4289 auto* max_stream_duration =
4290 route1->mutable_route()->mutable_max_stream_duration();
4291 duration = max_stream_duration->mutable_max_stream_duration();
4292 duration->set_seconds(kTimeoutMaxStreamDurationSecond);
4293 duration->set_nanos(kTimeoutNano);
4294 duration = max_stream_duration->mutable_grpc_timeout_header_max();
4295 duration->set_seconds(kTimeoutGrpcTimeoutHeaderMaxSecond);
4296 duration->set_nanos(kTimeoutNano);
4297 // route 2: Set max_stream_duration of 2.5 seconds
4298 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
4299 route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
4300 route2->mutable_route()->set_cluster(kNewCluster2Name);
4301 max_stream_duration = route2->mutable_route()->mutable_max_stream_duration();
4302 duration = max_stream_duration->mutable_max_stream_duration();
4303 duration->set_seconds(kTimeoutMaxStreamDurationSecond);
4304 duration->set_nanos(kTimeoutNano);
4305 // route 3: No timeout values in route configuration
4306 auto* route3 = new_route_config.mutable_virtual_hosts(0)->add_routes();
4307 route3->mutable_match()->set_path("/grpc.testing.EchoTestService/Echo");
4308 route3->mutable_route()->set_cluster(kNewCluster3Name);
4309 // Set listener and route config.
4310 SetListenerAndRouteConfiguration(0, std::move(listener), new_route_config);
4311 // Test grpc_timeout_header_max of 1.5 seconds applied
4312 auto t0 = system_clock::now();
4313 CheckRpcSendFailure(1,
4314 RpcOptions()
4315 .set_rpc_service(SERVICE_ECHO1)
4316 .set_rpc_method(METHOD_ECHO1)
4317 .set_wait_for_ready(true)
4318 .set_timeout_ms(kTimeoutApplicationSecond * 1000),
4319 StatusCode::DEADLINE_EXCEEDED);
4320 auto ellapsed_nano_seconds =
4321 std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
4322 t0);
4323 EXPECT_GT(ellapsed_nano_seconds.count(),
4324 kTimeoutGrpcTimeoutHeaderMaxSecond * 1000000000 + kTimeoutNano);
4325 EXPECT_LT(ellapsed_nano_seconds.count(),
4326 kTimeoutMaxStreamDurationSecond * 1000000000);
4327 // Test max_stream_duration of 2.5 seconds applied
4328 t0 = system_clock::now();
4329 CheckRpcSendFailure(1,
4330 RpcOptions()
4331 .set_rpc_service(SERVICE_ECHO2)
4332 .set_rpc_method(METHOD_ECHO2)
4333 .set_wait_for_ready(true)
4334 .set_timeout_ms(kTimeoutApplicationSecond * 1000),
4335 StatusCode::DEADLINE_EXCEEDED);
4336 ellapsed_nano_seconds = std::chrono::duration_cast<std::chrono::nanoseconds>(
4337 system_clock::now() - t0);
4338 EXPECT_GT(ellapsed_nano_seconds.count(),
4339 kTimeoutMaxStreamDurationSecond * 1000000000 + kTimeoutNano);
4340 EXPECT_LT(ellapsed_nano_seconds.count(),
4341 kTimeoutHttpMaxStreamDurationSecond * 1000000000);
4342 // Test http_stream_duration of 3.5 seconds applied
4343 t0 = system_clock::now();
4344 CheckRpcSendFailure(1,
4345 RpcOptions().set_wait_for_ready(true).set_timeout_ms(
4346 kTimeoutApplicationSecond * 1000),
4347 StatusCode::DEADLINE_EXCEEDED);
4348 ellapsed_nano_seconds = std::chrono::duration_cast<std::chrono::nanoseconds>(
4349 system_clock::now() - t0);
4350 EXPECT_GT(ellapsed_nano_seconds.count(),
4351 kTimeoutHttpMaxStreamDurationSecond * 1000000000 + kTimeoutNano);
4352 EXPECT_LT(ellapsed_nano_seconds.count(),
4353 kTimeoutApplicationSecond * 1000000000);
4354 gpr_unsetenv("GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT");
4355 }
4356
TEST_P(LdsRdsTest,XdsRoutingXdsTimeoutDisabled)4357 TEST_P(LdsRdsTest, XdsRoutingXdsTimeoutDisabled) {
4358 const int64_t kTimeoutMillis = 500;
4359 const int64_t kTimeoutNano = kTimeoutMillis * 1000000;
4360 const int64_t kTimeoutGrpcTimeoutHeaderMaxSecond = 1;
4361 const int64_t kTimeoutApplicationSecond = 4;
4362 SetNextResolution({});
4363 SetNextResolutionForLbChannelAllBalancers();
4364 // Populate new EDS resources.
4365 AdsServiceImpl::EdsResourceArgs args({
4366 {"locality0", {g_port_saver->GetPort()}},
4367 });
4368 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4369 RouteConfiguration new_route_config = default_route_config_;
4370 // route 1: Set grpc_timeout_header_max of 1.5
4371 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4372 auto* max_stream_duration =
4373 route1->mutable_route()->mutable_max_stream_duration();
4374 auto* duration = max_stream_duration->mutable_grpc_timeout_header_max();
4375 duration->set_seconds(kTimeoutGrpcTimeoutHeaderMaxSecond);
4376 duration->set_nanos(kTimeoutNano);
4377 SetRouteConfiguration(0, new_route_config);
4378 // Test grpc_timeout_header_max of 1.5 seconds is not applied
4379 gpr_timespec t0 = gpr_now(GPR_CLOCK_MONOTONIC);
4380 gpr_timespec est_timeout_time = gpr_time_add(
4381 t0, gpr_time_from_millis(
4382 kTimeoutGrpcTimeoutHeaderMaxSecond * 1000 + kTimeoutMillis,
4383 GPR_TIMESPAN));
4384 CheckRpcSendFailure(1,
4385 RpcOptions()
4386 .set_rpc_service(SERVICE_ECHO1)
4387 .set_rpc_method(METHOD_ECHO1)
4388 .set_wait_for_ready(true)
4389 .set_timeout_ms(kTimeoutApplicationSecond * 1000),
4390 StatusCode::DEADLINE_EXCEEDED);
4391 gpr_timespec timeout_time = gpr_now(GPR_CLOCK_MONOTONIC);
4392 EXPECT_GT(gpr_time_cmp(timeout_time, est_timeout_time), 0);
4393 }
4394
TEST_P(LdsRdsTest,XdsRoutingHttpTimeoutDisabled)4395 TEST_P(LdsRdsTest, XdsRoutingHttpTimeoutDisabled) {
4396 const int64_t kTimeoutMillis = 500;
4397 const int64_t kTimeoutNano = kTimeoutMillis * 1000000;
4398 const int64_t kTimeoutHttpMaxStreamDurationSecond = 3;
4399 const int64_t kTimeoutApplicationSecond = 4;
4400 SetNextResolution({});
4401 SetNextResolutionForLbChannelAllBalancers();
4402 // Populate new EDS resources.
4403 AdsServiceImpl::EdsResourceArgs args({
4404 {"locality0", {g_port_saver->GetPort()}},
4405 });
4406 // Construct listener.
4407 auto listener = default_listener_;
4408 HttpConnectionManager http_connection_manager;
4409 // Set up HTTP max_stream_duration of 3.5 seconds
4410 auto* duration =
4411 http_connection_manager.mutable_common_http_protocol_options()
4412 ->mutable_max_stream_duration();
4413 duration->set_seconds(kTimeoutHttpMaxStreamDurationSecond);
4414 duration->set_nanos(kTimeoutNano);
4415 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
4416 http_connection_manager);
4417 SetListenerAndRouteConfiguration(0, std::move(listener),
4418 default_route_config_);
4419 // Test http_stream_duration of 3.5 seconds is not applied
4420 auto t0 = gpr_now(GPR_CLOCK_MONOTONIC);
4421 auto est_timeout_time = gpr_time_add(
4422 t0, gpr_time_from_millis(
4423 kTimeoutHttpMaxStreamDurationSecond * 1000 + kTimeoutMillis,
4424 GPR_TIMESPAN));
4425 CheckRpcSendFailure(1,
4426 RpcOptions().set_wait_for_ready(true).set_timeout_ms(
4427 kTimeoutApplicationSecond * 1000),
4428 StatusCode::DEADLINE_EXCEEDED);
4429 auto timeout_time = gpr_now(GPR_CLOCK_MONOTONIC);
4430 EXPECT_GT(gpr_time_cmp(timeout_time, est_timeout_time), 0);
4431 }
4432
TEST_P(LdsRdsTest,XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0)4433 TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
4434 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT", "true");
4435 const int64_t kTimeoutNano = 500000000;
4436 const int64_t kTimeoutMaxStreamDurationSecond = 2;
4437 const int64_t kTimeoutHttpMaxStreamDurationSecond = 3;
4438 const int64_t kTimeoutApplicationSecond = 4;
4439 const char* kNewCluster1Name = "new_cluster_1";
4440 const char* kNewEdsService1Name = "new_eds_service_name_1";
4441 const char* kNewCluster2Name = "new_cluster_2";
4442 const char* kNewEdsService2Name = "new_eds_service_name_2";
4443 SetNextResolution({});
4444 SetNextResolutionForLbChannelAllBalancers();
4445 // Populate new EDS resources.
4446 AdsServiceImpl::EdsResourceArgs args({
4447 {"locality0", {g_port_saver->GetPort()}},
4448 });
4449 AdsServiceImpl::EdsResourceArgs args1({
4450 {"locality0", {g_port_saver->GetPort()}},
4451 });
4452 AdsServiceImpl::EdsResourceArgs args2({
4453 {"locality0", {g_port_saver->GetPort()}},
4454 });
4455 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4456 balancers_[0]->ads_service()->SetEdsResource(
4457 BuildEdsResource(args1, kNewEdsService1Name));
4458 balancers_[0]->ads_service()->SetEdsResource(
4459 BuildEdsResource(args2, kNewEdsService2Name));
4460 // Populate new CDS resources.
4461 Cluster new_cluster1 = default_cluster_;
4462 new_cluster1.set_name(kNewCluster1Name);
4463 new_cluster1.mutable_eds_cluster_config()->set_service_name(
4464 kNewEdsService1Name);
4465 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4466 Cluster new_cluster2 = default_cluster_;
4467 new_cluster2.set_name(kNewCluster2Name);
4468 new_cluster2.mutable_eds_cluster_config()->set_service_name(
4469 kNewEdsService2Name);
4470 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4471 // Construct listener.
4472 auto listener = default_listener_;
4473 HttpConnectionManager http_connection_manager;
4474 // Set up HTTP max_stream_duration of 3.5 seconds
4475 auto* duration =
4476 http_connection_manager.mutable_common_http_protocol_options()
4477 ->mutable_max_stream_duration();
4478 duration->set_seconds(kTimeoutHttpMaxStreamDurationSecond);
4479 duration->set_nanos(kTimeoutNano);
4480 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
4481 http_connection_manager);
4482 // Construct route config.
4483 RouteConfiguration new_route_config = default_route_config_;
4484 // route 1: Set max_stream_duration of 2.5 seconds, Set
4485 // grpc_timeout_header_max of 0
4486 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4487 route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1");
4488 route1->mutable_route()->set_cluster(kNewCluster1Name);
4489 auto* max_stream_duration =
4490 route1->mutable_route()->mutable_max_stream_duration();
4491 duration = max_stream_duration->mutable_max_stream_duration();
4492 duration->set_seconds(kTimeoutMaxStreamDurationSecond);
4493 duration->set_nanos(kTimeoutNano);
4494 duration = max_stream_duration->mutable_grpc_timeout_header_max();
4495 duration->set_seconds(0);
4496 duration->set_nanos(0);
4497 // route 2: Set max_stream_duration to 0
4498 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
4499 route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
4500 route2->mutable_route()->set_cluster(kNewCluster2Name);
4501 max_stream_duration = route2->mutable_route()->mutable_max_stream_duration();
4502 duration = max_stream_duration->mutable_max_stream_duration();
4503 duration->set_seconds(0);
4504 duration->set_nanos(0);
4505 // Set listener and route config.
4506 SetListenerAndRouteConfiguration(0, std::move(listener), new_route_config);
4507 // Test application timeout is applied for route 1
4508 auto t0 = system_clock::now();
4509 CheckRpcSendFailure(1,
4510 RpcOptions()
4511 .set_rpc_service(SERVICE_ECHO1)
4512 .set_rpc_method(METHOD_ECHO1)
4513 .set_wait_for_ready(true)
4514 .set_timeout_ms(kTimeoutApplicationSecond * 1000),
4515 StatusCode::DEADLINE_EXCEEDED);
4516 auto ellapsed_nano_seconds =
4517 std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
4518 t0);
4519 EXPECT_GT(ellapsed_nano_seconds.count(),
4520 kTimeoutApplicationSecond * 1000000000);
4521 // Test application timeout is applied for route 2
4522 t0 = system_clock::now();
4523 CheckRpcSendFailure(1,
4524 RpcOptions()
4525 .set_rpc_service(SERVICE_ECHO2)
4526 .set_rpc_method(METHOD_ECHO2)
4527 .set_wait_for_ready(true)
4528 .set_timeout_ms(kTimeoutApplicationSecond * 1000),
4529 StatusCode::DEADLINE_EXCEEDED);
4530 ellapsed_nano_seconds = std::chrono::duration_cast<std::chrono::nanoseconds>(
4531 system_clock::now() - t0);
4532 EXPECT_GT(ellapsed_nano_seconds.count(),
4533 kTimeoutApplicationSecond * 1000000000);
4534 gpr_unsetenv("GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT");
4535 }
4536
TEST_P(LdsRdsTest,XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0)4537 TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0) {
4538 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT", "true");
4539 const int64_t kTimeoutApplicationSecond = 4;
4540 SetNextResolution({});
4541 SetNextResolutionForLbChannelAllBalancers();
4542 // Populate new EDS resources.
4543 AdsServiceImpl::EdsResourceArgs args({
4544 {"locality0", {g_port_saver->GetPort()}},
4545 });
4546 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4547 HttpConnectionManager http_connection_manager;
4548 // Set up HTTP max_stream_duration to be explicit 0
4549 auto* duration =
4550 http_connection_manager.mutable_common_http_protocol_options()
4551 ->mutable_max_stream_duration();
4552 duration->set_seconds(0);
4553 duration->set_nanos(0);
4554 auto listener = default_listener_;
4555 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
4556 http_connection_manager);
4557 // Set listener and route config.
4558 SetListenerAndRouteConfiguration(0, std::move(listener),
4559 default_route_config_);
4560 // Test application timeout is applied for route 1
4561 auto t0 = system_clock::now();
4562 CheckRpcSendFailure(1,
4563 RpcOptions().set_wait_for_ready(true).set_timeout_ms(
4564 kTimeoutApplicationSecond * 1000),
4565 StatusCode::DEADLINE_EXCEEDED);
4566 auto ellapsed_nano_seconds =
4567 std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
4568 t0);
4569 EXPECT_GT(ellapsed_nano_seconds.count(),
4570 kTimeoutApplicationSecond * 1000000000);
4571 gpr_unsetenv("GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT");
4572 }
4573
4574 // Test to ensure application-specified deadline won't be affected when
4575 // the xDS config does not specify a timeout.
TEST_P(LdsRdsTest,XdsRoutingWithOnlyApplicationTimeout)4576 TEST_P(LdsRdsTest, XdsRoutingWithOnlyApplicationTimeout) {
4577 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT", "true");
4578 const int64_t kTimeoutApplicationSecond = 4;
4579 SetNextResolution({});
4580 SetNextResolutionForLbChannelAllBalancers();
4581 // Populate new EDS resources.
4582 AdsServiceImpl::EdsResourceArgs args({
4583 {"locality0", {g_port_saver->GetPort()}},
4584 });
4585 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4586 auto t0 = system_clock::now();
4587 CheckRpcSendFailure(1,
4588 RpcOptions().set_wait_for_ready(true).set_timeout_ms(
4589 kTimeoutApplicationSecond * 1000),
4590 StatusCode::DEADLINE_EXCEEDED);
4591 auto ellapsed_nano_seconds =
4592 std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
4593 t0);
4594 EXPECT_GT(ellapsed_nano_seconds.count(),
4595 kTimeoutApplicationSecond * 1000000000);
4596 gpr_unsetenv("GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT");
4597 }
4598
TEST_P(LdsRdsTest,XdsRoutingHeadersMatching)4599 TEST_P(LdsRdsTest, XdsRoutingHeadersMatching) {
4600 const char* kNewClusterName = "new_cluster";
4601 const char* kNewEdsServiceName = "new_eds_service_name";
4602 const size_t kNumEcho1Rpcs = 100;
4603 const size_t kNumEchoRpcs = 5;
4604 SetNextResolution({});
4605 SetNextResolutionForLbChannelAllBalancers();
4606 // Populate new EDS resources.
4607 AdsServiceImpl::EdsResourceArgs args({
4608 {"locality0", GetBackendPorts(0, 1)},
4609 });
4610 AdsServiceImpl::EdsResourceArgs args1({
4611 {"locality0", GetBackendPorts(1, 2)},
4612 });
4613 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4614 balancers_[0]->ads_service()->SetEdsResource(
4615 BuildEdsResource(args1, kNewEdsServiceName));
4616 // Populate new CDS resources.
4617 Cluster new_cluster = default_cluster_;
4618 new_cluster.set_name(kNewClusterName);
4619 new_cluster.mutable_eds_cluster_config()->set_service_name(
4620 kNewEdsServiceName);
4621 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
4622 // Populating Route Configurations for LDS.
4623 RouteConfiguration route_config = default_route_config_;
4624 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4625 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4626 auto* header_matcher1 = route1->mutable_match()->add_headers();
4627 header_matcher1->set_name("header1");
4628 header_matcher1->set_exact_match("POST,PUT,GET");
4629 auto* header_matcher2 = route1->mutable_match()->add_headers();
4630 header_matcher2->set_name("header2");
4631 header_matcher2->mutable_safe_regex_match()->set_regex("[a-z]*");
4632 auto* header_matcher3 = route1->mutable_match()->add_headers();
4633 header_matcher3->set_name("header3");
4634 header_matcher3->mutable_range_match()->set_start(1);
4635 header_matcher3->mutable_range_match()->set_end(1000);
4636 auto* header_matcher4 = route1->mutable_match()->add_headers();
4637 header_matcher4->set_name("header4");
4638 header_matcher4->set_present_match(false);
4639 auto* header_matcher5 = route1->mutable_match()->add_headers();
4640 header_matcher5->set_name("header5");
4641 header_matcher5->set_prefix_match("/grpc");
4642 auto* header_matcher6 = route1->mutable_match()->add_headers();
4643 header_matcher6->set_name("header6");
4644 header_matcher6->set_suffix_match(".cc");
4645 header_matcher6->set_invert_match(true);
4646 route1->mutable_route()->set_cluster(kNewClusterName);
4647 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
4648 default_route->mutable_match()->set_prefix("");
4649 default_route->mutable_route()->set_cluster(kDefaultClusterName);
4650 SetRouteConfiguration(0, route_config);
4651 std::vector<std::pair<std::string, std::string>> metadata = {
4652 {"header1", "POST"}, {"header2", "blah"},
4653 {"header3", "1"}, {"header5", "/grpc.testing.EchoTest1Service/"},
4654 {"header1", "PUT"}, {"header6", "grpc.java"},
4655 {"header1", "GET"},
4656 };
4657 const auto header_match_rpc_options = RpcOptions()
4658 .set_rpc_service(SERVICE_ECHO1)
4659 .set_rpc_method(METHOD_ECHO1)
4660 .set_metadata(std::move(metadata));
4661 // Make sure all backends are up.
4662 WaitForAllBackends(0, 1);
4663 WaitForAllBackends(1, 2, true, header_match_rpc_options);
4664 // Send RPCs.
4665 CheckRpcSendOk(kNumEchoRpcs);
4666 CheckRpcSendOk(kNumEcho1Rpcs, header_match_rpc_options);
4667 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4668 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
4669 EXPECT_EQ(0, backends_[0]->backend_service2()->request_count());
4670 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4671 EXPECT_EQ(kNumEcho1Rpcs, backends_[1]->backend_service1()->request_count());
4672 EXPECT_EQ(0, backends_[1]->backend_service2()->request_count());
4673 const auto& response_state = RouteConfigurationResponseState(0);
4674 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
4675 }
4676
TEST_P(LdsRdsTest,XdsRoutingHeadersMatchingSpecialHeaderContentType)4677 TEST_P(LdsRdsTest, XdsRoutingHeadersMatchingSpecialHeaderContentType) {
4678 const char* kNewClusterName = "new_cluster";
4679 const char* kNewEdsServiceName = "new_eds_service_name";
4680 const size_t kNumEchoRpcs = 100;
4681 SetNextResolution({});
4682 SetNextResolutionForLbChannelAllBalancers();
4683 // Populate new EDS resources.
4684 AdsServiceImpl::EdsResourceArgs args({
4685 {"locality0", GetBackendPorts(0, 1)},
4686 });
4687 AdsServiceImpl::EdsResourceArgs args1({
4688 {"locality0", GetBackendPorts(1, 2)},
4689 });
4690 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4691 balancers_[0]->ads_service()->SetEdsResource(
4692 BuildEdsResource(args1, kNewEdsServiceName));
4693 // Populate new CDS resources.
4694 Cluster new_cluster = default_cluster_;
4695 new_cluster.set_name(kNewClusterName);
4696 new_cluster.mutable_eds_cluster_config()->set_service_name(
4697 kNewEdsServiceName);
4698 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
4699 // Populating Route Configurations for LDS.
4700 RouteConfiguration route_config = default_route_config_;
4701 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4702 route1->mutable_match()->set_prefix("");
4703 auto* header_matcher1 = route1->mutable_match()->add_headers();
4704 header_matcher1->set_name("content-type");
4705 header_matcher1->set_exact_match("notapplication/grpc");
4706 route1->mutable_route()->set_cluster(kNewClusterName);
4707 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
4708 default_route->mutable_match()->set_prefix("");
4709 auto* header_matcher2 = default_route->mutable_match()->add_headers();
4710 header_matcher2->set_name("content-type");
4711 header_matcher2->set_exact_match("application/grpc");
4712 default_route->mutable_route()->set_cluster(kDefaultClusterName);
4713 SetRouteConfiguration(0, route_config);
4714 // Make sure the backend is up.
4715 WaitForAllBackends(0, 1);
4716 // Send RPCs.
4717 CheckRpcSendOk(kNumEchoRpcs);
4718 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4719 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4720 const auto& response_state = RouteConfigurationResponseState(0);
4721 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
4722 }
4723
TEST_P(LdsRdsTest,XdsRoutingHeadersMatchingSpecialCasesToIgnore)4724 TEST_P(LdsRdsTest, XdsRoutingHeadersMatchingSpecialCasesToIgnore) {
4725 const char* kNewCluster1Name = "new_cluster_1";
4726 const char* kNewEdsService1Name = "new_eds_service_name_1";
4727 const char* kNewCluster2Name = "new_cluster_2";
4728 const char* kNewEdsService2Name = "new_eds_service_name_2";
4729 const size_t kNumEchoRpcs = 100;
4730 SetNextResolution({});
4731 SetNextResolutionForLbChannelAllBalancers();
4732 // Populate new EDS resources.
4733 AdsServiceImpl::EdsResourceArgs args({
4734 {"locality0", GetBackendPorts(0, 1)},
4735 });
4736 AdsServiceImpl::EdsResourceArgs args1({
4737 {"locality0", GetBackendPorts(1, 2)},
4738 });
4739 AdsServiceImpl::EdsResourceArgs args2({
4740 {"locality0", GetBackendPorts(2, 3)},
4741 });
4742 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4743 balancers_[0]->ads_service()->SetEdsResource(
4744 BuildEdsResource(args1, kNewEdsService1Name));
4745 balancers_[0]->ads_service()->SetEdsResource(
4746 BuildEdsResource(args2, kNewEdsService2Name));
4747 // Populate new CDS resources.
4748 Cluster new_cluster1 = default_cluster_;
4749 new_cluster1.set_name(kNewCluster1Name);
4750 new_cluster1.mutable_eds_cluster_config()->set_service_name(
4751 kNewEdsService1Name);
4752 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4753 Cluster new_cluster2 = default_cluster_;
4754 new_cluster2.set_name(kNewCluster2Name);
4755 new_cluster2.mutable_eds_cluster_config()->set_service_name(
4756 kNewEdsService2Name);
4757 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4758 // Populating Route Configurations for LDS.
4759 RouteConfiguration route_config = default_route_config_;
4760 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4761 route1->mutable_match()->set_prefix("");
4762 auto* header_matcher1 = route1->mutable_match()->add_headers();
4763 header_matcher1->set_name("grpc-foo-bin");
4764 header_matcher1->set_present_match(true);
4765 route1->mutable_route()->set_cluster(kNewCluster1Name);
4766 auto route2 = route_config.mutable_virtual_hosts(0)->add_routes();
4767 route2->mutable_match()->set_prefix("");
4768 auto* header_matcher2 = route2->mutable_match()->add_headers();
4769 header_matcher2->set_name("grpc-previous-rpc-attempts");
4770 header_matcher2->set_present_match(true);
4771 route2->mutable_route()->set_cluster(kNewCluster2Name);
4772 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
4773 default_route->mutable_match()->set_prefix("");
4774 default_route->mutable_route()->set_cluster(kDefaultClusterName);
4775 SetRouteConfiguration(0, route_config);
4776 // Send headers which will mismatch each route
4777 std::vector<std::pair<std::string, std::string>> metadata = {
4778 {"grpc-foo-bin", "grpc-foo-bin"},
4779 {"grpc-previous-rpc-attempts", "grpc-previous-rpc-attempts"},
4780 };
4781 WaitForAllBackends(0, 1);
4782 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_metadata(metadata));
4783 // Verify that only the default backend got RPCs since all previous routes
4784 // were mismatched.
4785 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4786 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4787 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4788 const auto& response_state = RouteConfigurationResponseState(0);
4789 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
4790 }
4791
TEST_P(LdsRdsTest,XdsRoutingRuntimeFractionMatching)4792 TEST_P(LdsRdsTest, XdsRoutingRuntimeFractionMatching) {
4793 const char* kNewClusterName = "new_cluster";
4794 const char* kNewEdsServiceName = "new_eds_service_name";
4795 const size_t kNumRpcs = 1000;
4796 SetNextResolution({});
4797 SetNextResolutionForLbChannelAllBalancers();
4798 // Populate new EDS resources.
4799 AdsServiceImpl::EdsResourceArgs args({
4800 {"locality0", GetBackendPorts(0, 1)},
4801 });
4802 AdsServiceImpl::EdsResourceArgs args1({
4803 {"locality0", GetBackendPorts(1, 2)},
4804 });
4805 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4806 balancers_[0]->ads_service()->SetEdsResource(
4807 BuildEdsResource(args1, kNewEdsServiceName));
4808 // Populate new CDS resources.
4809 Cluster new_cluster = default_cluster_;
4810 new_cluster.set_name(kNewClusterName);
4811 new_cluster.mutable_eds_cluster_config()->set_service_name(
4812 kNewEdsServiceName);
4813 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
4814 // Populating Route Configurations for LDS.
4815 RouteConfiguration route_config = default_route_config_;
4816 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4817 route1->mutable_match()
4818 ->mutable_runtime_fraction()
4819 ->mutable_default_value()
4820 ->set_numerator(25);
4821 route1->mutable_route()->set_cluster(kNewClusterName);
4822 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
4823 default_route->mutable_match()->set_prefix("");
4824 default_route->mutable_route()->set_cluster(kDefaultClusterName);
4825 SetRouteConfiguration(0, route_config);
4826 WaitForAllBackends(0, 2);
4827 CheckRpcSendOk(kNumRpcs);
4828 const int default_backend_count =
4829 backends_[0]->backend_service()->request_count();
4830 const int matched_backend_count =
4831 backends_[1]->backend_service()->request_count();
4832 const double kErrorTolerance = 0.2;
4833 EXPECT_THAT(default_backend_count,
4834 ::testing::AllOf(
4835 ::testing::Ge(kNumRpcs * 75 / 100 * (1 - kErrorTolerance)),
4836 ::testing::Le(kNumRpcs * 75 / 100 * (1 + kErrorTolerance))));
4837 EXPECT_THAT(matched_backend_count,
4838 ::testing::AllOf(
4839 ::testing::Ge(kNumRpcs * 25 / 100 * (1 - kErrorTolerance)),
4840 ::testing::Le(kNumRpcs * 25 / 100 * (1 + kErrorTolerance))));
4841 const auto& response_state = RouteConfigurationResponseState(0);
4842 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
4843 }
4844
TEST_P(LdsRdsTest,XdsRoutingHeadersMatchingUnmatchCases)4845 TEST_P(LdsRdsTest, XdsRoutingHeadersMatchingUnmatchCases) {
4846 const char* kNewCluster1Name = "new_cluster_1";
4847 const char* kNewEdsService1Name = "new_eds_service_name_1";
4848 const char* kNewCluster2Name = "new_cluster_2";
4849 const char* kNewEdsService2Name = "new_eds_service_name_2";
4850 const char* kNewCluster3Name = "new_cluster_3";
4851 const char* kNewEdsService3Name = "new_eds_service_name_3";
4852 const size_t kNumEcho1Rpcs = 100;
4853 const size_t kNumEchoRpcs = 5;
4854 SetNextResolution({});
4855 SetNextResolutionForLbChannelAllBalancers();
4856 // Populate new EDS resources.
4857 AdsServiceImpl::EdsResourceArgs args({
4858 {"locality0", GetBackendPorts(0, 1)},
4859 });
4860 AdsServiceImpl::EdsResourceArgs args1({
4861 {"locality0", GetBackendPorts(1, 2)},
4862 });
4863 AdsServiceImpl::EdsResourceArgs args2({
4864 {"locality0", GetBackendPorts(2, 3)},
4865 });
4866 AdsServiceImpl::EdsResourceArgs args3({
4867 {"locality0", GetBackendPorts(3, 4)},
4868 });
4869 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4870 balancers_[0]->ads_service()->SetEdsResource(
4871 BuildEdsResource(args1, kNewEdsService1Name));
4872 balancers_[0]->ads_service()->SetEdsResource(
4873 BuildEdsResource(args2, kNewEdsService2Name));
4874 balancers_[0]->ads_service()->SetEdsResource(
4875 BuildEdsResource(args3, kNewEdsService3Name));
4876 // Populate new CDS resources.
4877 Cluster new_cluster1 = default_cluster_;
4878 new_cluster1.set_name(kNewCluster1Name);
4879 new_cluster1.mutable_eds_cluster_config()->set_service_name(
4880 kNewEdsService1Name);
4881 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4882 Cluster new_cluster2 = default_cluster_;
4883 new_cluster2.set_name(kNewCluster2Name);
4884 new_cluster2.mutable_eds_cluster_config()->set_service_name(
4885 kNewEdsService2Name);
4886 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4887 Cluster new_cluster3 = default_cluster_;
4888 new_cluster3.set_name(kNewCluster3Name);
4889 new_cluster3.mutable_eds_cluster_config()->set_service_name(
4890 kNewEdsService3Name);
4891 balancers_[0]->ads_service()->SetCdsResource(new_cluster3);
4892 // Populating Route Configurations for LDS.
4893 RouteConfiguration route_config = default_route_config_;
4894 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4895 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4896 auto* header_matcher1 = route1->mutable_match()->add_headers();
4897 header_matcher1->set_name("header1");
4898 header_matcher1->set_exact_match("POST");
4899 route1->mutable_route()->set_cluster(kNewCluster1Name);
4900 auto route2 = route_config.mutable_virtual_hosts(0)->add_routes();
4901 route2->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4902 auto* header_matcher2 = route2->mutable_match()->add_headers();
4903 header_matcher2->set_name("header2");
4904 header_matcher2->mutable_range_match()->set_start(1);
4905 header_matcher2->mutable_range_match()->set_end(1000);
4906 route2->mutable_route()->set_cluster(kNewCluster2Name);
4907 auto route3 = route_config.mutable_virtual_hosts(0)->add_routes();
4908 route3->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4909 auto* header_matcher3 = route3->mutable_match()->add_headers();
4910 header_matcher3->set_name("header3");
4911 header_matcher3->mutable_safe_regex_match()->set_regex("[a-z]*");
4912 route3->mutable_route()->set_cluster(kNewCluster3Name);
4913 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
4914 default_route->mutable_match()->set_prefix("");
4915 default_route->mutable_route()->set_cluster(kDefaultClusterName);
4916 SetRouteConfiguration(0, route_config);
4917 // Send headers which will mismatch each route
4918 std::vector<std::pair<std::string, std::string>> metadata = {
4919 {"header1", "POST"},
4920 {"header2", "1000"},
4921 {"header3", "123"},
4922 {"header1", "GET"},
4923 };
4924 WaitForAllBackends(0, 1);
4925 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_metadata(metadata));
4926 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
4927 .set_rpc_service(SERVICE_ECHO1)
4928 .set_rpc_method(METHOD_ECHO1)
4929 .set_metadata(metadata));
4930 // Verify that only the default backend got RPCs since all previous routes
4931 // were mismatched.
4932 for (size_t i = 1; i < 4; ++i) {
4933 EXPECT_EQ(0, backends_[i]->backend_service()->request_count());
4934 EXPECT_EQ(0, backends_[i]->backend_service1()->request_count());
4935 EXPECT_EQ(0, backends_[i]->backend_service2()->request_count());
4936 }
4937 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4938 EXPECT_EQ(kNumEcho1Rpcs, backends_[0]->backend_service1()->request_count());
4939 EXPECT_EQ(0, backends_[0]->backend_service2()->request_count());
4940 const auto& response_state = RouteConfigurationResponseState(0);
4941 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
4942 }
4943
TEST_P(LdsRdsTest,XdsRoutingChangeRoutesWithoutChangingClusters)4944 TEST_P(LdsRdsTest, XdsRoutingChangeRoutesWithoutChangingClusters) {
4945 const char* kNewClusterName = "new_cluster";
4946 const char* kNewEdsServiceName = "new_eds_service_name";
4947 SetNextResolution({});
4948 SetNextResolutionForLbChannelAllBalancers();
4949 // Populate new EDS resources.
4950 AdsServiceImpl::EdsResourceArgs args({
4951 {"locality0", GetBackendPorts(0, 1)},
4952 });
4953 AdsServiceImpl::EdsResourceArgs args1({
4954 {"locality0", GetBackendPorts(1, 2)},
4955 });
4956 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4957 balancers_[0]->ads_service()->SetEdsResource(
4958 BuildEdsResource(args1, kNewEdsServiceName));
4959 // Populate new CDS resources.
4960 Cluster new_cluster = default_cluster_;
4961 new_cluster.set_name(kNewClusterName);
4962 new_cluster.mutable_eds_cluster_config()->set_service_name(
4963 kNewEdsServiceName);
4964 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
4965 // Populating Route Configurations for LDS.
4966 RouteConfiguration route_config = default_route_config_;
4967 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4968 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4969 route1->mutable_route()->set_cluster(kNewClusterName);
4970 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
4971 default_route->mutable_match()->set_prefix("");
4972 default_route->mutable_route()->set_cluster(kDefaultClusterName);
4973 SetRouteConfiguration(0, route_config);
4974 // Make sure all backends are up and that requests for each RPC
4975 // service go to the right backends.
4976 WaitForAllBackends(0, 1, false);
4977 WaitForAllBackends(1, 2, false, RpcOptions().set_rpc_service(SERVICE_ECHO1));
4978 WaitForAllBackends(0, 1, false, RpcOptions().set_rpc_service(SERVICE_ECHO2));
4979 // Requests for services Echo and Echo2 should have gone to backend 0.
4980 EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
4981 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
4982 EXPECT_EQ(1, backends_[0]->backend_service2()->request_count());
4983 // Requests for service Echo1 should have gone to backend 1.
4984 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4985 EXPECT_EQ(1, backends_[1]->backend_service1()->request_count());
4986 EXPECT_EQ(0, backends_[1]->backend_service2()->request_count());
4987 // Now send an update that changes the first route to match a
4988 // different RPC service, and wait for the client to make the change.
4989 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest2Service/");
4990 SetRouteConfiguration(0, route_config);
4991 WaitForAllBackends(1, 2, true, RpcOptions().set_rpc_service(SERVICE_ECHO2));
4992 // Now repeat the earlier test, making sure all traffic goes to the
4993 // right place.
4994 WaitForAllBackends(0, 1, false);
4995 WaitForAllBackends(0, 1, false, RpcOptions().set_rpc_service(SERVICE_ECHO1));
4996 WaitForAllBackends(1, 2, false, RpcOptions().set_rpc_service(SERVICE_ECHO2));
4997 // Requests for services Echo and Echo1 should have gone to backend 0.
4998 EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
4999 EXPECT_EQ(1, backends_[0]->backend_service1()->request_count());
5000 EXPECT_EQ(0, backends_[0]->backend_service2()->request_count());
5001 // Requests for service Echo2 should have gone to backend 1.
5002 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5003 EXPECT_EQ(0, backends_[1]->backend_service1()->request_count());
5004 EXPECT_EQ(1, backends_[1]->backend_service2()->request_count());
5005 }
5006
5007 using CdsTest = BasicTest;
5008
5009 // Tests that CDS client should send an ACK upon correct CDS response.
TEST_P(CdsTest,Vanilla)5010 TEST_P(CdsTest, Vanilla) {
5011 SetNextResolution({});
5012 SetNextResolutionForLbChannelAllBalancers();
5013 (void)SendRpc();
5014 EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state,
5015 AdsServiceImpl::ResponseState::ACKED);
5016 }
5017
5018 // Tests that CDS client should send a NACK if the cluster type in CDS response
5019 // is other than EDS.
TEST_P(CdsTest,WrongClusterType)5020 TEST_P(CdsTest, WrongClusterType) {
5021 auto cluster = default_cluster_;
5022 cluster.set_type(Cluster::STATIC);
5023 balancers_[0]->ads_service()->SetCdsResource(cluster);
5024 SetNextResolution({});
5025 SetNextResolutionForLbChannelAllBalancers();
5026 CheckRpcSendFailure();
5027 const auto& response_state =
5028 balancers_[0]->ads_service()->cds_response_state();
5029 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5030 EXPECT_EQ(response_state.error_message, "DiscoveryType is not EDS.");
5031 }
5032
5033 // Tests that CDS client should send a NACK if the eds_config in CDS response is
5034 // other than ADS.
TEST_P(CdsTest,WrongEdsConfig)5035 TEST_P(CdsTest, WrongEdsConfig) {
5036 auto cluster = default_cluster_;
5037 cluster.mutable_eds_cluster_config()->mutable_eds_config()->mutable_self();
5038 balancers_[0]->ads_service()->SetCdsResource(cluster);
5039 SetNextResolution({});
5040 SetNextResolutionForLbChannelAllBalancers();
5041 CheckRpcSendFailure();
5042 const auto& response_state =
5043 balancers_[0]->ads_service()->cds_response_state();
5044 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5045 EXPECT_EQ(response_state.error_message, "EDS ConfigSource is not ADS.");
5046 }
5047
5048 // Tests that CDS client should send a NACK if the lb_policy in CDS response is
5049 // other than ROUND_ROBIN.
TEST_P(CdsTest,WrongLbPolicy)5050 TEST_P(CdsTest, WrongLbPolicy) {
5051 auto cluster = default_cluster_;
5052 cluster.set_lb_policy(Cluster::LEAST_REQUEST);
5053 balancers_[0]->ads_service()->SetCdsResource(cluster);
5054 SetNextResolution({});
5055 SetNextResolutionForLbChannelAllBalancers();
5056 CheckRpcSendFailure();
5057 const auto& response_state =
5058 balancers_[0]->ads_service()->cds_response_state();
5059 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5060 EXPECT_EQ(response_state.error_message, "LB policy is not ROUND_ROBIN.");
5061 }
5062
5063 // Tests that CDS client should send a NACK if the lrs_server in CDS response is
5064 // other than SELF.
TEST_P(CdsTest,WrongLrsServer)5065 TEST_P(CdsTest, WrongLrsServer) {
5066 auto cluster = default_cluster_;
5067 cluster.mutable_lrs_server()->mutable_ads();
5068 balancers_[0]->ads_service()->SetCdsResource(cluster);
5069 SetNextResolution({});
5070 SetNextResolutionForLbChannelAllBalancers();
5071 CheckRpcSendFailure();
5072 const auto& response_state =
5073 balancers_[0]->ads_service()->cds_response_state();
5074 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5075 EXPECT_EQ(response_state.error_message, "LRS ConfigSource is not self.");
5076 }
5077
5078 using EdsTest = BasicTest;
5079
5080 // Tests that EDS client should send a NACK if the EDS update contains
5081 // sparse priorities.
TEST_P(EdsTest,NacksSparsePriorityList)5082 TEST_P(EdsTest, NacksSparsePriorityList) {
5083 SetNextResolution({});
5084 SetNextResolutionForLbChannelAllBalancers();
5085 AdsServiceImpl::EdsResourceArgs args({
5086 {"locality0", GetBackendPorts(), kDefaultLocalityWeight, 1},
5087 });
5088 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5089 CheckRpcSendFailure();
5090 const auto& response_state =
5091 balancers_[0]->ads_service()->eds_response_state();
5092 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5093 EXPECT_EQ(response_state.error_message,
5094 "EDS update includes sparse priority list");
5095 }
5096
5097 // In most of our tests, we use different names for different resource
5098 // types, to make sure that there are no cut-and-paste errors in the code
5099 // that cause us to look at data for the wrong resource type. So we add
5100 // this test to make sure that the EDS resource name defaults to the
5101 // cluster name if not specified in the CDS resource.
TEST_P(EdsTest,EdsServiceNameDefaultsToClusterName)5102 TEST_P(EdsTest, EdsServiceNameDefaultsToClusterName) {
5103 AdsServiceImpl::EdsResourceArgs args({
5104 {"locality0", GetBackendPorts()},
5105 });
5106 balancers_[0]->ads_service()->SetEdsResource(
5107 BuildEdsResource(args, kDefaultClusterName));
5108 Cluster cluster = default_cluster_;
5109 cluster.mutable_eds_cluster_config()->clear_service_name();
5110 balancers_[0]->ads_service()->SetCdsResource(cluster);
5111 SetNextResolution({});
5112 SetNextResolutionForLbChannelAllBalancers();
5113 CheckRpcSendOk();
5114 }
5115
5116 class TimeoutTest : public BasicTest {
5117 protected:
SetUp()5118 void SetUp() override {
5119 xds_resource_does_not_exist_timeout_ms_ = 500;
5120 BasicTest::SetUp();
5121 }
5122 };
5123
5124 // Tests that LDS client times out when no response received.
TEST_P(TimeoutTest,Lds)5125 TEST_P(TimeoutTest, Lds) {
5126 balancers_[0]->ads_service()->SetResourceIgnore(kLdsTypeUrl);
5127 SetNextResolution({});
5128 SetNextResolutionForLbChannelAllBalancers();
5129 CheckRpcSendFailure();
5130 }
5131
TEST_P(TimeoutTest,Rds)5132 TEST_P(TimeoutTest, Rds) {
5133 balancers_[0]->ads_service()->SetResourceIgnore(kRdsTypeUrl);
5134 SetNextResolution({});
5135 SetNextResolutionForLbChannelAllBalancers();
5136 CheckRpcSendFailure();
5137 }
5138
5139 // Tests that CDS client times out when no response received.
TEST_P(TimeoutTest,Cds)5140 TEST_P(TimeoutTest, Cds) {
5141 balancers_[0]->ads_service()->SetResourceIgnore(kCdsTypeUrl);
5142 SetNextResolution({});
5143 SetNextResolutionForLbChannelAllBalancers();
5144 CheckRpcSendFailure();
5145 }
5146
TEST_P(TimeoutTest,Eds)5147 TEST_P(TimeoutTest, Eds) {
5148 balancers_[0]->ads_service()->SetResourceIgnore(kEdsTypeUrl);
5149 SetNextResolution({});
5150 SetNextResolutionForLbChannelAllBalancers();
5151 CheckRpcSendFailure();
5152 }
5153
5154 using LocalityMapTest = BasicTest;
5155
5156 // Tests that the localities in a locality map are picked according to their
5157 // weights.
TEST_P(LocalityMapTest,WeightedRoundRobin)5158 TEST_P(LocalityMapTest, WeightedRoundRobin) {
5159 SetNextResolution({});
5160 SetNextResolutionForLbChannelAllBalancers();
5161 const size_t kNumRpcs = 5000;
5162 const int kLocalityWeight0 = 2;
5163 const int kLocalityWeight1 = 8;
5164 const int kTotalLocalityWeight = kLocalityWeight0 + kLocalityWeight1;
5165 const double kLocalityWeightRate0 =
5166 static_cast<double>(kLocalityWeight0) / kTotalLocalityWeight;
5167 const double kLocalityWeightRate1 =
5168 static_cast<double>(kLocalityWeight1) / kTotalLocalityWeight;
5169 // ADS response contains 2 localities, each of which contains 1 backend.
5170 AdsServiceImpl::EdsResourceArgs args({
5171 {"locality0", GetBackendPorts(0, 1), kLocalityWeight0},
5172 {"locality1", GetBackendPorts(1, 2), kLocalityWeight1},
5173 });
5174 balancers_[0]->ads_service()->SetEdsResource(
5175 BuildEdsResource(args, DefaultEdsServiceName()));
5176 // Wait for both backends to be ready.
5177 WaitForAllBackends(0, 2);
5178 // Send kNumRpcs RPCs.
5179 CheckRpcSendOk(kNumRpcs);
5180 // The locality picking rates should be roughly equal to the expectation.
5181 const double locality_picked_rate_0 =
5182 static_cast<double>(backends_[0]->backend_service()->request_count()) /
5183 kNumRpcs;
5184 const double locality_picked_rate_1 =
5185 static_cast<double>(backends_[1]->backend_service()->request_count()) /
5186 kNumRpcs;
5187 const double kErrorTolerance = 0.2;
5188 EXPECT_THAT(locality_picked_rate_0,
5189 ::testing::AllOf(
5190 ::testing::Ge(kLocalityWeightRate0 * (1 - kErrorTolerance)),
5191 ::testing::Le(kLocalityWeightRate0 * (1 + kErrorTolerance))));
5192 EXPECT_THAT(locality_picked_rate_1,
5193 ::testing::AllOf(
5194 ::testing::Ge(kLocalityWeightRate1 * (1 - kErrorTolerance)),
5195 ::testing::Le(kLocalityWeightRate1 * (1 + kErrorTolerance))));
5196 }
5197
5198 // Tests that we correctly handle a locality containing no endpoints.
TEST_P(LocalityMapTest,LocalityContainingNoEndpoints)5199 TEST_P(LocalityMapTest, LocalityContainingNoEndpoints) {
5200 SetNextResolution({});
5201 SetNextResolutionForLbChannelAllBalancers();
5202 const size_t kNumRpcs = 5000;
5203 // EDS response contains 2 localities, one with no endpoints.
5204 AdsServiceImpl::EdsResourceArgs args({
5205 {"locality0", GetBackendPorts()},
5206 {"locality1", {}},
5207 });
5208 balancers_[0]->ads_service()->SetEdsResource(
5209 BuildEdsResource(args, DefaultEdsServiceName()));
5210 // Wait for both backends to be ready.
5211 WaitForAllBackends();
5212 // Send kNumRpcs RPCs.
5213 CheckRpcSendOk(kNumRpcs);
5214 // All traffic should go to the reachable locality.
5215 EXPECT_EQ(backends_[0]->backend_service()->request_count(),
5216 kNumRpcs / backends_.size());
5217 EXPECT_EQ(backends_[1]->backend_service()->request_count(),
5218 kNumRpcs / backends_.size());
5219 EXPECT_EQ(backends_[2]->backend_service()->request_count(),
5220 kNumRpcs / backends_.size());
5221 EXPECT_EQ(backends_[3]->backend_service()->request_count(),
5222 kNumRpcs / backends_.size());
5223 }
5224
5225 // EDS update with no localities.
TEST_P(LocalityMapTest,NoLocalities)5226 TEST_P(LocalityMapTest, NoLocalities) {
5227 SetNextResolution({});
5228 SetNextResolutionForLbChannelAllBalancers();
5229 balancers_[0]->ads_service()->SetEdsResource(
5230 BuildEdsResource({}, DefaultEdsServiceName()));
5231 Status status = SendRpc();
5232 EXPECT_FALSE(status.ok());
5233 EXPECT_EQ(status.error_code(), StatusCode::UNAVAILABLE);
5234 }
5235
5236 // Tests that the locality map can work properly even when it contains a large
5237 // number of localities.
TEST_P(LocalityMapTest,StressTest)5238 TEST_P(LocalityMapTest, StressTest) {
5239 SetNextResolution({});
5240 SetNextResolutionForLbChannelAllBalancers();
5241 const size_t kNumLocalities = 100;
5242 // The first ADS response contains kNumLocalities localities, each of which
5243 // contains backend 0.
5244 AdsServiceImpl::EdsResourceArgs args;
5245 for (size_t i = 0; i < kNumLocalities; ++i) {
5246 std::string name = absl::StrCat("locality", i);
5247 AdsServiceImpl::EdsResourceArgs::Locality locality(name,
5248 {backends_[0]->port()});
5249 args.locality_list.emplace_back(std::move(locality));
5250 }
5251 balancers_[0]->ads_service()->SetEdsResource(
5252 BuildEdsResource(args, DefaultEdsServiceName()));
5253 // The second ADS response contains 1 locality, which contains backend 1.
5254 args = AdsServiceImpl::EdsResourceArgs({
5255 {"locality0", GetBackendPorts(1, 2)},
5256 });
5257 std::thread delayed_resource_setter(
5258 std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
5259 BuildEdsResource(args, DefaultEdsServiceName()), 60 * 1000));
5260 // Wait until backend 0 is ready, before which kNumLocalities localities are
5261 // received and handled by the xds policy.
5262 WaitForBackend(0, /*reset_counters=*/false);
5263 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
5264 // Wait until backend 1 is ready, before which kNumLocalities localities are
5265 // removed by the xds policy.
5266 WaitForBackend(1);
5267 delayed_resource_setter.join();
5268 }
5269
5270 // Tests that the localities in a locality map are picked correctly after update
5271 // (addition, modification, deletion).
TEST_P(LocalityMapTest,UpdateMap)5272 TEST_P(LocalityMapTest, UpdateMap) {
5273 SetNextResolution({});
5274 SetNextResolutionForLbChannelAllBalancers();
5275 const size_t kNumRpcs = 3000;
5276 // The locality weight for the first 3 localities.
5277 const std::vector<int> kLocalityWeights0 = {2, 3, 4};
5278 const double kTotalLocalityWeight0 =
5279 std::accumulate(kLocalityWeights0.begin(), kLocalityWeights0.end(), 0);
5280 std::vector<double> locality_weight_rate_0;
5281 locality_weight_rate_0.reserve(kLocalityWeights0.size());
5282 for (int weight : kLocalityWeights0) {
5283 locality_weight_rate_0.push_back(weight / kTotalLocalityWeight0);
5284 }
5285 // Delete the first locality, keep the second locality, change the third
5286 // locality's weight from 4 to 2, and add a new locality with weight 6.
5287 const std::vector<int> kLocalityWeights1 = {3, 2, 6};
5288 const double kTotalLocalityWeight1 =
5289 std::accumulate(kLocalityWeights1.begin(), kLocalityWeights1.end(), 0);
5290 std::vector<double> locality_weight_rate_1 = {
5291 0 /* placeholder for locality 0 */};
5292 for (int weight : kLocalityWeights1) {
5293 locality_weight_rate_1.push_back(weight / kTotalLocalityWeight1);
5294 }
5295 AdsServiceImpl::EdsResourceArgs args({
5296 {"locality0", GetBackendPorts(0, 1), 2},
5297 {"locality1", GetBackendPorts(1, 2), 3},
5298 {"locality2", GetBackendPorts(2, 3), 4},
5299 });
5300 balancers_[0]->ads_service()->SetEdsResource(
5301 BuildEdsResource(args, DefaultEdsServiceName()));
5302 // Wait for the first 3 backends to be ready.
5303 WaitForAllBackends(0, 3);
5304 gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
5305 // Send kNumRpcs RPCs.
5306 CheckRpcSendOk(kNumRpcs);
5307 gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
5308 // The picking rates of the first 3 backends should be roughly equal to the
5309 // expectation.
5310 std::vector<double> locality_picked_rates;
5311 for (size_t i = 0; i < 3; ++i) {
5312 locality_picked_rates.push_back(
5313 static_cast<double>(backends_[i]->backend_service()->request_count()) /
5314 kNumRpcs);
5315 }
5316 const double kErrorTolerance = 0.2;
5317 for (size_t i = 0; i < 3; ++i) {
5318 gpr_log(GPR_INFO, "Locality %" PRIuPTR " rate %f", i,
5319 locality_picked_rates[i]);
5320 EXPECT_THAT(
5321 locality_picked_rates[i],
5322 ::testing::AllOf(
5323 ::testing::Ge(locality_weight_rate_0[i] * (1 - kErrorTolerance)),
5324 ::testing::Le(locality_weight_rate_0[i] * (1 + kErrorTolerance))));
5325 }
5326 args = AdsServiceImpl::EdsResourceArgs({
5327 {"locality1", GetBackendPorts(1, 2), 3},
5328 {"locality2", GetBackendPorts(2, 3), 2},
5329 {"locality3", GetBackendPorts(3, 4), 6},
5330 });
5331 balancers_[0]->ads_service()->SetEdsResource(
5332 BuildEdsResource(args, DefaultEdsServiceName()));
5333 // Backend 3 hasn't received any request.
5334 EXPECT_EQ(0U, backends_[3]->backend_service()->request_count());
5335 // Wait until the locality update has been processed, as signaled by backend 3
5336 // receiving a request.
5337 WaitForAllBackends(3, 4);
5338 gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
5339 // Send kNumRpcs RPCs.
5340 CheckRpcSendOk(kNumRpcs);
5341 gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
5342 // Backend 0 no longer receives any request.
5343 EXPECT_EQ(0U, backends_[0]->backend_service()->request_count());
5344 // The picking rates of the last 3 backends should be roughly equal to the
5345 // expectation.
5346 locality_picked_rates = {0 /* placeholder for backend 0 */};
5347 for (size_t i = 1; i < 4; ++i) {
5348 locality_picked_rates.push_back(
5349 static_cast<double>(backends_[i]->backend_service()->request_count()) /
5350 kNumRpcs);
5351 }
5352 for (size_t i = 1; i < 4; ++i) {
5353 gpr_log(GPR_INFO, "Locality %" PRIuPTR " rate %f", i,
5354 locality_picked_rates[i]);
5355 EXPECT_THAT(
5356 locality_picked_rates[i],
5357 ::testing::AllOf(
5358 ::testing::Ge(locality_weight_rate_1[i] * (1 - kErrorTolerance)),
5359 ::testing::Le(locality_weight_rate_1[i] * (1 + kErrorTolerance))));
5360 }
5361 }
5362
5363 // Tests that we don't fail RPCs when replacing all of the localities in
5364 // a given priority.
TEST_P(LocalityMapTest,ReplaceAllLocalitiesInPriority)5365 TEST_P(LocalityMapTest, ReplaceAllLocalitiesInPriority) {
5366 SetNextResolution({});
5367 SetNextResolutionForLbChannelAllBalancers();
5368 AdsServiceImpl::EdsResourceArgs args({
5369 {"locality0", GetBackendPorts(0, 1)},
5370 });
5371 balancers_[0]->ads_service()->SetEdsResource(
5372 BuildEdsResource(args, DefaultEdsServiceName()));
5373 args = AdsServiceImpl::EdsResourceArgs({
5374 {"locality1", GetBackendPorts(1, 2)},
5375 });
5376 std::thread delayed_resource_setter(
5377 std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
5378 BuildEdsResource(args, DefaultEdsServiceName()), 5000));
5379 // Wait for the first backend to be ready.
5380 WaitForBackend(0);
5381 // Keep sending RPCs until we switch over to backend 1, which tells us
5382 // that we received the update. No RPCs should fail during this
5383 // transition.
5384 WaitForBackend(1, /*reset_counters=*/true, /*require_success=*/true);
5385 delayed_resource_setter.join();
5386 }
5387
5388 class FailoverTest : public BasicTest {
5389 public:
SetUp()5390 void SetUp() override {
5391 BasicTest::SetUp();
5392 ResetStub(500);
5393 }
5394 };
5395
5396 // Localities with the highest priority are used when multiple priority exist.
TEST_P(FailoverTest,ChooseHighestPriority)5397 TEST_P(FailoverTest, ChooseHighestPriority) {
5398 SetNextResolution({});
5399 SetNextResolutionForLbChannelAllBalancers();
5400 AdsServiceImpl::EdsResourceArgs args({
5401 {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 1},
5402 {"locality1", GetBackendPorts(1, 2), kDefaultLocalityWeight, 2},
5403 {"locality2", GetBackendPorts(2, 3), kDefaultLocalityWeight, 3},
5404 {"locality3", GetBackendPorts(3, 4), kDefaultLocalityWeight, 0},
5405 });
5406 balancers_[0]->ads_service()->SetEdsResource(
5407 BuildEdsResource(args, DefaultEdsServiceName()));
5408 WaitForBackend(3, false);
5409 for (size_t i = 0; i < 3; ++i) {
5410 EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
5411 }
5412 }
5413
5414 // Does not choose priority with no endpoints.
TEST_P(FailoverTest,DoesNotUsePriorityWithNoEndpoints)5415 TEST_P(FailoverTest, DoesNotUsePriorityWithNoEndpoints) {
5416 SetNextResolution({});
5417 SetNextResolutionForLbChannelAllBalancers();
5418 AdsServiceImpl::EdsResourceArgs args({
5419 {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 1},
5420 {"locality1", GetBackendPorts(1, 2), kDefaultLocalityWeight, 2},
5421 {"locality2", GetBackendPorts(2, 3), kDefaultLocalityWeight, 3},
5422 {"locality3", {}, kDefaultLocalityWeight, 0},
5423 });
5424 balancers_[0]->ads_service()->SetEdsResource(
5425 BuildEdsResource(args, DefaultEdsServiceName()));
5426 WaitForBackend(0, false);
5427 for (size_t i = 1; i < 3; ++i) {
5428 EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
5429 }
5430 }
5431
5432 // Does not choose locality with no endpoints.
TEST_P(FailoverTest,DoesNotUseLocalityWithNoEndpoints)5433 TEST_P(FailoverTest, DoesNotUseLocalityWithNoEndpoints) {
5434 SetNextResolution({});
5435 SetNextResolutionForLbChannelAllBalancers();
5436 AdsServiceImpl::EdsResourceArgs args({
5437 {"locality0", {}, kDefaultLocalityWeight, 0},
5438 {"locality1", GetBackendPorts(), kDefaultLocalityWeight, 0},
5439 });
5440 balancers_[0]->ads_service()->SetEdsResource(
5441 BuildEdsResource(args, DefaultEdsServiceName()));
5442 // Wait for all backends to be used.
5443 std::tuple<int, int, int> counts = WaitForAllBackends();
5444 // Make sure no RPCs failed in the transition.
5445 EXPECT_EQ(0, std::get<1>(counts));
5446 }
5447
5448 // If the higher priority localities are not reachable, failover to the highest
5449 // priority among the rest.
TEST_P(FailoverTest,Failover)5450 TEST_P(FailoverTest, Failover) {
5451 SetNextResolution({});
5452 SetNextResolutionForLbChannelAllBalancers();
5453 AdsServiceImpl::EdsResourceArgs args({
5454 {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 1},
5455 {"locality1", GetBackendPorts(1, 2), kDefaultLocalityWeight, 2},
5456 {"locality2", GetBackendPorts(2, 3), kDefaultLocalityWeight, 3},
5457 {"locality3", GetBackendPorts(3, 4), kDefaultLocalityWeight, 0},
5458 });
5459 ShutdownBackend(3);
5460 ShutdownBackend(0);
5461 balancers_[0]->ads_service()->SetEdsResource(
5462 BuildEdsResource(args, DefaultEdsServiceName()));
5463 WaitForBackend(1, false);
5464 for (size_t i = 0; i < 4; ++i) {
5465 if (i == 1) continue;
5466 EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
5467 }
5468 }
5469
5470 // If a locality with higher priority than the current one becomes ready,
5471 // switch to it.
TEST_P(FailoverTest,SwitchBackToHigherPriority)5472 TEST_P(FailoverTest, SwitchBackToHigherPriority) {
5473 SetNextResolution({});
5474 SetNextResolutionForLbChannelAllBalancers();
5475 const size_t kNumRpcs = 100;
5476 AdsServiceImpl::EdsResourceArgs args({
5477 {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 1},
5478 {"locality1", GetBackendPorts(1, 2), kDefaultLocalityWeight, 2},
5479 {"locality2", GetBackendPorts(2, 3), kDefaultLocalityWeight, 3},
5480 {"locality3", GetBackendPorts(3, 4), kDefaultLocalityWeight, 0},
5481 });
5482 ShutdownBackend(3);
5483 ShutdownBackend(0);
5484 balancers_[0]->ads_service()->SetEdsResource(
5485 BuildEdsResource(args, DefaultEdsServiceName()));
5486 WaitForBackend(1, false);
5487 for (size_t i = 0; i < 4; ++i) {
5488 if (i == 1) continue;
5489 EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
5490 }
5491 StartBackend(0);
5492 WaitForBackend(0);
5493 CheckRpcSendOk(kNumRpcs);
5494 EXPECT_EQ(kNumRpcs, backends_[0]->backend_service()->request_count());
5495 }
5496
5497 // The first update only contains unavailable priorities. The second update
5498 // contains available priorities.
TEST_P(FailoverTest,UpdateInitialUnavailable)5499 TEST_P(FailoverTest, UpdateInitialUnavailable) {
5500 SetNextResolution({});
5501 SetNextResolutionForLbChannelAllBalancers();
5502 AdsServiceImpl::EdsResourceArgs args({
5503 {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 0},
5504 {"locality1", GetBackendPorts(1, 2), kDefaultLocalityWeight, 1},
5505 });
5506 balancers_[0]->ads_service()->SetEdsResource(
5507 BuildEdsResource(args, DefaultEdsServiceName()));
5508 args = AdsServiceImpl::EdsResourceArgs({
5509 {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 0},
5510 {"locality1", GetBackendPorts(1, 2), kDefaultLocalityWeight, 1},
5511 {"locality2", GetBackendPorts(2, 3), kDefaultLocalityWeight, 2},
5512 {"locality3", GetBackendPorts(3, 4), kDefaultLocalityWeight, 3},
5513 });
5514 ShutdownBackend(0);
5515 ShutdownBackend(1);
5516 std::thread delayed_resource_setter(
5517 std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
5518 BuildEdsResource(args, DefaultEdsServiceName()), 1000));
5519 gpr_timespec deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
5520 gpr_time_from_millis(500, GPR_TIMESPAN));
5521 // Send 0.5 second worth of RPCs.
5522 do {
5523 CheckRpcSendFailure();
5524 } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0);
5525 WaitForBackend(2, false);
5526 for (size_t i = 0; i < 4; ++i) {
5527 if (i == 2) continue;
5528 EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
5529 }
5530 delayed_resource_setter.join();
5531 }
5532
5533 // Tests that after the localities' priorities are updated, we still choose the
5534 // highest READY priority with the updated localities.
TEST_P(FailoverTest,UpdatePriority)5535 TEST_P(FailoverTest, UpdatePriority) {
5536 SetNextResolution({});
5537 SetNextResolutionForLbChannelAllBalancers();
5538 const size_t kNumRpcs = 100;
5539 AdsServiceImpl::EdsResourceArgs args({
5540 {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 1},
5541 {"locality1", GetBackendPorts(1, 2), kDefaultLocalityWeight, 2},
5542 {"locality2", GetBackendPorts(2, 3), kDefaultLocalityWeight, 3},
5543 {"locality3", GetBackendPorts(3, 4), kDefaultLocalityWeight, 0},
5544 });
5545 balancers_[0]->ads_service()->SetEdsResource(
5546 BuildEdsResource(args, DefaultEdsServiceName()));
5547 args = AdsServiceImpl::EdsResourceArgs({
5548 {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 2},
5549 {"locality1", GetBackendPorts(1, 2), kDefaultLocalityWeight, 0},
5550 {"locality2", GetBackendPorts(2, 3), kDefaultLocalityWeight, 1},
5551 {"locality3", GetBackendPorts(3, 4), kDefaultLocalityWeight, 3},
5552 });
5553 std::thread delayed_resource_setter(
5554 std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
5555 BuildEdsResource(args, DefaultEdsServiceName()), 1000));
5556 WaitForBackend(3, false);
5557 for (size_t i = 0; i < 3; ++i) {
5558 EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
5559 }
5560 WaitForBackend(1);
5561 CheckRpcSendOk(kNumRpcs);
5562 EXPECT_EQ(kNumRpcs, backends_[1]->backend_service()->request_count());
5563 delayed_resource_setter.join();
5564 }
5565
5566 // Moves all localities in the current priority to a higher priority.
TEST_P(FailoverTest,MoveAllLocalitiesInCurrentPriorityToHigherPriority)5567 TEST_P(FailoverTest, MoveAllLocalitiesInCurrentPriorityToHigherPriority) {
5568 SetNextResolution({});
5569 SetNextResolutionForLbChannelAllBalancers();
5570 // First update:
5571 // - Priority 0 is locality 0, containing backend 0, which is down.
5572 // - Priority 1 is locality 1, containing backends 1 and 2, which are up.
5573 ShutdownBackend(0);
5574 AdsServiceImpl::EdsResourceArgs args({
5575 {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 0},
5576 {"locality1", GetBackendPorts(1, 3), kDefaultLocalityWeight, 1},
5577 });
5578 balancers_[0]->ads_service()->SetEdsResource(
5579 BuildEdsResource(args, DefaultEdsServiceName()));
5580 // Second update:
5581 // - Priority 0 contains both localities 0 and 1.
5582 // - Priority 1 is not present.
5583 // - We add backend 3 to locality 1, just so we have a way to know
5584 // when the update has been seen by the client.
5585 args = AdsServiceImpl::EdsResourceArgs({
5586 {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 0},
5587 {"locality1", GetBackendPorts(1, 4), kDefaultLocalityWeight, 0},
5588 });
5589 std::thread delayed_resource_setter(
5590 std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
5591 BuildEdsResource(args, DefaultEdsServiceName()), 1000));
5592 // When we get the first update, all backends in priority 0 are down,
5593 // so we will create priority 1. Backends 1 and 2 should have traffic,
5594 // but backend 3 should not.
5595 WaitForAllBackends(1, 3, false);
5596 EXPECT_EQ(0UL, backends_[3]->backend_service()->request_count());
5597 // When backend 3 gets traffic, we know the second update has been seen.
5598 WaitForBackend(3);
5599 // The ADS service of balancer 0 got at least 1 response.
5600 EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
5601 AdsServiceImpl::ResponseState::NOT_SENT);
5602 delayed_resource_setter.join();
5603 }
5604
5605 using DropTest = BasicTest;
5606
5607 // Tests that RPCs are dropped according to the drop config.
TEST_P(DropTest,Vanilla)5608 TEST_P(DropTest, Vanilla) {
5609 SetNextResolution({});
5610 SetNextResolutionForLbChannelAllBalancers();
5611 const size_t kNumRpcs = 5000;
5612 const uint32_t kDropPerMillionForLb = 100000;
5613 const uint32_t kDropPerMillionForThrottle = 200000;
5614 const double kDropRateForLb = kDropPerMillionForLb / 1000000.0;
5615 const double kDropRateForThrottle = kDropPerMillionForThrottle / 1000000.0;
5616 const double KDropRateForLbAndThrottle =
5617 kDropRateForLb + (1 - kDropRateForLb) * kDropRateForThrottle;
5618 // The ADS response contains two drop categories.
5619 AdsServiceImpl::EdsResourceArgs args({
5620 {"locality0", GetBackendPorts()},
5621 });
5622 args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
5623 {kThrottleDropType, kDropPerMillionForThrottle}};
5624 balancers_[0]->ads_service()->SetEdsResource(
5625 BuildEdsResource(args, DefaultEdsServiceName()));
5626 WaitForAllBackends();
5627 // Send kNumRpcs RPCs and count the drops.
5628 size_t num_drops = 0;
5629 for (size_t i = 0; i < kNumRpcs; ++i) {
5630 EchoResponse response;
5631 const Status status = SendRpc(RpcOptions(), &response);
5632 if (!status.ok() &&
5633 status.error_message() == "Call dropped by load balancing policy") {
5634 ++num_drops;
5635 } else {
5636 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
5637 << " message=" << status.error_message();
5638 EXPECT_EQ(response.message(), kRequestMessage);
5639 }
5640 }
5641 // The drop rate should be roughly equal to the expectation.
5642 const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
5643 const double kErrorTolerance = 0.2;
5644 EXPECT_THAT(
5645 seen_drop_rate,
5646 ::testing::AllOf(
5647 ::testing::Ge(KDropRateForLbAndThrottle * (1 - kErrorTolerance)),
5648 ::testing::Le(KDropRateForLbAndThrottle * (1 + kErrorTolerance))));
5649 }
5650
5651 // Tests that drop config is converted correctly from per hundred.
TEST_P(DropTest,DropPerHundred)5652 TEST_P(DropTest, DropPerHundred) {
5653 SetNextResolution({});
5654 SetNextResolutionForLbChannelAllBalancers();
5655 const size_t kNumRpcs = 5000;
5656 const uint32_t kDropPerHundredForLb = 10;
5657 const double kDropRateForLb = kDropPerHundredForLb / 100.0;
5658 // The ADS response contains one drop category.
5659 AdsServiceImpl::EdsResourceArgs args({
5660 {"locality0", GetBackendPorts()},
5661 });
5662 args.drop_categories = {{kLbDropType, kDropPerHundredForLb}};
5663 args.drop_denominator = FractionalPercent::HUNDRED;
5664 balancers_[0]->ads_service()->SetEdsResource(
5665 BuildEdsResource(args, DefaultEdsServiceName()));
5666 WaitForAllBackends();
5667 // Send kNumRpcs RPCs and count the drops.
5668 size_t num_drops = 0;
5669 for (size_t i = 0; i < kNumRpcs; ++i) {
5670 EchoResponse response;
5671 const Status status = SendRpc(RpcOptions(), &response);
5672 if (!status.ok() &&
5673 status.error_message() == "Call dropped by load balancing policy") {
5674 ++num_drops;
5675 } else {
5676 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
5677 << " message=" << status.error_message();
5678 EXPECT_EQ(response.message(), kRequestMessage);
5679 }
5680 }
5681 // The drop rate should be roughly equal to the expectation.
5682 const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
5683 const double kErrorTolerance = 0.2;
5684 EXPECT_THAT(
5685 seen_drop_rate,
5686 ::testing::AllOf(::testing::Ge(kDropRateForLb * (1 - kErrorTolerance)),
5687 ::testing::Le(kDropRateForLb * (1 + kErrorTolerance))));
5688 }
5689
5690 // Tests that drop config is converted correctly from per ten thousand.
TEST_P(DropTest,DropPerTenThousand)5691 TEST_P(DropTest, DropPerTenThousand) {
5692 SetNextResolution({});
5693 SetNextResolutionForLbChannelAllBalancers();
5694 const size_t kNumRpcs = 5000;
5695 const uint32_t kDropPerTenThousandForLb = 1000;
5696 const double kDropRateForLb = kDropPerTenThousandForLb / 10000.0;
5697 // The ADS response contains one drop category.
5698 AdsServiceImpl::EdsResourceArgs args({
5699 {"locality0", GetBackendPorts()},
5700 });
5701 args.drop_categories = {{kLbDropType, kDropPerTenThousandForLb}};
5702 args.drop_denominator = FractionalPercent::TEN_THOUSAND;
5703 balancers_[0]->ads_service()->SetEdsResource(
5704 BuildEdsResource(args, DefaultEdsServiceName()));
5705 WaitForAllBackends();
5706 // Send kNumRpcs RPCs and count the drops.
5707 size_t num_drops = 0;
5708 for (size_t i = 0; i < kNumRpcs; ++i) {
5709 EchoResponse response;
5710 const Status status = SendRpc(RpcOptions(), &response);
5711 if (!status.ok() &&
5712 status.error_message() == "Call dropped by load balancing policy") {
5713 ++num_drops;
5714 } else {
5715 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
5716 << " message=" << status.error_message();
5717 EXPECT_EQ(response.message(), kRequestMessage);
5718 }
5719 }
5720 // The drop rate should be roughly equal to the expectation.
5721 const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
5722 const double kErrorTolerance = 0.2;
5723 EXPECT_THAT(
5724 seen_drop_rate,
5725 ::testing::AllOf(::testing::Ge(kDropRateForLb * (1 - kErrorTolerance)),
5726 ::testing::Le(kDropRateForLb * (1 + kErrorTolerance))));
5727 }
5728
5729 // Tests that drop is working correctly after update.
TEST_P(DropTest,Update)5730 TEST_P(DropTest, Update) {
5731 SetNextResolution({});
5732 SetNextResolutionForLbChannelAllBalancers();
5733 const size_t kNumRpcs = 3000;
5734 const uint32_t kDropPerMillionForLb = 100000;
5735 const uint32_t kDropPerMillionForThrottle = 200000;
5736 const double kDropRateForLb = kDropPerMillionForLb / 1000000.0;
5737 const double kDropRateForThrottle = kDropPerMillionForThrottle / 1000000.0;
5738 const double KDropRateForLbAndThrottle =
5739 kDropRateForLb + (1 - kDropRateForLb) * kDropRateForThrottle;
5740 // The first ADS response contains one drop category.
5741 AdsServiceImpl::EdsResourceArgs args({
5742 {"locality0", GetBackendPorts()},
5743 });
5744 args.drop_categories = {{kLbDropType, kDropPerMillionForLb}};
5745 balancers_[0]->ads_service()->SetEdsResource(
5746 BuildEdsResource(args, DefaultEdsServiceName()));
5747 WaitForAllBackends();
5748 // Send kNumRpcs RPCs and count the drops.
5749 size_t num_drops = 0;
5750 gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
5751 for (size_t i = 0; i < kNumRpcs; ++i) {
5752 EchoResponse response;
5753 const Status status = SendRpc(RpcOptions(), &response);
5754 if (!status.ok() &&
5755 status.error_message() == "Call dropped by load balancing policy") {
5756 ++num_drops;
5757 } else {
5758 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
5759 << " message=" << status.error_message();
5760 EXPECT_EQ(response.message(), kRequestMessage);
5761 }
5762 }
5763 gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
5764 // The drop rate should be roughly equal to the expectation.
5765 double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
5766 gpr_log(GPR_INFO, "First batch drop rate %f", seen_drop_rate);
5767 const double kErrorTolerance = 0.3;
5768 EXPECT_THAT(
5769 seen_drop_rate,
5770 ::testing::AllOf(::testing::Ge(kDropRateForLb * (1 - kErrorTolerance)),
5771 ::testing::Le(kDropRateForLb * (1 + kErrorTolerance))));
5772 // The second ADS response contains two drop categories, send an update EDS
5773 // response.
5774 args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
5775 {kThrottleDropType, kDropPerMillionForThrottle}};
5776 balancers_[0]->ads_service()->SetEdsResource(
5777 BuildEdsResource(args, DefaultEdsServiceName()));
5778 // Wait until the drop rate increases to the middle of the two configs, which
5779 // implies that the update has been in effect.
5780 const double kDropRateThreshold =
5781 (kDropRateForLb + KDropRateForLbAndThrottle) / 2;
5782 size_t num_rpcs = kNumRpcs;
5783 while (seen_drop_rate < kDropRateThreshold) {
5784 EchoResponse response;
5785 const Status status = SendRpc(RpcOptions(), &response);
5786 ++num_rpcs;
5787 if (!status.ok() &&
5788 status.error_message() == "Call dropped by load balancing policy") {
5789 ++num_drops;
5790 } else {
5791 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
5792 << " message=" << status.error_message();
5793 EXPECT_EQ(response.message(), kRequestMessage);
5794 }
5795 seen_drop_rate = static_cast<double>(num_drops) / num_rpcs;
5796 }
5797 // Send kNumRpcs RPCs and count the drops.
5798 num_drops = 0;
5799 gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
5800 for (size_t i = 0; i < kNumRpcs; ++i) {
5801 EchoResponse response;
5802 const Status status = SendRpc(RpcOptions(), &response);
5803 if (!status.ok() &&
5804 status.error_message() == "Call dropped by load balancing policy") {
5805 ++num_drops;
5806 } else {
5807 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
5808 << " message=" << status.error_message();
5809 EXPECT_EQ(response.message(), kRequestMessage);
5810 }
5811 }
5812 gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
5813 // The new drop rate should be roughly equal to the expectation.
5814 seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
5815 gpr_log(GPR_INFO, "Second batch drop rate %f", seen_drop_rate);
5816 EXPECT_THAT(
5817 seen_drop_rate,
5818 ::testing::AllOf(
5819 ::testing::Ge(KDropRateForLbAndThrottle * (1 - kErrorTolerance)),
5820 ::testing::Le(KDropRateForLbAndThrottle * (1 + kErrorTolerance))));
5821 }
5822
5823 // Tests that all the RPCs are dropped if any drop category drops 100%.
TEST_P(DropTest,DropAll)5824 TEST_P(DropTest, DropAll) {
5825 SetNextResolution({});
5826 SetNextResolutionForLbChannelAllBalancers();
5827 const size_t kNumRpcs = 1000;
5828 const uint32_t kDropPerMillionForLb = 100000;
5829 const uint32_t kDropPerMillionForThrottle = 1000000;
5830 // The ADS response contains two drop categories.
5831 AdsServiceImpl::EdsResourceArgs args;
5832 args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
5833 {kThrottleDropType, kDropPerMillionForThrottle}};
5834 balancers_[0]->ads_service()->SetEdsResource(
5835 BuildEdsResource(args, DefaultEdsServiceName()));
5836 // Send kNumRpcs RPCs and all of them are dropped.
5837 for (size_t i = 0; i < kNumRpcs; ++i) {
5838 EchoResponse response;
5839 const Status status = SendRpc(RpcOptions(), &response);
5840 EXPECT_EQ(status.error_code(), StatusCode::UNAVAILABLE);
5841 EXPECT_EQ(status.error_message(), "Call dropped by load balancing policy");
5842 }
5843 }
5844
5845 class BalancerUpdateTest : public XdsEnd2endTest {
5846 public:
BalancerUpdateTest()5847 BalancerUpdateTest() : XdsEnd2endTest(4, 3) {}
5848 };
5849
5850 // Tests that the old LB call is still used after the balancer address update as
5851 // long as that call is still alive.
TEST_P(BalancerUpdateTest,UpdateBalancersButKeepUsingOriginalBalancer)5852 TEST_P(BalancerUpdateTest, UpdateBalancersButKeepUsingOriginalBalancer) {
5853 SetNextResolution({});
5854 SetNextResolutionForLbChannelAllBalancers();
5855 AdsServiceImpl::EdsResourceArgs args({
5856 {"locality0", {backends_[0]->port()}},
5857 });
5858 balancers_[0]->ads_service()->SetEdsResource(
5859 BuildEdsResource(args, DefaultEdsServiceName()));
5860 args = AdsServiceImpl::EdsResourceArgs({
5861 {"locality0", {backends_[1]->port()}},
5862 });
5863 balancers_[1]->ads_service()->SetEdsResource(
5864 BuildEdsResource(args, DefaultEdsServiceName()));
5865 // Wait until the first backend is ready.
5866 WaitForBackend(0);
5867 // Send 10 requests.
5868 gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
5869 CheckRpcSendOk(10);
5870 gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
5871 // All 10 requests should have gone to the first backend.
5872 EXPECT_EQ(10U, backends_[0]->backend_service()->request_count());
5873 // The ADS service of balancer 0 sent at least 1 response.
5874 EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
5875 AdsServiceImpl::ResponseState::NOT_SENT);
5876 EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
5877 AdsServiceImpl::ResponseState::NOT_SENT)
5878 << "Error Message:"
5879 << balancers_[1]->ads_service()->eds_response_state().error_message;
5880 EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
5881 AdsServiceImpl::ResponseState::NOT_SENT)
5882 << "Error Message:"
5883 << balancers_[2]->ads_service()->eds_response_state().error_message;
5884 gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 ==========");
5885 SetNextResolutionForLbChannel({balancers_[1]->port()});
5886 gpr_log(GPR_INFO, "========= UPDATE 1 DONE ==========");
5887 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
5888 gpr_timespec deadline = gpr_time_add(
5889 gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_millis(10000, GPR_TIMESPAN));
5890 // Send 10 seconds worth of RPCs
5891 do {
5892 CheckRpcSendOk();
5893 } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0);
5894 // The current LB call is still working, so xds continued using it to the
5895 // first balancer, which doesn't assign the second backend.
5896 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
5897 // The ADS service of balancer 0 sent at least 1 response.
5898 EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
5899 AdsServiceImpl::ResponseState::NOT_SENT);
5900 EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
5901 AdsServiceImpl::ResponseState::NOT_SENT)
5902 << "Error Message:"
5903 << balancers_[1]->ads_service()->eds_response_state().error_message;
5904 EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
5905 AdsServiceImpl::ResponseState::NOT_SENT)
5906 << "Error Message:"
5907 << balancers_[2]->ads_service()->eds_response_state().error_message;
5908 }
5909
5910 // Tests that the old LB call is still used after multiple balancer address
5911 // updates as long as that call is still alive. Send an update with the same set
5912 // of LBs as the one in SetUp() in order to verify that the LB channel inside
5913 // xds keeps the initial connection (which by definition is also present in the
5914 // update).
TEST_P(BalancerUpdateTest,Repeated)5915 TEST_P(BalancerUpdateTest, Repeated) {
5916 SetNextResolution({});
5917 SetNextResolutionForLbChannelAllBalancers();
5918 AdsServiceImpl::EdsResourceArgs args({
5919 {"locality0", {backends_[0]->port()}},
5920 });
5921 balancers_[0]->ads_service()->SetEdsResource(
5922 BuildEdsResource(args, DefaultEdsServiceName()));
5923 args = AdsServiceImpl::EdsResourceArgs({
5924 {"locality0", {backends_[1]->port()}},
5925 });
5926 balancers_[1]->ads_service()->SetEdsResource(
5927 BuildEdsResource(args, DefaultEdsServiceName()));
5928 // Wait until the first backend is ready.
5929 WaitForBackend(0);
5930 // Send 10 requests.
5931 gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
5932 CheckRpcSendOk(10);
5933 gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
5934 // All 10 requests should have gone to the first backend.
5935 EXPECT_EQ(10U, backends_[0]->backend_service()->request_count());
5936 // The ADS service of balancer 0 sent at least 1 response.
5937 EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
5938 AdsServiceImpl::ResponseState::NOT_SENT);
5939 EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
5940 AdsServiceImpl::ResponseState::NOT_SENT)
5941 << "Error Message:"
5942 << balancers_[1]->ads_service()->eds_response_state().error_message;
5943 EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
5944 AdsServiceImpl::ResponseState::NOT_SENT)
5945 << "Error Message:"
5946 << balancers_[2]->ads_service()->eds_response_state().error_message;
5947 std::vector<int> ports;
5948 ports.emplace_back(balancers_[0]->port());
5949 ports.emplace_back(balancers_[1]->port());
5950 ports.emplace_back(balancers_[2]->port());
5951 gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 ==========");
5952 SetNextResolutionForLbChannel(ports);
5953 gpr_log(GPR_INFO, "========= UPDATE 1 DONE ==========");
5954 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
5955 gpr_timespec deadline = gpr_time_add(
5956 gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_millis(10000, GPR_TIMESPAN));
5957 // Send 10 seconds worth of RPCs
5958 do {
5959 CheckRpcSendOk();
5960 } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0);
5961 // xds continued using the original LB call to the first balancer, which
5962 // doesn't assign the second backend.
5963 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
5964 ports.clear();
5965 ports.emplace_back(balancers_[0]->port());
5966 ports.emplace_back(balancers_[1]->port());
5967 gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 2 ==========");
5968 SetNextResolutionForLbChannel(ports);
5969 gpr_log(GPR_INFO, "========= UPDATE 2 DONE ==========");
5970 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
5971 deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
5972 gpr_time_from_millis(10000, GPR_TIMESPAN));
5973 // Send 10 seconds worth of RPCs
5974 do {
5975 CheckRpcSendOk();
5976 } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0);
5977 // xds continued using the original LB call to the first balancer, which
5978 // doesn't assign the second backend.
5979 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
5980 }
5981
5982 // Tests that if the balancer is down, the RPCs will still be sent to the
5983 // backends according to the last balancer response, until a new balancer is
5984 // reachable.
TEST_P(BalancerUpdateTest,DeadUpdate)5985 TEST_P(BalancerUpdateTest, DeadUpdate) {
5986 SetNextResolution({});
5987 SetNextResolutionForLbChannel({balancers_[0]->port()});
5988 AdsServiceImpl::EdsResourceArgs args({
5989 {"locality0", {backends_[0]->port()}},
5990 });
5991 balancers_[0]->ads_service()->SetEdsResource(
5992 BuildEdsResource(args, DefaultEdsServiceName()));
5993 args = AdsServiceImpl::EdsResourceArgs({
5994 {"locality0", {backends_[1]->port()}},
5995 });
5996 balancers_[1]->ads_service()->SetEdsResource(
5997 BuildEdsResource(args, DefaultEdsServiceName()));
5998 // Start servers and send 10 RPCs per server.
5999 gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
6000 CheckRpcSendOk(10);
6001 gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
6002 // All 10 requests should have gone to the first backend.
6003 EXPECT_EQ(10U, backends_[0]->backend_service()->request_count());
6004 // The ADS service of balancer 0 sent at least 1 response.
6005 EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
6006 AdsServiceImpl::ResponseState::NOT_SENT);
6007 EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
6008 AdsServiceImpl::ResponseState::NOT_SENT)
6009 << "Error Message:"
6010 << balancers_[1]->ads_service()->eds_response_state().error_message;
6011 EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
6012 AdsServiceImpl::ResponseState::NOT_SENT)
6013 << "Error Message:"
6014 << balancers_[2]->ads_service()->eds_response_state().error_message;
6015 // Kill balancer 0
6016 gpr_log(GPR_INFO, "********** ABOUT TO KILL BALANCER 0 *************");
6017 balancers_[0]->Shutdown();
6018 gpr_log(GPR_INFO, "********** KILLED BALANCER 0 *************");
6019 // This is serviced by the existing child policy.
6020 gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
6021 CheckRpcSendOk(10);
6022 gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
6023 // All 10 requests should again have gone to the first backend.
6024 EXPECT_EQ(20U, backends_[0]->backend_service()->request_count());
6025 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
6026 // The ADS service of no balancers sent anything
6027 EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state().state,
6028 AdsServiceImpl::ResponseState::NOT_SENT)
6029 << "Error Message:"
6030 << balancers_[0]->ads_service()->eds_response_state().error_message;
6031 EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
6032 AdsServiceImpl::ResponseState::NOT_SENT)
6033 << "Error Message:"
6034 << balancers_[1]->ads_service()->eds_response_state().error_message;
6035 EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
6036 AdsServiceImpl::ResponseState::NOT_SENT)
6037 << "Error Message:"
6038 << balancers_[2]->ads_service()->eds_response_state().error_message;
6039 gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 ==========");
6040 SetNextResolutionForLbChannel({balancers_[1]->port()});
6041 gpr_log(GPR_INFO, "========= UPDATE 1 DONE ==========");
6042 // Wait until update has been processed, as signaled by the second backend
6043 // receiving a request. In the meantime, the client continues to be serviced
6044 // (by the first backend) without interruption.
6045 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
6046 WaitForBackend(1);
6047 // This is serviced by the updated RR policy
6048 backends_[1]->backend_service()->ResetCounters();
6049 gpr_log(GPR_INFO, "========= BEFORE THIRD BATCH ==========");
6050 CheckRpcSendOk(10);
6051 gpr_log(GPR_INFO, "========= DONE WITH THIRD BATCH ==========");
6052 // All 10 requests should have gone to the second backend.
6053 EXPECT_EQ(10U, backends_[1]->backend_service()->request_count());
6054 // The ADS service of balancer 1 sent at least 1 response.
6055 EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state().state,
6056 AdsServiceImpl::ResponseState::NOT_SENT)
6057 << "Error Message:"
6058 << balancers_[0]->ads_service()->eds_response_state().error_message;
6059 EXPECT_GT(balancers_[1]->ads_service()->eds_response_state().state,
6060 AdsServiceImpl::ResponseState::NOT_SENT);
6061 EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
6062 AdsServiceImpl::ResponseState::NOT_SENT)
6063 << "Error Message:"
6064 << balancers_[2]->ads_service()->eds_response_state().error_message;
6065 }
6066
6067 class ClientLoadReportingTest : public XdsEnd2endTest {
6068 public:
ClientLoadReportingTest()6069 ClientLoadReportingTest() : XdsEnd2endTest(4, 1, 3) {}
6070 };
6071
6072 // Tests that the load report received at the balancer is correct.
TEST_P(ClientLoadReportingTest,Vanilla)6073 TEST_P(ClientLoadReportingTest, Vanilla) {
6074 if (!GetParam().use_xds_resolver()) {
6075 balancers_[0]->lrs_service()->set_cluster_names({kServerName});
6076 }
6077 SetNextResolution({});
6078 SetNextResolutionForLbChannel({balancers_[0]->port()});
6079 const size_t kNumRpcsPerAddress = 10;
6080 const size_t kNumFailuresPerAddress = 3;
6081 // TODO(juanlishen): Partition the backends after multiple localities is
6082 // tested.
6083 AdsServiceImpl::EdsResourceArgs args({
6084 {"locality0", GetBackendPorts()},
6085 });
6086 balancers_[0]->ads_service()->SetEdsResource(
6087 BuildEdsResource(args, DefaultEdsServiceName()));
6088 // Wait until all backends are ready.
6089 int num_ok = 0;
6090 int num_failure = 0;
6091 int num_drops = 0;
6092 std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
6093 // Send kNumRpcsPerAddress RPCs per server.
6094 CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
6095 CheckRpcSendFailure(kNumFailuresPerAddress * num_backends_,
6096 RpcOptions().set_server_fail(true));
6097 // Check that each backend got the right number of requests.
6098 for (size_t i = 0; i < backends_.size(); ++i) {
6099 EXPECT_EQ(kNumRpcsPerAddress + kNumFailuresPerAddress,
6100 backends_[i]->backend_service()->request_count());
6101 }
6102 // The load report received at the balancer should be correct.
6103 std::vector<ClientStats> load_report =
6104 balancers_[0]->lrs_service()->WaitForLoadReport();
6105 ASSERT_EQ(load_report.size(), 1UL);
6106 ClientStats& client_stats = load_report.front();
6107 EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok,
6108 client_stats.total_successful_requests());
6109 EXPECT_EQ(0U, client_stats.total_requests_in_progress());
6110 EXPECT_EQ((kNumRpcsPerAddress + kNumFailuresPerAddress) * num_backends_ +
6111 num_ok + num_failure,
6112 client_stats.total_issued_requests());
6113 EXPECT_EQ(kNumFailuresPerAddress * num_backends_ + num_failure,
6114 client_stats.total_error_requests());
6115 EXPECT_EQ(0U, client_stats.total_dropped_requests());
6116 // The LRS service got a single request, and sent a single response.
6117 EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count());
6118 EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count());
6119 }
6120
6121 // Tests send_all_clusters.
TEST_P(ClientLoadReportingTest,SendAllClusters)6122 TEST_P(ClientLoadReportingTest, SendAllClusters) {
6123 balancers_[0]->lrs_service()->set_send_all_clusters(true);
6124 SetNextResolution({});
6125 SetNextResolutionForLbChannel({balancers_[0]->port()});
6126 const size_t kNumRpcsPerAddress = 10;
6127 const size_t kNumFailuresPerAddress = 3;
6128 // TODO(juanlishen): Partition the backends after multiple localities is
6129 // tested.
6130 AdsServiceImpl::EdsResourceArgs args({
6131 {"locality0", GetBackendPorts()},
6132 });
6133 balancers_[0]->ads_service()->SetEdsResource(
6134 BuildEdsResource(args, DefaultEdsServiceName()));
6135 // Wait until all backends are ready.
6136 int num_ok = 0;
6137 int num_failure = 0;
6138 int num_drops = 0;
6139 std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
6140 // Send kNumRpcsPerAddress RPCs per server.
6141 CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
6142 CheckRpcSendFailure(kNumFailuresPerAddress * num_backends_,
6143 RpcOptions().set_server_fail(true));
6144 // Check that each backend got the right number of requests.
6145 for (size_t i = 0; i < backends_.size(); ++i) {
6146 EXPECT_EQ(kNumRpcsPerAddress + kNumFailuresPerAddress,
6147 backends_[i]->backend_service()->request_count());
6148 }
6149 // The load report received at the balancer should be correct.
6150 std::vector<ClientStats> load_report =
6151 balancers_[0]->lrs_service()->WaitForLoadReport();
6152 ASSERT_EQ(load_report.size(), 1UL);
6153 ClientStats& client_stats = load_report.front();
6154 EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok,
6155 client_stats.total_successful_requests());
6156 EXPECT_EQ(0U, client_stats.total_requests_in_progress());
6157 EXPECT_EQ((kNumRpcsPerAddress + kNumFailuresPerAddress) * num_backends_ +
6158 num_ok + num_failure,
6159 client_stats.total_issued_requests());
6160 EXPECT_EQ(kNumFailuresPerAddress * num_backends_ + num_failure,
6161 client_stats.total_error_requests());
6162 EXPECT_EQ(0U, client_stats.total_dropped_requests());
6163 // The LRS service got a single request, and sent a single response.
6164 EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count());
6165 EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count());
6166 }
6167
6168 // Tests that we don't include stats for clusters that are not requested
6169 // by the LRS server.
TEST_P(ClientLoadReportingTest,HonorsClustersRequestedByLrsServer)6170 TEST_P(ClientLoadReportingTest, HonorsClustersRequestedByLrsServer) {
6171 balancers_[0]->lrs_service()->set_cluster_names({"bogus"});
6172 SetNextResolution({});
6173 SetNextResolutionForLbChannel({balancers_[0]->port()});
6174 const size_t kNumRpcsPerAddress = 100;
6175 AdsServiceImpl::EdsResourceArgs args({
6176 {"locality0", GetBackendPorts()},
6177 });
6178 balancers_[0]->ads_service()->SetEdsResource(
6179 BuildEdsResource(args, DefaultEdsServiceName()));
6180 // Wait until all backends are ready.
6181 int num_ok = 0;
6182 int num_failure = 0;
6183 int num_drops = 0;
6184 std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
6185 // Send kNumRpcsPerAddress RPCs per server.
6186 CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
6187 // Each backend should have gotten 100 requests.
6188 for (size_t i = 0; i < backends_.size(); ++i) {
6189 EXPECT_EQ(kNumRpcsPerAddress,
6190 backends_[i]->backend_service()->request_count());
6191 }
6192 // The LRS service got a single request, and sent a single response.
6193 EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count());
6194 EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count());
6195 // The load report received at the balancer should be correct.
6196 std::vector<ClientStats> load_report =
6197 balancers_[0]->lrs_service()->WaitForLoadReport();
6198 ASSERT_EQ(load_report.size(), 0UL);
6199 }
6200
6201 // Tests that if the balancer restarts, the client load report contains the
6202 // stats before and after the restart correctly.
TEST_P(ClientLoadReportingTest,BalancerRestart)6203 TEST_P(ClientLoadReportingTest, BalancerRestart) {
6204 if (!GetParam().use_xds_resolver()) {
6205 balancers_[0]->lrs_service()->set_cluster_names({kServerName});
6206 }
6207 SetNextResolution({});
6208 SetNextResolutionForLbChannel({balancers_[0]->port()});
6209 const size_t kNumBackendsFirstPass = backends_.size() / 2;
6210 const size_t kNumBackendsSecondPass =
6211 backends_.size() - kNumBackendsFirstPass;
6212 AdsServiceImpl::EdsResourceArgs args({
6213 {"locality0", GetBackendPorts(0, kNumBackendsFirstPass)},
6214 });
6215 balancers_[0]->ads_service()->SetEdsResource(
6216 BuildEdsResource(args, DefaultEdsServiceName()));
6217 // Wait until all backends returned by the balancer are ready.
6218 int num_ok = 0;
6219 int num_failure = 0;
6220 int num_drops = 0;
6221 std::tie(num_ok, num_failure, num_drops) =
6222 WaitForAllBackends(/* start_index */ 0,
6223 /* stop_index */ kNumBackendsFirstPass);
6224 std::vector<ClientStats> load_report =
6225 balancers_[0]->lrs_service()->WaitForLoadReport();
6226 ASSERT_EQ(load_report.size(), 1UL);
6227 ClientStats client_stats = std::move(load_report.front());
6228 EXPECT_EQ(static_cast<size_t>(num_ok),
6229 client_stats.total_successful_requests());
6230 EXPECT_EQ(0U, client_stats.total_requests_in_progress());
6231 EXPECT_EQ(0U, client_stats.total_error_requests());
6232 EXPECT_EQ(0U, client_stats.total_dropped_requests());
6233 // Shut down the balancer.
6234 balancers_[0]->Shutdown();
6235 // We should continue using the last EDS response we received from the
6236 // balancer before it was shut down.
6237 // Note: We need to use WaitForAllBackends() here instead of just
6238 // CheckRpcSendOk(kNumBackendsFirstPass), because when the balancer
6239 // shuts down, the XdsClient will generate an error to the
6240 // ServiceConfigWatcher, which will cause the xds resolver to send a
6241 // no-op update to the LB policy. When this update gets down to the
6242 // round_robin child policy for the locality, it will generate a new
6243 // subchannel list, which resets the start index randomly. So we need
6244 // to be a little more permissive here to avoid spurious failures.
6245 ResetBackendCounters();
6246 int num_started = std::get<0>(WaitForAllBackends(
6247 /* start_index */ 0, /* stop_index */ kNumBackendsFirstPass));
6248 // Now restart the balancer, this time pointing to the new backends.
6249 balancers_[0]->Start();
6250 args = AdsServiceImpl::EdsResourceArgs({
6251 {"locality0", GetBackendPorts(kNumBackendsFirstPass)},
6252 });
6253 balancers_[0]->ads_service()->SetEdsResource(
6254 BuildEdsResource(args, DefaultEdsServiceName()));
6255 // Wait for queries to start going to one of the new backends.
6256 // This tells us that we're now using the new serverlist.
6257 std::tie(num_ok, num_failure, num_drops) =
6258 WaitForAllBackends(/* start_index */ kNumBackendsFirstPass);
6259 num_started += num_ok + num_failure + num_drops;
6260 // Send one RPC per backend.
6261 CheckRpcSendOk(kNumBackendsSecondPass);
6262 num_started += kNumBackendsSecondPass;
6263 // Check client stats.
6264 load_report = balancers_[0]->lrs_service()->WaitForLoadReport();
6265 ASSERT_EQ(load_report.size(), 1UL);
6266 client_stats = std::move(load_report.front());
6267 EXPECT_EQ(num_started, client_stats.total_successful_requests());
6268 EXPECT_EQ(0U, client_stats.total_requests_in_progress());
6269 EXPECT_EQ(0U, client_stats.total_error_requests());
6270 EXPECT_EQ(0U, client_stats.total_dropped_requests());
6271 }
6272
6273 class ClientLoadReportingWithDropTest : public XdsEnd2endTest {
6274 public:
ClientLoadReportingWithDropTest()6275 ClientLoadReportingWithDropTest() : XdsEnd2endTest(4, 1, 20) {}
6276 };
6277
6278 // Tests that the drop stats are correctly reported by client load reporting.
TEST_P(ClientLoadReportingWithDropTest,Vanilla)6279 TEST_P(ClientLoadReportingWithDropTest, Vanilla) {
6280 if (!GetParam().use_xds_resolver()) {
6281 balancers_[0]->lrs_service()->set_cluster_names({kServerName});
6282 }
6283 SetNextResolution({});
6284 SetNextResolutionForLbChannelAllBalancers();
6285 const size_t kNumRpcs = 3000;
6286 const uint32_t kDropPerMillionForLb = 100000;
6287 const uint32_t kDropPerMillionForThrottle = 200000;
6288 const double kDropRateForLb = kDropPerMillionForLb / 1000000.0;
6289 const double kDropRateForThrottle = kDropPerMillionForThrottle / 1000000.0;
6290 const double KDropRateForLbAndThrottle =
6291 kDropRateForLb + (1 - kDropRateForLb) * kDropRateForThrottle;
6292 // The ADS response contains two drop categories.
6293 AdsServiceImpl::EdsResourceArgs args({
6294 {"locality0", GetBackendPorts()},
6295 });
6296 args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
6297 {kThrottleDropType, kDropPerMillionForThrottle}};
6298 balancers_[0]->ads_service()->SetEdsResource(
6299 BuildEdsResource(args, DefaultEdsServiceName()));
6300 int num_ok = 0;
6301 int num_failure = 0;
6302 int num_drops = 0;
6303 std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
6304 const size_t num_warmup = num_ok + num_failure + num_drops;
6305 // Send kNumRpcs RPCs and count the drops.
6306 for (size_t i = 0; i < kNumRpcs; ++i) {
6307 EchoResponse response;
6308 const Status status = SendRpc(RpcOptions(), &response);
6309 if (!status.ok() &&
6310 status.error_message() == "Call dropped by load balancing policy") {
6311 ++num_drops;
6312 } else {
6313 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
6314 << " message=" << status.error_message();
6315 EXPECT_EQ(response.message(), kRequestMessage);
6316 }
6317 }
6318 // The drop rate should be roughly equal to the expectation.
6319 const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
6320 const double kErrorTolerance = 0.2;
6321 EXPECT_THAT(
6322 seen_drop_rate,
6323 ::testing::AllOf(
6324 ::testing::Ge(KDropRateForLbAndThrottle * (1 - kErrorTolerance)),
6325 ::testing::Le(KDropRateForLbAndThrottle * (1 + kErrorTolerance))));
6326 // Check client stats.
6327 const size_t total_rpc = num_warmup + kNumRpcs;
6328 ClientStats client_stats;
6329 do {
6330 std::vector<ClientStats> load_reports =
6331 balancers_[0]->lrs_service()->WaitForLoadReport();
6332 for (const auto& load_report : load_reports) {
6333 client_stats += load_report;
6334 }
6335 } while (client_stats.total_issued_requests() +
6336 client_stats.total_dropped_requests() <
6337 total_rpc);
6338 EXPECT_EQ(num_drops, client_stats.total_dropped_requests());
6339 EXPECT_THAT(
6340 client_stats.dropped_requests(kLbDropType),
6341 ::testing::AllOf(
6342 ::testing::Ge(total_rpc * kDropRateForLb * (1 - kErrorTolerance)),
6343 ::testing::Le(total_rpc * kDropRateForLb * (1 + kErrorTolerance))));
6344 EXPECT_THAT(client_stats.dropped_requests(kThrottleDropType),
6345 ::testing::AllOf(
6346 ::testing::Ge(total_rpc * (1 - kDropRateForLb) *
6347 kDropRateForThrottle * (1 - kErrorTolerance)),
6348 ::testing::Le(total_rpc * (1 - kDropRateForLb) *
6349 kDropRateForThrottle * (1 + kErrorTolerance))));
6350 }
6351
TestTypeName(const::testing::TestParamInfo<TestType> & info)6352 std::string TestTypeName(const ::testing::TestParamInfo<TestType>& info) {
6353 return info.param.AsString();
6354 }
6355
6356 // TestType params:
6357 // - use_xds_resolver
6358 // - enable_load_reporting
6359 // - enable_rds_testing = false
6360 // - use_v2 = false
6361
6362 INSTANTIATE_TEST_SUITE_P(XdsTest, BasicTest,
6363 ::testing::Values(TestType(false, true),
6364 TestType(false, false),
6365 TestType(true, false),
6366 TestType(true, true)),
6367 &TestTypeName);
6368
6369 // Run with both fake resolver and xds resolver.
6370 // Don't run with load reporting or v2 or RDS, since they are irrelevant to
6371 // the tests.
6372 INSTANTIATE_TEST_SUITE_P(XdsTest, SecureNamingTest,
6373 ::testing::Values(TestType(false, false),
6374 TestType(true, false)),
6375 &TestTypeName);
6376
6377 // LDS depends on XdsResolver.
6378 INSTANTIATE_TEST_SUITE_P(XdsTest, LdsTest,
6379 ::testing::Values(TestType(true, false),
6380 TestType(true, true)),
6381 &TestTypeName);
6382
6383 // LDS/RDS commmon tests depend on XdsResolver.
6384 INSTANTIATE_TEST_SUITE_P(XdsTest, LdsRdsTest,
6385 ::testing::Values(TestType(true, false),
6386 TestType(true, true),
6387 TestType(true, false, true),
6388 TestType(true, true, true),
6389 // Also test with xDS v2.
6390 TestType(true, true, true, true)),
6391 &TestTypeName);
6392
6393 // CDS depends on XdsResolver.
6394 INSTANTIATE_TEST_SUITE_P(XdsTest, CdsTest,
6395 ::testing::Values(TestType(true, false),
6396 TestType(true, true)),
6397 &TestTypeName);
6398
6399 // EDS could be tested with or without XdsResolver, but the tests would
6400 // be the same either way, so we test it only with XdsResolver.
6401 INSTANTIATE_TEST_SUITE_P(XdsTest, EdsTest,
6402 ::testing::Values(TestType(true, false),
6403 TestType(true, true)),
6404 &TestTypeName);
6405
6406 // Test initial resource timeouts for each resource type.
6407 // Do this only for XdsResolver with RDS enabled, so that we can test
6408 // all resource types.
6409 // Run with V3 only, since the functionality is no different in V2.
6410 INSTANTIATE_TEST_SUITE_P(XdsTest, TimeoutTest,
6411 ::testing::Values(TestType(true, false, true)),
6412 &TestTypeName);
6413
6414 // XdsResolverOnlyTest depends on XdsResolver.
6415 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsResolverOnlyTest,
6416 ::testing::Values(TestType(true, false),
6417 TestType(true, true)),
6418 &TestTypeName);
6419
6420 // XdsResolverLoadReprtingOnlyTest depends on XdsResolver and load reporting.
6421 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsResolverLoadReportingOnlyTest,
6422 ::testing::Values(TestType(true, true)),
6423 &TestTypeName);
6424
6425 INSTANTIATE_TEST_SUITE_P(XdsTest, LocalityMapTest,
6426 ::testing::Values(TestType(false, true),
6427 TestType(false, false),
6428 TestType(true, false),
6429 TestType(true, true)),
6430 &TestTypeName);
6431
6432 INSTANTIATE_TEST_SUITE_P(XdsTest, FailoverTest,
6433 ::testing::Values(TestType(false, true),
6434 TestType(false, false),
6435 TestType(true, false),
6436 TestType(true, true)),
6437 &TestTypeName);
6438
6439 INSTANTIATE_TEST_SUITE_P(XdsTest, DropTest,
6440 ::testing::Values(TestType(false, true),
6441 TestType(false, false),
6442 TestType(true, false),
6443 TestType(true, true)),
6444 &TestTypeName);
6445
6446 INSTANTIATE_TEST_SUITE_P(XdsTest, BalancerUpdateTest,
6447 ::testing::Values(TestType(false, true),
6448 TestType(false, false),
6449 TestType(true, true)),
6450 &TestTypeName);
6451
6452 // Load reporting tests are not run with load reporting disabled.
6453 INSTANTIATE_TEST_SUITE_P(XdsTest, ClientLoadReportingTest,
6454 ::testing::Values(TestType(false, true),
6455 TestType(true, true)),
6456 &TestTypeName);
6457
6458 // Load reporting tests are not run with load reporting disabled.
6459 INSTANTIATE_TEST_SUITE_P(XdsTest, ClientLoadReportingWithDropTest,
6460 ::testing::Values(TestType(false, true),
6461 TestType(true, true)),
6462 &TestTypeName);
6463
6464 } // namespace
6465 } // namespace testing
6466 } // namespace grpc
6467
main(int argc,char ** argv)6468 int main(int argc, char** argv) {
6469 grpc::testing::TestEnvironment env(argc, argv);
6470 ::testing::InitGoogleTest(&argc, argv);
6471 grpc::testing::WriteBootstrapFiles();
6472 grpc::testing::g_port_saver = new grpc::testing::PortSaver();
6473 const auto result = RUN_ALL_TESTS();
6474 return result;
6475 }
6476